From 724b0fb2466ea6ac786075d15ae8364c3adab41b Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 29 Jun 2017 10:05:02 +0800 Subject: [PATCH 01/36] add initial files for deployment --- .../deploy/ctc_beam_search_decoder.cpp | 143 ++++++++++++++++++ .../deploy/ctc_beam_search_decoder.h | 19 +++ .../deploy/ctc_beam_search_decoder.i | 22 +++ deep_speech_2/deploy/decoder_setup.py | 58 +++++++ deep_speech_2/deploy/scorer.cpp | 82 ++++++++++ deep_speech_2/deploy/scorer.h | 22 +++ deep_speech_2/deploy/scorer.i | 8 + deep_speech_2/deploy/scorer_setup.py | 54 +++++++ 8 files changed, 408 insertions(+) create mode 100644 deep_speech_2/deploy/ctc_beam_search_decoder.cpp create mode 100644 deep_speech_2/deploy/ctc_beam_search_decoder.h create mode 100644 deep_speech_2/deploy/ctc_beam_search_decoder.i create mode 100644 deep_speech_2/deploy/decoder_setup.py create mode 100644 deep_speech_2/deploy/scorer.cpp create mode 100644 deep_speech_2/deploy/scorer.h create mode 100644 deep_speech_2/deploy/scorer.i create mode 100644 deep_speech_2/deploy/scorer_setup.py diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp new file mode 100644 index 0000000000..297c7c24b2 --- /dev/null +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include +#include "ctc_beam_search_decoder.h" + +template +bool pair_comp_first_rev(const std::pair a, const std::pair b) { + return a.first > b.first; +} + +template +bool pair_comp_second_rev(const std::pair a, const std::pair b) { + return a.second > b.second; +} + +/* CTC beam search decoder in C++, the interface is consistent with the original + decoder in Python version. +*/ +std::vector > + ctc_beam_search_decoder(std::vector > probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id, + double cutoff_prob, + Scorer *ext_scorer, + bool nproc + ) +{ + int num_time_steps = probs_seq.size(); + + // assign space ID + std::vector::iterator it = std::find(vocabulary.begin(), vocabulary.end(), " "); + int space_id = it-vocabulary.begin(); + if(space_id >= vocabulary.size()) { + std::cout<<"The character space is not in the vocabulary!"; + exit(1); + } + + // initialize + // two sets containing selected and candidate prefixes respectively + std::map prefix_set_prev, prefix_set_next; + // probability of prefixes ending with blank and non-blank + std::map probs_b_prev, probs_nb_prev; + std::map probs_b_cur, probs_nb_cur; + prefix_set_prev["\t"] = 1.0; + probs_b_prev["\t"] = 1.0; + probs_nb_prev["\t"] = 0.0; + + for (int time_step=0; time_step prob = probs_seq[time_step]; + + std::vector > prob_idx; + for (int i=0; i(i, prob[i])); + } + // pruning of vacobulary + if (cutoff_prob < 1.0) { + std::sort(prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + float cum_prob = 0.0; + int cutoff_len = 0; + for (int i=0; i= cutoff_prob) break; + } + prob_idx = std::vector >(prob_idx.begin(), prob_idx.begin()+cutoff_len); + } + // extend prefix + for (std::map::iterator it = prefix_set_prev.begin(); + it != prefix_set_prev.end(); it++) { + std::string l = it->first; + if( prefix_set_next.find(l) == prefix_set_next.end()) { + probs_b_cur[l] = probs_nb_cur[l] = 0.0; + } + + for (int index=0; index 1) { + score = ext_scorer->get_score(l.substr(1)); + } + probs_nb_cur[l_plus] += score * prob_c * ( + probs_b_prev[l] + probs_nb_prev[l]); + } else { + probs_nb_cur[l_plus] += prob_c * ( + probs_b_prev[l] + probs_nb_prev[l]); + } + prefix_set_next[l_plus] = probs_nb_cur[l_plus]+probs_b_cur[l_plus]; + } + } + + prefix_set_next[l] = probs_b_cur[l]+probs_nb_cur[l]; + } + + probs_b_prev = probs_b_cur; + probs_nb_prev = probs_nb_cur; + std::vector > + prefix_vec_next(prefix_set_next.begin(), prefix_set_next.end()); + std::sort(prefix_vec_next.begin(), prefix_vec_next.end(), pair_comp_second_rev); + int k = beam_size + (prefix_vec_next.begin(), prefix_vec_next.begin()+k); + } + + // post processing + std::vector > beam_result; + for (std::map::iterator it = prefix_set_prev.begin(); + it != prefix_set_prev.end(); it++) { + if (it->second > 0.0 && it->first.size() > 1) { + double prob = it->second; + std::string sentence = it->first.substr(1); + // scoring the last word + if (ext_scorer != NULL && sentence[sentence.size()-1] != ' ') { + prob = prob * ext_scorer->get_score(sentence); + } + double log_prob = log(it->second); + beam_result.push_back(std::pair(log_prob, it->first)); + } + } + // sort the result and return + std::sort(beam_result.begin(), beam_result.end(), pair_comp_first_rev); + return beam_result; +} diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.h b/deep_speech_2/deploy/ctc_beam_search_decoder.h new file mode 100644 index 0000000000..d23252aceb --- /dev/null +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.h @@ -0,0 +1,19 @@ +#ifndef CTC_BEAM_SEARCH_DECODER_H_ +#define CTC_BEAM_SEARCH_DECODER_H_ + +#include +#include +#include +#include "scorer.h" + +std::vector > + ctc_beam_search_decoder(std::vector > probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id=0, + double cutoff_prob=1.0, + Scorer *ext_scorer=NULL, + bool nproc=false + ); + +#endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.i b/deep_speech_2/deploy/ctc_beam_search_decoder.i new file mode 100644 index 0000000000..09e893d38e --- /dev/null +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.i @@ -0,0 +1,22 @@ +%module swig_ctc_beam_search_decoder +%{ +#include "ctc_beam_search_decoder.h" +%} + +%include "std_vector.i" +%include "std_pair.i" +%include "std_string.i" + +namespace std{ + %template(DoubleVector) std::vector; + %template(IntVector) std::vector; + %template(StringVector) std::vector; + %template(VectorOfStructVector) std::vector >; + %template(FloatVector) std::vector; + %template(Pair) std::pair; + %template(PairFloatStringVector) std::vector >; + %template(PairDoubleStringVector) std::vector >; +} + +%import scorer.h +%include "ctc_beam_search_decoder.h" diff --git a/deep_speech_2/deploy/decoder_setup.py b/deep_speech_2/deploy/decoder_setup.py new file mode 100644 index 0000000000..5201172b1f --- /dev/null +++ b/deep_speech_2/deploy/decoder_setup.py @@ -0,0 +1,58 @@ +from setuptools import setup, Extension +import glob +import platform +import os + + +def compile_test(header, library): + dummy_path = os.path.join(os.path.dirname(__file__), "dummy") + command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\"" + return os.system(command) == 0 + + +FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob( + 'util/double-conversion/*.cc') +FILES = [ + fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) +] + +LIBS = ['stdc++'] +if platform.system() != 'Darwin': + LIBS.append('rt') + +ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6'] + +if compile_test('zlib.h', 'z'): + ARGS.append('-DHAVE_ZLIB') + LIBS.append('z') + +if compile_test('bzlib.h', 'bz2'): + ARGS.append('-DHAVE_BZLIB') + LIBS.append('bz2') + +if compile_test('lzma.h', 'lzma'): + ARGS.append('-DHAVE_XZLIB') + LIBS.append('lzma') + +os.system('swig -python -c++ ./ctc_beam_search_decoder.i') + +ctc_beam_search_decoder_module = [ + Extension( + name='_swig_ctc_beam_search_decoder', + sources=FILES + [ + 'scorer.cpp', 'ctc_beam_search_decoder_wrap.cxx', + 'ctc_beam_search_decoder.cpp' + ], + language='C++', + include_dirs=['.'], + libraries=LIBS, + extra_compile_args=ARGS) +] + +setup( + name='swig_ctc_beam_search_decoder', + version='0.1', + author='Yibing Liu', + description="""CTC beam search decoder""", + ext_modules=ctc_beam_search_decoder_module, + py_modules=['swig_ctc_beam_search_decoder'], ) diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp new file mode 100644 index 0000000000..9cb6805567 --- /dev/null +++ b/deep_speech_2/deploy/scorer.cpp @@ -0,0 +1,82 @@ +#include + +#include "scorer.h" +#include "lm/model.hh" +#include "util/tokenize_piece.hh" +#include "util/string_piece.hh" + +using namespace lm::ngram; + +Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { + this->_alpha = alpha; + this->_beta = beta; + this->_language_model = new Model(lm_model_path.c_str()); +} + +Scorer::~Scorer(){ + delete (Model *)this->_language_model; +} + +inline void strip(std::string &str, char ch=' ') { + if (str.size() == 0) return; + int start = 0; + int end = str.size()-1; + for (int i=0; i=0; i--) { + if (str[i] == ch) { + end --; + } else { + break; + } + } + + if (start == 0 && end == str.size()-1) return; + if (start > end) { + std::string emp_str; + str = emp_str; + } else { + str = str.substr(start, end-start+1); + } +} + +int Scorer::word_count(std::string sentence) { + strip(sentence); + int cnt = 0; + for (int i=0; i 0) cnt ++; + return cnt; +} + +double Scorer::language_model_score(std::string sentence) { + Model *model = (Model *)this->_language_model; + State state, out_state; + lm::FullScoreReturn ret; + state = model->BeginSentenceState(); + + for (util::TokenIter it(sentence, ' '); it; ++it){ + lm::WordIndex vocab = model->GetVocabulary().Index(*it); + ret = model->FullScore(state, vocab, out_state); + state = out_state; + } + double score = ret.prob; + + return pow(10, score); +} + +double Scorer::get_score(std::string sentence) { + double lm_score = language_model_score(sentence); + int word_cnt = word_count(sentence); + + double final_score = pow(lm_score, _alpha) * pow(word_cnt, _beta); + return final_score; +} diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h new file mode 100644 index 0000000000..47261bb519 --- /dev/null +++ b/deep_speech_2/deploy/scorer.h @@ -0,0 +1,22 @@ +#ifndef SCORER_H_ +#define SCORER_H_ + +#include + + +class Scorer{ +private: + float _alpha; + float _beta; + void *_language_model; + +public: + Scorer(){} + Scorer(float alpha, float beta, std::string lm_model_path); + ~Scorer(); + int word_count(std::string); + double language_model_score(std::string); + double get_score(std::string); +}; + +#endif diff --git a/deep_speech_2/deploy/scorer.i b/deep_speech_2/deploy/scorer.i new file mode 100644 index 0000000000..8380e15a60 --- /dev/null +++ b/deep_speech_2/deploy/scorer.i @@ -0,0 +1,8 @@ +%module swig_scorer +%{ +#include "scorer.h" +%} + +%include "std_string.i" + +%include "scorer.h" diff --git a/deep_speech_2/deploy/scorer_setup.py b/deep_speech_2/deploy/scorer_setup.py new file mode 100644 index 0000000000..c0006e0717 --- /dev/null +++ b/deep_speech_2/deploy/scorer_setup.py @@ -0,0 +1,54 @@ +from setuptools import setup, Extension +import glob +import platform +import os + + +def compile_test(header, library): + dummy_path = os.path.join(os.path.dirname(__file__), "dummy") + command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\"" + return os.system(command) == 0 + + +FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob( + 'util/double-conversion/*.cc') +FILES = [ + fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) +] + +LIBS = ['stdc++'] +if platform.system() != 'Darwin': + LIBS.append('rt') + +ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6'] + +if compile_test('zlib.h', 'z'): + ARGS.append('-DHAVE_ZLIB') + LIBS.append('z') + +if compile_test('bzlib.h', 'bz2'): + ARGS.append('-DHAVE_BZLIB') + LIBS.append('bz2') + +if compile_test('lzma.h', 'lzma'): + ARGS.append('-DHAVE_XZLIB') + LIBS.append('lzma') + +os.system('swig -python -c++ ./scorer.i') + +ext_modules = [ + Extension( + name='_swig_scorer', + sources=FILES + ['scorer_wrap.cxx', 'scorer.cpp'], + language='C++', + include_dirs=['.'], + libraries=LIBS, + extra_compile_args=ARGS) +] + +setup( + name='swig_scorer', + version='0.1', + ext_modules=ext_modules, + include_package_data=True, + py_modules=['swig_scorer'], ) From 348d6bba64423a1ca0f1532f4cf878811b65760f Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 29 Jun 2017 11:19:39 +0800 Subject: [PATCH 02/36] add deploy.py --- deep_speech_2/deploy.py | 194 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 deep_speech_2/deploy.py diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py new file mode 100644 index 0000000000..3272371bf8 --- /dev/null +++ b/deep_speech_2/deploy.py @@ -0,0 +1,194 @@ +"""Deployment for DeepSpeech2 model.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import gzip +import distutils.util +import multiprocessing +import paddle.v2 as paddle +from data_utils.data import DataGenerator +from model import deep_speech2 +from swig_ctc_beam_search_decoder import * +from swig_scorer import Scorer +from error_rate import wer +import utils + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--num_samples", + default=100, + type=int, + help="Number of samples for inference. (default: %(default)s)") +parser.add_argument( + "--num_conv_layers", + default=2, + type=int, + help="Convolution layer number. (default: %(default)s)") +parser.add_argument( + "--num_rnn_layers", + default=3, + type=int, + help="RNN layer number. (default: %(default)s)") +parser.add_argument( + "--rnn_layer_size", + default=512, + type=int, + help="RNN layer cell number. (default: %(default)s)") +parser.add_argument( + "--use_gpu", + default=True, + type=distutils.util.strtobool, + help="Use gpu or not. (default: %(default)s)") +parser.add_argument( + "--num_threads_data", + default=multiprocessing.cpu_count(), + type=int, + help="Number of cpu threads for preprocessing data. (default: %(default)s)") +parser.add_argument( + "--mean_std_filepath", + default='mean_std.npz', + type=str, + help="Manifest path for normalizer. (default: %(default)s)") +parser.add_argument( + "--decode_manifest_path", + default='datasets/manifest.test', + type=str, + help="Manifest path for decoding. (default: %(default)s)") +parser.add_argument( + "--model_filepath", + default='ds2_new_models_0628/params.pass-51.tar.gz', + type=str, + help="Model filepath. (default: %(default)s)") +parser.add_argument( + "--vocab_filepath", + default='datasets/vocab/eng_vocab.txt', + type=str, + help="Vocabulary filepath. (default: %(default)s)") +parser.add_argument( + "--decode_method", + default='beam_search', + type=str, + help="Method for ctc decoding: best_path or beam_search. (default: %(default)s)" +) +parser.add_argument( + "--beam_size", + default=500, + type=int, + help="Width for beam search decoding. (default: %(default)d)") +parser.add_argument( + "--num_results_per_sample", + default=1, + type=int, + help="Number of output per sample in beam search. (default: %(default)d)") +parser.add_argument( + "--language_model_path", + default="lm/data/en.00.UNKNOWN.klm", + type=str, + help="Path for language model. (default: %(default)s)") +parser.add_argument( + "--alpha", + default=0.26, + type=float, + help="Parameter associated with language model. (default: %(default)f)") +parser.add_argument( + "--beta", + default=0.1, + type=float, + help="Parameter associated with word count. (default: %(default)f)") +parser.add_argument( + "--cutoff_prob", + default=0.99, + type=float, + help="The cutoff probability of pruning" + "in beam search. (default: %(default)f)") +args = parser.parse_args() + + +def infer(): + """Deployment for DeepSpeech2.""" + # initialize data generator + data_generator = DataGenerator( + vocab_filepath=args.vocab_filepath, + mean_std_filepath=args.mean_std_filepath, + augmentation_config='{}', + num_threads=args.num_threads_data) + + # create network config + # paddle.data_type.dense_array is used for variable batch input. + # The size 161 * 161 is only an placeholder value and the real shape + # of input batch data will be induced during training. + audio_data = paddle.layer.data( + name="audio_spectrogram", type=paddle.data_type.dense_array(161 * 161)) + text_data = paddle.layer.data( + name="transcript_text", + type=paddle.data_type.integer_value_sequence(data_generator.vocab_size)) + output_probs = deep_speech2( + audio_data=audio_data, + text_data=text_data, + dict_size=data_generator.vocab_size, + num_conv_layers=args.num_conv_layers, + num_rnn_layers=args.num_rnn_layers, + rnn_size=args.rnn_layer_size, + is_inference=True) + + # load parameters + parameters = paddle.parameters.Parameters.from_tar( + gzip.open(args.model_filepath)) + + # prepare infer data + batch_reader = data_generator.batch_reader_creator( + manifest_path=args.decode_manifest_path, + batch_size=args.num_samples, + min_batch_size=1, + sortagrad=False, + shuffle_method=None) + infer_data = batch_reader().next() + + # run inference + infer_results = paddle.infer( + output_layer=output_probs, parameters=parameters, input=infer_data) + num_steps = len(infer_results) // len(infer_data) + probs_split = [ + infer_results[i * num_steps:(i + 1) * num_steps] + for i in xrange(len(infer_data)) + ] + + # targe transcription + target_transcription = [ + ''.join( + [data_generator.vocab_list[index] for index in infer_data[i][1]]) + for i, probs in enumerate(probs_split) + ] + + ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) + ## decode and print + + wer_sum, wer_counter = 0, 0 + for i, probs in enumerate(probs_split): + beam_result = ctc_beam_search_decoder( + probs.tolist(), + args.beam_size, + data_generator.vocab_list, + len(data_generator.vocab_list), + args.cutoff_prob, + ext_scorer, ) + + print("\nTarget Transcription:\t%s" % target_transcription[i]) + print("Beam %d: %f \t%s" % (0, beam_result[0][0], beam_result[0][1])) + wer_cur = wer(target_transcription[i], beam_result[0][1]) + wer_sum += wer_cur + wer_counter += 1 + print("cur wer = %f , average wer = %f" % + (wer_cur, wer_sum / wer_counter)) + + +def main(): + utils.print_arguments(args) + paddle.init(use_gpu=args.use_gpu, trainer_count=1) + infer() + + +if __name__ == '__main__': + main() From a506198fc7551d1ff824c4e7bb326b60daf42ff7 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 4 Jul 2017 19:15:34 +0800 Subject: [PATCH 03/36] fix bugs --- deep_speech_2/deploy.py | 5 ++-- .../deploy/ctc_beam_search_decoder.cpp | 28 +++++++++---------- deep_speech_2/deploy/scorer.cpp | 14 +++++----- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 3272371bf8..d8a7e5b277 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -58,7 +58,7 @@ help="Manifest path for decoding. (default: %(default)s)") parser.add_argument( "--model_filepath", - default='ds2_new_models_0628/params.pass-51.tar.gz', + default='checkpoints/params.latest.tar.gz', type=str, help="Model filepath. (default: %(default)s)") parser.add_argument( @@ -162,9 +162,10 @@ def infer(): for i, probs in enumerate(probs_split) ] + # external scorer ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) - ## decode and print + ## decode and print wer_sum, wer_counter = 0, 0 for i, probs in enumerate(probs_split): beam_result = ctc_beam_search_decoder( diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp index 297c7c24b2..68d1a8457e 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp @@ -15,10 +15,10 @@ bool pair_comp_second_rev(const std::pair a, const std::pair b) return a.second > b.second; } -/* CTC beam search decoder in C++, the interface is consistent with the original +/* CTC beam search decoder in C++, the interface is consistent with the original decoder in Python version. */ -std::vector > +std::vector > ctc_beam_search_decoder(std::vector > probs_seq, int beam_size, std::vector vocabulary, @@ -29,15 +29,15 @@ std::vector > ) { int num_time_steps = probs_seq.size(); - - // assign space ID + + // assign space ID std::vector::iterator it = std::find(vocabulary.begin(), vocabulary.end(), " "); int space_id = it-vocabulary.begin(); if(space_id >= vocabulary.size()) { std::cout<<"The character space is not in the vocabulary!"; - exit(1); + exit(1); } - + // initialize // two sets containing selected and candidate prefixes respectively std::map prefix_set_prev, prefix_set_next; @@ -47,7 +47,7 @@ std::vector > prefix_set_prev["\t"] = 1.0; probs_b_prev["\t"] = 1.0; probs_nb_prev["\t"] = 0.0; - + for (int time_step=0; time_step > } prob_idx = std::vector >(prob_idx.begin(), prob_idx.begin()+cutoff_len); } - // extend prefix - for (std::map::iterator it = prefix_set_prev.begin(); + // extend prefix + for (std::map::iterator it = prefix_set_prev.begin(); it != prefix_set_prev.end(); it++) { std::string l = it->first; if( prefix_set_next.find(l) == prefix_set_next.end()) { @@ -109,12 +109,12 @@ std::vector > } } - prefix_set_next[l] = probs_b_cur[l]+probs_nb_cur[l]; + prefix_set_next[l] = probs_b_cur[l]+probs_nb_cur[l]; } probs_b_prev = probs_b_cur; probs_nb_prev = probs_nb_cur; - std::vector > + std::vector > prefix_vec_next(prefix_set_next.begin(), prefix_set_next.end()); std::sort(prefix_vec_next.begin(), prefix_vec_next.end(), pair_comp_second_rev); int k = beam_size > // post processing std::vector > beam_result; - for (std::map::iterator it = prefix_set_prev.begin(); + for (std::map::iterator it = prefix_set_prev.begin(); it != prefix_set_prev.end(); it++) { if (it->second > 0.0 && it->first.size() > 1) { double prob = it->second; @@ -133,8 +133,8 @@ std::vector > if (ext_scorer != NULL && sentence[sentence.size()-1] != ' ') { prob = prob * ext_scorer->get_score(sentence); } - double log_prob = log(it->second); - beam_result.push_back(std::pair(log_prob, it->first)); + double log_prob = log(prob); + beam_result.push_back(std::pair(log_prob, sentence)); } } // sort the result and return diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 9cb6805567..d7f68d71f8 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -35,7 +35,7 @@ inline void strip(std::string &str, char ch=' ') { break; } } - + if (start == 0 && end == str.size()-1) return; if (start > end) { std::string emp_str; @@ -47,13 +47,12 @@ inline void strip(std::string &str, char ch=' ') { int Scorer::word_count(std::string sentence) { strip(sentence); - int cnt = 0; + int cnt = 1; for (int i=0; i 0) cnt ++; return cnt; } @@ -68,15 +67,16 @@ double Scorer::language_model_score(std::string sentence) { ret = model->FullScore(state, vocab, out_state); state = out_state; } - double score = ret.prob; - - return pow(10, score); + //log10 prob + double log_prob = ret.prob; + + return log_prob; } double Scorer::get_score(std::string sentence) { double lm_score = language_model_score(sentence); int word_cnt = word_count(sentence); - double final_score = pow(lm_score, _alpha) * pow(word_cnt, _beta); + double final_score = pow(10, _alpha*lm_score) * pow(word_cnt, _beta); return final_score; } From 1ca38140e1f4142e219f5609d0c4dfb9d9337b18 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 6 Jul 2017 11:25:05 +0800 Subject: [PATCH 04/36] code cleanup for the deployment decoder --- .../deploy/ctc_beam_search_decoder.cpp | 72 ++++++++++++------- .../deploy/ctc_beam_search_decoder.h | 34 ++++++--- deep_speech_2/deploy/decoder_setup.py | 7 +- deep_speech_2/deploy/scorer.cpp | 14 +++- deep_speech_2/deploy/scorer.h | 20 +++++- deep_speech_2/deploy/scorer_setup.py | 6 +- 6 files changed, 105 insertions(+), 48 deletions(-) diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp index 68d1a8457e..a684b30a61 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp @@ -6,35 +6,47 @@ #include "ctc_beam_search_decoder.h" template -bool pair_comp_first_rev(const std::pair a, const std::pair b) { +bool pair_comp_first_rev(const std::pair a, const std::pair b) +{ return a.first > b.first; } template -bool pair_comp_second_rev(const std::pair a, const std::pair b) { +bool pair_comp_second_rev(const std::pair a, const std::pair b) +{ return a.second > b.second; } -/* CTC beam search decoder in C++, the interface is consistent with the original - decoder in Python version. -*/ std::vector > - ctc_beam_search_decoder(std::vector > probs_seq, - int beam_size, - std::vector vocabulary, - int blank_id, - double cutoff_prob, - Scorer *ext_scorer, - bool nproc - ) -{ + ctc_beam_search_decoder(std::vector > probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id, + double cutoff_prob, + Scorer *ext_scorer, + bool nproc) { + // dimension check int num_time_steps = probs_seq.size(); + for (int i=0; i vocabulary.size()) { + std::cout<<"Invalid blank_id!"<::iterator it = std::find(vocabulary.begin(), vocabulary.end(), " "); - int space_id = it-vocabulary.begin(); + std::vector::iterator it = std::find(vocabulary.begin(), + vocabulary.end(), " "); + int space_id = it - vocabulary.begin(); if(space_id >= vocabulary.size()) { - std::cout<<"The character space is not in the vocabulary!"; + std::cout<<"The character space is not in the vocabulary!"< > } // pruning of vacobulary if (cutoff_prob < 1.0) { - std::sort(prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + std::sort(prob_idx.begin(), prob_idx.end(), + pair_comp_second_rev); float cum_prob = 0.0; int cutoff_len = 0; for (int i=0; i > cutoff_len += 1; if (cum_prob >= cutoff_prob) break; } - prob_idx = std::vector >(prob_idx.begin(), prob_idx.begin()+cutoff_len); + prob_idx = std::vector >( prob_idx.begin(), + prob_idx.begin() + cutoff_len); } // extend prefix for (std::map::iterator it = prefix_set_prev.begin(); @@ -82,11 +96,11 @@ std::vector > int c = prob_idx[index].first; double prob_c = prob_idx[index].second; if (c == blank_id) { - probs_b_cur[l] += prob_c*(probs_b_prev[l]+probs_nb_prev[l]); + probs_b_cur[l] += prob_c * (probs_b_prev[l] + probs_nb_prev[l]); } else { std::string last_char = l.substr(l.size()-1, 1); std::string new_char = vocabulary[c]; - std::string l_plus = l+new_char; + std::string l_plus = l + new_char; if( prefix_set_next.find(l_plus) == prefix_set_next.end()) { probs_b_cur[l_plus] = probs_nb_cur[l_plus] = 0.0; @@ -105,19 +119,22 @@ std::vector > probs_nb_cur[l_plus] += prob_c * ( probs_b_prev[l] + probs_nb_prev[l]); } - prefix_set_next[l_plus] = probs_nb_cur[l_plus]+probs_b_cur[l_plus]; + prefix_set_next[l_plus] = probs_nb_cur[l_plus] + probs_b_cur[l_plus]; } } - prefix_set_next[l] = probs_b_cur[l]+probs_nb_cur[l]; + prefix_set_next[l] = probs_b_cur[l] + probs_nb_cur[l]; } probs_b_prev = probs_b_cur; probs_nb_prev = probs_nb_cur; std::vector > - prefix_vec_next(prefix_set_next.begin(), prefix_set_next.end()); - std::sort(prefix_vec_next.begin(), prefix_vec_next.end(), pair_comp_second_rev); - int k = beam_size); + int k = beam_size (prefix_vec_next.begin(), prefix_vec_next.begin()+k); } @@ -138,6 +155,7 @@ std::vector > } } // sort the result and return - std::sort(beam_result.begin(), beam_result.end(), pair_comp_first_rev); + std::sort(beam_result.begin(), beam_result.end(), + pair_comp_first_rev); return beam_result; } diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.h b/deep_speech_2/deploy/ctc_beam_search_decoder.h index d23252aceb..a4bb6aa741 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.h +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.h @@ -6,14 +6,30 @@ #include #include "scorer.h" -std::vector > - ctc_beam_search_decoder(std::vector > probs_seq, - int beam_size, - std::vector vocabulary, - int blank_id=0, - double cutoff_prob=1.0, - Scorer *ext_scorer=NULL, - bool nproc=false - ); +/* CTC Beam Search Decoder, the interface is consistent with the + * original decoder in Python version. + + * Parameters: + * probs_seq: 2-D vector that each element is a vector of probabilities + * over vocabulary of one time step. + * beam_size: The width of beam search. + * vocabulary: A vector of vocabulary. + * blank_id: ID of blank. + * cutoff_prob: Cutoff probability of pruning + * ext_scorer: External scorer to evaluate a prefix. + * nproc: Whether this function used in multiprocessing. + * Return: + * A vector that each element is a pair of score and decoding result, + * in desending order. +*/ +std::vector > + ctc_beam_search_decoder(std::vector > probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id, + double cutoff_prob=1.0, + Scorer *ext_scorer=NULL, + bool nproc=false + ); #endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/deploy/decoder_setup.py b/deep_speech_2/deploy/decoder_setup.py index 5201172b1f..4ed603b252 100644 --- a/deep_speech_2/deploy/decoder_setup.py +++ b/deep_speech_2/deploy/decoder_setup.py @@ -10,8 +10,8 @@ def compile_test(header, library): return os.system(command) == 0 -FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob( - 'util/double-conversion/*.cc') +FILES = glob.glob('kenlm/util/*.cc') + glob.glob('kenlm/lm/*.cc') + glob.glob( + 'kenlm/util/double-conversion/*.cc') FILES = [ fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) ] @@ -44,7 +44,7 @@ def compile_test(header, library): 'ctc_beam_search_decoder.cpp' ], language='C++', - include_dirs=['.'], + include_dirs=['.', './kenlm'], libraries=LIBS, extra_compile_args=ARGS) ] @@ -52,7 +52,6 @@ def compile_test(header, library): setup( name='swig_ctc_beam_search_decoder', version='0.1', - author='Yibing Liu', description="""CTC beam search decoder""", ext_modules=ctc_beam_search_decoder_module, py_modules=['swig_ctc_beam_search_decoder'], ) diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index d7f68d71f8..1b843402bf 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -1,5 +1,4 @@ #include - #include "scorer.h" #include "lm/model.hh" #include "util/tokenize_piece.hh" @@ -17,6 +16,13 @@ Scorer::~Scorer(){ delete (Model *)this->_language_model; } +/* Strip a input sentence + * Parameters: + * str: A reference to the objective string + * ch: The character to prune + * Return: + * void + */ inline void strip(std::string &str, char ch=' ') { if (str.size() == 0) return; int start = 0; @@ -69,10 +75,14 @@ double Scorer::language_model_score(std::string sentence) { } //log10 prob double log_prob = ret.prob; - return log_prob; } +void Scorer::reset_params(float alpha, float beta) { + this->_alpha = alpha; + this->_beta = beta; +} + double Scorer::get_score(std::string sentence) { double lm_score = language_model_score(sentence); int word_cnt = word_count(sentence); diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 47261bb519..7b305772c5 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -3,20 +3,34 @@ #include +/* External scorer to evaluate a prefix or a complete sentence + * when a new word appended during decoding, consisting of word + * count and language model scoring. + * Example: + * Scorer ext_scorer(alpha, beta, "path_to_language_model.klm"); + * double score = ext_scorer.get_score("sentence_to_score"); + */ class Scorer{ private: float _alpha; float _beta; void *_language_model; + // word insertion term + int word_count(std::string); + // n-gram language model scoring + double language_model_score(std::string); + public: Scorer(){} Scorer(float alpha, float beta, std::string lm_model_path); ~Scorer(); - int word_count(std::string); - double language_model_score(std::string); + + // reset params alpha & beta + void reset_params(float alpha, float beta); + // get the final score double get_score(std::string); }; -#endif +#endif //SCORER_H_ diff --git a/deep_speech_2/deploy/scorer_setup.py b/deep_speech_2/deploy/scorer_setup.py index c0006e0717..3bb582724a 100644 --- a/deep_speech_2/deploy/scorer_setup.py +++ b/deep_speech_2/deploy/scorer_setup.py @@ -10,8 +10,8 @@ def compile_test(header, library): return os.system(command) == 0 -FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob( - 'util/double-conversion/*.cc') +FILES = glob.glob('kenlm/util/*.cc') + glob.glob('kenlm/lm/*.cc') + glob.glob( + 'kenlm/util/double-conversion/*.cc') FILES = [ fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) ] @@ -41,7 +41,7 @@ def compile_test(header, library): name='_swig_scorer', sources=FILES + ['scorer_wrap.cxx', 'scorer.cpp'], language='C++', - include_dirs=['.'], + include_dirs=['.', './kenlm'], libraries=LIBS, extra_compile_args=ARGS) ] From 34f98e044b6499d4ec175e1f35a55a1cc03c2437 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 6 Jul 2017 12:18:09 +0800 Subject: [PATCH 05/36] add setup and README for deployment --- deep_speech_2/deploy/README.md | 38 ++++++++++++++++++++++++++++++++++ deep_speech_2/deploy/setup.sh | 11 ++++++++++ 2 files changed, 49 insertions(+) create mode 100644 deep_speech_2/deploy/README.md create mode 100644 deep_speech_2/deploy/setup.sh diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md new file mode 100644 index 0000000000..c8dbd1c125 --- /dev/null +++ b/deep_speech_2/deploy/README.md @@ -0,0 +1,38 @@ +### Installation +The setup of the decoder for deployment depends on the source code of [kenlm](https://github.com/kpu/kenlm/), first clone it to current directory (i.e., `deep_speech_2/deploy`) + +```shell +git clone https://github.com/kpu/kenlm.git +``` + +Then run the setup + +```shell +sh setup.sh +``` + +After the installation succeeds, go back to the parent directory + +``` +cd .. +``` + +### Deployment + +For GPU deployment + +``` +CUDA_VISIBLE_DEVICES=0 python deploy.py +``` + +For CPU deployment + +``` +python deploy.py --use_gpu=False +``` + +More help for arguments + +``` +python deploy.py --help +``` diff --git a/deep_speech_2/deploy/setup.sh b/deep_speech_2/deploy/setup.sh new file mode 100644 index 0000000000..e84cd9235d --- /dev/null +++ b/deep_speech_2/deploy/setup.sh @@ -0,0 +1,11 @@ +echo "Run decoder setup ..." + +python decoder_setup.py install +rm -r ./build + +echo "\nRun scorer setup ..." + +python scorer_setup.py install +rm -r ./build + +echo "\nFinish the installation of decoder and scorer." From ae05535e3e2fa7aa85c17f9712151b3da04daf8d Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 10 Jul 2017 11:34:47 +0800 Subject: [PATCH 06/36] enable loading language model in multiple format --- deep_speech_2/deploy.py | 6 +++++- deep_speech_2/deploy/scorer.cpp | 18 ++++++++++++------ deep_speech_2/deploy/setup.sh | 4 ++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index d8a7e5b277..02152b499f 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -14,6 +14,7 @@ from swig_scorer import Scorer from error_rate import wer import utils +import time parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -74,7 +75,7 @@ ) parser.add_argument( "--beam_size", - default=500, + default=200, type=int, help="Width for beam search decoding. (default: %(default)d)") parser.add_argument( @@ -166,6 +167,7 @@ def infer(): ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) ## decode and print + time_begin = time.time() wer_sum, wer_counter = 0, 0 for i, probs in enumerate(probs_split): beam_result = ctc_beam_search_decoder( @@ -183,6 +185,8 @@ def infer(): wer_counter += 1 print("cur wer = %f , average wer = %f" % (wer_cur, wer_sum / wer_counter)) + time_end = time.time() + print("total time = %f" % (time_end - time_begin)) def main(): diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 1b843402bf..d438ec1bda 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -1,4 +1,5 @@ #include +#include #include "scorer.h" #include "lm/model.hh" #include "util/tokenize_piece.hh" @@ -9,11 +10,16 @@ using namespace lm::ngram; Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { this->_alpha = alpha; this->_beta = beta; - this->_language_model = new Model(lm_model_path.c_str()); + + if (access(lm_model_path.c_str(), F_OK) != 0) { + std::cout<<"Invalid language model path!"<_language_model = LoadVirtual(lm_model_path.c_str()); } Scorer::~Scorer(){ - delete (Model *)this->_language_model; + delete (lm::base::Model *)this->_language_model; } /* Strip a input sentence @@ -63,14 +69,14 @@ int Scorer::word_count(std::string sentence) { } double Scorer::language_model_score(std::string sentence) { - Model *model = (Model *)this->_language_model; + lm::base::Model *model = (lm::base::Model *)this->_language_model; State state, out_state; lm::FullScoreReturn ret; - state = model->BeginSentenceState(); + model->BeginSentenceWrite(&state); for (util::TokenIter it(sentence, ' '); it; ++it){ - lm::WordIndex vocab = model->GetVocabulary().Index(*it); - ret = model->FullScore(state, vocab, out_state); + lm::WordIndex wid = model->BaseVocabulary().Index(*it); + ret = model->BaseFullScore(&state, wid, &out_state); state = out_state; } //log10 prob diff --git a/deep_speech_2/deploy/setup.sh b/deep_speech_2/deploy/setup.sh index e84cd9235d..423f5b8922 100644 --- a/deep_speech_2/deploy/setup.sh +++ b/deep_speech_2/deploy/setup.sh @@ -3,9 +3,9 @@ echo "Run decoder setup ..." python decoder_setup.py install rm -r ./build -echo "\nRun scorer setup ..." +echo "Run scorer setup ..." python scorer_setup.py install rm -r ./build -echo "\nFinish the installation of decoder and scorer." +echo "Finish the installation of decoder and scorer." From 4e5b345de8cc0bafbe27804f2d7d9d4294f7cfb6 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 27 Jul 2017 10:02:54 +0800 Subject: [PATCH 07/36] change probs' computation into log scale & add best path decoder --- deep_speech_2/deploy/__init__.py | 0 .../deploy/ctc_beam_search_decoder.cpp | 189 ++++++++++++++---- .../deploy/ctc_beam_search_decoder.h | 4 + deep_speech_2/deploy/scorer.cpp | 9 +- deep_speech_2/deploy/scorer.h | 2 +- deep_speech_2/deploy/swig_decoder.py | 22 ++ 6 files changed, 180 insertions(+), 46 deletions(-) create mode 100644 deep_speech_2/deploy/__init__.py create mode 100644 deep_speech_2/deploy/swig_decoder.py diff --git a/deep_speech_2/deploy/__init__.py b/deep_speech_2/deploy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp index a684b30a61..af6414a97c 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.cpp @@ -3,8 +3,11 @@ #include #include #include +#include #include "ctc_beam_search_decoder.h" +typedef float log_prob_type; + template bool pair_comp_first_rev(const std::pair a, const std::pair b) { @@ -17,6 +20,65 @@ bool pair_comp_second_rev(const std::pair a, const std::pair b) return a.second > b.second; } +template +T log_sum_exp(T x, T y) +{ + static T num_min = -std::numeric_limits::max(); + if (x <= -num_min) return y; + if (y <= -num_min) return x; + T xmax = std::max(x, y); + return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; +} + +std::string ctc_best_path_decoder(std::vector > probs_seq, + std::vector vocabulary) { + // dimension check + int num_time_steps = probs_seq.size(); + for (int i=0; i max_idx_vec; + double max_prob = 0.0; + int max_idx = 0; + for (int i=0; i idx_vec; + for (int i=0; i0) && max_idx_vec[i]!=max_idx_vec[i-1])) { + std::cout< > ctc_beam_search_decoder(std::vector > probs_seq, int beam_size, @@ -52,106 +114,147 @@ std::vector > // initialize // two sets containing selected and candidate prefixes respectively - std::map prefix_set_prev, prefix_set_next; + std::map prefix_set_prev, prefix_set_next; // probability of prefixes ending with blank and non-blank - std::map probs_b_prev, probs_nb_prev; - std::map probs_b_cur, probs_nb_cur; - prefix_set_prev["\t"] = 1.0; - probs_b_prev["\t"] = 1.0; - probs_nb_prev["\t"] = 0.0; + std::map log_probs_b_prev, log_probs_nb_prev; + std::map log_probs_b_cur, log_probs_nb_cur; + + static log_prob_type NUM_MAX = std::numeric_limits::max(); + prefix_set_prev["\t"] = 0.0; + log_probs_b_prev["\t"] = 0.0; + log_probs_nb_prev["\t"] = -NUM_MAX; for (int time_step=0; time_step prob = probs_seq[time_step]; std::vector > prob_idx; for (int i=0; i(i, prob[i])); } + // pruning of vacobulary + int cutoff_len = prob.size(); if (cutoff_prob < 1.0) { - std::sort(prob_idx.begin(), prob_idx.end(), + std::sort(prob_idx.begin(), + prob_idx.end(), pair_comp_second_rev); - float cum_prob = 0.0; - int cutoff_len = 0; + double cum_prob = 0.0; + cutoff_len = 0; for (int i=0; i= cutoff_prob) break; } prob_idx = std::vector >( prob_idx.begin(), - prob_idx.begin() + cutoff_len); + prob_idx.begin() + cutoff_len); } + + std::vector > log_prob_idx; + for (int i=0; i + (prob_idx[i].first, log(prob_idx[i].second))); + } + // extend prefix - for (std::map::iterator it = prefix_set_prev.begin(); + for (std::map::iterator + it = prefix_set_prev.begin(); it != prefix_set_prev.end(); it++) { std::string l = it->first; if( prefix_set_next.find(l) == prefix_set_next.end()) { - probs_b_cur[l] = probs_nb_cur[l] = 0.0; + log_probs_b_cur[l] = log_probs_nb_cur[l] = -NUM_MAX; } - for (int index=0; index 1) { - score = ext_scorer->get_score(l.substr(1)); + score = ext_scorer->get_score(l.substr(1), true); } - probs_nb_cur[l_plus] += score * prob_c * ( - probs_b_prev[l] + probs_nb_prev[l]); + log_probs_prev = log_sum_exp(log_probs_b_prev[l], + log_probs_nb_prev[l]); + log_probs_nb_cur[l_plus] = log_sum_exp( + log_probs_nb_cur[l_plus], + score + log_prob_c + log_probs_prev + ); } else { - probs_nb_cur[l_plus] += prob_c * ( - probs_b_prev[l] + probs_nb_prev[l]); + log_probs_prev = log_sum_exp(log_probs_b_prev[l], + log_probs_nb_prev[l]); + log_probs_nb_cur[l_plus] = log_sum_exp( + log_probs_nb_cur[l_plus], + log_prob_c+log_probs_prev + ); } - prefix_set_next[l_plus] = probs_nb_cur[l_plus] + probs_b_cur[l_plus]; + prefix_set_next[l_plus] = log_sum_exp( + log_probs_nb_cur[l_plus], + log_probs_b_cur[l_plus] + ); } } - prefix_set_next[l] = probs_b_cur[l] + probs_nb_cur[l]; + prefix_set_next[l] = log_sum_exp(log_probs_b_cur[l], + log_probs_nb_cur[l]); } - probs_b_prev = probs_b_cur; - probs_nb_prev = probs_nb_cur; - std::vector > + log_probs_b_prev = log_probs_b_cur; + log_probs_nb_prev = log_probs_nb_cur; + std::vector > prefix_vec_next(prefix_set_next.begin(), prefix_set_next.end()); std::sort(prefix_vec_next.begin(), prefix_vec_next.end(), - pair_comp_second_rev); - int k = beam_size - (prefix_vec_next.begin(), prefix_vec_next.begin()+k); + pair_comp_second_rev); + int num_prefixes_next = prefix_vec_next.size(); + int k = beam_size ( + prefix_vec_next.begin(), + prefix_vec_next.begin() + k + ); } // post processing std::vector > beam_result; - for (std::map::iterator it = prefix_set_prev.begin(); - it != prefix_set_prev.end(); it++) { - if (it->second > 0.0 && it->first.size() > 1) { - double prob = it->second; + for (std::map::iterator + it = prefix_set_prev.begin(); it != prefix_set_prev.end(); it++) { + if (it->second > -NUM_MAX && it->first.size() > 1) { + log_prob_type log_prob = it->second; std::string sentence = it->first.substr(1); // scoring the last word if (ext_scorer != NULL && sentence[sentence.size()-1] != ' ') { - prob = prob * ext_scorer->get_score(sentence); + log_prob = log_prob + ext_scorer->get_score(sentence, true); + } + if (log_prob > -NUM_MAX) { + std::pair cur_result(log_prob, sentence); + beam_result.push_back(cur_result); } - double log_prob = log(prob); - beam_result.push_back(std::pair(log_prob, sentence)); } } // sort the result and return diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.h b/deep_speech_2/deploy/ctc_beam_search_decoder.h index a4bb6aa741..de7e7791d7 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.h +++ b/deep_speech_2/deploy/ctc_beam_search_decoder.h @@ -31,5 +31,9 @@ std::vector > Scorer *ext_scorer=NULL, bool nproc=false ); +/* CTC Best Path Decoder + */ +std::string ctc_best_path_decoder(std::vector > probs_seq, + std::vector vocabulary); #endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index d438ec1bda..e9a74b989a 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -89,10 +89,15 @@ void Scorer::reset_params(float alpha, float beta) { this->_beta = beta; } -double Scorer::get_score(std::string sentence) { +double Scorer::get_score(std::string sentence, bool log) { double lm_score = language_model_score(sentence); int word_cnt = word_count(sentence); - double final_score = pow(10, _alpha*lm_score) * pow(word_cnt, _beta); + double final_score = 0.0; + if (log == false) { + final_score = pow(10, _alpha*lm_score) * pow(word_cnt, _beta); + } else { + final_score = _alpha*lm_score*std::log(10) + _beta*std::log(word_cnt); + } return final_score; } diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 7b305772c5..a18e119bcf 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -30,7 +30,7 @@ class Scorer{ // reset params alpha & beta void reset_params(float alpha, float beta); // get the final score - double get_score(std::string); + double get_score(std::string, bool log=false); }; #endif //SCORER_H_ diff --git a/deep_speech_2/deploy/swig_decoder.py b/deep_speech_2/deploy/swig_decoder.py new file mode 100644 index 0000000000..fed23c9efc --- /dev/null +++ b/deep_speech_2/deploy/swig_decoder.py @@ -0,0 +1,22 @@ +"""Contains various CTC decoders in SWIG.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from swig_ctc_beam_search_decoder import ctc_beam_search_decoder as beam_search_decoder +from swig_ctc_beam_search_decoder import ctc_best_path_decoder as best_path__decoder + + +def ctc_best_path_decoder(probs_seq, vocabulary): + best_path__decoder(probs_seq.to_list(), vocabulary) + + +def ctc_beam_search_decoder( + probs_seq, + beam_size, + vocabulary, + blank_id, + cutoff_prob=1.0, + ext_scoring_func=None, ): + beam_search_decoder(probs_seq.to_list(), beam_size, vocabulary, blank_id, + cutoff_prob, ext_scoring_func) From 908932f3ded2a3b8bf9bb4d7bd44f0986785767d Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 3 Aug 2017 11:58:09 +0800 Subject: [PATCH 08/36] refine the interface of decoders in swig --- deep_speech_2/deploy.py | 20 ++--- ...am_search_decoder.cpp => ctc_decoders.cpp} | 24 +++--- ...c_beam_search_decoder.h => ctc_decoders.h} | 11 ++- ...c_beam_search_decoder.i => ctc_decoders.i} | 6 +- deep_speech_2/deploy/decoder_setup.py | 16 ++-- deep_speech_2/deploy/scorer.cpp | 12 +-- deep_speech_2/deploy/scorer.h | 10 +-- deep_speech_2/deploy/swig_decoders.py | 86 +++++++++++++++++++ 8 files changed, 137 insertions(+), 48 deletions(-) rename deep_speech_2/deploy/{ctc_beam_search_decoder.cpp => ctc_decoders.cpp} (94%) rename deep_speech_2/deploy/{ctc_beam_search_decoder.h => ctc_decoders.h} (79%) rename deep_speech_2/deploy/{ctc_beam_search_decoder.i => ctc_decoders.i} (84%) create mode 100644 deep_speech_2/deploy/swig_decoders.py diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 02152b499f..70a9b9efee 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -10,8 +10,8 @@ import paddle.v2 as paddle from data_utils.data import DataGenerator from model import deep_speech2 -from swig_ctc_beam_search_decoder import * -from swig_scorer import Scorer +from deploy.swig_decoders import * +from swig_scorer import LmScorer from error_rate import wer import utils import time @@ -85,7 +85,7 @@ help="Number of output per sample in beam search. (default: %(default)d)") parser.add_argument( "--language_model_path", - default="lm/data/en.00.UNKNOWN.klm", + default="lm/data/common_crawl_00.prune01111.trie.klm", type=str, help="Path for language model. (default: %(default)s)") parser.add_argument( @@ -164,19 +164,19 @@ def infer(): ] # external scorer - ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) + ext_scorer = LmScorer(args.alpha, args.beta, args.language_model_path) ## decode and print time_begin = time.time() wer_sum, wer_counter = 0, 0 for i, probs in enumerate(probs_split): beam_result = ctc_beam_search_decoder( - probs.tolist(), - args.beam_size, - data_generator.vocab_list, - len(data_generator.vocab_list), - args.cutoff_prob, - ext_scorer, ) + probs_seq=probs, + beam_size=args.beam_size, + vocabulary=data_generator.vocab_list, + blank_id=len(data_generator.vocab_list), + cutoff_prob=args.cutoff_prob, + ext_scoring_func=ext_scorer, ) print("\nTarget Transcription:\t%s" % target_transcription[i]) print("Beam %d: %f \t%s" % (0, beam_result[0][0], beam_result[0][1])) diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp b/deep_speech_2/deploy/ctc_decoders.cpp similarity index 94% rename from deep_speech_2/deploy/ctc_beam_search_decoder.cpp rename to deep_speech_2/deploy/ctc_decoders.cpp index af6414a97c..4cff6d5e54 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -4,9 +4,9 @@ #include #include #include -#include "ctc_beam_search_decoder.h" +#include "ctc_decoders.h" -typedef float log_prob_type; +typedef double log_prob_type; template bool pair_comp_first_rev(const std::pair a, const std::pair b) @@ -24,8 +24,8 @@ template T log_sum_exp(T x, T y) { static T num_min = -std::numeric_limits::max(); - if (x <= -num_min) return y; - if (y <= -num_min) return x; + if (x <= num_min) return y; + if (y <= num_min) return x; T xmax = std::max(x, y); return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; } @@ -55,17 +55,13 @@ std::string ctc_best_path_decoder(std::vector > probs_seq, } } max_idx_vec.push_back(max_idx); - std::cout< idx_vec; for (int i=0; i0) && max_idx_vec[i]!=max_idx_vec[i-1])) { - std::cout< > probs_seq, std::string best_path_result; for (int i=0; i > std::vector vocabulary, int blank_id, double cutoff_prob, - Scorer *ext_scorer, + LmScorer *ext_scorer, bool nproc) { // dimension check int num_time_steps = probs_seq.size(); for (int i=0; i vocabulary.size()) { - std::cout<<"Invalid blank_id!"< > vocabulary.end(), " "); int space_id = it - vocabulary.begin(); if(space_id >= vocabulary.size()) { - std::cout<<"The character space is not in the vocabulary!"< > std::vector vocabulary, int blank_id, double cutoff_prob=1.0, - Scorer *ext_scorer=NULL, + LmScorer *ext_scorer=NULL, bool nproc=false ); + /* CTC Best Path Decoder + * + * Parameters: + * probs_seq: 2-D vector that each element is a vector of probabilities + * over vocabulary of one time step. + * vocabulary: A vector of vocabulary. + * Return: + * A vector that each element is a pair of score and decoding result, + * in desending order. */ std::string ctc_best_path_decoder(std::vector > probs_seq, std::vector vocabulary); diff --git a/deep_speech_2/deploy/ctc_beam_search_decoder.i b/deep_speech_2/deploy/ctc_decoders.i similarity index 84% rename from deep_speech_2/deploy/ctc_beam_search_decoder.i rename to deep_speech_2/deploy/ctc_decoders.i index 09e893d38e..c7d05238e5 100644 --- a/deep_speech_2/deploy/ctc_beam_search_decoder.i +++ b/deep_speech_2/deploy/ctc_decoders.i @@ -1,6 +1,6 @@ -%module swig_ctc_beam_search_decoder +%module swig_ctc_decoders %{ -#include "ctc_beam_search_decoder.h" +#include "ctc_decoders.h" %} %include "std_vector.i" @@ -19,4 +19,4 @@ namespace std{ } %import scorer.h -%include "ctc_beam_search_decoder.h" +%include "ctc_decoders.h" diff --git a/deep_speech_2/deploy/decoder_setup.py b/deep_speech_2/deploy/decoder_setup.py index 4ed603b252..aed45faafc 100644 --- a/deep_speech_2/deploy/decoder_setup.py +++ b/deep_speech_2/deploy/decoder_setup.py @@ -34,15 +34,13 @@ def compile_test(header, library): ARGS.append('-DHAVE_XZLIB') LIBS.append('lzma') -os.system('swig -python -c++ ./ctc_beam_search_decoder.i') +os.system('swig -python -c++ ./ctc_decoders.i') ctc_beam_search_decoder_module = [ Extension( - name='_swig_ctc_beam_search_decoder', - sources=FILES + [ - 'scorer.cpp', 'ctc_beam_search_decoder_wrap.cxx', - 'ctc_beam_search_decoder.cpp' - ], + name='_swig_ctc_decoders', + sources=FILES + + ['scorer.cpp', 'ctc_decoders_wrap.cxx', 'ctc_decoders.cpp'], language='C++', include_dirs=['.', './kenlm'], libraries=LIBS, @@ -50,8 +48,8 @@ def compile_test(header, library): ] setup( - name='swig_ctc_beam_search_decoder', + name='swig_ctc_decoders', version='0.1', - description="""CTC beam search decoder""", + description="""CTC decoders""", ext_modules=ctc_beam_search_decoder_module, - py_modules=['swig_ctc_beam_search_decoder'], ) + py_modules=['swig_ctc_decoders'], ) diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index e9a74b989a..7a66daad9c 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -7,7 +7,7 @@ using namespace lm::ngram; -Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { +LmScorer::LmScorer(float alpha, float beta, std::string lm_model_path) { this->_alpha = alpha; this->_beta = beta; @@ -18,7 +18,7 @@ Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { this->_language_model = LoadVirtual(lm_model_path.c_str()); } -Scorer::~Scorer(){ +LmScorer::~LmScorer(){ delete (lm::base::Model *)this->_language_model; } @@ -57,7 +57,7 @@ inline void strip(std::string &str, char ch=' ') { } } -int Scorer::word_count(std::string sentence) { +int LmScorer::word_count(std::string sentence) { strip(sentence); int cnt = 1; for (int i=0; i_language_model; State state, out_state; lm::FullScoreReturn ret; @@ -84,12 +84,12 @@ double Scorer::language_model_score(std::string sentence) { return log_prob; } -void Scorer::reset_params(float alpha, float beta) { +void LmScorer::reset_params(float alpha, float beta) { this->_alpha = alpha; this->_beta = beta; } -double Scorer::get_score(std::string sentence, bool log) { +double LmScorer::get_score(std::string sentence, bool log) { double lm_score = language_model_score(sentence); int word_cnt = word_count(sentence); diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index a18e119bcf..90a1a84a0a 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -8,10 +8,10 @@ * count and language model scoring. * Example: - * Scorer ext_scorer(alpha, beta, "path_to_language_model.klm"); + * LmScorer ext_scorer(alpha, beta, "path_to_language_model.klm"); * double score = ext_scorer.get_score("sentence_to_score"); */ -class Scorer{ +class LmScorer{ private: float _alpha; float _beta; @@ -23,9 +23,9 @@ class Scorer{ double language_model_score(std::string); public: - Scorer(){} - Scorer(float alpha, float beta, std::string lm_model_path); - ~Scorer(); + LmScorer(){} + LmScorer(float alpha, float beta, std::string lm_model_path); + ~LmScorer(); // reset params alpha & beta void reset_params(float alpha, float beta); diff --git a/deep_speech_2/deploy/swig_decoders.py b/deep_speech_2/deploy/swig_decoders.py new file mode 100644 index 0000000000..8e4a39252b --- /dev/null +++ b/deep_speech_2/deploy/swig_decoders.py @@ -0,0 +1,86 @@ +"""Wrapper for various CTC decoders in SWIG.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import swig_ctc_decoders +import multiprocessing + + +def ctc_best_path_decoder(probs_seq, vocabulary): + """Wrapper for ctc best path decoder in swig. + + :param probs_seq: 2-D list of probability distributions over each time + step, with each element being a list of normalized + probabilities over vocabulary and blank. + :type probs_seq: 2-D list + :param vocabulary: Vocabulary list. + :type vocabulary: list + :return: Decoding result string. + :rtype: basestring + """ + return swig_ctc_decoders.ctc_best_path_decoder(probs_seq.tolist(), + vocabulary) + + +def ctc_beam_search_decoder( + probs_seq, + beam_size, + vocabulary, + blank_id, + cutoff_prob=1.0, + ext_scoring_func=None, ): + """Wrapper for CTC Beam Search Decoder. + + :param probs_seq: 2-D list of probability distributions over each time + step, with each element being a list of normalized + probabilities over vocabulary and blank. + :type probs_seq: 2-D list + :param beam_size: Width for beam search. + :type beam_size: int + :param vocabulary: Vocabulary list. + :type vocabulary: list + :param blank_id: ID of blank. + :type blank_id: int + :param cutoff_prob: Cutoff probability in pruning, + default 1.0, no pruning. + :type cutoff_prob: float + :param ext_scoring_func: External scoring function for + partially decoded sentence, e.g. word count + or language model. + :type external_scoring_func: callable + :return: List of tuples of log probability and sentence as decoding + results, in descending order of the probability. + :rtype: list + """ + return swig_ctc_decoders.ctc_beam_search_decoder( + probs_seq.tolist(), beam_size, vocabulary, blank_id, cutoff_prob, + ext_scoring_func) + + +def ctc_beam_search_decoder_batch(probs_split, + beam_size, + vocabulary, + blank_id, + num_processes, + cutoff_prob=1.0, + ext_scoring_func=None): + """Wrapper for CTC beam search decoder in batch + """ + + # TODO: to resolve PicklingError + + if not num_processes > 0: + raise ValueError("Number of processes must be positive!") + + pool = multiprocessing.Pool(processes=num_processes) + results = [] + for i, probs_list in enumerate(probs_split): + args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, + ext_scoring_func) + results.append(pool.apply_async(ctc_beam_search_decoder, args)) + + pool.close() + pool.join() + beam_search_results = [result.get() for result in results] + return beam_search_results From ac3a49c0ba1cdc2ecfbabcf2b3a32facc0e8841e Mon Sep 17 00:00:00 2001 From: Yibing Liu <352748861@qq.com> Date: Thu, 3 Aug 2017 14:46:18 +0800 Subject: [PATCH 09/36] Delete swig_decoder.py --- deep_speech_2/deploy/swig_decoder.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 deep_speech_2/deploy/swig_decoder.py diff --git a/deep_speech_2/deploy/swig_decoder.py b/deep_speech_2/deploy/swig_decoder.py deleted file mode 100644 index fed23c9efc..0000000000 --- a/deep_speech_2/deploy/swig_decoder.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Contains various CTC decoders in SWIG.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from swig_ctc_beam_search_decoder import ctc_beam_search_decoder as beam_search_decoder -from swig_ctc_beam_search_decoder import ctc_best_path_decoder as best_path__decoder - - -def ctc_best_path_decoder(probs_seq, vocabulary): - best_path__decoder(probs_seq.to_list(), vocabulary) - - -def ctc_beam_search_decoder( - probs_seq, - beam_size, - vocabulary, - blank_id, - cutoff_prob=1.0, - ext_scoring_func=None, ): - beam_search_decoder(probs_seq.to_list(), beam_size, vocabulary, blank_id, - cutoff_prob, ext_scoring_func) From 9ff48b0567e41977ae4f64c22724bb47af142758 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 22 Aug 2017 16:19:57 +0800 Subject: [PATCH 10/36] reorganize cpp files --- deep_speech_2/deploy.py | 6 +++--- deep_speech_2/deploy/ctc_decoders.cpp | 4 +++- deep_speech_2/deploy/ctc_decoders.h | 2 +- deep_speech_2/deploy/ctc_decoders.i | 1 + deep_speech_2/deploy/decoder_setup.py | 6 ++++-- deep_speech_2/deploy/decoder_utils.cpp | 5 +++++ deep_speech_2/deploy/decoder_utils.h | 15 ++++++++++++++ deep_speech_2/deploy/scorer.cpp | 12 +++++------ deep_speech_2/deploy/scorer.h | 10 ++++----- deep_speech_2/deploy/swig_decoders.py | 28 ++++++++++++++++++++++++-- 10 files changed, 69 insertions(+), 20 deletions(-) create mode 100644 deep_speech_2/deploy/decoder_utils.cpp create mode 100644 deep_speech_2/deploy/decoder_utils.h diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 70a9b9efee..091d82892b 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -11,7 +11,7 @@ from data_utils.data import DataGenerator from model import deep_speech2 from deploy.swig_decoders import * -from swig_scorer import LmScorer +from swig_scorer import Scorer from error_rate import wer import utils import time @@ -19,7 +19,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num_samples", - default=100, + default=10, type=int, help="Number of samples for inference. (default: %(default)s)") parser.add_argument( @@ -164,7 +164,7 @@ def infer(): ] # external scorer - ext_scorer = LmScorer(args.alpha, args.beta, args.language_model_path) + ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) ## decode and print time_begin = time.time() diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 4cff6d5e54..75555c018b 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -5,9 +5,11 @@ #include #include #include "ctc_decoders.h" +#include "decoder_utils.h" typedef double log_prob_type; + template bool pair_comp_first_rev(const std::pair a, const std::pair b) { @@ -81,7 +83,7 @@ std::vector > std::vector vocabulary, int blank_id, double cutoff_prob, - LmScorer *ext_scorer, + Scorer *ext_scorer, bool nproc) { // dimension check int num_time_steps = probs_seq.size(); diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/deploy/ctc_decoders.h index da08a2c58a..50a6014f0a 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/deploy/ctc_decoders.h @@ -28,7 +28,7 @@ std::vector > std::vector vocabulary, int blank_id, double cutoff_prob=1.0, - LmScorer *ext_scorer=NULL, + Scorer *ext_scorer=NULL, bool nproc=false ); diff --git a/deep_speech_2/deploy/ctc_decoders.i b/deep_speech_2/deploy/ctc_decoders.i index c7d05238e5..8c9dd1643d 100644 --- a/deep_speech_2/deploy/ctc_decoders.i +++ b/deep_speech_2/deploy/ctc_decoders.i @@ -19,4 +19,5 @@ namespace std{ } %import scorer.h +%import decoder_utils.h %include "ctc_decoders.h" diff --git a/deep_speech_2/deploy/decoder_setup.py b/deep_speech_2/deploy/decoder_setup.py index aed45faafc..146538f557 100644 --- a/deep_speech_2/deploy/decoder_setup.py +++ b/deep_speech_2/deploy/decoder_setup.py @@ -39,8 +39,10 @@ def compile_test(header, library): ctc_beam_search_decoder_module = [ Extension( name='_swig_ctc_decoders', - sources=FILES + - ['scorer.cpp', 'ctc_decoders_wrap.cxx', 'ctc_decoders.cpp'], + sources=FILES + [ + 'scorer.cpp', 'ctc_decoders_wrap.cxx', 'ctc_decoders.cpp', + 'decoder_utils.cpp' + ], language='C++', include_dirs=['.', './kenlm'], libraries=LIBS, diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp new file mode 100644 index 0000000000..82e4cd1467 --- /dev/null +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -0,0 +1,5 @@ +#include +#include +#include +#include "decoder_utils.h" + diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h new file mode 100644 index 0000000000..6d58bf1f30 --- /dev/null +++ b/deep_speech_2/deploy/decoder_utils.h @@ -0,0 +1,15 @@ +#ifndef DECODER_UTILS_H +#define DECODER_UTILS_H +#pragma once +#include + +/* +template +bool pair_comp_first_rev(const std::pair a, const std::pair b); + +template +bool pair_comp_second_rev(const std::pair a, const std::pair b); + +template T log_sum_exp(T x, T y); +*/ +#endif // DECODER_UTILS_H diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 7a66daad9c..e9a74b989a 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -7,7 +7,7 @@ using namespace lm::ngram; -LmScorer::LmScorer(float alpha, float beta, std::string lm_model_path) { +Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { this->_alpha = alpha; this->_beta = beta; @@ -18,7 +18,7 @@ LmScorer::LmScorer(float alpha, float beta, std::string lm_model_path) { this->_language_model = LoadVirtual(lm_model_path.c_str()); } -LmScorer::~LmScorer(){ +Scorer::~Scorer(){ delete (lm::base::Model *)this->_language_model; } @@ -57,7 +57,7 @@ inline void strip(std::string &str, char ch=' ') { } } -int LmScorer::word_count(std::string sentence) { +int Scorer::word_count(std::string sentence) { strip(sentence); int cnt = 1; for (int i=0; i_language_model; State state, out_state; lm::FullScoreReturn ret; @@ -84,12 +84,12 @@ double LmScorer::language_model_score(std::string sentence) { return log_prob; } -void LmScorer::reset_params(float alpha, float beta) { +void Scorer::reset_params(float alpha, float beta) { this->_alpha = alpha; this->_beta = beta; } -double LmScorer::get_score(std::string sentence, bool log) { +double Scorer::get_score(std::string sentence, bool log) { double lm_score = language_model_score(sentence); int word_cnt = word_count(sentence); diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 90a1a84a0a..a18e119bcf 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -8,10 +8,10 @@ * count and language model scoring. * Example: - * LmScorer ext_scorer(alpha, beta, "path_to_language_model.klm"); + * Scorer ext_scorer(alpha, beta, "path_to_language_model.klm"); * double score = ext_scorer.get_score("sentence_to_score"); */ -class LmScorer{ +class Scorer{ private: float _alpha; float _beta; @@ -23,9 +23,9 @@ class LmScorer{ double language_model_score(std::string); public: - LmScorer(){} - LmScorer(float alpha, float beta, std::string lm_model_path); - ~LmScorer(); + Scorer(){} + Scorer(float alpha, float beta, std::string lm_model_path); + ~Scorer(); // reset params alpha & beta void reset_params(float alpha, float beta); diff --git a/deep_speech_2/deploy/swig_decoders.py b/deep_speech_2/deploy/swig_decoders.py index 8e4a39252b..0247c0c9ea 100644 --- a/deep_speech_2/deploy/swig_decoders.py +++ b/deep_speech_2/deploy/swig_decoders.py @@ -4,7 +4,8 @@ from __future__ import print_function import swig_ctc_decoders -import multiprocessing +#import multiprocessing +from pathos.multiprocessing import Pool def ctc_best_path_decoder(probs_seq, vocabulary): @@ -73,14 +74,37 @@ def ctc_beam_search_decoder_batch(probs_split, if not num_processes > 0: raise ValueError("Number of processes must be positive!") - pool = multiprocessing.Pool(processes=num_processes) + pool = Pool(processes=num_processes) results = [] + args_list = [] for i, probs_list in enumerate(probs_split): args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, ext_scoring_func) + args_list.append(args) results.append(pool.apply_async(ctc_beam_search_decoder, args)) pool.close() pool.join() beam_search_results = [result.get() for result in results] + """ + len_args = len(probs_split) + beam_search_results = pool.map(ctc_beam_search_decoder, + probs_split, + [beam_size for i in xrange(len_args)], + [vocabulary for i in xrange(len_args)], + [blank_id for i in xrange(len_args)], + [cutoff_prob for i in xrange(len_args)], + [ext_scoring_func for i in xrange(len_args)] + ) + """ + ''' + processes = [mp.Process(target=ctc_beam_search_decoder, + args=(probs_list, beam_size, vocabulary, blank_id, cutoff_prob, + ext_scoring_func) for probs_list in probs_split] + for p in processes: + p.start() + for p in processes: + p.join() + beam_search_results = [] + ''' return beam_search_results From 32047c72cfcbfa13080f2d56297217d5eb5414a2 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 22 Aug 2017 18:52:49 +0800 Subject: [PATCH 11/36] refine wrapper for swig and simplify setup --- deep_speech_2/deploy.py | 6 +-- deep_speech_2/deploy/README.md | 11 ++-- .../deploy/{ctc_decoders.i => decoders.i} | 5 +- deep_speech_2/deploy/scorer.i | 8 --- deep_speech_2/deploy/scorer_setup.py | 54 ------------------- .../deploy/{decoder_setup.py => setup.py} | 17 +++--- deep_speech_2/deploy/setup.sh | 11 ---- ...g_decoders.py => swig_decoders_wrapper.py} | 52 ++++++++---------- 8 files changed, 40 insertions(+), 124 deletions(-) rename deep_speech_2/deploy/{ctc_decoders.i => decoders.i} (91%) delete mode 100644 deep_speech_2/deploy/scorer.i delete mode 100644 deep_speech_2/deploy/scorer_setup.py rename deep_speech_2/deploy/{decoder_setup.py => setup.py} (75%) delete mode 100644 deep_speech_2/deploy/setup.sh rename deep_speech_2/deploy/{swig_decoders.py => swig_decoders_wrapper.py} (68%) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 091d82892b..2d29973fbb 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -10,8 +10,7 @@ import paddle.v2 as paddle from data_utils.data import DataGenerator from model import deep_speech2 -from deploy.swig_decoders import * -from swig_scorer import Scorer +from deploy.swig_decoders_wrapper import * from error_rate import wer import utils import time @@ -164,7 +163,8 @@ def infer(): ] # external scorer - ext_scorer = Scorer(args.alpha, args.beta, args.language_model_path) + ext_scorer = Scorer( + alpha=args.alpha, beta=args.beta, model_path=args.language_model_path) ## decode and print time_begin = time.time() diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index c8dbd1c125..cf0c04391c 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -1,19 +1,16 @@ ### Installation -The setup of the decoder for deployment depends on the source code of [kenlm](https://github.com/kpu/kenlm/), first clone it to current directory (i.e., `deep_speech_2/deploy`) +The setup of the decoder for deployment depends on the source code of [kenlm](https://github.com/kpu/kenlm/) and [openfst](http://www.openfst.org/twiki/bin/view/FST/WebHome), first clone kenlm and download openfst to current directory (i.e., `deep_speech_2/deploy`) ```shell git clone https://github.com/kpu/kenlm.git +wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz +tar -xzvf openfst-1.6.3.tar.gz ``` Then run the setup ```shell -sh setup.sh -``` - -After the installation succeeds, go back to the parent directory - -``` +python setup.py install cd .. ``` diff --git a/deep_speech_2/deploy/ctc_decoders.i b/deep_speech_2/deploy/decoders.i similarity index 91% rename from deep_speech_2/deploy/ctc_decoders.i rename to deep_speech_2/deploy/decoders.i index 8c9dd1643d..04736e09e8 100644 --- a/deep_speech_2/deploy/ctc_decoders.i +++ b/deep_speech_2/deploy/decoders.i @@ -1,5 +1,6 @@ -%module swig_ctc_decoders +%module swig_decoders %{ +#include "scorer.h" #include "ctc_decoders.h" %} @@ -18,6 +19,6 @@ namespace std{ %template(PairDoubleStringVector) std::vector >; } -%import scorer.h %import decoder_utils.h +%include "scorer.h" %include "ctc_decoders.h" diff --git a/deep_speech_2/deploy/scorer.i b/deep_speech_2/deploy/scorer.i deleted file mode 100644 index 8380e15a60..0000000000 --- a/deep_speech_2/deploy/scorer.i +++ /dev/null @@ -1,8 +0,0 @@ -%module swig_scorer -%{ -#include "scorer.h" -%} - -%include "std_string.i" - -%include "scorer.h" diff --git a/deep_speech_2/deploy/scorer_setup.py b/deep_speech_2/deploy/scorer_setup.py deleted file mode 100644 index 3bb582724a..0000000000 --- a/deep_speech_2/deploy/scorer_setup.py +++ /dev/null @@ -1,54 +0,0 @@ -from setuptools import setup, Extension -import glob -import platform -import os - - -def compile_test(header, library): - dummy_path = os.path.join(os.path.dirname(__file__), "dummy") - command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\"" - return os.system(command) == 0 - - -FILES = glob.glob('kenlm/util/*.cc') + glob.glob('kenlm/lm/*.cc') + glob.glob( - 'kenlm/util/double-conversion/*.cc') -FILES = [ - fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) -] - -LIBS = ['stdc++'] -if platform.system() != 'Darwin': - LIBS.append('rt') - -ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6'] - -if compile_test('zlib.h', 'z'): - ARGS.append('-DHAVE_ZLIB') - LIBS.append('z') - -if compile_test('bzlib.h', 'bz2'): - ARGS.append('-DHAVE_BZLIB') - LIBS.append('bz2') - -if compile_test('lzma.h', 'lzma'): - ARGS.append('-DHAVE_XZLIB') - LIBS.append('lzma') - -os.system('swig -python -c++ ./scorer.i') - -ext_modules = [ - Extension( - name='_swig_scorer', - sources=FILES + ['scorer_wrap.cxx', 'scorer.cpp'], - language='C++', - include_dirs=['.', './kenlm'], - libraries=LIBS, - extra_compile_args=ARGS) -] - -setup( - name='swig_scorer', - version='0.1', - ext_modules=ext_modules, - include_package_data=True, - py_modules=['swig_scorer'], ) diff --git a/deep_speech_2/deploy/decoder_setup.py b/deep_speech_2/deploy/setup.py similarity index 75% rename from deep_speech_2/deploy/decoder_setup.py rename to deep_speech_2/deploy/setup.py index 146538f557..077cabd086 100644 --- a/deep_speech_2/deploy/decoder_setup.py +++ b/deep_speech_2/deploy/setup.py @@ -20,7 +20,7 @@ def compile_test(header, library): if platform.system() != 'Darwin': LIBS.append('rt') -ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6'] +ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6', '-std=c++11'] if compile_test('zlib.h', 'z'): ARGS.append('-DHAVE_ZLIB') @@ -34,24 +34,21 @@ def compile_test(header, library): ARGS.append('-DHAVE_XZLIB') LIBS.append('lzma') -os.system('swig -python -c++ ./ctc_decoders.i') +os.system('swig -python -c++ ./decoders.i') ctc_beam_search_decoder_module = [ Extension( - name='_swig_ctc_decoders', - sources=FILES + [ - 'scorer.cpp', 'ctc_decoders_wrap.cxx', 'ctc_decoders.cpp', - 'decoder_utils.cpp' - ], + name='_swig_decoders', + sources=FILES + glob.glob('*.cxx') + glob.glob('*.cpp'), language='C++', - include_dirs=['.', './kenlm'], + include_dirs=['.', './kenlm', './openfst-1.6.3/src/include'], libraries=LIBS, extra_compile_args=ARGS) ] setup( - name='swig_ctc_decoders', + name='swig_decoders', version='0.1', description="""CTC decoders""", ext_modules=ctc_beam_search_decoder_module, - py_modules=['swig_ctc_decoders'], ) + py_modules=['swig_decoders'], ) diff --git a/deep_speech_2/deploy/setup.sh b/deep_speech_2/deploy/setup.sh deleted file mode 100644 index 423f5b8922..0000000000 --- a/deep_speech_2/deploy/setup.sh +++ /dev/null @@ -1,11 +0,0 @@ -echo "Run decoder setup ..." - -python decoder_setup.py install -rm -r ./build - -echo "Run scorer setup ..." - -python scorer_setup.py install -rm -r ./build - -echo "Finish the installation of decoder and scorer." diff --git a/deep_speech_2/deploy/swig_decoders.py b/deep_speech_2/deploy/swig_decoders_wrapper.py similarity index 68% rename from deep_speech_2/deploy/swig_decoders.py rename to deep_speech_2/deploy/swig_decoders_wrapper.py index 0247c0c9ea..54c4301475 100644 --- a/deep_speech_2/deploy/swig_decoders.py +++ b/deep_speech_2/deploy/swig_decoders_wrapper.py @@ -3,9 +3,25 @@ from __future__ import division from __future__ import print_function -import swig_ctc_decoders -#import multiprocessing -from pathos.multiprocessing import Pool +import swig_decoders +import multiprocessing + + +class Scorer(swig_decoders.Scorer): + """Wrapper for Scorer. + + :param alpha: Parameter associated with language model. Don't use + language model when alpha = 0. + :type alpha: float + :param beta: Parameter associated with word count. Don't use word + count when beta = 0. + :type beta: float + :model_path: Path to load language model. + :type model_path: basestring + """ + + def __init__(self, alpha, beta, model_path): + swig_decoders.Scorer.__init__(self, alpha, beta, model_path) def ctc_best_path_decoder(probs_seq, vocabulary): @@ -20,8 +36,7 @@ def ctc_best_path_decoder(probs_seq, vocabulary): :return: Decoding result string. :rtype: basestring """ - return swig_ctc_decoders.ctc_best_path_decoder(probs_seq.tolist(), - vocabulary) + return swig_decoders.ctc_best_path_decoder(probs_seq.tolist(), vocabulary) def ctc_beam_search_decoder( @@ -54,9 +69,9 @@ def ctc_beam_search_decoder( results, in descending order of the probability. :rtype: list """ - return swig_ctc_decoders.ctc_beam_search_decoder( - probs_seq.tolist(), beam_size, vocabulary, blank_id, cutoff_prob, - ext_scoring_func) + return swig_decoders.ctc_beam_search_decoder(probs_seq.tolist(), beam_size, + vocabulary, blank_id, + cutoff_prob, ext_scoring_func) def ctc_beam_search_decoder_batch(probs_split, @@ -86,25 +101,4 @@ def ctc_beam_search_decoder_batch(probs_split, pool.close() pool.join() beam_search_results = [result.get() for result in results] - """ - len_args = len(probs_split) - beam_search_results = pool.map(ctc_beam_search_decoder, - probs_split, - [beam_size for i in xrange(len_args)], - [vocabulary for i in xrange(len_args)], - [blank_id for i in xrange(len_args)], - [cutoff_prob for i in xrange(len_args)], - [ext_scoring_func for i in xrange(len_args)] - ) - """ - ''' - processes = [mp.Process(target=ctc_beam_search_decoder, - args=(probs_list, beam_size, vocabulary, blank_id, cutoff_prob, - ext_scoring_func) for probs_list in probs_split] - for p in processes: - p.start() - for p in processes: - p.join() - beam_search_results = [] - ''' return beam_search_results From f41375b9aca8bf05775822cdd509ba2700308ad4 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 23 Aug 2017 11:03:44 +0800 Subject: [PATCH 12/36] add the support of parallel beam search decoding in deployment --- deep_speech_2/deploy.py | 31 +++++++-- deep_speech_2/deploy/README.md | 15 ++++- deep_speech_2/deploy/ctc_decoders.cpp | 44 ++++++++++++- deep_speech_2/deploy/ctc_decoders.h | 53 +++++++++++---- deep_speech_2/deploy/decoders.i | 2 + deep_speech_2/deploy/setup.py | 6 +- deep_speech_2/deploy/swig_decoders_wrapper.py | 64 +++++++++++-------- 7 files changed, 160 insertions(+), 55 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 2d29973fbb..76b6160529 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num_samples", - default=10, + default=32, type=int, help="Number of samples for inference. (default: %(default)s)") parser.add_argument( @@ -46,6 +46,11 @@ default=multiprocessing.cpu_count(), type=int, help="Number of cpu threads for preprocessing data. (default: %(default)s)") +parser.add_argument( + "--num_processes_beam_search", + default=multiprocessing.cpu_count(), + type=int, + help="Number of cpu processes for beam search. (default: %(default)s)") parser.add_argument( "--mean_std_filepath", default='mean_std.npz', @@ -70,8 +75,8 @@ "--decode_method", default='beam_search', type=str, - help="Method for ctc decoding: best_path or beam_search. (default: %(default)s)" -) + help="Method for ctc decoding: beam_search or beam_search_batch. " + "(default: %(default)s)") parser.add_argument( "--beam_size", default=200, @@ -169,15 +174,28 @@ def infer(): ## decode and print time_begin = time.time() wer_sum, wer_counter = 0, 0 - for i, probs in enumerate(probs_split): - beam_result = ctc_beam_search_decoder( - probs_seq=probs, + batch_beam_results = [] + if args.decode_method == 'beam_search': + for i, probs in enumerate(probs_split): + beam_result = ctc_beam_search_decoder( + probs_seq=probs, + beam_size=args.beam_size, + vocabulary=data_generator.vocab_list, + blank_id=len(data_generator.vocab_list), + cutoff_prob=args.cutoff_prob, + ext_scoring_func=ext_scorer, ) + batch_beam_results += [beam_result] + else: + batch_beam_results = ctc_beam_search_decoder_batch( + probs_split=probs_split, beam_size=args.beam_size, vocabulary=data_generator.vocab_list, blank_id=len(data_generator.vocab_list), + num_processes=args.num_processes_beam_search, cutoff_prob=args.cutoff_prob, ext_scoring_func=ext_scorer, ) + for i, beam_result in enumerate(batch_beam_results): print("\nTarget Transcription:\t%s" % target_transcription[i]) print("Beam %d: %f \t%s" % (0, beam_result[0][0], beam_result[0][1])) wer_cur = wer(target_transcription[i], beam_result[0][1]) @@ -185,6 +203,7 @@ def infer(): wer_counter += 1 print("cur wer = %f , average wer = %f" % (wer_cur, wer_sum / wer_counter)) + time_end = time.time() print("total time = %f" % (time_end - time_begin)) diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index cf0c04391c..98dde7a604 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -1,12 +1,25 @@ ### Installation -The setup of the decoder for deployment depends on the source code of [kenlm](https://github.com/kpu/kenlm/) and [openfst](http://www.openfst.org/twiki/bin/view/FST/WebHome), first clone kenlm and download openfst to current directory (i.e., `deep_speech_2/deploy`) +The build of the decoder for deployment depends on several open-sourced projects, first clone or download them to current directory (i.e., `deep_speech_2/deploy`) + +- [**KenLM**](https://github.com/kpu/kenlm/): Faster and Smaller Language Model Queries ```shell git clone https://github.com/kpu/kenlm.git +``` + +- [**OpenFst**](http://www.openfst.org/twiki/bin/view/FST/WebHome): A library for finite-state transducers + +```shell wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz tar -xzvf openfst-1.6.3.tar.gz ``` +- [**ThreadPool**](http://progsch.net/wordpress/): A library for C++ thread pool + +```shell +git clone https://github.com/progschj/ThreadPool.git +``` + Then run the setup ```shell diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 75555c018b..b22a45a70d 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -6,6 +6,7 @@ #include #include "ctc_decoders.h" #include "decoder_utils.h" +#include "ThreadPool.h" typedef double log_prob_type; @@ -33,7 +34,8 @@ T log_sum_exp(T x, T y) } std::string ctc_best_path_decoder(std::vector > probs_seq, - std::vector vocabulary) { + std::vector vocabulary) +{ // dimension check int num_time_steps = probs_seq.size(); for (int i=0; i > std::vector vocabulary, int blank_id, double cutoff_prob, - Scorer *ext_scorer, - bool nproc) { + Scorer *ext_scorer) +{ // dimension check int num_time_steps = probs_seq.size(); for (int i=0; i > pair_comp_first_rev); return beam_result; } + + +std::vector>> + ctc_beam_search_decoder_batch( + std::vector>> probs_split, + int beam_size, + std::vector vocabulary, + int blank_id, + int num_processes, + double cutoff_prob, + Scorer *ext_scorer + ) +{ + if (num_processes <= 0) { + std::cout << "num_processes must be nonnegative!" << std::endl; + exit(1); + } + // thread pool + ThreadPool pool(num_processes); + // number of samples + int batch_size = probs_split.size(); + // enqueue the tasks of decoding + std::vector>>> res; + for (int i = 0; i < batch_size; i++) { + res.emplace_back( + pool.enqueue(ctc_beam_search_decoder, probs_split[i], + beam_size, vocabulary, blank_id, cutoff_prob, ext_scorer) + ); + } + // get decoding results + std::vector>> batch_results; + for (int i = 0; i < batch_size; i++) { + batch_results.emplace_back(res[i].get()); + } + return batch_results; +} diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/deploy/ctc_decoders.h index 50a6014f0a..2389038205 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/deploy/ctc_decoders.h @@ -6,8 +6,20 @@ #include #include "scorer.h" -/* CTC Beam Search Decoder, the interface is consistent with the - * original decoder in Python version. +/* CTC Best Path Decoder + * + * Parameters: + * probs_seq: 2-D vector that each element is a vector of probabilities + * over vocabulary of one time step. + * vocabulary: A vector of vocabulary. + * Return: + * A vector that each element is a pair of score and decoding result, + * in desending order. + */ +std::string ctc_best_path_decoder(std::vector > probs_seq, + std::vector vocabulary); + +/* CTC Beam Search Decoder * Parameters: * probs_seq: 2-D vector that each element is a vector of probabilities @@ -17,7 +29,6 @@ * blank_id: ID of blank. * cutoff_prob: Cutoff probability of pruning * ext_scorer: External scorer to evaluate a prefix. - * nproc: Whether this function used in multiprocessing. * Return: * A vector that each element is a pair of score and decoding result, * in desending order. @@ -28,21 +39,35 @@ std::vector > std::vector vocabulary, int blank_id, double cutoff_prob=1.0, - Scorer *ext_scorer=NULL, - bool nproc=false + Scorer *ext_scorer=NULL ); -/* CTC Best Path Decoder - * +/* CTC Beam Search Decoder for batch data, the interface is consistent with the + * original decoder in Python version. + * Parameters: - * probs_seq: 2-D vector that each element is a vector of probabilities - * over vocabulary of one time step. + * probs_seq: 3-D vector that each element is a 2-D vector that can be used + * by ctc_beam_search_decoder(). + * . + * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. + * blank_id: ID of blank. + * num_processes: Number of threads for beam search. + * cutoff_prob: Cutoff probability of pruning + * ext_scorer: External scorer to evaluate a prefix. * Return: - * A vector that each element is a pair of score and decoding result, - * in desending order. - */ -std::string ctc_best_path_decoder(std::vector > probs_seq, - std::vector vocabulary); + * A 2-D vector that each element is a vector of decoding result for one + * sample. +*/ +std::vector>> + ctc_beam_search_decoder_batch(std::vector>> probs_split, + int beam_size, + std::vector vocabulary, + int blank_id, + int num_processes, + double cutoff_prob=1.0, + Scorer *ext_scorer=NULL + ); + #endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/deploy/decoders.i b/deep_speech_2/deploy/decoders.i index 04736e09e8..34da1eca68 100644 --- a/deep_speech_2/deploy/decoders.i +++ b/deep_speech_2/deploy/decoders.i @@ -17,6 +17,8 @@ namespace std{ %template(Pair) std::pair; %template(PairFloatStringVector) std::vector >; %template(PairDoubleStringVector) std::vector >; + %template(PairDoubleStringVector2) std::vector > >; + %template(DoubleVector3) std::vector > >; } %import decoder_utils.h diff --git a/deep_speech_2/deploy/setup.py b/deep_speech_2/deploy/setup.py index 077cabd086..1342478b27 100644 --- a/deep_speech_2/deploy/setup.py +++ b/deep_speech_2/deploy/setup.py @@ -36,12 +36,12 @@ def compile_test(header, library): os.system('swig -python -c++ ./decoders.i') -ctc_beam_search_decoder_module = [ +decoders_module = [ Extension( name='_swig_decoders', sources=FILES + glob.glob('*.cxx') + glob.glob('*.cpp'), language='C++', - include_dirs=['.', './kenlm', './openfst-1.6.3/src/include'], + include_dirs=['.', 'kenlm', 'openfst-1.6.3/src/include', 'ThreadPool'], libraries=LIBS, extra_compile_args=ARGS) ] @@ -50,5 +50,5 @@ def compile_test(header, library): name='swig_decoders', version='0.1', description="""CTC decoders""", - ext_modules=ctc_beam_search_decoder_module, + ext_modules=decoders_module, py_modules=['swig_decoders'], ) diff --git a/deep_speech_2/deploy/swig_decoders_wrapper.py b/deep_speech_2/deploy/swig_decoders_wrapper.py index 54c4301475..51f3173b2b 100644 --- a/deep_speech_2/deploy/swig_decoders_wrapper.py +++ b/deep_speech_2/deploy/swig_decoders_wrapper.py @@ -4,7 +4,6 @@ from __future__ import print_function import swig_decoders -import multiprocessing class Scorer(swig_decoders.Scorer): @@ -39,14 +38,13 @@ def ctc_best_path_decoder(probs_seq, vocabulary): return swig_decoders.ctc_best_path_decoder(probs_seq.tolist(), vocabulary) -def ctc_beam_search_decoder( - probs_seq, - beam_size, - vocabulary, - blank_id, - cutoff_prob=1.0, - ext_scoring_func=None, ): - """Wrapper for CTC Beam Search Decoder. +def ctc_beam_search_decoder(probs_seq, + beam_size, + vocabulary, + blank_id, + cutoff_prob=1.0, + ext_scoring_func=None): + """Wrapper for the CTC Beam Search Decoder. :param probs_seq: 2-D list of probability distributions over each time step, with each element being a list of normalized @@ -81,24 +79,34 @@ def ctc_beam_search_decoder_batch(probs_split, num_processes, cutoff_prob=1.0, ext_scoring_func=None): - """Wrapper for CTC beam search decoder in batch - """ - - # TODO: to resolve PicklingError - - if not num_processes > 0: - raise ValueError("Number of processes must be positive!") + """Wrapper for the batched CTC beam search decoder. - pool = Pool(processes=num_processes) - results = [] - args_list = [] - for i, probs_list in enumerate(probs_split): - args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, - ext_scoring_func) - args_list.append(args) - results.append(pool.apply_async(ctc_beam_search_decoder, args)) + :param probs_seq: 3-D list with each element as an instance of 2-D list + of probabilities used by ctc_beam_search_decoder(). + :type probs_seq: 3-D list + :param beam_size: Width for beam search. + :type beam_size: int + :param vocabulary: Vocabulary list. + :type vocabulary: list + :param blank_id: ID of blank. + :type blank_id: int + :param num_processes: Number of parallel processes. + :type num_processes: int + :param cutoff_prob: Cutoff probability in pruning, + default 1.0, no pruning. + :param num_processes: Number of parallel processes. + :type num_processes: int + :type cutoff_prob: float + :param ext_scoring_func: External scoring function for + partially decoded sentence, e.g. word count + or language model. + :type external_scoring_function: callable + :return: List of tuples of log probability and sentence as decoding + results, in descending order of the probability. + :rtype: list + """ + probs_split = [probs_seq.tolist() for probs_seq in probs_split] - pool.close() - pool.join() - beam_search_results = [result.get() for result in results] - return beam_search_results + return swig_decoders.ctc_beam_search_decoder_batch( + probs_split, beam_size, vocabulary, blank_id, num_processes, + cutoff_prob, ext_scoring_func) From a96c65095b985b9787662a3acd3a8150d19d8d0f Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 23 Aug 2017 11:06:27 +0800 Subject: [PATCH 13/36] Refactor scorer and move utility functions to decoder_util.h --- deep_speech_2/deploy/README.md | 2 + deep_speech_2/deploy/ctc_decoders.cpp | 23 ---- deep_speech_2/deploy/decoder_utils.cpp | 7 ++ deep_speech_2/deploy/decoder_utils.h | 33 ++++-- deep_speech_2/deploy/decoders.i | 9 +- deep_speech_2/deploy/scorer.cpp | 148 +++++++++++-------------- deep_speech_2/deploy/scorer.h | 69 ++++++++---- 7 files changed, 154 insertions(+), 137 deletions(-) diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index cf0c04391c..162a396a43 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -7,6 +7,8 @@ wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz tar -xzvf openfst-1.6.3.tar.gz ``` +Compiling for python interface requires swig, please make sure swig being installed. + Then run the setup ```shell diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 75555c018b..836fb435d1 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -9,29 +9,6 @@ typedef double log_prob_type; - -template -bool pair_comp_first_rev(const std::pair a, const std::pair b) -{ - return a.first > b.first; -} - -template -bool pair_comp_second_rev(const std::pair a, const std::pair b) -{ - return a.second > b.second; -} - -template -T log_sum_exp(T x, T y) -{ - static T num_min = -std::numeric_limits::max(); - if (x <= num_min) return y; - if (y <= num_min) return x; - T xmax = std::max(x, y); - return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; -} - std::string ctc_best_path_decoder(std::vector > probs_seq, std::vector vocabulary) { // dimension check diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index 82e4cd1467..d616d7c66c 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -3,3 +3,10 @@ #include #include "decoder_utils.h" +size_t get_utf8_str_len(const std::string& str) { + size_t str_len = 0; + for (char c : str) { + str_len += ((c & 0xc0) != 0x80); + } + return str_len; +} diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index 6d58bf1f30..9419e005a1 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -1,15 +1,32 @@ -#ifndef DECODER_UTILS_H -#define DECODER_UTILS_H -#pragma once +#ifndef DECODER_UTILS_H_ +#define DECODER_UTILS_H_ + #include -/* template -bool pair_comp_first_rev(const std::pair a, const std::pair b); +bool pair_comp_first_rev(const std::pair &a, const std::pair &b) +{ + return a.first > b.first; +} template -bool pair_comp_second_rev(const std::pair a, const std::pair b); +bool pair_comp_second_rev(const std::pair &a, const std::pair &b) +{ + return a.second > b.second; +} + +template +T log_sum_exp(const T &x, const T &y) +{ + static T num_min = -std::numeric_limits::max(); + if (x <= num_min) return y; + if (y <= num_min) return x; + T xmax = std::max(x, y); + return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; +} + +// Get length of utf8 encoding string +// See: http://stackoverflow.com/a/4063229 +size_t get_utf8_str_len(const std::string& str); -template T log_sum_exp(T x, T y); -*/ #endif // DECODER_UTILS_H diff --git a/deep_speech_2/deploy/decoders.i b/deep_speech_2/deploy/decoders.i index 04736e09e8..ed7c85e67d 100644 --- a/deep_speech_2/deploy/decoders.i +++ b/deep_speech_2/deploy/decoders.i @@ -2,13 +2,15 @@ %{ #include "scorer.h" #include "ctc_decoders.h" +#include "decoder_utils.h" %} %include "std_vector.i" %include "std_pair.i" %include "std_string.i" +%import "decoder_utils.h" -namespace std{ +namespace std { %template(DoubleVector) std::vector; %template(IntVector) std::vector; %template(StringVector) std::vector; @@ -19,6 +21,9 @@ namespace std{ %template(PairDoubleStringVector) std::vector >; } -%import decoder_utils.h +%template(IntDoublePairCompSecondRev) pair_comp_second_rev; +%template(StringDoublePairCompSecondRev) pair_comp_second_rev; +%template(DoubleStringPairCompFirstRev) pair_comp_first_rev; + %include "scorer.h" %include "ctc_decoders.h" diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index e9a74b989a..17bb6e10d9 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -1,103 +1,89 @@ #include #include #include "scorer.h" -#include "lm/model.hh" -#include "util/tokenize_piece.hh" -#include "util/string_piece.hh" +#include "decoder_utils.h" -using namespace lm::ngram; - -Scorer::Scorer(float alpha, float beta, std::string lm_model_path) { - this->_alpha = alpha; - this->_beta = beta; - - if (access(lm_model_path.c_str(), F_OK) != 0) { - std::cout<<"Invalid language model path!"<_language_model = LoadVirtual(lm_model_path.c_str()); +Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { + this->alpha = alpha; + this->beta = beta; + _is_character_based = true; + _language_model = nullptr; + _max_order = 0; + // load language model + load_LM(lm_path.c_str()); } -Scorer::~Scorer(){ - delete (lm::base::Model *)this->_language_model; +Scorer::~Scorer() { + if (_language_model != nullptr) + delete static_cast(_language_model); } -/* Strip a input sentence - * Parameters: - * str: A reference to the objective string - * ch: The character to prune - * Return: - * void - */ -inline void strip(std::string &str, char ch=' ') { - if (str.size() == 0) return; - int start = 0; - int end = str.size()-1; - for (int i=0; i=0; i--) { - if (str[i] == ch) { - end --; - } else { - break; + RetriveStrEnumerateVocab enumerate; + Config config; + config.enumerate_vocab = &enumerate; + _language_model = lm::ngram::LoadVirtual(filename, config); + _max_order = static_cast(_language_model)->Order(); + _vocabulary = enumerate.vocabulary; + for (size_t i = 0; i < _vocabulary.size(); ++i) { + if (_is_character_based + && _vocabulary[i] != UNK_TOKEN + && _vocabulary[i] != START_TOKEN + && _vocabulary[i] != END_TOKEN + && get_utf8_str_len(enumerate.vocabulary[i]) > 1) { + _is_character_based = false; } } - - if (start == 0 && end == str.size()-1) return; - if (start > end) { - std::string emp_str; - str = emp_str; - } else { - str = str.substr(start, end-start+1); - } } -int Scorer::word_count(std::string sentence) { - strip(sentence); - int cnt = 1; - for (int i=0; i& words) { + lm::base::Model* model = static_cast(_language_model); + double cond_prob; + State state, tmp_state, out_state; + // avoid to inserting in begin + model->NullContextWrite(&state); + for (size_t i = 0; i < words.size(); ++i) { + lm::WordIndex word_index = model->BaseVocabulary().Index(words[i]); + // encounter OOV + if (word_index == 0) { + return OOV_SCOER; } - } - return cnt; -} - -double Scorer::language_model_score(std::string sentence) { - lm::base::Model *model = (lm::base::Model *)this->_language_model; - State state, out_state; - lm::FullScoreReturn ret; - model->BeginSentenceWrite(&state); - - for (util::TokenIter it(sentence, ' '); it; ++it){ - lm::WordIndex wid = model->BaseVocabulary().Index(*it); - ret = model->BaseFullScore(&state, wid, &out_state); + cond_prob = model->BaseScore(&state, word_index, &out_state); + tmp_state = state; state = out_state; + out_state = tmp_state; } - //log10 prob - double log_prob = ret.prob; - return log_prob; + // log10 prob + return cond_prob; } -void Scorer::reset_params(float alpha, float beta) { - this->_alpha = alpha; - this->_beta = beta; +double Scorer::get_sent_log_prob(const std::vector& words) { + std::vector sentence; + if (words.size() == 0) { + for (size_t i = 0; i < _max_order; ++i) { + sentence.push_back(START_TOKEN); + } + } else { + for (size_t i = 0; i < _max_order - 1; ++i) { + sentence.push_back(START_TOKEN); + } + sentence.insert(sentence.end(), words.begin(), words.end()); + } + sentence.push_back(END_TOKEN); + return get_log_prob(sentence); } -double Scorer::get_score(std::string sentence, bool log) { - double lm_score = language_model_score(sentence); - int word_cnt = word_count(sentence); - - double final_score = 0.0; - if (log == false) { - final_score = pow(10, _alpha*lm_score) * pow(word_cnt, _beta); - } else { - final_score = _alpha*lm_score*std::log(10) + _beta*std::log(word_cnt); +double Scorer::get_log_prob(const std::vector& words) { + assert(words.size() > _max_order); + double score = 0.0; + for (size_t i = 0; i < words.size() - _max_order + 1; ++i) { + std::vector ngram(words.begin() + i, + words.begin() + i + _max_order); + score += get_log_cond_prob(ngram); } - return final_score; + return score; } diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index a18e119bcf..a650d37537 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -2,35 +2,58 @@ #define SCORER_H_ #include +#include +#include +#include "lm/enumerate_vocab.hh" +#include "lm/word_index.hh" +#include "lm/virtual_interface.hh" +#include "util/string_piece.hh" -/* External scorer to evaluate a prefix or a complete sentence - * when a new word appended during decoding, consisting of word - * count and language model scoring. +const double OOV_SCOER = -1000.0; +const std::string START_TOKEN = ""; +const std::string UNK_TOKEN = ""; +const std::string END_TOKEN = ""; - * Example: - * Scorer ext_scorer(alpha, beta, "path_to_language_model.klm"); - * double score = ext_scorer.get_score("sentence_to_score"); - */ -class Scorer{ -private: - float _alpha; - float _beta; - void *_language_model; + // Implement a callback to retrive string vocabulary. +class RetriveStrEnumerateVocab : public lm::EnumerateVocab { +public: + RetriveStrEnumerateVocab() {} - // word insertion term - int word_count(std::string); - // n-gram language model scoring - double language_model_score(std::string); + void Add(lm::WordIndex index, const StringPiece& str) { + vocabulary.push_back(std::string(str.data(), str.length())); + } + + std::vector vocabulary; +}; +// External scorer to query languange score for n-gram or sentence. +// Example: +// Scorer scorer(alpha, beta, "path_of_language_model"); +// scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); +// scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); +class Scorer{ public: - Scorer(){} - Scorer(float alpha, float beta, std::string lm_model_path); + Scorer(double alpha, double beta, const std::string& lm_path); ~Scorer(); + double get_log_cond_prob(const std::vector& words); + double get_sent_log_prob(const std::vector& words); + size_t get_max_order() { return _max_order; } + bool is_character_based() { return _is_character_based; } + std::vector get_vocab() { return _vocabulary; } + + // expose to decoder + double alpha; + double beta; - // reset params alpha & beta - void reset_params(float alpha, float beta); - // get the final score - double get_score(std::string, bool log=false); +protected: + void load_LM(const char* filename); + double get_log_prob(const std::vector& words); + +private: + void* _language_model; + bool _is_character_based; + size_t _max_order; + std::vector _vocabulary; }; -#endif //SCORER_H_ +#endif // SCORER_H_ From 89c4a9627e440e2d4961e9a5771a139bcb673fb8 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 23 Aug 2017 14:41:41 +0800 Subject: [PATCH 14/36] Make setup.py to support parallel processing. --- deep_speech_2/deploy/README.md | 2 +- deep_speech_2/deploy/scorer.cpp | 7 +++- deep_speech_2/deploy/setup.py | 70 ++++++++++++++++++++++++++++++--- 3 files changed, 70 insertions(+), 9 deletions(-) diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index 9bd55dd9a0..90809ad354 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -25,7 +25,7 @@ git clone https://github.com/progschj/ThreadPool.git Then run the setup ```shell -python setup.py install +python setup.py install --num_processes 4 cd .. ``` diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 17bb6e10d9..233b4766dc 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -1,5 +1,8 @@ #include #include +#include "lm/config.hh" +#include "lm/state.hh" +#include "lm/model.hh" #include "scorer.h" #include "decoder_utils.h" @@ -24,7 +27,7 @@ void Scorer::load_LM(const char* filename) { exit(1); } RetriveStrEnumerateVocab enumerate; - Config config; + lm::ngram::Config config; config.enumerate_vocab = &enumerate; _language_model = lm::ngram::LoadVirtual(filename, config); _max_order = static_cast(_language_model)->Order(); @@ -43,7 +46,7 @@ void Scorer::load_LM(const char* filename) { double Scorer::get_log_cond_prob(const std::vector& words) { lm::base::Model* model = static_cast(_language_model); double cond_prob; - State state, tmp_state, out_state; + lm::ngram::State state, tmp_state, out_state; // avoid to inserting in begin model->NullContextWrite(&state); for (size_t i = 0; i < words.size(); ++i) { diff --git a/deep_speech_2/deploy/setup.py b/deep_speech_2/deploy/setup.py index 1342478b27..7a4b7e02c3 100644 --- a/deep_speech_2/deploy/setup.py +++ b/deep_speech_2/deploy/setup.py @@ -1,17 +1,75 @@ -from setuptools import setup, Extension +"""Script to build and install decoder package.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from setuptools import setup, Extension, distutils import glob import platform -import os +import os, sys +import multiprocessing.pool +import argparse + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--num_processes", + default=1, + type=int, + help="Number of cpu processes to build package. (default: %(default)d)") +args = parser.parse_known_args() + +# reconstruct sys.argv to pass to setup below +sys.argv = [sys.argv[0]] + args[1] + + +# monkey-patch for parallel compilation +# See: https://stackoverflow.com/a/13176803 +def parallelCCompile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + # parallel code + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # convert to list, imap is evaluated on-demand + thread_pool = multiprocessing.pool.ThreadPool(args[0].num_processes) + list(thread_pool.imap(_single_compile, objects)) + return objects def compile_test(header, library): dummy_path = os.path.join(os.path.dirname(__file__), "dummy") - command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\"" + command = "bash -c \"g++ -include " + header \ + + " -l" + library + " -x c++ - <<<'int main() {}' -o " \ + + dummy_path + " >/dev/null 2>/dev/null && rm " \ + + dummy_path + " 2>/dev/null\"" return os.system(command) == 0 -FILES = glob.glob('kenlm/util/*.cc') + glob.glob('kenlm/lm/*.cc') + glob.glob( - 'kenlm/util/double-conversion/*.cc') +# hack compile to support parallel compiling +distutils.ccompiler.CCompiler.compile = parallelCCompile + +FILES = glob.glob('kenlm/util/*.cc') \ + + glob.glob('kenlm/lm/*.cc') \ + + glob.glob('kenlm/util/double-conversion/*.cc') + +FILES += glob.glob('openfst-1.6.3/src/lib/*.cc') + FILES = [ fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) ] @@ -40,7 +98,7 @@ def compile_test(header, library): Extension( name='_swig_decoders', sources=FILES + glob.glob('*.cxx') + glob.glob('*.cpp'), - language='C++', + language='c++', include_dirs=['.', 'kenlm', 'openfst-1.6.3/src/include', 'ThreadPool'], libraries=LIBS, extra_compile_args=ARGS) From bbbc988535e53e89d1263d5f0fc00a6db6bb7a32 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 23 Aug 2017 16:57:25 +0800 Subject: [PATCH 15/36] adapt to the last three commits --- deep_speech_2/deploy/README.md | 2 +- deep_speech_2/deploy/scorer.cpp | 85 +++++++++++++++++++++++++++++++++ deep_speech_2/deploy/scorer.h | 10 +++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index 90809ad354..9f2be76e88 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -14,7 +14,7 @@ wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz tar -xzvf openfst-1.6.3.tar.gz ``` -- [**swig**]: Compiling for python interface requires swig, please make sure swig being installed. +- [**SWIG**](http://www.swig.org): Compiling for python interface requires swig, please make sure swig being installed. - [**ThreadPool**](http://progsch.net/wordpress/): A library for C++ thread pool diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 233b4766dc..a1be7e0f6c 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -3,9 +3,13 @@ #include "lm/config.hh" #include "lm/state.hh" #include "lm/model.hh" +#include "util/tokenize_piece.hh" +#include "util/string_piece.hh" #include "scorer.h" #include "decoder_utils.h" +using namespace lm::ngram; + Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { this->alpha = alpha; this->beta = beta; @@ -90,3 +94,84 @@ double Scorer::get_log_prob(const std::vector& words) { } return score; } + +/* Strip a input sentence + * Parameters: + * str: A reference to the objective string + * ch: The character to prune + * Return: + * void + */ +inline void strip(std::string &str, char ch=' ') { + if (str.size() == 0) return; + int start = 0; + int end = str.size()-1; + for (int i=0; i=0; i--) { + if (str[i] == ch) { + end --; + } else { + break; + } + } + + if (start == 0 && end == str.size()-1) return; + if (start > end) { + std::string emp_str; + str = emp_str; + } else { + str = str.substr(start, end-start+1); + } +} + +int Scorer::word_count(std::string sentence) { + strip(sentence); + int cnt = 1; + for (int i=0; i_language_model; + State state, out_state; + lm::FullScoreReturn ret; + model->BeginSentenceWrite(&state); + + for (util::TokenIter it(sentence, ' '); it; ++it){ + lm::WordIndex wid = model->BaseVocabulary().Index(*it); + ret = model->BaseFullScore(&state, wid, &out_state); + state = out_state; + } + //log10 prob + double log_prob = ret.prob; + return log_prob; +} + +void Scorer::reset_params(float alpha, float beta) { + this->alpha = alpha; + this->beta = beta; +} + +double Scorer::get_score(std::string sentence, bool log) { + double lm_score = get_log_cond_prob(sentence); + int word_cnt = word_count(sentence); + + double final_score = 0.0; + if (log == false) { + final_score = pow(10, alpha * lm_score) * pow(word_cnt, beta); + } else { + final_score = alpha * lm_score * std::log(10) + + beta * std::log(word_cnt); + } + return final_score; +} diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index a650d37537..a52420046f 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -30,6 +30,7 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { // Example: // Scorer scorer(alpha, beta, "path_of_language_model"); // scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); +// scorer.get_log_cond_prob("this a sentence"); // scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); class Scorer{ public: @@ -40,7 +41,14 @@ class Scorer{ size_t get_max_order() { return _max_order; } bool is_character_based() { return _is_character_based; } std::vector get_vocab() { return _vocabulary; } - + // word insertion term + int word_count(std::string); + // get the log cond prob of the last word + double get_log_cond_prob(std::string); + // reset params alpha & beta + void reset_params(float alpha, float beta); + // get the final score + double get_score(std::string, bool log=false); // expose to decoder double alpha; double beta; From d68732b7a1b5b90532ff7aa23b0a9dfab6a3e7eb Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 24 Aug 2017 11:14:56 +0800 Subject: [PATCH 16/36] convert data structure for prefix from map to trie tree --- deep_speech_2/deploy.py | 9 +- deep_speech_2/deploy/ctc_decoders.cpp | 250 ++++++++++++++----------- deep_speech_2/deploy/decoder_utils.cpp | 70 +++++++ deep_speech_2/deploy/decoder_utils.h | 14 ++ deep_speech_2/deploy/path_trie.cpp | 153 +++++++++++++++ deep_speech_2/deploy/path_trie.h | 59 ++++++ deep_speech_2/deploy/scorer.cpp | 39 ++++ deep_speech_2/deploy/scorer.h | 13 ++ 8 files changed, 492 insertions(+), 115 deletions(-) create mode 100644 deep_speech_2/deploy/path_trie.cpp create mode 100644 deep_speech_2/deploy/path_trie.h diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 76b6160529..833c5c20cd 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num_samples", - default=32, + default=5, type=int, help="Number of samples for inference. (default: %(default)s)") parser.add_argument( @@ -79,7 +79,7 @@ "(default: %(default)s)") parser.add_argument( "--beam_size", - default=200, + default=20, type=int, help="Width for beam search decoding. (default: %(default)d)") parser.add_argument( @@ -104,7 +104,7 @@ help="Parameter associated with word count. (default: %(default)f)") parser.add_argument( "--cutoff_prob", - default=0.99, + default=1.0, type=float, help="The cutoff probability of pruning" "in beam search. (default: %(default)f)") @@ -183,7 +183,8 @@ def infer(): vocabulary=data_generator.vocab_list, blank_id=len(data_generator.vocab_list), cutoff_prob=args.cutoff_prob, - ext_scoring_func=ext_scorer, ) + # ext_scoring_func=ext_scorer, + ) batch_beam_results += [beam_result] else: batch_beam_results = ctc_beam_search_decoder_batch( diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index fd553be61e..30e855258d 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -4,11 +4,13 @@ #include #include #include +#include "fst/fstlib.h" #include "ctc_decoders.h" #include "decoder_utils.h" +#include "path_trie.h" #include "ThreadPool.h" -typedef double log_prob_type; +typedef float log_prob_type; std::string ctc_best_path_decoder(std::vector > probs_seq, std::vector vocabulary) @@ -89,24 +91,30 @@ std::vector > exit(1); } - // initialize - // two sets containing selected and candidate prefixes respectively - std::map prefix_set_prev, prefix_set_next; - // probability of prefixes ending with blank and non-blank - std::map log_probs_b_prev, log_probs_nb_prev; - std::map log_probs_b_cur, log_probs_nb_cur; - - static log_prob_type NUM_MAX = std::numeric_limits::max(); - prefix_set_prev["\t"] = 0.0; - log_probs_b_prev["\t"] = 0.0; - log_probs_nb_prev["\t"] = -NUM_MAX; - - for (int time_step=0; time_step prob = probs_seq[time_step]; + static log_prob_type POS_INF = std::numeric_limits::max(); + static log_prob_type NEG_INF = -POS_INF; + static log_prob_type NUM_MIN = std::numeric_limits::min(); + + // init + PathTrie root; + root._log_prob_b_prev = 0.0; + root._score = 0.0; + std::vector prefixes; + prefixes.push_back(&root); + + if ( ext_scorer != nullptr && !ext_scorer->is_character_based()) { + if (ext_scorer->dictionary == nullptr) { + // TODO: init dictionary + } + auto fst_dict = static_cast(ext_scorer->dictionary); + fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); + root.set_dictionary(dict_ptr); + auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); + root.set_matcher(matcher); + } + for (int time_step = 0; time_step < num_time_steps; time_step++) { + std::vector prob = probs_seq[time_step]; std::vector > prob_idx; for (int i=0; i(i, prob[i])); @@ -132,113 +140,134 @@ std::vector > std::vector > log_prob_idx; for (int i=0; i - (prob_idx[i].first, log(prob_idx[i].second))); + (prob_idx[i].first, log(prob_idx[i].second + NUM_MIN))); } - // extend prefix - for (std::map::iterator - it = prefix_set_prev.begin(); - it != prefix_set_prev.end(); it++) { - std::string l = it->first; - if( prefix_set_next.find(l) == prefix_set_next.end()) { - log_probs_b_cur[l] = log_probs_nb_cur[l] = -NUM_MAX; - } + // loop over chars + for (int index = 0; index < log_prob_idx.size(); index++) { + auto c = log_prob_idx[index].first; + log_prob_type log_prob_c = log_prob_idx[index].second; + //log_prob_type log_probs_prev; - for (int index=0; index_log_prob_b_cur = log_sum_exp( + prefix->_log_prob_b_cur, + log_prob_c + prefix->_score); + continue; + } + // repeated character + if (c == prefix->_character) { + prefix->_log_prob_nb_cur = log_sum_exp( + prefix->_log_prob_nb_cur, + log_prob_c + prefix->_log_prob_nb_prev + ); + } + // get new prefix + auto prefix_new = prefix->get_path_trie(c); + + if (prefix_new != nullptr) { + float log_p = NEG_INF; + + if (c == prefix->_character + && prefix->_log_prob_b_prev > NEG_INF) { + log_p = log_prob_c + prefix->_log_prob_b_prev; + } else if (c != prefix->_character) { + log_p = log_prob_c + prefix->_score; } - if (last_char == new_char) { - log_probs_nb_cur[l_plus] = log_sum_exp( - log_probs_nb_cur[l_plus], - log_prob_c+log_probs_b_prev[l] - ); - log_probs_nb_cur[l] = log_sum_exp( - log_probs_nb_cur[l], - log_prob_c+log_probs_nb_prev[l] - ); - } else if (new_char == " ") { - float score = 0.0; - if (ext_scorer != NULL && l.size() > 1) { - score = ext_scorer->get_score(l.substr(1), true); + + // language model scoring + if (ext_scorer != nullptr && + (c == space_id || ext_scorer->is_character_based()) ) { + PathTrie *prefix_to_score = nullptr; + + // don't score the space + if (ext_scorer->is_character_based()) { + prefix_to_score = prefix_new; + } else { + prefix_to_score = prefix; } - log_probs_prev = log_sum_exp(log_probs_b_prev[l], - log_probs_nb_prev[l]); - log_probs_nb_cur[l_plus] = log_sum_exp( - log_probs_nb_cur[l_plus], - score + log_prob_c + log_probs_prev - ); - } else { - log_probs_prev = log_sum_exp(log_probs_b_prev[l], - log_probs_nb_prev[l]); - log_probs_nb_cur[l_plus] = log_sum_exp( - log_probs_nb_cur[l_plus], - log_prob_c+log_probs_prev - ); + + double score = 0.0; + std::vector ngram; + ngram = ext_scorer->make_ngram(prefix_to_score); + score = ext_scorer->get_log_cond_prob(ngram) * + ext_scorer->alpha; + + log_p += score; + log_p += ext_scorer->beta; + } - prefix_set_next[l_plus] = log_sum_exp( - log_probs_nb_cur[l_plus], - log_probs_b_cur[l_plus] - ); + prefix_new->_log_prob_nb_cur = log_sum_exp( + prefix_new->_log_prob_nb_cur, log_p); } } - prefix_set_next[l] = log_sum_exp(log_probs_b_cur[l], - log_probs_nb_cur[l]); + } // end of loop over chars + + prefixes.clear(); + // update log probabilities + root.iterate_to_vec(prefixes); + + // sort prefixes by score + if (prefixes.size() >= beam_size) { + std::nth_element(prefixes.begin(), + prefixes.begin() + beam_size, + prefixes.end(), + prefix_compare); + + for (size_t i = beam_size; i < prefixes.size(); i++) { + prefixes[i]->remove(); + } + } + } + + for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { + double approx_ctc = prefixes[i]->_score; + + // remove word insert: + std::vector output; + prefixes[i]->get_path_vec(output); + size_t prefix_length = output.size(); + // remove language model weight: + if (ext_scorer != nullptr) { + // auto words = split_labels(output); + // approx_ctc = approx_ctc - path_length * ext_scorer->beta; + // approx_ctc -= (_lm->get_sent_log_prob(words)) * ext_scorer->alpha; } - log_probs_b_prev = log_probs_b_cur; - log_probs_nb_prev = log_probs_nb_cur; - std::vector > - prefix_vec_next(prefix_set_next.begin(), - prefix_set_next.end()); - std::sort(prefix_vec_next.begin(), - prefix_vec_next.end(), - pair_comp_second_rev); - int num_prefixes_next = prefix_vec_next.size(); - int k = beam_size ( - prefix_vec_next.begin(), - prefix_vec_next.begin() + k - ); + prefixes[i]->_approx_ctc = approx_ctc; } - // post processing - std::vector > beam_result; - for (std::map::iterator - it = prefix_set_prev.begin(); it != prefix_set_prev.end(); it++) { - if (it->second > -NUM_MAX && it->first.size() > 1) { - log_prob_type log_prob = it->second; - std::string sentence = it->first.substr(1); - // scoring the last word - if (ext_scorer != NULL && sentence[sentence.size()-1] != ' ') { - log_prob = log_prob + ext_scorer->get_score(sentence, true); - } - if (log_prob > -NUM_MAX) { - std::pair cur_result(log_prob, sentence); - beam_result.push_back(cur_result); - } + // allow for the post processing + std::vector space_prefixes; + if (space_prefixes.empty()) { + for (size_t i = 0; i < beam_size && i< prefixes.size(); i++) { + space_prefixes.push_back(prefixes[i]); } } - // sort the result and return - std::sort(beam_result.begin(), beam_result.end(), - pair_comp_first_rev); - return beam_result; -} + + std::sort(space_prefixes.begin(), space_prefixes.end(), prefix_compare); + std::vector > output_vecs; + for (size_t i = 0; i < beam_size && i < space_prefixes.size(); i++) { + std::vector output; + space_prefixes[i]->get_path_vec(output); + // convert index to string + std::string output_str; + for (int j = 0; j < output.size(); j++) { + output_str += vocabulary[output[j]]; + } + std::pair output_pair(space_prefixes[i]->_score, + output_str); + output_vecs.emplace_back( + output_pair + ); + } + + return output_vecs; + } std::vector>> @@ -250,8 +279,7 @@ std::vector>> int num_processes, double cutoff_prob, Scorer *ext_scorer - ) -{ + ) { if (num_processes <= 0) { std::cout << "num_processes must be nonnegative!" << std::endl; exit(1); diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index d616d7c66c..366c8d3556 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -10,3 +10,73 @@ size_t get_utf8_str_len(const std::string& str) { } return str_len; } + +//------------------------------------------------------- +// Overriding less than operator for sorting +//------------------------------------------------------- +bool prefix_compare(const PathTrie* x, const PathTrie* y) { + if (x->_score == y->_score) { + if (x->_character == y->_character) { + return false; + } else { + return (x->_character < y->_character); + } + } else { + return x->_score > y->_score; + } +} //---------- End path_compare --------------------------- + +// -------------------------------------------------------------- +// Adds word to fst without copying entire dictionary +// -------------------------------------------------------------- +void add_word_to_fst(const std::vector& word, + fst::StdVectorFst* dictionary) { + if (dictionary->NumStates() == 0) { + fst::StdVectorFst::StateId start = dictionary->AddState(); + assert(start == 0); + dictionary->SetStart(start); + } + fst::StdVectorFst::StateId src = dictionary->Start(); + fst::StdVectorFst::StateId dst; + for (auto c : word) { + dst = dictionary->AddState(); + dictionary->AddArc(src, fst::StdArc(c, c, 0, dst)); + src = dst; + } + dictionary->SetFinal(dst, fst::StdArc::Weight::One()); +} // ------------ End of add_word_to_fst ----------------------- + +// --------------------------------------------------------- +// Adds a word to the dictionary FST based on char_map +// --------------------------------------------------------- +bool addWordToDictionary(const std::string& word, + const std::unordered_map& char_map, + bool add_space, + int SPACE, + fst::StdVectorFst* dictionary) { + /* + auto characters = UTF8_split(word); + + std::vector int_word; + + for (auto& c : characters) { + if (c == " ") { + int_word.push_back(SPACE); + } else { + auto int_c = char_map.find(c); + if (int_c != char_map.end()) { + int_word.push_back(int_c->second); + } else { + return false; // return without adding + } + } + } + + if (add_space) { + int_word.push_back(SPACE); + } + + add_word_to_fst(int_word, dictionary); + */ + return true; +} // -------------- End of addWordToDictionary ------------ diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index 9419e005a1..d5e7d1860e 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -2,6 +2,7 @@ #define DECODER_UTILS_H_ #include +#include "path_trie.h" template bool pair_comp_first_rev(const std::pair &a, const std::pair &b) @@ -25,8 +26,21 @@ T log_sum_exp(const T &x, const T &y) return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; } +//------------------------------------------------------- +// Overriding less than operator for sorting +//------------------------------------------------------- +bool prefix_compare(const PathTrie* x, const PathTrie* y); + // Get length of utf8 encoding string // See: http://stackoverflow.com/a/4063229 size_t get_utf8_str_len(const std::string& str); +void add_word_to_fst(const std::vector& word, + fst::StdVectorFst* dictionary); + +bool addWordToDictionary(const std::string& word, + const std::unordered_map& char_map, + bool add_space, + int SPACE, + fst::StdVectorFst* dictionary); #endif // DECODER_UTILS_H diff --git a/deep_speech_2/deploy/path_trie.cpp b/deep_speech_2/deploy/path_trie.cpp new file mode 100644 index 0000000000..6cf7ae515d --- /dev/null +++ b/deep_speech_2/deploy/path_trie.cpp @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include + +#include "path_trie.h" +#include "decoder_utils.h" + +PathTrie::PathTrie() { + float lowest = -1.0*std::numeric_limits::max(); + _log_prob_b_prev = lowest; + _log_prob_nb_prev = lowest; + _log_prob_b_cur = lowest; + _log_prob_nb_cur = lowest; + _score = lowest; + + _ROOT = -1; + _character = _ROOT; + _exists = true; + _parent = nullptr; + _dictionary = nullptr; + _dictionary_state = 0; + _has_dictionary = false; + _matcher = nullptr; // finds arcs in FST +} + +PathTrie::~PathTrie() { + for (auto child : _children) { + delete child.second; + } +} + +PathTrie* PathTrie::get_path_trie(int new_char, bool reset) { + auto child = _children.begin(); + for (child = _children.begin(); child != _children.end(); ++child) { + if (child->first == new_char) { + break; + } + } + if ( child != _children.end() ) { + if (!child->second->_exists) { + child->second->_exists = true; + float lowest = -1.0*std::numeric_limits::max(); + child->second->_log_prob_b_prev = lowest; + child->second->_log_prob_nb_prev = lowest; + child->second->_log_prob_b_cur = lowest; + child->second->_log_prob_nb_cur = lowest; + } + return (child->second); + } else { + if (_has_dictionary) { + _matcher->SetState(_dictionary_state); + bool found = _matcher->Find(new_char); + if (!found) { + // Adding this character causes word outside dictionary + auto FSTZERO = fst::TropicalWeight::Zero(); + auto final_weight = _dictionary->Final(_dictionary_state); + bool is_final = (final_weight != FSTZERO); + if (is_final && reset) { + _dictionary_state = _dictionary->Start(); + } + return nullptr; + } else { + PathTrie* new_path = new PathTrie; + new_path->_character = new_char; + new_path->_parent = this; + new_path->_dictionary = _dictionary; + new_path->_dictionary_state = _matcher->Value().nextstate; + new_path->_has_dictionary = true; + new_path->_matcher = _matcher; + _children.push_back(std::make_pair(new_char, new_path)); + return new_path; + } + } else { + PathTrie* new_path = new PathTrie; + new_path->_character = new_char; + new_path->_parent = this; + _children.push_back(std::make_pair(new_char, new_path)); + return new_path; + } + } +} + +PathTrie* PathTrie::get_path_vec(std::vector& output) { + return get_path_vec(output, _ROOT); +} + +PathTrie* PathTrie::get_path_vec(std::vector& output, + int stop, + size_t max_steps /*= std::numeric_limits::max() */) { + if (_character == stop || + _character == _ROOT || + output.size() == max_steps) { + std::reverse(output.begin(), output.end()); + return this; + } else { + output.push_back(_character); + return _parent->get_path_vec(output, stop, max_steps); + } +} + +void PathTrie::iterate_to_vec( + std::vector& output) { + if (_exists) { + _log_prob_b_prev = _log_prob_b_cur; + _log_prob_nb_prev = _log_prob_nb_cur; + + _log_prob_b_cur = -1.0 * std::numeric_limits::max(); + _log_prob_nb_cur = -1.0 * std::numeric_limits::max(); + + _score = log_sum_exp(_log_prob_b_prev, _log_prob_nb_prev); + output.push_back(this); + } + for (auto child : _children) { + child.second->iterate_to_vec(output); + } +} + +//------------------------------------------------------- +// Effectively removes node +//------------------------------------------------------- +void PathTrie::remove() { + _exists = false; + + if (_children.size() == 0) { + auto child = _parent->_children.begin(); + for (child = _parent->_children.begin(); + child != _parent->_children.end(); ++child) { + if (child->first == _character) { + _parent->_children.erase(child); + break; + } + } + + if ( _parent->_children.size() == 0 && !_parent->_exists ) { + _parent->remove(); + } + + delete this; + } +} + +void PathTrie::set_dictionary(fst::StdVectorFst* dictionary) { + _dictionary = dictionary; + _dictionary_state = dictionary->Start(); + _has_dictionary = true; +} + +using FSTMATCH = fst::SortedMatcher; +void PathTrie::set_matcher(std::shared_ptr matcher) { + _matcher = matcher; +} diff --git a/deep_speech_2/deploy/path_trie.h b/deep_speech_2/deploy/path_trie.h new file mode 100644 index 0000000000..7b378e3f97 --- /dev/null +++ b/deep_speech_2/deploy/path_trie.h @@ -0,0 +1,59 @@ +#ifndef PATH_TRIE_H +#define PATH_TRIE_H +#pragma once +#include +#include +#include +#include +#include +#include + +using FSTMATCH = fst::SortedMatcher; + +class PathTrie { +public: + PathTrie(); + ~PathTrie(); + + PathTrie* get_path_trie(int new_char, bool reset = true); + + PathTrie* get_path_vec(std::vector &output); + + PathTrie* get_path_vec(std::vector& output, + int stop, + size_t max_steps = std::numeric_limits::max()); + + void iterate_to_vec(std::vector &output); + + void set_dictionary(fst::StdVectorFst* dictionary); + + void set_matcher(std::shared_ptr matcher); + + bool is_empty() { + return _ROOT == _character; + } + + void remove(); + + float _log_prob_b_prev; + float _log_prob_nb_prev; + float _log_prob_b_cur; + float _log_prob_nb_cur; + float _score; + float _approx_ctc; + + + int _ROOT; + int _character; + bool _exists; + + PathTrie *_parent; + std::vector > _children; + + fst::StdVectorFst* _dictionary; + fst::StdVectorFst::StateId _dictionary_state; + bool _has_dictionary; + std::shared_ptr _matcher; +}; + +#endif // PATH_TRIE_H diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index a1be7e0f6c..4dc8b253f5 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -175,3 +175,42 @@ double Scorer::get_score(std::string sentence, bool log) { } return final_score; } + +//-------------------------------------------------- +// Turn indices back into strings of chars +//-------------------------------------------------- +std::vector Scorer::make_ngram(PathTrie* prefix) { + /* + std::vector ngram; + PathTrie* current_node = prefix; + PathTrie* new_node = nullptr; + + for (int order = 0; order < _max_order; order++) { + std::vector prefix_vec; + + if (_is_character_based) { + new_node = current_node->get_path_vec(prefix_vec, ' ', 1); + current_node = new_node; + } else { + new_node = current_node->getPathVec(prefix_vec, ' '); + current_node = new_node->_parent; // Skipping spaces + } + + // reconstruct word + std::string word = vec2str(prefix_vec); + ngram.push_back(word); + + if (new_node->_character == -1) { + // No more spaces, but still need order + for (int i = 0; i < max_order - order - 1; i++) { + ngram.push_back(""); + } + break; + } + } + std::reverse(ngram.begin(), ngram.end()); + */ + std::vector ngram; + ngram.push_back("this"); + return ngram; +} //---------------- End makeNgrams ------------------ diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index a52420046f..f0efbca99d 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -4,10 +4,12 @@ #include #include #include +#include #include "lm/enumerate_vocab.hh" #include "lm/word_index.hh" #include "lm/virtual_interface.hh" #include "util/string_piece.hh" +#include "path_trie.h" const double OOV_SCOER = -1000.0; const std::string START_TOKEN = ""; @@ -49,18 +51,29 @@ class Scorer{ void reset_params(float alpha, float beta); // get the final score double get_score(std::string, bool log=false); + // make ngram + std::vector make_ngram(PathTrie* prefix); // expose to decoder double alpha; double beta; + // fst dictionary + void* dictionary; protected: void load_LM(const char* filename); double get_log_prob(const std::vector& words); private: + void _init_char_list(); + void _init_char_map(); + void* _language_model; bool _is_character_based; size_t _max_order; + + std::vector _char_list; + std::unordered_map _char_map; + std::vector _vocabulary; }; From 955d293233c09f87c0a5603af5640dce18d1c19b Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 29 Aug 2017 12:27:30 +0800 Subject: [PATCH 17/36] enable finite-state transducer in beam search decoding --- deep_speech_2/deploy.py | 8 +- deep_speech_2/deploy/ctc_decoders.cpp | 15 ++- deep_speech_2/deploy/decoder_utils.cpp | 30 +++++- deep_speech_2/deploy/decoder_utils.h | 4 +- deep_speech_2/deploy/scorer.cpp | 143 +++++++++++++++++++++++-- deep_speech_2/deploy/scorer.h | 11 +- 6 files changed, 189 insertions(+), 22 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 833c5c20cd..d43ab1e0fc 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num_samples", - default=5, + default=4, type=int, help="Number of samples for inference. (default: %(default)s)") parser.add_argument( @@ -89,7 +89,8 @@ help="Number of output per sample in beam search. (default: %(default)d)") parser.add_argument( "--language_model_path", - default="lm/data/common_crawl_00.prune01111.trie.klm", + default="/home/work/liuyibing/lm_bak/common_crawl_00.prune01111.trie.klm", + #default="ptb_all.arpa", type=str, help="Path for language model. (default: %(default)s)") parser.add_argument( @@ -183,8 +184,7 @@ def infer(): vocabulary=data_generator.vocab_list, blank_id=len(data_generator.vocab_list), cutoff_prob=args.cutoff_prob, - # ext_scoring_func=ext_scorer, - ) + ext_scoring_func=ext_scorer, ) batch_beam_results += [beam_result] else: batch_beam_results = ctc_beam_search_decoder_batch( diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 30e855258d..d84f5b16ba 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -103,10 +103,13 @@ std::vector > prefixes.push_back(&root); if ( ext_scorer != nullptr && !ext_scorer->is_character_based()) { - if (ext_scorer->dictionary == nullptr) { + if (ext_scorer->_dictionary == nullptr) { // TODO: init dictionary + ext_scorer->set_char_map(vocabulary); + // add_space should be true? + ext_scorer->fill_dictionary(true); } - auto fst_dict = static_cast(ext_scorer->dictionary); + auto fst_dict = static_cast(ext_scorer->_dictionary); fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); root.set_dictionary(dict_ptr); auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); @@ -288,6 +291,14 @@ std::vector>> ThreadPool pool(num_processes); // number of samples int batch_size = probs_split.size(); + // dictionary init + if ( ext_scorer != nullptr) { + if (ext_scorer->_dictionary == nullptr) { + // TODO: init dictionary + ext_scorer->set_char_map(vocabulary); + ext_scorer->fill_dictionary(true); + } + } // enqueue the tasks of decoding std::vector>>> res; for (int i = 0; i < batch_size; i++) { diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index 366c8d3556..0ec86d6bc2 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -11,6 +11,32 @@ size_t get_utf8_str_len(const std::string& str) { return str_len; } +//------------------------------------------------------ +//Splits string into vector of strings representing +//UTF-8 characters (not same as chars) +//------------------------------------------------------ +std::vector UTF8_split(const std::string& str) +{ + std::vector result; + std::string out_str; + + for (char c : str) + { + if ((c & 0xc0) != 0x80) //new UTF-8 character + { + if (!out_str.empty()) + { + result.push_back(out_str); + out_str.clear(); + } + } + + out_str.append(1, c); + } + result.push_back(out_str); + return result; +} + //------------------------------------------------------- // Overriding less than operator for sorting //------------------------------------------------------- @@ -49,12 +75,11 @@ void add_word_to_fst(const std::vector& word, // --------------------------------------------------------- // Adds a word to the dictionary FST based on char_map // --------------------------------------------------------- -bool addWordToDictionary(const std::string& word, +bool add_word_to_dictionary(const std::string& word, const std::unordered_map& char_map, bool add_space, int SPACE, fst::StdVectorFst* dictionary) { - /* auto characters = UTF8_split(word); std::vector int_word; @@ -77,6 +102,5 @@ bool addWordToDictionary(const std::string& word, } add_word_to_fst(int_word, dictionary); - */ return true; } // -------------- End of addWordToDictionary ------------ diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index d5e7d1860e..b61cdfbfe6 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -35,10 +35,12 @@ bool prefix_compare(const PathTrie* x, const PathTrie* y); // See: http://stackoverflow.com/a/4063229 size_t get_utf8_str_len(const std::string& str); +std::vector UTF8_split(const std::string &str); + void add_word_to_fst(const std::vector& word, fst::StdVectorFst* dictionary); -bool addWordToDictionary(const std::string& word, +bool add_word_to_dictionary(const std::string& word, const std::unordered_map& char_map, bool add_space, int SPACE, diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 4dc8b253f5..ad33a0cdaf 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -15,7 +15,9 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { this->beta = beta; _is_character_based = true; _language_model = nullptr; + _dictionary = nullptr; _max_order = 0; + _SPACE = -1; // load language model load_LM(lm_path.c_str()); } @@ -23,6 +25,8 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { Scorer::~Scorer() { if (_language_model != nullptr) delete static_cast(_language_model); + if (_dictionary != nullptr) + delete static_cast(_dictionary); } void Scorer::load_LM(const char* filename) { @@ -176,11 +180,83 @@ double Scorer::get_score(std::string sentence, bool log) { return final_score; } -//-------------------------------------------------- -// Turn indices back into strings of chars -//-------------------------------------------------- +std::string Scorer::vec2str(const std::vector& input) { + std::string word; + for (auto ind : input) { + word += _char_list[ind]; + } + return word; +} + + +std::vector +Scorer::split_labels(const std::vector &labels) { + if (labels.empty()) + return {}; + + std::string s = vec2str(labels); + std::vector words; + if (_is_character_based) { + words = UTF8_split(s); + } else { + words = split_str(s, " "); + } + return words; +} + +// Split a string into a list of strings on a given string +// delimiter. NB: delimiters on beginning / end of string are +// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. +std::vector Scorer::split_str(const std::string &s, + const std::string &delim) { + std::vector result; + std::size_t start = 0, delim_len = delim.size(); + while (true) { + std::size_t end = s.find(delim, start); + if (end == std::string::npos) { + if (start < s.size()) { + result.push_back(s.substr(start)); + } + break; + } + if (end > start) { + result.push_back(s.substr(start, end - start)); + } + start = end + delim_len; + } + return result; +} + +//--------------------------------------------------- +// Add index to char list for searching language model +//--------------------------------------------------- +void Scorer::set_char_map(std::vector char_list) { + _char_list = char_list; + std::string _SPACE_STR = " "; + + for (unsigned int i = 0; i < _char_list.size(); i++) { + // if (_char_list[i] == _BLANK_STR) { + // _BLANK = i; + // } else + if (_char_list[i] == _SPACE_STR) { + _SPACE = i; + } + } + + _char_map.clear(); + for(unsigned int i = 0; i < _char_list.size(); i++) + { + if(i == (unsigned int)_SPACE){ + _char_map[' '] = i; + } + else if(_char_list[i].size() == 1){ + _char_map[_char_list[i][0]] = i; + } + } + +} //------------- End of set_char_map ---------------- + std::vector Scorer::make_ngram(PathTrie* prefix) { - /* std::vector ngram; PathTrie* current_node = prefix; PathTrie* new_node = nullptr; @@ -189,10 +265,10 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { std::vector prefix_vec; if (_is_character_based) { - new_node = current_node->get_path_vec(prefix_vec, ' ', 1); + new_node = current_node->get_path_vec(prefix_vec, _SPACE, 1); current_node = new_node; } else { - new_node = current_node->getPathVec(prefix_vec, ' '); + new_node = current_node->get_path_vec(prefix_vec, _SPACE); current_node = new_node->_parent; // Skipping spaces } @@ -202,15 +278,60 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { if (new_node->_character == -1) { // No more spaces, but still need order - for (int i = 0; i < max_order - order - 1; i++) { + for (int i = 0; i < _max_order - order - 1; i++) { ngram.push_back(""); } break; } } std::reverse(ngram.begin(), ngram.end()); - */ - std::vector ngram; - ngram.push_back("this"); return ngram; -} //---------------- End makeNgrams ------------------ +} + +//--------------------------------------------------------- +// Helper function to populate Trie with a vocab using the +// char_list for maping from string to int +//--------------------------------------------------------- +void Scorer::fill_dictionary(bool add_space) { + + fst::StdVectorFst dictionary; + // First reverse char_list so ints can be accessed by chars + std::unordered_map char_map; + for (unsigned int i = 0; i < _char_list.size(); i++) { + char_map[_char_list[i]] = i; + } + + // For each unigram convert to ints and put in trie + int vocab_size = 0; + for (const auto& word : _vocabulary) { + bool added = add_word_to_dictionary(word, + char_map, + add_space, + _SPACE, + &dictionary); + vocab_size += added ? 1 : 0; + } + + std::cerr << "Vocab Size " << vocab_size << std::endl; + + // Simplify FST + + // This gets rid of "epsilon" transitions in the FST. + // These are transitions that don't require a string input to be taken. + // Getting rid of them is necessary to make the FST determinisitc, but + // can greatly increase the size of the FST + fst::RmEpsilon(&dictionary); + fst::StdVectorFst* new_dict = new fst::StdVectorFst; + + // This makes the FST deterministic, meaning for any string input there's + // only one possible state the FST could be in. It is assumed our + // dictionary is deterministic when using it. + // (lest we'd have to check for multiple transitions at each state) + fst::Determinize(dictionary, new_dict); + + // Finds the simplest equivalent fst. This is unnecessary but decreases + // memory usage of the dictionary + fst::Minimize(new_dict); + _dictionary = new_dict; + +} diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index f0efbca99d..9ba55dd6de 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -53,15 +53,23 @@ class Scorer{ double get_score(std::string, bool log=false); // make ngram std::vector make_ngram(PathTrie* prefix); + // fill dictionary for fst + void fill_dictionary(bool add_space); + // set char map + void set_char_map(std::vector char_list); // expose to decoder double alpha; double beta; // fst dictionary - void* dictionary; + void* _dictionary; protected: void load_LM(const char* filename); double get_log_prob(const std::vector& words); + std::string vec2str(const std::vector &input); + std::vector split_labels(const std::vector &labels); + std::vector split_str(const std::string &s, + const std::string &delim); private: void _init_char_list(); @@ -71,6 +79,7 @@ class Scorer{ bool _is_character_based; size_t _max_order; + unsigned int _SPACE; std::vector _char_list; std::unordered_map _char_map; From 20d13a4d0811a27b7482a56e79866dded7900865 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 29 Aug 2017 18:54:15 +0800 Subject: [PATCH 18/36] streamline source code --- deep_speech_2/deploy/ctc_decoders.cpp | 67 +++++++++++--------------- deep_speech_2/deploy/decoder_utils.cpp | 27 ++++++++++- deep_speech_2/deploy/decoder_utils.h | 19 +++++--- deep_speech_2/deploy/path_trie.cpp | 27 +++++------ deep_speech_2/deploy/scorer.cpp | 65 +++++-------------------- deep_speech_2/deploy/scorer.h | 9 +--- 6 files changed, 92 insertions(+), 122 deletions(-) diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index d84f5b16ba..da37708af5 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -10,8 +10,6 @@ #include "path_trie.h" #include "ThreadPool.h" -typedef float log_prob_type; - std::string ctc_best_path_decoder(std::vector > probs_seq, std::vector vocabulary) { @@ -19,8 +17,8 @@ std::string ctc_best_path_decoder(std::vector > probs_seq, int num_time_steps = probs_seq.size(); for (int i=0; i > probs_seq, std::vector max_idx_vec; double max_prob = 0.0; int max_idx = 0; - for (int i=0; i > probs_seq, } std::vector idx_vec; - for (int i=0; i0) && max_idx_vec[i]!=max_idx_vec[i-1])) { + for (int i = 0; i < max_idx_vec.size(); i++) { + if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i-1])) { idx_vec.push_back(max_idx_vec[i]); } } std::string best_path_result; - for (int i=0; i > { // dimension check int num_time_steps = probs_seq.size(); - for (int i=0; i > std::vector::iterator it = std::find(vocabulary.begin(), vocabulary.end(), " "); int space_id = it - vocabulary.begin(); + // if no space in vocabulary if(space_id >= vocabulary.size()) { - std::cout << " The character space is not in the vocabulary!"<::max(); - static log_prob_type NEG_INF = -POS_INF; - static log_prob_type NUM_MIN = std::numeric_limits::min(); - // init PathTrie root; - root._log_prob_b_prev = 0.0; - root._score = 0.0; + root._score = root._log_prob_b_prev = 0.0; std::vector prefixes; prefixes.push_back(&root); @@ -140,17 +133,17 @@ std::vector > prob_idx.begin() + cutoff_len); } - std::vector > log_prob_idx; - for (int i=0; i - (prob_idx[i].first, log(prob_idx[i].second + NUM_MIN))); + std::vector > log_prob_idx; + for (int i = 0; i < cutoff_len; i++) { + log_prob_idx.push_back(std::pair + (prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); } // loop over chars for (int index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; - log_prob_type log_prob_c = log_prob_idx[index].second; - //log_prob_type log_probs_prev; + float log_prob_c = log_prob_idx[index].second; + //float log_probs_prev; for (int i = 0; i < prefixes.size() && i > if (c == prefix->_character) { prefix->_log_prob_nb_cur = log_sum_exp( prefix->_log_prob_nb_cur, - log_prob_c + prefix->_log_prob_nb_prev - ); + log_prob_c + prefix->_log_prob_nb_prev); } // get new prefix auto prefix_new = prefix->get_path_trie(c); if (prefix_new != nullptr) { - float log_p = NEG_INF; + float log_p = -NUM_FLT_INF; if (c == prefix->_character - && prefix->_log_prob_b_prev > NEG_INF) { + && prefix->_log_prob_b_prev > -NUM_FLT_INF) { log_p = log_prob_c + prefix->_log_prob_b_prev; } else if (c != prefix->_character) { log_p = log_prob_c + prefix->_score; @@ -201,7 +193,6 @@ std::vector > log_p += score; log_p += ext_scorer->beta; - } prefix_new->_log_prob_nb_cur = log_sum_exp( prefix_new->_log_prob_nb_cur, log_p); @@ -273,7 +264,7 @@ std::vector > } -std::vector>> +std::vector > > ctc_beam_search_decoder_batch( std::vector>> probs_split, int beam_size, @@ -292,12 +283,12 @@ std::vector>> // number of samples int batch_size = probs_split.size(); // dictionary init - if ( ext_scorer != nullptr) { - if (ext_scorer->_dictionary == nullptr) { - // TODO: init dictionary - ext_scorer->set_char_map(vocabulary); - ext_scorer->fill_dictionary(true); - } + if ( ext_scorer != nullptr + && !ext_scorer->is_character_based() + && ext_scorer->_dictionary == nullptr) { + // init dictionary + ext_scorer->set_char_map(vocabulary); + ext_scorer->fill_dictionary(true); } // enqueue the tasks of decoding std::vector>>> res; @@ -308,7 +299,7 @@ std::vector>> ); } // get decoding results - std::vector>> batch_results; + std::vector > > batch_results; for (int i = 0; i < batch_size; i++) { batch_results.emplace_back(res[i].get()); } diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index 0ec86d6bc2..39beb811e1 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -15,7 +15,7 @@ size_t get_utf8_str_len(const std::string& str) { //Splits string into vector of strings representing //UTF-8 characters (not same as chars) //------------------------------------------------------ -std::vector UTF8_split(const std::string& str) +std::vector split_utf8_str(const std::string& str) { std::vector result; std::string out_str; @@ -37,6 +37,29 @@ std::vector UTF8_split(const std::string& str) return result; } +// Split a string into a list of strings on a given string +// delimiter. NB: delimiters on beginning / end of string are +// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. +std::vector split_str(const std::string &s, + const std::string &delim) { + std::vector result; + std::size_t start = 0, delim_len = delim.size(); + while (true) { + std::size_t end = s.find(delim, start); + if (end == std::string::npos) { + if (start < s.size()) { + result.push_back(s.substr(start)); + } + break; + } + if (end > start) { + result.push_back(s.substr(start, end - start)); + } + start = end + delim_len; + } + return result; +} + //------------------------------------------------------- // Overriding less than operator for sorting //------------------------------------------------------- @@ -80,7 +103,7 @@ bool add_word_to_dictionary(const std::string& word, bool add_space, int SPACE, fst::StdVectorFst* dictionary) { - auto characters = UTF8_split(word); + auto characters = split_utf8_str(word); std::vector int_word; diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index b61cdfbfe6..936605868f 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -4,14 +4,19 @@ #include #include "path_trie.h" +const float NUM_FLT_INF = std::numeric_limits::max(); +const float NUM_FLT_MIN = std::numeric_limits::min(); + template -bool pair_comp_first_rev(const std::pair &a, const std::pair &b) +bool pair_comp_first_rev(const std::pair &a, + const std::pair &b) { return a.first > b.first; } template -bool pair_comp_second_rev(const std::pair &a, const std::pair &b) +bool pair_comp_second_rev(const std::pair &a, + const std::pair &b) { return a.second > b.second; } @@ -26,16 +31,18 @@ T log_sum_exp(const T &x, const T &y) return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; } -//------------------------------------------------------- -// Overriding less than operator for sorting -//------------------------------------------------------- + +// Functor for prefix comparsion bool prefix_compare(const PathTrie* x, const PathTrie* y); // Get length of utf8 encoding string // See: http://stackoverflow.com/a/4063229 size_t get_utf8_str_len(const std::string& str); -std::vector UTF8_split(const std::string &str); +std::vector split_str(const std::string &s, + const std::string &delim); + +std::vector split_utf8_str(const std::string &str); void add_word_to_fst(const std::vector& word, fst::StdVectorFst* dictionary); diff --git a/deep_speech_2/deploy/path_trie.cpp b/deep_speech_2/deploy/path_trie.cpp index 6cf7ae515d..b841831d71 100644 --- a/deep_speech_2/deploy/path_trie.cpp +++ b/deep_speech_2/deploy/path_trie.cpp @@ -8,12 +8,11 @@ #include "decoder_utils.h" PathTrie::PathTrie() { - float lowest = -1.0*std::numeric_limits::max(); - _log_prob_b_prev = lowest; - _log_prob_nb_prev = lowest; - _log_prob_b_cur = lowest; - _log_prob_nb_cur = lowest; - _score = lowest; + _log_prob_b_prev = -NUM_FLT_INF; + _log_prob_nb_prev = -NUM_FLT_INF; + _log_prob_b_cur = -NUM_FLT_INF; + _log_prob_nb_cur = -NUM_FLT_INF; + _score = -NUM_FLT_INF; _ROOT = -1; _character = _ROOT; @@ -41,11 +40,10 @@ PathTrie* PathTrie::get_path_trie(int new_char, bool reset) { if ( child != _children.end() ) { if (!child->second->_exists) { child->second->_exists = true; - float lowest = -1.0*std::numeric_limits::max(); - child->second->_log_prob_b_prev = lowest; - child->second->_log_prob_nb_prev = lowest; - child->second->_log_prob_b_cur = lowest; - child->second->_log_prob_nb_cur = lowest; + child->second->_log_prob_b_prev = -NUM_FLT_INF; + child->second->_log_prob_nb_prev = -NUM_FLT_INF; + child->second->_log_prob_b_cur = -NUM_FLT_INF; + child->second->_log_prob_nb_cur = -NUM_FLT_INF; } return (child->second); } else { @@ -106,8 +104,8 @@ void PathTrie::iterate_to_vec( _log_prob_b_prev = _log_prob_b_cur; _log_prob_nb_prev = _log_prob_nb_cur; - _log_prob_b_cur = -1.0 * std::numeric_limits::max(); - _log_prob_nb_cur = -1.0 * std::numeric_limits::max(); + _log_prob_b_cur = -NUM_FLT_INF; + _log_prob_nb_cur = -NUM_FLT_INF; _score = log_sum_exp(_log_prob_b_prev, _log_prob_nb_prev); output.push_back(this); @@ -117,9 +115,6 @@ void PathTrie::iterate_to_vec( } } -//------------------------------------------------------- -// Effectively removes node -//------------------------------------------------------- void PathTrie::remove() { _exists = false; diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index ad33a0cdaf..41f3894ab5 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -17,7 +17,7 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { _language_model = nullptr; _dictionary = nullptr; _max_order = 0; - _SPACE = -1; + _SPACE_ID = -1; // load language model load_LM(lm_path.c_str()); } @@ -61,7 +61,7 @@ double Scorer::get_log_cond_prob(const std::vector& words) { lm::WordIndex word_index = model->BaseVocabulary().Index(words[i]); // encounter OOV if (word_index == 0) { - return OOV_SCOER; + return OOV_SCORE; } cond_prob = model->BaseScore(&state, word_index, &out_state); tmp_state = state; @@ -197,64 +197,27 @@ Scorer::split_labels(const std::vector &labels) { std::string s = vec2str(labels); std::vector words; if (_is_character_based) { - words = UTF8_split(s); + words = split_utf8_str(s); } else { words = split_str(s, " "); } return words; } -// Split a string into a list of strings on a given string -// delimiter. NB: delimiters on beginning / end of string are -// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. -std::vector Scorer::split_str(const std::string &s, - const std::string &delim) { - std::vector result; - std::size_t start = 0, delim_len = delim.size(); - while (true) { - std::size_t end = s.find(delim, start); - if (end == std::string::npos) { - if (start < s.size()) { - result.push_back(s.substr(start)); - } - break; - } - if (end > start) { - result.push_back(s.substr(start, end - start)); - } - start = end + delim_len; - } - return result; -} - -//--------------------------------------------------- -// Add index to char list for searching language model -//--------------------------------------------------- void Scorer::set_char_map(std::vector char_list) { _char_list = char_list; - std::string _SPACE_STR = " "; - - for (unsigned int i = 0; i < _char_list.size(); i++) { - // if (_char_list[i] == _BLANK_STR) { - // _BLANK = i; - // } else - if (_char_list[i] == _SPACE_STR) { - _SPACE = i; - } - } - _char_map.clear(); + for(unsigned int i = 0; i < _char_list.size(); i++) { - if(i == (unsigned int)_SPACE){ + if (_char_list[i] == " ") { + _SPACE_ID = i; _char_map[' '] = i; - } - else if(_char_list[i].size() == 1){ + } else if(_char_list[i].size() == 1){ _char_map[_char_list[i][0]] = i; } } - -} //------------- End of set_char_map ---------------- +} std::vector Scorer::make_ngram(PathTrie* prefix) { std::vector ngram; @@ -265,10 +228,10 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { std::vector prefix_vec; if (_is_character_based) { - new_node = current_node->get_path_vec(prefix_vec, _SPACE, 1); + new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID, 1); current_node = new_node; } else { - new_node = current_node->get_path_vec(prefix_vec, _SPACE); + new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID); current_node = new_node->_parent; // Skipping spaces } @@ -279,7 +242,7 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { if (new_node->_character == -1) { // No more spaces, but still need order for (int i = 0; i < _max_order - order - 1; i++) { - ngram.push_back(""); + ngram.push_back(START_TOKEN); } break; } @@ -288,10 +251,6 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { return ngram; } -//--------------------------------------------------------- -// Helper function to populate Trie with a vocab using the -// char_list for maping from string to int -//--------------------------------------------------------- void Scorer::fill_dictionary(bool add_space) { fst::StdVectorFst dictionary; @@ -307,7 +266,7 @@ void Scorer::fill_dictionary(bool add_space) { bool added = add_word_to_dictionary(word, char_map, add_space, - _SPACE, + _SPACE_ID, &dictionary); vocab_size += added ? 1 : 0; } diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 9ba55dd6de..17a5f1aa6c 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -11,7 +11,7 @@ #include "util/string_piece.hh" #include "path_trie.h" -const double OOV_SCOER = -1000.0; +const double OOV_SCORE = -1000.0; const std::string START_TOKEN = ""; const std::string UNK_TOKEN = ""; const std::string END_TOKEN = ""; @@ -68,18 +68,13 @@ class Scorer{ double get_log_prob(const std::vector& words); std::string vec2str(const std::vector &input); std::vector split_labels(const std::vector &labels); - std::vector split_str(const std::string &s, - const std::string &delim); private: - void _init_char_list(); - void _init_char_map(); - void* _language_model; bool _is_character_based; size_t _max_order; - unsigned int _SPACE; + int _SPACE_ID; std::vector _char_list; std::unordered_map _char_map; From 09f4c6e117cee7eee20cec9a095c6d8a17819d6d Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Tue, 29 Aug 2017 19:22:52 +0800 Subject: [PATCH 19/36] remove unused functions in Scorer --- deep_speech_2/deploy/ctc_decoders.cpp | 6 +- deep_speech_2/deploy/scorer.cpp | 85 ++------------------------- deep_speech_2/deploy/scorer.h | 9 +-- 3 files changed, 8 insertions(+), 92 deletions(-) diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index da37708af5..9304c780b1 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -96,13 +96,13 @@ std::vector > prefixes.push_back(&root); if ( ext_scorer != nullptr && !ext_scorer->is_character_based()) { - if (ext_scorer->_dictionary == nullptr) { + if (ext_scorer->dictionary == nullptr) { // TODO: init dictionary ext_scorer->set_char_map(vocabulary); // add_space should be true? ext_scorer->fill_dictionary(true); } - auto fst_dict = static_cast(ext_scorer->_dictionary); + auto fst_dict = static_cast(ext_scorer->dictionary); fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); root.set_dictionary(dict_ptr); auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); @@ -285,7 +285,7 @@ std::vector > > // dictionary init if ( ext_scorer != nullptr && !ext_scorer->is_character_based() - && ext_scorer->_dictionary == nullptr) { + && ext_scorer->dictionary == nullptr) { // init dictionary ext_scorer->set_char_map(vocabulary); ext_scorer->fill_dictionary(true); diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index 41f3894ab5..ced71995ba 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -15,7 +15,7 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { this->beta = beta; _is_character_based = true; _language_model = nullptr; - _dictionary = nullptr; + dictionary = nullptr; _max_order = 0; _SPACE_ID = -1; // load language model @@ -25,8 +25,8 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { Scorer::~Scorer() { if (_language_model != nullptr) delete static_cast(_language_model); - if (_dictionary != nullptr) - delete static_cast(_dictionary); + if (dictionary != nullptr) + delete static_cast(dictionary); } void Scorer::load_LM(const char* filename) { @@ -99,87 +99,11 @@ double Scorer::get_log_prob(const std::vector& words) { return score; } -/* Strip a input sentence - * Parameters: - * str: A reference to the objective string - * ch: The character to prune - * Return: - * void - */ -inline void strip(std::string &str, char ch=' ') { - if (str.size() == 0) return; - int start = 0; - int end = str.size()-1; - for (int i=0; i=0; i--) { - if (str[i] == ch) { - end --; - } else { - break; - } - } - - if (start == 0 && end == str.size()-1) return; - if (start > end) { - std::string emp_str; - str = emp_str; - } else { - str = str.substr(start, end-start+1); - } -} - -int Scorer::word_count(std::string sentence) { - strip(sentence); - int cnt = 1; - for (int i=0; i_language_model; - State state, out_state; - lm::FullScoreReturn ret; - model->BeginSentenceWrite(&state); - - for (util::TokenIter it(sentence, ' '); it; ++it){ - lm::WordIndex wid = model->BaseVocabulary().Index(*it); - ret = model->BaseFullScore(&state, wid, &out_state); - state = out_state; - } - //log10 prob - double log_prob = ret.prob; - return log_prob; -} - void Scorer::reset_params(float alpha, float beta) { this->alpha = alpha; this->beta = beta; } -double Scorer::get_score(std::string sentence, bool log) { - double lm_score = get_log_cond_prob(sentence); - int word_cnt = word_count(sentence); - - double final_score = 0.0; - if (log == false) { - final_score = pow(10, alpha * lm_score) * pow(word_cnt, beta); - } else { - final_score = alpha * lm_score * std::log(10) - + beta * std::log(word_cnt); - } - return final_score; -} - std::string Scorer::vec2str(const std::vector& input) { std::string word; for (auto ind : input) { @@ -188,7 +112,6 @@ std::string Scorer::vec2str(const std::vector& input) { return word; } - std::vector Scorer::split_labels(const std::vector &labels) { if (labels.empty()) @@ -291,6 +214,6 @@ void Scorer::fill_dictionary(bool add_space) { // Finds the simplest equivalent fst. This is unnecessary but decreases // memory usage of the dictionary fst::Minimize(new_dict); - _dictionary = new_dict; + this->dictionary = new_dict; } diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 17a5f1aa6c..e5bfecaf86 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -42,15 +42,8 @@ class Scorer{ double get_sent_log_prob(const std::vector& words); size_t get_max_order() { return _max_order; } bool is_character_based() { return _is_character_based; } - std::vector get_vocab() { return _vocabulary; } - // word insertion term - int word_count(std::string); - // get the log cond prob of the last word - double get_log_cond_prob(std::string); // reset params alpha & beta void reset_params(float alpha, float beta); - // get the final score - double get_score(std::string, bool log=false); // make ngram std::vector make_ngram(PathTrie* prefix); // fill dictionary for fst @@ -61,7 +54,7 @@ class Scorer{ double alpha; double beta; // fst dictionary - void* _dictionary; + void* dictionary; protected: void load_LM(const char* filename); From b5c4d83218db152189ae2d0c3e161e026839b8e3 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 30 Aug 2017 13:01:44 +0800 Subject: [PATCH 20/36] add min cutoff & top n cutoff --- deep_speech_2/deploy.py | 14 +++- deep_speech_2/deploy/ctc_decoders.cpp | 71 ++++++++++++------- deep_speech_2/deploy/ctc_decoders.h | 2 + deep_speech_2/deploy/scorer.h | 2 +- deep_speech_2/deploy/swig_decoders_wrapper.py | 22 ++++-- 5 files changed, 75 insertions(+), 36 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index d43ab1e0fc..60bdcb0c57 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -18,7 +18,7 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--num_samples", - default=4, + default=10, type=int, help="Number of samples for inference. (default: %(default)s)") parser.add_argument( @@ -95,12 +95,12 @@ help="Path for language model. (default: %(default)s)") parser.add_argument( "--alpha", - default=0.26, + default=1.5, type=float, help="Parameter associated with language model. (default: %(default)f)") parser.add_argument( "--beta", - default=0.1, + default=0.3, type=float, help="Parameter associated with word count. (default: %(default)f)") parser.add_argument( @@ -109,6 +109,12 @@ type=float, help="The cutoff probability of pruning" "in beam search. (default: %(default)f)") +parser.add_argument( + "--cutoff_top_n", + default=40, + type=int, + help="The cutoff number of pruning" + "in beam search. (default: %(default)f)") args = parser.parse_args() @@ -184,6 +190,7 @@ def infer(): vocabulary=data_generator.vocab_list, blank_id=len(data_generator.vocab_list), cutoff_prob=args.cutoff_prob, + cutoff_top_n=args.cutoff_top_n, ext_scoring_func=ext_scorer, ) batch_beam_results += [beam_result] else: @@ -194,6 +201,7 @@ def infer(): blank_id=len(data_generator.vocab_list), num_processes=args.num_processes_beam_search, cutoff_prob=args.cutoff_prob, + cutoff_top_n=args.cutoff_top_n, ext_scoring_func=ext_scorer, ) for i, beam_result in enumerate(batch_beam_results): diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 9304c780b1..7933b01d04 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -62,6 +62,7 @@ std::vector > std::vector vocabulary, int blank_id, double cutoff_prob, + int cutoff_top_n, Scorer *ext_scorer) { // dimension check @@ -116,19 +117,33 @@ std::vector > prob_idx.push_back(std::pair(i, prob[i])); } + float min_cutoff = -NUM_FLT_INF; + bool full_beam = false; + if (ext_scorer != nullptr) { + int num_prefixes = std::min((int)prefixes.size(), beam_size); + std::sort(prefixes.begin(), prefixes.begin() + num_prefixes, + prefix_compare); + min_cutoff = prefixes[num_prefixes-1]->_score + log(prob[blank_id]) + - std::max(0.0, ext_scorer->beta); + full_beam = (num_prefixes == beam_size); + } + // pruning of vacobulary int cutoff_len = prob.size(); - if (cutoff_prob < 1.0) { + if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) { std::sort(prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); - double cum_prob = 0.0; - cutoff_len = 0; - for (int i=0; i= cutoff_prob) break; + if (cutoff_prob < 1.0) { + double cum_prob = 0.0; + cutoff_len = 0; + for (int i=0; i= cutoff_prob) break; + } } + cutoff_len = std::min(cutoff_len, cutoff_top_n); prob_idx = std::vector >( prob_idx.begin(), prob_idx.begin() + cutoff_len); } @@ -138,15 +153,17 @@ std::vector > log_prob_idx.push_back(std::pair (prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); } - // loop over chars for (int index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; float log_prob_c = log_prob_idx[index].second; - //float log_probs_prev; for (int i = 0; i < prefixes.size() && i_score < min_cutoff) { + break; + } // blank if (c == blank_id) { prefix->_log_prob_b_cur = log_sum_exp( @@ -178,7 +195,7 @@ std::vector > (c == space_id || ext_scorer->is_character_based()) ) { PathTrie *prefix_to_score = nullptr; - // don't score the space + // skip scoring the space if (ext_scorer->is_character_based()) { prefix_to_score = prefix_new; } else { @@ -202,10 +219,10 @@ std::vector > } // end of loop over chars prefixes.clear(); - // update log probabilities + // update log probs root.iterate_to_vec(prefixes); - // sort prefixes by score + // preserve top beam_size prefixes if (prefixes.size() >= beam_size) { std::nth_element(prefixes.begin(), prefixes.begin() + beam_size, @@ -218,18 +235,20 @@ std::vector > } } + // compute aproximate ctc score as the return score for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { double approx_ctc = prefixes[i]->_score; - // remove word insert: - std::vector output; - prefixes[i]->get_path_vec(output); - size_t prefix_length = output.size(); - // remove language model weight: if (ext_scorer != nullptr) { - // auto words = split_labels(output); - // approx_ctc = approx_ctc - path_length * ext_scorer->beta; - // approx_ctc -= (_lm->get_sent_log_prob(words)) * ext_scorer->alpha; + std::vector output; + prefixes[i]->get_path_vec(output); + size_t prefix_length = output.size(); + auto words = ext_scorer->split_labels(output); + // remove word insert + approx_ctc = approx_ctc - prefix_length * ext_scorer->beta; + // remove language model weight: + approx_ctc -= (ext_scorer->get_sent_log_prob(words)) + * ext_scorer->alpha; } prefixes[i]->_approx_ctc = approx_ctc; @@ -253,11 +272,9 @@ std::vector > for (int j = 0; j < output.size(); j++) { output_str += vocabulary[output[j]]; } - std::pair output_pair(space_prefixes[i]->_score, - output_str); - output_vecs.emplace_back( - output_pair - ); + std::pair + output_pair(-space_prefixes[i]->_approx_ctc, output_str); + output_vecs.emplace_back(output_pair); } return output_vecs; @@ -272,6 +289,7 @@ std::vector > > int blank_id, int num_processes, double cutoff_prob, + int cutoff_top_n, Scorer *ext_scorer ) { if (num_processes <= 0) { @@ -295,7 +313,8 @@ std::vector > > for (int i = 0; i < batch_size; i++) { res.emplace_back( pool.enqueue(ctc_beam_search_decoder, probs_split[i], - beam_size, vocabulary, blank_id, cutoff_prob, ext_scorer) + beam_size, vocabulary, blank_id, cutoff_prob, + cutoff_top_n, ext_scorer) ); } // get decoding results diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/deploy/ctc_decoders.h index 2389038205..f339cbd074 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/deploy/ctc_decoders.h @@ -39,6 +39,7 @@ std::vector > std::vector vocabulary, int blank_id, double cutoff_prob=1.0, + int cutoff_top_n=40, Scorer *ext_scorer=NULL ); @@ -66,6 +67,7 @@ std::vector>> int blank_id, int num_processes, double cutoff_prob=1.0, + int cutoff_top_n=40, Scorer *ext_scorer=NULL ); diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index e5bfecaf86..7d7ce430b1 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -50,6 +50,7 @@ class Scorer{ void fill_dictionary(bool add_space); // set char map void set_char_map(std::vector char_list); + std::vector split_labels(const std::vector &labels); // expose to decoder double alpha; double beta; @@ -60,7 +61,6 @@ class Scorer{ void load_LM(const char* filename); double get_log_prob(const std::vector& words); std::string vec2str(const std::vector &input); - std::vector split_labels(const std::vector &labels); private: void* _language_model; diff --git a/deep_speech_2/deploy/swig_decoders_wrapper.py b/deep_speech_2/deploy/swig_decoders_wrapper.py index 51f3173b2b..b44fae0aee 100644 --- a/deep_speech_2/deploy/swig_decoders_wrapper.py +++ b/deep_speech_2/deploy/swig_decoders_wrapper.py @@ -43,6 +43,7 @@ def ctc_beam_search_decoder(probs_seq, vocabulary, blank_id, cutoff_prob=1.0, + cutoff_top_n=40, ext_scoring_func=None): """Wrapper for the CTC Beam Search Decoder. @@ -59,6 +60,10 @@ def ctc_beam_search_decoder(probs_seq, :param cutoff_prob: Cutoff probability in pruning, default 1.0, no pruning. :type cutoff_prob: float + :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n + characters with highest probs in vocabulary will be + used in beam search, default 40. + :type cutoff_top_n: int :param ext_scoring_func: External scoring function for partially decoded sentence, e.g. word count or language model. @@ -67,9 +72,9 @@ def ctc_beam_search_decoder(probs_seq, results, in descending order of the probability. :rtype: list """ - return swig_decoders.ctc_beam_search_decoder(probs_seq.tolist(), beam_size, - vocabulary, blank_id, - cutoff_prob, ext_scoring_func) + return swig_decoders.ctc_beam_search_decoder( + probs_seq.tolist(), beam_size, vocabulary, blank_id, cutoff_prob, + cutoff_top_n, ext_scoring_func) def ctc_beam_search_decoder_batch(probs_split, @@ -78,6 +83,7 @@ def ctc_beam_search_decoder_batch(probs_split, blank_id, num_processes, cutoff_prob=1.0, + cutoff_top_n=40, ext_scoring_func=None): """Wrapper for the batched CTC beam search decoder. @@ -92,11 +98,15 @@ def ctc_beam_search_decoder_batch(probs_split, :type blank_id: int :param num_processes: Number of parallel processes. :type num_processes: int - :param cutoff_prob: Cutoff probability in pruning, + :param cutoff_prob: Cutoff probability in vocabulary pruning, default 1.0, no pruning. + :type cutoff_prob: float + :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n + characters with highest probs in vocabulary will be + used in beam search, default 40. + :type cutoff_top_n: int :param num_processes: Number of parallel processes. :type num_processes: int - :type cutoff_prob: float :param ext_scoring_func: External scoring function for partially decoded sentence, e.g. word count or language model. @@ -109,4 +119,4 @@ def ctc_beam_search_decoder_batch(probs_split, return swig_decoders.ctc_beam_search_decoder_batch( probs_split, beam_size, vocabulary, blank_id, num_processes, - cutoff_prob, ext_scoring_func) + cutoff_prob, cutoff_top_n, ext_scoring_func) From beb0c0753bd0fa814538e53ca2528075877132cb Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 30 Aug 2017 18:29:21 +0800 Subject: [PATCH 21/36] clean up code & update README for decoder in deployment --- deep_speech_2/deploy.py | 43 +++++++++++-------- deep_speech_2/deploy/README.md | 13 ++++-- deep_speech_2/deploy/ctc_decoders.cpp | 57 +++++++++++++++----------- deep_speech_2/deploy/ctc_decoders.h | 6 ++- deep_speech_2/deploy/decoder_utils.cpp | 28 +++---------- deep_speech_2/deploy/decoder_utils.h | 11 ++++- deep_speech_2/deploy/path_trie.cpp | 2 +- deep_speech_2/deploy/scorer.h | 16 +++++++- 8 files changed, 106 insertions(+), 70 deletions(-) diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py index 60bdcb0c57..11972f5f7d 100644 --- a/deep_speech_2/deploy.py +++ b/deep_speech_2/deploy.py @@ -9,7 +9,7 @@ import multiprocessing import paddle.v2 as paddle from data_utils.data import DataGenerator -from model import deep_speech2 +from layer import deep_speech2 from deploy.swig_decoders_wrapper import * from error_rate import wer import utils @@ -79,7 +79,7 @@ "(default: %(default)s)") parser.add_argument( "--beam_size", - default=20, + default=500, type=int, help="Width for beam search decoding. (default: %(default)d)") parser.add_argument( @@ -89,8 +89,7 @@ help="Number of output per sample in beam search. (default: %(default)d)") parser.add_argument( "--language_model_path", - default="/home/work/liuyibing/lm_bak/common_crawl_00.prune01111.trie.klm", - #default="ptb_all.arpa", + default="lm/data/common_crawl_00.prune01111.trie.klm", type=str, help="Path for language model. (default: %(default)s)") parser.add_argument( @@ -136,14 +135,13 @@ def infer(): text_data = paddle.layer.data( name="transcript_text", type=paddle.data_type.integer_value_sequence(data_generator.vocab_size)) - output_probs = deep_speech2( + output_probs, _ = deep_speech2( audio_data=audio_data, text_data=text_data, dict_size=data_generator.vocab_size, num_conv_layers=args.num_conv_layers, num_rnn_layers=args.num_rnn_layers, - rnn_size=args.rnn_layer_size, - is_inference=True) + rnn_size=args.rnn_layer_size) # load parameters parameters = paddle.parameters.Parameters.from_tar( @@ -159,8 +157,10 @@ def infer(): infer_data = batch_reader().next() # run inference - infer_results = paddle.infer( - output_layer=output_probs, parameters=parameters, input=infer_data) + inferer = paddle.inference.Inference( + output_layer=output_probs, parameters=parameters) + infer_results = inferer.infer(input=infer_data) + num_steps = len(infer_results) // len(infer_data) probs_split = [ infer_results[i * num_steps:(i + 1) * num_steps] @@ -178,17 +178,29 @@ def infer(): ext_scorer = Scorer( alpha=args.alpha, beta=args.beta, model_path=args.language_model_path) + # from unicode to string + vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] + + # The below two steps, i.e. setting char map and filling dictionary of + # FST will be completed implicitly when ext_scorer first used.But to save + # the time of decoding the first audio sample, they are done in advance. + ext_scorer.set_char_map(vocab_list) + # only for ward based language model + ext_scorer.fill_dictionary(True) + + # for word error rate metric + wer_sum, wer_counter = 0.0, 0 + ## decode and print time_begin = time.time() - wer_sum, wer_counter = 0, 0 batch_beam_results = [] if args.decode_method == 'beam_search': for i, probs in enumerate(probs_split): beam_result = ctc_beam_search_decoder( probs_seq=probs, beam_size=args.beam_size, - vocabulary=data_generator.vocab_list, - blank_id=len(data_generator.vocab_list), + vocabulary=vocab_list, + blank_id=len(vocab_list), cutoff_prob=args.cutoff_prob, cutoff_top_n=args.cutoff_top_n, ext_scoring_func=ext_scorer, ) @@ -197,8 +209,8 @@ def infer(): batch_beam_results = ctc_beam_search_decoder_batch( probs_split=probs_split, beam_size=args.beam_size, - vocabulary=data_generator.vocab_list, - blank_id=len(data_generator.vocab_list), + vocabulary=vocab_list, + blank_id=len(vocab_list), num_processes=args.num_processes_beam_search, cutoff_prob=args.cutoff_prob, cutoff_top_n=args.cutoff_top_n, @@ -213,8 +225,7 @@ def infer(): print("cur wer = %f , average wer = %f" % (wer_cur, wer_sum / wer_counter)) - time_end = time.time() - print("total time = %f" % (time_end - time_begin)) + print("time for decoding = %f" % (time.time() - time_begin)) def main(): diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/deploy/README.md index 9f2be76e88..e817be105d 100644 --- a/deep_speech_2/deploy/README.md +++ b/deep_speech_2/deploy/README.md @@ -1,5 +1,9 @@ + +The decoders for deployment developed in C++ are a better alternative for the prototype decoders in Pytthon, with more powerful performance in both speed and accuracy. + ### Installation -The build of the decoder for deployment depends on several open-sourced projects, first clone or download them to current directory (i.e., `deep_speech_2/deploy`) + +The build depends on several open-sourced projects, first clone or download them to current directory (i.e., `deep_speech_2/deploy`) - [**KenLM**](https://github.com/kpu/kenlm/): Faster and Smaller Language Model Queries @@ -14,7 +18,6 @@ wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz tar -xzvf openfst-1.6.3.tar.gz ``` -- [**SWIG**](http://www.swig.org): Compiling for python interface requires swig, please make sure swig being installed. - [**ThreadPool**](http://progsch.net/wordpress/): A library for C++ thread pool @@ -22,6 +25,8 @@ tar -xzvf openfst-1.6.3.tar.gz git clone https://github.com/progschj/ThreadPool.git ``` +- [**SWIG**](http://www.swig.org): A tool that provides the Python interface for the decoders, please make sure it being installed. + Then run the setup ```shell @@ -29,7 +34,9 @@ python setup.py install --num_processes 4 cd .. ``` -### Deployment +### Usage + +The decoders for deployment share almost the same interface with the prototye decoders in Python. After the installation succeeds, these decoders are very convenient for call in Python, and a complete example in ```deploy.py``` can be refered. For GPU deployment diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 7933b01d04..4e94edfbbc 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -90,26 +90,32 @@ std::vector > space_id = -2; } - // init + // init prefixes' root PathTrie root; root._score = root._log_prob_b_prev = 0.0; std::vector prefixes; prefixes.push_back(&root); - if ( ext_scorer != nullptr && !ext_scorer->is_character_based()) { - if (ext_scorer->dictionary == nullptr) { - // TODO: init dictionary + if ( ext_scorer != nullptr) { + if (ext_scorer->is_char_map_empty()) { ext_scorer->set_char_map(vocabulary); - // add_space should be true? - ext_scorer->fill_dictionary(true); } - auto fst_dict = static_cast(ext_scorer->dictionary); - fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); - root.set_dictionary(dict_ptr); - auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); - root.set_matcher(matcher); + if (!ext_scorer->is_character_based()) { + if (ext_scorer->dictionary == nullptr) { + // fill dictionary for fst + ext_scorer->fill_dictionary(true); + } + auto fst_dict = static_cast + (ext_scorer->dictionary); + fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); + root.set_dictionary(dict_ptr); + auto matcher = std::make_shared + (*dict_ptr, fst::MATCH_INPUT); + root.set_matcher(matcher); + } } + // prefix search over time for (int time_step = 0; time_step < num_time_steps; time_step++) { std::vector prob = probs_seq[time_step]; std::vector > prob_idx; @@ -147,12 +153,12 @@ std::vector > prob_idx = std::vector >( prob_idx.begin(), prob_idx.begin() + cutoff_len); } - std::vector > log_prob_idx; for (int i = 0; i < cutoff_len; i++) { log_prob_idx.push_back(std::pair (prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); } + // loop over chars for (int index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; @@ -214,15 +220,14 @@ std::vector > prefix_new->_log_prob_nb_cur = log_sum_exp( prefix_new->_log_prob_nb_cur, log_p); } - } - + } // end of loop over prefix } // end of loop over chars prefixes.clear(); // update log probs root.iterate_to_vec(prefixes); - // preserve top beam_size prefixes + // only preserve top beam_size prefixes if (prefixes.size() >= beam_size) { std::nth_element(prefixes.begin(), prefixes.begin() + beam_size, @@ -233,7 +238,7 @@ std::vector > prefixes[i]->remove(); } } - } + } // end of loop over time // compute aproximate ctc score as the return score for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { @@ -300,14 +305,19 @@ std::vector > > ThreadPool pool(num_processes); // number of samples int batch_size = probs_split.size(); - // dictionary init - if ( ext_scorer != nullptr - && !ext_scorer->is_character_based() - && ext_scorer->dictionary == nullptr) { - // init dictionary - ext_scorer->set_char_map(vocabulary); - ext_scorer->fill_dictionary(true); + + // scorer filling up + if ( ext_scorer != nullptr) { + if (ext_scorer->is_char_map_empty()) { + ext_scorer->set_char_map(vocabulary); + } + if(!ext_scorer->is_character_based() + && ext_scorer->dictionary == nullptr) { + // init dictionary + ext_scorer->fill_dictionary(true); + } } + // enqueue the tasks of decoding std::vector>>> res; for (int i = 0; i < batch_size; i++) { @@ -317,6 +327,7 @@ std::vector > > cutoff_top_n, ext_scorer) ); } + // get decoding results std::vector > > batch_results; for (int i = 0; i < batch_size; i++) { diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/deploy/ctc_decoders.h index f339cbd074..58d2b7895e 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/deploy/ctc_decoders.h @@ -27,7 +27,8 @@ std::string ctc_best_path_decoder(std::vector > probs_seq, * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. * blank_id: ID of blank. - * cutoff_prob: Cutoff probability of pruning + * cutoff_prob: Cutoff probability for pruning. + * cutoff_top_n: Cutoff number for pruning. * ext_scorer: External scorer to evaluate a prefix. * Return: * A vector that each element is a pair of score and decoding result, @@ -54,7 +55,8 @@ std::vector > * vocabulary: A vector of vocabulary. * blank_id: ID of blank. * num_processes: Number of threads for beam search. - * cutoff_prob: Cutoff probability of pruning + * cutoff_prob: Cutoff probability for pruning. + * cutoff_top_n: Cutoff number for pruning. * ext_scorer: External scorer to evaluate a prefix. * Return: * A 2-D vector that each element is a vector of decoding result for one diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index 39beb811e1..37674f71e6 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -11,10 +11,6 @@ size_t get_utf8_str_len(const std::string& str) { return str_len; } -//------------------------------------------------------ -//Splits string into vector of strings representing -//UTF-8 characters (not same as chars) -//------------------------------------------------------ std::vector split_utf8_str(const std::string& str) { std::vector result; @@ -37,9 +33,6 @@ std::vector split_utf8_str(const std::string& str) return result; } -// Split a string into a list of strings on a given string -// delimiter. NB: delimiters on beginning / end of string are -// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. std::vector split_str(const std::string &s, const std::string &delim) { std::vector result; @@ -60,9 +53,6 @@ std::vector split_str(const std::string &s, return result; } -//------------------------------------------------------- -// Overriding less than operator for sorting -//------------------------------------------------------- bool prefix_compare(const PathTrie* x, const PathTrie* y) { if (x->_score == y->_score) { if (x->_character == y->_character) { @@ -73,11 +63,8 @@ bool prefix_compare(const PathTrie* x, const PathTrie* y) { } else { return x->_score > y->_score; } -} //---------- End path_compare --------------------------- +} -// -------------------------------------------------------------- -// Adds word to fst without copying entire dictionary -// -------------------------------------------------------------- void add_word_to_fst(const std::vector& word, fst::StdVectorFst* dictionary) { if (dictionary->NumStates() == 0) { @@ -93,15 +80,12 @@ void add_word_to_fst(const std::vector& word, src = dst; } dictionary->SetFinal(dst, fst::StdArc::Weight::One()); -} // ------------ End of add_word_to_fst ----------------------- +} -// --------------------------------------------------------- -// Adds a word to the dictionary FST based on char_map -// --------------------------------------------------------- bool add_word_to_dictionary(const std::string& word, const std::unordered_map& char_map, bool add_space, - int SPACE, + int SPACE_ID, fst::StdVectorFst* dictionary) { auto characters = split_utf8_str(word); @@ -109,7 +93,7 @@ bool add_word_to_dictionary(const std::string& word, for (auto& c : characters) { if (c == " ") { - int_word.push_back(SPACE); + int_word.push_back(SPACE_ID); } else { auto int_c = char_map.find(c); if (int_c != char_map.end()) { @@ -121,9 +105,9 @@ bool add_word_to_dictionary(const std::string& word, } if (add_space) { - int_word.push_back(SPACE); + int_word.push_back(SPACE_ID); } add_word_to_fst(int_word, dictionary); return true; -} // -------------- End of addWordToDictionary ------------ +} diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index 936605868f..829ea76d0c 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -7,6 +7,7 @@ const float NUM_FLT_INF = std::numeric_limits::max(); const float NUM_FLT_MIN = std::numeric_limits::min(); +// Function template for comparing two pairs template bool pair_comp_first_rev(const std::pair &a, const std::pair &b) @@ -31,7 +32,6 @@ T log_sum_exp(const T &x, const T &y) return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; } - // Functor for prefix comparsion bool prefix_compare(const PathTrie* x, const PathTrie* y); @@ -39,17 +39,24 @@ bool prefix_compare(const PathTrie* x, const PathTrie* y); // See: http://stackoverflow.com/a/4063229 size_t get_utf8_str_len(const std::string& str); +// Split a string into a list of strings on a given string +// delimiter. NB: delimiters on beginning / end of string are +// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. std::vector split_str(const std::string &s, const std::string &delim); +// Splits string into vector of strings representing +// UTF-8 characters (not same as chars) std::vector split_utf8_str(const std::string &str); +// Add a word in index to the dicionary of fst void add_word_to_fst(const std::vector& word, fst::StdVectorFst* dictionary); +// Add a word in string to dictionary bool add_word_to_dictionary(const std::string& word, const std::unordered_map& char_map, bool add_space, - int SPACE, + int SPACE_ID, fst::StdVectorFst* dictionary); #endif // DECODER_UTILS_H diff --git a/deep_speech_2/deploy/path_trie.cpp b/deep_speech_2/deploy/path_trie.cpp index b841831d71..b22f2a4713 100644 --- a/deep_speech_2/deploy/path_trie.cpp +++ b/deep_speech_2/deploy/path_trie.cpp @@ -86,7 +86,7 @@ PathTrie* PathTrie::get_path_vec(std::vector& output) { PathTrie* PathTrie::get_path_vec(std::vector& output, int stop, - size_t max_steps /*= std::numeric_limits::max() */) { + size_t max_steps) { if (_character == stop || _character == _ROOT || output.size() == max_steps) { diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index 7d7ce430b1..e3d61a71c1 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -32,34 +32,48 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { // Example: // Scorer scorer(alpha, beta, "path_of_language_model"); // scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); -// scorer.get_log_cond_prob("this a sentence"); // scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); class Scorer{ public: Scorer(double alpha, double beta, const std::string& lm_path); ~Scorer(); + double get_log_cond_prob(const std::vector& words); + double get_sent_log_prob(const std::vector& words); + size_t get_max_order() { return _max_order; } + + bool is_char_map_empty() {return _char_map.size() == 0; } + bool is_character_based() { return _is_character_based; } + // reset params alpha & beta void reset_params(float alpha, float beta); + // make ngram std::vector make_ngram(PathTrie* prefix); + // fill dictionary for fst void fill_dictionary(bool add_space); + // set char map void set_char_map(std::vector char_list); + std::vector split_labels(const std::vector &labels); + // expose to decoder double alpha; double beta; + // fst dictionary void* dictionary; protected: void load_LM(const char* filename); + double get_log_prob(const std::vector& words); + std::string vec2str(const std::vector &input); private: From 103a6ac5f9bc1aaa1754271138f2e00b93f9bfbe Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 6 Sep 2017 18:18:53 +0800 Subject: [PATCH 22/36] format C++ source code --- deep_speech_2/deploy/ctc_decoders.cpp | 592 ++++++++++++------------- deep_speech_2/deploy/ctc_decoders.h | 44 +- deep_speech_2/deploy/decoder_utils.cpp | 160 ++++--- deep_speech_2/deploy/decoder_utils.h | 44 +- deep_speech_2/deploy/path_trie.cpp | 209 +++++---- deep_speech_2/deploy/path_trie.h | 62 ++- deep_speech_2/deploy/scorer.cpp | 331 +++++++------- deep_speech_2/deploy/scorer.h | 86 ++-- 8 files changed, 749 insertions(+), 779 deletions(-) diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/deploy/ctc_decoders.cpp index 4e94edfbbc..cedb943ea2 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/deploy/ctc_decoders.cpp @@ -1,337 +1,329 @@ -#include -#include +#include "ctc_decoders.h" #include -#include #include +#include #include -#include "fst/fstlib.h" -#include "ctc_decoders.h" +#include +#include +#include "ThreadPool.h" #include "decoder_utils.h" +#include "fst/fstlib.h" #include "path_trie.h" -#include "ThreadPool.h" -std::string ctc_best_path_decoder(std::vector > probs_seq, - std::vector vocabulary) -{ - // dimension check - int num_time_steps = probs_seq.size(); - for (int i=0; i> probs_seq, + std::vector vocabulary) { + // dimension check + int num_time_steps = probs_seq.size(); + for (int i = 0; i < num_time_steps; i++) { + if (probs_seq[i].size() != vocabulary.size() + 1) { + std::cout << "The shape of probs_seq does not match" + << " with the shape of the vocabulary!" << std::endl; + exit(1); } - - int blank_id = vocabulary.size(); - - std::vector max_idx_vec; - double max_prob = 0.0; - int max_idx = 0; - for (int i = 0; i < num_time_steps; i++) { - for (int j = 0; j < probs_seq[i].size(); j++) { - if (max_prob < probs_seq[i][j]) { - max_idx = j; - max_prob = probs_seq[i][j]; - } - } - max_idx_vec.push_back(max_idx); - max_prob = 0.0; - max_idx = 0; + } + + int blank_id = vocabulary.size(); + + std::vector max_idx_vec; + double max_prob = 0.0; + int max_idx = 0; + for (int i = 0; i < num_time_steps; i++) { + for (int j = 0; j < probs_seq[i].size(); j++) { + if (max_prob < probs_seq[i][j]) { + max_idx = j; + max_prob = probs_seq[i][j]; + } } - - std::vector idx_vec; - for (int i = 0; i < max_idx_vec.size(); i++) { - if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i-1])) { - idx_vec.push_back(max_idx_vec[i]); - } + max_idx_vec.push_back(max_idx); + max_prob = 0.0; + max_idx = 0; + } + + std::vector idx_vec; + for (int i = 0; i < max_idx_vec.size(); i++) { + if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i - 1])) { + idx_vec.push_back(max_idx_vec[i]); } + } - std::string best_path_result; - for (int i = 0; i < idx_vec.size(); i++) { - if (idx_vec[i] != blank_id) { - best_path_result += vocabulary[idx_vec[i]]; - } + std::string best_path_result; + for (int i = 0; i < idx_vec.size(); i++) { + if (idx_vec[i] != blank_id) { + best_path_result += vocabulary[idx_vec[i]]; } - return best_path_result; + } + return best_path_result; } -std::vector > - ctc_beam_search_decoder(std::vector > probs_seq, - int beam_size, - std::vector vocabulary, - int blank_id, - double cutoff_prob, - int cutoff_top_n, - Scorer *ext_scorer) -{ - // dimension check - int num_time_steps = probs_seq.size(); - for (int i = 0; i < num_time_steps; i++) { - if (probs_seq[i].size() != vocabulary.size() + 1) { - std::cout << " The shape of probs_seq does not match" - << " with the shape of the vocabulary!" << std::endl; - exit(1); - } +std::vector> ctc_beam_search_decoder( + std::vector> probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id, + double cutoff_prob, + int cutoff_top_n, + Scorer *extscorer) { + // dimension check + int num_time_steps = probs_seq.size(); + for (int i = 0; i < num_time_steps; i++) { + if (probs_seq[i].size() != vocabulary.size() + 1) { + std::cout << " The shape of probs_seq does not match" + << " with the shape of the vocabulary!" << std::endl; + exit(1); } - - // blank_id check - if (blank_id > vocabulary.size()) { - std::cout << " Invalid blank_id! " << std::endl; - exit(1); + } + + // blank_id check + if (blank_id > vocabulary.size()) { + std::cout << " Invalid blank_id! " << std::endl; + exit(1); + } + + // assign space ID + std::vector::iterator it = + std::find(vocabulary.begin(), vocabulary.end(), " "); + int space_id = it - vocabulary.begin(); + // if no space in vocabulary + if (space_id >= vocabulary.size()) { + space_id = -2; + } + + // init prefixes' root + PathTrie root; + root.score = root.log_prob_b_prev = 0.0; + std::vector prefixes; + prefixes.push_back(&root); + + if (extscorer != nullptr) { + if (extscorer->is_char_map_empty()) { + extscorer->set_char_map(vocabulary); } - - // assign space ID - std::vector::iterator it = std::find(vocabulary.begin(), - vocabulary.end(), " "); - int space_id = it - vocabulary.begin(); - // if no space in vocabulary - if(space_id >= vocabulary.size()) { - space_id = -2; + if (!extscorer->is_character_based()) { + if (extscorer->dictionary == nullptr) { + // fill dictionary for fst + extscorer->fill_dictionary(true); + } + auto fst_dict = static_cast(extscorer->dictionary); + fst::StdVectorFst *dict_ptr = fst_dict->Copy(true); + root.set_dictionary(dict_ptr); + auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); + root.set_matcher(matcher); + } + } + + // prefix search over time + for (int time_step = 0; time_step < num_time_steps; time_step++) { + std::vector prob = probs_seq[time_step]; + std::vector> prob_idx; + for (int i = 0; i < prob.size(); i++) { + prob_idx.push_back(std::pair(i, prob[i])); } - // init prefixes' root - PathTrie root; - root._score = root._log_prob_b_prev = 0.0; - std::vector prefixes; - prefixes.push_back(&root); + float min_cutoff = -NUM_FLT_INF; + bool full_beam = false; + if (extscorer != nullptr) { + int num_prefixes = std::min((int)prefixes.size(), beam_size); + std::sort( + prefixes.begin(), prefixes.begin() + num_prefixes, prefix_compare); + min_cutoff = prefixes[num_prefixes - 1]->score + log(prob[blank_id]) - + std::max(0.0, extscorer->beta); + full_beam = (num_prefixes == beam_size); + } - if ( ext_scorer != nullptr) { - if (ext_scorer->is_char_map_empty()) { - ext_scorer->set_char_map(vocabulary); - } - if (!ext_scorer->is_character_based()) { - if (ext_scorer->dictionary == nullptr) { - // fill dictionary for fst - ext_scorer->fill_dictionary(true); - } - auto fst_dict = static_cast - (ext_scorer->dictionary); - fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); - root.set_dictionary(dict_ptr); - auto matcher = std::make_shared - (*dict_ptr, fst::MATCH_INPUT); - root.set_matcher(matcher); + // pruning of vacobulary + int cutoff_len = prob.size(); + if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) { + std::sort( + prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + if (cutoff_prob < 1.0) { + double cum_prob = 0.0; + cutoff_len = 0; + for (int i = 0; i < prob_idx.size(); i++) { + cum_prob += prob_idx[i].second; + cutoff_len += 1; + if (cum_prob >= cutoff_prob) break; } + } + cutoff_len = std::min(cutoff_len, cutoff_top_n); + prob_idx = std::vector>( + prob_idx.begin(), prob_idx.begin() + cutoff_len); + } + std::vector> log_prob_idx; + for (int i = 0; i < cutoff_len; i++) { + log_prob_idx.push_back(std::pair( + prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); } - // prefix search over time - for (int time_step = 0; time_step < num_time_steps; time_step++) { - std::vector prob = probs_seq[time_step]; - std::vector > prob_idx; - for (int i=0; i(i, prob[i])); - } + // loop over chars + for (int index = 0; index < log_prob_idx.size(); index++) { + auto c = log_prob_idx[index].first; + float log_prob_c = log_prob_idx[index].second; - float min_cutoff = -NUM_FLT_INF; - bool full_beam = false; - if (ext_scorer != nullptr) { - int num_prefixes = std::min((int)prefixes.size(), beam_size); - std::sort(prefixes.begin(), prefixes.begin() + num_prefixes, - prefix_compare); - min_cutoff = prefixes[num_prefixes-1]->_score + log(prob[blank_id]) - - std::max(0.0, ext_scorer->beta); - full_beam = (num_prefixes == beam_size); - } - - // pruning of vacobulary - int cutoff_len = prob.size(); - if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) { - std::sort(prob_idx.begin(), - prob_idx.end(), - pair_comp_second_rev); - if (cutoff_prob < 1.0) { - double cum_prob = 0.0; - cutoff_len = 0; - for (int i=0; i= cutoff_prob) break; - } - } - cutoff_len = std::min(cutoff_len, cutoff_top_n); - prob_idx = std::vector >( prob_idx.begin(), - prob_idx.begin() + cutoff_len); - } - std::vector > log_prob_idx; - for (int i = 0; i < cutoff_len; i++) { - log_prob_idx.push_back(std::pair - (prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); - } + for (int i = 0; i < prefixes.size() && i < beam_size; i++) { + auto prefix = prefixes[i]; - // loop over chars - for (int index = 0; index < log_prob_idx.size(); index++) { - auto c = log_prob_idx[index].first; - float log_prob_c = log_prob_idx[index].second; - - for (int i = 0; i < prefixes.size() && i_score < min_cutoff) { - break; - } - // blank - if (c == blank_id) { - prefix->_log_prob_b_cur = log_sum_exp( - prefix->_log_prob_b_cur, - log_prob_c + prefix->_score); - continue; - } - // repeated character - if (c == prefix->_character) { - prefix->_log_prob_nb_cur = log_sum_exp( - prefix->_log_prob_nb_cur, - log_prob_c + prefix->_log_prob_nb_prev); - } - // get new prefix - auto prefix_new = prefix->get_path_trie(c); - - if (prefix_new != nullptr) { - float log_p = -NUM_FLT_INF; - - if (c == prefix->_character - && prefix->_log_prob_b_prev > -NUM_FLT_INF) { - log_p = log_prob_c + prefix->_log_prob_b_prev; - } else if (c != prefix->_character) { - log_p = log_prob_c + prefix->_score; - } - - // language model scoring - if (ext_scorer != nullptr && - (c == space_id || ext_scorer->is_character_based()) ) { - PathTrie *prefix_to_score = nullptr; - - // skip scoring the space - if (ext_scorer->is_character_based()) { - prefix_to_score = prefix_new; - } else { - prefix_to_score = prefix; - } - - double score = 0.0; - std::vector ngram; - ngram = ext_scorer->make_ngram(prefix_to_score); - score = ext_scorer->get_log_cond_prob(ngram) * - ext_scorer->alpha; - - log_p += score; - log_p += ext_scorer->beta; - } - prefix_new->_log_prob_nb_cur = log_sum_exp( - prefix_new->_log_prob_nb_cur, log_p); - } - } // end of loop over prefix - } // end of loop over chars - - prefixes.clear(); - // update log probs - root.iterate_to_vec(prefixes); - - // only preserve top beam_size prefixes - if (prefixes.size() >= beam_size) { - std::nth_element(prefixes.begin(), - prefixes.begin() + beam_size, - prefixes.end(), - prefix_compare); - - for (size_t i = beam_size; i < prefixes.size(); i++) { - prefixes[i]->remove(); - } + if (full_beam && log_prob_c + prefix->score < min_cutoff) { + break; } - } // end of loop over time - - // compute aproximate ctc score as the return score - for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { - double approx_ctc = prefixes[i]->_score; - - if (ext_scorer != nullptr) { - std::vector output; - prefixes[i]->get_path_vec(output); - size_t prefix_length = output.size(); - auto words = ext_scorer->split_labels(output); - // remove word insert - approx_ctc = approx_ctc - prefix_length * ext_scorer->beta; - // remove language model weight: - approx_ctc -= (ext_scorer->get_sent_log_prob(words)) - * ext_scorer->alpha; + // blank + if (c == blank_id) { + prefix->log_prob_b_cur = + log_sum_exp(prefix->log_prob_b_cur, log_prob_c + prefix->score); + continue; + } + // repeated character + if (c == prefix->character) { + prefix->log_prob_nb_cur = log_sum_exp( + prefix->log_prob_nb_cur, log_prob_c + prefix->log_prob_nb_prev); } + // get new prefix + auto prefix_new = prefix->get_path_trie(c); + + if (prefix_new != nullptr) { + float log_p = -NUM_FLT_INF; + + if (c == prefix->character && + prefix->log_prob_b_prev > -NUM_FLT_INF) { + log_p = log_prob_c + prefix->log_prob_b_prev; + } else if (c != prefix->character) { + log_p = log_prob_c + prefix->score; + } + + // language model scoring + if (extscorer != nullptr && + (c == space_id || extscorer->is_character_based())) { + PathTrie *prefix_toscore = nullptr; + + // skip scoring the space + if (extscorer->is_character_based()) { + prefix_toscore = prefix_new; + } else { + prefix_toscore = prefix; + } - prefixes[i]->_approx_ctc = approx_ctc; - } + double score = 0.0; + std::vector ngram; + ngram = extscorer->make_ngram(prefix_toscore); + score = extscorer->get_log_cond_prob(ngram) * extscorer->alpha; - // allow for the post processing - std::vector space_prefixes; - if (space_prefixes.empty()) { - for (size_t i = 0; i < beam_size && i< prefixes.size(); i++) { - space_prefixes.push_back(prefixes[i]); + log_p += score; + log_p += extscorer->beta; + } + prefix_new->log_prob_nb_cur = + log_sum_exp(prefix_new->log_prob_nb_cur, log_p); } + } // end of loop over prefix + } // end of loop over chars + + prefixes.clear(); + // update log probs + root.iterate_to_vec(prefixes); + + // only preserve top beam_size prefixes + if (prefixes.size() >= beam_size) { + std::nth_element(prefixes.begin(), + prefixes.begin() + beam_size, + prefixes.end(), + prefix_compare); + + for (size_t i = beam_size; i < prefixes.size(); i++) { + prefixes[i]->remove(); + } } - - std::sort(space_prefixes.begin(), space_prefixes.end(), prefix_compare); - std::vector > output_vecs; - for (size_t i = 0; i < beam_size && i < space_prefixes.size(); i++) { - std::vector output; - space_prefixes[i]->get_path_vec(output); - // convert index to string - std::string output_str; - for (int j = 0; j < output.size(); j++) { - output_str += vocabulary[output[j]]; - } - std::pair - output_pair(-space_prefixes[i]->_approx_ctc, output_str); - output_vecs.emplace_back(output_pair); + } // end of loop over time + + // compute aproximate ctc score as the return score + for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { + double approx_ctc = prefixes[i]->score; + + if (extscorer != nullptr) { + std::vector output; + prefixes[i]->get_path_vec(output); + size_t prefix_length = output.size(); + auto words = extscorer->split_labels(output); + // remove word insert + approx_ctc = approx_ctc - prefix_length * extscorer->beta; + // remove language model weight: + approx_ctc -= (extscorer->get_sent_log_prob(words)) * extscorer->alpha; } - return output_vecs; - } - - -std::vector > > - ctc_beam_search_decoder_batch( - std::vector>> probs_split, - int beam_size, - std::vector vocabulary, - int blank_id, - int num_processes, - double cutoff_prob, - int cutoff_top_n, - Scorer *ext_scorer - ) { - if (num_processes <= 0) { - std::cout << "num_processes must be nonnegative!" << std::endl; - exit(1); + prefixes[i]->approx_ctc = approx_ctc; + } + + // allow for the post processing + std::vector space_prefixes; + if (space_prefixes.empty()) { + for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { + space_prefixes.push_back(prefixes[i]); } - // thread pool - ThreadPool pool(num_processes); - // number of samples - int batch_size = probs_split.size(); - - // scorer filling up - if ( ext_scorer != nullptr) { - if (ext_scorer->is_char_map_empty()) { - ext_scorer->set_char_map(vocabulary); - } - if(!ext_scorer->is_character_based() - && ext_scorer->dictionary == nullptr) { - // init dictionary - ext_scorer->fill_dictionary(true); - } + } + + std::sort(space_prefixes.begin(), space_prefixes.end(), prefix_compare); + std::vector> output_vecs; + for (size_t i = 0; i < beam_size && i < space_prefixes.size(); i++) { + std::vector output; + space_prefixes[i]->get_path_vec(output); + // convert index to string + std::string output_str; + for (int j = 0; j < output.size(); j++) { + output_str += vocabulary[output[j]]; } + std::pair output_pair(-space_prefixes[i]->approx_ctc, + output_str); + output_vecs.emplace_back(output_pair); + } - // enqueue the tasks of decoding - std::vector>>> res; - for (int i = 0; i < batch_size; i++) { - res.emplace_back( - pool.enqueue(ctc_beam_search_decoder, probs_split[i], - beam_size, vocabulary, blank_id, cutoff_prob, - cutoff_top_n, ext_scorer) - ); - } + return output_vecs; +} - // get decoding results - std::vector > > batch_results; - for (int i = 0; i < batch_size; i++) { - batch_results.emplace_back(res[i].get()); +std::vector>> +ctc_beam_search_decoder_batch( + std::vector>> probs_split, + int beam_size, + std::vector vocabulary, + int blank_id, + int num_processes, + double cutoff_prob, + int cutoff_top_n, + Scorer *extscorer) { + if (num_processes <= 0) { + std::cout << "num_processes must be nonnegative!" << std::endl; + exit(1); + } + // thread pool + ThreadPool pool(num_processes); + // number of samples + int batch_size = probs_split.size(); + + // scorer filling up + if (extscorer != nullptr) { + if (extscorer->is_char_map_empty()) { + extscorer->set_char_map(vocabulary); + } + if (!extscorer->is_character_based() && + extscorer->dictionary == nullptr) { + // init dictionary + extscorer->fill_dictionary(true); } - return batch_results; + } + + // enqueue the tasks of decoding + std::vector>>> res; + for (int i = 0; i < batch_size; i++) { + res.emplace_back(pool.enqueue(ctc_beam_search_decoder, + probs_split[i], + beam_size, + vocabulary, + blank_id, + cutoff_prob, + cutoff_top_n, + extscorer)); + } + + // get decoding results + std::vector>> batch_results; + for (int i = 0; i < batch_size; i++) { + batch_results.emplace_back(res[i].get()); + } + return batch_results; } diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/deploy/ctc_decoders.h index 58d2b7895e..78edefb77f 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/deploy/ctc_decoders.h @@ -1,9 +1,9 @@ #ifndef CTC_BEAM_SEARCH_DECODER_H_ #define CTC_BEAM_SEARCH_DECODER_H_ -#include #include #include +#include #include "scorer.h" /* CTC Best Path Decoder @@ -16,8 +16,8 @@ * A vector that each element is a pair of score and decoding result, * in desending order. */ -std::string ctc_best_path_decoder(std::vector > probs_seq, - std::vector vocabulary); +std::string ctc_best_path_decoder(std::vector> probs_seq, + std::vector vocabulary); /* CTC Beam Search Decoder @@ -34,15 +34,14 @@ std::string ctc_best_path_decoder(std::vector > probs_seq, * A vector that each element is a pair of score and decoding result, * in desending order. */ -std::vector > - ctc_beam_search_decoder(std::vector > probs_seq, - int beam_size, - std::vector vocabulary, - int blank_id, - double cutoff_prob=1.0, - int cutoff_top_n=40, - Scorer *ext_scorer=NULL - ); +std::vector> ctc_beam_search_decoder( + std::vector> probs_seq, + int beam_size, + std::vector vocabulary, + int blank_id, + double cutoff_prob = 1.0, + int cutoff_top_n = 40, + Scorer *ext_scorer = NULL); /* CTC Beam Search Decoder for batch data, the interface is consistent with the * original decoder in Python version. @@ -63,15 +62,14 @@ std::vector > * sample. */ std::vector>> - ctc_beam_search_decoder_batch(std::vector>> probs_split, - int beam_size, - std::vector vocabulary, - int blank_id, - int num_processes, - double cutoff_prob=1.0, - int cutoff_top_n=40, - Scorer *ext_scorer=NULL - ); - +ctc_beam_search_decoder_batch( + std::vector>> probs_split, + int beam_size, + std::vector vocabulary, + int blank_id, + int num_processes, + double cutoff_prob = 1.0, + int cutoff_top_n = 40, + Scorer *ext_scorer = NULL); -#endif // CTC_BEAM_SEARCH_DECODER_H_ +#endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/deploy/decoder_utils.cpp index 37674f71e6..bed0f623f9 100644 --- a/deep_speech_2/deploy/decoder_utils.cpp +++ b/deep_speech_2/deploy/decoder_utils.cpp @@ -1,113 +1,111 @@ -#include +#include "decoder_utils.h" #include #include -#include "decoder_utils.h" +#include size_t get_utf8_str_len(const std::string& str) { - size_t str_len = 0; - for (char c : str) { - str_len += ((c & 0xc0) != 0x80); - } - return str_len; + size_t str_len = 0; + for (char c : str) { + str_len += ((c & 0xc0) != 0x80); + } + return str_len; } -std::vector split_utf8_str(const std::string& str) -{ +std::vector split_utf8_str(const std::string& str) { std::vector result; std::string out_str; - for (char c : str) + for (char c : str) { + if ((c & 0xc0) != 0x80) // new UTF-8 character { - if ((c & 0xc0) != 0x80) //new UTF-8 character - { - if (!out_str.empty()) - { - result.push_back(out_str); - out_str.clear(); - } - } - - out_str.append(1, c); + if (!out_str.empty()) { + result.push_back(out_str); + out_str.clear(); + } } + + out_str.append(1, c); + } result.push_back(out_str); return result; } -std::vector split_str(const std::string &s, - const std::string &delim) { - std::vector result; - std::size_t start = 0, delim_len = delim.size(); - while (true) { - std::size_t end = s.find(delim, start); - if (end == std::string::npos) { - if (start < s.size()) { - result.push_back(s.substr(start)); - } - break; - } - if (end > start) { - result.push_back(s.substr(start, end - start)); - } - start = end + delim_len; +std::vector split_str(const std::string& s, + const std::string& delim) { + std::vector result; + std::size_t start = 0, delim_len = delim.size(); + while (true) { + std::size_t end = s.find(delim, start); + if (end == std::string::npos) { + if (start < s.size()) { + result.push_back(s.substr(start)); + } + break; + } + if (end > start) { + result.push_back(s.substr(start, end - start)); } - return result; + start = end + delim_len; + } + return result; } -bool prefix_compare(const PathTrie* x, const PathTrie* y) { - if (x->_score == y->_score) { - if (x->_character == y->_character) { - return false; - } else { - return (x->_character < y->_character); - } +bool prefix_compare(const PathTrie* x, const PathTrie* y) { + if (x->score == y->score) { + if (x->character == y->character) { + return false; } else { - return x->_score > y->_score; + return (x->character < y->character); } + } else { + return x->score > y->score; + } } void add_word_to_fst(const std::vector& word, fst::StdVectorFst* dictionary) { - if (dictionary->NumStates() == 0) { - fst::StdVectorFst::StateId start = dictionary->AddState(); - assert(start == 0); - dictionary->SetStart(start); - } - fst::StdVectorFst::StateId src = dictionary->Start(); - fst::StdVectorFst::StateId dst; - for (auto c : word) { - dst = dictionary->AddState(); - dictionary->AddArc(src, fst::StdArc(c, c, 0, dst)); - src = dst; - } - dictionary->SetFinal(dst, fst::StdArc::Weight::One()); + if (dictionary->NumStates() == 0) { + fst::StdVectorFst::StateId start = dictionary->AddState(); + assert(start == 0); + dictionary->SetStart(start); + } + fst::StdVectorFst::StateId src = dictionary->Start(); + fst::StdVectorFst::StateId dst; + for (auto c : word) { + dst = dictionary->AddState(); + dictionary->AddArc(src, fst::StdArc(c, c, 0, dst)); + src = dst; + } + dictionary->SetFinal(dst, fst::StdArc::Weight::One()); } -bool add_word_to_dictionary(const std::string& word, - const std::unordered_map& char_map, - bool add_space, - int SPACE_ID, - fst::StdVectorFst* dictionary) { - auto characters = split_utf8_str(word); +bool add_word_to_dictionary( + const std::string& word, + const std::unordered_map& char_map, + bool add_space, + int SPACE_ID, + fst::StdVectorFst* dictionary) { + auto characters = split_utf8_str(word); - std::vector int_word; + std::vector int_word; - for (auto& c : characters) { - if (c == " ") { - int_word.push_back(SPACE_ID); - } else { - auto int_c = char_map.find(c); - if (int_c != char_map.end()) { - int_word.push_back(int_c->second); - } else { - return false; // return without adding - } - } + for (auto& c : characters) { + if (c == " ") { + int_word.push_back(SPACE_ID); + } else { + auto int_c = char_map.find(c); + if (int_c != char_map.end()) { + int_word.push_back(int_c->second); + } else { + return false; // return without adding + } } + } - if (add_space) { - int_word.push_back(SPACE_ID); - } + if (add_space) { + int_word.push_back(SPACE_ID); + } - add_word_to_fst(int_word, dictionary); - return true; + add_word_to_fst(int_word, dictionary); + return true; } diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/deploy/decoder_utils.h index 829ea76d0c..51985c86eb 100644 --- a/deep_speech_2/deploy/decoder_utils.h +++ b/deep_speech_2/deploy/decoder_utils.h @@ -10,34 +10,31 @@ const float NUM_FLT_MIN = std::numeric_limits::min(); // Function template for comparing two pairs template bool pair_comp_first_rev(const std::pair &a, - const std::pair &b) -{ - return a.first > b.first; + const std::pair &b) { + return a.first > b.first; } template bool pair_comp_second_rev(const std::pair &a, - const std::pair &b) -{ - return a.second > b.second; + const std::pair &b) { + return a.second > b.second; } template -T log_sum_exp(const T &x, const T &y) -{ - static T num_min = -std::numeric_limits::max(); - if (x <= num_min) return y; - if (y <= num_min) return x; - T xmax = std::max(x, y); - return std::log(std::exp(x-xmax) + std::exp(y-xmax)) + xmax; +T log_sum_exp(const T &x, const T &y) { + static T num_min = -std::numeric_limits::max(); + if (x <= num_min) return y; + if (y <= num_min) return x; + T xmax = std::max(x, y); + return std::log(std::exp(x - xmax) + std::exp(y - xmax)) + xmax; } // Functor for prefix comparsion -bool prefix_compare(const PathTrie* x, const PathTrie* y); +bool prefix_compare(const PathTrie *x, const PathTrie *y); // Get length of utf8 encoding string // See: http://stackoverflow.com/a/4063229 -size_t get_utf8_str_len(const std::string& str); +size_t get_utf8_str_len(const std::string &str); // Split a string into a list of strings on a given string // delimiter. NB: delimiters on beginning / end of string are @@ -50,13 +47,14 @@ std::vector split_str(const std::string &s, std::vector split_utf8_str(const std::string &str); // Add a word in index to the dicionary of fst -void add_word_to_fst(const std::vector& word, - fst::StdVectorFst* dictionary); +void add_word_to_fst(const std::vector &word, + fst::StdVectorFst *dictionary); // Add a word in string to dictionary -bool add_word_to_dictionary(const std::string& word, - const std::unordered_map& char_map, - bool add_space, - int SPACE_ID, - fst::StdVectorFst* dictionary); -#endif // DECODER_UTILS_H +bool add_word_to_dictionary( + const std::string &word, + const std::unordered_map &char_map, + bool add_space, + int SPACE_ID, + fst::StdVectorFst *dictionary); +#endif // DECODER_UTILS_H diff --git a/deep_speech_2/deploy/path_trie.cpp b/deep_speech_2/deploy/path_trie.cpp index b22f2a4713..db0b20cb5d 100644 --- a/deep_speech_2/deploy/path_trie.cpp +++ b/deep_speech_2/deploy/path_trie.cpp @@ -4,145 +4,142 @@ #include #include -#include "path_trie.h" #include "decoder_utils.h" +#include "path_trie.h" PathTrie::PathTrie() { - _log_prob_b_prev = -NUM_FLT_INF; - _log_prob_nb_prev = -NUM_FLT_INF; - _log_prob_b_cur = -NUM_FLT_INF; - _log_prob_nb_cur = -NUM_FLT_INF; - _score = -NUM_FLT_INF; - - _ROOT = -1; - _character = _ROOT; - _exists = true; - _parent = nullptr; - _dictionary = nullptr; - _dictionary_state = 0; - _has_dictionary = false; - _matcher = nullptr; // finds arcs in FST + log_prob_b_prev = -NUM_FLT_INF; + log_prob_nb_prev = -NUM_FLT_INF; + log_prob_b_cur = -NUM_FLT_INF; + log_prob_nb_cur = -NUM_FLT_INF; + score = -NUM_FLT_INF; + + _ROOT = -1; + character = _ROOT; + _exists = true; + parent = nullptr; + _dictionary = nullptr; + _dictionary_state = 0; + _has_dictionary = false; + _matcher = nullptr; // finds arcs in FST } PathTrie::~PathTrie() { - for (auto child : _children) { - delete child.second; - } + for (auto child : _children) { + delete child.second; + } } PathTrie* PathTrie::get_path_trie(int new_char, bool reset) { - auto child = _children.begin(); - for (child = _children.begin(); child != _children.end(); ++child) { - if (child->first == new_char) { - break; - } + auto child = _children.begin(); + for (child = _children.begin(); child != _children.end(); ++child) { + if (child->first == new_char) { + break; } - if ( child != _children.end() ) { - if (!child->second->_exists) { - child->second->_exists = true; - child->second->_log_prob_b_prev = -NUM_FLT_INF; - child->second->_log_prob_nb_prev = -NUM_FLT_INF; - child->second->_log_prob_b_cur = -NUM_FLT_INF; - child->second->_log_prob_nb_cur = -NUM_FLT_INF; + } + if (child != _children.end()) { + if (!child->second->_exists) { + child->second->_exists = true; + child->second->log_prob_b_prev = -NUM_FLT_INF; + child->second->log_prob_nb_prev = -NUM_FLT_INF; + child->second->log_prob_b_cur = -NUM_FLT_INF; + child->second->log_prob_nb_cur = -NUM_FLT_INF; + } + return (child->second); + } else { + if (_has_dictionary) { + _matcher->SetState(_dictionary_state); + bool found = _matcher->Find(new_char); + if (!found) { + // Adding this character causes word outside dictionary + auto FSTZERO = fst::TropicalWeight::Zero(); + auto final_weight = _dictionary->Final(_dictionary_state); + bool is_final = (final_weight != FSTZERO); + if (is_final && reset) { + _dictionary_state = _dictionary->Start(); } - return (child->second); + return nullptr; + } else { + PathTrie* new_path = new PathTrie; + new_path->character = new_char; + new_path->parent = this; + new_path->_dictionary = _dictionary; + new_path->_dictionary_state = _matcher->Value().nextstate; + new_path->_has_dictionary = true; + new_path->_matcher = _matcher; + _children.push_back(std::make_pair(new_char, new_path)); + return new_path; + } } else { - if (_has_dictionary) { - _matcher->SetState(_dictionary_state); - bool found = _matcher->Find(new_char); - if (!found) { - // Adding this character causes word outside dictionary - auto FSTZERO = fst::TropicalWeight::Zero(); - auto final_weight = _dictionary->Final(_dictionary_state); - bool is_final = (final_weight != FSTZERO); - if (is_final && reset) { - _dictionary_state = _dictionary->Start(); - } - return nullptr; - } else { - PathTrie* new_path = new PathTrie; - new_path->_character = new_char; - new_path->_parent = this; - new_path->_dictionary = _dictionary; - new_path->_dictionary_state = _matcher->Value().nextstate; - new_path->_has_dictionary = true; - new_path->_matcher = _matcher; - _children.push_back(std::make_pair(new_char, new_path)); - return new_path; - } - } else { - PathTrie* new_path = new PathTrie; - new_path->_character = new_char; - new_path->_parent = this; - _children.push_back(std::make_pair(new_char, new_path)); - return new_path; - } + PathTrie* new_path = new PathTrie; + new_path->character = new_char; + new_path->parent = this; + _children.push_back(std::make_pair(new_char, new_path)); + return new_path; } + } } PathTrie* PathTrie::get_path_vec(std::vector& output) { - return get_path_vec(output, _ROOT); + return get_path_vec(output, _ROOT); } PathTrie* PathTrie::get_path_vec(std::vector& output, - int stop, - size_t max_steps) { - if (_character == stop || - _character == _ROOT || - output.size() == max_steps) { - std::reverse(output.begin(), output.end()); - return this; - } else { - output.push_back(_character); - return _parent->get_path_vec(output, stop, max_steps); - } + int stop, + size_t max_steps) { + if (character == stop || character == _ROOT || output.size() == max_steps) { + std::reverse(output.begin(), output.end()); + return this; + } else { + output.push_back(character); + return parent->get_path_vec(output, stop, max_steps); + } } -void PathTrie::iterate_to_vec( - std::vector& output) { - if (_exists) { - _log_prob_b_prev = _log_prob_b_cur; - _log_prob_nb_prev = _log_prob_nb_cur; +void PathTrie::iterate_to_vec(std::vector& output) { + if (_exists) { + log_prob_b_prev = log_prob_b_cur; + log_prob_nb_prev = log_prob_nb_cur; - _log_prob_b_cur = -NUM_FLT_INF; - _log_prob_nb_cur = -NUM_FLT_INF; + log_prob_b_cur = -NUM_FLT_INF; + log_prob_nb_cur = -NUM_FLT_INF; - _score = log_sum_exp(_log_prob_b_prev, _log_prob_nb_prev); - output.push_back(this); - } - for (auto child : _children) { - child.second->iterate_to_vec(output); - } + score = log_sum_exp(log_prob_b_prev, log_prob_nb_prev); + output.push_back(this); + } + for (auto child : _children) { + child.second->iterate_to_vec(output); + } } void PathTrie::remove() { - _exists = false; - - if (_children.size() == 0) { - auto child = _parent->_children.begin(); - for (child = _parent->_children.begin(); - child != _parent->_children.end(); ++child) { - if (child->first == _character) { - _parent->_children.erase(child); - break; - } - } - - if ( _parent->_children.size() == 0 && !_parent->_exists ) { - _parent->remove(); - } + _exists = false; + + if (_children.size() == 0) { + auto child = parent->_children.begin(); + for (child = parent->_children.begin(); child != parent->_children.end(); + ++child) { + if (child->first == character) { + parent->_children.erase(child); + break; + } + } - delete this; + if (parent->_children.size() == 0 && !parent->_exists) { + parent->remove(); } + + delete this; + } } void PathTrie::set_dictionary(fst::StdVectorFst* dictionary) { - _dictionary = dictionary; - _dictionary_state = dictionary->Start(); - _has_dictionary = true; + _dictionary = dictionary; + _dictionary_state = dictionary->Start(); + _has_dictionary = true; } using FSTMATCH = fst::SortedMatcher; void PathTrie::set_matcher(std::shared_ptr matcher) { - _matcher = matcher; + _matcher = matcher; } diff --git a/deep_speech_2/deploy/path_trie.h b/deep_speech_2/deploy/path_trie.h index 7b378e3f97..cac524a3f1 100644 --- a/deep_speech_2/deploy/path_trie.h +++ b/deep_speech_2/deploy/path_trie.h @@ -1,59 +1,57 @@ #ifndef PATH_TRIE_H #define PATH_TRIE_H #pragma once +#include #include #include #include #include #include -#include using FSTMATCH = fst::SortedMatcher; class PathTrie { public: - PathTrie(); - ~PathTrie(); - - PathTrie* get_path_trie(int new_char, bool reset = true); + PathTrie(); + ~PathTrie(); - PathTrie* get_path_vec(std::vector &output); + PathTrie* get_path_trie(int new_char, bool reset = true); - PathTrie* get_path_vec(std::vector& output, - int stop, - size_t max_steps = std::numeric_limits::max()); + PathTrie* get_path_vec(std::vector& output); - void iterate_to_vec(std::vector &output); + PathTrie* get_path_vec(std::vector& output, + int stop, + size_t max_steps = std::numeric_limits::max()); - void set_dictionary(fst::StdVectorFst* dictionary); + void iterate_to_vec(std::vector& output); - void set_matcher(std::shared_ptr matcher); + void set_dictionary(fst::StdVectorFst* dictionary); - bool is_empty() { - return _ROOT == _character; - } + void set_matcher(std::shared_ptr matcher); - void remove(); + bool is_empty() { return _ROOT == character; } - float _log_prob_b_prev; - float _log_prob_nb_prev; - float _log_prob_b_cur; - float _log_prob_nb_cur; - float _score; - float _approx_ctc; + void remove(); + float log_prob_b_prev; + float log_prob_nb_prev; + float log_prob_b_cur; + float log_prob_nb_cur; + float score; + float approx_ctc; + int character; + PathTrie* parent; - int _ROOT; - int _character; - bool _exists; +private: + int _ROOT; + bool _exists; - PathTrie *_parent; - std::vector > _children; + std::vector> _children; - fst::StdVectorFst* _dictionary; - fst::StdVectorFst::StateId _dictionary_state; - bool _has_dictionary; - std::shared_ptr _matcher; + fst::StdVectorFst* _dictionary; + fst::StdVectorFst::StateId _dictionary_state; + bool _has_dictionary; + std::shared_ptr _matcher; }; -#endif // PATH_TRIE_H +#endif // PATH_TRIE_H diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/deploy/scorer.cpp index ced71995ba..8651eb61f9 100644 --- a/deep_speech_2/deploy/scorer.cpp +++ b/deep_speech_2/deploy/scorer.cpp @@ -1,219 +1,208 @@ -#include +#include "scorer.h" #include +#include +#include "decoder_utils.h" #include "lm/config.hh" -#include "lm/state.hh" #include "lm/model.hh" -#include "util/tokenize_piece.hh" +#include "lm/state.hh" #include "util/string_piece.hh" -#include "scorer.h" -#include "decoder_utils.h" +#include "util/tokenize_piece.hh" using namespace lm::ngram; Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { - this->alpha = alpha; - this->beta = beta; - _is_character_based = true; - _language_model = nullptr; - dictionary = nullptr; - _max_order = 0; - _SPACE_ID = -1; - // load language model - load_LM(lm_path.c_str()); + this->alpha = alpha; + this->beta = beta; + _is_character_based = true; + _language_model = nullptr; + dictionary = nullptr; + _max_order = 0; + _SPACE_ID = -1; + // load language model + load_LM(lm_path.c_str()); } Scorer::~Scorer() { - if (_language_model != nullptr) - delete static_cast(_language_model); - if (dictionary != nullptr) - delete static_cast(dictionary); + if (_language_model != nullptr) + delete static_cast(_language_model); + if (dictionary != nullptr) delete static_cast(dictionary); } void Scorer::load_LM(const char* filename) { - if (access(filename, F_OK) != 0) { - std::cerr << "Invalid language model file !!!" << std::endl; - exit(1); - } - RetriveStrEnumerateVocab enumerate; - lm::ngram::Config config; - config.enumerate_vocab = &enumerate; - _language_model = lm::ngram::LoadVirtual(filename, config); - _max_order = static_cast(_language_model)->Order(); - _vocabulary = enumerate.vocabulary; - for (size_t i = 0; i < _vocabulary.size(); ++i) { - if (_is_character_based - && _vocabulary[i] != UNK_TOKEN - && _vocabulary[i] != START_TOKEN - && _vocabulary[i] != END_TOKEN - && get_utf8_str_len(enumerate.vocabulary[i]) > 1) { - _is_character_based = false; - } + if (access(filename, F_OK) != 0) { + std::cerr << "Invalid language model file !!!" << std::endl; + exit(1); + } + RetriveStrEnumerateVocab enumerate; + lm::ngram::Config config; + config.enumerate_vocab = &enumerate; + _language_model = lm::ngram::LoadVirtual(filename, config); + _max_order = static_cast(_language_model)->Order(); + _vocabulary = enumerate.vocabulary; + for (size_t i = 0; i < _vocabulary.size(); ++i) { + if (_is_character_based && _vocabulary[i] != UNK_TOKEN && + _vocabulary[i] != START_TOKEN && _vocabulary[i] != END_TOKEN && + get_utf8_str_len(enumerate.vocabulary[i]) > 1) { + _is_character_based = false; } + } } double Scorer::get_log_cond_prob(const std::vector& words) { - lm::base::Model* model = static_cast(_language_model); - double cond_prob; - lm::ngram::State state, tmp_state, out_state; - // avoid to inserting in begin - model->NullContextWrite(&state); - for (size_t i = 0; i < words.size(); ++i) { - lm::WordIndex word_index = model->BaseVocabulary().Index(words[i]); - // encounter OOV - if (word_index == 0) { - return OOV_SCORE; - } - cond_prob = model->BaseScore(&state, word_index, &out_state); - tmp_state = state; - state = out_state; - out_state = tmp_state; + lm::base::Model* model = static_cast(_language_model); + double cond_prob; + lm::ngram::State state, tmp_state, out_state; + // avoid to inserting in begin + model->NullContextWrite(&state); + for (size_t i = 0; i < words.size(); ++i) { + lm::WordIndex word_index = model->BaseVocabulary().Index(words[i]); + // encounter OOV + if (word_index == 0) { + return OOV_SCORE; } - // log10 prob - return cond_prob; + cond_prob = model->BaseScore(&state, word_index, &out_state); + tmp_state = state; + state = out_state; + out_state = tmp_state; + } + // log10 prob + return cond_prob; } double Scorer::get_sent_log_prob(const std::vector& words) { - std::vector sentence; - if (words.size() == 0) { - for (size_t i = 0; i < _max_order; ++i) { - sentence.push_back(START_TOKEN); - } - } else { - for (size_t i = 0; i < _max_order - 1; ++i) { - sentence.push_back(START_TOKEN); - } - sentence.insert(sentence.end(), words.begin(), words.end()); + std::vector sentence; + if (words.size() == 0) { + for (size_t i = 0; i < _max_order; ++i) { + sentence.push_back(START_TOKEN); } - sentence.push_back(END_TOKEN); - return get_log_prob(sentence); + } else { + for (size_t i = 0; i < _max_order - 1; ++i) { + sentence.push_back(START_TOKEN); + } + sentence.insert(sentence.end(), words.begin(), words.end()); + } + sentence.push_back(END_TOKEN); + return get_log_prob(sentence); } double Scorer::get_log_prob(const std::vector& words) { - assert(words.size() > _max_order); - double score = 0.0; - for (size_t i = 0; i < words.size() - _max_order + 1; ++i) { - std::vector ngram(words.begin() + i, - words.begin() + i + _max_order); - score += get_log_cond_prob(ngram); - } - return score; + assert(words.size() > _max_order); + double score = 0.0; + for (size_t i = 0; i < words.size() - _max_order + 1; ++i) { + std::vector ngram(words.begin() + i, + words.begin() + i + _max_order); + score += get_log_cond_prob(ngram); + } + return score; } void Scorer::reset_params(float alpha, float beta) { - this->alpha = alpha; - this->beta = beta; + this->alpha = alpha; + this->beta = beta; } std::string Scorer::vec2str(const std::vector& input) { - std::string word; - for (auto ind : input) { - word += _char_list[ind]; - } - return word; + std::string word; + for (auto ind : input) { + word += _char_list[ind]; + } + return word; } -std::vector -Scorer::split_labels(const std::vector &labels) { - if (labels.empty()) - return {}; - - std::string s = vec2str(labels); - std::vector words; - if (_is_character_based) { - words = split_utf8_str(s); - } else { - words = split_str(s, " "); - } - return words; +std::vector Scorer::split_labels(const std::vector& labels) { + if (labels.empty()) return {}; + + std::string s = vec2str(labels); + std::vector words; + if (_is_character_based) { + words = split_utf8_str(s); + } else { + words = split_str(s, " "); + } + return words; } void Scorer::set_char_map(std::vector char_list) { - _char_list = char_list; - _char_map.clear(); - - for(unsigned int i = 0; i < _char_list.size(); i++) - { - if (_char_list[i] == " ") { - _SPACE_ID = i; - _char_map[' '] = i; - } else if(_char_list[i].size() == 1){ - _char_map[_char_list[i][0]] = i; - } + _char_list = char_list; + _char_map.clear(); + + for (unsigned int i = 0; i < _char_list.size(); i++) { + if (_char_list[i] == " ") { + _SPACE_ID = i; + _char_map[' '] = i; + } else if (_char_list[i].size() == 1) { + _char_map[_char_list[i][0]] = i; } + } } std::vector Scorer::make_ngram(PathTrie* prefix) { - std::vector ngram; - PathTrie* current_node = prefix; - PathTrie* new_node = nullptr; - - for (int order = 0; order < _max_order; order++) { - std::vector prefix_vec; - - if (_is_character_based) { - new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID, 1); - current_node = new_node; - } else { - new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID); - current_node = new_node->_parent; // Skipping spaces - } - - // reconstruct word - std::string word = vec2str(prefix_vec); - ngram.push_back(word); - - if (new_node->_character == -1) { - // No more spaces, but still need order - for (int i = 0; i < _max_order - order - 1; i++) { - ngram.push_back(START_TOKEN); - } - break; - } - } - std::reverse(ngram.begin(), ngram.end()); - return ngram; -} - -void Scorer::fill_dictionary(bool add_space) { + std::vector ngram; + PathTrie* current_node = prefix; + PathTrie* new_node = nullptr; - fst::StdVectorFst dictionary; - // First reverse char_list so ints can be accessed by chars - std::unordered_map char_map; - for (unsigned int i = 0; i < _char_list.size(); i++) { - char_map[_char_list[i]] = i; - } + for (int order = 0; order < _max_order; order++) { + std::vector prefix_vec; - // For each unigram convert to ints and put in trie - int vocab_size = 0; - for (const auto& word : _vocabulary) { - bool added = add_word_to_dictionary(word, - char_map, - add_space, - _SPACE_ID, - &dictionary); - vocab_size += added ? 1 : 0; + if (_is_character_based) { + new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID, 1); + current_node = new_node; + } else { + new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID); + current_node = new_node->parent; // Skipping spaces } - std::cerr << "Vocab Size " << vocab_size << std::endl; - - // Simplify FST - - // This gets rid of "epsilon" transitions in the FST. - // These are transitions that don't require a string input to be taken. - // Getting rid of them is necessary to make the FST determinisitc, but - // can greatly increase the size of the FST - fst::RmEpsilon(&dictionary); - fst::StdVectorFst* new_dict = new fst::StdVectorFst; + // reconstruct word + std::string word = vec2str(prefix_vec); + ngram.push_back(word); - // This makes the FST deterministic, meaning for any string input there's - // only one possible state the FST could be in. It is assumed our - // dictionary is deterministic when using it. - // (lest we'd have to check for multiple transitions at each state) - fst::Determinize(dictionary, new_dict); - - // Finds the simplest equivalent fst. This is unnecessary but decreases - // memory usage of the dictionary - fst::Minimize(new_dict); - this->dictionary = new_dict; + if (new_node->character == -1) { + // No more spaces, but still need order + for (int i = 0; i < _max_order - order - 1; i++) { + ngram.push_back(START_TOKEN); + } + break; + } + } + std::reverse(ngram.begin(), ngram.end()); + return ngram; +} +void Scorer::fill_dictionary(bool add_space) { + fst::StdVectorFst dictionary; + // First reverse char_list so ints can be accessed by chars + std::unordered_map char_map; + for (unsigned int i = 0; i < _char_list.size(); i++) { + char_map[_char_list[i]] = i; + } + + // For each unigram convert to ints and put in trie + int vocab_size = 0; + for (const auto& word : _vocabulary) { + bool added = add_word_to_dictionary( + word, char_map, add_space, _SPACE_ID, &dictionary); + vocab_size += added ? 1 : 0; + } + + std::cerr << "Vocab Size " << vocab_size << std::endl; + + // Simplify FST + + // This gets rid of "epsilon" transitions in the FST. + // These are transitions that don't require a string input to be taken. + // Getting rid of them is necessary to make the FST determinisitc, but + // can greatly increase the size of the FST + fst::RmEpsilon(&dictionary); + fst::StdVectorFst* new_dict = new fst::StdVectorFst; + + // This makes the FST deterministic, meaning for any string input there's + // only one possible state the FST could be in. It is assumed our + // dictionary is deterministic when using it. + // (lest we'd have to check for multiple transitions at each state) + fst::Determinize(dictionary, new_dict); + + // Finds the simplest equivalent fst. This is unnecessary but decreases + // memory usage of the dictionary + fst::Minimize(new_dict); + this->dictionary = new_dict; } diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/deploy/scorer.h index e3d61a71c1..0c78b98707 100644 --- a/deep_speech_2/deploy/scorer.h +++ b/deep_speech_2/deploy/scorer.h @@ -1,31 +1,31 @@ #ifndef SCORER_H_ #define SCORER_H_ -#include #include -#include +#include #include +#include #include "lm/enumerate_vocab.hh" -#include "lm/word_index.hh" #include "lm/virtual_interface.hh" -#include "util/string_piece.hh" +#include "lm/word_index.hh" #include "path_trie.h" +#include "util/string_piece.hh" const double OOV_SCORE = -1000.0; const std::string START_TOKEN = ""; const std::string UNK_TOKEN = ""; const std::string END_TOKEN = ""; - // Implement a callback to retrive string vocabulary. +// Implement a callback to retrive string vocabulary. class RetriveStrEnumerateVocab : public lm::EnumerateVocab { public: - RetriveStrEnumerateVocab() {} + RetriveStrEnumerateVocab() {} - void Add(lm::WordIndex index, const StringPiece& str) { - vocabulary.push_back(std::string(str.data(), str.length())); - } + void Add(lm::WordIndex index, const StringPiece& str) { + vocabulary.push_back(std::string(str.data(), str.length())); + } - std::vector vocabulary; + std::vector vocabulary; }; // External scorer to query languange score for n-gram or sentence. @@ -33,59 +33,59 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { // Scorer scorer(alpha, beta, "path_of_language_model"); // scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); // scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); -class Scorer{ +class Scorer { public: - Scorer(double alpha, double beta, const std::string& lm_path); - ~Scorer(); + Scorer(double alpha, double beta, const std::string& lm_path); + ~Scorer(); - double get_log_cond_prob(const std::vector& words); + double get_log_cond_prob(const std::vector& words); - double get_sent_log_prob(const std::vector& words); + double get_sent_log_prob(const std::vector& words); - size_t get_max_order() { return _max_order; } + size_t get_max_order() { return _max_order; } - bool is_char_map_empty() {return _char_map.size() == 0; } + bool is_char_map_empty() { return _char_map.size() == 0; } - bool is_character_based() { return _is_character_based; } + bool is_character_based() { return _is_character_based; } - // reset params alpha & beta - void reset_params(float alpha, float beta); + // reset params alpha & beta + void reset_params(float alpha, float beta); - // make ngram - std::vector make_ngram(PathTrie* prefix); + // make ngram + std::vector make_ngram(PathTrie* prefix); - // fill dictionary for fst - void fill_dictionary(bool add_space); + // fill dictionary for fst + void fill_dictionary(bool add_space); - // set char map - void set_char_map(std::vector char_list); + // set char map + void set_char_map(std::vector char_list); - std::vector split_labels(const std::vector &labels); + std::vector split_labels(const std::vector& labels); - // expose to decoder - double alpha; - double beta; + // expose to decoder + double alpha; + double beta; - // fst dictionary - void* dictionary; + // fst dictionary + void* dictionary; protected: - void load_LM(const char* filename); + void load_LM(const char* filename); - double get_log_prob(const std::vector& words); + double get_log_prob(const std::vector& words); - std::string vec2str(const std::vector &input); + std::string vec2str(const std::vector& input); private: - void* _language_model; - bool _is_character_based; - size_t _max_order; + void* _language_model; + bool _is_character_based; + size_t _max_order; - int _SPACE_ID; - std::vector _char_list; - std::unordered_map _char_map; + int _SPACE_ID; + std::vector _char_list; + std::unordered_map _char_map; - std::vector _vocabulary; + std::vector _vocabulary; }; -#endif // SCORER_H_ +#endif // SCORER_H_ From 5a318e999d9624c25127b8eb004553b57831b2fd Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 8 Sep 2017 15:20:23 +0800 Subject: [PATCH 23/36] adapt to the new folder structure of DS2 --- deep_speech_2/examples/librispeech/generate.sh | 6 +++--- deep_speech_2/examples/librispeech/run_test.sh | 8 ++++---- deep_speech_2/infer.py | 4 +++- deep_speech_2/models/model.py | 12 ++++++++---- .../{deploy => models/swig_decoders}/README.md | 0 .../{deploy => models/swig_decoders}/__init__.py | 0 .../{deploy => models/swig_decoders}/_init_paths.py | 0 .../swig_decoders}/ctc_decoders.cpp | 4 ++-- .../{deploy => models/swig_decoders}/ctc_decoders.h | 2 +- .../swig_decoders}/decoder_utils.cpp | 0 .../{deploy => models/swig_decoders}/decoder_utils.h | 0 .../{deploy => models/swig_decoders}/decoders.i | 0 .../{deploy => models/swig_decoders}/path_trie.cpp | 0 .../{deploy => models/swig_decoders}/path_trie.h | 0 .../{deploy => models/swig_decoders}/scorer.cpp | 0 .../{deploy => models/swig_decoders}/scorer.h | 0 .../{deploy => models/swig_decoders}/setup.py | 0 .../{deploy => models}/swig_decoders_wrapper.py | 4 ++-- deep_speech_2/test.py | 3 ++- 19 files changed, 25 insertions(+), 18 deletions(-) rename deep_speech_2/{deploy => models/swig_decoders}/README.md (100%) rename deep_speech_2/{deploy => models/swig_decoders}/__init__.py (100%) rename deep_speech_2/{deploy => models/swig_decoders}/_init_paths.py (100%) rename deep_speech_2/{deploy => models/swig_decoders}/ctc_decoders.cpp (98%) rename deep_speech_2/{deploy => models/swig_decoders}/ctc_decoders.h (96%) rename deep_speech_2/{deploy => models/swig_decoders}/decoder_utils.cpp (100%) rename deep_speech_2/{deploy => models/swig_decoders}/decoder_utils.h (100%) rename deep_speech_2/{deploy => models/swig_decoders}/decoders.i (100%) rename deep_speech_2/{deploy => models/swig_decoders}/path_trie.cpp (100%) rename deep_speech_2/{deploy => models/swig_decoders}/path_trie.h (100%) rename deep_speech_2/{deploy => models/swig_decoders}/scorer.cpp (100%) rename deep_speech_2/{deploy => models/swig_decoders}/scorer.h (100%) rename deep_speech_2/{deploy => models/swig_decoders}/setup.py (100%) rename deep_speech_2/{deploy => models}/swig_decoders_wrapper.py (97%) diff --git a/deep_speech_2/examples/librispeech/generate.sh b/deep_speech_2/examples/librispeech/generate.sh index a34b7bc100..752aafb6a3 100644 --- a/deep_speech_2/examples/librispeech/generate.sh +++ b/deep_speech_2/examples/librispeech/generate.sh @@ -12,9 +12,9 @@ python -u infer.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/librispeech/run_test.sh b/deep_speech_2/examples/librispeech/run_test.sh index 5a14cb6821..350db8f020 100644 --- a/deep_speech_2/examples/librispeech/run_test.sh +++ b/deep_speech_2/examples/librispeech/run_test.sh @@ -3,7 +3,7 @@ pushd ../.. CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ -python -u evaluate.py \ +python -u test.py \ --batch_size=128 \ --trainer_count=8 \ --beam_size=500 \ @@ -12,9 +12,9 @@ python -u evaluate.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/infer.py b/deep_speech_2/infer.py index 1ce969ae07..44ee93581d 100644 --- a/deep_speech_2/infer.py +++ b/deep_speech_2/infer.py @@ -84,6 +84,8 @@ def infer(): use_gru=args.use_gru, pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) + + vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] result_transcripts = ds2_model.infer_batch( infer_data=infer_data, decoding_method=args.decoding_method, @@ -91,7 +93,7 @@ def infer(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=data_generator.vocab_list, + vocab_list=vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) diff --git a/deep_speech_2/models/model.py b/deep_speech_2/models/model.py index 93c4c41bf7..b239d5f39f 100644 --- a/deep_speech_2/models/model.py +++ b/deep_speech_2/models/model.py @@ -8,8 +8,9 @@ import time import gzip import paddle.v2 as paddle -from lm.lm_scorer import LmScorer -from models.decoder import ctc_greedy_decoder, ctc_beam_search_decoder +from models.swig_decoders_wrapper import Scorer +from models.swig_decoders_wrapper import ctc_greedy_decoder +from models.swig_decoders_wrapper import ctc_beam_search_decoder_batch from models.network import deep_speech_v2_network @@ -199,9 +200,12 @@ def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, elif decoding_method == "ctc_beam_search": # initialize external scorer if self._ext_scorer == None: - self._ext_scorer = LmScorer(beam_alpha, beam_beta, - language_model_path) + self._ext_scorer = Scorer(beam_alpha, beam_beta, + language_model_path) self._loaded_lm_path = language_model_path + self._ext_scorer.set_char_map(vocab_list) + if (not self._ext_scorer.is_character_based()): + self._ext_scorer.fill_dictionary(True) else: self._ext_scorer.reset_params(beam_alpha, beam_beta) assert self._loaded_lm_path == language_model_path diff --git a/deep_speech_2/deploy/README.md b/deep_speech_2/models/swig_decoders/README.md similarity index 100% rename from deep_speech_2/deploy/README.md rename to deep_speech_2/models/swig_decoders/README.md diff --git a/deep_speech_2/deploy/__init__.py b/deep_speech_2/models/swig_decoders/__init__.py similarity index 100% rename from deep_speech_2/deploy/__init__.py rename to deep_speech_2/models/swig_decoders/__init__.py diff --git a/deep_speech_2/deploy/_init_paths.py b/deep_speech_2/models/swig_decoders/_init_paths.py similarity index 100% rename from deep_speech_2/deploy/_init_paths.py rename to deep_speech_2/models/swig_decoders/_init_paths.py diff --git a/deep_speech_2/deploy/ctc_decoders.cpp b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp similarity index 98% rename from deep_speech_2/deploy/ctc_decoders.cpp rename to deep_speech_2/models/swig_decoders/ctc_decoders.cpp index cedb943ea2..e60e669659 100644 --- a/deep_speech_2/deploy/ctc_decoders.cpp +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp @@ -10,8 +10,8 @@ #include "fst/fstlib.h" #include "path_trie.h" -std::string ctc_best_path_decoder(std::vector> probs_seq, - std::vector vocabulary) { +std::string ctc_greedy_decoder(std::vector> probs_seq, + std::vector vocabulary) { // dimension check int num_time_steps = probs_seq.size(); for (int i = 0; i < num_time_steps; i++) { diff --git a/deep_speech_2/deploy/ctc_decoders.h b/deep_speech_2/models/swig_decoders/ctc_decoders.h similarity index 96% rename from deep_speech_2/deploy/ctc_decoders.h rename to deep_speech_2/models/swig_decoders/ctc_decoders.h index 78edefb77f..a0028a3247 100644 --- a/deep_speech_2/deploy/ctc_decoders.h +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.h @@ -16,7 +16,7 @@ * A vector that each element is a pair of score and decoding result, * in desending order. */ -std::string ctc_best_path_decoder(std::vector> probs_seq, +std::string ctc_greedy_decoder(std::vector> probs_seq, std::vector vocabulary); /* CTC Beam Search Decoder diff --git a/deep_speech_2/deploy/decoder_utils.cpp b/deep_speech_2/models/swig_decoders/decoder_utils.cpp similarity index 100% rename from deep_speech_2/deploy/decoder_utils.cpp rename to deep_speech_2/models/swig_decoders/decoder_utils.cpp diff --git a/deep_speech_2/deploy/decoder_utils.h b/deep_speech_2/models/swig_decoders/decoder_utils.h similarity index 100% rename from deep_speech_2/deploy/decoder_utils.h rename to deep_speech_2/models/swig_decoders/decoder_utils.h diff --git a/deep_speech_2/deploy/decoders.i b/deep_speech_2/models/swig_decoders/decoders.i similarity index 100% rename from deep_speech_2/deploy/decoders.i rename to deep_speech_2/models/swig_decoders/decoders.i diff --git a/deep_speech_2/deploy/path_trie.cpp b/deep_speech_2/models/swig_decoders/path_trie.cpp similarity index 100% rename from deep_speech_2/deploy/path_trie.cpp rename to deep_speech_2/models/swig_decoders/path_trie.cpp diff --git a/deep_speech_2/deploy/path_trie.h b/deep_speech_2/models/swig_decoders/path_trie.h similarity index 100% rename from deep_speech_2/deploy/path_trie.h rename to deep_speech_2/models/swig_decoders/path_trie.h diff --git a/deep_speech_2/deploy/scorer.cpp b/deep_speech_2/models/swig_decoders/scorer.cpp similarity index 100% rename from deep_speech_2/deploy/scorer.cpp rename to deep_speech_2/models/swig_decoders/scorer.cpp diff --git a/deep_speech_2/deploy/scorer.h b/deep_speech_2/models/swig_decoders/scorer.h similarity index 100% rename from deep_speech_2/deploy/scorer.h rename to deep_speech_2/models/swig_decoders/scorer.h diff --git a/deep_speech_2/deploy/setup.py b/deep_speech_2/models/swig_decoders/setup.py similarity index 100% rename from deep_speech_2/deploy/setup.py rename to deep_speech_2/models/swig_decoders/setup.py diff --git a/deep_speech_2/deploy/swig_decoders_wrapper.py b/deep_speech_2/models/swig_decoders_wrapper.py similarity index 97% rename from deep_speech_2/deploy/swig_decoders_wrapper.py rename to deep_speech_2/models/swig_decoders_wrapper.py index b44fae0aee..202440bfba 100644 --- a/deep_speech_2/deploy/swig_decoders_wrapper.py +++ b/deep_speech_2/models/swig_decoders_wrapper.py @@ -23,7 +23,7 @@ def __init__(self, alpha, beta, model_path): swig_decoders.Scorer.__init__(self, alpha, beta, model_path) -def ctc_best_path_decoder(probs_seq, vocabulary): +def ctc_greedy_decoder(probs_seq, vocabulary): """Wrapper for ctc best path decoder in swig. :param probs_seq: 2-D list of probability distributions over each time @@ -35,7 +35,7 @@ def ctc_best_path_decoder(probs_seq, vocabulary): :return: Decoding result string. :rtype: basestring """ - return swig_decoders.ctc_best_path_decoder(probs_seq.tolist(), vocabulary) + return swig_decoders.ctc_greedy_decoder(probs_seq.tolist(), vocabulary) def ctc_beam_search_decoder(probs_seq, diff --git a/deep_speech_2/test.py b/deep_speech_2/test.py index 747e40df87..ec5d17f30d 100644 --- a/deep_speech_2/test.py +++ b/deep_speech_2/test.py @@ -85,6 +85,7 @@ def evaluate(): pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) + vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] error_rate_func = cer if args.error_rate_type == 'cer' else wer error_sum, num_ins = 0.0, 0 for infer_data in batch_reader(): @@ -95,7 +96,7 @@ def evaluate(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=data_generator.vocab_list, + vocab_list=vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) target_transcripts = [ From f8c7d46c1c78968657984687ad99f26862b1825d Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 8 Sep 2017 17:27:56 +0800 Subject: [PATCH 24/36] format header includes & update setup info --- deep_speech_2/README.md | 10 + deep_speech_2/deploy.py | 238 ------------------ deep_speech_2/models/swig_decoders/README.md | 57 ----- .../models/swig_decoders/ctc_decoders.cpp | 18 +- .../models/swig_decoders/ctc_decoders.h | 15 +- .../models/swig_decoders/decoder_utils.cpp | 1 + .../models/swig_decoders/path_trie.cpp | 3 +- .../models/swig_decoders/path_trie.h | 4 +- deep_speech_2/models/swig_decoders/scorer.cpp | 7 +- deep_speech_2/models/swig_decoders/scorer.h | 18 +- deep_speech_2/models/swig_decoders/setup.sh | 21 ++ 11 files changed, 71 insertions(+), 321 deletions(-) delete mode 100644 deep_speech_2/deploy.py delete mode 100644 deep_speech_2/models/swig_decoders/README.md create mode 100644 deep_speech_2/models/swig_decoders/setup.sh diff --git a/deep_speech_2/README.md b/deep_speech_2/README.md index db07d8c201..2cc12690b4 100644 --- a/deep_speech_2/README.md +++ b/deep_speech_2/README.md @@ -82,6 +82,16 @@ sh run.sh cd .. ``` +### Setup decoders + +```shell +cd models/swig_decoders +sh setup.sh +cd ../.. +``` + +These commands will install the decoders that translate the ouptut probability vectors of DS2 model to text data, incuding CTC greedy decoder, CTC beam search decoder and its batch version. + ### Inference For GPU inference diff --git a/deep_speech_2/deploy.py b/deep_speech_2/deploy.py deleted file mode 100644 index 11972f5f7d..0000000000 --- a/deep_speech_2/deploy.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Deployment for DeepSpeech2 model.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import gzip -import distutils.util -import multiprocessing -import paddle.v2 as paddle -from data_utils.data import DataGenerator -from layer import deep_speech2 -from deploy.swig_decoders_wrapper import * -from error_rate import wer -import utils -import time - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument( - "--num_samples", - default=10, - type=int, - help="Number of samples for inference. (default: %(default)s)") -parser.add_argument( - "--num_conv_layers", - default=2, - type=int, - help="Convolution layer number. (default: %(default)s)") -parser.add_argument( - "--num_rnn_layers", - default=3, - type=int, - help="RNN layer number. (default: %(default)s)") -parser.add_argument( - "--rnn_layer_size", - default=512, - type=int, - help="RNN layer cell number. (default: %(default)s)") -parser.add_argument( - "--use_gpu", - default=True, - type=distutils.util.strtobool, - help="Use gpu or not. (default: %(default)s)") -parser.add_argument( - "--num_threads_data", - default=multiprocessing.cpu_count(), - type=int, - help="Number of cpu threads for preprocessing data. (default: %(default)s)") -parser.add_argument( - "--num_processes_beam_search", - default=multiprocessing.cpu_count(), - type=int, - help="Number of cpu processes for beam search. (default: %(default)s)") -parser.add_argument( - "--mean_std_filepath", - default='mean_std.npz', - type=str, - help="Manifest path for normalizer. (default: %(default)s)") -parser.add_argument( - "--decode_manifest_path", - default='datasets/manifest.test', - type=str, - help="Manifest path for decoding. (default: %(default)s)") -parser.add_argument( - "--model_filepath", - default='checkpoints/params.latest.tar.gz', - type=str, - help="Model filepath. (default: %(default)s)") -parser.add_argument( - "--vocab_filepath", - default='datasets/vocab/eng_vocab.txt', - type=str, - help="Vocabulary filepath. (default: %(default)s)") -parser.add_argument( - "--decode_method", - default='beam_search', - type=str, - help="Method for ctc decoding: beam_search or beam_search_batch. " - "(default: %(default)s)") -parser.add_argument( - "--beam_size", - default=500, - type=int, - help="Width for beam search decoding. (default: %(default)d)") -parser.add_argument( - "--num_results_per_sample", - default=1, - type=int, - help="Number of output per sample in beam search. (default: %(default)d)") -parser.add_argument( - "--language_model_path", - default="lm/data/common_crawl_00.prune01111.trie.klm", - type=str, - help="Path for language model. (default: %(default)s)") -parser.add_argument( - "--alpha", - default=1.5, - type=float, - help="Parameter associated with language model. (default: %(default)f)") -parser.add_argument( - "--beta", - default=0.3, - type=float, - help="Parameter associated with word count. (default: %(default)f)") -parser.add_argument( - "--cutoff_prob", - default=1.0, - type=float, - help="The cutoff probability of pruning" - "in beam search. (default: %(default)f)") -parser.add_argument( - "--cutoff_top_n", - default=40, - type=int, - help="The cutoff number of pruning" - "in beam search. (default: %(default)f)") -args = parser.parse_args() - - -def infer(): - """Deployment for DeepSpeech2.""" - # initialize data generator - data_generator = DataGenerator( - vocab_filepath=args.vocab_filepath, - mean_std_filepath=args.mean_std_filepath, - augmentation_config='{}', - num_threads=args.num_threads_data) - - # create network config - # paddle.data_type.dense_array is used for variable batch input. - # The size 161 * 161 is only an placeholder value and the real shape - # of input batch data will be induced during training. - audio_data = paddle.layer.data( - name="audio_spectrogram", type=paddle.data_type.dense_array(161 * 161)) - text_data = paddle.layer.data( - name="transcript_text", - type=paddle.data_type.integer_value_sequence(data_generator.vocab_size)) - output_probs, _ = deep_speech2( - audio_data=audio_data, - text_data=text_data, - dict_size=data_generator.vocab_size, - num_conv_layers=args.num_conv_layers, - num_rnn_layers=args.num_rnn_layers, - rnn_size=args.rnn_layer_size) - - # load parameters - parameters = paddle.parameters.Parameters.from_tar( - gzip.open(args.model_filepath)) - - # prepare infer data - batch_reader = data_generator.batch_reader_creator( - manifest_path=args.decode_manifest_path, - batch_size=args.num_samples, - min_batch_size=1, - sortagrad=False, - shuffle_method=None) - infer_data = batch_reader().next() - - # run inference - inferer = paddle.inference.Inference( - output_layer=output_probs, parameters=parameters) - infer_results = inferer.infer(input=infer_data) - - num_steps = len(infer_results) // len(infer_data) - probs_split = [ - infer_results[i * num_steps:(i + 1) * num_steps] - for i in xrange(len(infer_data)) - ] - - # targe transcription - target_transcription = [ - ''.join( - [data_generator.vocab_list[index] for index in infer_data[i][1]]) - for i, probs in enumerate(probs_split) - ] - - # external scorer - ext_scorer = Scorer( - alpha=args.alpha, beta=args.beta, model_path=args.language_model_path) - - # from unicode to string - vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] - - # The below two steps, i.e. setting char map and filling dictionary of - # FST will be completed implicitly when ext_scorer first used.But to save - # the time of decoding the first audio sample, they are done in advance. - ext_scorer.set_char_map(vocab_list) - # only for ward based language model - ext_scorer.fill_dictionary(True) - - # for word error rate metric - wer_sum, wer_counter = 0.0, 0 - - ## decode and print - time_begin = time.time() - batch_beam_results = [] - if args.decode_method == 'beam_search': - for i, probs in enumerate(probs_split): - beam_result = ctc_beam_search_decoder( - probs_seq=probs, - beam_size=args.beam_size, - vocabulary=vocab_list, - blank_id=len(vocab_list), - cutoff_prob=args.cutoff_prob, - cutoff_top_n=args.cutoff_top_n, - ext_scoring_func=ext_scorer, ) - batch_beam_results += [beam_result] - else: - batch_beam_results = ctc_beam_search_decoder_batch( - probs_split=probs_split, - beam_size=args.beam_size, - vocabulary=vocab_list, - blank_id=len(vocab_list), - num_processes=args.num_processes_beam_search, - cutoff_prob=args.cutoff_prob, - cutoff_top_n=args.cutoff_top_n, - ext_scoring_func=ext_scorer, ) - - for i, beam_result in enumerate(batch_beam_results): - print("\nTarget Transcription:\t%s" % target_transcription[i]) - print("Beam %d: %f \t%s" % (0, beam_result[0][0], beam_result[0][1])) - wer_cur = wer(target_transcription[i], beam_result[0][1]) - wer_sum += wer_cur - wer_counter += 1 - print("cur wer = %f , average wer = %f" % - (wer_cur, wer_sum / wer_counter)) - - print("time for decoding = %f" % (time.time() - time_begin)) - - -def main(): - utils.print_arguments(args) - paddle.init(use_gpu=args.use_gpu, trainer_count=1) - infer() - - -if __name__ == '__main__': - main() diff --git a/deep_speech_2/models/swig_decoders/README.md b/deep_speech_2/models/swig_decoders/README.md deleted file mode 100644 index e817be105d..0000000000 --- a/deep_speech_2/models/swig_decoders/README.md +++ /dev/null @@ -1,57 +0,0 @@ - -The decoders for deployment developed in C++ are a better alternative for the prototype decoders in Pytthon, with more powerful performance in both speed and accuracy. - -### Installation - -The build depends on several open-sourced projects, first clone or download them to current directory (i.e., `deep_speech_2/deploy`) - -- [**KenLM**](https://github.com/kpu/kenlm/): Faster and Smaller Language Model Queries - -```shell -git clone https://github.com/kpu/kenlm.git -``` - -- [**OpenFst**](http://www.openfst.org/twiki/bin/view/FST/WebHome): A library for finite-state transducers - -```shell -wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz -tar -xzvf openfst-1.6.3.tar.gz -``` - - -- [**ThreadPool**](http://progsch.net/wordpress/): A library for C++ thread pool - -```shell -git clone https://github.com/progschj/ThreadPool.git -``` - -- [**SWIG**](http://www.swig.org): A tool that provides the Python interface for the decoders, please make sure it being installed. - -Then run the setup - -```shell -python setup.py install --num_processes 4 -cd .. -``` - -### Usage - -The decoders for deployment share almost the same interface with the prototye decoders in Python. After the installation succeeds, these decoders are very convenient for call in Python, and a complete example in ```deploy.py``` can be refered. - -For GPU deployment - -``` -CUDA_VISIBLE_DEVICES=0 python deploy.py -``` - -For CPU deployment - -``` -python deploy.py --use_gpu=False -``` - -More help for arguments - -``` -python deploy.py --help -``` diff --git a/deep_speech_2/models/swig_decoders/ctc_decoders.cpp b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp index e60e669659..4c9a45d9e6 100644 --- a/deep_speech_2/models/swig_decoders/ctc_decoders.cpp +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp @@ -1,17 +1,21 @@ #include "ctc_decoders.h" + #include #include #include #include #include #include + +#include "fst/fstlib.h" #include "ThreadPool.h" + #include "decoder_utils.h" -#include "fst/fstlib.h" #include "path_trie.h" -std::string ctc_greedy_decoder(std::vector> probs_seq, - std::vector vocabulary) { +std::string ctc_greedy_decoder( + const std::vector>& probs_seq, + const std::vector& vocabulary) { // dimension check int num_time_steps = probs_seq.size(); for (int i = 0; i < num_time_steps; i++) { @@ -56,7 +60,7 @@ std::string ctc_greedy_decoder(std::vector> probs_seq, } std::vector> ctc_beam_search_decoder( - std::vector> probs_seq, + const std::vector>& probs_seq, int beam_size, std::vector vocabulary, int blank_id, @@ -64,7 +68,7 @@ std::vector> ctc_beam_search_decoder( int cutoff_top_n, Scorer *extscorer) { // dimension check - int num_time_steps = probs_seq.size(); + size_t num_time_steps = probs_seq.size(); for (int i = 0; i < num_time_steps; i++) { if (probs_seq[i].size() != vocabulary.size() + 1) { std::cout << " The shape of probs_seq does not match" @@ -278,9 +282,9 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( - std::vector>> probs_split, + const std::vector>>& probs_split, int beam_size, - std::vector vocabulary, + const std::vector& vocabulary, int blank_id, int num_processes, double cutoff_prob, diff --git a/deep_speech_2/models/swig_decoders/ctc_decoders.h b/deep_speech_2/models/swig_decoders/ctc_decoders.h index a0028a3247..5b4bb79326 100644 --- a/deep_speech_2/models/swig_decoders/ctc_decoders.h +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.h @@ -4,6 +4,7 @@ #include #include #include + #include "scorer.h" /* CTC Best Path Decoder @@ -16,8 +17,9 @@ * A vector that each element is a pair of score and decoding result, * in desending order. */ -std::string ctc_greedy_decoder(std::vector> probs_seq, - std::vector vocabulary); +std::string ctc_greedy_decoder( + const std::vector>& probs_seq, + const std::vector& vocabulary); /* CTC Beam Search Decoder @@ -35,7 +37,7 @@ std::string ctc_greedy_decoder(std::vector> probs_seq, * in desending order. */ std::vector> ctc_beam_search_decoder( - std::vector> probs_seq, + const std::vector>& probs_seq, int beam_size, std::vector vocabulary, int blank_id, @@ -43,8 +45,7 @@ std::vector> ctc_beam_search_decoder( int cutoff_top_n = 40, Scorer *ext_scorer = NULL); -/* CTC Beam Search Decoder for batch data, the interface is consistent with the - * original decoder in Python version. +/* CTC Beam Search Decoder for batch data * Parameters: * probs_seq: 3-D vector that each element is a 2-D vector that can be used @@ -63,9 +64,9 @@ std::vector> ctc_beam_search_decoder( */ std::vector>> ctc_beam_search_decoder_batch( - std::vector>> probs_split, + const std::vector>>& probs_split, int beam_size, - std::vector vocabulary, + const std::vector& vocabulary, int blank_id, int num_processes, double cutoff_prob = 1.0, diff --git a/deep_speech_2/models/swig_decoders/decoder_utils.cpp b/deep_speech_2/models/swig_decoders/decoder_utils.cpp index bed0f623f9..d25c4deb45 100644 --- a/deep_speech_2/models/swig_decoders/decoder_utils.cpp +++ b/deep_speech_2/models/swig_decoders/decoder_utils.cpp @@ -1,4 +1,5 @@ #include "decoder_utils.h" + #include #include #include diff --git a/deep_speech_2/models/swig_decoders/path_trie.cpp b/deep_speech_2/models/swig_decoders/path_trie.cpp index db0b20cb5d..9e68c0f154 100644 --- a/deep_speech_2/models/swig_decoders/path_trie.cpp +++ b/deep_speech_2/models/swig_decoders/path_trie.cpp @@ -1,3 +1,5 @@ +#include "path_trie.h" + #include #include #include @@ -5,7 +7,6 @@ #include #include "decoder_utils.h" -#include "path_trie.h" PathTrie::PathTrie() { log_prob_b_prev = -NUM_FLT_INF; diff --git a/deep_speech_2/models/swig_decoders/path_trie.h b/deep_speech_2/models/swig_decoders/path_trie.h index cac524a3f1..e581ca73c1 100644 --- a/deep_speech_2/models/swig_decoders/path_trie.h +++ b/deep_speech_2/models/swig_decoders/path_trie.h @@ -1,12 +1,12 @@ #ifndef PATH_TRIE_H #define PATH_TRIE_H #pragma once -#include #include #include #include #include #include +#include using FSTMATCH = fst::SortedMatcher; @@ -45,12 +45,12 @@ class PathTrie { private: int _ROOT; bool _exists; + bool _has_dictionary; std::vector> _children; fst::StdVectorFst* _dictionary; fst::StdVectorFst::StateId _dictionary_state; - bool _has_dictionary; std::shared_ptr _matcher; }; diff --git a/deep_speech_2/models/swig_decoders/scorer.cpp b/deep_speech_2/models/swig_decoders/scorer.cpp index 8651eb61f9..a713b0dffc 100644 --- a/deep_speech_2/models/swig_decoders/scorer.cpp +++ b/deep_speech_2/models/swig_decoders/scorer.cpp @@ -1,13 +1,16 @@ #include "scorer.h" + #include #include -#include "decoder_utils.h" + #include "lm/config.hh" #include "lm/model.hh" #include "lm/state.hh" #include "util/string_piece.hh" #include "util/tokenize_piece.hh" +#include "decoder_utils.h" + using namespace lm::ngram; Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { @@ -122,7 +125,7 @@ std::vector Scorer::split_labels(const std::vector& labels) { return words; } -void Scorer::set_char_map(std::vector char_list) { +void Scorer::set_char_map(const std::vector& char_list) { _char_list = char_list; _char_map.clear(); diff --git a/deep_speech_2/models/swig_decoders/scorer.h b/deep_speech_2/models/swig_decoders/scorer.h index 0c78b98707..b99a99b720 100644 --- a/deep_speech_2/models/swig_decoders/scorer.h +++ b/deep_speech_2/models/swig_decoders/scorer.h @@ -5,12 +5,14 @@ #include #include #include + #include "lm/enumerate_vocab.hh" #include "lm/virtual_interface.hh" #include "lm/word_index.hh" -#include "path_trie.h" #include "util/string_piece.hh" +#include "path_trie.h" + const double OOV_SCORE = -1000.0; const std::string START_TOKEN = ""; const std::string UNK_TOKEN = ""; @@ -28,11 +30,13 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { std::vector vocabulary; }; -// External scorer to query languange score for n-gram or sentence. -// Example: -// Scorer scorer(alpha, beta, "path_of_language_model"); -// scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); -// scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); +/* External scorer to query languange score for n-gram or sentence. + * + * Example: + * Scorer scorer(alpha, beta, "path_of_language_model"); + * scorer.get_log_cond_prob({ "WORD1", "WORD2", "WORD3" }); + * scorer.get_sent_log_prob({ "WORD1", "WORD2", "WORD3" }); + */ class Scorer { public: Scorer(double alpha, double beta, const std::string& lm_path); @@ -58,7 +62,7 @@ class Scorer { void fill_dictionary(bool add_space); // set char map - void set_char_map(std::vector char_list); + void set_char_map(const std::vector& char_list); std::vector split_labels(const std::vector& labels); diff --git a/deep_speech_2/models/swig_decoders/setup.sh b/deep_speech_2/models/swig_decoders/setup.sh new file mode 100644 index 0000000000..069f51d6eb --- /dev/null +++ b/deep_speech_2/models/swig_decoders/setup.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ ! -d kenlm ]; then + git clone https://github.com/luotao1/kenlm.git + echo -e "\n" +fi + +if [ ! -d openfst-1.6.3 ]; then + echo "Download and extract openfst ..." + wget http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.3.tar.gz + tar -xzvf openfst-1.6.3.tar.gz + echo -e "\n" +fi + +if [ ! -d ThreadPool ]; then + git clone https://github.com/progschj/ThreadPool.git + echo -e "\n" +fi + +echo "Install decoders ..." +python setup.py install --num_processes 4 From 41e9e59d698238e87ed365d82c57b62ae3c2b486 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 8 Sep 2017 20:35:25 +0800 Subject: [PATCH 25/36] append some comments --- .../models/swig_decoders/ctc_decoders.cpp | 15 +++++------ .../models/swig_decoders/ctc_decoders.h | 17 ++++++------ .../models/swig_decoders/decoder_utils.cpp | 24 ++++++++--------- .../models/swig_decoders/decoder_utils.h | 19 ++++++++----- .../models/swig_decoders/path_trie.cpp | 2 +- .../models/swig_decoders/path_trie.h | 11 ++++++++ deep_speech_2/models/swig_decoders/scorer.cpp | 27 ++++++++++--------- deep_speech_2/models/swig_decoders/scorer.h | 27 ++++++++++--------- 8 files changed, 80 insertions(+), 62 deletions(-) diff --git a/deep_speech_2/models/swig_decoders/ctc_decoders.cpp b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp index 4c9a45d9e6..1097991299 100644 --- a/deep_speech_2/models/swig_decoders/ctc_decoders.cpp +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.cpp @@ -14,8 +14,8 @@ #include "path_trie.h" std::string ctc_greedy_decoder( - const std::vector>& probs_seq, - const std::vector& vocabulary) { + const std::vector> &probs_seq, + const std::vector &vocabulary) { // dimension check int num_time_steps = probs_seq.size(); for (int i = 0; i < num_time_steps; i++) { @@ -60,7 +60,7 @@ std::string ctc_greedy_decoder( } std::vector> ctc_beam_search_decoder( - const std::vector>& probs_seq, + const std::vector> &probs_seq, int beam_size, std::vector vocabulary, int blank_id, @@ -104,7 +104,7 @@ std::vector> ctc_beam_search_decoder( } if (!extscorer->is_character_based()) { if (extscorer->dictionary == nullptr) { - // fill dictionary for fst + // fill dictionary for fst with space extscorer->fill_dictionary(true); } auto fst_dict = static_cast(extscorer->dictionary); @@ -282,9 +282,9 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( - const std::vector>>& probs_split, + const std::vector>> &probs_split, int beam_size, - const std::vector& vocabulary, + const std::vector &vocabulary, int blank_id, int num_processes, double cutoff_prob, @@ -304,8 +304,7 @@ ctc_beam_search_decoder_batch( if (extscorer->is_char_map_empty()) { extscorer->set_char_map(vocabulary); } - if (!extscorer->is_character_based() && - extscorer->dictionary == nullptr) { + if (!extscorer->is_character_based() && extscorer->dictionary == nullptr) { // init dictionary extscorer->fill_dictionary(true); } diff --git a/deep_speech_2/models/swig_decoders/ctc_decoders.h b/deep_speech_2/models/swig_decoders/ctc_decoders.h index 5b4bb79326..b8c512bda8 100644 --- a/deep_speech_2/models/swig_decoders/ctc_decoders.h +++ b/deep_speech_2/models/swig_decoders/ctc_decoders.h @@ -14,12 +14,11 @@ * over vocabulary of one time step. * vocabulary: A vector of vocabulary. * Return: - * A vector that each element is a pair of score and decoding result, - * in desending order. + * The decoding result in string */ std::string ctc_greedy_decoder( - const std::vector>& probs_seq, - const std::vector& vocabulary); + const std::vector> &probs_seq, + const std::vector &vocabulary); /* CTC Beam Search Decoder @@ -37,7 +36,7 @@ std::string ctc_greedy_decoder( * in desending order. */ std::vector> ctc_beam_search_decoder( - const std::vector>& probs_seq, + const std::vector> &probs_seq, int beam_size, std::vector vocabulary, int blank_id, @@ -59,14 +58,14 @@ std::vector> ctc_beam_search_decoder( * cutoff_top_n: Cutoff number for pruning. * ext_scorer: External scorer to evaluate a prefix. * Return: - * A 2-D vector that each element is a vector of decoding result for one - * sample. + * A 2-D vector that each element is a vector of beam search decoding + * result for one audio sample. */ std::vector>> ctc_beam_search_decoder_batch( - const std::vector>>& probs_split, + const std::vector>> &probs_split, int beam_size, - const std::vector& vocabulary, + const std::vector &vocabulary, int blank_id, int num_processes, double cutoff_prob = 1.0, diff --git a/deep_speech_2/models/swig_decoders/decoder_utils.cpp b/deep_speech_2/models/swig_decoders/decoder_utils.cpp index d25c4deb45..989b067e7b 100644 --- a/deep_speech_2/models/swig_decoders/decoder_utils.cpp +++ b/deep_speech_2/models/swig_decoders/decoder_utils.cpp @@ -4,7 +4,7 @@ #include #include -size_t get_utf8_str_len(const std::string& str) { +size_t get_utf8_str_len(const std::string &str) { size_t str_len = 0; for (char c : str) { str_len += ((c & 0xc0) != 0x80); @@ -12,7 +12,7 @@ size_t get_utf8_str_len(const std::string& str) { return str_len; } -std::vector split_utf8_str(const std::string& str) { +std::vector split_utf8_str(const std::string &str) { std::vector result; std::string out_str; @@ -31,8 +31,8 @@ std::vector split_utf8_str(const std::string& str) { return result; } -std::vector split_str(const std::string& s, - const std::string& delim) { +std::vector split_str(const std::string &s, + const std::string &delim) { std::vector result; std::size_t start = 0, delim_len = delim.size(); while (true) { @@ -51,7 +51,7 @@ std::vector split_str(const std::string& s, return result; } -bool prefix_compare(const PathTrie* x, const PathTrie* y) { +bool prefix_compare(const PathTrie *x, const PathTrie *y) { if (x->score == y->score) { if (x->character == y->character) { return false; @@ -63,8 +63,8 @@ bool prefix_compare(const PathTrie* x, const PathTrie* y) { } } -void add_word_to_fst(const std::vector& word, - fst::StdVectorFst* dictionary) { +void add_word_to_fst(const std::vector &word, + fst::StdVectorFst *dictionary) { if (dictionary->NumStates() == 0) { fst::StdVectorFst::StateId start = dictionary->AddState(); assert(start == 0); @@ -81,16 +81,16 @@ void add_word_to_fst(const std::vector& word, } bool add_word_to_dictionary( - const std::string& word, - const std::unordered_map& char_map, + const std::string &word, + const std::unordered_map &char_map, bool add_space, int SPACE_ID, - fst::StdVectorFst* dictionary) { + fst::StdVectorFst *dictionary) { auto characters = split_utf8_str(word); std::vector int_word; - for (auto& c : characters) { + for (auto &c : characters) { if (c == " ") { int_word.push_back(SPACE_ID); } else { @@ -108,5 +108,5 @@ bool add_word_to_dictionary( } add_word_to_fst(int_word, dictionary); - return true; + return true; // return with successful adding } diff --git a/deep_speech_2/models/swig_decoders/decoder_utils.h b/deep_speech_2/models/swig_decoders/decoder_utils.h index 51985c86eb..d4ee36e1bf 100644 --- a/deep_speech_2/models/swig_decoders/decoder_utils.h +++ b/deep_speech_2/models/swig_decoders/decoder_utils.h @@ -14,12 +14,14 @@ bool pair_comp_first_rev(const std::pair &a, return a.first > b.first; } +// Function template for comparing two pairs template bool pair_comp_second_rev(const std::pair &a, const std::pair &b) { return a.second > b.second; } +// Return the sum of two probabilities in log scale template T log_sum_exp(const T &x, const T &y) { static T num_min = -std::numeric_limits::max(); @@ -32,18 +34,21 @@ T log_sum_exp(const T &x, const T &y) { // Functor for prefix comparsion bool prefix_compare(const PathTrie *x, const PathTrie *y); -// Get length of utf8 encoding string -// See: http://stackoverflow.com/a/4063229 +/* Get length of utf8 encoding string + * See: http://stackoverflow.com/a/4063229 + */ size_t get_utf8_str_len(const std::string &str); -// Split a string into a list of strings on a given string -// delimiter. NB: delimiters on beginning / end of string are -// trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. +/* Split a string into a list of strings on a given string + * delimiter. NB: delimiters on beginning / end of string are + * trimmed. Eg, "FooBarFoo" split on "Foo" returns ["Bar"]. + */ std::vector split_str(const std::string &s, const std::string &delim); -// Splits string into vector of strings representing -// UTF-8 characters (not same as chars) +/* Splits string into vector of strings representing + * UTF-8 characters (not same as chars) + */ std::vector split_utf8_str(const std::string &str); // Add a word in index to the dicionary of fst diff --git a/deep_speech_2/models/swig_decoders/path_trie.cpp b/deep_speech_2/models/swig_decoders/path_trie.cpp index 9e68c0f154..6a1f6170f5 100644 --- a/deep_speech_2/models/swig_decoders/path_trie.cpp +++ b/deep_speech_2/models/swig_decoders/path_trie.cpp @@ -22,7 +22,7 @@ PathTrie::PathTrie() { _dictionary = nullptr; _dictionary_state = 0; _has_dictionary = false; - _matcher = nullptr; // finds arcs in FST + _matcher = nullptr; } PathTrie::~PathTrie() { diff --git a/deep_speech_2/models/swig_decoders/path_trie.h b/deep_speech_2/models/swig_decoders/path_trie.h index e581ca73c1..6f150e4205 100644 --- a/deep_speech_2/models/swig_decoders/path_trie.h +++ b/deep_speech_2/models/swig_decoders/path_trie.h @@ -10,27 +10,36 @@ using FSTMATCH = fst::SortedMatcher; +/* Trie tree for prefix storing and manipulating, with a dictionary in + * finite-state transducer for spelling correction. + */ class PathTrie { public: PathTrie(); ~PathTrie(); + // get new prefix after appending new char PathTrie* get_path_trie(int new_char, bool reset = true); + // get the prefix in index from root to current node PathTrie* get_path_vec(std::vector& output); + // get the prefix in index from some stop node to current nodel PathTrie* get_path_vec(std::vector& output, int stop, size_t max_steps = std::numeric_limits::max()); + // update log probs void iterate_to_vec(std::vector& output); + // set dictionary for FST void set_dictionary(fst::StdVectorFst* dictionary); void set_matcher(std::shared_ptr matcher); bool is_empty() { return _ROOT == character; } + // remove current path from root void remove(); float log_prob_b_prev; @@ -49,8 +58,10 @@ class PathTrie { std::vector> _children; + // pointer to dictionary of FST fst::StdVectorFst* _dictionary; fst::StdVectorFst::StateId _dictionary_state; + // true if finding ars in FST std::shared_ptr _matcher; }; diff --git a/deep_speech_2/models/swig_decoders/scorer.cpp b/deep_speech_2/models/swig_decoders/scorer.cpp index a713b0dffc..75919c3c9e 100644 --- a/deep_speech_2/models/swig_decoders/scorer.cpp +++ b/deep_speech_2/models/swig_decoders/scorer.cpp @@ -68,7 +68,7 @@ double Scorer::get_log_cond_prob(const std::vector& words) { state = out_state; out_state = tmp_state; } - // log10 prob + // return log10 prob return cond_prob; } @@ -189,23 +189,26 @@ void Scorer::fill_dictionary(bool add_space) { std::cerr << "Vocab Size " << vocab_size << std::endl; - // Simplify FST + /* Simplify FST - // This gets rid of "epsilon" transitions in the FST. - // These are transitions that don't require a string input to be taken. - // Getting rid of them is necessary to make the FST determinisitc, but - // can greatly increase the size of the FST + * This gets rid of "epsilon" transitions in the FST. + * These are transitions that don't require a string input to be taken. + * Getting rid of them is necessary to make the FST determinisitc, but + * can greatly increase the size of the FST + */ fst::RmEpsilon(&dictionary); fst::StdVectorFst* new_dict = new fst::StdVectorFst; - // This makes the FST deterministic, meaning for any string input there's - // only one possible state the FST could be in. It is assumed our - // dictionary is deterministic when using it. - // (lest we'd have to check for multiple transitions at each state) + /* This makes the FST deterministic, meaning for any string input there's + * only one possible state the FST could be in. It is assumed our + * dictionary is deterministic when using it. + * (lest we'd have to check for multiple transitions at each state) + */ fst::Determinize(dictionary, new_dict); - // Finds the simplest equivalent fst. This is unnecessary but decreases - // memory usage of the dictionary + /* Finds the simplest equivalent fst. This is unnecessary but decreases + * memory usage of the dictionary + */ fst::Minimize(new_dict); this->dictionary = new_dict; } diff --git a/deep_speech_2/models/swig_decoders/scorer.h b/deep_speech_2/models/swig_decoders/scorer.h index b99a99b720..1b4857e38e 100644 --- a/deep_speech_2/models/swig_decoders/scorer.h +++ b/deep_speech_2/models/swig_decoders/scorer.h @@ -23,14 +23,15 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { public: RetriveStrEnumerateVocab() {} - void Add(lm::WordIndex index, const StringPiece& str) { + void Add(lm::WordIndex index, const StringPiece &str) { vocabulary.push_back(std::string(str.data(), str.length())); } std::vector vocabulary; }; -/* External scorer to query languange score for n-gram or sentence. +/* External scorer to query score for n-gram or sentence, including language + * model scoring and word insertion. * * Example: * Scorer scorer(alpha, beta, "path_of_language_model"); @@ -39,12 +40,12 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { */ class Scorer { public: - Scorer(double alpha, double beta, const std::string& lm_path); + Scorer(double alpha, double beta, const std::string &lm_path); ~Scorer(); - double get_log_cond_prob(const std::vector& words); + double get_log_cond_prob(const std::vector &words); - double get_sent_log_prob(const std::vector& words); + double get_sent_log_prob(const std::vector &words); size_t get_max_order() { return _max_order; } @@ -56,32 +57,32 @@ class Scorer { void reset_params(float alpha, float beta); // make ngram - std::vector make_ngram(PathTrie* prefix); + std::vector make_ngram(PathTrie *prefix); // fill dictionary for fst void fill_dictionary(bool add_space); // set char map - void set_char_map(const std::vector& char_list); + void set_char_map(const std::vector &char_list); - std::vector split_labels(const std::vector& labels); + std::vector split_labels(const std::vector &labels); // expose to decoder double alpha; double beta; // fst dictionary - void* dictionary; + void *dictionary; protected: - void load_LM(const char* filename); + void load_LM(const char *filename); - double get_log_prob(const std::vector& words); + double get_log_prob(const std::vector &words); - std::string vec2str(const std::vector& input); + std::string vec2str(const std::vector &input); private: - void* _language_model; + void *_language_model; bool _is_character_based; size_t _max_order; From 52a862d6349f2de80e201ca3e844846aae99085b Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Wed, 13 Sep 2017 23:08:30 +0800 Subject: [PATCH 26/36] add __init__.py in decoders/swig --- deep_speech_2/decoders/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 deep_speech_2/decoders/__init__.py diff --git a/deep_speech_2/decoders/__init__.py b/deep_speech_2/decoders/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 552dd5287538a01690a2f5ead7bde1c0797a3128 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Thu, 14 Sep 2017 11:46:59 +0800 Subject: [PATCH 27/36] move deprecated decoders --- .../{model_utils/decoder.py => decoders/decoder_deprecated.py} | 0 .../lm_scorer.py => decoders/lm_scorer_deprecated.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename deep_speech_2/{model_utils/decoder.py => decoders/decoder_deprecated.py} (100%) rename deep_speech_2/{model_utils/lm_scorer.py => decoders/lm_scorer_deprecated.py} (100%) diff --git a/deep_speech_2/model_utils/decoder.py b/deep_speech_2/decoders/decoder_deprecated.py similarity index 100% rename from deep_speech_2/model_utils/decoder.py rename to deep_speech_2/decoders/decoder_deprecated.py diff --git a/deep_speech_2/model_utils/lm_scorer.py b/deep_speech_2/decoders/lm_scorer_deprecated.py similarity index 100% rename from deep_speech_2/model_utils/lm_scorer.py rename to deep_speech_2/decoders/lm_scorer_deprecated.py From 0bda37cb8639c916f2b15a674747f1ed68b27161 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Fri, 15 Sep 2017 22:30:40 +0800 Subject: [PATCH 28/36] refine by following review comments --- deep_speech_2/README.md | 13 -- .../data_utils/featurizer/text_featurizer.py | 2 + deep_speech_2/decoders/swig/ctc_decoders.cpp | 156 ++++++++---------- deep_speech_2/decoders/swig/ctc_decoders.h | 24 +-- deep_speech_2/decoders/swig/decoder_utils.h | 16 ++ deep_speech_2/decoders/swig_wrapper.py | 16 +- .../examples/librispeech/run_test_golden.sh | 8 +- deep_speech_2/infer.py | 9 +- deep_speech_2/model_utils/model.py | 1 - deep_speech_2/setup.sh | 9 + deep_speech_2/test.py | 9 +- deep_speech_2/utils/utility.sh | 2 +- 12 files changed, 129 insertions(+), 136 deletions(-) diff --git a/deep_speech_2/README.md b/deep_speech_2/README.md index db940639a3..758799716a 100644 --- a/deep_speech_2/README.md +++ b/deep_speech_2/README.md @@ -24,8 +24,6 @@ ## Installation -### Basic setup - Please make sure the above [prerequisites](#prerequisites) have been satisfied before moving on. ```bash @@ -34,16 +32,6 @@ cd models/deep_speech_2 sh setup.sh ``` -### Decoders setup - -```bash -cd decoders/swig -sh setup.sh -cd ../.. -``` - -These commands will install the decoders that translate the ouptut probability vectors of DS2 model to text data, incuding CTC greedy decoder, CTC beam search decoder and its batch version. And a detailed usuage about them will be given in the following sections. - ## Getting Started Several shell scripts provided in `./examples` will help us to quickly give it a try, for most major modules, including data preparation, model training, case inference and model evaluation, with a few public dataset (e.g. [LibriSpeech](http://www.openslr.org/12/), [Aishell](http://www.openslr.org/33)). Reading these examples will also help you to understand how to make it work with your own data. @@ -189,7 +177,6 @@ Data augmentation has often been a highly effective technique to boost the deep Six optional augmentation components are provided to be selected, configured and inserted into the processing pipeline. ### Inference - - Volume Perturbation - Speed Perturbation - Shifting Perturbation diff --git a/deep_speech_2/data_utils/featurizer/text_featurizer.py b/deep_speech_2/data_utils/featurizer/text_featurizer.py index 89202163ca..95dc637e0d 100644 --- a/deep_speech_2/data_utils/featurizer/text_featurizer.py +++ b/deep_speech_2/data_utils/featurizer/text_featurizer.py @@ -22,6 +22,8 @@ class TextFeaturizer(object): def __init__(self, vocab_filepath): self._vocab_dict, self._vocab_list = self._load_vocabulary_from_file( vocab_filepath) + # from unicode to string + self._vocab_list = [chars.encode("utf-8") for chars in self._vocab_list] def featurize(self, text): """Convert text string to a list of token indices in char-level.Note diff --git a/deep_speech_2/decoders/swig/ctc_decoders.cpp b/deep_speech_2/decoders/swig/ctc_decoders.cpp index b52394b6e1..e86bfe0f2c 100644 --- a/deep_speech_2/decoders/swig/ctc_decoders.cpp +++ b/deep_speech_2/decoders/swig/ctc_decoders.cpp @@ -17,41 +17,38 @@ std::string ctc_greedy_decoder( const std::vector> &probs_seq, const std::vector &vocabulary) { // dimension check - int num_time_steps = probs_seq.size(); - for (int i = 0; i < num_time_steps; i++) { - if (probs_seq[i].size() != vocabulary.size() + 1) { - std::cout << "The shape of probs_seq does not match" - << " with the shape of the vocabulary!" << std::endl; - exit(1); - } + size_t num_time_steps = probs_seq.size(); + for (size_t i = 0; i < num_time_steps; i++) { + VALID_CHECK_EQ(probs_seq[i].size(), + vocabulary.size() + 1, + "The shape of probs_seq does not match with " + "the shape of the vocabulary"); } - int blank_id = vocabulary.size(); + size_t blank_id = vocabulary.size(); - std::vector max_idx_vec; - double max_prob = 0.0; - int max_idx = 0; - for (int i = 0; i < num_time_steps; i++) { - for (int j = 0; j < probs_seq[i].size(); j++) { + std::vector max_idx_vec; + for (size_t i = 0; i < num_time_steps; i++) { + double max_prob = 0.0; + size_t max_idx = 0; + for (size_t j = 0; j < probs_seq[i].size(); j++) { if (max_prob < probs_seq[i][j]) { max_idx = j; max_prob = probs_seq[i][j]; } } max_idx_vec.push_back(max_idx); - max_prob = 0.0; - max_idx = 0; } - std::vector idx_vec; - for (int i = 0; i < max_idx_vec.size(); i++) { + std::vector idx_vec; + for (size_t i = 0; i < max_idx_vec.size(); i++) { if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i - 1])) { idx_vec.push_back(max_idx_vec[i]); } } std::string best_path_result; - for (int i = 0; i < idx_vec.size(); i++) { + for (size_t i = 0; i < idx_vec.size(); i++) { if (idx_vec[i] != blank_id) { best_path_result += vocabulary[idx_vec[i]]; } @@ -61,29 +58,24 @@ std::string ctc_greedy_decoder( std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, - int beam_size, + const size_t beam_size, std::vector vocabulary, - int blank_id, - double cutoff_prob, - int cutoff_top_n, - Scorer *extscorer) { + const double cutoff_prob, + const size_t cutoff_top_n, + Scorer *ext_scorer) { // dimension check size_t num_time_steps = probs_seq.size(); - for (int i = 0; i < num_time_steps; i++) { - if (probs_seq[i].size() != vocabulary.size() + 1) { - std::cout << " The shape of probs_seq does not match" - << " with the shape of the vocabulary!" << std::endl; - exit(1); - } + for (size_t i = 0; i < num_time_steps; i++) { + VALID_CHECK_EQ(probs_seq[i].size(), + vocabulary.size() + 1, + "The shape of probs_seq does not match with " + "the shape of the vocabulary"); } - // blank_id check - if (blank_id > vocabulary.size()) { - std::cout << " Invalid blank_id! " << std::endl; - exit(1); - } + // assign blank id + size_t blank_id = vocabulary.size(); - // assign space ID + // assign space id std::vector::iterator it = std::find(vocabulary.begin(), vocabulary.end(), " "); int space_id = it - vocabulary.begin(); @@ -98,16 +90,16 @@ std::vector> ctc_beam_search_decoder( std::vector prefixes; prefixes.push_back(&root); - if (extscorer != nullptr) { - if (extscorer->is_char_map_empty()) { - extscorer->set_char_map(vocabulary); + if (ext_scorer != nullptr) { + if (ext_scorer->is_char_map_empty()) { + ext_scorer->set_char_map(vocabulary); } - if (!extscorer->is_character_based()) { - if (extscorer->dictionary == nullptr) { + if (!ext_scorer->is_character_based()) { + if (ext_scorer->dictionary == nullptr) { // fill dictionary for fst with space - extscorer->fill_dictionary(true); + ext_scorer->fill_dictionary(true); } - auto fst_dict = static_cast(extscorer->dictionary); + auto fst_dict = static_cast(ext_scorer->dictionary); fst::StdVectorFst *dict_ptr = fst_dict->Copy(true); root.set_dictionary(dict_ptr); auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); @@ -116,33 +108,33 @@ std::vector> ctc_beam_search_decoder( } // prefix search over time - for (int time_step = 0; time_step < num_time_steps; time_step++) { + for (size_t time_step = 0; time_step < num_time_steps; time_step++) { std::vector prob = probs_seq[time_step]; std::vector> prob_idx; - for (int i = 0; i < prob.size(); i++) { + for (size_t i = 0; i < prob.size(); i++) { prob_idx.push_back(std::pair(i, prob[i])); } float min_cutoff = -NUM_FLT_INF; bool full_beam = false; - if (extscorer != nullptr) { - int num_prefixes = std::min((int)prefixes.size(), beam_size); + if (ext_scorer != nullptr) { + size_t num_prefixes = std::min(prefixes.size(), beam_size); std::sort( prefixes.begin(), prefixes.begin() + num_prefixes, prefix_compare); min_cutoff = prefixes[num_prefixes - 1]->score + log(prob[blank_id]) - - std::max(0.0, extscorer->beta); + std::max(0.0, ext_scorer->beta); full_beam = (num_prefixes == beam_size); } // pruning of vacobulary - int cutoff_len = prob.size(); + size_t cutoff_len = prob.size(); if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) { std::sort( prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); if (cutoff_prob < 1.0) { double cum_prob = 0.0; cutoff_len = 0; - for (int i = 0; i < prob_idx.size(); i++) { + for (size_t i = 0; i < prob_idx.size(); i++) { cum_prob += prob_idx[i].second; cutoff_len += 1; if (cum_prob >= cutoff_prob) break; @@ -152,18 +144,18 @@ std::vector> ctc_beam_search_decoder( prob_idx = std::vector>( prob_idx.begin(), prob_idx.begin() + cutoff_len); } - std::vector> log_prob_idx; - for (int i = 0; i < cutoff_len; i++) { + std::vector> log_prob_idx; + for (size_t i = 0; i < cutoff_len; i++) { log_prob_idx.push_back(std::pair( prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); } // loop over chars - for (int index = 0; index < log_prob_idx.size(); index++) { + for (size_t index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; float log_prob_c = log_prob_idx[index].second; - for (int i = 0; i < prefixes.size() && i < beam_size; i++) { + for (size_t i = 0; i < prefixes.size() && i < beam_size; i++) { auto prefix = prefixes[i]; if (full_beam && log_prob_c + prefix->score < min_cutoff) { @@ -194,12 +186,12 @@ std::vector> ctc_beam_search_decoder( } // language model scoring - if (extscorer != nullptr && - (c == space_id || extscorer->is_character_based())) { + if (ext_scorer != nullptr && + (c == space_id || ext_scorer->is_character_based())) { PathTrie *prefix_toscore = nullptr; // skip scoring the space - if (extscorer->is_character_based()) { + if (ext_scorer->is_character_based()) { prefix_toscore = prefix_new; } else { prefix_toscore = prefix; @@ -207,11 +199,11 @@ std::vector> ctc_beam_search_decoder( double score = 0.0; std::vector ngram; - ngram = extscorer->make_ngram(prefix_toscore); - score = extscorer->get_log_cond_prob(ngram) * extscorer->alpha; + ngram = ext_scorer->make_ngram(prefix_toscore); + score = ext_scorer->get_log_cond_prob(ngram) * ext_scorer->alpha; log_p += score; - log_p += extscorer->beta; + log_p += ext_scorer->beta; } prefix_new->log_prob_nb_cur = log_sum_exp(prefix_new->log_prob_nb_cur, log_p); @@ -240,15 +232,15 @@ std::vector> ctc_beam_search_decoder( for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) { double approx_ctc = prefixes[i]->score; - if (extscorer != nullptr) { + if (ext_scorer != nullptr) { std::vector output; prefixes[i]->get_path_vec(output); size_t prefix_length = output.size(); - auto words = extscorer->split_labels(output); + auto words = ext_scorer->split_labels(output); // remove word insert - approx_ctc = approx_ctc - prefix_length * extscorer->beta; + approx_ctc = approx_ctc - prefix_length * ext_scorer->beta; // remove language model weight: - approx_ctc -= (extscorer->get_sent_log_prob(words)) * extscorer->alpha; + approx_ctc -= (ext_scorer->get_sent_log_prob(words)) * ext_scorer->alpha; } prefixes[i]->approx_ctc = approx_ctc; @@ -269,7 +261,7 @@ std::vector> ctc_beam_search_decoder( space_prefixes[i]->get_path_vec(output); // convert index to string std::string output_str; - for (int j = 0; j < output.size(); j++) { + for (size_t j = 0; j < output.size(); j++) { output_str += vocabulary[output[j]]; } std::pair output_pair(-space_prefixes[i]->approx_ctc, @@ -283,49 +275,45 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, - int beam_size, + const size_t beam_size, const std::vector &vocabulary, - int blank_id, - int num_processes, - double cutoff_prob, - int cutoff_top_n, - Scorer *extscorer) { - if (num_processes <= 0) { - std::cout << "num_processes must be nonnegative!" << std::endl; - exit(1); - } + const size_t num_processes, + const double cutoff_prob, + const size_t cutoff_top_n, + Scorer *ext_scorer) { + VALID_CHECK_GT(num_processes, 0, "num_processes must be nonnegative!"); // thread pool ThreadPool pool(num_processes); // number of samples - int batch_size = probs_split.size(); + size_t batch_size = probs_split.size(); // scorer filling up - if (extscorer != nullptr) { - if (extscorer->is_char_map_empty()) { - extscorer->set_char_map(vocabulary); + if (ext_scorer != nullptr) { + if (ext_scorer->is_char_map_empty()) { + ext_scorer->set_char_map(vocabulary); } - if (!extscorer->is_character_based() && extscorer->dictionary == nullptr) { + if (!ext_scorer->is_character_based() && + ext_scorer->dictionary == nullptr) { // init dictionary - extscorer->fill_dictionary(true); + ext_scorer->fill_dictionary(true); } } // enqueue the tasks of decoding std::vector>>> res; - for (int i = 0; i < batch_size; i++) { + for (size_t i = 0; i < batch_size; i++) { res.emplace_back(pool.enqueue(ctc_beam_search_decoder, probs_split[i], beam_size, vocabulary, - blank_id, cutoff_prob, cutoff_top_n, - extscorer)); + ext_scorer)); } // get decoding results std::vector>> batch_results; - for (int i = 0; i < batch_size; i++) { + for (size_t i = 0; i < batch_size; i++) { batch_results.emplace_back(res[i].get()); } return batch_results; diff --git a/deep_speech_2/decoders/swig/ctc_decoders.h b/deep_speech_2/decoders/swig/ctc_decoders.h index b8c512bda8..6384c8a8fc 100644 --- a/deep_speech_2/decoders/swig/ctc_decoders.h +++ b/deep_speech_2/decoders/swig/ctc_decoders.h @@ -27,21 +27,21 @@ std::string ctc_greedy_decoder( * over vocabulary of one time step. * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. - * blank_id: ID of blank. * cutoff_prob: Cutoff probability for pruning. * cutoff_top_n: Cutoff number for pruning. - * ext_scorer: External scorer to evaluate a prefix. + * ext_scorer: External scorer to evaluate a prefix, which consists of + * n-gram language model scoring and word insertion term. + * Default null, decoding the input sample without scorer. * Return: * A vector that each element is a pair of score and decoding result, * in desending order. */ std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, - int beam_size, + const size_t beam_size, std::vector vocabulary, - int blank_id, - double cutoff_prob = 1.0, - int cutoff_top_n = 40, + const double cutoff_prob = 1.0, + const size_t cutoff_top_n = 40, Scorer *ext_scorer = NULL); /* CTC Beam Search Decoder for batch data @@ -52,11 +52,12 @@ std::vector> ctc_beam_search_decoder( * . * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. - * blank_id: ID of blank. * num_processes: Number of threads for beam search. * cutoff_prob: Cutoff probability for pruning. * cutoff_top_n: Cutoff number for pruning. - * ext_scorer: External scorer to evaluate a prefix. + * ext_scorer: External scorer to evaluate a prefix, which consists of + * n-gram language model scoring and word insertion term. + * Default null, decoding the input sample without scorer. * Return: * A 2-D vector that each element is a vector of beam search decoding * result for one audio sample. @@ -64,12 +65,11 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, - int beam_size, + const size_t beam_size, const std::vector &vocabulary, - int blank_id, - int num_processes, + const size_t num_processes, double cutoff_prob = 1.0, - int cutoff_top_n = 40, + const size_t cutoff_top_n = 40, Scorer *ext_scorer = NULL); #endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/decoders/swig/decoder_utils.h b/deep_speech_2/decoders/swig/decoder_utils.h index d4ee36e1bf..015646ddd7 100644 --- a/deep_speech_2/decoders/swig/decoder_utils.h +++ b/deep_speech_2/decoders/swig/decoder_utils.h @@ -7,6 +7,22 @@ const float NUM_FLT_INF = std::numeric_limits::max(); const float NUM_FLT_MIN = std::numeric_limits::min(); +// check if __A == _B +#define VALID_CHECK_EQ(__A, __B, __ERR) \ + if ((__A) != (__B)) { \ + std::ostringstream str; \ + str << (__A) << " != " << (__B) << ", "; \ + throw std::runtime_error(str.str() + __ERR); \ + } + +// check if __A > __B +#define VALID_CHECK_GT(__A, __B, __ERR) \ + if ((__A) <= (__B)) { \ + std::ostringstream str; \ + str << (__A) << " <= " << (__B) << ", "; \ + throw std::runtime_error(str.str() + __ERR); \ + } + // Function template for comparing two pairs template bool pair_comp_first_rev(const std::pair &a, diff --git a/deep_speech_2/decoders/swig_wrapper.py b/deep_speech_2/decoders/swig_wrapper.py index 202440bfba..54ed249f38 100644 --- a/deep_speech_2/decoders/swig_wrapper.py +++ b/deep_speech_2/decoders/swig_wrapper.py @@ -41,7 +41,6 @@ def ctc_greedy_decoder(probs_seq, vocabulary): def ctc_beam_search_decoder(probs_seq, beam_size, vocabulary, - blank_id, cutoff_prob=1.0, cutoff_top_n=40, ext_scoring_func=None): @@ -55,8 +54,6 @@ def ctc_beam_search_decoder(probs_seq, :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list - :param blank_id: ID of blank. - :type blank_id: int :param cutoff_prob: Cutoff probability in pruning, default 1.0, no pruning. :type cutoff_prob: float @@ -72,15 +69,14 @@ def ctc_beam_search_decoder(probs_seq, results, in descending order of the probability. :rtype: list """ - return swig_decoders.ctc_beam_search_decoder( - probs_seq.tolist(), beam_size, vocabulary, blank_id, cutoff_prob, - cutoff_top_n, ext_scoring_func) + return swig_decoders.ctc_beam_search_decoder(probs_seq.tolist(), beam_size, + vocabulary, cutoff_prob, + cutoff_top_n, ext_scoring_func) def ctc_beam_search_decoder_batch(probs_split, beam_size, vocabulary, - blank_id, num_processes, cutoff_prob=1.0, cutoff_top_n=40, @@ -94,8 +90,6 @@ def ctc_beam_search_decoder_batch(probs_split, :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list - :param blank_id: ID of blank. - :type blank_id: int :param num_processes: Number of parallel processes. :type num_processes: int :param cutoff_prob: Cutoff probability in vocabulary pruning, @@ -118,5 +112,5 @@ def ctc_beam_search_decoder_batch(probs_split, probs_split = [probs_seq.tolist() for probs_seq in probs_split] return swig_decoders.ctc_beam_search_decoder_batch( - probs_split, beam_size, vocabulary, blank_id, num_processes, - cutoff_prob, cutoff_top_n, ext_scoring_func) + probs_split, beam_size, vocabulary, num_processes, cutoff_prob, + cutoff_top_n, ext_scoring_func) diff --git a/deep_speech_2/examples/librispeech/run_test_golden.sh b/deep_speech_2/examples/librispeech/run_test_golden.sh index 080c3c0622..e539bd0137 100644 --- a/deep_speech_2/examples/librispeech/run_test_golden.sh +++ b/deep_speech_2/examples/librispeech/run_test_golden.sh @@ -31,13 +31,13 @@ python -u test.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---test_manifest='data/tiny/manifest.test-clean' \ +--test_manifest='data/librispeech/manifest.test-clean' \ --mean_std_path='models/librispeech/mean_std.npz' \ --vocab_path='models/librispeech/vocab.txt' \ --model_path='models/librispeech/params.tar.gz' \ diff --git a/deep_speech_2/infer.py b/deep_speech_2/infer.py index 48c4ef493a..5da1db970c 100644 --- a/deep_speech_2/infer.py +++ b/deep_speech_2/infer.py @@ -21,9 +21,9 @@ add_arg('num_conv_layers', int, 2, "# of convolution layers.") add_arg('num_rnn_layers', int, 3, "# of recurrent layers.") add_arg('rnn_layer_size', int, 2048, "# of recurrent cells per layer.") -add_arg('alpha', float, 0.36, "Coef of LM for beam search.") -add_arg('beta', float, 0.25, "Coef of WC for beam search.") -add_arg('cutoff_prob', float, 0.99, "Cutoff probability for pruning.") +add_arg('alpha', float, 2.15, "Coef of LM for beam search.") +add_arg('beta', float, 0.35, "Coef of WC for beam search.") +add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") add_arg('use_gru', bool, False, "Use GRUs instead of simple RNNs.") add_arg('use_gpu', bool, True, "Use GPU or not.") add_arg('share_rnn_weights',bool, True, "Share input-hidden weights across " @@ -85,7 +85,6 @@ def infer(): pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) - vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] result_transcripts = ds2_model.infer_batch( infer_data=infer_data, decoding_method=args.decoding_method, @@ -93,7 +92,7 @@ def infer(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=vocab_list, + vocab_list=data_generator.vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) diff --git a/deep_speech_2/model_utils/model.py b/deep_speech_2/model_utils/model.py index 5812afca6e..1a9910e9d9 100644 --- a/deep_speech_2/model_utils/model.py +++ b/deep_speech_2/model_utils/model.py @@ -214,7 +214,6 @@ def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, probs_split=probs_split, vocabulary=vocab_list, beam_size=beam_size, - blank_id=len(vocab_list), num_processes=num_processes, ext_scoring_func=self._ext_scorer, cutoff_prob=cutoff_prob) diff --git a/deep_speech_2/setup.sh b/deep_speech_2/setup.sh index 6c8a709941..dcb3e0fbc2 100644 --- a/deep_speech_2/setup.sh +++ b/deep_speech_2/setup.sh @@ -26,4 +26,13 @@ if [ $? != 0 ]; then rm libsndfile-1.0.28.tar.gz fi +# install decoders +python -c "import swig_decoders" +if [ $? != 0 ]; then + pushd decoders/swig > /dev/null + sh setup.sh + popd > /dev/null +fi + + echo "Install all dependencies successfully." diff --git a/deep_speech_2/test.py b/deep_speech_2/test.py index 499f71f62a..76efb4d1e1 100644 --- a/deep_speech_2/test.py +++ b/deep_speech_2/test.py @@ -22,9 +22,9 @@ add_arg('num_conv_layers', int, 2, "# of convolution layers.") add_arg('num_rnn_layers', int, 3, "# of recurrent layers.") add_arg('rnn_layer_size', int, 2048, "# of recurrent cells per layer.") -add_arg('alpha', float, 0.36, "Coef of LM for beam search.") -add_arg('beta', float, 0.25, "Coef of WC for beam search.") -add_arg('cutoff_prob', float, 0.99, "Cutoff probability for pruning.") +add_arg('alpha', float, 2.15, "Coef of LM for beam search.") +add_arg('beta', float, 0.35, "Coef of WC for beam search.") +add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") add_arg('use_gru', bool, False, "Use GRUs instead of simple RNNs.") add_arg('use_gpu', bool, True, "Use GPU or not.") add_arg('share_rnn_weights',bool, True, "Share input-hidden weights across " @@ -85,7 +85,6 @@ def evaluate(): pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) - vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] error_rate_func = cer if args.error_rate_type == 'cer' else wer error_sum, num_ins = 0.0, 0 for infer_data in batch_reader(): @@ -96,7 +95,7 @@ def evaluate(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=vocab_list, + vocab_list=data_generator.vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) target_transcripts = [ diff --git a/deep_speech_2/utils/utility.sh b/deep_speech_2/utils/utility.sh index c8121126a1..aa0ec002bc 100644 --- a/deep_speech_2/utils/utility.sh +++ b/deep_speech_2/utils/utility.sh @@ -13,7 +13,7 @@ download() { wget -c $URL -P `dirname "$TARGET"` md5_result=`md5sum $TARGET | awk -F[' '] '{print $1}'` - if [ $MD5 -ne $md5_result ]; then + if [ ! $MD5 == $md5_result ]; then echo "Fail to download the language model!" return 1 fi From 15728d044b2e16d86b8176e5850052f7af536951 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Sat, 16 Sep 2017 12:38:58 +0800 Subject: [PATCH 29/36] expose param cutoff_top_n --- .../data_utils/featurizer/text_featurizer.py | 2 -- deep_speech_2/decoders/decoder_deprecated.py | 20 ++++++++----------- .../decoders/lm_scorer_deprecated.py | 2 +- deep_speech_2/decoders/swig/ctc_decoders.cpp | 2 +- .../examples/librispeech/run_infer.sh | 1 + .../examples/librispeech/run_infer_golden.sh | 1 + .../examples/librispeech/run_test_golden.sh | 1 + deep_speech_2/infer.py | 9 +++++++-- deep_speech_2/model_utils/model.py | 11 +++++++--- deep_speech_2/test.py | 9 +++++++-- 10 files changed, 35 insertions(+), 23 deletions(-) diff --git a/deep_speech_2/data_utils/featurizer/text_featurizer.py b/deep_speech_2/data_utils/featurizer/text_featurizer.py index 95dc637e0d..89202163ca 100644 --- a/deep_speech_2/data_utils/featurizer/text_featurizer.py +++ b/deep_speech_2/data_utils/featurizer/text_featurizer.py @@ -22,8 +22,6 @@ class TextFeaturizer(object): def __init__(self, vocab_filepath): self._vocab_dict, self._vocab_list = self._load_vocabulary_from_file( vocab_filepath) - # from unicode to string - self._vocab_list = [chars.encode("utf-8") for chars in self._vocab_list] def featurize(self, text): """Convert text string to a list of token indices in char-level.Note diff --git a/deep_speech_2/decoders/decoder_deprecated.py b/deep_speech_2/decoders/decoder_deprecated.py index ffba2731a0..6474316329 100644 --- a/deep_speech_2/decoders/decoder_deprecated.py +++ b/deep_speech_2/decoders/decoder_deprecated.py @@ -42,8 +42,8 @@ def ctc_greedy_decoder(probs_seq, vocabulary): def ctc_beam_search_decoder(probs_seq, beam_size, vocabulary, - blank_id, cutoff_prob=1.0, + cutoff_top_n=40, ext_scoring_func=None, nproc=False): """CTC Beam search decoder. @@ -66,8 +66,6 @@ def ctc_beam_search_decoder(probs_seq, :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list - :param blank_id: ID of blank. - :type blank_id: int :param cutoff_prob: Cutoff probability in pruning, default 1.0, no pruning. :type cutoff_prob: float @@ -87,9 +85,8 @@ def ctc_beam_search_decoder(probs_seq, raise ValueError("The shape of prob_seq does not match with the " "shape of the vocabulary.") - # blank_id check - if not blank_id < len(probs_seq[0]): - raise ValueError("blank_id shouldn't be greater than probs dimension") + # blank_id assign + blank_id = len(vocabulary) # If the decoder called in the multiprocesses, then use the global scorer # instantiated in ctc_beam_search_decoder_batch(). @@ -114,7 +111,7 @@ def ctc_beam_search_decoder(probs_seq, prob_idx = list(enumerate(probs_seq[time_step])) cutoff_len = len(prob_idx) #If pruning is enabled - if cutoff_prob < 1.0: + if cutoff_prob < 1.0 or cutoff_top_n < cutoff_len: prob_idx = sorted(prob_idx, key=lambda asd: asd[1], reverse=True) cutoff_len, cum_prob = 0, 0.0 for i in xrange(len(prob_idx)): @@ -122,6 +119,7 @@ def ctc_beam_search_decoder(probs_seq, cutoff_len += 1 if cum_prob >= cutoff_prob: break + cutoff_len = min(cutoff_top_n, cutoff_top_n) prob_idx = prob_idx[0:cutoff_len] for l in prefix_set_prev: @@ -191,9 +189,9 @@ def ctc_beam_search_decoder(probs_seq, def ctc_beam_search_decoder_batch(probs_split, beam_size, vocabulary, - blank_id, num_processes, cutoff_prob=1.0, + cutoff_top_n=40, ext_scoring_func=None): """CTC beam search decoder using multiple processes. @@ -204,8 +202,6 @@ def ctc_beam_search_decoder_batch(probs_split, :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list - :param blank_id: ID of blank. - :type blank_id: int :param num_processes: Number of parallel processes. :type num_processes: int :param cutoff_prob: Cutoff probability in pruning, @@ -232,8 +228,8 @@ def ctc_beam_search_decoder_batch(probs_split, pool = multiprocessing.Pool(processes=num_processes) results = [] for i, probs_list in enumerate(probs_split): - args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, None, - nproc) + args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, + cutoff_top_n, None, nproc) results.append(pool.apply_async(ctc_beam_search_decoder, args)) pool.close() diff --git a/deep_speech_2/decoders/lm_scorer_deprecated.py b/deep_speech_2/decoders/lm_scorer_deprecated.py index 463e96d665..c6a661030d 100644 --- a/deep_speech_2/decoders/lm_scorer_deprecated.py +++ b/deep_speech_2/decoders/lm_scorer_deprecated.py @@ -8,7 +8,7 @@ import numpy as np -class LmScorer(object): +class Scorer(object): """External scorer to evaluate a prefix or whole sentence in beam search decoding, including the score from n-gram language model and word count. diff --git a/deep_speech_2/decoders/swig/ctc_decoders.cpp b/deep_speech_2/decoders/swig/ctc_decoders.cpp index 86598eee6e..35425fbca3 100644 --- a/deep_speech_2/decoders/swig/ctc_decoders.cpp +++ b/deep_speech_2/decoders/swig/ctc_decoders.cpp @@ -128,7 +128,7 @@ std::vector> ctc_beam_search_decoder( // pruning of vacobulary size_t cutoff_len = prob.size(); - if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) { + if (cutoff_prob < 1.0 || cutoff_top_n < cutoff_len) { std::sort( prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); if (cutoff_prob < 1.0) { diff --git a/deep_speech_2/examples/librispeech/run_infer.sh b/deep_speech_2/examples/librispeech/run_infer.sh index fa177933af..b6f254a0bf 100644 --- a/deep_speech_2/examples/librispeech/run_infer.sh +++ b/deep_speech_2/examples/librispeech/run_infer.sh @@ -24,6 +24,7 @@ python -u infer.py \ --alpha=2.15 \ --beta=0.35 \ --cutoff_prob=1.0 \ +--cutoff_top_n=40 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/librispeech/run_infer_golden.sh b/deep_speech_2/examples/librispeech/run_infer_golden.sh index 20dfc65ee8..9336edebb0 100644 --- a/deep_speech_2/examples/librispeech/run_infer_golden.sh +++ b/deep_speech_2/examples/librispeech/run_infer_golden.sh @@ -33,6 +33,7 @@ python -u infer.py \ --alpha=2.15 \ --beta=0.35 \ --cutoff_prob=1.0 \ +--cutoff_top_n=40 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/librispeech/run_test_golden.sh b/deep_speech_2/examples/librispeech/run_test_golden.sh index e539bd0137..6aed4cfca1 100644 --- a/deep_speech_2/examples/librispeech/run_test_golden.sh +++ b/deep_speech_2/examples/librispeech/run_test_golden.sh @@ -34,6 +34,7 @@ python -u test.py \ --alpha=2.15 \ --beta=0.35 \ --cutoff_prob=1.0 \ +--cutoff_top_n=40 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/infer.py b/deep_speech_2/infer.py index 5da1db970c..1064fd25a0 100644 --- a/deep_speech_2/infer.py +++ b/deep_speech_2/infer.py @@ -23,7 +23,8 @@ add_arg('rnn_layer_size', int, 2048, "# of recurrent cells per layer.") add_arg('alpha', float, 2.15, "Coef of LM for beam search.") add_arg('beta', float, 0.35, "Coef of WC for beam search.") -add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") +add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") +add_arg('cutoff_top_n', int, 40, "Cutoff number for pruning.") add_arg('use_gru', bool, False, "Use GRUs instead of simple RNNs.") add_arg('use_gpu', bool, True, "Use GPU or not.") add_arg('share_rnn_weights',bool, True, "Share input-hidden weights across " @@ -85,6 +86,9 @@ def infer(): pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) + # decoders only accept string encoded in utf-8 + vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] + result_transcripts = ds2_model.infer_batch( infer_data=infer_data, decoding_method=args.decoding_method, @@ -92,7 +96,8 @@ def infer(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=data_generator.vocab_list, + cutoff_top_n=args.cutoff_top_n, + vocab_list=vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) diff --git a/deep_speech_2/model_utils/model.py b/deep_speech_2/model_utils/model.py index 1a9910e9d9..4f5021a6d5 100644 --- a/deep_speech_2/model_utils/model.py +++ b/deep_speech_2/model_utils/model.py @@ -148,8 +148,8 @@ def infer_loss_batch(self, infer_data): return self._loss_inferer.infer(input=infer_data) def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, - beam_size, cutoff_prob, vocab_list, language_model_path, - num_processes): + beam_size, cutoff_prob, cutoff_top_n, vocab_list, + language_model_path, num_processes): """Model inference. Infer the transcription for a batch of speech utterances. @@ -169,6 +169,10 @@ def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, :param cutoff_prob: Cutoff probability in pruning, default 1.0, no pruning. :type cutoff_prob: float + :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n + characters with highest probs in vocabulary will be + used in beam search, default 40. + :type cutoff_top_n: int :param vocab_list: List of tokens in the vocabulary, for decoding. :type vocab_list: list :param language_model_path: Filepath for language model. @@ -216,7 +220,8 @@ def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, beam_size=beam_size, num_processes=num_processes, ext_scoring_func=self._ext_scorer, - cutoff_prob=cutoff_prob) + cutoff_prob=cutoff_prob, + cutoff_top_n=cutoff_top_n) results = [result[0][1] for result in beam_search_results] else: diff --git a/deep_speech_2/test.py b/deep_speech_2/test.py index 76efb4d1e1..c564bb85db 100644 --- a/deep_speech_2/test.py +++ b/deep_speech_2/test.py @@ -24,7 +24,8 @@ add_arg('rnn_layer_size', int, 2048, "# of recurrent cells per layer.") add_arg('alpha', float, 2.15, "Coef of LM for beam search.") add_arg('beta', float, 0.35, "Coef of WC for beam search.") -add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") +add_arg('cutoff_prob', float, 1.0, "Cutoff probability for pruning.") +add_arg('cutoff_top_n', int, 40, "Cutoff number for pruning.") add_arg('use_gru', bool, False, "Use GRUs instead of simple RNNs.") add_arg('use_gpu', bool, True, "Use GPU or not.") add_arg('share_rnn_weights',bool, True, "Share input-hidden weights across " @@ -85,6 +86,9 @@ def evaluate(): pretrained_model_path=args.model_path, share_rnn_weights=args.share_rnn_weights) + # decoders only accept string encoded in utf-8 + vocab_list = [chars.encode("utf-8") for chars in data_generator.vocab_list] + error_rate_func = cer if args.error_rate_type == 'cer' else wer error_sum, num_ins = 0.0, 0 for infer_data in batch_reader(): @@ -95,7 +99,8 @@ def evaluate(): beam_beta=args.beta, beam_size=args.beam_size, cutoff_prob=args.cutoff_prob, - vocab_list=data_generator.vocab_list, + cutoff_top_n=args.cutoff_top_n, + vocab_list=vocab_list, language_model_path=args.lang_model_path, num_processes=args.num_proc_bsearch) target_transcripts = [ From e6740af49bb633aaefb93c4b89100f084cfd3667 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Sun, 17 Sep 2017 19:05:04 +0800 Subject: [PATCH 30/36] adjust scorer's init & add logging for scorer & separate long functions --- deep_speech_2/README.md | 1 - ...r_deprecated.py => decoders_deprecated.py} | 6 +- ...rer_deprecated.py => scorer_deprecated.py} | 0 ...coders.cpp => ctc_beam_search_decoder.cpp} | 164 +++--------------- ...c_decoders.h => ctc_beam_search_decoder.h} | 29 +--- .../decoders/swig/ctc_greedy_decoder.cpp | 45 +++++ .../decoders/swig/ctc_greedy_decoder.h | 20 +++ deep_speech_2/decoders/swig/decoder_utils.cpp | 65 +++++++ deep_speech_2/decoders/swig/decoder_utils.h | 39 +++-- deep_speech_2/decoders/swig/decoders.i | 6 +- deep_speech_2/decoders/swig/path_trie.h | 9 +- deep_speech_2/decoders/swig/scorer.cpp | 42 +++-- deep_speech_2/decoders/swig/scorer.h | 35 ++-- deep_speech_2/decoders/swig/setup.py | 13 +- deep_speech_2/decoders/swig/setup.sh | 2 +- deep_speech_2/decoders/swig_wrapper.py | 22 +-- deep_speech_2/examples/tiny/run_infer.sh | 6 +- .../examples/tiny/run_infer_golden.sh | 6 +- deep_speech_2/examples/tiny/run_test.sh | 6 +- .../examples/tiny/run_test_golden.sh | 6 +- deep_speech_2/infer.py | 1 + deep_speech_2/model_utils/model.py | 25 ++- deep_speech_2/test.py | 1 + 23 files changed, 310 insertions(+), 239 deletions(-) rename deep_speech_2/decoders/{decoder_deprecated.py => decoders_deprecated.py} (98%) rename deep_speech_2/decoders/{lm_scorer_deprecated.py => scorer_deprecated.py} (100%) rename deep_speech_2/decoders/swig/{ctc_decoders.cpp => ctc_beam_search_decoder.cpp} (55%) rename deep_speech_2/decoders/swig/{ctc_decoders.h => ctc_beam_search_decoder.h} (75%) create mode 100644 deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp create mode 100644 deep_speech_2/decoders/swig/ctc_greedy_decoder.h diff --git a/deep_speech_2/README.md b/deep_speech_2/README.md index 758799716a..9d9d4c77e8 100644 --- a/deep_speech_2/README.md +++ b/deep_speech_2/README.md @@ -176,7 +176,6 @@ Data augmentation has often been a highly effective technique to boost the deep Six optional augmentation components are provided to be selected, configured and inserted into the processing pipeline. -### Inference - Volume Perturbation - Speed Perturbation - Shifting Perturbation diff --git a/deep_speech_2/decoders/decoder_deprecated.py b/deep_speech_2/decoders/decoders_deprecated.py similarity index 98% rename from deep_speech_2/decoders/decoder_deprecated.py rename to deep_speech_2/decoders/decoders_deprecated.py index 6474316329..17b28b0d02 100644 --- a/deep_speech_2/decoders/decoder_deprecated.py +++ b/deep_speech_2/decoders/decoders_deprecated.py @@ -119,7 +119,7 @@ def ctc_beam_search_decoder(probs_seq, cutoff_len += 1 if cum_prob >= cutoff_prob: break - cutoff_len = min(cutoff_top_n, cutoff_top_n) + cutoff_len = min(cutoff_len, cutoff_top_n) prob_idx = prob_idx[0:cutoff_len] for l in prefix_set_prev: @@ -228,8 +228,8 @@ def ctc_beam_search_decoder_batch(probs_split, pool = multiprocessing.Pool(processes=num_processes) results = [] for i, probs_list in enumerate(probs_split): - args = (probs_list, beam_size, vocabulary, blank_id, cutoff_prob, - cutoff_top_n, None, nproc) + args = (probs_list, beam_size, vocabulary, cutoff_prob, cutoff_top_n, + None, nproc) results.append(pool.apply_async(ctc_beam_search_decoder, args)) pool.close() diff --git a/deep_speech_2/decoders/lm_scorer_deprecated.py b/deep_speech_2/decoders/scorer_deprecated.py similarity index 100% rename from deep_speech_2/decoders/lm_scorer_deprecated.py rename to deep_speech_2/decoders/scorer_deprecated.py diff --git a/deep_speech_2/decoders/swig/ctc_decoders.cpp b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp similarity index 55% rename from deep_speech_2/decoders/swig/ctc_decoders.cpp rename to deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp index 35425fbca3..36d1698713 100644 --- a/deep_speech_2/decoders/swig/ctc_decoders.cpp +++ b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp @@ -1,4 +1,4 @@ -#include "ctc_decoders.h" +#include "ctc_beam_search_decoder.h" #include #include @@ -9,59 +9,19 @@ #include "ThreadPool.h" #include "fst/fstlib.h" +#include "fst/log.h" #include "decoder_utils.h" #include "path_trie.h" -std::string ctc_greedy_decoder( - const std::vector> &probs_seq, - const std::vector &vocabulary) { - // dimension check - size_t num_time_steps = probs_seq.size(); - for (size_t i = 0; i < num_time_steps; ++i) { - VALID_CHECK_EQ(probs_seq[i].size(), - vocabulary.size() + 1, - "The shape of probs_seq does not match with " - "the shape of the vocabulary"); - } - - size_t blank_id = vocabulary.size(); - - std::vector max_idx_vec; - for (size_t i = 0; i < num_time_steps; ++i) { - double max_prob = 0.0; - size_t max_idx = 0; - for (size_t j = 0; j < probs_seq[i].size(); j++) { - if (max_prob < probs_seq[i][j]) { - max_idx = j; - max_prob = probs_seq[i][j]; - } - } - max_idx_vec.push_back(max_idx); - } - - std::vector idx_vec; - for (size_t i = 0; i < max_idx_vec.size(); ++i) { - if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i - 1])) { - idx_vec.push_back(max_idx_vec[i]); - } - } - - std::string best_path_result; - for (size_t i = 0; i < idx_vec.size(); ++i) { - if (idx_vec[i] != blank_id) { - best_path_result += vocabulary[idx_vec[i]]; - } - } - return best_path_result; -} +using FSTMATCH = fst::SortedMatcher; std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, - const size_t beam_size, + size_t beam_size, std::vector vocabulary, - const double cutoff_prob, - const size_t cutoff_top_n, + double cutoff_prob, + size_t cutoff_top_n, Scorer *ext_scorer) { // dimension check size_t num_time_steps = probs_seq.size(); @@ -80,7 +40,7 @@ std::vector> ctc_beam_search_decoder( std::find(vocabulary.begin(), vocabulary.end(), " "); int space_id = it - vocabulary.begin(); // if no space in vocabulary - if (space_id >= vocabulary.size()) { + if ((size_t)space_id >= vocabulary.size()) { space_id = -2; } @@ -90,30 +50,17 @@ std::vector> ctc_beam_search_decoder( std::vector prefixes; prefixes.push_back(&root); - if (ext_scorer != nullptr) { - if (ext_scorer->is_char_map_empty()) { - ext_scorer->set_char_map(vocabulary); - } - if (!ext_scorer->is_character_based()) { - if (ext_scorer->dictionary == nullptr) { - // fill dictionary for fst with space - ext_scorer->fill_dictionary(true); - } - auto fst_dict = static_cast(ext_scorer->dictionary); - fst::StdVectorFst *dict_ptr = fst_dict->Copy(true); - root.set_dictionary(dict_ptr); - auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); - root.set_matcher(matcher); - } + if (ext_scorer != nullptr && !ext_scorer->is_character_based()) { + auto fst_dict = static_cast(ext_scorer->dictionary); + fst::StdVectorFst *dict_ptr = fst_dict->Copy(true); + root.set_dictionary(dict_ptr); + auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); + root.set_matcher(matcher); } // prefix search over time - for (size_t time_step = 0; time_step < num_time_steps; time_step++) { - std::vector prob = probs_seq[time_step]; - std::vector> prob_idx; - for (size_t i = 0; i < prob.size(); ++i) { - prob_idx.push_back(std::pair(i, prob[i])); - } + for (size_t time_step = 0; time_step < num_time_steps; ++time_step) { + auto &prob = probs_seq[time_step]; float min_cutoff = -NUM_FLT_INF; bool full_beam = false; @@ -121,43 +68,20 @@ std::vector> ctc_beam_search_decoder( size_t num_prefixes = std::min(prefixes.size(), beam_size); std::sort( prefixes.begin(), prefixes.begin() + num_prefixes, prefix_compare); - min_cutoff = prefixes[num_prefixes - 1]->score + log(prob[blank_id]) - - std::max(0.0, ext_scorer->beta); + min_cutoff = prefixes[num_prefixes - 1]->score + + std::log(prob[blank_id]) - std::max(0.0, ext_scorer->beta); full_beam = (num_prefixes == beam_size); } - // pruning of vacobulary - size_t cutoff_len = prob.size(); - if (cutoff_prob < 1.0 || cutoff_top_n < cutoff_len) { - std::sort( - prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); - if (cutoff_prob < 1.0) { - double cum_prob = 0.0; - cutoff_len = 0; - for (size_t i = 0; i < prob_idx.size(); ++i) { - cum_prob += prob_idx[i].second; - cutoff_len += 1; - if (cum_prob >= cutoff_prob) break; - } - } - cutoff_len = std::min(cutoff_len, cutoff_top_n); - prob_idx = std::vector>( - prob_idx.begin(), prob_idx.begin() + cutoff_len); - } - std::vector> log_prob_idx; - for (size_t i = 0; i < cutoff_len; ++i) { - log_prob_idx.push_back(std::pair( - prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); - } - + std::vector> log_prob_idx = + get_pruned_log_probs(prob, cutoff_prob, cutoff_top_n); // loop over chars for (size_t index = 0; index < log_prob_idx.size(); index++) { auto c = log_prob_idx[index].first; - float log_prob_c = log_prob_idx[index].second; + auto log_prob_c = log_prob_idx[index].second; for (size_t i = 0; i < prefixes.size() && i < beam_size; ++i) { auto prefix = prefixes[i]; - if (full_beam && log_prob_c + prefix->score < min_cutoff) { break; } @@ -189,7 +113,6 @@ std::vector> ctc_beam_search_decoder( if (ext_scorer != nullptr && (c == space_id || ext_scorer->is_character_based())) { PathTrie *prefix_toscore = nullptr; - // skip scoring the space if (ext_scorer->is_character_based()) { prefix_toscore = prefix_new; @@ -201,7 +124,6 @@ std::vector> ctc_beam_search_decoder( std::vector ngram; ngram = ext_scorer->make_ngram(prefix_toscore); score = ext_scorer->get_log_cond_prob(ngram) * ext_scorer->alpha; - log_p += score; log_p += ext_scorer->beta; } @@ -221,57 +143,33 @@ std::vector> ctc_beam_search_decoder( prefixes.begin() + beam_size, prefixes.end(), prefix_compare); - for (size_t i = beam_size; i < prefixes.size(); ++i) { prefixes[i]->remove(); } } } // end of loop over time - // compute aproximate ctc score as the return score + // compute aproximate ctc score as the return score, without affecting the + // return order of decoding result. To delete when decoder gets stable. for (size_t i = 0; i < beam_size && i < prefixes.size(); ++i) { double approx_ctc = prefixes[i]->score; - if (ext_scorer != nullptr) { std::vector output; prefixes[i]->get_path_vec(output); - size_t prefix_length = output.size(); + auto prefix_length = output.size(); auto words = ext_scorer->split_labels(output); // remove word insert approx_ctc = approx_ctc - prefix_length * ext_scorer->beta; // remove language model weight: approx_ctc -= (ext_scorer->get_sent_log_prob(words)) * ext_scorer->alpha; } - prefixes[i]->approx_ctc = approx_ctc; } - // allow for the post processing - std::vector space_prefixes; - if (space_prefixes.empty()) { - for (size_t i = 0; i < beam_size && i < prefixes.size(); ++i) { - space_prefixes.push_back(prefixes[i]); - } - } - - std::sort(space_prefixes.begin(), space_prefixes.end(), prefix_compare); - std::vector> output_vecs; - for (size_t i = 0; i < beam_size && i < space_prefixes.size(); ++i) { - std::vector output; - space_prefixes[i]->get_path_vec(output); - // convert index to string - std::string output_str; - for (size_t j = 0; j < output.size(); j++) { - output_str += vocabulary[output[j]]; - } - std::pair output_pair(-space_prefixes[i]->approx_ctc, - output_str); - output_vecs.emplace_back(output_pair); - } - - return output_vecs; + return get_beam_search_result(prefixes, vocabulary, beam_size); } + std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, @@ -287,18 +185,6 @@ ctc_beam_search_decoder_batch( // number of samples size_t batch_size = probs_split.size(); - // scorer filling up - if (ext_scorer != nullptr) { - if (ext_scorer->is_char_map_empty()) { - ext_scorer->set_char_map(vocabulary); - } - if (!ext_scorer->is_character_based() && - ext_scorer->dictionary == nullptr) { - // init dictionary - ext_scorer->fill_dictionary(true); - } - } - // enqueue the tasks of decoding std::vector>>> res; for (size_t i = 0; i < batch_size; ++i) { diff --git a/deep_speech_2/decoders/swig/ctc_decoders.h b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h similarity index 75% rename from deep_speech_2/decoders/swig/ctc_decoders.h rename to deep_speech_2/decoders/swig/ctc_beam_search_decoder.h index 6384c8a8fc..c800384e5b 100644 --- a/deep_speech_2/decoders/swig/ctc_decoders.h +++ b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h @@ -7,19 +7,6 @@ #include "scorer.h" -/* CTC Best Path Decoder - * - * Parameters: - * probs_seq: 2-D vector that each element is a vector of probabilities - * over vocabulary of one time step. - * vocabulary: A vector of vocabulary. - * Return: - * The decoding result in string - */ -std::string ctc_greedy_decoder( - const std::vector> &probs_seq, - const std::vector &vocabulary); - /* CTC Beam Search Decoder * Parameters: @@ -38,11 +25,11 @@ std::string ctc_greedy_decoder( */ std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, - const size_t beam_size, + size_t beam_size, std::vector vocabulary, - const double cutoff_prob = 1.0, - const size_t cutoff_top_n = 40, - Scorer *ext_scorer = NULL); + double cutoff_prob = 1.0, + size_t cutoff_top_n = 40, + Scorer *ext_scorer = nullptr); /* CTC Beam Search Decoder for batch data @@ -65,11 +52,11 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, - const size_t beam_size, + size_t beam_size, const std::vector &vocabulary, - const size_t num_processes, + size_t num_processes, double cutoff_prob = 1.0, - const size_t cutoff_top_n = 40, - Scorer *ext_scorer = NULL); + size_t cutoff_top_n = 40, + Scorer *ext_scorer = nullptr); #endif // CTC_BEAM_SEARCH_DECODER_H_ diff --git a/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp b/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp new file mode 100644 index 0000000000..c4c94539ee --- /dev/null +++ b/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp @@ -0,0 +1,45 @@ +#include "ctc_greedy_decoder.h" +#include "decoder_utils.h" + +std::string ctc_greedy_decoder( + const std::vector> &probs_seq, + const std::vector &vocabulary) { + // dimension check + size_t num_time_steps = probs_seq.size(); + for (size_t i = 0; i < num_time_steps; ++i) { + VALID_CHECK_EQ(probs_seq[i].size(), + vocabulary.size() + 1, + "The shape of probs_seq does not match with " + "the shape of the vocabulary"); + } + + size_t blank_id = vocabulary.size(); + + std::vector max_idx_vec(num_time_steps, 0); + std::vector idx_vec; + for (size_t i = 0; i < num_time_steps; ++i) { + double max_prob = 0.0; + size_t max_idx = 0; + const std::vector &probs_step = probs_seq[i]; + for (size_t j = 0; j < probs_step.size(); ++j) { + if (max_prob < probs_step[j]) { + max_idx = j; + max_prob = probs_step[j]; + } + } + // id with maximum probability in current step + max_idx_vec[i] = max_idx; + // deduplicate + if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i - 1])) { + idx_vec.push_back(max_idx_vec[i]); + } + } + + std::string best_path_result; + for (size_t i = 0; i < idx_vec.size(); ++i) { + if (idx_vec[i] != blank_id) { + best_path_result += vocabulary[idx_vec[i]]; + } + } + return best_path_result; +} diff --git a/deep_speech_2/decoders/swig/ctc_greedy_decoder.h b/deep_speech_2/decoders/swig/ctc_greedy_decoder.h new file mode 100644 index 0000000000..043742f26e --- /dev/null +++ b/deep_speech_2/decoders/swig/ctc_greedy_decoder.h @@ -0,0 +1,20 @@ +#ifndef CTC_GREEDY_DECODER_H +#define CTC_GREEDY_DECODER_H + +#include +#include + +/* CTC Greedy (Best Path) Decoder + * + * Parameters: + * probs_seq: 2-D vector that each element is a vector of probabilities + * over vocabulary of one time step. + * vocabulary: A vector of vocabulary. + * Return: + * The decoding result in string + */ +std::string ctc_greedy_decoder( + const std::vector> &probs_seq, + const std::vector &vocabulary); + +#endif // CTC_GREEDY_DECODER_H diff --git a/deep_speech_2/decoders/swig/decoder_utils.cpp b/deep_speech_2/decoders/swig/decoder_utils.cpp index 989b067e7b..665fcc22f8 100644 --- a/deep_speech_2/decoders/swig/decoder_utils.cpp +++ b/deep_speech_2/decoders/swig/decoder_utils.cpp @@ -4,6 +4,71 @@ #include #include +std::vector> get_pruned_log_probs( + const std::vector &prob_step, + double cutoff_prob, + size_t cutoff_top_n) { + std::vector> prob_idx; + for (size_t i = 0; i < prob_step.size(); ++i) { + prob_idx.push_back(std::pair(i, prob_step[i])); + } + // pruning of vacobulary + size_t cutoff_len = prob_step.size(); + if (cutoff_prob < 1.0 || cutoff_top_n < cutoff_len) { + std::sort( + prob_idx.begin(), prob_idx.end(), pair_comp_second_rev); + if (cutoff_prob < 1.0) { + double cum_prob = 0.0; + cutoff_len = 0; + for (size_t i = 0; i < prob_idx.size(); ++i) { + cum_prob += prob_idx[i].second; + cutoff_len += 1; + if (cum_prob >= cutoff_prob) break; + } + } + cutoff_len = std::min(cutoff_len, cutoff_top_n); + prob_idx = std::vector>( + prob_idx.begin(), prob_idx.begin() + cutoff_len); + } + std::vector> log_prob_idx; + for (size_t i = 0; i < cutoff_len; ++i) { + log_prob_idx.push_back(std::pair( + prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN))); + } + return log_prob_idx; +} + + +std::vector> get_beam_search_result( + const std::vector &prefixes, + const std::vector &vocabulary, + size_t beam_size) { + // allow for the post processing + std::vector space_prefixes; + if (space_prefixes.empty()) { + for (size_t i = 0; i < beam_size && i < prefixes.size(); ++i) { + space_prefixes.push_back(prefixes[i]); + } + } + + std::sort(space_prefixes.begin(), space_prefixes.end(), prefix_compare); + std::vector> output_vecs; + for (size_t i = 0; i < beam_size && i < space_prefixes.size(); ++i) { + std::vector output; + space_prefixes[i]->get_path_vec(output); + // convert index to string + std::string output_str; + for (size_t j = 0; j < output.size(); j++) { + output_str += vocabulary[output[j]]; + } + std::pair output_pair(-space_prefixes[i]->approx_ctc, + output_str); + output_vecs.emplace_back(output_pair); + } + + return output_vecs; +} + size_t get_utf8_str_len(const std::string &str) { size_t str_len = 0; for (char c : str) { diff --git a/deep_speech_2/decoders/swig/decoder_utils.h b/deep_speech_2/decoders/swig/decoder_utils.h index 015646ddd7..932ffb12f7 100644 --- a/deep_speech_2/decoders/swig/decoder_utils.h +++ b/deep_speech_2/decoders/swig/decoder_utils.h @@ -3,25 +3,26 @@ #include #include "path_trie.h" +#include "fst/log.h" const float NUM_FLT_INF = std::numeric_limits::max(); const float NUM_FLT_MIN = std::numeric_limits::min(); -// check if __A == _B -#define VALID_CHECK_EQ(__A, __B, __ERR) \ - if ((__A) != (__B)) { \ - std::ostringstream str; \ - str << (__A) << " != " << (__B) << ", "; \ - throw std::runtime_error(str.str() + __ERR); \ +// inline function for validation check +inline void check( + bool x, const char *expr, const char *file, int line, const char *err) { + if (!x) { + std::cout << "[" << file << ":" << line << "] "; + LOG(FATAL) << "\"" << expr << "\" check failed. " << err; } +} + +#define VALID_CHECK(x, info) \ + check(static_cast(x), #x, __FILE__, __LINE__, info) +#define VALID_CHECK_EQ(x, y, info) VALID_CHECK((x) == (y), info) +#define VALID_CHECK_GT(x, y, info) VALID_CHECK((x) > (y), info) +#define VALID_CHECK_LT(x, y, info) VALID_CHECK((x) < (y), info) -// check if __A > __B -#define VALID_CHECK_GT(__A, __B, __ERR) \ - if ((__A) <= (__B)) { \ - std::ostringstream str; \ - str << (__A) << " <= " << (__B) << ", "; \ - throw std::runtime_error(str.str() + __ERR); \ - } // Function template for comparing two pairs template @@ -47,6 +48,18 @@ T log_sum_exp(const T &x, const T &y) { return std::log(std::exp(x - xmax) + std::exp(y - xmax)) + xmax; } +// Get pruned probability vector for each time step's beam search +std::vector> get_pruned_log_probs( + const std::vector &prob_step, + double cutoff_prob, + size_t cutoff_top_n); + +// Get beam search result from prefixes in trie tree +std::vector> get_beam_search_result( + const std::vector &prefixes, + const std::vector &vocabulary, + size_t beam_size); + // Functor for prefix comparsion bool prefix_compare(const PathTrie *x, const PathTrie *y); diff --git a/deep_speech_2/decoders/swig/decoders.i b/deep_speech_2/decoders/swig/decoders.i index 8059199d14..4227d4a375 100644 --- a/deep_speech_2/decoders/swig/decoders.i +++ b/deep_speech_2/decoders/swig/decoders.i @@ -1,7 +1,8 @@ %module swig_decoders %{ #include "scorer.h" -#include "ctc_decoders.h" +#include "ctc_greedy_decoder.h" +#include "ctc_beam_search_decoder.h" #include "decoder_utils.h" %} @@ -28,4 +29,5 @@ namespace std { %template(DoubleStringPairCompFirstRev) pair_comp_first_rev; %include "scorer.h" -%include "ctc_decoders.h" +%include "ctc_greedy_decoder.h" +%include "ctc_beam_search_decoder.h" diff --git a/deep_speech_2/decoders/swig/path_trie.h b/deep_speech_2/decoders/swig/path_trie.h index ddeccd9108..b4f5bc4baa 100644 --- a/deep_speech_2/decoders/swig/path_trie.h +++ b/deep_speech_2/decoders/swig/path_trie.h @@ -1,14 +1,13 @@ #ifndef PATH_TRIE_H #define PATH_TRIE_H -#pragma once -#include + #include #include #include #include #include -using FSTMATCH = fst::SortedMatcher; +#include "fst/fstlib.h" /* Trie tree for prefix storing and manipulating, with a dictionary in * finite-state transducer for spelling correction. @@ -35,7 +34,7 @@ class PathTrie { // set dictionary for FST void set_dictionary(fst::StdVectorFst* dictionary); - void set_matcher(std::shared_ptr matcher); + void set_matcher(std::shared_ptr>); bool is_empty() { return _ROOT == character; } @@ -62,7 +61,7 @@ class PathTrie { fst::StdVectorFst* _dictionary; fst::StdVectorFst::StateId _dictionary_state; // true if finding ars in FST - std::shared_ptr _matcher; + std::shared_ptr> _matcher; }; #endif // PATH_TRIE_H diff --git a/deep_speech_2/decoders/swig/scorer.cpp b/deep_speech_2/decoders/swig/scorer.cpp index 75919c3c9e..6b28034435 100644 --- a/deep_speech_2/decoders/swig/scorer.cpp +++ b/deep_speech_2/decoders/swig/scorer.cpp @@ -13,29 +13,47 @@ using namespace lm::ngram; -Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { +Scorer::Scorer(double alpha, + double beta, + const std::string& lm_path, + const std::vector& vocab_list) { this->alpha = alpha; this->beta = beta; _is_character_based = true; _language_model = nullptr; dictionary = nullptr; _max_order = 0; + _dict_size = 0; _SPACE_ID = -1; - // load language model - load_LM(lm_path.c_str()); + + setup(lm_path, vocab_list); } Scorer::~Scorer() { - if (_language_model != nullptr) + if (_language_model != nullptr) { delete static_cast(_language_model); - if (dictionary != nullptr) delete static_cast(dictionary); + } + if (dictionary != nullptr) { + delete static_cast(dictionary); + } } -void Scorer::load_LM(const char* filename) { - if (access(filename, F_OK) != 0) { - std::cerr << "Invalid language model file !!!" << std::endl; - exit(1); +void Scorer::setup(const std::string& lm_path, + const std::vector& vocab_list) { + // load language model + load_lm(lm_path); + // set char map for scorer + set_char_map(vocab_list); + // fill the dictionary for FST + if (!is_character_based()) { + fill_dictionary(true); } +} + +void Scorer::load_lm(const std::string& lm_path) { + const char* filename = lm_path.c_str(); + VALID_CHECK_EQ(access(filename, F_OK), 0, "Invalid language model path"); + RetriveStrEnumerateVocab enumerate; lm::ngram::Config config; config.enumerate_vocab = &enumerate; @@ -180,14 +198,14 @@ void Scorer::fill_dictionary(bool add_space) { } // For each unigram convert to ints and put in trie - int vocab_size = 0; + int dict_size = 0; for (const auto& word : _vocabulary) { bool added = add_word_to_dictionary( word, char_map, add_space, _SPACE_ID, &dictionary); - vocab_size += added ? 1 : 0; + dict_size += added ? 1 : 0; } - std::cerr << "Vocab Size " << vocab_size << std::endl; + _dict_size = dict_size; /* Simplify FST diff --git a/deep_speech_2/decoders/swig/scorer.h b/deep_speech_2/decoders/swig/scorer.h index 1b4857e38e..72544da7b5 100644 --- a/deep_speech_2/decoders/swig/scorer.h +++ b/deep_speech_2/decoders/swig/scorer.h @@ -40,31 +40,32 @@ class RetriveStrEnumerateVocab : public lm::EnumerateVocab { */ class Scorer { public: - Scorer(double alpha, double beta, const std::string &lm_path); + Scorer(double alpha, + double beta, + const std::string &lm_path, + const std::vector &vocabulary); ~Scorer(); double get_log_cond_prob(const std::vector &words); double get_sent_log_prob(const std::vector &words); - size_t get_max_order() { return _max_order; } + size_t get_max_order() const { return _max_order; } - bool is_char_map_empty() { return _char_map.size() == 0; } + size_t get_dict_size() const { return _dict_size; } - bool is_character_based() { return _is_character_based; } + bool is_char_map_empty() const { return _char_map.size() == 0; } + + bool is_character_based() const { return _is_character_based; } // reset params alpha & beta void reset_params(float alpha, float beta); - // make ngram + // make ngram for a given prefix std::vector make_ngram(PathTrie *prefix); - // fill dictionary for fst - void fill_dictionary(bool add_space); - - // set char map - void set_char_map(const std::vector &char_list); - + // trransform the labels in index to the vector of words (word based lm) or + // the vector of characters (character based lm) std::vector split_labels(const std::vector &labels); // expose to decoder @@ -75,7 +76,16 @@ class Scorer { void *dictionary; protected: - void load_LM(const char *filename); + void setup(const std::string &lm_path, + const std::vector &vocab_list); + + void load_lm(const std::string &lm_path); + + // fill dictionary for fst + void fill_dictionary(bool add_space); + + // set char map + void set_char_map(const std::vector &char_list); double get_log_prob(const std::vector &words); @@ -85,6 +95,7 @@ class Scorer { void *_language_model; bool _is_character_based; size_t _max_order; + size_t _dict_size; int _SPACE_ID; std::vector _char_list; diff --git a/deep_speech_2/decoders/swig/setup.py b/deep_speech_2/decoders/swig/setup.py index 7a4b7e02c3..8af9ff3047 100644 --- a/deep_speech_2/decoders/swig/setup.py +++ b/deep_speech_2/decoders/swig/setup.py @@ -70,8 +70,11 @@ def compile_test(header, library): FILES += glob.glob('openfst-1.6.3/src/lib/*.cc') +# FILES + glob.glob('glog/src/*.cc') FILES = [ - fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc')) + fn for fn in FILES + if not (fn.endswith('main.cc') or fn.endswith('test.cc') or fn.endswith( + 'unittest.cc')) ] LIBS = ['stdc++'] @@ -99,7 +102,13 @@ def compile_test(header, library): name='_swig_decoders', sources=FILES + glob.glob('*.cxx') + glob.glob('*.cpp'), language='c++', - include_dirs=['.', 'kenlm', 'openfst-1.6.3/src/include', 'ThreadPool'], + include_dirs=[ + '.', + 'kenlm', + 'openfst-1.6.3/src/include', + 'ThreadPool', + #'glog/src' + ], libraries=LIBS, extra_compile_args=ARGS) ] diff --git a/deep_speech_2/decoders/swig/setup.sh b/deep_speech_2/decoders/swig/setup.sh index 069f51d6eb..78ae2b2011 100644 --- a/deep_speech_2/decoders/swig/setup.sh +++ b/deep_speech_2/decoders/swig/setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [ ! -d kenlm ]; then git clone https://github.com/luotao1/kenlm.git diff --git a/deep_speech_2/decoders/swig_wrapper.py b/deep_speech_2/decoders/swig_wrapper.py index 54ed249f38..5ebcd133c9 100644 --- a/deep_speech_2/decoders/swig_wrapper.py +++ b/deep_speech_2/decoders/swig_wrapper.py @@ -13,14 +13,14 @@ class Scorer(swig_decoders.Scorer): language model when alpha = 0. :type alpha: float :param beta: Parameter associated with word count. Don't use word - count when beta = 0. + count when beta = 0. :type beta: float :model_path: Path to load language model. :type model_path: basestring """ - def __init__(self, alpha, beta, model_path): - swig_decoders.Scorer.__init__(self, alpha, beta, model_path) + def __init__(self, alpha, beta, model_path, vocabulary): + swig_decoders.Scorer.__init__(self, alpha, beta, model_path, vocabulary) def ctc_greedy_decoder(probs_seq, vocabulary): @@ -58,12 +58,12 @@ def ctc_beam_search_decoder(probs_seq, default 1.0, no pruning. :type cutoff_prob: float :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n - characters with highest probs in vocabulary will be - used in beam search, default 40. + characters with highest probs in vocabulary will be + used in beam search, default 40. :type cutoff_top_n: int :param ext_scoring_func: External scoring function for - partially decoded sentence, e.g. word count - or language model. + partially decoded sentence, e.g. word count + or language model. :type external_scoring_func: callable :return: List of tuples of log probability and sentence as decoding results, in descending order of the probability. @@ -96,14 +96,14 @@ def ctc_beam_search_decoder_batch(probs_split, default 1.0, no pruning. :type cutoff_prob: float :param cutoff_top_n: Cutoff number in pruning, only top cutoff_top_n - characters with highest probs in vocabulary will be - used in beam search, default 40. + characters with highest probs in vocabulary will be + used in beam search, default 40. :type cutoff_top_n: int :param num_processes: Number of parallel processes. :type num_processes: int :param ext_scoring_func: External scoring function for - partially decoded sentence, e.g. word count - or language model. + partially decoded sentence, e.g. word count + or language model. :type external_scoring_function: callable :return: List of tuples of log probability and sentence as decoding results, in descending order of the probability. diff --git a/deep_speech_2/examples/tiny/run_infer.sh b/deep_speech_2/examples/tiny/run_infer.sh index 1d33bfbba2..1e90f6081c 100644 --- a/deep_speech_2/examples/tiny/run_infer.sh +++ b/deep_speech_2/examples/tiny/run_infer.sh @@ -21,9 +21,9 @@ python -u infer.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/tiny/run_infer_golden.sh b/deep_speech_2/examples/tiny/run_infer_golden.sh index 32e9d8623f..40bb30337b 100644 --- a/deep_speech_2/examples/tiny/run_infer_golden.sh +++ b/deep_speech_2/examples/tiny/run_infer_golden.sh @@ -30,9 +30,9 @@ python -u infer.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/tiny/run_test.sh b/deep_speech_2/examples/tiny/run_test.sh index f9c3cc11ce..868a045f4e 100644 --- a/deep_speech_2/examples/tiny/run_test.sh +++ b/deep_speech_2/examples/tiny/run_test.sh @@ -22,9 +22,9 @@ python -u test.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/examples/tiny/run_test_golden.sh b/deep_speech_2/examples/tiny/run_test_golden.sh index 080c3c0622..1a4731dd1c 100644 --- a/deep_speech_2/examples/tiny/run_test_golden.sh +++ b/deep_speech_2/examples/tiny/run_test_golden.sh @@ -31,9 +31,9 @@ python -u test.py \ --num_conv_layers=2 \ --num_rnn_layers=3 \ --rnn_layer_size=2048 \ ---alpha=0.36 \ ---beta=0.25 \ ---cutoff_prob=0.99 \ +--alpha=2.15 \ +--beta=0.35 \ +--cutoff_prob=1.0 \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ diff --git a/deep_speech_2/infer.py b/deep_speech_2/infer.py index 1064fd25a0..e635f6d0f9 100644 --- a/deep_speech_2/infer.py +++ b/deep_speech_2/infer.py @@ -112,6 +112,7 @@ def infer(): print("Current error rate [%s] = %f" % (args.error_rate_type, error_rate_func(target, result))) + ds2_model.logger.info("finish inference") def main(): print_arguments(args) diff --git a/deep_speech_2/model_utils/model.py b/deep_speech_2/model_utils/model.py index 4f5021a6d5..66b161c3e9 100644 --- a/deep_speech_2/model_utils/model.py +++ b/deep_speech_2/model_utils/model.py @@ -6,6 +6,7 @@ import sys import os import time +import logging import gzip import paddle.v2 as paddle from decoders.swig_wrapper import Scorer @@ -13,6 +14,9 @@ from decoders.swig_wrapper import ctc_beam_search_decoder_batch from model_utils.network import deep_speech_v2_network +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s') + class DeepSpeech2Model(object): """DeepSpeech2Model class. @@ -43,6 +47,8 @@ def __init__(self, vocab_size, num_conv_layers, num_rnn_layers, self._inferer = None self._loss_inferer = None self._ext_scorer = None + self.logger = logging.getLogger("") + self.logger.setLevel(level=logging.INFO) def train(self, train_batch_reader, @@ -204,16 +210,25 @@ def infer_batch(self, infer_data, decoding_method, beam_alpha, beam_beta, elif decoding_method == "ctc_beam_search": # initialize external scorer if self._ext_scorer == None: - self._ext_scorer = Scorer(beam_alpha, beam_beta, - language_model_path) self._loaded_lm_path = language_model_path - self._ext_scorer.set_char_map(vocab_list) - if (not self._ext_scorer.is_character_based()): - self._ext_scorer.fill_dictionary(True) + self.logger.info("begin to initialize the external scorer " + "for decoding") + self._ext_scorer = Scorer(beam_alpha, beam_beta, + language_model_path, vocab_list) + + lm_char_based = self._ext_scorer.is_character_based() + lm_max_order = self._ext_scorer.get_max_order() + lm_dict_size = self._ext_scorer.get_dict_size() + self.logger.info("language model: " + "is_character_based = %d," % lm_char_based + + " max_order = %d," % lm_max_order + + " dict_size = %d" % lm_dict_size) + self.logger.info("end initializing scorer. Start decoding ...") else: self._ext_scorer.reset_params(beam_alpha, beam_beta) assert self._loaded_lm_path == language_model_path # beam search decode + num_processes = min(num_processes, len(probs_split)) beam_search_results = ctc_beam_search_decoder_batch( probs_split=probs_split, vocabulary=vocab_list, diff --git a/deep_speech_2/test.py b/deep_speech_2/test.py index c564bb85db..40f0795a13 100644 --- a/deep_speech_2/test.py +++ b/deep_speech_2/test.py @@ -115,6 +115,7 @@ def evaluate(): print("Final error rate [%s] (%d/%d) = %f" % (args.error_rate_type, num_ins, num_ins, error_sum / num_ins)) + ds2_model.logger.info("finish evaluation") def main(): print_arguments(args) From 8c5576d991b900d73f5d64b30ef782680492c473 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Sun, 17 Sep 2017 21:30:59 +0800 Subject: [PATCH 31/36] format varabiables' name & add more comments --- .../decoders/swig/ctc_beam_search_decoder.cpp | 15 ++-- .../decoders/swig/ctc_beam_search_decoder.h | 9 +- deep_speech_2/decoders/swig/path_trie.cpp | 76 ++++++++--------- deep_speech_2/decoders/swig/path_trie.h | 16 ++-- deep_speech_2/decoders/swig/scorer.cpp | 82 +++++++++---------- deep_speech_2/decoders/swig/scorer.h | 39 +++++---- deep_speech_2/decoders/swig_wrapper.py | 18 ++-- 7 files changed, 129 insertions(+), 126 deletions(-) diff --git a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp index 36d1698713..5c8373beaa 100644 --- a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp @@ -18,8 +18,8 @@ using FSTMATCH = fst::SortedMatcher; std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, + const std::vector &vocabulary, size_t beam_size, - std::vector vocabulary, double cutoff_prob, size_t cutoff_top_n, Scorer *ext_scorer) { @@ -36,8 +36,7 @@ std::vector> ctc_beam_search_decoder( size_t blank_id = vocabulary.size(); // assign space id - std::vector::iterator it = - std::find(vocabulary.begin(), vocabulary.end(), " "); + auto it = std::find(vocabulary.begin(), vocabulary.end(), " "); int space_id = it - vocabulary.begin(); // if no space in vocabulary if ((size_t)space_id >= vocabulary.size()) { @@ -173,11 +172,11 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, - const size_t beam_size, const std::vector &vocabulary, - const size_t num_processes, - const double cutoff_prob, - const size_t cutoff_top_n, + size_t beam_size, + size_t num_processes, + double cutoff_prob, + size_t cutoff_top_n, Scorer *ext_scorer) { VALID_CHECK_GT(num_processes, 0, "num_processes must be nonnegative!"); // thread pool @@ -190,8 +189,8 @@ ctc_beam_search_decoder_batch( for (size_t i = 0; i < batch_size; ++i) { res.emplace_back(pool.enqueue(ctc_beam_search_decoder, probs_split[i], - beam_size, vocabulary, + beam_size, cutoff_prob, cutoff_top_n, ext_scorer)); diff --git a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h index c800384e5b..6fdd15517e 100644 --- a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h +++ b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.h @@ -12,8 +12,8 @@ * Parameters: * probs_seq: 2-D vector that each element is a vector of probabilities * over vocabulary of one time step. - * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. + * beam_size: The width of beam search. * cutoff_prob: Cutoff probability for pruning. * cutoff_top_n: Cutoff number for pruning. * ext_scorer: External scorer to evaluate a prefix, which consists of @@ -25,8 +25,8 @@ */ std::vector> ctc_beam_search_decoder( const std::vector> &probs_seq, + const std::vector &vocabulary, size_t beam_size, - std::vector vocabulary, double cutoff_prob = 1.0, size_t cutoff_top_n = 40, Scorer *ext_scorer = nullptr); @@ -36,9 +36,8 @@ std::vector> ctc_beam_search_decoder( * Parameters: * probs_seq: 3-D vector that each element is a 2-D vector that can be used * by ctc_beam_search_decoder(). - * . - * beam_size: The width of beam search. * vocabulary: A vector of vocabulary. + * beam_size: The width of beam search. * num_processes: Number of threads for beam search. * cutoff_prob: Cutoff probability for pruning. * cutoff_top_n: Cutoff number for pruning. @@ -52,8 +51,8 @@ std::vector> ctc_beam_search_decoder( std::vector>> ctc_beam_search_decoder_batch( const std::vector>> &probs_split, - size_t beam_size, const std::vector &vocabulary, + size_t beam_size, size_t num_processes, double cutoff_prob = 1.0, size_t cutoff_top_n = 40, diff --git a/deep_speech_2/decoders/swig/path_trie.cpp b/deep_speech_2/decoders/swig/path_trie.cpp index 6a1f6170f5..fdff328611 100644 --- a/deep_speech_2/decoders/swig/path_trie.cpp +++ b/deep_speech_2/decoders/swig/path_trie.cpp @@ -15,32 +15,32 @@ PathTrie::PathTrie() { log_prob_nb_cur = -NUM_FLT_INF; score = -NUM_FLT_INF; - _ROOT = -1; - character = _ROOT; - _exists = true; + ROOT_ = -1; + character = ROOT_; + exists_ = true; parent = nullptr; - _dictionary = nullptr; - _dictionary_state = 0; - _has_dictionary = false; - _matcher = nullptr; + dictionary_ = nullptr; + dictionary_state_ = 0; + has_dictionary_ = false; + matcher_ = nullptr; } PathTrie::~PathTrie() { - for (auto child : _children) { + for (auto child : children_) { delete child.second; } } PathTrie* PathTrie::get_path_trie(int new_char, bool reset) { - auto child = _children.begin(); - for (child = _children.begin(); child != _children.end(); ++child) { + auto child = children_.begin(); + for (child = children_.begin(); child != children_.end(); ++child) { if (child->first == new_char) { break; } } - if (child != _children.end()) { - if (!child->second->_exists) { - child->second->_exists = true; + if (child != children_.end()) { + if (!child->second->exists_) { + child->second->exists_ = true; child->second->log_prob_b_prev = -NUM_FLT_INF; child->second->log_prob_nb_prev = -NUM_FLT_INF; child->second->log_prob_b_cur = -NUM_FLT_INF; @@ -48,47 +48,47 @@ PathTrie* PathTrie::get_path_trie(int new_char, bool reset) { } return (child->second); } else { - if (_has_dictionary) { - _matcher->SetState(_dictionary_state); - bool found = _matcher->Find(new_char); + if (has_dictionary_) { + matcher_->SetState(dictionary_state_); + bool found = matcher_->Find(new_char); if (!found) { // Adding this character causes word outside dictionary auto FSTZERO = fst::TropicalWeight::Zero(); - auto final_weight = _dictionary->Final(_dictionary_state); + auto final_weight = dictionary_->Final(dictionary_state_); bool is_final = (final_weight != FSTZERO); if (is_final && reset) { - _dictionary_state = _dictionary->Start(); + dictionary_state_ = dictionary_->Start(); } return nullptr; } else { PathTrie* new_path = new PathTrie; new_path->character = new_char; new_path->parent = this; - new_path->_dictionary = _dictionary; - new_path->_dictionary_state = _matcher->Value().nextstate; - new_path->_has_dictionary = true; - new_path->_matcher = _matcher; - _children.push_back(std::make_pair(new_char, new_path)); + new_path->dictionary_ = dictionary_; + new_path->dictionary_state_ = matcher_->Value().nextstate; + new_path->has_dictionary_ = true; + new_path->matcher_ = matcher_; + children_.push_back(std::make_pair(new_char, new_path)); return new_path; } } else { PathTrie* new_path = new PathTrie; new_path->character = new_char; new_path->parent = this; - _children.push_back(std::make_pair(new_char, new_path)); + children_.push_back(std::make_pair(new_char, new_path)); return new_path; } } } PathTrie* PathTrie::get_path_vec(std::vector& output) { - return get_path_vec(output, _ROOT); + return get_path_vec(output, ROOT_); } PathTrie* PathTrie::get_path_vec(std::vector& output, int stop, size_t max_steps) { - if (character == stop || character == _ROOT || output.size() == max_steps) { + if (character == stop || character == ROOT_ || output.size() == max_steps) { std::reverse(output.begin(), output.end()); return this; } else { @@ -98,7 +98,7 @@ PathTrie* PathTrie::get_path_vec(std::vector& output, } void PathTrie::iterate_to_vec(std::vector& output) { - if (_exists) { + if (exists_) { log_prob_b_prev = log_prob_b_cur; log_prob_nb_prev = log_prob_nb_cur; @@ -108,25 +108,25 @@ void PathTrie::iterate_to_vec(std::vector& output) { score = log_sum_exp(log_prob_b_prev, log_prob_nb_prev); output.push_back(this); } - for (auto child : _children) { + for (auto child : children_) { child.second->iterate_to_vec(output); } } void PathTrie::remove() { - _exists = false; + exists_ = false; - if (_children.size() == 0) { - auto child = parent->_children.begin(); - for (child = parent->_children.begin(); child != parent->_children.end(); + if (children_.size() == 0) { + auto child = parent->children_.begin(); + for (child = parent->children_.begin(); child != parent->children_.end(); ++child) { if (child->first == character) { - parent->_children.erase(child); + parent->children_.erase(child); break; } } - if (parent->_children.size() == 0 && !parent->_exists) { + if (parent->children_.size() == 0 && !parent->exists_) { parent->remove(); } @@ -135,12 +135,12 @@ void PathTrie::remove() { } void PathTrie::set_dictionary(fst::StdVectorFst* dictionary) { - _dictionary = dictionary; - _dictionary_state = dictionary->Start(); - _has_dictionary = true; + dictionary_ = dictionary; + dictionary_state_ = dictionary->Start(); + has_dictionary_ = true; } using FSTMATCH = fst::SortedMatcher; void PathTrie::set_matcher(std::shared_ptr matcher) { - _matcher = matcher; + matcher_ = matcher; } diff --git a/deep_speech_2/decoders/swig/path_trie.h b/deep_speech_2/decoders/swig/path_trie.h index b4f5bc4baa..7fd715d26d 100644 --- a/deep_speech_2/decoders/swig/path_trie.h +++ b/deep_speech_2/decoders/swig/path_trie.h @@ -36,7 +36,7 @@ class PathTrie { void set_matcher(std::shared_ptr>); - bool is_empty() { return _ROOT == character; } + bool is_empty() { return ROOT_ == character; } // remove current path from root void remove(); @@ -51,17 +51,17 @@ class PathTrie { PathTrie* parent; private: - int _ROOT; - bool _exists; - bool _has_dictionary; + int ROOT_; + bool exists_; + bool has_dictionary_; - std::vector> _children; + std::vector> children_; // pointer to dictionary of FST - fst::StdVectorFst* _dictionary; - fst::StdVectorFst::StateId _dictionary_state; + fst::StdVectorFst* dictionary_; + fst::StdVectorFst::StateId dictionary_state_; // true if finding ars in FST - std::shared_ptr> _matcher; + std::shared_ptr> matcher_; }; #endif // PATH_TRIE_H diff --git a/deep_speech_2/decoders/swig/scorer.cpp b/deep_speech_2/decoders/swig/scorer.cpp index 6b28034435..27c31fa714 100644 --- a/deep_speech_2/decoders/swig/scorer.cpp +++ b/deep_speech_2/decoders/swig/scorer.cpp @@ -19,19 +19,19 @@ Scorer::Scorer(double alpha, const std::vector& vocab_list) { this->alpha = alpha; this->beta = beta; - _is_character_based = true; - _language_model = nullptr; + is_character_based_ = true; + language_model_ = nullptr; dictionary = nullptr; - _max_order = 0; - _dict_size = 0; - _SPACE_ID = -1; + max_order_ = 0; + dict_size_ = 0; + SPACE_ID_ = -1; setup(lm_path, vocab_list); } Scorer::~Scorer() { - if (_language_model != nullptr) { - delete static_cast(_language_model); + if (language_model_ != nullptr) { + delete static_cast(language_model_); } if (dictionary != nullptr) { delete static_cast(dictionary); @@ -57,20 +57,20 @@ void Scorer::load_lm(const std::string& lm_path) { RetriveStrEnumerateVocab enumerate; lm::ngram::Config config; config.enumerate_vocab = &enumerate; - _language_model = lm::ngram::LoadVirtual(filename, config); - _max_order = static_cast(_language_model)->Order(); - _vocabulary = enumerate.vocabulary; - for (size_t i = 0; i < _vocabulary.size(); ++i) { - if (_is_character_based && _vocabulary[i] != UNK_TOKEN && - _vocabulary[i] != START_TOKEN && _vocabulary[i] != END_TOKEN && + language_model_ = lm::ngram::LoadVirtual(filename, config); + max_order_ = static_cast(language_model_)->Order(); + vocabulary_ = enumerate.vocabulary; + for (size_t i = 0; i < vocabulary_.size(); ++i) { + if (is_character_based_ && vocabulary_[i] != UNK_TOKEN && + vocabulary_[i] != START_TOKEN && vocabulary_[i] != END_TOKEN && get_utf8_str_len(enumerate.vocabulary[i]) > 1) { - _is_character_based = false; + is_character_based_ = false; } } } double Scorer::get_log_cond_prob(const std::vector& words) { - lm::base::Model* model = static_cast(_language_model); + lm::base::Model* model = static_cast(language_model_); double cond_prob; lm::ngram::State state, tmp_state, out_state; // avoid to inserting in begin @@ -93,11 +93,11 @@ double Scorer::get_log_cond_prob(const std::vector& words) { double Scorer::get_sent_log_prob(const std::vector& words) { std::vector sentence; if (words.size() == 0) { - for (size_t i = 0; i < _max_order; ++i) { + for (size_t i = 0; i < max_order_; ++i) { sentence.push_back(START_TOKEN); } } else { - for (size_t i = 0; i < _max_order - 1; ++i) { + for (size_t i = 0; i < max_order_ - 1; ++i) { sentence.push_back(START_TOKEN); } sentence.insert(sentence.end(), words.begin(), words.end()); @@ -107,11 +107,11 @@ double Scorer::get_sent_log_prob(const std::vector& words) { } double Scorer::get_log_prob(const std::vector& words) { - assert(words.size() > _max_order); + assert(words.size() > max_order_); double score = 0.0; - for (size_t i = 0; i < words.size() - _max_order + 1; ++i) { + for (size_t i = 0; i < words.size() - max_order_ + 1; ++i) { std::vector ngram(words.begin() + i, - words.begin() + i + _max_order); + words.begin() + i + max_order_); score += get_log_cond_prob(ngram); } return score; @@ -125,7 +125,7 @@ void Scorer::reset_params(float alpha, float beta) { std::string Scorer::vec2str(const std::vector& input) { std::string word; for (auto ind : input) { - word += _char_list[ind]; + word += char_list_[ind]; } return word; } @@ -135,7 +135,7 @@ std::vector Scorer::split_labels(const std::vector& labels) { std::string s = vec2str(labels); std::vector words; - if (_is_character_based) { + if (is_character_based_) { words = split_utf8_str(s); } else { words = split_str(s, " "); @@ -144,15 +144,15 @@ std::vector Scorer::split_labels(const std::vector& labels) { } void Scorer::set_char_map(const std::vector& char_list) { - _char_list = char_list; - _char_map.clear(); - - for (unsigned int i = 0; i < _char_list.size(); i++) { - if (_char_list[i] == " ") { - _SPACE_ID = i; - _char_map[' '] = i; - } else if (_char_list[i].size() == 1) { - _char_map[_char_list[i][0]] = i; + char_list_ = char_list; + char_map_.clear(); + + for (size_t i = 0; i < char_list_.size(); i++) { + if (char_list_[i] == " ") { + SPACE_ID_ = i; + char_map_[' '] = i; + } else if (char_list_[i].size() == 1) { + char_map_[char_list_[i][0]] = i; } } } @@ -162,14 +162,14 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { PathTrie* current_node = prefix; PathTrie* new_node = nullptr; - for (int order = 0; order < _max_order; order++) { + for (int order = 0; order < max_order_; order++) { std::vector prefix_vec; - if (_is_character_based) { - new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID, 1); + if (is_character_based_) { + new_node = current_node->get_path_vec(prefix_vec, SPACE_ID_, 1); current_node = new_node; } else { - new_node = current_node->get_path_vec(prefix_vec, _SPACE_ID); + new_node = current_node->get_path_vec(prefix_vec, SPACE_ID_); current_node = new_node->parent; // Skipping spaces } @@ -179,7 +179,7 @@ std::vector Scorer::make_ngram(PathTrie* prefix) { if (new_node->character == -1) { // No more spaces, but still need order - for (int i = 0; i < _max_order - order - 1; i++) { + for (int i = 0; i < max_order_ - order - 1; i++) { ngram.push_back(START_TOKEN); } break; @@ -193,19 +193,19 @@ void Scorer::fill_dictionary(bool add_space) { fst::StdVectorFst dictionary; // First reverse char_list so ints can be accessed by chars std::unordered_map char_map; - for (unsigned int i = 0; i < _char_list.size(); i++) { - char_map[_char_list[i]] = i; + for (size_t i = 0; i < char_list_.size(); i++) { + char_map[char_list_[i]] = i; } // For each unigram convert to ints and put in trie int dict_size = 0; - for (const auto& word : _vocabulary) { + for (const auto& word : vocabulary_) { bool added = add_word_to_dictionary( - word, char_map, add_space, _SPACE_ID, &dictionary); + word, char_map, add_space, SPACE_ID_, &dictionary); dict_size += added ? 1 : 0; } - _dict_size = dict_size; + dict_size_ = dict_size; /* Simplify FST diff --git a/deep_speech_2/decoders/swig/scorer.h b/deep_speech_2/decoders/swig/scorer.h index 72544da7b5..6183646359 100644 --- a/deep_speech_2/decoders/swig/scorer.h +++ b/deep_speech_2/decoders/swig/scorer.h @@ -18,7 +18,7 @@ const std::string START_TOKEN = ""; const std::string UNK_TOKEN = ""; const std::string END_TOKEN = ""; -// Implement a callback to retrive string vocabulary. +// Implement a callback to retrive the dictionary of language model. class RetriveStrEnumerateVocab : public lm::EnumerateVocab { public: RetriveStrEnumerateVocab() {} @@ -50,13 +50,14 @@ class Scorer { double get_sent_log_prob(const std::vector &words); - size_t get_max_order() const { return _max_order; } + // return the max order + size_t get_max_order() const { return max_order_; } - size_t get_dict_size() const { return _dict_size; } + // return the dictionary size of language model + size_t get_dict_size() const { return dict_size_; } - bool is_char_map_empty() const { return _char_map.size() == 0; } - - bool is_character_based() const { return _is_character_based; } + // retrun true if the language model is character based + bool is_character_based() const { return is_character_based_; } // reset params alpha & beta void reset_params(float alpha, float beta); @@ -68,20 +69,23 @@ class Scorer { // the vector of characters (character based lm) std::vector split_labels(const std::vector &labels); - // expose to decoder + // language model weight double alpha; + // word insertion weight double beta; - // fst dictionary + // pointer to the dictionary of FST void *dictionary; protected: + // necessary setup: load language model, set char map, fill FST's dictionary void setup(const std::string &lm_path, const std::vector &vocab_list); + // load language model from given path void load_lm(const std::string &lm_path); - // fill dictionary for fst + // fill dictionary for FST void fill_dictionary(bool add_space); // set char map @@ -89,19 +93,20 @@ class Scorer { double get_log_prob(const std::vector &words); + // translate the vector in index to string std::string vec2str(const std::vector &input); private: - void *_language_model; - bool _is_character_based; - size_t _max_order; - size_t _dict_size; + void *language_model_; + bool is_character_based_; + size_t max_order_; + size_t dict_size_; - int _SPACE_ID; - std::vector _char_list; - std::unordered_map _char_map; + int SPACE_ID_; + std::vector char_list_; + std::unordered_map char_map_; - std::vector _vocabulary; + std::vector vocabulary_; }; #endif // SCORER_H_ diff --git a/deep_speech_2/decoders/swig_wrapper.py b/deep_speech_2/decoders/swig_wrapper.py index 5ebcd133c9..0a92112582 100644 --- a/deep_speech_2/decoders/swig_wrapper.py +++ b/deep_speech_2/decoders/swig_wrapper.py @@ -39,8 +39,8 @@ def ctc_greedy_decoder(probs_seq, vocabulary): def ctc_beam_search_decoder(probs_seq, - beam_size, vocabulary, + beam_size, cutoff_prob=1.0, cutoff_top_n=40, ext_scoring_func=None): @@ -50,10 +50,10 @@ def ctc_beam_search_decoder(probs_seq, step, with each element being a list of normalized probabilities over vocabulary and blank. :type probs_seq: 2-D list - :param beam_size: Width for beam search. - :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list + :param beam_size: Width for beam search. + :type beam_size: int :param cutoff_prob: Cutoff probability in pruning, default 1.0, no pruning. :type cutoff_prob: float @@ -69,14 +69,14 @@ def ctc_beam_search_decoder(probs_seq, results, in descending order of the probability. :rtype: list """ - return swig_decoders.ctc_beam_search_decoder(probs_seq.tolist(), beam_size, - vocabulary, cutoff_prob, + return swig_decoders.ctc_beam_search_decoder(probs_seq.tolist(), vocabulary, + beam_size, cutoff_prob, cutoff_top_n, ext_scoring_func) def ctc_beam_search_decoder_batch(probs_split, - beam_size, vocabulary, + beam_size, num_processes, cutoff_prob=1.0, cutoff_top_n=40, @@ -86,10 +86,10 @@ def ctc_beam_search_decoder_batch(probs_split, :param probs_seq: 3-D list with each element as an instance of 2-D list of probabilities used by ctc_beam_search_decoder(). :type probs_seq: 3-D list - :param beam_size: Width for beam search. - :type beam_size: int :param vocabulary: Vocabulary list. :type vocabulary: list + :param beam_size: Width for beam search. + :type beam_size: int :param num_processes: Number of parallel processes. :type num_processes: int :param cutoff_prob: Cutoff probability in vocabulary pruning, @@ -112,5 +112,5 @@ def ctc_beam_search_decoder_batch(probs_split, probs_split = [probs_seq.tolist() for probs_seq in probs_split] return swig_decoders.ctc_beam_search_decoder_batch( - probs_split, beam_size, vocabulary, num_processes, cutoff_prob, + probs_split, vocabulary, beam_size, num_processes, cutoff_prob, cutoff_top_n, ext_scoring_func) From 98d35b9258c9800879247f27f8cdf54ceb7056b4 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 18 Sep 2017 13:19:02 +0800 Subject: [PATCH 32/36] adjust to pass ci --- deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp | 3 +-- deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp | 2 +- deep_speech_2/decoders/swig/ctc_greedy_decoder.h | 4 ++-- deep_speech_2/decoders/swig/decoder_utils.cpp | 3 +-- deep_speech_2/decoders/swig/decoder_utils.h | 2 +- deep_speech_2/decoders/swig/path_trie.cpp | 2 ++ deep_speech_2/decoders/swig/scorer.cpp | 4 +++- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp index 5c8373beaa..624784b05e 100644 --- a/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp +++ b/deep_speech_2/decoders/swig/ctc_beam_search_decoder.cpp @@ -9,7 +9,6 @@ #include "ThreadPool.h" #include "fst/fstlib.h" -#include "fst/log.h" #include "decoder_utils.h" #include "path_trie.h" @@ -130,7 +129,7 @@ std::vector> ctc_beam_search_decoder( log_sum_exp(prefix_new->log_prob_nb_cur, log_p); } } // end of loop over prefix - } // end of loop over chars + } // end of loop over vocabulary prefixes.clear(); // update log probs diff --git a/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp b/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp index c4c94539ee..03449d7391 100644 --- a/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp +++ b/deep_speech_2/decoders/swig/ctc_greedy_decoder.cpp @@ -27,7 +27,7 @@ std::string ctc_greedy_decoder( max_prob = probs_step[j]; } } - // id with maximum probability in current step + // id with maximum probability in current time step max_idx_vec[i] = max_idx; // deduplicate if ((i == 0) || ((i > 0) && max_idx_vec[i] != max_idx_vec[i - 1])) { diff --git a/deep_speech_2/decoders/swig/ctc_greedy_decoder.h b/deep_speech_2/decoders/swig/ctc_greedy_decoder.h index 043742f26e..5e64f692e5 100644 --- a/deep_speech_2/decoders/swig/ctc_greedy_decoder.h +++ b/deep_speech_2/decoders/swig/ctc_greedy_decoder.h @@ -14,7 +14,7 @@ * The decoding result in string */ std::string ctc_greedy_decoder( - const std::vector> &probs_seq, - const std::vector &vocabulary); + const std::vector>& probs_seq, + const std::vector& vocabulary); #endif // CTC_GREEDY_DECODER_H diff --git a/deep_speech_2/decoders/swig/decoder_utils.cpp b/deep_speech_2/decoders/swig/decoder_utils.cpp index 665fcc22f8..70a1592889 100644 --- a/deep_speech_2/decoders/swig/decoder_utils.cpp +++ b/deep_speech_2/decoders/swig/decoder_utils.cpp @@ -23,10 +23,9 @@ std::vector> get_pruned_log_probs( for (size_t i = 0; i < prob_idx.size(); ++i) { cum_prob += prob_idx[i].second; cutoff_len += 1; - if (cum_prob >= cutoff_prob) break; + if (cum_prob >= cutoff_prob || cutoff_len >= cutoff_top_n) break; } } - cutoff_len = std::min(cutoff_len, cutoff_top_n); prob_idx = std::vector>( prob_idx.begin(), prob_idx.begin() + cutoff_len); } diff --git a/deep_speech_2/decoders/swig/decoder_utils.h b/deep_speech_2/decoders/swig/decoder_utils.h index 932ffb12f7..72821c187f 100644 --- a/deep_speech_2/decoders/swig/decoder_utils.h +++ b/deep_speech_2/decoders/swig/decoder_utils.h @@ -2,8 +2,8 @@ #define DECODER_UTILS_H_ #include -#include "path_trie.h" #include "fst/log.h" +#include "path_trie.h" const float NUM_FLT_INF = std::numeric_limits::max(); const float NUM_FLT_MIN = std::numeric_limits::min(); diff --git a/deep_speech_2/decoders/swig/path_trie.cpp b/deep_speech_2/decoders/swig/path_trie.cpp index fdff328611..40d9097055 100644 --- a/deep_speech_2/decoders/swig/path_trie.cpp +++ b/deep_speech_2/decoders/swig/path_trie.cpp @@ -19,9 +19,11 @@ PathTrie::PathTrie() { character = ROOT_; exists_ = true; parent = nullptr; + dictionary_ = nullptr; dictionary_state_ = 0; has_dictionary_ = false; + matcher_ = nullptr; } diff --git a/deep_speech_2/decoders/swig/scorer.cpp b/deep_speech_2/decoders/swig/scorer.cpp index 27c31fa714..686c67c77e 100644 --- a/deep_speech_2/decoders/swig/scorer.cpp +++ b/deep_speech_2/decoders/swig/scorer.cpp @@ -19,9 +19,11 @@ Scorer::Scorer(double alpha, const std::vector& vocab_list) { this->alpha = alpha; this->beta = beta; + + dictionary = nullptr; is_character_based_ = true; language_model_ = nullptr; - dictionary = nullptr; + max_order_ = 0; dict_size_ = 0; SPACE_ID_ = -1; From d7a97526467b4999e9b24ade60a61de3e30b55bc Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 18 Sep 2017 14:41:00 +0800 Subject: [PATCH 33/36] specify clang_format to ver3.9 --- .clang_format.hook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang_format.hook b/.clang_format.hook index 40d70f56cf..4cbc972bbd 100755 --- a/.clang_format.hook +++ b/.clang_format.hook @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -readonly VERSION="3.8" +readonly VERSION="3.9" version=$(clang-format -version) From cc2f91f7f2708afd2157ec43ec3282d0d96f08a4 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 18 Sep 2017 16:20:57 +0800 Subject: [PATCH 34/36] disable the make output of libsndfile in setup --- deep_speech_2/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep_speech_2/setup.sh b/deep_speech_2/setup.sh index 209539399a..894aaea987 100644 --- a/deep_speech_2/setup.sh +++ b/deep_speech_2/setup.sh @@ -20,7 +20,7 @@ if [ $? != 0 ]; then fi tar -zxvf libsndfile-1.0.28.tar.gz cd libsndfile-1.0.28 - ./configure && make && make install + ./configure > /dev/null && make > /dev/null && make install > /dev/null cd .. rm -rf libsndfile-1.0.28 rm libsndfile-1.0.28.tar.gz From f1cd6727952408e5a760f39b76876bb35bbbc71a Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 18 Sep 2017 19:32:03 +0800 Subject: [PATCH 35/36] use cd instead of pushd in setup.sh --- deep_speech_2/setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deep_speech_2/setup.sh b/deep_speech_2/setup.sh index 894aaea987..7c40415db3 100644 --- a/deep_speech_2/setup.sh +++ b/deep_speech_2/setup.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#! /usr/bin/env bash # install python dependencies if [ -f "requirements.txt" ]; then @@ -29,9 +29,9 @@ fi # install decoders python -c "import swig_decoders" if [ $? != 0 ]; then - pushd decoders/swig > /dev/null + cd decoders/swig > /dev/null sh setup.sh - popd > /dev/null + cd - > /dev/null fi From 9db0d25ff4b34f2ac36dd3f058dbe12362831a25 Mon Sep 17 00:00:00 2001 From: Yibing Liu Date: Mon, 18 Sep 2017 22:37:36 +0800 Subject: [PATCH 36/36] pass unittest for deprecated decoders --- .../{model_utils => decoders}/tests/test_decoders.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename deep_speech_2/{model_utils => decoders}/tests/test_decoders.py (93%) diff --git a/deep_speech_2/model_utils/tests/test_decoders.py b/deep_speech_2/decoders/tests/test_decoders.py similarity index 93% rename from deep_speech_2/model_utils/tests/test_decoders.py rename to deep_speech_2/decoders/tests/test_decoders.py index adf36eefc3..d522b5efa0 100644 --- a/deep_speech_2/model_utils/tests/test_decoders.py +++ b/deep_speech_2/decoders/tests/test_decoders.py @@ -4,7 +4,7 @@ from __future__ import print_function import unittest -from model_utils import decoder +from decoders import decoders_deprecated as decoder class TestDecoders(unittest.TestCase): @@ -66,16 +66,14 @@ def test_beam_search_decoder_1(self): beam_result = decoder.ctc_beam_search_decoder( probs_seq=self.probs_seq1, beam_size=self.beam_size, - vocabulary=self.vocab_list, - blank_id=len(self.vocab_list)) + vocabulary=self.vocab_list) self.assertEqual(beam_result[0][1], self.beam_search_result[0]) def test_beam_search_decoder_2(self): beam_result = decoder.ctc_beam_search_decoder( probs_seq=self.probs_seq2, beam_size=self.beam_size, - vocabulary=self.vocab_list, - blank_id=len(self.vocab_list)) + vocabulary=self.vocab_list) self.assertEqual(beam_result[0][1], self.beam_search_result[1]) def test_beam_search_decoder_batch(self): @@ -83,7 +81,6 @@ def test_beam_search_decoder_batch(self): probs_split=[self.probs_seq1, self.probs_seq2], beam_size=self.beam_size, vocabulary=self.vocab_list, - blank_id=len(self.vocab_list), num_processes=24) self.assertEqual(beam_results[0][0][1], self.beam_search_result[0]) self.assertEqual(beam_results[1][0][1], self.beam_search_result[1])