diff --git a/README.md b/README.md index 9b56794085d..143ebd6a79f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ $ make base.en gcc -pthread -O3 -mavx -mavx2 -mfma -mf16c -c ggml.c g++ -pthread -O3 -std=c++11 -c main.cpp -g++ -o main ggml.o main.o +g++ -pthread -o main ggml.o main.o ./main -h usage: ./main [options] @@ -40,22 +40,17 @@ options: -h, --help show this help message and exit -s SEED, --seed SEED RNG seed (default: -1) -t N, --threads N number of threads to use during computation (default: 4) - -T N, --tokens N maximum number of tokens to generate per iteration (default: 64) -v, --verbose verbose output --translate translate from source language to english -ps, --print_special print special tokens + -nt, --no_timestamps do not print timestamps -l LANG, --language LANG spoken language (default: en) -m FNAME, --model FNAME model path (default: models/ggml-base.en.bin) -f FNAME, --file FNAME input WAV file path (default: samples/jfk.wav) bash ./download-ggml-model.sh base.en Downloading ggml model base.en ... -models/ggml-base.en.bin 100%[=====================================>] 141.11M 7.84MB/s in 18s -Done! Model 'base.en' saved in 'models/ggml-base.en.bin' -You can now use it like this: - - $ ./main -m models/ggml-base.en.bin -f samples/jfk.wav - +Model base.en already exists. Skipping download. =============================================== Running base.en on all samples in ./samples ... @@ -86,16 +81,17 @@ whisper_model_load: model size = 140.54 MB log_mel_spectrogram: n_sample = 176000, n_len = 1100 log_mel_spectrogram: recording length: 11.000000 s -main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe ... +main: processing 176000 samples (11.0 sec), 4 threads, lang = english, task = transcribe, timestamps = 1 ... - And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country. +[00:00.000 --> 00:11.000] And so my fellow Americans ask not what your country can do for you. Ask what you can do for your country. -main: load time = 71.89 ms -main: mel time = 36.95 ms + +main: load time = 61.78 ms +main: mel time = 41.74 ms main: sample time = 2.10 ms -main: encode time = 700.94 ms / 116.82 ms per layer -main: decode time = 86.14 ms -main: total time = 898.72 ms +main: encode time = 718.60 ms / 119.77 ms per layer +main: decode time = 83.55 ms +main: total time = 908.15 ms ``` The command downloads the `base.en` model converted to custom `ggml` format and runs the inference on all `.wav` samples in the folder `samples`. @@ -131,10 +127,73 @@ For example, you can use `ffmpeg` like this: ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav ``` +Here is another example of transcribing a [3:24 min speech](https://upload.wikimedia.org/wikipedia/commons/1/1f/George_W_Bush_Columbia_FINAL.ogg) in less than a minute, using `medium.en` model: + +```bash +$ ./main -m models/ggml-medium.en.bin -f samples/gb1.wav -t 8 +whisper_model_load: loading model from 'models/ggml-medium.en.bin' +whisper_model_load: n_vocab = 51864 +whisper_model_load: n_audio_ctx = 1500 +whisper_model_load: n_audio_state = 1024 +whisper_model_load: n_audio_head = 16 +whisper_model_load: n_audio_layer = 24 +whisper_model_load: n_text_ctx = 448 +whisper_model_load: n_text_state = 1024 +whisper_model_load: n_text_head = 16 +whisper_model_load: n_text_layer = 24 +whisper_model_load: n_mels = 80 +whisper_model_load: f16 = 1 +whisper_model_load: type = 4 +whisper_model_load: mem_required = 2786.00 MB +whisper_model_load: adding 1607 extra tokens +whisper_model_load: ggml ctx size = 1644.97 MB +whisper_model_load: memory size = 182.62 MB +whisper_model_load: model size = 1462.12 MB +log_mel_spectrogram: n_sample = 3179750, n_len = 19873 +log_mel_spectrogram: recording length: 198.734375 s + +main: processing 3179750 samples (198.7 sec), 8 threads, lang = english, task = transcribe, timestamps = 1 ... + +[00:00.000 --> 00:08.000] My fellow Americans, this day has brought terrible news and great sadness to our country. +[00:08.000 --> 00:17.000] At 9 o'clock this morning, Mission Control in Houston lost contact with our Space Shuttle Columbia. +[00:17.000 --> 00:24.000] A short time later, debris was seen falling from the skies above Texas. +[00:24.000 --> 00:29.000] The Columbia's lost. There are no survivors. +[00:29.000 --> 00:32.000] On board was a crew of seven. +[00:32.000 --> 00:43.000] Colonel Rick Husband, Lieutenant Colonel Michael Anderson, Commander Laurel Clark, Captain David Brown, Commander William McCool, +[00:43.000 --> 00:52.000] Dr. Kultner Aschavla, and Elon Ramon, a Colonel in the Israeli Air Force. +[00:52.000 --> 00:58.000] These men and women assumed great risk in the service to all humanity. +[00:58.000 --> 01:06.000] In an age when space flight has come to seem almost routine, it is easy to overlook the dangers of travel by rocket +[01:06.000 --> 01:12.000] and the difficulties of navigating the fierce outer atmosphere of the Earth. +[01:12.000 --> 01:22.000] These astronauts knew the dangers, and they faced them willingly, knowing they had a high and noble purpose in life. +[01:22.000 --> 01:30.000] Because of their courage, endearing, and idealism, we will miss them all the more. +[01:30.000 --> 01:40.000] All Americans today are thinking as well of the families of these men and women who have been given this sudden shock and grief. +[01:40.000 --> 01:45.000] You're not alone. Our entire nation agrees with you. +[01:45.000 --> 01:52.000] And those you love will always have the respect and gratitude of this country. +[01:52.000 --> 01:56.000] The cause in which they died will continue. +[01:56.000 --> 02:07.000] Mankind is led into the darkness beyond our world by the inspiration of discovery and the longing to understand. +[02:07.000 --> 02:11.000] Our journey into space will go on. +[02:11.000 --> 02:16.000] In the skies today, we saw destruction and tragedy. +[02:16.000 --> 02:22.000] Yet farther than we can see, there is comfort and hope. +[02:22.000 --> 02:31.000] In the words of the prophet Isaiah, "Lift your eyes and look to the heavens who created all these. +[02:31.000 --> 02:39.000] He who brings out the starry hosts one by one and calls them each by name." +[02:39.000 --> 02:46.000] Because of his great power and mighty strength, not one of them is missing. +[02:46.000 --> 02:55.000] The same creator who names the stars also knows the names of the seven souls we mourn today. +[02:55.000 --> 03:05.000] The crew of the shuttle Columbia did not return safely to Earth, yet we can pray that all are safely home. +[03:05.000 --> 03:14.000] May God bless the grieving families and may God continue to bless America. +[03:14.000 --> 03:24.000] [Music] + + +main: load time = 438.55 ms +main: mel time = 440.22 ms +main: sample time = 32.23 ms +main: encode time = 42329.63 ms / 1763.73 ms per layer +main: decode time = 15190.00 ms +main: total time = 58444.63 ms +``` + ## Limitations - Very basic greedy sampling scheme - always pick up the top token -- No timestamps - Inference only - Runs on the CPU - Only mono-channel 16-bit WAV is supported diff --git a/main.cpp b/main.cpp index 326a8a70d96..ac205315638 100644 --- a/main.cpp +++ b/main.cpp @@ -206,6 +206,7 @@ struct whisper_vocab { id token_sot = 50257; id token_prev = 50360; id token_solm = 50361; // ?? + id token_not = 50362; // no timestamps id token_beg = 50363; // available tasks @@ -217,17 +218,20 @@ struct whisper_vocab { } }; +struct whisper_result { + whisper_vocab::id id; + int64_t t; +}; + // command-line parameters struct whisper_params { int32_t seed = -1; // RNG seed, not used currently int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); - // sampling parameter - used for the greedy strategy - int32_t max_tokens_per_iter = 64; - bool verbose = false; bool translate = false; bool print_special_tokens = false; + bool no_timestamps = false; std::string language = "en"; std::string model = "models/ggml-base.en.bin"; @@ -244,8 +248,6 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) { params.seed = std::stoi(argv[++i]); } else if (arg == "-t" || arg == "--threads") { params.n_threads = std::stoi(argv[++i]); - } else if (arg == "-T" || arg == "--tokens") { - params.max_tokens_per_iter = std::stoi(argv[++i]); } else if (arg == "-v" || arg == "--verbose") { params.verbose = true; } else if (arg == "--translate") { @@ -259,6 +261,8 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) { } } else if (arg == "-ps" || arg == "--print_special") { params.print_special_tokens = true; + } else if (arg == "-nt" || arg == "--no_timestamps") { + params.no_timestamps = true; } else if (arg == "-m" || arg == "--model") { params.model = argv[++i]; } else if (arg == "-f" || arg == "--file") { @@ -284,10 +288,10 @@ void whisper_print_usage(int argc, char ** argv, const whisper_params & params) fprintf(stderr, " -h, --help show this help message and exit\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); - fprintf(stderr, " -T N, --tokens N maximum number of tokens to generate per iteration (default: %d)\n", params.max_tokens_per_iter); fprintf(stderr, " -v, --verbose verbose output\n"); fprintf(stderr, " --translate translate from source language to english\n"); fprintf(stderr, " -ps, --print_special print special tokens\n"); + fprintf(stderr, " -nt, --no_timestamps do not print timestamps\n"); fprintf(stderr, " -l LANG, --language LANG spoken language (default: %s)\n", params.language.c_str()); fprintf(stderr, " -m FNAME, --model FNAME model path (default: %s)\n", params.model.c_str()); fprintf(stderr, " -f FNAME, --file FNAME input WAV file path (default: %s)\n", params.fname_inp.c_str()); @@ -591,6 +595,7 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe vocab.token_sot++; vocab.token_prev++; vocab.token_solm++; + vocab.token_not++; vocab.token_beg++; } @@ -605,6 +610,8 @@ bool whisper_model_load(const std::string & fname, whisper_model & model, whispe word = "[_SOT_]"; } else if (i == vocab.token_prev) { word = "[_PREV_]"; + } else if (i == vocab.token_not) { + word = "[_NOT_]"; } else if (i == vocab.token_beg) { word = "[_BEG_]"; } else { @@ -1842,15 +1849,13 @@ bool whisper_decode( // TODO: temperature whisper_vocab::id whisper_sample_best( const whisper_vocab & vocab, - const float * probs, - double temp, - int offset = 0) { + const float * probs) { int n_logits = vocab.id_to_token.size(); std::vector> probs_id; probs_id.reserve(n_logits); - for (int i = offset; i < n_logits; i++) { + for (int i = 0; i < n_logits; i++) { probs_id.push_back(std::make_pair(probs[i], i)); } @@ -1872,13 +1877,49 @@ whisper_vocab::id whisper_sample_best( //} int res = 0; - while (probs_id[res].second == vocab.token_solm && res < (int) probs_id.size() - 1) { + while ((probs_id[res].second == vocab.token_sot || + probs_id[res].second == vocab.token_solm || + probs_id[res].second == vocab.token_not) && + res < (int) probs_id.size() - 1) { res++; } return probs_id[res].second; } +// samples only from the timestamps tokens +whisper_vocab::id whisper_sample_timestamp( + const whisper_vocab & vocab, + const float * probs) { + int n_logits = vocab.id_to_token.size(); + + std::vector> probs_id; + probs_id.reserve(n_logits); + + for (int i = vocab.token_beg + 1; i < n_logits; i++) { + probs_id.push_back(std::make_pair(probs[i], i)); + } + + const int top_k = 10; + + // find the top K tokens + std::partial_sort( + probs_id.begin(), + probs_id.begin() + top_k, probs_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + probs_id.resize(top_k); + + //printf("\n"); + //for (int i = 0; i < (int) probs_id.size(); i++) { + // printf("%d: '%s' %f, %d\n", i, vocab.id_to_token.at(probs_id[i].second).c_str(), probs_id[i].first, probs_id[i].second); + //} + + return probs_id[0].second; +} + // Cooley-Tukey FFT // poor man's implmentation - use something better // input is real-valued @@ -2032,6 +2073,20 @@ bool log_mel_spectrogram( return true; } +// 500 -> 00:05.000 +// 6000 -> 01:00.000 +std::string to_timestamp(int64_t t) { + int64_t sec = t/100; + int64_t msec = t - sec*100; + int64_t min = sec/60; + sec = sec - min*60; + + char buf[32]; + snprintf(buf, sizeof(buf), "%02d:%02d.%03d", (int) min, (int) sec, (int) msec); + + return std::string(buf); +} + int main(int argc, char ** argv) { const int64_t t_main_start_us = ggml_time_us(); @@ -2051,7 +2106,7 @@ int main(int argc, char ** argv) { int64_t t_load_us = 0; int64_t t_mel_us = 0; - int64_t t_sample_us = 0; + int64_t t_sample_us = 0; int64_t t_encode_us = 0; int64_t t_decode_us = 0; @@ -2128,10 +2183,12 @@ int main(int argc, char ** argv) { printf("%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__); } } - printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s ...\n", + printf("%s: processing %d samples (%.1f sec), %d threads, lang = %s, task = %s, timestamps = %d ...\n", __func__, int(pcmf32.size()), float(pcmf32.size())/SAMPLE_RATE, params.n_threads, g_lang.at(params.language).second.c_str(), - params.translate ? "translate" : "transcribe"); + params.translate ? "translate" : "transcribe", + params.no_timestamps ? 0 : 1); + printf("\n"); } // the accumulated text context so far @@ -2148,6 +2205,9 @@ int main(int argc, char ** argv) { } } + // the generated text including timestamps + std::vector result_all; + // main loop int seek = 0; while (true) { @@ -2165,7 +2225,7 @@ int main(int argc, char ** argv) { return 1; } - t_encode_us = ggml_time_us() - t_start_us; + t_encode_us += ggml_time_us() - t_start_us; } std::vector probs; @@ -2192,11 +2252,16 @@ int main(int argc, char ** argv) { int seek_delta = 100*CHUNK_SIZE; whisper_vocab::id last_id = 0; + //printf("\n\n"); //for (int i = 0; i < prompt.size(); i++) { // printf("%s: prompt[%d] = %s\n", __func__, i, vocab.id_to_token[prompt[i]].c_str()); //} + //printf("\n\n"); + + // the accumulated transcription in the current interation + int result_len = 0; + std::vector result_cur; - printf("\n"); for (int i = 0; i < model.hparams.n_text_ctx/2; ++i) { // decode if (prompt.size() > 0) { @@ -2216,63 +2281,118 @@ int main(int argc, char ** argv) { // very basic greedy sampling strategy: // // - always take the most probable token - // - if we have accumulated more than 'params.max_tokens_per_iter' -> pick most probable timestamp token - // and advance the sliding window by that amount - // - in the meantime, if we encounter 2 consecutive timestamp tokens, we advance the sliding window too // // more sophisticated sampling strategies could be implemented here, but we keep it simple // feel free to experiment! // { - // sample next token - const float temp = 1.0; // TODO - const int n_vocab = model.hparams.n_vocab; - whisper_vocab::id id = 0; + whisper_vocab::id id = 0; + whisper_vocab::id tid = vocab.token_beg; { const int64_t t_start_sample_us = ggml_time_us(); - id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab), temp, i > params.max_tokens_per_iter ? vocab.token_beg : 0); + id = whisper_sample_best(vocab, probs.data() + (probs.size() - n_vocab)); + if (i > 0) { + tid = whisper_sample_timestamp(vocab, probs.data() + (probs.size() - n_vocab)); + } t_sample_us += ggml_time_us() - t_start_sample_us; } - // end of text token - if (id == vocab.token_eot) { - break; - } - - // 2 consecutive time tokens - if (id > vocab.token_beg && last_id > vocab.token_beg) { + // update sliding window + if (id > vocab.token_beg) { seek_delta = 2*(id - vocab.token_beg); - done = true; + result_len = i + 1; } last_id = id; // add it to the context prompt.push_back(id); - prompt_past.push_back(id); - } + result_cur.push_back({ id, seek + 2*(tid - vocab.token_beg) }); - // display text - for (auto id : prompt) { - if (params.print_special_tokens == false && id >= vocab.token_eot) { - continue; + // end of text token + if (id == vocab.token_eot) { + break; } - printf("%s", vocab.id_to_token[id].c_str()); } - fflush(stdout); if (done) { break; } } + result_cur.resize(result_len); + result_all.insert(result_all.end(), result_cur.begin(), result_cur.end()); + + for (const auto & r : result_cur) { + prompt_past.push_back(r.id); + } + + // print the text from this iteration + if (result_cur.size() > 0) { + auto t0 = result_cur.front().t; + + std::string text = ""; + for (int i = 0; i < result_cur.size(); i++) { + if (params.print_special_tokens == false && result_cur[i].id >= vocab.token_eot) { + } else { + text += vocab.id_to_token[result_cur[i].id]; + } + if (result_cur[i].id > vocab.token_beg) { + const auto t1 = result_cur[i].t; + if (!text.empty()) { + if (params.no_timestamps) { + printf ("%s", text.c_str()); + fflush(stdout); + } else { + printf ("[%s --> %s] %s\n", to_timestamp(t0).c_str(), to_timestamp(t1).c_str(), text.c_str()); + } + } + text = ""; + while (result_cur[i].id > vocab.token_beg && i < result_cur.size()) { + i++; + } + i--; + t0 = result_cur[i].t; + } + } + + if (!text.empty()) { + printf ("[%s --> %s] %s\n", to_timestamp(t0).c_str(), to_timestamp(seek + seek_delta).c_str(), text.c_str()); + } + } + seek += seek_delta; } + // WIP: attempt for per-token timestamps + //if (!params.no_timestamps && result_all.size() > 0) { + // const int64_t dt = 500; // 5 second intervals + + // int i0 = 0; + + // int64_t t0 = result_all[0].t; + // int64_t t1 = t0; + + // printf("\n\n"); + // for (int i = 0; i < result_all.size(); ++i) { + // printf("'%s' -> %lld\n", vocab.id_to_token[result_all[i].id].c_str(), result_all[i].t); + // if (result_all[i].t - t0 > dt) { + // t1 = result_all[i - 1].t; + // printf("[%s --> %s] ", to_timestamp(t0).c_str(), to_timestamp(t1).c_str()); + // for (int j = i0; j < i; ++j) { + // printf("%s", vocab.id_to_token.at(result_all[j].id).c_str()); + // } + // printf("\n"); + // i0 = i; + // t0 = result_all[i].t; + // } + // } + //} + // report timing { const int64_t t_main_end_us = ggml_time_us();