From 4ffca750653acc19e31254a924b09defe3e49b1c Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sat, 10 Nov 2018 17:51:18 +0000 Subject: [PATCH 001/838] Fix #343 --- src/3rd_party/CLI/App.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/3rd_party/CLI/App.hpp b/src/3rd_party/CLI/App.hpp index b943ef1a4..555f480c1 100644 --- a/src/3rd_party/CLI/App.hpp +++ b/src/3rd_party/CLI/App.hpp @@ -1591,6 +1591,7 @@ class App { // Unlimited vector parser if(num < 0) { + bool emptyVectorArgs = true; while(!args.empty() && _recognize(args.back()) == detail::Classifer::NONE) { if(collected >= -num) { // We could break here for allow extras, but we don't @@ -1603,12 +1604,22 @@ class App { parse_order_.push_back(op.get()); args.pop_back(); collected++; + emptyVectorArgs = false; } // Allow -- to end an unlimited list and "eat" it if(!args.empty() && _recognize(args.back()) == detail::Classifer::POSITIONAL_MARK) args.pop_back(); + if(emptyVectorArgs) { + if(op->get_implicit()) { + op->add_result(op->get_implicitval()); + parse_order_.push_back(op.get()); + } else if (op->get_expected() < 0) { + parse_order_.push_back(op.get()); + throw ArgumentMismatch(op->get_name(), op->get_expected(), 0); + } + } } else { while(num > 0 && !args.empty()) { num--; From af0ce265d3ddacaf619a0c42263c14dd99c7987d Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 13:47:25 +0000 Subject: [PATCH 002/838] Log invalid options from config files --- src/common/cli_wrapper.cpp | 9 +++++---- src/common/cli_wrapper.h | 7 +++++-- src/common/config_parser.cpp | 7 +++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 28826bb28..702aa8940 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -3,6 +3,7 @@ #include "common/logging.h" #include "common/options.h" #include "common/timer.h" +#include "common/utils.h" #include "common/version.h" namespace marian { @@ -126,8 +127,8 @@ std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) return header; } -bool CLIWrapper::updateConfig(const YAML::Node &config) { - bool success = true; +void CLIWrapper::updateConfig(const YAML::Node &config, const std::string& errorMsg) { + std::vector invalidKeys; auto cmdOptions = getParsedOptionNames(); for(auto it : config) { auto key = it.first.as(); @@ -138,10 +139,10 @@ bool CLIWrapper::updateConfig(const YAML::Node &config) { config_[key] = YAML::Clone(it.second); options_[key].modified = true; } else { - success = false; + invalidKeys.push_back(key); } } - return success; + ABORT_IF(!invalidKeys.empty(), errorMsg + std::string(": ") + utils::join(invalidKeys, ", ")); } std::string CLIWrapper::dumpConfig(bool skipDefault /*= false*/) const { diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index cf47a3106..4e8701cc2 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -9,6 +9,7 @@ #include #include #include +#include namespace marian { @@ -232,9 +233,11 @@ class CLIWrapper { * This should be a preferred way of updating config options as the class keeps track of options, * which values have changed. * - * @param node YAML config with new default values for options + * @param config YAML config with new default values for options + * @param errorMsg error message printed if config contains undefined keys. The message is + * appended with ": * " */ - bool updateConfig(const YAML::Node &config); + void updateConfig(const YAML::Node &config, const std::string &errorMsg); // Get textual YAML representation of the config std::string dumpConfig(bool skipDefault = false) const; diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index d39f3b1ad..a55d5d50c 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -666,8 +666,8 @@ void ConfigParser::expandAliases(cli::CLIWrapper& cli) { } if(config) { - auto success = cli.updateConfig(config); - ABORT_IF(!success, "Unknown option(s) in aliases, check if aliases consist of correct options"); + cli.updateConfig(config, + "Unknown option(s) in aliases, check if aliases consist of correct options"); } } @@ -703,8 +703,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { auto configPaths = findConfigPaths(); if(!configPaths.empty()) { auto config = loadConfigFiles(configPaths); - auto success = cli.updateConfig(config); - ABORT_IF(!success, "There are option(s) in a config file that are not expected"); + cli.updateConfig(config, "There are option(s) in a config file that are not expected"); } if(get("interpolate-env-vars")) { From 3a88bd6d759836de269b19a8869aa36bf3f1084e Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 14:06:53 +0000 Subject: [PATCH 003/838] Handle 'version' in config options --- src/common/cli_wrapper.cpp | 6 +++++- src/common/config.cpp | 11 +++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 702aa8940..ce24f211a 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -135,6 +135,10 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string& error // skip options specified via command-line to allow overwriting them if(cmdOptions.count(key)) continue; + // skip special option "version" that possibly can be loaded from model.npz:special.yml + if(key == "version") + continue; + if(options_.count(key)) { config_[key] = YAML::Clone(it.second); options_[key].modified = true; @@ -142,7 +146,7 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string& error invalidKeys.push_back(key); } } - ABORT_IF(!invalidKeys.empty(), errorMsg + std::string(": ") + utils::join(invalidKeys, ", ")); + ABORT_IF(!invalidKeys.empty(), errorMsg + ": " + utils::join(invalidKeys, ", ")); } std::string CLIWrapper::dumpConfig(bool skipDefault /*= false*/) const { diff --git a/src/common/config.cpp b/src/common/config.cpp index e5208b0d5..83c214307 100755 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -95,15 +95,14 @@ void Config::initialize(int argc, char** argv, cli::mode mode, bool validate) { version, buildVersion()); else - LOG(info, - "[config] Loaded model has been created with Marian {}", - version); + LOG(info, "[config] Loaded model has been created with Marian {}", version); + + // Remove "version" from config to make it consistent among different start-up scenarios + config_.remove("version"); } // If this is a newly started training else if(mode == cli::mode::training) { - LOG(info, - "[config] Model is being created with Marian {}", - buildVersion()); + LOG(info, "[config] Model is being created with Marian {}", buildVersion()); } } From a1c1cb9154bf5ae524ab8b233c539b21377d2212 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 15:33:01 +0000 Subject: [PATCH 004/838] Use shorter error message for aliases --- src/common/config_parser.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index a55d5d50c..74b73dc67 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -666,8 +666,7 @@ void ConfigParser::expandAliases(cli::CLIWrapper& cli) { } if(config) { - cli.updateConfig(config, - "Unknown option(s) in aliases, check if aliases consist of correct options"); + cli.updateConfig(config, "Unknown option(s) in aliases"); } } From 2000d1a1eba50849878d6a8065eda0492451aa26 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 15:38:16 +0000 Subject: [PATCH 005/838] Remove unnecessary from_config() --- src/models/model_factory.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 2ec7fe752..c97303d1a 100644 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -53,7 +53,5 @@ typedef Accumulator encoder_decoder; Ptr by_type(std::string type, usage, Ptr options); Ptr from_options(Ptr options, usage); - -Ptr from_config(Ptr config, usage); } // namespace models } // namespace marian From ad1bc691eae5464e60fe3aeaa5115ff7984ac09a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 15:40:30 +0000 Subject: [PATCH 006/838] Remove unnecessary init() --- src/models/model_task.h | 1 - src/translator/translator.h | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/models/model_task.h b/src/models/model_task.h index 932fba620..1fcf6eca5 100644 --- a/src/models/model_task.h +++ b/src/models/model_task.h @@ -9,7 +9,6 @@ struct ModelTask { }; struct ModelServiceTask { - virtual void init() = 0; virtual std::string run(const std::string&) = 0; }; } // namespace marian diff --git a/src/translator/translator.h b/src/translator/translator.h index 9f973113f..5fbfe8aac 100755 --- a/src/translator/translator.h +++ b/src/translator/translator.h @@ -145,9 +145,7 @@ class TranslateService : public ModelServiceTask { public: virtual ~TranslateService() {} - TranslateService(Ptr options) : options_(options) { init(); } - - void init() override { + TranslateService(Ptr options) : options_(options) { // initialize vocabs options_->set("inference", true); From 2829e2b1d5453644b97b627b29735d0853edf1d8 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 15:54:41 +0000 Subject: [PATCH 007/838] Add regression-tests as a submodule --- .gitmodules | 3 +++ regression-tests | 1 + 2 files changed, 4 insertions(+) create mode 160000 regression-tests diff --git a/.gitmodules b/.gitmodules index 5c3c00f1e..efe6bc5ec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,9 @@ [submodule "examples"] path = examples url = https://github.com/marian-nmt/marian-examples +[submodule "regression-tests"] + path = regression-tests + url = https://github.com/marian-nmt/marian-regression-tests [submodule "src/3rd_party/sentencepiece"] path = src/3rd_party/sentencepiece url = https://github.com/marian-nmt/sentencepiece diff --git a/regression-tests b/regression-tests new file mode 160000 index 000000000..edc6a6dde --- /dev/null +++ b/regression-tests @@ -0,0 +1 @@ +Subproject commit edc6a6ddec3deb693fba5909539ac78000c5b8a2 From b6c265a6f197790f23041cec5e91779f029e3b28 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 16:11:47 +0000 Subject: [PATCH 008/838] Update submodule with regression tests --- regression-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-tests b/regression-tests index edc6a6dde..febdc3f56 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit edc6a6ddec3deb693fba5909539ac78000c5b8a2 +Subproject commit febdc3f56f75929b1f7b5b38a4b9b96ea8f648e7 From 77a97911a239195f1e267d1af6917c1553c1ecb3 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 16:51:28 +0000 Subject: [PATCH 009/838] Merge split() and splitAny() --- src/common/utils.cpp | 46 +++++++++++++++----------------------------- src/common/utils.h | 9 +++++---- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index b47cd1f4e..7860ee3a6 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -26,21 +26,25 @@ void trimLeft(std::string& s) { CLI::detail::ltrim(s, " \t\n"); } -// @TODO: use more functions from CLI instead of own implementations void split(const std::string& line, std::vector& pieces, const std::string del /*= " "*/, - bool keepEmpty) { + bool keepEmpty /*= false*/, + bool anyOf /*= false*/) { size_t begin = 0; size_t pos = 0; std::string token; - while((pos = line.find(del, begin)) != std::string::npos) { + size_t delSize = anyOf ? 1 : del.size(); + while(true) { + pos = anyOf ? line.find_first_of(del, begin) : line.find(del, begin); + if(pos == std::string::npos) + break; if(pos >= begin) { token = line.substr(begin, pos - begin); if(token.size() > 0 || keepEmpty) pieces.push_back(token); } - begin = pos + del.size(); + begin = pos + delSize; } if(pos >= begin) { token = line.substr(begin, pos - begin); @@ -51,45 +55,27 @@ void split(const std::string& line, std::vector split(const std::string& line, const std::string del /*= " "*/, - bool keepEmpty) { + bool keepEmpty /*= false*/, + bool anyOf /*= false*/) { std::vector pieces; - split(line, pieces, del, keepEmpty); + split(line, pieces, del, keepEmpty, anyOf); return pieces; } -// @TODO: splitAny() shares all but 2 expressions with split(). Merge them. void splitAny(const std::string& line, std::vector& pieces, const std::string del /*= " "*/, - bool keepEmpty) { - size_t begin = 0; - size_t pos = 0; - std::string token; - while((pos = line.find_first_of(del, begin)) != std::string::npos) { - if(pos >= begin) { - token = line.substr(begin, pos - begin); - if(token.size() > 0 || keepEmpty) - pieces.push_back(token); - } - begin = pos + 1; - } - if(pos >= begin) { - token = line.substr(begin, pos - begin); - if(token.size() > 0 || keepEmpty) - pieces.push_back(token); - } + bool keepEmpty /*= false*/) { + split(line, pieces, del, keepEmpty, /*anyOf =*/true); } std::vector splitAny(const std::string& line, const std::string del /*= " "*/, - bool keepEmpty) { - std::vector pieces; - splitAny(line, pieces, del, keepEmpty); - return pieces; + bool keepEmpty /*= false*/) { + return split(line, del, keepEmpty, /*anyOf =*/true); } -std::string join(const std::vector& words, - const std::string& del /*= " "*/) { +std::string join(const std::vector& words, const std::string& del /*= " "*/) { std::stringstream ss; if(words.empty()) { return ""; diff --git a/src/common/utils.h b/src/common/utils.h index cd4fc6de3..a782abe1c 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -13,11 +13,13 @@ void trimRight(std::string& s); void split(const std::string& line, std::vector& pieces, const std::string del = " ", - bool keepEmpty = false); + bool keepEmpty = false, + bool anyOf = false); std::vector split(const std::string& line, const std::string del = " ", - bool keepEmpty = false); + bool keepEmpty = false, + bool anyOf = false); void splitAny(const std::string& line, std::vector& pieces, @@ -28,8 +30,7 @@ std::vector splitAny(const std::string& line, const std::string del = " ", bool keepEmpty = false); -std::string join(const std::vector& words, - const std::string& del = " "); +std::string join(const std::vector& words, const std::string& del = " "); std::string exec(const std::string& cmd); From 8c23cbd57a32e2d2dec911b4c536cb92692bf407 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 17:07:28 +0000 Subject: [PATCH 010/838] Simplify split() calls --- src/common/config.cpp | 7 +++---- src/data/corpus_nbest.cpp | 6 ++---- src/data/default_vocab.cpp | 10 ++++------ src/rescorer/score_collector.cpp | 3 +-- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/common/config.cpp b/src/common/config.cpp index 83c214307..043a757f5 100755 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -155,10 +155,9 @@ void Config::log() { std::string configString = out.c_str(); // print YAML prepending each line with [config] - std::vector results; - utils::split(configString, results, "\n"); - for(auto& r : results) - LOG(info, "[config] {}", r); + auto lines = utils::split(configString, "\n"); + for(auto& line : lines) + LOG(info, "[config] {}", line); } // Parse the device-spec parameters (--num-devices, --devices, --cpu-threads) into an array of diff --git a/src/data/corpus_nbest.cpp b/src/data/corpus_nbest.cpp index 328c3c0d7..c5a85a231 100644 --- a/src/data/corpus_nbest.cpp +++ b/src/data/corpus_nbest.cpp @@ -15,8 +15,7 @@ CorpusNBest::CorpusNBest(std::vector paths, : CorpusBase(paths, vocabs, options) {} int numFromNbest(const std::string& line) { - std::vector fields; - utils::split(line, fields, " ||| ", true); + auto fields = utils::split(line, " ||| ", true); ABORT_IF(fields.size() < 4, "Too few fields ({}) in line \"{}\", is this a correct n-best list?", fields.size(), @@ -25,8 +24,7 @@ int numFromNbest(const std::string& line) { } std::string lineFromNbest(const std::string& line) { - std::vector fields; - utils::split(line, fields, " ||| ", true); + auto fields = utils::split(line, " ||| ", true); ABORT_IF(fields.size() < 4, "Too few fields ({}) in line \"{}\", is this a correct n-best list?", fields.size(), diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 1ce055db8..4cd8a929e 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -36,7 +36,7 @@ class DefaultVocab : public VocabBase { VocabFreqOrderer(const std::unordered_map& counter) : counter_(counter) {} - // order first by decreasing frequency, + // order first by decreasing frequency, // if frequencies are the same order lexicographically by vocabulary string bool operator()(const std::string& a, const std::string& b) const { return counter_.at(a) > counter_.at(b) || (counter_.at(a) == counter_.at(b) && a < b); @@ -56,8 +56,7 @@ class DefaultVocab : public VocabBase { } Words encode(const std::string& line, bool addEOS, bool /*inference*/) const override { - std::vector lineTokens; - utils::split(line, lineTokens, " "); + auto lineTokens = utils::split(line, " "); return (*this)(lineTokens, addEOS); } @@ -205,7 +204,7 @@ class DefaultVocab : public VocabBase { "Vocabulary file '{}' exists. Not overwriting", path.string()); } - + std::unordered_map counter; for(const auto& trainPath : trainPaths) addCounts(counter, trainPath); @@ -223,8 +222,7 @@ class DefaultVocab : public VocabBase { std::string line; while(getline(*trainStrm, line)) { - std::vector toks; - utils::split(line, toks, " "); + auto toks = utils::split(line, " "); for(const std::string& tok : toks) { auto iter = counter.find(tok); diff --git a/src/rescorer/score_collector.cpp b/src/rescorer/score_collector.cpp index ac118a6a4..28d582609 100644 --- a/src/rescorer/score_collector.cpp +++ b/src/rescorer/score_collector.cpp @@ -113,8 +113,7 @@ std::string ScoreCollectorNBest::addToNBest(const std::string nbest, const std::string feature, float score, const data::SoftAlignment& align) { - std::vector fields; - utils::split(nbest, fields, "|||"); + auto fields = utils::split(nbest, "|||"); std::stringstream ss; if(!alignment_.empty() && !align.empty()) ss << " " << getAlignment(align) << " |||"; From 8b33487c8899b2137d2713b84fbebfde8a3977ac Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 18:09:20 +0000 Subject: [PATCH 011/838] Separate unit tests and apps --- src/tests/CMakeLists.txt | 60 ++++++------------- src/tests/{cli_test.cpp => cli.cpp} | 0 src/tests/{conv_test.cu => conv.cu} | 0 src/tests/{conv_char_test.cu => conv_char.cu} | 0 src/tests/{dropout_test.cpp => dropout.cpp} | 0 src/tests/{logger_test.cpp => logger.cpp} | 0 src/tests/{pooling_test.cpp => pooling.cpp} | 0 src/tests/{sqlite_test.cpp => sqlite.cpp} | 0 src/tests/{tensor_test.cu => tensor.cu} | 0 src/tests/units/CMakeLists.txt | 19 ++++++ src/tests/{ => units}/attention_tests.cpp | 0 src/tests/{ => units}/graph_tests.cpp | 0 src/tests/{ => units}/operator_tests.cpp | 0 src/tests/{ => units}/rnn_tests.cpp | 0 src/tests/{ => units}/run_tests.cpp | 0 15 files changed, 37 insertions(+), 42 deletions(-) rename src/tests/{cli_test.cpp => cli.cpp} (100%) rename src/tests/{conv_test.cu => conv.cu} (100%) rename src/tests/{conv_char_test.cu => conv_char.cu} (100%) rename src/tests/{dropout_test.cpp => dropout.cpp} (100%) rename src/tests/{logger_test.cpp => logger.cpp} (100%) rename src/tests/{pooling_test.cpp => pooling.cpp} (100%) rename src/tests/{sqlite_test.cpp => sqlite.cpp} (100%) rename src/tests/{tensor_test.cu => tensor.cu} (100%) create mode 100644 src/tests/units/CMakeLists.txt rename src/tests/{ => units}/attention_tests.cpp (100%) rename src/tests/{ => units}/graph_tests.cpp (100%) rename src/tests/{ => units}/operator_tests.cpp (100%) rename src/tests/{ => units}/rnn_tests.cpp (100%) rename src/tests/{ => units}/run_tests.cpp (100%) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 347a18b78..2df971407 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -1,49 +1,25 @@ # Unit tests -set(UNIT_TESTS - graph_tests - operator_tests - rnn_tests - attention_tests -) - -foreach(test ${UNIT_TESTS}) - add_executable("run_${test}" run_tests.cpp "${test}.cpp") - target_link_libraries("run_${test}" marian ${EXT_LIBS} Catch) - - if(CUDA_FOUND) - target_link_libraries("run_${test}" marian marian_cuda ${EXT_LIBS} Catch) - endif(CUDA_FOUND) - - add_test(NAME ${test} COMMAND "run_${test}") -endforeach(test) +add_subdirectory(units) # Testing apps -add_executable(logger_test logger_test.cpp) -add_executable(dropout_test dropout_test.cpp) -add_executable(prod_test prod.cpp) -add_executable(cli_test cli_test.cpp) - -if(CUDA_FOUND) -add_executable(pooling_test pooling_test.cpp) -target_link_libraries(pooling_test marian ${EXT_LIBS} Catch) -target_link_libraries(pooling_test marian marian_cuda ${EXT_LIBS} Catch) -endif(CUDA_FOUND) - -add_executable(sqlite_test sqlite_test.cpp) +set(APP_TESTS + logger + dropout + sqlite + prod + cli + pooling +) -foreach(exec - logger_test - dropout_test - sqlite_test - prod_test - cli_test - ) - target_link_libraries(${exec} marian ${EXT_LIBS} Catch) - if(CUDA_FOUND) - target_link_libraries(${exec} marian marian_cuda ${EXT_LIBS} Catch) - endif(CUDA_FOUND) +foreach(test ${APP_TESTS}) + add_executable("test_${test}" "${test}.cpp") - set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") -endforeach(exec) + if(CUDA_FOUND) + target_link_libraries("test_${test}" marian marian_cuda ${EXT_LIBS}) + else(CUDA_FOUND) + target_link_libraries("test_${test}" marian ${EXT_LIBS}) + endif(CUDA_FOUND) + set_target_properties("test_${test}" PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") +endforeach(test) diff --git a/src/tests/cli_test.cpp b/src/tests/cli.cpp similarity index 100% rename from src/tests/cli_test.cpp rename to src/tests/cli.cpp diff --git a/src/tests/conv_test.cu b/src/tests/conv.cu similarity index 100% rename from src/tests/conv_test.cu rename to src/tests/conv.cu diff --git a/src/tests/conv_char_test.cu b/src/tests/conv_char.cu similarity index 100% rename from src/tests/conv_char_test.cu rename to src/tests/conv_char.cu diff --git a/src/tests/dropout_test.cpp b/src/tests/dropout.cpp similarity index 100% rename from src/tests/dropout_test.cpp rename to src/tests/dropout.cpp diff --git a/src/tests/logger_test.cpp b/src/tests/logger.cpp similarity index 100% rename from src/tests/logger_test.cpp rename to src/tests/logger.cpp diff --git a/src/tests/pooling_test.cpp b/src/tests/pooling.cpp similarity index 100% rename from src/tests/pooling_test.cpp rename to src/tests/pooling.cpp diff --git a/src/tests/sqlite_test.cpp b/src/tests/sqlite.cpp similarity index 100% rename from src/tests/sqlite_test.cpp rename to src/tests/sqlite.cpp diff --git a/src/tests/tensor_test.cu b/src/tests/tensor.cu similarity index 100% rename from src/tests/tensor_test.cu rename to src/tests/tensor.cu diff --git a/src/tests/units/CMakeLists.txt b/src/tests/units/CMakeLists.txt new file mode 100644 index 000000000..7d842c427 --- /dev/null +++ b/src/tests/units/CMakeLists.txt @@ -0,0 +1,19 @@ +# Unit tests +set(UNIT_TESTS + graph_tests + operator_tests + rnn_tests + attention_tests +) + +foreach(test ${UNIT_TESTS}) + add_executable("run_${test}" run_tests.cpp "${test}.cpp") + + if(CUDA_FOUND) + target_link_libraries("run_${test}" marian marian_cuda ${EXT_LIBS} Catch) + else(CUDA_FOUND) + target_link_libraries("run_${test}" marian ${EXT_LIBS} Catch) + endif(CUDA_FOUND) + + add_test(NAME ${test} COMMAND "run_${test}") +endforeach(test) diff --git a/src/tests/attention_tests.cpp b/src/tests/units/attention_tests.cpp similarity index 100% rename from src/tests/attention_tests.cpp rename to src/tests/units/attention_tests.cpp diff --git a/src/tests/graph_tests.cpp b/src/tests/units/graph_tests.cpp similarity index 100% rename from src/tests/graph_tests.cpp rename to src/tests/units/graph_tests.cpp diff --git a/src/tests/operator_tests.cpp b/src/tests/units/operator_tests.cpp similarity index 100% rename from src/tests/operator_tests.cpp rename to src/tests/units/operator_tests.cpp diff --git a/src/tests/rnn_tests.cpp b/src/tests/units/rnn_tests.cpp similarity index 100% rename from src/tests/rnn_tests.cpp rename to src/tests/units/rnn_tests.cpp diff --git a/src/tests/run_tests.cpp b/src/tests/units/run_tests.cpp similarity index 100% rename from src/tests/run_tests.cpp rename to src/tests/units/run_tests.cpp From 96fbb77a1c8ea4338f4f1e9a0346fb9adefa132f Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 21:34:40 +0000 Subject: [PATCH 012/838] Clean TextInput --- src/data/corpus.cpp | 2 +- src/data/corpus_base.cpp | 4 +-- src/data/dataset.h | 1 + src/data/text_input.cpp | 53 ++++++++++++++++++---------------------- src/data/text_input.h | 4 +-- 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7a7a846e1..38b503e06 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -18,7 +18,7 @@ Corpus::Corpus(std::vector paths, : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")) {} SentenceTuple Corpus::next() { - for (;;) { // (this is a retry loop for skipping invalid sentences) + for(;;) { // (this is a retry loop for skipping invalid sentences) // get index of the current sentence size_t curId = pos_; // note: at end, pos_ == total size // if corpus has been shuffled, ids_ contains sentence indexes diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index c97043135..2a088537a 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -101,7 +101,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) std::set paths; // contains all paths that are used for training the vocabulary size_t size; // contains the maximum vocabulary size }; - + // Group training files based on vocabulary path. If the same // vocab path corresponds to different training files, this means // that a single vocab should combine tokens from all files. @@ -120,7 +120,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) auto pathsAndSize = groupVocab[vocabPaths[i]]; std::vector groupedPaths(pathsAndSize.paths.begin(), pathsAndSize.paths.end()); int vocSize = vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size); - + // TODO: this is not nice as it modifies the option object and needs to expose the changes // outside the corpus as models need to know about the vocabulary size; extract the vocab // creation functionality from the class. diff --git a/src/data/dataset.h b/src/data/dataset.h index 343789544..830fcb3bc 100755 --- a/src/data/dataset.h +++ b/src/data/dataset.h @@ -15,6 +15,7 @@ class DatasetBase { protected: std::vector paths_; Ptr options_; + // Data processing may differ in training/inference settings bool inference_{false}; diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp index 78fbc7af0..85ae3a770 100755 --- a/src/data/text_input.cpp +++ b/src/data/text_input.cpp @@ -5,9 +5,7 @@ namespace marian { namespace data { TextIterator::TextIterator() : pos_(-1), tup_(0) {} - -TextIterator::TextIterator(TextInput& corpus) - : corpus_(&corpus), pos_(0), tup_(corpus_->next()) {} +TextIterator::TextIterator(TextInput& corpus) : corpus_(&corpus), pos_(0), tup_(corpus_->next()) {} void TextIterator::increment() { tup_ = corpus_->next(); @@ -25,40 +23,37 @@ const SentenceTuple& TextIterator::dereference() const { TextInput::TextInput(std::vector inputs, std::vector> vocabs, Ptr options) - // TODO: fix this: input text is stored in an inherited variable named - // paths_ that is very confusing - : DatasetBase(inputs, options), - vocabs_(vocabs) { + : DatasetBase(inputs, options), vocabs_(vocabs) { + // note: inputs are automatically stored in the inherited variable named paths_, but these are + // texts not paths! for(const auto& text : paths_) files_.emplace_back(new std::istringstream(text)); } +// TextInput is mainly used for inference in the server mode, not for training, so skipping too long +// or ill-formed inputs is not necessary here SentenceTuple TextInput::next() { - // @TODO: This code mixes two patterns (while and early exit). Fix that. - bool cont = true; - while(cont) { - // get index of the current sentence - size_t curId = pos_++; - - // fill up the sentence tuple with sentences from all input files - SentenceTuple tup(curId); - for(size_t i = 0; i < files_.size(); ++i) { - std::string line; - io::InputFileStream dummyStream(*files_[i]); - if(io::getline(dummyStream, line)) { - Words words = vocabs_[i]->encode(line, /*addEOS =*/ true, /*inference =*/ inference_); - if(words.empty()) - words.push_back(0); - tup.push_back(words); - } + // get index of the current sentence + size_t curId = pos_++; + + // fill up the sentence tuple with source and/or target sentences + SentenceTuple tup(curId); + for(size_t i = 0; i < files_.size(); ++i) { + io::InputFileStream dummyStream(*files_[i]); + std::string line; + if(io::getline(dummyStream, line)) { + Words words = vocabs_[i]->encode(line, /*addEOS =*/ true, /*inference =*/ inference_); + if(words.empty()) + words.push_back(DEFAULT_EOS_ID); + tup.push_back(words); } - - // continue only if each input file has provided an example - cont = tup.size() == files_.size(); - if(cont) - return tup; } + + // check if each input file provided an example + if(tup.size() == files_.size()) + return tup; return SentenceTuple(0); } + } // namespace data } // namespace marian diff --git a/src/data/text_input.h b/src/data/text_input.h index bf1bc1ebc..bdbcf64e9 100755 --- a/src/data/text_input.h +++ b/src/data/text_input.h @@ -36,9 +36,7 @@ class TextInput : public DatasetBase { public: typedef SentenceTuple Sample; - TextInput(std::vector inputs, - std::vector> vocabs, - Ptr options); + TextInput(std::vector inputs, std::vector> vocabs, Ptr options); Sample next() override; From 741406e1e53cd615b339e1a951566d3cd4d3a64a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 21:46:56 +0000 Subject: [PATCH 013/838] Make --learn-rate a float --- src/common/config_parser.cpp | 4 ++-- src/optimizers/optimizers.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 74b73dc67..df986b53c 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -314,9 +314,9 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Use synchronous SGD instead of asynchronous for multi-gpu training"); // learning rate options - cli.add("--learn-rate,-l", + cli.add("--learn-rate,-l", "Learning rate", - 0.0001); + 0.0001f); cli.add("--lr-report", "Report learning rate for each update"); diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index d7a58be31..be077c0f5 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -270,7 +270,7 @@ void Adam::resetStats() { } Ptr Optimizer(Ptr options) { - float lrate = (float)options->get("learn-rate"); // @TODO: should this be ? + float lrate = options->get("learn-rate"); auto params = options->has("optimizer-params") ? options->get>("optimizer-params") : std::vector({}); From d7dcaa599f2904a01efcec34b861a51c81e75b73 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 21:48:50 +0000 Subject: [PATCH 014/838] Make --clip-norm a float --- src/common/config_parser.cpp | 2 +- src/optimizers/optimizers.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index df986b53c..0745cbb5e 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -351,7 +351,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--label-smoothing", "Epsilon for label smoothing (0 to disable)"); - cli.add("--clip-norm", + cli.add("--clip-norm", "Clip gradient norm to argcli.add(0 to disable)", 1.f); cli.add("--exponential-smoothing", diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index be077c0f5..0fad790a1 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -276,7 +276,7 @@ Ptr Optimizer(Ptr options) { : std::vector({}); Ptr clipper = nullptr; - float clipNorm = (float)options->get("clip-norm"); // @TODO: should this be ? + float clipNorm = options->get("clip-norm"); if(clipNorm > 0) clipper = Clipper(clipNorm); From 2d1030159f955c4644966b2950d5e4f08d76aeaf Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 10 Dec 2018 21:49:30 +0000 Subject: [PATCH 015/838] Fix description of --clip-norm --- src/common/config_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 0745cbb5e..ce09f6806 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -352,7 +352,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--label-smoothing", "Epsilon for label smoothing (0 to disable)"); cli.add("--clip-norm", - "Clip gradient norm to argcli.add(0 to disable)", + "Clip gradient norm to arg (0 to disable)", 1.f); cli.add("--exponential-smoothing", "Maintain smoothed version of parameters for validation and saving with smoothing factor. 0 to disable", From 07d7f32254241eb741f663ccc0f095586d63a491 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 11 Dec 2018 11:12:26 +0000 Subject: [PATCH 016/838] A few cosmetic changes in training state --- src/training/scheduler.h | 30 ++++++++++-------- src/training/training_state.h | 57 ++++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index dee62496d..acf8b5243 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -16,7 +16,8 @@ class Scheduler : public TrainingObserver { Ptr state_; - timer::Timer timer_, heartBeatTimer_; + timer::Timer timer_; + timer::Timer heartBeatTimer_; float getLearningRate(TrainingState& state) { float baselr = options_->get("learn-rate"); @@ -24,7 +25,8 @@ class Scheduler : public TrainingObserver { float mult1 = 1.f; auto warmup = SchedulingParameter::parse(options_->get("lr-warmup")); if(warmup) { - ABORT_IF(state.warmupStart && state.warmupStart.unit != warmup.unit, "lr-warmup and warmup-start must have the same unit"); + ABORT_IF(state.warmupStart && state.warmupStart.unit != warmup.unit, + "lr-warmup and warmup-start must have the same unit"); auto bno = state.getProgressIn(warmup.unit) - state.warmupStart.n; mult1 = std::min(1.f, (float)bno / (float)warmup.n); } @@ -32,7 +34,9 @@ class Scheduler : public TrainingObserver { float mult2 = 1.f; auto decayGoogle = SchedulingParameter::parse(options_->get("lr-decay-inv-sqrt")); if(decayGoogle) { - mult2 = std::min(1.f, (float)(std::sqrt(decayGoogle.n) / std::sqrt(state.getProgressIn(decayGoogle.unit)))); + mult2 = std::min( + 1.f, + (float)(std::sqrt(decayGoogle.n) / std::sqrt(state.getProgressIn(decayGoogle.unit)))); } baselr = baselr * mult1 * mult2; @@ -45,8 +49,7 @@ class Scheduler : public TrainingObserver { } public: - Scheduler(Ptr options, Ptr state) - : options_(options), state_(state) { + Scheduler(Ptr options, Ptr state) : options_(options), state_(state) { state_->eta = getLearningRate(*state); } @@ -163,14 +166,15 @@ class Scheduler : public TrainingObserver { } void update(float cost, const std::vector>& batches, Ptr mpi = nullptr) { - state_->rememberPreviousProgress(); // note: epoch increases happen at the wrong place, hence -freq parameters do not support epoch units + state_->rememberPreviousProgress(); // note: epoch increases happen at the wrong place, hence + // -freq parameters do not support epoch units state_->validated = false; size_t batchSize = 0; // number of sentences in batch size_t batchLabels = 0; // number of target words in batch for(const auto& batch : batches) { - if (batch) { // (nullptr is allowed as result of split) + if(batch) { // (nullptr is allowed as result of split) batchSize += batch->size(); batchLabels += batch->words(-1); } @@ -178,8 +182,9 @@ class Scheduler : public TrainingObserver { // extrapolate cost across MPI processes, so that we have numbers in the right range // When doing the actual log, we then aggregate across MPI processes to get the accurate number. - if (mpi) - cost *= mpi->numMPIProcesses(); // @BUGBUG: this is presently correct for ce-sum, but possibly not the av-based losses + if(mpi) + cost *= mpi->numMPIProcesses(); // @BUGBUG: this is presently correct for ce-sum, but + // possibly not the av-based losses // reconstruct sum cost, for displaying epoch-level averages instead of minibatch-level auto costType = options_->get("cost-type"); @@ -208,13 +213,13 @@ class Scheduler : public TrainingObserver { if(state_->enteredNewPeriodOf(options_->get("disp-freq")) || state_->batches <= options_->get("disp-first")) { // if MPI then aggregate precise cost across workers - if (mpi) { + if(mpi) { //LOG(info, "all-reducing cost from {}", state_->costSum); state_->costSum /= mpi->numMPIProcesses(); // undo the extra scaling mpi->allReduce(&state_->costSum, &state_->costSum, 1, MPI_FLOAT, MPI_SUM); //LOG(info, "all-reduced cost to {}", state_->costSum); } - if (mpi && mpi->myMPIRank() != 0) + if(mpi && mpi->myMPIRank() != 0) ; // skip the report on alternate worker processes else if(dispLabelCounts) { if(options_->get("lr-report")) { // if true then show the learning rate @@ -419,8 +424,7 @@ class Scheduler : public TrainingObserver { if(factor > 0.0) { if(options_->get("lr-decay-strategy") == "stalled") { - size_t startStalled - = options_->get>("lr-decay-start").front(); + size_t startStalled = options_->get>("lr-decay-start").front(); if(startStalled && state.stalled && state.stalled % startStalled == 0) { state.factor *= factor; state.eta = baselr * state.factor; diff --git a/src/training/training_state.h b/src/training/training_state.h index 2deefe471..c7c2a7771 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -25,20 +25,22 @@ enum class SchedulingUnit { updates, // "u": number of updates so far (batches) epochs // "e": number of epochs begun so far (very first epoch is 1) }; + struct SchedulingParameter { size_t n{0}; // number of steps measured in 'unit' SchedulingUnit unit{SchedulingUnit::updates}; // unit of value // parses scheduling parameters of the form NU where N=unsigned int and U=unit - // Examples of valid inputs: "16000u" (16000 updates), "32000000t" (32 million target labels), "100e" (100 epochs). + // Examples of valid inputs: "16000u" (16000 updates), "32000000t" (32 million target labels), + // "100e" (100 epochs). static SchedulingParameter parse(std::string param) { SchedulingParameter res; - if (!param.empty() && param.back() >= 'a') { - switch (param.back()) { - case 't': res.unit = SchedulingUnit::trgLabels; break; - case 'u': res.unit = SchedulingUnit::updates; break; - case 'e': res.unit = SchedulingUnit::epochs; break; - default: ABORT("invalid unit '{}' in {}", param.back(), param); + if(!param.empty() && param.back() >= 'a') { + switch(param.back()) { + case 't': res.unit = SchedulingUnit::trgLabels; break; + case 'u': res.unit = SchedulingUnit::updates; break; + case 'e': res.unit = SchedulingUnit::epochs; break; + default: ABORT("invalid unit '{}' in {}", param.back(), param); } param.pop_back(); } @@ -49,11 +51,11 @@ struct SchedulingParameter { operator bool() const { return n > 0; } // check whether it is specified operator std::string() const { // convert back for storing in config - switch (unit) { - case SchedulingUnit::trgLabels: return std::to_string(n) + "t"; - case SchedulingUnit::updates : return std::to_string(n) + "u"; - case SchedulingUnit::epochs : return std::to_string(n) + "e"; - default: ABORT("corrupt enum value"); + switch(unit) { + case SchedulingUnit::trgLabels: return std::to_string(n) + "t"; + case SchedulingUnit::updates : return std::to_string(n) + "u"; + case SchedulingUnit::epochs : return std::to_string(n) + "e"; + default: ABORT("corrupt enum value for scheduling unit"); } } }; @@ -120,11 +122,11 @@ class TrainingState { // return the totals count that corresponds to the given unit (batches, labels, or epochs) size_t getProgressIn(SchedulingUnit u) const { - switch (u) { - case SchedulingUnit::trgLabels: return labelsTotal; - case SchedulingUnit::updates : return batches; - case SchedulingUnit::epochs : return epochs; - default: ABORT("corrupt enum value"); + switch(u) { + case SchedulingUnit::trgLabels: return labelsTotal; + case SchedulingUnit::updates : return batches; + case SchedulingUnit::epochs : return epochs; + default: ABORT("corrupt enum value"); } } @@ -137,11 +139,11 @@ class TrainingState { } size_t getPreviousProgressIn(SchedulingUnit u) const { - switch (u) { - case SchedulingUnit::trgLabels: return prevLabelsTotal; - case SchedulingUnit::updates : return prevBatches; - case SchedulingUnit::epochs : return prevEpochs; - default: ABORT("corrupt enum value"); + switch(u) { + case SchedulingUnit::trgLabels: return prevLabelsTotal; + case SchedulingUnit::updates : return prevBatches; + case SchedulingUnit::epochs : return prevEpochs; + default: ABORT("corrupt enum value"); } } @@ -149,6 +151,7 @@ class TrainingState { // unit in which that parameter is given. There are a few edge cases: // - this function will be called many times within the same epoch // - labelsTotal does not increment by 1, so simple modulus does not work + // // So instead of modulus==0, this function compares the previous progress/period // to the current, and triggers if they differ (i.e. the border between two // periods was crossed). This requires that rememberPreviousProgress() is called @@ -158,7 +161,8 @@ class TrainingState { bool enteredNewPeriodOf(std::string schedulingParam) const { auto period = SchedulingParameter::parse(schedulingParam); ABORT_IF(period.unit == SchedulingUnit::epochs, - "Unit {} is not supported for frequency parameters (the one(s) with value {})", schedulingParam); + "Unit {} is not supported for frequency parameters (the one(s) with value {})", + schedulingParam); auto previousProgress = getPreviousProgressIn(period.unit); auto progress = getProgressIn(period.unit); return period && progress / period.n != previousProgress / period.n; @@ -204,13 +208,16 @@ class TrainingState { epochs = config["epochs"].as(); batches = config["batches"].as(); batchesEpoch = config["batches-epoch"].as(); - // (different serialization name for back compat) + // different serialization name for backward compatibility samplesEpoch = config["samples"].as(); - // (optional for back compat) + + // clang-format off + // optional for backward compatibility labelsTotal = config["labels-total"] ? config["labels-total"].as() : 0; prevLabelsTotal = config["prev-labels-total"] ? config["prev-labels-total"].as() : 0; prevBatches = config["prev-batches"] ? config["prev-batches"].as() : 0; prevEpochs = config["prev-epochs"] ? config["prev-epochs"].as() : 0; + // clang-format on stalled = config["stalled"].as(); maxStalled = config["stalled-max"].as(); From d7919a898be79b150d2fe9183c614fb6322f9827 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 11 Dec 2018 11:19:43 +0000 Subject: [PATCH 017/838] Make --lr-decay a float --- src/common/config_parser.cpp | 2 +- src/common/config_validator.cpp | 2 +- src/training/scheduler.h | 11 ++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index ce09f6806..ceb744bb7 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -320,7 +320,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--lr-report", "Report learning rate for each update"); - cli.add("--lr-decay", + cli.add("--lr-decay", "Per-update decay factor for learning rate: lr <- lr * arg (0 to disable)"); cli.add("--lr-decay-strategy", "Strategy for learning rate decaying: epoch, batches, stalled, epoch+batches, epoch+stalled", diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 5086c7263..5f2eae09c 100755 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -89,7 +89,7 @@ void ConfigValidator::validateOptionsTraining() const { "There should be as many validation sets as training sets"); // validations for learning rate decaying - ABORT_IF(get("lr-decay") > 1.0, "Learning rate decay factor greater than 1.0 is unusual"); + ABORT_IF(get("lr-decay") > 1.f, "Learning rate decay factor greater than 1.0 is unusual"); auto strategy = get("lr-decay-strategy"); diff --git a/src/training/scheduler.h b/src/training/scheduler.h index acf8b5243..efc597410 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -318,7 +318,7 @@ class Scheduler : public TrainingObserver { } void actAfterEpoch(TrainingState& state) override { - float factor = (float)options_->get("lr-decay"); // @TODO: ? + float factor = options_->get("lr-decay"); float baselr = getLearningRate(state); state.eta = baselr * state.factor; @@ -352,10 +352,7 @@ class Scheduler : public TrainingObserver { if(decay) { state.factor *= factor; state.eta = baselr * state.factor; - LOG(info, - "Decaying learning rate to {} in epoch {}", - state.eta, - state.epochs); + LOG(info, "Decaying learning rate to {} in epoch {}", state.eta, state.epochs); state.reset = options_->get("lr-decay-reset-optimizer"); if(state.reset) @@ -370,7 +367,7 @@ class Scheduler : public TrainingObserver { } void actAfterBatches(TrainingState& state) override { - float factor = (float)options_->get("lr-decay"); // @TODO: ? + float factor = options_->get("lr-decay"); state.reset = false; float baselr = getLearningRate(state); @@ -416,7 +413,7 @@ class Scheduler : public TrainingObserver { } void actAfterStalled(TrainingState& state) override { - float factor = (float)options_->get("lr-decay"); // @TODO: ? + float factor = options_->get("lr-decay"); state.reset = false; float baselr = getLearningRate(state); From 980233bb04140e310b7d348ae402a957f9824869 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 11 Dec 2018 11:37:50 +0000 Subject: [PATCH 018/838] Add TODOs in scheduler --- src/training/scheduler.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index efc597410..b7506e8f2 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -41,6 +41,7 @@ class Scheduler : public TrainingObserver { baselr = baselr * mult1 * mult2; + // TODO: why lr-warmup-start-rate is extracted from options_ instead of using state.warmupStart? float lrStart = options_->get("lr-warmup-start-rate"); if(lrStart > 0) baselr = baselr - lrStart * mult1 * mult2 + lrStart * mult2; @@ -393,6 +394,7 @@ class Scheduler : public TrainingObserver { if(options_->get("lr-decay-repeat-warmup")) { LOG(info, "Restarting learning rate warmup"); + // TODO: avoid repeating this many times and minimize calls to options_->get state.warmupStart.n = state.getProgressIn(SchedulingParameter::parse(options_->get("lr-warmup")).unit); } } From 84af8c0e50f135f674aefc9718942990415449bf Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 12:26:14 +0000 Subject: [PATCH 019/838] Fix providing default value for cpu-threads --- src/common/config_parser.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index ceb744bb7..d0c9310a5 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -560,12 +560,12 @@ void ConfigParser::addSuboptionsDevices(cli::CLIWrapper& cli) { #endif #ifdef CUDA_FOUND cli.add("--cpu-threads", - "Use CPU-based computation with this many independent threads, 0 means GPU-based computation") - ->default_val("0")->implicit_val("1"); + "Use CPU-based computation with this many independent threads, 0 means GPU-based computation", + 0)->implicit_val("1"); #else cli.add("--cpu-threads", - "Use CPU-based computation with this many independent threads, 0 means GPU-based computation") - ->default_val("1"); + "Use CPU-based computation with this many independent threads, 0 means GPU-based computation", + 1); #endif // clang-format on } From cf6cfc370e604f1241a1166b32c68f603bc59836 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 12:46:03 +0000 Subject: [PATCH 020/838] Replace add_nondefault() with add() for --log and --log-time-zone --- src/common/config_parser.cpp | 4 +-- src/common/logging.cpp | 55 ++++++++++++++---------------------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index d0c9310a5..637200471 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -58,12 +58,12 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { cli.add("--workspace,-w", "Preallocate arg MB of work space", defaultWorkspace); - cli.add_nondefault("--log", + cli.add("--log", "Log training process information to file given by arg"); cli.add("--log-level", "Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off", "info"); - cli.add_nondefault("--log-time-zone", + cli.add("--log-time-zone", "Set time zone for the date shown on logging"); cli.add("--quiet", "Suppress all logging to stderr. Logging to files still works"); diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 0170d633c..0f8eefcb1 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -14,26 +14,22 @@ #define noinline __attribute__((noinline)) #endif -std::shared_ptr createStderrLogger( - const std::string& name, - const std::string& pattern, - const std::vector& files, - bool quiet) { +std::shared_ptr createStderrLogger(const std::string& name, + const std::string& pattern, + const std::vector& files, + bool quiet) { std::vector sinks; auto stderr_sink = spdlog::sinks::stderr_sink_mt::instance(); - if(!quiet) sinks.push_back(stderr_sink); for(auto&& file : files) { - auto file_sink - = std::make_shared(file, true); + auto file_sink = std::make_shared(file, true); sinks.push_back(file_sink); } - auto logger - = std::make_shared(name, begin(sinks), end(sinks)); + auto logger = std::make_shared(name, begin(sinks), end(sinks)); spdlog::register_logger(logger); logger->set_pattern(pattern); @@ -56,9 +52,7 @@ bool setLoggingLevel(spdlog::logger& logger, std::string const level) { else if(level == "off") logger.set_level(spdlog::level::off); else { - logger.warn("Unknown log level '{}' for logger '{}'", - level.c_str(), - logger.name().c_str()); + logger.warn("Unknown log level '{}' for logger '{}'", level.c_str(), logger.name().c_str()); return false; } return true; @@ -69,7 +63,7 @@ void createLoggers(const marian::Config* options) { std::vector generalLogs; std::vector validLogs; - if(options && options->has("log")) { + if(options && !options->get("log").empty()) { generalLogs.push_back(options->get("log")); #ifndef _WIN32 // can't open the same file twice in Windows for some reason @@ -77,16 +71,13 @@ void createLoggers(const marian::Config* options) { #endif } - if(options && options->has("valid-log") - && !options->get("valid-log").empty()) { + if(options && options->has("valid-log") && !options->get("valid-log").empty()) { validLogs.push_back(options->get("valid-log")); } bool quiet = options && options->get("quiet"); - Logger general{ - createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; - Logger valid{ - createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; + Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; + Logger valid{createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; if(options && options->has("log-level")) { std::string loglevel = options->get("log-level"); @@ -95,34 +86,30 @@ void createLoggers(const marian::Config* options) { setLoggingLevel(*valid, loglevel); } - if (options && options->has("log-time-zone")) { + if(options && !options->get("log-time-zone").empty()) { std::string timezone = options->get("log-time-zone"); - if (timezone != "") { #ifdef _WIN32 #define setenv(var, val, over) SetEnvironmentVariableA(var, val) // ignoring over flag #endif - setenv("TZ", timezone.c_str(), true); - tzset(); - } + setenv("TZ", timezone.c_str(), true); + tzset(); } setErrorHandlers(); } static void unhandledException() { - if (std::current_exception()) { + if(std::current_exception()) { try { - throw; // rethrow so that we can get access to what() - } - catch (const std::exception& e) { + throw; // rethrow so that we can get access to what() + } catch(const std::exception& e) { ABORT("Unhandled {}: {}", typeid(e).name(), e.what()); - } - catch (...) { + } catch(...) { ABORT("Unhandled exception"); } - } - else + } else { std::abort(); + } } static void setErrorHandlers() { @@ -144,7 +131,7 @@ static void setErrorHandlers() { // This is called upon initializing MPI. It is needed to associated error messages to ranks. void switchtoMultinodeLogging(std::string nodeIdStr) { Logger log = spdlog::get("general"); - if (log) + if(log) log->set_pattern("[%Y-%m-%d %T " + nodeIdStr + "] %v"); } From 3abba2873a13bee3e555eaf667c78aea5ba69c85 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 12:52:31 +0000 Subject: [PATCH 021/838] Replace add_nondefault() with add() for --dump-config --- src/common/config_parser.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 637200471..166858c49 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -77,8 +77,9 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { "allow the use of environment variables in paths, of the form ${VAR_NAME}"); cli.add("--relative-paths", "All paths are relative to the config file location"); - cli.add_nondefault("--dump-config", - "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") + cli.add("--dump-config", + "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal", + "false") ->implicit_val("full"); // clang-format on } @@ -717,7 +718,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { // remove extra config files from the config to avoid redundancy config_.remove("config"); - if(has("dump-config") && get("dump-config") != "false") { + if(get("dump-config") != "false") { bool skipDefault = get("dump-config") == "minimal"; config_.remove("dump-config"); std::cout << cli.dumpConfig(skipDefault) << std::endl; From a0794f46bd088e4c924b033cd6e1d2f6e071ab19 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 13:53:02 +0000 Subject: [PATCH 022/838] Replace add_nondefault() with add() for --valid-log --- src/common/config_parser.cpp | 2 +- src/common/logging.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 166858c49..a6f38467d 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -446,7 +446,7 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { cli.add("--keep-best", "Keep best model for each validation metric"); - cli.add_nondefault("--valid-log", + cli.add("--valid-log", "Log validation scores to file given by arg"); // clang-format on } diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 0f8eefcb1..586336025 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -71,7 +71,7 @@ void createLoggers(const marian::Config* options) { #endif } - if(options && options->has("valid-log") && !options->get("valid-log").empty()) { + if(options && !options->get("valid-log").empty()) { validLogs.push_back(options->get("valid-log")); } From 0a24ca422cae59919445b038763a098afca81d4a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 13:53:36 +0000 Subject: [PATCH 023/838] Rename options to config for clarity --- src/common/logging.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 586336025..430000228 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -59,35 +59,35 @@ bool setLoggingLevel(spdlog::logger& logger, std::string const level) { } static void setErrorHandlers(); -void createLoggers(const marian::Config* options) { +void createLoggers(const marian::Config* config) { std::vector generalLogs; std::vector validLogs; - if(options && !options->get("log").empty()) { - generalLogs.push_back(options->get("log")); + if(config && !config->get("log").empty()) { + generalLogs.push_back(config->get("log")); #ifndef _WIN32 // can't open the same file twice in Windows for some reason - validLogs.push_back(options->get("log")); + validLogs.push_back(config->get("log")); #endif } - if(options && !options->get("valid-log").empty()) { - validLogs.push_back(options->get("valid-log")); + if(config && !config->get("valid-log").empty()) { + validLogs.push_back(config->get("valid-log")); } - bool quiet = options && options->get("quiet"); + bool quiet = config && config->get("quiet"); Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; Logger valid{createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; - if(options && options->has("log-level")) { - std::string loglevel = options->get("log-level"); + if(config && config->has("log-level")) { + std::string loglevel = config->get("log-level"); if(!setLoggingLevel(*general, loglevel)) return; setLoggingLevel(*valid, loglevel); } - if(options && !options->get("log-time-zone").empty()) { - std::string timezone = options->get("log-time-zone"); + if(config && !config->get("log-time-zone").empty()) { + std::string timezone = config->get("log-time-zone"); #ifdef _WIN32 #define setenv(var, val, over) SetEnvironmentVariableA(var, val) // ignoring over flag #endif From fefd438402a7a73c4c57e79f5b52f1b380472722 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 14:01:13 +0000 Subject: [PATCH 024/838] Fix --valid-log --- src/common/logging.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 430000228..5e431cdda 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -71,7 +71,8 @@ void createLoggers(const marian::Config* config) { #endif } - if(config && !config->get("valid-log").empty()) { + // valid-log is available only for training + if(config && config->has("valid-log") && !config->get("valid-log").empty()) { validLogs.push_back(config->get("valid-log")); } From c2b6d10c658cfd47563b00828b925fac1235d282 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 14:52:12 +0000 Subject: [PATCH 025/838] Add Options::nonempty() --- src/common/options.h | 25 +++++++++++++++++++++++++ src/tests/cli.cpp | 7 ++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/common/options.h b/src/common/options.h index 9f1bb8dba..b413f7db3 100755 --- a/src/common/options.h +++ b/src/common/options.h @@ -92,6 +92,31 @@ class Options { return defaultValue; } + /** + * @brief Check if a sequence or string option is defined and nonempty + * + * Aborts if the option does not store a sequence or string value. Returns false if an option with + * the given key does not exist. + * + * @param key option name + * + * @return true if the option is defined and is a nonempty sequence or string + */ + bool nonempty(const std::string& key) const { + if(!has(key)) { + return false; + } + if(options_[key].IsSequence()) { + return options_[key].size() != 0; + } + try { + return !options_[key].as().empty(); + } catch(const YAML::BadConversion& e) { + ABORT("Option '{}' is neither a sequence nor a text"); + } + return false; + } + bool has(const std::string& key) const { return options_[key]; } }; diff --git a/src/tests/cli.cpp b/src/tests/cli.cpp index 2f40bb133..c3210514b 100644 --- a/src/tests/cli.cpp +++ b/src/tests/cli.cpp @@ -41,7 +41,7 @@ int main(int argc, char** argv) { { auto w = New(options); w->add("-i,--int", "help message")->implicit_val("555")->default_val("123"); - w->add("-s,--str", "help message")->default_val("foo"); + w->add("-s,--str", "help message"); w->add>("-v,--vec", "help message")->expected(-2); w->switchGroup("My group"); w->add>("--defvec,-d", "help message")->default_val("foo"); @@ -67,5 +67,10 @@ int main(int argc, char** argv) { YAML::Emitter emit; OutputYaml(options->getYaml(), emit); std::cout << emit.c_str() << std::endl; + + std::cout << "===" << std::endl; + std::cout << "vec/str.nonempty? " << options->nonempty("vec") << " " << options->nonempty("str") << std::endl; + std::cout << "vec/str.has? " << options->has("vec") << " " << options->has("str") << std::endl; + return 0; } From ea51591671f2e444f0d15098ee9602511cbf5539 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 14:55:16 +0000 Subject: [PATCH 026/838] Replace add_nondefault() with add() for --data-weighting --- src/common/config_parser.cpp | 2 +- src/data/corpus.h | 2 +- src/data/corpus_base.cpp | 2 +- src/data/corpus_base.h | 2 +- src/data/corpus_sqlite.cpp | 3 +-- src/data/corpus_sqlite.h | 2 +- src/layers/weight.cpp | 2 +- src/models/costs.h | 10 +++++----- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index a6f38467d..ab4b2b7aa 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -367,7 +367,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--guided-alignment-weight", "Weight for guided alignment cost", 0.1); - cli.add_nondefault("--data-weighting", + cli.add("--data-weighting", "Path to a file with sentence or word weights"); cli.add("--data-weighting-type", "Processing level for data weighting: sentence, word", diff --git a/src/data/corpus.h b/src/data/corpus.h index 119f3aab2..e916403d9 100755 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -98,7 +98,7 @@ class Corpus : public CorpusBase { if(options_->get("guided-alignment", std::string("none")) != "none" && alignFileIdx_) addAlignmentsToBatch(batch, batchVector); - if(options_->has("data-weighting") && weightFileIdx_) + if(options_->nonempty("data-weighting") && weightFileIdx_) addWeightsToBatch(batch, batchVector); return batch; diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index 2a088537a..11304030a 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -174,7 +174,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) ABORT_IF(files_.back()->empty(), "File with alignments '{}' is empty", path); } - if(training && options_->has("data-weighting")) { + if(training && options_->nonempty("data-weighting")) { auto path = options_->get("data-weighting"); ABORT_IF(!filesystem::exists(path), "Weight file does not exist"); diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 8ecdf2337..9eb5a2984 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -337,7 +337,7 @@ class CorpusBatch : public Batch { batch->setGuidedAlignment(std::move(alignment)); } - if(options->has("data-weighting")) { + if(options->nonempty("data-weighting")) { auto weightsSize = batchSize; if(options->get("data-weighting-type") != "sentence") weightsSize *= lengths.back(); diff --git a/src/data/corpus_sqlite.cpp b/src/data/corpus_sqlite.cpp index cbab750eb..714a70f01 100644 --- a/src/data/corpus_sqlite.cpp +++ b/src/data/corpus_sqlite.cpp @@ -25,8 +25,7 @@ void CorpusSQLite::fillSQLite() { if(options_->get("sqlite") == "temporary") { LOG(info, "[sqlite] Creating temporary database in {}", tempDir); - db_.reset( - new SQLite::Database("", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)); + db_.reset(new SQLite::Database("", SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)); db_->exec("PRAGMA temp_store_directory = '" + tempDir + "';"); fill = true; diff --git a/src/data/corpus_sqlite.h b/src/data/corpus_sqlite.h index 641da2a8e..ba40ecb49 100755 --- a/src/data/corpus_sqlite.h +++ b/src/data/corpus_sqlite.h @@ -104,7 +104,7 @@ class CorpusSQLite : public CorpusBase { if(options_->has("guided-alignment") && alignFileIdx_) addAlignmentsToBatch(batch, batchVector); - if(options_->has("data-weighting") && weightFileIdx_) + if(options_->nonempty("data-weighting") && weightFileIdx_) addWeightsToBatch(batch, batchVector); return batch; diff --git a/src/layers/weight.cpp b/src/layers/weight.cpp index ab7fe0729..98e97e4e9 100755 --- a/src/layers/weight.cpp +++ b/src/layers/weight.cpp @@ -3,7 +3,7 @@ namespace marian { Ptr WeightingFactory(Ptr options) { - ABORT_IF(!options->has("data-weighting"), + ABORT_IF(!options->nonempty("data-weighting"), "No data-weighting specified in options"); return New(options->get("data-weighting-type")); } diff --git a/src/models/costs.h b/src/models/costs.h index d38a445e8..dd8459b7d 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -33,9 +33,9 @@ class EncoderDecoderCE : public CostBase { loss_ = LossFactory(options_, inference_); toBeWeighted_ - = (options_->has("data-weighting") && !inference_) - || (options_->has("dynamic-weighting") - && options_->get("dynamic-weighting") && !inference_); + = (options_->nonempty("data-weighting") && !inference_) + || (options_->has("dynamic-weighting") && options_->get("dynamic-weighting") + && !inference_); if(toBeWeighted_) weighter_ = WeightingFactory(options_); } @@ -116,7 +116,7 @@ class LogSoftmaxStep : public CostStep { virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) auto logits = state->getLogProbs(); - + auto logprobs = logsoftmax(logits); state->setLogProbs(logprobs); @@ -131,7 +131,7 @@ class GumbelSoftmaxStep : public CostStep { public: virtual Ptr apply(Ptr state) override { auto logits = state->getLogProbs(); - + auto logprobs = logsoftmax(logits + constant_like(logits, inits::gumbel)); state->setLogProbs(logprobs); From 69b795e238e67541375193135154f21d9302c6e6 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 12 Dec 2018 16:47:13 +0000 Subject: [PATCH 027/838] Replace some add_nondefault() with add() --- src/common/config_parser.cpp | 6 +++--- src/common/config_validator.cpp | 3 ++- src/models/decoder.h | 2 +- src/models/s2s.h | 2 +- src/models/transformer.h | 6 +++--- src/optimizers/optimizers.cpp | 4 +--- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_multinode.h | 2 +- src/training/graph_group_multinode_sync.h | 2 +- src/training/graph_group_singleton.h | 2 +- src/training/graph_group_sync.cpp | 2 +- 11 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index ab4b2b7aa..37b75ac45 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -97,7 +97,7 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "model.npz"); if(mode_ == cli::mode::training) { - cli.add_nondefault("--pretrained-model", + cli.add("--pretrained-model", "Path prefix for pre-trained model to initialize model weights"); } } @@ -305,7 +305,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--optimizer,-o", "Optimization algorithm: sgd, adagrad, adam", "adam"); - cli.add_nondefault>("--optimizer-params", + cli.add>("--optimizer-params", "Parameters for optimization algorithm, e.g. betas for adam"); cli.add("--optimizer-delay", "SGD update delay, 1 = no delay", @@ -552,7 +552,7 @@ void ConfigParser::addSuboptionsDevices(cli::CLIWrapper& cli) { cli.add>("--devices,-d", "Specifies GPU ID(s) to use for training. Defaults to 0..num-devices-1", std::vector({"0"})); - cli.add_nondefault("--num-devices", + cli.add("--num-devices", "Number of GPUs to use for this process. Defaults to length(devices) or 1"); #ifdef USE_NCCL if(mode_ == cli::mode::training) diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 5f2eae09c..8a6845633 100755 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -72,7 +72,8 @@ void ConfigValidator::validateOptionsTraining() const { auto trainSets = get>("train-sets"); ABORT_IF(has("embedding-vectors") - && get>("embedding-vectors").size() != trainSets.size(), + && get>("embedding-vectors").size() != trainSets.size() + && !get>("embedding-vectors").empty(), "There should be as many embedding vector files as training sets"); filesystem::Path modelPath(get("model")); diff --git a/src/models/decoder.h b/src/models/decoder.h index 39e56e1f3..c76ea763c 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -50,7 +50,7 @@ class DecoderBase { if(options_->has("embedding-fix-trg")) yEmbFactory("fixed", opt("embedding-fix-trg")); - if(options_->has("embedding-vectors")) { + if(options_->nonempty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); yEmbFactory("embFile", embFiles[batchIndex_]) // ("normalization", opt("embedding-normalization")); diff --git a/src/models/s2s.h b/src/models/s2s.h index 92ba9a7d1..a1087a824 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -136,7 +136,7 @@ class EncoderS2S : public EncoderBase { if(options_->has("embedding-fix-src")) embFactory("fixed", opt("embedding-fix-src")); - if(options_->has("embedding-vectors")) { + if(options_->nonempty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory // ("embFile", embFiles[batchIndex_]) // diff --git a/src/models/transformer.h b/src/models/transformer.h index c30d06c8e..3cd7140a8 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -500,13 +500,13 @@ class EncoderTransformer : public Transformer { int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb); - if (opt("tied-embeddings-src") || opt("tied-embeddings-all")) + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); - if (options_->has("embedding-fix-src")) + if(options_->has("embedding-fix-src")) embFactory("fixed", opt("embedding-fix-src")); - if (options_->has("embedding-vectors")) { + if(options_->nonempty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory("embFile", embFiles[subBatchIndex]) ("normalization", opt("embedding-normalization")); diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index 0fad790a1..b037963e7 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -271,9 +271,7 @@ void Adam::resetStats() { Ptr Optimizer(Ptr options) { float lrate = options->get("learn-rate"); - auto params = options->has("optimizer-params") - ? options->get>("optimizer-params") - : std::vector({}); + auto params = options->get>("optimizer-params", std::vector({})); Ptr clipper = nullptr; float clipNorm = options->get("clip-norm"); diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 1d0042019..ec722b0ad 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -325,7 +325,7 @@ void AsyncGraphGroup::load() { setFn(i, data.begin() + begin, data.begin() + end); } }); - } else if(options_->has("pretrained-model")) { + } else if(options_->nonempty("pretrained-model")) { std::string nameInit = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index c86225ebc..d4158bd07 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -397,7 +397,7 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { size_t i = 0; for(auto graph : clientGraphs_) clientBuilders_[i++]->load(graph, name); - } else if(options_->has("pretrained-model")) { + } else if(options_->nonempty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index bfef2050c..21613e2c0 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -163,7 +163,7 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { size_t i = 0; for(auto graph : clientGraphs_) clientBuilders_[i++]->load(graph, name); - } else if(options_->has("pretrained-model")) { + } else if(options_->nonempty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 1eb589d91..eb0a3f4ae 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -69,7 +69,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { /*scatterStateFn=*/[&](const std::vector& data, const OptimizerBase::ScatterStateSetFunc& setFn) { setFn(/*localDeviceIndex=*/0, data.begin(), data.end()); }); - } else if(options_->has("pretrained-model")) { + } else if(options_->nonempty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 6699484c1..fca75fb4b 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -253,7 +253,7 @@ void SyncGraphGroup::load() /*override*/ { [&](const std::vector& optimizerStateVector, const OptimizerBase::ScatterStateSetFunc& setShardFn) { comm_->scatterState(optimizerStateVector, setShardFn); }); - } else if(options_->has("pretrained-model")) { + } else if(options_->nonempty("pretrained-model")) { std::string nameInit = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", From 4cf97df51e462d2d80e2a296828596b493f058b6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 12 Dec 2018 18:40:46 -0800 Subject: [PATCH 028/838] minibatch-size warmup (manually merged over from fseide/covbias); minibatches are now fed in GPU-sized chunks rather than a massive joint batch for all GPUs in the update; Adam hyper-parameter adjustment limited to learning rate, as momentum adjustment is counterproductive for MB scaling; log output now includes the last batch size; log output now shows current best for stalled validation metrics; bug fix: Adam optimizer should persist denominators; bug fix: Adam and Adagrad should use correct element size when persisting; min and max renamed to minimum and maximum, for consistency with other toolkits; pathie now compiles in manual VS Project --- src/3rd_party/ExceptionWithCallStack.h | 2 + .../pathie-cpp/src/entry_iterator.cpp | 2 +- src/3rd_party/pathie-cpp/src/path.cpp | 12 +- src/3rd_party/sentencepiece | 2 +- src/command/marian_train.cpp | 2 + src/common/config_parser.cpp | 21 +- src/common/definitions.h | 10 +- src/common/file_stream.h | 9 +- src/common/filesystem.h | 2 +- src/common/io.cpp | 4 +- src/common/logging.cpp | 6 +- src/common/logging.h | 10 + src/common/utils.cpp | 35 ++ src/common/utils.h | 4 + src/data/batch.h | 2 +- src/data/batch_generator.h | 34 +- src/data/batch_stats.h | 13 + src/data/corpus.cpp | 2 +- src/data/corpus.h | 4 +- src/data/corpus_base.h | 61 +-- src/examples/mnist/dataset.h | 2 +- src/functional/tmp.h | 21 + src/graph/expression_graph.h | 4 +- src/graph/expression_operators.h | 4 +- src/graph/node_operators.h | 2 +- src/graph/node_operators_binary.h | 6 +- src/graph/node_operators_unary.h | 0 src/optimizers/optimizers.cpp | 96 +++-- src/optimizers/optimizers.h | 54 ++- src/tensors/gpu/add.cu | 0 src/tensors/gpu/element.inc | 6 + src/tensors/tensor_allocator.h | 2 +- src/training/communicator.cpp | 4 +- src/training/communicator.h | 1 - src/training/communicator_nccl.h | 67 ++-- src/training/exponential_smoothing.h | 30 +- src/training/graph_group.h | 9 +- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_async.h | 2 +- src/training/graph_group_multinode.h | 2 +- src/training/graph_group_multinode_sync.h | 2 +- src/training/graph_group_singleton.h | 4 +- src/training/graph_group_sync.cpp | 225 ++++++++--- src/training/graph_group_sync.h | 10 +- src/training/scheduler.h | 112 +++++- src/training/training.h | 8 +- src/training/training_state.h | 13 +- src/training/validator.h | 3 +- vs/Marian.sln | 376 +----------------- vs/Marian.vcxproj | 203 +++++++++- vs/Marian.vcxproj.filters | 278 ++++++++++++- 51 files changed, 1132 insertions(+), 653 deletions(-) mode change 100644 => 100755 src/3rd_party/ExceptionWithCallStack.h mode change 100644 => 100755 src/3rd_party/pathie-cpp/src/entry_iterator.cpp mode change 100644 => 100755 src/3rd_party/pathie-cpp/src/path.cpp mode change 100644 => 100755 src/functional/tmp.h mode change 100644 => 100755 src/graph/expression_operators.h mode change 100644 => 100755 src/graph/node_operators.h mode change 100644 => 100755 src/graph/node_operators_unary.h mode change 100644 => 100755 src/tensors/gpu/add.cu mode change 100644 => 100755 src/tensors/gpu/element.inc diff --git a/src/3rd_party/ExceptionWithCallStack.h b/src/3rd_party/ExceptionWithCallStack.h old mode 100644 new mode 100755 index 5b961bd9b..488b1277f --- a/src/3rd_party/ExceptionWithCallStack.h +++ b/src/3rd_party/ExceptionWithCallStack.h @@ -5,6 +5,8 @@ // ExceptionWithCallStack.h - debug util functions // +#pragma once + #include namespace Microsoft { namespace MSR { namespace CNTK { diff --git a/src/3rd_party/pathie-cpp/src/entry_iterator.cpp b/src/3rd_party/pathie-cpp/src/entry_iterator.cpp old mode 100644 new mode 100755 index e2ecb2fe1..24e5b1101 --- a/src/3rd_party/pathie-cpp/src/entry_iterator.cpp +++ b/src/3rd_party/pathie-cpp/src/entry_iterator.cpp @@ -178,7 +178,7 @@ entry_iterator& entry_iterator::operator++(int) /// Same as the other operator++(). entry_iterator& entry_iterator::operator++() { - return (operator++()); + return (operator++(0)); } /** diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp old mode 100644 new mode 100755 index 3dc1e14bb..12aa02703 --- a/src/3rd_party/pathie-cpp/src/path.cpp +++ b/src/3rd_party/pathie-cpp/src/path.cpp @@ -51,7 +51,7 @@ #include //#include // Currently not in msys2 -// @TODO: This is a hack to make it compile under Windows, check if this is save. +// @TODO: This is a hack to make it compile under Windows, check if this is safe. #define F_OK 0 #elif defined(_PATHIE_UNIX) @@ -1546,7 +1546,7 @@ bool Path::is_directory() const throw(Pathie::ErrnoError(errsav)); } - return s.st_mode & S_IFDIR; + return (s.st_mode & S_IFDIR) != 0; #else #error Unsupported system. #endif @@ -1590,7 +1590,7 @@ bool Path::is_file() const throw(Pathie::ErrnoError(errno)); } - return s.st_mode & S_IFREG; + return (s.st_mode & S_IFREG) != 0; #else #error Unsupported system. #endif @@ -1710,9 +1710,9 @@ void Path::remove() const * function uses the apropriate native Win32API function * calls accordingly therefore. */ if (is_directory()) - result = RemoveDirectoryW(utf16.c_str()); + result = RemoveDirectoryW(utf16.c_str()) != 0; else - result = DeleteFileW(utf16.c_str()); + result = DeleteFileW(utf16.c_str()) != 0; if (!result) { DWORD err = GetLastError(); @@ -3282,7 +3282,7 @@ bool Path::fnmatch(const std::string& pattern, int flags /* = 0 */) const #elif defined(_WIN32) std::wstring utf16path = utf8_to_utf16(m_path); std::wstring utf16pattern = utf8_to_utf16(pattern); - return PathMatchSpecW(utf16path.c_str(), utf16pattern.c_str()); + return PathMatchSpecW(utf16path.c_str(), utf16pattern.c_str()) != 0; #else #error Unsupported system. #endif diff --git a/src/3rd_party/sentencepiece b/src/3rd_party/sentencepiece index 1a38d26a1..21309542e 160000 --- a/src/3rd_party/sentencepiece +++ b/src/3rd_party/sentencepiece @@ -1 +1 @@ -Subproject commit 1a38d26a13cc67b1aae641d4983b624bef6d5305 +Subproject commit 21309542e69e1821ff8e905fa60d8852ac12a73f diff --git a/src/command/marian_train.cpp b/src/command/marian_train.cpp index 889806244..9f255e5c3 100755 --- a/src/command/marian_train.cpp +++ b/src/command/marian_train.cpp @@ -11,6 +11,8 @@ #include "training/graph_group_multinode.h" #endif +#include "3rd_party/ExceptionWithCallStack.h" + int main(int argc, char** argv) { using namespace marian; diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index d39f3b1ad..06eabd320 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -335,9 +335,10 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Reset running statistics of optimizer whenever learning rate decays"); cli.add("--lr-decay-repeat-warmup", "Repeat learning rate warmup when learning rate is decayed"); - cli.add("--lr-decay-inv-sqrt", - "Decrease learning rate at arg / sqrt(no. batches) starting at arg (append 't' or 'e' for sqrt(target labels or epochs))", - "0"); + cli.add>("--lr-decay-inv-sqrt", + "Decrease learning rate at arg / sqrt(no. batches) starting at arg (append 't' or 'e' for sqrt(target labels or epochs)). " + "Add second argument to define the starting point", + {"0"}); cli.add("--lr-warmup", "Increase learning rate linearly for arg first batches (append 't' for arg first target labels)", @@ -354,9 +355,10 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--clip-norm", "Clip gradient norm to argcli.add(0 to disable)", 1.f); - cli.add("--exponential-smoothing", - "Maintain smoothed version of parameters for validation and saving with smoothing factor. 0 to disable", - 0)->implicit_val("1e-4"); + cli.add>("--exponential-smoothing", + "Maintain smoothed version of parameters for validation and saving with smoothing factor. 0 to disable. " + "Add a second number to specify a reference batch size (in target words).", + { 0.f })->implicit_val("1e-4"); cli.add("--guided-alignment", "Path to a file with word alignments. Use guided alignment to guide attention or 'none'", "none"); @@ -604,6 +606,13 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { cli.add("--shuffle-in-ram", "Keep shuffled corpus in RAM, do not write to temp file"); + + cli.add>("--mini-batch-warmup", + "linear ramp-up of MB size, up to this #updates (append 't' for up to this #target labels);" + "optional second number is reference batch size at which to stop scaling up (instead of full batch size)", + {"0"}); + cli.add("--mini-batch-track-lr", + "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); // clang-format on } diff --git a/src/common/definitions.h b/src/common/definitions.h index 3b4a6eddb..101200fff 100755 --- a/src/common/definitions.h +++ b/src/common/definitions.h @@ -51,8 +51,16 @@ struct DeviceId { DeviceId() : no{0}, type{DeviceType::gpu} {} DeviceId(size_t no_, DeviceType type_) : no(no_), type(type_) {} + std::string typeAsString() const { + return (type == DeviceType::gpu ? "gpu" : "cpu"); + } + + operator std::string() const { + return typeAsString() + std::to_string(no); + } + friend std::ostream& operator<<(std::ostream& out, DeviceId deviceId) { - out << (deviceId.type == DeviceType::gpu ? "gpu" : "cpu") << deviceId.no; + out << std::string(deviceId); return out; } diff --git a/src/common/file_stream.h b/src/common/file_stream.h index 9abe7e238..52fa8e9c6 100755 --- a/src/common/file_stream.h +++ b/src/common/file_stream.h @@ -178,11 +178,9 @@ class InputFileStream { bool empty() { return istream_->peek() == std::ifstream::traits_type::eof(); } void setbufsize(size_t size) const { -#ifdef 0 // this is buggy, do nothing istream_->rdbuf()->pubsetbuf(0, 0); - readBuf_.reset(new char[size]); - istream_->rdbuf()->pubsetbuf(readBuf_.get(), 0); -#endif + readBuf_.resize(size); + istream_->rdbuf()->pubsetbuf(readBuf_.data(), readBuf_.size()); } template @@ -206,9 +204,8 @@ class InputFileStream { std::unique_ptr istream_; boost::iostreams::file_descriptor_source fds_; + mutable std::vector readBuf_; // for setbuf() std::unique_ptr> fdsBuffer_; - - mutable UPtr readBuf_; // for setbuf() }; // wrapper around std::getline() that handles Windows input files with extra CR diff --git a/src/common/filesystem.h b/src/common/filesystem.h index f9c061046..08d5995f1 100755 --- a/src/common/filesystem.h +++ b/src/common/filesystem.h @@ -12,7 +12,7 @@ #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -#include "3rd_party/pathie-cpp/include/path.hpp" +#include "3rd_party/pathie-cpp/include/path.hpp" // @TODO: update to latest Pathie #include "3rd_party/pathie-cpp/include/errors.hpp" #ifdef __GNUC__ diff --git a/src/common/io.cpp b/src/common/io.cpp index ae768e183..ee19105f4 100755 --- a/src/common/io.cpp +++ b/src/common/io.cpp @@ -128,10 +128,12 @@ void saveItemsNpz(const std::string& fileName, const std::vector& items) { std::vector npzItems; for(auto& item : items) { std::vector shape(item.shape.begin(), item.shape.end()); - char type = 'f'; + char type; if(item.type == Type::float32) type = cnpy::map_type(typeid(float)); + else if(item.type == Type::float64) + type = cnpy::map_type(typeid(double)); else if(item.type == Type::int8) type = cnpy::map_type(typeid(char)); else diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 0170d633c..da583878d 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -84,7 +84,7 @@ void createLoggers(const marian::Config* options) { bool quiet = options && options->get("quiet"); Logger general{ - createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; + createStderrLogger("general", "[%Y-%m-%d %T %t] %v", generalLogs, quiet)}; Logger valid{ createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; @@ -115,7 +115,7 @@ static void unhandledException() { throw; // rethrow so that we can get access to what() } catch (const std::exception& e) { - ABORT("Unhandled {}: {}", typeid(e).name(), e.what()); + ABORT("Unhandled exception of type '{}': {}", typeid(e).name(), e.what()); } catch (...) { ABORT("Unhandled exception"); @@ -145,7 +145,7 @@ static void setErrorHandlers() { void switchtoMultinodeLogging(std::string nodeIdStr) { Logger log = spdlog::get("general"); if (log) - log->set_pattern("[%Y-%m-%d %T " + nodeIdStr + "] %v"); + log->set_pattern("[%Y-%m-%d %T " + nodeIdStr + ":%t] %v"); } diff --git a/src/common/logging.h b/src/common/logging.h index cdaa806c8..2889d6c7f 100755 --- a/src/common/logging.h +++ b/src/common/logging.h @@ -21,6 +21,16 @@ namespace marian { */ #define LOG(level, ...) checkedLog("general", #level, __VA_ARGS__) +// variant that prints the log message only upon the first time the call site is executed +#define LOG_ONCE(level, ...) do { \ + static bool logged = false; \ + if (!logged) \ + { \ + logged = true; \ + LOG(level, __VA_ARGS__); \ + } \ +} while(0) + /** * Prints logging message regarding validation into stderr and a file specified * with `--valid-log` option. diff --git a/src/common/utils.cpp b/src/common/utils.cpp index b47cd1f4e..992c74ac8 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -149,5 +149,40 @@ bool endsWith(const std::string& text, const std::string& suffix) { && !text.compare(text.size() - suffix.size(), suffix.size(), suffix); } +std::string toUpper(const std::string& s) { + std::locale loc; + std::string res; res.reserve(s.capacity()); + for (auto c : s) // @BUGBUG: This won't work with UTF-8 characters. + res.push_back((char)std::toupper(c, loc)); + return res; +} + +double parseDouble(std::string s) { + double res; + char c; // dummy char--if we succeed to parse this, then there were extraneous characters after the number + auto rc = sscanf(s.c_str(), "%lf%c", &res, &c); + ABORT_IF(rc != 1, "Mal-formed number: {}", s); + return res; +} + +// parses a user-friendly number that can have commas and (some) units +double parseNumber(std::string param) { + // get unit prefix + double factor = 1.; + if (!param.empty() && param.back() >= 'A') { + switch (param.back()) { + case 'k': factor = 1.e3; break; + case 'M': factor = 1.e6; break; + case 'G': factor = 1.e9; break; + case 'T': factor = 1.e12; break; + default: ABORT("Invalid or unsupported unit prefix '{}' in {}", param.back(), param); + } + param.pop_back(); + } + // we allow users to place commas in numbers (note: we are not actually verifying that they are in the right place) + std::remove_if(param.begin(), param.end(), [](char c) { return c == ','; }); + return factor * parseDouble(param); +} + } // namespace utils } // namespace marian diff --git a/src/common/utils.h b/src/common/utils.h index cd4fc6de3..7c4d56432 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -38,5 +38,9 @@ std::pair hostnameAndProcessId(); std::string withCommas(size_t n); bool endsWith(const std::string& text, const std::string& suffix); +std::string toUpper(const std::string& s); +double parseDouble(std::string s); +double parseNumber(std::string s); + } // namespace utils } // namespace marian diff --git a/src/data/batch.h b/src/data/batch.h index a7832585f..f4b870177 100755 --- a/src/data/batch.h +++ b/src/data/batch.h @@ -19,7 +19,7 @@ class Batch { virtual void debug(){}; - virtual std::vector> split(size_t n) = 0; + virtual std::vector> split(size_t n, size_t sizeLimit = SIZE_MAX) = 0; const std::vector& getSentenceIds() const { return sentenceIds_; } void setSentenceIds(const std::vector& ids) { sentenceIds_ = ids; } diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h index 4578c3bb7..488d3e55b 100755 --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -56,7 +56,7 @@ class BatchGenerator : public RNGEngine { typedef typename DataSet::batch_ptr BatchPtr; typedef typename DataSet::Sample Sample; - typedef std::vector Samples; // @TODO: type names should be capitalized + typedef std::vector Samples; typedef BatchIterator iterator; friend iterator; @@ -83,7 +83,6 @@ class BatchGenerator : public RNGEngine { // this runs on a bg thread; sequencing is handled by caller, but locking is done in here std::deque fetchBatches() { - //LOG(info, "fillBatches entered"); typedef typename Sample::value_type Item; auto itemCmp = [](const Item& sa, const Item& sb) { return sa.size() < sb.size(); }; // sort by element length, not content @@ -118,8 +117,6 @@ class BatchGenerator : public RNGEngine { size_t maxBatchSize = options_->get("mini-batch"); size_t maxSize = maxBatchSize * options_->get("maxi-batch"); - // LOG(info, "Preloading batches"); - // consume data from corpus into maxi-batch (single sentences) // sorted into specified order (due to queue) if(newlyPrepared_) { @@ -141,8 +138,6 @@ class BatchGenerator : public RNGEngine { } size_t numSentencesRead = maxiBatch->size(); - // LOG(info, "Turning samples into batches"); - // construct the actual batches and place them in the queue Samples batchVector; size_t currentWords = 0; @@ -152,7 +147,6 @@ class BatchGenerator : public RNGEngine { // process all loaded sentences in order of increasing length // @TODO: we could just use a vector and do a sort() here; would make the cost more explicit - //LOG(info, "begin form batches, #lines = {}", maxiBatch->size()); const size_t mbWords = options_->get("mini-batch-words", 0); const bool useDynamicBatching = options_->has("mini-batch-fit"); BatchStats::const_iterator cachedStatsIter; @@ -205,15 +199,25 @@ class BatchGenerator : public RNGEngine { } // turn rest into batch + // @BUGBUG: This can create a very small batch, which with ce-mean-words can artificially + // inflate the contribution of the sames in the batch, causing instability. + // I think a good alternative would be to carry over the left-over sentences into the next round. if(!batchVector.empty()) tempBatches.push_back(data_->toBatch(batchVector)); - //LOG(info, "end form batches, #tempBatches = {}", tempBatches.size()); // Shuffle the batches if(shuffle_) { std::shuffle(tempBatches.begin(), tempBatches.end(), eng_); } - LOG(debug, "[data] fetched {} batches with {} sentences.", tempBatches.size(), numSentencesRead); + double totalSent{}, totalLabels{}; + for (auto& b : tempBatches) { + totalSent += (double)b->size(); + totalLabels += (double)b->words(-1); + } + auto totalDenom = tempBatches.empty() ? 1 : tempBatches.size(); // (make 0/0 = 0) + LOG(info, "[data] fetched {} batches with {} sentences. Per batch: {} sentences, {} labels.", + tempBatches.size(), numSentencesRead, + (double)totalSent / (double)totalDenom, (double)totalLabels / (double)totalDenom); return tempBatches; } @@ -300,6 +304,18 @@ class BatchGenerator : public RNGEngine { return true; } + + // this is needed for dynamic MB scaling. Returns 0 if size is not known in words. + size_t estimateTypicalTrgBatchWords() const { + const size_t mbWords = options_->get("mini-batch-words", 0); + const bool useDynamicBatching = options_->has("mini-batch-fit"); + if (useDynamicBatching && stats_) + return stats_->estimateTypicalTrgWords(); + else if (mbWords) + return mbWords; + else + return 0; + } }; class CorpusBatchGenerator : public BatchGenerator, diff --git a/src/data/batch_stats.h b/src/data/batch_stats.h index 50791ed0b..581f83df2 100755 --- a/src/data/batch_stats.h +++ b/src/data/batch_stats.h @@ -49,6 +49,19 @@ class BatchStats { map_[lengths] = batchSize; } + // return a rough minibatch size in labels + // We average over all (batch sizes * max trg length). + size_t estimateTypicalTrgWords() const { + size_t sum = 0; + for (const auto& entry : map_) { + auto maxTrgLength = entry.first.back(); + auto numSentences = entry.second; + auto numLabels = numSentences * maxTrgLength; + sum += numLabels; + } + return sum / map_.size(); + } + // helpers for multi-node --note: presently unused, but keeping them around for later use // serialize into a flat vector, for MPI data exchange std::vector flatten() const { diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7a7a846e1..7f51b12ba 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -102,7 +102,7 @@ void Corpus::restore(Ptr ts) { } void Corpus::shuffleData(const std::vector& paths) { - LOG(info, "[data] Shuffling files"); + LOG(info, "[data] Shuffling data"); size_t numStreams = paths.size(); diff --git a/src/data/corpus.h b/src/data/corpus.h index 119f3aab2..59d82ef55 100755 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -63,8 +63,8 @@ class Corpus : public CorpusBase { std::vector sentenceIds; - std::vector maxDims; - for(auto& ex : batchVector) { + std::vector maxDims; // @TODO: What's this? widths? maxLengths? + for(auto& ex : batchVector) { // @TODO: rename 'ex' to 'sample' or 'sentenceTuple' if(maxDims.size() < ex.size()) maxDims.resize(ex.size(), 0); for(size_t i = 0; i < ex.size(); ++i) { diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 8ecdf2337..4d9fd4cc9 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -164,48 +164,51 @@ class SubBatch { */ size_t batchWidth() { return width_; }; /** - * @brief The total number of words in the batch, considering the mask. + * @brief The total number of words in the batch (not counting masked-out words). */ size_t batchWords() { return words_; } /** - * @brief Splits the subbatch into subbatches of equal size. + * @brief Splits the stream into sub-batches of equal size (except for last). * - * @param n Number of splits + * @param n number of sub-batches to split into * - * @return Vector of pointers to new subbatches. + * @param sizeLimit Pretend the batch only has this many sentences. Used for MB-size ramp-up. + * + * @return Vector of pointers to new sub-batches (or nullptrs where run out of sub-batches) * * @see marian::data::Batch::split(size_t n) */ - std::vector> split(size_t n) { - ABORT_IF(size_ == 0, "Encoutered sub-batch size of 0"); + std::vector> split(size_t n, size_t sizeLimit /*or SIZE_MAX*/) { + ABORT_IF(size_ == 0, "Encountered sub-batch size of 0"); - size_t subSize = (size_t)(std::ceil(size_ / (float)n)); + auto size = std::min(size_, sizeLimit); // if limit is given then pretend the batch only has that many sentences + size_t targetSubSize = (size_t)(std::ceil(size / (float)n)); // aim at forming sub-batches of this #sentences std::vector> splits; - for(size_t pos = 0; pos < size_; pos += subSize) { - size_t size = std::min(subSize, size_ - pos); + for(size_t pos = 0; pos < size; pos += targetSubSize) { // loop over ranges of size targetSubSize to form sub-batches of this size + size_t subSize = std::min(targetSubSize, size - pos); // actual number of sentences can be smaller at the end - // determine actual width + // determine actual width (=max length) of this sub-batch, which may be smaller than the overall max length size_t subWidth = 0; for(size_t j = 0; j < width_; ++j) { - for(size_t i = 0; i < size; ++i) { + for(size_t i = 0; i < subSize; ++i) { if(mask_[j * size_ + (pos + i)] != 0) if (subWidth < j + 1) subWidth = j + 1; } } //if (subWidth < width_) - // LOG(info, "[data] sub-batch {} of {} wide batch has effective width of {}", pos / subSize, width_, subWidth); + // LOG(info, "[data] sub-batch {} of {} wide batch has effective width of {}", pos / targetSize, width_, subWidth); // create sub-batch - auto sb = New(size, subWidth, vocab_); + auto sb = New(subSize, subWidth, vocab_); size_t words = 0; for(size_t j = 0; j < subWidth; ++j) { - for(size_t i = 0; i < size; ++i) { - sb->data()[j * size + i] = indices_[j * size_ + (pos + i)]; - sb->mask()[j * size + i] = mask_[j * size_ + (pos + i)]; + for(size_t i = 0; i < subSize; ++i) { + sb->data()[j * subSize + i] = indices_[j * size_ + (pos + i)]; + sb->mask()[j * subSize + i] = mask_[j * size_ + (pos + i)]; if(mask_[j * size_ + (pos + i)] != 0) words++; @@ -263,8 +266,8 @@ class CorpusBatch : public Batch { size_t size() const override { return subBatches_[0]->batchSize(); } /** - * @brief The total number of words for the longest sentence in the batch plus - * one. Pass which=0 for source and -1 for target. + * @brief The total number of words in the batch (not counting masked-out words). + * Pass which=0 for source words and -1 for target words. */ size_t words(int which = 0) const override { return subBatches_[which >= 0 ? which @@ -349,25 +352,27 @@ class CorpusBatch : public Batch { } /** - * @brief Splits the batch into batches of equal size. + * @brief Splits the batch into batches of equal size (except for last). + * + * @param n number of sub-batches to split into * - * @param n number of splits + * @param sizeLimit Clip batch content to the first sizeLimit sentences in the batch * - * @return Vector of pointers to new batches. + * @return Vector of pointers to new sub-batches (or nullptrs where run out of sub-batches) * * @see marian::data::SubBatch::split(size_t n) */ - std::vector> split(size_t n) override { + std::vector> split(size_t n, size_t sizeLimit /*=SIZE_MAX*/) override { ABORT_IF(size() == 0, "Encoutered batch size of 0"); - std::vector>> subs; - // split each subbatch separately - for(auto subBatch : subBatches_) { - size_t i = 0; - for(auto splitSubBatch : subBatch->split(n)) { + std::vector>> subs; // [subBatchIndex][streamIndex] + // split each stream separately + for(auto batchStream : subBatches_) { + size_t i = 0; // index into split batch + for(auto splitSubBatch : batchStream->split(n, sizeLimit)) { if(subs.size() <= i) subs.resize(i + 1); - subs[i++].push_back(splitSubBatch); + subs[i++].push_back(splitSubBatch); // this forms tuples across streams } } diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h index 2152f850d..4a4f5ae45 100755 --- a/src/examples/mnist/dataset.h +++ b/src/examples/mnist/dataset.h @@ -63,7 +63,7 @@ class DataBatch : public Batch { void push_back(Input input) { inputs_.push_back(input); } - virtual std::vector> split(size_t /*n*/) override { ABORT("Not implemented"); } + virtual std::vector> split(size_t /*n*/, size_t /*sizeLimit*/) override { ABORT("Not implemented"); } Data& features() { return inputs_[0].data(); } diff --git a/src/functional/tmp.h b/src/functional/tmp.h old mode 100644 new mode 100755 index 720901bce..083836603 --- a/src/functional/tmp.h +++ b/src/functional/tmp.h @@ -81,6 +81,27 @@ struct FApply<4, Functor> { } }; +template +struct FApply<5, Functor> { + __HDI__ static float apply( + Functor functor, + functional::Array, 5>& in, + const functional::Array& indices) { + return functor(in[0][indices[0]], + in[1][indices[1]], + in[2][indices[2]], + in[3][indices[3]], + in[4][indices[4]]); + } + + __HDI__ static float apply( + Functor functor, + functional::Array, 5>& in, + int index) { + return functor(in[0][index], in[1][index], in[2][index], in[3][index], in[4][index]); + } +}; + template __HDI__ float apply(Functor functor, functional::Array, K>& in, diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index 815d90804..bf4f256d3 100755 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -63,7 +63,7 @@ class Tensors { tensors_->allocate(node->grad(), node->shape(), node->value_type()); } - void free(Tensor& tensor) { tensors_->free(tensor); } + void free(const Tensor& tensor) { tensors_->free(tensor); } // @TODO: get rid of this, not really used or can be done better Ptr allocator() { return tensors_->allocator(); } @@ -437,7 +437,7 @@ class ExpressionGraph : public std::enable_shared_from_this { tensors_->allocateBackward(node); } - void free(Tensor& tensor) { + void free(const Tensor& tensor) { if(tensors_) tensors_->free(tensor); } diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h old mode 100644 new mode 100755 index 6c0a11b39..6a887aa0a --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -66,9 +66,9 @@ Expr operator/(Expr a, float b); Expr logaddexp(Expr a, Expr b); -Expr max(Expr a, Expr b); // TODO: haggle over the name (max vs. elementMax) +Expr maximum(Expr a, Expr b); -Expr min(Expr a, Expr b); // TODO: haggle over the name +Expr minimum(Expr a, Expr b); Expr dot(Expr a, Expr b, diff --git a/src/graph/node_operators.h b/src/graph/node_operators.h old mode 100644 new mode 100755 index 018d05331..01c7f5904 --- a/src/graph/node_operators.h +++ b/src/graph/node_operators.h @@ -50,7 +50,7 @@ struct ParamNode : public Node { ~ParamNode() {} virtual size_t allocate() override { - ABORT_IF(!val_, "Parameters should be allocated by their graph"); + ABORT_IF(!val_, "Parameters should be allocated by their graph. Parameter {} was not", name_); return 0; } diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 7da85443b..0f972bc79 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -42,7 +42,7 @@ class DotNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(outShape.size() - 1, shapeB[shapeB.size() - 1]); ABORT_IF(shapeA[shapeA.size() - 1] != shapeB[shapeB.size() - 2], - "matrix product requires dimensions to match"); + "Matrix product requires dimensions to match"); return outShape; } @@ -165,7 +165,7 @@ class AffineNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(outShape.size() - 1, shapeB[shapeB.size() - 1]); ABORT_IF(shapeA[shapeA.size() - 1] != shapeB[shapeB.size() - 2], - "matrix product requires dimensions to match"); + "Matrix product requires dimensions to match"); return outShape; } @@ -309,7 +309,7 @@ class DotBatchedNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(-1, shapeB[-1]); ABORT_IF(shapeA[-1] != shapeB[-2], - "matrix product requires dimensions to match"); + "Batched matrix product requires dimensions to match"); return outShape; } diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h old mode 100644 new mode 100755 diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index d7a58be31..807135949 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -2,10 +2,12 @@ #include "common/io.h" #include "tensors/tensor_operators.h" +#include namespace marian { -void Sgd::updateImpl(Tensor params, Tensor grads) { +void Sgd::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { + actualMBSize, refMBSize; // (no correction for base update needed beyond using ce-sum) using namespace functional; Element(_1 -= eta_ * _2, params, @@ -14,9 +16,10 @@ void Sgd::updateImpl(Tensor params, Tensor grads) { params->getBackend()->synchronize(); } -// Aagrad +// Adagrad -void Adagrad::updateImpl(Tensor params, Tensor grads) { +void Adagrad::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { + ABORT_IF(actualMBSize != refMBSize, "Adagrad does not support rational hyper-parameter adjustment"); if(!alloc_) alloc_ = New(params->getBackend()); @@ -62,7 +65,7 @@ void Adagrad::load(const std::string& name, if(item.name == "adagrad_gt") { vGt.resize(totalSize); std::copy( - (float*)item.data(), (float*)item.data() + totalSize, vGt.begin()); + (float*)item.data(), ((float*)item.data()) + totalSize, vGt.begin()); } } if(vGt.empty()) { @@ -109,7 +112,7 @@ void Adagrad::save(const std::string& name, item.type = Type::float32; item.bytes.resize(vGt.size() * sizeOf(item.type)); std::copy( - (char*)vGt.data(), (char*)vGt.data() + vGt.size(), item.bytes.begin()); + (char*)vGt.data(), (char*)(vGt.data() + vGt.size()), item.bytes.begin()); io::saveItems(name, {item}); } @@ -121,7 +124,8 @@ void Adagrad::resetStats() { // Adam -void Adam::updateImpl(Tensor params, Tensor grads) { +void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { + // lazy allocation if(!alloc_) alloc_ = New(params->getBackend()); @@ -130,29 +134,42 @@ void Adam::updateImpl(Tensor params, Tensor grads) { alloc_->reserveExact(2 * params->memory()->size()); alloc_->allocate(mt_, {1, elements}); mt_->set(0.f); - alloc_->allocate(vt_, {1, elements}); vt_->set(0.f); } - t_++; - float denom1 = 1 - (float)std::pow(beta1_, t_); - float denom2 = 1 - (float)std::pow(beta2_, t_); + double Tref = (double)refMBSize; + double T = (double)actualMBSize; - using namespace functional; + // adjust for minibatch-size changes if Adam parameters are given a reference size (else do nothing) + double eta = eta_ * (T/Tref); + double beta1 = beta1_; + double beta2 = beta2_; + double decay = w_ ; - Element(_1 = (beta1_ * _1) + ((1 - beta1_) * _2), mt_, grads); - Element(_1 = (beta2_ * _1) + ((1 - beta2_) * (_2 * _2)), vt_, grads); + // denominators. At steady state: =1. This recursion does the same as the Adam beta correction term. + denom1_ = (beta1 * denom1_) + (1 - beta1); // momentum smoothing + denom2_ = (beta2 * denom2_) + (1 - beta2); // RMS normalization - Element(_1 -= eta_ // learning-rate: x_t = x_{t-1} - \eta * (...) - * ((_2 / denom1) // 1st moment: m_{t-1} - / (sqrt(_3 / denom2) + eps_) // 2nd moment: \sqrt(v_{t-1}) - + w_ * _1), // weight-decay: w * x_{t-1} - params, - mt_, - vt_); + LOG_ONCE(info, "[adam] First update: Tref = {}, T = {}, eta = {} -> {}, beta = {}, {}", Tref, T, eta_, eta, beta1, beta2); - params->getBackend()->synchronize(); + // numerators. Divide by T to convert ce-sum gradient to avg gradient. + using namespace functional; + Element(_1 = ((float)beta1 * _1) + float((1 - beta1) / T ) * _2, mt_, grads); // momentum smoothing. At steady state: =smoothed avg gradient + Element(_1 = ((float)beta2 * _1) + float((1 - beta2) / T / T) * (_2 * _2), vt_, grads); // RMS normalization. At steady state: =mean square of the avg gradients + + // apply Adam normalization + float etaf = (float)eta, denom1f = (float)denom1_, denom2f = (float)denom2_, decayf = (float)decay; // (get casts out of Element expression for readability) + Element(_1 -= etaf // learning-rate: x_t = x_{t-1} - \eta * (...) + * (( ( _2 / denom1f) // momentum-smoothed per-sample gradient: m_{t-1} + / (sqrt(_3 / denom2f) + eps_)) // normalize by RMS: \sqrt(v_{t-1}) + + decayf * _1), // weight-decay: w * x_{t-1} + params, // =_1 + mt_, // =_2 + vt_ // =_3 + ); + + params->getBackend()->synchronize(); // @TODO: This should not be in here. Maybe in the wrapper. Why is it needed at all? } void Adam::load(const std::string& name, @@ -168,6 +185,7 @@ void Adam::load(const std::string& name, std::vector vMt; std::vector vVt; + std::array vDenoms; auto items = io::loadItems(name); for(auto item : items) { @@ -178,12 +196,18 @@ void Adam::load(const std::string& name, if(item.name == "adam_mt") { vMt.resize(totalSize); std::copy( - (float*)item.data(), (float*)item.data() + totalSize, vMt.begin()); + (float*)item.data(), ((float*)item.data()) + totalSize, vMt.begin()); } - if(item.name == "adam_vt") { + else if(item.name == "adam_vt") { vVt.resize(totalSize); std::copy( - (float*)item.data(), (float*)item.data() + totalSize, vVt.begin()); + (float*)item.data(), ((float*)item.data()) + totalSize, vVt.begin()); + } + else if(item.name == "adam_denoms") { + ABORT_IF(totalSize != 2, "adam_denoms should have 2 entries"); + std::copy( + (double*)item.data(), ((double*)item.data()) + totalSize, vDenoms.begin()); + // Back compat note: Old files lacked "adam_denoms". For those, vDenoms will remain 0, which reproduces the old behavior. } } if(vMt.empty() || vVt.empty()) { @@ -212,6 +236,9 @@ void Adam::load(const std::string& name, auto opt = std::dynamic_pointer_cast(opts[id]); opt->vt_->set(std::vector(begin, end)); }); + + denom1_ = vDenoms[0]; + denom2_ = vDenoms[1]; //LOG(info, "done loading Adam params"); } @@ -248,7 +275,7 @@ void Adam::save(const std::string& name, itemMt.type = Type::float32; itemMt.bytes.resize(vMt.size() * sizeOf(itemMt.type)); std::copy( - (char*)vMt.data(), (char*)vMt.data() + vMt.size(), itemMt.bytes.begin()); + (char*)vMt.data(), (char*)(vMt.data() + vMt.size()), itemMt.bytes.begin()); io::Item itemVt; itemVt.name = "adam_vt"; @@ -256,9 +283,19 @@ void Adam::save(const std::string& name, itemVt.type = Type::float32; itemVt.bytes.resize(vVt.size() * sizeOf(itemVt.type)); std::copy( - (char*)vVt.data(), (char*)vVt.data() + vVt.size(), itemVt.bytes.begin()); + (char*)vVt.data(), (char*)(vVt.data() + vVt.size()), itemVt.bytes.begin()); + + // @TODO: this pattern is duplicated several times; refactor it + std::array vDenoms{denom1_, denom2_}; + io::Item itemDenoms; + itemDenoms.name = "adam_denoms"; + itemDenoms.shape = Shape({1, (int)vDenoms.size()}); + itemDenoms.type = Type::float64; + itemDenoms.bytes.resize(vDenoms.size() * sizeOf(itemDenoms.type)); + std::copy( + (char*)vDenoms.data(), (char*)(vDenoms.data() + vDenoms.size()), itemDenoms.bytes.begin()); - io::saveItems(name, {itemMt, itemVt}); + io::saveItems(name, {itemMt, itemVt, itemDenoms}); } void Adam::resetStats() { @@ -267,6 +304,9 @@ void Adam::resetStats() { if(vt_) vt_->set(0.f); + + denom1_ = 0; // @BUGBUG: or 1 or refMBSize if so specified. Fix once we have proper parameterization for that. + denom2_ = 0; } Ptr Optimizer(Ptr options) { @@ -287,7 +327,7 @@ Ptr Optimizer(Ptr options) { } else if(opt == "adagrad") { return Optimizer(lrate, clipper, params); } else if(opt == "adam") { - return Optimizer(lrate, clipper, params); + return Optimizer(lrate, clipper, params); // @TODO: parse the parameters here, or just pass the options object } else { ABORT("Unknown optimizer: {}", opt); } diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h index 9b83afb6d..76e2780eb 100755 --- a/src/optimizers/optimizers.h +++ b/src/optimizers/optimizers.h @@ -21,19 +21,29 @@ class OptimizerBase : public TrainingObserver { OptimizerBase(float eta, Ptr clipper = nullptr) : eta_(eta), clipper_(clipper) {} - void update(Ptr graph) { + static constexpr size_t mbSizeNotProvided = SIZE_MAX; + + void update(Ptr graph, size_t mbSize = mbSizeNotProvided) { Tensor p = graph->params()->vals(); Tensor g = graph->params()->grads(); - update(p, g); + update(p, g, mbSize); } - void update(Tensor params, Tensor grads) { + void update(Tensor params, Tensor grads, size_t mbSize = mbSizeNotProvided) { if(clipper_) clipper_->clip(grads); - // In case we want to add a multiply factor to our learning rate - updateImpl(params, grads); + size_t refMBSize = refMBSize_; + if (refMBSize == 0) { // optimizer not configured to use hyper-parameter auto-adjustment + refMBSize = mbSize = 1; // neutral settings that keep the standard behavior + } + else { // optimizer is configured to auto-adjust hyper-parameters + ABORT_IF(mbSize == mbSizeNotProvided, "Using rational optimizer auto-adjustment with trainer that does not provide MB size"); + // note: this behavior is only meaningful if using the ce-sum criterion + } + + updateImpl(params, grads, mbSize, refMBSize); } virtual void init(TrainingState& state) override { @@ -78,7 +88,7 @@ class OptimizerBase : public TrainingObserver { bool /*isMainProcess*/ = true) {} protected: - virtual void updateImpl(Tensor params, Tensor grads) = 0; + virtual void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) = 0; virtual void parseParams(const std::vector& params) = 0; virtual void resetStats() = 0; @@ -86,6 +96,8 @@ class OptimizerBase : public TrainingObserver { float eta_; // Clip gradient norm Ptr clipper_; + // Reference MB size. This enables automatic adjustment of optimizer hyper-parameters to MB size. + size_t refMBSize_{0}; // 0 means no adjustment }; /** @@ -97,7 +109,7 @@ class Sgd : public OptimizerBase { : OptimizerBase(eta, clipper) {} private: - void updateImpl(Tensor params, Tensor grads) override; + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; virtual void parseParams(const std::vector& /*params*/) override {} virtual void resetStats() override {} @@ -123,7 +135,7 @@ class Adagrad : public OptimizerBase { bool /*isMainProcess*/ = true) override; private: - void updateImpl(Tensor params, Tensor grads) override; + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; void resetStats() override; void parseParams(const std::vector& params) override { @@ -140,11 +152,13 @@ class Adagrad : public OptimizerBase { * @brief Adam optimizer * * https://arxiv.org/pdf/1412.6980v8.pdf + * + * with Frank's modifications for automatic hyper-parameter adjustment. */ class Adam : public OptimizerBase { public: Adam(float eta, Ptr clipper = nullptr) - : OptimizerBase(eta, clipper), t_(0) {} + : OptimizerBase(eta, clipper) {} void load(const std::string& name, const std::vector>& opts, @@ -156,9 +170,11 @@ class Adam : public OptimizerBase { bool isMainProcess = true) override; private: - void updateImpl(Tensor params, Tensor grads) override; + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; void resetStats() override; + // Adam parameters: + // [beta1, beta2, eps, w, refMBSize] virtual void parseParams(const std::vector& params) override { if(params.size() > 0) beta1_ = params[0]; @@ -169,15 +185,29 @@ class Adam : public OptimizerBase { // weighted decay for AdamW, to be explored, disabled by default if(params.size() > 3) - w_ = params[3]; + w_ = params[3]; // default (disabled): 0 + + // automatic learning-rate adjustment + // If users provide, in addition to the hyper-parameters, a reference minibatch size, + // that these hyper-parameters were originally tuned for, then the learning-rate gets + // adjusted accordingly. Note: Requires user to also use ce-sum criterion. + if(params.size() > 4) { + refMBSize_ = (size_t)params[4]; // default (disabled): 0 + LOG(info, "Note: Modified Adam optimizer: automatically adjusting learning rate as if minibatch size was {}", refMBSize_); + } } + // hyper-parameters float beta1_ = 0.9f; float beta2_ = 0.999f; float eps_ = 1e-8f; float w_ = 0.0f; - size_t t_; + // CPU-side running accumulators + double denom1_ = 0; + double denom2_ = 0; + + // GPU-side running accumulators Ptr alloc_; Tensor mt_; Tensor vt_; diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc old mode 100644 new mode 100755 index 50ae97936..96debcc2f --- a/src/tensors/gpu/element.inc +++ b/src/tensors/gpu/element.inc @@ -48,3 +48,9 @@ template void Element, BinaryFunctor, UnaryFunctor>>>>>>(Assign, UnaryFunctor>>>>>, marian::Tensor); template void Element, UnaryFunctor, Capture>>>, Capture>>>>>(Assign, UnaryFunctor, Capture>>>, Capture>>>>, marian::Tensor); template void Element, BinaryFunctor, Capture>, Capture>>>(Assign, BinaryFunctor, Capture>, Capture> >, marian::Tensor); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> > > >, Capture>, BinaryFunctor, Assignee<4> > > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> > > >, Capture>, BinaryFunctor, Assignee<4> > > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); diff --git a/src/tensors/tensor_allocator.h b/src/tensors/tensor_allocator.h index 3a2d99e60..d65b50e7c 100755 --- a/src/tensors/tensor_allocator.h +++ b/src/tensors/tensor_allocator.h @@ -74,7 +74,7 @@ class TensorAllocator { } } - void free(Tensor& t) { allocator_->free(t->memory()); } + void free(const Tensor& t) { allocator_->free(t->memory()); } Tensor asTensor() { auto mem = allocator_->memory(); diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp index e158fcb28..35835a0ed 100755 --- a/src/training/communicator.cpp +++ b/src/training/communicator.cpp @@ -72,7 +72,7 @@ class MPIWrapper : public IMPIWrapper public: MPIWrapper(bool multiThreaded) { - int requiredThreadingMode = multiThreaded ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE; + int requiredThreadingMode = multiThreaded ? MPI_THREAD_MULTIPLE : MPI_THREAD_FUNNELED; // FUNNELED means only one thread ever calls MPI int argc = 1; char* argv[] = { const_cast("this.exe") }; char** argvp = argv; // dummy argc/argv since MPI_Init needs something here int providedThreadingMode; @@ -124,6 +124,8 @@ class MPIWrapper : public IMPIWrapper HANDLE_MPI_ERROR(MPI_Recv(buf, (int)count, datatype, (int)sourceRank, tag, comm, status)); } virtual void allReduce(const void* sendbuf, void* recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) const override { + if (sendbuf == recvbuf) + sendbuf = MPI_IN_PLACE; // MSMPI requires this HANDLE_MPI_ERROR(MPI_Allreduce(sendbuf, recvbuf, (int)count, datatype, op, comm)); } virtual void finalize() override { diff --git a/src/training/communicator.h b/src/training/communicator.h index eef136f2d..7e705cfac 100755 --- a/src/training/communicator.h +++ b/src/training/communicator.h @@ -203,7 +203,6 @@ class DefaultCommunicator : public ICommunicator { void swapParams(const std::vector& paramShards) const override { // Update all graphs with parameter shard - auto gather = [this, paramShards](size_t idx, size_t begin, size_t end) { ABORT_IF(end - begin != paramShards[idx]->size(), "inconsistent shard size (swapParams, [{}], {} vs {})??", idx, end-begin, paramShards[idx]->size()); // Copy parameter shard to each graph, apart from last graph diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h index bf1f07dfc..09b70f3e6 100755 --- a/src/training/communicator_nccl.h +++ b/src/training/communicator_nccl.h @@ -4,10 +4,11 @@ #include "3rd_party/threadpool.h" #include "tensors/gpu/cuda_helpers.h" +#include "common/timer.h" + // Generated by NCCL make files in build/nccl/include; // include dir has been set in CMake files. NCCL add version number etc. #include "nccl.h" - #include #if (NCCL_MAJOR<3 || NCCL_MINOR<2) @@ -44,6 +45,14 @@ class NCCLCommunicator : public ICommunicator { } } + void synchronizeAllOnNullStream() const { + for (int i = 0; i < graphs_.size(); ++i) { + auto backend = graphs_[i]->params()->vals()->getBackend(); + backend->setDevice(); + backend->synchronize(); // note: synchronize() does not set the device by itself + } + } + std::string mpiIdStr() const { // (for logging) return mpi_ ? mpi_->idStr() : ""; } @@ -150,6 +159,16 @@ class NCCLCommunicator : public ICommunicator { CUDA_CHECK(cudaStreamCreate(&streams_[i])); } + // Note: due to a bug in NCCL 2.3.5, NCCL's allocation of shared memory intermittently fails with + // Failed, NCCL error 2 'unhandled system error' - ncclGroupEnd() + // include/shm.h:26 NCCL WARN Unable to allocate shared memory (4263936 bytes) : Interrupted system call + // This is caused by SIGPROF signals being raised, causing EINTR, which NCCL does not handle. + // Reported as Issue #137 on the NCCL Github, and supposedly fixed for 2.3.7 (to be verified). + // To work around, we disable the SIGPROF signal during NCCL initialization. +#define SIG_BAD 27 // SIGPROF + BlockSignal blockThread(SIG_BAD, pthread_sigmask); // Note: I don't know yet which of these two makes the difference. + BlockSignal blockProc(SIG_BAD, sigprocmask); // So for now just block both. + // set up NCCL // Since we want to use MPI, we cannot use NCCL's handy convenience function. Instead, we must go the laborious route. // cf. https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/index.html#multidevprothrd @@ -160,35 +179,19 @@ class NCCLCommunicator : public ICommunicator { NCCL_CHECK(ncclGetUniqueId(&uniqueId)); if (mpi_) { - //LOG(info, "[{}] before bcast", mpiIdStr()); static_assert(sizeof(uniqueId) == NCCL_UNIQUE_ID_BYTES, "wrong NCCL_UNIQUE_ID_BYTES??"); // (this value is used in NVidia examples) mpi_->bCast(&uniqueId, sizeof(uniqueId), MPI_BYTE, 0); - //LOG(info, "[{}] after bcast", mpiIdStr()); } - //mpiBarrier(); // should not be needed since bCast is a barrier - - // Note: due to a bug in NCCL 2.3.5, NCCL's allocation of shared memory intermittently fails with - // Failed, NCCL error 2 'unhandled system error' - ncclGroupEnd() - // include/shm.h:26 NCCL WARN Unable to allocate shared memory (4263936 bytes) : Interrupted system call - // This is caused by SIGPROF signals being raised, causing EINTR, which NCCL does not handle. - // Reported as Issue #137 on the NCCL Github. - // To work around, we disable the SIGPROF signal during NCCL initialization. -#define SIG_BAD 27 // SIGPROF - BlockSignal blockThread(SIG_BAD, pthread_sigmask); // Note: I don't know yet which of these two makes the difference. - BlockSignal blockProc(SIG_BAD, sigprocmask); // So for now just block both. - groupStart(); for (int localDeviceIndex = 0; localDeviceIndex < devices_.size(); localDeviceIndex++) { CUDA_CHECK(cudaSetDevice(devices_[localDeviceIndex])); - //LOG(info, "[{}] ncclCommInitRank {} out of {}: GPU[{}]", mpiIdStr(), myNcclRank(localDeviceIndex), numNcclRanks(), localDeviceIndex); NCCL_CHECK(ncclCommInitRank(&comms_[localDeviceIndex], numNcclRanks(), uniqueId, myNcclRank(localDeviceIndex))); - //LOG(info, "[{}] done ncclCommInitRank {} out of {}, GPU[{}]", mpiIdStr(), myNcclRank(localDeviceIndex), numNcclRanks(), localDeviceIndex); } groupEnd(); mpiBarrier(); // (synchronize the log messages) - LOG(debug, "NCCLCommunicator constructed successfully for {}", mpiIdStr()); + LOG(info, "NCCLCommunicator constructed successfully."); mpiBarrier(); // (synchronize the log messages) } @@ -206,61 +209,46 @@ class NCCLCommunicator : public ICommunicator { for(size_t i = 0; i < graphs_.size(); ++i) { size_t begin, end; std::tie (begin, end) = localShardRange(i); - //std::cerr << "[" << mpiIdStr() << "] foreach " << begin << " " << end << std::endl; -try{ if (parallel) threadResults_[i] = threadPool_.enqueue(func, i, begin, end); - //group.emplace_back(func, i, begin, end); - //threadPool_.enqueue([&](size_t i){ - // func(i, begin, end); - //}, i); else func(i, begin, end); -} -catch (const std::exception& e) // something leaks thread handles -{ - // keeping this around, in case the error still happens --@TODO: remove once this has not been observed anymore - LOG(info, "caught exception in foreach {}", i); - system("ps -T -A"); - throw; -} } if (parallel) for(size_t i = 0; i < graphs_.size(); ++i) threadResults_[i].wait(); - //for(auto& t : group) // (note: group is empty is not parallel) - // t.join(); } void scatterReduce() const override { + synchronizeAllOnNullStream(); + groupStart(); for(int i = 0; i < graphs_.size(); ++i) { size_t begin, end; std::tie (begin, end) = localShardRange(i); - //std::cerr << "[" << mpiIdStr() << "] scatterReduce " << begin << " " << end << std::endl; auto grads = graphs_[i]->params()->grads(); const auto* sendbuf = grads->data(); auto* recvbuf = grads->subtensor(begin, end-begin)->data(); size_t bufsize = shardSize(); + ABORT_IF(grads->subtensor(begin, end-begin)->size() != bufsize, "unexpected subtensor size??"); NCCL_CHECK(ncclReduceScatter(sendbuf, recvbuf, bufsize, ncclFloat, ncclSum, comms_[i], streams_[i])); } groupEnd(); - //std::cerr << "scatterReduce submitted" << std::endl; synchronizeAll(); - //std::cerr << "scatterReduce completed" << std::endl; } // This distributes all 64 model shards to all 64 GPUs. // @TODO: For unknown reasons, this takes longer than any other operation incl. scatterReduce(). // But both should have the same number of data transfers of the same size. void allGather() const override { + synchronizeAllOnNullStream(); + groupStart(); for(int i = 0; i < graphs_.size(); ++i) { size_t begin, end; std::tie (begin, end) = localShardRange(i); - //std::cerr << "[" << mpiIdStr() << "] allGather " << begin << " " << end << std::endl; auto vals = graphs_[i]->params()->vals(); const auto* sendbuf = vals->subtensor(begin, end-begin)->data(); @@ -281,14 +269,12 @@ catch (const std::exception& e) // something leaks thread handles auto distributedParams = gatherState([&](size_t localDeviceIndex) { std::vector tmp; distributedParamShards[localDeviceIndex]->get(tmp); - //LOG(info, "[{}] swapParams.getFn({}) -> size {}, ({}, {}, {}, ...)", mpiIdStr(), localDeviceIndex, tmp.size(), tmp[0], tmp[1], tmp[2]); return tmp; }); // Now all MPI processes hold an identical copy of a concatenation of all distributedParamShards[] across local and remote devices. std::vector localParams; graphs_[0]->params()->vals()->get(localParams); // Now all MPI processes hold an identical copy of params() (remember, we assumed all devices hold the same params()). - //LOG(info, "[{}] swapParams: distributedParams.size = {}, localParams.size = {}", mpiIdStr(), distributedParams.size(), localParams.size()); ABORT_IF(distributedParams.size() != localParams.size(), "distributed sharded and local params have different size??"); // swap @@ -331,7 +317,6 @@ catch (const std::exception& e) // something leaks thread handles tmp = getFn(localDeviceIndex); localData.insert(localData.end(), tmp.begin(), tmp.end()); } - //LOG(info, "[{}] gatherState: localData.size = {}", mpiIdStr(), localData.size()); // second, concatenate across MPI processes // Note that all local devices occupy consecutive ncclRanks in order. std::vector data; diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h index bc2f2761d..fb1d3b4ac 100755 --- a/src/training/exponential_smoothing.h +++ b/src/training/exponential_smoothing.h @@ -3,6 +3,7 @@ #include "common/definitions.h" #include "functional/functional.h" #include "tensors/tensor_operators.h" +#include "optimizers/optimizers.h" namespace marian { @@ -12,18 +13,33 @@ namespace marian { */ class ExponentialSmoothing { public: - ExponentialSmoothing(float decay = 0.0f) - : mvAvg_{decay > 0}, mvDecay_{decay} {} + ExponentialSmoothing(Ptr options) { + auto args = options->get>("exponential-smoothing"); + ABORT_IF(args.size() < 1 || args.size() > 2, "exponential-smoothing parameter must be one or two numbers"); + mvDecayBy_ = args[0]; + if (args.size() > 1) + refBatchTrgWords_ = (size_t)args[1]; + mvAvg_ = (mvDecayBy_ > 0); + } protected: - void updateAvgParams(Tensor paramsAvg, Tensor params, size_t batches) { + void updateAvgParams(Tensor paramsAvg, Tensor params, size_t batches, size_t actualBatchTrgWords = OptimizerBase::mbSizeNotProvided) { + double beta = 1. - mvDecayBy_; + // correction term if batch size is different from what mvDecayBy_ was specified for + if (refBatchTrgWords_) { + ABORT_IF(actualBatchTrgWords == OptimizerBase::mbSizeNotProvided, + "This graph-group type does not support reference batch size specification for exponential-smoothing"); + beta = pow(beta, (double)actualBatchTrgWords / (double)refBatchTrgWords_); + } + // reduce effect of decay parameter in early training stages + float decayBy = std::max(1.f - (float)beta, + 1.f - (float)(batches + 1) / (float)(batches + 10)); using namespace functional; - float decay = std::max(mvDecay_, - 1.f - (float)(batches + 1) / (float)(batches + 10)); - Element(_1 = ((1.f - decay) * _1) + (decay * _2), paramsAvg, params); + Element(_1 = ((1.f - decayBy) * _1) + (decayBy * _2), paramsAvg, params); } bool mvAvg_{false}; - float mvDecay_{1e-4f}; + float mvDecayBy_{1e-4f}; // decay prior model by this factor + size_t refBatchTrgWords_{0}; // mvDecayBy_ is specified for this batch size (in target words) }; } // namespace marian diff --git a/src/training/graph_group.h b/src/training/graph_group.h index fc372adce..46a317c32 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -33,6 +33,10 @@ class GraphGroup { virtual void save(bool isFinal = false) = 0; + void validate() { + ABORT_IF(finalized_, "Training has already finished."); + } + virtual void finalize() { finalized_ = true; } @@ -48,6 +52,7 @@ class GraphGroup { * The actual allowed size is then determined by multiplying it with the * number of devices, which is passed in as the 'multiplier'. */ + // @TODO: Can this be made const? It seems wrong to have a stateful method that still returns a result. virtual Ptr collectStats(Ptr graph, Ptr model, size_t multiplier = 1) { @@ -194,10 +199,8 @@ class MultiNodeGraphGroupBase : public GraphGroup { } virtual void finalize() override { - if (mpi_) { + if (mpi_) finalizeMPI(std::move(mpi_)); - ABORT_IF(mpi_, "MPI not finalized??"); - } Base::finalize(); } }; diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 1d0042019..b272a4b7f 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -7,7 +7,7 @@ namespace marian { AsyncGraphGroup::AsyncGraphGroup(Ptr config) : GraphGroup(config), - ExponentialSmoothing{options_->get("exponential-smoothing")}, + ExponentialSmoothing(options_), devices_{Config::getDevices(options_)}, shardSync_(devices_.size()), optimizerDelay_{options_->get("optimizer-delay")} { diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h index d3af0f22b..713e30064 100755 --- a/src/training/graph_group_async.h +++ b/src/training/graph_group_async.h @@ -55,7 +55,7 @@ class AsyncGraphGroup : public GraphGroup, public ExponentialSmoothing { AsyncGraphGroup(Ptr config); void update(Ptr batch) override { - ABORT_IF(finalized_, "Training has already finished"); + validate(); execute(batch); } diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index c86225ebc..39a20e062 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -376,7 +376,7 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { * Update any client model with given batch if batch is assigned to this node. */ void update(Ptr batch) override { - ABORT_IF(finalized_, "Training has already finished"); + validate(); // Only take batch assigned to this node if(batchIter_ % mpi_->numMPIProcesses() == (size_t)mpi_->myMPIRank()) { execute(batch); diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index bfef2050c..7685b6789 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -143,7 +143,7 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { * Update any client model with given batch if batch is assigned to this node. */ void update(Ptr batch) override { - ABORT_IF(finalized_, "Training has already finished"); + validate(); if(batchIter_ % mpi_->numMPIProcesses() == mpi_->myMPIRank()) { // Only take batch assigned to this node execute(batch); } diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 1eb589d91..9e417b7d6 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -25,7 +25,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { public: SingletonGraph(Ptr config) : GraphGroup(config), - ExponentialSmoothing(options_->get("exponential-smoothing")) { + ExponentialSmoothing(config) { // Get device ID auto devices = Config::getDevices(options_); ABORT_IF(devices.size() != 1, "Only one device ID should be provided for singleton training"); @@ -40,7 +40,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { } void update(Ptr batch) override { - ABORT_IF(finalized_, "Training has already finished"); + validate(); execute(batch); } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 6699484c1..cedf5c54e 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -4,7 +4,7 @@ namespace marian { SyncGraphGroup::SyncGraphGroup(Ptr config) : GraphGroup(config), - ExponentialSmoothing{options_->get("exponential-smoothing")}, + ExponentialSmoothing(config), delay_{options_->get("optimizer-delay")} { // @TODO: rename to something else; delay means delayed updated, not accumulation mpi_ = initMPI(/*multiThreaded=*/false); // when not running under MPI, this will be a fake object that represents a one-MPI-process setup @@ -25,9 +25,16 @@ SyncGraphGroup::SyncGraphGroup(Ptr config) // This part of the code will not special-case any of this here. // Rather, it is assumed that the communicator knows to reduce unnecessary transfers to no-ops. comm_ = createCommunicator(graphs_, /*noNccl=*/options_->get("no-nccl", false), /*mpi=*/mpi_); + + auto type = utils::toUpper(devices_.front().typeAsString()) + "s"; + if (mpi_->numMPIProcesses() > 1) + LOG(info, "[training] Using {} {}, distributed over {} MPI processes", mpi_->numMPIProcesses() * devices_.size(), type, mpi_->numMPIProcesses()); + else + LOG(info, "[training] Using {} {}", devices_.size(), type); } void SyncGraphGroup::setScheduler(Ptr scheduler) /*override*/ { + validate(); scheduler_ = scheduler; // optimizer has to be registered last to see changes of learning rate // @TODO: ^^Fix this comment. Either it refers to the scheduler, or it should be moved. Which one? @@ -101,31 +108,144 @@ void SyncGraphGroup::initializeAvg() { } Ptr SyncGraphGroup::collectStats() { - // @TODO: This should only run on MPI process 0. Also we can share vv this vv expression with update(). - size_t multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_; - return GraphGroup::collectStats(graphs_[0], builders_[0], multiplier); + // @TODO: This is an incompatible change. Decide how to handle that. + //size_t multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_; + return GraphGroup::collectStats(graphs_[0], builders_[0]/*, multiplier*/); +} + +// helper for MB scaling: quantize the ratio with a given error margin +static double roundUpRatio(double ratio) { + if (ratio == 0) + return ratio; + // find largest power of two that fits into ratio + double p = 1; + while (p*2 < ratio) + p *= 2; + // round up to nearest multiple of a largest power of 2 where relative error is within margin + // 25% error margin seems acceptable: + // - using a 25% larger MB size should not break convergence + // - @TODO: not using the first 25% of the next block is OK since those are dominated by data exchange + double maxError = 0.25; + while (p >= 1) { + double proposedRatio = ceil(ratio / p) * p; + double error = (proposedRatio - ratio) / ratio; + if (fabs(error) <= maxError) + return proposedRatio; + p /= 2; + } + return ratio; +} + +// helper routine that handles accumulation and load-balancing of sub-batches to fill all devices +// It adds 'newBatch' to 'pendingBatches_', and if sufficient batches have been queued, then +// returns 'pendingBatches_' in 'subBatches' and resets it. If not, it returns false. +bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector>& subBatches) { + pendingBatches_.push_back(newBatch); + size_t warpSize = devices_.size() * mpi_->numMPIProcesses(); // warp := set of batches processed concurrently across GPus and workers + + size_t pendingTrgWords = 0; // diagnosics only: compute how many target labels are pending so far + for (const auto& batch : pendingBatches_) + pendingTrgWords += batch->wordsTrg(); + + // MB-size warm-up and dynamic scaling + double ratio; + bool isDynamic = scheduler_->tryGetDynamicMBSizeMultiplier(ratio); + if (isDynamic) + ratio = roundUpRatio(ratio); // round up to full batches if within a certain error margin --@BUGBUG: Not invariant w.r.t. GPU size, as ratio is relative to what fits into 1 GPU + else // if dynamic scaling not enabled, then fill each GPU with a batch + ratio = (double)(delay_ * warpSize); + if (pendingBatches_.size() < ratio) + return false; // not enough data yet + + // now we have enough to fill at least 'ratio' batches + if (pendingBatches_.size() == ratio) + return true; // nothing to do, e.g. warm-up not enabled + + // warm-up is happening + LOG_ONCE(info, "[training] Mini-batch-warmup enabled"); + + // shorten all batches a little to accurately reflect ratio + // e.g. ratio = 3.3 for 4 batches: Reduce each by 3.3/4 + // Alternatively, we could just shorten the last 'warp', but that would not be invariant to warp size. + size_t before = 0, after = 0; + for (auto& batch : pendingBatches_) { + auto reducedBatchSize = (size_t)ceil((double)batch->size() * ratio / (double)pendingBatches_.size()); + size_t minSize = 1; + if (pendingBatches_.size() == 1) { // enforce a minimum (only needed/correct if still in first batch) + size_t minTrgWords = 256; // don't go below this number of target words, as it seems excessive --@TODO: parameterize? + minSize = 1 + (minTrgWords * batch->size() - 1) / batch->wordsTrg(); // approximately convert minTrgWords into a #sentences + } + reducedBatchSize = std::max(reducedBatchSize, minSize); + before += batch->wordsTrg(); + if (reducedBatchSize < batch->size()) + batch = batch->split(/*numSubBatches=*/1, reducedBatchSize).front(); + after += batch->wordsTrg(); + } + + // load-balance: distribute the last numWarps-group's batches over GPUs + // This is tricky since batches do not have the same length, therefore we can only split, but not merge. + auto numWarps = (pendingBatches_.size() - 1) / warpSize + 1; // = ceil(#buffers / (#GPUs * #workers)) + auto availableBatches = numWarps * warpSize; // we got this many GPUs anyways, so we better make use of them + if (pendingBatches_.size() < availableBatches) { + // we are not using all available GPUs -> try to load-balance a bit better + auto fullBatches = (numWarps - 1) * warpSize; + auto expandLast = pendingBatches_.size() - fullBatches; + auto toLast = availableBatches - fullBatches; + LOG(info, "attempt to redistribute {} last batches over {}", expandLast, toLast); + auto splitInto = toLast / expandLast; // unfortunately we can only split in integer ratios + // @TODO: We can do better since the last batch is typically smaller. + if (splitInto > 1) { + // split each of last numWarps's batches into 'splitInto' batches + // pop them first + std::vector> batchesToSplit; + while (pendingBatches_.size() > fullBatches) { + batchesToSplit.push_back(pendingBatches_.back()); + pendingBatches_.pop_back(); + } + // now split them + for (auto& batchToSplit : batchesToSplit) { + LOG(info, "{}-way splitting batchToSplit with size {}", splitInto, batchToSplit->size()); + auto splitBatches = batchToSplit->split(splitInto); + for (auto& splitBatch : splitBatches) { + LOG(info, " -> getting batchToSplit with size {}", splitBatch->size()); + pendingBatches_.push_back(splitBatch); + } + } + } + ABORT_IF(pendingBatches_.size() > availableBatches, "somehow split into too many batches??"); + } + subBatches = std::move(pendingBatches_); + + // @TODO: sort by width, so that in case of delay > 1, each GPU gets about the same size + return true; } -void SyncGraphGroup::update(Ptr batch) /*override*/ { - ABORT_IF(finalized_, "Training has already finished"); +void SyncGraphGroup::update(Ptr newBatch) /*override*/ { + validate(); - // distribute the batch over (delay, local device, MPI rank) - size_t numSubBatches = delay_ * devices_.size() * mpi_->numMPIProcesses(); - auto subBatches = batch->split(numSubBatches); - subBatches.resize(numSubBatches); // pad with nullptrs if out of data + std::vector> subBatches; + bool gotSubBatches = tryGetSubBatches(newBatch, subBatches); + + // not enough data yet: return right away + if (!gotSubBatches) + return; // Helper to access the subBatches array - auto getSubBatch = [&](size_t t, size_t localDeviceIndex, size_t rank) { + auto getSubBatch = [&](size_t t, size_t localDeviceIndex, size_t rank) -> Ptr { // 't' (the delay) should be slowest changing dimension. If subBatches are sorted by // length, then grouping sentences of similar length into the same delay step can // reduce unnecessary time spent in padding. - return subBatches[(t * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex]; + auto index = (t * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex; + if (index < subBatches.size()) + return subBatches[index]; + else + return nullptr; }; // Upon very first execution, reset everything if(first_) { - LOG(debug, "[{}] Processing first minibatch. Batches are processed as {} processes x {} GPUs/process x {} delay steps", - mpi_->idStr(), mpi_->numMPIProcesses(), devices_.size(), delay_); + LOG(info, "[training] Processing first minibatch. Batches are processed as {} processes x {} GPUs/process", + mpi_->numMPIProcesses(), devices_.size()); initialize(subBatches.front()); if(mvAvg_ && paramsAvg_.empty()) initializeAvg(); @@ -133,33 +253,34 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { } // Compute gradients - // This happens in multiple steps in case of delay_ > 1. + // This happens in multiple steps in case of delay > 1. std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device - for (size_t t = 0; t < delay_; t++) { + for (size_t t = 0; getSubBatch(t, 0, 0); t++) { // @TODO: rename 't' to 'delay' // Execute single forward/backward step auto forwardBackward = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { auto graph = graphs_[localDeviceIndex]; auto subBatch = getSubBatch(t, localDeviceIndex, mpi_->myMPIRank()); if(subBatch) { - timer::Timer timer; auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); - //LOG(info, timer.format(2, "after build: %ws")); graph->forward(); - //LOG(info, timer.format(2, "after forward (no sync): %ws")); localDeviceCosts[localDeviceIndex] += costNode->scalar(); graph->backward(/*zero=*/t == 0); // only reset gradients to 0 if t = 0 - //LOG(info, timer.format(2, "after backward (no sync): %ws")); - //localDeviceCosts[localDeviceIndex] += costNode->scalar(); // moved here for time measurements; @TODO: move this back - //LOG(info, timer.format(2, "after scalar() (that's a sync): %ws")); } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets +#if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead + graph->params()->allocateBackward(); + if (t == 0) // these have already been sized + graph->params()->set_zero_adjoint(); +#else + graph->clear(); // instead of build() graph->forward(); graph->backward(/*zero=*/t == 0); +#endif } }; - comm_->foreach(forwardBackward); // compute gradients in parallel on each device. Aggregate if delay_ > 1. + comm_->foreach(forwardBackward); // compute gradients in parallel on each device. Aggregate if delay > 1. } // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. @@ -177,23 +298,25 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { Element(_1 = _1 / (float)div, curGrad); } + // determine num words for dynamic hyper-parameter adjustment + size_t mbWords = OptimizerBase::mbSizeNotProvided; + if (options_->get("cost-type") == "ce-sum") { // presently only supported for ce-sum + mbWords = 0; + for (const auto& batch : subBatches) + mbWords += batch->words(-1); // @TODO: use wordsTrg (it's the same) + } + // actual model update - shardOpt_[idx]->update(curParam, curGrad); + shardOpt_[idx]->update(curParam, curGrad, mbWords); if(mvAvg_) updateAvgParams( - paramsAvg_[idx], curParam, scheduler_->numberOfBatches()); + paramsAvg_[idx], curParam, scheduler_->numberOfBatches(), mbWords); }; - timer::Timer timer; comm_->scatterReduce(); // reduce gradients across all devices (globally) into shards - //LOG(info, timer.format(2, "after scatterReduce (has sync): %ws")); comm_->foreach(update); // per-shard model-update - //LOG(info, timer.format(2, "after model update (no sync): %ws")); - //graphs_.front()->getBackend()->synchronize(); // @TODO: This is strictly for time measurement. Make sure it doesn't accidentally stay in here!! - //LOG(info, timer.format(2, "after model update sync (which is unnecessary except for time measurements): %ws")); comm_->allGather(); // distribute param value shards back - //LOG(info, timer.format(2, "after allGather (has sync): %ws")); // cost across all local devices (scheduler will aggregate cross-process) float localCost = 0; @@ -202,7 +325,7 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { // if localCost is average-based, we need to turn the sum over devices into an average as well if(options_->get("cost-type") != "ce-sum") - localCost /= numSubBatches; + localCost /= subBatches.size(); if(scheduler_) { // track and log localCost @@ -224,6 +347,7 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { } void SyncGraphGroup::load() /*override*/ { + validate(); // This function loads the main parameters in the graphs. // In case of exponential smoothing, we also need to restore paramsAvg_. @@ -253,10 +377,11 @@ void SyncGraphGroup::load() /*override*/ { [&](const std::vector& optimizerStateVector, const OptimizerBase::ScatterStateSetFunc& setShardFn) { comm_->scatterState(optimizerStateVector, setShardFn); }); + LOG(info, "[training] Model reloaded from {}", name); } else if(options_->has("pretrained-model")) { std::string nameInit = options_->get("pretrained-model"); LOG(info, - "Initialize model weights with the pre-trained model {}", + "[training] Initializing model weights with the pre-trained model {}", nameInit); size_t i = 0; @@ -267,44 +392,34 @@ void SyncGraphGroup::load() /*override*/ { } void SyncGraphGroup::save(bool final) /*override*/ { + validate(); barrier(); // (for better grouping of log messages) - //LOG(info, "[{}] save() line {}!", this->mpi_->idStr(), __LINE__); // do final validation if(final && scheduler_) { // bring the smoothed model in // Note that it is sharded. For multi-node, it is sharded over multiple machines, so this is a network access. // Also note that the swap must run on all MPI processes concurrently, although only one actually validates. - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); swapParamsAvg(); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); if (isMainProcess()) // in multi-node, only first MPI process saves the model (they are all identical) scheduler_->validate(graphs_, true); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); swapParamsAvg(); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); } std::string name = options_->get("model"); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); barrier(); // (for better grouping of log messages) - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); // if smoothing then save original (unsmoothed) parameters as well - // @TODO: Check whether we are reloading the correct file (the unsmoothed one). if(mvAvg_ && paramsAvg_.size() > 0 && isMainProcess()) // only save from one MPI process // Save the original parameters in model.npz.orig.npz builders_[0]->save(graphs_[0], name + ".orig.npz", true); // Temporarily switch to the averaged parameters // Note: the smoothed model is sharded across GPUs, and across MPI processes if applicable. This brings it into MPI process[*].device[*] - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); swapParamsAvg(); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); // save main model file if (isMainProcess()) { // only save from one MPI process // if not overwrite then save a copy with number of updates in the model pathname - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); if(!options_->get("overwrite") && !final) { std::string numberOfBatches = scheduler_ ? std::to_string(scheduler_->numberOfBatches()) @@ -313,40 +428,34 @@ void SyncGraphGroup::save(bool final) /*override*/ { nameOverwrite.replace(name.size() - 4, 4, ".iter" + numberOfBatches + ".npz"); // @TODO: use insert? builders_[0]->save(graphs_[0], nameOverwrite); } - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); // save main model file builders_[0]->save(graphs_[0], name, true); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); // save scheduler-related state if (scheduler_) scheduler_->save(name); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); } // Switch back to the original parameters - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); swapParamsAvg(); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); -#if 0 // temporary, for testing of saving distributed models; must be identical to .orig.npz - if(mvAvg_ && paramsAvg_.size() > 0 && isMainProcess()) - builders_[0]->save(graphs_[0], name + ".orig_after_swapping.npz", true); -#endif - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); barrier(); // (for better grouping of log messages) - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); // persist optimizer state - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); + LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); shardOpt_[0]->save(name + ".optimizer.npz", shardOpt_, [&](const OptimizerBase::GatherStateGetFunc& getShardFn) { return comm_->gatherState(getShardFn); }, isMainProcess()); - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); + LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); barrier(); // (for better grouping of log messages) - //LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); +} + +void SyncGraphGroup::finalize() /*override*/ { + validate(); + finalizeMPI(std::move(mpi_)); + Base::finalize(); } } // namespace marian diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 478166996..5d4b11912 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -7,6 +7,7 @@ namespace marian { class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { + using Base = GraphGroup; const size_t delay_{ 1 }; // optimizer-delay parameter Ptr comm_; // [not null] communicator, e.g. NCCLCommunicator @@ -23,7 +24,10 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { std::vector> paramsAllocs_; // [deviceIndex] we must hold a reference to the memory until this class dies // @TODO: move this nto ExponentialSmoothing, together with paramsAvg_? - bool first_{ true }; // gets interpreted and cleared by update() + // state for update() + bool first_{ true }; // gets interpreted and cleared by update() + std::vector> pendingBatches_; // in case of delay, multi-worker, and/or multi-GPU, we buffer up batches + size_t typicalTrgWords_{}; // typical batch size in words (labels); remembered from collectStats() void initialize(const Ptr& exampleBatch); void initializeAvg(); @@ -32,6 +36,8 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void barrier() const { mpi_->barrier(); } // (we need this several times) void swapParamsAvg() { if (mvAvg_ && paramsAvg_.size() > 0) comm_->swapParams(paramsAvg_); } // note: must call this on all MPI ranks in parallel + bool tryGetSubBatches(Ptr newBatch, std::vector>& subBatches); + public: SyncGraphGroup(Ptr config); @@ -42,6 +48,8 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void load() override; void save(bool final = false) override; + void finalize() override; + Ptr collectStats(); // @TODO: consider to make this a virtual as well? Currently it is a template dispatch }; diff --git a/src/training/scheduler.h b/src/training/scheduler.h index dee62496d..783845c8c 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -13,12 +13,34 @@ class Scheduler : public TrainingObserver { std::vector> validators_; bool first_{true}; + size_t typicalTrgBatchWords_{0}; // for dynamic batch sizing Ptr state_; timer::Timer timer_, heartBeatTimer_; - float getLearningRate(TrainingState& state) { + // determine LR decay factor from --lr-decay-inv-sqrt option + float getLearningRateDecayFactor(const TrainingState& state) const { + auto args = options_->get>("lr-decay-inv-sqrt"); + ABORT_IF(args.empty() || args.size() > 2, "--lr-decay-inv-sqrt argument must be one or two numbers with units"); + auto decayGoogle = SchedulingParameter::parse(args[0]); + size_t progress = state.getProgressIn(decayGoogle.unit); + size_t start = decayGoogle.n; + if (args.size() > 1) { + auto decayStart = SchedulingParameter::parse(args[1]); + ABORT_IF(decayStart && decayStart.unit != decayGoogle.unit, "both --lr-decay-inv-sqrt arguments must have the same unit"); + start = decayStart.n; + } + if (decayGoogle && progress > start) { + progress = progress - start + decayGoogle.n; // shift so that we get 1 at progress==start + return (float)(std::sqrt((double)decayGoogle.n / (double)progress)); + } + else + return 1.f; + } + + // determine the dynamically adjusted learning rate, incl. warm-up and decay + float getLearningRate(const TrainingState& state) const { float baselr = options_->get("learn-rate"); float mult1 = 1.f; @@ -29,11 +51,7 @@ class Scheduler : public TrainingObserver { mult1 = std::min(1.f, (float)bno / (float)warmup.n); } - float mult2 = 1.f; - auto decayGoogle = SchedulingParameter::parse(options_->get("lr-decay-inv-sqrt")); - if(decayGoogle) { - mult2 = std::min(1.f, (float)(std::sqrt(decayGoogle.n) / std::sqrt(state.getProgressIn(decayGoogle.unit)))); - } + float mult2 = getLearningRateDecayFactor(state); baselr = baselr * mult1 * mult2; @@ -45,6 +63,54 @@ class Scheduler : public TrainingObserver { } public: + void setTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for tryGetDynamicMBSizeMultiplier() + typicalTrgBatchWords_ = typicalTrgBatchWords; + LOG(info, "batch size estimate is {} target words", typicalTrgBatchWords_); + } + + // determine dynamic MB size, if respective parameters are given (return false if not) + bool tryGetDynamicMBSizeMultiplier(double /*out*/ &ratio) const { + auto mbWarmupOpts = options_->get>("mini-batch-warmup"); + ABORT_IF(mbWarmupOpts.empty() || mbWarmupOpts.size() > 2, "--mini-batch-warmup argument must be one or two numbers with units"); + auto mbWarmup = SchedulingParameter::parse(mbWarmupOpts[0]); + if (!mbWarmup) + return false; + + ratio = 1.0; + // mini-batch-warmup + LOG_ONCE(info, "[scheduler] Mini-batch size warmup {}", std::string(mbWarmup)); + + // This scales MB size up from the start. + // now scale batch size relative to progress within warm-up period + size_t progress = state_->getProgressIn(mbWarmup.unit); // number of updates/labels processed + auto progressRatio = (double)progress / (double)mbWarmup.n; // where are we relatively within target warm-up period + if (mbWarmup.unit == SchedulingUnit::trgLabels) + progressRatio = std::sqrt(progressRatio); + // apply ratio to actual batch size + ratio *= progressRatio; + + // adjust for reference batch size if given + // At progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever + // the actual batch size is. We approximate the latter as typicalTrgBatchWords_, and scale ratio accordingly. + if (mbWarmupOpts.size() > 1) { + ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences + auto refBatchLabels = (size_t)std::stoull(mbWarmupOpts[1]); + LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords_); + ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords_; + } + + // dynamic MB-size tracking with learning rate + // As LR goes down, MB gets ramped up by the same ratio, which has been found to be safe. + auto mbTracking = options_->get("mini-batch-track-lr"); + if (mbTracking) { + auto lrFactor = getLearningRateDecayFactor(*state_); + if (lrFactor != 1) + LOG_ONCE(info, "[scheduler] Dynamic mini-batch size adjustment enabled and kicking in"); + ratio /= lrFactor; + } + return true; + } + Scheduler(Ptr options, Ptr state) : options_(options), state_(state) { state_->eta = getLearningRate(*state); @@ -120,12 +186,13 @@ class Scheduler : public TrainingObserver { float value = validator->validate(graphs); if(validator->stalled() > 0) { LOG_VALID(info, - "Ep. {} : Up. {} : {} : {} : stalled {} times", + "Ep. {} : Up. {} : {} : {} : stalled {} times (last best: {})", state_->epochs, state_->batches, validator->type(), value, - validator->stalled()); + validator->stalled(), + validator->lastBest()); } else { LOG_VALID(info, "Ep. {} : Up. {} : {} : {} : new best", @@ -170,13 +237,12 @@ class Scheduler : public TrainingObserver { size_t batchLabels = 0; // number of target words in batch for(const auto& batch : batches) { - if (batch) { // (nullptr is allowed as result of split) - batchSize += batch->size(); - batchLabels += batch->words(-1); - } + batchSize += batch->size(); + batchLabels += batch->words(-1); } - // extrapolate cost across MPI processes, so that we have numbers in the right range + // Since batchLabels is counted across all MPI processes, we also should temporarily + // extrapolate cost across MPI processes, to have numbers in the right range. // When doing the actual log, we then aggregate across MPI processes to get the accurate number. if (mpi) cost *= mpi->numMPIProcesses(); // @BUGBUG: this is presently correct for ce-sum, but possibly not the av-based losses @@ -203,42 +269,42 @@ class Scheduler : public TrainingObserver { state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed - state_->newBatch(); + state_->newUpdate(batches.size()); if(state_->enteredNewPeriodOf(options_->get("disp-freq")) || state_->batches <= options_->get("disp-first")) { // if MPI then aggregate precise cost across workers if (mpi) { - //LOG(info, "all-reducing cost from {}", state_->costSum); state_->costSum /= mpi->numMPIProcesses(); // undo the extra scaling mpi->allReduce(&state_->costSum, &state_->costSum, 1, MPI_FLOAT, MPI_SUM); - //LOG(info, "all-reduced cost to {}", state_->costSum); } if (mpi && mpi->myMPIRank() != 0) ; // skip the report on alternate worker processes else if(dispLabelCounts) { if(options_->get("lr-report")) { // if true then show the learning rate LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} after {} : Time {:.2f}s : {:.2f} " + "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} @ {} after {} : Time {:.2f}s : {:.2f} " "words/s : L.r. {:.4e}", state_->epochs, state_->batches, utils::withCommas(state_->samplesEpoch), state_->costSum / state_->costCount, utils::withCommas(state_->costCount), // show cost as "av * count" + batchLabels, utils::withCommas(state_->labelsTotal), timer_.elapsed(), state_->wordsDisp / timer_.elapsed(), state_->eta); } else { LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} after {} : Time {:.2f}s : {:.2f} " + "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} @ {} after {} : Time {:.2f}s : {:.2f} " "words/s", state_->epochs, state_->batches, utils::withCommas(state_->samplesEpoch), state_->costSum / state_->costCount, utils::withCommas(state_->costCount), + batchLabels, utils::withCommas(state_->labelsTotal), timer_.elapsed(), state_->wordsDisp / timer_.elapsed()); @@ -272,12 +338,14 @@ class Scheduler : public TrainingObserver { } // progress heartbeat for MS-internal Philly compute cluster // This environment variable exists when running on the cluster. + using namespace std::chrono; if((!mpi || mpi->myMPIRank() == 0) && getenv("PHILLY_JOB_ID") && heartBeatTimer_.elapsed() >= 10) { - printf("PROGRESS: %.2f%%\nEVALERR: %.7f\n", (double)state_->epochs, state_->costSum / state_->costCount), fflush(stdout); -#if 0 - LOG(info, "heart beat after {} updates", state_->batches); -#endif + printf("PROGRESS: %.2f%%\nEVALERR: %.7f%%\n", + (double)state_->epochs, + state_->costSum / state_->costCount / (mpi ? mpi->numMPIProcesses() : 1)); + fflush(stdout); + std::cout << "MBSIZE: " << batchLabels << " after " << state_->batches << " updates = " << state_->labelsTotal << " labels" << std::endl << std::flush; heartBeatTimer_.start(); } } diff --git a/src/training/training.h b/src/training/training.h index 2209b96e2..174768542 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -56,7 +56,9 @@ class Train : public ModelTask { } auto batchGenerator = New(dataset, options_, stats); + scheduler->registerTrainingObserver(batchGenerator); + scheduler->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); // needed for dynamic MB scaling auto model = New(options_); model->setScheduler(scheduler); @@ -85,12 +87,14 @@ class Train : public ModelTask { } scheduler->finished(); - model->finalize(); - // Avoid saving the model twice if it has been loaded and training did not // progress if(!trainState->loaded) model->save(true); + + // finalize, including communicating successful completion to MPI + // @BUGBUG: This is wrong for async, but needed for sync. How to solve it? + model->finalize(); } }; } // namespace marian diff --git a/src/training/training_state.h b/src/training/training_state.h index 2deefe471..65cefcbce 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -2,6 +2,7 @@ #include "common/definitions.h" #include "common/filesystem.h" +#include "common/utils.h" #include #include @@ -42,7 +43,9 @@ struct SchedulingParameter { } param.pop_back(); } - res.n = (size_t)std::stoull(param); + double number = utils::parseNumber(param); + res.n = (size_t)number; + ABORT_IF(number != (double)res.n, "Scheduling parameters must be whole numbers"); return res; } @@ -62,9 +65,9 @@ class TrainingState { public: // Current epoch size_t epochs{1}; - // The total number of batches (=updates) processed since beginning of training --@TODO: rename to 'updates' + // The total number of updates since beginning of training --@TODO: rename to 'updates' size_t batches{0}; - // The number of batches seen in this epoch --@TODO: rename to 'updatesEpoch' or 'updatesInCurrentEpoch' + // The number of batches seen in this epoch --note: not updates; an update can consist of multiple batches size_t batchesEpoch{0}; // The number of sentences seen in this epoch --@TODO: rename to 'sentencesEpoch' size_t samplesEpoch{0}; @@ -172,9 +175,9 @@ class TrainingState { batchesEpoch = 0; } - void newBatch() { + void newUpdate(size_t batchesInUpdate) { ++batches; - ++batchesEpoch; + batchesEpoch += batchesInUpdate; loaded = false; validated = false; for(auto observer : observers_) diff --git a/src/training/validator.h b/src/training/validator.h index efed50cb0..682495410 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -137,7 +137,7 @@ class Validator : public ValidatorBase { lastBest_ = val; if(options_->get("keep-best")) keepBest(graphs); - } else { + } else if (lastBest_ != val) { // (special case 0 at start) @TODO: needed? Seems stall count gets reset each time it does improve. If not needed, remove "if(...)" again. stalled_++; } } @@ -166,7 +166,6 @@ class CrossEntropyValidator : public Validator { protected: virtual float validateBG(const std::vector>& graphs) override { - auto ctype = options_->get("cost-type"); options_->set("cost-type", "ce-sum"); diff --git a/vs/Marian.sln b/vs/Marian.sln index c2eb49c99..9cbbcefb7 100755 --- a/vs/Marian.sln +++ b/vs/Marian.sln @@ -1,377 +1,25 @@ + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ALL_BUILD", "ALL_BUILD.vcxproj", "{5216F769-E887-369E-AD1E-D6A1F69E834E}" - ProjectSection(ProjectDependencies) = postProject - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33} = {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33} - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {5AF43E07-5917-3D8F-9BF0-B41F698242EA} = {5AF43E07-5917-3D8F-9BF0-B41F698242EA} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {3CD61EAE-244E-33AB-8C7D-F5182481E033} = {3CD61EAE-244E-33AB-8C7D-F5182481E033} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {25A05D30-AFC2-3F0E-B475-0B2B81530151} = {25A05D30-AFC2-3F0E-B475-0B2B81530151} - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7} = {8A6B1F60-8E2D-3171-828B-07E732C8E7D7} - {3784D69C-33A9-33A7-A557-F809EF2F4D34} = {3784D69C-33A9-33A7-A557-F809EF2F4D34} - {EA3973A2-F92E-3124-9817-81B2458EC8DC} = {EA3973A2-F92E-3124-9817-81B2458EC8DC} - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D} = {36953645-6D01-37E4-ACF7-D3F9BFFCA49D} - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162} = {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - {5857EF98-C87F-3197-A399-F0F9A20913FC} = {5857EF98-C87F-3197-A399-F0F9A20913FC} - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F} = {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F} - {FBB107B9-523B-3094-95CF-A103E2388006} = {FBB107B9-523B-3094-95CF-A103E2388006} - {5B4A6D26-C638-3350-9E1A-0F987C448DEC} = {5B4A6D26-C638-3350-9E1A-0F987C448DEC} - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F} = {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F} - {1134F859-3DE4-34B1-924F-82CA38D4D4F3} = {1134F859-3DE4-34B1-924F-82CA38D4D4F3} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "INSTALL", "INSTALL.vcxproj", "{9DAF8CA3-052E-3480-A332-34676CAE852B}" - ProjectSection(ProjectDependencies) = postProject - {5216F769-E887-369E-AD1E-D6A1F69E834E} = {5216F769-E887-369E-AD1E-D6A1F69E834E} - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PACKAGE", "PACKAGE.vcxproj", "{3A3C6EA5-65CD-324E-90F4-6B4D70DD5A37}" - ProjectSection(ProjectDependencies) = postProject - {5216F769-E887-369E-AD1E-D6A1F69E834E} = {5216F769-E887-369E-AD1E-D6A1F69E834E} - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SQLiteCpp", "src\3rd_party\SQLiteCpp\SQLiteCpp.vcxproj", "{17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZERO_CHECK", "ZERO_CHECK.vcxproj", "{806A44E1-15D4-3368-B0B9-2A6CC352D505}" - ProjectSection(ProjectDependencies) = postProject - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libyaml-cpp", "src\3rd_party\yaml-cpp\libyaml-cpp.vcxproj", "{5AF43E07-5917-3D8F-9BF0-B41F698242EA}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian", "src\marian.vcxproj", "{885D3D2B-7278-30EF-BB1B-50E83D1635C4}" - ProjectSection(ProjectDependencies) = postProject - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33} = {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33} - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {5AF43E07-5917-3D8F-9BF0-B41F698242EA} = {5AF43E07-5917-3D8F-9BF0-B41F698242EA} - {55A27783-64A4-3AA7-A4B1-49C4B628F18C} = {55A27783-64A4-3AA7-A4B1-49C4B628F18C} - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162} = {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162} - {1134F859-3DE4-34B1-924F-82CA38D4D4F3} = {1134F859-3DE4-34B1-924F-82CA38D4D4F3} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_conv", "src\marian_conv.vcxproj", "{3CD61EAE-244E-33AB-8C7D-F5182481E033}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_cuda", "src\marian_cuda.vcxproj", "{97131187-E592-3981-886F-222EE20FB669}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_decoder", "src\marian_decoder.vcxproj", "{25A05D30-AFC2-3F0E-B475-0B2B81530151}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_scorer", "src\marian_scorer.vcxproj", "{8A6B1F60-8E2D-3171-828B-07E732C8E7D7}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_server", "src\marian_server.vcxproj", "{3784D69C-33A9-33A7-A557-F809EF2F4D34}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_train", "src\marian_train.vcxproj", "{EA3973A2-F92E-3124-9817-81B2458EC8DC}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_version", "src\marian_version.vcxproj", "{55A27783-64A4-3AA7-A4B1-49C4B628F18C}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "marian_vocab", "src\marian_vocab.vcxproj", "{36953645-6D01-37E4-ACF7-D3F9BFFCA49D}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {885D3D2B-7278-30EF-BB1B-50E83D1635C4} = {885D3D2B-7278-30EF-BB1B-50E83D1635C4} - {97131187-E592-3981-886F-222EE20FB669} = {97131187-E592-3981-886F-222EE20FB669} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pathie-cpp", "src\3rd_party\pathie-cpp\pathie-cpp.vcxproj", "{F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sentencepiece-static", "src\3rd_party\sentencepiece\src\sentencepiece-static.vcxproj", "{D9D20410-4011-370C-8E15-A6F5C311F337}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sentencepiece_train-static", "src\3rd_party\sentencepiece\src\sentencepiece_train-static.vcxproj", "{4A20AD5F-7334-31D3-B31D-9AAF53CC6678}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "spm_decode", "src\3rd_party\sentencepiece\src\spm_decode.vcxproj", "{5857EF98-C87F-3197-A399-F0F9A20913FC}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "spm_encode", "src\3rd_party\sentencepiece\src\spm_encode.vcxproj", "{F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "spm_export_vocab", "src\3rd_party\sentencepiece\src\spm_export_vocab.vcxproj", "{FBB107B9-523B-3094-95CF-A103E2388006}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "spm_normalize", "src\3rd_party\sentencepiece\src\spm_normalize.vcxproj", "{5B4A6D26-C638-3350-9E1A-0F987C448DEC}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "spm_train", "src\3rd_party\sentencepiece\src\spm_train.vcxproj", "{11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - {D9D20410-4011-370C-8E15-A6F5C311F337} = {D9D20410-4011-370C-8E15-A6F5C311F337} - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} = {4A20AD5F-7334-31D3-B31D-9AAF53CC6678} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "src\3rd_party\zlib\zlib.vcxproj", "{1134F859-3DE4-34B1-924F-82CA38D4D4F3}" - ProjectSection(ProjectDependencies) = postProject - {806A44E1-15D4-3368-B0B9-2A6CC352D505} = {806A44E1-15D4-3368-B0B9-2A6CC352D505} - EndProjectSection +VisualStudioVersion = 15.0.27703.2047 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Marian", "Marian.vcxproj", "{E2F320FE-0C01-4C80-810C-3A92205A29DC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 Release|x64 = Release|x64 - MinSizeRel|x64 = MinSizeRel|x64 - RelWithDebInfo|x64 = RelWithDebInfo|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5216F769-E887-369E-AD1E-D6A1F69E834E}.Debug|x64.ActiveCfg = Debug|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.Debug|x64.Build.0 = Debug|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.Release|x64.ActiveCfg = Release|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.Release|x64.Build.0 = Release|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {5216F769-E887-369E-AD1E-D6A1F69E834E}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {9DAF8CA3-052E-3480-A332-34676CAE852B}.Debug|x64.ActiveCfg = Debug|x64 - {9DAF8CA3-052E-3480-A332-34676CAE852B}.Release|x64.ActiveCfg = Release|x64 - {9DAF8CA3-052E-3480-A332-34676CAE852B}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {9DAF8CA3-052E-3480-A332-34676CAE852B}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {3A3C6EA5-65CD-324E-90F4-6B4D70DD5A37}.Debug|x64.ActiveCfg = Debug|x64 - {3A3C6EA5-65CD-324E-90F4-6B4D70DD5A37}.Release|x64.ActiveCfg = Release|x64 - {3A3C6EA5-65CD-324E-90F4-6B4D70DD5A37}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {3A3C6EA5-65CD-324E-90F4-6B4D70DD5A37}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.Debug|x64.ActiveCfg = Debug|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.Debug|x64.Build.0 = Debug|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.Release|x64.ActiveCfg = Release|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.Release|x64.Build.0 = Release|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {17E8F84B-76CD-326B-B50A-C4F3C3A8CE33}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.Debug|x64.ActiveCfg = Debug|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.Debug|x64.Build.0 = Debug|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.Release|x64.ActiveCfg = Release|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.Release|x64.Build.0 = Release|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {806A44E1-15D4-3368-B0B9-2A6CC352D505}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.Debug|x64.ActiveCfg = Debug|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.Debug|x64.Build.0 = Debug|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.Release|x64.ActiveCfg = Release|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.Release|x64.Build.0 = Release|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {5AF43E07-5917-3D8F-9BF0-B41F698242EA}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.Debug|x64.ActiveCfg = Debug|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.Debug|x64.Build.0 = Debug|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.Release|x64.ActiveCfg = Release|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.Release|x64.Build.0 = Release|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {885D3D2B-7278-30EF-BB1B-50E83D1635C4}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.Debug|x64.ActiveCfg = Debug|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.Debug|x64.Build.0 = Debug|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.Release|x64.ActiveCfg = Release|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.Release|x64.Build.0 = Release|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {3CD61EAE-244E-33AB-8C7D-F5182481E033}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {97131187-E592-3981-886F-222EE20FB669}.Debug|x64.ActiveCfg = Debug|x64 - {97131187-E592-3981-886F-222EE20FB669}.Debug|x64.Build.0 = Debug|x64 - {97131187-E592-3981-886F-222EE20FB669}.Release|x64.ActiveCfg = Release|x64 - {97131187-E592-3981-886F-222EE20FB669}.Release|x64.Build.0 = Release|x64 - {97131187-E592-3981-886F-222EE20FB669}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {97131187-E592-3981-886F-222EE20FB669}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {97131187-E592-3981-886F-222EE20FB669}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {97131187-E592-3981-886F-222EE20FB669}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.Debug|x64.ActiveCfg = Debug|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.Debug|x64.Build.0 = Debug|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.Release|x64.ActiveCfg = Release|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.Release|x64.Build.0 = Release|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {25A05D30-AFC2-3F0E-B475-0B2B81530151}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.Debug|x64.ActiveCfg = Debug|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.Debug|x64.Build.0 = Debug|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.Release|x64.ActiveCfg = Release|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.Release|x64.Build.0 = Release|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {8A6B1F60-8E2D-3171-828B-07E732C8E7D7}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.Debug|x64.ActiveCfg = Debug|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.Debug|x64.Build.0 = Debug|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.Release|x64.ActiveCfg = Release|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.Release|x64.Build.0 = Release|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {3784D69C-33A9-33A7-A557-F809EF2F4D34}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.Debug|x64.ActiveCfg = Debug|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.Debug|x64.Build.0 = Debug|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.Release|x64.ActiveCfg = Release|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.Release|x64.Build.0 = Release|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {EA3973A2-F92E-3124-9817-81B2458EC8DC}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.Debug|x64.ActiveCfg = Debug|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.Debug|x64.Build.0 = Debug|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.Release|x64.ActiveCfg = Release|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.Release|x64.Build.0 = Release|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {55A27783-64A4-3AA7-A4B1-49C4B628F18C}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.Debug|x64.ActiveCfg = Debug|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.Debug|x64.Build.0 = Debug|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.Release|x64.ActiveCfg = Release|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.Release|x64.Build.0 = Release|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {36953645-6D01-37E4-ACF7-D3F9BFFCA49D}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.Debug|x64.ActiveCfg = Debug|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.Debug|x64.Build.0 = Debug|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.Release|x64.ActiveCfg = Release|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.Release|x64.Build.0 = Release|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {F4AD2C38-E6B9-3C4A-A281-4AB7440D6162}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.Debug|x64.ActiveCfg = Debug|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.Debug|x64.Build.0 = Debug|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.Release|x64.ActiveCfg = Release|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.Release|x64.Build.0 = Release|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {D9D20410-4011-370C-8E15-A6F5C311F337}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.Debug|x64.ActiveCfg = Debug|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.Debug|x64.Build.0 = Debug|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.Release|x64.ActiveCfg = Release|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.Release|x64.Build.0 = Release|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {4A20AD5F-7334-31D3-B31D-9AAF53CC6678}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.Debug|x64.ActiveCfg = Debug|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.Debug|x64.Build.0 = Debug|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.Release|x64.ActiveCfg = Release|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.Release|x64.Build.0 = Release|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {5857EF98-C87F-3197-A399-F0F9A20913FC}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.Debug|x64.ActiveCfg = Debug|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.Debug|x64.Build.0 = Debug|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.Release|x64.ActiveCfg = Release|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.Release|x64.Build.0 = Release|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {F6E7B14E-D9E6-343C-B58D-CA0381A3BB8F}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.Debug|x64.ActiveCfg = Debug|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.Debug|x64.Build.0 = Debug|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.Release|x64.ActiveCfg = Release|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.Release|x64.Build.0 = Release|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {FBB107B9-523B-3094-95CF-A103E2388006}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.Debug|x64.ActiveCfg = Debug|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.Debug|x64.Build.0 = Debug|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.Release|x64.ActiveCfg = Release|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.Release|x64.Build.0 = Release|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {5B4A6D26-C638-3350-9E1A-0F987C448DEC}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.Debug|x64.ActiveCfg = Debug|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.Debug|x64.Build.0 = Debug|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.Release|x64.ActiveCfg = Release|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.Release|x64.Build.0 = Release|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {11AB9AE9-CF65-341B-B425-9EDFC4E2F22F}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.Debug|x64.ActiveCfg = Debug|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.Debug|x64.Build.0 = Debug|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.Release|x64.ActiveCfg = Release|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.Release|x64.Build.0 = Release|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 - {1134F859-3DE4-34B1-924F-82CA38D4D4F3}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 + {E2F320FE-0C01-4C80-810C-3A92205A29DC}.Debug|x64.ActiveCfg = Debug|x64 + {E2F320FE-0C01-4C80-810C-3A92205A29DC}.Debug|x64.Build.0 = Debug|x64 + {E2F320FE-0C01-4C80-810C-3A92205A29DC}.Release|x64.ActiveCfg = Release|x64 + {E2F320FE-0C01-4C80-810C-3A92205A29DC}.Release|x64.Build.0 = Release|x64 EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {A73289FB-DB51-3D6F-802E-B474CC102EDA} + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection - GlobalSection(ExtensibilityAddIns) = postSolution + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8CA1BE8F-87A9-4094-B549-E8C790F79D8C} EndGlobalSection EndGlobal diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index bebb987bf..15486147a 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -76,7 +76,7 @@ Console true - zlib.lib; msmpi.lib; mkl_intel_ilp64.lib; mkl_sequential.lib; mkl_core.lib; kernel32.lib; user32.lib; gdi32.lib; winspool.lib; comdlg32.lib; advapi32.lib; shell32.lib; ole32.lib; oleaut32.lib; uuid.lib; odbc32.lib; odbccp32.lib; %(AdditionalDependencies) + zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) 100000000 true @@ -106,13 +106,20 @@ true true true - zlib.lib; msmpi.lib; mkl_intel_ilp64.lib; mkl_sequential.lib; mkl_core.lib; kernel32.lib; user32.lib; gdi32.lib; winspool.lib; comdlg32.lib; advapi32.lib; shell32.lib; ole32.lib; oleaut32.lib; uuid.lib; odbc32.lib; odbccp32.lib; %(AdditionalDependencies) + zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) 100000000 true + + + + + + + true true @@ -316,6 +323,103 @@ + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + + + + + + true true @@ -459,7 +563,6 @@ - @@ -620,8 +723,6 @@ - - @@ -738,7 +839,6 @@ - @@ -808,6 +908,96 @@ + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + + true true @@ -846,6 +1036,7 @@ + true true diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 12aa5b4e4..ce6d8441b 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -31,9 +31,6 @@ 3rd_party\cnpy - - 3rd_party\svd - tensors @@ -421,8 +418,26 @@ rescorer - - command + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src + + + 3rd_party\pathie-cpp\src @@ -640,12 +655,6 @@ 3rd_party\spdlog\tests - - 3rd_party\svd - - - 3rd_party\svd - 3rd_party\yaml-cpp @@ -988,9 +997,6 @@ models - - models - models @@ -1345,6 +1351,118 @@ 3rd_party\sentencepiece\src + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\nccl\src\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + + + 3rd_party\pathie-cpp\include + @@ -1386,9 +1504,6 @@ {880c8f51-3306-4d80-a682-7242341b0041} - - {880c8f51-3306-4d80-a682-7242341b0044} - {880c8f51-3306-4d80-a682-7242341b0047} @@ -1482,6 +1597,36 @@ {638bf0e1-4f83-4b37-9077-2be549d75909} + + {0ba105eb-79fb-4e2a-8940-f1ecebbcd4fe} + + + {fbc17f5e-3f10-44a9-b3ad-66ce12573174} + + + {c6036c35-5848-4fd5-b1a0-59e2042cbb69} + + + {7b9a131d-9e0a-4c28-8a51-08232ff2e35e} + + + {0bd9cca8-660b-46f6-aac6-691fb50245f0} + + + {2beba56f-5dda-4994-bef0-16170b6552b4} + + + {ac585624-4e66-42cd-8e4e-62cb90029610} + + + {825beb7c-2997-408b-af81-34ab5f14593a} + + + {db1dd5a2-f331-495d-9e3b-6dc1c01528ab} + + + {5d5ee615-192f-4b7f-bdfd-fb8316ceabc8} + @@ -1524,10 +1669,109 @@ 3rd_party\sentencepiece\src + + 3rd_party\nccl\src + + + 3rd_party\nccl\src + + + 3rd_party\nccl\src + + + 3rd_party\nccl\src + + + 3rd_party\nccl\src + + + 3rd_party\nccl\src + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\collectives\device + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\misc + + + 3rd_party\nccl\src\transport + + + 3rd_party\nccl\src\transport + + + 3rd_party\nccl\src\transport + + + 3rd_party\nccl\src\transport + + + 3rd_party\nccl\src\transport + + + 3rd_party\pathie-cpp + + + 3rd_party\pathie-cpp + + + 3rd_party\pathie-cpp + 3rd_party\sentencepiece\src + + 3rd_party\pathie-cpp + \ No newline at end of file From 9d3028599ebae1ba51e2323602be3eb7e89a0165 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 09:48:58 +0000 Subject: [PATCH 029/838] Replace more add_nondefault() with add() --- src/common/config_parser.cpp | 12 ++++++------ src/common/config_validator.cpp | 7 ++++--- src/data/shortlist.h | 3 +-- src/training/training.h | 2 +- src/training/validator.h | 18 +++++++++--------- src/translator/output_collector.h | 4 +--- src/translator/scorers.cpp | 4 ++-- src/translator/translator.h | 2 +- 8 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 37b75ac45..4641d31a1 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -374,7 +374,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "sentence"); // embedding options - cli.add_nondefault>("--embedding-vectors", + cli.add>("--embedding-vectors", "Paths to files with custom source and target embedding vectors"); cli.add("--embedding-normalization", "Normalize values from custom embedding vectors to [-1, 1]"); @@ -397,7 +397,7 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { cli.switchGroup("Validation set options"); // clang-format off - cli.add_nondefault>("--valid-sets", + cli.add>("--valid-sets", "Paths to validation corpora: source target"); cli.add("--valid-freq", "Validate model every arg updates (append 't' for every arg target labels)", @@ -436,12 +436,12 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { 1000); // options for validation script - cli.add_nondefault("--valid-script-path", + cli.add("--valid-script-path", "Path to external validation script." " It should print a single score to stdout." " If the option is used with validating translation, the output" " translation file will be passed as a first argument"); - cli.add_nondefault("--valid-translation-output", + cli.add("--valid-translation-output", "Path to store the translation"); cli.add("--keep-best", @@ -492,9 +492,9 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { cli.add("--skip-cost", "Ignore model cost during translation, not recommended for beam-size > 1"); - cli.add_nondefault>("--shortlist", + cli.add>("--shortlist", "Use softmax shortlist: path first best prune"); - cli.add_nondefault>("--weights", + cli.add>("--weights", "Scorer weights"); cli.add("--output-sampling", "Noise output layer with gumbel noise", diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 8a6845633..4b3bd4315 100755 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -85,9 +85,10 @@ void ConfigValidator::validateOptionsTraining() const { ABORT_IF(!modelDir.empty() && !filesystem::isDirectory(modelDir), "Model directory does not exist"); - ABORT_IF( - has("valid-sets") && get>("valid-sets").size() != trainSets.size(), - "There should be as many validation sets as training sets"); + ABORT_IF(has("valid-sets") + && get>("valid-sets").size() != trainSets.size() + && !get>("valid-sets").empty(), + "There should be as many validation sets as training sets"); // validations for learning rate decaying ABORT_IF(get("lr-decay") > 1.f, "Learning rate decay factor greater than 1.0 is unusual"); diff --git a/src/data/shortlist.h b/src/data/shortlist.h index 55fe4f5bc..4763d6589 100755 --- a/src/data/shortlist.h +++ b/src/data/shortlist.h @@ -182,8 +182,7 @@ class LexicalShortlistGenerator : public ShortlistGenerator { srcIdx_(srcIdx), trgIdx_(trgIdx), shared_(shared) { - std::vector vals - = options_->get>("shortlist"); + std::vector vals = options_->get>("shortlist"); ABORT_IF(vals.empty(), "No path to filter path given"); std::string fname = vals[0]; diff --git a/src/training/training.h b/src/training/training.h index 2209b96e2..fd077da6a 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -49,7 +49,7 @@ class Train : public ModelTask { auto trainState = New(options_->get("learn-rate")); auto scheduler = New(options_, trainState); - if((options_->has("valid-sets") || options_->has("valid-script-path")) + if((options_->nonempty("valid-sets") || options_->nonempty("valid-script-path")) && SchedulingParameter::parse(options_->get("valid-freq"))) { for(auto validator : Validators(dataset->getVocabs(), options_)) scheduler->addValidator(validator); diff --git a/src/training/validator.h b/src/training/validator.h index efed50cb0..351bf15d2 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -104,7 +104,6 @@ class Validator : public ValidatorBase { public: virtual float validate(const std::vector>& graphs) override { - for(auto graph : graphs) graph->setInference(true); @@ -224,7 +223,7 @@ class ScriptValidator : public Validator { : Validator(vocabs, options, false) { builder_ = models::from_options(options_, models::usage::raw); - ABORT_IF(!options_->has("valid-script-path"), "valid-script metric but no script given"); + ABORT_IF(!options_->nonempty("valid-script-path"), "valid-script metric but no script given"); } virtual float validate(const std::vector>& graphs) override { @@ -255,9 +254,8 @@ class TranslationValidator : public Validator { quiet_(options_->get("quiet-translation")) { builder_ = models::from_options(options_, models::usage::translation); - if(!options_->has("valid-script-path")) - LOG_VALID(warn, - "No post-processing script given for validating translator"); + if(!options_->nonempty("valid-script-path")) + LOG_VALID(warn, "No post-processing script given for validating translator"); createBatchGenerator(/*isTranslating=*/true); } @@ -287,7 +285,7 @@ class TranslationValidator : public Validator { std::string fileName; Ptr tempFile; - if(options_->has("valid-translation-output")) { + if(options_->nonempty("valid-translation-output")) { fileName = options_->get("valid-translation-output"); } else { tempFile.reset(new io::TemporaryFile(options_->get("tempdir"), false)); @@ -303,7 +301,9 @@ class TranslationValidator : public Validator { timer::Timer timer; { auto printer = New(options_, vocabs_.back()); - auto collector = options_->has("valid-translation-output") + // @TODO: This can be simplified. If there is no "valid-translation-output", fileName already + // contains the name of temporary file that should be used? + auto collector = options_->nonempty("valid-translation-output") ? New(fileName) : New(*tempFile); @@ -358,7 +358,7 @@ class TranslationValidator : public Validator { float val = 0.0f; // Run post-processing script if given - if(options_->has("valid-script-path")) { + if(options_->nonempty("valid-script-path")) { auto command = options_->get("valid-script-path") + " " + fileName; auto valStr = utils::exec(command); val = (float)std::atof(valStr.c_str()); @@ -442,7 +442,7 @@ class BleuValidator : public Validator { auto printer = New(options_, vocabs_.back()); Ptr collector; - if(options_->has("valid-translation-output")) { + if(options_->nonempty("valid-translation-output")) { auto fileName = options_->get("valid-translation-output"); collector = New(fileName); // for debugging } diff --git a/src/translator/output_collector.h b/src/translator/output_collector.h index 51b47159c..c616f03ff 100755 --- a/src/translator/output_collector.h +++ b/src/translator/output_collector.h @@ -49,9 +49,7 @@ class OutputCollector { OutputCollector(std::string outFile); template - OutputCollector(T&& arg) - : nextId_(0), - outStrm_(new io::OutputFileStream(arg)) {} + OutputCollector(T&& arg) : nextId_(0), outStrm_(new io::OutputFileStream(arg)) {} OutputCollector(const OutputCollector&) = delete; diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp index 9afa1d7b8..6c0031bff 100755 --- a/src/translator/scorers.cpp +++ b/src/translator/scorers.cpp @@ -53,7 +53,7 @@ std::vector> createScorers(Ptr options) { auto models = options->get>("models"); std::vector weights(models.size(), 1.f); - if(options->has("weights")) + if(options->nonempty("weights")) weights = options->get>("weights"); size_t i = 0; @@ -83,7 +83,7 @@ std::vector> createScorers(Ptr options, const std::vector> scorers; std::vector weights(ptrs.size(), 1.f); - if(options->has("weights")) + if(options->nonempty("weights")) weights = options->get>("weights"); size_t i = 0; diff --git a/src/translator/translator.h b/src/translator/translator.h index 5fbfe8aac..877312b81 100755 --- a/src/translator/translator.h +++ b/src/translator/translator.h @@ -41,7 +41,7 @@ class Translate : public ModelTask { trgVocab_->load(vocabs.back()); auto srcVocab = corpus_->getVocabs()[0]; - if(options_->has("shortlist")) + if(options_->nonempty("shortlist")) shortlistGenerator_ = New( options_, srcVocab, trgVocab_, 0, 1, vocabs.front() == vocabs.back()); From 49d1cf85fa402a11442a6fc9940a93a0e4672c67 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 13:09:53 +0000 Subject: [PATCH 030/838] Replace add_nondefault() with add() for --alignment and --summary --- src/common/config_parser.cpp | 6 +++--- src/microsoft/quicksand.cpp | 2 +- src/models/transformer.h | 2 +- src/rescorer/rescorer.h | 13 ++++++------- src/translator/beam_search.h | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 4641d31a1..2e73ad5d7 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -479,7 +479,7 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { "Allow unknown words to appear in output"); cli.add("--n-best", "Generate n-best list"); - cli.add_nondefault("--alignment", + cli.add("--alignment", "Return word alignment. Possible values: 0.0-1.0, hard, soft") ->implicit_val("1"); @@ -531,10 +531,10 @@ void ConfigParser::addOptionsScoring(cli::CLIWrapper& cli) { "Feature name to be inserted into n-best list", "Score"); cli.add("--normalize,-n", "Divide translation score by translation length"); - cli.add_nondefault("--summary", + cli.add("--summary", "Only print total cost, possible values: cross-entropy (ce-mean), ce-mean-words, ce-sum, perplexity") ->implicit_val("cross-entropy"); - cli.add_nondefault("--alignment", + cli.add("--alignment", "Return word alignments. Possible values: 0.0-1.0, hard, soft") ->implicit_val("1"), diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 962bda182..2a82263f7 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -136,7 +136,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { auto score = std::get<2>(result); // determine alignment if present AlignmentSets alignmentSets; - if (options_->has("alignment")) + if (options_->nonempty("alignment")) { float alignmentThreshold; auto alignment = options_->get("alignment"); // @TODO: this logic now exists three times in Marian diff --git a/src/models/transformer.h b/src/models/transformer.h index 3cd7140a8..ee5020e26 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -789,7 +789,7 @@ class DecoderTransformer : public Transformer { // decoding or scoring return the attention weights of one head of the last layer. // @TODO: maybe allow to return average or max over all heads? bool saveAttentionWeights = false; - if(j == 0 && (options_->get("guided-alignment", std::string("none")) != "none" || options_->has("alignment"))) { + if(j == 0 && (options_->get("guided-alignment", std::string("none")) != "none" || options_->nonempty("alignment"))) { size_t attLayer = decDepth - 1; std::string gaStr = options_->get("transformer-guided-alignment-layer", "last"); if(gaStr != "last") diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index fa456856a..a837d15dc 100644 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -49,10 +49,10 @@ class Rescore : public ModelTask { public: Rescore(Ptr options) : options_(options) { - ABORT_IF(options_->has("summary") && options_->has("alignment"), + ABORT_IF(options_->nonempty("summary") && options_->nonempty("alignment"), "Alignments can not be produced with summarized score"); - ABORT_IF(options_->has("summary") && options_->get("normalize"), + ABORT_IF(options_->nonempty("summary") && options_->get("normalize"), "Normalization by length cannot be used with summary scores"); options_->set("inference", true); @@ -99,12 +99,11 @@ class Rescore : public ModelTask { New(options_)) : New(options_); - std::string alignment = options_->get("alignment", ""); - bool summarize = options_->has("summary"); + auto alignment = options_->get("alignment", ""); + auto summary = options_->get("summary", ""); + bool summarize = !summary.empty(); bool normalize = options_->get("normalize"); - std::string summary = summarize ? options_->get("summary") : "cross-entropy"; - float sumCost = 0; size_t sumWords = 0; size_t sumSamples = 0; @@ -126,7 +125,7 @@ class Rescore : public ModelTask { // @TODO: normalize by length as in normalize // Once we have Frank's concept of ce-sum with sample size by words we will return a pair - // here which will make it trivial to report all variants. + // here which will make it trivial to report all variants. auto costNode = builder->build(graph, batch); graph->forward(); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 798312fed..50bf5ccbc 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -42,7 +42,7 @@ class BeamSearch { Beams newBeams(beams.size()); std::vector align; - if(options_->has("alignment")) + if(options_->nonempty("alignment")) // Use alignments from the first scorer, even if ensemble align = scorers_[0]->getAlignment(); From f078e1717dc9bd374ebb8210397bd2d317a1c1d9 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 14:10:48 +0000 Subject: [PATCH 031/838] Remove --special-vocab --- src/common/cli_wrapper.cpp | 12 ++++++++++-- src/common/config.cpp | 6 +++--- src/common/config_parser.cpp | 5 ++--- src/models/encoder_decoder.cpp | 1 - 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index ce24f211a..df647bfe4 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -9,6 +9,14 @@ namespace marian { namespace cli { +// clang-format off +const std::unordered_set DEPRECIATED_OPTIONS = { + "version", + "special-vocab" +}; +// clang-format on + + /* static uint16_t guess_terminal_width(uint16_t max_width, uint16_t default_width) { uint16_t cols = 0; @@ -135,8 +143,8 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string& error // skip options specified via command-line to allow overwriting them if(cmdOptions.count(key)) continue; - // skip special option "version" that possibly can be loaded from model.npz:special.yml - if(key == "version") + // skip options that might exist in config files generated by older versions of Marian + if(DEPRECIATED_OPTIONS.count(key)) continue; if(options_.count(key)) { diff --git a/src/common/config.cpp b/src/common/config.cpp index 043a757f5..c1a99cad9 100755 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -33,13 +33,13 @@ void Config::initialize(int argc, char** argv, cli::mode mode, bool validate) { // echo version and command line LOG(info, "[marian] Marian {}", buildVersion()); std::string cmdLine; - for (int i = 0; i < argc; i++) { + for(int i = 0; i < argc; i++) { std::string arg = argv[i]; std::string quote; // attempt to quote special chars - if (arg.empty() || arg.find_first_of(" #`\"'\\${}|&^?*!()%><") != std::string::npos) + if(arg.empty() || arg.find_first_of(" #`\"'\\${}|&^?*!()%><") != std::string::npos) quote = "'"; arg = regex::regex_replace(arg, regex::regex("'"), "'\\''"); - if (!cmdLine.empty()) + if(!cmdLine.empty()) cmdLine.push_back(' '); cmdLine += quote + arg + quote; } diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 2e73ad5d7..bce007d5a 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -22,7 +22,8 @@ namespace marian { -// TODO: move to CLIWrapper +// TODO: Move this to CLIWrapper and allow to mark options as paths in the same place they are +// defined // clang-format off const std::set PATHS = { "model", @@ -146,8 +147,6 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Train right-to-left model"); cli.add("--best-deep", "Use Edinburgh deep RNN configuration (s2s)"); - cli.add_nondefault>("--special-vocab", - "Model-specific special vocabulary ids"); cli.add("--tied-embeddings", "Tie target embeddings and output embeddings in output layer"); cli.add("--tied-embeddings-src", diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 3edb7b335..49593a15c 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -23,7 +23,6 @@ EncoderDecoder::EncoderDecoder(Ptr options) "skip", "layer-normalization", "right-left", - "special-vocab", "tied-embeddings", "tied-embeddings-src", "tied-embeddings-all"}; From 219ea012e4de76a4dc735b9420f46c171be6c07c Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 14:47:02 +0000 Subject: [PATCH 032/838] Make --port available only for marian-server --- src/command/marian_server.cpp | 2 +- src/command/marian_train.cpp | 2 +- src/common/config.cpp | 26 +++++++++++++------------- src/common/config.h | 2 +- src/common/config_parser.cpp | 19 +++++++++++++++---- src/common/config_parser.h | 7 +++++-- src/common/config_validator.cpp | 3 +++ 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/command/marian_server.cpp b/src/command/marian_server.cpp index a9a1b0be1..5df944d71 100644 --- a/src/command/marian_server.cpp +++ b/src/command/marian_server.cpp @@ -12,7 +12,7 @@ int main(int argc, char **argv) { using namespace marian; // Initialize translation task - auto options = parseOptions(argc, argv, cli::mode::translation, true); + auto options = parseOptions(argc, argv, cli::mode::server, true); auto task = New>(options); // Initialize web server diff --git a/src/command/marian_train.cpp b/src/command/marian_train.cpp index 889806244..6be2abbba 100755 --- a/src/command/marian_train.cpp +++ b/src/command/marian_train.cpp @@ -14,7 +14,7 @@ int main(int argc, char** argv) { using namespace marian; - auto options = parseOptions(argc, argv); + auto options = parseOptions(argc, argv, cli::mode::training); // selects MultiNodeGraphGroup family // diff --git a/src/common/config.cpp b/src/common/config.cpp index c1a99cad9..ddfb5199e 100755 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -16,7 +16,7 @@ size_t Config::seed = (size_t)time(0); Config::Config(int argc, char** argv, - cli::mode mode /*= cli::mode::training*/, + cli::mode mode, bool validate /*= true*/) { initialize(argc, argv, mode, validate); } @@ -56,7 +56,17 @@ void Config::initialize(int argc, char** argv, cli::mode mode, bool validate) { } // load model parameters - if(mode != cli::mode::translation) { + if(mode == cli::mode::translation || mode == cli::mode::server) { + auto model = get>("models")[0]; + try { + if(!get("ignore-model-config")) + loadModelParameters(model); + } catch(std::runtime_error& ) { + LOG(info, "[config] No model configuration found in model file"); + } + } + // if cli::mode::training or cli::mode::scoring + else { auto model = get("model"); if(filesystem::exists(model) && !get("no-reload")) { try { @@ -67,16 +77,6 @@ void Config::initialize(int argc, char** argv, cli::mode mode, bool validate) { } } } - // if cli::mode::translation - else { - auto model = get>("models")[0]; - try { - if(!get("ignore-model-config")) - loadModelParameters(model); - } catch(std::runtime_error& ) { - LOG(info, "[config] No model configuration found in model file"); - } - } // echo full configuration log(); @@ -264,7 +264,7 @@ std::vector Config::getDevices(Ptr options, Ptr parseOptions(int argc, char** argv, - cli::mode mode /*= cli::mode::training*/, + cli::mode mode, bool validate /*= true*/) { auto config = New(argc, argv, mode, validate); auto options = New(); diff --git a/src/common/config.h b/src/common/config.h index ec462a532..326dbcf1d 100755 --- a/src/common/config.h +++ b/src/common/config.h @@ -115,7 +115,7 @@ class Config { */ Ptr parseOptions(int argc, char** argv, - cli::mode mode = cli::mode::training, + cli::mode mode, bool validate = true); } // namespace marian diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index bce007d5a..cea33ac47 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -85,6 +85,14 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { // clang-format on } +void ConfigParser::addOptionsServer(cli::CLIWrapper& cli) { + // clang-format off + cli.add("--port,-p", + "Port number for web socket server", + 8080); + // clang-format on +} + void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.switchGroup("Model options"); @@ -499,9 +507,6 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { "Noise output layer with gumbel noise", false); - // TODO: the options should be available only in server - cli.add_nondefault("--port,-p", - "Port number for web socket server"); // add ULR settings addSuboptionsULR(cli); @@ -561,7 +566,8 @@ void ConfigParser::addSuboptionsDevices(cli::CLIWrapper& cli) { #ifdef CUDA_FOUND cli.add("--cpu-threads", "Use CPU-based computation with this many independent threads, 0 means GPU-based computation", - 0)->implicit_val("1"); + 0) + ->implicit_val("1"); #else cli.add("--cpu-threads", "Use CPU-based computation with this many independent threads, 0 means GPU-based computation", @@ -678,6 +684,8 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { 40); addOptionsGeneral(cli); + if(modeServer_) + addOptionsServer(cli); addOptionsModel(cli); // clang-format off @@ -692,6 +700,9 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { case cli::mode::scoring: addOptionsScoring(cli); break; + default: + ABORT("wrong CLI mode"); + break; } // clang-format on diff --git a/src/common/config_parser.h b/src/common/config_parser.h index de1cb70e1..ff305f738 100755 --- a/src/common/config_parser.h +++ b/src/common/config_parser.h @@ -14,7 +14,7 @@ namespace marian { namespace cli { -enum struct mode { training, translation, scoring }; +enum struct mode { training, translation, scoring, server }; } // namespace cli /** @@ -25,7 +25,8 @@ enum struct mode { training, translation, scoring }; class ConfigParser { public: ConfigParser(int argc, char** argv, cli::mode mode, bool validate = false) - : mode_(mode) { + : modeServer_(mode == cli::mode::server), + mode_(mode == cli::mode::server ? cli::mode::translation : mode) { parseOptions(argc, argv, validate); } @@ -51,6 +52,7 @@ class ConfigParser { YAML::Node getConfig() const; private: + bool modeServer_; cli::mode mode_; YAML::Node config_; @@ -68,6 +70,7 @@ class ConfigParser { } void addOptionsGeneral(cli::CLIWrapper&); + void addOptionsServer(cli::CLIWrapper&); void addOptionsModel(cli::CLIWrapper&); void addOptionsTraining(cli::CLIWrapper&); void addOptionsValidation(cli::CLIWrapper&); diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 4b3bd4315..9345b0a56 100755 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -28,6 +28,9 @@ void ConfigValidator::validateOptions(cli::mode mode) const { validateOptionsParallelData(); validateOptionsTraining(); break; + default: + ABORT("wrong CLI mode"); + break; } // clang-format on From eaa0350091f31cdd90deb52db0c8f5adee2670e3 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 15:13:29 +0000 Subject: [PATCH 033/838] Remove CLIWrapper::add_nondefault() --- src/common/cli_wrapper.h | 55 ++++++++-------------------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 4e8701cc2..263d62120 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -159,8 +159,7 @@ class CLIWrapper { args, help, val, - /*defaulted =*/true, - /*addToConfig =*/true); + /*defaulted =*/true); } /** @@ -186,33 +185,7 @@ class CLIWrapper { args, help, T(), - /*defaulted =*/false, - /*addToConfig =*/true); - } - - /** - * @brief Define a non-defaulted option - * - * The option will not be present in the config file unless given as a - * command-line argument. - * - * @param args Comma-separated list of short and long option names - * @param help Help message - * - * @return Option object - * - * @TODO: consider removing this method during final refactorization of - * command-line/config parsers in the future as all options should either - * have a default value or be non-defaulted - */ - template - CLI::Option *add_nondefault(const std::string &args, const std::string &help) { - return add_option(keyName(args), - args, - help, - T(), - /*defaulted =*/false, - /*addToConfig =*/false); + /*defaulted =*/false); } /** @@ -256,11 +229,9 @@ class CLIWrapper { const std::string &args, const std::string &help, T val, - bool defaulted, - bool addToConfig) { - // define YAML entry if requested - if(addToConfig) - config_[key] = val; + bool defaulted) { + // add key to YAML + config_[key] = val; // create option tuple CLIOptionTuple option; @@ -305,11 +276,9 @@ class CLIWrapper { const std::string &args, const std::string &help, T val, - bool defaulted, - bool addToConfig) { - // define YAML entry if requested - if(addToConfig) - config_[key] = val; + bool defaulted) { + // add key to YAML + config_[key] = val; // create option tuple CLIOptionTuple option; @@ -364,11 +333,9 @@ class CLIWrapper { const std::string &args, const std::string &help, T val, - bool defaulted, - bool addToConfig) { - // define YAML entry if requested - if(addToConfig) - config_[key] = val; + bool defaulted) { + // add key to YAML + config_[key] = val; // create option tuple CLIOptionTuple option; From 6407e1b5372fe0f723ab05bf8f81c43528bab19a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 15:19:27 +0000 Subject: [PATCH 034/838] Fix misspelling --- src/common/config_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index cea33ac47..466693332 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -281,7 +281,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Display information every arg updates (append 't' for every arg target labels)", "1000u"); cli.add("--disp-first", - "Display nformation for the first arg updates"); + "Display information for the first arg updates"); cli.add("--disp-label-counts", "Display label counts when logging loss progress"); cli.add("--save-freq", From c733eb1e352a6883aa89a813d7e50d3e1c61cd5b Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 13 Dec 2018 16:09:10 +0000 Subject: [PATCH 035/838] Rename add_option to addOption --- src/common/cli_wrapper.h | 54 +++++++++++++++++++----------------- src/common/config_parser.cpp | 11 +++----- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 263d62120..1590441a0 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -147,6 +147,8 @@ class CLIWrapper { /** * @brief Define an option with a default value * + * Explicit default values will appear in help messages. + * * @param args Comma-separated list of short and long option names * @param help Help message * @param val Default value @@ -155,11 +157,11 @@ class CLIWrapper { */ template CLI::Option *add(const std::string &args, const std::string &help, T val) { - return add_option(keyName(args), - args, - help, - val, - /*defaulted =*/true); + return addOption(keyName(args), + args, + help, + val, + /*defaulted =*/true); } /** @@ -171,6 +173,8 @@ class CLIWrapper { * option is 0, for a string is an empty string, and for a vector is an empty * vector. * + * Implicit default values will *NOT* appear in help messages. + * * @param args Comma-separated list of short and long option names * @param help Help message * @@ -181,11 +185,11 @@ class CLIWrapper { */ template CLI::Option *add(const std::string &args, const std::string &help) { - return add_option(keyName(args), - args, - help, - T(), - /*defaulted =*/false); + return addOption(keyName(args), + args, + help, + T(), + /*defaulted =*/false); } /** @@ -225,11 +229,11 @@ class CLIWrapper { // options with numeric and string-like values CLI::enable_if_t::value && !CLI::is_vector::value, CLI::detail::enabler> = CLI::detail::dummy> - CLI::Option *add_option(const std::string &key, - const std::string &args, - const std::string &help, - T val, - bool defaulted) { + CLI::Option *addOption(const std::string &key, + const std::string &args, + const std::string &help, + T val, + bool defaulted) { // add key to YAML config_[key] = val; @@ -272,11 +276,11 @@ class CLIWrapper { template ::value, CLI::detail::enabler> = CLI::detail::dummy> - CLI::Option *add_option(const std::string &key, - const std::string &args, - const std::string &help, - T val, - bool defaulted) { + CLI::Option *addOption(const std::string &key, + const std::string &args, + const std::string &help, + T val, + bool defaulted) { // add key to YAML config_[key] = val; @@ -329,11 +333,11 @@ class CLIWrapper { template ::value, CLI::detail::enabler> = CLI::detail::dummy> - CLI::Option *add_option(const std::string &key, - const std::string &args, - const std::string &help, - T val, - bool defaulted) { + CLI::Option *addOption(const std::string &key, + const std::string &args, + const std::string &help, + T val, + bool defaulted) { // add key to YAML config_[key] = val; diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 466693332..a4699e023 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -79,8 +79,7 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { cli.add("--relative-paths", "All paths are relative to the config file location"); cli.add("--dump-config", - "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal", - "false") + "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") ->implicit_val("full"); // clang-format on } @@ -628,8 +627,7 @@ void ConfigParser::addSuboptionsULR(cli::CLIWrapper& cli) { // clang-format off // support for universal encoder ULR https://arxiv.org/pdf/1802.05368.pdf cli.add("--ulr", - "Enable ULR (Universal Language Representation)", - false); + "Enable ULR (Universal Language Representation)"); // reading pre-trained universal embeddings for multi-sources. // Note that source and target here is relative to ULR not the translation langs // queries: EQ in Fig2 : is the unified embeddings projected to one space. @@ -642,8 +640,7 @@ void ConfigParser::addSuboptionsULR(cli::CLIWrapper& cli) { "Path to file with universal sources embeddings of traget keys from projection into universal space", ""); cli.add("--ulr-trainable-transformation", - "Make Query Transformation Matrix A trainable", - false); + "Make Query Transformation Matrix A trainable"); cli.add("--ulr-dim-emb", "ULR monolingual embeddings dimension"); cli.add("--ulr-dropout", @@ -728,7 +725,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { // remove extra config files from the config to avoid redundancy config_.remove("config"); - if(get("dump-config") != "false") { + if(!get("dump-config").empty() && get("dump-config") != "false") { bool skipDefault = get("dump-config") == "minimal"; config_.remove("dump-config"); std::cout << cli.dumpConfig(skipDefault) << std::endl; From 58c493a37d1455f7ea2d8637c216ba301eb86318 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 13:55:53 -0800 Subject: [PATCH 036/838] removed reference-MB-size arguments throughout, and replaced it with a single argument --mini-batch-words-ref honored at multiple places --- src/common/config_parser.cpp | 61 ++++++++++++----------- src/optimizers/optimizers.cpp | 23 ++++----- src/optimizers/optimizers.h | 72 ++++++++++++++-------------- src/training/exponential_smoothing.h | 7 +-- src/training/scheduler.h | 11 ++--- 5 files changed, 87 insertions(+), 87 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 06eabd320..377e8d2e6 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -108,7 +108,7 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "amun"); cli.add>("--dim-vocabs", "Maximum items in vocabulary ordered by rank, 0 uses all items in the provided/created vocabulary file", - std::vector({0, 0})); + {0, 0}); cli.add("--dim-emb", "Size of embedding vector", 512); @@ -205,11 +205,11 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Number of highway network layers after max-pooling in char-s2s model", 4); cli.add>("--char-conv-filters-num", - "Numbers of convolution filters of correspoding width in char-s2s model", - std::vector({200, 200, 250, 250, 300, 300, 300, 300})); + "Numbers of convolution filters of corresponding width in char-s2s model", + {200, 200, 250, 250, 300, 300, 300, 300}); cli.add>("--char-conv-filters-widths", "Convolution window widths in char-s2s model", - std::vector({1, 2, 3, 4, 5, 6, 7, 8})); + {1, 2, 3, 4, 5, 6, 7, 8}); #endif if(mode_ == cli::mode::training) { @@ -305,17 +305,19 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Optimization algorithm: sgd, adagrad, adam", "adam"); cli.add_nondefault>("--optimizer-params", - "Parameters for optimization algorithm, e.g. betas for adam"); - cli.add("--optimizer-delay", - "SGD update delay, 1 = no delay", - 1); + "Parameters for optimization algorithm, e.g. betas for Adam. " + "Auto-adjusted to --mini-batch-words-ref if given"); + cli.add("--optimizer-delay", + "SGD update delay, 1 = no delay. Can be fractional, e.g. 0.1 to use only 10% of each batch", + 1.); cli.add("--sync-sgd", "Use synchronous SGD instead of asynchronous for multi-gpu training"); // learning rate options cli.add("--learn-rate,-l", - "Learning rate", + "Learning rate. " + "Auto-adjusted to --mini-batch-words-ref if given", 0.0001); cli.add("--lr-report", "Report learning rate for each update"); @@ -327,7 +329,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "epoch+stalled"); cli.add>("--lr-decay-start", "The first number of (epoch, batches, stalled) validations to start learning rate decaying (tuple)", - std::vector({10,1})); + {10,1}); cli.add("--lr-decay-freq", "Learning rate decaying frequency for batches, requires --lr-decay-strategy to be batches", 50000); @@ -337,8 +339,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Repeat learning rate warmup when learning rate is decayed"); cli.add>("--lr-decay-inv-sqrt", "Decrease learning rate at arg / sqrt(no. batches) starting at arg (append 't' or 'e' for sqrt(target labels or epochs)). " - "Add second argument to define the starting point", - {"0"}); + "Add second argument to define the starting point (default: same as first value)", + {"0"}); cli.add("--lr-warmup", "Increase learning rate linearly for arg first batches (append 't' for arg first target labels)", @@ -355,10 +357,10 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--clip-norm", "Clip gradient norm to argcli.add(0 to disable)", 1.f); - cli.add>("--exponential-smoothing", + cli.add("--exponential-smoothing", "Maintain smoothed version of parameters for validation and saving with smoothing factor. 0 to disable. " - "Add a second number to specify a reference batch size (in target words).", - { 0.f })->implicit_val("1e-4"); + "Auto-adjusted to --mini-batch-words-ref if given.", + 0.f)->implicit_val("1e-4"); cli.add("--guided-alignment", "Path to a file with word alignments. Use guided alignment to guide attention or 'none'", "none"); @@ -405,8 +407,8 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { "10000u"); cli.add>("--valid-metrics", "Metric to use during validation: cross-entropy, ce-mean-words, perplexity, valid-script, " - " translation, bleu, bleu-detok. Multiple metrics can be specified", - std::vector({"cross-entropy"})); + "translation, bleu, bleu-detok. Multiple metrics can be specified", + {"cross-entropy"}); cli.add("--early-stopping", "Stop if the first validation metric does not improve for arg consecutive validation steps", 10); @@ -458,7 +460,7 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { // clang-format off cli.add>("--input,-i", "Paths to input file(s), stdin by default", - std::vector({"stdin"})); + {"stdin"}); cli.add("--output,-o", "Path to output file, stdout by default", "stdout"); @@ -496,10 +498,10 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { cli.add_nondefault>("--shortlist", "Use softmax shortlist: path first best prune"); cli.add_nondefault>("--weights", - "Scorer weights"); + "Scorer weights"); cli.add("--output-sampling", - "Noise output layer with gumbel noise", - false); + "Noise output layer with gumbel noise", + false); // TODO: the options should be available only in server cli.add_nondefault("--port,-p", @@ -523,9 +525,9 @@ void ConfigParser::addOptionsScoring(cli::CLIWrapper& cli) { "Path to output file, stdout by default", "stdout"); cli.add>("--vocabs,-v", - "Paths to vocabulary files have to correspond to --train-sets." - " If this parameter is not supplied we look for vocabulary files source.{yml,json} and target.{yml,json}." - " If these files do not exists they are created"); + "Paths to vocabulary files have to correspond to --train-sets. " + "If this parameter is not supplied we look for vocabulary files source.{yml,json} and target.{yml,json}. " + "If these files do not exists they are created"); cli.add("--n-best", "Score n-best list instead of plain text corpus"); cli.add("--n-best-feature", @@ -552,7 +554,7 @@ void ConfigParser::addSuboptionsDevices(cli::CLIWrapper& cli) { // clang-format off cli.add>("--devices,-d", "Specifies GPU ID(s) to use for training. Defaults to 0..num-devices-1", - std::vector({"0"})); + {"0"}); cli.add_nondefault("--num-devices", "Number of GPUs to use for this process. Defaults to length(devices) or 1"); #ifdef USE_NCCL @@ -607,9 +609,12 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { cli.add("--shuffle-in-ram", "Keep shuffled corpus in RAM, do not write to temp file"); - cli.add>("--mini-batch-warmup", - "linear ramp-up of MB size, up to this #updates (append 't' for up to this #target labels);" - "optional second number is reference batch size at which to stop scaling up (instead of full batch size)", + cli.add("--mini-batch-words-ref", + "If given, the following hyper parameters are adjusted as-if we had this mini-batch size: " + "--learn-rate, --optimizer-params, --exponential-smoothing, --mini-batch-warmup"); + cli.add("--mini-batch-warmup", + "Linear ramp-up of MB size, up to this #updates (append 't' for up to this #target labels). " + "Auto-adjusted to --mini-batch-words-ref if given", {"0"}); cli.add("--mini-batch-track-lr", "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index 807135949..75501a370 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -6,8 +6,8 @@ namespace marian { -void Sgd::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { - actualMBSize, refMBSize; // (no correction for base update needed beyond using ce-sum) +void Sgd::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) { + actualMBSize, refMBWords; // (no correction for base update needed beyond using ce-sum) using namespace functional; Element(_1 -= eta_ * _2, params, @@ -18,8 +18,8 @@ void Sgd::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t re // Adagrad -void Adagrad::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { - ABORT_IF(actualMBSize != refMBSize, "Adagrad does not support rational hyper-parameter adjustment"); +void Adagrad::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) { + ABORT_IF(actualMBSize != refMBWords, "Adagrad does not support rational hyper-parameter adjustment"); if(!alloc_) alloc_ = New(params->getBackend()); @@ -124,7 +124,7 @@ void Adagrad::resetStats() { // Adam -void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) { +void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) { // lazy allocation if(!alloc_) alloc_ = New(params->getBackend()); @@ -138,7 +138,7 @@ void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t r vt_->set(0.f); } - double Tref = (double)refMBSize; + double Tref = (double)refMBWords; double T = (double)actualMBSize; // adjust for minibatch-size changes if Adam parameters are given a reference size (else do nothing) @@ -305,7 +305,7 @@ void Adam::resetStats() { if(vt_) vt_->set(0.f); - denom1_ = 0; // @BUGBUG: or 1 or refMBSize if so specified. Fix once we have proper parameterization for that. + denom1_ = 0; // @BUGBUG: or 1 or refMBWords if so specified. Fix once we have proper parameterization for that. denom2_ = 0; } @@ -314,6 +314,7 @@ Ptr Optimizer(Ptr options) { auto params = options->has("optimizer-params") ? options->get>("optimizer-params") : std::vector({}); + size_t refMBWordsParam = options->get("mini-batch-words-ref"); // adjust hyper-parameters as if our MB size (in target labels) was this value Ptr clipper = nullptr; float clipNorm = (float)options->get("clip-norm"); // @TODO: should this be ? @@ -323,13 +324,13 @@ Ptr Optimizer(Ptr options) { auto opt = options->get("optimizer"); if(opt == "sgd") { - return Optimizer(lrate, clipper, params); + return Optimizer(lrate, refMBWordsParam, clipper, params); } else if(opt == "adagrad") { - return Optimizer(lrate, clipper, params); + return Optimizer(lrate, refMBWordsParam, clipper, params); } else if(opt == "adam") { - return Optimizer(lrate, clipper, params); // @TODO: parse the parameters here, or just pass the options object + return Optimizer(lrate, refMBWordsParam, clipper, params); } else { - ABORT("Unknown optimizer: {}", opt); + ABORT("Unknown optimizer kind: {}", opt); } } } // namespace marian diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h index 76e2780eb..cc8bbae57 100755 --- a/src/optimizers/optimizers.h +++ b/src/optimizers/optimizers.h @@ -18,8 +18,16 @@ namespace marian { */ class OptimizerBase : public TrainingObserver { public: - OptimizerBase(float eta, Ptr clipper = nullptr) - : eta_(eta), clipper_(clipper) {} + OptimizerBase(float eta, size_t refMBWordsParam, Ptr clipper) + : eta_(eta), clipper_(clipper), refMBWordsParam_(refMBWordsParam) { + + // automatic learning-rate adjustment + // If users provide, in addition to the hyper-parameters, a reference minibatch size, + // that these hyper-parameters were originally tuned for, then the learning-rate gets + // adjusted accordingly. Note: Requires user to also use ce-sum criterion. + if (refMBWordsParam_ != 0) + LOG(info, "Note: Learning rate gets automatically adjusted as if minibatch size was {}", refMBWordsParam_); + } static constexpr size_t mbSizeNotProvided = SIZE_MAX; @@ -34,16 +42,16 @@ class OptimizerBase : public TrainingObserver { if(clipper_) clipper_->clip(grads); - size_t refMBSize = refMBSize_; - if (refMBSize == 0) { // optimizer not configured to use hyper-parameter auto-adjustment - refMBSize = mbSize = 1; // neutral settings that keep the standard behavior + size_t refMBWords = refMBWordsParam_; + if (refMBWords == 0) { // optimizer not configured to use hyper-parameter auto-adjustment + refMBWords = mbSize = 1; // neutral settings that keep the standard behavior } else { // optimizer is configured to auto-adjust hyper-parameters ABORT_IF(mbSize == mbSizeNotProvided, "Using rational optimizer auto-adjustment with trainer that does not provide MB size"); // note: this behavior is only meaningful if using the ce-sum criterion } - updateImpl(params, grads, mbSize, refMBSize); + updateImpl(params, grads, mbSize, refMBWords); } virtual void init(TrainingState& state) override { @@ -68,7 +76,7 @@ class OptimizerBase : public TrainingObserver { resetStats(); } - void setParams(const std::vector& params) { parseParams(params); } + virtual void setParams(const std::vector& params) = 0; typedef std::function::const_iterator /*begin*/, @@ -88,8 +96,7 @@ class OptimizerBase : public TrainingObserver { bool /*isMainProcess*/ = true) {} protected: - virtual void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) = 0; - virtual void parseParams(const std::vector& params) = 0; + virtual void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) = 0; virtual void resetStats() = 0; // Learning rate @@ -97,7 +104,7 @@ class OptimizerBase : public TrainingObserver { // Clip gradient norm Ptr clipper_; // Reference MB size. This enables automatic adjustment of optimizer hyper-parameters to MB size. - size_t refMBSize_{0}; // 0 means no adjustment + size_t refMBWordsParam_{0}; // 0 means no adjustment }; /** @@ -105,13 +112,13 @@ class OptimizerBase : public TrainingObserver { */ class Sgd : public OptimizerBase { public: - Sgd(float eta, Ptr clipper = nullptr) - : OptimizerBase(eta, clipper) {} + Sgd(float eta, size_t refMBWordsParam = 0, Ptr clipper = nullptr) + : OptimizerBase(eta, refMBWordsParam, clipper) {} + virtual void setParams(const std::vector& /*params*/) override {} private: - void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) override; - virtual void parseParams(const std::vector& /*params*/) override {} virtual void resetStats() override {} }; @@ -122,8 +129,8 @@ class Sgd : public OptimizerBase { */ class Adagrad : public OptimizerBase { public: - Adagrad(float eta, Ptr clipper = nullptr) - : OptimizerBase(eta, clipper) {} + Adagrad(float eta, size_t refMBWordsParam = 0, Ptr clipper = nullptr) + : OptimizerBase(eta, refMBWordsParam, clipper) {} void load(const std::string& name, const std::vector>& opts, @@ -134,15 +141,15 @@ class Adagrad : public OptimizerBase { const GatherStateFunc& gatherFn, bool /*isMainProcess*/ = true) override; -private: - void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; - void resetStats() override; - - void parseParams(const std::vector& params) override { + void setParams(const std::vector& params) override { if(params.size() > 0) eps_ = params[0]; } +private: + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) override; + void resetStats() override; + float eps_ = 1e-8f; Ptr alloc_; Tensor gt_; @@ -157,8 +164,8 @@ class Adagrad : public OptimizerBase { */ class Adam : public OptimizerBase { public: - Adam(float eta, Ptr clipper = nullptr) - : OptimizerBase(eta, clipper) {} + Adam(float eta, size_t refMBWordsParam = 0, Ptr clipper = nullptr) + : OptimizerBase(eta, refMBWordsParam, clipper) {} void load(const std::string& name, const std::vector>& opts, @@ -170,12 +177,12 @@ class Adam : public OptimizerBase { bool isMainProcess = true) override; private: - void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBSize) override; + void updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t refMBWords) override; void resetStats() override; // Adam parameters: - // [beta1, beta2, eps, w, refMBSize] - virtual void parseParams(const std::vector& params) override { + // [beta1, beta2, eps, w, refMBWords] + virtual void setParams(const std::vector& params) override { if(params.size() > 0) beta1_ = params[0]; if(params.size() > 1) @@ -186,15 +193,6 @@ class Adam : public OptimizerBase { // weighted decay for AdamW, to be explored, disabled by default if(params.size() > 3) w_ = params[3]; // default (disabled): 0 - - // automatic learning-rate adjustment - // If users provide, in addition to the hyper-parameters, a reference minibatch size, - // that these hyper-parameters were originally tuned for, then the learning-rate gets - // adjusted accordingly. Note: Requires user to also use ce-sum criterion. - if(params.size() > 4) { - refMBSize_ = (size_t)params[4]; // default (disabled): 0 - LOG(info, "Note: Modified Adam optimizer: automatically adjusting learning rate as if minibatch size was {}", refMBSize_); - } } // hyper-parameters @@ -214,10 +212,10 @@ class Adam : public OptimizerBase { }; template -Ptr Optimizer(float eta, +Ptr Optimizer(float eta, size_t refMBWordsParam = 0, Ptr clipper = nullptr, std::vector params = {}) { - auto opt = Ptr(new Algorithm(eta, clipper)); + auto opt = Ptr(new Algorithm(eta, refMBWordsParam, clipper)); opt->setParams(params); return opt; } diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h index fb1d3b4ac..b018fe9ea 100755 --- a/src/training/exponential_smoothing.h +++ b/src/training/exponential_smoothing.h @@ -14,11 +14,8 @@ namespace marian { class ExponentialSmoothing { public: ExponentialSmoothing(Ptr options) { - auto args = options->get>("exponential-smoothing"); - ABORT_IF(args.size() < 1 || args.size() > 2, "exponential-smoothing parameter must be one or two numbers"); - mvDecayBy_ = args[0]; - if (args.size() > 1) - refBatchTrgWords_ = (size_t)args[1]; + mvDecayBy_ = options->get("exponential-smoothing"); + refBatchTrgWords_ = options->get("mini-batch-words-ref"); // adjust as if our MB size (in target labels) was this value mvAvg_ = (mvDecayBy_ > 0); } diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 783845c8c..2176e85d7 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -70,9 +70,7 @@ class Scheduler : public TrainingObserver { // determine dynamic MB size, if respective parameters are given (return false if not) bool tryGetDynamicMBSizeMultiplier(double /*out*/ &ratio) const { - auto mbWarmupOpts = options_->get>("mini-batch-warmup"); - ABORT_IF(mbWarmupOpts.empty() || mbWarmupOpts.size() > 2, "--mini-batch-warmup argument must be one or two numbers with units"); - auto mbWarmup = SchedulingParameter::parse(mbWarmupOpts[0]); + auto mbWarmup = SchedulingParameter::parse(options_->get("mini-batch-warmup")); if (!mbWarmup) return false; @@ -89,13 +87,14 @@ class Scheduler : public TrainingObserver { // apply ratio to actual batch size ratio *= progressRatio; + // @TODO: move this out // adjust for reference batch size if given // At progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever // the actual batch size is. We approximate the latter as typicalTrgBatchWords_, and scale ratio accordingly. - if (mbWarmupOpts.size() > 1) { - ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences - auto refBatchLabels = (size_t)std::stoull(mbWarmupOpts[1]); + auto refBatchLabels = options_->get("mini-batch-words-ref"); + if (refBatchLabels != 0) { LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords_); + ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords_; } From 12fe1a654919174a4d753f0bc84cf77a34413312 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 14:10:28 -0800 Subject: [PATCH 037/838] bug fix: optimizer-delay is now a double, and should be read out as such --- src/optimizers/optimizers.h | 6 +++--- src/training/exponential_smoothing.h | 2 +- src/training/graph_group_async.cpp | 3 ++- src/training/graph_group_multinode_sync.h | 2 +- src/training/graph_group_sync.cpp | 5 +++-- src/training/graph_group_sync.h | 2 +- src/training/scheduler.h | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h index cc8bbae57..51e43f871 100755 --- a/src/optimizers/optimizers.h +++ b/src/optimizers/optimizers.h @@ -19,7 +19,7 @@ namespace marian { class OptimizerBase : public TrainingObserver { public: OptimizerBase(float eta, size_t refMBWordsParam, Ptr clipper) - : eta_(eta), clipper_(clipper), refMBWordsParam_(refMBWordsParam) { + : eta_(eta), refMBWordsParam_(refMBWordsParam), clipper_(clipper) { // automatic learning-rate adjustment // If users provide, in addition to the hyper-parameters, a reference minibatch size, @@ -101,10 +101,10 @@ class OptimizerBase : public TrainingObserver { // Learning rate float eta_; - // Clip gradient norm - Ptr clipper_; // Reference MB size. This enables automatic adjustment of optimizer hyper-parameters to MB size. size_t refMBWordsParam_{0}; // 0 means no adjustment + // Clip gradient norm + Ptr clipper_; }; /** diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h index b018fe9ea..e24c1e74b 100755 --- a/src/training/exponential_smoothing.h +++ b/src/training/exponential_smoothing.h @@ -37,6 +37,6 @@ class ExponentialSmoothing { bool mvAvg_{false}; float mvDecayBy_{1e-4f}; // decay prior model by this factor - size_t refBatchTrgWords_{0}; // mvDecayBy_ is specified for this batch size (in target words) + size_t refBatchTrgWords_{0}; // mvDecayBy_ is specified for this batch size (in target words) (0 means not specified) }; } // namespace marian diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index b272a4b7f..e5547cb40 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -10,7 +10,8 @@ AsyncGraphGroup::AsyncGraphGroup(Ptr config) ExponentialSmoothing(options_), devices_{Config::getDevices(options_)}, shardSync_(devices_.size()), - optimizerDelay_{options_->get("optimizer-delay")} { + optimizerDelay_((size_t)options_->get("optimizer-delay")) { + ABORT_IF((double)optimizerDelay_ != options_->get("optimizer-delay"), "AsyncGraphGroup does not support fractional values for --optimizer-delay"); pool_.reset(new ThreadPool(devices_.size(), devices_.size())); for(auto device : devices_) { diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index 7685b6789..c4b3683d2 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -133,7 +133,7 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { */ MultiNodeGraphGroupSync(Ptr options) : Base(options), - tau_{options_->get("optimizer-delay")}, + tau_{(size_t)options_->get("optimizer-delay")}, syncOptimizer_{Optimizer(options_)}, movingAvg_{options_->get("exponential-smoothing") > 0}, mvDecay_{options_->get("exponential-smoothing")} { diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index cedf5c54e..e96e85607 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -5,7 +5,7 @@ namespace marian { SyncGraphGroup::SyncGraphGroup(Ptr config) : GraphGroup(config), ExponentialSmoothing(config), - delay_{options_->get("optimizer-delay")} { // @TODO: rename to something else; delay means delayed updated, not accumulation + delay_{options_->get("optimizer-delay")} { // @TODO: rename to something else; delay means delayed updated, not accumulation mpi_ = initMPI(/*multiThreaded=*/false); // when not running under MPI, this will be a fake object that represents a one-MPI-process setup @@ -153,7 +153,8 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector comm_; // [not null] communicator, e.g. NCCLCommunicator Ptr mpi_; // [not null] all MPI-like communication goes through this (this is a dummy implementation if no MPI run) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 2176e85d7..873086dda 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -87,7 +87,7 @@ class Scheduler : public TrainingObserver { // apply ratio to actual batch size ratio *= progressRatio; - // @TODO: move this out + // @TODO: move this out, and just return ratio instead // adjust for reference batch size if given // At progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever // the actual batch size is. We approximate the latter as typicalTrgBatchWords_, and scale ratio accordingly. From 0cf43717f476be89d642506b837009e4a52c562d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 14:16:29 -0800 Subject: [PATCH 038/838] bug fix: optimizer-delay is now a double, and should be read out as such --- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_multinode.h | 2 +- src/training/graph_group_sync.cpp | 13 ++++++++++++- src/training/scheduler.h | 15 ++++----------- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index e5547cb40..4c85e168a 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -11,7 +11,7 @@ AsyncGraphGroup::AsyncGraphGroup(Ptr config) devices_{Config::getDevices(options_)}, shardSync_(devices_.size()), optimizerDelay_((size_t)options_->get("optimizer-delay")) { - ABORT_IF((double)optimizerDelay_ != options_->get("optimizer-delay"), "AsyncGraphGroup does not support fractional values for --optimizer-delay"); + ABORT_IF((double)optimizerDelay_ != options_->get("optimizer-delay"), "AsyncGraphGroup presently does not implement fractional values for --optimizer-delay"); pool_.reset(new ThreadPool(devices_.size(), devices_.size())); for(auto device : devices_) { diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index 39a20e062..873155df7 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -354,7 +354,7 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { MultiNodeGraphGroup(Ptr options) : Base(options), clientCommOverlap{options_->get("multi-node-overlap")}, - tau_{options_->get("optimizer-delay")} { } + tau_{(size_t)options_->get("optimizer-delay")} { } /** * (Destructor) Shut down server shard thread and (if comm. overlap enabled) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index e96e85607..44db971d5 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -154,7 +154,18 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectorget("mini-batch-words-ref"); + if (refBatchLabels != 0) { + auto typicalTrgBatchWords = scheduler_->getTypicalTrgBatchWords(); + LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords); + ABORT_IF(typicalTrgBatchWords == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences + ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords; + } + if (pendingBatches_.size() < ratio) return false; // not enough data yet diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 873086dda..08cd753a1 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -63,10 +63,14 @@ class Scheduler : public TrainingObserver { } public: + // @TODO: move this out from here void setTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for tryGetDynamicMBSizeMultiplier() typicalTrgBatchWords_ = typicalTrgBatchWords; LOG(info, "batch size estimate is {} target words", typicalTrgBatchWords_); } + size_t getTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for tryGetDynamicMBSizeMultiplier() + return typicalTrgBatchWords; + } // determine dynamic MB size, if respective parameters are given (return false if not) bool tryGetDynamicMBSizeMultiplier(double /*out*/ &ratio) const { @@ -87,17 +91,6 @@ class Scheduler : public TrainingObserver { // apply ratio to actual batch size ratio *= progressRatio; - // @TODO: move this out, and just return ratio instead - // adjust for reference batch size if given - // At progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever - // the actual batch size is. We approximate the latter as typicalTrgBatchWords_, and scale ratio accordingly. - auto refBatchLabels = options_->get("mini-batch-words-ref"); - if (refBatchLabels != 0) { - LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords_); - ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences - ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords_; - } - // dynamic MB-size tracking with learning rate // As LR goes down, MB gets ramped up by the same ratio, which has been found to be safe. auto mbTracking = options_->get("mini-batch-track-lr"); From 10438e87c436fc9e3244dff18732fb3c2bbefebe Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 14:35:45 -0800 Subject: [PATCH 039/838] typicalTrgBatchWords_ moved from Scheduler to SyncGraphGroup --- src/training/graph_group.h | 5 +++++ src/training/graph_group_sync.cpp | 31 +++++++++++++------------------ src/training/scheduler.h | 10 ---------- src/training/training.h | 11 +++++++---- 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 46a317c32..94fad7a86 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -21,6 +21,7 @@ class GraphGroup { Ptr opt_; // the optimizer Ptr scheduler_; // scheduler that keeps track of how much has been processed bool finalized_{false}; // 'true' if training has completed (further updates are no longer allowed) + size_t typicalTrgBatchWords_{ 0 }; // for dynamic batch sizing public: GraphGroup(Ptr options) : options_(options), opt_(Optimizer(options)) {} @@ -110,6 +111,10 @@ class GraphGroup { } return stats; } + + void setTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for dynamic MB scaling + typicalTrgBatchWords_ = typicalTrgBatchWords; + } }; /** diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 44db971d5..5d2327e37 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -57,10 +57,10 @@ void SyncGraphGroup::initialize(const Ptr& exampleBatch) { ThreadPool pool(graphs_.size() - 1, graphs_.size() - 1); for(size_t i = 1; i < graphs_.size(); ++i) { auto init = [&](size_t i) { - // initialize t-th graph and weights + // initialize i-th graph and weights builders_[i]->build(graphs_[i], exampleBatch); graphs_[i]->forward(); - // overwrite weights of t-th graph with weights from 0th graph + // overwrite weights of i-th graph with weights from 0-th graph graphs_[i]->params()->vals()->copyFrom(graphs_[0]->params()->vals()); }; pool.enqueue(init, i); @@ -143,10 +143,6 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectornumMPIProcesses(); // warp := set of batches processed concurrently across GPus and workers - size_t pendingTrgWords = 0; // diagnosics only: compute how many target labels are pending so far - for (const auto& batch : pendingBatches_) - pendingTrgWords += batch->wordsTrg(); - // MB-size warm-up and dynamic scaling double ratio; bool isDynamic = scheduler_->tryGetDynamicMBSizeMultiplier(ratio); @@ -160,10 +156,9 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectorget("mini-batch-words-ref"); if (refBatchLabels != 0) { - auto typicalTrgBatchWords = scheduler_->getTypicalTrgBatchWords(); - LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords); - ABORT_IF(typicalTrgBatchWords == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences - ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords; + LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords_); + ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences + ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords_; } if (pendingBatches_.size() < ratio) @@ -243,11 +238,11 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { return; // Helper to access the subBatches array - auto getSubBatch = [&](size_t t, size_t localDeviceIndex, size_t rank) -> Ptr { - // 't' (the delay) should be slowest changing dimension. If subBatches are sorted by + auto getSubBatch = [&](size_t delay, size_t localDeviceIndex, size_t rank) -> Ptr { + // 'delay' should be slowest changing dimension. If subBatches are sorted by // length, then grouping sentences of similar length into the same delay step can // reduce unnecessary time spent in padding. - auto index = (t * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex; + auto index = (delay * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex; if (index < subBatches.size()) return subBatches[index]; else @@ -267,27 +262,27 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // Compute gradients // This happens in multiple steps in case of delay > 1. std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device - for (size_t t = 0; getSubBatch(t, 0, 0); t++) { // @TODO: rename 't' to 'delay' + for (size_t delay = 0; getSubBatch(delay, 0, 0); delay++) { // Execute single forward/backward step auto forwardBackward = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { auto graph = graphs_[localDeviceIndex]; - auto subBatch = getSubBatch(t, localDeviceIndex, mpi_->myMPIRank()); + auto subBatch = getSubBatch(delay, localDeviceIndex, mpi_->myMPIRank()); if(subBatch) { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); localDeviceCosts[localDeviceIndex] += costNode->scalar(); - graph->backward(/*zero=*/t == 0); // only reset gradients to 0 if t = 0 + graph->backward(/*zero=*/delay == 0); // only reset gradients to 0 if delay = 0 } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets #if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead graph->params()->allocateBackward(); - if (t == 0) // these have already been sized + if (delay == 0) // these have already been sized graph->params()->set_zero_adjoint(); #else graph->clear(); // instead of build() graph->forward(); - graph->backward(/*zero=*/t == 0); + graph->backward(/*zero=*/delay == 0); #endif } }; diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 08cd753a1..f1b65bd0e 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -13,7 +13,6 @@ class Scheduler : public TrainingObserver { std::vector> validators_; bool first_{true}; - size_t typicalTrgBatchWords_{0}; // for dynamic batch sizing Ptr state_; @@ -63,15 +62,6 @@ class Scheduler : public TrainingObserver { } public: - // @TODO: move this out from here - void setTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for tryGetDynamicMBSizeMultiplier() - typicalTrgBatchWords_ = typicalTrgBatchWords; - LOG(info, "batch size estimate is {} target words", typicalTrgBatchWords_); - } - size_t getTypicalTrgBatchWords(size_t typicalTrgBatchWords) { // needed for tryGetDynamicMBSizeMultiplier() - return typicalTrgBatchWords; - } - // determine dynamic MB size, if respective parameters are given (return false if not) bool tryGetDynamicMBSizeMultiplier(double /*out*/ &ratio) const { auto mbWarmup = SchedulingParameter::parse(options_->get("mini-batch-warmup")); diff --git a/src/training/training.h b/src/training/training.h index 174768542..17512efd4 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -58,10 +58,10 @@ class Train : public ModelTask { auto batchGenerator = New(dataset, options_, stats); scheduler->registerTrainingObserver(batchGenerator); - scheduler->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); // needed for dynamic MB scaling auto model = New(options_); model->setScheduler(scheduler); + model->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); // needed for dynamic MB scaling model->load(); // @TODO: shuffle_ as a private attribute in BG @@ -69,16 +69,19 @@ class Train : public ModelTask { bool restored = !options_->get("no-restore-corpus") && batchGenerator->restore(trainState, shuffle); + // -- main training loop scheduler->started(); while(scheduler->keepGoing()) { if(!restored) batchGenerator->prepare(shuffle); restored = false; - // @TODO: try to use for(auto ...) - for(auto batchIt = std::begin(*batchGenerator); - batchIt != std::end(*batchGenerator) && scheduler->keepGoing(); + // main training loop for one epoch + for(auto batchIt = std::begin(*batchGenerator); // @TODO: try to use for(auto ...) + batchIt != std::end(*batchGenerator); batchIt++) { + if (!scheduler->keepGoing()) + break; model->update(*batchIt); } From 2cd890809871581f5bdd1993f3439f1a2404aadd Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 17:49:01 -0800 Subject: [PATCH 040/838] streamlined all cases for dynamic scaling and read sizes --- src/common/config_parser.cpp | 2 +- src/data/batch_stats.h | 4 +- src/training/graph_group.h | 4 +- src/training/graph_group_multinode_sync.h | 2 +- src/training/graph_group_sync.cpp | 133 +++++++++++++--------- src/training/graph_group_sync.h | 5 +- src/training/scheduler.h | 38 ++++--- src/training/training.h | 9 +- 8 files changed, 118 insertions(+), 79 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 377e8d2e6..b5ea081e8 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -308,7 +308,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Parameters for optimization algorithm, e.g. betas for Adam. " "Auto-adjusted to --mini-batch-words-ref if given"); cli.add("--optimizer-delay", - "SGD update delay, 1 = no delay. Can be fractional, e.g. 0.1 to use only 10% of each batch", + "SGD update delay (#batches between updates). 1 = no delay. Can be fractional, e.g. 0.1 to use only 10% of each batch", 1.); cli.add("--sync-sgd", diff --git a/src/data/batch_stats.h b/src/data/batch_stats.h index 581f83df2..2fc642938 100755 --- a/src/data/batch_stats.h +++ b/src/data/batch_stats.h @@ -39,11 +39,11 @@ class BatchStats { return it->second; } - void add(Ptr batch, size_t multiplier = 1) { + void add(Ptr batch, double multiplier = 1.) { std::vector lengths; for(size_t i = 0; i < batch->sets(); ++i) lengths.push_back((*batch)[i]->batchWidth()); - size_t batchSize = batch->size() * multiplier; + size_t batchSize = (size_t)ceil((double)batch->size() * multiplier); if(map_[lengths] < batchSize) map_[lengths] = batchSize; diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 94fad7a86..c1d6c818a 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -21,7 +21,7 @@ class GraphGroup { Ptr opt_; // the optimizer Ptr scheduler_; // scheduler that keeps track of how much has been processed bool finalized_{false}; // 'true' if training has completed (further updates are no longer allowed) - size_t typicalTrgBatchWords_{ 0 }; // for dynamic batch sizing + size_t typicalTrgBatchWords_{ 0 }; // for dynamic batch sizing: typical batch size in words public: GraphGroup(Ptr options) : options_(options), opt_(Optimizer(options)) {} @@ -56,7 +56,7 @@ class GraphGroup { // @TODO: Can this be made const? It seems wrong to have a stateful method that still returns a result. virtual Ptr collectStats(Ptr graph, Ptr model, - size_t multiplier = 1) { + double multiplier = 1.) { auto stats = New(); size_t numFiles diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index c4b3683d2..7b93b2a84 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -222,7 +222,7 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { */ Ptr collectStats() { return GraphGroup::collectStats( - clientGraphs_[0], clientBuilders_[0], devices_.size()); + clientGraphs_[0], clientBuilders_[0], (double)devices_.size()); } }; } // namespace marian diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 5d2327e37..f25e321c3 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -108,9 +108,17 @@ void SyncGraphGroup::initializeAvg() { } Ptr SyncGraphGroup::collectStats() { - // @TODO: This is an incompatible change. Decide how to handle that. - //size_t multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_; - return GraphGroup::collectStats(graphs_[0], builders_[0]/*, multiplier*/); + // This function determines the granularity in which the reader provides data. + // If no mini-batch-fit, then user provides a constant number. It reads that much. We won't get into this function. + // If mini-batch-fit, then we get here and set miniBatchFitMultiplier_. Then... + // If dynamic MB scaling, then we want fine-grained minibatches of the size of one GPU. + // If not, we prefer a single large batch that can be split into equal-size parts over GPUs, + // so that we have perfect load balancing and read precisely as much as we need (no waste). + double multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_; + bool isDynamic = scheduler_->isDynamicMBSizeScaling(); + double readerMultiplier = isDynamic ? 1. : multiplier; // multiplier applied already by reader + updateMultiplier_ = isDynamic ? multiplier : 1.; // multiplier applied later in update() + return GraphGroup::collectStats(graphs_[0], builders_[0], readerMultiplier); } // helper for MB scaling: quantize the ratio with a given error margin @@ -140,76 +148,99 @@ static double roundUpRatio(double ratio) { // It adds 'newBatch' to 'pendingBatches_', and if sufficient batches have been queued, then // returns 'pendingBatches_' in 'subBatches' and resets it. If not, it returns false. bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector>& subBatches) { - pendingBatches_.push_back(newBatch); + // The reader delivers in chunks of these sizes, according to case: + // - no dynamic MB-size scaling: + // - reader batch size = update batch size, with... + // - mini-batch-fit: + // - update batch size = what fits into all GPUs, times decay_ to allow experimenting with fractional sizes + // - no mini-batch-fit: + // - update batch size = user-specified size (user guarantees that it fits if distributed over delay_ GPUs) + // - dynamic MB-size scaling: + // - update batch size = aggregate reader batch size * (dynamic progress-based ratio * reference adjustment), with... + // - mini-batch-fit: + // - aggregate reader batch size = equal to what fits into one GPU * warpSize * delay_ + // - no mini-batch-fit: + // - aggregate reader batch size = user-specified size (user guarantees that it fits if distributed over delay_ GPUs) + // - reference adjustment = + // - reference batch size specified: (reference batch size / typical aggregate reader batch size) + // - no ref size specified: 1 + size_t warpSize = devices_.size() * mpi_->numMPIProcesses(); // warp := set of batches processed concurrently across GPus and workers - // MB-size warm-up and dynamic scaling - double ratio; - bool isDynamic = scheduler_->tryGetDynamicMBSizeMultiplier(ratio); - if (isDynamic) - ratio = roundUpRatio(ratio); // round up to full batches if within a certain error margin --@BUGBUG: Not invariant w.r.t. GPU size, as ratio is relative to what fits into 1 GPU - else // if dynamic scaling not enabled, then fill each GPU with a batch - ratio = delay_ * (double)warpSize; // note: delay_ may be fractional - - // adjust for reference batch size if given - // At progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever - // the actual batch size is. We approximate the latter as typicalTrgBatchWords, and scale ratio accordingly. + // if not dynamic then return the big batch, but first split it over GPUs as it may be too large + if (!scheduler_->isDynamicMBSizeScaling()) { + // If mini-batch-fit, then the read batch is (devices_.size() * mpi_->numMPIProcesses() * delay_) + // times what fits one GPU. If not mini-batch-fit, it is whatever the user has specified, which + // is the user's responsibility to guarantee that it fits into 'delay_' warps. + // Distribute evenly over all GPUs we have, using multiple warps if needed. + size_t numWarps = (size_t)ceil(delay_); + subBatches = newBatch->split(numWarps * warpSize); + return true; + } + LOG_ONCE(info, "[training] Dynamic mini-batch scaling enabled"); + + // if dynamic and mini-batch-fit, then we get batches in the size of what fits into one GPU + pendingBatches_.push_back(newBatch); + + // what ratio do we want, based on current training progress schedule? + double ratio = scheduler_->getDynamicMBSizeMultiplier(); + + // relative to what base? (what does ratio == 1 mean) + ratio *= updateMultiplier_; // if mini-batch-fit, this is = warpSize * delay_, otherwise 1 + + // If a reference is given, then at progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever + // the actual batch size is. Since we cannot know the future actual batch sizes that will be delivered + // by the reader, we approximate them with (typicalTrgBatchWords * updateMultiplier), and scale ratio accordingly. auto refBatchLabels = options_->get("mini-batch-words-ref"); if (refBatchLabels != 0) { - LOG_ONCE(info, "[scheduler] Scaling to {} reference labels. Typical actual batch words is {}", refBatchLabels, typicalTrgBatchWords_); - ABORT_IF(typicalTrgBatchWords_ == 0, "dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences - ratio *= (double)refBatchLabels / (double)typicalTrgBatchWords_; + LOG_ONCE(info, "[scheduler] Scaling to {} reference labels, using actual-batch-word estimate of {}", refBatchLabels, typicalTrgBatchWords_); + ABORT_IF(typicalTrgBatchWords_ == 0, "Dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences + ratio *= (double)refBatchLabels / (double)(typicalTrgBatchWords_ * updateMultiplier_); } + // round up to full batches if within a certain error margin --@BUGBUG: Not invariant w.r.t. GPU size, as ratio is relative to what fits into 1 GPU + ratio = roundUpRatio(ratio); + if (pendingBatches_.size() < ratio) return false; // not enough data yet // now we have enough to fill at least 'ratio' batches - if (pendingBatches_.size() == ratio) - return true; // nothing to do, e.g. warm-up not enabled - - // warm-up is happening - LOG_ONCE(info, "[training] Mini-batch-warmup enabled"); + // @BUGBUG: We do not handle the case that fixed MB size * ratio exceeds GPU memory (we'd need to split that). - // shorten all batches a little to accurately reflect ratio - // e.g. ratio = 3.3 for 4 batches: Reduce each by 3.3/4 + // in fact, we got too much, so make up for it by shortening all batches to accurately reflect desried ratio + // e.g. ratio = 3.3 for 4 batches -> Reduce each by 3.3/4 // Alternatively, we could just shorten the last 'warp', but that would not be invariant to warp size. - size_t before = 0, after = 0; for (auto& batch : pendingBatches_) { auto reducedBatchSize = (size_t)ceil((double)batch->size() * ratio / (double)pendingBatches_.size()); size_t minSize = 1; if (pendingBatches_.size() == 1) { // enforce a minimum (only needed/correct if still in first batch) - size_t minTrgWords = 256; // don't go below this number of target words, as it seems excessive --@TODO: parameterize? + size_t minTrgWords = 256; // don't go below this number of target words, as it seems excessive --@TODO: parameterize? minSize = 1 + (minTrgWords * batch->size() - 1) / batch->wordsTrg(); // approximately convert minTrgWords into a #sentences } reducedBatchSize = std::max(reducedBatchSize, minSize); - before += batch->wordsTrg(); if (reducedBatchSize < batch->size()) batch = batch->split(/*numSubBatches=*/1, reducedBatchSize).front(); - after += batch->wordsTrg(); } // load-balance: distribute the last numWarps-group's batches over GPUs // This is tricky since batches do not have the same length, therefore we can only split, but not merge. auto numWarps = (pendingBatches_.size() - 1) / warpSize + 1; // = ceil(#buffers / (#GPUs * #workers)) - auto availableBatches = numWarps * warpSize; // we got this many GPUs anyways, so we better make use of them - if (pendingBatches_.size() < availableBatches) { - // we are not using all available GPUs -> try to load-balance a bit better - auto fullBatches = (numWarps - 1) * warpSize; - auto expandLast = pendingBatches_.size() - fullBatches; - auto toLast = availableBatches - fullBatches; - LOG(info, "attempt to redistribute {} last batches over {}", expandLast, toLast); - auto splitInto = toLast / expandLast; // unfortunately we can only split in integer ratios - // @TODO: We can do better since the last batch is typically smaller. - if (splitInto > 1) { + auto availableGPUSlots = numWarps * warpSize; // we will run this many GPUs: better use them all + if (pendingBatches_.size() < availableGPUSlots) { + // last warp does not use all available GPUs: try to re-balance + auto fullWarpsBatches = (numWarps - 1) * warpSize; // number of batches in all but the last warp. Those warps that are fully used. + auto lastWarpSize = pendingBatches_.size() - fullWarpsBatches; // the last warp is possibly not fully used + LOG(info, "attempt to redistribute {} last batches over {}", lastWarpSize, warpSize); + auto splitInto = warpSize / lastWarpSize; + if (splitInto > 1) { // unfortunately we can only split in integer ratios // split each of last numWarps's batches into 'splitInto' batches // pop them first std::vector> batchesToSplit; - while (pendingBatches_.size() > fullBatches) { + while (pendingBatches_.size() > fullWarpsBatches) { batchesToSplit.push_back(pendingBatches_.back()); pendingBatches_.pop_back(); } - // now split them + // now split them and push them back for (auto& batchToSplit : batchesToSplit) { LOG(info, "{}-way splitting batchToSplit with size {}", splitInto, batchToSplit->size()); auto splitBatches = batchToSplit->split(splitInto); @@ -219,7 +250,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector availableBatches, "somehow split into too many batches??"); + ABORT_IF(pendingBatches_.size() > availableGPUSlots, "somehow split into too many batches??"); } subBatches = std::move(pendingBatches_); @@ -238,15 +269,15 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { return; // Helper to access the subBatches array - auto getSubBatch = [&](size_t delay, size_t localDeviceIndex, size_t rank) -> Ptr { - // 'delay' should be slowest changing dimension. If subBatches are sorted by + auto getSubBatch = [&](size_t warp, size_t localDeviceIndex, size_t rank) -> Ptr { + // Warp should be slowest changing dimension. If subBatches are sorted by // length, then grouping sentences of similar length into the same delay step can // reduce unnecessary time spent in padding. - auto index = (delay * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex; + auto index = (warp * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex; if (index < subBatches.size()) return subBatches[index]; else - return nullptr; + return nullptr; // null if we reached beyond the end }; // Upon very first execution, reset everything @@ -262,27 +293,27 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // Compute gradients // This happens in multiple steps in case of delay > 1. std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device - for (size_t delay = 0; getSubBatch(delay, 0, 0); delay++) { + for (size_t warp = 0; getSubBatch(warp, 0, 0); warp++) { // Execute single forward/backward step auto forwardBackward = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { auto graph = graphs_[localDeviceIndex]; - auto subBatch = getSubBatch(delay, localDeviceIndex, mpi_->myMPIRank()); + auto subBatch = getSubBatch(warp, localDeviceIndex, mpi_->myMPIRank()); if(subBatch) { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); localDeviceCosts[localDeviceIndex] += costNode->scalar(); - graph->backward(/*zero=*/delay == 0); // only reset gradients to 0 if delay = 0 + graph->backward(/*zero=*/warp == 0); // only reset gradients to 0 if warp = 0 } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets #if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead graph->params()->allocateBackward(); - if (delay == 0) // these have already been sized + if (warp == 0) // these have already been sized graph->params()->set_zero_adjoint(); #else graph->clear(); // instead of build() graph->forward(); - graph->backward(/*zero=*/delay == 0); + graph->backward(/*zero=*/warp == 0); #endif } }; diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 45f90112e..6bb3a4861 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -26,8 +26,9 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { // state for update() bool first_{ true }; // gets interpreted and cleared by update() - std::vector> pendingBatches_; // in case of delay, multi-worker, and/or multi-GPU, we buffer up batches - size_t typicalTrgWords_{}; // typical batch size in words (labels); remembered from collectStats() + std::vector> pendingBatches_; // in case of dynamic MB-size scaling, we temporarly buffer up batches across update() calls until enough + size_t typicalTrgWords_{}; // typical batch size in words (labels), 0 if unknown (e.g. specified in sentences) + double updateMultiplier_{1}; // multiplier not applied in collectStats() (no multiplier if not mini-batch-fit) void initialize(const Ptr& exampleBatch); void initializeAvg(); diff --git a/src/training/scheduler.h b/src/training/scheduler.h index f1b65bd0e..802282354 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -62,24 +62,30 @@ class Scheduler : public TrainingObserver { } public: - // determine dynamic MB size, if respective parameters are given (return false if not) - bool tryGetDynamicMBSizeMultiplier(double /*out*/ &ratio) const { + // test if any parameters specify dynamic MB scaling + bool isDynamicMBSizeScaling() const { auto mbWarmup = SchedulingParameter::parse(options_->get("mini-batch-warmup")); - if (!mbWarmup) - return false; + auto mbTracking = options_->get("mini-batch-track-lr"); + return mbWarmup || mbTracking; + } - ratio = 1.0; - // mini-batch-warmup - LOG_ONCE(info, "[scheduler] Mini-batch size warmup {}", std::string(mbWarmup)); + // determine dynamic MB scaling factor + double getDynamicMBSizeMultiplier() const { + double ratio = 1.0; - // This scales MB size up from the start. - // now scale batch size relative to progress within warm-up period - size_t progress = state_->getProgressIn(mbWarmup.unit); // number of updates/labels processed - auto progressRatio = (double)progress / (double)mbWarmup.n; // where are we relatively within target warm-up period - if (mbWarmup.unit == SchedulingUnit::trgLabels) - progressRatio = std::sqrt(progressRatio); - // apply ratio to actual batch size - ratio *= progressRatio; + auto mbWarmup = SchedulingParameter::parse(options_->get("mini-batch-warmup")); + if (mbWarmup) { + // mini-batch-warmup + LOG_ONCE(info, "[scheduler] Mini-batch size warmup {}", std::string(mbWarmup)); + // This scales MB size up from the start, relative to progress within warm-up period. + size_t progress = state_->getProgressIn(mbWarmup.unit); // number of updates/labels processed + auto progressRatio = (double)progress / (double)mbWarmup.n; // where are we relatively within target warm-up period + // if unit is labels, then account for the fact that our increment itself is not constant + if (mbWarmup.unit == SchedulingUnit::trgLabels) + progressRatio = std::sqrt(progressRatio); + // apply ratio to actual batch size + ratio *= progressRatio; + } // dynamic MB-size tracking with learning rate // As LR goes down, MB gets ramped up by the same ratio, which has been found to be safe. @@ -90,7 +96,7 @@ class Scheduler : public TrainingObserver { LOG_ONCE(info, "[scheduler] Dynamic mini-batch size adjustment enabled and kicking in"); ratio /= lrFactor; } - return true; + return ratio; } Scheduler(Ptr options, Ptr state) diff --git a/src/training/training.h b/src/training/training.h index 17512efd4..401f6bdf0 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -34,6 +34,9 @@ class Train : public ModelTask { dataset->prepare(); + auto trainState = New(options_->get("learn-rate")); + auto scheduler = New(options_, trainState); + Ptr stats; if(options_->get("mini-batch-fit")) { LOG(info, @@ -42,13 +45,11 @@ class Train : public ModelTask { options_->get("mini-batch-fit-step")); // @TODO, better fake batch with vocabulary auto model = New(options_); - THREAD_GUARD(stats = model->collectStats()); + model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling + stats = model->collectStats(); LOG(info, "[batching] Done"); } - auto trainState = New(options_->get("learn-rate")); - auto scheduler = New(options_, trainState); - if((options_->has("valid-sets") || options_->has("valid-script-path")) && SchedulingParameter::parse(options_->get("valid-freq"))) { for(auto validator : Validators(dataset->getVocabs(), options_)) From fbf6aa4bebc3dee78b09a88799bc295330b204ba Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 17:57:26 -0800 Subject: [PATCH 041/838] (minor) --- src/data/batch_stats.h | 3 +-- src/training/graph_group_sync.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/data/batch_stats.h b/src/data/batch_stats.h index 2fc642938..bba055d5b 100755 --- a/src/data/batch_stats.h +++ b/src/data/batch_stats.h @@ -56,8 +56,7 @@ class BatchStats { for (const auto& entry : map_) { auto maxTrgLength = entry.first.back(); auto numSentences = entry.second; - auto numLabels = numSentences * maxTrgLength; - sum += numLabels; + sum += numSentences * maxTrgLength; } return sum / map_.size(); } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index f25e321c3..107c9a4bf 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -182,7 +182,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectorgetDynamicMBSizeMultiplier(); // relative to what base? (what does ratio == 1 mean) From b31c9eecd9d217c831777c1db581e59165840356 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 18:35:52 -0800 Subject: [PATCH 042/838] moved gradient resetting into scatterReduce function, in prep for residuals --- src/graph/parameters.h | 1 + src/training/communicator.h | 8 ++++---- src/training/communicator_nccl.h | 14 +++++++++++--- src/training/graph_group_sync.cpp | 12 ++++++------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/graph/parameters.h b/src/graph/parameters.h index 32f88a1e4..71c538fcd 100755 --- a/src/graph/parameters.h +++ b/src/graph/parameters.h @@ -73,6 +73,7 @@ class Parameters { for(auto p : params_) if(!p->grad()) grads_->allocate(p->grad(), p->shape()); + grads()->set(0.f); } } diff --git a/src/training/communicator.h b/src/training/communicator.h index 7e705cfac..4190401a8 100755 --- a/src/training/communicator.h +++ b/src/training/communicator.h @@ -37,8 +37,8 @@ class ICommunicator { virtual void foreach(const ForeachFunc& func, bool parallel = true) const = 0; // @TODO: We probably can still share foreach() between the two implementations. Just need to move some helper functions from the .cu file. - virtual void scatterReduce() const = 0; // reduce param gradients and scatter into gradient shards - virtual void allGather() const = 0; // redistribute value shards into param values + virtual void scatterReduceAndResetGrads() const = 0; // reduce param gradients and scatter into gradient shards + virtual void allGatherParams() const = 0; // redistribute value shards into param values virtual void swapParams(const std::vector& paramShards) const = 0; @@ -153,7 +153,7 @@ class DefaultCommunicator : public ICommunicator { t.join(); } - void scatterReduce() const override { + void scatterReduceAndResetGrads() const override { const_cast(this)->lazyInit(); int totalSize = (int)graphs_[0]->params()->vals()->size(); @@ -178,7 +178,7 @@ class DefaultCommunicator : public ICommunicator { foreach(scatter); } - void allGather() const override { + void allGatherParams() const override { int totalSize = (int)graphs_[0]->params()->vals()->size(); int shardSize = (int)ceil(totalSize / (float)graphs_.size()); diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h index 09b70f3e6..1b9512102 100755 --- a/src/training/communicator_nccl.h +++ b/src/training/communicator_nccl.h @@ -219,7 +219,7 @@ class NCCLCommunicator : public ICommunicator { threadResults_[i].wait(); } - void scatterReduce() const override { + void scatterReduceAndResetGrads() const override { synchronizeAllOnNullStream(); groupStart(); @@ -237,12 +237,20 @@ class NCCLCommunicator : public ICommunicator { } groupEnd(); synchronizeAll(); + + // reset gradients + // In the future, we can keep quantization residuals here straight in the grads themselves. + auto resetGrads = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { + auto graph = graphs_[localDeviceIndex]; + graph->params()->set_zero_adjoint(); + }; + foreach(resetGrads); } // This distributes all 64 model shards to all 64 GPUs. - // @TODO: For unknown reasons, this takes longer than any other operation incl. scatterReduce(). + // @TODO: For unknown reasons, this takes longer than any other operation incl. scatterReduceAndResetGrads(). // But both should have the same number of data transfers of the same size. - void allGather() const override { + void allGatherParams() const override { synchronizeAllOnNullStream(); groupStart(); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 107c9a4bf..9aa5d288e 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -303,17 +303,17 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); localDeviceCosts[localDeviceIndex] += costNode->scalar(); - graph->backward(/*zero=*/warp == 0); // only reset gradients to 0 if warp = 0 + graph->backward(/*zero=*/false); // gradients are reset by the scatterReduce op } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets #if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead graph->params()->allocateBackward(); - if (warp == 0) // these have already been sized - graph->params()->set_zero_adjoint(); + //if (warp == 0) // these have already been sized + // graph->params()->set_zero_adjoint(); #else graph->clear(); // instead of build() graph->forward(); - graph->backward(/*zero=*/warp == 0); + graph->backward(/*zero=*/false); #endif } }; @@ -352,9 +352,9 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { paramsAvg_[idx], curParam, scheduler_->numberOfBatches(), mbWords); }; - comm_->scatterReduce(); // reduce gradients across all devices (globally) into shards + comm_->scatterReduceAndResetGrads(); // reduce gradients across all devices (globally) into shards comm_->foreach(update); // per-shard model-update - comm_->allGather(); // distribute param value shards back + comm_->allGatherParams(); // distribute param value shards back // cost across all local devices (scheduler will aggregate cross-process) float localCost = 0; From e1f81ee97f6ca44ef1c2a33f195659fa68f5d11e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 19:21:17 -0800 Subject: [PATCH 043/838] bug fix: reduceScatter should not reset its on result --- src/training/communicator_nccl.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h index 1b9512102..dbfe2d877 100755 --- a/src/training/communicator_nccl.h +++ b/src/training/communicator_nccl.h @@ -240,9 +240,14 @@ class NCCLCommunicator : public ICommunicator { // reset gradients // In the future, we can keep quantization residuals here straight in the grads themselves. - auto resetGrads = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { - auto graph = graphs_[localDeviceIndex]; - graph->params()->set_zero_adjoint(); + auto resetGrads = [&](size_t i, size_t begin, size_t end) { + auto grads = graphs_[i]->params()->grads(); + auto size = grads->size(); + // reset everything outside the shard that we reduce in + if (begin > 0) + grads->subtensor(0, begin)->set(0.f); + if (end < size) + grads->subtensor(end, size - end)->set(0.f); }; foreach(resetGrads); } From d216d1e5b562612590202f9fee831e167203e662 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 20:05:42 -0800 Subject: [PATCH 044/838] bug fix: need to reset gradientttttttt shard as well --- src/training/graph_group_sync.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 9aa5d288e..d948612f2 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -303,6 +303,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); localDeviceCosts[localDeviceIndex] += costNode->scalar(); + //graph->backward(/*zero=*/warp == 0); // gradients are reset by the scatterReduce op graph->backward(/*zero=*/false); // gradients are reset by the scatterReduce op } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets @@ -346,6 +347,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // actual model update shardOpt_[idx]->update(curParam, curGrad, mbWords); + curGrad->set(0.f); if(mvAvg_) updateAvgParams( From 8dcebdbe30acf5a24d401a19988fd2827006d0d5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 13 Dec 2018 21:17:08 -0800 Subject: [PATCH 045/838] initialize() now also allocates and resets the gradients, and uses foreach() --- src/graph/parameters.h | 1 - src/training/graph_group_sync.cpp | 48 +++++++++++++++++-------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/graph/parameters.h b/src/graph/parameters.h index 71c538fcd..32f88a1e4 100755 --- a/src/graph/parameters.h +++ b/src/graph/parameters.h @@ -73,7 +73,6 @@ class Parameters { for(auto p : params_) if(!p->grad()) grads_->allocate(p->grad(), p->shape()); - grads()->set(0.f); } } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index d948612f2..339d46b1f 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -45,28 +45,34 @@ void SyncGraphGroup::setScheduler(Ptr scheduler) /*override*/ { } void SyncGraphGroup::initialize(const Ptr& exampleBatch) { - // Initialize 0th graph with random weights in one forward step - // @TODO: Why do we need the THREAD_GUARD here? Why not run this on the main thread? - THREAD_GUARD({ - builders_[0]->build(graphs_[0], exampleBatch); - graphs_[0]->forward(); + // Initialize graphs with random weights in one forward step + // Also allocate and clear the gradients + comm_->foreach([&](size_t i, size_t /*begin*/, size_t /*end*/) { + builders_[i]->build(graphs_[i], exampleBatch); + graphs_[i]->forward(); + graphs_[i]->params()->allocateBackward(); + graphs_[i]->params()->set_zero_adjoint(); }); - // Copy weights from 0th graph to all other graphs + // Copy weights from 0-th graph to all other graphs // to have equal weights across devices - ThreadPool pool(graphs_.size() - 1, graphs_.size() - 1); - for(size_t i = 1; i < graphs_.size(); ++i) { - auto init = [&](size_t i) { - // initialize i-th graph and weights - builders_[i]->build(graphs_[i], exampleBatch); - graphs_[i]->forward(); - // overwrite weights of i-th graph with weights from 0-th graph + comm_->foreach([&](size_t i, size_t /*begin*/, size_t /*end*/) { + if (i > 0) graphs_[i]->params()->vals()->copyFrom(graphs_[0]->params()->vals()); - }; - pool.enqueue(init, i); - } - // ThreadPool destructor waits until completion of all tasks. - // @TODO: can we use comm_->foreach()? + }); + //ThreadPool pool(graphs_.size() - 1, graphs_.size() - 1); + //for(size_t i = 1; i < graphs_.size(); ++i) { + // auto init = [&](size_t i) { + // // initialize i-th graph and weights + // builders_[i]->build(graphs_[i], exampleBatch); + // graphs_[i]->forward(); + // // overwrite weights of i-th graph with weights from 0-th graph + // graphs_[i]->params()->vals()->copyFrom(graphs_[0]->params()->vals()); + // }; + // pool.enqueue(init, i); + //} + //// ThreadPool destructor waits until completion of all tasks. + //// @TODO: can we use comm_->foreach()? } void SyncGraphGroup::initializeAvg() { @@ -308,7 +314,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets #if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead - graph->params()->allocateBackward(); + //graph->params()->allocateBackward(); //if (warp == 0) // these have already been sized // graph->params()->set_zero_adjoint(); #else @@ -355,8 +361,8 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { }; comm_->scatterReduceAndResetGrads(); // reduce gradients across all devices (globally) into shards - comm_->foreach(update); // per-shard model-update - comm_->allGatherParams(); // distribute param value shards back + comm_->foreach(update); // per-shard model-update + comm_->allGatherParams(); // distribute param value shards back // cost across all local devices (scheduler will aggregate cross-process) float localCost = 0; From a5f0d2eebd375284aabbd08b5d4ff1f01519e1d3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 14 Dec 2018 01:33:31 -0800 Subject: [PATCH 046/838] (minor cleanup) --- src/data/batch_generator.h | 4 ++-- src/training/graph_group_sync.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h index 488d3e55b..7d02b8303 100755 --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -211,8 +211,8 @@ class BatchGenerator : public RNGEngine { } double totalSent{}, totalLabels{}; for (auto& b : tempBatches) { - totalSent += (double)b->size(); - totalLabels += (double)b->words(-1); + totalSent += (double)b->size(); + totalLabels += (double)b->words(-1); } auto totalDenom = tempBatches.empty() ? 1 : tempBatches.size(); // (make 0/0 = 0) LOG(info, "[data] fetched {} batches with {} sentences. Per batch: {} sentences, {} labels.", diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 339d46b1f..d51840b71 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -330,9 +330,9 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. // Update parameter shard with gradient shard - auto update = [&](size_t idx, size_t begin, size_t end) { - auto curGrad = graphs_[idx]->params()->grads()->subtensor(begin, end-begin); - auto curParam = graphs_[idx]->params()->vals()->subtensor(begin, end-begin); + auto update = [&](size_t i, size_t begin, size_t end) { + auto curGrad = graphs_[i]->params()->grads()->subtensor(begin, end-begin); + auto curParam = graphs_[i]->params()->vals()->subtensor(begin, end-begin); // if individual gradients were averages, then need to average again over all subBatches auto div = subBatches.size(); @@ -348,16 +348,16 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if (options_->get("cost-type") == "ce-sum") { // presently only supported for ce-sum mbWords = 0; for (const auto& batch : subBatches) - mbWords += batch->words(-1); // @TODO: use wordsTrg (it's the same) + mbWords += batch->wordsTrg(); } // actual model update - shardOpt_[idx]->update(curParam, curGrad, mbWords); + shardOpt_[i]->update(curParam, curGrad, mbWords); curGrad->set(0.f); if(mvAvg_) updateAvgParams( - paramsAvg_[idx], curParam, scheduler_->numberOfBatches(), mbWords); + paramsAvg_[i], curParam, scheduler_->numberOfBatches(), mbWords); }; comm_->scatterReduceAndResetGrads(); // reduce gradients across all devices (globally) into shards From 9cc5b176db0f7bd3c23d7e49a6b46d9826441f12 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 14 Dec 2018 15:11:34 -0800 Subject: [PATCH 047/838] set nvcc host compiler to match general compiler --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 28e648aa9..b54766ae9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ else(CMAKE_BUILD_TYPE STREQUAL "Debug") endif(CMAKE_BUILD_TYPE STREQUAL "Debug") if(NOT MSVC) # @TODO: add warnings here too - list(APPEND CUDA_NVCC_FLAGS -std=c++11; -Xcompiler\ -fPIC; -Xcompiler\ -Wno-unused-result; -Xcompiler\ -Wno-deprecated; -Xcompiler\ -Wno-pragmas; -Xcompiler\ -Wno-unused-value; -Xcompiler\ -Werror;) + list(APPEND CUDA_NVCC_FLAGS -ccbin ${CMAKE_C_COMPILER}; -std=c++11; -Xcompiler\ -fPIC; -Xcompiler\ -Wno-unused-result; -Xcompiler\ -Wno-deprecated; -Xcompiler\ -Wno-pragmas; -Xcompiler\ -Wno-unused-value; -Xcompiler\ -Werror;) else() list(APPEND CUDA_NVCC_FLAGS -Xcompiler\ /FS; ) endif() From 48597eeb9f5340e7ca367634c82b8743e53cb7ef Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 17 Dec 2018 11:46:27 -0800 Subject: [PATCH 048/838] temporarily added override to enable/disable TensorCores; some cleanup --- src/tensors/gpu/prod.cu | 32 ++++++++++++++++++++++++++-- src/training/exponential_smoothing.h | 1 + src/training/graph_group_sync.cpp | 8 +++---- src/training/training.h | 2 +- 4 files changed, 36 insertions(+), 7 deletions(-) mode change 100644 => 100755 src/tensors/gpu/prod.cu diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu old mode 100644 new mode 100755 index 9558a67f2..5b2889328 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -11,6 +11,32 @@ namespace marian { namespace gpu { +static void setTensorMode(cublasHandle_t cublasHandle) { + static int mode = 0; // 1: use TC; -1: do not use TC; 0: not set yet + if (mode == 0) { // multi-thread note: this is sort-of thread-safe, since multiple threads would determine the same value + const char* var = getenv("ENABLE_CUBLAS_TENSOR_OP_MATH_FP32"); + if (!var) + var = "0"; + switch(var[0]) { + case '0': mode = -1; break; + case '1': mode = 1; break; + default: ABORT("Invalid ENABLE_CUBLAS_TENSOR_OP_MATH_FP32={}", var); + } + if (mode > 0) { // try whether it can be set --@TODO: check whether this actually works + cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); + cublasMath_t actual = CUBLAS_DEFAULT_MATH; + cublasGetMathMode(cublasHandle, &actual); + if (actual != CUBLAS_TENSOR_OP_MATH) { + LOG(info, "WARNING: TensorCores requested but not available"); + mode = -1; + } + } + if (mode > 0) + LOG(info, "TensorCores enabled"); + } + cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH); +} + void Prod(marian::Tensor C, const marian::Tensor& A, const marian::Tensor& B, @@ -45,7 +71,8 @@ void Prod(marian::Tensor C, ->getCublasHandle(); #if CUDA_VERSION >= 9000 - cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); + setTensorMode(cublasHandle); + //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif cublasSgemm(cublasHandle, @@ -170,7 +197,8 @@ void ProdBatched(marian::Tensor C, CudaCopy(cptr.data(), cptr.data() + cptr.size(), mp_cptr->data()); #if CUDA_VERSION >= 9000 - cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); + setTensorMode(cublasHandle); + //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif cublasSgemmBatched(cublasHandle, opB, diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h index e24c1e74b..f8786fca1 100755 --- a/src/training/exponential_smoothing.h +++ b/src/training/exponential_smoothing.h @@ -24,6 +24,7 @@ class ExponentialSmoothing { double beta = 1. - mvDecayBy_; // correction term if batch size is different from what mvDecayBy_ was specified for if (refBatchTrgWords_) { + LOG_ONCE(info, "Exponential smoothing gets automatically adjusted as if update size was {} target words", refBatchTrgWords_); ABORT_IF(actualBatchTrgWords == OptimizerBase::mbSizeNotProvided, "This graph-group type does not support reference batch size specification for exponential-smoothing"); beta = pow(beta, (double)actualBatchTrgWords / (double)refBatchTrgWords_); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index d51840b71..de2e34b17 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -231,12 +231,12 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector 1) { // unfortunately we can only split in integer ratios // split each of last numWarps's batches into 'splitInto' batches @@ -256,7 +256,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector availableGPUSlots, "somehow split into too many batches??"); + ABORT_IF(pendingBatches_.size() > availableDevices, "somehow split into too many batches??"); } subBatches = std::move(pendingBatches_); diff --git a/src/training/training.h b/src/training/training.h index 401f6bdf0..ce9c5d4cf 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -47,7 +47,7 @@ class Train : public ModelTask { auto model = New(options_); model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling stats = model->collectStats(); - LOG(info, "[batching] Done"); + LOG(info, "[batching] Done. Typical MB size is {} target words", stats->estimateTypicalTrgWords()); } if((options_->has("valid-sets") || options_->has("valid-script-path")) From fe35160259cb0b103e67e0e8791d90957c3b799b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 18 Dec 2018 09:30:38 -0800 Subject: [PATCH 049/838] temporarily allows to enable/disable TensorCores --- src/tensors/gpu/prod.cu | 4 ++-- src/training/graph_group_sync.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 5b2889328..9da32e65a 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -16,7 +16,7 @@ static void setTensorMode(cublasHandle_t cublasHandle) { if (mode == 0) { // multi-thread note: this is sort-of thread-safe, since multiple threads would determine the same value const char* var = getenv("ENABLE_CUBLAS_TENSOR_OP_MATH_FP32"); if (!var) - var = "0"; + var = "1"; switch(var[0]) { case '0': mode = -1; break; case '1': mode = 1; break; @@ -32,7 +32,7 @@ static void setTensorMode(cublasHandle_t cublasHandle) { } } if (mode > 0) - LOG(info, "TensorCores enabled"); + LOG(info, "16-bit TensorCores enabled for float32 matrix operations"); } cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH); } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index de2e34b17..638642786 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -236,7 +236,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector 1) { // unfortunately we can only split in integer ratios // split each of last numWarps's batches into 'splitInto' batches @@ -248,10 +248,10 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectorsize()); + //LOG(info, "{}-way splitting batchToSplit with size {}", splitInto, batchToSplit->size()); auto splitBatches = batchToSplit->split(splitInto); for (auto& splitBatch : splitBatches) { - LOG(info, " -> getting batchToSplit with size {}", splitBatch->size()); + //LOG(info, " -> getting batchToSplit with size {}", splitBatch->size()); pendingBatches_.push_back(splitBatch); } } @@ -288,7 +288,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // Upon very first execution, reset everything if(first_) { - LOG(info, "[training] Processing first minibatch. Batches are processed as {} processes x {} GPUs/process", + LOG(info, "[training] Batches are processed as {} process(es) x {} devices/process", mpi_->numMPIProcesses(), devices_.size()); initialize(subBatches.front()); if(mvAvg_ && paramsAvg_.empty()) From e627684b48497102935af685e5ef79bcc16ce6c5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 18 Dec 2018 14:44:07 -0800 Subject: [PATCH 050/838] new experimental option --mini-batch-overstuff --- src/common/config_parser.cpp | 3 ++ src/optimizers/optimizers.cpp | 2 +- src/training/graph_group_sync.cpp | 65 ++++++++++++++++++++----------- src/training/graph_group_sync.h | 2 +- src/training/scheduler.h | 19 ++++----- 5 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index b5ea081e8..77421d691 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -618,6 +618,9 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { {"0"}); cli.add("--mini-batch-track-lr", "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); + cli.add("--mini-batch-overstuff", + "Stuff this much more data into a minibatch, but scale down the LR and progress counter", + 1.0); // clang-format on } diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index 75501a370..685c917f9 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -138,8 +138,8 @@ void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t r vt_->set(0.f); } - double Tref = (double)refMBWords; double T = (double)actualMBSize; + double Tref = (double)refMBWords; // adjust for minibatch-size changes if Adam parameters are given a reference size (else do nothing) double eta = eta_ * (T/Tref); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 638642786..a2e530ec6 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -153,7 +153,8 @@ static double roundUpRatio(double ratio) { // helper routine that handles accumulation and load-balancing of sub-batches to fill all devices // It adds 'newBatch' to 'pendingBatches_', and if sufficient batches have been queued, then // returns 'pendingBatches_' in 'subBatches' and resets it. If not, it returns false. -bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector>& subBatches) { +bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, double overstuff, + std::vector>& subBatches, size_t& numReadBatches) { // The reader delivers in chunks of these sizes, according to case: // - no dynamic MB-size scaling: // - reader batch size = update batch size, with... @@ -181,6 +182,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vectorsplit(numWarps * warpSize); + numReadBatches = 1; return true; } LOG_ONCE(info, "[training] Dynamic mini-batch scaling enabled"); @@ -204,6 +206,9 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector newBatch, std::vector Reduce each by 3.3/4 // Alternatively, we could just shorten the last 'warp', but that would not be invariant to warp size. for (auto& batch : pendingBatches_) { @@ -267,13 +274,29 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, std::vector newBatch) /*override*/ { validate(); + double overstuff = options_->get("mini-batch-overstuff"); + if (overstuff != 1) + LOG_ONCE(info, "Overstuffing minibatches by a factor of {}", overstuff); std::vector> subBatches; - bool gotSubBatches = tryGetSubBatches(newBatch, subBatches); + size_t numReadBatches; // actual #batches delivered by reader, for restoring from checkpoint --@TODO: reader should checkpoint itself; should not go via the scheduler + bool gotSubBatches = tryGetSubBatches(newBatch, overstuff, subBatches, numReadBatches); // not enough data yet: return right away if (!gotSubBatches) return; + // determine num words for dynamic hyper-parameter adjustment + // @TODO: We can return these directly from tryGetSubBatches() + size_t batchSize = 0; + size_t batchTrgWords = 0; + for (const auto& batch : subBatches) { + batchSize += batch->size(); + batchTrgWords += batch->wordsTrg(); + } + // effective batch size: batch should be weighted like this + size_t effectiveBatchTrgWords = (size_t)ceil(batchTrgWords / overstuff); + size_t effectiveBatchSize = (size_t)ceil(batchSize / overstuff); + // Helper to access the subBatches array auto getSubBatch = [&](size_t warp, size_t localDeviceIndex, size_t rank) -> Ptr { // Warp should be slowest changing dimension. If subBatches are sorted by @@ -301,7 +324,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device for (size_t warp = 0; getSubBatch(warp, 0, 0); warp++) { // Execute single forward/backward step - auto forwardBackward = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { + comm_->foreach([&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { // parallel across devices. Aggregate for warp > 1. auto graph = graphs_[localDeviceIndex]; auto subBatch = getSubBatch(warp, localDeviceIndex, mpi_->myMPIRank()); @@ -323,41 +346,37 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { graph->backward(/*zero=*/false); #endif } - }; - - comm_->foreach(forwardBackward); // compute gradients in parallel on each device. Aggregate if delay > 1. + }); } // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. + // If individual gradients were averages, then need to average again over all subBatches + auto div = subBatches.size(); + if (options_->get("cost-type") == "ce-sum") + div = 1; + // Update parameter shard with gradient shard auto update = [&](size_t i, size_t begin, size_t end) { auto curGrad = graphs_[i]->params()->grads()->subtensor(begin, end-begin); auto curParam = graphs_[i]->params()->vals()->subtensor(begin, end-begin); - // if individual gradients were averages, then need to average again over all subBatches - auto div = subBatches.size(); - if (options_->get("cost-type") == "ce-sum") - div = 1; if(div != 1) { using namespace functional; - Element(_1 = _1 / (float)div, curGrad); - } - - // determine num words for dynamic hyper-parameter adjustment - size_t mbWords = OptimizerBase::mbSizeNotProvided; - if (options_->get("cost-type") == "ce-sum") { // presently only supported for ce-sum - mbWords = 0; - for (const auto& batch : subBatches) - mbWords += batch->wordsTrg(); + Element(_1 = _1 / (float)div, curGrad); // average once again in case of ce-mean* } // actual model update - shardOpt_[i]->update(curParam, curGrad, mbWords); + auto updateTrgWords = + /*if*/(options_->get("cost-type") == "ce-sum") ? + effectiveBatchTrgWords // if overstuffing then scale the LR down by that factor --@TODO: how about momentum? + /*else*/: + OptimizerBase::mbSizeNotProvided; + shardOpt_[i]->update(curParam, curGrad, updateTrgWords); curGrad->set(0.f); if(mvAvg_) updateAvgParams( - paramsAvg_[i], curParam, scheduler_->numberOfBatches(), mbWords); + paramsAvg_[i], curParam, scheduler_->numberOfBatches(), updateTrgWords); }; comm_->scatterReduceAndResetGrads(); // reduce gradients across all devices (globally) into shards @@ -375,7 +394,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if(scheduler_) { // track and log localCost - scheduler_->update(localCost, subBatches, mpi_); + scheduler_->update(localCost, numReadBatches, effectiveBatchSize, effectiveBatchTrgWords, mpi_); // save intermediate model (and optimizer state) to file if(scheduler_->saving()) diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 6bb3a4861..22e5c6f28 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -37,7 +37,7 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void barrier() const { mpi_->barrier(); } // (we need this several times) void swapParamsAvg() { if (mvAvg_ && paramsAvg_.size() > 0) comm_->swapParams(paramsAvg_); } // note: must call this on all MPI ranks in parallel - bool tryGetSubBatches(Ptr newBatch, std::vector>& subBatches); + bool tryGetSubBatches(Ptr newBatch, double overstuff, std::vector>& subBatches, size_t& numReadBatches); public: SyncGraphGroup(Ptr config); diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 802282354..5d8189758 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -214,21 +214,18 @@ class Scheduler : public TrainingObserver { } void update(float cost, Ptr batch) { - update(cost, std::vector>({batch})); + update(cost, + /*numReadBatches=*/1, /*batchSize=*/batch->size(), /*batchLabels=*/batch->wordsTrg()); } - void update(float cost, const std::vector>& batches, Ptr mpi = nullptr) { + void update(float cost, + size_t numReadBatches, // number of batches read by the reader (for seeking in case of restart) + size_t batchSize, // total number of sentences in batch + size_t batchLabels, // total number of target words in batch + Ptr mpi = nullptr) { state_->rememberPreviousProgress(); // note: epoch increases happen at the wrong place, hence -freq parameters do not support epoch units state_->validated = false; - size_t batchSize = 0; // number of sentences in batch - size_t batchLabels = 0; // number of target words in batch - - for(const auto& batch : batches) { - batchSize += batch->size(); - batchLabels += batch->words(-1); - } - // Since batchLabels is counted across all MPI processes, we also should temporarily // extrapolate cost across MPI processes, to have numbers in the right range. // When doing the actual log, we then aggregate across MPI processes to get the accurate number. @@ -257,7 +254,7 @@ class Scheduler : public TrainingObserver { state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed - state_->newUpdate(batches.size()); + state_->newUpdate(numReadBatches); if(state_->enteredNewPeriodOf(options_->get("disp-freq")) || state_->batches <= options_->get("disp-first")) { From 9fb672b6de176180ab74a184597b05e55f223ac8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 18 Dec 2018 14:55:52 -0800 Subject: [PATCH 051/838] (minor fix in overstuffing) --- src/training/graph_group_sync.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index a2e530ec6..6e0b4d482 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -331,7 +331,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if(subBatch) { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); - localDeviceCosts[localDeviceIndex] += costNode->scalar(); + localDeviceCosts[localDeviceIndex] += costNode->scalar() / overstuff; //graph->backward(/*zero=*/warp == 0); // gradients are reset by the scatterReduce op graph->backward(/*zero=*/false); // gradients are reset by the scatterReduce op } From c0b4dff958d122dcc09dd405a1227f1a038333f8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 18 Dec 2018 16:13:18 -0800 Subject: [PATCH 052/838] bug fix: overstuff must scale down both gradient and denominator --- src/training/graph_group_sync.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 6e0b4d482..d2fc13f1c 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -351,9 +351,9 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. // If individual gradients were averages, then need to average again over all subBatches - auto div = subBatches.size(); + float div = (float)subBatches.size(); if (options_->get("cost-type") == "ce-sum") - div = 1; + div = (float)overstuff; // Update parameter shard with gradient shard auto update = [&](size_t i, size_t begin, size_t end) { @@ -362,7 +362,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if(div != 1) { using namespace functional; - Element(_1 = _1 / (float)div, curGrad); // average once again in case of ce-mean* + Element(_1 = _1 / div, curGrad); // average once again in case of ce-mean* } // actual model update From da57654bc4dfe9988665ab0a44fd3c7ac2633e54 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 19 Dec 2018 15:18:46 +0000 Subject: [PATCH 053/838] Add aliases and --problem --- src/common/cli_wrapper.h | 44 ++++++++++++++++++++++++--- src/common/config_parser.cpp | 59 ++++++++++++++++++++++++++++++------ src/common/config_parser.h | 3 +- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 1590441a0..d06bdd718 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -51,9 +51,7 @@ class CLIFormatter : public CLI::Formatter { // @TODO: in this file review the use of naked pointers. We use Ptr anywhere else, // what's up with that? -/** - * The helper structure storing an option object, the associated variable and creation index. - */ +// The helper structure storing an option object, the associated variable and creation index struct CLIOptionTuple { CLI::Option *opt; Ptr var; @@ -61,6 +59,13 @@ struct CLIOptionTuple { bool modified{false}; }; +// Helper structure used for aliases and storing an option key, value, and YAML node +struct CLIAliasTuple { + std::string key; + std::string value; + YAML::Node config; +}; + /** * @brief The class used to define and parse command-line arguments. * @@ -82,9 +87,12 @@ class CLIWrapper { std::unordered_map options_; // Counter for created options size_t counter_{0}; + // List of alias tuples + std::vector aliases_; // Command-line argument parser Ptr app_; + // Name of the default option group std::string defaultGroup_{""}; // Name of the current option group @@ -192,6 +200,25 @@ class CLIWrapper { /*defaulted =*/false); } + /** + * @brief Define an alias that is a shortcut for a set of options + * + * Option values are compared as std::string. + * + * @param key Option name + * @param value Option value that trigger the alias + * @param fun Function initializing options + */ + void alias(const std::string &key, + const std::string &value, + const std::function &fun) { + ABORT_IF(!options_.count(key), "Option '{}' is not defined so alias can not be created", key); + aliases_.resize(aliases_.size() + 1); + aliases_.back().key = key; + aliases_.back().value = value; + fun(aliases_.back().config); + } + /** * Switch to different option group or to the default group if argument is empty. * @@ -202,6 +229,15 @@ class CLIWrapper { // Parse command-line arguments. Handles --help and --version options void parse(int argc, char **argv); + void parseAliases(const YAML::Node &config) { + for(const auto& alias : aliases_) { + if(config[alias.key] && config[alias.key].as() == alias.value) { + updateConfig(alias.config, + "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); + } + } + } + /* * @brief Overwrite values for unparsed options * @@ -212,7 +248,7 @@ class CLIWrapper { * * @param config YAML config with new default values for options * @param errorMsg error message printed if config contains undefined keys. The message is - * appended with ": * " + * appended with ": " */ void updateConfig(const YAML::Node &config, const std::string &errorMsg); diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index a4699e023..f1f9a71de 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -394,8 +394,12 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--multi-node-overlap", "Overlap model computations with MPI communication", true); + // add ULR settings addSuboptionsULR(cli); + + cli.add("--problem", + "Use predefined set of options. Possible values: transformer"); // clang-format on } @@ -652,11 +656,10 @@ void ConfigParser::addSuboptionsULR(cli::CLIWrapper& cli) { // clang-format on } -void ConfigParser::expandAliases(cli::CLIWrapper& cli) { - YAML::Node config; +void ConfigParser::addAliases(cli::CLIWrapper& cli) { // The order of aliases does matter as later options overwrite earlier - if(config_["best-deep"].as()) { + cli.alias("best-deep", "true", [](YAML::Node& config) { config["layer-normalization"] = true; config["tied-embeddings"] = true; config["enc-type"] = "alternating"; @@ -666,11 +669,42 @@ void ConfigParser::expandAliases(cli::CLIWrapper& cli) { config["dec-cell-high-depth"] = 2; config["dec-depth"] = 4; config["skip"] = true; - } - - if(config) { - cli.updateConfig(config, "Unknown option(s) in aliases"); - } + }); + + cli.alias("problem", "transformer", [](YAML::Node& config) { + config["type"] = "transformer"; + config["enc-depth"] = 6; + config["dec-depth"] = 6; + config["transformer-heads"] = 8; + config["learn-rate"] = 0.0003; + config["cost-type"] = "ce-mean-words"; + config["lr-warmup"] = 16000; + config["lr-decay-inv-sqrt"] = 16000; + config["transformer-dropout"] = 0.1; + config["label-smoothing"] = 0.1; + config["clip-norm"] = 5; + }); + + cli.alias("problem", "transformer-big", [](YAML::Node& config) { + config["type"] = "transformer"; + config["enc-depth"] = 6; + config["dec-depth"] = 6; + config["dim-emb"] = 1024; + config["transformer-dim-ffn"] = 4096; + config["transformer-heads"] = 16; + config["transformer-postprocess"] = "dan"; + config["transformer-preprocess"] = "d"; + config["transformer-ffn-activation"] = "relu"; + config["learn-rate"] = 0.0002; + config["cost-type"] = "ce-mean-words"; + config["lr-warmup"] = 8000; + config["lr-decay-inv-sqrt"] = 8000; + config["transformer-dropout"] = 0.1; + config["transformer-attention-dropout"] = 0.1; + config["transformer-ffn-dropout"] = 0.1; + config["label-smoothing"] = 0.1; + config["clip-norm"] = 5; + }); } void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { @@ -690,6 +724,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { case cli::mode::training: addOptionsTraining(cli); addOptionsValidation(cli); + addAliases(cli); break; case cli::mode::translation: addOptionsTranslation(cli); @@ -718,6 +753,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { } if(doValidate) { + // TODO: Do not check some constraints if --dump-config, e.g. -t // this aborts the program on first validation error ConfigValidator(config_).validateOptions(mode_); } @@ -725,14 +761,17 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { // remove extra config files from the config to avoid redundancy config_.remove("config"); + // TODO: Consider expanding aliases after dumping config + cli.parseAliases(config_); + config_.remove("best-deep"); + config_.remove("problem"); + if(!get("dump-config").empty() && get("dump-config") != "false") { bool skipDefault = get("dump-config") == "minimal"; config_.remove("dump-config"); std::cout << cli.dumpConfig(skipDefault) << std::endl; exit(0); } - - expandAliases(cli); } std::vector ConfigParser::findConfigPaths() { diff --git a/src/common/config_parser.h b/src/common/config_parser.h index ff305f738..d6822cc1a 100755 --- a/src/common/config_parser.h +++ b/src/common/config_parser.h @@ -77,11 +77,12 @@ class ConfigParser { void addOptionsTranslation(cli::CLIWrapper&); void addOptionsScoring(cli::CLIWrapper&); + void addAliases(cli::CLIWrapper&); + void addSuboptionsDevices(cli::CLIWrapper&); void addSuboptionsBatching(cli::CLIWrapper&); void addSuboptionsInputLength(cli::CLIWrapper&); void addSuboptionsULR(cli::CLIWrapper&); - void expandAliases(cli::CLIWrapper&); // Extract paths to all config files found in the config object. // Look at --config option and model.npz.yml files. From 680857507873c8590eaaf6f1f1aa08cce0fcdffd Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 19 Dec 2018 19:26:40 +0000 Subject: [PATCH 054/838] Add --dump-config explain --- src/common/config_parser.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index f1f9a71de..f4d86c567 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -79,7 +79,8 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { cli.add("--relative-paths", "All paths are relative to the config file location"); cli.add("--dump-config", - "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") + "Dump current (modified) configuration to stdout and exit. " + "Possible values: full, minimal, explain") ->implicit_val("full"); // clang-format on } @@ -761,17 +762,24 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { // remove extra config files from the config to avoid redundancy config_.remove("config"); - // TODO: Consider expanding aliases after dumping config - cli.parseAliases(config_); - config_.remove("best-deep"); - config_.remove("problem"); - if(!get("dump-config").empty() && get("dump-config") != "false") { - bool skipDefault = get("dump-config") == "minimal"; + auto type = get("dump-config"); config_.remove("dump-config"); - std::cout << cli.dumpConfig(skipDefault) << std::endl; + + if(type == "explain") { + cli.parseAliases(config_); + config_.remove("best-deep"); + config_.remove("problem"); + } + + bool minimal = (type == "minimal" || type == "explain"); + std::cout << cli.dumpConfig(minimal) << std::endl; exit(0); } + + cli.parseAliases(config_); + config_.remove("best-deep"); + config_.remove("problem"); } std::vector ConfigParser::findConfigPaths() { From 81b2c7acbe5e64c24ef42693ab324ced51a67af8 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Wed, 19 Dec 2018 19:39:55 +0000 Subject: [PATCH 055/838] Do not require --train-sets if --dump-config is given --- src/common/config_validator.cpp | 9 ++++++++- src/common/config_validator.h | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 9345b0a56..1566e1cfc 100755 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -10,7 +10,10 @@ bool ConfigValidator::has(const std::string& key) const { return config_[key]; } -ConfigValidator::ConfigValidator(const YAML::Node& config) : config_(config) {} +ConfigValidator::ConfigValidator(const YAML::Node& config) + : config_(config), + dump_(config["dump-config"] && !config["dump-config"].as().empty() + && config["dump-config"].as() != "false") {} ConfigValidator::~ConfigValidator() {} @@ -55,6 +58,10 @@ void ConfigValidator::validateOptionsTranslation() const { } void ConfigValidator::validateOptionsParallelData() const { + // Do not check these constraints if only goal is to dump config + if(dump_) + return; + auto trainSets = get>("train-sets"); ABORT_IF(trainSets.empty(), "No train sets given in config file or on command line"); diff --git a/src/common/config_validator.h b/src/common/config_validator.h index 59cd11864..fb40ea6c6 100644 --- a/src/common/config_validator.h +++ b/src/common/config_validator.h @@ -5,11 +5,10 @@ namespace marian { -// TODO: Finally refactorize Config, Options, ConfigParser and ConfigValidator -// classes. class ConfigValidator { private: const YAML::Node& config_; + bool dump_{false}; bool has(const std::string& key) const; From b2be66cf11eab7bbd2acecc65108b30d7636c6d1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 20 Dec 2018 16:58:22 -0800 Subject: [PATCH 056/838] new experimental parameter --understuff; cleaned up MPI uninitialization --- src/common/config_parser.cpp | 9 ++++-- src/training/graph_group_sync.cpp | 50 ++++++++++++++++++++++++------- src/training/graph_group_sync.h | 9 ++++-- src/training/training.h | 6 ++-- 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 77421d691..70a30a5fd 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -618,9 +618,12 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { {"0"}); cli.add("--mini-batch-track-lr", "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); - cli.add("--mini-batch-overstuff", - "Stuff this much more data into a minibatch, but scale down the LR and progress counter", - 1.0); + cli.add("--mini-batch-overstuff", + "[experimental] Stuff this much more data into a minibatch, but scale down the LR and progress counter", + 1); + cli.add("--mini-batch-understuff", + "[experimental] Break each batch into this many updates", + 1); // clang-format on } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index d2fc13f1c..08503ae89 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -153,7 +153,7 @@ static double roundUpRatio(double ratio) { // helper routine that handles accumulation and load-balancing of sub-batches to fill all devices // It adds 'newBatch' to 'pendingBatches_', and if sufficient batches have been queued, then // returns 'pendingBatches_' in 'subBatches' and resets it. If not, it returns false. -bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, double overstuff, +bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, size_t overstuff, std::vector>& subBatches, size_t& numReadBatches) { // The reader delivers in chunks of these sizes, according to case: // - no dynamic MB-size scaling: @@ -207,7 +207,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, double overstuf } // overstuff: blow up ratio by a factor, which we later factor into the learning rate - ratio *= overstuff; + ratio *= (double)overstuff; // round up to full batches if within a certain error margin --@BUGBUG: Not invariant w.r.t. GPU size, as ratio is relative to what fits into 1 GPU ratio = roundUpRatio(ratio); @@ -274,7 +274,7 @@ bool SyncGraphGroup::tryGetSubBatches(Ptr newBatch, double overstuf void SyncGraphGroup::update(Ptr newBatch) /*override*/ { validate(); - double overstuff = options_->get("mini-batch-overstuff"); + size_t overstuff = options_->get("mini-batch-overstuff"); if (overstuff != 1) LOG_ONCE(info, "Overstuffing minibatches by a factor of {}", overstuff); std::vector> subBatches; @@ -285,6 +285,30 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if (!gotSubBatches) return; + // for testing the hypothesis that one can always go smaller. This is independent of overstuff. + size_t understuff = options_->get("mini-batch-understuff"); + if (understuff != 1) + LOG_ONCE(info, "Understuffing minibatches by a factor of {}", understuff); + if (understuff == 1) + update(subBatches, numReadBatches); + else { + std::vector> subBatches1; + for (auto& b : subBatches) { + auto bbs = b->split(understuff); + for (auto& bb : bbs) + subBatches1.push_back(bb); + } + for (size_t i = 0; i < understuff; i++) { + std::vector> subBatchRange(subBatches1.begin() + i * subBatches1.size() / understuff, subBatches1.begin() + (i+1) * subBatches1.size() / understuff); + if (!subBatchRange.empty()) + update(subBatchRange, numReadBatches * (i+1) / understuff - numReadBatches * i / understuff); + } + } +} + +void SyncGraphGroup::update(std::vector> subBatches, size_t numReadBatches) { + size_t overstuff = options_->get("mini-batch-overstuff"); + //size_t understuff = options_->get("mini-batch-understuff"); // determine num words for dynamic hyper-parameter adjustment // @TODO: We can return these directly from tryGetSubBatches() size_t batchSize = 0; @@ -293,9 +317,9 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { batchSize += batch->size(); batchTrgWords += batch->wordsTrg(); } - // effective batch size: batch should be weighted like this - size_t effectiveBatchTrgWords = (size_t)ceil(batchTrgWords / overstuff); - size_t effectiveBatchSize = (size_t)ceil(batchSize / overstuff); + // effective batch size: batch should be weighted like this. This will weight down the learning rate. + size_t effectiveBatchTrgWords = (size_t)ceil(batchTrgWords / (double)overstuff); + size_t effectiveBatchSize = (size_t)ceil(batchSize / (double)overstuff); // Helper to access the subBatches array auto getSubBatch = [&](size_t warp, size_t localDeviceIndex, size_t rank) -> Ptr { @@ -331,7 +355,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if(subBatch) { auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); - localDeviceCosts[localDeviceIndex] += costNode->scalar() / overstuff; + localDeviceCosts[localDeviceIndex] += costNode->scalar() / (float)overstuff; //graph->backward(/*zero=*/warp == 0); // gradients are reset by the scatterReduce op graph->backward(/*zero=*/false); // gradients are reset by the scatterReduce op } @@ -353,7 +377,7 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { // If individual gradients were averages, then need to average again over all subBatches float div = (float)subBatches.size(); if (options_->get("cost-type") == "ce-sum") - div = (float)overstuff; + div = (float)overstuff; // (note: with Adam, a constant here makes no difference) // Update parameter shard with gradient shard auto update = [&](size_t i, size_t begin, size_t end) { @@ -362,13 +386,13 @@ void SyncGraphGroup::update(Ptr newBatch) /*override*/ { if(div != 1) { using namespace functional; - Element(_1 = _1 / div, curGrad); // average once again in case of ce-mean* + Element(_1 = _1 / div, curGrad); // average once again for ce-mean*, or scale down if overstuffed for ce-sum } // actual model update auto updateTrgWords = /*if*/(options_->get("cost-type") == "ce-sum") ? - effectiveBatchTrgWords // if overstuffing then scale the LR down by that factor --@TODO: how about momentum? + effectiveBatchTrgWords // if overstuffing then bring the count back to the original value /*else*/: OptimizerBase::mbSizeNotProvided; shardOpt_[i]->update(curParam, curGrad, updateTrgWords); @@ -519,8 +543,12 @@ void SyncGraphGroup::save(bool final) /*override*/ { void SyncGraphGroup::finalize() /*override*/ { validate(); - finalizeMPI(std::move(mpi_)); Base::finalize(); } +SyncGraphGroup::~SyncGraphGroup() /*override*/ { + comm_ = nullptr; + finalizeMPI(std::move(mpi_)); +} + } // namespace marian diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 22e5c6f28..d5f818c54 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -37,11 +37,15 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void barrier() const { mpi_->barrier(); } // (we need this several times) void swapParamsAvg() { if (mvAvg_ && paramsAvg_.size() > 0) comm_->swapParams(paramsAvg_); } // note: must call this on all MPI ranks in parallel - bool tryGetSubBatches(Ptr newBatch, double overstuff, std::vector>& subBatches, size_t& numReadBatches); + bool tryGetSubBatches(Ptr newBatch, size_t overstuff, std::vector>& subBatches, size_t& numReadBatches); + void update(std::vector> subBatches, size_t numReadBatches); public: SyncGraphGroup(Ptr config); + Ptr collectStats(); + // @TODO: consider to make this a virtual as well? Currently it is a template dispatch + void setScheduler(Ptr scheduler) override; void update(Ptr batch) override; @@ -51,7 +55,6 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void finalize() override; - Ptr collectStats(); - // @TODO: consider to make this a virtual as well? Currently it is a template dispatch + ~SyncGraphGroup() override; }; } // namespace marian diff --git a/src/training/training.h b/src/training/training.h index ce9c5d4cf..b2290db75 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -91,14 +91,12 @@ class Train : public ModelTask { } scheduler->finished(); + model->finalize(); + // Avoid saving the model twice if it has been loaded and training did not // progress if(!trainState->loaded) model->save(true); - - // finalize, including communicating successful completion to MPI - // @BUGBUG: This is wrong for async, but needed for sync. How to solve it? - model->finalize(); } }; } // namespace marian From 7843c0c135b0d1fbe0e30c3102c52be17883dc9d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 20 Dec 2018 17:01:20 -0800 Subject: [PATCH 057/838] small clean-ups to get through regression tests, part 1 --- src/common/logging.cpp | 2 +- src/data/batch_generator.h | 2 +- src/training/graph_group_sync.cpp | 12 +++++++----- src/training/graph_group_sync.h | 2 ++ src/training/scheduler.h | 5 ++--- src/training/training.h | 6 ++---- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/common/logging.cpp b/src/common/logging.cpp index da583878d..203acfe79 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -84,7 +84,7 @@ void createLoggers(const marian::Config* options) { bool quiet = options && options->get("quiet"); Logger general{ - createStderrLogger("general", "[%Y-%m-%d %T %t] %v", generalLogs, quiet)}; + createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; Logger valid{ createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h index 7d02b8303..ce673cd2a 100755 --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -215,7 +215,7 @@ class BatchGenerator : public RNGEngine { totalLabels += (double)b->words(-1); } auto totalDenom = tempBatches.empty() ? 1 : tempBatches.size(); // (make 0/0 = 0) - LOG(info, "[data] fetched {} batches with {} sentences. Per batch: {} sentences, {} labels.", + LOG(debug, "[data] fetched {} batches with {} sentences. Per batch: {} sentences, {} labels.", tempBatches.size(), numSentencesRead, (double)totalSent / (double)totalDenom, (double)totalLabels / (double)totalDenom); return tempBatches; diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 638642786..30d163661 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -438,7 +438,7 @@ void SyncGraphGroup::load() /*override*/ { } void SyncGraphGroup::save(bool final) /*override*/ { - validate(); + // validate(); @TODO: get rid of this everywhere (SyncGraphGroup) barrier(); // (for better grouping of log messages) // do final validation if(final && scheduler_) { @@ -487,21 +487,23 @@ void SyncGraphGroup::save(bool final) /*override*/ { barrier(); // (for better grouping of log messages) // persist optimizer state - LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); shardOpt_[0]->save(name + ".optimizer.npz", shardOpt_, [&](const OptimizerBase::GatherStateGetFunc& getShardFn) { return comm_->gatherState(getShardFn); }, isMainProcess()); - LOG(info, "[{}] save() line {}", this->mpi_->idStr(), __LINE__); - + barrier(); // (for better grouping of log messages) } void SyncGraphGroup::finalize() /*override*/ { validate(); - finalizeMPI(std::move(mpi_)); Base::finalize(); } + +SyncGraphGroup::~SyncGraphGroup() /*override*/ { + comm_ = nullptr; + finalizeMPI(std::move(mpi_)); +} } // namespace marian diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 6bb3a4861..2171cbf0f 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -53,5 +53,7 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { Ptr collectStats(); // @TODO: consider to make this a virtual as well? Currently it is a template dispatch + + ~SyncGraphGroup() override; }; } // namespace marian diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 802282354..3d46de096 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -174,13 +174,12 @@ class Scheduler : public TrainingObserver { float value = validator->validate(graphs); if(validator->stalled() > 0) { LOG_VALID(info, - "Ep. {} : Up. {} : {} : {} : stalled {} times (last best: {})", + "Ep. {} : Up. {} : {} : {} : stalled {} times", /*"(last best: {})",*/ // @TODO (LOGGING CHANGE) state_->epochs, state_->batches, validator->type(), value, - validator->stalled(), - validator->lastBest()); + validator->stalled() /*, validator->lastBest()*/); // @TODO (LOGGING CHANGE) } else { LOG_VALID(info, "Ep. {} : Up. {} : {} : {} : new best", diff --git a/src/training/training.h b/src/training/training.h index ce9c5d4cf..b2290db75 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -91,14 +91,12 @@ class Train : public ModelTask { } scheduler->finished(); + model->finalize(); + // Avoid saving the model twice if it has been loaded and training did not // progress if(!trainState->loaded) model->save(true); - - // finalize, including communicating successful completion to MPI - // @BUGBUG: This is wrong for async, but needed for sync. How to solve it? - model->finalize(); } }; } // namespace marian From 496ed37d56f6e1af92b8fbff1db9fb0e04f226b5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 20 Dec 2018 17:10:24 -0800 Subject: [PATCH 058/838] bug fix (perf): loop over warps should not synchronize worker threads after each warp --- src/training/graph_group_sync.cpp | 40 +++++++++++++------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 08503ae89..c6ef37486 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -346,32 +346,24 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num // Compute gradients // This happens in multiple steps in case of delay > 1. std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device - for (size_t warp = 0; getSubBatch(warp, 0, 0); warp++) { - // Execute single forward/backward step - comm_->foreach([&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { // parallel across devices. Aggregate for warp > 1. - auto graph = graphs_[localDeviceIndex]; + comm_->foreach([&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { // parallel across devices. Aggregate for warp > 1. + auto graph = graphs_[localDeviceIndex]; + // reset gradient --presently done outside + //graph->params()->allocateBackward(); + //if (warp == 0) // these have already been sized + // graph->params()->set_zero_adjoint(); + for (size_t warp = 0; ; warp++) { + // Execute single forward/backward step auto subBatch = getSubBatch(warp, localDeviceIndex, mpi_->myMPIRank()); + if (!subBatch) + break; - if(subBatch) { - auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); - graph->forward(); - localDeviceCosts[localDeviceIndex] += costNode->scalar() / (float)overstuff; - //graph->backward(/*zero=*/warp == 0); // gradients are reset by the scatterReduce op - graph->backward(/*zero=*/false); // gradients are reset by the scatterReduce op - } - else { // empty batch: execute do-nothing fw-bw step for proper inits and resets -#if 1 // @TODO: double-check whether the #else branch is the same; and if so, use it instead - //graph->params()->allocateBackward(); - //if (warp == 0) // these have already been sized - // graph->params()->set_zero_adjoint(); -#else - graph->clear(); // instead of build() - graph->forward(); - graph->backward(/*zero=*/false); -#endif - } - }); - } + auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); + graph->forward(); + localDeviceCosts[localDeviceIndex] += costNode->scalar() / (float)overstuff; + graph->backward(/*zero=*/false); // (gradients are reset before we get here) + } + }); // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. // If individual gradients were averages, then need to average again over all subBatches From 0af017e2e2aaae0897be8e19e93ea1a57ff00df4 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 20 Dec 2018 17:14:06 -0800 Subject: [PATCH 059/838] move pointer to right sentencepiece version --- src/3rd_party/sentencepiece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rd_party/sentencepiece b/src/3rd_party/sentencepiece index 21309542e..61ae30fa8 160000 --- a/src/3rd_party/sentencepiece +++ b/src/3rd_party/sentencepiece @@ -1 +1 @@ -Subproject commit 21309542e69e1821ff8e905fa60d8852ac12a73f +Subproject commit 61ae30fa8313c72ea36c41f1ea0402c7ed8c2fe0 From 3293dddb4f5e56ddb8540727ed05201309256d1c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 20 Dec 2018 18:23:16 -0800 Subject: [PATCH 060/838] restore old validation behaviour --- src/training/validator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/validator.h b/src/training/validator.h index 682495410..2b76d6578 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -137,7 +137,7 @@ class Validator : public ValidatorBase { lastBest_ = val; if(options_->get("keep-best")) keepBest(graphs); - } else if (lastBest_ != val) { // (special case 0 at start) @TODO: needed? Seems stall count gets reset each time it does improve. If not needed, remove "if(...)" again. + } else /* if (lastBest_ != val) */ { // (special case 0 at start) @TODO: needed? Seems stall count gets reset each time it does improve. If not needed, remove "if(...)" again. stalled_++; } } From c5c25974109c2af7175c4792ebd2b0fb382cc3fd Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 21 Dec 2018 11:52:52 -0800 Subject: [PATCH 061/838] upcast single element to single element sequence in config if vector is expected --- src/common/cli_wrapper.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 28826bb28..6b7262c79 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -135,8 +135,28 @@ bool CLIWrapper::updateConfig(const YAML::Node &config) { if(cmdOptions.count(key)) continue; if(options_.count(key)) { - config_[key] = YAML::Clone(it.second); - options_[key].modified = true; + // this is a default value, so it has a node type + if(config_[key]) { // types don't match, handle this + if(config_[key].Type() != it.second.Type()) { + // default value is a sequence and incoming node is a scalar, hence we can upcast to single element sequence + if(config_[key].Type() == YAML::NodeType::Sequence && it.second.Type() == YAML::NodeType::Scalar) { + YAML::Node sequence; + sequence.push_back(YAML::Clone(it.second)); + + // overwrite so default values are replaced too + config_[key] = sequence; + options_[key].modified = true; + } else { // Cannot convert other non-matching types, e.g. scalar <- list should fail + success = false; + } + } else { // types match, go ahead + config_[key] = YAML::Clone(it.second); + options_[key].modified = true; + } + } else { + config_[key] = YAML::Clone(it.second); + options_[key].modified = true; + } } else { success = false; } From f8a89dcceacd882cc69dcb91b21499bfeaa78d31 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 21 Dec 2018 13:01:25 -0800 Subject: [PATCH 062/838] adjust comments --- src/common/cli_wrapper.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 6b7262c79..1c7ad16bb 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -135,16 +135,14 @@ bool CLIWrapper::updateConfig(const YAML::Node &config) { if(cmdOptions.count(key)) continue; if(options_.count(key)) { - // this is a default value, so it has a node type - if(config_[key]) { // types don't match, handle this - if(config_[key].Type() != it.second.Type()) { + if(config_[key]) { // it exists, so this is a default value, hence it has a node type + if(config_[key].Type() != it.second.Type()) { // types don't match, handle this // default value is a sequence and incoming node is a scalar, hence we can upcast to single element sequence if(config_[key].Type() == YAML::NodeType::Sequence && it.second.Type() == YAML::NodeType::Scalar) { + // create single element sequence YAML::Node sequence; sequence.push_back(YAML::Clone(it.second)); - - // overwrite so default values are replaced too - config_[key] = sequence; + config_[key] = sequence; // overwrite to replace default values options_[key].modified = true; } else { // Cannot convert other non-matching types, e.g. scalar <- list should fail success = false; From c10f2c91bc07c0f6b1bc6f9f76b7e8345e28a166 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 21 Dec 2018 14:29:18 -0800 Subject: [PATCH 063/838] [experiental] exponent for warmup; compensation term for ramp-up of exponential smoothing --- src/common/config_parser.cpp | 3 +++ src/training/exponential_smoothing.h | 4 ++++ src/training/graph_group_sync.cpp | 12 +++++++----- src/training/scheduler.h | 5 +++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 70a30a5fd..14c266c8b 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -618,6 +618,9 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { {"0"}); cli.add("--mini-batch-track-lr", "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); + cli.add("--mini-batch-warmup-exp", + "[experimental] Soften initial ramp-up curve with this exponent (until ramp-up reached 100%)", + 1.); cli.add("--mini-batch-overstuff", "[experimental] Stuff this much more data into a minibatch, but scale down the LR and progress counter", 1); diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h index f8786fca1..30e64430b 100755 --- a/src/training/exponential_smoothing.h +++ b/src/training/exponential_smoothing.h @@ -28,6 +28,10 @@ class ExponentialSmoothing { ABORT_IF(actualBatchTrgWords == OptimizerBase::mbSizeNotProvided, "This graph-group type does not support reference batch size specification for exponential-smoothing"); beta = pow(beta, (double)actualBatchTrgWords / (double)refBatchTrgWords_); + // If actual size differs from reference, then try to estimate the equivalent number of batches. + // E.g. if MB size is growing over time, then this is an overestimate, which would diminish the + // effect overly quickly, but in a range where that should be OK. + batches = std::max(batches, batches * actualBatchTrgWords / refBatchTrgWords_); // @BUGBUG: Does not consider that batch size is changing } // reduce effect of decay parameter in early training stages float decayBy = std::max(1.f - (float)beta, diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index a11d48c54..b89efde87 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -344,14 +344,13 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num } // Compute gradients - // This happens in multiple steps in case of delay > 1. std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device comm_->foreach([&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { // parallel across devices. Aggregate for warp > 1. auto graph = graphs_[localDeviceIndex]; // reset gradient --presently done outside //graph->params()->allocateBackward(); - //if (warp == 0) // these have already been sized - // graph->params()->set_zero_adjoint(); + //graph->params()->set_zero_adjoint(); + // This happens in multiple steps if there are more subbatches than devices. for (size_t warp = 0; ; warp++) { // Execute single forward/backward step auto subBatch = getSubBatch(warp, localDeviceIndex, mpi_->myMPIRank()); @@ -535,10 +534,13 @@ void SyncGraphGroup::finalize() /*override*/ { validate(); Base::finalize(); } - + SyncGraphGroup::~SyncGraphGroup() /*override*/ { comm_ = nullptr; - finalizeMPI(std::move(mpi_)); + static int c = 0; + c++; + if (c == 2) // HACKHACK: MPI can only be closed once + finalizeMPI(std::move(mpi_)); } } // namespace marian diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 2f2fc5a21..14f2bb07f 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -83,6 +83,11 @@ class Scheduler : public TrainingObserver { // if unit is labels, then account for the fact that our increment itself is not constant if (mbWarmup.unit == SchedulingUnit::trgLabels) progressRatio = std::sqrt(progressRatio); + // soften the very initial ramp-up if requested + if (progressRatio < 1) { + double exp = options_->get("mini-batch-warmup-exp"); + progressRatio = pow(progressRatio, exp); // e.g. 1.5 => linear ramp-up -> sublinear ramp-up + } // apply ratio to actual batch size ratio *= progressRatio; } From 776aa2a7d38af110ee80d9e4339495e0c04e7057 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 21 Dec 2018 15:10:17 -0800 Subject: [PATCH 064/838] bug fix: MPI must only be finalized once --- src/training/graph_group.h | 14 ++------------ src/training/graph_group_async.cpp | 3 ++- src/training/graph_group_async.h | 2 +- src/training/graph_group_async_drop.h | 4 ++-- src/training/graph_group_multinode.h | 4 ++-- src/training/graph_group_multinode_sync.h | 4 ++-- src/training/graph_group_singleton.h | 3 ++- src/training/graph_group_sync.cpp | 17 +++-------------- src/training/graph_group_sync.h | 4 +--- src/training/training.h | 17 ++++++++++------- 10 files changed, 27 insertions(+), 45 deletions(-) diff --git a/src/training/graph_group.h b/src/training/graph_group.h index c1d6c818a..9a888bda8 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -136,11 +136,8 @@ class MultiNodeGraphGroupBase : public GraphGroup { std::vector> clientGraphs_; // [num local GPUs] public: - MultiNodeGraphGroupBase(Ptr options) - : Base(options) { - - // Setup MPI - setupMPI(); + MultiNodeGraphGroupBase(Ptr options, Ptr mpi) + : Base(options), mpi_(mpi) { // Set up devices for this node std::vector devices; // set of GPU device ids for this MPI process @@ -157,13 +154,6 @@ class MultiNodeGraphGroupBase : public GraphGroup { } } - /** - * Setup MPI world size and rank of this node. - */ - void setupMPI() { - mpi_ = initMPI(/*multiThreaded=*/!options_->get("sync-sgd")); - } - /** * Load the GPU configuration of this node (i.e. which GPUs to use) and the * number of GPUs on the other nodes. diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 4c85e168a..ca7f7b487 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -5,12 +5,13 @@ namespace marian { -AsyncGraphGroup::AsyncGraphGroup(Ptr config) +AsyncGraphGroup::AsyncGraphGroup(Ptr config, Ptr mpi) : GraphGroup(config), ExponentialSmoothing(options_), devices_{Config::getDevices(options_)}, shardSync_(devices_.size()), optimizerDelay_((size_t)options_->get("optimizer-delay")) { + ABORT_IF(mpi->numMPIProcesses() != 1, "AsyncGraphGroup presently does not support multiple MPI processes"); ABORT_IF((double)optimizerDelay_ != options_->get("optimizer-delay"), "AsyncGraphGroup presently does not implement fractional values for --optimizer-delay"); pool_.reset(new ThreadPool(devices_.size(), devices_.size())); diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h index 713e30064..d77fff7f3 100755 --- a/src/training/graph_group_async.h +++ b/src/training/graph_group_async.h @@ -52,7 +52,7 @@ class AsyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void execute(Ptr batch); public: - AsyncGraphGroup(Ptr config); + AsyncGraphGroup(Ptr config, Ptr mpi); void update(Ptr batch) override { validate(); diff --git a/src/training/graph_group_async_drop.h b/src/training/graph_group_async_drop.h index 7d22208b3..3e313440f 100755 --- a/src/training/graph_group_async_drop.h +++ b/src/training/graph_group_async_drop.h @@ -30,8 +30,8 @@ class AsyncGraphGroupDrop : public AsyncGraphGroup { int device_id) override; public: - AsyncGraphGroupDrop(Ptr options) - : AsyncGraphGroup(options), + AsyncGraphGroupDrop(Ptr options, Ptr mpi) + : AsyncGraphGroup(options, mpi), dropping_warmup{options->get("grad-dropping-warmup")}, droping_rate{options->get("grad-dropping-rate")}, dropping_momentum{options->get("grad-dropping-momentum")} {} diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index 873155df7..a05787e56 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -351,8 +351,8 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { /** * (Constructor) Call super class and initialize client graphs and builders. */ - MultiNodeGraphGroup(Ptr options) - : Base(options), + MultiNodeGraphGroup(Ptr options, Ptr mpi) + : Base(options, mpi), clientCommOverlap{options_->get("multi-node-overlap")}, tau_{(size_t)options_->get("optimizer-delay")} { } diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index 7b93b2a84..cdb95ae6c 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -131,8 +131,8 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { /** * (Constructor) Call super class and initialize client graphs and builders. */ - MultiNodeGraphGroupSync(Ptr options) - : Base(options), + MultiNodeGraphGroupSync(Ptr options, Ptr mpi) + : Base(options, mpi), tau_{(size_t)options_->get("optimizer-delay")}, syncOptimizer_{Optimizer(options_)}, movingAvg_{options_->get("exponential-smoothing") > 0}, diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 9e417b7d6..f9e67fb90 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -23,9 +23,10 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { void execute(Ptr batch); public: - SingletonGraph(Ptr config) + SingletonGraph(Ptr config, Ptr mpi) : GraphGroup(config), ExponentialSmoothing(config) { + ABORT_IF(mpi->numMPIProcesses() != 1, "SingletonGraph does not support multiple MPI processes"); // Get device ID auto devices = Config::getDevices(options_); ABORT_IF(devices.size() != 1, "Only one device ID should be provided for singleton training"); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index b89efde87..c00a4fc2f 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -2,12 +2,9 @@ namespace marian { -SyncGraphGroup::SyncGraphGroup(Ptr config) - : GraphGroup(config), - ExponentialSmoothing(config), - delay_{options_->get("optimizer-delay")} { // @TODO: rename to something else; delay means delayed updated, not accumulation - - mpi_ = initMPI(/*multiThreaded=*/false); // when not running under MPI, this will be a fake object that represents a one-MPI-process setup +SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) + : GraphGroup(config), ExponentialSmoothing(config), + delay_{options_->get("optimizer-delay")}, mpi_(mpi) { // @TODO: rename delay_ to something else; delay means delayed updated, not accumulation devices_ = Config::getDevices(options_, mpi_->myMPIRank(), mpi_->numMPIProcesses()); for(auto device : devices_) { @@ -535,12 +532,4 @@ void SyncGraphGroup::finalize() /*override*/ { Base::finalize(); } -SyncGraphGroup::~SyncGraphGroup() /*override*/ { - comm_ = nullptr; - static int c = 0; - c++; - if (c == 2) // HACKHACK: MPI can only be closed once - finalizeMPI(std::move(mpi_)); -} - } // namespace marian diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index d7ecec2ff..03ee8c4d2 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -41,7 +41,7 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void update(std::vector> subBatches, size_t numReadBatches); public: - SyncGraphGroup(Ptr config); + SyncGraphGroup(Ptr config, Ptr mpi); void setScheduler(Ptr scheduler) override; @@ -54,7 +54,5 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { Ptr collectStats(); // @TODO: consider to make this a virtual as well? Currently it is a template dispatch - - ~SyncGraphGroup() override; }; } // namespace marian diff --git a/src/training/training.h b/src/training/training.h index b2290db75..c388d6195 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -36,15 +36,15 @@ class Train : public ModelTask { auto trainState = New(options_->get("learn-rate")); auto scheduler = New(options_, trainState); + auto mpi = initMPI(/*multiThreaded=*/!options_->get("sync-sgd")); // @TODO: do we need the multiThreaded distinction at all? Ptr stats; if(options_->get("mini-batch-fit")) { LOG(info, - "[batching] Collecting statistics for batch fitting with step size " - "{}", + "[batching] Collecting statistics for batch fitting with step size {}", options_->get("mini-batch-fit-step")); // @TODO, better fake batch with vocabulary - auto model = New(options_); + auto model = New(options_, mpi); model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling stats = model->collectStats(); LOG(info, "[batching] Done. Typical MB size is {} target words", stats->estimateTypicalTrgWords()); @@ -60,7 +60,7 @@ class Train : public ModelTask { scheduler->registerTrainingObserver(batchGenerator); - auto model = New(options_); + auto model = New(options_, mpi); model->setScheduler(scheduler); model->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); // needed for dynamic MB scaling model->load(); @@ -91,12 +91,15 @@ class Train : public ModelTask { } scheduler->finished(); - model->finalize(); + model->finalize(); // allow async to sync before final save --@TODO: rename, or move into save() - // Avoid saving the model twice if it has been loaded and training did not - // progress + // Avoid saving the model twice if it has been loaded and training did not progress if(!trainState->loaded) model->save(true); + + // Signal success to a potential MPI runner + model = nullptr; // release any reference to MPI that model may hold + finalizeMPI(std::move(mpi)); } }; } // namespace marian From fdd1e4c9cd8dc0ede09228061d9824909d7e901a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 24 Dec 2018 12:09:37 +0000 Subject: [PATCH 065/838] Fix implicit values for vectors --- src/3rd_party/CLI/App.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/3rd_party/CLI/App.hpp b/src/3rd_party/CLI/App.hpp index 555f480c1..fe430c16c 100644 --- a/src/3rd_party/CLI/App.hpp +++ b/src/3rd_party/CLI/App.hpp @@ -1613,8 +1613,10 @@ class App { if(emptyVectorArgs) { if(op->get_implicit()) { - op->add_result(op->get_implicitval()); - parse_order_.push_back(op.get()); + for(const auto& ival : detail::split_up(op->get_implicitval())) { + op->add_result(ival); + parse_order_.push_back(op.get()); + } } else if (op->get_expected() < 0) { parse_order_.push_back(op.get()); throw ArgumentMismatch(op->get_name(), op->get_expected(), 0); From 0d0fe5906a5f9e16494894ca5018071a2800a88d Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 24 Dec 2018 13:10:57 +0000 Subject: [PATCH 066/838] Add comment for parseAliases --- src/common/cli_wrapper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index d06bdd718..57400f6ca 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -229,8 +229,10 @@ class CLIWrapper { // Parse command-line arguments. Handles --help and --version options void parse(int argc, char **argv); + // Expand aliases based on arguments parsed with parse(int, char**) method void parseAliases(const YAML::Node &config) { for(const auto& alias : aliases_) { + // note: options values are always compared as strings if(config[alias.key] && config[alias.key].as() == alias.value) { updateConfig(alias.config, "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); From 9b2d58ab5ab309011a28281399ef2285019f9502 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 24 Dec 2018 13:26:55 +0000 Subject: [PATCH 067/838] Remove parsed aliases automatically --- src/common/cli_wrapper.cpp | 21 +++++++++++++++++++++ src/common/cli_wrapper.h | 18 ++++++++---------- src/common/config_parser.cpp | 8 ++------ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index df647bfe4..65f34aa84 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -128,6 +128,27 @@ void CLIWrapper::parse(int argc, char **argv) { } } +void CLIWrapper::parseAliases() { + if(aliases_.empty()) + return; + + std::set aliasKeys; + for(const auto& alias : aliases_) { + // Note: options values are always compared as strings + if(config_[alias.key] && config_[alias.key].as() == alias.value) { + updateConfig(alias.config, + "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); + } + aliasKeys.insert(alias.key); + } + + // Remove aliases from the config + for(const auto& key : aliasKeys) { + config_.remove(key); + } +} + + std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) { std::string header = "Error: " + std::string(e.what()) + "\n"; if(app->get_help_ptr() != nullptr) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 57400f6ca..a748c560e 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -229,16 +229,14 @@ class CLIWrapper { // Parse command-line arguments. Handles --help and --version options void parse(int argc, char **argv); - // Expand aliases based on arguments parsed with parse(int, char**) method - void parseAliases(const YAML::Node &config) { - for(const auto& alias : aliases_) { - // note: options values are always compared as strings - if(config[alias.key] && config[alias.key].as() == alias.value) { - updateConfig(alias.config, - "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); - } - } - } + /** + * @brief Expand aliases based on arguments parsed with parse(int, char**) + * + * Should be called after parse(int, char**) to take an effect. + * + * All options defined as aliases are removed from the config object. + */ + void parseAliases(); /* * @brief Overwrite values for unparsed options diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index f4d86c567..355641cb2 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -767,9 +767,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { config_.remove("dump-config"); if(type == "explain") { - cli.parseAliases(config_); - config_.remove("best-deep"); - config_.remove("problem"); + cli.parseAliases(); } bool minimal = (type == "minimal" || type == "explain"); @@ -777,9 +775,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { exit(0); } - cli.parseAliases(config_); - config_.remove("best-deep"); - config_.remove("problem"); + cli.parseAliases(); } std::vector ConfigParser::findConfigPaths() { From a9a0f65380622f1c90944d0570f3cc94d02a3b29 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 24 Dec 2018 13:47:51 +0000 Subject: [PATCH 068/838] Fix options in transformer-big --- src/common/config_parser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 355641cb2..3a8f054af 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -701,8 +701,8 @@ void ConfigParser::addAliases(cli::CLIWrapper& cli) { config["lr-warmup"] = 8000; config["lr-decay-inv-sqrt"] = 8000; config["transformer-dropout"] = 0.1; - config["transformer-attention-dropout"] = 0.1; - config["transformer-ffn-dropout"] = 0.1; + config["transformer-dropout-attention"] = 0.1; + config["transformer-dropout-ffn"] = 0.1; config["label-smoothing"] = 0.1; config["clip-norm"] = 5; }); From b013bab0d3a18ade9d3abc362f7368ed93e6014a Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 24 Dec 2018 14:00:20 +0000 Subject: [PATCH 069/838] Make --problem a vector --- src/common/cli_wrapper.cpp | 17 ++++++++++++++--- src/common/config_parser.cpp | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 65f34aa84..27c8031a8 100755 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -134,12 +134,23 @@ void CLIWrapper::parseAliases() { std::set aliasKeys; for(const auto& alias : aliases_) { - // Note: options values are always compared as strings - if(config_[alias.key] && config_[alias.key].as() == alias.value) { + if(config_[alias.key]) { + bool expand = false; + if(config_[alias.key].IsSequence()) { + // Note: options values are always extracted as vectors and compared as strings + auto aliasOpts = config_[alias.key].as>(); + expand = std::find(aliasOpts.begin(), aliasOpts.end(), alias.value) != aliasOpts.end(); + } else { + // Note: options values are always compared as strings + expand = config_[alias.key].as() == alias.value; + } + + if(expand) { updateConfig(alias.config, "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); + } + aliasKeys.insert(alias.key); } - aliasKeys.insert(alias.key); } // Remove aliases from the config diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 3a8f054af..1e75e7590 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -399,8 +399,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { // add ULR settings addSuboptionsULR(cli); - cli.add("--problem", - "Use predefined set of options. Possible values: transformer"); + cli.add>("--problem", + "Use predefined set of options. Possible values: transformer, transformer-big"); // clang-format on } From 71cf7009d612912c83b44f36dbfaee305a136544 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 26 Dec 2018 11:04:44 -0800 Subject: [PATCH 070/838] changed back to log last best in validation --- src/training/scheduler.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 14f2bb07f..5db0cfdea 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -179,12 +179,12 @@ class Scheduler : public TrainingObserver { float value = validator->validate(graphs); if(validator->stalled() > 0) { LOG_VALID(info, - "Ep. {} : Up. {} : {} : {} : stalled {} times", /*"(last best: {})",*/ // @TODO (LOGGING CHANGE) + "Ep. {} : Up. {} : {} : {} : stalled {} times (last best: {})", // @TODO (LOGGING CHANGE) state_->epochs, state_->batches, validator->type(), value, - validator->stalled() /*, validator->lastBest()*/); // @TODO (LOGGING CHANGE) + validator->stalled(), validator->lastBest()); // @TODO (LOGGING CHANGE) } else { LOG_VALID(info, "Ep. {} : Up. {} : {} : {} : new best", From 8fec73b7712e8083b0427724fc658976c2e18a36 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 26 Dec 2018 11:50:24 -0800 Subject: [PATCH 071/838] enable last-best reporting --- src/training/scheduler.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 2f2fc5a21..10d171874 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -174,12 +174,12 @@ class Scheduler : public TrainingObserver { float value = validator->validate(graphs); if(validator->stalled() > 0) { LOG_VALID(info, - "Ep. {} : Up. {} : {} : {} : stalled {} times", /*"(last best: {})",*/ // @TODO (LOGGING CHANGE) + "Ep. {} : Up. {} : {} : {} : stalled {} times (last best: {})", state_->epochs, state_->batches, validator->type(), value, - validator->stalled() /*, validator->lastBest()*/); // @TODO (LOGGING CHANGE) + validator->stalled(), validator->lastBest()); } else { LOG_VALID(info, "Ep. {} : Up. {} : {} : {} : new best", From 1e143f61fb9c46c0b33d834510c7379e36fe22a6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 26 Dec 2018 14:20:16 -0800 Subject: [PATCH 072/838] added elementwise comparison operators (lt(), eq(), ); bug fix: Maximum and Minimum got it backwards; streamlined the operator tests a little --- src/functional/predicates.h | 15 +++---- src/graph/expression_operators.cpp | 21 ++++++++++ src/graph/expression_operators.h | 25 +++++++++++- src/graph/node_operators_binary.h | 29 ++++++++++++++ src/tensors/gpu/element.cu | 0 src/tensors/gpu/element.inc | 11 ++++- src/tests/operator_tests.cpp | 64 +++++++++++++++++------------- src/tests/prod.cpp | 1 - vs/Marian.vcxproj | 58 +++++++++++++++++++++++++++ vs/Marian.vcxproj.filters | 51 ++++++++++++++++++++++++ 10 files changed, 235 insertions(+), 40 deletions(-) mode change 100644 => 100755 src/tensors/gpu/element.cu mode change 100644 => 100755 src/tests/operator_tests.cpp mode change 100644 => 100755 src/tests/prod.cpp diff --git a/src/functional/predicates.h b/src/functional/predicates.h index fce5da473..4d92a0224 100755 --- a/src/functional/predicates.h +++ b/src/functional/predicates.h @@ -98,15 +98,12 @@ BINARY(Div, operator/, x / y); BINARY(LogAddExp, logaddexp, - (/*if*/ (x < y) - ? // Note: This may not be ideal for CUDA; cf. CNTK implementation - (y + log1pf(expf(x - y))) - /*else*/ - : (x + log1pf(expf(y - x))))); -BINARY(Maximum, - max, - (x > y) ? y : x); // note: std::max not available on CUDA it seems -BINARY(Minimum, min, (x < y) ? y : x); + (/*if*/ (x < y) ? // Note: This may not be ideal for CUDA; cf. CNTK implementation + (y + log1pf(expf(x - y))) + /*else*/ : + (x + log1pf(expf(y - x))))); +BINARY(Maximum, max, (x > y) ? x : y); // Note: std::max not available on CUDA it seems +BINARY(Minimum, min, (x < y) ? x : y); UNARY(Negate, operator!, !x); BINARY(Eq, operator==, x == y); diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 2f4f5ecf9..4870a6664 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -107,6 +107,27 @@ Expr minimum(Expr a, Expr b) { return Expression(a, b); } +Expr lt(Expr a, Expr b) { return Expression(a, b, -1, false); } +Expr eq(Expr a, Expr b) { return Expression(a, b, 0, false); } +Expr gt(Expr a, Expr b) { return Expression(a, b, 1, false); } +Expr ge(Expr a, Expr b) { return Expression(a, b, -1, true); } +Expr ne(Expr a, Expr b) { return Expression(a, b, 0, true); } +Expr le(Expr a, Expr b) { return Expression(a, b, 1, true); } + +Expr lt(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, -1, false); } +Expr eq(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 0, false); } +Expr gt(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 1, false); } +Expr ge(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, -1, true); } +Expr ne(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 0, true); } +Expr le(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 1, true); } + +Expr lt(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), -1, false); } +Expr eq(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 0, false); } +Expr gt(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 1, false); } +Expr ge(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), -1, true); } +Expr ne(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 0, true); } +Expr le(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 1, true); } + /*********************************************************/ Expr operator+(Expr a, float b) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 6a887aa0a..ab1b364b8 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -66,10 +66,33 @@ Expr operator/(Expr a, float b); Expr logaddexp(Expr a, Expr b); +// Note: Following numpy, minimum() is element-wise, while min() is along an axis in both Numpy and PyTorch. Expr maximum(Expr a, Expr b); - Expr minimum(Expr a, Expr b); +// Note: We cannot overload the relational operators, as they also mean something for Expr itself. +// Note: These names follow PyTorch convention. +Expr lt(Expr a, Expr b); +Expr eq(Expr a, Expr b); +Expr gt(Expr a, Expr b); +Expr ge(Expr a, Expr b); +Expr ne(Expr a, Expr b); +Expr le(Expr a, Expr b); + +Expr lt(float a, Expr b); +Expr eq(float a, Expr b); +Expr gt(float a, Expr b); +Expr ge(float a, Expr b); +Expr ne(float a, Expr b); +Expr le(float a, Expr b); + +Expr lt(Expr a, float b); +Expr eq(Expr a, float b); +Expr gt(Expr a, float b); +Expr ge(Expr a, float b); +Expr ne(Expr a, float b); +Expr le(Expr a, float b); + Expr dot(Expr a, Expr b, bool transA = false, diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 0f972bc79..868ee4ebd 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -814,6 +814,35 @@ struct MinimumNodeOp : public ElementBinaryNodeOp { const std::string type() override { return "min"; } }; +struct CmpNodeOp : public ElementBinaryNodeOp { + CmpNodeOp(Expr a, Expr b, int cmp_, bool not_) : ElementBinaryNodeOp(a, b), cmp_(cmp_), not_(not_) { + setTrainable(false); // has no gradient + } + + NodeOps forwardOps() override { + using namespace functional; + + return { + NodeOp(Element(_1 = ((((_2 > _3) - (_2 < _3)) == (float)cmp_) != not_), + val_, child(0)->val(), child(1)->val()))}; + } + + NodeOps backwardOps() override { return {}; } + + const std::string type() override { + switch (cmp_) { + case -1: return not_ ? "ge" : "lt"; + case 0: return not_ ? "ne" : "eq"; + case 1: return not_ ? "le" : "gt"; + } + ABORT("Should not get here??"); + } + +private: + int cmp_; // -1: less; 0: equal; 1: greater + bool not_; // invert result if true +}; + // In each j-th row, take the corresponding j-th label index i from indices and compute: // For each vocabulary item v, the only non-zero element in a row in the sum is the item // that matches the label indexed by i (the picked element). diff --git a/src/tensors/gpu/element.cu b/src/tensors/gpu/element.cu old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc index 96debcc2f..66f763018 100755 --- a/src/tensors/gpu/element.inc +++ b/src/tensors/gpu/element.inc @@ -52,5 +52,12 @@ template void Element, BinaryFunctor, Bin template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); -template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); -template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Element, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr >(marian::functional::Assign, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr, std::shared_ptr); +// How to add new specializations: +// When you use a new specialization, it will cause a link error of this form (example): +// .../src/tensors/tensor_operators.h:41: undefined reference to `void marian::gpu::Element ( ... )' +// To fix this, copy the line with the error message in here and: +// - replace up to including "undefined reference to `" by "template" +// - replace final ' by a semicolon diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp old mode 100644 new mode 100755 index 7c17ec8fd..30f4dfb70 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -1,11 +1,13 @@ #include "catch.hpp" #include "graph/expression_graph.h" #include "graph/expression_operators.h" +#include using namespace marian; void tests(DeviceType device) { - auto floatApprox = [](float x, float y) { return x == Approx(y); }; + auto floatApprox = [](float x, float y) -> bool { return x == Approx(y); }; + auto floatEqual = [](float x, float y) -> bool { return x == y; }; Config::seed = 1234; @@ -38,38 +40,46 @@ void tests(DeviceType device) { std::vector vA({1, -2, 3, -4}); std::vector vB({0.5, 1.5}); - std::vector vAdd({1.5, -0.5, 3.5, -2.5}); - std::vector vMinus({-0.5, 3.5, -2.5, 5.5}); - std::vector vMult({0.5, -3.0, 1.5, -6.0}); - std::vector vDiv({2.0f, -1.33333f, 6.0f, -2.66667f}); - auto a = graph->constant({2, 2, 1}, inits::from_vector(vA)); auto b = graph->constant({2, 1}, inits::from_vector(vB)); - auto add = a + b; - auto minus = b - a; - auto mult = a * b; - auto div = a / b; + auto compare = [&](Expr res, std::function f, bool exactMatch) -> bool { + if (res->shape() != Shape({ 2, 2, 1 })) + return false; + res->val()->get(values); + std::vector ref{f(vA[0], vB[0]), f(vA[1], vB[1]), f(vA[2], vB[0]), f(vA[3], vB[1])}; + return std::equal(values.begin(), values.end(), ref.begin(), exactMatch ? floatEqual : floatApprox); + }; + + auto rplus = a + b; + auto rminus = a - b; + auto rmult = a * b; + auto rdiv = a / b; + auto rlae = logaddexp(a, b); + auto rmax = maximum(a, b); + auto rmin = minimum(a, b); + auto rlt = lt(a, b); + auto req = eq(a, b); + auto rgt = gt(a, b); + auto rge = ge(a, b); + auto rne = ne(a, b); + auto rle = le(a, b); graph->forward(); - CHECK(add->shape() == Shape({2, 2, 1})); - CHECK(minus->shape() == Shape({2, 2, 1})); - CHECK(mult->shape() == Shape({2, 2, 1})); - CHECK(div->shape() == Shape({2, 2, 1})); - - add->val()->get(values); - CHECK( values == vAdd ); - - minus->val()->get(values); - CHECK( values == vMinus ); - - mult->val()->get(values); - CHECK( values == vMult ); - - div->val()->get(values); - CHECK( std::equal(values.begin(), values.end(), - vDiv.begin(), floatApprox) ); + CHECK(compare(rplus, [](float a, float b) {return a + b;}, true)); + CHECK(compare(rminus, [](float a, float b) {return a - b;}, true)); + CHECK(compare(rmult, [](float a, float b) {return a * b;}, true)); + CHECK(compare(rdiv, [](float a, float b) {return a / b;}, /*exactMatch=*/false)); + CHECK(compare(rlae, [](float a, float b) {return logf(expf(a) + expf(b));}, /*exactMatch=*/false)); + CHECK(compare(rmax, [](float a, float b) {return std::max(a, b);}, true)); + CHECK(compare(rmin, [](float a, float b) {return std::min(a, b);}, true)); + CHECK(compare(rlt, [](float a, float b) {return a < b;}, true)); + CHECK(compare(req, [](float a, float b) {return a == b;}, true)); + CHECK(compare(rgt, [](float a, float b) {return a > b;}, true)); + CHECK(compare(rge, [](float a, float b) {return a >= b;}, true)); + CHECK(compare(rne, [](float a, float b) {return a != b;}, true)); + CHECK(compare(rle, [](float a, float b) {return a <= b;}, true)); } SECTION("transposing and reshaping") { diff --git a/src/tests/prod.cpp b/src/tests/prod.cpp old mode 100644 new mode 100755 index 5ade67168..c329d403e --- a/src/tests/prod.cpp +++ b/src/tests/prod.cpp @@ -68,6 +68,5 @@ int main(int argc, char** argv) { } } - return 0; } diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 15486147a..300e4c91b 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -588,6 +588,50 @@ + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + @@ -1032,6 +1076,19 @@ true + + true + true + + + true + true + + + + true + true + @@ -1041,6 +1098,7 @@ true true + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index ce6d8441b..e488f7a06 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -439,6 +439,39 @@ 3rd_party\pathie-cpp\src + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + @@ -1627,6 +1660,9 @@ {5d5ee615-192f-4b7f-bdfd-fb8316ceabc8} + + {a86d650a-2268-43d9-9d74-cb17cd6b534b} + @@ -1765,6 +1801,18 @@ 3rd_party\pathie-cpp + + tests + + + tests + + + tests + + + tests + @@ -1773,5 +1821,8 @@ 3rd_party\pathie-cpp + + tests + \ No newline at end of file From efc20adacc9e88785fc8f42c98b002ebf6a47f3a Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 26 Dec 2018 14:20:16 -0800 Subject: [PATCH 073/838] added elementwise comparison operators (lt(), eq(), ); bug fix: Maximum and Minimum got it backwards; streamlined the operator tests a little --- src/functional/predicates.h | 15 +++---- src/graph/expression_operators.cpp | 21 ++++++++++ src/graph/expression_operators.h | 25 +++++++++++- src/graph/node_operators_binary.h | 29 ++++++++++++++ src/tensors/gpu/element.cu | 0 src/tensors/gpu/element.inc | 11 ++++- src/tests/operator_tests.cpp | 64 +++++++++++++++++------------- src/tests/prod.cpp | 1 - vs/Marian.vcxproj | 58 +++++++++++++++++++++++++++ vs/Marian.vcxproj.filters | 51 ++++++++++++++++++++++++ 10 files changed, 235 insertions(+), 40 deletions(-) mode change 100644 => 100755 src/tensors/gpu/element.cu mode change 100644 => 100755 src/tests/operator_tests.cpp mode change 100644 => 100755 src/tests/prod.cpp diff --git a/src/functional/predicates.h b/src/functional/predicates.h index fce5da473..4d92a0224 100755 --- a/src/functional/predicates.h +++ b/src/functional/predicates.h @@ -98,15 +98,12 @@ BINARY(Div, operator/, x / y); BINARY(LogAddExp, logaddexp, - (/*if*/ (x < y) - ? // Note: This may not be ideal for CUDA; cf. CNTK implementation - (y + log1pf(expf(x - y))) - /*else*/ - : (x + log1pf(expf(y - x))))); -BINARY(Maximum, - max, - (x > y) ? y : x); // note: std::max not available on CUDA it seems -BINARY(Minimum, min, (x < y) ? y : x); + (/*if*/ (x < y) ? // Note: This may not be ideal for CUDA; cf. CNTK implementation + (y + log1pf(expf(x - y))) + /*else*/ : + (x + log1pf(expf(y - x))))); +BINARY(Maximum, max, (x > y) ? x : y); // Note: std::max not available on CUDA it seems +BINARY(Minimum, min, (x < y) ? x : y); UNARY(Negate, operator!, !x); BINARY(Eq, operator==, x == y); diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 2f4f5ecf9..4870a6664 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -107,6 +107,27 @@ Expr minimum(Expr a, Expr b) { return Expression(a, b); } +Expr lt(Expr a, Expr b) { return Expression(a, b, -1, false); } +Expr eq(Expr a, Expr b) { return Expression(a, b, 0, false); } +Expr gt(Expr a, Expr b) { return Expression(a, b, 1, false); } +Expr ge(Expr a, Expr b) { return Expression(a, b, -1, true); } +Expr ne(Expr a, Expr b) { return Expression(a, b, 0, true); } +Expr le(Expr a, Expr b) { return Expression(a, b, 1, true); } + +Expr lt(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, -1, false); } +Expr eq(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 0, false); } +Expr gt(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 1, false); } +Expr ge(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, -1, true); } +Expr ne(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 0, true); } +Expr le(float a, Expr b) { return Expression(b->graph()->constant({}, inits::from_value(a), b->value_type()), b, 1, true); } + +Expr lt(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), -1, false); } +Expr eq(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 0, false); } +Expr gt(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 1, false); } +Expr ge(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), -1, true); } +Expr ne(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 0, true); } +Expr le(Expr a, float b) { return Expression(a, a->graph()->constant({}, inits::from_value(b), a->value_type()), 1, true); } + /*********************************************************/ Expr operator+(Expr a, float b) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 6a887aa0a..ab1b364b8 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -66,10 +66,33 @@ Expr operator/(Expr a, float b); Expr logaddexp(Expr a, Expr b); +// Note: Following numpy, minimum() is element-wise, while min() is along an axis in both Numpy and PyTorch. Expr maximum(Expr a, Expr b); - Expr minimum(Expr a, Expr b); +// Note: We cannot overload the relational operators, as they also mean something for Expr itself. +// Note: These names follow PyTorch convention. +Expr lt(Expr a, Expr b); +Expr eq(Expr a, Expr b); +Expr gt(Expr a, Expr b); +Expr ge(Expr a, Expr b); +Expr ne(Expr a, Expr b); +Expr le(Expr a, Expr b); + +Expr lt(float a, Expr b); +Expr eq(float a, Expr b); +Expr gt(float a, Expr b); +Expr ge(float a, Expr b); +Expr ne(float a, Expr b); +Expr le(float a, Expr b); + +Expr lt(Expr a, float b); +Expr eq(Expr a, float b); +Expr gt(Expr a, float b); +Expr ge(Expr a, float b); +Expr ne(Expr a, float b); +Expr le(Expr a, float b); + Expr dot(Expr a, Expr b, bool transA = false, diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 0f972bc79..868ee4ebd 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -814,6 +814,35 @@ struct MinimumNodeOp : public ElementBinaryNodeOp { const std::string type() override { return "min"; } }; +struct CmpNodeOp : public ElementBinaryNodeOp { + CmpNodeOp(Expr a, Expr b, int cmp_, bool not_) : ElementBinaryNodeOp(a, b), cmp_(cmp_), not_(not_) { + setTrainable(false); // has no gradient + } + + NodeOps forwardOps() override { + using namespace functional; + + return { + NodeOp(Element(_1 = ((((_2 > _3) - (_2 < _3)) == (float)cmp_) != not_), + val_, child(0)->val(), child(1)->val()))}; + } + + NodeOps backwardOps() override { return {}; } + + const std::string type() override { + switch (cmp_) { + case -1: return not_ ? "ge" : "lt"; + case 0: return not_ ? "ne" : "eq"; + case 1: return not_ ? "le" : "gt"; + } + ABORT("Should not get here??"); + } + +private: + int cmp_; // -1: less; 0: equal; 1: greater + bool not_; // invert result if true +}; + // In each j-th row, take the corresponding j-th label index i from indices and compute: // For each vocabulary item v, the only non-zero element in a row in the sum is the item // that matches the label indexed by i (the picked element). diff --git a/src/tensors/gpu/element.cu b/src/tensors/gpu/element.cu old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc index 96debcc2f..66f763018 100755 --- a/src/tensors/gpu/element.inc +++ b/src/tensors/gpu/element.inc @@ -52,5 +52,12 @@ template void Element, BinaryFunctor, Bin template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Assignee<4> >, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); -template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); -template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Element, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr >(marian::functional::Assign, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr, std::shared_ptr); +// How to add new specializations: +// When you use a new specialization, it will cause a link error of this form (example): +// .../src/tensors/tensor_operators.h:41: undefined reference to `void marian::gpu::Element ( ... )' +// To fix this, copy the line with the error message in here and: +// - replace up to including "undefined reference to `" by "template" +// - replace final ' by a semicolon diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp old mode 100644 new mode 100755 index 7c17ec8fd..30f4dfb70 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -1,11 +1,13 @@ #include "catch.hpp" #include "graph/expression_graph.h" #include "graph/expression_operators.h" +#include using namespace marian; void tests(DeviceType device) { - auto floatApprox = [](float x, float y) { return x == Approx(y); }; + auto floatApprox = [](float x, float y) -> bool { return x == Approx(y); }; + auto floatEqual = [](float x, float y) -> bool { return x == y; }; Config::seed = 1234; @@ -38,38 +40,46 @@ void tests(DeviceType device) { std::vector vA({1, -2, 3, -4}); std::vector vB({0.5, 1.5}); - std::vector vAdd({1.5, -0.5, 3.5, -2.5}); - std::vector vMinus({-0.5, 3.5, -2.5, 5.5}); - std::vector vMult({0.5, -3.0, 1.5, -6.0}); - std::vector vDiv({2.0f, -1.33333f, 6.0f, -2.66667f}); - auto a = graph->constant({2, 2, 1}, inits::from_vector(vA)); auto b = graph->constant({2, 1}, inits::from_vector(vB)); - auto add = a + b; - auto minus = b - a; - auto mult = a * b; - auto div = a / b; + auto compare = [&](Expr res, std::function f, bool exactMatch) -> bool { + if (res->shape() != Shape({ 2, 2, 1 })) + return false; + res->val()->get(values); + std::vector ref{f(vA[0], vB[0]), f(vA[1], vB[1]), f(vA[2], vB[0]), f(vA[3], vB[1])}; + return std::equal(values.begin(), values.end(), ref.begin(), exactMatch ? floatEqual : floatApprox); + }; + + auto rplus = a + b; + auto rminus = a - b; + auto rmult = a * b; + auto rdiv = a / b; + auto rlae = logaddexp(a, b); + auto rmax = maximum(a, b); + auto rmin = minimum(a, b); + auto rlt = lt(a, b); + auto req = eq(a, b); + auto rgt = gt(a, b); + auto rge = ge(a, b); + auto rne = ne(a, b); + auto rle = le(a, b); graph->forward(); - CHECK(add->shape() == Shape({2, 2, 1})); - CHECK(minus->shape() == Shape({2, 2, 1})); - CHECK(mult->shape() == Shape({2, 2, 1})); - CHECK(div->shape() == Shape({2, 2, 1})); - - add->val()->get(values); - CHECK( values == vAdd ); - - minus->val()->get(values); - CHECK( values == vMinus ); - - mult->val()->get(values); - CHECK( values == vMult ); - - div->val()->get(values); - CHECK( std::equal(values.begin(), values.end(), - vDiv.begin(), floatApprox) ); + CHECK(compare(rplus, [](float a, float b) {return a + b;}, true)); + CHECK(compare(rminus, [](float a, float b) {return a - b;}, true)); + CHECK(compare(rmult, [](float a, float b) {return a * b;}, true)); + CHECK(compare(rdiv, [](float a, float b) {return a / b;}, /*exactMatch=*/false)); + CHECK(compare(rlae, [](float a, float b) {return logf(expf(a) + expf(b));}, /*exactMatch=*/false)); + CHECK(compare(rmax, [](float a, float b) {return std::max(a, b);}, true)); + CHECK(compare(rmin, [](float a, float b) {return std::min(a, b);}, true)); + CHECK(compare(rlt, [](float a, float b) {return a < b;}, true)); + CHECK(compare(req, [](float a, float b) {return a == b;}, true)); + CHECK(compare(rgt, [](float a, float b) {return a > b;}, true)); + CHECK(compare(rge, [](float a, float b) {return a >= b;}, true)); + CHECK(compare(rne, [](float a, float b) {return a != b;}, true)); + CHECK(compare(rle, [](float a, float b) {return a <= b;}, true)); } SECTION("transposing and reshaping") { diff --git a/src/tests/prod.cpp b/src/tests/prod.cpp old mode 100644 new mode 100755 index 5ade67168..c329d403e --- a/src/tests/prod.cpp +++ b/src/tests/prod.cpp @@ -68,6 +68,5 @@ int main(int argc, char** argv) { } } - return 0; } diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 15486147a..300e4c91b 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -588,6 +588,50 @@ + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + + + true + true + @@ -1032,6 +1076,19 @@ true + + true + true + + + true + true + + + + true + true + @@ -1041,6 +1098,7 @@ true true + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index ce6d8441b..e488f7a06 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -439,6 +439,39 @@ 3rd_party\pathie-cpp\src + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + + + tests + @@ -1627,6 +1660,9 @@ {5d5ee615-192f-4b7f-bdfd-fb8316ceabc8} + + {a86d650a-2268-43d9-9d74-cb17cd6b534b} + @@ -1765,6 +1801,18 @@ 3rd_party\pathie-cpp + + tests + + + tests + + + tests + + + tests + @@ -1773,5 +1821,8 @@ 3rd_party\pathie-cpp + + tests + \ No newline at end of file From f6de599980cd3d645dc57b6162270281ed33f454 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 26 Dec 2018 16:35:56 -0800 Subject: [PATCH 074/838] generalized step() to narrow() and sliceView(), new class Slice; bug fix: SliceViewNodeOp should use correct size for memory piece; new operation stopGradient() --- src/common/shape.h | 27 ++++++++++++ src/graph/expression_operators.cpp | 11 ++++- src/graph/expression_operators.h | 10 ++++- src/graph/node_operators_unary.h | 71 ++++++++++++++++-------------- src/tests/operator_tests.cpp | 31 ++++++++++++- vs/Marian.vcxproj | 1 - vs/Marian.vcxproj.filters | 3 -- 7 files changed, 113 insertions(+), 41 deletions(-) diff --git a/src/common/shape.h b/src/common/shape.h index 89a120fe6..20fb26070 100755 --- a/src/common/shape.h +++ b/src/common/shape.h @@ -12,6 +12,22 @@ namespace marian { +struct Slice // Python-like slice/index descriptor +{ + Slice(int b, int e, int s) : begin(b), end(e), stride(s) {} + Slice(int b, int e) : Slice(b, e, 1) {} + Slice() : Slice(0, END) {} + Slice(int i) : Slice(i, i + 1) {} + Slice(const Slice& other) : Slice(other.begin, other.end, other.stride) {} + const Slice& operator=(const Slice& other) { begin = other.begin; end = other.end; stride = other.stride; return *this; } + const Slice& operator=(int i) { begin = i; end = i + 1; stride = 1; return *this; } + bool operator==(const Slice& other) const { return begin == other.begin && end == other.end && stride == other.stride; } + bool operator!=(const Slice& other) const { return !(*this == other); } + /*const*/ int begin, end, stride; + static const int END = INT_MAX; +}; +typedef std::vector Slices; + struct Shape { private: std::vector shape_; @@ -147,6 +163,17 @@ struct Shape { return ax; } + Slice slice(Slice slice, int ax) const { // interpret negative and special values in Slice + int n = dim(ax); + if (slice.begin < 0) + slice.begin += n; + if (slice.end < 0) + slice.end += n; + else if (slice.end == Slice::END) + slice.end = n; + return slice; + } + static Shape broadcast(const std::vector& shapes) { size_t maxDims = 0; for(auto& s : shapes) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 4870a6664..c1b298f8d 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -232,6 +232,13 @@ Expr flatten_2d(Expr a) { return Expression(a, shape); } +Expr stopGradient(Expr a) { + // implemented as a dummy reshape that is not trainable + auto res = reshape(a, a->shape()); + res->setTrainable(false); + return res; +} + Expr constant_like(Expr a, const NodeInitializer& init) { const auto& shape = a->shape(); auto graph = a->graph(); @@ -442,8 +449,8 @@ Expr swapAxes(Expr x, int axis1, int axis2) return transpose(x, axes); } -Expr step(Expr a, int step, int axis) { - return Expression(a, step, axis); +Expr sliceView(Expr a, const Slice& slice, int axis) { // numpy __getitem__ semantics + return Expression(a, slice, axis); } Expr cross_entropy(Expr a, Expr indices) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index ab1b364b8..49a48f6f3 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -134,6 +134,8 @@ Expr constant_like(Expr a, const NodeInitializer& init); Expr flatten(Expr a); Expr flatten_2d(Expr a); +Expr stopGradient(Expr a); + Expr rows(Expr a, Expr indices); Expr rows(Expr a, const std::vector& indices); @@ -163,7 +165,13 @@ Expr scalar_product(Expr a, Expr b, int ax = 0); Expr weighted_average(Expr in, Expr weights, int ax = 0); -Expr step(Expr a, int step, int axis); +Expr sliceView(Expr a, const Slice& slice, int axis); +static inline Expr narrow(Expr a, int start, int length, int axis) { // PyTorch name + return sliceView(a, Slice(start, start + length), axis); +} +static inline Expr step(Expr a, int step, int axis) { + return sliceView(a, Slice(step), axis); +} Expr sqrt(Expr a, float eps = 0.f); Expr square(Expr a); diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index b8b192082..462d81cf7 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -737,29 +737,36 @@ class ReshapeNodeOp : public UnaryNodeOp { } }; -class StepNodeOp : public UnaryNodeOp { +// narrow an axis to [begin, end) +// The resulting object must be consecutive in memory. +class SliceViewNodeOp : public UnaryNodeOp { private: - Expr stepNode_; - int step_; - int axis_; + Expr viewedNode_; // viewed underlying node + Slice slice_; // index range + int axis_; // and axis along which it is viewed + size_t byteOffset_, byteSize_; // viewed segment in bytes (memory-consecutive) public: - StepNodeOp(Expr a, int step, int axis) - : UnaryNodeOp(a, newShape(a, axis)), stepNode_(a), step_(step) { + SliceViewNodeOp(Expr a, Slice slice, int axis) + : UnaryNodeOp(a, newShape(a, slice, axis)), viewedNode_(a), slice_(slice), axis_(axis) { Node::destroy_ = false; - } - - Shape newShape(Expr a, int axis) { - Shape outShape = a->shape(); - - axis_ = outShape.axis(axis); -#if 0 // this check currently fails in translation; I think should not fail for - // step==0 - for(int i = 0; i < axis_; ++i) - ABORT_IF(outShape[i] != 1, "non-consecutive slices are presently not supported by step()"); -#endif - outShape.set(axis_, 1); - + auto byteStride = a->shape().stride(axis) * sizeOf(value_type()); + byteOffset_ = slice.begin * byteStride; + byteSize_ = shape()[axis] * byteStride; + } + + static Shape newShape(Expr a, Slice& slice, int& axis) { // note: normalizes slice and axis in-place + const auto& shape = a->shape(); + axis = shape.axis(axis); // normalize negative axis + slice = shape.slice(slice, axis); // normalize negative slice values + // enforce consecutive memory + if (slice.begin != 0 || slice.end != shape[axis] || slice.stride != 1) { // unless it's a no-op + ABORT_IF(slice.stride != 1, "Strides other than 1 are presently not supported by sliceView()"); + for(int i = 0; i < axis; ++i) + ABORT_IF(shape[i] != 1, "Non-consecutive slices are presently not supported by sliceView()"); + } + Shape outShape = shape; + outShape.set(axis, slice.end - slice.begin); return outShape; } @@ -769,36 +776,34 @@ class StepNodeOp : public UnaryNodeOp { void forward() override {} void backward() override {} - void init_dependent() override { stepNode_->init_dependent(); } + void init_dependent() override { viewedNode_->init_dependent(); } - void set_zero_adjoint() override { stepNode_->set_zero_adjoint(); } + void set_zero_adjoint() override { viewedNode_->set_zero_adjoint(); } // lazily allocate and zero out gradient (only runs once) Tensor& val() override { - auto childVal = stepNode_->val(); - size_t offset = step_ * shape().elements() * sizeof(float); - auto mem = New(childVal->memory()->data() + offset, - childVal->memory()->size()); + auto childVal = viewedNode_->val(); + auto mem = New(childVal->memory()->data() + byteOffset_, byteSize_); val_.reset(new TensorBase(mem, shape(), childVal->getBackend())); return val_; }; Tensor& grad() override { - auto childGrad = stepNode_->grad(); - size_t offset = step_ * shape().elements() * sizeof(float); - auto mem = New(childGrad->memory()->data() + offset, - childGrad->memory()->size()); + auto childGrad = viewedNode_->grad(); + auto mem = New(childGrad->memory()->data() + byteOffset_, byteSize_); adj_.reset(new TensorBase(mem, shape(), childGrad->getBackend())); return adj_; }; - const std::string type() override { return "step"; } + const std::string type() override { return "sliceView"; } const std::string color() override { return "grey"; } virtual size_t hash() override { if(!hash_) { hash_ = NaryNodeOp::hash(); - util::hash_combine(hash_, step_); + util::hash_combine(hash_, slice_.begin); + util::hash_combine(hash_, slice_.end); + util::hash_combine(hash_, slice_.stride); util::hash_combine(hash_, axis_); } return hash_; @@ -807,10 +812,10 @@ class StepNodeOp : public UnaryNodeOp { virtual bool equal(Expr node) override { if(!NaryNodeOp::equal(node)) return false; - Ptr cnode = std::dynamic_pointer_cast(node); + Ptr cnode = std::dynamic_pointer_cast(node); if(!cnode) return false; - if(step_ != cnode->step_) + if(slice_ != cnode->slice_) return false; if(axis_ != cnode->axis_) return false; diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 30f4dfb70..40867d851 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -102,6 +102,8 @@ void tests(DeviceType device) { auto t4 = transpose(reshape(a, {2, 1, 2, 2}), {1, 3, 2, 0}); auto t5 = transpose(reshape(a, {2, 1, 2, 2}), {2, 0, 1, 3}); + auto t6 = stopGradient(a); + graph->forward(); CHECK(t1->shape() == Shape({4, 2})); @@ -109,6 +111,7 @@ void tests(DeviceType device) { CHECK(t3->shape() == Shape({2, 2, 2})); CHECK(t4->shape() == Shape({1, 2, 2, 2})); CHECK(t5->shape() == Shape({2, 2, 1, 2})); + CHECK(t6->shape() == a->shape()); t1->val()->get(values); CHECK( values == vT1 ); @@ -124,6 +127,10 @@ void tests(DeviceType device) { t5->val()->get(values); CHECK( values == vT5 ); + + t6->val()->get(values); + CHECK(values == vA); + CHECK(!t6->trainable()); } SECTION("softmax and logsoftmax") { @@ -531,7 +538,7 @@ void tests(DeviceType device) { CHECK( values == values2 ); } - SECTION("select operator") { + SECTION("select, step, sliceView operators") { using Indices = std::vector; graph->clear(); @@ -545,6 +552,9 @@ void tests(DeviceType device) { std::vector vD1(vB4); std::vector vD2({5, -6, 11, -12}); std::vector vD3({1, -2, 5, -6, 7, -8, 11, -12}); + std::vector vS1({7, -8, 9}); + std::vector vS2({-4, 5, -6, 7, -8, 9}); + std::vector vS3({7, -8, 9, -10, 11, -12}); auto A = graph->param("4x3", {4,3}, inits::from_vector(in)); auto B1 = select(A, Indices({0}), 0); @@ -556,6 +566,11 @@ void tests(DeviceType device) { auto D1 = select(C, Indices({0}), 0); auto D2 = select(C, Indices({2}), -2); auto D3 = select(C, Indices({0,2}), 1); + + auto S1 = step(A, 2, 0); + auto S2 = narrow(A, 1, -1, 0); + auto S3 = sliceView(A, Slice(-2, Slice::END), 0); + graph->forward(); CHECK(B1->shape() == Shape({1, 3})); @@ -587,6 +602,20 @@ void tests(DeviceType device) { CHECK(D3->shape() == Shape({2, 2, 2})); D3->val()->get(values); CHECK( values == vD3 ); + + values.clear(); + + CHECK(S1->shape() == Shape({1,3})); + S1->val()->get(values); + CHECK(values == vS1); + + CHECK(S2->shape() == Shape({2,3})); + S2->val()->get(values); + CHECK(values == vS2); + + CHECK(S3->shape() == Shape({3,3})); // -> 2,3 + S3->val()->get(values); + CHECK(values == vS3); } SECTION("rows/cols as select operations") { diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 300e4c91b..c13d7841d 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -691,7 +691,6 @@ - diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index e488f7a06..745e130da 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -478,9 +478,6 @@ 3rd_party - - 3rd_party - 3rd_party From 1718a88419a1e1f5841d4f4a715eca409f7eab9e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 26 Dec 2018 17:12:23 -0800 Subject: [PATCH 075/838] bug fix in op tests: narrow() takes length, not end, and its args should be unsigned --- src/graph/expression_operators.h | 4 ++-- src/tests/operator_tests.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 49a48f6f3..7c3797069 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -166,8 +166,8 @@ Expr scalar_product(Expr a, Expr b, int ax = 0); Expr weighted_average(Expr in, Expr weights, int ax = 0); Expr sliceView(Expr a, const Slice& slice, int axis); -static inline Expr narrow(Expr a, int start, int length, int axis) { // PyTorch name - return sliceView(a, Slice(start, start + length), axis); +static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name + return sliceView(a, Slice((int)start, (int)(start + length)), axis); } static inline Expr step(Expr a, int step, int axis) { return sliceView(a, Slice(step), axis); diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 40867d851..0ece15a23 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -568,7 +568,7 @@ void tests(DeviceType device) { auto D3 = select(C, Indices({0,2}), 1); auto S1 = step(A, 2, 0); - auto S2 = narrow(A, 1, -1, 0); + auto S2 = narrow(A, 1, 2, 0); auto S3 = sliceView(A, Slice(-2, Slice::END), 0); graph->forward(); @@ -613,7 +613,7 @@ void tests(DeviceType device) { S2->val()->get(values); CHECK(values == vS2); - CHECK(S3->shape() == Shape({3,3})); // -> 2,3 + CHECK(S3->shape() == Shape({2,3})); S3->val()->get(values); CHECK(values == vS3); } From 7c206c45275eedc8c31328b9a1b685e9d9867254 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 31 Dec 2018 15:33:35 -0800 Subject: [PATCH 076/838] MB-size warmup now stops once it reached 1.0 --- src/optimizers/optimizers.cpp | 22 ++++++++++++++++++++++ src/training/scheduler.h | 6 +++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index 685c917f9..c4e9951f0 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -169,6 +169,28 @@ void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t r vt_ // =_3 ); + // log the magnitude (with good ole CPU code) + static int logCount = 0; + logCount++; + if (logCount <= 10 || logCount % 100 == 0) { + std::vector g, mt, vt; + grads->get(g); mt_->get(mt); vt_->get(vt); + size_t n = g.size(); ABORT_IF(mt.size() != g.size() || vt.size() != g.size(), "mismatching sizes??"); + double gs = 0, ms = 0; // square sum + for (size_t i = 0; i < n; i++) { + auto gi = g[i] / T; // raw average gradient + auto mi = mt[i] / denom1f; // momentum-smoothed average gradient + auto di = sqrtf(vt[i] / denom2f) + eps_; + gi /= di; // RMS-normalize both + mi /= di; + gs += gi * gi; + ms += mi * mi; + } + auto grms = std::sqrt(gs / n); // hmm... expecting 1.0 here, no? + auto mrms = std::sqrt(ms / n); + LOG(info, "Adam[{}]: grms = {}, mrms = {}", logCount, grms, mrms); + } + params->getBackend()->synchronize(); // @TODO: This should not be in here. Maybe in the wrapper. Why is it needed at all? } diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 51320af03..cad01db20 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -83,13 +83,13 @@ class Scheduler : public TrainingObserver { // if unit is labels, then account for the fact that our increment itself is not constant if (mbWarmup.unit == SchedulingUnit::trgLabels) progressRatio = std::sqrt(progressRatio); - // soften the very initial ramp-up if requested if (progressRatio < 1) { + // soften the very initial ramp-up if requested double exp = options_->get("mini-batch-warmup-exp"); progressRatio = pow(progressRatio, exp); // e.g. 1.5 => linear ramp-up -> sublinear ramp-up + // apply ratio to actual batch size + ratio *= progressRatio; } - // apply ratio to actual batch size - ratio *= progressRatio; } // dynamic MB-size tracking with learning rate From 35aa00f8b6b6385b469c51717a41aa6d75d860d2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 31 Dec 2018 16:15:59 -0800 Subject: [PATCH 077/838] refactored state.eta update; bug fix: dynamic MB scaling should consider state-based LR decay --- src/training/scheduler.h | 60 +++++++++++++++++++---------------- src/training/training_state.h | 11 +++++-- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 10d171874..d59644aae 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -18,8 +18,8 @@ class Scheduler : public TrainingObserver { timer::Timer timer_, heartBeatTimer_; - // determine LR decay factor from --lr-decay-inv-sqrt option - float getLearningRateDecayFactor(const TrainingState& state) const { + // determine scheduled LR decay factor (--lr-decay-inv-sqrt option) + float getScheduledLRDecayFactor(const TrainingState& state) const { auto args = options_->get>("lr-decay-inv-sqrt"); ABORT_IF(args.empty() || args.size() > 2, "--lr-decay-inv-sqrt argument must be one or two numbers with units"); auto decayGoogle = SchedulingParameter::parse(args[0]); @@ -38,27 +38,33 @@ class Scheduler : public TrainingObserver { return 1.f; } - // determine the dynamically adjusted learning rate, incl. warm-up and decay - float getLearningRate(const TrainingState& state) const { + // update current learning rate in state.eta + // This considers + // - base LR (--learn-rate) + // - LR warm-up (--lr-warmup, --lr=warmup-start-rate) + // - scheduled LR decay (--lr-decay-inv-sqrt) + // - state-based LR decay (--lr-decay, --lr-decay-strategy) + void updateLearningRate(TrainingState& state) const { float baselr = options_->get("learn-rate"); - float mult1 = 1.f; - auto warmup = SchedulingParameter::parse(options_->get("lr-warmup")); - if(warmup) { - ABORT_IF(state.warmupStart && state.warmupStart.unit != warmup.unit, "lr-warmup and warmup-start must have the same unit"); - auto bno = state.getProgressIn(warmup.unit) - state.warmupStart.n; - mult1 = std::min(1.f, (float)bno / (float)warmup.n); + // warm-up factor + float warmupFactor = 1.f; + auto warmupParam = SchedulingParameter::parse(options_->get("lr-warmup")); + if(warmupParam) { + ABORT_IF(state.warmupStart && state.warmupStart.unit != warmupParam.unit, "lr-warmup and warmup-start must have the same unit"); + auto bno = state.getProgressIn(warmupParam.unit) - state.warmupStart.n; + warmupFactor = std::min(1.f, (float)bno / (float)warmupParam.n); } - float mult2 = getLearningRateDecayFactor(state); - - baselr = baselr * mult1 * mult2; - float lrStart = options_->get("lr-warmup-start-rate"); - if(lrStart > 0) - baselr = baselr - lrStart * mult1 * mult2 + lrStart * mult2; + baselr = lrStart + (baselr - lrStart) * warmupFactor; // linear interpolation between lr-warmup-start-rate to learn-rate + + // schedule-based decay factor (--lr-decay-inv-sqrt) + float scheduledDecayFactor = getScheduledLRDecayFactor(state); + baselr = baselr * scheduledDecayFactor; - return baselr; + // factor in state-based decay and set final LR as state.eta + state.updateEta(baselr); } public: @@ -91,7 +97,7 @@ class Scheduler : public TrainingObserver { // As LR goes down, MB gets ramped up by the same ratio, which has been found to be safe. auto mbTracking = options_->get("mini-batch-track-lr"); if (mbTracking) { - auto lrFactor = getLearningRateDecayFactor(*state_); + auto lrFactor = getScheduledLRDecayFactor(*state_) * state_->factor; // (don't include lr-warmup) if (lrFactor != 1) LOG_ONCE(info, "[scheduler] Dynamic mini-batch size adjustment enabled and kicking in"); ratio /= lrFactor; @@ -101,7 +107,8 @@ class Scheduler : public TrainingObserver { Scheduler(Ptr options, Ptr state) : options_(options), state_(state) { - state_->eta = getLearningRate(*state); + ABORT_IF(state_->factor != 1, "state.factor unexpectedly not 1 at this point??"); + updateLearningRate(*state); } bool keepGoing() { @@ -367,8 +374,7 @@ class Scheduler : public TrainingObserver { void actAfterEpoch(TrainingState& state) override { float factor = (float)options_->get("lr-decay"); // @TODO: ? - float baselr = getLearningRate(state); - state.eta = baselr * state.factor; + updateLearningRate(state); if(factor > 0.0) { bool decay = false; @@ -398,7 +404,7 @@ class Scheduler : public TrainingObserver { if(decay) { state.factor *= factor; - state.eta = baselr * state.factor; + updateLearningRate(state); LOG(info, "Decaying learning rate to {} in epoch {}", state.eta, @@ -420,8 +426,7 @@ class Scheduler : public TrainingObserver { float factor = (float)options_->get("lr-decay"); // @TODO: ? state.reset = false; - float baselr = getLearningRate(state); - state.eta = baselr * state.factor; + updateLearningRate(state); if(factor > 0.0) { if(options_->get("lr-decay-strategy") == "batches") { @@ -431,7 +436,7 @@ class Scheduler : public TrainingObserver { if(start > 0 && freq > 0 && state.batches >= start && ((state.batches - start) % freq == 0)) { state.factor *= factor; - state.eta = baselr * state.factor; + updateLearningRate(state); LOG(info, "Decaying learning rate to {} after {} batches", state.eta, @@ -466,8 +471,7 @@ class Scheduler : public TrainingObserver { float factor = (float)options_->get("lr-decay"); // @TODO: ? state.reset = false; - float baselr = getLearningRate(state); - state.eta = baselr * state.factor; + updateLearningRate(state); if(factor > 0.0) { if(options_->get("lr-decay-strategy") == "stalled") { @@ -475,7 +479,7 @@ class Scheduler : public TrainingObserver { = options_->get>("lr-decay-start").front(); if(startStalled && state.stalled && state.stalled % startStalled == 0) { state.factor *= factor; - state.eta = baselr * state.factor; + updateLearningRate(state); LOG(info, "Decaying learning rate to {} after stalled {} time(s)", state.eta, diff --git a/src/training/training_state.h b/src/training/training_state.h index 65cefcbce..33cf92f84 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -89,9 +89,12 @@ class TrainingState { // Reset optimizer parameters bool reset{false}; - // Learning rate + // Current learning rate, representing all adjustment processes and factors float eta; - // Multiplication factor for learning rate + void updateEta(float dynamicBaseLR) { // note: no other function may write to 'eta' (besides load()) + eta = dynamicBaseLR * factor; + } + // State-based multiplication factor for learning rate float factor{1.f}; SchedulingParameter warmupStart; // has same unit as lr-warmup @@ -114,7 +117,9 @@ class TrainingState { // Set flag if the model was validated in the current batch bool validated{false}; - TrainingState(float learnRate) : eta(learnRate) {} + TrainingState(float learnRate) { + updateEta(learnRate); + } void registerObserver(Ptr observer) { observers_.push_back(observer); From 734097b3dfbac37cceb8fda3c186122f2a2c4183 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 2 Jan 2019 16:55:13 -0800 Subject: [PATCH 078/838] first attempts at BERT --- src/models/classifier.h | 137 +++++++++++++++++++++ src/models/costs.h | 47 ++++++++ src/models/decoder.h | 2 + src/models/encoder.h | 2 + src/models/encoder_classifier.h | 208 ++++++++++++++++++++++++++++++++ src/models/encoder_decoder.cpp | 2 +- src/models/encoder_decoder.h | 8 +- src/models/model_factory.cpp | 30 ++++- src/models/model_factory.h | 34 ++++++ src/models/states.h | 23 ++++ src/models/transformer.h | 90 +++++++++++--- 11 files changed, 560 insertions(+), 23 deletions(-) create mode 100644 src/models/classifier.h create mode 100644 src/models/encoder_classifier.h diff --git a/src/models/classifier.h b/src/models/classifier.h new file mode 100644 index 000000000..21fe8b6f9 --- /dev/null +++ b/src/models/classifier.h @@ -0,0 +1,137 @@ +#pragma once + +#include "marian.h" +#include "models/states.h" +#include "layers/constructors.h" +#include "layers/factory.h" + +namespace marian { + +class ClassifierBase { +protected: + Ptr options_; + std::string prefix_{"classifier"}; + bool inference_{false}; + size_t batchIndex_{0}; + +public: + ClassifierBase(Ptr options) + : options_(options), + prefix_(options->get("prefix", "classifier")), + inference_(options->get("inference", false)), + batchIndex_(options->get("index", 1)) {} // assume that training input has batch index 0 and labels has 1 + + virtual ~ClassifierBase() {} + + virtual Ptr apply(Ptr, Ptr, const std::vector>&) = 0; + + template + T opt(const std::string& key) const { + return options_->get(key); + } + + virtual void clear() = 0; +}; + +// This is a model that pretrains BERT for classification +class BertMaskedLM : public ClassifierBase { +public: + BertMaskedLM(Ptr options) : ClassifierBase(options) {} + + Ptr apply(Ptr graph, Ptr bertBatch, const std::vector>& encoderStates) override { + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence + + // Since this is a classifier we are not masking anything on the target. We can (mis)use the mask to hold + // indices of words in the encoder that have been masked out for BERT's masked LM training. + // Masks are floats, hence the conversion to IndexType. + const auto& maskedRowsFloats = (*bertBatch)[batchIndex_]->mask(); + std::vector maskedRows(maskedRowsFloats.size()); + std::copy(maskedRowsFloats.begin(), maskedRowsFloats.end(), maskedRows.begin()); + + classEmbeddings = rows(classEmbeddings, graph->indices(maskedRows)); // subselect stuff that has actually been masked out; + + int dimModel = classEmbeddings->shape()[-1]; + + int dimTrgVoc = opt>("dim-vocabs")[0]; // set to first one, @TODO: make this modular. + + auto layerTanh = mlp::dense(graph) // + ("dim", dimModel) // + ("activation", mlp::act::tanh); // + + auto layerOut = mlp::output(graph) // + ("dim", dimTrgVoc); + + if(opt("tied-embeddings") || opt("tied-embeddings-all")) { + std::string tiedPrefix = prefix_ + "_Wemb"; + if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) + tiedPrefix = "Wemb"; + layerOut.tie_transposed("W", tiedPrefix); + } + + // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] + // assemble layers into MLP and apply to embeddings, decoder context and + // aligned source context + auto output = mlp::mlp(graph) // + ("prefix", prefix_ + "_ff_logit_bert_out") // + .push_back(layerTanh) // + .push_back(layerOut) // + .construct(); + + auto logits = output->apply(classEmbeddings); + + auto state = New(); + state->setLogProbs(logits); + + // filled automatically during masking, these are the vocab indices of masked words + const auto& classLabels = (*bertBatch)[batchIndex_]->data(); + state->setTargetIndices(graph->indices(classLabels)); + + return state; + } + + virtual void clear() override {} +}; + +// This is a model that uses a pre-trained BERT model to build a classifier on top of the encoder +class BertClassifier : public ClassifierBase { +public: + BertClassifier(Ptr options) : ClassifierBase(options) {} + + // The batch has been filled with external classifier labels, @TODO: figure out how to do that + Ptr apply(Ptr graph, Ptr bertBatch, const std::vector>& encoderStates) override { + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence + + int dimModel = classEmbeddings->shape()[-1]; + int dimTrgCls = opt("bert-classes"); + + auto output = mlp::mlp(graph) // + ("prefix", prefix_ + "_ff_logit") // + .push_back(mlp::dense(graph) // + ("dim", dimModel) // + ("activation", mlp::act::tanh)) // + .push_back(mlp::dense(graph) // + ("dim", dimTrgCls)) // + .construct(); + + auto logits = output->apply(classEmbeddings); // class logits for each batch entry + + auto state = New(); + state->setLogProbs(logits); + + // filled externally + const auto& classLabels = (*bertBatch)[batchIndex_]->data(); + state->setTargetIndices(graph->indices(classLabels)); + + return state; + } + + virtual void clear() override {} +}; + +} \ No newline at end of file diff --git a/src/models/costs.h b/src/models/costs.h index d38a445e8..4d9845768 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -5,6 +5,7 @@ #include "layers/loss.h" #include "layers/weight.h" #include "models/encoder_decoder.h" +#include "models/encoder_classifier.h" namespace marian { namespace models { @@ -72,6 +73,37 @@ class EncoderDecoderCE : public CostBase { } }; +class EncoderClassifierCE : public CostBase { +protected: + Ptr options_; + bool inference_{false}; + Ptr loss_; + +public: + EncoderClassifierCE(Ptr options) + : options_(options), inference_(options->get("inference", false)) { + loss_ = LossFactory(options_, inference_); + } + + Expr apply(Ptr model, + Ptr graph, + Ptr batch, + bool clearGraph = true) override { + + auto enccls = std::static_pointer_cast(model); + auto corpusBatch = std::static_pointer_cast(batch); + + auto state = enccls->apply(graph, corpusBatch, clearGraph); + + Expr cost = loss_->getCost(state->getLogProbs(), + state->getTargetIndices(), + /*mask=*/nullptr, + /*weights=*/nullptr); + + return cost; + } +}; + class Trainer : public ModelBase { protected: Ptr model_; @@ -231,5 +263,20 @@ inline Ptr add_cost(Ptr encdec, default: return encdec; } } + +inline Ptr add_cost(Ptr enccls, + Ptr options) { + switch(options->get("usage", usage::raw)) { + case usage::training: + return New(enccls, New(options)); + case usage::scoring: + return New(enccls, New(options)); + case usage::translation: + ABORT("Classifier cannot be used for translation"); + case usage::raw: + default: return enccls; + } +} + } // namespace models } // namespace marian diff --git a/src/models/decoder.h b/src/models/decoder.h index 39e56e1f3..ddedb4836 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -24,6 +24,8 @@ class DecoderBase { inference_(options->get("inference", false)), batchIndex_(options->get("index", 1)) {} + virtual ~DecoderBase() {} + virtual Ptr startState(Ptr, Ptr batch, std::vector>&) diff --git a/src/models/encoder.h b/src/models/encoder.h index 6d1ee852e..79424ef28 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -80,6 +80,8 @@ class EncoderBase { inference_(options->get("inference", false)), batchIndex_(options->get("index", 0)) {} + virtual ~EncoderBase() {} + virtual Ptr build(Ptr, Ptr) = 0; diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h new file mode 100644 index 000000000..aae6fe235 --- /dev/null +++ b/src/models/encoder_classifier.h @@ -0,0 +1,208 @@ +#pragma once + +#include "marian.h" + +#include "models/encoder.h" +#include "models/classifier.h" +#include "models/model_base.h" +#include "models/states.h" + +namespace marian { + +class EncoderClassifierBase : public models::ModelBase { +public: + virtual ~EncoderClassifierBase() {} + + virtual void load(Ptr graph, + const std::string& name, + bool markedReloaded = true) override + = 0; + + virtual void mmap(Ptr graph, + const void* ptr, + bool markedReloaded = true) + = 0; + + virtual void save(Ptr graph, + const std::string& name, + bool saveTranslatorConfig = false) override + = 0; + + virtual void clear(Ptr graph) override = 0; + + virtual Ptr apply(Ptr, Ptr, bool) = 0; + + + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override + = 0; + + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) + = 0; + + virtual Ptr getOptions() = 0; +}; + +class EncoderClassifier : public EncoderClassifierBase { +protected: + Ptr options_; + + std::string prefix_; + + std::vector> encoders_; + std::vector> classifiers_; + + bool inference_{false}; + + std::set modelFeatures_; + + Config::YamlNode getModelParameters() { + Config::YamlNode modelParams; + for(auto& key : modelFeatures_) + modelParams[key] = options_->getYaml()[key]; + + if(options_->has("original-type")) + modelParams["type"] = options_->getYaml()["original-type"]; + + modelParams["version"] = buildVersion(); + return modelParams; + } + + std::string getModelParametersAsString() { + auto yaml = getModelParameters(); + YAML::Emitter out; + cli::OutputYaml(yaml, out); + return std::string(out.c_str()); + } + +// virtual void createClassifierConfig(const std::string& name) {} + +public: + typedef data::Corpus dataset_type; + + // @TODO: lots of code-duplication with EncoderDecoder + EncoderClassifier(Ptr options) + : options_(options), + prefix_(options->get("prefix", "")), + inference_(options->get("inference", false)) { + modelFeatures_ = {"type", + "dim-vocabs", + "dim-emb", + "dim-rnn", + "enc-cell", + "enc-type", + "enc-cell-depth", + "enc-depth", + "dec-depth", + "dec-cell", + "dec-cell-base-depth", + "dec-cell-high-depth", + "skip", + "layer-normalization", + "right-left", + "special-vocab", + "tied-embeddings", + "tied-embeddings-src", + "tied-embeddings-all"}; + + modelFeatures_.insert("transformer-heads"); + modelFeatures_.insert("transformer-no-projection"); + modelFeatures_.insert("transformer-dim-ffn"); + modelFeatures_.insert("transformer-ffn-depth"); + modelFeatures_.insert("transformer-ffn-activation"); + modelFeatures_.insert("transformer-dim-aan"); + modelFeatures_.insert("transformer-aan-depth"); + modelFeatures_.insert("transformer-aan-activation"); + modelFeatures_.insert("transformer-aan-nogate"); + modelFeatures_.insert("transformer-preprocess"); + modelFeatures_.insert("transformer-postprocess"); + modelFeatures_.insert("transformer-postprocess-emb"); + modelFeatures_.insert("transformer-decoder-autoreg"); + modelFeatures_.insert("transformer-tied-layers"); + modelFeatures_.insert("transformer-guided-alignment-layer"); + } + + virtual Ptr getOptions() override { return options_; } + + std::vector>& getEncoders() { return encoders_; } + std::vector>& getClassifiers() { return classifiers_; } + + void push_back(Ptr encoder) { encoders_.push_back(encoder); } + void push_back(Ptr classifier) { classifiers_.push_back(classifier); } + + void load(Ptr graph, + const std::string& name, + bool markedReloaded) override { + graph->load(name, markedReloaded && !opt("ignore-model-config", false)); + } + + void mmap(Ptr graph, + const void* ptr, + bool markedReloaded) override { + graph->mmap(ptr, markedReloaded && !opt("ignore-model-config", false)); + } + + void save(Ptr graph, + const std::string& name, + bool saveModelConfig) override { + LOG(info, "Saving model weights and runtime parameters to {}", name); + graph->save(name , getModelParametersAsString()); + } + + void clear(Ptr graph) override { + graph->clear(); + + for(auto& enc : encoders_) + enc->clear(); + for(auto& cls : classifiers_) + cls->clear(); + } + + template + T opt(const std::string& key) { + return options_->get(key); + } + + template + T opt(const std::string& key, const T& def) { + return options_->get(key, def); + } + + template + void set(std::string key, T value) { + options_->set(key, value); + } + + /*********************************************************************/ + + Ptr apply(Ptr graph, Ptr batch, bool clearGraph) override { + if(clearGraph) + clear(graph); + + std::vector> encoderStates; + for(auto& encoder : encoders_) + encoderStates.push_back(encoder->build(graph, batch)); + + return classifiers_[0]->apply(graph, batch, encoderStates); + } + + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto state = apply(graph, batch, clearGraph); + // returns raw logits + return state->getLogProbs(); + } + + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto corpusBatch = std::static_pointer_cast(batch); + return build(graph, corpusBatch, clearGraph); + } +}; + +} // namespace marian diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 3edb7b335..e8f93bc1c 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -1,4 +1,4 @@ -#include "encoder_decoder.h" +#include "models/encoder_decoder.h" #include "common/cli_helper.h" #include "common/version.h" diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index 5818c2370..0ea89b4f6 100644 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -2,10 +2,10 @@ #include "marian.h" -#include "decoder.h" -#include "encoder.h" -#include "model_base.h" -#include "states.h" +#include "models/decoder.h" +#include "models/encoder.h" +#include "models/model_base.h" +#include "models/states.h" namespace marian { diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index d42f07c8b..28178e768 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -1,7 +1,8 @@ #include "marian.h" -#include "models/encoder_decoder.h" #include "models/model_factory.h" +#include "models/encoder_decoder.h" +#include "models/encoder_classifier.h" #include "models/costs.h" @@ -49,6 +50,13 @@ Ptr DecoderFactory::construct() { ABORT("Unknown decoder type"); } +Ptr ClassifierFactory::construct() { + if(options_->get("type") == "bert") + return New(options_); + ABORT("Unknown classifier type"); +} + + Ptr EncoderDecoderFactory::construct() { Ptr encdec; @@ -69,6 +77,18 @@ Ptr EncoderDecoderFactory::construct() { return add_cost(encdec, options_); } +Ptr EncoderClassifierFactory::construct() { + auto enccls = New(options_); + + for(auto& ef : encoders_) + enccls->push_back(ef(options_).construct()); + + for(auto& cf : classifiers_) + enccls->push_back(cf(options_).construct()); + + return add_cost(enccls, options_); +} + Ptr by_type(std::string type, usage use, Ptr options) { // clang-format off if(type == "s2s" || type == "amun" || type == "nematus") { @@ -197,6 +217,14 @@ Ptr by_type(std::string type, usage use, Ptr options) { .construct(); } + if(type == "bert") { + return models::encoder_classifier()(options) + ("usage", use) + .push_back(models::encoder()("type", "transformer")("original-type", type)) + .push_back(models::classifier()("index", 2)) // indices 0 and 1 used by encoder + .construct(); + } + #ifdef COMPILE_EXAMPLES // @TODO: examples should be compiled optionally if(type == "mnist-ffnn") { diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 2ec7fe752..72c3a4a61 100644 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -4,6 +4,7 @@ #include "layers/factory.h" #include "models/encoder_decoder.h" +#include "models/encoder_classifier.h" namespace marian { namespace models { @@ -26,6 +27,15 @@ class DecoderFactory : public Factory { typedef Accumulator decoder; +class ClassifierFactory : public Factory { +public: + ClassifierFactory(Ptr graph = nullptr) : Factory(graph) {} + + virtual Ptr construct(); +}; + +typedef Accumulator classifier; + class EncoderDecoderFactory : public Factory { private: std::vector encoders_; @@ -50,6 +60,30 @@ class EncoderDecoderFactory : public Factory { typedef Accumulator encoder_decoder; +class EncoderClassifierFactory : public Factory { +private: + std::vector encoders_; + std::vector classifiers_; + +public: + EncoderClassifierFactory(Ptr graph = nullptr) + : Factory(graph) {} + + Accumulator push_back(encoder enc) { + encoders_.push_back(enc); + return Accumulator(*this); + } + + Accumulator push_back(classifier cls) { + classifiers_.push_back(cls); + return Accumulator(*this); + } + + virtual Ptr construct(); +}; + +typedef Accumulator encoder_classifier; + Ptr by_type(std::string type, usage, Ptr options); Ptr from_options(Ptr options, usage); diff --git a/src/models/states.h b/src/models/states.h index 964617340..cb94e4bb7 100755 --- a/src/models/states.h +++ b/src/models/states.h @@ -99,4 +99,27 @@ class DecoderState { virtual void blacklist(Expr /*totalCosts*/, Ptr /*batch*/) {} }; + +class ClassifierState { +private: + Expr logProbs_; + std::vector> encStates_; + Ptr batch_; + + Expr targetMask_; + Expr targetIndices_; + +public: + virtual Expr getLogProbs() const { return logProbs_; } + virtual void setLogProbs(Expr logProbs) { logProbs_ = logProbs; } + + virtual Expr getTargetIndices() const { return targetIndices_; }; + + virtual void setTargetIndices(Expr targetIndices) { targetIndices_ = targetIndices; } + + virtual Expr getTargetMask() const { return targetMask_; }; + + virtual void setTargetMask(Expr targetMask) { targetMask_ = targetMask; } +}; + } // namespace marian diff --git a/src/models/transformer.h b/src/models/transformer.h index c30d06c8e..baf7d261b 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -47,26 +47,81 @@ class Transformer : public EncoderOrDecoderBase { static Expr transposeTimeBatch(Expr input) { return transpose(input, {0, 2, 1, 3}); } - Expr addPositionalEmbeddings(Expr input, int start = 0) const { + Expr addPositionalEmbeddings(Expr input, int start = 0, bool learnedPosEmbeddings = false) const { int dimEmb = input->shape()[-1]; int dimWords = input->shape()[-3]; - float num_timescales = (float)dimEmb / 2; - float log_timescale_increment = std::log(10000.f) / (num_timescales - 1.f); + Expr embeddings = input; + if(learnedPosEmbeddings) { + int maxLength = opt("max-length"); - std::vector vPos(dimEmb * dimWords, 0); - for(int p = start; p < dimWords + start; ++p) { - for(int i = 0; i < num_timescales; ++i) { - float v = p * std::exp(i * -log_timescale_increment); - vPos[(p - start) * dimEmb + i] = std::sin(v); - vPos[(p - start) * dimEmb + (int)num_timescales + i] = std::cos(v); // @TODO: is int vs. float correct for num_timescales? + auto posEmbFactory = embedding(graph_) + ("prefix", "Wpos") // share positional embeddings across all encoders/decorders + ("dimVocab", maxLength) + ("dimEmb", dimEmb) + .construct(); + + std::vector positions(dimWords); + std::iota(positions.begin(), positions.end(), 0); // fill with increasing numbers until current length + + auto signal = rows(posEmbFactory, graph_->indices(positions)); + embeddings = embeddings + signal; + } else { + float num_timescales = (float)dimEmb / 2; + float log_timescale_increment = std::log(10000.f) / (num_timescales - 1.f); + + std::vector vPos(dimEmb * dimWords, 0); + for(int p = start; p < dimWords + start; ++p) { + for(int i = 0; i < num_timescales; ++i) { + float v = p * std::exp(i * -log_timescale_increment); + vPos[(p - start) * dimEmb + i] = std::sin(v); + vPos[(p - start) * dimEmb + (int)num_timescales + i] = std::cos(v); // @TODO: is int vs. float correct for num_timescales? + } } + + // shared across batch entries + auto signal = graph_->constant({dimWords, 1, dimEmb}, inits::from_vector(vPos)); + embeddings = embeddings + signal; + } + + return embeddings; + } + + Expr addSentenceEmbeddings(Expr input, int start, Ptr batch) const { + Expr embeddings = input; + + // do nothing if we are not training with sentence pairs + if(opt("bert-sentencepair", false)) { + int dimEmb = input->shape()[-1]; + + auto sentEmbFactory = embedding(graph_) + ("prefix", "Wsent") + ("dimVocab", 2) // sentence A or sentence B + ("dimEmb", dimEmb) + .construct(); + + const auto& sentenceIndices = (*batch)[1]->data(); // @TODO: do not use constant index + + // @TODO: note this is going to be really slow due to atomicAdd in backward step + // with only two classes; + // instead two masked reduce operations, maybe in parallel streams? + auto signal = rows(sentEmbFactory, graph_->indices(sentenceIndices)); + + embeddings = embeddings + signal; } + return embeddings; + } + + Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const { + bool iAmBert = opt("original-type", "undefined") == "bert"; + bool learnedPosEmbeddings = opt("transformer-learned-positions", false) || iAmBert; - // shared across batch entries - auto signal - = graph_->constant({dimWords, 1, dimEmb}, inits::from_vector(vPos)); - return input + signal; + input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); + + if(iAmBert && batch) + input = addSentenceEmbeddings(input, start, batch); + + return input; } Expr triangleMask(int length) const { @@ -532,8 +587,7 @@ class EncoderTransformer : public Transformer { std::tie(batchEmbeddings, batchMask) = EncoderBase::ulrLookup(graph_, embeddings, batch); } - else - { + else { auto embeddings = wordEmbeddings(batchIndex_); std::tie(batchEmbeddings, batchMask) = EncoderBase::lookup(graph_, embeddings, batch); @@ -546,7 +600,9 @@ class EncoderTransformer : public Transformer { } // according to paper embeddings are scaled up by \sqrt(d_m) auto scaledEmbeddings = std::sqrt((float)dimEmb) * batchEmbeddings; - scaledEmbeddings = addPositionalEmbeddings(scaledEmbeddings); + + scaledEmbeddings = addSpecialEmbeddings(scaledEmbeddings, /*start=*/0, batch); + // reorganize batch and timestep scaledEmbeddings = atleast_nd(scaledEmbeddings, 4); batchMask = atleast_nd(batchMask, 4); @@ -698,7 +754,7 @@ class DecoderTransformer : public Transformer { int startPos = (int)state->getPosition(); scaledEmbeddings - = addPositionalEmbeddings(scaledEmbeddings, startPos); + = addSpecialEmbeddings(scaledEmbeddings, startPos); scaledEmbeddings = atleast_nd(scaledEmbeddings, 4); From 963305b1c62b96a7bd43053289311830ebc7f4ca Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 3 Jan 2019 13:31:39 -0800 Subject: [PATCH 079/838] multi-objective learning for classifiers --- src/data/corpus_base.h | 16 ++++++++---- src/models/classifier.h | 23 +++++++---------- src/models/costs.h | 14 ++++++++--- src/models/encoder_classifier.h | 15 +++++++---- src/models/model_factory.cpp | 21 ++++++++++++---- src/models/transformer.h | 44 +++++++++++++++------------------ 6 files changed, 77 insertions(+), 56 deletions(-) diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 8ecdf2337..765d39c78 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -110,7 +110,7 @@ class SentenceTuple { */ class SubBatch { private: - std::vector indices_; + std::vector indices_; std::vector mask_; size_t size_; @@ -128,7 +128,7 @@ class SubBatch { * @param width Number of words in the longest sentence */ SubBatch(size_t size, size_t width, const Ptr& vocab) - : indices_(size * width, 0), + : indices_(1, Words(size * width, 0)), mask_(size * width, 0), size_(size), width_(width), @@ -142,7 +142,12 @@ class SubBatch { * idx_{w,0},idx_{w,1},\dots,idx_{w,s}\f$, where \f$w\f$ is the number of * words (width) and \f$s\f$ is the number of sentences (size). */ - std::vector& data() { return indices_; } + std::vector& data() { return indices_[0]; } + std::vector& data(int index) { return indices_[index]; } + + void setNumFactors(size_t num) { indices_.resize(num, Words(size_ * width_, 0)); } // @TODO: for now factors have no vocabulary, they are generated from the other inputs + size_t numFactors() { return indices_.size(); } + /** * @brief Flat masking vector; 0 is used for masked words. * @@ -204,8 +209,9 @@ class SubBatch { size_t words = 0; for(size_t j = 0; j < subWidth; ++j) { for(size_t i = 0; i < size; ++i) { - sb->data()[j * size + i] = indices_[j * size_ + (pos + i)]; - sb->mask()[j * size + i] = mask_[j * size_ + (pos + i)]; + for(size_t k = 0; k < sb->numFactors(); ++k) + sb->data(k)[j * size + i] = indices_[k][j * size_ + (pos + i)]; + sb->mask()[j * size + i] = mask_[j * size_ + (pos + i)]; if(mask_[j * size_ + (pos + i)] != 0) words++; diff --git a/src/models/classifier.h b/src/models/classifier.h index 21fe8b6f9..ea92cf669 100644 --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -42,8 +42,7 @@ class BertMaskedLM : public ClassifierBase { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); auto context = encoderStates[0]->getContext(); - auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence - + // Since this is a classifier we are not masking anything on the target. We can (mis)use the mask to hold // indices of words in the encoder that have been masked out for BERT's masked LM training. // Masks are floats, hence the conversion to IndexType. @@ -51,32 +50,27 @@ class BertMaskedLM : public ClassifierBase { std::vector maskedRows(maskedRowsFloats.size()); std::copy(maskedRowsFloats.begin(), maskedRowsFloats.end(), maskedRows.begin()); - classEmbeddings = rows(classEmbeddings, graph->indices(maskedRows)); // subselect stuff that has actually been masked out; + auto classEmbeddings = rows(context, graph->indices(maskedRows)); // subselect stuff that has actually been masked out; int dimModel = classEmbeddings->shape()[-1]; - int dimTrgVoc = opt>("dim-vocabs")[0]; // set to first one, @TODO: make this modular. + int dimVoc = opt>("dim-vocabs")[batchIndex_ - 1]; // unsafe auto layerTanh = mlp::dense(graph) // ("dim", dimModel) // ("activation", mlp::act::tanh); // auto layerOut = mlp::output(graph) // - ("dim", dimTrgVoc); + ("dim", dimVoc); - if(opt("tied-embeddings") || opt("tied-embeddings-all")) { - std::string tiedPrefix = prefix_ + "_Wemb"; - if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) - tiedPrefix = "Wemb"; - layerOut.tie_transposed("W", tiedPrefix); - } + layerOut.tie_transposed("W", "Wemb"); // We are a BERT model, hence tie with input // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context auto output = mlp::mlp(graph) // ("prefix", prefix_ + "_ff_logit_bert_out") // - .push_back(layerTanh) // + .push_back(layerTanh) // @TODO: do we actually need this? .push_back(layerOut) // .construct(); @@ -96,6 +90,7 @@ class BertMaskedLM : public ClassifierBase { }; // This is a model that uses a pre-trained BERT model to build a classifier on top of the encoder +// Can be used for next sentence prediction task class BertClassifier : public ClassifierBase { public: BertClassifier(Ptr options) : ClassifierBase(options) {} @@ -114,7 +109,7 @@ class BertClassifier : public ClassifierBase { ("prefix", prefix_ + "_ff_logit") // .push_back(mlp::dense(graph) // ("dim", dimModel) // - ("activation", mlp::act::tanh)) // + ("activation", mlp::act::tanh)) // @TODO: do we actually need this? .push_back(mlp::dense(graph) // ("dim", dimTrgCls)) // .construct(); @@ -124,7 +119,7 @@ class BertClassifier : public ClassifierBase { auto state = New(); state->setLogProbs(logits); - // filled externally + // filled externally, for BERT these are NextSentence prediction labels const auto& classLabels = (*bertBatch)[batchIndex_]->data(); state->setTargetIndices(graph->indices(classLabels)); diff --git a/src/models/costs.h b/src/models/costs.h index 4d9845768..da190e9ca 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -93,13 +93,21 @@ class EncoderClassifierCE : public CostBase { auto enccls = std::static_pointer_cast(model); auto corpusBatch = std::static_pointer_cast(batch); - auto state = enccls->apply(graph, corpusBatch, clearGraph); + auto states = enccls->apply(graph, corpusBatch, clearGraph); - Expr cost = loss_->getCost(state->getLogProbs(), - state->getTargetIndices(), + Expr cost = loss_->getCost(states[0]->getLogProbs(), + states[0]->getTargetIndices(), /*mask=*/nullptr, /*weights=*/nullptr); + // multi-objective training + for(int i = 1; i < states.size(); ++i) { + cost = cost + loss_->getCost(states[i]->getLogProbs(), + states[i]->getTargetIndices(), + /*mask=*/nullptr, + /*weights=*/nullptr); + } + return cost; } }; diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index aae6fe235..48f5d6c0e 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -30,7 +30,7 @@ class EncoderClassifierBase : public models::ModelBase { virtual void clear(Ptr graph) override = 0; - virtual Ptr apply(Ptr, Ptr, bool) = 0; + virtual std::vector> apply(Ptr, Ptr, bool) = 0; virtual Expr build(Ptr graph, @@ -178,7 +178,7 @@ class EncoderClassifier : public EncoderClassifierBase { /*********************************************************************/ - Ptr apply(Ptr graph, Ptr batch, bool clearGraph) override { + std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { if(clearGraph) clear(graph); @@ -186,15 +186,20 @@ class EncoderClassifier : public EncoderClassifierBase { for(auto& encoder : encoders_) encoderStates.push_back(encoder->build(graph, batch)); - return classifiers_[0]->apply(graph, batch, encoderStates); + std::vector> classifierStates; + for(auto& classifier : classifiers_) + classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); + + return classifierStates; } virtual Expr build(Ptr graph, Ptr batch, bool clearGraph = true) override { - auto state = apply(graph, batch, clearGraph); + ABORT("Don't use this"); + auto states = apply(graph, batch, clearGraph); // returns raw logits - return state->getLogProbs(); + return states[0]->getLogProbs(); } virtual Expr build(Ptr graph, diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 28178e768..4aaf9cd52 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -51,8 +51,10 @@ Ptr DecoderFactory::construct() { } Ptr ClassifierFactory::construct() { - if(options_->get("type") == "bert") + if(options_->get("type") == "bert-masked-lm") return New(options_); + if(options_->get("type") == "bert-classifier") + return New(options_); ABORT("Unknown classifier type"); } @@ -218,10 +220,19 @@ Ptr by_type(std::string type, usage use, Ptr options) { } if(type == "bert") { - return models::encoder_classifier()(options) - ("usage", use) - .push_back(models::encoder()("type", "transformer")("original-type", type)) - .push_back(models::classifier()("index", 2)) // indices 0 and 1 used by encoder + return models::encoder_classifier()(options) // + ("usage", use) // + .push_back(models::encoder() // + ("type", "transformer") // + ("original-type", type) // + ("index", 0)) // close to original transformer encoder + .push_back(models::classifier() // + ("type", "bert-masked-lm") // + ("index", 1)) // multi-task learning with MaskedLM + .push_back(models::classifier() // + ("type", "bert-classifier") // + ("bert-classes", 2) // + ("index", 2)) // and next sentence prediction .construct(); } diff --git a/src/models/transformer.h b/src/models/transformer.h index baf7d261b..2c37034a6 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -25,7 +25,7 @@ class Transformer : public EncoderOrDecoderBase { typedef EncoderOrDecoderBase Base; protected: - using Base::options_; using Base::inference_; + using Base::options_; using Base::inference_; using Base::batchIndex_; std::unordered_map cache_; // attention weights produced by step() @@ -87,29 +87,22 @@ class Transformer : public EncoderOrDecoderBase { return embeddings; } - Expr addSentenceEmbeddings(Expr input, int start, Ptr batch) const { - Expr embeddings = input; - - // do nothing if we are not training with sentence pairs - if(opt("bert-sentencepair", false)) { - int dimEmb = input->shape()[-1]; - - auto sentEmbFactory = embedding(graph_) - ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B - ("dimEmb", dimEmb) - .construct(); + Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { + int dimEmb = embeddings->shape()[-1]; - const auto& sentenceIndices = (*batch)[1]->data(); // @TODO: do not use constant index - - // @TODO: note this is going to be really slow due to atomicAdd in backward step - // with only two classes; - // instead two masked reduce operations, maybe in parallel streams? - auto signal = rows(sentEmbFactory, graph_->indices(sentenceIndices)); + auto sentenceEmbeddings = embedding(graph_) + ("prefix", "Wsent") + ("dimVocab", 2) // sentence A or sentence B + ("dimEmb", dimEmb) + .construct(); - embeddings = embeddings + signal; - } - return embeddings; + const auto& sentenceIndices = (*batch)[batchIndex_]->data(1); + + // @TODO: note this is going to be really slow due to atomicAdd in backward step + // with only two classes; + // instead two masked reduce operations, maybe in parallel streams? + auto signal = rows(sentenceEmbeddings, graph_->indices(sentenceIndices)); + return embeddings + signal; } Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const { @@ -118,7 +111,7 @@ class Transformer : public EncoderOrDecoderBase { input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); - if(iAmBert && batch) + if(iAmBert) input = addSentenceEmbeddings(input, start, batch); return input; @@ -555,7 +548,10 @@ class EncoderTransformer : public Transformer { int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb); - if (opt("tied-embeddings-src") || opt("tied-embeddings-all")) + + bool iAmBert = opt("original-type") == "bert"; + + if (opt("tied-embeddings-src") || opt("tied-embeddings-all") || iAmBert) embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); From a99c8b89ae560e8546ace7535acc311c2eab9b48 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 3 Jan 2019 15:29:12 -0800 Subject: [PATCH 080/838] move bert-specifig code to bert.h --- src/data/corpus_base.h | 16 ++-- src/models/bert.h | 131 ++++++++++++++++++++++++++++++++ src/models/classifier.h | 96 ----------------------- src/models/encoder_classifier.h | 2 +- src/models/model_factory.cpp | 18 +++-- src/models/transformer.h | 10 ++- 6 files changed, 156 insertions(+), 117 deletions(-) create mode 100644 src/models/bert.h diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 765d39c78..8ecdf2337 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -110,7 +110,7 @@ class SentenceTuple { */ class SubBatch { private: - std::vector indices_; + std::vector indices_; std::vector mask_; size_t size_; @@ -128,7 +128,7 @@ class SubBatch { * @param width Number of words in the longest sentence */ SubBatch(size_t size, size_t width, const Ptr& vocab) - : indices_(1, Words(size * width, 0)), + : indices_(size * width, 0), mask_(size * width, 0), size_(size), width_(width), @@ -142,12 +142,7 @@ class SubBatch { * idx_{w,0},idx_{w,1},\dots,idx_{w,s}\f$, where \f$w\f$ is the number of * words (width) and \f$s\f$ is the number of sentences (size). */ - std::vector& data() { return indices_[0]; } - std::vector& data(int index) { return indices_[index]; } - - void setNumFactors(size_t num) { indices_.resize(num, Words(size_ * width_, 0)); } // @TODO: for now factors have no vocabulary, they are generated from the other inputs - size_t numFactors() { return indices_.size(); } - + std::vector& data() { return indices_; } /** * @brief Flat masking vector; 0 is used for masked words. * @@ -209,9 +204,8 @@ class SubBatch { size_t words = 0; for(size_t j = 0; j < subWidth; ++j) { for(size_t i = 0; i < size; ++i) { - for(size_t k = 0; k < sb->numFactors(); ++k) - sb->data(k)[j * size + i] = indices_[k][j * size_ + (pos + i)]; - sb->mask()[j * size + i] = mask_[j * size_ + (pos + i)]; + sb->data()[j * size + i] = indices_[j * size_ + (pos + i)]; + sb->mask()[j * size + i] = mask_[j * size_ + (pos + i)]; if(mask_[j * size_ + (pos + i)] != 0) words++; diff --git a/src/models/bert.h b/src/models/bert.h new file mode 100644 index 000000000..9ce80635f --- /dev/null +++ b/src/models/bert.h @@ -0,0 +1,131 @@ +#pragma once + +#include "data/corpus_base.h" +#include "models/encoder_classifier.h" + +namespace marian { +namespace data { + +class BertBatch : public CorpusBatch { +private: + std::vector maskedPositions_; + std::vector maskedIndices_; + std::vector sentenceIndices_; + + void init() { + ABORT("Not implemented"); + } + +public: + BertBatch(Ptr batch) : CorpusBatch(*batch) { + std::cerr << "Creating BERT batch" << std::endl; + init(); + } + + const std::vector& bertMaskedPositions() { return maskedPositions_; } + const std::vector& bertMaskedIndices() { return maskedIndices_; } + const std::vector& bertSentenceIndices() { return sentenceIndices_; } +}; + +} + +class BertEncoderClassifier : public EncoderClassifier { +public: + BertEncoderClassifier(Ptr options) : EncoderClassifier(options) {} + + std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + Ptr bertBatch = New(batch); // intercept batch and anotate with BERT-specific concepts + return EncoderClassifier::apply(graph, bertBatch, clearGraph); + } +}; + +// Can be used for next sentence prediction task +class BertClassifier : public ClassifierBase { +public: + BertClassifier(Ptr options) : ClassifierBase(options) {} + + // The batch has been filled with external classifier labels, @TODO: figure out how to do that + Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence + + int dimModel = classEmbeddings->shape()[-1]; + int dimTrgCls = opt("classifier-classes"); + + auto output = mlp::mlp(graph) // + ("prefix", prefix_ + "_ff_logit") // + .push_back(mlp::dense(graph) // + ("dim", dimModel) // + ("activation", mlp::act::tanh)) // @TODO: do we actually need this? + .push_back(mlp::dense(graph) // + ("dim", dimTrgCls)) // + .construct(); + + auto logits = output->apply(classEmbeddings); // class logits for each batch entry + + auto state = New(); + state->setLogProbs(logits); + + // filled externally, for BERT these are NextSentence prediction labels + const auto& classLabels = (*batch)[batchIndex_]->data(); + state->setTargetIndices(graph->indices(classLabels)); + + return state; + } + + virtual void clear() override {} +}; + +// This is a model that pretrains BERT for classification +class BertMaskedLM : public ClassifierBase { +public: + BertMaskedLM(Ptr options) : ClassifierBase(options) {} + + Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { + Ptr bertBatch = std::dynamic_pointer_cast(batch); + + ABORT_IF(!bertBatch, "Batch could not be converted to batch for BERT training"); + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + + auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries + auto bertMaskedIndices = graph->indices(bertBatch->bertMaskedIndices()); // vocab ids of entries that have been masked + + auto classEmbeddings = rows(context, bertMaskedPositions); // subselect stuff that has actually been masked out; + + int dimModel = classEmbeddings->shape()[-1]; + + int dimVoc = opt>("dim-vocabs")[batchIndex_]; + + auto layerTanh = mlp::dense(graph) // + ("dim", dimModel) // + ("activation", mlp::act::tanh); // + auto layerOut = mlp::output(graph) // + ("dim", dimVoc); // + layerOut.tie_transposed("W", "Wemb"); // We are a BERT model, hence tie with input + + // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] + // assemble layers into MLP and apply to embeddings, decoder context and + // aligned source context + auto output = mlp::mlp(graph) // + ("prefix", prefix_ + "_ff_logit_maskedlm_out") // + .push_back(layerTanh) // @TODO: do we actually need this? + .push_back(layerOut) // + .construct(); + + auto logits = output->apply(classEmbeddings); + + auto state = New(); + state->setLogProbs(logits); + state->setTargetIndices(bertMaskedIndices); + + return state; + } + + virtual void clear() override {} +}; + +} \ No newline at end of file diff --git a/src/models/classifier.h b/src/models/classifier.h index ea92cf669..61c4d64e3 100644 --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -33,100 +33,4 @@ class ClassifierBase { virtual void clear() = 0; }; -// This is a model that pretrains BERT for classification -class BertMaskedLM : public ClassifierBase { -public: - BertMaskedLM(Ptr options) : ClassifierBase(options) {} - - Ptr apply(Ptr graph, Ptr bertBatch, const std::vector>& encoderStates) override { - ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); - - auto context = encoderStates[0]->getContext(); - - // Since this is a classifier we are not masking anything on the target. We can (mis)use the mask to hold - // indices of words in the encoder that have been masked out for BERT's masked LM training. - // Masks are floats, hence the conversion to IndexType. - const auto& maskedRowsFloats = (*bertBatch)[batchIndex_]->mask(); - std::vector maskedRows(maskedRowsFloats.size()); - std::copy(maskedRowsFloats.begin(), maskedRowsFloats.end(), maskedRows.begin()); - - auto classEmbeddings = rows(context, graph->indices(maskedRows)); // subselect stuff that has actually been masked out; - - int dimModel = classEmbeddings->shape()[-1]; - - int dimVoc = opt>("dim-vocabs")[batchIndex_ - 1]; // unsafe - - auto layerTanh = mlp::dense(graph) // - ("dim", dimModel) // - ("activation", mlp::act::tanh); // - - auto layerOut = mlp::output(graph) // - ("dim", dimVoc); - - layerOut.tie_transposed("W", "Wemb"); // We are a BERT model, hence tie with input - - // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] - // assemble layers into MLP and apply to embeddings, decoder context and - // aligned source context - auto output = mlp::mlp(graph) // - ("prefix", prefix_ + "_ff_logit_bert_out") // - .push_back(layerTanh) // @TODO: do we actually need this? - .push_back(layerOut) // - .construct(); - - auto logits = output->apply(classEmbeddings); - - auto state = New(); - state->setLogProbs(logits); - - // filled automatically during masking, these are the vocab indices of masked words - const auto& classLabels = (*bertBatch)[batchIndex_]->data(); - state->setTargetIndices(graph->indices(classLabels)); - - return state; - } - - virtual void clear() override {} -}; - -// This is a model that uses a pre-trained BERT model to build a classifier on top of the encoder -// Can be used for next sentence prediction task -class BertClassifier : public ClassifierBase { -public: - BertClassifier(Ptr options) : ClassifierBase(options) {} - - // The batch has been filled with external classifier labels, @TODO: figure out how to do that - Ptr apply(Ptr graph, Ptr bertBatch, const std::vector>& encoderStates) override { - ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); - - auto context = encoderStates[0]->getContext(); - auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence - - int dimModel = classEmbeddings->shape()[-1]; - int dimTrgCls = opt("bert-classes"); - - auto output = mlp::mlp(graph) // - ("prefix", prefix_ + "_ff_logit") // - .push_back(mlp::dense(graph) // - ("dim", dimModel) // - ("activation", mlp::act::tanh)) // @TODO: do we actually need this? - .push_back(mlp::dense(graph) // - ("dim", dimTrgCls)) // - .construct(); - - auto logits = output->apply(classEmbeddings); // class logits for each batch entry - - auto state = New(); - state->setLogProbs(logits); - - // filled externally, for BERT these are NextSentence prediction labels - const auto& classLabels = (*bertBatch)[batchIndex_]->data(); - state->setTargetIndices(graph->indices(classLabels)); - - return state; - } - - virtual void clear() override {} -}; - } \ No newline at end of file diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 48f5d6c0e..7d7570213 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -178,7 +178,7 @@ class EncoderClassifier : public EncoderClassifierBase { /*********************************************************************/ - std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + virtual std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { if(clearGraph) clear(graph); diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 4aaf9cd52..49a489ff6 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -3,6 +3,7 @@ #include "models/model_factory.h" #include "models/encoder_decoder.h" #include "models/encoder_classifier.h" +#include "models/bert.h" #include "models/costs.h" @@ -80,7 +81,12 @@ Ptr EncoderDecoderFactory::construct() { } Ptr EncoderClassifierFactory::construct() { - auto enccls = New(options_); + Ptr enccls; + if(options_->get("type") == "bert") { + enccls = New(options_); + } else { + enccls = New(options_); + } for(auto& ef : encoders_) enccls->push_back(ef(options_).construct()); @@ -226,13 +232,13 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("type", "transformer") // ("original-type", type) // ("index", 0)) // close to original transformer encoder - .push_back(models::classifier() // - ("type", "bert-masked-lm") // - ("index", 1)) // multi-task learning with MaskedLM .push_back(models::classifier() // ("type", "bert-classifier") // - ("bert-classes", 2) // - ("index", 2)) // and next sentence prediction + ("classifier-classes", 2) // + ("index", 1)) // next sentence prediction + .push_back(models::classifier() // + ("type", "bert-masked-lm") // + ("index", 0)) // multi-task learning with MaskedLM .construct(); } diff --git a/src/models/transformer.h b/src/models/transformer.h index 2c37034a6..d702dbad1 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -11,6 +11,7 @@ #include "models/encoder.h" #include "models/states.h" #include "models/transformer_factory.h" +#include "models/bert.h" #include "rnn/constructors.h" namespace marian { @@ -88,6 +89,10 @@ class Transformer : public EncoderOrDecoderBase { } Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { + Ptr bertBatch = std::dynamic_pointer_cast(batch); + + ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); + int dimEmb = embeddings->shape()[-1]; auto sentenceEmbeddings = embedding(graph_) @@ -96,12 +101,11 @@ class Transformer : public EncoderOrDecoderBase { ("dimEmb", dimEmb) .construct(); - const auto& sentenceIndices = (*batch)[batchIndex_]->data(1); - // @TODO: note this is going to be really slow due to atomicAdd in backward step // with only two classes; // instead two masked reduce operations, maybe in parallel streams? - auto signal = rows(sentenceEmbeddings, graph_->indices(sentenceIndices)); + auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); + auto signal = rows(sentenceEmbeddings, sentenceIndices); return embeddings + signal; } From 9b84dbf51bdcdeb427b782807e4172a4ff00a054 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 3 Jan 2019 16:21:55 -0800 Subject: [PATCH 081/838] move more bert stuff to bert.h --- src/models/bert.h | 33 ++++++++++++++++++++++++++ src/models/model_factory.cpp | 7 ++++-- src/models/transformer.h | 46 +++++++----------------------------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 9ce80635f..37005875a 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -2,6 +2,7 @@ #include "data/corpus_base.h" #include "models/encoder_classifier.h" +#include "models/transformer.h" namespace marian { namespace data { @@ -39,6 +40,38 @@ class BertEncoderClassifier : public EncoderClassifier { } }; +class BertEncoder : public EncoderTransformer { +public: + BertEncoder(Ptr options) : EncoderTransformer(options) {} + + Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { + Ptr bertBatch = std::dynamic_pointer_cast(batch); + + ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); + + int dimEmb = embeddings->shape()[-1]; + + auto sentenceEmbeddings = embedding(graph_) + ("prefix", "Wsent") + ("dimVocab", 2) // sentence A or sentence B + ("dimEmb", dimEmb) + .construct(); + + // @TODO: note this is going to be really slow due to atomicAdd in backward step + // with only two classes; + // instead two masked reduce operations, maybe in parallel streams? + auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); + auto signal = rows(sentenceEmbeddings, sentenceIndices); + return embeddings + signal; + } + + virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { + input = addPositionalEmbeddings(input, start, true); // true for BERT + input = addSentenceEmbeddings(input, start, batch); + return input; + } +}; + // Can be used for next sentence prediction task class BertClassifier : public ClassifierBase { public: diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 49a489ff6..1146c8523 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -39,6 +39,10 @@ Ptr EncoderFactory::construct() { // return New(options_); return NewEncoderTransformer(options_); + if(options_->get("type") == "bert-encoder") + // return New(options_); + return New(options_); + ABORT("Unknown encoder type"); } @@ -229,8 +233,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "transformer") // - ("original-type", type) // + ("type", "bert-encoder") // ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // diff --git a/src/models/transformer.h b/src/models/transformer.h index d702dbad1..6d7f7a894 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -11,7 +11,6 @@ #include "models/encoder.h" #include "models/states.h" #include "models/transformer_factory.h" -#include "models/bert.h" #include "rnn/constructors.h" namespace marian { @@ -88,36 +87,10 @@ class Transformer : public EncoderOrDecoderBase { return embeddings; } - Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { - Ptr bertBatch = std::dynamic_pointer_cast(batch); - - ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); - - int dimEmb = embeddings->shape()[-1]; - - auto sentenceEmbeddings = embedding(graph_) - ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B - ("dimEmb", dimEmb) - .construct(); - - // @TODO: note this is going to be really slow due to atomicAdd in backward step - // with only two classes; - // instead two masked reduce operations, maybe in parallel streams? - auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); - auto signal = rows(sentenceEmbeddings, sentenceIndices); - return embeddings + signal; - } - - Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const { - bool iAmBert = opt("original-type", "undefined") == "bert"; - bool learnedPosEmbeddings = opt("transformer-learned-positions", false) || iAmBert; - + virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const { + batch; + bool learnedPosEmbeddings = opt("transformer-learned-positions", false); input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); - - if(iAmBert) - input = addSentenceEmbeddings(input, start, batch); - return input; } @@ -529,6 +502,7 @@ class Transformer : public EncoderOrDecoderBase { class EncoderTransformer : public Transformer { public: EncoderTransformer(Ptr options) : Transformer(options) {} + virtual ~EncoderTransformer() {} // returns the embedding matrix based on options // and based on batchIndex_. @@ -569,13 +543,13 @@ class EncoderTransformer : public Transformer { return embFactory.construct(); } - Ptr build(Ptr graph, - Ptr batch) override { + virtual Ptr build(Ptr graph, + Ptr batch) override { graph_ = graph; return apply(batch); } - Ptr apply(Ptr batch) { + virtual Ptr apply(Ptr batch) { int dimEmb = opt("dim-emb"); int dimBatch = (int)batch->size(); int dimSrcWords = (int)(*batch)[batchIndex_]->batchWidth(); @@ -637,7 +611,7 @@ class EncoderTransformer : public Transformer { return New(context, batchMask, batch); } - void clear() override {} + virtual void clear() override {} }; class TransformerState : public DecoderState { @@ -753,9 +727,7 @@ class DecoderTransformer : public Transformer { // Used for position embeddings and creating new decoder states. int startPos = (int)state->getPosition(); - scaledEmbeddings - = addSpecialEmbeddings(scaledEmbeddings, startPos); - + scaledEmbeddings = addSpecialEmbeddings(scaledEmbeddings, startPos); scaledEmbeddings = atleast_nd(scaledEmbeddings, 4); // reorganize batch and timestep From 13e8640d0fa894b725cc009afd1b5390fd79f137 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 7 Jan 2019 09:33:49 -0800 Subject: [PATCH 082/838] (spelling error) --- src/training/scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index bcf7175e9..d6c527900 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -486,7 +486,7 @@ class Scheduler : public TrainingObserver { state.factor *= factor; updateLearningRate(state); LOG(info, - "Decaying learning rate to {} after stalled {} time(s)", + "Decaying learning rate to {} after having stalled {} time(s)", state.eta, state.stalled); From d765d06f50e3de741a3f9b2c47fb4d987f3b5aa3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 7 Jan 2019 09:42:26 -0800 Subject: [PATCH 083/838] removed some experimental code --- src/common/config_parser.cpp | 3 --- src/optimizers/optimizers.cpp | 22 ---------------------- src/training/scheduler.h | 9 ++------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 14c266c8b..70a30a5fd 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -618,9 +618,6 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { {"0"}); cli.add("--mini-batch-track-lr", "Dynamically track mini-batch size inverse to actual learning rate (not considering lr-warmup)"); - cli.add("--mini-batch-warmup-exp", - "[experimental] Soften initial ramp-up curve with this exponent (until ramp-up reached 100%)", - 1.); cli.add("--mini-batch-overstuff", "[experimental] Stuff this much more data into a minibatch, but scale down the LR and progress counter", 1); diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index c4e9951f0..685c917f9 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -169,28 +169,6 @@ void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t r vt_ // =_3 ); - // log the magnitude (with good ole CPU code) - static int logCount = 0; - logCount++; - if (logCount <= 10 || logCount % 100 == 0) { - std::vector g, mt, vt; - grads->get(g); mt_->get(mt); vt_->get(vt); - size_t n = g.size(); ABORT_IF(mt.size() != g.size() || vt.size() != g.size(), "mismatching sizes??"); - double gs = 0, ms = 0; // square sum - for (size_t i = 0; i < n; i++) { - auto gi = g[i] / T; // raw average gradient - auto mi = mt[i] / denom1f; // momentum-smoothed average gradient - auto di = sqrtf(vt[i] / denom2f) + eps_; - gi /= di; // RMS-normalize both - mi /= di; - gs += gi * gi; - ms += mi * mi; - } - auto grms = std::sqrt(gs / n); // hmm... expecting 1.0 here, no? - auto mrms = std::sqrt(ms / n); - LOG(info, "Adam[{}]: grms = {}, mrms = {}", logCount, grms, mrms); - } - params->getBackend()->synchronize(); // @TODO: This should not be in here. Maybe in the wrapper. Why is it needed at all? } diff --git a/src/training/scheduler.h b/src/training/scheduler.h index d6c527900..1fadad0be 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -83,19 +83,14 @@ class Scheduler : public TrainingObserver { if (mbWarmup) { // mini-batch-warmup LOG_ONCE(info, "[scheduler] Mini-batch size warmup {}", std::string(mbWarmup)); - // This scales MB size up from the start, relative to progress within warm-up period. + // This ramps up MB size at start, relative to progress within warm-up period. size_t progress = state_->getProgressIn(mbWarmup.unit); // number of updates/labels processed auto progressRatio = (double)progress / (double)mbWarmup.n; // where are we relatively within target warm-up period // if unit is labels, then account for the fact that our increment itself is not constant if (mbWarmup.unit == SchedulingUnit::trgLabels) progressRatio = std::sqrt(progressRatio); - if (progressRatio < 1) { - // soften the very initial ramp-up if requested - double exp = options_->get("mini-batch-warmup-exp"); - progressRatio = pow(progressRatio, exp); // e.g. 1.5 => linear ramp-up -> sublinear ramp-up - // apply ratio to actual batch size + if (progressRatio < 1) ratio *= progressRatio; - } } // dynamic MB-size tracking with learning rate From 3800c03d096fed196e46111ba6a411c042679825 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 7 Jan 2019 14:06:28 -0800 Subject: [PATCH 084/838] classifier input --- src/common/config_parser.cpp | 4 ++++ src/data/corpus.cpp | 2 +- src/data/corpus_base.cpp | 22 ++++++++++++++++++++-- src/data/corpus_base.h | 9 ++++++++- src/data/corpus_nbest.cpp | 4 ++-- src/data/corpus_sqlite.cpp | 2 +- src/models/bert.h | 8 ++++---- src/models/encoder_classifier.h | 1 + src/models/encoder_decoder.cpp | 1 + src/models/model_factory.cpp | 13 ++++++++++++- 10 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index d39f3b1ad..2d77661bf 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -143,6 +143,10 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Enable layer normalization"); cli.add("--right-left", "Train right-to-left model"); + cli.add>("--input-types", + "Provide type of input data if different than 'sequence'. " + "Possible values: sequence, labels. You need to provide one type per input.", + {}); cli.add("--best-deep", "Use Edinburgh deep RNN configuration (s2s)"); cli.add_nondefault>("--special-vocab", diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7a7a846e1..ffa0a24c6 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -55,7 +55,7 @@ SentenceTuple Corpus::next() { } else if(i > 0 && i == weightFileIdx_) { addWeightsToSentenceTuple(line, tup); } else { - addWordsToSentenceTuple(line, i, tup); + addWordsToSentenceTuple(line, i, tup, addEOS_[i]); } } diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index c9df1c42d..a8297c638 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -57,6 +57,23 @@ CorpusBase::CorpusBase(Ptr options, bool translate) else paths_ = options_->get>("input"); + addEOS_.resize(paths_.size(), true); + + // @TODO: think if this should be checked and processed here or in a validation step in config? + auto inputTypes = options_->get>("input-types", {}); // empty list by default + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types are specified ({}) you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + // Currently input types affects only EOS symbol + for(int i = 0; i < inputTypes.size(); ++i) + if(inputTypes[i] == "labels") + addEOS_[i] = false; + else if(inputTypes[i] == "sequence") + addEOS_[i] = true; + else + ABORT("Unknown input type {}: {}", i, inputTypes[i]); + std::vector vocabPaths; if(!options_->get>("vocabs").empty()) vocabPaths = options_->get>("vocabs"); @@ -189,12 +206,13 @@ CorpusBase::CorpusBase(Ptr options, bool translate) void CorpusBase::addWordsToSentenceTuple(const std::string& line, size_t i, - SentenceTuple& tup) const { + SentenceTuple& tup, + bool addEOS) const { // This turns a string in to a sequence of numerical word ids. Depending // on the vocabulary type, this can be non-trivial, e.g. when SentencePiece // is used. - Words words = vocabs_[i]->encode(line, /*addEOS =*/ true, inference_); + Words words = vocabs_[i]->encode(line, /*addEOS =*/ addEOS, inference_); if(words.empty()) words.push_back(0); diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 8ecdf2337..be33036bf 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -513,6 +513,12 @@ class CorpusBase std::vector> files_; std::vector> vocabs_; + /** + * brief Determines if a EOS symbol should be added. By default this is true for any sequence, + * but should be false for instance for classifier labels. This is set per input stream, hence a vector. + */ + std::vector addEOS_; + size_t pos_{0}; size_t maxLength_{0}; @@ -537,7 +543,8 @@ class CorpusBase */ void addWordsToSentenceTuple(const std::string& line, size_t i, - SentenceTuple& tup) const; + SentenceTuple& tup, + bool addEOS) const; /** * @brief Helper function parsing a line with word alignments and adding them * to the sentence tuple. diff --git a/src/data/corpus_nbest.cpp b/src/data/corpus_nbest.cpp index 328c3c0d7..4fd560992 100644 --- a/src/data/corpus_nbest.cpp +++ b/src/data/corpus_nbest.cpp @@ -61,9 +61,9 @@ SentenceTuple CorpusNBest::next() { "Too few lines in input {}", i); } - addWordsToSentenceTuple(lastLines_[i], i, tup); + addWordsToSentenceTuple(lastLines_[i], i, tup, addEOS_[i]); } - addWordsToSentenceTuple(curr_text, last, tup); + addWordsToSentenceTuple(curr_text, last, tup, addEOS_[last]); lastNum_ = curr_num; } diff --git a/src/data/corpus_sqlite.cpp b/src/data/corpus_sqlite.cpp index cbab750eb..7127aafda 100644 --- a/src/data/corpus_sqlite.cpp +++ b/src/data/corpus_sqlite.cpp @@ -120,7 +120,7 @@ SentenceTuple CorpusSQLite::next() { } else if(i > 0 && i == weightFileIdx_) { addWeightsToSentenceTuple(line, tup); } else { - addWordsToSentenceTuple(line, i, tup); + addWordsToSentenceTuple(line, i, tup, addEOS_[i]); } } diff --git a/src/models/bert.h b/src/models/bert.h index 37005875a..a1c7d4588 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -85,7 +85,7 @@ class BertClassifier : public ClassifierBase { auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence int dimModel = classEmbeddings->shape()[-1]; - int dimTrgCls = opt("classifier-classes"); + int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels auto output = mlp::mlp(graph) // ("prefix", prefix_ + "_ff_logit") // @@ -143,10 +143,10 @@ class BertMaskedLM : public ClassifierBase { // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context - auto output = mlp::mlp(graph) // + auto output = mlp::mlp(graph) // ("prefix", prefix_ + "_ff_logit_maskedlm_out") // - .push_back(layerTanh) // @TODO: do we actually need this? - .push_back(layerOut) // + .push_back(layerTanh) // @TODO: do we actually need this? + .push_back(layerOut) // .construct(); auto logits = output->apply(classEmbeddings); diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 7d7570213..9a650311a 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -103,6 +103,7 @@ class EncoderClassifier : public EncoderClassifierBase { "skip", "layer-normalization", "right-left", + "input-types", "special-vocab", "tied-embeddings", "tied-embeddings-src", diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index e8f93bc1c..047d960d1 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -23,6 +23,7 @@ EncoderDecoder::EncoderDecoder(Ptr options) "skip", "layer-normalization", "right-left", + "input-types", "special-vocab", "tied-embeddings", "tied-embeddings-src", diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 1146c8523..d800308ee 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -237,7 +237,6 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // - ("classifier-classes", 2) // ("index", 1)) // next sentence prediction .push_back(models::classifier() // ("type", "bert-masked-lm") // @@ -245,6 +244,18 @@ Ptr by_type(std::string type, usage use, Ptr options) { .construct(); } + if(type == "bert-classifier") { + return models::encoder_classifier()(options) // + ("usage", use) // + .push_back(models::encoder() // + ("type", "bert-encoder") // + ("index", 0)) // close to original transformer encoder + .push_back(models::classifier() // + ("type", "bert-classifier") // + ("index", 1)) // next sentence prediction + .construct(); + } + #ifdef COMPILE_EXAMPLES // @TODO: examples should be compiled optionally if(type == "mnist-ffnn") { From bf41d04947d4eb1f308de248ec6220421874a50b Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 7 Jan 2019 16:09:50 -0800 Subject: [PATCH 085/838] fix errors for first classifier training --- src/data/corpus_base.cpp | 20 ++++++++++++++++++-- src/models/bert.h | 19 ++++++++++++------- src/models/model_factory.cpp | 4 ++-- src/models/transformer.h | 5 ++--- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index a8297c638..319d5f2ac 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -43,6 +43,23 @@ CorpusBase::CorpusBase(const std::vector& paths, files_.emplace_back(new io::InputFileStream(path)); ABORT_IF(files_.back()->empty(), "File '{}' is empty", path); } + + addEOS_.resize(paths_.size(), true); + // @TODo: think if this should be checked and processed here or in a validation step in config? + auto inputTypes = options_->get>("input-types", {}); // empty list by default + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types are specified ({}) you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + // Currently input types affects only EOS symbol + for(int i = 0; i < inputTypes.size(); ++i) + if(inputTypes[i] == "labels") + addEOS_[i] = false; + else if(inputTypes[i] == "sequence") + addEOS_[i] = true; + else + ABORT("Unknown input type {}: {}", i, inputTypes[i]); + } CorpusBase::CorpusBase(Ptr options, bool translate) @@ -58,8 +75,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) paths_ = options_->get>("input"); addEOS_.resize(paths_.size(), true); - - // @TODO: think if this should be checked and processed here or in a validation step in config? + // @TODo: think if this should be checked and processed here or in a validation step in config? auto inputTypes = options_->get>("input-types", {}); // empty list by default ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), "Input types are specified ({}) you need to specify one per input ({})", diff --git a/src/models/bert.h b/src/models/bert.h index a1c7d4588..b25f608be 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -40,6 +40,7 @@ class BertEncoderClassifier : public EncoderClassifier { } }; +// @TODO: this should be in transformer.h class BertEncoder : public EncoderTransformer { public: BertEncoder(Ptr options) : EncoderTransformer(options) {} @@ -50,6 +51,8 @@ class BertEncoder : public EncoderTransformer { ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); int dimEmb = embeddings->shape()[-1]; + int dimBatch = embeddings->shape()[-2]; + int dimWords = embeddings->shape()[-3]; auto sentenceEmbeddings = embedding(graph_) ("prefix", "Wsent") @@ -62,6 +65,7 @@ class BertEncoder : public EncoderTransformer { // instead two masked reduce operations, maybe in parallel streams? auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); auto signal = rows(sentenceEmbeddings, sentenceIndices); + signal = reshape(signal, {dimWords, dimBatch, dimEmb}); return embeddings + signal; } @@ -87,13 +91,14 @@ class BertClassifier : public ClassifierBase { int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels - auto output = mlp::mlp(graph) // - ("prefix", prefix_ + "_ff_logit") // - .push_back(mlp::dense(graph) // - ("dim", dimModel) // - ("activation", mlp::act::tanh)) // @TODO: do we actually need this? - .push_back(mlp::dense(graph) // - ("dim", dimTrgCls)) // + auto output = mlp::mlp(graph) // + .push_back(mlp::dense(graph) // + ("prefix", prefix_ + "_ff_logit_l1") // + ("dim", dimModel) // + ("activation", mlp::act::tanh)) // @TODO: do we actually need this? + .push_back(mlp::output(graph) // + ("dim", dimTrgCls)) // + ("prefix", prefix_ + "_ff_logit_l2") // .construct(); auto logits = output->apply(classEmbeddings); // class logits for each batch entry diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index d800308ee..2b19b4990 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -233,7 +233,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // + ("type", "bert-encoder") // transformer encoder for now ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // @@ -248,7 +248,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // + ("type", "transformer") // ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // diff --git a/src/models/transformer.h b/src/models/transformer.h index 6d7f7a894..188080a99 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -65,6 +65,7 @@ class Transformer : public EncoderOrDecoderBase { std::iota(positions.begin(), positions.end(), 0); // fill with increasing numbers until current length auto signal = rows(posEmbFactory, graph_->indices(positions)); + signal = reshape(signal, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; } else { float num_timescales = (float)dimEmb / 2; @@ -527,9 +528,7 @@ class EncoderTransformer : public Transformer { int dimEmb = opt("dim-emb"); auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb); - bool iAmBert = opt("original-type") == "bert"; - - if (opt("tied-embeddings-src") || opt("tied-embeddings-all") || iAmBert) + if (opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); From 9023bf8b90c0d5ca68890dc2683dce50e7ab637f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 8 Jan 2019 17:15:39 -0800 Subject: [PATCH 086/838] fixed build break due to MPI change in examples --- src/examples/mnist/mnist_ffnn.cpp | 0 src/examples/mnist/training.h | 7 +++++- vs/Marian.vcxproj | 20 +++++++++++++++ vs/Marian.vcxproj.filters | 42 +++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/examples/mnist/mnist_ffnn.cpp mode change 100644 => 100755 src/examples/mnist/training.h diff --git a/src/examples/mnist/mnist_ffnn.cpp b/src/examples/mnist/mnist_ffnn.cpp old mode 100644 new mode 100755 diff --git a/src/examples/mnist/training.h b/src/examples/mnist/training.h old mode 100644 new mode 100755 index 8cdacb5e0..64ccfcbd0 --- a/src/examples/mnist/training.h +++ b/src/examples/mnist/training.h @@ -30,8 +30,11 @@ class TrainMNIST : public ModelTask { auto scheduler = New(options_, trainState); scheduler->addValidator(New(options_)); + // Multi-node training + auto mpi = initMPI(/*multiThreaded=*/false); + // Prepare model - auto model = New(options_); + auto model = New(options_, mpi); model->setScheduler(scheduler); model->load(); @@ -47,6 +50,8 @@ class TrainMNIST : public ModelTask { scheduler->increaseEpoch(); } scheduler->finished(); + model = nullptr; + finalizeMPI(std::move(mpi)); } }; } // namespace marian diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index c13d7841d..c77cbd006 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -563,6 +563,15 @@ + + true + true + + + + true + true + @@ -687,6 +696,11 @@ + + + + + @@ -1049,6 +1063,11 @@ true true + + + + + true @@ -1097,6 +1116,7 @@ true true + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 745e130da..9e5cbf5f4 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -472,6 +472,15 @@ tests + + examples\mnist + + + examples\iris + + + examples\iris + @@ -1493,6 +1502,21 @@ 3rd_party\pathie-cpp\include + + examples\mnist + + + examples\mnist + + + examples\mnist + + + examples\mnist + + + examples\mnist + @@ -1810,6 +1834,21 @@ tests + + examples\mnist + + + examples\iris + + + examples + + + examples + + + examples + @@ -1821,5 +1860,8 @@ tests + + examples + \ No newline at end of file From 664e29ff6ec4a83d34f42c3398de52ac9181a17d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 8 Jan 2019 17:59:05 -0800 Subject: [PATCH 087/838] removed operator-> for Factories --- src/layers/constructors.h | 2 -- src/models/s2s.h | 8 ++++---- src/rnn/constructors.h | 2 -- src/tests/attention_tests.cpp | 2 +- src/tests/rnn_tests.cpp | 8 ++++---- vs/Marian.vcxproj | 5 ++++- 6 files changed, 13 insertions(+), 14 deletions(-) mode change 100644 => 100755 src/layers/constructors.h mode change 100644 => 100755 src/tests/attention_tests.cpp mode change 100644 => 100755 src/tests/rnn_tests.cpp diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100644 new mode 100755 index c063f44c8..16765341f --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -146,8 +146,6 @@ class MLPFactory : public Factory { return mlp; } - Ptr operator->() { return construct(); } - template Accumulator push_back(const LF& lf) { layers_.push_back(New(lf)); diff --git a/src/models/s2s.h b/src/models/s2s.h index 92ba9a7d1..b4600892f 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -86,8 +86,8 @@ class EncoderS2S : public EncoderBase { rnnBw.push_back(stacked); } - auto context = concatenate({rnnFw->transduce(embeddings, mask), - rnnBw->transduce(embeddings, mask)}, + auto context = concatenate({rnnFw.construct()->transduce(embeddings, mask), + rnnBw.construct()->transduce(embeddings, mask)}, /*axis =*/ -1); if(second > 0) { @@ -114,7 +114,7 @@ class EncoderS2S : public EncoderBase { } // transduce context to new context - context = rnnUni->transduce(context); + context = rnnUni.construct()->transduce(context); } return context; } @@ -269,7 +269,7 @@ class DecoderS2S : public DecoderBase { && opt("original-type") == "nematus") // ); - start = mlp->apply(meanContexts); + start = mlp.construct()->apply(meanContexts); } else { int dimBatch = (int)batch->size(); int dimRnn = opt("dim-rnn"); diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index ec0e1bd85..657d8ad16 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -187,8 +187,6 @@ class RNNFactory : public Factory { return rnn; } - Ptr operator->() { return construct(); } - template Accumulator push_back(const F& f) { layerFactories_.push_back(New(f)); diff --git a/src/tests/attention_tests.cpp b/src/tests/attention_tests.cpp old mode 100644 new mode 100755 index eff555729..1f4ae5b6d --- a/src/tests/attention_tests.cpp +++ b/src/tests/attention_tests.cpp @@ -61,7 +61,7 @@ void tests(DeviceType type) { .push_back(rnn::cell(graph)) // .construct(); - auto context = rnn->transduce(input, mask); + auto context = rnn.construct()->transduce(input, mask); auto encState = New(context, mask, nullptr); diff --git a/src/tests/rnn_tests.cpp b/src/tests/rnn_tests.cpp old mode 100644 new mode 100755 index 87639c860..145828788 --- a/src/tests/rnn_tests.cpp +++ b/src/tests/rnn_tests.cpp @@ -51,7 +51,7 @@ void tests(DeviceType type) { .push_back(rnn::cell(graph)) // .construct(); - auto output = rnn->transduce(input); + auto output = rnn.construct()->transduce(input); graph->forward(); @@ -161,8 +161,8 @@ void tests(DeviceType type) { rnnBw.push_back(stacked); } - auto context = concatenate({rnnFw->transduce(input, mask), - rnnBw->transduce(input, mask)}, + auto context = concatenate({rnnFw.construct()->transduce(input, mask), + rnnBw.construct()->transduce(input, mask)}, /*axis =*/ input->shape().size() - 1); if(second > 0) { @@ -188,7 +188,7 @@ void tests(DeviceType type) { } // transduce context to new context - context = rnnUni->transduce(context); + context = rnnUni.construct()->transduce(context); } return context; }; diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index c77cbd006..64faded63 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -567,7 +567,10 @@ true true - + + true + true + true true From 209d38724ca5e2b38baa8ff81e98a06a1924fad7 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 9 Jan 2019 09:21:36 -0800 Subject: [PATCH 088/838] fix batch-size-related error, choose correct axis for classifier --- src/common/config_parser.cpp | 2 + src/data/corpus_base.cpp | 25 ++++---- src/data/corpus_base.h | 4 +- src/data/default_vocab.cpp | 96 +++++++++++++--------------- src/data/vocab.cpp | 15 ++++- src/graph/node_operators_binary.h | 3 + src/layers/loss.cpp | 37 +++++------ src/models/bert.h | 2 +- src/tensors/cpu/tensor_operators.cpp | 20 +++--- src/training/graph_group.h | 24 +++++-- src/training/scheduler.h | 14 ++-- 11 files changed, 131 insertions(+), 111 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 2d77661bf..1910c85de 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -280,6 +280,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Display nformation for the first arg updates"); cli.add("--disp-label-counts", "Display label counts when logging loss progress"); + cli.add("--disp-wps-index", + "Display words-per-second ratio based on i-th sub-batch (-1 is last)", -1); cli.add("--save-freq", "Save model file every arg updates (append 't' for every arg target labels)", "10000u"); diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index 319d5f2ac..b35bb032b 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -77,18 +77,6 @@ CorpusBase::CorpusBase(Ptr options, bool translate) addEOS_.resize(paths_.size(), true); // @TODo: think if this should be checked and processed here or in a validation step in config? auto inputTypes = options_->get>("input-types", {}); // empty list by default - ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), - "Input types are specified ({}) you need to specify one per input ({})", - inputTypes.size(), - paths_.size()); - // Currently input types affects only EOS symbol - for(int i = 0; i < inputTypes.size(); ++i) - if(inputTypes[i] == "labels") - addEOS_[i] = false; - else if(inputTypes[i] == "sequence") - addEOS_[i] = true; - else - ABORT("Unknown input type {}: {}", i, inputTypes[i]); std::vector vocabPaths; if(!options_->get>("vocabs").empty()) @@ -162,6 +150,19 @@ CorpusBase::CorpusBase(Ptr options, bool translate) vocabs_.emplace_back(vocab); } } + + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types are specified ({}) you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + // Currently input types affects only EOS symbol + for(int i = 0; i < inputTypes.size(); ++i) + if(inputTypes[i] == "labels") + addEOS_[i] = false; + else if(inputTypes[i] == "sequence") + addEOS_[i] = true; + else + ABORT("Unknown input type {}: {}", i, inputTypes[i]); } if(translate) { diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index be33036bf..b8bc556cc 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -47,7 +47,7 @@ class SentenceTuple { /** * @brief Adds a new sentence at the end of the tuple. * - * @param words A vector of word indexes. + * @param words A vector of word indices. */ void push_back(const Words& words) { tuple_.push_back(words); } @@ -460,7 +460,7 @@ class CorpusBatch : public Batch { std::cerr << "batches: " << sets() << std::endl; if(!sentenceIds_.empty()) { - std::cerr << "indexes: "; + std::cerr << "indices: "; for(auto id : sentenceIds_) std::cerr << id << " "; std::cerr << std::endl; diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index e344eadf9..bac5c72ec 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -112,8 +112,6 @@ class DefaultVocab : public VocabBase { ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath); } - std::unordered_set seenSpecial; - id2str_.reserve(vocab.size()); for(auto&& pair : vocab) { auto str = pair.first; @@ -126,59 +124,13 @@ class DefaultVocab : public VocabBase { } ABORT_IF(id2str_.empty(), "Empty vocabulary: ", vocabPath); - // look up ids for and , which are required - // The name backCompatStr is alternatively accepted for Yaml vocabs if id - // equals backCompatId. - auto getRequiredWordId = [&](const std::string& str, - const std::string& backCompatStr, - Word backCompatId) { - // back compat with Nematus Yaml dicts - if(isJson) { - // if word id 0 or 1 is either empty or has the Nematus-convention string, - // then use it - if(backCompatId < id2str_.size() - && (id2str_[backCompatId].empty() - || id2str_[backCompatId] == backCompatStr)) { - LOG(info, - "[data] Using unused word id {} for {}", - backCompatStr, - backCompatId, - str); - return backCompatId; - } - } - auto iter = str2id_.find(str); - ABORT_IF(iter == str2id_.end(), - "DefaultVocabulary file {} is expected to contain an entry for {}", - vocabPath, - str); - return iter->second; - }; - eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, DEFAULT_EOS_ID); - unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, DEFAULT_UNK_ID); - - // some special symbols for hard attention - if(!seenSpecial.empty()) { - auto requireWord = [&](Word id, const std::string& str) { - auto iter = str2id_.find(str); - // word already in vocab: must be at right index, else fail - if(iter != str2id_.end()) - ABORT_IF(iter->second != id, - "special vocabulary entry '{}' is expected to have id {}", - str, - id); - else - insertWord(id, str); - }; - // @TODO: the hard-att code has not yet been updated to accept EOS at any id - requireWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR); - } + addRequiredVocabulary(vocabPath, isJson); return std::max(id2str_.size(), maxSize); } // for fakeBatch() - void createFake() override { + virtual void createFake() override { eosId_ = insertWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR); unkId_ = insertWord(DEFAULT_UNK_ID, DEFAULT_UNK_STR); } @@ -214,6 +166,39 @@ class DefaultVocab : public VocabBase { private: + virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) { + // look up ids for and , which are required + // The name backCompatStr is alternatively accepted for Yaml vocabs if id + // equals backCompatId. + auto getRequiredWordId = [&](const std::string& str, + const std::string& backCompatStr, + Word backCompatId) { + // back compat with Nematus Yaml dicts + if(isJson) { + // if word id 0 or 1 is either empty or has the Nematus-convention string, + // then use it + if(backCompatId < id2str_.size() + && (id2str_[backCompatId].empty() + || id2str_[backCompatId] == backCompatStr)) { + LOG(info, + "[data] Using unused word id {} for {}", + backCompatStr, + backCompatId, + str); + return backCompatId; + } + } + auto iter = str2id_.find(str); + ABORT_IF(iter == str2id_.end(), + "DefaultVocabulary file {} is expected to contain an entry for {}", + vocabPath, + str); + return iter->second; + }; + eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, DEFAULT_EOS_ID); + unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, DEFAULT_UNK_ID); + } + void addCounts(std::unordered_map& counter, const std::string& trainPath) { std::unique_ptr trainStrm( @@ -298,8 +283,19 @@ class DefaultVocab : public VocabBase { }; }; +// This is a vocabulary class that does not enforce or . +// This is used for class lists in a classifier. +class LabelsVocab : public DefaultVocab { +private: + virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override {} // Do nothing. +}; + Ptr createDefaultVocab() { return New(); } +Ptr createLabelsVocab() { + return New(); +} + } diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 62b021a3f..1e6a341b2 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -4,17 +4,26 @@ namespace marian { Ptr createDefaultVocab(); +Ptr createLabelsVocab(); Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); // @TODO: make each vocab peek on type Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); - return vocab ? vocab : createDefaultVocab(); + if(vocab) { + return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it + } else { + auto inputType = options->get>("input-types")[batchIndex]; + if(inputType == "labels") + return createLabelsVocab(); + else + return createDefaultVocab(); + } } size_t Vocab::loadOrCreate(const std::string& vocabPath, - const std::vector& trainPaths, - size_t maxSize) { + const std::vector& trainPaths, + size_t maxSize) { size_t size = 0; if(vocabPath.empty()) { // No vocabulary path was given, attempt to first find a vocabulary diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 7da85443b..8bd518bec 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -821,6 +821,9 @@ struct MinimumNodeOp : public ElementBinaryNodeOp { struct CrossEntropyNodeOp : public NaryNodeOp { CrossEntropyNodeOp(Expr a, Expr indices) : NaryNodeOp({a, indices}, newShape(a)) { matchOrAbort(indices->value_type()); + int rows = a->shape().elements() / a->shape()[-1]; + int labels = indices->shape().elements(); + ABORT_IF(rows != labels, "Number of examples and labels does not match: {} != {}", rows, labels); } Shape newShape(Expr a) { diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 03b796823..4dbbb99ee 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -64,14 +64,14 @@ Expr CrossEntropyMeanWordsLoss::getCost(Expr logits, Expr mask, Expr weights) { auto ce = getCrossEntropy(logits, indices, mask, weights); - // if(weights) { - // return (sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) - // / sum(sum(mask * weights, /*axis =*/ -3), /*axis =*/ -2)); - // } - // else { + if(mask) { return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch / sum(sum(mask, /*axis =*/ -3), /*axis =*/ -2); // divide by number of words (sum over mask) - // } + } else { + return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch + / ce->shape().elements(); // no mask, hence divide by all number of all elements + } + } Expr CrossEntropySumLoss::getCost(Expr logits, @@ -79,13 +79,7 @@ Expr CrossEntropySumLoss::getCost(Expr logits, Expr mask, Expr weights) { auto ce = getCrossEntropy(logits, indices, mask, weights); - // if(weights) { - // return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) - // / mean(mean(mask * weights, /*axis =*/ -3), /*axis =*/ -2); - // } - // else { - return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2); - // } + return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2); } Expr PerplexityLoss::getCost(Expr logits, @@ -93,14 +87,13 @@ Expr PerplexityLoss::getCost(Expr logits, Expr mask, Expr weights) { auto ce = getCrossEntropy(logits, indices, mask, weights); - // if(weights) { - // return exp(sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) - // / sum(sum(mask * weights, /*axis =*/ -3), /*axis =*/ -2)); - // } - // else { + if(mask) { return exp(sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch / sum(sum(mask, /*axis =*/ -3), /*axis =*/ -2)); // divide by number of words (sum over mask) - // } + } else { + return exp(sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch + / ce->shape().elements()); // divide by number of words (sum over mask) + } } Expr CrossEntropyRescoreLoss::getCost(Expr logits, @@ -117,7 +110,11 @@ Expr CrossEntropyRescoreMeanLoss::getCost(Expr logits, Expr weights) { auto ce = getCrossEntropy(logits, indices, mask, weights); // divide by number of words in sentence - return -sum(ce, /*axis =*/ -3) / sum(mask, /*axis =*/ -3); + if(mask) { + return -sum(ce, /*axis =*/ -3) / sum(mask, /*axis =*/ -3); + } else { + return -sum(ce, /*axis =*/ -3) / ce->shape()[-3]; + } } } // namespace marian diff --git a/src/models/bert.h b/src/models/bert.h index b25f608be..7dfc83c90 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -86,7 +86,7 @@ class BertClassifier : public ClassifierBase { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); auto context = encoderStates[0]->getContext(); - auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-2); // [CLS] symbol is first symbol in each sequence + auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-3); // [CLS] symbol is first symbol in each sequence int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index a3b6bf9a6..4cc5daf0d 100755 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -661,36 +661,34 @@ void GRUFastBackward(std::vector outputs, } } -void CrossEntropyPick(Tensor out_, Tensor in_, Tensor pick_) { - matchOrAbort(pick_->type()); +void CrossEntropyPick(Tensor out, Tensor in, Tensor pick) { + matchOrAbort(pick->type()); - float* out = out_->data(); // Shape& outShape = out_->shape(); - const float* in = in_->data(); - Shape& inShape = in_->shape(); + Shape& inShape = in->shape(); int rows = inShape.elements() / inShape.back(); int cols = inShape.back(); -#pragma omp parallel for + #pragma omp parallel for for(int j = 0; j < rows; ++j) { - const float* sp = in + j * cols; + const float* sp = in->data() + j * cols; float max = sp[0]; -#pragma omp simd reduction(max : max) + #pragma omp simd reduction(max : max) for(int i = 1; i < cols; ++i) { max = std::max(max, sp[i]); } float sum = 0.f; -#pragma omp simd reduction(+ : sum) + #pragma omp simd reduction(+ : sum) for(int i = 0; i < cols; ++i) { sum += std::exp(sp[i] - max); } // cross-entropy - int i = (int)pick_->data()[j]; + IndexType i = pick->data()[j]; // This appears to be safe i.e. that i >= 0 && i < cols is known - out[j] = std::log(sum) - sp[i] + max; + out->data()[j] = std::log(sum) - sp[i] + max; } } diff --git a/src/training/graph_group.h b/src/training/graph_group.h index fc372adce..2bd98d40b 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -53,8 +53,7 @@ class GraphGroup { size_t multiplier = 1) { auto stats = New(); - size_t numFiles - = options_->get>("train-sets").size(); + size_t numFiles = options_->get>("train-sets").size(); // Initialize first batch to step size size_t first = options_->get("mini-batch-fit-step"); @@ -65,31 +64,42 @@ class GraphGroup { size_t maxLength = options_->get("max-length"); maxLength = (size_t)(std::ceil(maxLength / (float)step) * step); - // @TODO: ugly - auto toptions = New(); - toptions->merge(options_); + // restrict maximum length for labels to 1 + std::vector localMaxes(numFiles, maxLength); + auto inputTypes = options_->get>("input-types", {}); + for(int i = 0; i < inputTypes.size(); ++i) + if(inputTypes[i] == "labels") + localMaxes[i] = 1; + size_t maxBatch = 512; bool fits = true; while(fits) { std::vector lengths(numFiles, first); - auto batch = data::CorpusBatch::fakeBatch(lengths, maxBatch, toptions); + for(int j = 0; j < lengths.size(); ++j) // apply length restrictions + lengths[j] = std::min(lengths[j], localMaxes[j]); + + auto batch = data::CorpusBatch::fakeBatch(lengths, maxBatch, options_); auto cost = model->build(graph, batch); fits = graph->fits(); if(fits) maxBatch *= 2; } + // Do a binary search for maxmimum batch size that fits into given workspace memory + // for a tested sentence length. for(size_t i = step; i <= maxLength; i += step) { size_t start = 1; size_t end = maxBatch; std::vector lengths(numFiles, i); + for(int j = 0; j < lengths.size(); ++j) // apply length restrictions + lengths[j] = std::min(lengths[j], localMaxes[j]); fits = true; do { size_t current = (start + end) / 2; - auto batch = data::CorpusBatch::fakeBatch(lengths, current, toptions); + auto batch = data::CorpusBatch::fakeBatch(lengths, current, options_); auto cost = model->build(graph, batch); fits = graph->fits(); diff --git a/src/training/scheduler.h b/src/training/scheduler.h index dee62496d..1585fd8a4 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -10,11 +10,11 @@ namespace marian { class Scheduler : public TrainingObserver { private: Ptr options_; + Ptr state_; std::vector> validators_; bool first_{true}; - - Ptr state_; + int dispIndex_{-1}; timer::Timer timer_, heartBeatTimer_; @@ -46,7 +46,8 @@ class Scheduler : public TrainingObserver { public: Scheduler(Ptr options, Ptr state) - : options_(options), state_(state) { + : options_(options), state_(state), + dispIndex_{options_->get("disp-wps-index", -1)} { state_->eta = getLearningRate(*state); } @@ -168,11 +169,13 @@ class Scheduler : public TrainingObserver { size_t batchSize = 0; // number of sentences in batch size_t batchLabels = 0; // number of target words in batch + size_t batchDisp = 0; // number of words in chosen sub-batch, last by default unless set differently in dispIndex_. Used for displaying speed. for(const auto& batch : batches) { if (batch) { // (nullptr is allowed as result of split) - batchSize += batch->size(); + batchSize += batch->size(); batchLabels += batch->words(-1); + batchDisp += batch->words(dispIndex_); } } @@ -199,7 +202,8 @@ class Scheduler : public TrainingObserver { state_->costSum += cost * batchSize; state_->costCount += batchSize; } - state_->wordsDisp += batchLabels; // target words processed since last display, for speed display + + state_->wordsDisp += batchDisp; // words at given input processed since last display, for speed display state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed From b978ca51b3d6d085ada9bbe482a6a4f9a79cc5e3 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 9 Jan 2019 10:52:11 -0800 Subject: [PATCH 089/838] accuracy validator --- src/models/bert.h | 2 +- src/models/encoder_classifier.h | 1 - src/training/validator.cpp | 3 ++ src/training/validator.h | 88 +++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 7dfc83c90..f0a1f8eec 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -96,7 +96,7 @@ class BertClassifier : public ClassifierBase { ("prefix", prefix_ + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh)) // @TODO: do we actually need this? - .push_back(mlp::output(graph) // + .push_back(mlp::output(graph) // ("dim", dimTrgCls)) // ("prefix", prefix_ + "_ff_logit_l2") // .construct(); diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 9a650311a..0bf048b93 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -197,7 +197,6 @@ class EncoderClassifier : public EncoderClassifierBase { virtual Expr build(Ptr graph, Ptr batch, bool clearGraph = true) override { - ABORT("Don't use this"); auto states = apply(graph, batch, clearGraph); // returns raw logits return states[0]->getLogProbs(); diff --git a/src/training/validator.cpp b/src/training/validator.cpp index a3f55c71b..f63c9d0b5 100644 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -31,6 +31,9 @@ std::vector>> Validators( } else if(metric == "bleu-detok") { auto validator = New(vocabs, config, true); validators.push_back(validator); + } else if(metric == "accuracy") { + auto validator = New(vocabs, config); + validators.push_back(validator); } else { LOG_VALID(warn, "Unrecognized validation metric: {}", metric); } diff --git a/src/training/validator.h b/src/training/validator.h index efed50cb0..1ff6d59ee 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -218,6 +218,94 @@ class CrossEntropyValidator : public Validator { } }; +class AccuracyValidator : public Validator { +public: + AccuracyValidator(std::vector> vocabs, Ptr options) + : Validator(vocabs, options, /*lowerIsBetter=*/false) { + createBatchGenerator(/*isTranslating=*/false); + + // @TODO: check if this is required. + Ptr opts = New(); + opts->merge(options); + opts->set("inference", true); + builder_ = models::from_options(opts, models::usage::raw); + } + + std::string type() override { return "accuracy"; } + +protected: + virtual float validateBG(const std::vector>& graphs) override { + + size_t correct = 0; + size_t totalLabels = 0; + size_t batchId = 0; + + { + threadPool_.reserve(graphs.size()); + + TaskBarrier taskBarrier; + for(auto batch : *batchGenerator_) { + auto task = [=, &correct, &totalLabels](size_t id) { + thread_local Ptr graph; + thread_local auto builder = models::from_options(options_, models::usage::raw); + + if(!graph) { + graph = graphs[id % graphs.size()]; + } + + // Future: requires argmax implementation and integer arithmetics + // builder->clear(graph); + // auto predicted = argmax(builder->build(graph, batch), /*axis*/-1); + // auto labels = graph->indices(batch->back()->data()); + // auto correct = sum(flatten(predicted) == labels); + // graph->forward(); + + // std::unique_lock lock(mutex_); + // totalLabels += labels->shape().elements(); + // correct += correct->scalar(); + + builder->clear(graph); + auto logits = builder->build(graph, batch); + graph->forward(); + + std::vector vLogits; + logits->val()->get(vLogits); + const auto& labels = batch->back()->data(); + + IndexType cols = logits->shape()[-1]; + + size_t thisCorrect = 0; + size_t thisLabels = labels.size(); + + for(int i = 0; i < thisLabels; ++i) { + // CPU-side Argmax + IndexType bestIndex = 0; + float bestValue = std::numeric_limits::lowest(); + for(IndexType j = 0; j < cols; ++j) { + float currValue = vLogits[i * cols + j]; + if(currValue > bestValue) { + bestValue = currValue; + bestIndex = j; + } + } + thisCorrect += (size_t)(bestIndex == labels[i]); + } + + std::unique_lock lock(mutex_); + totalLabels += thisLabels; + correct += thisCorrect; + }; + + taskBarrier.push_back(threadPool_.enqueue(task, batchId)); + batchId++; + } + // ~TaskBarrier waits until all are done + } + + return (float)correct / (float)totalLabels; + } +}; + class ScriptValidator : public Validator { public: ScriptValidator(std::vector> vocabs, Ptr options) From c077d2cf163298064567515b6fb5fed94895a187 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 13:47:40 -0800 Subject: [PATCH 090/838] towards turning embedding into a more regular layer --- src/layers/factory.h | 0 src/layers/generic.h | 66 ++++++++++++++++++++++++++++++++-------- src/models/decoder.h | 38 +++++++++-------------- src/models/encoder.h | 22 ++------------ src/models/s2s.h | 7 ++--- src/models/transformer.h | 9 +++--- 6 files changed, 77 insertions(+), 65 deletions(-) mode change 100644 => 100755 src/layers/factory.h diff --git a/src/layers/factory.h b/src/layers/factory.h old mode 100644 new mode 100755 diff --git a/src/layers/generic.h b/src/layers/generic.h index b5c53b462..bc3930aad 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -17,15 +17,14 @@ enum struct act : int { linear, tanh, sigmoid, ReLU, LeakyReLU, PReLU, swish }; YAML_REGISTER_TYPE(marian::mlp::act, int) namespace marian { -namespace mlp { -class Layer { +class BaseLayer { // @TODO: better name. Ideally change uses of Layer to IUnaryLayer; then explicit derive from IUnaryLayer in each layer protected: Ptr graph_; Ptr options_; public: - Layer(Ptr graph, Ptr options) + BaseLayer(Ptr graph, Ptr options) : graph_(graph), options_(options) {} template @@ -37,11 +36,19 @@ class Layer { T opt(const std::string key, T defaultValue) { return options_->get(key, defaultValue); } +}; +namespace mlp { +struct IUnaryLayer { virtual Expr apply(const std::vector&) = 0; virtual Expr apply(Expr) = 0; }; +class Layer : public BaseLayer, public IUnaryLayer { +public: + Layer(Ptr graph, Ptr options) : BaseLayer(graph, options) { } +}; + class Dense : public Layer { public: Dense(Ptr graph, Ptr options) @@ -162,10 +169,16 @@ class Output : public Layer { } // namespace mlp -struct EmbeddingFactory : public Factory { - EmbeddingFactory(Ptr graph) : Factory(graph) {} +struct IEmbedding { + virtual std::tuple apply(Ptr subBatch) const = 0; + // alternative version from index vector, and with batch dim + virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; +}; - Expr construct() { +class Embedding : public BaseLayer, public IEmbedding { + Expr E_; +public: + Embedding(Ptr graph, Ptr options) : BaseLayer(graph, options) { std::string name = opt("prefix"); int dimVoc = opt("dimVocab"); int dimEmb = opt("dimEmb"); @@ -173,18 +186,45 @@ struct EmbeddingFactory : public Factory { bool fixed = opt("fixed", false); NodeInitializer initFunc = inits::glorot_uniform; - if (options_->has("embFile")) { - std::string file = opt("embFile"); - if (!file.empty()) { - bool norm = opt("normalization", false); - initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm); + if (options_->has("embFile")) { + std::string file = opt("embFile"); + if (!file.empty()) { + bool norm = opt("normalization", false); + initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm); + } } + + E_ = graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed); } - - return graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed); + + std::tuple apply(Ptr subBatch) const override final { + auto graph = E_->graph(); + int dimBatch = (int)subBatch->batchSize(); + int dimEmb = E_->shape()[-1]; + int dimWords = (int)subBatch->batchWidth(); + // @TODO: merge this with below. Currently can't only due to the extra beam dimension + auto chosenEmbeddings = rows(E_, subBatch->data()); + auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb }); + auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, + inits::from_vector(subBatch->mask())); + return std::make_tuple(batchEmbeddings, batchMask); + } + + // special version used in decoding + Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const override final { + int dimEmb = E_->shape()[-1]; + auto selectedEmbs = rows(E_, embIdx); + return reshape(selectedEmbs, { dimBeam, 1, dimBatch, dimEmb }); } }; +struct EmbeddingFactory : public Factory { + EmbeddingFactory(Ptr graph) : Factory(graph) {} + + Ptr construct() { + return New(graph_, options_); + } +}; struct ULREmbeddingFactory : public Factory { ULREmbeddingFactory(Ptr graph) : Factory(graph) {} diff --git a/src/models/decoder.h b/src/models/decoder.h index 39e56e1f3..67ab8a214 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -59,16 +59,9 @@ class DecoderBase { auto yEmb = yEmbFactory.construct(); auto subBatch = (*batch)[batchIndex_]; - int dimBatch = (int)subBatch->batchSize(); - int dimWords = (int)subBatch->batchWidth(); - auto chosenEmbeddings = rows(yEmb, subBatch->data()); - - auto y - = reshape(chosenEmbeddings, {dimWords, dimBatch, opt("dim-emb")}); - - auto yMask = graph->constant({dimWords, dimBatch, 1}, - inits::from_vector(subBatch->mask())); + Expr y, yMask; std::tie + (y, yMask) = yEmb->apply(subBatch); Expr yData; if(shortlist_) { @@ -92,24 +85,23 @@ class DecoderBase { int dimTrgEmb = opt("dim-emb"); int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - // embeddings are loaded from model during translation, no fixing required - auto yEmbFactory = embedding(graph) // - ("dimVocab", dimTrgVoc) // - ("dimEmb", dimTrgEmb); - - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - yEmbFactory("prefix", "Wemb"); - else - yEmbFactory("prefix", prefix_ + "_Wemb"); - - auto yEmb = yEmbFactory.construct(); - Expr selectedEmbs; if(embIdx.empty()) { selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); } else { - selectedEmbs = rows(yEmb, embIdx); - selectedEmbs = reshape(selectedEmbs, {dimBeam, 1, dimBatch, dimTrgEmb}); + // embeddings are loaded from model during translation, no fixing required + auto yEmbFactory = embedding(graph) // + ("dimVocab", dimTrgVoc) // + ("dimEmb", dimTrgEmb); + + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + yEmbFactory("prefix", "Wemb"); + else + yEmbFactory("prefix", prefix_ + "_Wemb"); + + auto yEmb = yEmbFactory.construct(); + + selectedEmbs = yEmb->apply(embIdx, dimBatch, dimBeam); } state->setTargetEmbeddings(selectedEmbs); } diff --git a/src/models/encoder.h b/src/models/encoder.h index 6d1ee852e..f8f16ed11 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -12,26 +12,8 @@ class EncoderBase { bool inference_{false}; size_t batchIndex_{0}; - // @TODO: This used to be virtual, but is never overridden. - // virtual - std::tuple lookup(Ptr graph, - Expr srcEmbeddings, - Ptr batch) const { - auto subBatch = (*batch)[batchIndex_]; - int dimBatch = (int)subBatch->batchSize(); - int dimEmb = srcEmbeddings->shape()[-1]; - int dimWords = (int)subBatch->batchWidth(); - auto chosenEmbeddings = rows(srcEmbeddings, subBatch->data()); - auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb }); - auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, - inits::from_vector(subBatch->mask())); - - return std::make_tuple(batchEmbeddings, batchMask); - } - - std::tuple ulrLookup(Ptr graph, - std::vector urlEmbeddings, - Ptr batch) const { + std::tuple ulrLookup(std::vector urlEmbeddings, Ptr batch) const { + auto graph = urlEmbeddings.front()->graph(); auto subBatch = (*batch)[batchIndex_]; // is their a better way to do this? assert(urlEmbeddings.size() == 6); diff --git a/src/models/s2s.h b/src/models/s2s.h index b4600892f..de3754057 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -119,7 +119,7 @@ class EncoderS2S : public EncoderBase { return context; } - Expr buildSourceEmbeddings(Ptr graph) { + Ptr buildSourceEmbeddings(Ptr graph) { // create source embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); @@ -153,9 +153,8 @@ class EncoderS2S : public EncoderBase { auto embeddings = buildSourceEmbeddings(graph); // select embeddings that occur in the batch - Expr batchEmbeddings, batchMask; - std::tie(batchEmbeddings, batchMask) - = EncoderBase::lookup(graph, embeddings, batch); + Expr batchEmbeddings, batchMask; std::tie + (batchEmbeddings, batchMask) = embeddings->apply((*batch)[batchIndex_]); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); diff --git a/src/models/transformer.h b/src/models/transformer.h index c30d06c8e..523add21a 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -495,7 +495,7 @@ class EncoderTransformer : public Transformer { return embFactory.construct(); } - Expr wordEmbeddings(size_t subBatchIndex) const { + Ptr createWordEmbeddingLayer(size_t subBatchIndex) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); @@ -530,13 +530,12 @@ class EncoderTransformer : public Transformer { if (options_->has("ulr") && options_->get("ulr") == true) { auto embeddings = ULREmbeddings(); // embedding uses ULR std::tie(batchEmbeddings, batchMask) - = EncoderBase::ulrLookup(graph_, embeddings, batch); + = EncoderBase::ulrLookup(embeddings, batch); } else { - auto embeddings = wordEmbeddings(batchIndex_); - std::tie(batchEmbeddings, batchMask) - = EncoderBase::lookup(graph_, embeddings, batch); + auto embedding = createWordEmbeddingLayer(batchIndex_); + std::tie(batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); } // apply dropout over source words float dropoutSrc = inference_ ? 0 : opt("dropout-src"); From b2e3427924cc58665cbd01261f720a0155a8aef5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 13:54:18 -0800 Subject: [PATCH 091/838] Ptr changed to interface Ptr --- src/layers/constructors.h | 10 +++++----- src/layers/generic.h | 24 +++++++++++------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index 16765341f..cf51ce55f 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -26,7 +26,7 @@ struct LayerFactory : public Factory { return as() != nullptr; } - virtual Ptr construct() = 0; + virtual Ptr construct() = 0; }; /** @@ -36,7 +36,7 @@ class DenseFactory : public LayerFactory { public: DenseFactory(Ptr graph) : LayerFactory(graph) {} - Ptr construct() override { + Ptr construct() override { auto dense = New(graph_, options_); return dense; } @@ -73,7 +73,7 @@ class OutputFactory : public LayerFactory { return Accumulator(*this); } - Ptr construct() override { + Ptr construct() override { auto output = New(graph_, options_); for(auto& p : tiedParamsTransposed_) output->tie_transposed(p.first, p.second); @@ -101,7 +101,7 @@ class MLP { Ptr graph_; Ptr options_; - std::vector> layers_; + std::vector> layers_; public: MLP(Ptr graph, Ptr options) @@ -123,7 +123,7 @@ class MLP { return output; } - void push_back(Ptr layer) { layers_.push_back(layer); } + void push_back(Ptr layer) { layers_.push_back(layer); } }; /** diff --git a/src/layers/generic.h b/src/layers/generic.h index bc3930aad..c4e2858eb 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -18,13 +18,15 @@ YAML_REGISTER_TYPE(marian::mlp::act, int) namespace marian { -class BaseLayer { // @TODO: better name. Ideally change uses of Layer to IUnaryLayer; then explicit derive from IUnaryLayer in each layer +// Each layer consists of Layer and one or more apply() functions according to +// a layer-type specific interface (different layers may require different signatures). +class Layer { protected: Ptr graph_; Ptr options_; public: - BaseLayer(Ptr graph, Ptr options) + Layer(Ptr graph, Ptr options) : graph_(graph), options_(options) {} template @@ -38,18 +40,14 @@ class BaseLayer { // @TODO: better name. Ideally change uses of Layer to IUnar } }; -namespace mlp { +// Simplest layer interface: Unary function. struct IUnaryLayer { - virtual Expr apply(const std::vector&) = 0; virtual Expr apply(Expr) = 0; + virtual Expr apply(const std::vector&) = 0; }; -class Layer : public BaseLayer, public IUnaryLayer { -public: - Layer(Ptr graph, Ptr options) : BaseLayer(graph, options) { } -}; - -class Dense : public Layer { +namespace mlp { +class Dense : public Layer, public IUnaryLayer { public: Dense(Ptr graph, Ptr options) : Layer(graph, options) {} @@ -116,7 +114,7 @@ class Dense : public Layer { Expr apply(Expr input) override { return apply(std::vector({input})); } }; -class Output : public Layer { +class Output : public Layer, public IUnaryLayer { private: std::map tiedParams_; Ptr shortlist_; @@ -175,10 +173,10 @@ struct IEmbedding { virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; }; -class Embedding : public BaseLayer, public IEmbedding { +class Embedding : public Layer, public IEmbedding { Expr E_; public: - Embedding(Ptr graph, Ptr options) : BaseLayer(graph, options) { + Embedding(Ptr graph, Ptr options) : Layer(graph, options) { std::string name = opt("prefix"); int dimVoc = opt("dimVocab"); int dimEmb = opt("dimEmb"); From ce1e3093e8161babc82a9792758cb7fc03057683 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 14:24:15 -0800 Subject: [PATCH 092/838] disentangled embedding layers and make them look like other layers --- src/layers/generic.h | 130 +++++++++++++++++++++++++++------------ src/models/encoder.h | 46 +------------- src/models/s2s.h | 2 +- src/models/transformer.h | 18 +++--- 4 files changed, 100 insertions(+), 96 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index c4e2858eb..32ea291cc 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -18,15 +18,16 @@ YAML_REGISTER_TYPE(marian::mlp::act, int) namespace marian { -// Each layer consists of Layer and one or more apply() functions according to -// a layer-type specific interface (different layers may require different signatures). -class Layer { +// Each layer consists of LayerBase and IXXXLayer which defines one or more apply() +// functions for the respective layer type (different layers may require different signatures). +// This base class contains configuration info for creating parameters and executing apply(). +class LayerBase { protected: Ptr graph_; Ptr options_; public: - Layer(Ptr graph, Ptr options) + LayerBase(Ptr graph, Ptr options) : graph_(graph), options_(options) {} template @@ -40,17 +41,24 @@ class Layer { } }; -// Simplest layer interface: Unary function. +// Simplest layer interface: Unary function struct IUnaryLayer { virtual Expr apply(Expr) = 0; virtual Expr apply(const std::vector&) = 0; }; +// Embedding from corpus sub-batch to (emb, mask) +struct IEmbeddingLayer { + virtual std::tuple apply(Ptr subBatch) const = 0; + // alternative version from index vector, and with batch dim + virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; +}; + namespace mlp { -class Dense : public Layer, public IUnaryLayer { +class Dense : public LayerBase, public IUnaryLayer { public: Dense(Ptr graph, Ptr options) - : Layer(graph, options) {} + : LayerBase(graph, options) {} Expr apply(const std::vector& inputs) override { ABORT_IF(inputs.empty(), "No inputs"); @@ -114,7 +122,7 @@ class Dense : public Layer, public IUnaryLayer { Expr apply(Expr input) override { return apply(std::vector({input})); } }; -class Output : public Layer, public IUnaryLayer { +class Output : public LayerBase, public IUnaryLayer { private: std::map tiedParams_; Ptr shortlist_; @@ -125,7 +133,7 @@ class Output : public Layer, public IUnaryLayer { public: Output(Ptr graph, Ptr options) - : Layer(graph, options) {} + : LayerBase(graph, options) {} void tie_transposed(const std::string& param, const std::string& tied) { tiedParams_[param] = graph_->get(tied); @@ -167,16 +175,10 @@ class Output : public Layer, public IUnaryLayer { } // namespace mlp -struct IEmbedding { - virtual std::tuple apply(Ptr subBatch) const = 0; - // alternative version from index vector, and with batch dim - virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; -}; - -class Embedding : public Layer, public IEmbedding { +class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; public: - Embedding(Ptr graph, Ptr options) : Layer(graph, options) { + Embedding(Ptr graph, Ptr options) : LayerBase(graph, options) { std::string name = opt("prefix"); int dimVoc = opt("dimVocab"); int dimEmb = opt("dimEmb"); @@ -216,25 +218,16 @@ class Embedding : public Layer, public IEmbedding { } }; -struct EmbeddingFactory : public Factory { - EmbeddingFactory(Ptr graph) : Factory(graph) {} - - Ptr construct() { - return New(graph_, options_); - } -}; - -struct ULREmbeddingFactory : public Factory { -ULREmbeddingFactory(Ptr graph) : Factory(graph) {} - - std::vector construct() { +class ULREmbedding : public LayerBase, public IEmbeddingLayer { + std::vector ulrEmbeddings_; // @TODO: These can now better be written as 6 named class members +public: + ULREmbedding(Ptr graph, Ptr options) : LayerBase(graph, options) { std::string name = "url_embed"; //opt("prefix"); int dimKeys = opt("dimTgtVoc"); int dimQueries = opt("dimSrcVoc"); int dimEmb = opt("dimEmb"); int dimUlrEmb = opt("dimUlrEmb"); // ULR mono embed size bool fixed = opt("fixed", false); - std::vector ulrEmbeds; NodeInitializer initFunc = inits::glorot_uniform; std::string queryFile = opt("ulrQueryFile"); std::string keyFile = opt("ulrKeysFile"); @@ -244,23 +237,23 @@ ULREmbeddingFactory(Ptr graph) : Factory(graph) {} name = "ulr_query"; fixed = true; auto query_embed = graph_->param(name, { dimQueries, dimUlrEmb }, initFunc, fixed); - ulrEmbeds.push_back(query_embed); + ulrEmbeddings_.push_back(query_embed); // keys embeds initFunc = inits::from_word2vec(keyFile, dimKeys, dimUlrEmb, false); name = "ulr_keys"; fixed = true; auto key_embed = graph_->param(name, { dimKeys, dimUlrEmb }, initFunc, fixed); - ulrEmbeds.push_back(key_embed); + ulrEmbeddings_.push_back(key_embed); // actual trainable embedding initFunc = inits::glorot_uniform; name = "ulr_embed"; fixed = false; auto ulr_embed = graph_->param(name, {dimKeys , dimEmb }, initFunc, fixed); // note the reverse dim - ulrEmbeds.push_back(ulr_embed); + ulrEmbeddings_.push_back(ulr_embed); // init trainable src embedding name = "ulr_src_embed"; auto ulr_src_embed = graph_->param(name, { dimQueries, dimEmb }, initFunc, fixed); - ulrEmbeds.push_back(ulr_src_embed); + ulrEmbeddings_.push_back(ulr_src_embed); // ulr transformation matrix //initFunc = inits::eye(1.f); // identity matrix - is it ok to init wiht identity or shall we make this to the fixed case only if (trainTrans) { @@ -273,18 +266,77 @@ ULREmbeddingFactory(Ptr graph) : Factory(graph) {} fixed = true; } name = "ulr_transform"; - auto ulr_transform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed); - ulrEmbeds.push_back(ulr_transform); + auto ulrTransform = graph_->param(name, { dimUlrEmb, dimUlrEmb }, initFunc, fixed); + ulrEmbeddings_.push_back(ulrTransform); initFunc = inits::from_value(1.f); // TBD: we should read sharable flags here - 1 means all sharable - 0 means no universal embeddings - should be zero for top freq only fixed = true; name = "ulr_shared"; auto share_embed = graph_->param(name, { dimQueries, 1 }, initFunc, fixed); - ulrEmbeds.push_back(share_embed); - + ulrEmbeddings_.push_back(share_embed); } + } + + std::tuple apply(Ptr subBatch) const override final { + auto queryEmbed = ulrEmbeddings_[0]; // Q : dimQueries*dimUlrEmb + auto keyEmbed = ulrEmbeddings_[1]; // K : dimKeys*dimUlrEmb + auto uniEmbed = ulrEmbeddings_[2]; // E : dimQueries*dimEmb + auto srcEmbed = ulrEmbeddings_[3]; // I : dimQueries*dimEmb + auto ulrTransform = ulrEmbeddings_[4]; // A : dimUlrEmb *dimUlrEmb + auto ulrSharable = ulrEmbeddings_[5]; // alpha : dimQueries*1 + int dimBatch = (int)subBatch->batchSize(); + int dimEmb = uniEmbed->shape()[-1]; + int dimWords = (int)subBatch->batchWidth(); + // D = K.A.QT + // dimm(K) = univ_tok_vocab*uni_embed_size + // dim A = uni_embed_size*uni_embed_size + // dim Q: uni_embed_size * total_merged_vocab_size + // dim D = univ_tok_vocab * total_merged_vocab_size + // note all above can be precombuted and serialized if A is not trainiable and during decoding (TBD) + // here we need to handle the mini-batch + // extract raws corresponding to Xs in this minibatch from Q + auto queryEmbeddings = rows(queryEmbed, subBatch->data()); + auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings + auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags + auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb*dimUlrEmb + auto sqrtDim=std::sqrt((float)queryEmbeddings->shape()[-1]); + qt = qt/sqrtDim; // normalize accordin to embed size to avoid dot prodcut growing large in magnitude with larger embeds sizes + auto z = dot(qt, keyEmbed, false, true); // query-key similarity + float dropProb = this->options_->get("ulr-dropout", 0.0f); // default no dropout + z = dropout(z, dropProb); + float tau = this->options_->get("ulr-softmax-temperature", 1.0f); // default no temperature + // temperature in softmax is to control randomness of predictions + // high temperature Softmax outputs are more close to each other + // low temperatures the softmax become more similar to "hardmax" + auto weights = softmax(z / tau); // assume default is dim=-1, what about temprature? - scaler ?? + auto chosenEmbeddings = dot(weights, uniEmbed); // AVERAGE + auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast + auto batchEmbeddings = reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb }); + auto graph = ulrEmbeddings_.front()->graph(); + auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, + inits::from_vector(subBatch->mask())); + return std::make_tuple(batchEmbeddings, batchMask); + } + + Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const override final { + embIdx; dimBatch; dimBeam; + ABORT("not implemented"); // ULR cannot be used for decoding + } +}; + +struct EmbeddingFactory : public Factory { + EmbeddingFactory(Ptr graph) : Factory(graph) {} + + Ptr construct() { + return New(graph_, options_); + } +}; + +struct ULREmbeddingFactory : public Factory { + ULREmbeddingFactory(Ptr graph) : Factory(graph) {} - return ulrEmbeds; + Ptr construct() { + return New(graph_, options_); } }; diff --git a/src/models/encoder.h b/src/models/encoder.h index f8f16ed11..8eab61fb4 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -11,50 +11,6 @@ class EncoderBase { std::string prefix_{"encoder"}; bool inference_{false}; size_t batchIndex_{0}; - - std::tuple ulrLookup(std::vector urlEmbeddings, Ptr batch) const { - auto graph = urlEmbeddings.front()->graph(); - auto subBatch = (*batch)[batchIndex_]; - // is their a better way to do this? - assert(urlEmbeddings.size() == 6); - auto queryEmbed = urlEmbeddings[0]; //Q : dimQueries*dimUlrEmb - auto keyEmbed = urlEmbeddings[1]; // K : dimKeys*dimUlrEmb - auto uniEmbed = urlEmbeddings[2]; // E : dimQueries*dimEmb - auto srcEmbed = urlEmbeddings[3]; // I : dimQueries*dimEmb - auto ulrTransform = urlEmbeddings[4]; //A : dimUlrEmb *dimUlrEmb - auto ulrSharable = urlEmbeddings[5]; //alpha : dimQueries*1 - int dimBatch = (int)subBatch->batchSize(); - int dimEmb = uniEmbed->shape()[-1]; - int dimWords = (int)subBatch->batchWidth(); - // D = K.A.QT - // dimm(K) = univ_tok_vocab*uni_embed_size - // dim A = uni_embed_size*uni_embed_size - // dim Q: uni_embed_size * total_merged_vocab_size - // dim D = univ_tok_vocab * total_merged_vocab_size - // note all above can be precombuted and serialized if A is not trainiabale and during decoding (TBD) - // here we need to handle the mini-batch - // extract raws corresponding to Xs in this mini batch from Q - auto queryEmbeddings = rows(queryEmbed, subBatch->data()); - auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings - auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags - auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb *dimUlrEmb - auto sqrtDim=std::sqrt((float)queryEmbeddings->shape()[-1]); - qt = qt/sqrtDim; // normalize accordin to embed size to avoid dot prodcut growing large in magintude with larger embeds sizes - auto z = dot(qt, keyEmbed, false, true); // query-key similarity - float dropProb = this->options_->get("ulr-dropout", 0.0f); // default no dropout - z = dropout(z, dropProb); - float tau = this->options_->get("ulr-softmax-temperature", 1.0f); // default no temperature - // temperature in softmax is to control randomness of predictions - // high temperature Softmax outputs are more close to each other - // low temperatures the softmax become more similar to "hardmax" - auto weights = softmax(z / tau); // assume default is dim=-1, what about temprature? - scaler ?? - auto chosenEmbeddings = dot(weights, uniEmbed); // AVERAGE - auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast - auto batchEmbeddings = reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb }); - auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, - inits::from_vector(subBatch->mask())); - return std::make_tuple(batchEmbeddings, batchMask); - } public: EncoderBase(Ptr options) : options_(options), @@ -62,7 +18,7 @@ class EncoderBase { inference_(options->get("inference", false)), batchIndex_(options->get("index", 0)) {} - virtual Ptr build(Ptr, Ptr) + virtual Ptr build(Ptr, Ptr) // @TODO: rename to apply()? = 0; template diff --git a/src/models/s2s.h b/src/models/s2s.h index de3754057..fe746877d 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -119,7 +119,7 @@ class EncoderS2S : public EncoderBase { return context; } - Ptr buildSourceEmbeddings(Ptr graph) { + Ptr buildSourceEmbeddings(Ptr graph) { // create source embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); diff --git a/src/models/transformer.h b/src/models/transformer.h index 523add21a..3292cabfe 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -481,7 +481,7 @@ class EncoderTransformer : public Transformer { // returns the embedding matrix based on options // and based on batchIndex_. - std::vector ULREmbeddings() const { + Ptr createULREmbeddingLayer() const { // standard encoder word embeddings int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt @@ -495,7 +495,7 @@ class EncoderTransformer : public Transformer { return embFactory.construct(); } - Ptr createWordEmbeddingLayer(size_t subBatchIndex) const { + Ptr createWordEmbeddingLayer(size_t subBatchIndex) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); @@ -527,16 +527,12 @@ class EncoderTransformer : public Transformer { // create the embedding matrix, considering tying and some other options // embed the source words in the batch Expr batchEmbeddings, batchMask; - if (options_->has("ulr") && options_->get("ulr") == true) { - auto embeddings = ULREmbeddings(); // embedding uses ULR - std::tie(batchEmbeddings, batchMask) - = EncoderBase::ulrLookup(embeddings, batch); - } + Ptr embedding; + if (options_->has("ulr") && options_->get("ulr") == true) + embedding = createULREmbeddingLayer(); // embedding uses ULR else - { - auto embedding = createWordEmbeddingLayer(batchIndex_); - std::tie(batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); - } + embedding = createWordEmbeddingLayer(batchIndex_); + std::tie(batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); // apply dropout over source words float dropoutSrc = inference_ ? 0 : opt("dropout-src"); if(dropoutSrc) { From 2e86b2250ba45d78df14c185a42014c465a34c4f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 14:27:14 -0800 Subject: [PATCH 093/838] updated CharS2S --- src/models/char_s2s.h | 7 +++---- src/models/s2s.h | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) mode change 100644 => 100755 src/models/char_s2s.h diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h old mode 100644 new mode 100755 index 6d5d1db16..928c6a18d --- a/src/models/char_s2s.h +++ b/src/models/char_s2s.h @@ -13,12 +13,11 @@ class CharS2SEncoder : public EncoderS2S { virtual Ptr build(Ptr graph, Ptr batch) override { - auto embeddings = buildSourceEmbeddings(graph); + auto embedding = createSourceEmbedding(graph); // select embeddings that occur in the batch - Expr batchEmbeddings, batchMask; - std::tie(batchEmbeddings, batchMask) - = EncoderBase::lookup(graph, embeddings, batch); + Expr batchEmbeddings, batchMask; std::tie + (batchEmbeddings, batchMask) = embedding->apply(batch->front()); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); diff --git a/src/models/s2s.h b/src/models/s2s.h index fe746877d..0522176fd 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -150,11 +150,11 @@ class EncoderS2S : public EncoderBase { virtual Ptr build(Ptr graph, Ptr batch) override { - auto embeddings = buildSourceEmbeddings(graph); + auto embedding = buildSourceEmbeddings(graph); // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = embeddings->apply((*batch)[batchIndex_]); + (batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); From e2ec8a68040326117a45c611580759c33e32629b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 14:28:11 -0800 Subject: [PATCH 094/838] updated CharS2S --- src/models/s2s.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/s2s.h b/src/models/s2s.h index 0522176fd..2ee090d6f 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -119,7 +119,7 @@ class EncoderS2S : public EncoderBase { return context; } - Ptr buildSourceEmbeddings(Ptr graph) { + Ptr createSourceEmbedding(Ptr graph) { // create source embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); @@ -150,7 +150,7 @@ class EncoderS2S : public EncoderBase { virtual Ptr build(Ptr graph, Ptr batch) override { - auto embedding = buildSourceEmbeddings(graph); + auto embedding = createSourceEmbedding(graph); // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie From 777045e6081b1c4adba908e5fa980cb7b76bdfd4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 15:09:18 -0800 Subject: [PATCH 095/838] removed graph_ from Factory --- src/graph/expression_operators.cpp | 7 +++--- src/layers/constructors.h | 20 +++++++-------- src/layers/factory.h | 4 +-- src/layers/generic.h | 8 +++--- src/models/decoder.h | 4 +-- src/models/model_factory.cpp | 29 +++++++++++----------- src/models/model_factory.h | 6 ++--- src/models/s2s.h | 17 +++++++------ src/models/transformer.h | 8 +++--- src/rnn/attention_constructors.h | 4 +-- src/rnn/constructors.h | 40 +++++++++++++++--------------- 11 files changed, 75 insertions(+), 72 deletions(-) mode change 100644 => 100755 src/models/model_factory.cpp mode change 100644 => 100755 src/models/model_factory.h mode change 100644 => 100755 src/rnn/attention_constructors.h diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index c1b298f8d..cbfd8054b 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -511,16 +511,17 @@ Expr highway(Expr y, Expr x, Expr t) { Expr highway(const std::string prefix, Expr x) { // clang-format off size_t outDim = x->shape()[-1]; - auto g = mlp::dense(x->graph()) + auto graph = x->graph(); + auto g = mlp::dense(graph) ("prefix", prefix + "_highway_d1") ("dim", outDim) ("activation", mlp::act::sigmoid) - .construct()->apply(x); + .construct(graph)->apply(x); auto relued = mlp::dense(x->graph()) ("prefix", prefix + "_highway_d2") ("dim", outDim) ("activation", mlp::act::ReLU) - .construct()->apply(x); + .construct(graph)->apply(x); return (g * relued) + ((1 - g) * x); // clang-format on } diff --git a/src/layers/constructors.h b/src/layers/constructors.h index cf51ce55f..f5d6aef6b 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -26,7 +26,7 @@ struct LayerFactory : public Factory { return as() != nullptr; } - virtual Ptr construct() = 0; + virtual Ptr construct(Ptr graph) = 0; }; /** @@ -36,13 +36,13 @@ class DenseFactory : public LayerFactory { public: DenseFactory(Ptr graph) : LayerFactory(graph) {} - Ptr construct() override { - auto dense = New(graph_, options_); + Ptr construct(Ptr graph) override { + auto dense = New(graph, options_); return dense; } DenseFactory clone() { - DenseFactory aClone(graph_); + DenseFactory aClone(nullptr); aClone.options_->merge(options_); return aClone; } @@ -73,8 +73,8 @@ class OutputFactory : public LayerFactory { return Accumulator(*this); } - Ptr construct() override { - auto output = New(graph_, options_); + Ptr construct(Ptr graph) override { + auto output = New(graph, options_); for(auto& p : tiedParamsTransposed_) output->tie_transposed(p.first, p.second); output->set_shortlist(shortlist_); @@ -82,7 +82,7 @@ class OutputFactory : public LayerFactory { } OutputFactory clone() { - OutputFactory aClone(graph_); + OutputFactory aClone(nullptr); aClone.options_->merge(options_); aClone.tiedParamsTransposed_ = tiedParamsTransposed_; aClone.shortlist_ = shortlist_; @@ -137,11 +137,11 @@ class MLPFactory : public Factory { public: MLPFactory(Ptr graph) : Factory(graph) {} - Ptr construct() { - auto mlp = New(graph_, options_); + Ptr construct(Ptr graph) { + auto mlp = New(graph, options_); for(auto layer : layers_) { layer->getOptions()->merge(options_); - mlp->push_back(layer->construct()); + mlp->push_back(layer->construct(graph)); } return mlp; } diff --git a/src/layers/factory.h b/src/layers/factory.h index 0e84fd168..6ebc63fe3 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -7,11 +7,11 @@ namespace marian { class Factory : public std::enable_shared_from_this { protected: Ptr options_; - Ptr graph_; + //Ptr graph_; public: Factory(Ptr graph) - : options_(New()), graph_(graph) {} + : options_(New())/*, graph_(graph)*/ {} virtual ~Factory() {} diff --git a/src/layers/generic.h b/src/layers/generic.h index 32ea291cc..c7a5c3bed 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -327,16 +327,16 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { struct EmbeddingFactory : public Factory { EmbeddingFactory(Ptr graph) : Factory(graph) {} - Ptr construct() { - return New(graph_, options_); + Ptr construct(Ptr graph) { + return New(graph, options_); } }; struct ULREmbeddingFactory : public Factory { ULREmbeddingFactory(Ptr graph) : Factory(graph) {} - Ptr construct() { - return New(graph_, options_); + Ptr construct(Ptr graph) { + return New(graph, options_); } }; diff --git a/src/models/decoder.h b/src/models/decoder.h index 67ab8a214..b982b84a3 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -56,7 +56,7 @@ class DecoderBase { ("normalization", opt("embedding-normalization")); } - auto yEmb = yEmbFactory.construct(); + auto yEmb = yEmbFactory.construct(graph); auto subBatch = (*batch)[batchIndex_]; @@ -99,7 +99,7 @@ class DecoderBase { else yEmbFactory("prefix", prefix_ + "_Wemb"); - auto yEmb = yEmbFactory.construct(); + auto yEmb = yEmbFactory.construct(graph); selectedEmbs = yEmb->apply(embIdx, dimBatch, dimBeam); } diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp old mode 100644 new mode 100755 index d42f07c8b..8198f46c0 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -24,7 +24,7 @@ namespace marian { namespace models { -Ptr EncoderFactory::construct() { +Ptr EncoderFactory::construct(Ptr graph) { if(options_->get("type") == "s2s") return New(options_); @@ -40,7 +40,7 @@ Ptr EncoderFactory::construct() { ABORT("Unknown encoder type"); } -Ptr DecoderFactory::construct() { +Ptr DecoderFactory::construct(Ptr graph) { if(options_->get("type") == "s2s") return New(options_); if(options_->get("type") == "transformer") @@ -49,7 +49,7 @@ Ptr DecoderFactory::construct() { ABORT("Unknown decoder type"); } -Ptr EncoderDecoderFactory::construct() { +Ptr EncoderDecoderFactory::construct(Ptr graph) { Ptr encdec; if(options_->get("type") == "amun") @@ -61,15 +61,16 @@ Ptr EncoderDecoderFactory::construct() { encdec = New(options_); for(auto& ef : encoders_) - encdec->push_back(ef(options_).construct()); + encdec->push_back(ef(options_).construct(graph)); for(auto& df : decoders_) - encdec->push_back(df(options_).construct()); + encdec->push_back(df(options_).construct(graph)); return add_cost(encdec, options_); } Ptr by_type(std::string type, usage use, Ptr options) { + Ptr graph = nullptr; // graph unknown at this stage // clang-format off if(type == "s2s" || type == "amun" || type == "nematus") { return models::encoder_decoder()(options) @@ -77,7 +78,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("original-type", type) .push_back(models::encoder()("type", "s2s")) .push_back(models::decoder()("type", "s2s")) - .construct(); + .construct(graph); } if(type == "transformer") { @@ -85,7 +86,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("usage", use) .push_back(models::encoder()("type", "transformer")) .push_back(models::decoder()("type", "transformer")) - .construct(); + .construct(graph); } if(type == "transformer_s2s") { @@ -94,7 +95,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("original-type", type) .push_back(models::encoder()("type", "transformer")) .push_back(models::decoder()("type", "s2s")) - .construct(); + .construct(graph); } if(type == "lm") { @@ -111,7 +112,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { .push_back(models::decoder() ("index", idx) ("dim-vocabs", dimVocabs)) - .construct(); + .construct(graph); } if(type == "multi-s2s") { @@ -128,7 +129,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ms2sFactory.push_back(models::decoder()("index", numEncoders)); - return ms2sFactory.construct(); + return ms2sFactory.construct(graph); } if(type == "shared-multi-s2s") { @@ -145,7 +146,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ms2sFactory.push_back(models::decoder()("index", numEncoders)); - return ms2sFactory.construct(); + return ms2sFactory.construct(graph); } if(type == "multi-transformer") { @@ -161,7 +162,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { } mtransFactory.push_back(models::decoder()("index", numEncoders)); - return mtransFactory.construct(); + return mtransFactory.construct(graph); } if(type == "shared-multi-transformer") { @@ -177,7 +178,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { } mtransFactory.push_back(models::decoder()("index", numEncoders)); - return mtransFactory.construct(); + return mtransFactory.construct(graph); } if(type == "lm-transformer") { @@ -194,7 +195,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { .push_back(models::decoder() ("index", idx) ("dim-vocabs", dimVocabs)) - .construct(); + .construct(graph); } #ifdef COMPILE_EXAMPLES diff --git a/src/models/model_factory.h b/src/models/model_factory.h old mode 100644 new mode 100755 index 2ec7fe752..724561cc7 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -12,7 +12,7 @@ class EncoderFactory : public Factory { public: EncoderFactory(Ptr graph = nullptr) : Factory(graph) {} - virtual Ptr construct(); + virtual Ptr construct(Ptr graph); }; typedef Accumulator encoder; @@ -21,7 +21,7 @@ class DecoderFactory : public Factory { public: DecoderFactory(Ptr graph = nullptr) : Factory(graph) {} - virtual Ptr construct(); + virtual Ptr construct(Ptr graph); }; typedef Accumulator decoder; @@ -45,7 +45,7 @@ class EncoderDecoderFactory : public Factory { return Accumulator(*this); } - virtual Ptr construct(); + virtual Ptr construct(Ptr graph); }; typedef Accumulator encoder_decoder; diff --git a/src/models/s2s.h b/src/models/s2s.h index 2ee090d6f..4dfe32d97 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -86,8 +86,8 @@ class EncoderS2S : public EncoderBase { rnnBw.push_back(stacked); } - auto context = concatenate({rnnFw.construct()->transduce(embeddings, mask), - rnnBw.construct()->transduce(embeddings, mask)}, + auto context = concatenate({rnnFw.construct(graph)->transduce(embeddings, mask), + rnnBw.construct(graph)->transduce(embeddings, mask)}, /*axis =*/ -1); if(second > 0) { @@ -114,7 +114,7 @@ class EncoderS2S : public EncoderBase { } // transduce context to new context - context = rnnUni.construct()->transduce(context); + context = rnnUni.construct(graph)->transduce(context); } return context; } @@ -143,7 +143,7 @@ class EncoderS2S : public EncoderBase { ("normalization", opt("embedding-normalization")); } - return embFactory.construct(); + return embFactory.construct(graph); } EncoderS2S(Ptr options) : EncoderBase(options) {} @@ -235,7 +235,7 @@ class DecoderS2S : public DecoderBase { rnn.push_back(highCell); } - return rnn.construct(); + return rnn.construct(graph); } public: @@ -266,9 +266,10 @@ class DecoderS2S : public DecoderBase { ("nematus-normalization", options_->has("original-type") && opt("original-type") == "nematus") // - ); + ) + .construct(graph); - start = mlp.construct()->apply(meanContexts); + start = mlp->apply(meanContexts); } else { int dimBatch = (int)batch->size(); int dimRnn = opt("dim-rnn"); @@ -351,7 +352,7 @@ class DecoderS2S : public DecoderBase { output_ = mlp::mlp(graph) // .push_back(hidden) // .push_back(last) - .construct(); + .construct(graph); } Expr logits; diff --git a/src/models/transformer.h b/src/models/transformer.h index 3292cabfe..ac925a793 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -456,7 +456,7 @@ class Transformer : public EncoderOrDecoderBase { ("dropout", dropoutRnn) // ("layer-normalization", opt("layer-normalization")) // .push_back(rnn::cell(graph_)) // - .construct(); + .construct(graph_); float dropProb = inference_ ? 0 : opt("transformer-dropout"); auto opsPre = opt("transformer-preprocess"); @@ -492,7 +492,7 @@ class EncoderTransformer : public Transformer { ("ulrTrainTransform", opt("ulr-trainable-transformation")) ("ulrQueryFile", opt("ulr-query-vectors")) ("ulrKeysFile", opt("ulr-keys-vectors")); - return embFactory.construct(); + return embFactory.construct(graph_); } Ptr createWordEmbeddingLayer(size_t subBatchIndex) const { @@ -511,7 +511,7 @@ class EncoderTransformer : public Transformer { embFactory("embFile", embFiles[subBatchIndex]) ("normalization", opt("embedding-normalization")); } - return embFactory.construct(); + return embFactory.construct(graph_); } Ptr build(Ptr graph, @@ -630,7 +630,7 @@ class DecoderTransformer : public Transformer { // aligned source context output_ = mlp::mlp(graph_) // .push_back(layerOut) // - .construct(); + .construct(graph_); } public: diff --git a/src/rnn/attention_constructors.h b/src/rnn/attention_constructors.h old mode 100644 new mode 100755 index 9fd1e966b..e377090a3 --- a/src/rnn/attention_constructors.h +++ b/src/rnn/attention_constructors.h @@ -17,9 +17,9 @@ class AttentionFactory : public InputFactory { public: AttentionFactory(Ptr graph) : InputFactory(graph) {} - Ptr construct() override { + Ptr construct(Ptr graph) override { ABORT_IF(!state_, "EncoderState not set"); - return New(graph_, options_, state_); + return New(graph, options_, state_); } Accumulator set_state(Ptr state) { diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index 657d8ad16..406c0c480 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -27,7 +27,7 @@ struct StackableFactory : public Factory { struct InputFactory : public StackableFactory { InputFactory(Ptr graph) : StackableFactory(graph) {} - virtual Ptr construct() = 0; + virtual Ptr construct(Ptr graph) = 0; }; class CellFactory : public StackableFactory { @@ -37,42 +37,42 @@ class CellFactory : public StackableFactory { public: CellFactory(Ptr graph) : StackableFactory(graph) {} - virtual Ptr construct() { + virtual Ptr construct(Ptr graph) { std::string type = options_->get("type"); if(type == "gru") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "gru-nematus") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "lstm") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "mlstm") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "mgru") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "tanh") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "relu") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "sru") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else if(type == "ssru") { - auto cell = New(graph_, options_); + auto cell = New(graph, options_); cell->setLazyInputs(inputs_); return cell; } else { @@ -81,7 +81,7 @@ class CellFactory : public StackableFactory { } CellFactory clone() { - CellFactory aClone(graph_); + CellFactory aClone(nullptr); aClone.options_->merge(options_); aClone.inputs_ = inputs_; return aClone; @@ -105,8 +105,8 @@ class StackedCellFactory : public CellFactory { public: StackedCellFactory(Ptr graph) : CellFactory(graph) {} - Ptr construct() override { - auto stacked = New(graph_, options_); + Ptr construct(Ptr graph) override { + auto stacked = New(graph, options_); int lastDimInput = options_->get("dimInput"); @@ -124,11 +124,11 @@ class StackedCellFactory : public CellFactory { for(auto f : inputs_) cellFactory->add_input(f); - stacked->push_back(cellFactory->construct()); + stacked->push_back(cellFactory->construct(graph)); } else { auto inputFactory = sf->as(); inputFactory->getOptions()->merge(options_); - auto input = inputFactory->construct(); + auto input = inputFactory->construct(graph); stacked->push_back(input); lastDimInput += input->dimOutput(); } @@ -152,8 +152,8 @@ class RNNFactory : public Factory { public: RNNFactory(Ptr graph) : Factory(graph) {} - Ptr construct() { - auto rnn = New(graph_, options_); + Ptr construct(Ptr graph) { + auto rnn = New(graph, options_); for(size_t i = 0; i < layerFactories_.size(); ++i) { auto lf = layerFactories_[i]; @@ -182,7 +182,7 @@ class RNNFactory : public Factory { lf->getOptions()->set("direction", (int)rnn::dir::backward); } - rnn->push_back(lf->construct()); + rnn->push_back(lf->construct(graph)); } return rnn; } @@ -194,7 +194,7 @@ class RNNFactory : public Factory { } RNNFactory clone() { - RNNFactory aClone(graph_); + RNNFactory aClone(nullptr); aClone.options_->merge(options_); for(auto lf : layerFactories_) aClone.push_back(lf->clone()); From 5d65214ee56e8e29b578d0670a06b503f0860a79 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 15:53:05 -0800 Subject: [PATCH 096/838] removed graph from Factory constructor (and derived classes') --- src/graph/expression_operators.cpp | 4 +-- src/layers/constructors.h | 15 +++------- src/layers/factory.h | 10 +++---- src/layers/generic.h | 4 --- src/models/decoder.h | 8 ++--- src/models/model_factory.h | 6 ++-- src/models/s2s.h | 47 +++++++++++++++--------------- src/models/transformer.h | 20 ++++++------- src/rnn/attention_constructors.h | 2 +- src/rnn/constructors.h | 13 ++------- 10 files changed, 55 insertions(+), 74 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index cbfd8054b..cdfeff271 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -512,12 +512,12 @@ Expr highway(const std::string prefix, Expr x) { // clang-format off size_t outDim = x->shape()[-1]; auto graph = x->graph(); - auto g = mlp::dense(graph) + auto g = mlp::dense() ("prefix", prefix + "_highway_d1") ("dim", outDim) ("activation", mlp::act::sigmoid) .construct(graph)->apply(x); - auto relued = mlp::dense(x->graph()) + auto relued = mlp::dense() ("prefix", prefix + "_highway_d2") ("dim", outDim) ("activation", mlp::act::ReLU) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index f5d6aef6b..650c814b0 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -10,7 +10,7 @@ namespace mlp { * Base class for layer factories, can be used in a multi-layer network factory. */ struct LayerFactory : public Factory { - LayerFactory(Ptr graph) : Factory(graph) {} + LayerFactory() : Factory() {} LayerFactory(const LayerFactory&) = default; LayerFactory(LayerFactory&&) = default; @@ -34,15 +34,12 @@ struct LayerFactory : public Factory { */ class DenseFactory : public LayerFactory { public: - DenseFactory(Ptr graph) : LayerFactory(graph) {} - Ptr construct(Ptr graph) override { - auto dense = New(graph, options_); - return dense; + return New(graph, options_); } DenseFactory clone() { - DenseFactory aClone(nullptr); + DenseFactory aClone; aClone.options_->merge(options_); return aClone; } @@ -60,8 +57,6 @@ class OutputFactory : public LayerFactory { Ptr shortlist_; public: - OutputFactory(Ptr graph) : LayerFactory(graph) {} - Accumulator tie_transposed(const std::string& param, const std::string& tied) { tiedParamsTransposed_.push_back({param, tied}); @@ -82,7 +77,7 @@ class OutputFactory : public LayerFactory { } OutputFactory clone() { - OutputFactory aClone(nullptr); + OutputFactory aClone; aClone.options_->merge(options_); aClone.tiedParamsTransposed_ = tiedParamsTransposed_; aClone.shortlist_ = shortlist_; @@ -135,8 +130,6 @@ class MLPFactory : public Factory { std::vector> layers_; public: - MLPFactory(Ptr graph) : Factory(graph) {} - Ptr construct(Ptr graph) { auto mlp = New(graph, options_); for(auto layer : layers_) { diff --git a/src/layers/factory.h b/src/layers/factory.h index 6ebc63fe3..4db11d605 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -7,11 +7,12 @@ namespace marian { class Factory : public std::enable_shared_from_this { protected: Ptr options_; - //Ptr graph_; public: - Factory(Ptr graph) - : options_(New())/*, graph_(graph)*/ {} + Factory() : options_(New()) {} + Factory(Ptr options) : Factory() { + options_->merge(options); + } virtual ~Factory() {} @@ -35,8 +36,7 @@ class Accumulator : public BaseFactory { typedef BaseFactory Factory; public: - Accumulator() : Factory(nullptr) {} - Accumulator(Ptr graph) : Factory(graph) {} + Accumulator() : Factory() {} Accumulator(const Factory& factory) : Factory(factory) {} Accumulator(const Accumulator&) = default; Accumulator(Accumulator&&) = default; diff --git a/src/layers/generic.h b/src/layers/generic.h index c7a5c3bed..80ec4ec0f 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -325,16 +325,12 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { }; struct EmbeddingFactory : public Factory { - EmbeddingFactory(Ptr graph) : Factory(graph) {} - Ptr construct(Ptr graph) { return New(graph, options_); } }; struct ULREmbeddingFactory : public Factory { - ULREmbeddingFactory(Ptr graph) : Factory(graph) {} - Ptr construct(Ptr graph) { return New(graph, options_); } diff --git a/src/models/decoder.h b/src/models/decoder.h index b982b84a3..99fbd4ad6 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -38,8 +38,8 @@ class DecoderBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - auto yEmbFactory = embedding(graph) // - ("dimVocab", dimVoc) // + auto yEmbFactory = embedding() // + ("dimVocab", dimVoc) // ("dimEmb", dimEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) @@ -90,8 +90,8 @@ class DecoderBase { selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); } else { // embeddings are loaded from model during translation, no fixing required - auto yEmbFactory = embedding(graph) // - ("dimVocab", dimTrgVoc) // + auto yEmbFactory = embedding() // + ("dimVocab", dimTrgVoc) // ("dimEmb", dimTrgEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 724561cc7..1f099a603 100755 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -10,7 +10,7 @@ namespace models { class EncoderFactory : public Factory { public: - EncoderFactory(Ptr graph = nullptr) : Factory(graph) {} + EncoderFactory(Ptr graph = nullptr) : Factory() {} virtual Ptr construct(Ptr graph); }; @@ -19,7 +19,7 @@ typedef Accumulator encoder; class DecoderFactory : public Factory { public: - DecoderFactory(Ptr graph = nullptr) : Factory(graph) {} + DecoderFactory(Ptr graph = nullptr) : Factory() {} virtual Ptr construct(Ptr graph); }; @@ -33,7 +33,7 @@ class EncoderDecoderFactory : public Factory { public: EncoderDecoderFactory(Ptr graph = nullptr) - : Factory(graph) {} + : Factory() {} Accumulator push_back(encoder enc) { encoders_.push_back(enc); diff --git a/src/models/s2s.h b/src/models/s2s.h index 4dfe32d97..e6300e228 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -34,7 +34,7 @@ class EncoderS2S : public EncoderBase { float dropoutRnn = inference_ ? 0 : opt("dropout-rnn"); - auto rnnFw = rnn::rnn(graph) // + auto rnnFw = rnn::rnn() // ("type", opt("enc-cell")) // ("direction", (int)forward) // ("dimInput", embeddings->shape()[-1]) // @@ -44,7 +44,7 @@ class EncoderS2S : public EncoderBase { ("skip", opt("skip")); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= opt("enc-cell-depth"); ++j) { std::string paramPrefix = prefix_ + "_bi"; if(i > 1) @@ -53,14 +53,14 @@ class EncoderS2S : public EncoderBase { paramPrefix += "_cell" + std::to_string(j); bool transition = (j > 1); - stacked.push_back(rnn::cell(graph) // + stacked.push_back(rnn::cell() // ("prefix", paramPrefix) // ("transition", transition)); } rnnFw.push_back(stacked); } - auto rnnBw = rnn::rnn(graph) // + auto rnnBw = rnn::rnn() // ("type", opt("enc-cell")) // ("direction", (int)backward) // ("dimInput", embeddings->shape()[-1]) // @@ -70,7 +70,7 @@ class EncoderS2S : public EncoderBase { ("skip", opt("skip")); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= opt("enc-cell-depth"); ++j) { std::string paramPrefix = prefix_ + "_bi_r"; if(i > 1) @@ -79,7 +79,7 @@ class EncoderS2S : public EncoderBase { paramPrefix += "_cell" + std::to_string(j); bool transition = (j > 1); - stacked.push_back(rnn::cell(graph) // + stacked.push_back(rnn::cell() // ("prefix", paramPrefix) // ("transition", transition)); } @@ -95,7 +95,7 @@ class EncoderS2S : public EncoderBase { // previous bidirectional RNN through multiple layers // construct RNN first - auto rnnUni = rnn::rnn(graph) // + auto rnnUni = rnn::rnn() // ("type", opt("enc-cell")) // ("dimInput", 2 * opt("dim-rnn")) // ("dimState", opt("dim-rnn")) // @@ -104,11 +104,11 @@ class EncoderS2S : public EncoderBase { ("skip", opt("skip")); for(int i = first + 1; i <= second + first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= opt("enc-cell-depth"); ++j) { std::string paramPrefix = prefix_ + "_l" + std::to_string(i) + "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnUni.push_back(stacked); } @@ -124,8 +124,8 @@ class EncoderS2S : public EncoderBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - auto embFactory = embedding(graph) // - ("dimVocab", dimVoc) // + auto embFactory = embedding() // + ("dimVocab", dimVoc) // ("dimEmb", dimEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) @@ -180,7 +180,7 @@ class DecoderS2S : public DecoderBase { Ptr constructDecoderRNN(Ptr graph, Ptr state) { float dropoutRnn = inference_ ? 0 : opt("dropout-rnn"); - auto rnn = rnn::rnn(graph) // + auto rnn = rnn::rnn() // ("type", opt("dec-cell")) // ("dimInput", opt("dim-emb")) // ("dimState", opt("dim-rnn")) // @@ -196,11 +196,11 @@ class DecoderS2S : public DecoderBase { size_t decoderHighDepth = opt("dec-cell-high-depth"); // setting up conditional (transitional) cell - auto baseCell = rnn::stacked_cell(graph); + auto baseCell = rnn::stacked_cell(); for(size_t i = 1; i <= decoderBaseDepth; ++i) { bool transition = (i > 2); auto paramPrefix = prefix_ + "_cell" + std::to_string(i); - baseCell.push_back(rnn::cell(graph) // + baseCell.push_back(rnn::cell() // ("prefix", paramPrefix) // ("final", i > 1) // ("transition", transition)); @@ -212,8 +212,7 @@ class DecoderS2S : public DecoderBase { auto encState = state->getEncoderStates()[k]; - baseCell.push_back( - rnn::attention(graph)("prefix", attPrefix).set_state(encState)); + baseCell.push_back(rnn::attention()("prefix", attPrefix).set_state(encState)); } } } @@ -223,12 +222,12 @@ class DecoderS2S : public DecoderBase { // Add more cells to RNN (stacked RNN) for(size_t i = 2; i <= decoderLayers; ++i) { // deep transition - auto highCell = rnn::stacked_cell(graph); + auto highCell = rnn::stacked_cell(); for(size_t j = 1; j <= decoderHighDepth; j++) { auto paramPrefix = prefix_ + "_l" + std::to_string(i) + "_cell" + std::to_string(j); - highCell.push_back(rnn::cell(graph)("prefix", paramPrefix)); + highCell.push_back(rnn::cell()("prefix", paramPrefix)); } // Add cell to RNN (more layers) @@ -257,14 +256,14 @@ class DecoderS2S : public DecoderBase { Expr start; if(!meanContexts.empty()) { // apply single layer network to mean to map into decoder space - auto mlp = mlp::mlp(graph).push_back( - mlp::dense(graph) // + auto mlp = mlp::mlp().push_back( + mlp::dense() // ("prefix", prefix_ + "_ff_state") // ("dim", opt("dim-rnn")) // ("activation", (int)mlp::act::tanh) // ("layer-normalization", opt("layer-normalization")) // ("nematus-normalization", - options_->has("original-type") + options_->has("original-type") && opt("original-type") == "nematus") // ) .construct(graph); @@ -322,7 +321,7 @@ class DecoderS2S : public DecoderBase { if(!output_) { // construct deep output multi-layer network layer-wise - auto hidden = mlp::dense(graph) // + auto hidden = mlp::dense() // ("prefix", prefix_ + "_ff_logit_l1") // ("dim", opt("dim-emb")) // ("activation", mlp::act::tanh) // @@ -333,7 +332,7 @@ class DecoderS2S : public DecoderBase { int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - auto last = mlp::output(graph) // + auto last = mlp::output() // ("prefix", prefix_ + "_ff_logit_l2") // ("dim", dimTrgVoc); @@ -349,7 +348,7 @@ class DecoderS2S : public DecoderBase { // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context - output_ = mlp::mlp(graph) // + output_ = mlp::mlp() // .push_back(hidden) // .push_back(last) .construct(graph); diff --git a/src/models/transformer.h b/src/models/transformer.h index ac925a793..f96ce257c 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -448,14 +448,14 @@ class Transformer : public EncoderOrDecoderBase { int /*startPos*/) const { float dropoutRnn = inference_ ? 0.f : opt("dropout-rnn"); - auto rnn = rnn::rnn(graph_) // + auto rnn = rnn::rnn() // ("type", opt("dec-cell")) // ("prefix", prefix) // ("dimInput", opt("dim-emb")) // ("dimState", opt("dim-emb")) // ("dropout", dropoutRnn) // ("layer-normalization", opt("layer-normalization")) // - .push_back(rnn::cell(graph_)) // + .push_back(rnn::cell()) // .construct(graph_); float dropProb = inference_ ? 0 : opt("transformer-dropout"); @@ -487,11 +487,11 @@ class EncoderTransformer : public Transformer { int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt int dimEmb = opt("dim-emb"); int dimUlrEmb = opt("ulr-dim-emb"); - auto embFactory = ulr_embedding(graph_)("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) - ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) - ("ulrTrainTransform", opt("ulr-trainable-transformation")) - ("ulrQueryFile", opt("ulr-query-vectors")) - ("ulrKeysFile", opt("ulr-keys-vectors")); + auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) + ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) + ("ulrTrainTransform", opt("ulr-trainable-transformation")) + ("ulrQueryFile", opt("ulr-query-vectors")) + ("ulrKeysFile", opt("ulr-keys-vectors")); return embFactory.construct(graph_); } @@ -499,7 +499,7 @@ class EncoderTransformer : public Transformer { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); - auto embFactory = embedding(graph_)("dimVocab", dimVoc)("dimEmb", dimEmb); + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); if (opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else @@ -611,7 +611,7 @@ class DecoderTransformer : public Transformer { int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - auto layerOut = mlp::output(graph_) // + auto layerOut = mlp::output() // ("prefix", prefix_ + "_ff_logit_out") // ("dim", dimTrgVoc); @@ -628,7 +628,7 @@ class DecoderTransformer : public Transformer { // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context - output_ = mlp::mlp(graph_) // + output_ = mlp::mlp() // .push_back(layerOut) // .construct(graph_); } diff --git a/src/rnn/attention_constructors.h b/src/rnn/attention_constructors.h index e377090a3..a878f57f6 100755 --- a/src/rnn/attention_constructors.h +++ b/src/rnn/attention_constructors.h @@ -15,7 +15,7 @@ class AttentionFactory : public InputFactory { Ptr state_; public: - AttentionFactory(Ptr graph) : InputFactory(graph) {} +// AttentionFactory(Ptr graph) : InputFactory(graph) {} Ptr construct(Ptr graph) override { ABORT_IF(!state_, "EncoderState not set"); diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index 406c0c480..3a5826d64 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -8,7 +8,7 @@ namespace marian { namespace rnn { struct StackableFactory : public Factory { - StackableFactory(Ptr graph) : Factory(graph) {} + StackableFactory() : Factory() {} StackableFactory(const StackableFactory&) = default; StackableFactory(StackableFactory&&) = default; @@ -26,7 +26,6 @@ struct StackableFactory : public Factory { }; struct InputFactory : public StackableFactory { - InputFactory(Ptr graph) : StackableFactory(graph) {} virtual Ptr construct(Ptr graph) = 0; }; @@ -35,8 +34,6 @@ class CellFactory : public StackableFactory { std::vector)>> inputs_; public: - CellFactory(Ptr graph) : StackableFactory(graph) {} - virtual Ptr construct(Ptr graph) { std::string type = options_->get("type"); if(type == "gru") { @@ -81,7 +78,7 @@ class CellFactory : public StackableFactory { } CellFactory clone() { - CellFactory aClone(nullptr); + CellFactory aClone; aClone.options_->merge(options_); aClone.inputs_ = inputs_; return aClone; @@ -103,8 +100,6 @@ class StackedCellFactory : public CellFactory { std::vector> stackableFactories_; public: - StackedCellFactory(Ptr graph) : CellFactory(graph) {} - Ptr construct(Ptr graph) override { auto stacked = New(graph, options_); @@ -150,8 +145,6 @@ class RNNFactory : public Factory { std::vector> layerFactories_; public: - RNNFactory(Ptr graph) : Factory(graph) {} - Ptr construct(Ptr graph) { auto rnn = New(graph, options_); for(size_t i = 0; i < layerFactories_.size(); ++i) { @@ -194,7 +187,7 @@ class RNNFactory : public Factory { } RNNFactory clone() { - RNNFactory aClone(nullptr); + RNNFactory aClone; aClone.options_->merge(options_); for(auto lf : layerFactories_) aClone.push_back(lf->clone()); From 1211c275581779e769174c3937cee7a993576864 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 16:33:55 -0800 Subject: [PATCH 097/838] minor further factoring for super-simple factories --- src/layers/factory.h | 8 ++++++++ src/layers/generic.h | 17 ++++------------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/layers/factory.h b/src/layers/factory.h index 4db11d605..e3b742442 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -31,6 +31,14 @@ class Factory : public std::enable_shared_from_this { } }; +// simplest form of Factory that just passes on options to the constructor of a layer type +template +struct ConstructingFactory : public Factory { + Ptr construct(Ptr graph) { + return New(graph, options_); + } +}; + template class Accumulator : public BaseFactory { typedef BaseFactory Factory; diff --git a/src/layers/generic.h b/src/layers/generic.h index 80ec4ec0f..054589570 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -55,6 +55,7 @@ struct IEmbeddingLayer { }; namespace mlp { + class Dense : public LayerBase, public IUnaryLayer { public: Dense(Ptr graph, Ptr options) @@ -98,7 +99,6 @@ class Dense : public LayerBase, public IUnaryLayer { outputs.push_back(layerNorm(dot(in, W), gamma, b)); } - } else { outputs.push_back(affine(in, W, b)); } @@ -219,7 +219,7 @@ class Embedding : public LayerBase, public IEmbeddingLayer { }; class ULREmbedding : public LayerBase, public IEmbeddingLayer { - std::vector ulrEmbeddings_; // @TODO: These can now better be written as 6 named class members + std::vector ulrEmbeddings_; // @TODO: These could now better be written as 6 named class members public: ULREmbedding(Ptr graph, Ptr options) : LayerBase(graph, options) { std::string name = "url_embed"; //opt("prefix"); @@ -324,17 +324,8 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { } }; -struct EmbeddingFactory : public Factory { - Ptr construct(Ptr graph) { - return New(graph, options_); - } -}; - -struct ULREmbeddingFactory : public Factory { - Ptr construct(Ptr graph) { - return New(graph, options_); - } -}; +typedef ConstructingFactory EmbeddingFactory; +typedef ConstructingFactory ULREmbeddingFactory; typedef Accumulator embedding; typedef Accumulator ulr_embedding; From 93c11dbd03b1d604472b6a0c5e08195b515469be Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 16:55:27 -0800 Subject: [PATCH 098/838] tied_transposed() no longer takes the name parameter which is always "W" --- src/layers/constructors.h | 7 +++---- src/layers/generic.h | 4 ++-- src/models/s2s.h | 2 +- src/models/transformer.h | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index 650c814b0..8c7345d4b 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -57,9 +57,8 @@ class OutputFactory : public LayerFactory { Ptr shortlist_; public: - Accumulator tie_transposed(const std::string& param, - const std::string& tied) { - tiedParamsTransposed_.push_back({param, tied}); + Accumulator tie_transposed(const std::string& tied) { + tiedParamsTransposed_.push_back({"W", tied}); return Accumulator(*this); } @@ -71,7 +70,7 @@ class OutputFactory : public LayerFactory { Ptr construct(Ptr graph) override { auto output = New(graph, options_); for(auto& p : tiedParamsTransposed_) - output->tie_transposed(p.first, p.second); + output->tie_transposed(p.second); output->set_shortlist(shortlist_); return output; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 054589570..15a1aec5e 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -135,8 +135,8 @@ class Output : public LayerBase, public IUnaryLayer { Output(Ptr graph, Ptr options) : LayerBase(graph, options) {} - void tie_transposed(const std::string& param, const std::string& tied) { - tiedParams_[param] = graph_->get(tied); + void tie_transposed(const std::string& tied) { + tiedParams_["W"] = graph_->get(tied); } void set_shortlist(Ptr shortlist) { shortlist_ = shortlist; } diff --git a/src/models/s2s.h b/src/models/s2s.h index e6300e228..58d1377cc 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -340,7 +340,7 @@ class DecoderS2S : public DecoderBase { std::string tiedPrefix = prefix_ + "_Wemb"; if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; - last.tie_transposed("W", tiedPrefix); + last.tie_transposed(tiedPrefix); } if(shortlist_) diff --git a/src/models/transformer.h b/src/models/transformer.h index f96ce257c..e979a9261 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -619,7 +619,7 @@ class DecoderTransformer : public Transformer { std::string tiedPrefix = prefix_ + "_Wemb"; if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; - layerOut.tie_transposed("W", tiedPrefix); + layerOut.tie_transposed(tiedPrefix); } if(shortlist_) From 3ed01fd3d03b7f2fac82898f6502e69d5bdddb4f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 9 Jan 2019 17:11:37 -0800 Subject: [PATCH 099/838] removed the no-longer-used map for Output::tiedParams, replaced with a single value --- src/layers/constructors.h | 15 +++++++-------- src/layers/generic.h | 15 +++++++-------- src/models/s2s.h | 4 ++-- src/models/transformer.h | 4 ++-- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index 8c7345d4b..d0ac34873 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -53,32 +53,31 @@ typedef Accumulator dense; */ class OutputFactory : public LayerFactory { protected: - std::vector> tiedParamsTransposed_; + std::string tiedTransposedName_; Ptr shortlist_; public: - Accumulator tie_transposed(const std::string& tied) { - tiedParamsTransposed_.push_back({"W", tied}); + Accumulator tieTransposed(const std::string& tied) { + tiedTransposedName_ = tied; return Accumulator(*this); } - Accumulator set_shortlist(Ptr shortlist) { + Accumulator setShortlist(Ptr shortlist) { shortlist_ = shortlist; return Accumulator(*this); } Ptr construct(Ptr graph) override { auto output = New(graph, options_); - for(auto& p : tiedParamsTransposed_) - output->tie_transposed(p.second); - output->set_shortlist(shortlist_); + output->tieTransposed(graph->get(tiedTransposedName_)); + output->setShortlist(shortlist_); return output; } OutputFactory clone() { OutputFactory aClone; aClone.options_->merge(options_); - aClone.tiedParamsTransposed_ = tiedParamsTransposed_; + aClone.tiedTransposedName_ = tiedTransposedName_; aClone.shortlist_ = shortlist_; return aClone; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 15a1aec5e..a9e2be016 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -124,7 +124,7 @@ class Dense : public LayerBase, public IUnaryLayer { class Output : public LayerBase, public IUnaryLayer { private: - std::map tiedParams_; + Expr tiedParam_; Ptr shortlist_; Expr W_; @@ -135,25 +135,24 @@ class Output : public LayerBase, public IUnaryLayer { Output(Ptr graph, Ptr options) : LayerBase(graph, options) {} - void tie_transposed(const std::string& tied) { - tiedParams_["W"] = graph_->get(tied); + void tieTransposed(Expr tied) { + tiedParam_ = tied; } - void set_shortlist(Ptr shortlist) { shortlist_ = shortlist; } + void setShortlist(Ptr shortlist) { shortlist_ = shortlist; } Expr apply(Expr input) override { if(!W_) { auto name = options_->get("prefix"); auto dim = options_->get("dim"); - std::string nameW = "W"; - if(tiedParams_.count(nameW)) { + if(tiedParam_) { transposeW_ = true; - W_ = tiedParams_[nameW]; + W_ = tiedParam_; if(shortlist_) W_ = rows(W_, shortlist_->indices()); } else { - W_ = graph_->param(name + "_" + nameW, + W_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform); if(shortlist_) diff --git a/src/models/s2s.h b/src/models/s2s.h index 58d1377cc..edda79bc8 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -340,11 +340,11 @@ class DecoderS2S : public DecoderBase { std::string tiedPrefix = prefix_ + "_Wemb"; if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; - last.tie_transposed(tiedPrefix); + last.tieTransposed(tiedPrefix); } if(shortlist_) - last.set_shortlist(shortlist_); + last.setShortlist(shortlist_); // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context diff --git a/src/models/transformer.h b/src/models/transformer.h index e979a9261..d91dfb7f8 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -619,11 +619,11 @@ class DecoderTransformer : public Transformer { std::string tiedPrefix = prefix_ + "_Wemb"; if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; - layerOut.tie_transposed(tiedPrefix); + layerOut.tieTransposed(tiedPrefix); } if(shortlist_) - layerOut.set_shortlist(shortlist_); + layerOut.setShortlist(shortlist_); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and From dd104c34f2d152ba8b09910cf7de9803f228081c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 10 Jan 2019 16:10:10 -0800 Subject: [PATCH 100/838] fix masking --- src/common/config_parser.cpp | 8 +- src/data/corpus_base.h | 9 +- src/data/default_vocab.cpp | 2 +- src/models/bert.h | 415 ++++++++++++-------- src/models/model_factory.cpp | 14 +- src/training/graph_group.h | 5 +- src/training/graph_group_async.cpp | 4 +- src/training/graph_group_async.h | 5 +- src/training/graph_group_multinode.cpp | 4 +- src/training/graph_group_multinode.h | 4 +- src/training/graph_group_multinode_sync.cpp | 3 +- src/training/graph_group_multinode_sync.h | 4 +- src/training/graph_group_singleton.h | 4 +- src/training/graph_group_sync.cpp | 4 +- src/training/graph_group_sync.h | 2 +- src/training/scheduler.h | 8 +- src/training/training.h | 5 +- 17 files changed, 291 insertions(+), 209 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 1910c85de..da6f7d6a9 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -201,6 +201,10 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Operation after each transformer layer: d = dropout, a = add, n = normalize", "dan"); + cli.add("--bert-mask-symbol", "Masking symbol for BERT masked-LM training", "[MASK]"); + cli.add("--bert-sep-symbol", "Sentence separator symbol for BERT next sentence prediction training", "[SEP]"); + cli.add("--bert-class-symbol", "Class symbol BERT classifier training", "[CLS]"); + cli.add("--bert-masking-fraction", "Fraction of masked out tokens during training", 0.15); #ifdef CUDNN cli.add("--char-stride", "Width of max-pooling layer after convolution layer in char-s2s model", @@ -280,8 +284,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Display nformation for the first arg updates"); cli.add("--disp-label-counts", "Display label counts when logging loss progress"); - cli.add("--disp-wps-index", - "Display words-per-second ratio based on i-th sub-batch (-1 is last)", -1); + cli.add("--disp-label-index", + "Display label counts based on i-th sub-batch (-1 is last)", -1); cli.add("--save-freq", "Save model file every arg updates (append 't' for every arg target labels)", "10000u"); diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index b8bc556cc..e25d7d451 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -226,7 +226,7 @@ class SubBatch { * such as guided alignments and sentence or word-leve weighting. */ class CorpusBatch : public Batch { -private: +protected: std::vector> subBatches_; std::vector guidedAlignment_; std::vector dataWeights_; @@ -307,17 +307,16 @@ class CorpusBatch : public Batch { * * @return Fake batch of the same size as the real batch. */ - static Ptr fakeBatch(std::vector& lengths, + static Ptr fakeBatch(const std::vector& lengths, + const std::vector>& vocabs, size_t batchSize, Ptr options) { std::vector> batches; size_t idx = 0; for(auto len : lengths) { - auto vocab = New(options, 0); - vocab->createFake(); // data: gets initialized to 0. No EOS symbol is distinguished. - auto sb = New(batchSize, len, vocab); + auto sb = New(batchSize, len, vocabs[idx]); // set word indices to different values to avoid same hashes std::fill(sb->data().begin(), sb->data().end(), (unsigned int)idx++); // mask: no items ask being masked out diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index bac5c72ec..96ae61e64 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -74,7 +74,7 @@ class DefaultVocab : public VocabBase { const std::string& operator[](Word id) const override { - ABORT_IF(id >= id2str_.size(), "Unknown word id: ", id); + ABORT_IF(id >= id2str_.size(), "Unknown word id: {}", id); return id2str_[id]; } diff --git a/src/models/bert.h b/src/models/bert.h index f0a1f8eec..232f7e88f 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -1,169 +1,248 @@ -#pragma once - -#include "data/corpus_base.h" -#include "models/encoder_classifier.h" -#include "models/transformer.h" - -namespace marian { -namespace data { - -class BertBatch : public CorpusBatch { -private: - std::vector maskedPositions_; - std::vector maskedIndices_; - std::vector sentenceIndices_; - - void init() { - ABORT("Not implemented"); - } - -public: - BertBatch(Ptr batch) : CorpusBatch(*batch) { - std::cerr << "Creating BERT batch" << std::endl; - init(); - } - - const std::vector& bertMaskedPositions() { return maskedPositions_; } - const std::vector& bertMaskedIndices() { return maskedIndices_; } - const std::vector& bertSentenceIndices() { return sentenceIndices_; } -}; - -} - -class BertEncoderClassifier : public EncoderClassifier { -public: - BertEncoderClassifier(Ptr options) : EncoderClassifier(options) {} - - std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { - Ptr bertBatch = New(batch); // intercept batch and anotate with BERT-specific concepts - return EncoderClassifier::apply(graph, bertBatch, clearGraph); - } -}; - -// @TODO: this should be in transformer.h -class BertEncoder : public EncoderTransformer { -public: - BertEncoder(Ptr options) : EncoderTransformer(options) {} - - Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { - Ptr bertBatch = std::dynamic_pointer_cast(batch); - - ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); - - int dimEmb = embeddings->shape()[-1]; - int dimBatch = embeddings->shape()[-2]; - int dimWords = embeddings->shape()[-3]; - - auto sentenceEmbeddings = embedding(graph_) - ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B - ("dimEmb", dimEmb) - .construct(); - - // @TODO: note this is going to be really slow due to atomicAdd in backward step - // with only two classes; - // instead two masked reduce operations, maybe in parallel streams? - auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); - auto signal = rows(sentenceEmbeddings, sentenceIndices); - signal = reshape(signal, {dimWords, dimBatch, dimEmb}); - return embeddings + signal; - } - - virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { - input = addPositionalEmbeddings(input, start, true); // true for BERT - input = addSentenceEmbeddings(input, start, batch); - return input; - } -}; - -// Can be used for next sentence prediction task -class BertClassifier : public ClassifierBase { -public: - BertClassifier(Ptr options) : ClassifierBase(options) {} - - // The batch has been filled with external classifier labels, @TODO: figure out how to do that - Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { - ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); - - auto context = encoderStates[0]->getContext(); - auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-3); // [CLS] symbol is first symbol in each sequence - - int dimModel = classEmbeddings->shape()[-1]; - int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels - - auto output = mlp::mlp(graph) // - .push_back(mlp::dense(graph) // - ("prefix", prefix_ + "_ff_logit_l1") // - ("dim", dimModel) // - ("activation", mlp::act::tanh)) // @TODO: do we actually need this? - .push_back(mlp::output(graph) // - ("dim", dimTrgCls)) // - ("prefix", prefix_ + "_ff_logit_l2") // - .construct(); - - auto logits = output->apply(classEmbeddings); // class logits for each batch entry - - auto state = New(); - state->setLogProbs(logits); - - // filled externally, for BERT these are NextSentence prediction labels - const auto& classLabels = (*batch)[batchIndex_]->data(); - state->setTargetIndices(graph->indices(classLabels)); - - return state; - } - - virtual void clear() override {} -}; - -// This is a model that pretrains BERT for classification -class BertMaskedLM : public ClassifierBase { -public: - BertMaskedLM(Ptr options) : ClassifierBase(options) {} - - Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { - Ptr bertBatch = std::dynamic_pointer_cast(batch); - - ABORT_IF(!bertBatch, "Batch could not be converted to batch for BERT training"); - ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); - - auto context = encoderStates[0]->getContext(); - - auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries - auto bertMaskedIndices = graph->indices(bertBatch->bertMaskedIndices()); // vocab ids of entries that have been masked - - auto classEmbeddings = rows(context, bertMaskedPositions); // subselect stuff that has actually been masked out; - - int dimModel = classEmbeddings->shape()[-1]; - - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - - auto layerTanh = mlp::dense(graph) // - ("dim", dimModel) // - ("activation", mlp::act::tanh); // - auto layerOut = mlp::output(graph) // - ("dim", dimVoc); // - layerOut.tie_transposed("W", "Wemb"); // We are a BERT model, hence tie with input - - // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] - // assemble layers into MLP and apply to embeddings, decoder context and - // aligned source context - auto output = mlp::mlp(graph) // - ("prefix", prefix_ + "_ff_logit_maskedlm_out") // - .push_back(layerTanh) // @TODO: do we actually need this? - .push_back(layerOut) // - .construct(); - - auto logits = output->apply(classEmbeddings); - - auto state = New(); - state->setLogProbs(logits); - state->setTargetIndices(bertMaskedIndices); - - return state; - } - - virtual void clear() override {} -}; - +#pragma once + +#include "data/corpus_base.h" +#include "models/encoder_classifier.h" +#include "models/transformer.h" +#include "data/rng_engine.h" + +namespace marian { +namespace data { + +class BertBatch : public CorpusBatch { +private: + std::mt19937& eng_; + + std::vector maskedPositions_; + std::vector maskedIndices_; + std::vector sentenceIndices_; + + std::string maskSymbol_; + std::string sepSymbol_; + std::string clsSymbol_; + + std::unique_ptr> randomWord_; + std::unique_ptr> randomPercent_; + + std::unordered_set dontMask_; + + Word maskOut(Word word, Word mask) { + auto subBatch = subBatches_.front(); + + int r = (*randomPercent_)(eng_); + if (r < 10) { // for 10% of cases return same word + return word; + } else if (r < 20) { // for 10% return random word + Word randWord = (*randomWord_)(eng_); + if(dontMask_.count(randWord) > 0) // the random word is a forbidden word + return mask; // hence return mask symbol + else + return randWord; // else return the random word + } else { // for 80% of words apply mask symbol + return mask; + } + } + +public: + BertBatch(Ptr batch, + std::mt19937& engine, + float maskFraction, + const std::string& maskSymbol, + const std::string& sepSymbol, + const std::string& clsSymbol) + : CorpusBatch(*batch), eng_(engine), + maskSymbol_(maskSymbol), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { + + auto subBatch = subBatches_.front(); + + randomWord_.reset(new std::uniform_int_distribution(0, subBatch->vocab()->size())); + randomPercent_.reset(new std::uniform_int_distribution(0, 100)); + + auto& words = subBatch->data(); + + Word maskId = (*subBatch->vocab())[maskSymbol_]; + Word clsId = (*subBatch->vocab())[clsSymbol_]; + Word sepId = (*subBatch->vocab())[sepSymbol_]; + + ABORT_IF(maskId == subBatch->vocab()->getUnkId(), + "BERT masking symbol {} not found in vocabulary", maskSymbol_); + + ABORT_IF(sepId == subBatch->vocab()->getUnkId(), + "BERT separator symbol {} not found in vocabulary", sepSymbol_); + + ABORT_IF(clsId == subBatch->vocab()->getUnkId(), + "BERT class symbol {} not found in vocabulary", clsSymbol_); + + dontMask_.insert(clsId); // don't mask class token + dontMask_.insert(sepId); // don't mask separator token + dontMask_.insert(subBatch->vocab()->getEosId()); // don't mask + // it's ok to mask + + std::vector selected; + selected.reserve(words.size()); + for(int i = 0; i < words.size(); ++i) // collect words among which we will mask + if(dontMask_.count(words[i]) == 0) // do not add indices of special words + selected.push_back(i); + std::shuffle(selected.begin(), selected.end(), eng_); + selected.resize(std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices + + for(int i : selected) { + maskedPositions_.push_back(i); // where is the original word? + maskedIndices_.push_back(words[i]); // what is the original word? + words[i] = maskOut(words[i], maskId); // mask that position + } + } + + const std::vector& bertMaskedPositions() { return maskedPositions_; } + const std::vector& bertMaskedIndices() { return maskedIndices_; } + const std::vector& bertSentenceIndices() { return sentenceIndices_; } +}; + +} + +class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { +public: + BertEncoderClassifier(Ptr options) + : EncoderClassifier(options) {} + + std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + // intercept batch and anotate with BERT-specific concepts + auto bertBatch = New(batch, + eng_, + opt("bert-masking-fraction"), + opt("bert-mask-symbol"), + opt("bert-sep-symbol"), + opt("bert-class-symbol")); + return EncoderClassifier::apply(graph, bertBatch, clearGraph); + } +}; + +// @TODO: this should be in transformer.h +class BertEncoder : public EncoderTransformer { +public: + BertEncoder(Ptr options) : EncoderTransformer(options) {} + + Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { + Ptr bertBatch = std::dynamic_pointer_cast(batch); + + ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); + + int dimEmb = embeddings->shape()[-1]; + int dimBatch = embeddings->shape()[-2]; + int dimWords = embeddings->shape()[-3]; + + auto sentenceEmbeddings = embedding(graph_) + ("prefix", "Wsent") + ("dimVocab", 2) // sentence A or sentence B + ("dimEmb", dimEmb) + .construct(); + + // @TODO: note this is going to be really slow due to atomicAdd in backward step + // with only two classes; + // instead two masked reduce operations, maybe in parallel streams? + auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); + auto signal = rows(sentenceEmbeddings, sentenceIndices); + signal = reshape(signal, {dimWords, dimBatch, dimEmb}); + return embeddings + signal; + } + + virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { + input = addPositionalEmbeddings(input, start, true); // true for BERT + input = addSentenceEmbeddings(input, start, batch); + return input; + } +}; + +// Can be used for next sentence prediction task +class BertClassifier : public ClassifierBase { +public: + BertClassifier(Ptr options) : ClassifierBase(options) {} + + // The batch has been filled with external classifier labels, @TODO: figure out how to do that + Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-3); // [CLS] symbol is first symbol in each sequence + + int dimModel = classEmbeddings->shape()[-1]; + int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels + + auto output = mlp::mlp(graph) // + .push_back(mlp::dense(graph) // + ("prefix", prefix_ + "_ff_logit_l1") // + ("dim", dimModel) // + ("activation", mlp::act::tanh)) // @TODO: do we actually need this? + .push_back(mlp::output(graph) // + ("dim", dimTrgCls)) // + ("prefix", prefix_ + "_ff_logit_l2") // + .construct(); + + auto logits = output->apply(classEmbeddings); // class logits for each batch entry + + auto state = New(); + state->setLogProbs(logits); + + // filled externally, for BERT these are NextSentence prediction labels + const auto& classLabels = (*batch)[batchIndex_]->data(); + state->setTargetIndices(graph->indices(classLabels)); + + return state; + } + + virtual void clear() override {} +}; + +// This is a model that pretrains BERT for classification +class BertMaskedLM : public ClassifierBase { +public: + BertMaskedLM(Ptr options) : ClassifierBase(options) {} + + Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { + Ptr bertBatch = std::dynamic_pointer_cast(batch); + + ABORT_IF(!bertBatch, "Batch could not be converted to batch for BERT training"); + ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); + + auto context = encoderStates[0]->getContext(); + + auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries + auto bertMaskedIndices = graph->indices(bertBatch->bertMaskedIndices()); // vocab ids of entries that have been masked + + int dimModel = context->shape()[-1]; + int dimBatch = context->shape()[-2]; + int dimTime = context->shape()[-3]; + + auto maskedEmbeddings = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out; + + int dimVoc = opt>("dim-vocabs")[batchIndex_]; + + auto layerTanh = mlp::dense(graph) // + ("prefix", prefix_ + "_ff_logit_maskedlm_out_l1") // + ("dim", dimModel) // + ("activation", mlp::act::tanh); // + auto layerOut = mlp::output(graph) // + ("prefix", prefix_ + "_ff_logit_maskedlm_out_l2") // + ("dim", dimVoc); // + layerOut.tie_transposed("W", "Wemb"); // We are a BERT model, hence tie with input + + // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] + // assemble layers into MLP and apply to embeddings, decoder context and + // aligned source context + auto output = mlp::mlp(graph) // + .push_back(layerTanh) // @TODO: do we actually need this? + .push_back(layerOut) // + .construct(); + + auto logits = output->apply(maskedEmbeddings); + + auto state = New(); + state->setLogProbs(logits); + state->setTargetIndices(bertMaskedIndices); + + return state; + } + + virtual void clear() override {} +}; + } \ No newline at end of file diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 2b19b4990..0d66b95d8 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -90,7 +90,7 @@ Ptr EncoderClassifierFactory::construct() { enccls = New(options_); } else { enccls = New(options_); - } + } for(auto& ef : encoders_) enccls->push_back(ef(options_).construct()); @@ -233,14 +233,14 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // transformer encoder for now + ("type", "transformer") // @TODO: replace with 'bert-encoder' ("index", 0)) // close to original transformer encoder - .push_back(models::classifier() // - ("type", "bert-classifier") // - ("index", 1)) // next sentence prediction .push_back(models::classifier() // ("type", "bert-masked-lm") // ("index", 0)) // multi-task learning with MaskedLM + .push_back(models::classifier() // + ("type", "bert-classifier") // + ("index", 1)) // next sentence prediction .construct(); } @@ -248,11 +248,11 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "transformer") // + ("type", "transformer") // ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // - ("index", 1)) // next sentence prediction + ("index", 1)) // next sentence prediction .construct(); } diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 2bd98d40b..ea1705698 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -50,6 +50,7 @@ class GraphGroup { */ virtual Ptr collectStats(Ptr graph, Ptr model, + const std::vector>& vocabs, size_t multiplier = 1) { auto stats = New(); @@ -79,7 +80,7 @@ class GraphGroup { for(int j = 0; j < lengths.size(); ++j) // apply length restrictions lengths[j] = std::min(lengths[j], localMaxes[j]); - auto batch = data::CorpusBatch::fakeBatch(lengths, maxBatch, options_); + auto batch = data::CorpusBatch::fakeBatch(lengths, vocabs, maxBatch, options_); auto cost = model->build(graph, batch); fits = graph->fits(); if(fits) @@ -99,7 +100,7 @@ class GraphGroup { do { size_t current = (start + end) / 2; - auto batch = data::CorpusBatch::fakeBatch(lengths, current, options_); + auto batch = data::CorpusBatch::fakeBatch(lengths, vocabs, current, options_); auto cost = model->build(graph, batch); fits = graph->fits(); diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 1d0042019..3195c5624 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -254,8 +254,8 @@ void AsyncGraphGroup::execute(Ptr batch) { if(optimizerDelay_ > 1) { std::vector fakeLength = {1, 1}; - auto fb = data::CorpusBatch::fakeBatch( - fakeLength, num_seen_sentences, NULL); + std::vector> vocabs; + auto fb = data::CorpusBatch::fakeBatch(fakeLength, vocabs, num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); scheduler_->update(cost, fb); diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h index d3af0f22b..e8c794edd 100755 --- a/src/training/graph_group_async.h +++ b/src/training/graph_group_async.h @@ -63,8 +63,9 @@ class AsyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void save(bool final = false) override; void save(Ptr, bool final = false); - Ptr collectStats() { - return GraphGroup::collectStats(graphs_[0], builders_[0]); + // @TODO: give it a fake batch generator which own vocabs instead of passing vocabs + Ptr collectStats(const std::vector>& vocabs) { + return GraphGroup::collectStats(graphs_[0], builders_[0], vocabs); } virtual void finalize() override; diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index f5cbafd36..bac76b945 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -597,8 +597,8 @@ void MultiNodeGraphGroup::execute(Ptr batch) { if(tau_ > 1) { std::vector fakeLength = {1, 1}; - auto fb = data::CorpusBatch::fakeBatch( - fakeLength, num_seen_sentences, NULL); + std::vector> vocabs; + auto fb = data::CorpusBatch::fakeBatch(fakeLength, vocabs, num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); scheduler_->update(cost, fb); } else { diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index c86225ebc..ae4f1400f 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -454,8 +454,8 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { /** * Collect statistics from first client's graph. */ - Ptr collectStats() { - return GraphGroup::collectStats(clientGraphs_[0], clientBuilders_[0]); + Ptr collectStats(const std::vector>& vocabs) { + return GraphGroup::collectStats(clientGraphs_[0], clientBuilders_[0], vocabs); } }; } // namespace marian diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp index 328657057..05abd6d5f 100755 --- a/src/training/graph_group_multinode_sync.cpp +++ b/src/training/graph_group_multinode_sync.cpp @@ -224,8 +224,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { if(tau_ > 1) { std::vector fakeLength = {1, 1}; - auto fb - = data::CorpusBatch::fakeBatch(fakeLength, num_seen_sentences, NULL); + auto fb = data::CorpusBatch::fakeBatch(fakeLength, std::vector>(), num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); scheduler_->update(cost, fb); } else { diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index bfef2050c..55474b3ed 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -220,9 +220,9 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { /** * Collect statistics from first client's graph. */ - Ptr collectStats() { + Ptr collectStats(const std::vector>& vocabs) { return GraphGroup::collectStats( - clientGraphs_[0], clientBuilders_[0], devices_.size()); + clientGraphs_[0], clientBuilders_[0], vocabs, devices_.size()); } }; } // namespace marian diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 1eb589d91..93a6c0699 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -125,8 +125,8 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { }); } - Ptr collectStats() { - return GraphGroup::collectStats(graph_, builder_); + Ptr collectStats(const std::vector>& vocabs) { + return GraphGroup::collectStats(graph_, builder_, vocabs); } virtual void finalize() override { finalized_ = true; } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 6699484c1..c3b025e31 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -100,10 +100,10 @@ void SyncGraphGroup::initializeAvg() { comm_->foreach(init, /*parallel=*/false); // @TODO: is sequential operation necessary here? (is the allocation stuff sufficiently reentrant or thread-separated?) } -Ptr SyncGraphGroup::collectStats() { +Ptr SyncGraphGroup::collectStats(const std::vector>& vocabs) { // @TODO: This should only run on MPI process 0. Also we can share vv this vv expression with update(). size_t multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_; - return GraphGroup::collectStats(graphs_[0], builders_[0], multiplier); + return GraphGroup::collectStats(graphs_[0], builders_[0], vocabs, multiplier); } void SyncGraphGroup::update(Ptr batch) /*override*/ { diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index 478166996..39008fda2 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -42,7 +42,7 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { void load() override; void save(bool final = false) override; - Ptr collectStats(); + Ptr collectStats(const std::vector>&); // @TODO: consider to make this a virtual as well? Currently it is a template dispatch }; } // namespace marian diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 1585fd8a4..1e9491cc8 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -47,7 +47,7 @@ class Scheduler : public TrainingObserver { public: Scheduler(Ptr options, Ptr state) : options_(options), state_(state), - dispIndex_{options_->get("disp-wps-index", -1)} { + dispIndex_{options_->get("disp-label-index", -1)} { state_->eta = getLearningRate(*state); } @@ -169,13 +169,11 @@ class Scheduler : public TrainingObserver { size_t batchSize = 0; // number of sentences in batch size_t batchLabels = 0; // number of target words in batch - size_t batchDisp = 0; // number of words in chosen sub-batch, last by default unless set differently in dispIndex_. Used for displaying speed. for(const auto& batch : batches) { if (batch) { // (nullptr is allowed as result of split) batchSize += batch->size(); - batchLabels += batch->words(-1); - batchDisp += batch->words(dispIndex_); + batchLabels += batch->words(dispIndex_); } } @@ -203,7 +201,7 @@ class Scheduler : public TrainingObserver { state_->costCount += batchSize; } - state_->wordsDisp += batchDisp; // words at given input processed since last display, for speed display + state_->wordsDisp += batchLabels; // words at given input processed since last display, for speed display state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed diff --git a/src/training/training.h b/src/training/training.h index 2209b96e2..94d8c8900 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -40,9 +40,10 @@ class Train : public ModelTask { "[batching] Collecting statistics for batch fitting with step size " "{}", options_->get("mini-batch-fit-step")); - // @TODO, better fake batch with vocabulary + // @TODO this should receive a function object that can generate a fake batch + // that way vocabs would not be exposed. auto model = New(options_); - THREAD_GUARD(stats = model->collectStats()); + THREAD_GUARD(stats = model->collectStats(dataset->getVocabs())); LOG(info, "[batching] Done"); } From 5b0885670b11a73a5122b086a2b7f9f37ccf8b92 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 10 Jan 2019 16:15:23 -0800 Subject: [PATCH 101/838] renamed function name --- src/models/bert.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 232f7e88f..1a7fb58e1 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -13,7 +13,7 @@ class BertBatch : public CorpusBatch { std::mt19937& eng_; std::vector maskedPositions_; - std::vector maskedIndices_; + std::vector maskedWords_; std::vector sentenceIndices_; std::string maskSymbol_; @@ -87,13 +87,13 @@ class BertBatch : public CorpusBatch { for(int i : selected) { maskedPositions_.push_back(i); // where is the original word? - maskedIndices_.push_back(words[i]); // what is the original word? + maskedWords_.push_back(words[i]); // what is the original word? words[i] = maskOut(words[i], maskId); // mask that position } } const std::vector& bertMaskedPositions() { return maskedPositions_; } - const std::vector& bertMaskedIndices() { return maskedIndices_; } + const std::vector& bertMaskedWords() { return maskedWords_; } const std::vector& bertSentenceIndices() { return sentenceIndices_; } }; @@ -206,7 +206,7 @@ class BertMaskedLM : public ClassifierBase { auto context = encoderStates[0]->getContext(); auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries - auto bertMaskedIndices = graph->indices(bertBatch->bertMaskedIndices()); // vocab ids of entries that have been masked + auto bertMaskedWords = graph->indices(bertBatch->bertMaskedWords()); // vocab ids of entries that have been masked int dimModel = context->shape()[-1]; int dimBatch = context->shape()[-2]; @@ -237,7 +237,7 @@ class BertMaskedLM : public ClassifierBase { auto state = New(); state->setLogProbs(logits); - state->setTargetIndices(bertMaskedIndices); + state->setTargetIndices(bertMaskedWords); return state; } From bc9851ccea0f3d55d4ca1f315c3e891de7ccf84e Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 10 Jan 2019 16:35:14 -0800 Subject: [PATCH 102/838] randomize fake batch content --- src/data/corpus_base.h | 6 ++++-- src/models/bert.h | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index e25d7d451..a1c70e63d 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -315,12 +315,14 @@ class CorpusBatch : public Batch { size_t idx = 0; for(auto len : lengths) { - // data: gets initialized to 0. No EOS symbol is distinguished. auto sb = New(batchSize, len, vocabs[idx]); // set word indices to different values to avoid same hashes - std::fill(sb->data().begin(), sb->data().end(), (unsigned int)idx++); + // rand() is OK, this does not affect state in any way + std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), + [&](Word) { return rand() % vocabs[idx]->size(); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); + idx++; batches.push_back(sb); } diff --git a/src/models/bert.h b/src/models/bert.h index 1a7fb58e1..69b826993 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -87,13 +87,13 @@ class BertBatch : public CorpusBatch { for(int i : selected) { maskedPositions_.push_back(i); // where is the original word? - maskedWords_.push_back(words[i]); // what is the original word? + maskedWords_.push_back(words[i]); // what is the original word? words[i] = maskOut(words[i], maskId); // mask that position } } const std::vector& bertMaskedPositions() { return maskedPositions_; } - const std::vector& bertMaskedWords() { return maskedWords_; } + const std::vector& bertMaskedWords() { return maskedWords_; } const std::vector& bertSentenceIndices() { return sentenceIndices_; } }; From c5839f07e0805cd645c86884fe758fac4c9a1c2d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 10 Jan 2019 21:14:55 -0800 Subject: [PATCH 103/838] towards factored embeddings; towards csr_dot() operation --- src/CMakeLists.txt | 1 + src/common/config_parser.cpp | 3 ++ src/graph/expression_operators.cpp | 7 ++++ src/graph/expression_operators.h | 2 ++ src/graph/node_operators_binary.h | 54 ++++++++++++++++++++++++++++++ src/layers/generic.h | 33 ++---------------- src/models/transformer.h | 4 +++ src/tensors/cpu/prod.cpp | 9 +++++ src/tensors/gpu/backend.h | 17 ++++------ src/tensors/gpu/prod.cu | 37 ++++++++++++++++++++ src/tensors/gpu/prod.h | 5 +++ src/tensors/tensor_operators.h | 1 + vs/Marian.vcxproj | 1 + vs/Marian.vcxproj.filters | 3 ++ 14 files changed, 136 insertions(+), 41 deletions(-) mode change 100644 => 100755 src/tensors/gpu/prod.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 03524117b..a3b2724ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,6 +47,7 @@ add_library(marian STATIC graph/node_initializers.cpp layers/convolution.cpp + layers/generic.cpp layers/loss.cpp layers/weight.cpp diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 70a30a5fd..e301fb30b 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -30,6 +30,7 @@ const std::set PATHS = { "train-sets", "vocabs", "embedding-vectors", + "embedding-factors", "valid-sets", "valid-script-path", "valid-log", @@ -385,6 +386,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Fix source embeddings. Affects all encoders"); cli.add("--embedding-fix-trg", "Fix target embeddings. Affects all decoders"); + cli.add_nondefault>("--embedding-factors", + "Paths to (factor map, factor list) file for factored embeddings"); cli.add("--multi-node", "Enable asynchronous multi-node training through MPI (and legacy sync if combined with --sync-sgd)"); diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index cdfeff271..12f5464a6 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -417,6 +417,13 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { } } +// multiply a CSR matrix A with a matrix B +// A[i,j] is at A_values[A_offsets[i]+k], where k is position of j in A_indices[A_offsets[i]:A_offsets[i+1]] +// Result shape is (Aoffsets.size() - 1, B->shape(-1)) +Expr csr_dot(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) { + return Expression(A_values, A_indices, A_offsets, B); +} + // swap the last two axes // @TODO: change to swapAxes(a, -1, -2) Expr transpose(Expr a) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 7c3797069..080fbf2d5 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -112,6 +112,8 @@ Expr affine(Expr a, bool transB = false, float scalar = 1.f); +Expr csr_dot(Expr Avalues, Expr Aindices, Expr Aoffsets, Expr B); + Expr transpose(Expr a); Expr transpose(Expr a, const std::vector& axes); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 868ee4ebd..e27aa6f74 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -409,6 +409,60 @@ class DotBatchedNodeOp : public NaryNodeOp { const std::string color() override { return "orange"; } }; +class CSRDotNodeOp : public NaryNodeOp { +public: + CSRDotNodeOp(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) + : NaryNodeOp({ A_values, A_indices, A_offsets, B }, newShape(A_values, A_indices, A_offsets, B)) { + matchOrAbort(A_indices->value_type()); + matchOrAbort(A_offsets->value_type()); + } + + Shape newShape(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) { + ABORT_IF(A_values->shape().size() != 1 || A_indices->shape().size() != 1 || A_offsets->shape().size() != 1, + "Sparse matrix components must all be vectors."); + auto outShape = B->shape(); + outShape.set(0, A_offsets->shape()[0] - 1); // A_offsets = A.numRows + 1 + return outShape; + } + + NodeOps forwardOps() override { + // C = dot(A, B) + return {NodeOp(CSRProd(val_, + child(0)->val(), + child(1)->val(), + child(2)->val(), + child(3)->val()))}; + } + + NodeOps backwardOps() override { +#if 1 + return {}; // this is coming next +#else + return {NodeOp(Prod(child(0)->grad(), + adj_, + child(1)->val(), + false, + true, + 1.0, + scalar_)), + NodeOp(Prod(child(1)->grad(), + child(0)->val(), + adj_, + true, + false, + 1.0, + scalar_))}; +#endif + } + + const std::string type() override { return "•"; } + + const std::string color() override { return "orange"; } +}; + + + + struct ScalarProductNodeOp : public NaryNodeOp { ScalarProductNodeOp(Expr a, Expr b, int axis) : NaryNodeOp({a, b}, newShape(a, b, axis)) {} diff --git a/src/layers/generic.h b/src/layers/generic.h index a9e2be016..0289e57b7 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -176,38 +176,11 @@ class Output : public LayerBase, public IUnaryLayer { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; + Ptr embeddingFactorMapping_; public: - Embedding(Ptr graph, Ptr options) : LayerBase(graph, options) { - std::string name = opt("prefix"); - int dimVoc = opt("dimVocab"); - int dimEmb = opt("dimEmb"); - - bool fixed = opt("fixed", false); - - NodeInitializer initFunc = inits::glorot_uniform; - if (options_->has("embFile")) { - std::string file = opt("embFile"); - if (!file.empty()) { - bool norm = opt("normalization", false); - initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm); - } - } - - E_ = graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed); - } + Embedding(Ptr graph, Ptr options); - std::tuple apply(Ptr subBatch) const override final { - auto graph = E_->graph(); - int dimBatch = (int)subBatch->batchSize(); - int dimEmb = E_->shape()[-1]; - int dimWords = (int)subBatch->batchWidth(); - // @TODO: merge this with below. Currently can't only due to the extra beam dimension - auto chosenEmbeddings = rows(E_, subBatch->data()); - auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb }); - auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, - inits::from_vector(subBatch->mask())); - return std::make_tuple(batchEmbeddings, batchMask); - } + std::tuple apply(Ptr subBatch) const override final; // special version used in decoding Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const override final { diff --git a/src/models/transformer.h b/src/models/transformer.h index d91dfb7f8..c142d0dd7 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -511,6 +511,10 @@ class EncoderTransformer : public Transformer { embFactory("embFile", embFiles[subBatchIndex]) ("normalization", opt("embedding-normalization")); } + if (options_->has("embedding-factors")) { + embFactory("embedding-factors", opt("embedding-factors")); + embFactory("vocab", opt>("vocabs")[subBatchIndex]); + } return embFactory.construct(graph_); } diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 69923f872..f3dcb4fee 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -167,5 +167,14 @@ void ProdWithBias(marian::Tensor C, cpu::int16::AddBias(C, bias); } +void CSRProd(marian::Tensor C, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B) { + C, A_values, A_indices, A_offsets, B; + ABORT("CSRProd is not yet implemented for CPU"); +} + } // namespace cpu } // namespace marian diff --git a/src/tensors/gpu/backend.h b/src/tensors/gpu/backend.h index ab1968c1f..87f6407c5 100755 --- a/src/tensors/gpu/backend.h +++ b/src/tensors/gpu/backend.h @@ -6,6 +6,7 @@ #include #include +#include #include namespace marian { @@ -15,11 +16,13 @@ class Backend : public marian::Backend { public: Backend(DeviceId deviceId, size_t seed) : marian::Backend(deviceId, seed) { setDevice(); - setHandles(); + cublasCreate(&cublasHandle_); + cusparseCreate(&cusparseHandle_); } ~Backend() { setDevice(); + cusparseDestroy(cusparseHandle_); cublasDestroy(cublasHandle_); } @@ -28,19 +31,11 @@ class Backend : public marian::Backend { void synchronize() override { cudaStreamSynchronize(0); } cublasHandle_t getCublasHandle() { return cublasHandle_; } + cusparseHandle_t getCusparseHandle() { return cusparseHandle_; } private: cublasHandle_t cublasHandle_; - - void setHandles() { - cublasHandle_ = create_handle(); - } - - cublasHandle_t create_handle() { - cublasHandle_t cublasHandle; - cublasCreate(&cublasHandle); - return cublasHandle; - } + cusparseHandle_t cusparseHandle_; }; } // namespace gpu } // namespace marian diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 9da32e65a..d2ac84a2a 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -1,5 +1,6 @@ #include +#include // clang-format off #include "tensors/gpu/prod.h" @@ -224,5 +225,41 @@ void ProdBatched(marian::Tensor C, allocator->free(mp_cptr); } +void CSRProd(marian::Tensor C, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B) { + cudaSetDevice(C->getDeviceId().no); + auto cusparseHandle = std::static_pointer_cast(C->getBackend()) + ->getCusparseHandle(); + const auto& shapeB = B->shape(); + const auto& shapeC = C->shape(); + int k = (int)shapeB[0]; // number of columns of sparse matrix A = #rows of B + int n = (int)shapeB.elements() / k; // number of columns of dense matrices B and C + int m = (int)A_offsets->shape().elements() - 1; // number of rows of sparse matrix A + ABORT_IF(m != shapeC[0], "CSR matrix has wrong number of rows"); + ABORT_IF(A_values->shape() != A_indices->shape(), "CSR constituents has inconsistent dimensions"); + int nnz = (int)A_values->shape().elements(); + float alpha = 1; + float beta = 0; + cusparseMatDescr_t descrA; + cusparseCreateMatDescr(&descrA); + cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); + cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); + auto rc = cusparseScsrmm(cusparseHandle, + /*transA=*/ CUSPARSE_OPERATION_NON_TRANSPOSE, + m, n, k, nnz, &alpha, descrA, + /*csrValA=*/ A_values->data(), + /*csrRowPtrA=*/ (int*)A_indices->data(), + /*csrColIndA=*/ (int*)A_offsets->data(), + B->data(), + /*ldb=*/ k, + &beta, + C->data(), + /*ldc=*/ m); + cusparseDestroyMatDescr(descrA); +} + } // namespace gpu } // namespace marian diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h old mode 100644 new mode 100755 index ce1f6cdca..f1c59bd9c --- a/src/tensors/gpu/prod.h +++ b/src/tensors/gpu/prod.h @@ -33,5 +33,10 @@ void ProdBatched(marian::Tensor C, bool transB, float beta = 0, float scalar = 1); +void CSRProd(marian::Tensor C, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B); } // namespace gpu } // namespace marian diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index 2cac284ab..692d3a2e0 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -77,6 +77,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) +DISPATCH5(CSRProd, marian::Tensor, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&) DISPATCH2(Softmax, marian::Tensor, marian::Tensor) DISPATCH3(SoftmaxGrad, marian::Tensor, marian::Tensor, marian::Tensor) diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 64faded63..c29fd372c 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -575,6 +575,7 @@ true true + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 9e5cbf5f4..c70527a1e 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -481,6 +481,9 @@ examples\iris + + layers + From 6305f225dc2c8f7f3f39875054a7f17a58ccf4c9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 10 Jan 2019 21:57:28 -0800 Subject: [PATCH 104/838] added gradient of csr_dot; added transA flag to CSRPrd --- src/graph/node_operators_binary.h | 31 ++++++------------------------ src/tensors/cpu/prod.cpp | 5 +++-- src/tensors/gpu/prod.cu | 32 +++++++++++++++++++------------ src/tensors/tensor_operators.h | 2 +- 4 files changed, 30 insertions(+), 40 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index e27aa6f74..411d89b15 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -428,31 +428,15 @@ class CSRDotNodeOp : public NaryNodeOp { NodeOps forwardOps() override { // C = dot(A, B) return {NodeOp(CSRProd(val_, - child(0)->val(), - child(1)->val(), - child(2)->val(), - child(3)->val()))}; + child(0)->val(), child(1)->val(), child(2)->val(), child(3)->val(), + /*transA=*/false, /*beta=*/0))}; } NodeOps backwardOps() override { -#if 1 - return {}; // this is coming next -#else - return {NodeOp(Prod(child(0)->grad(), - adj_, - child(1)->val(), - false, - true, - 1.0, - scalar_)), - NodeOp(Prod(child(1)->grad(), - child(0)->val(), - adj_, - true, - false, - 1.0, - scalar_))}; -#endif + return {nullptr, // can't backprop into the sparse matrix, as it would be dense + NodeOp(CSRProd(child(1)->grad(), + child(0)->val(), child(1)->val(), child(2)->val(), adj_, + /*transA=*/true, /*beta=*/1))}; } const std::string type() override { return "•"; } @@ -460,9 +444,6 @@ class CSRDotNodeOp : public NaryNodeOp { const std::string color() override { return "orange"; } }; - - - struct ScalarProductNodeOp : public NaryNodeOp { ScalarProductNodeOp(Expr a, Expr b, int axis) : NaryNodeOp({a, b}, newShape(a, b, axis)) {} diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index f3dcb4fee..478fe9901 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -171,8 +171,9 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, - const marian::Tensor& B) { - C, A_values, A_indices, A_offsets, B; + const marian::Tensor& B, + bool transA, float beta) { + C, A_values, A_indices, A_offsets, B, transA, beta; ABORT("CSRProd is not yet implemented for CPU"); } diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index d2ac84a2a..80e774864 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -229,35 +229,43 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, - const marian::Tensor& B) { + const marian::Tensor& B, + bool transA, float beta) { cudaSetDevice(C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); - const auto& shapeB = B->shape(); const auto& shapeC = C->shape(); - int k = (int)shapeB[0]; // number of columns of sparse matrix A = #rows of B - int n = (int)shapeB.elements() / k; // number of columns of dense matrices B and C - int m = (int)A_offsets->shape().elements() - 1; // number of rows of sparse matrix A - ABORT_IF(m != shapeC[0], "CSR matrix has wrong number of rows"); + const auto& shapeB = B->shape(); + auto numValues = A_values->shape().elements(); + auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length + auto rowsC = shapeC[0]; + auto colsC = shapeC.elements() / rowsC; + auto rowsB = shapeB[0]; + auto colsB = shapeB.elements() / rowsB; + auto rowsA = transA ? rowsB : numOffsets; // we don't know the dimension of the sparse axis from A directly + auto colsA = transA ? numOffsets : rowsB; + ABORT_IF((transA ? colsA : rowsA) != rowsC || (transA ? rowsA : colsA) != rowsB || colsB != colsC, "Inconsistent dimensions in CSR product"); ABORT_IF(A_values->shape() != A_indices->shape(), "CSR constituents has inconsistent dimensions"); - int nnz = (int)A_values->shape().elements(); float alpha = 1; - float beta = 0; cusparseMatDescr_t descrA; cusparseCreateMatDescr(&descrA); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); auto rc = cusparseScsrmm(cusparseHandle, - /*transA=*/ CUSPARSE_OPERATION_NON_TRANSPOSE, - m, n, k, nnz, &alpha, descrA, + transA ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, + /*m=*/ rowsA, // #rows of sparse A + /*n=*/ colsB, // #cols of dense B and C + /*k=*/ colsA, // #cols of sparse A + /*nnz=*/ (int)numValues, + &alpha, descrA, /*csrValA=*/ A_values->data(), /*csrRowPtrA=*/ (int*)A_indices->data(), /*csrColIndA=*/ (int*)A_offsets->data(), B->data(), - /*ldb=*/ k, + /*ldb=*/ rowsB, &beta, C->data(), - /*ldc=*/ m); + /*ldc=*/ rowsC); cusparseDestroyMatDescr(descrA); } diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index 692d3a2e0..d84ca7974 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -77,7 +77,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) -DISPATCH5(CSRProd, marian::Tensor, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&) +DISPATCH7(CSRProd, marian::Tensor, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, float) DISPATCH2(Softmax, marian::Tensor, marian::Tensor) DISPATCH3(SoftmaxGrad, marian::Tensor, marian::Tensor, marian::Tensor) From fab4fdf1f69256e579dc872bf6223e21f758b448 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 11 Jan 2019 08:31:51 -0800 Subject: [PATCH 105/838] learnable positional embeddings --- src/common/config_parser.cpp | 2 + src/graph/node_initializers.cpp | 21 ++++++ src/graph/node_initializers.h | 2 + src/models/bert.h | 64 ++++++++++++----- src/models/model_factory.cpp | 2 +- src/models/transformer.h | 18 +---- src/models/transformer_factory.h | 4 +- src/training/validator.cpp | 6 ++ src/training/validator.h | 118 +++++++++++++++++++++++++++++-- 9 files changed, 196 insertions(+), 41 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index da6f7d6a9..e745e99d5 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -200,6 +200,8 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.add("--transformer-postprocess", "Operation after each transformer layer: d = dropout, a = add, n = normalize", "dan"); + cli.add("--transformer-learned-positions", + "Use learned positional embeddings instead of trigonometric embeddings"); cli.add("--bert-mask-symbol", "Masking symbol for BERT masked-LM training", "[MASK]"); cli.add("--bert-sep-symbol", "Sentence separator symbol for BERT next sentence prediction training", "[SEP]"); diff --git a/src/graph/node_initializers.cpp b/src/graph/node_initializers.cpp index dc71562d8..84f60044f 100755 --- a/src/graph/node_initializers.cpp +++ b/src/graph/node_initializers.cpp @@ -157,6 +157,27 @@ NodeInitializer from_item(const io::Item& item) { } } +NodeInitializer positions(int start) { + return [start](Tensor t) { + int dimEmb = t->shape()[-1]; + int dimWords = t->size() / dimEmb; + + float numTimescales = (float)dimEmb / 2; + float logTimescaleIncrement = std::log(10000.f) / (numTimescales - 1.f); + + std::vector vPos(dimEmb * dimWords, 0); + for(int p = start; p < dimWords + start; ++p) { + for(int i = 0; i < numTimescales; ++i) { + float v = p * std::exp(i * -logTimescaleIncrement); + vPos[(p - start) * dimEmb + i ] = std::sin(v); + vPos[(p - start) * dimEmb + (int)numTimescales + i] = std::cos(v); // @TODO: is int vs. float correct for num_timescales? + } + } + + from_vector(vPos)(t); + }; +} + } // namespace inits } // namespace marian diff --git a/src/graph/node_initializers.h b/src/graph/node_initializers.h index 58690e4fb..185ab1191 100755 --- a/src/graph/node_initializers.h +++ b/src/graph/node_initializers.h @@ -42,6 +42,8 @@ NodeInitializer from_vector(const std::vector& v); NodeInitializer from_item(const io::Item& item); +NodeInitializer positions(int start); + NodeInitializer from_sparse_vector( std::pair, std::vector>& v); diff --git a/src/models/bert.h b/src/models/bert.h index 69b826993..9994f8668 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -11,7 +11,7 @@ namespace data { class BertBatch : public CorpusBatch { private: std::mt19937& eng_; - + std::vector maskedPositions_; std::vector maskedWords_; std::vector sentenceIndices_; @@ -35,7 +35,7 @@ class BertBatch : public CorpusBatch { Word randWord = (*randomWord_)(eng_); if(dontMask_.count(randWord) > 0) // the random word is a forbidden word return mask; // hence return mask symbol - else + else return randWord; // else return the random word } else { // for 80% of words apply mask symbol return mask; @@ -43,11 +43,11 @@ class BertBatch : public CorpusBatch { } public: - BertBatch(Ptr batch, + BertBatch(Ptr batch, std::mt19937& engine, float maskFraction, - const std::string& maskSymbol, - const std::string& sepSymbol, + const std::string& maskSymbol, + const std::string& sepSymbol, const std::string& clsSymbol) : CorpusBatch(*batch), eng_(engine), maskSymbol_(maskSymbol), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { @@ -68,7 +68,7 @@ class BertBatch : public CorpusBatch { ABORT_IF(sepId == subBatch->vocab()->getUnkId(), "BERT separator symbol {} not found in vocabulary", sepSymbol_); - + ABORT_IF(clsId == subBatch->vocab()->getUnkId(), "BERT class symbol {} not found in vocabulary", clsSymbol_); @@ -84,12 +84,26 @@ class BertBatch : public CorpusBatch { selected.push_back(i); std::shuffle(selected.begin(), selected.end(), eng_); selected.resize(std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices - + for(int i : selected) { maskedPositions_.push_back(i); // where is the original word? maskedWords_.push_back(words[i]); // what is the original word? words[i] = maskOut(words[i], maskId); // mask that position } + + int dimBatch = subBatch->batchSize(); + int dimWords = subBatch->batchWidth(); + + sentenceIndices_.resize(words.size()); + std::vector sentPos(dimBatch, 0); + for(int i = 0; i < dimWords; ++i) { + for(int j = 0; j < dimBatch; ++j) { + int k = i * dimBatch + j; + sentenceIndices_[k] = sentPos[j]; + if(words[k] == sepId) + sentPos[j]++; + } + } } const std::vector& bertMaskedPositions() { return maskedPositions_; } @@ -101,7 +115,7 @@ class BertBatch : public CorpusBatch { class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { public: - BertEncoderClassifier(Ptr options) + BertEncoderClassifier(Ptr options) : EncoderClassifier(options) {} std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { @@ -114,6 +128,11 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { opt("bert-class-symbol")); return EncoderClassifier::apply(graph, bertBatch, clearGraph); } + + // for externally created BertBatch for instance in BertValidator + std::vector> apply(Ptr graph, Ptr bertBatch, bool clearGraph) { + return EncoderClassifier::apply(graph, bertBatch, clearGraph); + } }; // @TODO: this should be in transformer.h @@ -121,7 +140,9 @@ class BertEncoder : public EncoderTransformer { public: BertEncoder(Ptr options) : EncoderTransformer(options) {} - Expr addSentenceEmbeddings(Expr embeddings, int start, Ptr batch) const { + Expr addSentenceEmbeddings(Expr embeddings, + Ptr batch, + bool learnedPosEmbeddings) const { Ptr bertBatch = std::dynamic_pointer_cast(batch); ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); @@ -130,24 +151,29 @@ class BertEncoder : public EncoderTransformer { int dimBatch = embeddings->shape()[-2]; int dimWords = embeddings->shape()[-3]; - auto sentenceEmbeddings = embedding(graph_) - ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B - ("dimEmb", dimEmb) - .construct(); + Expr sentenceEmbeddings; + if(learnedPosEmbeddings) { + sentenceEmbeddings = embedding(graph_) + ("prefix", "Wsent") + ("dimVocab", 2) // sentence A or sentence B + ("dimEmb", dimEmb) + .construct(); + } else { + // trigonometric positions, no backprob + sentenceEmbeddings = graph_->constant({2, dimEmb}, inits::positions(0)); + } - // @TODO: note this is going to be really slow due to atomicAdd in backward step - // with only two classes; - // instead two masked reduce operations, maybe in parallel streams? auto sentenceIndices = graph_->indices(bertBatch->bertSentenceIndices()); + auto signal = rows(sentenceEmbeddings, sentenceIndices); signal = reshape(signal, {dimWords, dimBatch, dimEmb}); return embeddings + signal; } virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { - input = addPositionalEmbeddings(input, start, true); // true for BERT - input = addSentenceEmbeddings(input, start, batch); + bool learnedPosEmbeddings = opt("transformer-learned-positions", true); + input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); + input = addSentenceEmbeddings(input, batch, learnedPosEmbeddings); return input; } }; diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 0d66b95d8..bf0de22d6 100644 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -233,7 +233,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "transformer") // @TODO: replace with 'bert-encoder' + ("type", "bert-encoder") // @TODO: replace with 'bert-encoder' ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-masked-lm") // diff --git a/src/models/transformer.h b/src/models/transformer.h index 188080a99..d57f23b95 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -68,20 +68,8 @@ class Transformer : public EncoderOrDecoderBase { signal = reshape(signal, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; } else { - float num_timescales = (float)dimEmb / 2; - float log_timescale_increment = std::log(10000.f) / (num_timescales - 1.f); - - std::vector vPos(dimEmb * dimWords, 0); - for(int p = start; p < dimWords + start; ++p) { - for(int i = 0; i < num_timescales; ++i) { - float v = p * std::exp(i * -log_timescale_increment); - vPos[(p - start) * dimEmb + i] = std::sin(v); - vPos[(p - start) * dimEmb + (int)num_timescales + i] = std::cos(v); // @TODO: is int vs. float correct for num_timescales? - } - } - - // shared across batch entries - auto signal = graph_->constant({dimWords, 1, dimEmb}, inits::from_vector(vPos)); + auto signal = graph_->constant({dimWords, 1, dimEmb}, + inits::positions(start)); embeddings = embeddings + signal; } @@ -573,7 +561,7 @@ class EncoderTransformer : public Transformer { } // according to paper embeddings are scaled up by \sqrt(d_m) auto scaledEmbeddings = std::sqrt((float)dimEmb) * batchEmbeddings; - + scaledEmbeddings = addSpecialEmbeddings(scaledEmbeddings, /*start=*/0, batch); // reorganize batch and timestep diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h index aa31e4d15..16ce91bdc 100755 --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -9,6 +9,6 @@ //#include "layers/factory.h" namespace marian { -Ptr NewEncoderTransformer(Ptr options); -Ptr NewDecoderTransformer(Ptr options); +static Ptr NewEncoderTransformer(Ptr options); +static Ptr NewDecoderTransformer(Ptr options); } // namespace marian diff --git a/src/training/validator.cpp b/src/training/validator.cpp index f63c9d0b5..14b854ff3 100644 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -34,6 +34,12 @@ std::vector>> Validators( } else if(metric == "accuracy") { auto validator = New(vocabs, config); validators.push_back(validator); + } else if(metric == "bert-lm-accuracy") { + auto validator = New(vocabs, config, true); + validators.push_back(validator); + } else if(metric == "bert-sentence-accuracy") { + auto validator = New(vocabs, config, false); + validators.push_back(validator); } else { LOG_VALID(warn, "Unrecognized validation metric: {}", metric); } diff --git a/src/training/validator.h b/src/training/validator.h index 1ff6d59ee..f22dd56e0 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -15,6 +15,7 @@ #include "translator/output_collector.h" #include "translator/output_printer.h" #include "translator/scorers.h" +#include "models/bert.h" #include #include @@ -261,9 +262,9 @@ class AccuracyValidator : public Validator { // graph->forward(); // std::unique_lock lock(mutex_); - // totalLabels += labels->shape().elements(); - // correct += correct->scalar(); - + // totalLabels += labels->shape().elements(); + // correct += correct->scalar(); + builder->clear(graph); auto logits = builder->build(graph, batch); graph->forward(); @@ -292,7 +293,7 @@ class AccuracyValidator : public Validator { } std::unique_lock lock(mutex_); - totalLabels += thisLabels; + totalLabels += thisLabels; correct += thisCorrect; }; @@ -306,6 +307,115 @@ class AccuracyValidator : public Validator { } }; +class BertAccuracyValidator : public Validator { +private: + bool evalMaskedLM_{true}; + +public: + BertAccuracyValidator(std::vector> vocabs, Ptr options, bool evalMaskedLM) + : Validator(vocabs, options, /*lowerIsBetter=*/false), + evalMaskedLM_(evalMaskedLM) { + createBatchGenerator(/*isTranslating=*/false); + + // @TODO: check if this is required. + Ptr opts = New(); + opts->merge(options); + opts->set("inference", true); + builder_ = models::from_options(opts, models::usage::raw); + } + + std::string type() override { + if(evalMaskedLM_) + return "bert-lm-accuracy"; + else + return "bert-sentence-accuracy"; + } + +protected: + virtual float validateBG(const std::vector>& graphs) override { + + size_t correct = 0; + size_t totalLabels = 0; + size_t batchId = 0; + + { + threadPool_.reserve(graphs.size()); + + TaskBarrier taskBarrier; + for(auto batch : *batchGenerator_) { + auto task = [=, &correct, &totalLabels](size_t id) { + thread_local Ptr graph; + thread_local auto builder = models::from_options(options_, models::usage::raw); + thread_local std::unique_ptr engine; + + if(!graph) { + graph = graphs[id % graphs.size()]; + } + + if(!engine) + engine.reset(new std::mt19937(Config::seed + id)); + + auto bertBatch = New(batch, + *engine, + options_->get("bert-masking-fraction"), + options_->get("bert-mask-symbol"), + options_->get("bert-sep-symbol"), + options_->get("bert-class-symbol")); + + builder->clear(graph); + auto classifierStates = std::dynamic_pointer_cast(builder)->apply(graph, bertBatch, true); + graph->forward(); + + auto maskedLMLogits = classifierStates[0]->getLogProbs(); + const auto& maskedLMLabels = bertBatch->bertMaskedWords(); + + auto sentenceLogits = classifierStates[1]->getLogProbs(); + const auto& sentenceLabels = bertBatch->back()->data(); + + auto count = [=, &correct, &totalLabels](Expr logits, const std::vector& labels) { + IndexType cols = logits->shape()[-1]; + size_t thisCorrect = 0; + size_t thisLabels = labels.size(); + + std::vector vLogits; + logits->val()->get(vLogits); + + for(int i = 0; i < thisLabels; ++i) { + // CPU-side Argmax + IndexType bestIndex = 0; + float bestValue = std::numeric_limits::lowest(); + for(IndexType j = 0; j < cols; ++j) { + float currValue = vLogits[i * cols + j]; + if(currValue > bestValue) { + bestValue = currValue; + bestIndex = j; + } + } + thisCorrect += (size_t)(bestIndex == labels[i]); + } + + std::unique_lock lock(mutex_); + totalLabels += thisLabels; + correct += thisCorrect; + }; + + if(evalMaskedLM_) + count(maskedLMLogits, maskedLMLabels); + else + count(sentenceLogits, sentenceLabels); + }; + + taskBarrier.push_back(threadPool_.enqueue(task, batchId)); + batchId++; + } + // ~TaskBarrier waits until all are done + } + + return (float)correct / (float)totalLabels; + } +}; + + class ScriptValidator : public Validator { public: ScriptValidator(std::vector> vocabs, Ptr options) From 060c959172afd112815d9d80e303186e5b9c2c74 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 13:36:01 -0800 Subject: [PATCH 106/838] minor fixes --- src/common/utils.cpp | 8 +++--- src/common/utils.h | 8 +++--- src/graph/node_operators_binary.h | 8 +++++- src/layers/constructors.h | 6 ++++ src/layers/generic.h | 6 ---- src/models/decoder.h | 42 +++++++++++++++------------- src/models/s2s.h | 1 + src/models/transformer.h | 21 ++++++++------ src/models/transformer_factory.h | 1 + src/optimizers/optimizers.h | 2 +- src/tensors/cpu/prod.cpp | 3 +- src/tensors/gpu/cuda_helpers.h | 6 ++++ src/tensors/gpu/prod.cu | 31 +++++++++++---------- src/tensors/gpu/prod.h | 4 ++- src/tests/operator_tests.cpp | 46 +++++++++++++++++++++++++++++-- src/training/communicator_nccl.h | 2 +- 16 files changed, 131 insertions(+), 64 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 992c74ac8..bde788359 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -29,7 +29,7 @@ void trimLeft(std::string& s) { // @TODO: use more functions from CLI instead of own implementations void split(const std::string& line, std::vector& pieces, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { size_t begin = 0; size_t pos = 0; @@ -50,7 +50,7 @@ void split(const std::string& line, } std::vector split(const std::string& line, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { std::vector pieces; split(line, pieces, del, keepEmpty); @@ -60,7 +60,7 @@ std::vector split(const std::string& line, // @TODO: splitAny() shares all but 2 expressions with split(). Merge them. void splitAny(const std::string& line, std::vector& pieces, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { size_t begin = 0; size_t pos = 0; @@ -81,7 +81,7 @@ void splitAny(const std::string& line, } std::vector splitAny(const std::string& line, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { std::vector pieces; splitAny(line, pieces, del, keepEmpty); diff --git a/src/common/utils.h b/src/common/utils.h index 7c4d56432..94113a0ec 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -12,20 +12,20 @@ void trimRight(std::string& s); void split(const std::string& line, std::vector& pieces, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::vector split(const std::string& line, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); void splitAny(const std::string& line, std::vector& pieces, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::vector splitAny(const std::string& line, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::string join(const std::vector& words, diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 411d89b15..61eb9ed46 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -419,7 +419,9 @@ class CSRDotNodeOp : public NaryNodeOp { Shape newShape(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) { ABORT_IF(A_values->shape().size() != 1 || A_indices->shape().size() != 1 || A_offsets->shape().size() != 1, - "Sparse matrix components must all be vectors."); + "Sparse matrix components must all be vectors"); + ABORT_IF(A_values->shape() != A_indices->shape(), + "Sparse matrix values and indices must have the same shape"); auto outShape = B->shape(); outShape.set(0, A_offsets->shape()[0] - 1); // A_offsets = A.numRows + 1 return outShape; @@ -434,9 +436,13 @@ class CSRDotNodeOp : public NaryNodeOp { NodeOps backwardOps() override { return {nullptr, // can't backprop into the sparse matrix, as it would be dense +#if 0 + nullptr}; +#else NodeOp(CSRProd(child(1)->grad(), child(0)->val(), child(1)->val(), child(2)->val(), adj_, /*transA=*/true, /*beta=*/1))}; +#endif } const std::string type() override { return "•"; } diff --git a/src/layers/constructors.h b/src/layers/constructors.h index d0ac34873..5ed7f3f53 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -147,4 +147,10 @@ class MLPFactory : public Factory { // @TODO: change naming convention. typedef Accumulator mlp; } // namespace mlp + +typedef ConstructingFactory EmbeddingFactory; +typedef ConstructingFactory ULREmbeddingFactory; + +typedef Accumulator embedding; +typedef Accumulator ulr_embedding; } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h index 0289e57b7..25b51dc13 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -295,10 +295,4 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { ABORT("not implemented"); // ULR cannot be used for decoding } }; - -typedef ConstructingFactory EmbeddingFactory; -typedef ConstructingFactory ULREmbeddingFactory; - -typedef Accumulator embedding; -typedef Accumulator ulr_embedding; } // namespace marian diff --git a/src/models/decoder.h b/src/models/decoder.h index 99fbd4ad6..abf6bbd1e 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -4,6 +4,7 @@ #include "states.h" #include "data/shortlist.h" +#include "layers/constructors.h" #include "layers/generic.h" namespace marian { @@ -31,6 +32,7 @@ class DecoderBase { virtual Ptr step(Ptr, Ptr) = 0; + std::vector> embedding_; // @TODO: move away, also rename virtual void embeddingsFromBatch(Ptr graph, Ptr state, Ptr batch) { @@ -38,30 +40,32 @@ class DecoderBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - auto yEmbFactory = embedding() // - ("dimVocab", dimVoc) // - ("dimEmb", dimEmb); - - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - yEmbFactory("prefix", "Wemb"); - else - yEmbFactory("prefix", prefix_ + "_Wemb"); - - if(options_->has("embedding-fix-trg")) - yEmbFactory("fixed", opt("embedding-fix-trg")); - - if(options_->has("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - yEmbFactory("embFile", embFiles[batchIndex_]) // - ("normalization", opt("embedding-normalization")); + // @TODO: code dup with EncoderTransformer + if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy + embedding_.resize(batch->sets()); + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + embFactory("prefix", "Wemb"); + else + embFactory("prefix", prefix_ + "_Wemb"); + if(options_->has("embedding-fix-trg")) + embFactory("fixed", opt("embedding-fix-trg")); + if(options_->has("embedding-vectors")) { + auto embFiles = opt>("embedding-vectors"); + embFactory("embFile", embFiles[batchIndex_]) // + ("normalization", opt("embedding-normalization")); + } + if (options_->has("embedding-factors")) { + embFactory("embedding-factors", opt>("embedding-factors")); + embFactory("vocab", opt>("vocabs")[batchIndex_]); + } + embedding_[batchIndex_] = embFactory.construct(graph); } - auto yEmb = yEmbFactory.construct(graph); - auto subBatch = (*batch)[batchIndex_]; Expr y, yMask; std::tie - (y, yMask) = yEmb->apply(subBatch); + (y, yMask) = embedding_[batchIndex_]->apply(subBatch); Expr yData; if(shortlist_) { diff --git a/src/models/s2s.h b/src/models/s2s.h index edda79bc8..2f4a45797 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -124,6 +124,7 @@ class EncoderS2S : public EncoderBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); + // @TODO: code dup with Decider and EncoderTransformer; actually diverged by now. Unify this. auto embFactory = embedding() // ("dimVocab", dimVoc) // ("dimEmb", dimEmb); diff --git a/src/models/transformer.h b/src/models/transformer.h index c142d0dd7..48335d1d0 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -6,7 +6,6 @@ #include "marian.h" #include "layers/constructors.h" -#include "layers/factory.h" #include "models/decoder.h" #include "models/encoder.h" #include "models/states.h" @@ -495,7 +494,7 @@ class EncoderTransformer : public Transformer { return embFactory.construct(graph_); } - Ptr createWordEmbeddingLayer(size_t subBatchIndex) const { + Ptr createSourceEmbeddingLayer(size_t subBatchIndex) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); @@ -512,7 +511,7 @@ class EncoderTransformer : public Transformer { ("normalization", opt("embedding-normalization")); } if (options_->has("embedding-factors")) { - embFactory("embedding-factors", opt("embedding-factors")); + embFactory("embedding-factors", opt>("embedding-factors")); embFactory("vocab", opt>("vocabs")[subBatchIndex]); } return embFactory.construct(graph_); @@ -524,6 +523,7 @@ class EncoderTransformer : public Transformer { return apply(batch); } + std::vector> embedding_; // @TODO: move away, also rename Ptr apply(Ptr batch) { int dimEmb = opt("dim-emb"); int dimBatch = (int)batch->size(); @@ -531,12 +531,15 @@ class EncoderTransformer : public Transformer { // create the embedding matrix, considering tying and some other options // embed the source words in the batch Expr batchEmbeddings, batchMask; - Ptr embedding; - if (options_->has("ulr") && options_->get("ulr") == true) - embedding = createULREmbeddingLayer(); // embedding uses ULR - else - embedding = createWordEmbeddingLayer(batchIndex_); - std::tie(batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); + + if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy + embedding_.resize(batch->sets()); + if (options_->has("ulr") && options_->get("ulr") == true) + embedding_[batchIndex_] = createULREmbeddingLayer(); // embedding uses ULR + else + embedding_[batchIndex_] = createSourceEmbeddingLayer(batchIndex_); + } + std::tie(batchEmbeddings, batchMask) = embedding_[batchIndex_]->apply((*batch)[batchIndex_]); // apply dropout over source words float dropoutSrc = inference_ ? 0 : opt("dropout-src"); if(dropoutSrc) { diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h index aa31e4d15..825c32b93 100755 --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -1,3 +1,4 @@ +// @TODO: rename to transformer.h eventually. This is not a Factory as in factory.h. #pragma once #include "marian.h" diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h index 51e43f871..44d603120 100755 --- a/src/optimizers/optimizers.h +++ b/src/optimizers/optimizers.h @@ -26,7 +26,7 @@ class OptimizerBase : public TrainingObserver { // that these hyper-parameters were originally tuned for, then the learning-rate gets // adjusted accordingly. Note: Requires user to also use ce-sum criterion. if (refMBWordsParam_ != 0) - LOG(info, "Note: Learning rate gets automatically adjusted as if minibatch size was {}", refMBWordsParam_); + LOG(info, "[optimizers] Learning rate gets automatically adjusted as if minibatch size was {}", refMBWordsParam_); } static constexpr size_t mbSizeNotProvided = SIZE_MAX; diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 478fe9901..43f99faef 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -172,7 +172,8 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, const marian::Tensor& B, - bool transA, float beta) { + bool transA, + float beta) { C, A_values, A_indices, A_offsets, B, transA, beta; ABORT("CSRProd is not yet implemented for CPU"); } diff --git a/src/tensors/gpu/cuda_helpers.h b/src/tensors/gpu/cuda_helpers.h index ba890490e..920cec8c4 100755 --- a/src/tensors/gpu/cuda_helpers.h +++ b/src/tensors/gpu/cuda_helpers.h @@ -13,6 +13,12 @@ const int MAX_BLOCKS = 65535; "CUDA error {} '{}' - {}:{}: {}", rc, cudaGetErrorString(rc), __FILE__, __LINE__, #expr); \ } while(0) +#define CUBLAS_CHECK(expr) do { \ + cublasStatus_t rc = (expr); \ + ABORT_IF(rc != CUBLAS_STATUS_SUCCESS, \ + "Cublas Error: {} - {}:{}: {}", rc, __FILE__, __LINE__, #expr); \ +} while(0) + #define CUSPARSE_CHECK(expr) do { \ cusparseStatus_t rc = (expr); \ ABORT_IF(rc != CUSPARSE_STATUS_SUCCESS, \ diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 80e774864..4acb01669 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -24,18 +24,18 @@ static void setTensorMode(cublasHandle_t cublasHandle) { default: ABORT("Invalid ENABLE_CUBLAS_TENSOR_OP_MATH_FP32={}", var); } if (mode > 0) { // try whether it can be set --@TODO: check whether this actually works - cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); + CUBLAS_CHECK(cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH)); cublasMath_t actual = CUBLAS_DEFAULT_MATH; cublasGetMathMode(cublasHandle, &actual); if (actual != CUBLAS_TENSOR_OP_MATH) { - LOG(info, "WARNING: TensorCores requested but not available"); + LOG(warn, "[gpu] TensorCores requested but not available"); mode = -1; } } if (mode > 0) - LOG(info, "16-bit TensorCores enabled for float32 matrix operations"); + LOG(info, "[gpu] 16-bit TensorCores enabled for float32 matrix operations"); } - cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH); + CUBLAS_CHECK(cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH)); } void Prod(marian::Tensor C, @@ -76,7 +76,7 @@ void Prod(marian::Tensor C, //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - cublasSgemm(cublasHandle, + CUBLAS_CHECK(cublasSgemm(cublasHandle, opB, opA, n, @@ -89,7 +89,7 @@ void Prod(marian::Tensor C, lda, &beta, C->data(), - ldc); + ldc)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -108,6 +108,7 @@ __global__ void gAddBias(float* out, } } +#if 0 // @TODO: remove, then rename from .cu to .cpp void AddBias(marian::Tensor C, const marian::Tensor bias) { cudaSetDevice(C->getDeviceId().no); @@ -117,9 +118,9 @@ void AddBias(marian::Tensor C, const marian::Tensor bias) { int threads = std::min(MAX_THREADS, length); int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); - gAddBias<<>>(C->data(), bias->data(), length, cols); + gAddBias<<>>(C->data(), bias->data(), length, cols); // @TODO: CUDA_CHECK - cudaStreamSynchronize(0); + CUDA_CHECK(cudaStreamSynchronize(0)); // @BUGBUG: Should not be here. Prod() also does not have this. } void ProdWithBias(marian::Tensor C, @@ -133,6 +134,7 @@ void ProdWithBias(marian::Tensor C, marian::gpu::Prod(C, A, B, transA, transB, beta, scalar); marian::gpu::AddBias(C, bias); } +#endif void ProdBatched(marian::Tensor C, Ptr allocator, @@ -201,7 +203,7 @@ void ProdBatched(marian::Tensor C, setTensorMode(cublasHandle); //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - cublasSgemmBatched(cublasHandle, + CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, opB, opA, n, @@ -215,7 +217,7 @@ void ProdBatched(marian::Tensor C, &beta, mp_cptr->data(), ldc, - batchC); + batchC)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -230,7 +232,8 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, const marian::Tensor& B, - bool transA, float beta) { + bool transA, + float beta) { cudaSetDevice(C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); @@ -248,10 +251,10 @@ void CSRProd(marian::Tensor C, ABORT_IF(A_values->shape() != A_indices->shape(), "CSR constituents has inconsistent dimensions"); float alpha = 1; cusparseMatDescr_t descrA; - cusparseCreateMatDescr(&descrA); + CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); - auto rc = cusparseScsrmm(cusparseHandle, + CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, transA ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, /*m=*/ rowsA, // #rows of sparse A /*n=*/ colsB, // #cols of dense B and C @@ -265,7 +268,7 @@ void CSRProd(marian::Tensor C, /*ldb=*/ rowsB, &beta, C->data(), - /*ldc=*/ rowsC); + /*ldc=*/ rowsC)); cusparseDestroyMatDescr(descrA); } diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h index f1c59bd9c..1443e53a8 100755 --- a/src/tensors/gpu/prod.h +++ b/src/tensors/gpu/prod.h @@ -37,6 +37,8 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, - const marian::Tensor& B); + const marian::Tensor& B, + bool transA, + float beta = 0); } // namespace gpu } // namespace marian diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 0ece15a23..5002a0ff7 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -302,9 +302,17 @@ void tests(DeviceType device) { graph->clear(); values.clear(); - std::vector vA({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); - std::vector vB({1, 2, 3, 4, 5, 6}); - std::vector vC({22, 28, 49, 64, 76, 100, 103, 136}); + std::vector vA({1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 10, 11, 12}); + std::vector vB({1, 2, + 3, 4, + 5, 6}); + std::vector vC({22, 28, + 49, 64, + 76, 100, + 103, 136}); auto A = graph->param("A", {2, 2, 3}, inits::from_vector(vA)); auto B = graph->param("B", {3, 2}, inits::from_vector(vB)); @@ -314,6 +322,38 @@ void tests(DeviceType device) { CHECK(C->shape() == Shape({2, 2, 2})); C->val()->get(values); CHECK(values == vC); + + // CSR dot product + std::vector vS({1, 0, 0, 1, + 0, 0, 1, 2}); + std::vector vR({1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 10, 11, 12}); + std::vector vSxR({11, 13, 15, + 27, 30, 33.1}); + std::vector SV; + std::vector SI, SO; + SO.push_back((IndexType)SI.size()); + for (IndexType i = 0; i < 2; i++) { // convert to CSR + for (IndexType j = 0; j < 4; j++) { + auto k = 4 * i + j; + if (vS[k] != 0) { + SV.push_back(vS[k]); + SI.push_back(j); + } + } + SO.push_back((IndexType)SI.size()); + } + auto R = graph->param("A", {4, 3}, inits::from_vector(vR)); + auto SxR = csr_dot( + graph->constant({ (int)SV.size() }, inits::from_vector(SV), Type::float32), + graph->constant({ (int)SI.size() }, inits::from_vector(SI), Type::uint32), + graph->constant({ (int)SO.size() }, inits::from_vector(SO), Type::uint32), + A); + CHECK(SxR->shape() == Shape({2, 3})); + SxR->val()->get(values); + CHECK(values == vSxR); } SECTION("affine transformation") { diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h index dbfe2d877..55b485287 100755 --- a/src/training/communicator_nccl.h +++ b/src/training/communicator_nccl.h @@ -191,7 +191,7 @@ class NCCLCommunicator : public ICommunicator { groupEnd(); mpiBarrier(); // (synchronize the log messages) - LOG(info, "NCCLCommunicator constructed successfully."); + LOG(info, "[comm] NCCLCommunicator constructed successfully."); mpiBarrier(); // (synchronize the log messages) } From e626f0a16727f934e39a7dda57f66bc6f09d235f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 14:02:31 -0800 Subject: [PATCH 107/838] updated tests w.r.t. earlier refactoring; towards test for csr_dot() --- src/tests/attention_tests.cpp | 8 ++--- src/tests/operator_tests.cpp | 18 ++++++----- src/tests/rnn_tests.cpp | 58 +++++++++++++++++------------------ 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/tests/attention_tests.cpp b/src/tests/attention_tests.cpp index 1f4ae5b6d..9b1b61859 100755 --- a/src/tests/attention_tests.cpp +++ b/src/tests/attention_tests.cpp @@ -53,15 +53,15 @@ void tests(DeviceType type) { auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); - auto rnn = rnn::rnn(graph) // + auto rnn = rnn::rnn() // ("prefix", "rnntest") // ("type", "gru") // ("dimInput", 16) // ("dimState", 8) // - .push_back(rnn::cell(graph)) // - .construct(); + .push_back(rnn::cell()) // + .construct(graph); - auto context = rnn.construct()->transduce(input, mask); + auto context = rnn->transduce(input, mask); auto encState = New(context, mask, nullptr); diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 5002a0ff7..814046159 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -317,11 +317,6 @@ void tests(DeviceType device) { auto A = graph->param("A", {2, 2, 3}, inits::from_vector(vA)); auto B = graph->param("B", {3, 2}, inits::from_vector(vB)); auto C = dot(A, B); - graph->forward(); - - CHECK(C->shape() == Shape({2, 2, 2})); - C->val()->get(values); - CHECK(values == vC); // CSR dot product std::vector vS({1, 0, 0, 1, @@ -335,7 +330,7 @@ void tests(DeviceType device) { std::vector SV; std::vector SI, SO; SO.push_back((IndexType)SI.size()); - for (IndexType i = 0; i < 2; i++) { // convert to CSR + for (IndexType i = 0; i < 2; i++) { // convert to CSR! for (IndexType j = 0; j < 4; j++) { auto k = 4 * i + j; if (vS[k] != 0) { @@ -345,12 +340,19 @@ void tests(DeviceType device) { } SO.push_back((IndexType)SI.size()); } - auto R = graph->param("A", {4, 3}, inits::from_vector(vR)); + auto R = graph->param("R", {4, 3}, inits::from_vector(vR)); auto SxR = csr_dot( graph->constant({ (int)SV.size() }, inits::from_vector(SV), Type::float32), graph->constant({ (int)SI.size() }, inits::from_vector(SI), Type::uint32), graph->constant({ (int)SO.size() }, inits::from_vector(SO), Type::uint32), - A); + R); + + graph->forward(); + + CHECK(C->shape() == Shape({ 2, 2, 2 })); + C->val()->get(values); + CHECK(values == vC); + CHECK(SxR->shape() == Shape({2, 3})); SxR->val()->get(values); CHECK(values == vSxR); diff --git a/src/tests/rnn_tests.cpp b/src/tests/rnn_tests.cpp index 145828788..8310f2332 100755 --- a/src/tests/rnn_tests.cpp +++ b/src/tests/rnn_tests.cpp @@ -43,15 +43,15 @@ void tests(DeviceType type) { auto input = graph->constant({4, 1, 4}, inits::glorot_uniform); - auto rnn = rnn::rnn(graph) // - ("prefix", "rnntest") // - ("type", "tanh") // - ("dimInput", 4) // - ("dimState", 4) // - .push_back(rnn::cell(graph)) // - .construct(); + auto rnn = rnn::rnn() // + ("prefix", "rnntest") // + ("type", "tanh") // + ("dimInput", 4) // + ("dimState", 4) // + .push_back(rnn::cell()) // + .construct(graph); - auto output = rnn.construct()->transduce(input); + auto output = rnn->transduce(input); graph->forward(); @@ -117,7 +117,7 @@ void tests(DeviceType type) { auto backward = type == "alternating" ? rnn::dir::alternating_backward : rnn::dir::backward; - auto rnnFw = rnn::rnn(graph) // + auto rnnFw = rnn::rnn() // ("type", cellType) // ("direction", forward) // ("dimInput", dimEmb) // @@ -126,7 +126,7 @@ void tests(DeviceType type) { ("skip", skip); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_bi"; if(i > 1) @@ -134,21 +134,21 @@ void tests(DeviceType type) { if(i > 1 || j > 1) paramPrefix += "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnFw.push_back(stacked); } - auto rnnBw = rnn::rnn(graph) // - ("type", cellType) // - ("direction", backward) // - ("dimInput", dimEmb) // - ("dimState", dimRnn) // - ("layer-normalization", layerNorm) // + auto rnnBw = rnn::rnn() // + ("type", cellType) // + ("direction", backward) // + ("dimInput", dimEmb) // + ("dimState", dimRnn) // + ("layer-normalization", layerNorm) // ("skip", skip); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_bi_r"; if(i > 1) @@ -156,13 +156,13 @@ void tests(DeviceType type) { if(i > 1 || j > 1) paramPrefix += "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnBw.push_back(stacked); } - auto context = concatenate({rnnFw.construct()->transduce(input, mask), - rnnBw.construct()->transduce(input, mask)}, + auto context = concatenate({rnnFw.construct(graph)->transduce(input, mask), + rnnBw.construct(graph)->transduce(input, mask)}, /*axis =*/ input->shape().size() - 1); if(second > 0) { @@ -170,25 +170,25 @@ void tests(DeviceType type) { // previous bidirectional RNN through multiple layers // construct RNN first - auto rnnUni = rnn::rnn(graph) // - ("type", cellType) // - ("dimInput", 2 * dimRnn) // - ("dimState", dimRnn) // - ("layer-normalization", layerNorm) // + auto rnnUni = rnn::rnn() // + ("type", cellType) // + ("dimInput", 2 * dimRnn) // + ("dimState", dimRnn) // + ("layer-normalization", layerNorm) // ("skip", skip); for(int i = first + 1; i <= second + first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_l" + std::to_string(i) + "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnUni.push_back(stacked); } // transduce context to new context - context = rnnUni.construct()->transduce(context); + context = rnnUni.construct(graph)->transduce(context); } return context; }; From 2c05491f7aa4fe172fba3f12ba8a69edcfe75ccb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 14:23:19 -0800 Subject: [PATCH 108/838] bug fix: sparse int arrays should be passed in correct order --- src/tensors/gpu/prod.cu | 4 ++-- src/tests/operator_tests.cpp | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 4acb01669..776079577 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -262,8 +262,8 @@ void CSRProd(marian::Tensor C, /*nnz=*/ (int)numValues, &alpha, descrA, /*csrValA=*/ A_values->data(), - /*csrRowPtrA=*/ (int*)A_indices->data(), - /*csrColIndA=*/ (int*)A_offsets->data(), + /*csrRowPtrA=*/ (int*)A_offsets->data(), + /*csrColIndA=*/ (int*)A_indices->data(), B->data(), /*ldb=*/ rowsB, &beta, diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 814046159..7ade98967 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -340,21 +340,27 @@ void tests(DeviceType device) { } SO.push_back((IndexType)SI.size()); } + auto S = graph->param("S", {2, 4}, inits::from_vector(vS)); auto R = graph->param("R", {4, 3}, inits::from_vector(vR)); - auto SxR = csr_dot( - graph->constant({ (int)SV.size() }, inits::from_vector(SV), Type::float32), - graph->constant({ (int)SI.size() }, inits::from_vector(SI), Type::uint32), - graph->constant({ (int)SO.size() }, inits::from_vector(SO), Type::uint32), + auto SxRs = csr_dot( + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), R); + auto SxRd = dot(S, R); graph->forward(); - CHECK(C->shape() == Shape({ 2, 2, 2 })); + CHECK(C->shape() == Shape({2, 2, 2})); C->val()->get(values); CHECK(values == vC); - CHECK(SxR->shape() == Shape({2, 3})); - SxR->val()->get(values); + CHECK(SxRd->shape() == Shape({2, 3})); + SxRd->val()->get(values); + CHECK(values == vSxR); + + CHECK(SxRs->shape() == Shape({2, 3})); + SxRs->val()->get(values); CHECK(values == vSxR); } From 08298bc4c9ad53e81debf04e51e0f517450e6c91 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 15:02:49 -0800 Subject: [PATCH 109/838] bug fix: dense matrices passed to cusparse are col-major --- src/tensors/gpu/prod.cu | 24 +++++++++++++++++++++++- src/tests/operator_tests.cpp | 28 ++++++++++++++-------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 776079577..5a98a04a6 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -250,11 +250,32 @@ void CSRProd(marian::Tensor C, ABORT_IF((transA ? colsA : rowsA) != rowsC || (transA ? rowsA : colsA) != rowsB || colsB != colsC, "Inconsistent dimensions in CSR product"); ABORT_IF(A_values->shape() != A_indices->shape(), "CSR constituents has inconsistent dimensions"); float alpha = 1; +#if 1 + // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. + // Hence, we compute C = spA * B as C' = B' * spA'. where B' and C' are + // column-major views on the data of B and C, and likewise, spA' is + // the CSR matrix reinterpreted as a CSC matrix. + CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + /*m=*/ colsB, // #rows of A = #cols of row-major B + /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*nnz=*/ (int)numValues, + &alpha, + /*A=*/ B->data(), + /*lda=*/ colsB, // stride + /*cscValB=*/ A_values->data(), // second arg + /*cscRowPtrB=*/ (int*)A_offsets->data(), + /*cscColIndB=*/ (int*)A_indices->data(), + &beta, + C->data(), + /*ldc=*/ colsC)); // stride +#else + // Incorrect code that assumes col-major matrices. Reuse that later for dense x sparse. cusparseMatDescr_t descrA; CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); - CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, + CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, transA ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, /*m=*/ rowsA, // #rows of sparse A /*n=*/ colsB, // #cols of dense B and C @@ -270,6 +291,7 @@ void CSRProd(marian::Tensor C, C->data(), /*ldc=*/ rowsC)); cusparseDestroyMatDescr(descrA); +#endif } } // namespace gpu diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 7ade98967..89cf0fb0b 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -320,18 +320,20 @@ void tests(DeviceType device) { // CSR dot product std::vector vS({1, 0, 0, 1, - 0, 0, 1, 2}); - std::vector vR({1, 2, 3, - 4, 5, 6, - 7, 8, 9, - 10, 11, 12}); - std::vector vSxR({11, 13, 15, - 27, 30, 33.1}); - std::vector SV; + 0, 0, 1, 1.5}); + std::vector vR({1, 2, 3, 1.2, 5.6, + 4, 5, 6, 2.3, 6.7, + 7, 8, 9, 3.4, 7.8, + 1, 1, 2, 4.5, 8.9}); + std::vector vSxR({2.0, 3.0, 5.0, 5.7 , 14.5 , + 8.5, 9.5, 12.0, 10.15, 21.15}); + auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); + auto R = graph->param("R", { 4, 5 }, inits::from_vector(vR)); + std::vector SV; // create CSR version of S std::vector SI, SO; SO.push_back((IndexType)SI.size()); - for (IndexType i = 0; i < 2; i++) { // convert to CSR! - for (IndexType j = 0; j < 4; j++) { + for (IndexType i = 0; i < S->shape()[0]; i++) { + for (IndexType j = 0; j < S->shape()[1]; j++) { auto k = 4 * i + j; if (vS[k] != 0) { SV.push_back(vS[k]); @@ -340,8 +342,6 @@ void tests(DeviceType device) { } SO.push_back((IndexType)SI.size()); } - auto S = graph->param("S", {2, 4}, inits::from_vector(vS)); - auto R = graph->param("R", {4, 3}, inits::from_vector(vR)); auto SxRs = csr_dot( graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), @@ -355,11 +355,11 @@ void tests(DeviceType device) { C->val()->get(values); CHECK(values == vC); - CHECK(SxRd->shape() == Shape({2, 3})); + CHECK(SxRd->shape() == Shape({2, 5})); SxRd->val()->get(values); CHECK(values == vSxR); - CHECK(SxRs->shape() == Shape({2, 3})); + CHECK(SxRs->shape() == Shape({2, 5})); SxRs->val()->get(values); CHECK(values == vSxR); } From 8fa02dbfac9a4c51d4337a8cf29c82b28c666036 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 16:08:07 -0800 Subject: [PATCH 110/838] scaffolding for csr_dot(transA) --- src/graph/expression_operators.cpp | 4 +- src/graph/expression_operators.h | 2 +- src/graph/node_operators_binary.h | 29 +++++++------ src/tensors/cpu/prod.cpp | 1 + src/tensors/gpu/prod.cu | 67 ++++++++++++++++++++---------- src/tensors/gpu/prod.h | 1 + src/tensors/tensor_operators.h | 2 +- src/tests/operator_tests.cpp | 29 ++++++++----- 8 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 12f5464a6..ce009f290 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -420,8 +420,8 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { // multiply a CSR matrix A with a matrix B // A[i,j] is at A_values[A_offsets[i]+k], where k is position of j in A_indices[A_offsets[i]:A_offsets[i+1]] // Result shape is (Aoffsets.size() - 1, B->shape(-1)) -Expr csr_dot(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) { - return Expression(A_values, A_indices, A_offsets, B); +Expr csr_dot(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA /*= false*/) { + return Expression(A_shape, A_values, A_indices, A_offsets, B, transA); } // swap the last two axes diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 080fbf2d5..c9a66daec 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -112,7 +112,7 @@ Expr affine(Expr a, bool transB = false, float scalar = 1.f); -Expr csr_dot(Expr Avalues, Expr Aindices, Expr Aoffsets, Expr B); +Expr csr_dot(const Shape& A_shape, Expr Avalues, Expr Aindices, Expr Aoffsets, Expr B, bool transA = false); Expr transpose(Expr a); Expr transpose(Expr a, const std::vector& axes); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 61eb9ed46..994b5004a 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -128,7 +128,7 @@ class DotNodeOp : public NaryNodeOp { scalar_))}; } - const std::string type() override { return "•"; } + const std::string type() override { return "dot"; } const std::string color() override { return "orange"; } }; @@ -404,48 +404,51 @@ class DotBatchedNodeOp : public NaryNodeOp { scalar_))}; } - const std::string type() override { return "•"; } + const std::string type() override { return "bdot"; } const std::string color() override { return "orange"; } }; class CSRDotNodeOp : public NaryNodeOp { + bool transA_; public: - CSRDotNodeOp(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) - : NaryNodeOp({ A_values, A_indices, A_offsets, B }, newShape(A_values, A_indices, A_offsets, B)) { + CSRDotNodeOp(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) + : NaryNodeOp({ A_values, A_indices, A_offsets, B }, newShape(A_shape, A_values, A_indices, A_offsets, B, transA)), transA_(transA) { matchOrAbort(A_indices->value_type()); matchOrAbort(A_offsets->value_type()); } - Shape newShape(Expr A_values, Expr A_indices, Expr A_offsets, Expr B) { + Shape newShape(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) { ABORT_IF(A_values->shape().size() != 1 || A_indices->shape().size() != 1 || A_offsets->shape().size() != 1, "Sparse matrix components must all be vectors"); ABORT_IF(A_values->shape() != A_indices->shape(), "Sparse matrix values and indices must have the same shape"); + ABORT_IF(A_shape.size() != 2, + "Sparse matrix must have rank 2"); + ABORT_IF(A_offsets->shape()[0] - 1 != A_shape[0], + "Sparse matrix offset vector has incorrect size"); auto outShape = B->shape(); - outShape.set(0, A_offsets->shape()[0] - 1); // A_offsets = A.numRows + 1 + outShape.set(0, transA ? A_shape[1] : A_shape[0]); return outShape; } NodeOps forwardOps() override { // C = dot(A, B) return {NodeOp(CSRProd(val_, + graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), child(3)->val(), - /*transA=*/false, /*beta=*/0))}; + /*transA=*/transA_, /*beta=*/0))}; } NodeOps backwardOps() override { return {nullptr, // can't backprop into the sparse matrix, as it would be dense -#if 0 - nullptr}; -#else NodeOp(CSRProd(child(1)->grad(), + graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), adj_, - /*transA=*/true, /*beta=*/1))}; -#endif + /*transA=*/!transA_, /*beta=*/1))}; } - const std::string type() override { return "•"; } + const std::string type() override { return "csr_dot"; } const std::string color() override { return "orange"; } }; diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 43f99faef..f17d23f84 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -168,6 +168,7 @@ void ProdWithBias(marian::Tensor C, } void CSRProd(marian::Tensor C, + Ptr /*allocator*/, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 5a98a04a6..9e986ef01 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -228,6 +228,7 @@ void ProdBatched(marian::Tensor C, } void CSRProd(marian::Tensor C, + Ptr allocator, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, @@ -237,39 +238,63 @@ void CSRProd(marian::Tensor C, cudaSetDevice(C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); + // dimensions const auto& shapeC = C->shape(); const auto& shapeB = B->shape(); - auto numValues = A_values->shape().elements(); - auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length auto rowsC = shapeC[0]; auto colsC = shapeC.elements() / rowsC; auto rowsB = shapeB[0]; auto colsB = shapeB.elements() / rowsB; - auto rowsA = transA ? rowsB : numOffsets; // we don't know the dimension of the sparse axis from A directly - auto colsA = transA ? numOffsets : rowsB; + auto rowsA = transA ? rowsB : rowsC; + auto colsA = transA ? rowsC : rowsB; ABORT_IF((transA ? colsA : rowsA) != rowsC || (transA ? rowsA : colsA) != rowsB || colsB != colsC, "Inconsistent dimensions in CSR product"); - ABORT_IF(A_values->shape() != A_indices->shape(), "CSR constituents has inconsistent dimensions"); + // sparse arrays + auto numValues = A_values->shape().elements(); + auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length + LOG(info, "n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); + ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch: n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); + ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch"); + ABORT_IF(A_values->shape() != A_indices->shape(), "CSR values and indices must have the same size"); float alpha = 1; -#if 1 // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. // Hence, we compute C = spA * B as C' = B' * spA'. where B' and C' are // column-major views on the data of B and C, and likewise, spA' is // the CSR matrix reinterpreted as a CSC matrix. - CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - /*m=*/ colsB, // #rows of A = #cols of row-major B - /*n=*/ rowsC, // #cols of B and C = #rows of row-major C - /*k=*/ rowsB, // #cols of A = #rows of row-major B - /*nnz=*/ (int)numValues, - &alpha, - /*A=*/ B->data(), - /*lda=*/ colsB, // stride - /*cscValB=*/ A_values->data(), // second arg - /*cscRowPtrB=*/ (int*)A_offsets->data(), - /*cscColIndB=*/ (int*)A_indices->data(), - &beta, - C->data(), - /*ldc=*/ colsC)); // stride -#else + if (transA) { + //// transpose the second argument + //ABORT("not implemented"); + //CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + // /*m=*/ colsB, // #rows of A = #cols of row-major B + // /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + // /*k=*/ rowsB, // #cols of A = #rows of row-major B + // /*nnz=*/ (int)numValues, + // &alpha, + // /*A=*/ B->data(), + // /*lda=*/ colsB, // stride + // /*cscValB=*/ A_values->data(), // second arg --these get replaced + // /*cscRowPtrB=*/ (int*)A_offsets->data(), + // /*cscColIndB=*/ (int*)A_indices->data(), + // &beta, + // C->data(), + // /*ldc=*/ colsC)); // stride + } + else { + CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + /*m=*/ colsB, // #rows of A = #cols of row-major B + /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*nnz=*/ (int)numValues, + &alpha, + /*A=*/ B->data(), + /*lda=*/ colsB, // stride + /*cscValB=*/ A_values->data(), // second arg + /*cscRowPtrB=*/ (int*)A_offsets->data(), + /*cscColIndB=*/ (int*)A_indices->data(), + &beta, + C->data(), + /*ldc=*/ colsC)); // stride + } +#if 0 // Incorrect code that assumes col-major matrices. Reuse that later for dense x sparse. cusparseMatDescr_t descrA; CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h index 1443e53a8..35bde4f79 100755 --- a/src/tensors/gpu/prod.h +++ b/src/tensors/gpu/prod.h @@ -34,6 +34,7 @@ void ProdBatched(marian::Tensor C, float beta = 0, float scalar = 1); void CSRProd(marian::Tensor C, + Ptr allocator, const marian::Tensor& A_values, const marian::Tensor& A_indices, const marian::Tensor& A_offsets, diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index d84ca7974..a9bd58480 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -77,7 +77,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) -DISPATCH7(CSRProd, marian::Tensor, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, float) +DISPATCH8(CSRProd, marian::Tensor, Ptr, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, float) DISPATCH2(Softmax, marian::Tensor, marian::Tensor) DISPATCH3(SoftmaxGrad, marian::Tensor, marian::Tensor, marian::Tensor) diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 89cf0fb0b..954f4adce 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -15,7 +15,7 @@ void tests(DeviceType device) { graph->setDevice({0, device}); graph->reserveWorkspaceMB(16); - std::vector values; + std::vector values, values2; SECTION("scalar multiplication") { graph->clear(); @@ -325,8 +325,6 @@ void tests(DeviceType device) { 4, 5, 6, 2.3, 6.7, 7, 8, 9, 3.4, 7.8, 1, 1, 2, 4.5, 8.9}); - std::vector vSxR({2.0, 3.0, 5.0, 5.7 , 14.5 , - 8.5, 9.5, 12.0, 10.15, 21.15}); auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); auto R = graph->param("R", { 4, 5 }, inits::from_vector(vR)); std::vector SV; // create CSR version of S @@ -343,25 +341,36 @@ void tests(DeviceType device) { SO.push_back((IndexType)SI.size()); } auto SxRs = csr_dot( + S->shape(), graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), R); auto SxRd = dot(S, R); + auto STxRs = csr_dot( // and transpose; use result of previous since dimensions match + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + SxRd, /*transA=*/true); + auto STxRd = dot(S, SxRd, /*transA=*/true); + + CHECK(C->shape() == Shape({2, 2, 2})); + CHECK(SxRs->shape() == SxRd->shape()); + CHECK(STxRs->shape() == STxRd->shape()); graph->forward(); - CHECK(C->shape() == Shape({2, 2, 2})); C->val()->get(values); CHECK(values == vC); - CHECK(SxRd->shape() == Shape({2, 5})); - SxRd->val()->get(values); - CHECK(values == vSxR); + SxRd->val()->get(values2); // dense + SxRs->val()->get(values); // sparse + CHECK(values == values2); // must be the same - CHECK(SxRs->shape() == Shape({2, 5})); - SxRs->val()->get(values); - CHECK(values == vSxR); + STxRd->val()->get(values2); + STxRs->val()->get(values); + CHECK(values == values2); } SECTION("affine transformation") { From 7e9eed4da6a8a0aa16ab8c40cec4bf17d6e2f0ca Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 16:25:01 -0800 Subject: [PATCH 111/838] csr_dot(transA) complete, with test passing --- src/graph/expression_operators.cpp | 2 +- src/tensors/gpu/prod.cu | 51 ++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index ce009f290..6828bf103 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -419,7 +419,7 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { // multiply a CSR matrix A with a matrix B // A[i,j] is at A_values[A_offsets[i]+k], where k is position of j in A_indices[A_offsets[i]:A_offsets[i+1]] -// Result shape is (Aoffsets.size() - 1, B->shape(-1)) +// @TODO: Define a proper sparse tensor type. Expr csr_dot(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA /*= false*/) { return Expression(A_shape, A_values, A_indices, A_offsets, B, transA); } diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 9e986ef01..e816ecf10 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -251,7 +251,6 @@ void CSRProd(marian::Tensor C, // sparse arrays auto numValues = A_values->shape().elements(); auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length - LOG(info, "n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch: n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch"); ABORT_IF(A_values->shape() != A_indices->shape(), "CSR values and indices must have the same size"); @@ -261,22 +260,40 @@ void CSRProd(marian::Tensor C, // column-major views on the data of B and C, and likewise, spA' is // the CSR matrix reinterpreted as a CSC matrix. if (transA) { - //// transpose the second argument - //ABORT("not implemented"); - //CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - // /*m=*/ colsB, // #rows of A = #cols of row-major B - // /*n=*/ rowsC, // #cols of B and C = #rows of row-major C - // /*k=*/ rowsB, // #cols of A = #rows of row-major B - // /*nnz=*/ (int)numValues, - // &alpha, - // /*A=*/ B->data(), - // /*lda=*/ colsB, // stride - // /*cscValB=*/ A_values->data(), // second arg --these get replaced - // /*cscRowPtrB=*/ (int*)A_offsets->data(), - // /*cscColIndB=*/ (int*)A_indices->data(), - // &beta, - // C->data(), - // /*ldc=*/ colsC)); // stride + // cusparse does not support this specific version of transpose; do it explicitly + auto At_values = allocator->alloc(numValues); + auto At_indices = allocator->alloc(numValues); + auto At_offsets = allocator->alloc(colsA + 1); + // transpose the second argument + CUSPARSE_CHECK(cusparseScsr2csc(cusparseHandle, + /*m=*/ rowsA, // number of rows of matrix + /*n=*/ colsA, // number of columns of matrix + /*nnz=*/ (int)numValues, + /*csrcVal=*/ A_values->data(), // second arg + /*csrcRowPtr=*/ (int*)A_offsets->data(), + /*csrcColInd=*/ (int*)A_indices->data(), + /*cscVal=*/ At_values->data(), // transposed version goes here + /*cscRowInd=*/ At_indices->data(), + /*cscColPtr=*/ At_offsets->data(), + /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, + /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); + CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + /*m=*/ colsB, // #rows of A = #cols of row-major B + /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*nnz=*/ (int)numValues, + &alpha, + /*A=*/ B->data(), + /*lda=*/ colsB, // stride + /*cscValB=*/ At_values->data(), // second arg, transposed + /*cscRowPtrB=*/ At_offsets->data(), + /*cscColIndB=*/ At_indices->data(), + &beta, + C->data(), + /*ldc=*/ colsC)); // stride + allocator->free(At_values); + allocator->free(At_indices); + allocator->free(At_offsets); } else { CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, From 9c7f977b632e418569a7b5cb49f00d9efa28b4cf Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 16:34:56 -0800 Subject: [PATCH 112/838] (comments) --- src/graph/node_operators_binary.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 994b5004a..357c24b69 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -436,15 +436,17 @@ class CSRDotNodeOp : public NaryNodeOp { // C = dot(A, B) return {NodeOp(CSRProd(val_, graph()->allocator(), - child(0)->val(), child(1)->val(), child(2)->val(), child(3)->val(), + child(0)->val(), child(1)->val(), child(2)->val(), + child(3)->val(), /*transA=*/transA_, /*beta=*/0))}; } NodeOps backwardOps() override { - return {nullptr, // can't backprop into the sparse matrix, as it would be dense - NodeOp(CSRProd(child(1)->grad(), + return {nullptr, // can't backprop into the sparse matrix (the gradient is dense) + NodeOp(CSRProd(child(3)->grad(), // child(3) = B graph()->allocator(), - child(0)->val(), child(1)->val(), child(2)->val(), adj_, + child(0)->val(), child(1)->val(), child(2)->val(), // children(0..2) = A + adj_, /*transA=*/!transA_, /*beta=*/1))}; } From eb2c5bccc77d4b271d33cccfbb156ca12b02dc06 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 16:48:29 -0800 Subject: [PATCH 113/838] csr_dot operation --- src/common/utils.cpp | 8 +- src/common/utils.h | 8 +- src/graph/expression_operators.cpp | 7 ++ src/graph/expression_operators.h | 2 + src/graph/node_operators_binary.h | 50 ++++++++++- src/optimizers/optimizers.h | 2 +- src/tensors/cpu/prod.cpp | 12 +++ src/tensors/gpu/backend.h | 17 ++-- src/tensors/gpu/cuda_helpers.h | 6 ++ src/tensors/gpu/prod.cu | 132 ++++++++++++++++++++++++++--- src/tensors/gpu/prod.h | 8 ++ src/tensors/tensor_operators.h | 1 + src/tests/attention_tests.cpp | 8 +- src/tests/operator_tests.cpp | 67 +++++++++++++-- src/tests/rnn_tests.cpp | 58 ++++++------- src/training/communicator_nccl.h | 2 +- 16 files changed, 317 insertions(+), 71 deletions(-) mode change 100644 => 100755 src/tensors/gpu/prod.h diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 992c74ac8..bde788359 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -29,7 +29,7 @@ void trimLeft(std::string& s) { // @TODO: use more functions from CLI instead of own implementations void split(const std::string& line, std::vector& pieces, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { size_t begin = 0; size_t pos = 0; @@ -50,7 +50,7 @@ void split(const std::string& line, } std::vector split(const std::string& line, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { std::vector pieces; split(line, pieces, del, keepEmpty); @@ -60,7 +60,7 @@ std::vector split(const std::string& line, // @TODO: splitAny() shares all but 2 expressions with split(). Merge them. void splitAny(const std::string& line, std::vector& pieces, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { size_t begin = 0; size_t pos = 0; @@ -81,7 +81,7 @@ void splitAny(const std::string& line, } std::vector splitAny(const std::string& line, - const std::string del /*= " "*/, + const std::string& del /*= " "*/, bool keepEmpty) { std::vector pieces; splitAny(line, pieces, del, keepEmpty); diff --git a/src/common/utils.h b/src/common/utils.h index 7c4d56432..94113a0ec 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -12,20 +12,20 @@ void trimRight(std::string& s); void split(const std::string& line, std::vector& pieces, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::vector split(const std::string& line, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); void splitAny(const std::string& line, std::vector& pieces, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::vector splitAny(const std::string& line, - const std::string del = " ", + const std::string& del = " ", bool keepEmpty = false); std::string join(const std::vector& words, diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index cdfeff271..6828bf103 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -417,6 +417,13 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { } } +// multiply a CSR matrix A with a matrix B +// A[i,j] is at A_values[A_offsets[i]+k], where k is position of j in A_indices[A_offsets[i]:A_offsets[i+1]] +// @TODO: Define a proper sparse tensor type. +Expr csr_dot(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA /*= false*/) { + return Expression(A_shape, A_values, A_indices, A_offsets, B, transA); +} + // swap the last two axes // @TODO: change to swapAxes(a, -1, -2) Expr transpose(Expr a) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 7c3797069..c9a66daec 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -112,6 +112,8 @@ Expr affine(Expr a, bool transB = false, float scalar = 1.f); +Expr csr_dot(const Shape& A_shape, Expr Avalues, Expr Aindices, Expr Aoffsets, Expr B, bool transA = false); + Expr transpose(Expr a); Expr transpose(Expr a, const std::vector& axes); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 868ee4ebd..357c24b69 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -128,7 +128,7 @@ class DotNodeOp : public NaryNodeOp { scalar_))}; } - const std::string type() override { return "•"; } + const std::string type() override { return "dot"; } const std::string color() override { return "orange"; } }; @@ -404,7 +404,53 @@ class DotBatchedNodeOp : public NaryNodeOp { scalar_))}; } - const std::string type() override { return "•"; } + const std::string type() override { return "bdot"; } + + const std::string color() override { return "orange"; } +}; + +class CSRDotNodeOp : public NaryNodeOp { + bool transA_; +public: + CSRDotNodeOp(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) + : NaryNodeOp({ A_values, A_indices, A_offsets, B }, newShape(A_shape, A_values, A_indices, A_offsets, B, transA)), transA_(transA) { + matchOrAbort(A_indices->value_type()); + matchOrAbort(A_offsets->value_type()); + } + + Shape newShape(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) { + ABORT_IF(A_values->shape().size() != 1 || A_indices->shape().size() != 1 || A_offsets->shape().size() != 1, + "Sparse matrix components must all be vectors"); + ABORT_IF(A_values->shape() != A_indices->shape(), + "Sparse matrix values and indices must have the same shape"); + ABORT_IF(A_shape.size() != 2, + "Sparse matrix must have rank 2"); + ABORT_IF(A_offsets->shape()[0] - 1 != A_shape[0], + "Sparse matrix offset vector has incorrect size"); + auto outShape = B->shape(); + outShape.set(0, transA ? A_shape[1] : A_shape[0]); + return outShape; + } + + NodeOps forwardOps() override { + // C = dot(A, B) + return {NodeOp(CSRProd(val_, + graph()->allocator(), + child(0)->val(), child(1)->val(), child(2)->val(), + child(3)->val(), + /*transA=*/transA_, /*beta=*/0))}; + } + + NodeOps backwardOps() override { + return {nullptr, // can't backprop into the sparse matrix (the gradient is dense) + NodeOp(CSRProd(child(3)->grad(), // child(3) = B + graph()->allocator(), + child(0)->val(), child(1)->val(), child(2)->val(), // children(0..2) = A + adj_, + /*transA=*/!transA_, /*beta=*/1))}; + } + + const std::string type() override { return "csr_dot"; } const std::string color() override { return "orange"; } }; diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h index 51e43f871..44d603120 100755 --- a/src/optimizers/optimizers.h +++ b/src/optimizers/optimizers.h @@ -26,7 +26,7 @@ class OptimizerBase : public TrainingObserver { // that these hyper-parameters were originally tuned for, then the learning-rate gets // adjusted accordingly. Note: Requires user to also use ce-sum criterion. if (refMBWordsParam_ != 0) - LOG(info, "Note: Learning rate gets automatically adjusted as if minibatch size was {}", refMBWordsParam_); + LOG(info, "[optimizers] Learning rate gets automatically adjusted as if minibatch size was {}", refMBWordsParam_); } static constexpr size_t mbSizeNotProvided = SIZE_MAX; diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 69923f872..f17d23f84 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -167,5 +167,17 @@ void ProdWithBias(marian::Tensor C, cpu::int16::AddBias(C, bias); } +void CSRProd(marian::Tensor C, + Ptr /*allocator*/, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B, + bool transA, + float beta) { + C, A_values, A_indices, A_offsets, B, transA, beta; + ABORT("CSRProd is not yet implemented for CPU"); +} + } // namespace cpu } // namespace marian diff --git a/src/tensors/gpu/backend.h b/src/tensors/gpu/backend.h index ab1968c1f..87f6407c5 100755 --- a/src/tensors/gpu/backend.h +++ b/src/tensors/gpu/backend.h @@ -6,6 +6,7 @@ #include #include +#include #include namespace marian { @@ -15,11 +16,13 @@ class Backend : public marian::Backend { public: Backend(DeviceId deviceId, size_t seed) : marian::Backend(deviceId, seed) { setDevice(); - setHandles(); + cublasCreate(&cublasHandle_); + cusparseCreate(&cusparseHandle_); } ~Backend() { setDevice(); + cusparseDestroy(cusparseHandle_); cublasDestroy(cublasHandle_); } @@ -28,19 +31,11 @@ class Backend : public marian::Backend { void synchronize() override { cudaStreamSynchronize(0); } cublasHandle_t getCublasHandle() { return cublasHandle_; } + cusparseHandle_t getCusparseHandle() { return cusparseHandle_; } private: cublasHandle_t cublasHandle_; - - void setHandles() { - cublasHandle_ = create_handle(); - } - - cublasHandle_t create_handle() { - cublasHandle_t cublasHandle; - cublasCreate(&cublasHandle); - return cublasHandle; - } + cusparseHandle_t cusparseHandle_; }; } // namespace gpu } // namespace marian diff --git a/src/tensors/gpu/cuda_helpers.h b/src/tensors/gpu/cuda_helpers.h index ba890490e..920cec8c4 100755 --- a/src/tensors/gpu/cuda_helpers.h +++ b/src/tensors/gpu/cuda_helpers.h @@ -13,6 +13,12 @@ const int MAX_BLOCKS = 65535; "CUDA error {} '{}' - {}:{}: {}", rc, cudaGetErrorString(rc), __FILE__, __LINE__, #expr); \ } while(0) +#define CUBLAS_CHECK(expr) do { \ + cublasStatus_t rc = (expr); \ + ABORT_IF(rc != CUBLAS_STATUS_SUCCESS, \ + "Cublas Error: {} - {}:{}: {}", rc, __FILE__, __LINE__, #expr); \ +} while(0) + #define CUSPARSE_CHECK(expr) do { \ cusparseStatus_t rc = (expr); \ ABORT_IF(rc != CUSPARSE_STATUS_SUCCESS, \ diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cu index 9da32e65a..e816ecf10 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cu @@ -1,5 +1,6 @@ #include +#include // clang-format off #include "tensors/gpu/prod.h" @@ -23,18 +24,18 @@ static void setTensorMode(cublasHandle_t cublasHandle) { default: ABORT("Invalid ENABLE_CUBLAS_TENSOR_OP_MATH_FP32={}", var); } if (mode > 0) { // try whether it can be set --@TODO: check whether this actually works - cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); + CUBLAS_CHECK(cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH)); cublasMath_t actual = CUBLAS_DEFAULT_MATH; cublasGetMathMode(cublasHandle, &actual); if (actual != CUBLAS_TENSOR_OP_MATH) { - LOG(info, "WARNING: TensorCores requested but not available"); + LOG(warn, "[gpu] TensorCores requested but not available"); mode = -1; } } if (mode > 0) - LOG(info, "16-bit TensorCores enabled for float32 matrix operations"); + LOG(info, "[gpu] 16-bit TensorCores enabled for float32 matrix operations"); } - cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH); + CUBLAS_CHECK(cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH)); } void Prod(marian::Tensor C, @@ -75,7 +76,7 @@ void Prod(marian::Tensor C, //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - cublasSgemm(cublasHandle, + CUBLAS_CHECK(cublasSgemm(cublasHandle, opB, opA, n, @@ -88,7 +89,7 @@ void Prod(marian::Tensor C, lda, &beta, C->data(), - ldc); + ldc)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -107,6 +108,7 @@ __global__ void gAddBias(float* out, } } +#if 0 // @TODO: remove, then rename from .cu to .cpp void AddBias(marian::Tensor C, const marian::Tensor bias) { cudaSetDevice(C->getDeviceId().no); @@ -116,9 +118,9 @@ void AddBias(marian::Tensor C, const marian::Tensor bias) { int threads = std::min(MAX_THREADS, length); int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); - gAddBias<<>>(C->data(), bias->data(), length, cols); + gAddBias<<>>(C->data(), bias->data(), length, cols); // @TODO: CUDA_CHECK - cudaStreamSynchronize(0); + CUDA_CHECK(cudaStreamSynchronize(0)); // @BUGBUG: Should not be here. Prod() also does not have this. } void ProdWithBias(marian::Tensor C, @@ -132,6 +134,7 @@ void ProdWithBias(marian::Tensor C, marian::gpu::Prod(C, A, B, transA, transB, beta, scalar); marian::gpu::AddBias(C, bias); } +#endif void ProdBatched(marian::Tensor C, Ptr allocator, @@ -200,7 +203,7 @@ void ProdBatched(marian::Tensor C, setTensorMode(cublasHandle); //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - cublasSgemmBatched(cublasHandle, + CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, opB, opA, n, @@ -214,7 +217,7 @@ void ProdBatched(marian::Tensor C, &beta, mp_cptr->data(), ldc, - batchC); + batchC)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -224,5 +227,114 @@ void ProdBatched(marian::Tensor C, allocator->free(mp_cptr); } +void CSRProd(marian::Tensor C, + Ptr allocator, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B, + bool transA, + float beta) { + cudaSetDevice(C->getDeviceId().no); + auto cusparseHandle = std::static_pointer_cast(C->getBackend()) + ->getCusparseHandle(); + // dimensions + const auto& shapeC = C->shape(); + const auto& shapeB = B->shape(); + auto rowsC = shapeC[0]; + auto colsC = shapeC.elements() / rowsC; + auto rowsB = shapeB[0]; + auto colsB = shapeB.elements() / rowsB; + auto rowsA = transA ? rowsB : rowsC; + auto colsA = transA ? rowsC : rowsB; + ABORT_IF((transA ? colsA : rowsA) != rowsC || (transA ? rowsA : colsA) != rowsB || colsB != colsC, "Inconsistent dimensions in CSR product"); + // sparse arrays + auto numValues = A_values->shape().elements(); + auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length + ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch: n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); + ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch"); + ABORT_IF(A_values->shape() != A_indices->shape(), "CSR values and indices must have the same size"); + float alpha = 1; + // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. + // Hence, we compute C = spA * B as C' = B' * spA'. where B' and C' are + // column-major views on the data of B and C, and likewise, spA' is + // the CSR matrix reinterpreted as a CSC matrix. + if (transA) { + // cusparse does not support this specific version of transpose; do it explicitly + auto At_values = allocator->alloc(numValues); + auto At_indices = allocator->alloc(numValues); + auto At_offsets = allocator->alloc(colsA + 1); + // transpose the second argument + CUSPARSE_CHECK(cusparseScsr2csc(cusparseHandle, + /*m=*/ rowsA, // number of rows of matrix + /*n=*/ colsA, // number of columns of matrix + /*nnz=*/ (int)numValues, + /*csrcVal=*/ A_values->data(), // second arg + /*csrcRowPtr=*/ (int*)A_offsets->data(), + /*csrcColInd=*/ (int*)A_indices->data(), + /*cscVal=*/ At_values->data(), // transposed version goes here + /*cscRowInd=*/ At_indices->data(), + /*cscColPtr=*/ At_offsets->data(), + /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, + /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); + CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + /*m=*/ colsB, // #rows of A = #cols of row-major B + /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*nnz=*/ (int)numValues, + &alpha, + /*A=*/ B->data(), + /*lda=*/ colsB, // stride + /*cscValB=*/ At_values->data(), // second arg, transposed + /*cscRowPtrB=*/ At_offsets->data(), + /*cscColIndB=*/ At_indices->data(), + &beta, + C->data(), + /*ldc=*/ colsC)); // stride + allocator->free(At_values); + allocator->free(At_indices); + allocator->free(At_offsets); + } + else { + CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + /*m=*/ colsB, // #rows of A = #cols of row-major B + /*n=*/ rowsC, // #cols of B and C = #rows of row-major C + /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*nnz=*/ (int)numValues, + &alpha, + /*A=*/ B->data(), + /*lda=*/ colsB, // stride + /*cscValB=*/ A_values->data(), // second arg + /*cscRowPtrB=*/ (int*)A_offsets->data(), + /*cscColIndB=*/ (int*)A_indices->data(), + &beta, + C->data(), + /*ldc=*/ colsC)); // stride + } +#if 0 + // Incorrect code that assumes col-major matrices. Reuse that later for dense x sparse. + cusparseMatDescr_t descrA; + CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); + cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); + cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); + CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, + transA ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, + /*m=*/ rowsA, // #rows of sparse A + /*n=*/ colsB, // #cols of dense B and C + /*k=*/ colsA, // #cols of sparse A + /*nnz=*/ (int)numValues, + &alpha, descrA, + /*csrValA=*/ A_values->data(), + /*csrRowPtrA=*/ (int*)A_offsets->data(), + /*csrColIndA=*/ (int*)A_indices->data(), + B->data(), + /*ldb=*/ rowsB, + &beta, + C->data(), + /*ldc=*/ rowsC)); + cusparseDestroyMatDescr(descrA); +#endif +} + } // namespace gpu } // namespace marian diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h old mode 100644 new mode 100755 index ce1f6cdca..35bde4f79 --- a/src/tensors/gpu/prod.h +++ b/src/tensors/gpu/prod.h @@ -33,5 +33,13 @@ void ProdBatched(marian::Tensor C, bool transB, float beta = 0, float scalar = 1); +void CSRProd(marian::Tensor C, + Ptr allocator, + const marian::Tensor& A_values, + const marian::Tensor& A_indices, + const marian::Tensor& A_offsets, + const marian::Tensor& B, + bool transA, + float beta = 0); } // namespace gpu } // namespace marian diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index 2cac284ab..a9bd58480 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -77,6 +77,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) +DISPATCH8(CSRProd, marian::Tensor, Ptr, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, float) DISPATCH2(Softmax, marian::Tensor, marian::Tensor) DISPATCH3(SoftmaxGrad, marian::Tensor, marian::Tensor, marian::Tensor) diff --git a/src/tests/attention_tests.cpp b/src/tests/attention_tests.cpp index 1f4ae5b6d..9b1b61859 100755 --- a/src/tests/attention_tests.cpp +++ b/src/tests/attention_tests.cpp @@ -53,15 +53,15 @@ void tests(DeviceType type) { auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); - auto rnn = rnn::rnn(graph) // + auto rnn = rnn::rnn() // ("prefix", "rnntest") // ("type", "gru") // ("dimInput", 16) // ("dimState", 8) // - .push_back(rnn::cell(graph)) // - .construct(); + .push_back(rnn::cell()) // + .construct(graph); - auto context = rnn.construct()->transduce(input, mask); + auto context = rnn->transduce(input, mask); auto encState = New(context, mask, nullptr); diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 0ece15a23..954f4adce 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -15,7 +15,7 @@ void tests(DeviceType device) { graph->setDevice({0, device}); graph->reserveWorkspaceMB(16); - std::vector values; + std::vector values, values2; SECTION("scalar multiplication") { graph->clear(); @@ -302,18 +302,75 @@ void tests(DeviceType device) { graph->clear(); values.clear(); - std::vector vA({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}); - std::vector vB({1, 2, 3, 4, 5, 6}); - std::vector vC({22, 28, 49, 64, 76, 100, 103, 136}); + std::vector vA({1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 10, 11, 12}); + std::vector vB({1, 2, + 3, 4, + 5, 6}); + std::vector vC({22, 28, + 49, 64, + 76, 100, + 103, 136}); auto A = graph->param("A", {2, 2, 3}, inits::from_vector(vA)); auto B = graph->param("B", {3, 2}, inits::from_vector(vB)); auto C = dot(A, B); - graph->forward(); + + // CSR dot product + std::vector vS({1, 0, 0, 1, + 0, 0, 1, 1.5}); + std::vector vR({1, 2, 3, 1.2, 5.6, + 4, 5, 6, 2.3, 6.7, + 7, 8, 9, 3.4, 7.8, + 1, 1, 2, 4.5, 8.9}); + auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); + auto R = graph->param("R", { 4, 5 }, inits::from_vector(vR)); + std::vector SV; // create CSR version of S + std::vector SI, SO; + SO.push_back((IndexType)SI.size()); + for (IndexType i = 0; i < S->shape()[0]; i++) { + for (IndexType j = 0; j < S->shape()[1]; j++) { + auto k = 4 * i + j; + if (vS[k] != 0) { + SV.push_back(vS[k]); + SI.push_back(j); + } + } + SO.push_back((IndexType)SI.size()); + } + auto SxRs = csr_dot( + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + R); + auto SxRd = dot(S, R); + auto STxRs = csr_dot( // and transpose; use result of previous since dimensions match + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + SxRd, /*transA=*/true); + auto STxRd = dot(S, SxRd, /*transA=*/true); CHECK(C->shape() == Shape({2, 2, 2})); + CHECK(SxRs->shape() == SxRd->shape()); + CHECK(STxRs->shape() == STxRd->shape()); + + graph->forward(); + C->val()->get(values); CHECK(values == vC); + + SxRd->val()->get(values2); // dense + SxRs->val()->get(values); // sparse + CHECK(values == values2); // must be the same + + STxRd->val()->get(values2); + STxRs->val()->get(values); + CHECK(values == values2); } SECTION("affine transformation") { diff --git a/src/tests/rnn_tests.cpp b/src/tests/rnn_tests.cpp index 145828788..8310f2332 100755 --- a/src/tests/rnn_tests.cpp +++ b/src/tests/rnn_tests.cpp @@ -43,15 +43,15 @@ void tests(DeviceType type) { auto input = graph->constant({4, 1, 4}, inits::glorot_uniform); - auto rnn = rnn::rnn(graph) // - ("prefix", "rnntest") // - ("type", "tanh") // - ("dimInput", 4) // - ("dimState", 4) // - .push_back(rnn::cell(graph)) // - .construct(); + auto rnn = rnn::rnn() // + ("prefix", "rnntest") // + ("type", "tanh") // + ("dimInput", 4) // + ("dimState", 4) // + .push_back(rnn::cell()) // + .construct(graph); - auto output = rnn.construct()->transduce(input); + auto output = rnn->transduce(input); graph->forward(); @@ -117,7 +117,7 @@ void tests(DeviceType type) { auto backward = type == "alternating" ? rnn::dir::alternating_backward : rnn::dir::backward; - auto rnnFw = rnn::rnn(graph) // + auto rnnFw = rnn::rnn() // ("type", cellType) // ("direction", forward) // ("dimInput", dimEmb) // @@ -126,7 +126,7 @@ void tests(DeviceType type) { ("skip", skip); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_bi"; if(i > 1) @@ -134,21 +134,21 @@ void tests(DeviceType type) { if(i > 1 || j > 1) paramPrefix += "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnFw.push_back(stacked); } - auto rnnBw = rnn::rnn(graph) // - ("type", cellType) // - ("direction", backward) // - ("dimInput", dimEmb) // - ("dimState", dimRnn) // - ("layer-normalization", layerNorm) // + auto rnnBw = rnn::rnn() // + ("type", cellType) // + ("direction", backward) // + ("dimInput", dimEmb) // + ("dimState", dimRnn) // + ("layer-normalization", layerNorm) // ("skip", skip); for(int i = 1; i <= first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_bi_r"; if(i > 1) @@ -156,13 +156,13 @@ void tests(DeviceType type) { if(i > 1 || j > 1) paramPrefix += "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnBw.push_back(stacked); } - auto context = concatenate({rnnFw.construct()->transduce(input, mask), - rnnBw.construct()->transduce(input, mask)}, + auto context = concatenate({rnnFw.construct(graph)->transduce(input, mask), + rnnBw.construct(graph)->transduce(input, mask)}, /*axis =*/ input->shape().size() - 1); if(second > 0) { @@ -170,25 +170,25 @@ void tests(DeviceType type) { // previous bidirectional RNN through multiple layers // construct RNN first - auto rnnUni = rnn::rnn(graph) // - ("type", cellType) // - ("dimInput", 2 * dimRnn) // - ("dimState", dimRnn) // - ("layer-normalization", layerNorm) // + auto rnnUni = rnn::rnn() // + ("type", cellType) // + ("dimInput", 2 * dimRnn) // + ("dimState", dimRnn) // + ("layer-normalization", layerNorm) // ("skip", skip); for(int i = first + 1; i <= second + first; ++i) { - auto stacked = rnn::stacked_cell(graph); + auto stacked = rnn::stacked_cell(); for(int j = 1; j <= cellDepth; ++j) { std::string paramPrefix = prefix + "_l" + std::to_string(i) + "_cell" + std::to_string(j); - stacked.push_back(rnn::cell(graph)("prefix", paramPrefix)); + stacked.push_back(rnn::cell()("prefix", paramPrefix)); } rnnUni.push_back(stacked); } // transduce context to new context - context = rnnUni.construct()->transduce(context); + context = rnnUni.construct(graph)->transduce(context); } return context; }; diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h index dbfe2d877..55b485287 100755 --- a/src/training/communicator_nccl.h +++ b/src/training/communicator_nccl.h @@ -191,7 +191,7 @@ class NCCLCommunicator : public ICommunicator { groupEnd(); mpiBarrier(); // (synchronize the log messages) - LOG(info, "NCCLCommunicator constructed successfully."); + LOG(info, "[comm] NCCLCommunicator constructed successfully."); mpiBarrier(); // (synchronize the log messages) } From 4edc577b70e2d529469dfd770d7c306a70e7fcea Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 11 Jan 2019 19:50:10 -0800 Subject: [PATCH 114/838] bug fix: decoder should also use factors for the input --- src/layers/generic.h | 7 ++----- src/models/decoder.h | 43 ++++++++++++++++--------------------------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index 25b51dc13..092567c94 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -177,17 +177,14 @@ class Output : public LayerBase, public IUnaryLayer { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; Ptr embeddingFactorMapping_; + Expr multiRows(const std::vector& data) const; public: Embedding(Ptr graph, Ptr options); std::tuple apply(Ptr subBatch) const override final; // special version used in decoding - Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const override final { - int dimEmb = E_->shape()[-1]; - auto selectedEmbs = rows(E_, embIdx); - return reshape(selectedEmbs, { dimBeam, 1, dimBatch, dimEmb }); - } + Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const override final; }; class ULREmbedding : public LayerBase, public IEmbeddingLayer { diff --git a/src/models/decoder.h b/src/models/decoder.h index abf6bbd1e..9b7aad9c3 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -15,6 +15,7 @@ class DecoderBase { std::string prefix_{"decoder"}; bool inference_{false}; size_t batchIndex_{1}; + std::vector> embedding_; // @TODO: find a more grammattical name Ptr shortlist_; @@ -32,17 +33,13 @@ class DecoderBase { virtual Ptr step(Ptr, Ptr) = 0; - std::vector> embedding_; // @TODO: move away, also rename - virtual void embeddingsFromBatch(Ptr graph, - Ptr state, - Ptr batch) { - - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - + void lazyCreateEmbedding(Ptr graph) { // @TODO: code dup with EncoderTransformer - if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy - embedding_.resize(batch->sets()); + if (embedding_.size() <= batchIndex_ || !embedding_[batchIndex_]) { // lazy + if (embedding_.size() <= batchIndex_) + embedding_.resize(batchIndex_ + 1); + int dimVoc = opt>("dim-vocabs")[batchIndex_]; + int dimEmb = opt("dim-emb"); auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); @@ -61,9 +58,14 @@ class DecoderBase { } embedding_[batchIndex_] = embFactory.construct(graph); } + } + virtual void embeddingsFromBatch(Ptr graph, + Ptr state, + Ptr batch) { auto subBatch = (*batch)[batchIndex_]; + lazyCreateEmbedding(graph); Expr y, yMask; std::tie (y, yMask) = embedding_[batchIndex_]->apply(subBatch); @@ -86,26 +88,13 @@ class DecoderBase { const std::vector& embIdx, int dimBatch, int dimBeam) { - int dimTrgEmb = opt("dim-emb"); - int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - Expr selectedEmbs; if(embIdx.empty()) { - selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); + int dimEmb = opt("dim-emb"); + selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); } else { - // embeddings are loaded from model during translation, no fixing required - auto yEmbFactory = embedding() // - ("dimVocab", dimTrgVoc) // - ("dimEmb", dimTrgEmb); - - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - yEmbFactory("prefix", "Wemb"); - else - yEmbFactory("prefix", prefix_ + "_Wemb"); - - auto yEmb = yEmbFactory.construct(graph); - - selectedEmbs = yEmb->apply(embIdx, dimBatch, dimBeam); + lazyCreateEmbedding(graph); + selectedEmbs = embedding_[batchIndex_]->apply(embIdx, dimBatch, dimBeam); } state->setTargetEmbeddings(selectedEmbs); } From 569b6639e23d34fd291efc78219bc77f0cdcca08 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sat, 12 Jan 2019 21:25:08 -0800 Subject: [PATCH 115/838] only scale trigonometric emeddings --- src/models/transformer.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/models/transformer.h b/src/models/transformer.h index d57f23b95..3791d3c80 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -67,9 +67,11 @@ class Transformer : public EncoderOrDecoderBase { auto signal = rows(posEmbFactory, graph_->indices(positions)); signal = reshape(signal, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; - } else { + } else { auto signal = graph_->constant({dimWords, 1, dimEmb}, inits::positions(start)); + // according to paper embeddings are scaled up by \sqrt(d_m) + embeddings = std::sqrt((float)dimEmb) * embeddings; embeddings = embeddings + signal; } @@ -537,7 +539,6 @@ class EncoderTransformer : public Transformer { } virtual Ptr apply(Ptr batch) { - int dimEmb = opt("dim-emb"); int dimBatch = (int)batch->size(); int dimSrcWords = (int)(*batch)[batchIndex_]->batchWidth(); // create the embedding matrix, considering tying and some other options @@ -559,15 +560,13 @@ class EncoderTransformer : public Transformer { int srcWords = batchEmbeddings->shape()[-3]; batchEmbeddings = dropout(batchEmbeddings, dropoutSrc, {srcWords, 1, 1}); } - // according to paper embeddings are scaled up by \sqrt(d_m) - auto scaledEmbeddings = std::sqrt((float)dimEmb) * batchEmbeddings; - - scaledEmbeddings = addSpecialEmbeddings(scaledEmbeddings, /*start=*/0, batch); + + batchEmbeddings = addSpecialEmbeddings(batchEmbeddings, /*start=*/0, batch); // reorganize batch and timestep - scaledEmbeddings = atleast_nd(scaledEmbeddings, 4); + batchEmbeddings = atleast_nd(batchEmbeddings, 4); batchMask = atleast_nd(batchMask, 4); - auto layer = transposeTimeBatch(scaledEmbeddings); // [-4: beam depth=1, -3: batch size, -2: max length, -1: vector dim] + auto layer = transposeTimeBatch(batchEmbeddings); // [-4: beam depth=1, -3: batch size, -2: max length, -1: vector dim] auto layerMask = reshape(transposeTimeBatch(batchMask), {1, dimBatch, 1, dimSrcWords}); // [-4: beam depth=1, -3: batch size, -2: vector dim=1, -1: max length] From fe56ba3f33c39231fe1661dd1df2603ee36c336c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 13 Jan 2019 10:34:08 -0800 Subject: [PATCH 116/838] first shot at factored embeddings --- src/layers/generic.cpp | 180 +++++++++++++++++++++++++++++++++++++++ src/models/transformer.h | 8 ++ 2 files changed, 188 insertions(+) create mode 100755 src/layers/generic.cpp diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp new file mode 100755 index 000000000..2b18a0255 --- /dev/null +++ b/src/layers/generic.cpp @@ -0,0 +1,180 @@ +#include "marian.h" + +#include "layers/generic.h" + +namespace marian { + struct CSRSparseTensor { // simplistic for now + Shape shape; + Expr values; // [k_i..k_{i+1}-1] -> value at [i,j] + Expr indices; // [k_i..k_{i+1}-1] -> j of non-null value + Expr offsets; // [i] -> k_i + }; + + class EmbeddingFactorMapping { + Vocab factorVocab_; // [factor name] -> factor index = row of E_ + Shape mappingShape; + std::vector> factorMap_; // [word index] -> set of factor indices + std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ + public: + // mapPath = path to file with entries in order of vocab entries of the form + // WORD FACTOR1 FACTOR2 FACTOR3... + // listPath = path to file that lists all FACTOR names + // vocab = original vocabulary + // Note: The WORD field in the map file is redundant. It is required for consistency checking only. + // Note: Presently, this implementation has the following short-comings + // - we do not group factors (to normalize the probs); instead we assume that the global softmax will normalize correctly + // - we do not handle binary features differently; we'd need to apply sigmoid(x) / sigmoid(-x) + EmbeddingFactorMapping(const std::string& mapPath, const std::string& factorVocabPath, const std::string& vocabPath) : + factorVocab_(New(), 0) { + // Note: We misuse the Vocab class a little. + // But it means that the factorVocab_ must contain and "". + Vocab vocab(New(), 0); + vocab.load(vocabPath); + factorVocab_.load(factorVocabPath); + // load and parse factorMap + factorMap_.resize(vocab.size()); + factorRefCounts_.resize(vocab.size()); + std::vector tokens; + io::InputFileStream in(mapPath); + std::string line; + size_t numTotalFactors = 0; + for (Word v = 0; io::getline(in, line); v++) { + tokens.clear(); // @BUGBUG: should be done in split() + utils::splitAny(line, tokens, " \t"); + ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[v], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + for (size_t i = 1; i < tokens.size(); i++) { + auto u = factorVocab_[tokens[i]]; + auto& m = factorMap_[v]; + m.push_back(u); + factorRefCounts_[u]++; + } + numTotalFactors += tokens.size() - 1; + } + LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, factorVocab_.size(), vocab.size()); + mappingShape = Shape({ (int)vocab.size(), (int)factorVocab_.size() }); + } + + size_t factorVocabSize() const { return factorVocab_.size(); } + + // create a CSR matrix M[V,U] from indices[] with + // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced + std::tuple/*weights*/, std::vector/*indices*/, std::vector/*offsets*/> csr_rows(const std::vector& words) const { + std::vector weights; + std::vector indices; + std::vector offsets; + offsets.reserve(words.size() + 1); + indices.reserve(words.size()); // (at least this many) + // loop over all input words, and select the corresponding set of unit indices into CSR format + offsets.push_back((IndexType)indices.size()); + for (auto v : words) { + const auto& m = factorMap_[v]; + for (auto u : m) { + indices.push_back(u); + weights.push_back(1.0f/(float)factorRefCounts_[u]); + } + offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset + } + return + std::tuple/*weights*/, std::vector/*indices*/, std::vector/*offsets*/> // (needed for unknown reasons) + { mappingShape, weights, indices, offsets }; + } + }; + + Embedding::Embedding(Ptr graph, Ptr options) : LayerBase(graph, options) { + std::string name = opt("prefix"); + int dimVoc = opt("dimVocab"); + int dimEmb = opt("dimEmb"); + + bool fixed = opt("fixed", false); + + if (options_->has("embedding-factors") && !embeddingFactorMapping_) { // (lazy init) + std::vector paths = opt>("embedding-factors"); + ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); + embeddingFactorMapping_ = New(paths[0], paths[1], opt("vocab")); + dimVoc = (int)embeddingFactorMapping_->factorVocabSize(); + } + + NodeInitializer initFunc = inits::glorot_uniform; + if (options_->has("embFile")) { + std::string file = opt("embFile"); + if (!file.empty()) { + bool norm = opt("normalization", false); + initFunc = inits::from_word2vec(file, dimVoc, dimEmb, norm); + } + } + + E_ = graph_->param(name, {dimVoc, dimEmb}, initFunc, fixed); + } + + // helper to embed a sequence of words (given as indices) via factored embeddings + /*private*/ Expr Embedding::multiRows(const std::vector& data) const + { + auto graph = E_->graph(); + Shape shape; std::vector weights; std::vector indices, offsets; std::tie + (shape, weights, indices, offsets) = embeddingFactorMapping_->csr_rows(data); + // multi-hot factor vectors are represented as a sparse CSR matrix + // [row index = word position index] -> set of factor indices for word at this position + return csr_dot( // the CSR matrix is passed in pieces + Shape({(int)offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), + graph->constant({(int)indices.size()}, inits::from_vector(weights), Type::float32), + graph->constant({(int)indices.size()}, inits::from_vector(indices), Type::uint32), + graph->constant({(int)offsets.size()}, inits::from_vector(offsets), Type::uint32), + E_); + } + + std::tuple Embedding::apply(Ptr subBatch) const /*override final*/ { + auto graph = E_->graph(); + int dimBatch = (int)subBatch->batchSize(); + int dimEmb = E_->shape()[-1]; + int dimWords = (int)subBatch->batchWidth(); + + // factored embeddings: + // - regular: + // - y = x @ E x:[B x 1ofV] ; E:[V x D] ; y:[B x D] + // - factored: + // - u = x @ M one-hot to U-dimensional multi-hot (all factors in one concatenated space) + // - each row of M contains the set of factors for one word => we want a CSR matrix + // - y = (x @ M) @ E (x:[B x 1ofV] ; M:[V x U]) ; E:[U x D] ; y:[B x D] + // - first compute x @ M on the CPU + // - (Uvalues, Uindices, Uoffsets) = csr_rows(Mvalues, Mindices, Moffsets, subBatch->data()): + // - shape (U, specifically) not actually needed here + // - foreach input x[i] + // - locate row M[i,*] + // - copy through its index values (std::vector) + // - create a matching ones vector (we can keep growing) + // - convert to GPU-side CSR matrix. CSR matrix now has #rows equal to len(x) + // - CSR matrix product with E + // - csr_dot(Uvalues, Uindices, Uoffsets, E_, transposeU) + // - double-check if all dimensions are specified. Probably not for transpose (which would be like csc_dot()). + // - weighting: + // - core factors' gradients are sums over all words that use the factors; + // - core factors' embeddings move very fast + // - words will need to make up for the move; rare words cannot + // - so, we multiply each factor with 1/refCount + // - core factors get weighed down a lot + // - no impact on gradients, as Adam makes up for it; embeddings still move fast just as before + // - but forward pass weighs them down, so that all factors are in a similar numeric range + // - if it is required to be in a different range, the embeddings can still learn that, but more slowly + + Expr chosenEmbeddings; + if (embeddingFactorMapping_) + chosenEmbeddings = multiRows(subBatch->data()); + else + chosenEmbeddings = rows(E_, subBatch->data()); + + auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb }); + auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, + inits::from_vector(subBatch->mask())); + return std::make_tuple(batchEmbeddings, batchMask); + } + + Expr Embedding::apply(const std::vector& embIdx, int dimBatch, int dimBeam) const /*override final*/ { + int dimEmb = E_->shape()[-1]; + Expr chosenEmbeddings; + if (embeddingFactorMapping_) + chosenEmbeddings = multiRows(embIdx); + else + chosenEmbeddings = rows(E_, embIdx); + return reshape(chosenEmbeddings, { dimBeam, 1, dimBatch, dimEmb }); + } +} // namespace marian diff --git a/src/models/transformer.h b/src/models/transformer.h index 48335d1d0..99999e8f6 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -627,6 +627,14 @@ class DecoderTransformer : public Transformer { if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; layerOut.tieTransposed(tiedPrefix); + // factored embeddings, simplistic version (which just adds the logits, like multiplying probs) + // z = h @ W // h:[B x D] ; W:[D x V] -> [B x V] + // with factors: + // z = h @ W @ M' // h:[B x D] ; W:[D x U] ; M':[U x V] -> [B x V] + // i.e. multiOutput(): + // output = dot_csr(output, M, transB=true) + // Should biases be done afterwards? Or maybe at two places? + // note: need to also specify output factors separately if not tied-embeddings or tied-embeddings-all } if(shortlist_) From e37a1f8a7dc71192ffbc9c7cbe7c00b13a9b16f8 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 15 Jan 2019 12:22:04 +0000 Subject: [PATCH 117/838] Rename variable --- src/common/cli_wrapper.h | 1 - src/common/config_parser.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index a748c560e..95eb3590d 100755 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -92,7 +92,6 @@ class CLIWrapper { // Command-line argument parser Ptr app_; - // Name of the default option group std::string defaultGroup_{""}; // Name of the current option group diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 1e75e7590..eef2e5b8f 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -763,14 +763,14 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { config_.remove("config"); if(!get("dump-config").empty() && get("dump-config") != "false") { - auto type = get("dump-config"); + auto dumpMode = get("dump-config"); config_.remove("dump-config"); - if(type == "explain") { + if(dumpMode == "explain") { cli.parseAliases(); } - bool minimal = (type == "minimal" || type == "explain"); + bool minimal = (dumpMode == "minimal" || dumpMode == "explain"); std::cout << cli.dumpConfig(minimal) << std::endl; exit(0); } From cc1c663e1c02a90b4e0267d0a50cc0e3f2616e06 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 12:53:50 -0800 Subject: [PATCH 118/838] bug fixes of factored source embeddings --- src/graph/node_operators_binary.h | 6 +++++- src/layers/generic.cpp | 20 +++++++++++++------- src/tensors/tensor.h | 4 ++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 357c24b69..860199d8e 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -428,6 +428,8 @@ class CSRDotNodeOp : public NaryNodeOp { ABORT_IF(A_offsets->shape()[0] - 1 != A_shape[0], "Sparse matrix offset vector has incorrect size"); auto outShape = B->shape(); + ABORT_IF((transA ? A_shape[0] : A_shape[1] != B->shape()[0]), + "Matrix product requires dimensions to match"); outShape.set(0, transA ? A_shape[1] : A_shape[0]); return outShape; } @@ -442,7 +444,9 @@ class CSRDotNodeOp : public NaryNodeOp { } NodeOps backwardOps() override { - return {nullptr, // can't backprop into the sparse matrix (the gradient is dense) + return {nullptr, // can't backprop into the sparse matrix pieces (the gradient is dense) + nullptr, + nullptr, NodeOp(CSRProd(child(3)->grad(), // child(3) = B graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), // children(0..2) = A diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2b18a0255..f651a3ce6 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -12,7 +12,6 @@ namespace marian { class EmbeddingFactorMapping { Vocab factorVocab_; // [factor name] -> factor index = row of E_ - Shape mappingShape; std::vector> factorMap_; // [word index] -> set of factor indices std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ public: @@ -33,7 +32,7 @@ namespace marian { factorVocab_.load(factorVocabPath); // load and parse factorMap factorMap_.resize(vocab.size()); - factorRefCounts_.resize(vocab.size()); + factorRefCounts_.resize(factorVocab_.size()); std::vector tokens; io::InputFileStream in(mapPath); std::string line; @@ -51,7 +50,6 @@ namespace marian { numTotalFactors += tokens.size() - 1; } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, factorVocab_.size(), vocab.size()); - mappingShape = Shape({ (int)vocab.size(), (int)factorVocab_.size() }); } size_t factorVocabSize() const { return factorVocab_.size(); } @@ -70,13 +68,13 @@ namespace marian { const auto& m = factorMap_[v]; for (auto u : m) { indices.push_back(u); - weights.push_back(1.0f/(float)factorRefCounts_[u]); + weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); } offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset } return std::tuple/*weights*/, std::vector/*indices*/, std::vector/*offsets*/> // (needed for unknown reasons) - { mappingShape, weights, indices, offsets }; + { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; } }; @@ -112,11 +110,19 @@ namespace marian { auto graph = E_->graph(); Shape shape; std::vector weights; std::vector indices, offsets; std::tie (shape, weights, indices, offsets) = embeddingFactorMapping_->csr_rows(data); +#if 0 // tests for special case of nop-op + ABORT_IF(data != indices, "bang"); + for (size_t i = 0; i < offsets.size(); i++) + ABORT_IF(offsets[i] != i, "boom"); + for (size_t i = 0; i < weights.size(); i++) + ABORT_IF(weights[i] != 1, "oops"); +#endif // multi-hot factor vectors are represented as a sparse CSR matrix // [row index = word position index] -> set of factor indices for word at this position + ABORT_IF(shape != Shape({(int)offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), "shape mismatch??"); return csr_dot( // the CSR matrix is passed in pieces - Shape({(int)offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), - graph->constant({(int)indices.size()}, inits::from_vector(weights), Type::float32), + shape, + graph->constant({(int)weights.size()}, inits::from_vector(weights), Type::float32), graph->constant({(int)indices.size()}, inits::from_vector(indices), Type::uint32), graph->constant({(int)offsets.size()}, inits::from_vector(offsets), Type::uint32), E_); diff --git a/src/tensors/tensor.h b/src/tensors/tensor.h index acc7e54c8..f77259cb9 100755 --- a/src/tensors/tensor.h +++ b/src/tensors/tensor.h @@ -140,6 +140,10 @@ class TensorBase : public std::enable_shared_from_this { template void set(const T* begin, const T* end) { + ABORT_IF(end - begin != shape_.elements(), + "Vector size ({}) and underlying type ({}) do not match", + end - begin, + std::string(shape_)); ABORT_IF(!matchType(type_), "Requested type ({}) and underlying type ({}) do not match", request(), From 9733db11fbbcebf2e7e3e7f840f3e496c6198407 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 15 Jan 2019 13:14:12 -0800 Subject: [PATCH 119/838] use ce-sum by default everywhere, but fake display as ce-mean --- src/layers/loss.cpp | 119 ++------ src/layers/loss.h | 283 ++++++++++++++++---- src/models/costs.h | 87 +++--- src/models/encoder_classifier.h | 28 +- src/models/encoder_decoder.cpp | 10 +- src/models/encoder_decoder.h | 30 +-- src/models/model_base.h | 7 +- src/models/transformer.h | 16 +- src/rescorer/rescorer.h | 6 +- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_multinode.cpp | 2 +- src/training/graph_group_multinode_sync.cpp | 2 +- src/training/graph_group_singleton.cpp | 2 +- src/training/graph_group_sync.cpp | 29 +- src/training/scheduler.h | 148 +++++----- src/training/training_state.h | 2 + src/training/validator.h | 20 +- 17 files changed, 446 insertions(+), 347 deletions(-) diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 4dbbb99ee..024b884c4 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -2,119 +2,38 @@ namespace marian { -Ptr LossFactory(Ptr options, bool inference) { +Ptr newLoss(Ptr options, bool inference) { float smoothing = inference ? 0.f : options->get("label-smoothing"); std::string costType = options->get("cost-type", "ce-mean"); if(costType == "ce-mean" || costType == "cross-entropy") { - return New(smoothing); + return New(smoothing); } else if(costType == "ce-mean-words") { - return New(smoothing); + return New(smoothing); } else if(costType == "ce-sum") { - return New(smoothing); + return New(smoothing); } else if(costType == "perplexity") { - return New(smoothing); + return New(smoothing); } else if(costType == "ce-rescore") { - return New(smoothing); + return New(); } else if(costType == "ce-rescore-mean") { - return New(smoothing); + return New(); } else { // same as ce-mean - return New(smoothing); + return New(smoothing); } } -Expr LossBase::getCrossEntropy(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = cross_entropy(logits, indices); +Ptr newMultiLoss(Ptr options) { + std::string multiLossType = options->get("multi-loss-type", "sum"); + if(multiLossType == "sum") + return New(); + else if(multiLossType == "scaled") + return New(); + else if(multiLossType == "normal") + return New(); + else + ABORT("Unknown multi-loss-type {}", multiLossType); - if(smoothing_ > 0) { - // @TODO: add this to CE kernels instead - auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; - } - - if(mask) - ce = ce * mask; - - if(weights) - ce = ce * weights; - - return ce; -} - -Expr CrossEntropyMeanLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - // Time axis (words): -3 - // Batch axis (sentences): -2 - // if(weights) { - // return sum(sum(ce, /*axis =*/ -3) /*axis =*/ -2); - // / sum(mean(mask * weights, /*axis =*/ -3) /*axis =*/ -2); - // } - // else { - return mean(sum(ce, /*axis =*/ -3), /*axis =*/ -2); - // } -} - -Expr CrossEntropyMeanWordsLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - if(mask) { - return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch - / sum(sum(mask, /*axis =*/ -3), /*axis =*/ -2); // divide by number of words (sum over mask) - } else { - return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch - / ce->shape().elements(); // no mask, hence divide by all number of all elements - } - -} - -Expr CrossEntropySumLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - return sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2); -} - -Expr PerplexityLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - if(mask) { - return exp(sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch - / sum(sum(mask, /*axis =*/ -3), /*axis =*/ -2)); // divide by number of words (sum over mask) - } else { - return exp(sum(sum(ce, /*axis =*/ -3), /*axis =*/ -2) // sum CE over all words in the batch - / ce->shape().elements()); // divide by number of words (sum over mask) - } -} - -Expr CrossEntropyRescoreLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - return -sum(ce, /*axis =*/ -3); -} - -Expr CrossEntropyRescoreMeanLoss::getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights) { - auto ce = getCrossEntropy(logits, indices, mask, weights); - // divide by number of words in sentence - if(mask) { - return -sum(ce, /*axis =*/ -3) / sum(mask, /*axis =*/ -3); - } else { - return -sum(ce, /*axis =*/ -3) / ce->shape()[-3]; - } + return nullptr; } } // namespace marian diff --git a/src/layers/loss.h b/src/layers/loss.h index ebf711470..eaa98da04 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -3,74 +3,263 @@ #include "marian.h" namespace marian { -class LossBase { + +class RationalLoss { protected: - float smoothing_; + Expr loss_; + Expr labels_; + + RationalLoss() = default; // protected public: - explicit LossBase(float smoothing = 0) : smoothing_(smoothing){}; - - Expr getCrossEntropy(Expr logits, Expr indices, Expr mask, Expr weights); - virtual Expr getCost(Expr logits, - Expr indices, - Expr mask, - Expr weights = nullptr) - = 0; + RationalLoss(Expr loss, Expr labels) + : loss_(loss), labels_(labels) {} + + RationalLoss(const RationalLoss& other) + : loss_(other.loss_), labels_(other.labels_) {} + + virtual ~RationalLoss() = default; + + Expr loss() const { return loss_; } + + template + void loss(std::vector& losses) const { + ABORT_IF(!loss_, "Loss has not been defined"); + loss_->val()->get(losses); + } + + template + T loss() const { // this will fail if loss is not a single value + ABORT_IF(!loss_, "Loss has not been defined"); + return loss_->val()->scalar(); + } + + Expr labels() const { return labels_; } + + template + void labels(std::vector& labels) const { + ABORT_IF(!labels_, "Labels have not been defined"); + labels_->val()->get(labels); + } + + template + T labels() const { // this will fail if loss is not a single value + ABORT_IF(!labels_, "Labels have not been defined"); + return labels_->val()->scalar(); + } + + size_t size() const { + ABORT_IF(!labels_, "Labels have not been defined"); + return labels_->shape().elements(); + } }; -/* - * @brief The cross entropy loss function - * - * A sum over words and average over sentences - */ -class CrossEntropyMeanLoss : public LossBase { +// POD for accumulating loss values after backprop +struct StaticLoss { + float loss; + float labels; + + StaticLoss() : loss(0.f), labels(0.f) {} + + StaticLoss(const RationalLoss& rl) + : loss(rl.loss()), labels(rl.labels()) {} + + StaticLoss& operator +=(const StaticLoss& other) { + loss = loss + other.loss; + labels = labels + other.labels; + return *this; + } +}; + +class MultiRationalLoss : public RationalLoss { +protected: + std::vector partialLosses_; + + virtual Expr accumulateLoss(const RationalLoss& current) = 0; + virtual Expr accumulateLabels(const RationalLoss& current) = 0; + public: - explicit CrossEntropyMeanLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + MultiRationalLoss() : RationalLoss() {} + + MultiRationalLoss(const RationalLoss& rl) + : RationalLoss(rl.loss(), rl.labels()) { + partialLosses_.push_back(rl); + } + + void push_back(const RationalLoss& current) { + loss_ = accumulateLoss(current); + labels_ = accumulateLabels(current); + partialLosses_.push_back(current); + } + + const RationalLoss& operator[](size_t i) { + return partialLosses_[i]; + } + + auto begin() -> decltype(partialLosses_.begin()) const { + return partialLosses_.begin(); + } + + auto end() -> decltype(partialLosses_.end()) const { + return partialLosses_.end(); + } + + size_t size() const { + return partialLosses_.size(); + } + }; -/* - * @brief The cross entropy loss function as an average over target tokens - */ -class CrossEntropyMeanWordsLoss : public LossBase { +class SumMultiRationalLoss : public MultiRationalLoss { +private: + virtual Expr accumulateLoss(const RationalLoss& current) override { + if(loss_) + return loss_ + current.loss(); + else + return current.loss(); + } + + virtual Expr accumulateLabels(const RationalLoss& current) override { + if(labels_) + return labels_ + current.labels(); + else + return current.labels(); + } + public: - explicit CrossEntropyMeanWordsLoss(float smoothing = 0) - : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + SumMultiRationalLoss() : MultiRationalLoss() {} + SumMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} }; -/* - * @brief The cross entropy loss function as a sum over target tokens - */ -class CrossEntropySumLoss : public LossBase { +class MeanMultiRationalLoss : public MultiRationalLoss { +private: + virtual Expr accumulateLoss(const RationalLoss& current) override { + if(loss_) + return loss_ + current.loss() / current.labels(); + else + return current.loss() / current.labels(); + } + + virtual Expr accumulateLabels(const RationalLoss& current) override { + if(labels_) + return labels_ + 1.f; // broadcast to size + else + return current.labels() / current.labels(); // 1, but with correct size + } + +public: + MeanMultiRationalLoss() : MultiRationalLoss() {} + MeanMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} +}; + +class ScaledMultiRationalLoss : public MultiRationalLoss { +private: + virtual Expr accumulateLoss(const RationalLoss& current) override { + if(loss_) { + const auto& first = partialLosses_.front(); + return loss_ + first.labels() * (current.loss() / current.labels()); // scale up/down to match scale of first loss + } else { + return current.loss(); // first reference loss, keeps to scale with this one + } + } + + virtual Expr accumulateLabels(const RationalLoss& current) override { + if(labels_) { + const auto& first = partialLosses_.front(); + return labels_ + first.labels() / current.labels(); // fractional label counts are OK + } else { + return current.labels(); + } + } + public: - explicit CrossEntropySumLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + ScaledMultiRationalLoss() : MultiRationalLoss() {} + ScaledMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} }; -/* - * @brief The perplexity loss function - */ -class PerplexityLoss : public LossBase { +Ptr newMultiLoss(Ptr options); + +//***********************************************************************************// +// This needs some to be refactored. Currentl easiest route for backwards compat. + +class LabelwiseLoss { +protected: + std::vector axes_; + + virtual Expr compute(Expr logits, Expr labelIndices, + Expr mask = nullptr, Expr labelWeights = nullptr) = 0; + + RationalLoss reduce(Expr loss, Expr labels) { + ABORT_IF(!loss, "Loss has not been computed"); + ABORT_IF(!labels, "Labels have not been computed"); + + Expr lossSum = loss; + Expr labelsSum = labels; + for(int i = 0; i < axes_.size(); ++i) { + lossSum = sum(lossSum, axes_[i]); + labelsSum = sum(labelsSum, axes_[i]); + } + + return RationalLoss(lossSum, labelsSum); + } + public: - explicit PerplexityLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + LabelwiseLoss(const std::vector& axes) + : axes_(axes) { } + + RationalLoss apply(Expr logits, Expr labelIndices, + Expr mask = nullptr, Expr labelWeights = nullptr) { + Expr loss = compute(logits, labelIndices, mask, labelWeights); + + Expr labels; + if(mask) + labels = mask; + else + labels = constant_like(loss, inits::ones); // we have no mask, assume all items are labels + + return reduce(loss, labels); + } }; -/* - * @brief The cross entropy loss function that keeps sentence-level costs - */ -class CrossEntropyRescoreLoss : public LossBase { +class CrossEntropyLoss : public LabelwiseLoss { public: - explicit CrossEntropyRescoreLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + CrossEntropyLoss(float smoothing) + : LabelwiseLoss(/*axes=*/{-2, -3}), // cross-entropy already reduces over axis -1 + smoothing_(smoothing) {} + + CrossEntropyLoss(const std::vector& axes, float smoothing) + : LabelwiseLoss(axes), // cross-entropy already reduces over axis -1 + smoothing_(smoothing) {} + +protected: + float smoothing_; + + Expr compute(Expr logits, Expr labelIndices, + Expr mask = nullptr, Expr labelWeights = nullptr) override { + Expr ce = cross_entropy(logits, labelIndices); + + if(smoothing_ > 0) { + // @TODO: add this to CE kernels instead + Expr ceq = mean(logsoftmax(logits), /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceq; + } + + if(mask) + ce = ce * mask; + + if(labelWeights) + ce = ce * labelWeights; + + return ce; + } }; -class CrossEntropyRescoreMeanLoss : public LossBase { +class RescorerLoss : public CrossEntropyLoss { public: - explicit CrossEntropyRescoreMeanLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + // sentence-wise CE, hence reduce only over time axis. + RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} }; -Ptr LossFactory(Ptr options, bool inference); +Ptr newLoss(Ptr options, bool inference); + } // namespace marian diff --git a/src/models/costs.h b/src/models/costs.h index da190e9ca..7f5f0743e 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -12,11 +12,10 @@ namespace models { class CostBase { public: - virtual Expr apply(Ptr model, - Ptr graph, - Ptr batch, - bool clearGraph = true) - = 0; + virtual Ptr apply(Ptr model, + Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; }; class EncoderDecoderCE : public CostBase { @@ -25,13 +24,13 @@ class EncoderDecoderCE : public CostBase { bool inference_{false}; bool toBeWeighted_{false}; - Ptr loss_; + Ptr loss_; Ptr weighter_; public: EncoderDecoderCE(Ptr options) : options_(options), inference_(options->get("inference", false)) { - loss_ = LossFactory(options_, inference_); + loss_ = newLoss(options_, inference_); toBeWeighted_ = (options_->has("data-weighting") && !inference_) @@ -41,7 +40,7 @@ class EncoderDecoderCE : public CostBase { weighter_ = WeightingFactory(options_); } - Expr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -54,11 +53,16 @@ class EncoderDecoderCE : public CostBase { if(toBeWeighted_) weights = weighter_->getWeights(graph, corpusBatch); - Expr cost; - cost = loss_->getCost(state->getLogProbs(), - state->getTargetIndices(), - state->getTargetMask(), - weights); + // multi-objective training + Ptr multiLoss = newMultiLoss(options_); + + // @TODO: adapt to multi-objective training + auto partialLoss = loss_->apply(state->getLogProbs(), + state->getTargetIndices(), + state->getTargetMask(), + weights); + multiLoss->push_back(partialLoss); + if(options_->get("guided-alignment", std::string("none")) != "none" && !inference_) { auto alignments = encdec->getDecoders()[0]->getAlignments(); @@ -66,9 +70,12 @@ class EncoderDecoderCE : public CostBase { auto att = concatenate(alignments, /*axis =*/ -1); - return cost + guidedAlignmentCost(graph, corpusBatch, options_, att); + // @TODO: represent this as an multi-objective training loss, which it actually is + // return multiLoss.loss() + guidedAlignmentCost(graph, corpusBatch, options_, att); + ABORT("Fix me"); + return multiLoss; } else { - return cost; + return multiLoss; } } }; @@ -77,15 +84,15 @@ class EncoderClassifierCE : public CostBase { protected: Ptr options_; bool inference_{false}; - Ptr loss_; + Ptr loss_; public: EncoderClassifierCE(Ptr options) : options_(options), inference_(options->get("inference", false)) { - loss_ = LossFactory(options_, inference_); + loss_ = newLoss(options_, inference_); } - Expr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -94,21 +101,17 @@ class EncoderClassifierCE : public CostBase { auto corpusBatch = std::static_pointer_cast(batch); auto states = enccls->apply(graph, corpusBatch, clearGraph); - - Expr cost = loss_->getCost(states[0]->getLogProbs(), - states[0]->getTargetIndices(), - /*mask=*/nullptr, - /*weights=*/nullptr); - + // multi-objective training - for(int i = 1; i < states.size(); ++i) { - cost = cost + loss_->getCost(states[i]->getLogProbs(), - states[i]->getTargetIndices(), - /*mask=*/nullptr, - /*weights=*/nullptr); + Ptr multiLoss = newMultiLoss(options_); + for(int i = 0; i < states.size(); ++i) { + auto partialLoss = loss_->apply(states[i]->getLogProbs(), + states[i]->getTargetIndices(), + /*mask=*/nullptr, + /*weights=*/nullptr); + multiLoss->push_back(partialLoss); } - - return cost; + return multiLoss; } }; @@ -135,9 +138,9 @@ class Trainer : public ModelBase { model_->save(graph, name, saveTranslatorConfig); } - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { return cost_->apply(model_, graph, batch, clearGraph); }; @@ -156,7 +159,7 @@ class LogSoftmaxStep : public CostStep { virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) auto logits = state->getLogProbs(); - + auto logprobs = logsoftmax(logits); state->setLogProbs(logprobs); @@ -171,7 +174,7 @@ class GumbelSoftmaxStep : public CostStep { public: virtual Ptr apply(Ptr state) override { auto logits = state->getLogProbs(); - + auto logprobs = logsoftmax(logits + constant_like(logits, inits::gumbel)); state->setLogProbs(logprobs); @@ -211,9 +214,9 @@ class Stepwise : public EncoderDecoderBase { virtual void clear(Ptr graph) override { encdec_->clear(graph); } - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto corpusBatch = std::static_pointer_cast(batch); return build(graph, corpusBatch, clearGraph); } @@ -234,9 +237,9 @@ class Stepwise : public EncoderDecoderBase { return cost_->apply(nextState); } - virtual Expr build(Ptr /*graph*/, - Ptr /*batch*/, - bool /*clearGraph*/ = true) override { + virtual Ptr build(Ptr /*graph*/, + Ptr /*batch*/, + bool /*clearGraph*/ = true) override { ABORT("Wrong wrapper. Use models::Trainer or models::Scorer"); return nullptr; } diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 0bf048b93..9c794144c 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -33,15 +33,13 @@ class EncoderClassifierBase : public models::ModelBase { virtual std::vector> apply(Ptr, Ptr, bool) = 0; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override - = 0; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) - = 0; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual Ptr getOptions() = 0; }; @@ -194,17 +192,17 @@ class EncoderClassifier : public EncoderClassifierBase { return classifierStates; } - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto states = apply(graph, batch, clearGraph); // returns raw logits - return states[0]->getLogProbs(); + return New(states[0]->getLogProbs(), nullptr); // @TODO: this should explode } - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto corpusBatch = std::static_pointer_cast(batch); return build(graph, corpusBatch, clearGraph); } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 047d960d1..70e71c836 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -183,16 +183,16 @@ Ptr EncoderDecoder::stepAll(Ptr graph, return nextState; } -Expr EncoderDecoder::build(Ptr graph, - Ptr batch, - bool clearGraph) { +Ptr EncoderDecoder::build(Ptr graph, + Ptr batch, + bool clearGraph) { auto state = stepAll(graph, batch, clearGraph); // returns raw logits - return state->getLogProbs(); + return New(state->getLogProbs(), state->getTargetMask()); // @TODO: hacky hack hack } -Expr EncoderDecoder::build(Ptr graph, +Ptr EncoderDecoder::build(Ptr graph, Ptr batch, bool clearGraph) { auto corpusBatch = std::static_pointer_cast(batch); diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index 0ea89b4f6..a33de385f 100644 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -28,14 +28,14 @@ class EncoderDecoderBase : public models::ModelBase { virtual void clear(Ptr graph) override = 0; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override - = 0; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; - virtual Ptr startState(Ptr graph, - Ptr batch) - = 0; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual Ptr startState(Ptr graph, + Ptr batch) = 0; virtual Ptr step(Ptr graph, Ptr state, @@ -45,10 +45,6 @@ class EncoderDecoderBase : public models::ModelBase { int beamSize) = 0; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) - = 0; virtual Ptr getOptions() = 0; @@ -158,13 +154,13 @@ class EncoderDecoder : public EncoderDecoderBase { Ptr batch, bool clearGraph = true); - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override; + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override; }; } // namespace marian diff --git a/src/models/model_base.h b/src/models/model_base.h index 47039841a..0701a8b4b 100644 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -2,6 +2,7 @@ #include #include "marian.h" +#include "layers/loss.h" namespace marian { namespace models { @@ -26,9 +27,9 @@ class ModelBase { bool saveTranslatorConfig = false) = 0; - virtual Expr build(Ptr graph, - Ptr batch, - bool clearGraph = true) + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual void clear(Ptr graph) = 0; diff --git a/src/models/transformer.h b/src/models/transformer.h index 3791d3c80..a5fba81aa 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -55,19 +55,25 @@ class Transformer : public EncoderOrDecoderBase { if(learnedPosEmbeddings) { int maxLength = opt("max-length"); + // Hack for translating with length longer than trained embeddings + Expr seenEmb = graph_->get("Wpos"); + int numPos = seenEmb ? seenEmb->shape()[-2] : maxLength; + auto posEmbFactory = embedding(graph_) ("prefix", "Wpos") // share positional embeddings across all encoders/decorders - ("dimVocab", maxLength) + ("dimVocab", numPos) ("dimEmb", dimEmb) .construct(); - std::vector positions(dimWords); - std::iota(positions.begin(), positions.end(), 0); // fill with increasing numbers until current length + // fill with increasing numbers until current length or maxPos + std::vector positions(dimWords, numPos - 1); + for(int i = 0; i < std::min(dimWords, numPos); ++i) + positions[i] = i; auto signal = rows(posEmbFactory, graph_->indices(positions)); signal = reshape(signal, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; - } else { + } else { auto signal = graph_->constant({dimWords, 1, dimEmb}, inits::positions(start)); // according to paper embeddings are scaled up by \sqrt(d_m) @@ -560,7 +566,7 @@ class EncoderTransformer : public Transformer { int srcWords = batchEmbeddings->shape()[-3]; batchEmbeddings = dropout(batchEmbeddings, dropoutSrc, {srcWords, 1, 1}); } - + batchEmbeddings = addSpecialEmbeddings(batchEmbeddings, /*start=*/0, batch); // reorganize batch and timestep diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index fa456856a..64ed3cde2 100644 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -29,7 +29,7 @@ class Rescorer { builder_->load(graph, modelFile); } - Expr build(Ptr graph, Ptr batch) { + Ptr build(Ptr graph, Ptr batch) { return builder_->build(graph, batch); } @@ -126,13 +126,13 @@ class Rescore : public ModelTask { // @TODO: normalize by length as in normalize // Once we have Frank's concept of ce-sum with sample size by words we will return a pair - // here which will make it trivial to report all variants. + // here which will make it trivial to report all variants. auto costNode = builder->build(graph, batch); graph->forward(); std::vector scores; - costNode->val()->get(scores); + costNode->loss(scores); // soft alignments for each sentence in the batch std::vector aligns(batch->size()); diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 3195c5624..b529089af 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -211,7 +211,7 @@ void AsyncGraphGroup::execute(Ptr batch) { } graph->forward(); - cost += costNode->scalar(); + cost += costNode->loss(); graph->backward(); Tensor gradients; diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index bac76b945..8adafd89a 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -537,7 +537,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { } graph->forward(); - cost += costNode->scalar(); + cost += costNode->loss(); num_seen_words += batch->words(); num_seen_sentences += batch->size(); graph->backward(); diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp index 05abd6d5f..f32fb0d28 100755 --- a/src/training/graph_group_multinode_sync.cpp +++ b/src/training/graph_group_multinode_sync.cpp @@ -195,7 +195,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { graph->forward(); { std::lock_guard guard(sumCostMutex_); - cost += costNode->scalar(); + cost += costNode->loss(); num_seen_words += batch->words(); num_seen_sentences += batch->size(); } diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp index ac6ef5d0c..46084c787 100755 --- a/src/training/graph_group_singleton.cpp +++ b/src/training/graph_group_singleton.cpp @@ -13,7 +13,7 @@ void SingletonGraph::execute(Ptr batch) { auto costNode = builder_->build(graph_, batch); graph_->forward(); - float cost = costNode->scalar(); + float cost = costNode->loss(); graph_->backward(); // Get batch stats diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index c3b025e31..dafd61246 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -134,7 +134,8 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { // Compute gradients // This happens in multiple steps in case of delay_ > 1. - std::vector localDeviceCosts(devices_.size(), 0.f); // [local device index] aggregate cost for each local device + std::vector localDeviceCosts(devices_.size()); + for (size_t t = 0; t < delay_; t++) { // Execute single forward/backward step auto forwardBackward = [&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { @@ -143,11 +144,12 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { if(subBatch) { timer::Timer timer; - auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); + auto rationalLoss = builders_[localDeviceIndex]->build(graph, subBatch); //LOG(info, timer.format(2, "after build: %ws")); graph->forward(); //LOG(info, timer.format(2, "after forward (no sync): %ws")); - localDeviceCosts[localDeviceIndex] += costNode->scalar(); + localDeviceCosts[localDeviceIndex] += *rationalLoss; // converts dynamic RationalLoss to StaticLoss + graph->backward(/*zero=*/t == 0); // only reset gradients to 0 if t = 0 //LOG(info, timer.format(2, "after backward (no sync): %ws")); //localDeviceCosts[localDeviceIndex] += costNode->scalar(); // moved here for time measurements; @TODO: move this back @@ -168,15 +170,6 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { auto curGrad = graphs_[idx]->params()->grads()->subtensor(begin, end-begin); auto curParam = graphs_[idx]->params()->vals()->subtensor(begin, end-begin); - // if individual gradients were averages, then need to average again over all subBatches - auto div = subBatches.size(); - if (options_->get("cost-type") == "ce-sum") - div = 1; - if(div != 1) { - using namespace functional; - Element(_1 = _1 / (float)div, curGrad); - } - // actual model update shardOpt_[idx]->update(curParam, curGrad); @@ -196,17 +189,13 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { //LOG(info, timer.format(2, "after allGather (has sync): %ws")); // cost across all local devices (scheduler will aggregate cross-process) - float localCost = 0; + StaticLoss localLoss; for(auto& c : localDeviceCosts) // localDeviceCosts is already summed up over delay steps - localCost += c; - - // if localCost is average-based, we need to turn the sum over devices into an average as well - if(options_->get("cost-type") != "ce-sum") - localCost /= numSubBatches; + localLoss += c; if(scheduler_) { - // track and log localCost - scheduler_->update(localCost, subBatches, mpi_); + // track and log localLoss, @TODO: rather pass StaticLoss object + scheduler_->update(localLoss.loss, localLoss.labels, subBatches, mpi_); // save intermediate model (and optimizer state) to file if(scheduler_->saving()) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 1e9491cc8..d35749901 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -44,6 +44,33 @@ class Scheduler : public TrainingObserver { return baselr; } + std::string displayLoss(std::string lossType, bool dispLabelCounts, Ptr state) { + std::stringstream ss; + ss << "Cost "; + ss << std::setprecision(8) << std::fixed; + + // @TODO: put a single loss formatting funciton into loss.h and reuse here to avoid code duplication + // @TODO: use dispLabelCounts with any display type? + if(lossType == "ce-mean-words") + ss << state->costSum / state->costCount; + else if(lossType == "ce-sum" && dispLabelCounts) + ss << state->costSum / state->costCount + << " * " + << utils::withCommas(state->costCount) + << " after " + << utils::withCommas(state->labelsTotal); + else if(lossType == "ce-sum" && !dispLabelCounts) + ss << state->costSum; + else if(lossType == "perplexity") + ss << std::exp(state->costSum / state->costCount); + else if(lossType == "cross-entropy" || lossType == "ce-mean") // backwards-compat, @TODO: get rid of this? + ss << state->costSum / state->samplesCount; + else + ABORT("Unknown loss type {}", lossType); + + return ss.str(); + } + public: Scheduler(Ptr options, Ptr state) : options_(options), state_(state), @@ -160,53 +187,44 @@ class Scheduler : public TrainingObserver { } void update(float cost, Ptr batch) { - update(cost, std::vector>({batch})); + ABORT("Fix me"); + update(cost, batch->words(-1), std::vector>({batch})); } - void update(float cost, const std::vector>& batches, Ptr mpi = nullptr) { + void update(float cost, float labels, const std::vector>& batches, Ptr mpi = nullptr) { state_->rememberPreviousProgress(); // note: epoch increases happen at the wrong place, hence -freq parameters do not support epoch units state_->validated = false; size_t batchSize = 0; // number of sentences in batch - size_t batchLabels = 0; // number of target words in batch + size_t batchLabels = (size_t)labels; // number of target words in batch + size_t batchWords = 0; for(const auto& batch : batches) { if (batch) { // (nullptr is allowed as result of split) - batchSize += batch->size(); - batchLabels += batch->words(dispIndex_); + batchSize += batch->size(); + batchWords += batch->words(dispIndex_); } } // extrapolate cost across MPI processes, so that we have numbers in the right range // When doing the actual log, we then aggregate across MPI processes to get the accurate number. if (mpi) - cost *= mpi->numMPIProcesses(); // @BUGBUG: this is presently correct for ce-sum, but possibly not the av-based losses + cost *= mpi->numMPIProcesses(); - // reconstruct sum cost, for displaying epoch-level averages instead of minibatch-level - auto costType = options_->get("cost-type"); - auto dispLabelCounts = options_->get( - "disp-label-counts"); // if true then show as "cost per label * number of labels" - if(dispLabelCounts) { - auto count = // what was cost normalized with originally? - /*if*/ (costType == "ce-sum") ? - 1 - /*else if*/ : ((costType == "ce-mean-words") ? - batchLabels - /*else*/ : // all others: treat like ce-mean (not correct for some) - batchSize); - state_->costSum += cost * count; // aggregate sum cost since last display - state_->costCount += batchLabels; // cost gets normalized w.r.t. this in display - } else { // (back compat) - state_->costSum += cost * batchSize; - state_->costCount += batchSize; - } + state_->costSum += cost; // aggregate sum cost since last display + state_->costCount += batchLabels; // cost gets normalized w.r.t. this in display + state_->samplesCount += batchSize; - state_->wordsDisp += batchLabels; // words at given input processed since last display, for speed display + state_->wordsDisp += batchWords; // words at given input processed since last display, for speed display state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed state_->newBatch(); + // reconstruct sum cost, for displaying epoch-level averages instead of minibatch-level + auto lossType = options_->get("cost-type"); + auto dispLabelCounts = options_->get("disp-label-counts"); // if true then show as "cost per label * number of labels" + if(state_->enteredNewPeriodOf(options_->get("disp-freq")) || state_->batches <= options_->get("disp-first")) { // if MPI then aggregate precise cost across workers @@ -216,62 +234,37 @@ class Scheduler : public TrainingObserver { mpi->allReduce(&state_->costSum, &state_->costSum, 1, MPI_FLOAT, MPI_SUM); //LOG(info, "all-reduced cost to {}", state_->costSum); } - if (mpi && mpi->myMPIRank() != 0) - ; // skip the report on alternate worker processes - else if(dispLabelCounts) { - if(options_->get("lr-report")) { // if true then show the learning rate - LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} after {} : Time {:.2f}s : {:.2f} " - "words/s : L.r. {:.4e}", - state_->epochs, - state_->batches, - utils::withCommas(state_->samplesEpoch), - state_->costSum / state_->costCount, - utils::withCommas(state_->costCount), // show cost as "av * count" - utils::withCommas(state_->labelsTotal), - timer_.elapsed(), - state_->wordsDisp / timer_.elapsed(), - state_->eta); - } else { - LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} * {} after {} : Time {:.2f}s : {:.2f} " - "words/s", - state_->epochs, - state_->batches, - utils::withCommas(state_->samplesEpoch), - state_->costSum / state_->costCount, - utils::withCommas(state_->costCount), - utils::withCommas(state_->labelsTotal), - timer_.elapsed(), - state_->wordsDisp / timer_.elapsed()); - } + if (mpi && mpi->myMPIRank() != 0) { + // skip the report on alternate worker processes + } else if(options_->get("lr-report")) { + LOG(info, + "Ep. {} : Up. {} : Sen. {} : {} : Time {:.2f}s : {:.2f} words/s : L.r. {:.4e}", + state_->epochs, + state_->batches, + utils::withCommas(state_->samplesEpoch), + displayLoss(lossType, dispLabelCounts, state_), + timer_.elapsed(), + state_->wordsDisp / timer_.elapsed(), + state_->eta); } else { - if(options_->get("lr-report")) { - LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} : Time {:.2f}s : {:.2f} words/s : L.r. {:.4e}", - state_->epochs, - state_->batches, - utils::withCommas(state_->samplesEpoch), - state_->costSum / state_->costCount, - timer_.elapsed(), - state_->wordsDisp / timer_.elapsed(), - state_->eta); - } else { - LOG(info, - "Ep. {} : Up. {} : Sen. {} : Cost {:.8f} : Time {:.2f}s : {:.2f} words/s", - state_->epochs, - state_->batches, - utils::withCommas(state_->samplesEpoch), - state_->costSum / state_->costCount, - timer_.elapsed(), - state_->wordsDisp / timer_.elapsed()); - } + LOG(info, + "Ep. {} : Up. {} : Sen. {} : {} : Time {:.2f}s : {:.2f} words/s", + state_->epochs, + state_->batches, + utils::withCommas(state_->samplesEpoch), + displayLoss(lossType, dispLabelCounts, state_), + timer_.elapsed(), + state_->wordsDisp / timer_.elapsed()); } + + timer_.start(); - state_->costSum = 0; - state_->costCount = 0; - state_->wordsDisp = 0; + state_->costSum = 0; + state_->costCount = 0; + state_->samplesCount = 0; + state_->wordsDisp = 0; } + // progress heartbeat for MS-internal Philly compute cluster // This environment variable exists when running on the cluster. if((!mpi || mpi->myMPIRank() == 0) && getenv("PHILLY_JOB_ID") @@ -293,6 +286,7 @@ class Scheduler : public TrainingObserver { state_->samplesEpoch = 0; state_->costSum = 0; state_->costCount = 0; + state_->samplesCount = 0; state_->wordsDisp = 0; } diff --git a/src/training/training_state.h b/src/training/training_state.h index 2deefe471..d4ecdb6cb 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -99,6 +99,8 @@ class TrainingState { size_t costCount{0}; // Number of words seen since last display, for speed measurement size_t wordsDisp{0}; + // Number of samples/sentences seen since last display + size_t samplesCount{0}; // The state of the random number generator from a batch generator std::string seedBatch; diff --git a/src/training/validator.h b/src/training/validator.h index f22dd56e0..052b7e15b 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -190,13 +190,14 @@ class CrossEntropyValidator : public Validator { } builder->clear(graph); - auto costNode = builder->build(graph, batch); + auto loss = builder->build(graph, batch); graph->forward(); std::unique_lock lock(mutex_); - cost += costNode->scalar(); + cost += loss->loss(); + words += (size_t)loss->labels(); + samples += batch->size(); - words += batch->back()->batchWords(); }; taskBarrier.push_back(threadPool_.enqueue(task, batchId)); @@ -254,7 +255,7 @@ class AccuracyValidator : public Validator { graph = graphs[id % graphs.size()]; } - // Future: requires argmax implementation and integer arithmetics + // @TODO: requires argmax implementation and integer arithmetics // builder->clear(graph); // auto predicted = argmax(builder->build(graph, batch), /*axis*/-1); // auto labels = graph->indices(batch->back()->data()); @@ -263,20 +264,21 @@ class AccuracyValidator : public Validator { // std::unique_lock lock(mutex_); // totalLabels += labels->shape().elements(); - // correct += correct->scalar(); + // correct += correct->scalar(); builder->clear(graph); - auto logits = builder->build(graph, batch); + Expr logits = builder->build(graph, batch)->loss(); graph->forward(); std::vector vLogits; logits->val()->get(vLogits); - const auto& labels = batch->back()->data(); + + const auto& groundTruth = batch->back()->data(); IndexType cols = logits->shape()[-1]; size_t thisCorrect = 0; - size_t thisLabels = labels.size(); + size_t thisLabels = groundTruth.size(); for(int i = 0; i < thisLabels; ++i) { // CPU-side Argmax @@ -289,7 +291,7 @@ class AccuracyValidator : public Validator { bestIndex = j; } } - thisCorrect += (size_t)(bestIndex == labels[i]); + thisCorrect += (size_t)(bestIndex == groundTruth[i]); } std::unique_lock lock(mutex_); From 31f4f721b41b8a867808babaf975092ea04c4d5a Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 13:25:36 -0800 Subject: [PATCH 120/838] scaffolding for new operation dot_csr(); renamed prod.cu to prod.cpp, since it no longer has CUDA code --- src/CMakeLists.txt | 2 +- src/graph/expression_operators.cpp | 8 +- src/graph/expression_operators.h | 1 + src/graph/node_operators_binary.h | 37 ++++---- src/tensors/cpu/prod.cpp | 3 +- src/tensors/gpu/{prod.cu => prod.cpp} | 119 +++++++++++++------------- src/tensors/gpu/prod.h | 1 + src/tensors/tensor_operators.h | 2 +- vs/Marian.vcxproj | 2 +- vs/Marian.vcxproj.filters | 6 +- 10 files changed, 98 insertions(+), 83 deletions(-) rename src/tensors/gpu/{prod.cu => prod.cpp} (74%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 03524117b..8c757f100 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -103,7 +103,7 @@ if(CUDA_FOUND) cuda_add_library(marian_cuda tensors/gpu/device.cu tensors/gpu/algorithm.cu - tensors/gpu/prod.cu + tensors/gpu/prod.cpp tensors/gpu/element.cu tensors/gpu/add.cu tensors/gpu/tensor_operators.cu diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 6828bf103..4694351b2 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -421,7 +421,13 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { // A[i,j] is at A_values[A_offsets[i]+k], where k is position of j in A_indices[A_offsets[i]:A_offsets[i+1]] // @TODO: Define a proper sparse tensor type. Expr csr_dot(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA /*= false*/) { - return Expression(A_shape, A_values, A_indices, A_offsets, B, transA); + return Expression(A_shape, A_values, A_indices, A_offsets, B, transA, /*swapOperands=*/false); +} + +// multiply a matrix A with a CSR matrix B +// @TODO: Define a proper sparse tensor type. +Expr dot_csr(Expr A, const Shape& B_shape, Expr B_values, Expr B_indices, Expr B_offsets, bool transB /*= false*/) { + return Expression(B_shape, B_values, B_indices, B_offsets, A, transB, /*swapOperands=*/true); } // swap the last two axes diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index c9a66daec..e2c1d9226 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -113,6 +113,7 @@ Expr affine(Expr a, float scalar = 1.f); Expr csr_dot(const Shape& A_shape, Expr Avalues, Expr Aindices, Expr Aoffsets, Expr B, bool transA = false); +Expr dot_csr(Expr A, const Shape& B_shape, Expr B_values, Expr B_indices, Expr B_offsets, bool transB = false); Expr transpose(Expr a); Expr transpose(Expr a, const std::vector& axes); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 357c24b69..19cf0253d 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -410,44 +410,49 @@ class DotBatchedNodeOp : public NaryNodeOp { }; class CSRDotNodeOp : public NaryNodeOp { - bool transA_; + bool transS_; + bool swapOperands_; public: - CSRDotNodeOp(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) - : NaryNodeOp({ A_values, A_indices, A_offsets, B }, newShape(A_shape, A_values, A_indices, A_offsets, B, transA)), transA_(transA) { - matchOrAbort(A_indices->value_type()); - matchOrAbort(A_offsets->value_type()); + CSRDotNodeOp(const Shape& S_shape, Expr S_values, Expr S_indices, Expr S_offsets, Expr D, bool transS, bool swapOperands) + : NaryNodeOp({ S_values, S_indices, S_offsets, D }, newShape(S_shape, S_values, S_indices, S_offsets, D, transS, swapOperands)), transS_(transS) { + matchOrAbort(S_indices->value_type()); + matchOrAbort(S_offsets->value_type()); } - Shape newShape(const Shape& A_shape, Expr A_values, Expr A_indices, Expr A_offsets, Expr B, bool transA) { - ABORT_IF(A_values->shape().size() != 1 || A_indices->shape().size() != 1 || A_offsets->shape().size() != 1, + Shape newShape(const Shape& S_shape, Expr S_values, Expr S_indices, Expr S_offsets, Expr D, bool transS, bool swapOperands) { + ABORT_IF(S_values->shape().size() != 1 || S_indices->shape().size() != 1 || S_offsets->shape().size() != 1, "Sparse matrix components must all be vectors"); - ABORT_IF(A_values->shape() != A_indices->shape(), + ABORT_IF(S_values->shape() != S_indices->shape(), "Sparse matrix values and indices must have the same shape"); - ABORT_IF(A_shape.size() != 2, + ABORT_IF(S_shape.size() != 2, "Sparse matrix must have rank 2"); - ABORT_IF(A_offsets->shape()[0] - 1 != A_shape[0], + ABORT_IF(S_offsets->shape()[0] - 1 != S_shape[0], "Sparse matrix offset vector has incorrect size"); - auto outShape = B->shape(); - outShape.set(0, transA ? A_shape[1] : A_shape[0]); + ABORT_IF(swapOperands, "swapOperands not yet implemented"); + auto outShape = D->shape(); + outShape.set(0, transS ? S_shape[1] : S_shape[0]); return outShape; } NodeOps forwardOps() override { - // C = dot(A, B) + // C = dot(D, S) if swapOperands else + // C = dot(A, D) return {NodeOp(CSRProd(val_, graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), child(3)->val(), - /*transA=*/transA_, /*beta=*/0))}; + /*transS=*/transS_, /*swapOperands=*/swapOperands_, /*beta=*/0))}; } NodeOps backwardOps() override { return {nullptr, // can't backprop into the sparse matrix (the gradient is dense) - NodeOp(CSRProd(child(3)->grad(), // child(3) = B + nullptr, + nullptr, + NodeOp(CSRProd(child(3)->grad(), // child(3) = D graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), // children(0..2) = A adj_, - /*transA=*/!transA_, /*beta=*/1))}; + /*transS=*/!transS_, /*swapOperands=*/swapOperands_, /*beta=*/1))}; } const std::string type() override { return "csr_dot"; } diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index f17d23f84..408e47913 100755 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -174,8 +174,9 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_offsets, const marian::Tensor& B, bool transA, + bool swapOperands, float beta) { - C, A_values, A_indices, A_offsets, B, transA, beta; + C, A_values, A_indices, A_offsets, B, transA, swapOperands, beta; ABORT("CSRProd is not yet implemented for CPU"); } diff --git a/src/tensors/gpu/prod.cu b/src/tensors/gpu/prod.cpp similarity index 74% rename from src/tensors/gpu/prod.cu rename to src/tensors/gpu/prod.cpp index e816ecf10..0d86cbbd6 100755 --- a/src/tensors/gpu/prod.cu +++ b/src/tensors/gpu/prod.cpp @@ -95,6 +95,7 @@ void Prod(marian::Tensor C, #endif } +#if 0 // @TODO: remove, then rename from .cu to .cpp __global__ void gAddBias(float* out, const float* bias, size_t length, @@ -108,7 +109,6 @@ __global__ void gAddBias(float* out, } } -#if 0 // @TODO: remove, then rename from .cu to .cpp void AddBias(marian::Tensor C, const marian::Tensor bias) { cudaSetDevice(C->getDeviceId().no); @@ -229,84 +229,85 @@ void ProdBatched(marian::Tensor C, void CSRProd(marian::Tensor C, Ptr allocator, - const marian::Tensor& A_values, - const marian::Tensor& A_indices, - const marian::Tensor& A_offsets, - const marian::Tensor& B, - bool transA, + const marian::Tensor& S_values, + const marian::Tensor& S_indices, + const marian::Tensor& S_offsets, + const marian::Tensor& D, + bool transS, + bool swapOperands, float beta) { cudaSetDevice(C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); // dimensions const auto& shapeC = C->shape(); - const auto& shapeB = B->shape(); + const auto& shapeD = D->shape(); + ABORT_IF(swapOperands, "swapOperands not yet implemented"); auto rowsC = shapeC[0]; auto colsC = shapeC.elements() / rowsC; - auto rowsB = shapeB[0]; - auto colsB = shapeB.elements() / rowsB; - auto rowsA = transA ? rowsB : rowsC; - auto colsA = transA ? rowsC : rowsB; - ABORT_IF((transA ? colsA : rowsA) != rowsC || (transA ? rowsA : colsA) != rowsB || colsB != colsC, "Inconsistent dimensions in CSR product"); + auto rowsD = shapeD[0]; + auto colsD = shapeD.elements() / rowsD; + auto rowsS = transS ? rowsD : rowsC; + auto colsS = transS ? rowsC : rowsD; + ABORT_IF((transS ? colsS : rowsS) != rowsC || (transS ? rowsS : colsS) != rowsD || colsD != colsC, "Inconsistent dimensions in CSR product"); // sparse arrays - auto numValues = A_values->shape().elements(); - auto numOffsets = A_offsets->shape().elements() - 1; // -1 since last value is length - ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch: n={}, transA={}, rowsB={}, rowsC={}", numOffsets,transA, rowsB, rowsC); - ABORT_IF(numOffsets != (transA ? rowsB : rowsC), "CSR offset array dimension mismatch"); - ABORT_IF(A_values->shape() != A_indices->shape(), "CSR values and indices must have the same size"); + auto numValues = S_values->shape().elements(); + auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length + ABORT_IF(numOffsets != rowsS, "CSR offset array dimension mismatch"); + ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); float alpha = 1; // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. - // Hence, we compute C = spA * B as C' = B' * spA'. where B' and C' are - // column-major views on the data of B and C, and likewise, spA' is + // Hence, we compute C = S * D as C' = D' * S'. where D' and C' are + // column-major views on the data of D and C, and likewise, S' is // the CSR matrix reinterpreted as a CSC matrix. - if (transA) { + if (transS) { // cusparse does not support this specific version of transpose; do it explicitly - auto At_values = allocator->alloc(numValues); - auto At_indices = allocator->alloc(numValues); - auto At_offsets = allocator->alloc(colsA + 1); + auto St_values = allocator->alloc(numValues); + auto St_indices = allocator->alloc(numValues); + auto St_offsets = allocator->alloc(colsS + 1); // transpose the second argument CUSPARSE_CHECK(cusparseScsr2csc(cusparseHandle, - /*m=*/ rowsA, // number of rows of matrix - /*n=*/ colsA, // number of columns of matrix + /*m=*/ rowsS, // number of rows of matrix + /*n=*/ colsS, // number of columns of matrix /*nnz=*/ (int)numValues, - /*csrcVal=*/ A_values->data(), // second arg - /*csrcRowPtr=*/ (int*)A_offsets->data(), - /*csrcColInd=*/ (int*)A_indices->data(), - /*cscVal=*/ At_values->data(), // transposed version goes here - /*cscRowInd=*/ At_indices->data(), - /*cscColPtr=*/ At_offsets->data(), + /*csrcVal=*/ S_values->data(), // second arg + /*csrcRowPtr=*/ (int*)S_offsets->data(), + /*csrcColInd=*/ (int*)S_indices->data(), + /*cscVal=*/ St_values->data(), // transposed version goes here + /*cscRowInd=*/ St_indices->data(), + /*cscColPtr=*/ St_offsets->data(), /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - /*m=*/ colsB, // #rows of A = #cols of row-major B - /*n=*/ rowsC, // #cols of B and C = #rows of row-major C - /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*m=*/ colsD, // #rows of A = #cols of row-major D + /*n=*/ rowsC, // #cols of D and C = #rows of row-major C + /*k=*/ rowsD, // #cols of A = #rows of row-major D /*nnz=*/ (int)numValues, &alpha, - /*A=*/ B->data(), - /*lda=*/ colsB, // stride - /*cscValB=*/ At_values->data(), // second arg, transposed - /*cscRowPtrB=*/ At_offsets->data(), - /*cscColIndB=*/ At_indices->data(), + /*A=*/ D->data(), + /*lda=*/ colsD, // stride + /*cscValB=*/ St_values->data(), // second arg, transposed + /*cscRowPtrB=*/ St_offsets->data(), + /*cscColIndB=*/ St_indices->data(), &beta, C->data(), /*ldc=*/ colsC)); // stride - allocator->free(At_values); - allocator->free(At_indices); - allocator->free(At_offsets); + allocator->free(St_values); + allocator->free(St_indices); + allocator->free(St_offsets); } else { CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - /*m=*/ colsB, // #rows of A = #cols of row-major B - /*n=*/ rowsC, // #cols of B and C = #rows of row-major C - /*k=*/ rowsB, // #cols of A = #rows of row-major B + /*m=*/ colsD, // #rows of A = #cols of row-major D + /*n=*/ rowsC, // #cols of D and C = #rows of row-major C + /*k=*/ rowsD, // #cols of A = #rows of row-major D /*nnz=*/ (int)numValues, &alpha, - /*A=*/ B->data(), - /*lda=*/ colsB, // stride - /*cscValB=*/ A_values->data(), // second arg - /*cscRowPtrB=*/ (int*)A_offsets->data(), - /*cscColIndB=*/ (int*)A_indices->data(), + /*A=*/ D->data(), + /*lda=*/ colsD, // stride + /*cscValB=*/ S_values->data(), // second arg + /*cscRowPtrB=*/ (int*)S_offsets->data(), + /*cscColIndB=*/ (int*)S_indices->data(), &beta, C->data(), /*ldc=*/ colsC)); // stride @@ -318,17 +319,17 @@ void CSRProd(marian::Tensor C, cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, - transA ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, - /*m=*/ rowsA, // #rows of sparse A - /*n=*/ colsB, // #cols of dense B and C - /*k=*/ colsA, // #cols of sparse A + transS ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, + /*m=*/ rowsS, // #rows of sparse A + /*n=*/ colsD, // #cols of dense D and C + /*k=*/ colsS, // #cols of sparse A /*nnz=*/ (int)numValues, &alpha, descrA, - /*csrValA=*/ A_values->data(), - /*csrRowPtrA=*/ (int*)A_offsets->data(), - /*csrColIndA=*/ (int*)A_indices->data(), - B->data(), - /*ldb=*/ rowsB, + /*csrValA=*/ S_values->data(), + /*csrRowPtrA=*/ (int*)S_offsets->data(), + /*csrColIndA=*/ (int*)S_indices->data(), + D->data(), + /*ldb=*/ rowsD, &beta, C->data(), /*ldc=*/ rowsC)); diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h index 35bde4f79..b7177649f 100755 --- a/src/tensors/gpu/prod.h +++ b/src/tensors/gpu/prod.h @@ -40,6 +40,7 @@ void CSRProd(marian::Tensor C, const marian::Tensor& A_offsets, const marian::Tensor& B, bool transA, + bool swapOperands, float beta = 0); } // namespace gpu } // namespace marian diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index a9bd58480..f7de2f205 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -77,7 +77,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) -DISPATCH8(CSRProd, marian::Tensor, Ptr, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, float) +DISPATCH9(CSRProd, marian::Tensor, Ptr, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, const marian::Tensor&, bool, bool, float) DISPATCH2(Softmax, marian::Tensor, marian::Tensor) DISPATCH3(SoftmaxGrad, marian::Tensor, marian::Tensor, marian::Tensor) diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 64faded63..a6c560d3a 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -1088,7 +1088,7 @@ true - + true diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 9e5cbf5f4..d9e56843f 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1711,9 +1711,6 @@ tensors\gpu - - tensors\gpu - tensors\gpu @@ -1849,6 +1846,9 @@ examples + + tensors\gpu + From 4b73840d547438de529ce579c7966d6a1364c775 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 13:29:35 -0800 Subject: [PATCH 121/838] ported one check from another branch --- src/graph/node_operators_binary.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 19cf0253d..1aaad51b3 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -428,8 +428,10 @@ class CSRDotNodeOp : public NaryNodeOp { "Sparse matrix must have rank 2"); ABORT_IF(S_offsets->shape()[0] - 1 != S_shape[0], "Sparse matrix offset vector has incorrect size"); - ABORT_IF(swapOperands, "swapOperands not yet implemented"); auto outShape = D->shape(); + ABORT_IF(swapOperands, "swapOperands not yet implemented"); + ABORT_IF((transS ? S_shape[0] : S_shape[1] != D->shape()[0]), + "Matrix product requires dimensions to match"); outShape.set(0, transS ? S_shape[1] : S_shape[0]); return outShape; } From 99a4e12425d0f734940f0e2215e7c02ac3731f9e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 13:43:34 -0800 Subject: [PATCH 122/838] CSRDotNodeOp::newShape() now handles swapOperands --- src/graph/node_operators_binary.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 1aaad51b3..a76df2a54 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -429,10 +429,9 @@ class CSRDotNodeOp : public NaryNodeOp { ABORT_IF(S_offsets->shape()[0] - 1 != S_shape[0], "Sparse matrix offset vector has incorrect size"); auto outShape = D->shape(); - ABORT_IF(swapOperands, "swapOperands not yet implemented"); - ABORT_IF((transS ? S_shape[0] : S_shape[1] != D->shape()[0]), + ABORT_IF(S_shape[transS == swapOperands ? 1 : 0] != outShape[-(int)swapOperands], "Matrix product requires dimensions to match"); - outShape.set(0, transS ? S_shape[1] : S_shape[0]); + outShape.set(-(int)swapOperands, S_shape[transS != swapOperands]); return outShape; } From acf4e860f0fd4cc4925e2c7248f594ab2476d778 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 14:14:51 -0800 Subject: [PATCH 123/838] CSRProd now determines dimensions for swapOperands case --- src/graph/node_operators_binary.h | 4 +- src/tensors/gpu/prod.cpp | 61 +++++++++++++++---------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index a76df2a54..e61cb1e32 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -409,6 +409,8 @@ class DotBatchedNodeOp : public NaryNodeOp { const std::string color() override { return "orange"; } }; +// Note: To reduce code duplication, we use the same NodeOp for C = op(S) x D and C = D x op(S). +// Set swapOperands to select the latter. class CSRDotNodeOp : public NaryNodeOp { bool transS_; bool swapOperands_; @@ -436,8 +438,6 @@ class CSRDotNodeOp : public NaryNodeOp { } NodeOps forwardOps() override { - // C = dot(D, S) if swapOperands else - // C = dot(A, D) return {NodeOp(CSRProd(val_, graph()->allocator(), child(0)->val(), child(1)->val(), child(2)->val(), diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index 0d86cbbd6..9758a4c43 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -239,32 +239,40 @@ void CSRProd(marian::Tensor C, cudaSetDevice(C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); - // dimensions + // interpret tensor dimensions as matrix dimensions const auto& shapeC = C->shape(); const auto& shapeD = D->shape(); - ABORT_IF(swapOperands, "swapOperands not yet implemented"); - auto rowsC = shapeC[0]; + // If swapOperands, S and D are swapped (C = D x S instead of C = S x D). + // In that case, in the next 6 lines, please read all dimensions as if they were reversed in order. + auto rowsC = shapeC[-(int)swapOperands]; auto colsC = shapeC.elements() / rowsC; - auto rowsD = shapeD[0]; + auto rowsD = shapeD[-(int)swapOperands]; auto colsD = shapeD.elements() / rowsD; - auto rowsS = transS ? rowsD : rowsC; - auto colsS = transS ? rowsC : rowsD; - ABORT_IF((transS ? colsS : rowsS) != rowsC || (transS ? rowsS : colsS) != rowsD || colsD != colsC, "Inconsistent dimensions in CSR product"); + auto rowsS = transS != swapOperands ? rowsD : rowsC; + auto colsS = transS != swapOperands ? rowsC : rowsD; + ABORT_IF(colsD != colsC, "Inconsistent outer dimensions in CSR product"); + if (swapOperands) { // make rowsX actual row dimensions again, likewise colsX + std::swap(rowsC, colsC); + std::swap(rowsD, colsD); + std::swap(rowsS, colsS); + } // sparse arrays auto numValues = S_values->shape().elements(); auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length - ABORT_IF(numOffsets != rowsS, "CSR offset array dimension mismatch"); + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); float alpha = 1; // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. // Hence, we compute C = S * D as C' = D' * S'. where D' and C' are // column-major views on the data of D and C, and likewise, S' is // the CSR matrix reinterpreted as a CSC matrix. + Ptr St_values, St_indices, St_offsets; if (transS) { - // cusparse does not support this specific version of transpose; do it explicitly - auto St_values = allocator->alloc(numValues); - auto St_indices = allocator->alloc(numValues); - auto St_offsets = allocator->alloc(colsS + 1); + // Cusparse gemmi() does not support this specific version of transpose, and csrmm() is non-deterministic. + // Hence, we transpose the matrix explicitly. + St_values = allocator->alloc(numValues); + St_indices = allocator->alloc(numValues); + St_offsets = allocator->alloc(colsS + 1); // transpose the second argument CUSPARSE_CHECK(cusparseScsr2csc(cusparseHandle, /*m=*/ rowsS, // number of rows of matrix @@ -278,23 +286,9 @@ void CSRProd(marian::Tensor C, /*cscColPtr=*/ St_offsets->data(), /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); - CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - /*m=*/ colsD, // #rows of A = #cols of row-major D - /*n=*/ rowsC, // #cols of D and C = #rows of row-major C - /*k=*/ rowsD, // #cols of A = #rows of row-major D - /*nnz=*/ (int)numValues, - &alpha, - /*A=*/ D->data(), - /*lda=*/ colsD, // stride - /*cscValB=*/ St_values->data(), // second arg, transposed - /*cscRowPtrB=*/ St_offsets->data(), - /*cscColIndB=*/ St_indices->data(), - &beta, - C->data(), - /*ldc=*/ colsC)); // stride - allocator->free(St_values); - allocator->free(St_indices); - allocator->free(St_offsets); + } + if (swapOperands) { + ABORT("CSRProd does not yet implement swapOperands"); } else { CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, @@ -305,13 +299,16 @@ void CSRProd(marian::Tensor C, &alpha, /*A=*/ D->data(), /*lda=*/ colsD, // stride - /*cscValB=*/ S_values->data(), // second arg - /*cscRowPtrB=*/ (int*)S_offsets->data(), - /*cscColIndB=*/ (int*)S_indices->data(), + /*cscValB=*/ transS ? St_values ->data() : S_values ->data(), // second arg + /*cscRowPtrB=*/ transS ? St_offsets->data() : (int*)S_offsets->data(), + /*cscColIndB=*/ transS ? St_indices->data() : (int*)S_indices->data(), &beta, C->data(), /*ldc=*/ colsC)); // stride } + if(St_values ) allocator->free(St_values ); + if(St_indices) allocator->free(St_indices); + if(St_offsets) allocator->free(St_offsets); #if 0 // Incorrect code that assumes col-major matrices. Reuse that later for dense x sparse. cusparseMatDescr_t descrA; From feed1c6edaf1d5c7f7d463217f151aa38e4026d4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 14:35:25 -0800 Subject: [PATCH 124/838] clarified some comments --- src/tensors/gpu/prod.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index 9758a4c43..0156c8602 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -227,6 +227,8 @@ void ProdBatched(marian::Tensor C, allocator->free(mp_cptr); } +// C = op(S) x D if not swapOperands else C = D x op(S) +// op(S) = S if not transA else S^T void CSRProd(marian::Tensor C, Ptr allocator, const marian::Tensor& S_values, @@ -262,10 +264,6 @@ void CSRProd(marian::Tensor C, ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); float alpha = 1; - // Marian uses row-major storage, but CUSPARSE/CUBLAS assume column-major. - // Hence, we compute C = S * D as C' = D' * S'. where D' and C' are - // column-major views on the data of D and C, and likewise, S' is - // the CSR matrix reinterpreted as a CSC matrix. Ptr St_values, St_indices, St_offsets; if (transS) { // Cusparse gemmi() does not support this specific version of transpose, and csrmm() is non-deterministic. @@ -288,13 +286,16 @@ void CSRProd(marian::Tensor C, /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); } if (swapOperands) { + // C = D x S ABORT("CSRProd does not yet implement swapOperands"); } else { + // C = S x D for row-major matrices + // Implemented via cusparse as C' = D' x S' where C' and D' are column-major. CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, - /*m=*/ colsD, // #rows of A = #cols of row-major D - /*n=*/ rowsC, // #cols of D and C = #rows of row-major C - /*k=*/ rowsD, // #cols of A = #rows of row-major D + /*m=*/ colsD, // #rows of first (col-major) factor = #cols of row-major D + /*n=*/ rowsC, // #cols of second (sparse) factor and (col-major) result = #rows of row-major C + /*k=*/ rowsD, // #cols of first (col-major) factor = #rows of row-major D /*nnz=*/ (int)numValues, &alpha, /*A=*/ D->data(), From 41d43039980b9a4b7ac2aad591ed98c6f7eb1207 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 15:06:45 -0800 Subject: [PATCH 125/838] added tests for dot_csr() --- src/graph/node_operators_binary.h | 8 ++--- src/tests/operator_tests.cpp | 58 ++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index e61cb1e32..fa2bd350f 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -42,7 +42,7 @@ class DotNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(outShape.size() - 1, shapeB[shapeB.size() - 1]); ABORT_IF(shapeA[shapeA.size() - 1] != shapeB[shapeB.size() - 2], - "Matrix product requires dimensions to match"); + "Matrix product requires inner dimensions to match"); return outShape; } @@ -165,7 +165,7 @@ class AffineNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(outShape.size() - 1, shapeB[shapeB.size() - 1]); ABORT_IF(shapeA[shapeA.size() - 1] != shapeB[shapeB.size() - 2], - "Matrix product requires dimensions to match"); + "Matrix product requires inner dimensions to match"); return outShape; } @@ -309,7 +309,7 @@ class DotBatchedNodeOp : public NaryNodeOp { Shape outShape = shapeA; outShape.set(-1, shapeB[-1]); ABORT_IF(shapeA[-1] != shapeB[-2], - "Batched matrix product requires dimensions to match"); + "Batched matrix product requires inner dimensions to match"); return outShape; } @@ -432,7 +432,7 @@ class CSRDotNodeOp : public NaryNodeOp { "Sparse matrix offset vector has incorrect size"); auto outShape = D->shape(); ABORT_IF(S_shape[transS == swapOperands ? 1 : 0] != outShape[-(int)swapOperands], - "Matrix product requires dimensions to match"); + "Matrix product requires inner dimensions to match {} {} {} {}", transS, swapOperands, std::string(S_shape), std::string(outShape)); outShape.set(-(int)swapOperands, S_shape[transS != swapOperands]); return outShape; } diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 954f4adce..e4610dd92 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -318,15 +318,16 @@ void tests(DeviceType device) { auto B = graph->param("B", {3, 2}, inits::from_vector(vB)); auto C = dot(A, B); - // CSR dot product - std::vector vS({1, 0, 0, 1, + // CSR dot product, tested against dense product on the same values + std::vector vS({1, 0, 0, 1, // sparse 0, 0, 1, 1.5}); - std::vector vR({1, 2, 3, 1.2, 5.6, + std::vector vD({1, 2, 3, 1.2, 5.6, // dense 4, 5, 6, 2.3, 6.7, 7, 8, 9, 3.4, 7.8, 1, 1, 2, 4.5, 8.9}); - auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); - auto R = graph->param("R", { 4, 5 }, inits::from_vector(vR)); + auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); + auto D = graph->param("D", { 4, 5 }, inits::from_vector(vD)); + auto DT = graph->param("DT", { 5, 4 }, inits::from_vector(vD)); // example matrix with transposed dimensions std::vector SV; // create CSR version of S std::vector SI, SO; SO.push_back((IndexType)SI.size()); @@ -340,37 +341,54 @@ void tests(DeviceType device) { } SO.push_back((IndexType)SI.size()); } - auto SxRs = csr_dot( + + auto SxDd = dot(S, D); + auto STxSxDd = dot(S, SxDd, /*transA=*/true); + auto SxDs = csr_dot( // sparse x dense + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + D); + auto STxSxDs = csr_dot( // transpose(sparse) x dense; we use result of previous since dimensions match S->shape(), graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - R); - auto SxRd = dot(S, R); - auto STxRs = csr_dot( // and transpose; use result of previous since dimensions match + SxDd, /*transS=*/true); + + auto DTxSTd = dot(DT, S, /*transA=*/false, /*transB=*/true); + auto DTxSTxSd = dot(DTxSTd, S); + auto DTxSTs = dot_csr( // dense x sparse + DT, S->shape(), graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - SxRd, /*transA=*/true); - auto STxRd = dot(S, SxRd, /*transA=*/true); + /*transS=*/true); + auto DTxSTxSs = dot_csr( // dense x transpose(sparse) + DTxSTd, + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32)); CHECK(C->shape() == Shape({2, 2, 2})); - CHECK(SxRs->shape() == SxRd->shape()); - CHECK(STxRs->shape() == STxRd->shape()); + CHECK(SxDs->shape() == SxDd->shape()); + CHECK(STxSxDs->shape() == STxSxDd->shape()); + CHECK(DTxSTs->shape() == DTxSTd->shape()); + CHECK(DTxSTxSs->shape() == DTxSTxSd->shape()); graph->forward(); C->val()->get(values); CHECK(values == vC); - SxRd->val()->get(values2); // dense - SxRs->val()->get(values); // sparse - CHECK(values == values2); // must be the same - - STxRd->val()->get(values2); - STxRs->val()->get(values); - CHECK(values == values2); + // dense and sparse operation results must be the same + SxDd ->val()->get(values2); SxDs ->val()->get(values); CHECK(values == values2); + STxSxDd ->val()->get(values2); STxSxDs ->val()->get(values); CHECK(values == values2); + DTxSTd ->val()->get(values2); DTxSTs ->val()->get(values); CHECK(values == values2); + DTxSTxSd->val()->get(values2); DTxSTxSs->val()->get(values); CHECK(values == values2); } SECTION("affine transformation") { From 1ffc8825ebb4714ec73d0f96d023d4f65431849f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 16:31:37 -0800 Subject: [PATCH 126/838] bug fix: CSRDotNodeOp should remember swapOperands arg; dot_csr() implemented and passes tests --- src/graph/node_operators_binary.h | 3 +- src/tensors/gpu/prod.cpp | 68 +++++++++++++++---------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index fa2bd350f..4ebda8d38 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -416,7 +416,8 @@ class CSRDotNodeOp : public NaryNodeOp { bool swapOperands_; public: CSRDotNodeOp(const Shape& S_shape, Expr S_values, Expr S_indices, Expr S_offsets, Expr D, bool transS, bool swapOperands) - : NaryNodeOp({ S_values, S_indices, S_offsets, D }, newShape(S_shape, S_values, S_indices, S_offsets, D, transS, swapOperands)), transS_(transS) { + : NaryNodeOp({ S_values, S_indices, S_offsets, D }, newShape(S_shape, S_values, S_indices, S_offsets, D, transS, swapOperands)), + transS_(transS), swapOperands_(swapOperands){ matchOrAbort(S_indices->value_type()); matchOrAbort(S_offsets->value_type()); } diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index 0156c8602..cb795258b 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -250,8 +250,8 @@ void CSRProd(marian::Tensor C, auto colsC = shapeC.elements() / rowsC; auto rowsD = shapeD[-(int)swapOperands]; auto colsD = shapeD.elements() / rowsD; - auto rowsS = transS != swapOperands ? rowsD : rowsC; - auto colsS = transS != swapOperands ? rowsC : rowsD; + auto rowsS = transS ? rowsD : rowsC; + auto colsS = transS ? rowsC : rowsD; ABORT_IF(colsD != colsC, "Inconsistent outer dimensions in CSR product"); if (swapOperands) { // make rowsX actual row dimensions again, likewise colsX std::swap(rowsC, colsC); @@ -265,9 +265,9 @@ void CSRProd(marian::Tensor C, ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); float alpha = 1; Ptr St_values, St_indices, St_offsets; - if (transS) { + if (transS != swapOperands) { // Cusparse gemmi() does not support this specific version of transpose, and csrmm() is non-deterministic. - // Hence, we transpose the matrix explicitly. + // Hence, we transpose the matrix explicitly. Note that gemmi() expects a CSC, while csrmm() a CSR. St_values = allocator->alloc(numValues); St_indices = allocator->alloc(numValues); St_offsets = allocator->alloc(colsS + 1); @@ -279,30 +279,51 @@ void CSRProd(marian::Tensor C, /*csrcVal=*/ S_values->data(), // second arg /*csrcRowPtr=*/ (int*)S_offsets->data(), /*csrcColInd=*/ (int*)S_indices->data(), - /*cscVal=*/ St_values->data(), // transposed version goes here + /*cscVal=*/ St_values ->data(), // transposed version goes here /*cscRowInd=*/ St_indices->data(), /*cscColPtr=*/ St_offsets->data(), /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); + std::swap(rowsS, colsS); // dims of the explicitly transposed object } if (swapOperands) { - // C = D x S - ABORT("CSRProd does not yet implement swapOperands"); + // C = D x S for row-major matrices + // Implemented via cusparse as C' = S' x D' ("csrmm") where C' and D' are column-major, and S' is CSC. + cusparseMatDescr_t descrA; + CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); + cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); + cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); + CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, + CUSPARSE_OPERATION_NON_TRANSPOSE, // (we explicitly transposed above) + /*m=*/ rowsS, // #rows of first (CSR) factor (the transpose was done explicitly) + /*n=*/ rowsC, // #cols of second (col-major) factor and (col-major) result = #rows of row-major C + /*k=*/ colsS, // #cols of first (CSR) factor + /*nnz=*/ (int)numValues, + &alpha, descrA, + /*csrValA=*/ St_values ? St_values ->data() : S_values ->data(), // second arg + /*csrRowPtrA=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), + /*csrColIndA=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), + D->data(), + /*ldb=*/ colsD, // stride + &beta, + C->data(), + /*ldc=*/ colsC)); // stride + cusparseDestroyMatDescr(descrA); } else { // C = S x D for row-major matrices - // Implemented via cusparse as C' = D' x S' where C' and D' are column-major. + // Implemented via cusparse as C' = D' x S' ("gemmi") where C' and D' are column-major. CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, /*m=*/ colsD, // #rows of first (col-major) factor = #cols of row-major D - /*n=*/ rowsC, // #cols of second (sparse) factor and (col-major) result = #rows of row-major C + /*n=*/ rowsC, // #cols of second (CSC) factor and (col-major) result = #rows of row-major C /*k=*/ rowsD, // #cols of first (col-major) factor = #rows of row-major D /*nnz=*/ (int)numValues, &alpha, /*A=*/ D->data(), /*lda=*/ colsD, // stride - /*cscValB=*/ transS ? St_values ->data() : S_values ->data(), // second arg - /*cscRowPtrB=*/ transS ? St_offsets->data() : (int*)S_offsets->data(), - /*cscColIndB=*/ transS ? St_indices->data() : (int*)S_indices->data(), + /*cscValB=*/ St_values ? St_values ->data() : S_values ->data(), // second arg + /*cscRowPtrB=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), + /*cscColIndB=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), &beta, C->data(), /*ldc=*/ colsC)); // stride @@ -310,29 +331,6 @@ void CSRProd(marian::Tensor C, if(St_values ) allocator->free(St_values ); if(St_indices) allocator->free(St_indices); if(St_offsets) allocator->free(St_offsets); -#if 0 - // Incorrect code that assumes col-major matrices. Reuse that later for dense x sparse. - cusparseMatDescr_t descrA; - CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); - cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); - cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO); - CUSPARSE_CHECK(cusparseScsrmm(cusparseHandle, - transS ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE, - /*m=*/ rowsS, // #rows of sparse A - /*n=*/ colsD, // #cols of dense D and C - /*k=*/ colsS, // #cols of sparse A - /*nnz=*/ (int)numValues, - &alpha, descrA, - /*csrValA=*/ S_values->data(), - /*csrRowPtrA=*/ (int*)S_offsets->data(), - /*csrColIndA=*/ (int*)S_indices->data(), - D->data(), - /*ldb=*/ rowsD, - &beta, - C->data(), - /*ldc=*/ rowsC)); - cusparseDestroyMatDescr(descrA); -#endif } } // namespace gpu From 2b6ed113b615b0d51accfe714e630dc3997e137b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 16:40:54 -0800 Subject: [PATCH 127/838] (comments) --- src/tensors/gpu/prod.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index cb795258b..d13081728 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -267,7 +267,8 @@ void CSRProd(marian::Tensor C, Ptr St_values, St_indices, St_offsets; if (transS != swapOperands) { // Cusparse gemmi() does not support this specific version of transpose, and csrmm() is non-deterministic. - // Hence, we transpose the matrix explicitly. Note that gemmi() expects a CSC, while csrmm() a CSR. + // Hence, we transpose the matrix explicitly. + // Note that gemmi() expects a CSC, while csrmm() a CSR; hence, the strange condition (transS != swapOperands) above. St_values = allocator->alloc(numValues); St_indices = allocator->alloc(numValues); St_offsets = allocator->alloc(colsS + 1); @@ -276,7 +277,7 @@ void CSRProd(marian::Tensor C, /*m=*/ rowsS, // number of rows of matrix /*n=*/ colsS, // number of columns of matrix /*nnz=*/ (int)numValues, - /*csrcVal=*/ S_values->data(), // second arg + /*csrcVal=*/ S_values ->data(), /*csrcRowPtr=*/ (int*)S_offsets->data(), /*csrcColInd=*/ (int*)S_indices->data(), /*cscVal=*/ St_values ->data(), // transposed version goes here @@ -284,11 +285,12 @@ void CSRProd(marian::Tensor C, /*cscColPtr=*/ St_offsets->data(), /*copyValues=*/ CUSPARSE_ACTION_NUMERIC, /*idxBase=*/ CUSPARSE_INDEX_BASE_ZERO)); - std::swap(rowsS, colsS); // dims of the explicitly transposed object + std::swap(rowsS, colsS); // these variables now represent the dims of the explicitly transposed object } if (swapOperands) { // C = D x S for row-major matrices - // Implemented via cusparse as C' = S' x D' ("csrmm") where C' and D' are column-major, and S' is CSC. + // Implemented via cusparse as C' = S' x D' ("csrmm") where C' and D' are column-major, + // and S' is CSR (if not transS then we make a transposed copy). cusparseMatDescr_t descrA; CUSPARSE_CHECK(cusparseCreateMatDescr(&descrA)); cusparseSetMatType (descrA, CUSPARSE_MATRIX_TYPE_GENERAL); @@ -300,7 +302,7 @@ void CSRProd(marian::Tensor C, /*k=*/ colsS, // #cols of first (CSR) factor /*nnz=*/ (int)numValues, &alpha, descrA, - /*csrValA=*/ St_values ? St_values ->data() : S_values ->data(), // second arg + /*csrValA=*/ St_values ? St_values ->data() : S_values ->data(), /*csrRowPtrA=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), /*csrColIndA=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), D->data(), @@ -321,7 +323,7 @@ void CSRProd(marian::Tensor C, &alpha, /*A=*/ D->data(), /*lda=*/ colsD, // stride - /*cscValB=*/ St_values ? St_values ->data() : S_values ->data(), // second arg + /*cscValB=*/ St_values ? St_values ->data() : S_values ->data(), /*cscRowPtrB=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), /*cscColIndB=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), &beta, From 148540310694d0d80218952202c0cb2357b0e6d7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 15 Jan 2019 16:42:37 -0800 Subject: [PATCH 128/838] (removed a debug message) --- src/graph/node_operators_binary.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 4ebda8d38..eb20033d9 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -433,7 +433,7 @@ class CSRDotNodeOp : public NaryNodeOp { "Sparse matrix offset vector has incorrect size"); auto outShape = D->shape(); ABORT_IF(S_shape[transS == swapOperands ? 1 : 0] != outShape[-(int)swapOperands], - "Matrix product requires inner dimensions to match {} {} {} {}", transS, swapOperands, std::string(S_shape), std::string(outShape)); + "Matrix product requires inner dimensions to match"); outShape.set(-(int)swapOperands, S_shape[transS != swapOperands]); return outShape; } From d6d281f84f3c02ce23b5d78e388bea7d3f1a4c8a Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 15 Jan 2019 22:43:00 -0800 Subject: [PATCH 129/838] fix first batch of regression tests --- src/common/config_parser.cpp | 4 +++- src/data/vocab.cpp | 9 ++++----- src/layers/loss.cpp | 7 ++++--- src/layers/loss.h | 12 ++++-------- src/models/costs.h | 7 +++++-- src/models/transformer.h | 13 ++++--------- src/training/scheduler.h | 26 ++++++++++++++++---------- src/training/training_state.h | 9 ++++++--- 8 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index e745e99d5..cfc7774a9 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -250,8 +250,10 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.switchGroup("Training options"); // clang-format off - cli.add("--cost-type", + cli.add("--cost-type", // @TODO: rename to loss-type "Optimization criterion: ce-mean, ce-mean-words, ce-sum, perplexity", "ce-mean"); + cli.add("--multi-loss-type", + "How to accumulate multi-objective losses: sum, scaled, mean", "sum"); cli.add("--overwrite", "Do not create model checkpoints, only overwrite main model file with last checkpoint. " "Reduces disk usage"); diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 1e6a341b2..97963e4d9 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -13,11 +13,10 @@ Ptr createVocab(const std::string& vocabPath, Ptr options, s if(vocab) { return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it } else { - auto inputType = options->get>("input-types")[batchIndex]; - if(inputType == "labels") - return createLabelsVocab(); - else - return createDefaultVocab(); + // check type of input, if not given, assume "sequence" + auto inputTypes = options->get>("input-types", {}); + std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; + return inputType == "labels" ? createLabelsVocab() : createDefaultVocab(); } } diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 024b884c4..f904f73ce 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -2,6 +2,7 @@ namespace marian { +// @TODO, simplify this. Currently here for back-compat Ptr newLoss(Ptr options, bool inference) { float smoothing = inference ? 0.f : options->get("label-smoothing"); std::string costType = options->get("cost-type", "ce-mean"); @@ -24,11 +25,11 @@ Ptr newLoss(Ptr options, bool inference) { Ptr newMultiLoss(Ptr options) { std::string multiLossType = options->get("multi-loss-type", "sum"); - if(multiLossType == "sum") + if(multiLossType == "sum") // sum of sums return New(); - else if(multiLossType == "scaled") + else if(multiLossType == "scaled") // sum of scaled sums, first element is reference scale return New(); - else if(multiLossType == "normal") + else if(multiLossType == "mean") // sum of means return New(); else ABORT("Unknown multi-loss-type {}", multiLossType); diff --git a/src/layers/loss.h b/src/layers/loss.h index eaa98da04..3e876a3bd 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -81,9 +81,8 @@ class MultiRationalLoss : public RationalLoss { public: MultiRationalLoss() : RationalLoss() {} - MultiRationalLoss(const RationalLoss& rl) - : RationalLoss(rl.loss(), rl.labels()) { - partialLosses_.push_back(rl); + MultiRationalLoss(const RationalLoss& rl) : RationalLoss() { + this->push_back(rl); } void push_back(const RationalLoss& current) { @@ -211,11 +210,8 @@ class LabelwiseLoss { Expr mask = nullptr, Expr labelWeights = nullptr) { Expr loss = compute(logits, labelIndices, mask, labelWeights); - Expr labels; - if(mask) - labels = mask; - else - labels = constant_like(loss, inits::ones); // we have no mask, assume all items are labels + Expr labels = mask ? mask // mask can be used as element-wise label count with broadcasting + : constant_like(loss, inits::ones); // we have no mask, assume all items are labels return reduce(loss, labels); } diff --git a/src/models/costs.h b/src/models/costs.h index 7f5f0743e..70bbe1768 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -10,6 +10,10 @@ namespace marian { namespace models { +// @TODO: this whole file is an unholy mess and needs to be refactored. +// Using MultiRationalLoss is a first improvement, but we can probably +// unify classifier and decoder costs. Also rethink step-wise cost. + class CostBase { public: virtual Ptr apply(Ptr model, @@ -56,14 +60,13 @@ class EncoderDecoderCE : public CostBase { // multi-objective training Ptr multiLoss = newMultiLoss(options_); - // @TODO: adapt to multi-objective training + // @TODO: adapt to multi-objective training with multiple decoders auto partialLoss = loss_->apply(state->getLogProbs(), state->getTargetIndices(), state->getTargetMask(), weights); multiLoss->push_back(partialLoss); - if(options_->get("guided-alignment", std::string("none")) != "none" && !inference_) { auto alignments = encdec->getDecoders()[0]->getAlignments(); ABORT_IF(alignments.empty(), "Model does not seem to support alignments"); diff --git a/src/models/transformer.h b/src/models/transformer.h index a5fba81aa..7cbb8d5d9 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -52,6 +52,7 @@ class Transformer : public EncoderOrDecoderBase { int dimWords = input->shape()[-3]; Expr embeddings = input; + if(learnedPosEmbeddings) { int maxLength = opt("max-length"); @@ -84,11 +85,9 @@ class Transformer : public EncoderOrDecoderBase { return embeddings; } - virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const { - batch; + virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr /*batch*/ = nullptr) const { bool learnedPosEmbeddings = opt("transformer-learned-positions", false); - input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); - return input; + return addPositionalEmbeddings(input, start, learnedPosEmbeddings); } Expr triangleMask(int length) const { @@ -706,20 +705,16 @@ class DecoderTransformer : public Transformer { //************************************************************************// - int dimEmb = embeddings->shape()[-1]; int dimBeam = 1; if(embeddings->shape().size() > 3) dimBeam = embeddings->shape()[-4]; - // according to paper embeddings are scaled by \sqrt(d_m) - auto scaledEmbeddings = std::sqrt((float)dimEmb) * embeddings; - // set current target token position during decoding or training. At training // this should be 0. During translation the current length of the translation. // Used for position embeddings and creating new decoder states. int startPos = (int)state->getPosition(); - scaledEmbeddings = addSpecialEmbeddings(scaledEmbeddings, startPos); + auto scaledEmbeddings = addSpecialEmbeddings(embeddings, startPos); scaledEmbeddings = atleast_nd(scaledEmbeddings, 4); // reorganize batch and timestep diff --git a/src/training/scheduler.h b/src/training/scheduler.h index d35749901..b03a4ec99 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -49,7 +49,7 @@ class Scheduler : public TrainingObserver { ss << "Cost "; ss << std::setprecision(8) << std::fixed; - // @TODO: put a single loss formatting funciton into loss.h and reuse here to avoid code duplication + // @TODO: put a single loss formatting function into loss.h and reuse here to avoid code duplication // @TODO: use dispLabelCounts with any display type? if(lossType == "ce-mean-words") ss << state->costSum / state->costCount; @@ -60,11 +60,11 @@ class Scheduler : public TrainingObserver { << " after " << utils::withCommas(state->labelsTotal); else if(lossType == "ce-sum" && !dispLabelCounts) - ss << state->costSum; + ss << state->costSum / state->updatesDisp; // average over batches else if(lossType == "perplexity") ss << std::exp(state->costSum / state->costCount); else if(lossType == "cross-entropy" || lossType == "ce-mean") // backwards-compat, @TODO: get rid of this? - ss << state->costSum / state->samplesCount; + ss << state->costSum / state->samplesDisp; else ABORT("Unknown loss type {}", lossType); @@ -187,7 +187,7 @@ class Scheduler : public TrainingObserver { } void update(float cost, Ptr batch) { - ABORT("Fix me"); + // @TODO: Check me and eventually remove me, currently here for back-compat. update(cost, batch->words(-1), std::vector>({batch})); } @@ -213,9 +213,11 @@ class Scheduler : public TrainingObserver { state_->costSum += cost; // aggregate sum cost since last display state_->costCount += batchLabels; // cost gets normalized w.r.t. this in display - state_->samplesCount += batchSize; + state_->updatesDisp += 1; + state_->samplesDisp += batchSize; state_->wordsDisp += batchWords; // words at given input processed since last display, for speed display + state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += batchLabels; // total labels processed @@ -261,7 +263,9 @@ class Scheduler : public TrainingObserver { timer_.start(); state_->costSum = 0; state_->costCount = 0; - state_->samplesCount = 0; + + state_->updatesDisp = 0; + state_->samplesDisp = 0; state_->wordsDisp = 0; } @@ -284,10 +288,12 @@ class Scheduler : public TrainingObserver { if(options_->get("no-restore-corpus")) { state_->samplesEpoch = 0; - state_->costSum = 0; - state_->costCount = 0; - state_->samplesCount = 0; - state_->wordsDisp = 0; + state_->costSum = 0; + state_->costCount = 0; + + state_->updatesDisp = 0; + state_->samplesDisp = 0; + state_->wordsDisp = 0; } state_->newLoad(); diff --git a/src/training/training_state.h b/src/training/training_state.h index d4ecdb6cb..6cd5686af 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -94,13 +94,16 @@ class TrainingState { // Sum of costs since last display float costSum{0}; - // Number of words/labels/samples (depending on cost-type) aggregated in + // Number of labels aggregated in // costSum since last display size_t costCount{0}; - // Number of words seen since last display, for speed measurement + + // Number of words seen since last display size_t wordsDisp{0}; // Number of samples/sentences seen since last display - size_t samplesCount{0}; + size_t samplesDisp{0}; + // Number of updates seen since last display + size_t updatesDisp{0}; // The state of the random number generator from a batch generator std::string seedBatch; From 9a090af595a136cec7344265e9c43640edfc374d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 15 Jan 2019 23:01:25 -0800 Subject: [PATCH 130/838] fix rescorer loss --- src/layers/loss.h | 16 +++++++++++----- src/models/transformer.h | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/layers/loss.h b/src/layers/loss.h index 3e876a3bd..99abcd029 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -179,7 +179,7 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { Ptr newMultiLoss(Ptr options); //***********************************************************************************// -// This needs some to be refactored. Currentl easiest route for backwards compat. +// This needs some to be refactored. Currentl easiest route for backwards compat. class LabelwiseLoss { protected: @@ -206,8 +206,8 @@ class LabelwiseLoss { LabelwiseLoss(const std::vector& axes) : axes_(axes) { } - RationalLoss apply(Expr logits, Expr labelIndices, - Expr mask = nullptr, Expr labelWeights = nullptr) { + virtual RationalLoss apply(Expr logits, Expr labelIndices, + Expr mask = nullptr, Expr labelWeights = nullptr) { Expr loss = compute(logits, labelIndices, mask, labelWeights); Expr labels = mask ? mask // mask can be used as element-wise label count with broadcasting @@ -230,7 +230,7 @@ class CrossEntropyLoss : public LabelwiseLoss { protected: float smoothing_; - Expr compute(Expr logits, Expr labelIndices, + virtual Expr compute(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { Expr ce = cross_entropy(logits, labelIndices); @@ -252,8 +252,14 @@ class CrossEntropyLoss : public LabelwiseLoss { class RescorerLoss : public CrossEntropyLoss { public: - // sentence-wise CE, hence reduce only over time axis. + // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} + + virtual RationalLoss apply(Expr logits, Expr labelIndices, + Expr mask = nullptr, Expr labelWeights = nullptr) override { + auto ce = CrossEntropyLoss::apply(logits, labelIndices, mask, labelWeights); + return RationalLoss(-ce.loss(), ce.labels()); // we report logprobs, hence negate + } }; Ptr newLoss(Ptr options, bool inference); diff --git a/src/models/transformer.h b/src/models/transformer.h index 7cbb8d5d9..c72d5a06e 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -71,6 +71,7 @@ class Transformer : public EncoderOrDecoderBase { for(int i = 0; i < std::min(dimWords, numPos); ++i) positions[i] = i; + // @TODO : test if embeddings should be scaled here too! auto signal = rows(posEmbFactory, graph_->indices(positions)); signal = reshape(signal, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; From 637f50837b7dc33c733b354525f50eb75c5f2de1 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 15 Jan 2019 23:14:48 -0800 Subject: [PATCH 131/838] add comments --- src/layers/loss.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/layers/loss.h b/src/layers/loss.h index 99abcd029..1f9543b44 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -54,7 +54,8 @@ class RationalLoss { } }; -// POD for accumulating loss values after backprop +// POD for accumulating loss values after forward/backward and to-be-used in +// Scheduler for updating statistics. struct StaticLoss { float loss; float labels; @@ -179,7 +180,8 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { Ptr newMultiLoss(Ptr options); //***********************************************************************************// -// This needs some to be refactored. Currentl easiest route for backwards compat. +// This needs some to be refactored. Currently easiest route for backwards compat, but +// still feels somewhat hacky. class LabelwiseLoss { protected: From d0e380cd623a15dc694705ce93f95296b3cdec07 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 16 Jan 2019 08:08:04 -0800 Subject: [PATCH 132/838] first version of factored outputs (closed vocab) --- src/layers/generic.cpp | 118 +++++++++++++++++++++++++++++---------- src/layers/generic.h | 32 ++--------- src/models/transformer.h | 14 +++-- 3 files changed, 105 insertions(+), 59 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index f651a3ce6..82c5bc7c9 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -11,10 +11,13 @@ namespace marian { }; class EmbeddingFactorMapping { - Vocab factorVocab_; // [factor name] -> factor index = row of E_ - std::vector> factorMap_; // [word index] -> set of factor indices - std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ public: + struct CSRData { + Shape shape; + std::vector weights; + std::vector indices; + std::vector offsets; + }; // mapPath = path to file with entries in order of vocab entries of the form // WORD FACTOR1 FACTOR2 FACTOR3... // listPath = path to file that lists all FACTOR names @@ -23,13 +26,19 @@ namespace marian { // Note: Presently, this implementation has the following short-comings // - we do not group factors (to normalize the probs); instead we assume that the global softmax will normalize correctly // - we do not handle binary features differently; we'd need to apply sigmoid(x) / sigmoid(-x) - EmbeddingFactorMapping(const std::string& mapPath, const std::string& factorVocabPath, const std::string& vocabPath) : - factorVocab_(New(), 0) { + EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { + std::vector paths = options->get>("embedding-factors"); + ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); + auto mapPath = paths[0]; + auto factorVocabPath = paths[1]; + auto vocabPath = options->get("vocab"); + // Note: We misuse the Vocab class a little. - // But it means that the factorVocab_ must contain and "". + // Specifically, it means that the factorVocab_ must contain and "". Vocab vocab(New(), 0); vocab.load(vocabPath); factorVocab_.load(factorVocabPath); + // load and parse factorMap factorMap_.resize(vocab.size()); factorRefCounts_.resize(factorVocab_.size()); @@ -50,13 +59,18 @@ namespace marian { numTotalFactors += tokens.size() - 1; } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, factorVocab_.size(), vocab.size()); + + // create the factor matrix + std::vector data(vocab.size()); + std::iota(data.begin(), data.end(), 0); + factorMatrix_ = csr_rows(data); // [V x U] } size_t factorVocabSize() const { return factorVocab_.size(); } // create a CSR matrix M[V,U] from indices[] with // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced - std::tuple/*weights*/, std::vector/*indices*/, std::vector/*offsets*/> csr_rows(const std::vector& words) const { + CSRData csr_rows(const std::vector& words) const { std::vector weights; std::vector indices; std::vector offsets; @@ -72,12 +86,69 @@ namespace marian { } offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset } - return - std::tuple/*weights*/, std::vector/*indices*/, std::vector/*offsets*/> // (needed for unknown reasons) - { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; + return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; } + + const CSRData& getFactorMatrix() const { return factorMatrix_; } + private: + Vocab factorVocab_; // [factor name] -> factor index = row of E_ + std::vector> factorMap_; // [word index] -> set of factor indices + std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ + CSRData factorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v }; + namespace mlp { + /*private*/ void Output::lazyConstruct(int inputDim) { + // We must construct lazily since we won't know tying nor input dim in constructor. + if (W_) + return; + + auto name = options_->get("prefix"); + auto dim = options_->get("dim"); + + if (options_->has("embedding-factors")) { + ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); + embeddingFactorMapping_ = New(options_); + dim = (int)embeddingFactorMapping_->factorVocabSize(); + LOG(info, "[embedding] Factored outputs enabled"); + } + + if(tiedParam_) { + transposeW_ = true; + W_ = tiedParam_; + if(shortlist_) + W_ = rows(W_, shortlist_->indices()); + } else { + W_ = graph_->param(name + "_W", + {inputDim, dim}, + inits::glorot_uniform); + if(shortlist_) + W_ = cols(W_, shortlist_->indices()); + } + + b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); + if(shortlist_) + b_ = cols(b_, shortlist_->indices()); + } + + Expr Output::apply(Expr input) /*override*/ { + lazyConstruct(input->shape()[-1]); + auto y = affine(input, W_, b_, false, transposeW_); + if (embeddingFactorMapping_) { + auto graph = input->graph(); + auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] + y = dot_csr( // the CSR matrix is passed in pieces + y, // [B x U] + factorMatrix.shape, + graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), + graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), + graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), + /*transB=*/ true); // -> [B x V] + } + return y; + } + } + Embedding::Embedding(Ptr graph, Ptr options) : LayerBase(graph, options) { std::string name = opt("prefix"); int dimVoc = opt("dimVocab"); @@ -85,11 +156,10 @@ namespace marian { bool fixed = opt("fixed", false); - if (options_->has("embedding-factors") && !embeddingFactorMapping_) { // (lazy init) - std::vector paths = opt>("embedding-factors"); - ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); - embeddingFactorMapping_ = New(paths[0], paths[1], opt("vocab")); + if (options_->has("embedding-factors")) { + embeddingFactorMapping_ = New(options_); dimVoc = (int)embeddingFactorMapping_->factorVocabSize(); + LOG(info, "[embedding] Factored embeddings enabled"); } NodeInitializer initFunc = inits::glorot_uniform; @@ -108,23 +178,15 @@ namespace marian { /*private*/ Expr Embedding::multiRows(const std::vector& data) const { auto graph = E_->graph(); - Shape shape; std::vector weights; std::vector indices, offsets; std::tie - (shape, weights, indices, offsets) = embeddingFactorMapping_->csr_rows(data); -#if 0 // tests for special case of nop-op - ABORT_IF(data != indices, "bang"); - for (size_t i = 0; i < offsets.size(); i++) - ABORT_IF(offsets[i] != i, "boom"); - for (size_t i = 0; i < weights.size(); i++) - ABORT_IF(weights[i] != 1, "oops"); -#endif + auto factoredData = embeddingFactorMapping_->csr_rows(data); // multi-hot factor vectors are represented as a sparse CSR matrix // [row index = word position index] -> set of factor indices for word at this position - ABORT_IF(shape != Shape({(int)offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), "shape mismatch??"); + ABORT_IF(factoredData.shape != Shape({(int)factoredData.offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), "shape mismatch??"); return csr_dot( // the CSR matrix is passed in pieces - shape, - graph->constant({(int)weights.size()}, inits::from_vector(weights), Type::float32), - graph->constant({(int)indices.size()}, inits::from_vector(indices), Type::uint32), - graph->constant({(int)offsets.size()}, inits::from_vector(offsets), Type::uint32), + factoredData.shape, + graph->constant({(int)factoredData.weights.size()}, inits::from_vector(factoredData.weights), Type::float32), + graph->constant({(int)factoredData.indices.size()}, inits::from_vector(factoredData.indices), Type::uint32), + graph->constant({(int)factoredData.offsets.size()}, inits::from_vector(factoredData.offsets), Type::uint32), E_); } diff --git a/src/layers/generic.h b/src/layers/generic.h index 092567c94..1ac9764f9 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -54,6 +54,8 @@ struct IEmbeddingLayer { virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; }; +class EmbeddingFactorMapping; + namespace mlp { class Dense : public LayerBase, public IUnaryLayer { @@ -126,11 +128,13 @@ class Output : public LayerBase, public IUnaryLayer { private: Expr tiedParam_; Ptr shortlist_; + Ptr embeddingFactorMapping_; Expr W_; Expr b_; bool transposeW_{false}; + void lazyConstruct(int inputDim); public: Output(Ptr graph, Ptr options) : LayerBase(graph, options) {} @@ -141,31 +145,7 @@ class Output : public LayerBase, public IUnaryLayer { void setShortlist(Ptr shortlist) { shortlist_ = shortlist; } - Expr apply(Expr input) override { - if(!W_) { - auto name = options_->get("prefix"); - auto dim = options_->get("dim"); - - if(tiedParam_) { - transposeW_ = true; - W_ = tiedParam_; - if(shortlist_) - W_ = rows(W_, shortlist_->indices()); - } else { - W_ = graph_->param(name + "_W", - {input->shape()[-1], dim}, - inits::glorot_uniform); - if(shortlist_) - W_ = cols(W_, shortlist_->indices()); - } - - b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); - if(shortlist_) - b_ = cols(b_, shortlist_->indices()); - } - - return affine(input, W_, b_, false, transposeW_); - } + Expr apply(Expr input) override; virtual Expr apply(const std::vector& /*inputs*/) override { ABORT("Not implemented"); @@ -176,7 +156,7 @@ class Output : public LayerBase, public IUnaryLayer { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; - Ptr embeddingFactorMapping_; + Ptr embeddingFactorMapping_; Expr multiRows(const std::vector& data) const; public: Embedding(Ptr graph, Ptr options); diff --git a/src/models/transformer.h b/src/models/transformer.h index 99999e8f6..0557ec19b 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -627,19 +627,23 @@ class DecoderTransformer : public Transformer { if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; layerOut.tieTransposed(tiedPrefix); + } + + if(shortlist_) + layerOut.setShortlist(shortlist_); + + if (options_->has("embedding-factors")) { // factored embeddings, simplistic version (which just adds the logits, like multiplying probs) // z = h @ W // h:[B x D] ; W:[D x V] -> [B x V] // with factors: // z = h @ W @ M' // h:[B x D] ; W:[D x U] ; M':[U x V] -> [B x V] // i.e. multiOutput(): // output = dot_csr(output, M, transB=true) - // Should biases be done afterwards? Or maybe at two places? - // note: need to also specify output factors separately if not tied-embeddings or tied-embeddings-all + // @BUGBUG: need to specify output factors separately if not tied-embeddings or tied-embeddings-all + layerOut("embedding-factors", opt>("embedding-factors")); + layerOut("vocab", opt>("vocabs")[batchIndex_]); } - if(shortlist_) - layerOut.setShortlist(shortlist_); - // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context From f6a183d23e7e7c4502d1a311f96dfd9b931d72a0 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 08:49:32 -0800 Subject: [PATCH 133/838] use rational loss for guided alignment --- src/layers/guided_alignment.h | 70 +++++++++++++++-------------------- src/layers/loss.cpp | 1 + src/layers/loss.h | 4 +- src/models/costs.h | 6 +-- src/training/scheduler.h | 1 + 5 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/layers/guided_alignment.h b/src/layers/guided_alignment.h index df8607cae..9fefab2ca 100755 --- a/src/layers/guided_alignment.h +++ b/src/layers/guided_alignment.h @@ -1,53 +1,41 @@ #pragma once -#include "marian.h" +#include "layers/loss.h" namespace marian { -static inline Expr guidedAlignmentCost(Ptr graph, - Ptr batch, - Ptr options, - Expr att) { - int dimBatch = att->shape()[-2]; - int dimSrc = att->shape()[-3]; - int dimTrg = att->shape()[-1]; +static inline RationalLoss guidedAlignmentCost(Ptr graph, + Ptr batch, + Ptr options, + Expr attention) { - //debug(att, "Attention"); - - auto aln = graph->constant(att->shape(), - inits::from_vector(batch->getGuidedAlignment())); - - //debug(aln, "Alignment"); - - std::string guidedCostType - = options->get("guided-alignment-cost"); - - std::string costType = options->get("cost-type"); - - int div = 1; - if(costType == "ce-mean-words") { - div = dimBatch * dimSrc * dimTrg; - } else if(costType == "perplexity") { - div = dimBatch * dimSrc * dimTrg; - } else if(costType == "ce-sum") { - div = 1; - } else { - div = dimBatch; - } - - Expr alnCost; + // @TODO: change "cost" to "loss" + std::string guidedLossType = options->get("guided-alignment-cost"); + float guidedScalar = options->get("guided-alignment-weight"); + float epsilon = 1e-6f; - if(guidedCostType == "mse") { - alnCost = sum(flatten(square(att - aln))) / (float)(2 * div); - } else if(guidedCostType == "mult") { - alnCost = -log(sum(flatten(att * aln)) + epsilon) / (float)div; - } else if(guidedCostType == "ce") { - alnCost = -sum(flatten(aln * log(att + epsilon))) / (float)div; + Expr alignment = constant_like(attention, inits::from_vector(batch->getGuidedAlignment())); + Expr alignmentLoss; // sum up loss over all attention/alignment positions + if(guidedLossType == "mse") { + alignmentLoss = sum(flatten(square(attention - alignment))) / 2.f; + } else if(guidedLossType == "mult") { + alignmentLoss = -log(sum(flatten(attention * alignment)) + epsilon); + } else if(guidedLossType == "ce") { + alignmentLoss = -sum(flatten(alignment * log(attention + epsilon))); } else { - ABORT("Unknown alignment cost type"); + ABORT("Unknown alignment cost type: {}", guidedLossType); } + + alignmentLoss = guidedScalar * alignmentLoss; // weigh by scalar - float guidedScalar = options->get("guided-alignment-weight"); - return guidedScalar * alnCost; + // every position is a label as they should all agree + float numLabels = alignment->shape().elements(); + + // create label node + Expr labels = graph->constant({1}, inits::from_value(numLabels)); + labels = guidedScalar * labels; // also weigh by scalar so labels and cost are in the same domain + + return RationalLoss(alignmentLoss, labels); } + } // namespace marian diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index f904f73ce..06f8dc59d 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -17,6 +17,7 @@ Ptr newLoss(Ptr options, bool inference) { } else if(costType == "ce-rescore") { return New(); } else if(costType == "ce-rescore-mean") { + ABORT("Check me"); return New(); } else { // same as ce-mean return New(smoothing); diff --git a/src/layers/loss.h b/src/layers/loss.h index 1f9543b44..8c8d170c0 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -1,6 +1,6 @@ #pragma once -#include "marian.h" +#include "graph/expression_operators.h" namespace marian { @@ -233,7 +233,7 @@ class CrossEntropyLoss : public LabelwiseLoss { float smoothing_; virtual Expr compute(Expr logits, Expr labelIndices, - Expr mask = nullptr, Expr labelWeights = nullptr) override { + Expr mask = nullptr, Expr labelWeights = nullptr) override { Expr ce = cross_entropy(logits, labelIndices); if(smoothing_ > 0) { diff --git a/src/models/costs.h b/src/models/costs.h index 70bbe1768..aaab3890c 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -73,9 +73,9 @@ class EncoderDecoderCE : public CostBase { auto att = concatenate(alignments, /*axis =*/ -1); - // @TODO: represent this as an multi-objective training loss, which it actually is - // return multiLoss.loss() + guidedAlignmentCost(graph, corpusBatch, options_, att); - ABORT("Fix me"); + auto alignmentLoss = guidedAlignmentCost(graph, corpusBatch, options_, att); + multiLoss->push_back(alignmentLoss); + return multiLoss; } else { return multiLoss; diff --git a/src/training/scheduler.h b/src/training/scheduler.h index b03a4ec99..41b765e45 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -51,6 +51,7 @@ class Scheduler : public TrainingObserver { // @TODO: put a single loss formatting function into loss.h and reuse here to avoid code duplication // @TODO: use dispLabelCounts with any display type? + // @TODO: bugbug cost-type ce-mean-words with multi-loss-type mean divides too much in display if(lossType == "ce-mean-words") ss << state->costSum / state->costCount; else if(lossType == "ce-sum" && dispLabelCounts) From 041dcccc54d2cdb08699fc83748c510cd72ceb5c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 08:58:08 -0800 Subject: [PATCH 134/838] add comments --- src/layers/guided_alignment.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/layers/guided_alignment.h b/src/layers/guided_alignment.h index 9fefab2ca..451647826 100755 --- a/src/layers/guided_alignment.h +++ b/src/layers/guided_alignment.h @@ -9,6 +9,10 @@ static inline RationalLoss guidedAlignmentCost(Ptr graph, Ptr options, Expr attention) { + // @TODO: there should be positional masking here ... on the other hand, positions that are not + // in a sentence should always agree (both being 0). Lack of masking affects label count only which is + // probably negligible? + // @TODO: change "cost" to "loss" std::string guidedLossType = options->get("guided-alignment-cost"); float guidedScalar = options->get("guided-alignment-weight"); @@ -28,12 +32,12 @@ static inline RationalLoss guidedAlignmentCost(Ptr graph, alignmentLoss = guidedScalar * alignmentLoss; // weigh by scalar - // every position is a label as they should all agree - float numLabels = alignment->shape().elements(); + // every position is a label as they should all agree, see caveat at the top. + size_t numLabels = alignment->shape().elements(); - // create label node - Expr labels = graph->constant({1}, inits::from_value(numLabels)); - labels = guidedScalar * labels; // also weigh by scalar so labels and cost are in the same domain + // Create label node, also weigh by scalar so labels and cost are in the same domain. + // Fractional label counts are OK + Expr labels = graph->constant({1}, inits::from_value(guidedScalar * numLabels)); // @TODO: introduce graph->value(...) ? return RationalLoss(alignmentLoss, labels); } From 1e205c242441d5c09ab8ea2393520517de9dc2ae Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 09:31:29 -0800 Subject: [PATCH 135/838] add comments --- src/models/costs.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/models/costs.h b/src/models/costs.h index aaab3890c..40d490b53 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -14,6 +14,11 @@ namespace models { // Using MultiRationalLoss is a first improvement, but we can probably // unify classifier and decoder costs. Also rethink step-wise cost. +// @TODO: inheritance and polymorphism is used here in a rather unclear way. +// E.g. returns Ptr which should be Ptr? +// Other functions return RationalLoss directly without Ptr<...>, but also +// they do not need polymorphism here. + class CostBase { public: virtual Ptr apply(Ptr model, @@ -28,6 +33,8 @@ class EncoderDecoderCE : public CostBase { bool inference_{false}; bool toBeWeighted_{false}; + + // @TODO: single loss seems wrong Ptr loss_; Ptr weighter_; @@ -68,14 +75,14 @@ class EncoderDecoderCE : public CostBase { multiLoss->push_back(partialLoss); if(options_->get("guided-alignment", std::string("none")) != "none" && !inference_) { - auto alignments = encdec->getDecoders()[0]->getAlignments(); - ABORT_IF(alignments.empty(), "Model does not seem to support alignments"); + auto attentionVectors = encdec->getDecoders()[0]->getAlignments(); + ABORT_IF(attentionVectors.empty(), "Model does not seem to support alignments"); - auto att = concatenate(alignments, /*axis =*/ -1); + auto attention = concatenate(attentionVectors, /*axis =*/ -1); - auto alignmentLoss = guidedAlignmentCost(graph, corpusBatch, options_, att); + auto alignmentLoss = guidedAlignmentCost(graph, corpusBatch, options_, attention); multiLoss->push_back(alignmentLoss); - + return multiLoss; } else { return multiLoss; @@ -87,6 +94,9 @@ class EncoderClassifierCE : public CostBase { protected: Ptr options_; bool inference_{false}; + + // @TODO: single loss seems wrong, especially since we support multiple objectives here, + // also not sure this needs to be a member at all. Ptr loss_; public: From 87e8d7526c14187dfb229ec2a4a284dd01e57029 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 10:39:29 -0800 Subject: [PATCH 136/838] use rationalLoss for updates --- src/training/graph_group_async.cpp | 17 ++++++++--------- src/training/graph_group_multinode.cpp | 16 +++++++--------- src/training/graph_group_multinode_sync.cpp | 15 ++++++--------- src/training/graph_group_singleton.cpp | 6 ++---- src/training/graph_group_sync.cpp | 13 ++++++------- src/training/scheduler.h | 21 ++++++++++----------- 6 files changed, 39 insertions(+), 49 deletions(-) diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index b529089af..5ac456229 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -192,7 +192,7 @@ void AsyncGraphGroup::execute(Ptr batch) { thread_local size_t num_seen_words = 0; thread_local size_t num_seen_sentences = 0; thread_local int t_id = 0; - thread_local float cost = 0; + thread_local StaticLoss loss; thread_local Tensor accGradients; thread_local Ptr accAlloc; @@ -204,14 +204,14 @@ void AsyncGraphGroup::execute(Ptr batch) { builder = builders_[i++]; } - auto costNode = builder->build(graph, batch); + auto lossNode = builder->build(graph, batch); if(t % optimizerDelay_ == 0) { fetchParams(graph->params()->vals(), params_, t_id); } graph->forward(); - cost += costNode->loss(); + loss += *lossNode; graph->backward(); Tensor gradients; @@ -249,23 +249,22 @@ void AsyncGraphGroup::execute(Ptr batch) { // Wait until the thread that wants to do validation is finished. pool_->wait_for_one(lock); - if(options_->get("cost-type") != "ce-sum") - cost /= optimizerDelay_; - if(optimizerDelay_ > 1) { std::vector fakeLength = {1, 1}; std::vector> vocabs; auto fb = data::CorpusBatch::fakeBatch(fakeLength, vocabs, num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); - scheduler_->update(cost, fb); + + scheduler_->update(loss, fb); num_seen_words = 0; num_seen_sentences = 0; } else { - scheduler_->update(cost, batch); + scheduler_->update(loss, batch); } - cost = 0; + loss.loss = 0; + loss.labels = 0; if(scheduler_->saving() || scheduler_->validating()) { // Wait with validation or saving until all other threads are done with diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index 8adafd89a..b5c01836d 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -516,7 +516,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { thread_local size_t my_id = 0; thread_local size_t t = 0; // only for scheduler statistic - thread_local float cost = 0; + thread_local StaticLoss loss; thread_local size_t num_seen_words = 0; thread_local size_t num_seen_sentences = 0; @@ -527,7 +527,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { builder = clientBuilders_[i++]; } - auto costNode = builder->build(graph, batch); + auto lossNode = builder->build(graph, batch); if(t == 0) { mpi_->barrier(); @@ -537,7 +537,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { } graph->forward(); - cost += costNode->loss(); + loss += *lossNode; num_seen_words += batch->words(); num_seen_sentences += batch->size(); graph->backward(); @@ -592,22 +592,20 @@ void MultiNodeGraphGroup::execute(Ptr batch) { // Wait until the thread that wants to do validation is finished. clientThreadPool_->wait_for_one(lock); - if(options_->get("cost-type") != "ce-sum") - cost /= tau_; - if(tau_ > 1) { std::vector fakeLength = {1, 1}; std::vector> vocabs; auto fb = data::CorpusBatch::fakeBatch(fakeLength, vocabs, num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); - scheduler_->update(cost, fb); + scheduler_->update(loss, fb); } else { - scheduler_->update(cost, batch); + scheduler_->update(loss, batch); } num_seen_words = 0; num_seen_sentences = 0; - cost = 0; + loss.loss = 0; + loss.labels = 0; if((scheduler_->saving() || scheduler_->validating())) { // Wait with validation or saving until all other threads are done with diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp index f32fb0d28..7d7f2d00b 100755 --- a/src/training/graph_group_multinode_sync.cpp +++ b/src/training/graph_group_multinode_sync.cpp @@ -175,7 +175,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { static int t = 0; - static float cost = 0; + static StaticLoss loss; static size_t num_seen_words = 0; static size_t num_seen_sentences = 0; @@ -185,7 +185,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { auto graph = clientGraphs_[my_id]; auto builder = clientBuilders_[my_id]; - auto costNode = builder->build(graph, batch); + auto lossNode = builder->build(graph, batch); if(t == 0) { if(my_id != 0) @@ -195,7 +195,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { graph->forward(); { std::lock_guard guard(sumCostMutex_); - cost += costNode->loss(); + loss += *lossNode; num_seen_words += batch->words(); num_seen_sentences += batch->size(); } @@ -219,21 +219,18 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { // Run scheduler (if enabled) if(t % tau_ == 0 && scheduler_) { - if(options_->get("cost-type") != "ce-sum") - cost /= (tau_ * devices_.size()); - if(tau_ > 1) { std::vector fakeLength = {1, 1}; auto fb = data::CorpusBatch::fakeBatch(fakeLength, std::vector>(), num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); - scheduler_->update(cost, fb); + scheduler_->update(loss, fb); } else { - scheduler_->update(cost, fullBatch); + scheduler_->update(loss, fullBatch); } num_seen_words = 0; num_seen_sentences = 0; - cost = 0; + loss = StaticLoss(); if((scheduler_->saving() || scheduler_->validating())) { // wait until other nodes are ready diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp index 46084c787..210834081 100755 --- a/src/training/graph_group_singleton.cpp +++ b/src/training/graph_group_singleton.cpp @@ -10,10 +10,8 @@ void SingletonGraph::setScheduler(Ptr scheduler) { } void SingletonGraph::execute(Ptr batch) { - auto costNode = builder_->build(graph_, batch); - + auto lossNode = builder_->build(graph_, batch); graph_->forward(); - float cost = costNode->loss(); graph_->backward(); // Get batch stats @@ -34,7 +32,7 @@ void SingletonGraph::execute(Ptr batch) { } if(scheduler_) { - scheduler_->update(cost, batch); + scheduler_->update(*lossNode, batch); if(scheduler_->validating()) { if(mvAvg_) { diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index dafd61246..e6b01c54c 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -134,7 +134,7 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { // Compute gradients // This happens in multiple steps in case of delay_ > 1. - std::vector localDeviceCosts(devices_.size()); + std::vector localDeviceLosses(devices_.size()); for (size_t t = 0; t < delay_; t++) { // Execute single forward/backward step @@ -148,11 +148,11 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { //LOG(info, timer.format(2, "after build: %ws")); graph->forward(); //LOG(info, timer.format(2, "after forward (no sync): %ws")); - localDeviceCosts[localDeviceIndex] += *rationalLoss; // converts dynamic RationalLoss to StaticLoss + localDeviceLosses[localDeviceIndex] += *rationalLoss; // converts dynamic RationalLoss to StaticLoss graph->backward(/*zero=*/t == 0); // only reset gradients to 0 if t = 0 //LOG(info, timer.format(2, "after backward (no sync): %ws")); - //localDeviceCosts[localDeviceIndex] += costNode->scalar(); // moved here for time measurements; @TODO: move this back + //localDeviceLosses[localDeviceIndex] += costNode->scalar(); // moved here for time measurements; @TODO: move this back //LOG(info, timer.format(2, "after scalar() (that's a sync): %ws")); } else { // empty batch: execute do-nothing fw-bw step for proper inits and resets @@ -190,12 +190,11 @@ void SyncGraphGroup::update(Ptr batch) /*override*/ { // cost across all local devices (scheduler will aggregate cross-process) StaticLoss localLoss; - for(auto& c : localDeviceCosts) // localDeviceCosts is already summed up over delay steps - localLoss += c; + for(auto& l : localDeviceLosses) // localDeviceLosses is already summed up over delay steps + localLoss += l; if(scheduler_) { - // track and log localLoss, @TODO: rather pass StaticLoss object - scheduler_->update(localLoss.loss, localLoss.labels, subBatches, mpi_); + scheduler_->update(localLoss, subBatches, mpi_); // save intermediate model (and optimizer state) to file if(scheduler_->saving()) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 41b765e45..7fc51da37 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -4,6 +4,7 @@ #include "training/training_state.h" #include "training/validator.h" #include "training/communicator.h" +#include "layers/loss.h" namespace marian { @@ -187,17 +188,15 @@ class Scheduler : public TrainingObserver { return 0; } - void update(float cost, Ptr batch) { - // @TODO: Check me and eventually remove me, currently here for back-compat. - update(cost, batch->words(-1), std::vector>({batch})); + void update(StaticLoss rationalLoss, Ptr batch) { + update(rationalLoss, std::vector>({batch})); } - void update(float cost, float labels, const std::vector>& batches, Ptr mpi = nullptr) { + void update(StaticLoss rationalLoss, const std::vector>& batches, Ptr mpi = nullptr) { state_->rememberPreviousProgress(); // note: epoch increases happen at the wrong place, hence -freq parameters do not support epoch units state_->validated = false; size_t batchSize = 0; // number of sentences in batch - size_t batchLabels = (size_t)labels; // number of target words in batch size_t batchWords = 0; for(const auto& batch : batches) { @@ -210,17 +209,17 @@ class Scheduler : public TrainingObserver { // extrapolate cost across MPI processes, so that we have numbers in the right range // When doing the actual log, we then aggregate across MPI processes to get the accurate number. if (mpi) - cost *= mpi->numMPIProcesses(); + rationalLoss.loss *= mpi->numMPIProcesses(); - state_->costSum += cost; // aggregate sum cost since last display - state_->costCount += batchLabels; // cost gets normalized w.r.t. this in display + state_->costSum += rationalLoss.loss; // aggregate sum cost since last display + state_->costCount += rationalLoss.labels; // cost gets normalized w.r.t. this in display state_->updatesDisp += 1; state_->samplesDisp += batchSize; - state_->wordsDisp += batchWords; // words at given input processed since last display, for speed display + state_->wordsDisp += batchWords; // words at given input processed since last display, for speed display - state_->samplesEpoch += batchSize; // sentences processed in this epoch - state_->labelsTotal += batchLabels; // total labels processed + state_->samplesEpoch += batchSize; // sentences processed in this epoch + state_->labelsTotal += rationalLoss.labels; // total labels processed state_->newBatch(); From 874274377406e77c0249a42ca9631c75713598c5 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 14:30:46 -0800 Subject: [PATCH 137/838] mnist now passes reg tests --- src/examples/mnist/model.h | 38 ++++++++++++++++++++++++++-------- src/examples/mnist/training.h | 2 +- src/examples/mnist/validator.h | 6 +++--- src/training/training.h | 2 +- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 71de15ec4..44f41e6ac 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -8,20 +8,23 @@ #include "graph/expression_graph.h" #include "models/costs.h" #include "models/model_base.h" +#include "layers/loss.h" #include "examples/mnist/dataset.h" namespace marian { namespace models { +// @TODO: looking at this file, simplify the new RationalLoss idea. Here it gets too complicated + class MNISTCrossEntropyCost : public CostBase { public: MNISTCrossEntropyCost() {} - Expr apply(Ptr model, - Ptr graph, - Ptr batch, - bool clearGraph = true) override { + Ptr apply(Ptr model, + Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto top = model->build(graph, batch, clearGraph); auto vfLabels = std::static_pointer_cast(batch)->labels(); @@ -31,7 +34,16 @@ class MNISTCrossEntropyCost : public CostBase { auto labels = graph->indices(vLabels); // Define a top-level node for training - return mean(cross_entropy(top, labels), /*axis =*/ 0); + // use CE loss + + auto loss = sum(cross_entropy(top->loss(), labels), /*axis =*/ 0); + auto labelsNum = graph->constant({1}, inits::from_value((float)vLabels.size())); + + // @TODO: simplify this + auto multiLoss = New(); + multiLoss->push_back({loss, labelsNum}); + + return multiLoss; } }; @@ -39,12 +51,16 @@ class MNISTLogsoftmax : public CostBase { public: MNISTLogsoftmax() {} - Expr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { auto top = model->build(graph, batch, clearGraph); - return logsoftmax(top); + + // @TODO: simplify this + auto multiLoss = New(); + multiLoss->push_back({logsoftmax(top->loss()), top->labels()}); + return multiLoss; } }; @@ -56,10 +72,14 @@ class MnistFeedForwardNet : public ModelBase { MnistFeedForwardNet(Ptr options, Args... args) : options_(options), inference_(options->get("inference", false)) {} - virtual Expr build(Ptr graph, + virtual Ptr build(Ptr graph, Ptr batch, bool /*clean*/ = false) override { - return construct(graph, batch, inference_); + + auto loss = construct(graph, batch, inference_); + auto labels = graph->constant({(int)batch->size(), 1}, inits::from_value(1.f)); + + return New(loss, labels); } void load(Ptr /*graph*/, const std::string& /*name*/, bool) override { diff --git a/src/examples/mnist/training.h b/src/examples/mnist/training.h index 64ccfcbd0..9e10bdd7e 100755 --- a/src/examples/mnist/training.h +++ b/src/examples/mnist/training.h @@ -28,7 +28,7 @@ class TrainMNIST : public ModelTask { // Prepare scheduler with validators auto trainState = New(options_->get("learn-rate")); auto scheduler = New(options_, trainState); - scheduler->addValidator(New(options_)); + scheduler->addValidator(New(options_)); // Multi-node training auto mpi = initMPI(/*multiThreaded=*/false); diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h index 907994ee7..f38a95cec 100644 --- a/src/examples/mnist/validator.h +++ b/src/examples/mnist/validator.h @@ -12,9 +12,9 @@ using namespace marian; namespace marian { -class AccuracyValidator : public Validator { +class MNISTAccuracyValidator : public Validator { public: - AccuracyValidator(Ptr options) : Validator(std::vector>(), options, false) { + MNISTAccuracyValidator(Ptr options) : Validator(std::vector>(), options, false) { createBatchGenerator(/*isTranslating=*/false); builder_ = models::from_options(options, models::usage::scoring); } @@ -35,7 +35,7 @@ class AccuracyValidator : public Validator { graphs[0]->forward(); std::vector scores; - probs->val()->get(scores); + probs->loss(scores); correct += countCorrect(scores, batch->labels()); samples += batch->size(); diff --git a/src/training/training.h b/src/training/training.h index b07d4c6e8..d0623cfde 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -43,7 +43,7 @@ class Train : public ModelTask { LOG(info, "[batching] Collecting statistics for batch fitting with step size {}", options_->get("mini-batch-fit-step")); - // @TODO this should receive a function object that can generate a fake batch + // @TODO this should receive a function object that can generate a fake batch; // that way vocabs would not be exposed. auto model = New(options_, mpi); model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling From 9a196afbfbf82d1f5de3e2c79e5e6046b1ea3fd8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 16 Jan 2019 15:05:37 -0800 Subject: [PATCH 138/838] fix missing restore parameters --- src/training/training_state.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/training/training_state.h b/src/training/training_state.h index 814b270ae..99c1621d5 100755 --- a/src/training/training_state.h +++ b/src/training/training_state.h @@ -237,9 +237,11 @@ class TrainingState { warmupStart = SchedulingParameter::parse(config["warmup-start"].as()); costSum = config["cost-sum"].as(); - // (different serialization name for back compat) - costCount = config["disp-samples"].as(); + costCount = config["cost-count"].as(); + wordsDisp = config["disp-words"].as(); + samplesDisp = config["disp-samples"].as(); + updatesDisp = config["disp-updates"].as(); seedBatch = config["seed-batch"].as(); seedCorpus = config["seed-corpus"].as(); @@ -270,7 +272,10 @@ class TrainingState { config["warmup-start"] = std::string(warmupStart); config["cost-sum"] = costSum; - config["disp-samples"] = costCount; + config["cost-count"] = costCount; + + config["disp-updates"] = updatesDisp; + config["disp-samples"] = samplesDisp; config["disp-words"] = wordsDisp; config["seed-batch"] = seedBatch; From b13e5aa550e3b2dce855d79b3a4d37a2513568e5 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 08:05:23 -0800 Subject: [PATCH 139/838] comments in loss --- src/layers/loss.h | 131 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 28 deletions(-) diff --git a/src/layers/loss.h b/src/layers/loss.h index 8c8d170c0..850dc6b78 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -4,6 +4,18 @@ namespace marian { +/** + * We represent loss as pair of expressions, where loss_ is usually a sum + * of all accumulated loss values per label and labels_ is the total number + * of labels over which the loss was collected. + * + * These two values can then be used to represent various cost variants - + * for instance label-wise cross-entropy or perplexity. Optimization is + * only performed with regard to the summed loss_. + * + * Since both, loss_ and labels_ are dynamic graph nodes they can be further + * combined into larger structures. See multi-objective losses below. + */ class RationalLoss { protected: Expr loss_; @@ -15,6 +27,10 @@ class RationalLoss { RationalLoss(Expr loss, Expr labels) : loss_(loss), labels_(labels) {} + RationalLoss(Expr loss, float labels) + : loss_(loss), + labels_(loss->graph()->constant({1}, inits::from_value(labels))) {} + RationalLoss(const RationalLoss& other) : loss_(other.loss_), labels_(other.labels_) {} @@ -54,16 +70,20 @@ class RationalLoss { } }; -// POD for accumulating loss values after forward/backward and to-be-used in -// Scheduler for updating statistics. +/** + * POD for accumulating loss values after forward/backward used in + * Scheduler for updating statistics. This can only be used after a + * successful forward step in a computation graph that owns the assigned + * RationalLoss object. + */ struct StaticLoss { float loss; float labels; StaticLoss() : loss(0.f), labels(0.f) {} - StaticLoss(const RationalLoss& rl) - : loss(rl.loss()), labels(rl.labels()) {} + StaticLoss(const RationalLoss& dynamic) + : loss(dynamic.loss()), labels(dynamic.labels()) {} StaticLoss& operator +=(const StaticLoss& other) { loss = loss + other.loss; @@ -72,11 +92,22 @@ struct StaticLoss { } }; +/** + * Base class for multi-objective losses which is a list of RationalLoss + * but also defines how to accumulate that list into a single RationalLoss + */ class MultiRationalLoss : public RationalLoss { protected: std::vector partialLosses_; + /** + * Accumulation rule for losses + */ virtual Expr accumulateLoss(const RationalLoss& current) = 0; + + /** + * Accumulation rule for labels + */ virtual Expr accumulateLabels(const RationalLoss& current) = 0; public: @@ -110,6 +141,11 @@ class MultiRationalLoss : public RationalLoss { }; +/** + * Simple sum of losses. + * Using this makes sense when the two loss types are similar in scale and + * number of labels. For instance two decoders over similarly sized vocabularies + */ class SumMultiRationalLoss : public MultiRationalLoss { private: virtual Expr accumulateLoss(const RationalLoss& current) override { @@ -131,27 +167,20 @@ class SumMultiRationalLoss : public MultiRationalLoss { SumMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} }; -class MeanMultiRationalLoss : public MultiRationalLoss { -private: - virtual Expr accumulateLoss(const RationalLoss& current) override { - if(loss_) - return loss_ + current.loss() / current.labels(); - else - return current.loss() / current.labels(); - } - - virtual Expr accumulateLabels(const RationalLoss& current) override { - if(labels_) - return labels_ + 1.f; // broadcast to size - else - return current.labels() / current.labels(); // 1, but with correct size - } - -public: - MeanMultiRationalLoss() : MultiRationalLoss() {} - MeanMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} -}; - +/** + * Scaled sum of losses. + * This can weigh losses equally by choosing the first loss_0 as a reference + * and scaling all remaining losses loss_i by labels_0 / labels_i. Labels are + * summed up by the same rule. By this we simulate a sum of losses at similar + * scales. Dividing by scaled label counts yields a value close to an equally + * weighted sum of means. + * + * L = sum_i^N L_i + N/M sum_j^M L_j + * + * We set labels to N. When reporting L/N this is equvalient to sum of means. + * Compare to sum of means below where N is factored into the loss, but labels + * are set to 1. + */ class ScaledMultiRationalLoss : public MultiRationalLoss { private: virtual Expr accumulateLoss(const RationalLoss& current) override { @@ -165,10 +194,9 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { virtual Expr accumulateLabels(const RationalLoss& current) override { if(labels_) { - const auto& first = partialLosses_.front(); - return labels_ + first.labels() / current.labels(); // fractional label counts are OK + return labels_; // Keep first label count // or: labels_ + first.labels() / current.labels(); } else { - return current.labels(); + return current.labels(); // This is the first loss } } @@ -177,12 +205,50 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { ScaledMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} }; +/** + * Sum of mean losses. + * Not really a rational loss as labels are factored into loss. Contribution of + * losses is equal, same as for ScaledMultiRationalLoss, just divided by different + * number of labels. See: + * + * L = (1/N sum_i^N L_i + 1/M sum_j^M L_j) = (sum_i^N L_i + N/M sum_j^M L_j) / N + * + * We set labels to 1. During reporting, we would see the same numbers, but gradients + * are scaled diffrently which may result in different learning curves. + */ +class MeanMultiRationalLoss : public MultiRationalLoss { +private: + virtual Expr accumulateLoss(const RationalLoss& current) override { + if(loss_) + return loss_ + current.loss() / current.labels(); + else + return current.loss() / current.labels(); + } + + virtual Expr accumulateLabels(const RationalLoss& current) override { + if(labels_) + return labels_; // keep the existing '1' + else + return current.labels()->graph()->ones({1}); // just '1' as labels are factored into loss_ + } + +public: + MeanMultiRationalLoss() : MultiRationalLoss() {} + MeanMultiRationalLoss(const RationalLoss& rl) : MultiRationalLoss(rl) {} +}; + +/** + * Factory for multi-objective rational loss functions + */ Ptr newMultiLoss(Ptr options); //***********************************************************************************// // This needs some to be refactored. Currently easiest route for backwards compat, but // still feels somewhat hacky. +/** + * Computes loss per label and then reduces to RationalLoss + */ class LabelwiseLoss { protected: std::vector axes_; @@ -219,6 +285,9 @@ class LabelwiseLoss { } }; +/** + * Cross entropy loss across last axis, summed up over batch and time dimensions + */ class CrossEntropyLoss : public LabelwiseLoss { public: CrossEntropyLoss(float smoothing) @@ -252,6 +321,9 @@ class CrossEntropyLoss : public LabelwiseLoss { } }; +/** + * Cross entropy in rescorer used for computing sentences-level log probabilities + */ class RescorerLoss : public CrossEntropyLoss { public: // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) @@ -264,6 +336,9 @@ class RescorerLoss : public CrossEntropyLoss { } }; +/** + * Factory for label-wise loss functions + */ Ptr newLoss(Ptr options, bool inference); } // namespace marian From c26f3bd1563711c18720dbfeef83b88d91093f23 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 08:42:28 -0800 Subject: [PATCH 140/838] more comments --- src/graph/node_initializers.h | 12 ++++- src/models/bert.h | 98 +++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/src/graph/node_initializers.h b/src/graph/node_initializers.h index 185ab1191..c7576cbb2 100755 --- a/src/graph/node_initializers.h +++ b/src/graph/node_initializers.h @@ -42,8 +42,6 @@ NodeInitializer from_vector(const std::vector& v); NodeInitializer from_item(const io::Item& item); -NodeInitializer positions(int start); - NodeInitializer from_sparse_vector( std::pair, std::vector>& v); @@ -53,6 +51,16 @@ NodeInitializer from_word2vec(const std::string& file, int dimVoc, int dimEmb, bool normalize = false); + +/** + * Computes Google's trigonometric positional embeddings + * starting from position 'start' taking into account + * batch and time dimensions of the tensor. + * + * Expected layout tensor layout {time, batch, model} + */ +NodeInitializer positions(int start); + } // namespace inits } // namespace marian diff --git a/src/models/bert.h b/src/models/bert.h index e74b9f633..615cbd570 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -6,8 +6,22 @@ #include "data/rng_engine.h" namespace marian { + +/** + * This file contains nearly all BERT-related code and adds BERT-funtionality + * on top of existing classes like TansformerEncoder and Classifier. + */ + namespace data { +/** + * BERT-specific mini-batch that computes masking for Masked LM training. + * Expects symbols [MASK], [SEP], [CLS] to be present in vocabularies unless + * other symbols are specified on the command line. + * + * This takes a normal CorpusBatch and extends it with additional data. Luckily + * all the BERT-functionality can be inferred from a CorpusBatch alone. + */ class BertBatch : public CorpusBatch { private: std::mt19937& eng_; @@ -20,14 +34,20 @@ class BertBatch : public CorpusBatch { std::string sepSymbol_; std::string clsSymbol_; + // Selects a random word from the vocabulary std::unique_ptr> randomWord_; + + // Selects a random integer between 0 and 99 std::unique_ptr> randomPercent_; + // Word ids of words that should not be masked, e.g. separators, padding std::unordered_set dontMask_; + // Masking function Word maskOut(Word word, Word mask) { auto subBatch = subBatches_.front(); + // @TODO: turn those threshold into parameters, adjustable from command line int r = (*randomPercent_)(eng_); if (r < 10) { // for 10% of cases return same word return word; @@ -43,6 +63,9 @@ class BertBatch : public CorpusBatch { } public: + + // Takes a corpus batch, random engine (for deterministic behavior) and the masking percentage. + // Also sets special vocabulary items given on command line. BertBatch(Ptr batch, std::mt19937& engine, float maskFraction, @@ -54,11 +77,15 @@ class BertBatch : public CorpusBatch { auto subBatch = subBatches_.front(); + // Initialize to sample random vocab id randomWord_.reset(new std::uniform_int_distribution(0, subBatch->vocab()->size())); + + // Intialize to sample random percentage randomPercent_.reset(new std::uniform_int_distribution(0, 100)); auto& words = subBatch->data(); + // Get word id of special symbols Word maskId = (*subBatch->vocab())[maskSymbol_]; Word clsId = (*subBatch->vocab())[clsSymbol_]; Word sepId = (*subBatch->vocab())[sepSymbol_]; @@ -82,7 +109,7 @@ class BertBatch : public CorpusBatch { for(int i = 0; i < words.size(); ++i) // collect words among which we will mask if(dontMask_.count(words[i]) == 0) // do not add indices of special words selected.push_back(i); - std::shuffle(selected.begin(), selected.end(), eng_); + std::shuffle(selected.begin(), selected.end(), eng_); // randomize positions selected.resize(std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices for(int i : selected) { @@ -94,14 +121,17 @@ class BertBatch : public CorpusBatch { int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); - sentenceIndices_.resize(words.size()); - std::vector sentPos(dimBatch, 0); - for(int i = 0; i < dimWords; ++i) { - for(int j = 0; j < dimBatch; ++j) { + // create indices for BERT sentence embeddings A and B + sentenceIndices_.resize(words.size()); // each word is either in sentence A or B + std::vector sentPos(dimBatch, 0); // initialize each batch entry with being A [0] + for(int i = 0; i < dimWords; ++i) { // advance word-wise + for(int j = 0; j < dimBatch; ++j) { // scan batch-wise int k = i * dimBatch + j; - sentenceIndices_[k] = sentPos[j]; - if(words[k] == sepId) - sentPos[j]++; + sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry + if(words[k] == sepId) { // if current word is a separator + sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) + ABORT_IF(sentPos[i] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[i]); + } } } } @@ -113,6 +143,10 @@ class BertBatch : public CorpusBatch { } +/** + * BERT-specific version of EncoderClassifier, mostly here to automatically convert a + * CorpusBatch to BertBatch. + */ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { public: BertEncoderClassifier(Ptr options) @@ -135,7 +169,11 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { } }; -// @TODO: this should be in transformer.h +/** + * BERT-specific modifications to EncoderTransformer + * Actually all that is needed is to intercept the creation of special embeddings, + * here sentence embeddings for sentence A and B. + */ class BertEncoder : public EncoderTransformer { public: BertEncoder(Ptr options) : EncoderTransformer(options) {} @@ -155,7 +193,7 @@ class BertEncoder : public EncoderTransformer { if(learnedPosEmbeddings) { auto sentenceEmbeddings = embedding() ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B + ("dimVocab", 2) // sentence A or sentence B, @TODO: should rather be a parameter ("dimEmb", dimEmb) .construct(graph_); signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); @@ -173,17 +211,20 @@ class BertEncoder : public EncoderTransformer { virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { bool learnedPosEmbeddings = opt("transformer-learned-positions", true); input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); - input = addSentenceEmbeddings(input, batch, learnedPosEmbeddings); + input = addSentenceEmbeddings(input, batch, learnedPosEmbeddings); // @TODO: separately set learnable pos and sent embeddings return input; } }; -// Can be used for next sentence prediction task +/** + * BERT-specific classifier + * Can be used for next sentence prediction task or other fine-tuned down-stream tasks + * Does not actually need a BertBatch, works with CorpusBatch + */ class BertClassifier : public ClassifierBase { public: BertClassifier(Ptr options) : ClassifierBase(options) {} - // The batch has been filled with external classifier labels, @TODO: figure out how to do that Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); @@ -193,12 +234,12 @@ class BertClassifier : public ClassifierBase { int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels - auto output = mlp::mlp() // - .push_back(mlp::dense() // + auto output = mlp::mlp() // + .push_back(mlp::dense() // ("prefix", prefix_ + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh)) // @TODO: do we actually need this? - .push_back(mlp::output() // + .push_back(mlp::output() // ("dim", dimTrgCls)) // ("prefix", prefix_ + "_ff_logit_l2") // .construct(graph); @@ -208,7 +249,7 @@ class BertClassifier : public ClassifierBase { auto state = New(); state->setLogProbs(logits); - // filled externally, for BERT these are NextSentence prediction labels + // Filled externally, for BERT these are NextSentence prediction labels const auto& classLabels = (*batch)[batchIndex_]->data(); state->setTargetIndices(graph->indices(classLabels)); @@ -218,7 +259,12 @@ class BertClassifier : public ClassifierBase { virtual void clear() override {} }; -// This is a model that pretrains BERT for classification +/** + * This is a model that pretrains BERT for classification. + * This is also a Classifier, but compared to the one above needs the BERT-specific information from BertBatch + * as this is self-generating its labels from the source. Labels are dynamically created as complements of the + * masking process. + */ class BertMaskedLM : public ClassifierBase { public: BertMaskedLM(Ptr options) : ClassifierBase(options) {} @@ -242,21 +288,21 @@ class BertMaskedLM : public ClassifierBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; - auto layerTanh = mlp::dense() // + auto layerTanh = mlp::dense() // ("prefix", prefix_ + "_ff_logit_maskedlm_out_l1") // - ("dim", dimModel) // - ("activation", mlp::act::tanh); // - auto layerOut = mlp::output() // + ("dim", dimModel) // + ("activation", mlp::act::tanh); // @TODO: again, check if this layer is present in original code + auto layerOut = mlp::output() // ("prefix", prefix_ + "_ff_logit_maskedlm_out_l2") // ("dim", dimVoc); // - layerOut.tieTransposed("Wemb"); // We are a BERT model, hence tie with input + layerOut.tieTransposed("Wemb"); // We are a BERT model, hence tie with input, @TODO: check if this is actually what Google does // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context - auto output = mlp::mlp() // - .push_back(layerTanh) // @TODO: do we actually need this? - .push_back(layerOut) // + auto output = mlp::mlp() // + .push_back(layerTanh) // + .push_back(layerOut) // .construct(graph); auto logits = output->apply(maskedEmbeddings); From 43eb1b2b4d2f80bb5224c29db7453eb13eea80d8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 08:52:38 -0800 Subject: [PATCH 141/838] more comments --- src/models/bert.h | 6 ++++-- src/models/classifier.h | 4 ++++ src/models/encoder_classifier.h | 12 +++++++++--- src/models/model_factory.cpp | 13 +++++-------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 615cbd570..f02b30c1a 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -156,7 +156,7 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { // intercept batch and anotate with BERT-specific concepts auto bertBatch = New(batch, eng_, - opt("bert-masking-fraction"), + opt("bert-masking-fraction", 0.15f), // 15% by default according to paper opt("bert-mask-symbol"), opt("bert-sep-symbol"), opt("bert-class-symbol")); @@ -219,7 +219,9 @@ class BertEncoder : public EncoderTransformer { /** * BERT-specific classifier * Can be used for next sentence prediction task or other fine-tuned down-stream tasks - * Does not actually need a BertBatch, works with CorpusBatch + * Does not actually need a BertBatch, works with CorpusBatch. + * + * @TODO: This is in fact so generic that we might move it out of here as the typical classifier implementation */ class BertClassifier : public ClassifierBase { public: diff --git a/src/models/classifier.h b/src/models/classifier.h index 61c4d64e3..86e01841e 100644 --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -7,6 +7,10 @@ namespace marian { +/** + * Simple base class for Classifiers + * Currently only implementations are in bert.h + */ class ClassifierBase { protected: Ptr options_; diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 9c794144c..43ed9e1ad 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -9,6 +9,14 @@ namespace marian { +/** + * Combines sequence encoders with generic classifiers + * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. + * Already has support for multi-objective training. + * + * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier + * multi-objective training. + */ class EncoderClassifierBase : public models::ModelBase { public: virtual ~EncoderClassifierBase() {} @@ -76,8 +84,6 @@ class EncoderClassifier : public EncoderClassifierBase { return std::string(out.c_str()); } -// virtual void createClassifierConfig(const std::string& name) {} - public: typedef data::Corpus dataset_type; @@ -197,7 +203,7 @@ class EncoderClassifier : public EncoderClassifierBase { bool clearGraph = true) override { auto states = apply(graph, batch, clearGraph); // returns raw logits - return New(states[0]->getLogProbs(), nullptr); // @TODO: this should explode + return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used } virtual Ptr build(Ptr graph, diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 0059aac88..a6ceb9988 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -36,11 +36,9 @@ Ptr EncoderFactory::construct(Ptr graph) { #endif if(options_->get("type") == "transformer") - // return New(options_); return NewEncoderTransformer(options_); if(options_->get("type") == "bert-encoder") - // return New(options_); return New(options_); ABORT("Unknown encoder type"); @@ -50,7 +48,6 @@ Ptr DecoderFactory::construct(Ptr graph) { if(options_->get("type") == "s2s") return New(options_); if(options_->get("type") == "transformer") - // return New(options_); return NewDecoderTransformer(options_); ABORT("Unknown decoder type"); } @@ -229,12 +226,12 @@ Ptr by_type(std::string type, usage use, Ptr options) { .construct(graph); } - if(type == "bert") { + if(type == "bert") { // for full BERT training return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // @TODO: replace with 'bert-encoder' - ("index", 0)) // close to original transformer encoder + ("type", "bert-encoder") // close to original transformer encoder + ("index", 0)) // .push_back(models::classifier() // ("type", "bert-masked-lm") // ("index", 0)) // multi-task learning with MaskedLM @@ -244,11 +241,11 @@ Ptr by_type(std::string type, usage use, Ptr options) { .construct(graph); } - if(type == "bert-classifier") { + if(type == "bert-classifier") { // for BERT fine-tuning on non-BERT classification task return models::encoder_classifier()(options) // ("usage", use) // .push_back(models::encoder() // - ("type", "transformer") // + ("type", "bert-encoder") // ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // ("type", "bert-classifier") // From 8ad7ef2748924d40627b67213dfe8d0a5c554473 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:02:09 -0800 Subject: [PATCH 142/838] more comments --- src/models/states.h | 4 ++++ src/training/scheduler.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/models/states.h b/src/models/states.h index cb94e4bb7..8e3972210 100755 --- a/src/models/states.h +++ b/src/models/states.h @@ -100,6 +100,10 @@ class DecoderState { virtual void blacklist(Expr /*totalCosts*/, Ptr /*batch*/) {} }; +/** + * Classifier output based on DecoderState + * @TODO: should be unified with DecoderState or not be used at all as Classifier do not really have stateful output. + */ class ClassifierState { private: Expr logProbs_; diff --git a/src/training/scheduler.h b/src/training/scheduler.h index d316c08e7..12ad1ae4d 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -257,6 +257,10 @@ class Scheduler : public TrainingObserver { update(rationalLoss, /*numReadBatches=*/1, /*batchSize=*/batch->size(), /*batchLabels=*/batch->wordsTrg()); } + // @TODO: go back to function which takes batch as an argument? The current arguments make it hard to choose + // which subbatch should be used for speed display. For sequence-classifiers it's more interesting to see the + // source-words consumed rather than the labels. We have a CLI option '--disp-label-index' (bad name?) which is + // now defunct. void update(StaticLoss rationalLoss, size_t numReadBatches, // number of batches read by the reader (for seeking in case of restart) size_t batchSize, // total number of sentences in batch From 97bc272118022334e342795bbfbf10a812cf8887 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:11:05 -0800 Subject: [PATCH 143/838] add StaticLoss to CE validation --- src/training/validator.h | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/training/validator.h b/src/training/validator.h index 078fb4f3b..bce37da28 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -168,11 +168,10 @@ class CrossEntropyValidator : public Validator { protected: virtual float validateBG(const std::vector>& graphs) override { auto ctype = options_->get("cost-type"); - options_->set("cost-type", "ce-sum"); + options_->set("cost-type", "ce-sum"); // @TODO: check if still needed, most likely not. - float cost = 0; + StaticLoss loss; size_t samples = 0; - size_t words = 0; size_t batchId = 0; { @@ -180,7 +179,7 @@ class CrossEntropyValidator : public Validator { TaskBarrier taskBarrier; for(auto batch : *batchGenerator_) { - auto task = [=, &cost, &samples, &words](size_t id) { + auto task = [=, &loss, &samples](size_t id) { thread_local Ptr graph; thread_local auto builder = models::from_options(options_, models::usage::scoring); @@ -189,13 +188,11 @@ class CrossEntropyValidator : public Validator { } builder->clear(graph); - auto loss = builder->build(graph, batch); + auto dynamicLoss = builder->build(graph, batch); graph->forward(); std::unique_lock lock(mutex_); - cost += loss->loss(); - words += (size_t)loss->labels(); - + loss += *dynamicLoss; samples += batch->size(); }; @@ -206,16 +203,16 @@ class CrossEntropyValidator : public Validator { } // get back to the original cost type - options_->set("cost-type", ctype); + options_->set("cost-type", ctype); // @TODO: check if still needed, most likely not. if(ctype == "perplexity") - return std::exp(cost / words); + return std::exp(loss.loss / loss.labels); if(ctype == "ce-mean-words") - return cost / words; + return loss.loss / loss.labels; if(ctype == "ce-sum") - return cost; + return loss.loss; else - return cost / samples; + return loss.loss / samples; // @TODO: back-compat, to be removed } }; From 175e7e90adddfc7753f400d52defb0265d9e5442 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:20:53 -0800 Subject: [PATCH 144/838] fix index --- src/models/bert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/bert.h b/src/models/bert.h index f02b30c1a..ffc599549 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -130,7 +130,7 @@ class BertBatch : public CorpusBatch { sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry if(words[k] == sepId) { // if current word is a separator sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) - ABORT_IF(sentPos[i] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[i]); + ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[i]); } } } From d912f9171d04ffa1f6793f16fbcd0e55bb8f84f0 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:21:17 -0800 Subject: [PATCH 145/838] fix index 2 --- src/models/bert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/bert.h b/src/models/bert.h index ffc599549..66b40dd92 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -130,7 +130,7 @@ class BertBatch : public CorpusBatch { sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry if(words[k] == sepId) { // if current word is a separator sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) - ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[i]); + ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[j]); } } } From 95b23d6492660559d3e5f2812896d454bbbef998 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:25:35 -0800 Subject: [PATCH 146/838] move sentence number check in BERT before assignment --- src/models/bert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/bert.h b/src/models/bert.h index 66b40dd92..904f6c716 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -127,10 +127,10 @@ class BertBatch : public CorpusBatch { for(int i = 0; i < dimWords; ++i) { // advance word-wise for(int j = 0; j < dimBatch; ++j) { // scan batch-wise int k = i * dimBatch + j; + ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[j] + 1); sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry if(words[k] == sepId) { // if current word is a separator sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) - ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[j]); } } } From d5720460e134d8bbb4e476106429d8befe9160ab Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 09:36:03 -0800 Subject: [PATCH 147/838] restrict max sentence embedding index to 1 for now --- src/models/bert.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 904f6c716..9c5c05bc2 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -121,15 +121,17 @@ class BertBatch : public CorpusBatch { int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); + int maxSentPos = 1; // Currently only two sentences allowed, if another separator is seen do not increase position index beyond 1 + // @TODO: make this configurable, see below for NextSentencePredictions task where we also restrict to 2. + // create indices for BERT sentence embeddings A and B sentenceIndices_.resize(words.size()); // each word is either in sentence A or B std::vector sentPos(dimBatch, 0); // initialize each batch entry with being A [0] for(int i = 0; i < dimWords; ++i) { // advance word-wise for(int j = 0; j < dimBatch; ++j) { // scan batch-wise int k = i * dimBatch + j; - ABORT_IF(sentPos[j] > 1, "Currently we only support sequences of up to two sentences in BERT, not {}", sentPos[j] + 1); - sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry - if(words[k] == sepId) { // if current word is a separator + sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry, max position 1. + if(words[k] == sepId && sentPos[j] < maxSentPos) { // if current word is a separator and not beyond range sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) } } From 699b710b7fa342dbf4c91dd2f7a70370f31ca75c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 10:13:32 -0800 Subject: [PATCH 148/838] add third sentence embedding for padding --- src/models/bert.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 9c5c05bc2..0591b3c08 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -121,7 +121,8 @@ class BertBatch : public CorpusBatch { int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); - int maxSentPos = 1; // Currently only two sentences allowed, if another separator is seen do not increase position index beyond 1 + int maxSentPos = 2; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] + // If another separator is seen do not increase position index beyond 2 but use padding. // @TODO: make this configurable, see below for NextSentencePredictions task where we also restrict to 2. // create indices for BERT sentence embeddings A and B @@ -132,7 +133,7 @@ class BertBatch : public CorpusBatch { int k = i * dimBatch + j; sentenceIndices_[k] = sentPos[j]; // set to current sentence position for batch entry, max position 1. if(words[k] == sepId && sentPos[j] < maxSentPos) { // if current word is a separator and not beyond range - sentPos[j]++; // then increase sentence position for batch entry (probably to B [1]) + sentPos[j]++; // then increase sentence position for batch entry (to B [1]) } } } @@ -195,7 +196,7 @@ class BertEncoder : public EncoderTransformer { if(learnedPosEmbeddings) { auto sentenceEmbeddings = embedding() ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B, @TODO: should rather be a parameter + ("dimVocab", 3) // sentence A or sentence B plus padding, @TODO: should rather be a parameter ("dimEmb", dimEmb) .construct(graph_); signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); From afcaf54651154b6afbee955553f655fe2e62fcac Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 17 Jan 2019 15:16:01 -0800 Subject: [PATCH 149/838] added --embedding-factors option also to decode and score commands --- src/common/config_parser.cpp | 4 ++++ src/layers/generic.cpp | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index e301fb30b..776042858 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -469,6 +469,8 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { "stdout"); cli.add>("--vocabs,-v", "Paths to vocabulary files have to correspond to --input"); + cli.add_nondefault>("--embedding-factors", + "Paths to (factor map, factor list) file for factored embeddings"); // decoding options cli.add("--beam-size,-b", "Beam size used during search with validating translator", @@ -531,6 +533,8 @@ void ConfigParser::addOptionsScoring(cli::CLIWrapper& cli) { "Paths to vocabulary files have to correspond to --train-sets. " "If this parameter is not supplied we look for vocabulary files source.{yml,json} and target.{yml,json}. " "If these files do not exists they are created"); + cli.add_nondefault>("--embedding-factors", + "Paths to (factor map, factor list) file for factored embeddings"); cli.add("--n-best", "Score n-best list instead of plain text corpus"); cli.add("--n-best-feature", diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 82c5bc7c9..92f6c59ee 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -133,7 +133,9 @@ namespace marian { Expr Output::apply(Expr input) /*override*/ { lazyConstruct(input->shape()[-1]); + auto y = affine(input, W_, b_, false, transposeW_); + if (embeddingFactorMapping_) { auto graph = input->graph(); auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] @@ -145,6 +147,7 @@ namespace marian { graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), /*transB=*/ true); // -> [B x V] } + return y; } } From e8403817ae9db20f4563c3085710acf42d9fb177 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 17 Jan 2019 15:52:27 -0800 Subject: [PATCH 150/838] mlp::Output can now be reset per batch (with correct caching of shortlist), and therefore no longer needs to be recreated for each batch --- src/layers/generic.h | 67 ++++++++++++++++++++++++++++------------ src/models/transformer.h | 27 +++++++--------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index a9e2be016..5c604205b 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -124,47 +124,74 @@ class Dense : public LayerBase, public IUnaryLayer { class Output : public LayerBase, public IUnaryLayer { private: - Expr tiedParam_; - Ptr shortlist_; - - Expr W_; + Expr W_; // parameters held by this layer Expr b_; + Expr cachedShortW_; // short-listed version, cached (cleared by clear()) + Expr cachedShortb_; // these match the current value of shortlist_ + + // optional parameters set/updated after construction + Expr tiedParam_; bool transposeW_{false}; + Ptr shortlist_; public: Output(Ptr graph, Ptr options) - : LayerBase(graph, options) {} + : LayerBase(graph, options) { + clear(); + } void tieTransposed(Expr tied) { - tiedParam_ = tied; + if (W_) + ABORT_IF(tiedParam_.get() != tied.get(), "Tied output projection cannot be changed once weights have been created"); + else + tiedParam_ = tied; } - void setShortlist(Ptr shortlist) { shortlist_ = shortlist; } + void setShortlist(Ptr shortlist) { + if (shortlist_) + ABORT_IF(shortlist.get() != shortlist_.get(), "Output shortlist cannot be changed except after clear()"); + else { + ABORT_IF(cachedShortW_ || cachedShortb_, "No shortlist but cached parameters??"); + shortlist_ = shortlist; + } + // cachedShortW_ and cachedShortb_ will be created lazily inside apply() + } + + // this is expected to be called in sync with graph->clear(), which invalidates + // cachedShortW_ and cachedShortb_ in the graph's short-term cache + void clear() { + shortlist_ = nullptr; + cachedShortW_ = nullptr; + cachedShortb_ = nullptr; + } Expr apply(Expr input) override { - if(!W_) { + if(!W_) { // create lazily because we need input's dimension auto name = options_->get("prefix"); auto dim = options_->get("dim"); if(tiedParam_) { - transposeW_ = true; W_ = tiedParam_; - if(shortlist_) - W_ = rows(W_, shortlist_->indices()); + transposeW_ = true; } else { - W_ = graph_->param(name + "_W", - {input->shape()[-1], dim}, - inits::glorot_uniform); - if(shortlist_) - W_ = cols(W_, shortlist_->indices()); + W_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform); + transposeW_ = false; } - b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); - if(shortlist_) - b_ = cols(b_, shortlist_->indices()); } - return affine(input, W_, b_, false, transposeW_); + if (shortlist_) { + if (!cachedShortW_) { // short versions of parameters are cached within one batch, then clear()ed + if(transposeW_) + cachedShortW_ = rows(W_, shortlist_->indices()); + else + cachedShortW_ = cols(W_, shortlist_->indices()); + cachedShortb_ = cols(b_, shortlist_->indices()); + } + return affine(input, cachedShortW_, cachedShortb_, false, transposeW_); + } + else + return affine(input, W_, b_, false, transposeW_); } virtual Expr apply(const std::vector& /*inputs*/) override { diff --git a/src/models/transformer.h b/src/models/transformer.h index d91dfb7f8..048ac03ae 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -601,17 +601,17 @@ class TransformerState : public DecoderState { class DecoderTransformer : public Transformer { private: - Ptr output_; + Ptr output_; private: - void LazyCreateOutputLayer() + void lazyCreateOutputLayer() { if(output_) // create it lazily return; int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - auto layerOut = mlp::output() // + auto outputFactory = mlp::output() // ("prefix", prefix_ + "_ff_logit_out") // ("dim", dimTrgVoc); @@ -619,18 +619,10 @@ class DecoderTransformer : public Transformer { std::string tiedPrefix = prefix_ + "_Wemb"; if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) tiedPrefix = "Wemb"; - layerOut.tieTransposed(tiedPrefix); + outputFactory.tieTransposed(tiedPrefix); } - if(shortlist_) - layerOut.setShortlist(shortlist_); - - // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] - // assemble layers into MLP and apply to embeddings, decoder context and - // aligned source context - output_ = mlp::mlp() // - .push_back(layerOut) // - .construct(graph_); + output_ = std::dynamic_pointer_cast(outputFactory.construct(graph_)); // (construct() returns only the underlying interface) } public: @@ -662,7 +654,7 @@ class DecoderTransformer : public Transformer { virtual Ptr step(Ptr graph, Ptr state) override { ABORT_IF(graph != graph_, "An inconsistent graph parameter was passed to step()"); - LazyCreateOutputLayer(); + lazyCreateOutputLayer(); return step(state); } @@ -818,7 +810,9 @@ class DecoderTransformer : public Transformer { //************************************************************************// // final feed-forward layer (output) - Expr logits = output_->apply(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] + if(shortlist_) + output_->setShortlist(shortlist_); + Expr logits = output_->apply(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab or shortlist dim] // return unormalized(!) probabilities Ptr nextState; @@ -840,7 +834,8 @@ class DecoderTransformer : public Transformer { } void clear() override { - output_ = nullptr; + if (output_) + output_->clear(); cache_.clear(); alignments_.clear(); } From a2ade1537877e89f0d527064737c95e38a8a104d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 17 Jan 2019 18:33:42 -0800 Subject: [PATCH 151/838] (comment) --- src/layers/generic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 0bc20434a..e99a5410f 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -28,10 +28,11 @@ namespace marian { // Factors are grouped // - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group // - factors within a group as multi-class and normalized that way - // - groups of size 1 are interpreted as sigmoids + // - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) // - one prefix must not contain another // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive + // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { std::vector paths = options->get>("embedding-factors"); ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); From 272253c2dc48458804682ef3061717af1a6a25bb Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 17 Jan 2019 20:23:40 -0800 Subject: [PATCH 152/838] address code review comments, part 1 --- src/common/config_parser.cpp | 8 +-- src/data/corpus.cpp | 2 +- src/data/corpus_base.cpp | 63 +++++++++------------ src/data/corpus_base.h | 18 +++--- src/data/corpus_nbest.cpp | 4 +- src/data/corpus_sqlite.cpp | 2 +- src/data/default_vocab.cpp | 6 +- src/data/vocab.cpp | 4 +- src/examples/mnist/model.h | 9 +-- src/graph/node_initializers.cpp | 3 +- src/graph/node_initializers.h | 11 +++- src/layers/generic.h | 5 +- src/layers/guided_alignment.h | 4 +- src/layers/loss.cpp | 12 +--- src/layers/loss.h | 4 +- src/models/bert.h | 97 ++++++++++++++++----------------- src/models/classifier.h | 3 +- src/models/costs.h | 14 +++-- src/models/encoder_classifier.h | 7 ++- src/models/encoder_decoder.cpp | 1 + src/models/transformer.h | 6 +- src/training/graph_group.h | 4 +- src/training/validator.h | 1 + 23 files changed, 140 insertions(+), 148 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index de7b52a51..e192b995a 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -145,7 +145,7 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { "Train right-to-left model"); cli.add>("--input-types", "Provide type of input data if different than 'sequence'. " - "Possible values: sequence, labels. You need to provide one type per input.", + "Possible values: sequence, class. You need to provide one type per input.", {}); cli.add("--best-deep", "Use Edinburgh deep RNN configuration (s2s)"); @@ -200,8 +200,8 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.add("--transformer-postprocess", "Operation after each transformer layer: d = dropout, a = add, n = normalize", "dan"); - cli.add("--transformer-learned-positions", - "Use learned positional embeddings instead of trigonometric embeddings"); + cli.add("--transformer-train-positions", + "Train positional embeddings instead of using static sinusoidal embeddings"); cli.add("--bert-mask-symbol", "Masking symbol for BERT masked-LM training", "[MASK]"); cli.add("--bert-sep-symbol", "Sentence separator symbol for BERT next sentence prediction training", "[SEP]"); @@ -289,7 +289,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { cli.add("--disp-label-counts", "Display label counts when logging loss progress"); cli.add("--disp-label-index", - "Display label counts based on i-th sub-batch (-1 is last)", -1); + "Display label counts based on i-th input stream (-1 is last)", -1); cli.add("--save-freq", "Save model file every arg updates (append 't' for every arg target labels)", "10000u"); diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7d1f27544..7f51b12ba 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -55,7 +55,7 @@ SentenceTuple Corpus::next() { } else if(i > 0 && i == weightFileIdx_) { addWeightsToSentenceTuple(line, tup); } else { - addWordsToSentenceTuple(line, i, tup, addEOS_[i]); + addWordsToSentenceTuple(line, i, tup); } } diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index b35bb032b..b34bacef9 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -44,22 +44,7 @@ CorpusBase::CorpusBase(const std::vector& paths, ABORT_IF(files_.back()->empty(), "File '{}' is empty", path); } - addEOS_.resize(paths_.size(), true); - // @TODo: think if this should be checked and processed here or in a validation step in config? - auto inputTypes = options_->get>("input-types", {}); // empty list by default - ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), - "Input types are specified ({}) you need to specify one per input ({})", - inputTypes.size(), - paths_.size()); - // Currently input types affects only EOS symbol - for(int i = 0; i < inputTypes.size(); ++i) - if(inputTypes[i] == "labels") - addEOS_[i] = false; - else if(inputTypes[i] == "sequence") - addEOS_[i] = true; - else - ABORT("Unknown input type {}: {}", i, inputTypes[i]); - + initEOS(); } CorpusBase::CorpusBase(Ptr options, bool translate) @@ -74,9 +59,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) else paths_ = options_->get>("input"); - addEOS_.resize(paths_.size(), true); - // @TODo: think if this should be checked and processed here or in a validation step in config? - auto inputTypes = options_->get>("input-types", {}); // empty list by default + initEOS(); std::vector vocabPaths; if(!options_->get>("vocabs").empty()) @@ -150,19 +133,6 @@ CorpusBase::CorpusBase(Ptr options, bool translate) vocabs_.emplace_back(vocab); } } - - ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), - "Input types are specified ({}) you need to specify one per input ({})", - inputTypes.size(), - paths_.size()); - // Currently input types affects only EOS symbol - for(int i = 0; i < inputTypes.size(); ++i) - if(inputTypes[i] == "labels") - addEOS_[i] = false; - else if(inputTypes[i] == "sequence") - addEOS_[i] = true; - else - ABORT("Unknown input type {}: {}", i, inputTypes[i]); } if(translate) { @@ -222,14 +192,13 @@ CorpusBase::CorpusBase(Ptr options, bool translate) } void CorpusBase::addWordsToSentenceTuple(const std::string& line, - size_t i, - SentenceTuple& tup, - bool addEOS) const { + size_t batchIndex, + SentenceTuple& tup) const { // This turns a string in to a sequence of numerical word ids. Depending // on the vocabulary type, this can be non-trivial, e.g. when SentencePiece // is used. - Words words = vocabs_[i]->encode(line, /*addEOS =*/ addEOS, inference_); + Words words = vocabs_[batchIndex]->encode(line, /*addEOS =*/ addEOS_[batchIndex], inference_); if(words.empty()) words.push_back(0); @@ -314,5 +283,27 @@ void CorpusBase::addWeightsToBatch(Ptr batch, batch->setDataWeights(weights); } + +void CorpusBase::initEOS() { + // Labels fed into sub-batches that are just class-labels, not sequence labels do not require to + // add a EOS symbol. Hence decision to add EOS is now based on input stream positions and correspoding + // input type. + + addEOS_.resize(paths_.size(), true); + // @TODO: think if this should be checked and processed here or in a validation step in config? + auto inputTypes = options_->get>("input-types", {}); // empty list by default + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types have been specified ({}), you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + for(int i = 0; i < inputTypes.size(); ++i) + if(inputTypes[i] == "class") + addEOS_[i] = false; + else if(inputTypes[i] == "sequence") + addEOS_[i] = true; + else + ABORT("Unknown input type {}: {}", i, inputTypes[i]); +} + } // namespace data } // namespace marian diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index e925f2dd0..f1ac99aef 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -316,16 +316,16 @@ class CorpusBatch : public Batch { Ptr options) { std::vector> batches; - size_t idx = 0; + size_t batchIndex = 0; for(auto len : lengths) { - auto sb = New(batchSize, len, vocabs[idx]); + auto sb = New(batchSize, len, vocabs[batchIndex]); // set word indices to different values to avoid same hashes // rand() is OK, this does not affect state in any way std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), - [&](Word) { return rand() % vocabs[idx]->size(); }); + [&](Word) { return rand() % vocabs[batchIndex]->size(); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); - idx++; + batchIndex++; batches.push_back(sb); } @@ -543,14 +543,18 @@ class CorpusBase */ size_t alignFileIdx_{0}; + /** + * @brief Determine if EOS symbol should be added to input + */ + void initEOS(); + /** * @brief Helper function converting a line of text into words using the i-th * vocabulary and adding them to the sentence tuple. */ void addWordsToSentenceTuple(const std::string& line, - size_t i, - SentenceTuple& tup, - bool addEOS) const; + size_t batchIndex, + SentenceTuple& tup) const; /** * @brief Helper function parsing a line with word alignments and adding them * to the sentence tuple. diff --git a/src/data/corpus_nbest.cpp b/src/data/corpus_nbest.cpp index 4fd560992..328c3c0d7 100644 --- a/src/data/corpus_nbest.cpp +++ b/src/data/corpus_nbest.cpp @@ -61,9 +61,9 @@ SentenceTuple CorpusNBest::next() { "Too few lines in input {}", i); } - addWordsToSentenceTuple(lastLines_[i], i, tup, addEOS_[i]); + addWordsToSentenceTuple(lastLines_[i], i, tup); } - addWordsToSentenceTuple(curr_text, last, tup, addEOS_[last]); + addWordsToSentenceTuple(curr_text, last, tup); lastNum_ = curr_num; } diff --git a/src/data/corpus_sqlite.cpp b/src/data/corpus_sqlite.cpp index 7127aafda..cbab750eb 100644 --- a/src/data/corpus_sqlite.cpp +++ b/src/data/corpus_sqlite.cpp @@ -120,7 +120,7 @@ SentenceTuple CorpusSQLite::next() { } else if(i > 0 && i == weightFileIdx_) { addWeightsToSentenceTuple(line, tup); } else { - addWordsToSentenceTuple(line, i, tup, addEOS_[i]); + addWordsToSentenceTuple(line, i, tup); } } diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 96ae61e64..4557830c1 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -285,7 +285,7 @@ class DefaultVocab : public VocabBase { // This is a vocabulary class that does not enforce or . // This is used for class lists in a classifier. -class LabelsVocab : public DefaultVocab { +class ClassVocab : public DefaultVocab { private: virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override {} // Do nothing. }; @@ -294,8 +294,8 @@ Ptr createDefaultVocab() { return New(); } -Ptr createLabelsVocab() { - return New(); +Ptr createClassVocab() { + return New(); } } diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 97963e4d9..266b14322 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -4,7 +4,7 @@ namespace marian { Ptr createDefaultVocab(); -Ptr createLabelsVocab(); +Ptr createClassVocab(); Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); // @TODO: make each vocab peek on type @@ -16,7 +16,7 @@ Ptr createVocab(const std::string& vocabPath, Ptr options, s // check type of input, if not given, assume "sequence" auto inputTypes = options->get>("input-types", {}); std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; - return inputType == "labels" ? createLabelsVocab() : createDefaultVocab(); + return inputType == "class" ? createClassVocab() : createDefaultVocab(); } } diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 44f41e6ac..876830186 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -37,13 +37,8 @@ class MNISTCrossEntropyCost : public CostBase { // use CE loss auto loss = sum(cross_entropy(top->loss(), labels), /*axis =*/ 0); - auto labelsNum = graph->constant({1}, inits::from_value((float)vLabels.size())); - - // @TODO: simplify this - auto multiLoss = New(); - multiLoss->push_back({loss, labelsNum}); - - return multiLoss; + + return New(RationalLoss({loss, (float)vLabels.size()})); } }; diff --git a/src/graph/node_initializers.cpp b/src/graph/node_initializers.cpp index 84f60044f..da38d2ae8 100755 --- a/src/graph/node_initializers.cpp +++ b/src/graph/node_initializers.cpp @@ -157,7 +157,8 @@ NodeInitializer from_item(const io::Item& item) { } } -NodeInitializer positions(int start) { +// Computes Google's sinusoidal position embeddings +NodeInitializer sinusoidalPositionEmbeddings(int start) { return [start](Tensor t) { int dimEmb = t->shape()[-1]; int dimWords = t->size() / dimEmb; diff --git a/src/graph/node_initializers.h b/src/graph/node_initializers.h index c7576cbb2..254f02ae2 100755 --- a/src/graph/node_initializers.h +++ b/src/graph/node_initializers.h @@ -53,13 +53,18 @@ NodeInitializer from_word2vec(const std::string& file, bool normalize = false); /** - * Computes Google's trigonometric positional embeddings + * Computes Google's sinusoidal position embeddings * starting from position 'start' taking into account * batch and time dimensions of the tensor. * - * Expected layout tensor layout {time, batch, model} + * Expected tensor layout {-2: time, -1: model} + * + * Usually gets later reshaped to {time, 1, model} and + * added with a broadcast to learned embeddings. Positional + * embeddings are the same for each batch entry and change + * over time. */ -NodeInitializer positions(int start); +NodeInitializer sinusoidalPositionEmbeddings(int start); } // namespace inits diff --git a/src/layers/generic.h b/src/layers/generic.h index 7f4164bee..5a9435084 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -202,9 +202,8 @@ class Embedding : public LayerBase, public IEmbeddingLayer { int dimBatch = (int)subBatch->batchSize(); int dimEmb = E_->shape()[-1]; int dimWords = (int)subBatch->batchWidth(); - // @TODO: merge this with below. Currently can't only due to the extra beam dimension - auto chosenEmbeddings = rows(E_, subBatch->data()); - auto batchEmbeddings = reshape(chosenEmbeddings, { dimWords, dimBatch, dimEmb }); + + auto batchEmbeddings = apply(subBatch->data(), { dimWords, dimBatch, dimEmb }); auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, inits::from_vector(subBatch->mask())); return std::make_tuple(batchEmbeddings, batchMask); diff --git a/src/layers/guided_alignment.h b/src/layers/guided_alignment.h index 451647826..8c912ddfc 100755 --- a/src/layers/guided_alignment.h +++ b/src/layers/guided_alignment.h @@ -37,9 +37,7 @@ static inline RationalLoss guidedAlignmentCost(Ptr graph, // Create label node, also weigh by scalar so labels and cost are in the same domain. // Fractional label counts are OK - Expr labels = graph->constant({1}, inits::from_value(guidedScalar * numLabels)); // @TODO: introduce graph->value(...) ? - - return RationalLoss(alignmentLoss, labels); + return RationalLoss(alignmentLoss, guidedScalar * numLabels); } } // namespace marian diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 06f8dc59d..86cd99d19 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -6,15 +6,8 @@ namespace marian { Ptr newLoss(Ptr options, bool inference) { float smoothing = inference ? 0.f : options->get("label-smoothing"); std::string costType = options->get("cost-type", "ce-mean"); - if(costType == "ce-mean" || costType == "cross-entropy") { - return New(smoothing); - } else if(costType == "ce-mean-words") { - return New(smoothing); - } else if(costType == "ce-sum") { - return New(smoothing); - } else if(costType == "perplexity") { - return New(smoothing); - } else if(costType == "ce-rescore") { + + if(costType == "ce-rescore") { return New(); } else if(costType == "ce-rescore-mean") { ABORT("Check me"); @@ -24,6 +17,7 @@ Ptr newLoss(Ptr options, bool inference) { } } +// see loss.h for detailed explanations of each class Ptr newMultiLoss(Ptr options) { std::string multiLossType = options->get("multi-loss-type", "sum"); if(multiLossType == "sum") // sum of sums diff --git a/src/layers/loss.h b/src/layers/loss.h index 850dc6b78..a82ca60fd 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -18,8 +18,8 @@ namespace marian { */ class RationalLoss { protected: - Expr loss_; - Expr labels_; + Expr loss_; // numerator + Expr labels_; // denominator RationalLoss() = default; // protected diff --git a/src/models/bert.h b/src/models/bert.h index 0591b3c08..72d744e9c 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -7,9 +7,9 @@ namespace marian { -/** +/** * This file contains nearly all BERT-related code and adds BERT-funtionality - * on top of existing classes like TansformerEncoder and Classifier. + * on top of existing classes like TansformerEncoder and Classifier. */ namespace data { @@ -17,15 +17,13 @@ namespace data { /** * BERT-specific mini-batch that computes masking for Masked LM training. * Expects symbols [MASK], [SEP], [CLS] to be present in vocabularies unless - * other symbols are specified on the command line. - * + * other symbols are specified in the config. + * * This takes a normal CorpusBatch and extends it with additional data. Luckily * all the BERT-functionality can be inferred from a CorpusBatch alone. */ class BertBatch : public CorpusBatch { private: - std::mt19937& eng_; - std::vector maskedPositions_; std::vector maskedWords_; std::vector sentenceIndices_; @@ -38,23 +36,24 @@ class BertBatch : public CorpusBatch { std::unique_ptr> randomWord_; // Selects a random integer between 0 and 99 - std::unique_ptr> randomPercent_; + std::unique_ptr> randomPercent_; // Word ids of words that should not be masked, e.g. separators, padding std::unordered_set dontMask_; - // Masking function - Word maskOut(Word word, Word mask) { + // Masking function, i.e. replaces a chosen word with either + // a [MASK] symbol, itself or a random word + Word maskOut(Word word, Word mask, std::mt19937& engine) { auto subBatch = subBatches_.front(); // @TODO: turn those threshold into parameters, adjustable from command line - int r = (*randomPercent_)(eng_); - if (r < 10) { // for 10% of cases return same word + float r = (*randomPercent_)(engine); + if (r < 0.1f) { // for 10% of cases return same word return word; - } else if (r < 20) { // for 10% return random word - Word randWord = (*randomWord_)(eng_); - if(dontMask_.count(randWord) > 0) // the random word is a forbidden word - return mask; // hence return mask symbol + } else if (r < 0.2f) { // for 10% return random word + Word randWord = (*randomWord_)(engine); + if(dontMask_.count(randWord) > 0) // some words, e.g. [CLS] or , may not be used as random words + return mask; // for those, return the mask symbol instead else return randWord; // else return the random word } else { // for 80% of words apply mask symbol @@ -72,36 +71,38 @@ class BertBatch : public CorpusBatch { const std::string& maskSymbol, const std::string& sepSymbol, const std::string& clsSymbol) - : CorpusBatch(*batch), eng_(engine), + : CorpusBatch(*batch), maskSymbol_(maskSymbol), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { + // BERT expects a textual first stream and a second stream with class labels auto subBatch = subBatches_.front(); + const auto& vocab = *subBatch->vocab(); // Initialize to sample random vocab id - randomWord_.reset(new std::uniform_int_distribution(0, subBatch->vocab()->size())); + randomWord_.reset(new std::uniform_int_distribution(0, vocab.size())); // Intialize to sample random percentage - randomPercent_.reset(new std::uniform_int_distribution(0, 100)); + randomPercent_.reset(new std::uniform_real_distribution(0.f, 1.f)); auto& words = subBatch->data(); // Get word id of special symbols - Word maskId = (*subBatch->vocab())[maskSymbol_]; - Word clsId = (*subBatch->vocab())[clsSymbol_]; - Word sepId = (*subBatch->vocab())[sepSymbol_]; + Word maskId = vocab[maskSymbol_]; + Word clsId = vocab[clsSymbol_]; + Word sepId = vocab[sepSymbol_]; - ABORT_IF(maskId == subBatch->vocab()->getUnkId(), + ABORT_IF(maskId == vocab.getUnkId(), "BERT masking symbol {} not found in vocabulary", maskSymbol_); - ABORT_IF(sepId == subBatch->vocab()->getUnkId(), + ABORT_IF(sepId == vocab.getUnkId(), "BERT separator symbol {} not found in vocabulary", sepSymbol_); - ABORT_IF(clsId == subBatch->vocab()->getUnkId(), + ABORT_IF(clsId == vocab.getUnkId(), "BERT class symbol {} not found in vocabulary", clsSymbol_); dontMask_.insert(clsId); // don't mask class token dontMask_.insert(sepId); // don't mask separator token - dontMask_.insert(subBatch->vocab()->getEosId()); // don't mask + dontMask_.insert(vocab.getEosId()); // don't mask // it's ok to mask std::vector selected; @@ -109,13 +110,13 @@ class BertBatch : public CorpusBatch { for(int i = 0; i < words.size(); ++i) // collect words among which we will mask if(dontMask_.count(words[i]) == 0) // do not add indices of special words selected.push_back(i); - std::shuffle(selected.begin(), selected.end(), eng_); // randomize positions + std::shuffle(selected.begin(), selected.end(), engine); // randomize positions selected.resize(std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices for(int i : selected) { - maskedPositions_.push_back(i); // where is the original word? - maskedWords_.push_back(words[i]); // what is the original word? - words[i] = maskOut(words[i], maskId); // mask that position + maskedPositions_.push_back(i); // where is the original word? + maskedWords_.push_back(words[i]); // what is the original word? + words[i] = maskOut(words[i], maskId, engine); // mask that position } int dimBatch = subBatch->batchSize(); @@ -148,15 +149,15 @@ class BertBatch : public CorpusBatch { /** * BERT-specific version of EncoderClassifier, mostly here to automatically convert a - * CorpusBatch to BertBatch. + * CorpusBatch to BertBatch. */ -class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { +class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { // @TODO: this random engine is not being serialized right now public: BertEncoderClassifier(Ptr options) : EncoderClassifier(options) {} std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { - // intercept batch and anotate with BERT-specific concepts + // intercept batch and annotate with BERT-specific concepts auto bertBatch = New(batch, eng_, opt("bert-masking-fraction", 0.15f), // 15% by default according to paper @@ -175,7 +176,7 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { /** * BERT-specific modifications to EncoderTransformer * Actually all that is needed is to intercept the creation of special embeddings, - * here sentence embeddings for sentence A and B. + * here sentence embeddings for sentence A and B. */ class BertEncoder : public EncoderTransformer { public: @@ -201,9 +202,9 @@ class BertEncoder : public EncoderTransformer { .construct(graph_); signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); } else { - // @TODO: factory for postional embeddings? - // trigonometric positions, no backprob - auto sentenceEmbeddingsExpr = graph_->constant({2, dimEmb}, inits::positions(0)); + // @TODO: factory for positional embeddings? + // constant sinusoidal position embeddings, no backprob + auto sentenceEmbeddingsExpr = graph_->constant({2, dimEmb}, inits::sinusoidalPositionEmbeddings(0)); signal = rows(sentenceEmbeddingsExpr, bertBatch->bertSentenceIndices()); signal = reshape(signal, {dimWords, dimBatch, dimEmb}); } @@ -212,18 +213,18 @@ class BertEncoder : public EncoderTransformer { } virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { - bool learnedPosEmbeddings = opt("transformer-learned-positions", true); - input = addPositionalEmbeddings(input, start, learnedPosEmbeddings); - input = addSentenceEmbeddings(input, batch, learnedPosEmbeddings); // @TODO: separately set learnable pos and sent embeddings + bool trainPosEmbeddings = opt("transformer-train-positions", true); + input = addPositionalEmbeddings(input, start, trainPosEmbeddings); + input = addSentenceEmbeddings(input, batch, trainPosEmbeddings); // @TODO: separately set learnable pos and sent embeddings return input; } }; /** - * BERT-specific classifier + * BERT-specific classifier * Can be used for next sentence prediction task or other fine-tuned down-stream tasks * Does not actually need a BertBatch, works with CorpusBatch. - * + * * @TODO: This is in fact so generic that we might move it out of here as the typical classifier implementation */ class BertClassifier : public ClassifierBase { @@ -266,9 +267,8 @@ class BertClassifier : public ClassifierBase { /** * This is a model that pretrains BERT for classification. - * This is also a Classifier, but compared to the one above needs the BERT-specific information from BertBatch - * as this is self-generating its labels from the source. Labels are dynamically created as complements of the - * masking process. + * This is also a Classifier, but compared to the BertClassifier above needs the BERT-specific information from BertBatch + * as this is self-generating its labels from the source. Labels are dynamically created as complements of the masking process. */ class BertMaskedLM : public ClassifierBase { public: @@ -277,7 +277,7 @@ class BertMaskedLM : public ClassifierBase { Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { Ptr bertBatch = std::dynamic_pointer_cast(batch); - ABORT_IF(!bertBatch, "Batch could not be converted to batch for BERT training"); + ABORT_IF(!bertBatch, "Batch must be BertBatch for BERT training"); ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); auto context = encoderStates[0]->getContext(); @@ -289,7 +289,7 @@ class BertMaskedLM : public ClassifierBase { int dimBatch = context->shape()[-2]; int dimTime = context->shape()[-3]; - auto maskedEmbeddings = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out; + auto maskedEmbeddings = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out int dimVoc = opt>("dim-vocabs")[batchIndex_]; @@ -302,15 +302,14 @@ class BertMaskedLM : public ClassifierBase { ("dim", dimVoc); // layerOut.tieTransposed("Wemb"); // We are a BERT model, hence tie with input, @TODO: check if this is actually what Google does - // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context auto output = mlp::mlp() // - .push_back(layerTanh) // + .push_back(layerTanh) // .push_back(layerOut) // .construct(graph); - auto logits = output->apply(maskedEmbeddings); + auto logits = output->apply(maskedEmbeddings); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] auto state = New(); state->setLogProbs(logits); diff --git a/src/models/classifier.h b/src/models/classifier.h index 86e01841e..dae9bcbb2 100644 --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -8,7 +8,7 @@ namespace marian { /** - * Simple base class for Classifiers + * Simple base class for Classifiers to be used in EncoderClassifier framework * Currently only implementations are in bert.h */ class ClassifierBase { @@ -34,6 +34,7 @@ class ClassifierBase { return options_->get(key); } + // Should be used to clear any batch-wise temporary objects if present virtual void clear() = 0; }; diff --git a/src/models/costs.h b/src/models/costs.h index 40d490b53..bfd62da7b 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -82,14 +82,14 @@ class EncoderDecoderCE : public CostBase { auto alignmentLoss = guidedAlignmentCost(graph, corpusBatch, options_, attention); multiLoss->push_back(alignmentLoss); - - return multiLoss; - } else { - return multiLoss; } + + return multiLoss; + } }; +// Wraps an EncoderClassifier so it can produce a cost from raw logits. @TODO: Needs refactoring class EncoderClassifierCE : public CostBase { protected: Ptr options_; @@ -284,7 +284,8 @@ inline Ptr add_cost(Ptr encdec, else return New(encdec, New()); case usage::raw: - default: return encdec; + default: + return encdec; } } @@ -298,7 +299,8 @@ inline Ptr add_cost(Ptr enccls, case usage::translation: ABORT("Classifier cannot be used for translation"); case usage::raw: - default: return enccls; + default: + return enccls; } } diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 43ed9e1ad..bc3d8f9f8 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -12,8 +12,8 @@ namespace marian { /** * Combines sequence encoders with generic classifiers * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. - * Already has support for multi-objective training. - * + * Already has support for multi-objective training. + * * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier * multi-objective training. */ @@ -128,6 +128,7 @@ class EncoderClassifier : public EncoderClassifierBase { modelFeatures_.insert("transformer-decoder-autoreg"); modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); + modelFeatures_.insert("transformer-train-positions"); } virtual Ptr getOptions() override { return options_; } @@ -194,7 +195,7 @@ class EncoderClassifier : public EncoderClassifierBase { std::vector> classifierStates; for(auto& classifier : classifiers_) classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); - + return classifierStates; } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 70e71c836..c4b96cc30 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -44,6 +44,7 @@ EncoderDecoder::EncoderDecoder(Ptr options) modelFeatures_.insert("transformer-decoder-autoreg"); modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); + modelFeatures_.insert("transformer-train-positions"); } std::vector>& EncoderDecoder::getEncoders() { diff --git a/src/models/transformer.h b/src/models/transformer.h index b3cb8e45f..8f54a2213 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -76,7 +76,7 @@ class Transformer : public EncoderOrDecoderBase { embeddings = embeddings + signal; } else { auto signal = graph_->constant({dimWords, 1, dimEmb}, - inits::positions(start)); + inits::sinusoidalPositionEmbeddings(start)); // according to paper embeddings are scaled up by \sqrt(d_m) embeddings = std::sqrt((float)dimEmb) * embeddings; embeddings = embeddings + signal; @@ -86,8 +86,8 @@ class Transformer : public EncoderOrDecoderBase { } virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr /*batch*/ = nullptr) const { - bool learnedPosEmbeddings = opt("transformer-learned-positions", false); - return addPositionalEmbeddings(input, start, learnedPosEmbeddings); + bool trainPosEmbeddings = opt("transformer-train-positions", false); + return addPositionalEmbeddings(input, start, trainPosEmbeddings); } Expr triangleMask(int length) const { diff --git a/src/training/graph_group.h b/src/training/graph_group.h index e9ad041a8..404789a6f 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -71,11 +71,11 @@ class GraphGroup { size_t maxLength = options_->get("max-length"); maxLength = (size_t)(std::ceil(maxLength / (float)step) * step); - // restrict maximum length for labels to 1 + // restrict maximum length for class labels to 1 std::vector localMaxes(numFiles, maxLength); auto inputTypes = options_->get>("input-types", {}); for(int i = 0; i < inputTypes.size(); ++i) - if(inputTypes[i] == "labels") + if(inputTypes[i] == "class") localMaxes[i] = 1; diff --git a/src/training/validator.h b/src/training/validator.h index bce37da28..358d3e4a5 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -216,6 +216,7 @@ class CrossEntropyValidator : public Validator { } }; +// Used for validating with classifiers. Compute prediction accuary versus groundtruth for a set of classes class AccuracyValidator : public Validator { public: AccuracyValidator(std::vector> vocabs, Ptr options) From a38d82217b4b29eed4a93bce9d8e943c3ddfdead Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 09:31:57 -0800 Subject: [PATCH 153/838] (comments) --- src/layers/generic.cpp | 8 +++++++- src/layers/loss.cpp | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index e99a5410f..347d4a833 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -33,6 +33,12 @@ namespace marian { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries + // Factor normalization + // - f = h * E' + b // hidden state projected onto all factors + // - f: [B x U] // U = number of factor units + // - normalization terms Z: [B x G] // G = number of groups + // - factor group matrix F: [U x G] // [u,g] 1 if factor u is in group g (one-hot); computed once + // - z = f - Z * F' = affine(Z, F, f, transB=true, alpha=-1) // This does it with only one extra copy EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { std::vector paths = options->get>("embedding-factors"); ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); @@ -96,7 +102,7 @@ namespace marian { groupCounts[g]++; } for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members are not consecutive in the factor vocabulary", groupPrefixes[g]); + //ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members are not consecutive in the factor vocabulary", groupPrefixes[g]); LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); } diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 03b796823..d9102456a 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -31,6 +31,7 @@ Expr LossBase::getCrossEntropy(Expr logits, if(smoothing_ > 0) { // @TODO: add this to CE kernels instead auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); + //auto ceq = mean(logits, /*axis=*/ -1) - logsum(logits), /*axis=*/ -1); ce = (1 - smoothing_) * ce - smoothing_ * ceq; } From 4cc6219f185fa91a02e33fbbec94fa4e4348001d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 09:52:12 -0800 Subject: [PATCH 154/838] towards unifying reduction operators --- src/graph/expression_operators.cpp | 9 ++- src/graph/node_operators_unary.h | 90 +++++++++++------------------- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 4694351b2..9e2772bcd 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -278,11 +278,16 @@ Expr select(Expr a, const std::vector& indices, int axis) { } Expr sum(Expr a, int ax) { - return Expression(a, ax); + return Expression(a, ax, ReduceNodeOpCode::sum); +} + +// log(sum(exp(a))) +Expr logSumExp(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::logSumExp); } Expr mean(Expr a, int ax) { - return Expression(a, ax); + return Expression(a, ax, ReduceNodeOpCode::mean); } Expr scalar_product(Expr a, Expr b, int ax) { diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 462d81cf7..1716292c9 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -412,20 +412,40 @@ struct LogSoftmaxNodeOp : public UnaryNodeOp { const std::string type() override { return "logsoftmax"; } }; +enum class ReduceNodeOpCode { + sum, mean, min, max, logSumExp +}; + struct SumNodeOp : public UnaryNodeOp { int axis_; + ReduceNodeOpCode opCode_; - SumNodeOp(Expr a, int axis) : UnaryNodeOp(a, newShape(a, axis)) {} + SumNodeOp(Expr a, int axis, ReduceNodeOpCode opCode) + : UnaryNodeOp(a, newShape(a, axis)), opCode_(opCode){} NodeOps forwardOps() override { using namespace functional; - return {NodeOp(Reduce(_1, val_, child(0)->val()))}; + switch (opCode_) { + case ReduceNodeOpCode::sum: + return {NodeOp(Reduce(_1, val_, child(0)->val()))}; + case ReduceNodeOpCode::mean: + return {NodeOp(Reduce(_1, (float)val_->shape().elements() / (float)child(0)->shape().elements(), val_, child(0)->val()))}; + default: + ABORT("Unexpected reduction op-code {}", (int)opCode_); + } } NodeOps backwardOps() override { using namespace functional; - return {NodeOp(Add(_1, child(0)->grad(), adj_))}; + switch (opCode_) { + case ReduceNodeOpCode::sum: + return {NodeOp(Add(_1, child(0)->grad(), adj_))}; + case ReduceNodeOpCode::mean: + return {NodeOp(Add(_1, (float)val_->shape().elements() / (float)child(0)->shape().elements(), child(0)->grad(), adj_))}; + default: + ABORT("Unexpected reduction op-code {}", (int)opCode_); + } } Shape newShape(Expr a, int axis) { @@ -436,59 +456,16 @@ struct SumNodeOp : public UnaryNodeOp { return shape; } - const std::string type() override { return "sum"; } - - const std::string color() override { return "orange"; } - - virtual size_t hash() override { - if(!hash_) { - hash_ = NaryNodeOp::hash(); - util::hash_combine(hash_, axis_); + const std::string type() override { + switch (opCode_) { + case ReduceNodeOpCode::sum: + return "sum"; + case ReduceNodeOpCode::mean: + return "mean"; + default: + ABORT("Unexpected reduction op-code {}", (int)opCode_); } - return hash_; - } - - virtual bool equal(Expr node) override { - if(!NaryNodeOp::equal(node)) - return false; - Ptr cnode = std::dynamic_pointer_cast(node); - if(!cnode) - return false; - if(axis_ != cnode->axis_) - return false; - return true; } -}; - -struct MeanNodeOp : public UnaryNodeOp { - int axis_; - - MeanNodeOp(Expr a, int axis) : UnaryNodeOp(a, newShape(a, axis)) {} - - NodeOps forwardOps() override { - using namespace functional; - int left = child(0)->shape().elements() / val_->shape().elements(); - float scale = 1.f / left; - - return {NodeOp(Reduce(_1, scale, val_, child(0)->val()))}; - } - - NodeOps backwardOps() override { - using namespace functional; - int left = child(0)->shape().elements() / val_->shape().elements(); - float scale = 1.f / left; - - return {NodeOp(Add(_1, scale, child(0)->grad(), adj_))}; - } - - Shape newShape(Expr a, int axis) { - Shape shape = a->shape(); - axis_ = shape.axis(axis); - shape.set(axis_, 1); - return shape; - } - - const std::string type() override { return "mean"; } const std::string color() override { return "orange"; } @@ -496,6 +473,7 @@ struct MeanNodeOp : public UnaryNodeOp { if(!hash_) { hash_ = NaryNodeOp::hash(); util::hash_combine(hash_, axis_); + util::hash_combine(hash_, (int)opCode_); } return hash_; } @@ -503,10 +481,10 @@ struct MeanNodeOp : public UnaryNodeOp { virtual bool equal(Expr node) override { if(!NaryNodeOp::equal(node)) return false; - Ptr cnode = std::dynamic_pointer_cast(node); + Ptr cnode = std::dynamic_pointer_cast(node); if(!cnode) return false; - if(axis_ != cnode->axis_) + if(axis_ != cnode->axis_ || opCode_ != cnode->opCode_) return false; return true; } From 4a8aa185db03073b8b7316f5bdf597c316b911cf Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 18 Jan 2019 10:11:40 -0800 Subject: [PATCH 155/838] address more comments, part 2 --- src/examples/mnist/model.h | 2 +- src/layers/loss.h | 62 +++++++++++++++------ src/models/encoder_decoder.h | 4 +- src/models/transformer.h | 8 ++- src/models/transformer_factory.h | 1 + src/rescorer/rescorer.h | 28 +++++----- src/tensors/cpu/tensor_operators.cpp | 31 +++++------ src/training/graph_group.h | 2 +- src/training/graph_group_async.cpp | 7 +-- src/training/graph_group_multinode.cpp | 3 +- src/training/graph_group_multinode_sync.cpp | 2 +- src/training/scheduler.h | 12 ++-- 12 files changed, 98 insertions(+), 64 deletions(-) diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 876830186..788df5810 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -71,7 +71,7 @@ class MnistFeedForwardNet : public ModelBase { Ptr batch, bool /*clean*/ = false) override { - auto loss = construct(graph, batch, inference_); + auto loss = construct(graph, batch, inference_); // @TODO: unify nomenclature, e.g. rather use apply auto labels = graph->constant({(int)batch->size(), 1}, inits::from_value(1.f)); return New(loss, labels); diff --git a/src/layers/loss.h b/src/layers/loss.h index a82ca60fd..2975864ca 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -29,7 +29,7 @@ class RationalLoss { RationalLoss(Expr loss, float labels) : loss_(loss), - labels_(loss->graph()->constant({1}, inits::from_value(labels))) {} + labels_(constant_like(loss, inits::from_value(labels))) {} RationalLoss(const RationalLoss& other) : loss_(other.loss_), labels_(other.labels_) {} @@ -64,6 +64,8 @@ class RationalLoss { return labels_->val()->scalar(); } + // @TODO: add a funtion for returning maybe ratio? + size_t size() const { ABORT_IF(!labels_, "Labels have not been defined"); return labels_->shape().elements(); @@ -90,8 +92,14 @@ struct StaticLoss { labels = labels + other.labels; return *this; } + + void reset() { + loss = 0.f; + labels = 0.f; + } }; +// @TODO: overthink interface /** * Base class for multi-objective losses which is a list of RationalLoss * but also defines how to accumulate that list into a single RationalLoss @@ -214,7 +222,7 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { * L = (1/N sum_i^N L_i + 1/M sum_j^M L_j) = (sum_i^N L_i + N/M sum_j^M L_j) / N * * We set labels to 1. During reporting, we would see the same numbers, but gradients - * are scaled diffrently which may result in different learning curves. + * are scaled differently which may result in different learning curves. */ class MeanMultiRationalLoss : public MultiRationalLoss { private: @@ -243,11 +251,11 @@ class MeanMultiRationalLoss : public MultiRationalLoss { Ptr newMultiLoss(Ptr options); //***********************************************************************************// -// This needs some to be refactored. Currently easiest route for backwards compat, but +// This needs to be refactored. Currently easiest route for backwards compat, but // still feels somewhat hacky. /** - * Computes loss per label and then reduces to RationalLoss + * Computes loss per given groundtruth label and then reduces to RationalLoss */ class LabelwiseLoss { protected: @@ -256,6 +264,7 @@ class LabelwiseLoss { virtual Expr compute(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) = 0; + // label counts are available, reduce together with loss to obtain counts RationalLoss reduce(Expr loss, Expr labels) { ABORT_IF(!loss, "Loss has not been computed"); ABORT_IF(!labels, "Labels have not been computed"); @@ -270,6 +279,19 @@ class LabelwiseLoss { return RationalLoss(lossSum, labelsSum); } + // label counts are not available, assume every element of tensor corresponds to label count 1 + RationalLoss reduce(Expr loss) { + ABORT_IF(!loss, "Loss has not been computed"); + + Expr lossSum = loss; + for(int i = 0; i < axes_.size(); ++i) + lossSum = sum(lossSum, axes_[i]); + + // reduction factor tells how over how many labels we reduced in total. + float reducedLabels = (float)loss->shape().elements() / (float)lossSum->shape().elements(); + return RationalLoss(lossSum, reducedLabels); + } + public: LabelwiseLoss(const std::vector& axes) : axes_(axes) { } @@ -278,10 +300,10 @@ class LabelwiseLoss { Expr mask = nullptr, Expr labelWeights = nullptr) { Expr loss = compute(logits, labelIndices, mask, labelWeights); - Expr labels = mask ? mask // mask can be used as element-wise label count with broadcasting - : constant_like(loss, inits::ones); // we have no mask, assume all items are labels - - return reduce(loss, labels); + if(mask) + return reduce(loss, mask); // mask can be used as element-wise label count with broadcasting + else + return reduce(loss); // we have no mask, assume all items are labels } }; @@ -290,25 +312,33 @@ class LabelwiseLoss { */ class CrossEntropyLoss : public LabelwiseLoss { public: - CrossEntropyLoss(float smoothing) + CrossEntropyLoss(float labelSmoothing) : LabelwiseLoss(/*axes=*/{-2, -3}), // cross-entropy already reduces over axis -1 - smoothing_(smoothing) {} + labelSmoothing_(labelSmoothing) {} - CrossEntropyLoss(const std::vector& axes, float smoothing) + CrossEntropyLoss(const std::vector& axes, float labelSmoothing) : LabelwiseLoss(axes), // cross-entropy already reduces over axis -1 - smoothing_(smoothing) {} + labelSmoothing_(labelSmoothing) {} protected: - float smoothing_; + float labelSmoothing_; // interpolation factor for label smoothing, see below virtual Expr compute(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { Expr ce = cross_entropy(logits, labelIndices); - if(smoothing_ > 0) { + if(labelSmoothing_ > 0) { // @TODO: add this to CE kernels instead + + // Label smoothing (see https://arxiv.org/pdf/1512.00567.pdf, section 7) + // We compute smoothed H(q',p) = (1 - eps) * H(q,p) + eps * H(u,p) where H(q,p) is the normal cross-entropy + // and H(u,p) penalizes deviation of p from u, u being uniform distribution over vocab V => u_v = 1/|V|. + // H(u,p) = - \sum_{v \in V} u_v * \log p_v = - 1/|V| \sum_{v \in V} \log \softmax_v => -mean(logsoftmax(logits)) + // ceq = -H(u,p) - avoid one kernel call by negating in the interpolation below Expr ceq = mean(logsoftmax(logits), /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; + + // H(q',p) = (1 - eps) * H(q,p) - eps * -H(u,p) + ce = (1 - labelSmoothing_) * ce - labelSmoothing_ * ceq; } if(mask) @@ -332,7 +362,7 @@ class RescorerLoss : public CrossEntropyLoss { virtual RationalLoss apply(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { auto ce = CrossEntropyLoss::apply(logits, labelIndices, mask, labelWeights); - return RationalLoss(-ce.loss(), ce.labels()); // we report logprobs, hence negate + return RationalLoss(ce.loss(), ce.labels()); } }; diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index a33de385f..b55e8bd1d 100644 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -34,7 +34,9 @@ class EncoderDecoderBase : public models::ModelBase { virtual Ptr build(Ptr graph, Ptr batch, - bool clearGraph = true) = 0; virtual Ptr startState(Ptr graph, + bool clearGraph = true) = 0; + + virtual Ptr startState(Ptr graph, Ptr batch) = 0; virtual Ptr step(Ptr graph, diff --git a/src/models/transformer.h b/src/models/transformer.h index 8f54a2213..b29ffc052 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -57,10 +57,14 @@ class Transformer : public EncoderOrDecoderBase { int maxLength = opt("max-length"); // Hack for translating with length longer than trained embeddings + // We check if the embedding matrix "Wpos" already exist so we can + // check the number of positions in that loaded parameter. + // We then have to restict the maximum length to the maximum positon + // and positions beyond this will be the maximum position. Expr seenEmb = graph_->get("Wpos"); int numPos = seenEmb ? seenEmb->shape()[-2] : maxLength; - auto posEmbFactory = embedding() + auto embeddingLayer = embedding() ("prefix", "Wpos") // share positional embeddings across all encoders/decorders ("dimVocab", numPos) ("dimEmb", dimEmb) @@ -72,7 +76,7 @@ class Transformer : public EncoderOrDecoderBase { positions[i] = i; // @TODO : test if embeddings should be scaled here too! - auto signal = posEmbFactory->apply(positions, {dimWords, 1, dimEmb}); + auto signal = embeddingLayer->apply(positions, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; } else { auto signal = graph_->constant({dimWords, 1, dimEmb}, diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h index 16ce91bdc..c2a7e13bb 100755 --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -9,6 +9,7 @@ //#include "layers/factory.h" namespace marian { +// @TODO: find out why static is required here to get to compile static Ptr NewEncoderTransformer(Ptr options); static Ptr NewDecoderTransformer(Ptr options); } // namespace marian diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 64ed3cde2..2c145c002 100644 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -105,7 +105,7 @@ class Rescore : public ModelTask { std::string summary = summarize ? options_->get("summary") : "cross-entropy"; - float sumCost = 0; + float sumLoss = 0; size_t sumWords = 0; size_t sumSamples = 0; size_t batchId = 0; @@ -115,7 +115,7 @@ class Rescore : public ModelTask { ThreadPool pool(graphs_.size(), graphs_.size()); for(auto batch : *batchGenerator) { - auto task = [=, &sumCost, &sumWords, &sumSamples, &smutex](size_t id) { + auto task = [=, &sumLoss, &sumWords, &sumSamples, &smutex](size_t id) { thread_local Ptr graph; thread_local Ptr builder; @@ -127,12 +127,12 @@ class Rescore : public ModelTask { // @TODO: normalize by length as in normalize // Once we have Frank's concept of ce-sum with sample size by words we will return a pair // here which will make it trivial to report all variants. - auto costNode = builder->build(graph, batch); + auto dynamicLoss = builder->build(graph, batch); graph->forward(); std::vector scores; - costNode->loss(scores); + dynamicLoss->loss(scores); // soft alignments for each sentence in the batch std::vector aligns(batch->size()); @@ -142,13 +142,15 @@ class Rescore : public ModelTask { std::unique_lock lock(smutex); for(auto s : scores) - sumCost += s; + sumLoss += s; sumWords += batch->back()->batchWords(); sumSamples += batch->size(); if(!summarize) { for(size_t i = 0; i < batch->size(); ++i) { - output->Write((long)batch->getSentenceIds()[i], scores[i], aligns[i]); + output->Write((long)batch->getSentenceIds()[i], + -1.f * scores[i], // report logProb while score is CE, hence negate + aligns[i]); } } @@ -168,22 +170,22 @@ class Rescore : public ModelTask { } if(normalize) { - LOG(info, "Total normalized log probs {} : Total sentences {} : Total words {}", sumCost, sumSamples, sumWords); + LOG(info, "Total normalized log probs {} : Total sentences {} : Total words {}", sumLoss, sumSamples, sumWords); LOG(warn, "Sum of normalized log probs is a sum of averages"); } else { - LOG(info, "Total log probs {} : Total sentences {} : Total words {}", sumCost, sumSamples, sumWords); + LOG(info, "Total log probs {} : Total sentences {} : Total words {}", sumLoss, sumSamples, sumWords); } - if(summarize) { + if(summarize) { // @TODO: use one function from loss float cost = 0; if(summary == "perplexity") - cost = std::exp(-(float)sumCost / (float)sumWords); + cost = std::exp(sumLoss / (float)sumWords); else if(summary == "ce-sum") - cost = -sumCost; + cost = sumLoss; else if(summary == "ce-mean-words") - cost = -(float)sumCost / (float)sumWords; + cost = sumLoss / (float)sumWords; else - cost = -sumCost / sumSamples; + cost = sumLoss / sumSamples; LOG(info, "Reporting {} summary", summary); std::cout << cost << std::endl; diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index 4cc5daf0d..e7fefdcfe 100755 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -661,8 +661,8 @@ void GRUFastBackward(std::vector outputs, } } -void CrossEntropyPick(Tensor out, Tensor in, Tensor pick) { - matchOrAbort(pick->type()); +void CrossEntropyPick(Tensor out, Tensor in, Tensor labelIndices) { + matchOrAbort(labelIndices->type()); // Shape& outShape = out_->shape(); Shape& inShape = in->shape(); @@ -685,31 +685,28 @@ void CrossEntropyPick(Tensor out, Tensor in, Tensor pick) { sum += std::exp(sp[i] - max); } - // cross-entropy - IndexType i = pick->data()[j]; + // Groundtruth label index + IndexType i = labelIndices->data()[j]; // This appears to be safe i.e. that i >= 0 && i < cols is known out->data()[j] = std::log(sum) - sp[i] + max; } } -void CrossEntropyPickBackward(Tensor out_, - Tensor adj_, - Tensor a, - Tensor pick_) { +void CrossEntropyPickBackward(Tensor out, + Tensor adj, + Tensor in, + Tensor labelIndices) { - matchOrAbort(pick_->type()); - float* out = out_->data(); - Shape& outShape = out_->shape(); - const float* adj = adj_->data(); - const float* in = a->data(); + matchOrAbort(labelIndices->type()); + Shape& outShape = out->shape(); int rows = outShape.elements() / outShape.back(); int cols = outShape.back(); #pragma omp parallel for for(int j = 0; j < rows; ++j) { - const float* sp = in + j * cols; - float* so = out + j * cols; + const float* sp = in->data() + j * cols; + float* so = out->data() + j * cols; float max = sp[0]; for(int i = 1; i < cols; ++i) { @@ -723,8 +720,8 @@ void CrossEntropyPickBackward(Tensor out_, // cross-entropy for(int i = 0; i < cols; ++i) { - float sub = (float)(i == (int)pick_->data()[j]); - so[i] += adj[j] * (std::exp(sp[i] - max) / sum - sub); + float sub = (float)(i == (int)labelIndices->data()[j]); // delta, true if label index and column index match + so[i] += adj->data()[j] * (std::exp(sp[i] - max) / sum - sub); } } } diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 404789a6f..f4617d782 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -71,7 +71,7 @@ class GraphGroup { size_t maxLength = options_->get("max-length"); maxLength = (size_t)(std::ceil(maxLength / (float)step) * step); - // restrict maximum length for class labels to 1 + // this should be only one class label per line on input, hence restricting length to 1 std::vector localMaxes(numFiles, maxLength); auto inputTypes = options_->get>("input-types", {}); for(int i = 0; i < inputTypes.size(); ++i) diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 6291c2ed7..c15adf3f6 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -206,14 +206,14 @@ void AsyncGraphGroup::execute(Ptr batch) { builder = builders_[i++]; } - auto lossNode = builder->build(graph, batch); + Ptr dynamicLoss = builder->build(graph, batch); if(t % optimizerDelay_ == 0) { fetchParams(graph->params()->vals(), params_, t_id); } graph->forward(); - loss += *lossNode; + loss += *dynamicLoss; graph->backward(); Tensor gradients; @@ -265,8 +265,7 @@ void AsyncGraphGroup::execute(Ptr batch) { scheduler_->update(loss, batch); } - loss.loss = 0; - loss.labels = 0; + loss.reset(); if(scheduler_->saving() || scheduler_->validating()) { // Wait with validation or saving until all other threads are done with diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index b5c01836d..255b5a2a5 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -604,8 +604,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { num_seen_words = 0; num_seen_sentences = 0; - loss.loss = 0; - loss.labels = 0; + loss.reset(); if((scheduler_->saving() || scheduler_->validating())) { // Wait with validation or saving until all other threads are done with diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp index 7d7f2d00b..904d614f9 100755 --- a/src/training/graph_group_multinode_sync.cpp +++ b/src/training/graph_group_multinode_sync.cpp @@ -230,7 +230,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { num_seen_words = 0; num_seen_sentences = 0; - loss = StaticLoss(); + loss.reset(); if((scheduler_->saving() || scheduler_->validating())) { // wait until other nodes are ready diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 12ad1ae4d..4e54f70b5 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -68,10 +68,10 @@ class Scheduler : public TrainingObserver { state.updateEta(baselr); } - std::string displayLoss(std::string lossType, - bool dispLabelCounts, - size_t batchLabels, - Ptr state) { + std::string formatLoss(std::string lossType, + bool dispLabelCounts, + size_t batchLabels, + Ptr state) { std::stringstream ss; ss << "Cost "; ss << std::setprecision(8) << std::fixed; @@ -308,7 +308,7 @@ class Scheduler : public TrainingObserver { state_->epochs, state_->batches, utils::withCommas(state_->samplesEpoch), - displayLoss(lossType, dispLabelCounts, batchLabels, state_), + formatLoss(lossType, dispLabelCounts, batchLabels, state_), timer_.elapsed(), state_->wordsDisp / timer_.elapsed(), state_->eta); @@ -318,7 +318,7 @@ class Scheduler : public TrainingObserver { state_->epochs, state_->batches, utils::withCommas(state_->samplesEpoch), - displayLoss(lossType, dispLabelCounts, 0, state_), // ignore batchLabels + formatLoss(lossType, dispLabelCounts, 0, state_), // ignore batchLabels timer_.elapsed(), state_->wordsDisp / timer_.elapsed()); } From 5c413f6e49215c523379c64fca8bb2bf7d1ed706 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 18 Jan 2019 10:40:40 -0800 Subject: [PATCH 156/838] address remaining comments from code review --- src/examples/mnist/model.h | 6 +-- src/layers/loss.h | 89 ++++++++++++++++++++------------------ src/tensors/gpu/add.cu | 5 --- src/training/scheduler.h | 4 +- src/training/validator.h | 4 +- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 788df5810..3b5be8778 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -54,7 +54,7 @@ class MNISTLogsoftmax : public CostBase { // @TODO: simplify this auto multiLoss = New(); - multiLoss->push_back({logsoftmax(top->loss()), top->labels()}); + multiLoss->push_back({logsoftmax(top->loss()), top->count()}); return multiLoss; } }; @@ -72,9 +72,9 @@ class MnistFeedForwardNet : public ModelBase { bool /*clean*/ = false) override { auto loss = construct(graph, batch, inference_); // @TODO: unify nomenclature, e.g. rather use apply - auto labels = graph->constant({(int)batch->size(), 1}, inits::from_value(1.f)); + auto count = graph->constant({(int)batch->size(), 1}, inits::from_value(1.f)); - return New(loss, labels); + return New(loss, count); } void load(Ptr /*graph*/, const std::string& /*name*/, bool) override { diff --git a/src/layers/loss.h b/src/layers/loss.h index 2975864ca..52f2155e1 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -6,33 +6,33 @@ namespace marian { /** * We represent loss as pair of expressions, where loss_ is usually a sum - * of all accumulated loss values per label and labels_ is the total number + * of all accumulated loss values per label and count_ is the total number * of labels over which the loss was collected. * * These two values can then be used to represent various cost variants - * for instance label-wise cross-entropy or perplexity. Optimization is * only performed with regard to the summed loss_. * - * Since both, loss_ and labels_ are dynamic graph nodes they can be further + * Since both, loss_ and count_ are dynamic graph nodes they can be further * combined into larger structures. See multi-objective losses below. */ class RationalLoss { protected: Expr loss_; // numerator - Expr labels_; // denominator + Expr count_; // denominator RationalLoss() = default; // protected public: - RationalLoss(Expr loss, Expr labels) - : loss_(loss), labels_(labels) {} + RationalLoss(Expr loss, Expr count) + : loss_(loss), count_(count) {} - RationalLoss(Expr loss, float labels) + RationalLoss(Expr loss, float count) : loss_(loss), - labels_(constant_like(loss, inits::from_value(labels))) {} + count_(constant_like(loss, inits::from_value(count))) {} RationalLoss(const RationalLoss& other) - : loss_(other.loss_), labels_(other.labels_) {} + : loss_(other.loss_), count_(other.count_) {} virtual ~RationalLoss() = default; @@ -50,25 +50,25 @@ class RationalLoss { return loss_->val()->scalar(); } - Expr labels() const { return labels_; } + Expr count() const { return count_; } template - void labels(std::vector& labels) const { - ABORT_IF(!labels_, "Labels have not been defined"); - labels_->val()->get(labels); + void count(std::vector& labels) const { + ABORT_IF(!count_, "Labels have not been defined"); + count_->val()->get(labels); } template - T labels() const { // this will fail if loss is not a single value - ABORT_IF(!labels_, "Labels have not been defined"); - return labels_->val()->scalar(); + T count() const { // this will fail if loss is not a single value + ABORT_IF(!count_, "Labels have not been defined"); + return count_->val()->scalar(); } // @TODO: add a funtion for returning maybe ratio? size_t size() const { - ABORT_IF(!labels_, "Labels have not been defined"); - return labels_->shape().elements(); + ABORT_IF(!count_, "Labels have not been defined"); + return count_->shape().elements(); } }; @@ -80,26 +80,25 @@ class RationalLoss { */ struct StaticLoss { float loss; - float labels; + float count; - StaticLoss() : loss(0.f), labels(0.f) {} + StaticLoss() : loss(0.f), count(0.f) {} StaticLoss(const RationalLoss& dynamic) - : loss(dynamic.loss()), labels(dynamic.labels()) {} + : loss(dynamic.loss()), count(dynamic.count()) {} StaticLoss& operator +=(const StaticLoss& other) { loss = loss + other.loss; - labels = labels + other.labels; + count = count + other.count; return *this; } void reset() { loss = 0.f; - labels = 0.f; + count = 0.f; } }; -// @TODO: overthink interface /** * Base class for multi-objective losses which is a list of RationalLoss * but also defines how to accumulate that list into a single RationalLoss @@ -110,13 +109,19 @@ class MultiRationalLoss : public RationalLoss { /** * Accumulation rule for losses + * In the default case this would just be a sum, see SumMultiRationalLoss, but there are + * special cases like ScaledMultiRationalLoss or MeanMultiRationalLoss (see below) where + * the accumulation is more complex. */ virtual Expr accumulateLoss(const RationalLoss& current) = 0; /** * Accumulation rule for labels + * Similar as above, the naive case is summation, but for instance MeanMultiRationalLoss + * is including all label counts in the loss hence label counts are always just 1 which is + * passed throught without summation or other modifications. */ - virtual Expr accumulateLabels(const RationalLoss& current) = 0; + virtual Expr accumulateCount(const RationalLoss& current) = 0; public: MultiRationalLoss() : RationalLoss() {} @@ -127,7 +132,7 @@ class MultiRationalLoss : public RationalLoss { void push_back(const RationalLoss& current) { loss_ = accumulateLoss(current); - labels_ = accumulateLabels(current); + count_ = accumulateCount(current); partialLosses_.push_back(current); } @@ -163,11 +168,11 @@ class SumMultiRationalLoss : public MultiRationalLoss { return current.loss(); } - virtual Expr accumulateLabels(const RationalLoss& current) override { - if(labels_) - return labels_ + current.labels(); + virtual Expr accumulateCount(const RationalLoss& current) override { + if(count_) + return count_ + current.count(); else - return current.labels(); + return current.count(); } public: @@ -178,7 +183,7 @@ class SumMultiRationalLoss : public MultiRationalLoss { /** * Scaled sum of losses. * This can weigh losses equally by choosing the first loss_0 as a reference - * and scaling all remaining losses loss_i by labels_0 / labels_i. Labels are + * and scaling all remaining losses loss_i by count_0 / count_i. Labels are * summed up by the same rule. By this we simulate a sum of losses at similar * scales. Dividing by scaled label counts yields a value close to an equally * weighted sum of means. @@ -194,17 +199,17 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { virtual Expr accumulateLoss(const RationalLoss& current) override { if(loss_) { const auto& first = partialLosses_.front(); - return loss_ + first.labels() * (current.loss() / current.labels()); // scale up/down to match scale of first loss + return loss_ + first.count() * (current.loss() / current.count()); // scale up/down to match scale of first loss } else { return current.loss(); // first reference loss, keeps to scale with this one } } - virtual Expr accumulateLabels(const RationalLoss& current) override { - if(labels_) { - return labels_; // Keep first label count // or: labels_ + first.labels() / current.labels(); + virtual Expr accumulateCount(const RationalLoss& current) override { + if(count_) { + return count_; // Keep first label count // or: count_ + first.count() / current.count(); } else { - return current.labels(); // This is the first loss + return current.count(); // This is the first loss } } @@ -228,16 +233,16 @@ class MeanMultiRationalLoss : public MultiRationalLoss { private: virtual Expr accumulateLoss(const RationalLoss& current) override { if(loss_) - return loss_ + current.loss() / current.labels(); + return loss_ + current.loss() / current.count(); else - return current.loss() / current.labels(); + return current.loss() / current.count(); } - virtual Expr accumulateLabels(const RationalLoss& current) override { - if(labels_) - return labels_; // keep the existing '1' + virtual Expr accumulateCount(const RationalLoss& current) override { + if(count_) + return count_; // keep the existing '1' else - return current.labels()->graph()->ones({1}); // just '1' as labels are factored into loss_ + return current.count()->graph()->ones({1}); // just '1' as labels are factored into loss_ } public: @@ -362,7 +367,7 @@ class RescorerLoss : public CrossEntropyLoss { virtual RationalLoss apply(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { auto ce = CrossEntropyLoss::apply(logits, labelIndices, mask, labelWeights); - return RationalLoss(ce.loss(), ce.labels()); + return RationalLoss(ce.loss(), ce.count()); } }; diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu index 2431948ee..81af79408 100755 --- a/src/tensors/gpu/add.cu +++ b/src/tensors/gpu/add.cu @@ -1,8 +1,3 @@ -/* All or part of this file was contributed by Intel under license: - * Copyright (C) 2017-2018 Intel Corporation - * SPDX-License-Identifier: MIT - */ - #include "tensors/gpu/add.h" #include "tensors/gpu/cuda_helpers.h" diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 4e54f70b5..891a2c0e7 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -277,14 +277,14 @@ class Scheduler : public TrainingObserver { rationalLoss.loss *= mpi->numMPIProcesses(); state_->costSum += rationalLoss.loss; // aggregate sum cost since last display - state_->costCount += rationalLoss.labels; // cost gets normalized w.r.t. this in display + state_->costCount += rationalLoss.count; // cost gets normalized w.r.t. this in display state_->updatesDisp += 1; state_->samplesDisp += batchSize; state_->wordsDisp += batchLabels; //@TODO: this is wrong // words at given input processed since last display, for speed display state_->samplesEpoch += batchSize; // sentences processed in this epoch - state_->labelsTotal += rationalLoss.labels; // total labels processed + state_->labelsTotal += rationalLoss.count; // total labels processed state_->newUpdate(numReadBatches); diff --git a/src/training/validator.h b/src/training/validator.h index 358d3e4a5..d09859b20 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -206,9 +206,9 @@ class CrossEntropyValidator : public Validator { options_->set("cost-type", ctype); // @TODO: check if still needed, most likely not. if(ctype == "perplexity") - return std::exp(loss.loss / loss.labels); + return std::exp(loss.loss / loss.count); if(ctype == "ce-mean-words") - return loss.loss / loss.labels; + return loss.loss / loss.count; if(ctype == "ce-sum") return loss.loss; else From fb656e566a06899b02aaf52a63dceac46a1c31b8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 18 Jan 2019 10:45:07 -0800 Subject: [PATCH 157/838] cleanup comments a bit --- src/layers/loss.h | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/layers/loss.h b/src/layers/loss.h index 52f2155e1..e92364762 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -18,7 +18,7 @@ namespace marian { */ class RationalLoss { protected: - Expr loss_; // numerator + Expr loss_; // numerator Expr count_; // denominator RationalLoss() = default; // protected @@ -79,8 +79,8 @@ class RationalLoss { * RationalLoss object. */ struct StaticLoss { - float loss; - float count; + float loss; // numerator + float count; // denominator StaticLoss() : loss(0.f), count(0.f) {} @@ -100,6 +100,7 @@ struct StaticLoss { }; /** + * @brief Base class for multi-objective losses * Base class for multi-objective losses which is a list of RationalLoss * but also defines how to accumulate that list into a single RationalLoss */ @@ -108,18 +109,18 @@ class MultiRationalLoss : public RationalLoss { std::vector partialLosses_; /** - * Accumulation rule for losses + * @brief Accumulation rule for losses * In the default case this would just be a sum, see SumMultiRationalLoss, but there are - * special cases like ScaledMultiRationalLoss or MeanMultiRationalLoss (see below) where - * the accumulation is more complex. + * special cases like ScaledMultiRationalLoss (scale other loses according to first label count) + * or MeanMultiRationalLoss (sum of means) where the accumulation is more complex. */ virtual Expr accumulateLoss(const RationalLoss& current) = 0; /** - * Accumulation rule for labels + * @brief Accumulation rule for labels * Similar as above, the naive case is summation, but for instance MeanMultiRationalLoss * is including all label counts in the loss hence label counts are always just 1 which is - * passed throught without summation or other modifications. + * passed through without summation or other modifications. */ virtual Expr accumulateCount(const RationalLoss& current) = 0; @@ -155,7 +156,7 @@ class MultiRationalLoss : public RationalLoss { }; /** - * Simple sum of losses. + * @brief Simple sum of losses. * Using this makes sense when the two loss types are similar in scale and * number of labels. For instance two decoders over similarly sized vocabularies */ @@ -181,7 +182,7 @@ class SumMultiRationalLoss : public MultiRationalLoss { }; /** - * Scaled sum of losses. + * @brief Scaled sum of losses. * This can weigh losses equally by choosing the first loss_0 as a reference * and scaling all remaining losses loss_i by count_0 / count_i. Labels are * summed up by the same rule. By this we simulate a sum of losses at similar @@ -219,7 +220,7 @@ class ScaledMultiRationalLoss : public MultiRationalLoss { }; /** - * Sum of mean losses. + * @brief Sum of mean losses. * Not really a rational loss as labels are factored into loss. Contribution of * losses is equal, same as for ScaledMultiRationalLoss, just divided by different * number of labels. See: @@ -251,7 +252,7 @@ class MeanMultiRationalLoss : public MultiRationalLoss { }; /** - * Factory for multi-objective rational loss functions + * @brief Factory for multi-objective rational loss functions */ Ptr newMultiLoss(Ptr options); @@ -260,7 +261,7 @@ Ptr newMultiLoss(Ptr options); // still feels somewhat hacky. /** - * Computes loss per given groundtruth label and then reduces to RationalLoss + * @brief Computes loss per given groundtruth label and then reduces to RationalLoss */ class LabelwiseLoss { protected: @@ -313,7 +314,7 @@ class LabelwiseLoss { }; /** - * Cross entropy loss across last axis, summed up over batch and time dimensions + * @brief Cross entropy loss across last axis, summed up over batch and time dimensions */ class CrossEntropyLoss : public LabelwiseLoss { public: @@ -357,7 +358,7 @@ class CrossEntropyLoss : public LabelwiseLoss { }; /** - * Cross entropy in rescorer used for computing sentences-level log probabilities + * @brief Cross entropy in rescorer used for computing sentences-level log probabilities */ class RescorerLoss : public CrossEntropyLoss { public: @@ -372,7 +373,7 @@ class RescorerLoss : public CrossEntropyLoss { }; /** - * Factory for label-wise loss functions + * @brief Factory for label-wise loss functions */ Ptr newLoss(Ptr options, bool inference); From 184b64475d8c93fabf9f66d906b1eec889a5bb1d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 11:17:05 -0800 Subject: [PATCH 158/838] first shot at extending Reduce() with a redunction functor (CPU only so far) --- src/functional/tmp.h | 27 ++++++++-------- src/graph/expression_operators.cpp | 32 +++++++++++++++---- src/graph/node_operators_unary.h | 51 ++++++++++++++++++++++-------- src/tensors/cpu/add.h | 37 +++++++++++----------- src/tensors/tensor_operators.h | 20 +++++++++++- 5 files changed, 116 insertions(+), 51 deletions(-) diff --git a/src/functional/tmp.h b/src/functional/tmp.h index 083836603..7c8f6fa11 100755 --- a/src/functional/tmp.h +++ b/src/functional/tmp.h @@ -118,55 +118,56 @@ __HDI__ float apply(Functor functor, /******************************************************************************/ +// @TODO: Rename this. It is a reduction loop. template struct Loop { - template + template __HDI__ static float result( - Functor functor, + Functor functor, float aggInit, AggFunctor aggFunctor, functional::Array, K>& in, const functional::Array& pAcc, const functional::Array& length, const functional::Array& dim) { - float sum = 0; + float agg = aggInit; functional::Array acc; for(int i = 0; i < length[N - n]; ++i) { for(size_t j = 0; j < K; ++j) { acc[j] = pAcc[j] + (dim[N - n] + i) * in[j].shape().bstride(N - n); } - sum += Loop::result(functor, in, acc, length, dim); + agg = aggFunctor(agg, Loop::result(functor, aggInit, aggFunctor, in, acc, length, dim)); } - return sum; + return agg; } }; template struct Loop<1, N, K> { - template + template __HDI__ static float result( - Functor functor, + Functor functor, float aggInit, AggFunctor aggFunctor, functional::Array, K>& in, const functional::Array& pAcc, const functional::Array& length, const functional::Array& dim) { - float sum = 0; + float agg = aggInit; functional::Array acc; for(int i = 0; i < length[N - 1]; ++i) { for(size_t j = 0; j < K; ++j) { acc[j] = pAcc[j] + (dim[N - 1] + i) * in[j].shape().bstride(N - 1); } - sum += apply(functor, in, acc); + agg = aggFunctor(agg, apply(functor, in, acc)); } - return sum; + return agg; } }; -template -__HDI__ float loops(Functor functor, +template +__HDI__ float loops(Functor functor, float aggInit, AggFunctor aggFunctor, functional::Array, K>& in, const functional::Array& length, const functional::Array& dim) { functional::Array acc = {0}; - return Loop::result(functor, in, acc, length, dim); + return Loop::result(functor, aggInit, aggFunctor, in, acc, length, dim); } } // namespace functional } // namespace marian diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 9e2772bcd..f3e1b75f7 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -278,16 +278,36 @@ Expr select(Expr a, const std::vector& indices, int axis) { } Expr sum(Expr a, int ax) { - return Expression(a, ax, ReduceNodeOpCode::sum); + return Expression(a, ax, ReduceNodeOpCode::sum); } -// log(sum(exp(a))) -Expr logSumExp(Expr a, int ax) { - return Expression(a, ax, ReduceNodeOpCode::logSumExp); +Expr mean(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::mean); } -Expr mean(Expr a, int ax) { - return Expression(a, ax, ReduceNodeOpCode::mean); +Expr std(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::std); +} + +Expr var(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::var); +} + +Expr max(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::max); +} + +Expr min(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::min); +} + +Expr prod(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::prod); +} + +// log(sum(exp(a))) +Expr logsumexp(Expr a, int ax) { + return Expression(a, ax, ReduceNodeOpCode::logSumExp); } Expr scalar_product(Expr a, Expr b, int ax) { diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 1716292c9..70a47d5e7 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -413,15 +413,19 @@ struct LogSoftmaxNodeOp : public UnaryNodeOp { }; enum class ReduceNodeOpCode { - sum, mean, min, max, logSumExp + sum, mean, std, var, min, max, prod, logSumExp }; -struct SumNodeOp : public UnaryNodeOp { +struct ReduceNodeOp : public UnaryNodeOp { int axis_; ReduceNodeOpCode opCode_; + int reducedDim_; // dimension of axis being reduced, e.g. used in mean() - SumNodeOp(Expr a, int axis, ReduceNodeOpCode opCode) - : UnaryNodeOp(a, newShape(a, axis)), opCode_(opCode){} + ReduceNodeOp(Expr a, int axis, ReduceNodeOpCode opCode) + : UnaryNodeOp(a, newShape(a, axis)), opCode_(opCode) + { + reducedDim_ = child(0)->shape().elements() / val_->shape().elements(); // e.g. used in mean() + } NodeOps forwardOps() override { using namespace functional; @@ -430,7 +434,20 @@ struct SumNodeOp : public UnaryNodeOp { case ReduceNodeOpCode::sum: return {NodeOp(Reduce(_1, val_, child(0)->val()))}; case ReduceNodeOpCode::mean: - return {NodeOp(Reduce(_1, (float)val_->shape().elements() / (float)child(0)->shape().elements(), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; + case ReduceNodeOpCode::std: + return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()); + Element(_1 = sqrt(_1), val_, val_))}; + case ReduceNodeOpCode::var: + return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; + case ReduceNodeOpCode::min: + return {NodeOp(Reduce(_1, FLT_MAX, min(_1,_2), val_, child(0)->val()))}; + case ReduceNodeOpCode::max: + return {NodeOp(Reduce(_1, -FLT_MAX, max(_1,_2), val_, child(0)->val()))}; + case ReduceNodeOpCode::prod: + return {NodeOp(Reduce(_1, 1.0f, _1 * _2, val_, child(0)->val()))}; + case ReduceNodeOpCode::logSumExp: + return {NodeOp(Reduce(_1, -FLT_MAX, logaddexp(_1,_2), val_, child(0)->val()))}; default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } @@ -442,7 +459,12 @@ struct SumNodeOp : public UnaryNodeOp { case ReduceNodeOpCode::sum: return {NodeOp(Add(_1, child(0)->grad(), adj_))}; case ReduceNodeOpCode::mean: - return {NodeOp(Add(_1, (float)val_->shape().elements() / (float)child(0)->shape().elements(), child(0)->grad(), adj_))}; + return {NodeOp(Add(_1, 1.0f / (float)reducedDim_, child(0)->grad(), adj_))}; + //case ReduceNodeOpCode::std: + //case ReduceNodeOpCode::var: + //case ReduceNodeOpCode::min: + //case ReduceNodeOpCode::max: + //case ReduceNodeOpCode::logSumExp: default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } @@ -458,12 +480,15 @@ struct SumNodeOp : public UnaryNodeOp { const std::string type() override { switch (opCode_) { - case ReduceNodeOpCode::sum: - return "sum"; - case ReduceNodeOpCode::mean: - return "mean"; - default: - ABORT("Unexpected reduction op-code {}", (int)opCode_); + case ReduceNodeOpCode::sum: return "sum"; + case ReduceNodeOpCode::mean: return "mean"; + case ReduceNodeOpCode::std: return "std"; + case ReduceNodeOpCode::var: return "var"; + case ReduceNodeOpCode::min: return "min"; + case ReduceNodeOpCode::max: return "max"; + case ReduceNodeOpCode::prod: return "prod"; + case ReduceNodeOpCode::logSumExp: return "logSumExp"; + default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } } @@ -481,7 +506,7 @@ struct SumNodeOp : public UnaryNodeOp { virtual bool equal(Expr node) override { if(!NaryNodeOp::equal(node)) return false; - Ptr cnode = std::dynamic_pointer_cast(node); + Ptr cnode = std::dynamic_pointer_cast(node); if(!cnode) return false; if(axis_ != cnode->axis_ || opCode_ != cnode->opCode_) diff --git a/src/tensors/cpu/add.h b/src/tensors/cpu/add.h index 38a0684dd..4bae5bb59 100755 --- a/src/tensors/cpu/add.h +++ b/src/tensors/cpu/add.h @@ -15,8 +15,8 @@ namespace marian { namespace cpu { -template -void gAddGeneric(Functor functor, +template +void gAggregateGeneric(Functor functor, float aggInit, AggFunctor aggFunctor, const functional::Shape full, functional::Tensor out, functional::Array, K> ins, @@ -34,16 +34,16 @@ void gAddGeneric(Functor functor, functional::Array dims; for(int index = 0; index < outLength; ++index) { if(same) { - out[index] += functional::apply(functor, ins, index) * scale; + out[index] = aggFunctor(out[index], functional::apply(functor, ins, index) * scale); } else { out.shape().dims(index, dims); - out[index] += functional::loops(functor, ins, len, dims) * scale; + out[index] = aggFunctor(out[index], functional::loops(functor, aggInit, aggFunctor, ins, len, dims) * scale); } } } -template -void gAddEqual(Functor functor, +template +void gAggregateEqual(Functor functor, AggFunctor aggFunctor, functional::Tensor out, functional::Array, K> ins, float scale, @@ -61,12 +61,12 @@ void gAddEqual(Functor functor, indices[i] = ins[i].shape().bindex(dims); } - out[index] += functional::apply(functor, ins, indices) * scale; + out[index] = aggFunctor(out[index], functional::apply(functor, ins, indices) * scale); } } -template -void gAddReduce(Functor functor, +template +void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggFunctor, const functional::Shape full, functional::Tensor out, functional::Array, K> ins, @@ -79,10 +79,10 @@ void gAddReduce(Functor functor, same = same && ins[i].shape().elements() == full.elements(); for(int j = 0; j < rows; ++j) { - float sum = 0; + float colSum = aggInit; if(same) { for(int id = 0; id < cols; ++id) - sum += functional::apply(functor, ins, j * cols + id); + colSum = aggFunctor(colSum, functional::apply(functor, ins, j * cols + id)); } else { functional::Array dims; for(int id = 0; id < cols; ++id) { @@ -90,15 +90,15 @@ void gAddReduce(Functor functor, functional::Array indices; for(size_t i = 0; i < K; ++i) indices[i] = ins[i].shape().bindex(dims); - sum += functional::apply(functor, ins, indices); + colSum = aggFunctor(colSum, functional::apply(functor, ins, indices)); } } - out[j] += sum * scale; + out[j] = aggFunctor(out[j], colSum * scale); } } -template -void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { +template +void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, float scale, marian::Tensor out, Tensors... tensors) { auto full = marian::Shape::broadcast({out, tensors...}); //int length = out->shape().elements(); @@ -111,15 +111,16 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { if(full.back() != 1 && out->shape().back() == 1) { //size_t m = full.elements() / length; //size_t k = full.back(); - cpu::gAddReduce(functor, full, gOut, gIns, scale); + cpu::gAggregateReduce(functor, aggInit, aggFunctor, full, gOut, gIns, scale); } else if(out->shape() == full) { bool broadcast = false; for(size_t i = 0; i < K; ++i) broadcast = broadcast || gOut.shape() != gIns[i].shape(); - cpu::gAddEqual(functor, gOut, gIns, scale, broadcast); + cpu::gAggregateEqual(functor, aggFunctor, gOut, gIns, scale, broadcast); } else { - cpu::gAddGeneric(functor, full, gOut, gIns, scale); + cpu::gAggregateGeneric(functor, aggInit, aggFunctor, full, gOut, gIns, scale); } } + } // namespace cpu } // namespace marian diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index f7de2f205..b2f31adc0 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -51,7 +51,7 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { gpu::Add(functor, scale, out, tensors...); else #endif - cpu::Add(functor, scale, out, tensors...); + cpu::Aggregate(functor, 0.0f, functional::_1 + functional::_2, scale, out, tensors...); } template @@ -59,6 +59,16 @@ void Add(Functor functor, marian::Tensor out, Tensors... tensors) { Add(functor, 1, out, tensors...); } +template +void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, marian::Tensor out, Tensors... tensors) { +#ifdef CUDA_FOUND + if(out->getBackend()->getDeviceId().type == DeviceType::gpu) + gpu::Aggregate(functor, aggInit, aggFunctor, out, tensors...); + else +#endif + cpu::Aggregate(functor, aggInit, aggFunctor, 1.0f, out, tensors...); +} + template void Reduce(Functor functor, float scale, @@ -74,6 +84,14 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { Add(functor, out, tensors...); } +template +void Reduce(Functor functor, float aggInit, AggFunctor aggFunctor, + marian::Tensor out, + Tensors... tensors) { + out->set(aggInit); + Aggregate(functor, aggInit, aggFunctor, out, tensors...); +} + // clang-format off DISPATCH7(Prod, marian::Tensor, const marian::Tensor&, const marian::Tensor&, bool, bool, float, float) DISPATCH8(ProdBatched, marian::Tensor, Ptr, const marian::Tensor, const marian::Tensor, bool, bool, float, float) From ecffcc4cfee8c21be1265e1921ba7c3425c0ece1 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 18 Jan 2019 13:51:22 -0800 Subject: [PATCH 159/838] small changes to make BERT fine-tuning work --- src/data/corpus_base.cpp | 25 +++++++++++----- src/data/corpus_base.h | 2 +- src/models/bert.h | 58 +++++++++++++++++++++++++++++------- src/models/model_factory.cpp | 4 +++ 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index b34bacef9..b6bd7042a 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -44,7 +44,7 @@ CorpusBase::CorpusBase(const std::vector& paths, ABORT_IF(files_.back()->empty(), "File '{}' is empty", path); } - initEOS(); + initEOS(/*training=*/true); } CorpusBase::CorpusBase(Ptr options, bool translate) @@ -59,7 +59,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) else paths_ = options_->get>("input"); - initEOS(); + initEOS(training); std::vector vocabPaths; if(!options_->get>("vocabs").empty()) @@ -284,7 +284,7 @@ void CorpusBase::addWeightsToBatch(Ptr batch, batch->setDataWeights(weights); } -void CorpusBase::initEOS() { +void CorpusBase::initEOS(bool training = true) { // Labels fed into sub-batches that are just class-labels, not sequence labels do not require to // add a EOS symbol. Hence decision to add EOS is now based on input stream positions and correspoding // input type. @@ -292,11 +292,20 @@ void CorpusBase::initEOS() { addEOS_.resize(paths_.size(), true); // @TODO: think if this should be checked and processed here or in a validation step in config? auto inputTypes = options_->get>("input-types", {}); // empty list by default - ABORT_IF(inputTypes.size() > 0 && inputTypes.size() != paths_.size(), - "Input types have been specified ({}), you need to specify one per input ({})", - inputTypes.size(), - paths_.size()); - for(int i = 0; i < inputTypes.size(); ++i) + + // make sure there is an input type for each path + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() < paths_.size(), + "Input types have been specified ({}), you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + + // make sure there is an equal number of input types and paths when training + ABORT_IF(training && inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types have been specified ({}), you need to specify one per input ({})", + inputTypes.size(), + paths_.size()); + + for(int i = 0; i < paths_.size(); ++i) if(inputTypes[i] == "class") addEOS_[i] = false; else if(inputTypes[i] == "sequence") diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index f1ac99aef..85e26d717 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -546,7 +546,7 @@ class CorpusBase /** * @brief Determine if EOS symbol should be added to input */ - void initEOS(); + void initEOS(bool training); /** * @brief Helper function converting a line of text into words using the i-th diff --git a/src/models/bert.h b/src/models/bert.h index 72d744e9c..288d06c8c 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -119,6 +119,28 @@ class BertBatch : public CorpusBatch { words[i] = maskOut(words[i], maskId, engine); // mask that position } + annotateSentenceIndices(); + } + + BertBatch(Ptr batch, + const std::string& sepSymbol, + const std::string& clsSymbol) + : CorpusBatch(*batch), + maskSymbol_("dummy"), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { + annotateSentenceIndices(); + } + + void annotateSentenceIndices() { + // BERT expects a textual first stream and a second stream with class labels + auto subBatch = subBatches_.front(); + const auto& vocab = *subBatch->vocab(); + auto& words = subBatch->data(); + + // Get word id of special symbols + Word sepId = vocab[sepSymbol_]; + ABORT_IF(sepId == vocab.getUnkId(), + "BERT separator symbol {} not found in vocabulary", sepSymbol_); + int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); @@ -157,13 +179,25 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { : EncoderClassifier(options) {} std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + std::string modelType = opt("type"); + // intercept batch and annotate with BERT-specific concepts - auto bertBatch = New(batch, - eng_, - opt("bert-masking-fraction", 0.15f), // 15% by default according to paper - opt("bert-mask-symbol"), - opt("bert-sep-symbol"), - opt("bert-class-symbol")); + Ptr bertBatch; + if(modelType == "bert") { // full BERT pre-training + bertBatch = New(batch, + eng_, + opt("bert-masking-fraction", 0.15f), // 15% by default according to paper + opt("bert-mask-symbol"), + opt("bert-sep-symbol"), + opt("bert-class-symbol")); + } else if(modelType == "bert-classifier") { // we are probably fine-tuning a BERT model for a classification task + bertBatch = New(batch, + opt("bert-sep-symbol"), + opt("bert-class-symbol")); // only annotate sentence separators + } else { + ABORT("Unknown BERT-style model: {}", modelType); + } + return EncoderClassifier::apply(graph, bertBatch, clearGraph); } @@ -185,9 +219,9 @@ class BertEncoder : public EncoderTransformer { Expr addSentenceEmbeddings(Expr embeddings, Ptr batch, bool learnedPosEmbeddings) const { + Ptr bertBatch = std::dynamic_pointer_cast(batch); - - ABORT_IF(!bertBatch, "Batch could not be converted for BERT training"); + ABORT_IF(!bertBatch, "Batch must be BertBatch for BERT training or fine-tuning"); int dimEmb = embeddings->shape()[-1]; int dimBatch = embeddings->shape()[-2]; @@ -240,14 +274,18 @@ class BertClassifier : public ClassifierBase { int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels + std::string finetune = ""; + if(opt("original-type") == "bert-classifier") // seems we are fine-tuning + finetune = "_finetune"; // change name so we do not relead BERT output layers for fine-tuning + auto output = mlp::mlp() // .push_back(mlp::dense() // - ("prefix", prefix_ + "_ff_logit_l1") // + ("prefix", prefix_ + finetune + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh)) // @TODO: do we actually need this? .push_back(mlp::output() // ("dim", dimTrgCls)) // - ("prefix", prefix_ + "_ff_logit_l2") // + ("prefix", prefix_ + finetune + "_ff_logit_l2") // .construct(graph); auto logits = output->apply(classEmbeddings); // class logits for each batch entry diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index a6ceb9988..66f2db928 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -84,6 +84,8 @@ Ptr EncoderClassifierFactory::construct(Ptr graph) { Ptr enccls; if(options_->get("type") == "bert") { enccls = New(options_); + } else if(options_->get("type") == "bert-classifier") { + enccls = New(options_); } else { enccls = New(options_); } @@ -228,6 +230,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { if(type == "bert") { // for full BERT training return models::encoder_classifier()(options) // + ("original-type", "bert") // so we can query this ("usage", use) // .push_back(models::encoder() // ("type", "bert-encoder") // close to original transformer encoder @@ -243,6 +246,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { if(type == "bert-classifier") { // for BERT fine-tuning on non-BERT classification task return models::encoder_classifier()(options) // + ("original-type", "bert-classifier") // so we can query this ("usage", use) // .push_back(models::encoder() // ("type", "bert-encoder") // From fb1b0cb1f7ded54e9921c8df7886a1e3304171d6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 14:29:34 -0800 Subject: [PATCH 160/838] (towards GPU aggregator) --- src/functional/tmp.h | 14 ++++++++++++++ src/graph/node_operators_unary.h | 6 +++--- src/tensors/gpu/add.h | 2 ++ 3 files changed, 19 insertions(+), 3 deletions(-) mode change 100644 => 100755 src/tensors/gpu/add.h diff --git a/src/functional/tmp.h b/src/functional/tmp.h index 7c8f6fa11..059f36ec4 100755 --- a/src/functional/tmp.h +++ b/src/functional/tmp.h @@ -169,5 +169,19 @@ __HDI__ float loops(Functor functor, float aggInit, AggFunctor aggFunctor, functional::Array acc = {0}; return Loop::result(functor, aggInit, aggFunctor, in, acc, length, dim); } + + +// dummy until changed Add to Agg +template +__HDI__ float loops(Functor functor, + functional::Array, K>& in, + const functional::Array& length, + const functional::Array& dim) { + (void)functor; (void)in; (void)length; (void)dim; + return 0; +} + + + } // namespace functional } // namespace marian diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 70a47d5e7..818aed0c6 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -441,13 +441,13 @@ struct ReduceNodeOp : public UnaryNodeOp { case ReduceNodeOpCode::var: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; case ReduceNodeOpCode::min: - return {NodeOp(Reduce(_1, FLT_MAX, min(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, std::numeric_limits::max(), min(_1,_2), val_, child(0)->val()))}; case ReduceNodeOpCode::max: - return {NodeOp(Reduce(_1, -FLT_MAX, max(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, std::numeric_limits::lowest(), max(_1,_2), val_, child(0)->val()))}; case ReduceNodeOpCode::prod: return {NodeOp(Reduce(_1, 1.0f, _1 * _2, val_, child(0)->val()))}; case ReduceNodeOpCode::logSumExp: - return {NodeOp(Reduce(_1, -FLT_MAX, logaddexp(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, std::numeric_limits::lowest(), logaddexp(_1,_2), val_, child(0)->val()))}; default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } diff --git a/src/tensors/gpu/add.h b/src/tensors/gpu/add.h old mode 100644 new mode 100755 index e5e22d88d..dbb2cf719 --- a/src/tensors/gpu/add.h +++ b/src/tensors/gpu/add.h @@ -8,5 +8,7 @@ namespace gpu { template void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors); +template +void Aggregate(Functor functor, float initAgg, AggFunctor aggFunctor, marian::Tensor out, Tensors... tensors); } } // namespace marian From 19cdedf7f888bd0befd0b6e8099473706ab21e73 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 15:04:02 -0800 Subject: [PATCH 161/838] resolved a template ambiguity, still not compiling on gcc for now --- src/graph/node_operators_unary.h | 21 +++++++++++---------- src/tensors/gpu/add.inc | 6 ++++++ src/tensors/gpu/element.inc | 1 + src/tensors/tensor_operators.h | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) mode change 100644 => 100755 src/tensors/gpu/add.inc diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 818aed0c6..27ca6f0d4 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -437,17 +437,17 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Reduce(_1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; case ReduceNodeOpCode::std: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()); - Element(_1 = sqrt(_1), val_, val_))}; + Element(_1 = sqrt(_1), val_))}; case ReduceNodeOpCode::var: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; case ReduceNodeOpCode::min: - return {NodeOp(Reduce(_1, std::numeric_limits::max(), min(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, min(_1,_2), std::numeric_limits::max(), val_, child(0)->val()))}; case ReduceNodeOpCode::max: - return {NodeOp(Reduce(_1, std::numeric_limits::lowest(), max(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, max(_1,_2), std::numeric_limits::lowest(), val_, child(0)->val()))}; case ReduceNodeOpCode::prod: - return {NodeOp(Reduce(_1, 1.0f, _1 * _2, val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, _1 * _2, 1.0f, val_, child(0)->val()))}; case ReduceNodeOpCode::logSumExp: - return {NodeOp(Reduce(_1, std::numeric_limits::lowest(), logaddexp(_1,_2), val_, child(0)->val()))}; + return {NodeOp(Reduce(_1, logaddexp(_1,_2), std::numeric_limits::lowest(), val_, child(0)->val()))}; default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } @@ -460,11 +460,12 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Add(_1, child(0)->grad(), adj_))}; case ReduceNodeOpCode::mean: return {NodeOp(Add(_1, 1.0f / (float)reducedDim_, child(0)->grad(), adj_))}; - //case ReduceNodeOpCode::std: - //case ReduceNodeOpCode::var: - //case ReduceNodeOpCode::min: - //case ReduceNodeOpCode::max: - //case ReduceNodeOpCode::logSumExp: + case ReduceNodeOpCode::std: + case ReduceNodeOpCode::var: + case ReduceNodeOpCode::min: + case ReduceNodeOpCode::max: + case ReduceNodeOpCode::logSumExp: + ABORT("Reduction op-code for {} not yet implemented", type()); default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc old mode 100644 new mode 100755 index 27f35b954..c2aeffb4b --- a/src/tensors/gpu/add.inc +++ b/src/tensors/gpu/add.inc @@ -1,3 +1,4 @@ +// see element.inc for instructions on how to maintain this using namespace functional; template void Add>, Assignee<2>>, marian::Tensor, marian::Tensor>(BinaryFunctor>, Assignee<2>>, float, marian::Tensor, marian::Tensor, marian::Tensor); template void Add>>, Assignee<2>>, marian::Tensor, marian::Tensor>(BinaryFunctor>>, Assignee<2>>, float, marian::Tensor, marian::Tensor, marian::Tensor); @@ -22,3 +23,8 @@ template void Add, template void Add, Assignee<3>>, Assignee<1>>, marian::Tensor, marian::Tensor, marian::Tensor>(BinaryFunctor, Assignee<3>>, Assignee<1>>, float, marian::Tensor, marian::Tensor, marian::Tensor, marian::Tensor); template void Add, UnaryFunctor, Assignee<3>>>>, marian::Tensor, marian::Tensor, marian::Tensor>(BinaryFunctor, UnaryFunctor, Assignee<3>>>>, float, marian::Tensor, marian::Tensor, marian::Tensor, marian::Tensor); template void Add, Assignee<2> >, std::shared_ptr, std::shared_ptr >(BinaryFunctor, Assignee<2> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::Assignee<1> >, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<1> >, float, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc index 66f763018..f3cdea282 100755 --- a/src/tensors/gpu/element.inc +++ b/src/tensors/gpu/element.inc @@ -55,6 +55,7 @@ template void Element, BinaryFunctor, Bin template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, BinaryFunctor >, Assignee<4> >, Capture> >, BinaryFunctor >, Assignee<4> >, Capture> > >, Capture>, BinaryFunctor, Assignee<4> >, Capture> > >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Element, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr >(marian::functional::Assign, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Element, marian::functional::UnaryFunctor > >>(marian::functional::Assign, marian::functional::UnaryFunctor > >, std::shared_ptr); // How to add new specializations: // When you use a new specialization, it will cause a link error of this form (example): // .../src/tensors/tensor_operators.h:41: undefined reference to `void marian::gpu::Element ( ... )' diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index b2f31adc0..dd32ead60 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -85,7 +85,7 @@ void Reduce(Functor functor, marian::Tensor out, Tensors... tensors) { } template -void Reduce(Functor functor, float aggInit, AggFunctor aggFunctor, +void Reduce(Functor functor, AggFunctor aggFunctor, float aggInit, marian::Tensor out, Tensors... tensors) { out->set(aggInit); From ef456f27f67eef7cad3552a83f0556fbb199b7c8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 18 Jan 2019 15:17:56 -0800 Subject: [PATCH 162/838] make mnist run again --- src/examples/mnist/model.h | 5 +++-- src/layers/loss.h | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 3b5be8778..0b07f99a0 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -37,8 +37,9 @@ class MNISTCrossEntropyCost : public CostBase { // use CE loss auto loss = sum(cross_entropy(top->loss(), labels), /*axis =*/ 0); - - return New(RationalLoss({loss, (float)vLabels.size()})); + auto multiLoss = New(); + multiLoss->push_back({loss, (float)vLabels.size()}); + return multiLoss; } }; diff --git a/src/layers/loss.h b/src/layers/loss.h index e92364762..4a28de7b2 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -128,12 +128,12 @@ class MultiRationalLoss : public RationalLoss { MultiRationalLoss() : RationalLoss() {} MultiRationalLoss(const RationalLoss& rl) : RationalLoss() { - this->push_back(rl); + push_back(rl); } - void push_back(const RationalLoss& current) { + virtual void push_back(const RationalLoss& current) { loss_ = accumulateLoss(current); - count_ = accumulateCount(current); + count_ = accumulateCount(current); partialLosses_.push_back(current); } From 53481ab6a55bbc2b4265fa32efc41ef45fdb690e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 17:16:17 -0800 Subject: [PATCH 163/838] towards gpu::Aggregate() --- src/tensors/gpu/add.cu | 66 ++++++++++++++++++++++++++++------ src/tensors/gpu/add.h | 2 +- src/tensors/gpu/add.inc | 8 ++--- src/tensors/tensor_operators.h | 2 +- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu index 2431948ee..42800b9f6 100755 --- a/src/tensors/gpu/add.cu +++ b/src/tensors/gpu/add.cu @@ -16,8 +16,8 @@ namespace marian { namespace gpu { -template -__global__ void gAddGeneric(Functor functor, +template +__global__ void gAggregateGeneric(Functor functor, AggFunctor aggFunctor, const functional::Shape full, functional::Tensor out, functional::Array, K> ins, @@ -46,8 +46,8 @@ __global__ void gAddGeneric(Functor functor, } } -template -__global__ void gAddEqual(Functor functor, +template +__global__ void gAggregateEqual(Functor functor, AggFunctor aggFunctor, functional::Tensor out, functional::Array, K> ins, float scale, @@ -72,8 +72,8 @@ __global__ void gAddEqual(Functor functor, } } -template -__global__ void gAddReduce(Functor functor, +template +__global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggFunctor, const functional::Shape full, functional::Tensor out, functional::Array, K> ins, @@ -92,7 +92,7 @@ __global__ void gAddReduce(Functor functor, float* _sum = _share + blockDim.x; if(same) { - _sum[threadIdx.x] = 0; + _sum[threadIdx.x] = aggInit; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; if(id < cols) @@ -100,7 +100,7 @@ __global__ void gAddReduce(Functor functor, } } else { functional::Array dims; - _sum[threadIdx.x] = 0; + _sum[threadIdx.x] = aggInit; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; @@ -129,6 +129,48 @@ __global__ void gAddReduce(Functor functor, } } +template +void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, float scale, marian::Tensor out, Tensors... tensors) { + cudaSetDevice(out->getDeviceId().no); + + auto full = marian::Shape::broadcast({out, tensors...}); + + int length = out->shape().elements(); + + constexpr size_t K = sizeof...(Tensors); + + functional::Tensor gOut = out; + functional::Array, K> gIns = {tensors...}; + + if(full.back() != 1 && out->shape().back() == 1) { + size_t m = full.elements() / length; + size_t k = full.back(); + + int blocks = std::min(MAX_BLOCKS, (int)m); + int threads = std::min(MAX_THREADS, (int)k); + int shared = sizeof(float) * threads * 2; + + gAggregateReduce<<>>(functor, aggInit, aggFunctor, full, gOut, gIns, scale); + + } else if(out->shape() == full) { + int threads = std::min(MAX_THREADS, length); + int blocks + = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); + + bool broadcast = false; + for(int i = 0; i < K; ++i) + broadcast = broadcast || gOut.shape() != gIns[i].shape(); + gAggregateEqual<<>>(functor, aggFunctor, gOut, gIns, scale, broadcast); + } else { + int threads = std::min(MAX_THREADS, length); + int blocks + = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); + + gAggregateGeneric<<>>(functor, aggFunctor, full, gOut, gIns, scale); + } +} + +// @TODO: this is a duplicate; can be removed, but need to redo all the add.inc entries... template void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { cudaSetDevice(out->getDeviceId().no); @@ -142,6 +184,8 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { functional::Tensor gOut = out; functional::Array, K> gIns = {tensors...}; + auto addFunctor = functional::_1 + functional::_2; + if(full.back() != 1 && out->shape().back() == 1) { size_t m = full.elements() / length; size_t k = full.back(); @@ -150,7 +194,7 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { int threads = std::min(MAX_THREADS, (int)k); int shared = sizeof(float) * threads * 2; - gAddReduce<<>>(functor, full, gOut, gIns, scale); + gAggregateReduce<<>>(functor, 0, addFunctor, full, gOut, gIns, scale); } else if(out->shape() == full) { int threads = std::min(MAX_THREADS, length); @@ -160,13 +204,13 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { bool broadcast = false; for(int i = 0; i < K; ++i) broadcast = broadcast || gOut.shape() != gIns[i].shape(); - gAddEqual<<>>(functor, gOut, gIns, scale, broadcast); + gAggregateEqual<<>>(functor, addFunctor, gOut, gIns, scale, broadcast); } else { int threads = std::min(MAX_THREADS, length); int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); - gAddGeneric<<>>(functor, full, gOut, gIns, scale); + gAggregateGeneric<<>>(functor, addFunctor, full, gOut, gIns, scale); } } diff --git a/src/tensors/gpu/add.h b/src/tensors/gpu/add.h index dbb2cf719..21e0bb96c 100755 --- a/src/tensors/gpu/add.h +++ b/src/tensors/gpu/add.h @@ -9,6 +9,6 @@ namespace gpu { template void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors); template -void Aggregate(Functor functor, float initAgg, AggFunctor aggFunctor, marian::Tensor out, Tensors... tensors); +void Aggregate(Functor functor, float initAgg, AggFunctor aggFunctor, float scale, marian::Tensor out, Tensors... tensors); } } // namespace marian diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc index c2aeffb4b..41bbe9a52 100755 --- a/src/tensors/gpu/add.inc +++ b/src/tensors/gpu/add.inc @@ -24,7 +24,7 @@ template void Add template void Add, UnaryFunctor, Assignee<3>>>>, marian::Tensor, marian::Tensor, marian::Tensor>(BinaryFunctor, UnaryFunctor, Assignee<3>>>>, float, marian::Tensor, marian::Tensor, marian::Tensor, marian::Tensor); template void Add, Assignee<2> >, std::shared_ptr, std::shared_ptr >(BinaryFunctor, Assignee<2> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Add, marian::functional::Assignee<1> >, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<1> >, float, std::shared_ptr, std::shared_ptr); -template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); -template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); -template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); -template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index dd32ead60..8dd17a5ce 100755 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -63,7 +63,7 @@ template void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, marian::Tensor out, Tensors... tensors) { #ifdef CUDA_FOUND if(out->getBackend()->getDeviceId().type == DeviceType::gpu) - gpu::Aggregate(functor, aggInit, aggFunctor, out, tensors...); + gpu::Aggregate(functor, aggInit, aggFunctor, 1.0f, out, tensors...); else #endif cpu::Aggregate(functor, aggInit, aggFunctor, 1.0f, out, tensors...); From 7ca0460bea75fe11978f594642e1dd353f0727f9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 17:27:04 -0800 Subject: [PATCH 164/838] completed gpu::Aggregate() --- src/functional/tmp.h | 14 -------------- src/tensors/gpu/add.cu | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/functional/tmp.h b/src/functional/tmp.h index 059f36ec4..7c8f6fa11 100755 --- a/src/functional/tmp.h +++ b/src/functional/tmp.h @@ -169,19 +169,5 @@ __HDI__ float loops(Functor functor, float aggInit, AggFunctor aggFunctor, functional::Array acc = {0}; return Loop::result(functor, aggInit, aggFunctor, in, acc, length, dim); } - - -// dummy until changed Add to Agg -template -__HDI__ float loops(Functor functor, - functional::Array, K>& in, - const functional::Array& length, - const functional::Array& dim) { - (void)functor; (void)in; (void)length; (void)dim; - return 0; -} - - - } // namespace functional } // namespace marian diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu index 42800b9f6..32c12783c 100755 --- a/src/tensors/gpu/add.cu +++ b/src/tensors/gpu/add.cu @@ -17,7 +17,7 @@ namespace marian { namespace gpu { template -__global__ void gAggregateGeneric(Functor functor, AggFunctor aggFunctor, +__global__ void gAggregateGeneric(Functor functor, float aggInit, AggFunctor aggFunctor, const functional::Shape full, functional::Tensor out, functional::Array, K> ins, @@ -37,10 +37,10 @@ __global__ void gAggregateGeneric(Functor functor, AggFunctor aggFunctor, int index = bid + blockDim.x * blockIdx.x + threadIdx.x; if(index < outLength) { if(same) { - out[index] += functional::apply(functor, ins, index) * scale; + out[index] = aggFunctor(out[index], functional::apply(functor, ins, index) * scale); } else { out.shape().dims(index, dims); - out[index] += functional::loops(functor, ins, len, dims) * scale; + out[index] = aggFunctor(out[index], functional::loops(functor, aggInit, aggFunctor, ins, len, dims) * scale); } } } @@ -67,7 +67,7 @@ __global__ void gAggregateEqual(Functor functor, AggFunctor aggFunctor, indices[i] = ins[i].shape().bindex(dims); } - out[index] += functional::apply(functor, ins, indices) * scale; + out[index] = aggFunctor(out[index], functional::apply(functor, ins, indices) * scale); } } } @@ -96,7 +96,7 @@ __global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggF for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; if(id < cols) - _sum[threadIdx.x] += functional::apply(functor, ins, j * cols + id); + _sum[threadIdx.x] = aggFunctor(_sum[threadIdx.x], functional::apply(functor, ins, j * cols + id)); } } else { functional::Array dims; @@ -109,7 +109,7 @@ __global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggF functional::Array indices; for(int i = 0; i < K; ++i) indices[i] = ins[i].shape().bindex(dims); - _sum[threadIdx.x] += functional::apply(functor, ins, indices); + _sum[threadIdx.x] = aggFunctor(_sum[threadIdx.x], functional::apply(functor, ins, indices)); } } } @@ -119,12 +119,12 @@ __global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggF __syncthreads(); int skip = (len + 1) >> 1; if(threadIdx.x < (len >> 1)) { - _sum[threadIdx.x] += _sum[threadIdx.x + skip]; + _sum[threadIdx.x] = aggFunctor(_sum[threadIdx.x], _sum[threadIdx.x + skip]); } len = (len + 1) >> 1; } __syncthreads(); - out[j] += _sum[0] * scale; + out[j] = aggFunctor(out[j], _sum[0] * scale); } } } @@ -166,7 +166,7 @@ void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, float scal int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); - gAggregateGeneric<<>>(functor, aggFunctor, full, gOut, gIns, scale); + gAggregateGeneric<<>>(functor, aggInit, aggFunctor, full, gOut, gIns, scale); } } @@ -210,7 +210,7 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); - gAggregateGeneric<<>>(functor, addFunctor, full, gOut, gIns, scale); + gAggregateGeneric<<>>(functor, 0, addFunctor, full, gOut, gIns, scale); } } From f211fffbe38b5d172a1671d155558802d78cfdfe Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 17:44:23 -0800 Subject: [PATCH 165/838] (minor bug fix) --- src/graph/node_operators_unary.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 27ca6f0d4..ed5c4c0ce 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -424,7 +424,8 @@ struct ReduceNodeOp : public UnaryNodeOp { ReduceNodeOp(Expr a, int axis, ReduceNodeOpCode opCode) : UnaryNodeOp(a, newShape(a, axis)), opCode_(opCode) { - reducedDim_ = child(0)->shape().elements() / val_->shape().elements(); // e.g. used in mean() + reducedDim_ = a->shape()[axis]; // e.g. used in mean() + ABORT_IF(reducedDim_ != a->shape().elements() / shape().elements(), "bug in determining reducedDim"); } NodeOps forwardOps() override { From 1932ada095126da267282ba9c81ecacf94ec8069 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 18:42:56 -0800 Subject: [PATCH 166/838] added tests for all reduction operators --- src/graph/expression_operators.cpp | 4 +-- src/graph/expression_operators.h | 9 +++-- src/graph/node_operators_unary.h | 20 +++++++---- src/tests/operator_tests.cpp | 54 +++++++++++++++++++++--------- 4 files changed, 61 insertions(+), 26 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index f3e1b75f7..4e5ac5787 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -286,11 +286,11 @@ Expr mean(Expr a, int ax) { } Expr std(Expr a, int ax) { - return Expression(a, ax, ReduceNodeOpCode::std); + return Expression(a - mean(a,ax), ax, ReduceNodeOpCode::rms); } Expr var(Expr a, int ax) { - return Expression(a, ax, ReduceNodeOpCode::var); + return Expression(a - mean(a, ax), ax, ReduceNodeOpCode::sumSqr); } Expr max(Expr a, int ax) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index e2c1d9226..51d4dfc0d 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -151,6 +151,13 @@ Expr select(Expr a, const std::vector& indices, int axis); /*********************************************************/ Expr sum(Expr a, int ax = 0); +Expr mean(Expr a, int ax = 0); +Expr std(Expr a, int ax); +Expr var(Expr a, int ax); +Expr max(Expr a, int ax); +Expr min(Expr a, int ax); +Expr prod(Expr a, int ax); +Expr logsumexp(Expr a, int ax); Expr softmax(Expr x, int axis = -1); @@ -160,8 +167,6 @@ Expr softmax(Expr a, Expr zeroOneMask, int axis = -1); Expr logsoftmax(Expr a); -Expr mean(Expr a, int ax = 0); - Expr cross_entropy(Expr a, Expr b); Expr scalar_product(Expr a, Expr b, int ax = 0); diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index ed5c4c0ce..086906a1c 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -413,7 +413,7 @@ struct LogSoftmaxNodeOp : public UnaryNodeOp { }; enum class ReduceNodeOpCode { - sum, mean, std, var, min, max, prod, logSumExp + sum, mean, rms, sumSqr, min, max, prod, logSumExp }; struct ReduceNodeOp : public UnaryNodeOp { @@ -436,10 +436,10 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Reduce(_1, val_, child(0)->val()))}; case ReduceNodeOpCode::mean: return {NodeOp(Reduce(_1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; - case ReduceNodeOpCode::std: + case ReduceNodeOpCode::rms: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()); Element(_1 = sqrt(_1), val_))}; - case ReduceNodeOpCode::var: + case ReduceNodeOpCode::sumSqr: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; case ReduceNodeOpCode::min: return {NodeOp(Reduce(_1, min(_1,_2), std::numeric_limits::max(), val_, child(0)->val()))}; @@ -461,12 +461,18 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Add(_1, child(0)->grad(), adj_))}; case ReduceNodeOpCode::mean: return {NodeOp(Add(_1, 1.0f / (float)reducedDim_, child(0)->grad(), adj_))}; - case ReduceNodeOpCode::std: - case ReduceNodeOpCode::var: + case ReduceNodeOpCode::rms: + case ReduceNodeOpCode::sumSqr: case ReduceNodeOpCode::min: case ReduceNodeOpCode::max: case ReduceNodeOpCode::logSumExp: ABORT("Reduction op-code for {} not yet implemented", type()); +// NDArrayView::NumericOperation({ const_cast(outputGradientValue)->shared_from_this(), +// const_cast( inputValues[0] )->shared_from_this(), +// const_cast(outputValue )->shared_from_this() }, alpha, +// Microsoft::MSR::CNTK::ElementWiseOperator::opElementwiseProductWithExpOfDiff, +// gradient, beta, +// Microsoft::MSR::CNTK::ElementWiseOperator::opSum); default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } @@ -484,8 +490,8 @@ struct ReduceNodeOp : public UnaryNodeOp { switch (opCode_) { case ReduceNodeOpCode::sum: return "sum"; case ReduceNodeOpCode::mean: return "mean"; - case ReduceNodeOpCode::std: return "std"; - case ReduceNodeOpCode::var: return "var"; + case ReduceNodeOpCode::rms: return "rms"; + case ReduceNodeOpCode::sumSqr: return "sumSqr"; case ReduceNodeOpCode::min: return "min"; case ReduceNodeOpCode::max: return "max"; case ReduceNodeOpCode::prod: return "prod"; diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index e4610dd92..a6e298ea1 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -204,12 +204,19 @@ void tests(DeviceType device) { graph->clear(); values.clear(); - std::vector vA({1, 2, 3, 4, 5, 6, 7, 8}); - std::vector vS1({6, 8, 10, 12}); - std::vector vS2({10, 26}); - - std::vector vW({2.77778f, 6.77778f}); - + std::vector vA({1, 6, 3, 8, + 5, 2, 7, 4}); + // import numpy as np + // a = np.array([[1, 6, 3, 8], [5, 2, 7, 4]]) + std::vector vS1({6, 8, 10, 12}); // s1 = np.sum(a, axis=0) + std::vector vS2({18, 18}); // np.sum(a, axis = 1) + std::vector vS4({2.6925824f, 1.80277564f}); // np.std(a, axis = 1) + std::vector vV5({7.25, 3.25}); // np.var(a, axis = 1) + std::vector vM6({8, 7}); // np.max(a, axis = 1) + std::vector vM7({1, 2}); // np.min(a, axis = 1) + std::vector vP8({144, 280}); // np.prod(a, axis = 1) + std::vector vL9({8.13364336f, 7.17551536f}); // np.log(np.sum(np.exp(a), axis=1)) + std::vector vW({5.0f, 4.55555556f}); // np.mean(a*s1,axis=-1) / np.mean(s1,axis=-1) auto a = graph->constant({2, 4}, inits::from_vector(vA)); @@ -218,6 +225,14 @@ void tests(DeviceType device) { auto m3 = mean(s1, /*axis=*/ 1); + auto s4 = marian::std(a, /*axis=*/ 1); + auto v5 = var(a, /*axis=*/ 1); + + auto m6 = max(a, /*axis=*/ 1); + auto m7 = min(a, /*axis=*/ 1); + auto p8 = prod(a, /*axis=*/ 1); + auto l9 = logsumexp(a, /*axis=*/ 1); + auto sp = scalar_product(s2, s2, /*axis=*/ 0); auto wa = weighted_average(a, s1, /*axis=*/ -1); @@ -227,21 +242,30 @@ void tests(DeviceType device) { CHECK(s1->shape() == Shape({1, 4})); CHECK(s2->shape() == Shape({2, 1})); CHECK(m3->shape() == Shape({1, 1})); + CHECK(s4->shape() == Shape({2, 1})); + CHECK(v5->shape() == Shape({2, 1})); + CHECK(m6->shape() == Shape({2, 1})); + CHECK(m7->shape() == Shape({2, 1})); + CHECK(p8->shape() == Shape({2, 1})); + CHECK(l9->shape() == Shape({2, 1})); CHECK(sp->shape() == Shape({1, 1})); CHECK(wa->shape() == Shape({2, 1})); - s1->val()->get(values); - CHECK( values == vS1 ); + s1->val()->get(values); CHECK(values == vS1); + s2->val()->get(values); CHECK(values == vS2); - s2->val()->get(values); - CHECK( values == vS2 ); + CHECK(m3->val()->scalar() == 9); - CHECK( m3->val()->scalar() == 9 ); - CHECK( sp->val()->scalar() == 776 ); + s4->val()->get(values); CHECK(std::equal(values.begin(), values.end(), vS4.begin(), floatApprox)); + v5->val()->get(values); CHECK(values == vV5); + m6->val()->get(values); CHECK(values == vM6); + m7->val()->get(values); CHECK(values == vM7); + p8->val()->get(values); CHECK(values == vP8); + l9->val()->get(values); CHECK(std::equal(values.begin(), values.end(), vL9.begin(), floatApprox)); - wa->val()->get(values); - CHECK( std::equal(values.begin(), values.end(), - vW.begin(), floatApprox) ); + CHECK(sp->val()->scalar() == 648); + + wa->val()->get(values); CHECK(std::equal(values.begin(), values.end(), vW.begin(), floatApprox)); } SECTION("concatenation") { From 7ce226739248bf3479c409159b4685a2bbb9dc8c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 19:08:40 -0800 Subject: [PATCH 167/838] added gradients for min(), max(), and logsumexp() --- src/graph/node_operators_unary.h | 17 +++++++++-------- src/tensors/gpu/add.inc | 2 ++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 086906a1c..2b9395436 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -463,16 +463,17 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Add(_1, 1.0f / (float)reducedDim_, child(0)->grad(), adj_))}; case ReduceNodeOpCode::rms: case ReduceNodeOpCode::sumSqr: + ABORT("Gradient of reduction op-code for {} not yet implemented", type()); case ReduceNodeOpCode::min: case ReduceNodeOpCode::max: - case ReduceNodeOpCode::logSumExp: - ABORT("Reduction op-code for {} not yet implemented", type()); -// NDArrayView::NumericOperation({ const_cast(outputGradientValue)->shared_from_this(), -// const_cast( inputValues[0] )->shared_from_this(), -// const_cast(outputValue )->shared_from_this() }, alpha, -// Microsoft::MSR::CNTK::ElementWiseOperator::opElementwiseProductWithExpOfDiff, -// gradient, beta, -// Microsoft::MSR::CNTK::ElementWiseOperator::opSum); + return {NodeOp(Add((_1 == _2) * _3, // adj_ gets routed into the min/max value --@REVIEW: is this correct? + child(0)->grad(), child(0)->val(), val_, adj_))}; + case ReduceNodeOpCode::logSumExp: // y = log(sum_j exp(x_j)) + return {NodeOp(Add(_1 * exp(_2 - _3), // dJ/dx_i = dJ/dy * 1/(sum_j exp(x_j)) exp(x_i) = dJ/dy * exp(x_i - y)) + child(0)->grad(), // out = dJ/dx_i + adj_, // _1 = dJ/dy + child(0)->val(), // _2 = x_i + val_))}; // _3 = y default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc index 41bbe9a52..2dac53996 100755 --- a/src/tensors/gpu/add.inc +++ b/src/tensors/gpu/add.inc @@ -28,3 +28,5 @@ template void marian::gpu::Aggregate, marian::fu template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); From 1c115ad6a9bfece4f778a25be0f7d3df59d23ac2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 19:37:13 -0800 Subject: [PATCH 168/838] added gradients for std() and var() --- src/graph/node_operators_unary.h | 30 +++++++++++++++++------------- src/tensors/gpu/add.inc | 2 ++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 2b9395436..18120485b 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -461,19 +461,23 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Add(_1, child(0)->grad(), adj_))}; case ReduceNodeOpCode::mean: return {NodeOp(Add(_1, 1.0f / (float)reducedDim_, child(0)->grad(), adj_))}; - case ReduceNodeOpCode::rms: - case ReduceNodeOpCode::sumSqr: - ABORT("Gradient of reduction op-code for {} not yet implemented", type()); - case ReduceNodeOpCode::min: - case ReduceNodeOpCode::max: - return {NodeOp(Add((_1 == _2) * _3, // adj_ gets routed into the min/max value --@REVIEW: is this correct? - child(0)->grad(), child(0)->val(), val_, adj_))}; - case ReduceNodeOpCode::logSumExp: // y = log(sum_j exp(x_j)) - return {NodeOp(Add(_1 * exp(_2 - _3), // dJ/dx_i = dJ/dy * 1/(sum_j exp(x_j)) exp(x_i) = dJ/dy * exp(x_i - y)) - child(0)->grad(), // out = dJ/dx_i - adj_, // _1 = dJ/dy - child(0)->val(), // _2 = x_i - val_))}; // _3 = y + case ReduceNodeOpCode::rms: // WARNING: UNTESTED!! + // y = (sum_j x_j^2)^0.5 + // dJ/dx_i = dJ/dy * 0.5 (sum_j x_j^2)^-0.5 * 2 x_i = dJ/dy * x_i / y --@REVIEW: is this correct? + // @TODO: do we need protection against div by 0? L'hospital rule? + return {NodeOp(Add(_1 * _2 / _3, child(0)->grad(), adj_, child(0)->val(), val_))}; + case ReduceNodeOpCode::sumSqr: // WARNING: UNTESTED!! + // y = sum_j x_j^2 + // dJ/dx_i = dJ/dy * sum_j dx_j^2/dx_i = dJ/dy * 2 dx_i --@REVIEW: is this correct? + return {NodeOp(Add(_1 * 2.0f * _2, child(0)->grad(), adj_, child(0)->val()))}; + case ReduceNodeOpCode::min: // WARNING: UNTESTED!! + case ReduceNodeOpCode::max: // WARNING: UNTESTED!! + // adj_ gets routed into the min/max value --@REVIEW: is this correct? + return {NodeOp(Add((_1 == _2) * _3, child(0)->grad(), child(0)->val(), val_, adj_))}; + case ReduceNodeOpCode::logSumExp: + // y = log(sum_j exp(x_j)) + // dJ/dx_i = dJ/dy * 1/(sum_j exp(x_j)) exp(x_i) = dJ/dy * exp(x_i - y)) --@REVIEW: is this correct? + return {NodeOp(Add(_1 * exp(_2 - _3), child(0)->grad(), adj_, child(0)->val(), val_))}; default: ABORT("Unexpected reduction op-code {}", (int)opCode_); } diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc index 2dac53996..69244dce0 100755 --- a/src/tensors/gpu/add.inc +++ b/src/tensors/gpu/add.inc @@ -30,3 +30,5 @@ template void marian::gpu::Aggregate, marian::fu template void marian::gpu::Aggregate, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, std::shared_ptr >(marian::functional::Assignee<1>, float, marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr); template void marian::gpu::Add, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Add, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::Capture>, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Capture>, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr); From 731aab417a1453ef1936aba8f5c8d1fe5a65a3e6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 19:39:23 -0800 Subject: [PATCH 169/838] bugbug: ReduceNodeOpCode::sumSqr should be meanSqr --- src/graph/expression_operators.cpp | 2 +- src/graph/node_operators_unary.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 4e5ac5787..75fac34a0 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -290,7 +290,7 @@ Expr std(Expr a, int ax) { } Expr var(Expr a, int ax) { - return Expression(a - mean(a, ax), ax, ReduceNodeOpCode::sumSqr); + return Expression(a - mean(a, ax), ax, ReduceNodeOpCode::meanSqr); } Expr max(Expr a, int ax) { diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 18120485b..03c19f19c 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -413,7 +413,7 @@ struct LogSoftmaxNodeOp : public UnaryNodeOp { }; enum class ReduceNodeOpCode { - sum, mean, rms, sumSqr, min, max, prod, logSumExp + sum, mean, rms, meanSqr, min, max, prod, logSumExp }; struct ReduceNodeOp : public UnaryNodeOp { @@ -439,7 +439,7 @@ struct ReduceNodeOp : public UnaryNodeOp { case ReduceNodeOpCode::rms: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()); Element(_1 = sqrt(_1), val_))}; - case ReduceNodeOpCode::sumSqr: + case ReduceNodeOpCode::meanSqr: return {NodeOp(Reduce(_1 * _1, 1.0f / (float)reducedDim_, val_, child(0)->val()))}; case ReduceNodeOpCode::min: return {NodeOp(Reduce(_1, min(_1,_2), std::numeric_limits::max(), val_, child(0)->val()))}; @@ -466,7 +466,7 @@ struct ReduceNodeOp : public UnaryNodeOp { // dJ/dx_i = dJ/dy * 0.5 (sum_j x_j^2)^-0.5 * 2 x_i = dJ/dy * x_i / y --@REVIEW: is this correct? // @TODO: do we need protection against div by 0? L'hospital rule? return {NodeOp(Add(_1 * _2 / _3, child(0)->grad(), adj_, child(0)->val(), val_))}; - case ReduceNodeOpCode::sumSqr: // WARNING: UNTESTED!! + case ReduceNodeOpCode::meanSqr: // WARNING: UNTESTED!! // y = sum_j x_j^2 // dJ/dx_i = dJ/dy * sum_j dx_j^2/dx_i = dJ/dy * 2 dx_i --@REVIEW: is this correct? return {NodeOp(Add(_1 * 2.0f * _2, child(0)->grad(), adj_, child(0)->val()))}; @@ -496,7 +496,7 @@ struct ReduceNodeOp : public UnaryNodeOp { case ReduceNodeOpCode::sum: return "sum"; case ReduceNodeOpCode::mean: return "mean"; case ReduceNodeOpCode::rms: return "rms"; - case ReduceNodeOpCode::sumSqr: return "sumSqr"; + case ReduceNodeOpCode::meanSqr: return "meanSqr"; case ReduceNodeOpCode::min: return "min"; case ReduceNodeOpCode::max: return "max"; case ReduceNodeOpCode::prod: return "prod"; From c97facfe6b92c1085aad5ddc6560f746ce98416c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 18 Jan 2019 19:46:49 -0800 Subject: [PATCH 170/838] (fixed an indentation) --- src/graph/node_operators_unary.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 03c19f19c..ef3654ea8 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -495,8 +495,8 @@ struct ReduceNodeOp : public UnaryNodeOp { switch (opCode_) { case ReduceNodeOpCode::sum: return "sum"; case ReduceNodeOpCode::mean: return "mean"; - case ReduceNodeOpCode::rms: return "rms"; - case ReduceNodeOpCode::meanSqr: return "meanSqr"; + case ReduceNodeOpCode::rms: return "rms"; + case ReduceNodeOpCode::meanSqr: return "meanSqr"; case ReduceNodeOpCode::min: return "min"; case ReduceNodeOpCode::max: return "max"; case ReduceNodeOpCode::prod: return "prod"; From 406e289820531691ef6b21b10b2277fd3cb22a51 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 10:30:11 -0800 Subject: [PATCH 171/838] switched to memory-saving implementation of smoothing --- src/graph/node_operators_unary.h | 2 +- src/layers/loss.cpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index ef3654ea8..a576fb3d9 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -476,7 +476,7 @@ struct ReduceNodeOp : public UnaryNodeOp { return {NodeOp(Add((_1 == _2) * _3, child(0)->grad(), child(0)->val(), val_, adj_))}; case ReduceNodeOpCode::logSumExp: // y = log(sum_j exp(x_j)) - // dJ/dx_i = dJ/dy * 1/(sum_j exp(x_j)) exp(x_i) = dJ/dy * exp(x_i - y)) --@REVIEW: is this correct? + // dJ/dx_i = dJ/dy * 1/(sum_j exp(x_j)) exp(x_i) = dJ/dy * exp(x_i - y)) --@REVIEW: is this correct? return {NodeOp(Add(_1 * exp(_2 - _3), child(0)->grad(), adj_, child(0)->val(), val_))}; default: ABORT("Unexpected reduction op-code {}", (int)opCode_); diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index d9102456a..e9709c691 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -27,11 +27,15 @@ Expr LossBase::getCrossEntropy(Expr logits, Expr mask, Expr weights) { auto ce = cross_entropy(logits, indices); + //auto ce = rows(logits, indices) - logsumexp(logits, /*axis=*/ -1); if(smoothing_ > 0) { // @TODO: add this to CE kernels instead +#if 0 auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); - //auto ceq = mean(logits, /*axis=*/ -1) - logsum(logits), /*axis=*/ -1); +#else // alternative that is cheaper memory-wise + auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); +#endif ce = (1 - smoothing_) * ce - smoothing_ * ceq; } From 3ceb4fac81ed1aaca7b45f01ba2caa7aa8bb60c8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 14:36:02 -0800 Subject: [PATCH 172/838] towards shared denominator for smoothed CE --- src/layers/loss.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index e9709c691..e87cac865 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -26,18 +26,25 @@ Expr LossBase::getCrossEntropy(Expr logits, Expr indices, Expr mask, Expr weights) { - auto ce = cross_entropy(logits, indices); - //auto ce = rows(logits, indices) - logsumexp(logits, /*axis=*/ -1); + Expr ce; if(smoothing_ > 0) { // @TODO: add this to CE kernels instead #if 0 + ce = cross_entropy(logits, indices); auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceq; #else // alternative that is cheaper memory-wise + ce = cross_entropy(logits, indices); auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + //auto ceq = mean(logits, /*axis=*/ -1) - Z; + //ce = (1 - smoothing_) * cols(logits, indices) // ce term + // - smoothing_ * mean(logits, /*axis=*/ -1) // smoothing term + // - logsumexp(logits, /*axis=*/ -1); // denominator #endif - ce = (1 - smoothing_) * ce - smoothing_ * ceq; } + else + ce = cross_entropy(logits, indices); if(mask) ce = ce * mask; From bc7957ad323d4f874acf4bfcc694b4d986495232 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 15:09:39 -0800 Subject: [PATCH 173/838] towards PyTorch names for select() (gather(), index_select()) --- src/graph/expression_graph.h | 2 +- src/graph/expression_operators.cpp | 44 +++++++++++++++--------------- src/graph/expression_operators.h | 27 +++++++++++------- src/graph/node_operators_binary.h | 16 ++++++----- src/models/transformer.h | 4 +-- src/rnn/types.h | 1 + src/tests/operator_tests.cpp | 21 +++++++------- 7 files changed, 63 insertions(+), 52 deletions(-) diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index bf4f256d3..fe8361618 100755 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -371,7 +371,7 @@ class ExpressionGraph : public std::enable_shared_from_this { Type::uint32); } // this version sets up the shape such that the indices are in a given axis - // Use this if you want to pass these indices to select(). + // Use this if you want to pass these indices to gather(). // indexee shape = (3, 2, 5, 2); axis = 1 -> resulting shape = (1, size of indicesVector, 1, 1) Expr indices(const std::vector& indicesVector, Expr indexee, int axis = -1) { Shape shape; diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 4694351b2..54e68960a 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -245,36 +245,40 @@ Expr constant_like(Expr a, const NodeInitializer& init) { return graph->constant(shape, init); } +Expr gather(Expr a, Expr indices, int axis) { + return Expression(a, indices, axis); +} + +Expr index_select(Expr a, const std::vector& indices, int axis) { + auto indexExpr = a->graph()->indices(indices, a, axis); + return gather(a, indexExpr, axis); +} + Expr rows(Expr a, Expr indices) { - // @TODO:: replace with `select(a, indices, -2)` - // as soon as select is efficient enough - return Expression(a, indices); + // @TODO:: replace with `index_select(a, indices, -2)` + // as soon as select is efficient enough + return Expression(a, indices); } Expr rows(Expr a, const std::vector& indices) { - auto indexExpr = a->graph()->indices(indices); - return rows(a, indexExpr); + auto indexExpr = a->graph()->indices(indices); + return rows(a, indexExpr); } - Expr cols(Expr a, Expr indices) { - // @TODO:: replace with `select(a, indices, -1)` - // as soon as select is efficient enough - return Expression(a, indices); + // @TODO:: replace with `index_select(a, indices, -1)` + // as soon as select is efficient enough + return Expression(a, indices); } Expr cols(Expr a, const std::vector& indices) { - auto indexExpr = a->graph()->indices(indices); - return cols(a, indexExpr); -} - -Expr select(Expr a, Expr indices, int axis) { - return Expression(a, indices, axis); + auto indexExpr = a->graph()->indices(indices); + return cols(a, indexExpr); } -Expr select(Expr a, const std::vector& indices, int axis) { - auto indexExpr = a->graph()->indices(indices, a, axis); - return select(a, indexExpr, axis); +Expr sliceView(Expr a, const Slice& slice, int axis) { // numpy __getitem__ semantics + // @TODO: If not memory-consecutive then fall back to index_select + return Expression(a, slice, axis); } Expr sum(Expr a, int ax) { @@ -462,10 +466,6 @@ Expr swapAxes(Expr x, int axis1, int axis2) return transpose(x, axes); } -Expr sliceView(Expr a, const Slice& slice, int axis) { // numpy __getitem__ semantics - return Expression(a, slice, axis); -} - Expr cross_entropy(Expr a, Expr indices) { return Expression(a, indices); } diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index e2c1d9226..f5379246d 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -139,14 +139,29 @@ Expr flatten_2d(Expr a); Expr stopGradient(Expr a); +Expr gather(Expr a, Expr indices, int axis); +Expr index_select(Expr a, const std::vector& indices, int axis); +static inline Expr index_select(Expr a, int index, int axis) { + // Until Marian supports strides, use this for indexing non-memory-consecutive + // slices, while sliceView() can be used for memory-consecutive ones. + return index_select(a, std::vector({(IndexType)index}), axis); +} + Expr rows(Expr a, Expr indices); Expr rows(Expr a, const std::vector& indices); Expr cols(Expr a, Expr indices); Expr cols(Expr a, const std::vector& indices); -Expr select(Expr a, Expr indices, int axis); -Expr select(Expr a, const std::vector& indices, int axis); +Expr sliceView(Expr a, const Slice& slice, int axis); + +static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name + return sliceView(a, Slice((int)start, (int)(start + length)), axis); +} + +static inline Expr step(Expr a, int step, int axis) { + return sliceView(a, Slice(step), axis); +} /*********************************************************/ @@ -168,14 +183,6 @@ Expr scalar_product(Expr a, Expr b, int ax = 0); Expr weighted_average(Expr in, Expr weights, int ax = 0); -Expr sliceView(Expr a, const Slice& slice, int axis); -static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name - return sliceView(a, Slice((int)start, (int)(start + length)), axis); -} -static inline Expr step(Expr a, int step, int axis) { - return sliceView(a, Slice(step), axis); -} - Expr sqrt(Expr a, float eps = 0.f); Expr square(Expr a); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index eb20033d9..10541b9de 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -540,8 +540,8 @@ struct RowsNodeOp : public NaryNodeOp { const std::string color() override { return "orange"; } }; -// This operation indexes a tensor along an axis. -// This is similar to the common gather() operation in other toolkits. +// This operation gathers elements of a tensor along an axis. +// This is like PyTorch gather(), except that Marian also implements broadcasting. // For example, this can be used for: // - Same index applied to all batch items (today's select()): // 'index' has 1 in the axes that match batch axes in the input, and axis set to the one axis that gets selected over. @@ -574,12 +574,14 @@ struct RowsNodeOp : public NaryNodeOp { // out[i][j][k] = input[i][index[i][j][k]][k] # if dim == 1 // out[i][j][k] = input[i][j][index[i][j][k]] # if dim == 2 // If 'a' and 'indices' do not have the same rank, then negative 'axis' is -// interpreted relative to 'a', and 'indices' must have the resulting axis. -// Broadcasting is supported as usual. +// interpreted relative to 'a'; then both shapes are left-padded to the same rank; +// and indexing happens along the axis that corresponds to 'axis' in the padded shapes. +// Broadcasting is supported as usual. This way, this function can implement both +// batched and non-batched selection. // @TODO: The current implementation does not support batched indices (third scenario above). // I.e. all axes of 'indices' except 'axis' must have dimension 1. -struct SelectNodeOp : public NaryNodeOp { - SelectNodeOp(Expr a, Expr indices, int axis) +struct GatherNodeOp : public NaryNodeOp { + GatherNodeOp(Expr a, Expr indices, int axis) : NaryNodeOp({a, indices}, newShape(a, indices, axis), a->value_type()), axis_(a->shape().axis(axis)) { matchOrAbort(indices->value_type()); @@ -628,7 +630,7 @@ struct SelectNodeOp : public NaryNodeOp { virtual bool equal(Expr node) override { if(!NaryNodeOp::equal(node)) return false; - Ptr cnode = std::dynamic_pointer_cast(node); + Ptr cnode = std::dynamic_pointer_cast(node); if(!cnode) return false; if(axis_ != cnode->axis_) diff --git a/src/models/transformer.h b/src/models/transformer.h index d91dfb7f8..166c68722 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -178,7 +178,7 @@ class Transformer : public EncoderOrDecoderBase { void collectOneHead(Expr weights, int dimBeam) { // select first head, this is arbitrary as the choice does not really matter - auto head0 = select(weights, std::vector({0}), -3); // @TODO: implement an index() or slice() operator and use that + auto head0 = index_select(weights, 0, -3); int dimBatchBeam = head0->shape()[-4]; int srcWords = head0->shape()[-1]; @@ -194,7 +194,7 @@ class Transformer : public EncoderOrDecoderBase { // @TODO: make splitting obsolete alignments_.clear(); for(int i = 0; i < trgWords; ++i) { - alignments_.push_back(select(head0, std::vector({(IndexType)i}), -1)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] + alignments_.push_back(index_select(head0, i, -1)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] } } diff --git a/src/rnn/types.h b/src/rnn/types.h index 672c600bb..bd44b98bb 100755 --- a/src/rnn/types.h +++ b/src/rnn/types.h @@ -34,6 +34,7 @@ struct State { ABORT_IF(dimTime != 1 && !isBatchMajor, "unexpected time extent for RNN state"); // (the reshape()/rows() trick won't work in this case) int numCols = isBatchMajor ? dimDepth * dimTime : dimDepth; + // @TODO: Can this complex operation be more easily written using index_select()? sel = reshape(sel, { sel->shape().elements() / numCols, numCols }); // [beamSize * dimBatch, dimDepth] or [beamSize * dimBatch, dimTime * dimDepth] sel = rows(sel, selIdx); sel = reshape(sel, { beamSize, isBatchMajor ? dimBatch : dimTime, isBatchMajor ? dimTime : dimBatch, dimDepth }); diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index e4610dd92..8873655fa 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -632,15 +632,16 @@ void tests(DeviceType device) { std::vector vS3({7, -8, 9, -10, 11, -12}); auto A = graph->param("4x3", {4,3}, inits::from_vector(in)); - auto B1 = select(A, Indices({0}), 0); - auto B2 = select(A, Indices({0}), 1); - auto B3 = select(A, Indices({1}), -1); - auto B4 = select(A, Indices({0, 1}), 0); + // @TODO: split this into index_select() and gather() + auto B1 = index_select(A, Indices({0}), 0); + auto B2 = index_select(A, Indices({0}), 1); + auto B3 = index_select(A, Indices({1}), -1); + auto B4 = index_select(A, Indices({0, 1}), 0); auto C = graph->param("2x3x2", {2, 3, 2}, inits::from_vector(in)); - auto D1 = select(C, Indices({0}), 0); - auto D2 = select(C, Indices({2}), -2); - auto D3 = select(C, Indices({0,2}), 1); + auto D1 = index_select(C, Indices({0}), 0); + auto D2 = index_select(C, Indices({2}), -2); + auto D3 = index_select(C, Indices({0,2}), 1); auto S1 = step(A, 2, 0); auto S2 = narrow(A, 1, 2, 0); @@ -693,7 +694,7 @@ void tests(DeviceType device) { CHECK(values == vS3); } - SECTION("rows/cols as select operations") { + SECTION("rows/cols as index_select operations") { graph->clear(); values.clear(); std::vector values2; @@ -703,9 +704,9 @@ void tests(DeviceType device) { auto A = graph->param("4x3", {4, 3}, inits::from_vector(vA)); auto B1 = rows(A, idx); - auto B2 = select(A, idx, 0); + auto B2 = index_select(A, idx, 0); auto C1 = cols(A, idx); - auto C2 = select(A, idx, 1); + auto C2 = index_select(A, idx, 1); graph->forward(); CHECK(B1->shape() == B2->shape()); From a14a6239490be8f0c9e397a9b5ecea209147d75f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 15:16:22 -0800 Subject: [PATCH 174/838] gather() (formerly select(tensor)) now requires the same rank for input and indices --- src/graph/node_operators_binary.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 10541b9de..b7c6d7064 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -541,7 +541,7 @@ struct RowsNodeOp : public NaryNodeOp { }; // This operation gathers elements of a tensor along an axis. -// This is like PyTorch gather(), except that Marian also implements broadcasting. +// This is like PyTorch gather(). // For example, this can be used for: // - Same index applied to all batch items (today's select()): // 'index' has 1 in the axes that match batch axes in the input, and axis set to the one axis that gets selected over. @@ -573,11 +573,7 @@ struct RowsNodeOp : public NaryNodeOp { // out[i][j][k] = input[index[i][j][k]][j][k] # if dim == 0 // out[i][j][k] = input[i][index[i][j][k]][k] # if dim == 1 // out[i][j][k] = input[i][j][index[i][j][k]] # if dim == 2 -// If 'a' and 'indices' do not have the same rank, then negative 'axis' is -// interpreted relative to 'a'; then both shapes are left-padded to the same rank; -// and indexing happens along the axis that corresponds to 'axis' in the padded shapes. -// Broadcasting is supported as usual. This way, this function can implement both -// batched and non-batched selection. +// 'a' and 'indices' must have the same rank. // @TODO: The current implementation does not support batched indices (third scenario above). // I.e. all axes of 'indices' except 'axis' must have dimension 1. struct GatherNodeOp : public NaryNodeOp { @@ -598,16 +594,14 @@ struct GatherNodeOp : public NaryNodeOp { } Shape newShape(Expr a, Expr indices, int axis) { - axis = a->shape().axis(axis); - auto indicesRank = indices->shape().size(); - ABORT_IF(axis >= indicesRank, "Axis {} is invalid for indices shape {}", axis, std::string(indices->shape())); Shape shape = a->shape(); - if (shape.size() < indicesRank) // pad - shape.resize(indicesRank); + auto rank = shape.size(); + ABORT_IF(rank != indices->shape().size(), "Mismatching shapes for input ({}) and indices ({})", std::string(shape), std::string(indices->shape())); + axis = a->shape().axis(axis); shape.set(axis, indices->shape()[axis]); #if 1 // presently, this implementation does not support batched indices - for (size_t i = 0; i < indicesRank; ++i) { - ABORT_IF(indices->shape()[i] != 1 && i + shape.size() - indicesRank != axis, + for (size_t i = 0; i < rank; ++i) { + ABORT_IF(indices->shape()[i] != 1 && i != axis, "Presently, select() does not implement batched indices"); } #endif From 6fc4d4b87f047e2b24fb5d5bfe3dbd68165382c9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 16:53:34 -0800 Subject: [PATCH 175/838] now routing rows() and cols() via index_select(), which then redistributes them to RowsNodeOp or ColsNodeOp; tests updated accordingly; bug fix: missed an axis normalization; bug fix: ReshapeNodeOp should pass on the value_type as to allow reshaping IndexType tensors --- src/graph/expression_operators.cpp | 45 +++++++++++++----------------- src/graph/expression_operators.h | 24 +++++++++++----- src/graph/node_operators_binary.h | 5 ++-- src/graph/node_operators_unary.h | 10 +++---- src/tests/operator_tests.cpp | 20 ++++++------- 5 files changed, 53 insertions(+), 51 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 54e68960a..27eef3eb4 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -249,35 +249,30 @@ Expr gather(Expr a, Expr indices, int axis) { return Expression(a, indices, axis); } -Expr index_select(Expr a, const std::vector& indices, int axis) { - auto indexExpr = a->graph()->indices(indices, a, axis); - return gather(a, indexExpr, axis); -} - -Expr rows(Expr a, Expr indices) { - // @TODO:: replace with `index_select(a, indices, -2)` - // as soon as select is efficient enough - return Expression(a, indices); -} - -Expr rows(Expr a, const std::vector& indices) { - auto indexExpr = a->graph()->indices(indices); - return rows(a, indexExpr); -} - -Expr cols(Expr a, Expr indices) { - // @TODO:: replace with `index_select(a, indices, -1)` - // as soon as select is efficient enough - return Expression(a, indices); +Expr index_select(Expr a, Expr indices, int axis) { + ABORT_IF(indices->shape().size() != 1, "Indices must be a 1D tensor"); + // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. + auto rank = a->shape().size(); + if (rank == 2) { + if (axis == 0) + return Expression(a, indices); + else if (axis == -1 || axis == 1) + return Expression(a, indices); + } + // Delegate to gather() for any other axis or non-matrix input. + Shape shape; + shape.resize(a->shape().size()); + shape.set(axis, indices->shape()[0]); + indices = reshape(indices, shape); // move index to axis + return gather(a, indices, axis); } - -Expr cols(Expr a, const std::vector& indices) { - auto indexExpr = a->graph()->indices(indices); - return cols(a, indexExpr); +Expr index_select(Expr a, const std::vector& indices, int axis) { + auto indexExpr = a->graph()->indices(indices); + return index_select(a, indexExpr, axis); } Expr sliceView(Expr a, const Slice& slice, int axis) { // numpy __getitem__ semantics - // @TODO: If not memory-consecutive then fall back to index_select + // @TODO: If not memory-consecutive then fall back to index_select(a, slice, axis) return Expression(a, slice, axis); } diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index f5379246d..6eba2b476 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -140,18 +140,28 @@ Expr flatten_2d(Expr a); Expr stopGradient(Expr a); Expr gather(Expr a, Expr indices, int axis); + +Expr index_select(Expr a, Expr indices, int axis); + +// convenience wrappers for index_select() Expr index_select(Expr a, const std::vector& indices, int axis); -static inline Expr index_select(Expr a, int index, int axis) { +static inline Expr index_select(Expr a, int index, int axis) { // scalar index version // Until Marian supports strides, use this for indexing non-memory-consecutive // slices, while sliceView() can be used for memory-consecutive ones. return index_select(a, std::vector({(IndexType)index}), axis); } - -Expr rows(Expr a, Expr indices); -Expr rows(Expr a, const std::vector& indices); - -Expr cols(Expr a, Expr indices); -Expr cols(Expr a, const std::vector& indices); +static inline Expr rows(Expr a, Expr indices) { + return index_select(a, indices, 0); +} +static inline Expr rows(Expr a, const std::vector& indices) { + return index_select(a, indices, 0); +} +static inline Expr cols(Expr a, Expr indices) { + return index_select(a, indices, -1); +} +static inline Expr cols(Expr a, const std::vector& indices) { + return index_select(a, indices, -1); +} Expr sliceView(Expr a, const Slice& slice, int axis); diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index b7c6d7064..8abba3c61 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -514,7 +514,7 @@ struct ScalarProductNodeOp : public NaryNodeOp { struct RowsNodeOp : public NaryNodeOp { RowsNodeOp(Expr a, Expr indices) : NaryNodeOp({a, indices}, newShape(a, indices->shape().elements())) { - matchOrAbort(indices->value_type()); + matchOrAbort(indices->value_type()); } NodeOps forwardOps() override { @@ -526,7 +526,6 @@ struct RowsNodeOp : public NaryNodeOp { return {NodeOp(PasteRows(child(0)->grad(), adj_, child(1)->val()))}; } - template Shape newShape(Expr a, size_t num) { Shape shape = a->shape(); ABORT_IF(shape.size() != 2, @@ -595,6 +594,7 @@ struct GatherNodeOp : public NaryNodeOp { Shape newShape(Expr a, Expr indices, int axis) { Shape shape = a->shape(); + axis = shape.axis(axis); auto rank = shape.size(); ABORT_IF(rank != indices->shape().size(), "Mismatching shapes for input ({}) and indices ({})", std::string(shape), std::string(indices->shape())); axis = a->shape().axis(axis); @@ -649,7 +649,6 @@ struct ColsNodeOp : public NaryNodeOp { return {NodeOp(PasteCols(child(0)->grad(), adj_, child(1)->val()))}; } - template Shape newShape(Expr a, size_t num) { Shape shape = a->shape(); shape.set(1, num); diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 462d81cf7..610928b2c 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -635,7 +635,6 @@ struct TransposeNodeOp : public UnaryNodeOp { return {NodeOp(TransposeNDGrad(child(0)->grad(), adj_, axesBw_))}; } - template Shape newShape(Expr a, const std::vector& axes) { Shape shape = a->shape(); @@ -680,8 +679,7 @@ class ReshapeNodeOp : public UnaryNodeOp { Expr reshapee_; public: - template - ReshapeNodeOp(Expr a, Shape shape) : UnaryNodeOp(a, shape), reshapee_(a) { + ReshapeNodeOp(Expr a, Shape shape) : UnaryNodeOp(a, shape, a->value_type()), reshapee_(a) { Node::destroy_ = false; } @@ -700,14 +698,14 @@ class ReshapeNodeOp : public UnaryNodeOp { Tensor& val() override { auto childVal = reshapee_->val(); val_.reset( - new TensorBase(childVal->memory(), shape(), childVal->getBackend())); + new TensorBase(childVal->memory(), shape(), childVal->type(), childVal->getBackend())); return val_; }; Tensor& grad() override { auto childGrad = reshapee_->grad(); adj_.reset( - new TensorBase(childGrad->memory(), shape(), childGrad->getBackend())); + new TensorBase(childGrad->memory(), shape(), childGrad->type(), childGrad->getBackend())); return adj_; }; @@ -748,7 +746,7 @@ class SliceViewNodeOp : public UnaryNodeOp { public: SliceViewNodeOp(Expr a, Slice slice, int axis) - : UnaryNodeOp(a, newShape(a, slice, axis)), viewedNode_(a), slice_(slice), axis_(axis) { + : UnaryNodeOp(a, newShape(a, slice, axis), a->value_type()), viewedNode_(a), slice_(slice), axis_(axis) { Node::destroy_ = false; auto byteStride = a->shape().stride(axis) * sizeOf(value_type()); byteOffset_ = slice.begin * byteStride; diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 8873655fa..f37189d61 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -634,13 +634,13 @@ void tests(DeviceType device) { auto A = graph->param("4x3", {4,3}, inits::from_vector(in)); // @TODO: split this into index_select() and gather() auto B1 = index_select(A, Indices({0}), 0); - auto B2 = index_select(A, Indices({0}), 1); - auto B3 = index_select(A, Indices({1}), -1); + auto B2 = index_select(A, 0, 1); + auto B3 = index_select(A, 1, -1); auto B4 = index_select(A, Indices({0, 1}), 0); auto C = graph->param("2x3x2", {2, 3, 2}, inits::from_vector(in)); - auto D1 = index_select(C, Indices({0}), 0); - auto D2 = index_select(C, Indices({2}), -2); + auto D1 = index_select(C, 0, 0); + auto D2 = index_select(C, 2, -2); auto D3 = index_select(C, Indices({0,2}), 1); auto S1 = step(A, 2, 0); @@ -694,19 +694,19 @@ void tests(DeviceType device) { CHECK(values == vS3); } - SECTION("rows/cols as index_select operations") { + SECTION("rows/cols as gather operations") { graph->clear(); values.clear(); std::vector values2; std::vector vA({0, .3333, -.2, -.3, 0, 4.5, 5.2, -10, 101.45, -100.05, 0, 1.05e-5}); - std::vector idx({0, 2}); + std::vector indices({0, 2}); auto A = graph->param("4x3", {4, 3}, inits::from_vector(vA)); - auto B1 = rows(A, idx); - auto B2 = index_select(A, idx, 0); - auto C1 = cols(A, idx); - auto C2 = index_select(A, idx, 1); + auto B1 = rows(A, indices); + auto B2 = gather(A, graph->indices(indices, A, 0), 0); + auto C1 = cols(A, indices); + auto C2 = gather(A, graph->indices(indices, A, 1), 1); graph->forward(); CHECK(B1->shape() == B2->shape()); From 55c07775e2832270a926ab6082fd7ca774c7ffc4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 19 Jan 2019 17:12:22 -0800 Subject: [PATCH 176/838] bug fix: SliceViewNodeOp should forward value_type() correctly --- src/graph/node_operators_unary.h | 4 ++-- src/tests/operator_tests.cpp | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 610928b2c..51433f933 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -781,14 +781,14 @@ class SliceViewNodeOp : public UnaryNodeOp { Tensor& val() override { auto childVal = viewedNode_->val(); auto mem = New(childVal->memory()->data() + byteOffset_, byteSize_); - val_.reset(new TensorBase(mem, shape(), childVal->getBackend())); + val_.reset(new TensorBase(mem, shape(), childVal->type(), childVal->getBackend())); return val_; }; Tensor& grad() override { auto childGrad = viewedNode_->grad(); auto mem = New(childGrad->memory()->data() + byteOffset_, byteSize_); - adj_.reset(new TensorBase(mem, shape(), childGrad->getBackend())); + adj_.reset(new TensorBase(mem, shape(), childGrad->type(), childGrad->getBackend())); return adj_; }; diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index f37189d61..839ab4313 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -632,7 +632,6 @@ void tests(DeviceType device) { std::vector vS3({7, -8, 9, -10, 11, -12}); auto A = graph->param("4x3", {4,3}, inits::from_vector(in)); - // @TODO: split this into index_select() and gather() auto B1 = index_select(A, Indices({0}), 0); auto B2 = index_select(A, 0, 1); auto B3 = index_select(A, 1, -1); From dad3afe8a5baeda015b4f13748de5be196742148 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 20 Jan 2019 18:59:40 -0800 Subject: [PATCH 177/838] various small fixes --- src/data/default_vocab.cpp | 44 ++++++++++++++++++++++++------ src/graph/expression_operators.cpp | 10 ++++--- src/graph/node_initializers.cpp | 36 +++++++++++++++++++++++- src/graph/node_initializers.h | 12 ++++---- src/layers/generic.h | 17 ++++++++---- src/models/bert.h | 16 ++++------- src/models/model_factory.cpp | 6 ++-- src/models/transformer.h | 13 +++++---- 8 files changed, 111 insertions(+), 43 deletions(-) diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 4557830c1..f5d009b2a 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -16,7 +16,7 @@ namespace marian { class DefaultVocab : public VocabBase { -private: +protected: typedef std::map Str2Id; Str2Id str2id_; @@ -36,7 +36,7 @@ class DefaultVocab : public VocabBase { VocabFreqOrderer(const std::unordered_map& counter) : counter_(counter) {} - // order first by decreasing frequency, + // order first by decreasing frequency, // if frequencies are the same order lexicographically by vocabulary string bool operator()(const std::string& a, const std::string& b) const { return counter_.at(a) > counter_.at(b) || (counter_.at(a) == counter_.at(b) && a < b); @@ -157,7 +157,7 @@ class DefaultVocab : public VocabBase { "Vocabulary file '{}' exists. Not overwriting", path.string()); } - + std::unordered_map counter; for(const auto& trainPath : trainPaths) addCounts(counter, trainPath); @@ -221,9 +221,9 @@ class DefaultVocab : public VocabBase { } } - void create(const std::string& vocabPath, - const std::unordered_map& counter, - size_t maxSize = 0) { + virtual void create(const std::string& vocabPath, + const std::unordered_map& counter, + size_t maxSize = 0) { std::vector vocabVec; for(auto& p : counter) @@ -283,11 +283,37 @@ class DefaultVocab : public VocabBase { }; }; -// This is a vocabulary class that does not enforce or . -// This is used for class lists in a classifier. +// This is a vocabulary class that does not enforce or . +// This is used for class lists in a classifier. class ClassVocab : public DefaultVocab { private: - virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override {} // Do nothing. + // Do nothing. + virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override {} + + // Not adding special class labels, only seen classes. + virtual void create(const std::string& vocabPath, + const std::unordered_map& counter, + size_t maxSize = 0) override { + + std::vector vocabVec; + for(auto& p : counter) + vocabVec.push_back(p.first); + std::sort(vocabVec.begin(), vocabVec.end(), VocabFreqOrderer(counter)); + + ABORT_IF(maxSize != 0 && vocabVec.size() != maxSize, + "Class vocab maxSize given ({}) has to match class vocab size ({})", + maxSize, vocabVec.size()); + + YAML::Node vocabYaml; + for(size_t i = 0; i < vocabVec.size(); ++i) + vocabYaml.force_insert(vocabVec[i], i); + + std::unique_ptr vocabStrm( + vocabPath == "stdout" ? new io::OutputFileStream(std::cout) + : new io::OutputFileStream(vocabPath) + ); + *vocabStrm << vocabYaml; + } }; Ptr createDefaultVocab() { diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 4694351b2..ecda67751 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -474,8 +474,9 @@ Expr plus(const std::vector&) { ABORT("Not implemented"); } -Expr swish(const std::vector&) { - ABORT("Not implemented"); +Expr swish(const std::vector& nodes) { + ABORT_IF(nodes.size() > 1, "Not implemented"); + return swish(nodes[0]); } Expr tanh(const std::vector& nodes) { @@ -486,8 +487,9 @@ Expr sigmoid(const std::vector&) { ABORT("Not implemented"); } -Expr relu(const std::vector&) { - ABORT("Not implemented"); +Expr relu(const std::vector& nodes) { + ABORT_IF(nodes.size() > 1, "Not implemented"); + return relu(nodes[0]); } Expr leakyrelu(const std::vector&) { diff --git a/src/graph/node_initializers.cpp b/src/graph/node_initializers.cpp index da38d2ae8..f405102ec 100755 --- a/src/graph/node_initializers.cpp +++ b/src/graph/node_initializers.cpp @@ -50,16 +50,50 @@ NodeInitializer normal(float mean, float stddev) { }; } +// @TODO: replace with parametered version below void glorot_uniform(Tensor tensor) { float scale = sqrtf(6.0f / (tensor->shape()[-2] + tensor->shape()[-1])); uniform(-scale, scale)(tensor); } +NodeInitializer glorot_uniform2(bool fanIn, bool fanOut) { + return [fanIn, fanOut](Tensor tensor) { + float scale = 1.f; + if(fanIn && fanOut) + scale = sqrtf(6.0f / (tensor->shape()[-2] + tensor->shape()[-1])); + else if(!fanIn && fanOut) + scale = sqrtf(3.0f / tensor->shape()[-1]); + else if(fanIn && !fanOut) + scale = sqrtf(3.0f / tensor->shape()[-2]); + else + ABORT("You need to set fanIn or fanOut or both to true"); + + uniform(-scale, scale)(tensor); + }; +} + +// @TODO: replace with parametered version below void glorot_normal(Tensor tensor) { float scale = sqrtf(2.0f / (tensor->shape()[-2] + tensor->shape()[-1])); normal(0.f, scale)(tensor); } +NodeInitializer glorot_normal2(bool fanIn, bool fanOut) { + return [fanIn, fanOut](Tensor tensor) { + float scale = 1.f; + if(fanIn && fanOut) + scale = sqrtf(2.0f / (tensor->shape()[-2] + tensor->shape()[-1])); + else if(!fanIn && fanOut) + scale = sqrtf(1.0f / tensor->shape()[-1]); + else if(fanIn && !fanOut) + scale = sqrtf(1.0f / tensor->shape()[-2]); + else + ABORT("You need to set fanIn or fanOut or both to true"); + + normal(0.f, scale)(tensor); + }; +} + NodeInitializer bernoulli(float prob, float scale) { return [prob, scale](Tensor tensor) { Bernoulli(tensor, prob, scale); @@ -157,7 +191,7 @@ NodeInitializer from_item(const io::Item& item) { } } -// Computes Google's sinusoidal position embeddings +// Computes Google's sinusoidal position embeddings NodeInitializer sinusoidalPositionEmbeddings(int start) { return [start](Tensor t) { int dimEmb = t->shape()[-1]; diff --git a/src/graph/node_initializers.h b/src/graph/node_initializers.h index 254f02ae2..fbb073482 100755 --- a/src/graph/node_initializers.h +++ b/src/graph/node_initializers.h @@ -26,8 +26,10 @@ NodeInitializer normal(float mean = 0.f, float stddev = 1.f); NodeInitializer uniform(float a = 0.f, float b = 1.f); void glorot_uniform(Tensor t); +NodeInitializer glorot_uniform2(bool fanIn = true, bool fanOut = true); void glorot_normal(Tensor t); +NodeInitializer glorot_normal2(bool fanIn = true, bool fanOut = true); NodeInitializer bernoulli(float p, float scale = 1.f); @@ -53,15 +55,15 @@ NodeInitializer from_word2vec(const std::string& file, bool normalize = false); /** - * Computes Google's sinusoidal position embeddings + * Computes Google's sinusoidal position embeddings * starting from position 'start' taking into account * batch and time dimensions of the tensor. - * + * * Expected tensor layout {-2: time, -1: model} - * - * Usually gets later reshaped to {time, 1, model} and + * + * Usually gets later reshaped to {time, 1, model} and * added with a broadcast to learned embeddings. Positional - * embeddings are the same for each batch entry and change + * embeddings are the same for each batch entry and change * over time. */ NodeInitializer sinusoidalPositionEmbeddings(int start); diff --git a/src/layers/generic.h b/src/layers/generic.h index 074494910..062b59cc8 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -50,7 +50,7 @@ struct IUnaryLayer { // Embedding from corpus sub-batch to (emb, mask) struct IEmbeddingLayer { virtual std::tuple apply(Ptr subBatch) const = 0; - + // alternative version from index vector, and with batch dims/shape virtual Expr apply(const std::vector& embIdx, const Shape& shape) const = 0; }; @@ -212,7 +212,9 @@ class Embedding : public LayerBase, public IEmbeddingLayer { bool fixed = opt("fixed", false); - NodeInitializer initFunc = inits::glorot_uniform; + // Embedding layer initialization should depend only on embedding size, hence fanIn=false + NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); + if (options_->has("embFile")) { std::string file = opt("embFile"); if (!file.empty()) { @@ -253,7 +255,10 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { int dimEmb = opt("dimEmb"); int dimUlrEmb = opt("dimUlrEmb"); // ULR mono embed size bool fixed = opt("fixed", false); - NodeInitializer initFunc = inits::glorot_uniform; + + // Embedding layer initialization should depend only on embedding size, hence fanIn=false + NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); + std::string queryFile = opt("ulrQueryFile"); std::string keyFile = opt("ulrKeysFile"); bool trainTrans = opt("ulrTrainTransform", false); @@ -326,15 +331,15 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb*dimUlrEmb auto sqrtDim=std::sqrt((float)queryEmbeddings->shape()[-1]); qt = qt/sqrtDim; // normalize accordin to embed size to avoid dot prodcut growing large in magnitude with larger embeds sizes - auto z = dot(qt, keyEmbed, false, true); // query-key similarity + auto z = dot(qt, keyEmbed, false, true); // query-key similarity float dropProb = this->options_->get("ulr-dropout", 0.0f); // default no dropout z = dropout(z, dropProb); float tau = this->options_->get("ulr-softmax-temperature", 1.0f); // default no temperature // temperature in softmax is to control randomness of predictions // high temperature Softmax outputs are more close to each other - // low temperatures the softmax become more similar to "hardmax" + // low temperatures the softmax become more similar to "hardmax" auto weights = softmax(z / tau); // assume default is dim=-1, what about temprature? - scaler ?? - auto chosenEmbeddings = dot(weights, uniEmbed); // AVERAGE + auto chosenEmbeddings = dot(weights, uniEmbed); // AVERAGE auto chosenEmbeddings_mix = srcEmbeddings + alpha * chosenEmbeddings; // this should be elementwise broadcast auto batchEmbeddings = reshape(chosenEmbeddings_mix, { dimWords, dimBatch, dimEmb }); auto graph = ulrEmbeddings_.front()->graph(); diff --git a/src/models/bert.h b/src/models/bert.h index 288d06c8c..849144b1a 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -191,7 +191,7 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { opt("bert-sep-symbol"), opt("bert-class-symbol")); } else if(modelType == "bert-classifier") { // we are probably fine-tuning a BERT model for a classification task - bertBatch = New(batch, + bertBatch = New(batch, opt("bert-sep-symbol"), opt("bert-class-symbol")); // only annotate sentence separators } else { @@ -219,7 +219,7 @@ class BertEncoder : public EncoderTransformer { Expr addSentenceEmbeddings(Expr embeddings, Ptr batch, bool learnedPosEmbeddings) const { - + Ptr bertBatch = std::dynamic_pointer_cast(batch); ABORT_IF(!bertBatch, "Batch must be BertBatch for BERT training or fine-tuning"); @@ -274,18 +274,14 @@ class BertClassifier : public ClassifierBase { int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels - std::string finetune = ""; - if(opt("original-type") == "bert-classifier") // seems we are fine-tuning - finetune = "_finetune"; // change name so we do not relead BERT output layers for fine-tuning - auto output = mlp::mlp() // .push_back(mlp::dense() // - ("prefix", prefix_ + finetune + "_ff_logit_l1") // + ("prefix", prefix_ + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh)) // @TODO: do we actually need this? .push_back(mlp::output() // ("dim", dimTrgCls)) // - ("prefix", prefix_ + finetune + "_ff_logit_l2") // + ("prefix", prefix_ + "_ff_logit_l2") // .construct(graph); auto logits = output->apply(classEmbeddings); // class logits for each batch entry @@ -332,11 +328,11 @@ class BertMaskedLM : public ClassifierBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; auto layerTanh = mlp::dense() // - ("prefix", prefix_ + "_ff_logit_maskedlm_out_l1") // + ("prefix", prefix_ + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh); // @TODO: again, check if this layer is present in original code auto layerOut = mlp::output() // - ("prefix", prefix_ + "_ff_logit_maskedlm_out_l2") // + ("prefix", prefix_ + "_ff_logit_l2") // ("dim", dimVoc); // layerOut.tieTransposed("Wemb"); // We are a BERT model, hence tie with input, @TODO: check if this is actually what Google does diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 66f2db928..9eb1931ca 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -234,11 +234,13 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("usage", use) // .push_back(models::encoder() // ("type", "bert-encoder") // close to original transformer encoder - ("index", 0)) // + ("index", 0)) // .push_back(models::classifier() // + ("prefix", "masked-lm") // prefix for parameter names ("type", "bert-masked-lm") // ("index", 0)) // multi-task learning with MaskedLM .push_back(models::classifier() // + ("prefix", "next-sentence") // prefix for parameter names ("type", "bert-classifier") // ("index", 1)) // next sentence prediction .construct(graph); @@ -246,7 +248,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { if(type == "bert-classifier") { // for BERT fine-tuning on non-BERT classification task return models::encoder_classifier()(options) // - ("original-type", "bert-classifier") // so we can query this + ("original-type", "bert-classifier") // so we can query this if needed ("usage", use) // .push_back(models::encoder() // ("type", "bert-encoder") // diff --git a/src/models/transformer.h b/src/models/transformer.h index bbdaa1343..a1cf4cb58 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -47,18 +47,18 @@ class Transformer : public EncoderOrDecoderBase { static Expr transposeTimeBatch(Expr input) { return transpose(input, {0, 2, 1, 3}); } - Expr addPositionalEmbeddings(Expr input, int start = 0, bool learnedPosEmbeddings = false) const { + Expr addPositionalEmbeddings(Expr input, int start = 0, bool trainPosEmbeddings = false) const { int dimEmb = input->shape()[-1]; int dimWords = input->shape()[-3]; Expr embeddings = input; - if(learnedPosEmbeddings) { + if(trainPosEmbeddings) { int maxLength = opt("max-length"); // Hack for translating with length longer than trained embeddings // We check if the embedding matrix "Wpos" already exist so we can - // check the number of positions in that loaded parameter. + // check the number of positions in that loaded parameter. // We then have to restict the maximum length to the maximum positon // and positions beyond this will be the maximum position. Expr seenEmb = graph_->get("Wpos"); @@ -75,14 +75,15 @@ class Transformer : public EncoderOrDecoderBase { for(int i = 0; i < std::min(dimWords, numPos); ++i) positions[i] = i; - // @TODO : test if embeddings should be scaled here too! auto signal = embeddingLayer->apply(positions, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; } else { - auto signal = graph_->constant({dimWords, 1, dimEmb}, - inits::sinusoidalPositionEmbeddings(start)); + // @TODO : test if embeddings should be scaled when trainable // according to paper embeddings are scaled up by \sqrt(d_m) embeddings = std::sqrt((float)dimEmb) * embeddings; + + auto signal = graph_->constant({dimWords, 1, dimEmb}, + inits::sinusoidalPositionEmbeddings(start)); embeddings = embeddings + signal; } From 5197012386427b1c75cd44c153a3661af82d2d30 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 20 Jan 2019 19:24:06 -0800 Subject: [PATCH 178/838] remove disp-label-index for now --- src/common/config_parser.cpp | 4 ++-- src/training/scheduler.h | 17 +++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index e192b995a..338d4b82c 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -288,8 +288,8 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Display nformation for the first arg updates"); cli.add("--disp-label-counts", "Display label counts when logging loss progress"); - cli.add("--disp-label-index", - "Display label counts based on i-th input stream (-1 is last)", -1); +// cli.add("--disp-label-index", +// "Display label counts based on i-th input stream (-1 is last)", -1); cli.add("--save-freq", "Save model file every arg updates (append 't' for every arg target labels)", "10000u"); diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 891a2c0e7..65bb88207 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -15,7 +15,6 @@ class Scheduler : public TrainingObserver { std::vector> validators_; bool first_{true}; - int dispIndex_{-1}; timer::Timer timer_, heartBeatTimer_; @@ -68,9 +67,9 @@ class Scheduler : public TrainingObserver { state.updateEta(baselr); } - std::string formatLoss(std::string lossType, - bool dispLabelCounts, - size_t batchLabels, + std::string formatLoss(std::string lossType, + bool dispLabelCounts, + size_t batchLabels, Ptr state) { std::stringstream ss; ss << "Cost "; @@ -139,8 +138,7 @@ class Scheduler : public TrainingObserver { } Scheduler(Ptr options, Ptr state) - : options_(options), state_(state), - dispIndex_{options_->get("disp-label-index", -1)} { + : options_(options), state_(state) { ABORT_IF(state_->factor != 1, "state.factor unexpectedly not 1 at this point??"); updateLearningRate(*state); } @@ -258,10 +256,9 @@ class Scheduler : public TrainingObserver { } // @TODO: go back to function which takes batch as an argument? The current arguments make it hard to choose - // which subbatch should be used for speed display. For sequence-classifiers it's more interesting to see the - // source-words consumed rather than the labels. We have a CLI option '--disp-label-index' (bad name?) which is - // now defunct. - void update(StaticLoss rationalLoss, + // which subbatch should be used for speed display. For sequence-classifiers it's more interesting to see the + // source-words consumed rather than the labels. + void update(StaticLoss rationalLoss, size_t numReadBatches, // number of batches read by the reader (for seeking in case of restart) size_t batchSize, // total number of sentences in batch size_t batchLabels, // total number of target words in batch From 1712ccb35490cbee5f31e394bb9a20200ff147f8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 20 Jan 2019 19:57:39 -0800 Subject: [PATCH 179/838] add missing validation check on input types --- src/data/corpus_base.cpp | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index b6bd7042a..74987882f 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -105,7 +105,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) std::set paths; // contains all paths that are used for training the vocabulary size_t size; // contains the maximum vocabulary size }; - + // Group training files based on vocabulary path. If the same // vocab path corresponds to different training files, this means // that a single vocab should combine tokens from all files. @@ -124,7 +124,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) auto pathsAndSize = groupVocab[vocabPaths[i]]; std::vector groupedPaths(pathsAndSize.paths.begin(), pathsAndSize.paths.end()); size_t vocSize = vocab->loadOrCreate(vocabPaths[i], groupedPaths, pathsAndSize.size); - + // TODO: this is not nice as it modifies the option object and needs to expose the changes // outside the corpus as models need to know about the vocabulary size; extract the vocab // creation functionality from the class. @@ -287,31 +287,36 @@ void CorpusBase::addWeightsToBatch(Ptr batch, void CorpusBase::initEOS(bool training = true) { // Labels fed into sub-batches that are just class-labels, not sequence labels do not require to // add a EOS symbol. Hence decision to add EOS is now based on input stream positions and correspoding - // input type. - + // input type. + addEOS_.resize(paths_.size(), true); // @TODO: think if this should be checked and processed here or in a validation step in config? auto inputTypes = options_->get>("input-types", {}); // empty list by default - + // make sure there is an input type for each path - ABORT_IF(inputTypes.size() > 0 && inputTypes.size() < paths_.size(), - "Input types have been specified ({}), you need to specify one per input ({})", - inputTypes.size(), + ABORT_IF(inputTypes.size() > 0 && inputTypes.size() < paths_.size(), + "Input types have been specified ({}), you need to specify one per input ({})", + inputTypes.size(), paths_.size()); // make sure there is an equal number of input types and paths when training - ABORT_IF(training && inputTypes.size() > 0 && inputTypes.size() != paths_.size(), - "Input types have been specified ({}), you need to specify one per input ({})", - inputTypes.size(), + ABORT_IF(training && inputTypes.size() > 0 && inputTypes.size() != paths_.size(), + "Input types have been specified ({}), you need to specify one per input ({})", + inputTypes.size(), paths_.size()); - + for(int i = 0; i < paths_.size(); ++i) - if(inputTypes[i] == "class") - addEOS_[i] = false; - else if(inputTypes[i] == "sequence") + if(inputTypes.size() > i) { + if(inputTypes[i] == "class") + addEOS_[i] = false; + else if(inputTypes[i] == "sequence") + addEOS_[i] = true; + else + ABORT("Unknown input type {}: {}", i, inputTypes[i]); + } else { + // No input type specified, assuming "sequence" addEOS_[i] = true; - else - ABORT("Unknown input type {}: {}", i, inputTypes[i]); + } } } // namespace data From e5e85e426a6b76c0411256c0cc4b9764e34f7e93 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 20 Jan 2019 23:41:39 -0800 Subject: [PATCH 180/838] formatting --- src/models/transformer.h | 2 +- src/optimizers/optimizers.cpp | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/models/transformer.h b/src/models/transformer.h index a1cf4cb58..057c7cca9 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -285,7 +285,7 @@ class Transformer : public EncoderOrDecoderBase { auto Wk = graph_->param(prefix + "_Wk", {dimModel, dimModel}, inits::glorot_uniform); auto bk = graph_->param(prefix + "_bk", {1, dimModel}, inits::zeros); - kh = affine(keys,Wk, bk); // [-4: beam depth, -3: batch size, -2: max length, -1: vector dim] + kh = affine(keys, Wk, bk); // [-4: beam depth, -3: batch size, -2: max length, -1: vector dim] kh = SplitHeads(kh, dimHeads); // [-4: batch size, -3: num heads, -2: max length, -1: split vector dim] cache_[prefix + "_keys"] = kh; } diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index 685c917f9..e1de4068a 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -55,7 +55,6 @@ void Adagrad::load(const std::string& name, std::vector vGt; - // @TODO: use new IO auto items = io::loadItems(name); for(auto item : items) { // get the size of gt_ @@ -64,8 +63,7 @@ void Adagrad::load(const std::string& name, // extract data into vectors if(item.name == "adagrad_gt") { vGt.resize(totalSize); - std::copy( - (float*)item.data(), ((float*)item.data()) + totalSize, vGt.begin()); + std::copy((float*)item.data(), ((float*)item.data()) + totalSize, vGt.begin()); } } if(vGt.empty()) { @@ -79,7 +77,7 @@ void Adagrad::load(const std::string& name, if(!opt->gt_) { if(!opt->alloc_) opt->alloc_ = New(backends[localDeviceIndex]); - auto size = end-begin; + auto size = end - begin; opt->alloc_->reserveExact(sizeof(float) * size); opt->alloc_->allocate(opt->gt_, {1, (int)size}); } @@ -111,8 +109,7 @@ void Adagrad::save(const std::string& name, item.shape = Shape({1, (int)vGt.size()}); item.type = Type::float32; item.bytes.resize(vGt.size() * sizeOf(item.type)); - std::copy( - (char*)vGt.data(), (char*)(vGt.data() + vGt.size()), item.bytes.begin()); + std::copy((char*)vGt.data(), (char*)(vGt.data() + vGt.size()), item.bytes.begin()); io::saveItems(name, {item}); } From e94b627b949f9b4a1a2f11d5811749ac27eb8478 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 21 Jan 2019 12:47:25 -0800 Subject: [PATCH 181/838] towards explicit normalization of output factors --- src/layers/generic.cpp | 49 +++++++++++++++++++++++++++++++----------- src/layers/loss.cpp | 3 ++- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 347d4a833..c0812ae7b 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -101,10 +101,20 @@ namespace marian { groupRanges_[g].second = u + 1; groupCounts[g]++; } + // determine if a factor needs explicit softmax normalization + groupNeedsNormalization_.resize(numGroups, false); for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - //ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members are not consecutive in the factor vocabulary", groupPrefixes[g]); LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); + // any factor that is not referenced in all words and is not a sigmoid needs normalization + if (g == 0) // @TODO: For now we assume that the main factor is used in all words. Test this. + continue; + if (groupCounts[g] == 1) // sigmoid factors have no normalizer + continue; + groupNeedsNormalization_[g] = true; // needed + ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], + "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); + LOG(info, "[embedding] Factor group '{}' needs needs explicit normalization ({}..{})", groupPrefixes[g], groupRanges_[g].first, groupRanges_[g].second-1); } // create the factor matrix @@ -143,7 +153,9 @@ namespace marian { std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ CSRData factorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v std::vector factorGroups_; // [u] -> group id of factor u + public: // @TODO: temporarily; later factor this properly std::vector> groupRanges_; // [group id] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. + std::vector groupNeedsNormalization_; // [group id] -> true if explicit softmax normalization is necessary }; namespace mlp { @@ -186,23 +198,34 @@ namespace marian { } return affine(input, cachedShortW_, cachedShortb_, false, transposeW_); } - else { - auto y = affine(input, W_, b_, false, transposeW_); + else if (embeddingFactorMapping_) { + auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] - if (embeddingFactorMapping_) { // note: presently mutually incompatible with shortlist_ - auto graph = input->graph(); - auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] - y = dot_csr( // the CSR matrix is passed in pieces - y, // [B x U] - factorMatrix.shape, - graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), - graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), - graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), - /*transB=*/ true); // -> [B x V] + auto graph = input->graph(); + auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] + y = dot_csr( // the CSR matrix is passed in pieces + y, // [B x U] + factorMatrix.shape, + graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), + graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), + graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), + /*transB=*/ true); // -> [B x V] + + // apply normalization factors + const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly + auto numGroups = groupRanges.size(); + for (size_t g = 0; g < numGroups; g++) { + if (!embeddingFactorMapping_->groupNeedsNormalization_[g]) + continue; + auto range = groupRanges[g]; + // need to compute log denominator over y[range] and subtract it from y[range] + auto groupLogits = narrow(y, range.first, range.second, /*axis=*/-1); } return y; } + else + return affine(input, W_, b_, false, transposeW_); } } diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index e87cac865..d11e4384c 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -37,6 +37,7 @@ Expr LossBase::getCrossEntropy(Expr logits, #else // alternative that is cheaper memory-wise ce = cross_entropy(logits, indices); auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceq; //auto ceq = mean(logits, /*axis=*/ -1) - Z; //ce = (1 - smoothing_) * cols(logits, indices) // ce term // - smoothing_ * mean(logits, /*axis=*/ -1) // smoothing term @@ -44,7 +45,7 @@ Expr LossBase::getCrossEntropy(Expr logits, #endif } else - ce = cross_entropy(logits, indices); + ce = cross_entropy(logits, indices); if(mask) ce = ce * mask; From e8448f6506df3420c0eba4cdcfcd43d1f2b309ff Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 21 Jan 2019 14:39:04 -0800 Subject: [PATCH 182/838] further disentangling of index operators --- src/graph/expression_operators.cpp | 33 +++++++- src/graph/expression_operators.h | 24 +++--- src/graph/node_operators_binary.h | 20 +++-- src/models/transformer.h | 2 +- src/tests/operator_tests.cpp | 116 ++++++++++++++--------------- 5 files changed, 112 insertions(+), 83 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 27eef3eb4..81310b438 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -245,10 +245,12 @@ Expr constant_like(Expr a, const NodeInitializer& init) { return graph->constant(shape, init); } +// gather() -- gather arbitrary elements along an axis; batched or non-batched Expr gather(Expr a, Expr indices, int axis) { return Expression(a, indices, axis); } +// index_select() -- gather arbitrary elements along an axis; unbatched (indices are specified as a 1D vector) Expr index_select(Expr a, Expr indices, int axis) { ABORT_IF(indices->shape().size() != 1, "Indices must be a 1D tensor"); // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. @@ -271,11 +273,38 @@ Expr index_select(Expr a, const std::vector& indices, int axis) { return index_select(a, indexExpr, axis); } -Expr sliceView(Expr a, const Slice& slice, int axis) { // numpy __getitem__ semantics - // @TODO: If not memory-consecutive then fall back to index_select(a, slice, axis) +static Expr sliceCopy(Expr a, const Slice& slice, int axis) { // copy a Slice via gather() + ABORT_IF(slice.stride < 0, "Negative strides are not supported yet"); + ABORT_IF(slice.begin == slice.end, "Empty slices are not allowed"); // @TODO: Or are they? + std::vector indices; + indices.reserve((slice.end - slice.begin - 1) / slice.stride + 1); + for (int i = slice.begin; i < slice.end; i += slice.stride) + indices.push_back((IndexType)i); + return gather(a, a->graph()->indices(indices, a, axis), axis); +} + +static Expr sliceView(Expr a, const Slice& slice, int axis) { // view a slice (must be memory-consecutive) return Expression(a, slice, axis); } +// slice() -- gather a slice along an axis (step size > 1 allowed) +Expr slice(Expr a, Slice slice, int axis) { // numpy __getslice__ semantics, but with axis parameter + const auto& shape = a->shape(); + axis = shape.axis(axis); // normalize negative axis + slice = shape.slice(slice, axis); // normalize negative slice values + if (slice.begin == 0 && slice.end == shape[axis] && slice.stride == 1) + return a; // it's a no-op +#if 1 // until strided views are supported, non-consecutive slices are implemented via gather() + if (slice.stride != 1) + return sliceCopy(a, slice, axis); + for (int i = 0; i < axis; ++i) { + if (shape[i] != 1) // this makes it non-consecutive + return sliceCopy(a, slice, axis); + } +#endif + return sliceView(a, slice, axis); +} + Expr sum(Expr a, int ax) { return Expression(a, ax); } diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 6eba2b476..5902f2af2 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -145,32 +145,28 @@ Expr index_select(Expr a, Expr indices, int axis); // convenience wrappers for index_select() Expr index_select(Expr a, const std::vector& indices, int axis); -static inline Expr index_select(Expr a, int index, int axis) { // scalar index version - // Until Marian supports strides, use this for indexing non-memory-consecutive - // slices, while sliceView() can be used for memory-consecutive ones. - return index_select(a, std::vector({(IndexType)index}), axis); -} static inline Expr rows(Expr a, Expr indices) { return index_select(a, indices, 0); } -static inline Expr rows(Expr a, const std::vector& indices) { - return index_select(a, indices, 0); +static inline Expr rows(Expr a, const std::vector& indexVector) { + return index_select(a, indexVector, 0); } static inline Expr cols(Expr a, Expr indices) { return index_select(a, indices, -1); } -static inline Expr cols(Expr a, const std::vector& indices) { - return index_select(a, indices, -1); +static inline Expr cols(Expr a, const std::vector& indexVector) { + return index_select(a, indexVector, -1); } -Expr sliceView(Expr a, const Slice& slice, int axis); +Expr slice(Expr a, Slice slice, int axis); -static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name - return sliceView(a, Slice((int)start, (int)(start + length)), axis); +// convenience wrappers for slice() +static inline Expr step(Expr a, int step, int axis) { // @TODO: name is too narrow + return slice(a, Slice(step), axis); } -static inline Expr step(Expr a, int step, int axis) { - return sliceView(a, Slice(step), axis); +static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name + return slice(a, Slice((int)start, (int)(start + length)), axis); } /*********************************************************/ diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 8abba3c61..22109fbc6 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -542,7 +542,7 @@ struct RowsNodeOp : public NaryNodeOp { // This operation gathers elements of a tensor along an axis. // This is like PyTorch gather(). // For example, this can be used for: -// - Same index applied to all batch items (today's select()): +// - Same index applied to all batch items: // 'index' has 1 in the axes that match batch axes in the input, and axis set to the one axis that gets selected over. // Example: Selecting Transformer head 0, i.e. return a[:,1,:,:] // axis = -3 @@ -557,7 +557,7 @@ struct RowsNodeOp : public NaryNodeOp { // idx: (#(B*S)#, 1) B=batch size, S=source length, idx values are in range 0..V-1 // out: ( (B*S) , E) out[b, s, e] == e[/*0,*/ idx[b, s, 0], e] // - Batched selection (x-ent scenario): Both 'index' and 'data' have matching batch axes. -// Example: Cross-entropy loss as -select(logSoftmax(logits), groundTruth, axis=-1): +// Example: Cross-entropy loss as -gather(logSoftmax(logits), groundTruth, axis=-1): // axis = -1 // lp : (B, T, V ) B=batch size, T=trg length, V=vocab size // idx: (B, T, #1#) idx values are in range 0..V-1 @@ -596,19 +596,23 @@ struct GatherNodeOp : public NaryNodeOp { Shape shape = a->shape(); axis = shape.axis(axis); auto rank = shape.size(); - ABORT_IF(rank != indices->shape().size(), "Mismatching shapes for input ({}) and indices ({})", std::string(shape), std::string(indices->shape())); + ABORT_IF(rank != indices->shape().size(), "Mismatching ranks for input ({}) and indices ({})", std::string(shape), std::string(indices->shape())); axis = a->shape().axis(axis); shape.set(axis, indices->shape()[axis]); -#if 1 // presently, this implementation does not support batched indices for (size_t i = 0; i < rank; ++i) { - ABORT_IF(indices->shape()[i] != 1 && i != axis, - "Presently, select() does not implement batched indices"); - } + if (i != axis) { + ABORT_IF(indices->shape()[i] != shape[i] && indices->shape()[i] != 1, + "Dimensions must match or broadcast for input ({}) and indices ({})", std::string(shape), std::string(indices->shape())); +#if 1 // presently, this implementation does not support batched indices + ABORT_IF(indices->shape()[i] != 1, + "Presently, gather() does not implement batched indices"); #endif + } + } return shape; } - const std::string type() override { return "select"; } + const std::string type() override { return "gather"; } const std::string color() override { return "orange"; } diff --git a/src/models/transformer.h b/src/models/transformer.h index 166c68722..3c540c12a 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -194,7 +194,7 @@ class Transformer : public EncoderOrDecoderBase { // @TODO: make splitting obsolete alignments_.clear(); for(int i = 0; i < trgWords; ++i) { - alignments_.push_back(index_select(head0, i, -1)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] + alignments_.push_back(marian::step(head0, i, -1)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] } } diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 839ab4313..34d8b03c9 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -613,84 +613,84 @@ void tests(DeviceType device) { CHECK( values == values2 ); } - SECTION("select, step, sliceView operators") { - using Indices = std::vector; + SECTION("select, step, slice operators") { + using IndexVector = std::vector; graph->clear(); values.clear(); - std::vector in({1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12}); + std::vector vA({ 1, -2, 3, + -4, 5, -6, + 7, -8, 9, + -10, 11, -12}); + std::vector vC({ 1, -2, // C = np.array([1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12]).reshape((2, 3, 2)) + 3, -4, + 5, -6, + + 7, -8, + 9, -10, + 11, -12 }); std::vector vB1({1, -2, 3}); std::vector vB2({1, -4, 7, -10}); std::vector vB3({-2, 5, -8, 11}); std::vector vB4({1, -2, 3, -4, 5, -6}); std::vector vD1(vB4); std::vector vD2({5, -6, 11, -12}); - std::vector vD3({1, -2, 5, -6, 7, -8, 11, -12}); + std::vector vD3({1, -2, 5, -6, 7, -8, 11, -12}); // C[:,(0,2),:] + //std::vector vD4({5, -6, 3, -4, 7, -8, 11, -12}); // [C[0,(2,1),:],C[1,(0,2),:]] std::vector vS1({7, -8, 9}); std::vector vS2({-4, 5, -6, 7, -8, 9}); std::vector vS3({7, -8, 9, -10, 11, -12}); - auto A = graph->param("4x3", {4,3}, inits::from_vector(in)); - auto B1 = index_select(A, Indices({0}), 0); - auto B2 = index_select(A, 0, 1); - auto B3 = index_select(A, 1, -1); - auto B4 = index_select(A, Indices({0, 1}), 0); - - auto C = graph->param("2x3x2", {2, 3, 2}, inits::from_vector(in)); - auto D1 = index_select(C, 0, 0); - auto D2 = index_select(C, 2, -2); - auto D3 = index_select(C, Indices({0,2}), 1); + auto A = graph->param("4x3", {4,3}, inits::from_vector(vA)); + auto B1a = index_select(A, IndexVector({0}), 0); // always uses gather() + auto B1b = step(A, 0, 0); // memory-consecutive view + auto B2 = step(A, 0, 1); // not memory-consecutive + auto B3 = step(A, 1, -1); + auto B4a = index_select(A, IndexVector({0, 1}), 0); + auto B4b = slice(A, Slice(0, 2), 0); // this is memory-consecutive + auto B5 = slice(A, Slice(0, 4), 0); // this is a no-op + CHECK(B1a->type() == "rows"); // actually optimized to rows() + CHECK(B1b->type() == "sliceView"); // must use view + CHECK(B2->type() == "gather"); // cannot use view + CHECK(B4a->type() == "rows"); + CHECK(B4b->type() == "sliceView"); // must use view + CHECK(B5.get() == A.get()); // must be no-op + + auto C = graph->param("2x3x2", {2, 3, 2}, inits::from_vector(vC)); + auto D1 = step(C, 0, 0); + auto D2 = step(C, 2, -2); + auto D3 = index_select(C, IndexVector({0, 2}), 1); // C[:,(0,2),:] + CHECK(D1->type() == "sliceView"); + CHECK(D2->type() == "gather"); + // enable this once gather() supports batched indices: + //auto D4 = gather(C, graph->constant({2, 2, 1}, // [C[0,(2,1),:],C[1,(0,2),:]] + // inits::from_vector(std::vector{ + // 2, 1, + // 0, 2 }), + // Type::uint32), 1); auto S1 = step(A, 2, 0); auto S2 = narrow(A, 1, 2, 0); - auto S3 = sliceView(A, Slice(-2, Slice::END), 0); + auto S3 = slice(A, Slice(-2, Slice::END), 0); graph->forward(); - CHECK(B1->shape() == Shape({1, 3})); - B1->val()->get(values); - CHECK( values == vB1 ); - - CHECK(B2->shape() == Shape({4, 1})); - B2->val()->get(values); - CHECK( values == vB2 ); - - CHECK(B3->shape() == Shape({4, 1})); - B3->val()->get(values); - CHECK( values == vB3 ); - - CHECK(B4->shape() == Shape({2, 3})); - B4->val()->get(values); - CHECK( values == vB4 ); - - values.clear(); - - CHECK(D1->shape() == Shape({1, 3, 2})); - D1->val()->get(values); - CHECK( values == vD1 ); - - CHECK(D2->shape() == Shape({2, 1, 2})); - D2->val()->get(values); - CHECK( values == vD2 ); - - CHECK(D3->shape() == Shape({2, 2, 2})); - D3->val()->get(values); - CHECK( values == vD3 ); - - values.clear(); - - CHECK(S1->shape() == Shape({1,3})); - S1->val()->get(values); - CHECK(values == vS1); - - CHECK(S2->shape() == Shape({2,3})); - S2->val()->get(values); - CHECK(values == vS2); - - CHECK(S3->shape() == Shape({2,3})); - S3->val()->get(values); - CHECK(values == vS3); + CHECK(B1a->shape() == Shape({1, 3})); B1a->val()->get(values); CHECK( values == vB1 ); + CHECK(B1b->shape() == Shape({1, 3})); B1b->val()->get(values); CHECK( values == vB1 ); + CHECK(B2->shape() == Shape({4, 1})); B2->val()->get(values); CHECK( values == vB2 ); + CHECK(B3->shape() == Shape({4, 1})); B3->val()->get(values); CHECK( values == vB3 ); + CHECK(B4a->shape() == Shape({2, 3})); B4a->val()->get(values); CHECK( values == vB4 ); + CHECK(B4b->shape() == Shape({2, 3})); B4b->val()->get(values); CHECK( values == vB4 ); + + CHECK(D1->shape() == Shape({1, 3, 2})); D1->val()->get(values); CHECK( values == vD1 ); + CHECK(D2->shape() == Shape({2, 1, 2})); D2->val()->get(values); CHECK( values == vD2 ); + CHECK(D3->shape() == Shape({2, 2, 2})); D3->val()->get(values); CHECK( values == vD3 ); + //CHECK(D4->shape() == Shape({2, 2, 2})); D4->val()->get(values); CHECK( values == vD4 ); + + CHECK(S1->shape() == Shape({1,3})); S1->val()->get(values); CHECK(values == vS1); + CHECK(S2->shape() == Shape({2,3})); S2->val()->get(values); CHECK(values == vS2); + CHECK(S3->shape() == Shape({2,3})); S3->val()->get(values); CHECK(values == vS3); } SECTION("rows/cols as gather operations") { From f5dbb61fb47cf1da713768536aa8b3f00c46c5da Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 21 Jan 2019 15:18:11 -0800 Subject: [PATCH 183/838] now normalizes to factor logits --- src/layers/generic.cpp | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index c0812ae7b..d61d8216c 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -199,29 +199,40 @@ namespace marian { return affine(input, cachedShortW_, cachedShortb_, false, transposeW_); } else if (embeddingFactorMapping_) { - auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] - auto graph = input->graph(); - auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] - y = dot_csr( // the CSR matrix is passed in pieces - y, // [B x U] - factorMatrix.shape, - graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), - graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), - graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), - /*transB=*/ true); // -> [B x V] + auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] - // apply normalization factors + // denominators (only for groups that don't normalize out naturally by the final softmax()) const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); for (size_t g = 0; g < numGroups; g++) { - if (!embeddingFactorMapping_->groupNeedsNormalization_[g]) + if (!embeddingFactorMapping_->groupNeedsNormalization_[g]) // @TODO: if we ever need it, we can combine multiple continue; auto range = groupRanges[g]; // need to compute log denominator over y[range] and subtract it from y[range] - auto groupLogits = narrow(y, range.first, range.second, /*axis=*/-1); + auto groupY = slice(y, Slice((int)range.first, (int)range.second), /*axis=*/-1); // [B... x Ug] + auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] + // y: [B... x U] + // m: [1 x U] // ones at positions of group members + auto yDim = y->shape()[-1]; + std::vector mVec(yDim, 0.0f); + for (size_t i = range.first; i < range.second; i++) + mVec[i] = 1.0f; + auto m = graph->constant({1, yDim}, inits::from_vector(mVec)); + auto Z = dot(groupZ, m); + y = y - Z; } + // sum up the unit logits across factors for each target word + auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] + y = dot_csr( + y, // [B x U] + factorMatrix.shape, + graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), + graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), + graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), + /*transB=*/ true); // -> [B x V] + return y; } else From fa38017e781898c2cd18a6cfdfe9a728522d12d8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 21 Jan 2019 15:55:45 -0800 Subject: [PATCH 184/838] (minor refactoring) --- src/layers/generic.cpp | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index d61d8216c..3f138d87e 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -33,12 +33,6 @@ namespace marian { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - // Factor normalization - // - f = h * E' + b // hidden state projected onto all factors - // - f: [B x U] // U = number of factor units - // - normalization terms Z: [B x G] // G = number of groups - // - factor group matrix F: [U x G] // [u,g] 1 if factor u is in group g (one-hot); computed once - // - z = f - Z * F' = affine(Z, F, f, transB=true, alpha=-1) // This does it with only one extra copy EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { std::vector paths = options->get>("embedding-factors"); ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); @@ -200,7 +194,7 @@ namespace marian { } else if (embeddingFactorMapping_) { auto graph = input->graph(); - auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] + auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits // denominators (only for groups that don't normalize out naturally by the final softmax()) const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly @@ -209,17 +203,17 @@ namespace marian { if (!embeddingFactorMapping_->groupNeedsNormalization_[g]) // @TODO: if we ever need it, we can combine multiple continue; auto range = groupRanges[g]; - // need to compute log denominator over y[range] and subtract it from y[range] - auto groupY = slice(y, Slice((int)range.first, (int)range.second), /*axis=*/-1); // [B... x Ug] - auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] // y: [B... x U] // m: [1 x U] // ones at positions of group members auto yDim = y->shape()[-1]; - std::vector mVec(yDim, 0.0f); + std::vector mVec(yDim, 0.0f); // @TODO: This vector should be produced by embeddingFactorMapping_ for (size_t i = range.first; i < range.second; i++) mVec[i] = 1.0f; - auto m = graph->constant({1, yDim}, inits::from_vector(mVec)); - auto Z = dot(groupZ, m); + // need to compute log denominator over y[range] and subtract it from y[range] + auto groupY = slice(y, Slice((int)range.first, (int)range.second), /*axis=*/-1); // [B... x Ug] + auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] + auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] + auto Z = dot(groupZ, m); // [B... x U] y = y - Z; } From d0d0b14736851201d6297e9c911046fd92837077 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 21 Jan 2019 22:27:21 -0800 Subject: [PATCH 185/838] mimic bert output layer for masked lm --- src/graph/expression_operators.cpp | 5 +-- src/models/bert.h | 49 +++++++++++++++++++----------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index ecda67751..8fa36719f 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -470,8 +470,9 @@ Expr cross_entropy(Expr a, Expr indices) { return Expression(a, indices); } -Expr plus(const std::vector&) { - ABORT("Not implemented"); +Expr plus(const std::vector& nodes) { + ABORT_IF(nodes.size() > 1, "Not implemented"); + return nodes[0]; } Expr swish(const std::vector& nodes) { diff --git a/src/models/bert.h b/src/models/bert.h index 849144b1a..4bf281dbc 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -323,27 +323,40 @@ class BertMaskedLM : public ClassifierBase { int dimBatch = context->shape()[-2]; int dimTime = context->shape()[-3]; - auto maskedEmbeddings = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out + auto maskedContext = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out int dimVoc = opt>("dim-vocabs")[batchIndex_]; - auto layerTanh = mlp::dense() // - ("prefix", prefix_ + "_ff_logit_l1") // - ("dim", dimModel) // - ("activation", mlp::act::tanh); // @TODO: again, check if this layer is present in original code - auto layerOut = mlp::output() // - ("prefix", prefix_ + "_ff_logit_l2") // - ("dim", dimVoc); // - layerOut.tieTransposed("Wemb"); // We are a BERT model, hence tie with input, @TODO: check if this is actually what Google does - - // assemble layers into MLP and apply to embeddings, decoder context and - // aligned source context - auto output = mlp::mlp() // - .push_back(layerTanh) // - .push_back(layerOut) // - .construct(graph); - - auto logits = output->apply(maskedEmbeddings); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] + std::string activationType = opt("transformer-ffn-activation"); + mlp::act activation; + if(activationType == "relu") + activation = mlp::act::ReLU; + else if(activationType == "swish") + activation = mlp::act::swish; + else + ABORT("Activation function {} not supported in BERT masked LM", activationType); + + auto layer1 = mlp::mlp() + .push_back(mlp::dense() + ("prefix", prefix_ + "_ff_logit_l1") + ("dim", dimModel) + ("activation", activation)) + .construct(graph); + + auto intermediate = layer1->apply(maskedContext); + + auto gamma = graph->param(prefix_ + "_ff_ln_scale", {1, dimModel}, inits::ones); + auto beta = graph->param(prefix_ + "_ff_ln_bias", {1, dimModel}, inits::zeros); + intermediate = layerNorm(intermediate, gamma, beta); + + auto layer2 = mlp::mlp() + .push_back(mlp::output() + ("prefix", prefix_ + "_ff_logit_l2") + ("dim", dimVoc) + .tieTransposed("Wemb")) + .construct(graph); + + auto logits = layer2->apply(intermediate); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] auto state = New(); state->setLogProbs(logits); From c1c175f99522da1611c0847c6fc3152d423a24fa Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 22 Jan 2019 14:21:42 -0800 Subject: [PATCH 186/838] added a log-linear weight to @C factor (#ifdef'ed out for now) --- src/layers/generic.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 3f138d87e..2941b689d 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -215,6 +215,12 @@ namespace marian { auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] auto Z = dot(groupZ, m); // [B... x U] y = y - Z; +#if 0 + // and a log-linear weight + auto name = options_->get("prefix"); + auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + y = y * ((llWeight - 1) * m + 1); +#endif } // sum up the unit logits across factors for each target word From 0a62d1e2d051105cb25a83cc3ab348e4b19c50df Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 22 Jan 2019 15:08:32 -0800 Subject: [PATCH 187/838] changed index operations' parameter lists to match PyTorch parameter order (axis before arg) --- src/common/shape.h | 2 +- src/graph/expression_operators.cpp | 30 +++++++++++----------- src/graph/expression_operators.h | 25 ++++++++++--------- src/graph/node_operators_binary.h | 6 ++--- src/graph/node_operators_unary.h | 6 ++--- src/models/transformer.h | 4 +-- src/rnn/rnn.h | 4 +-- src/tests/operator_tests.cpp | 40 +++++++++++++++--------------- 8 files changed, 59 insertions(+), 58 deletions(-) diff --git a/src/common/shape.h b/src/common/shape.h index 20fb26070..c8a4bdd3d 100755 --- a/src/common/shape.h +++ b/src/common/shape.h @@ -17,7 +17,7 @@ struct Slice // Python-like slice/index descriptor Slice(int b, int e, int s) : begin(b), end(e), stride(s) {} Slice(int b, int e) : Slice(b, e, 1) {} Slice() : Slice(0, END) {} - Slice(int i) : Slice(i, i + 1) {} + explicit Slice(int i) : Slice(i, i + 1) {} Slice(const Slice& other) : Slice(other.begin, other.end, other.stride) {} const Slice& operator=(const Slice& other) { begin = other.begin; end = other.end; stride = other.stride; return *this; } const Slice& operator=(int i) { begin = i; end = i + 1; stride = 1; return *this; } diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 81310b438..db820062a 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -246,17 +246,17 @@ Expr constant_like(Expr a, const NodeInitializer& init) { } // gather() -- gather arbitrary elements along an axis; batched or non-batched -Expr gather(Expr a, Expr indices, int axis) { - return Expression(a, indices, axis); +Expr gather(Expr a, int axis, Expr indices) { + return Expression(a, axis, indices); } // index_select() -- gather arbitrary elements along an axis; unbatched (indices are specified as a 1D vector) -Expr index_select(Expr a, Expr indices, int axis) { +Expr index_select(Expr a, int axis, Expr indices) { ABORT_IF(indices->shape().size() != 1, "Indices must be a 1D tensor"); // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. auto rank = a->shape().size(); if (rank == 2) { - if (axis == 0) + if (axis == 0 || axis == 2) return Expression(a, indices); else if (axis == -1 || axis == 1) return Expression(a, indices); @@ -266,29 +266,29 @@ Expr index_select(Expr a, Expr indices, int axis) { shape.resize(a->shape().size()); shape.set(axis, indices->shape()[0]); indices = reshape(indices, shape); // move index to axis - return gather(a, indices, axis); + return gather(a, axis, indices); } -Expr index_select(Expr a, const std::vector& indices, int axis) { +Expr index_select(Expr a, int axis, const std::vector& indices) { auto indexExpr = a->graph()->indices(indices); - return index_select(a, indexExpr, axis); + return index_select(a, axis, indexExpr); } -static Expr sliceCopy(Expr a, const Slice& slice, int axis) { // copy a Slice via gather() +static Expr sliceCopy(Expr a, int axis, const Slice& slice) { // copy a Slice via gather() ABORT_IF(slice.stride < 0, "Negative strides are not supported yet"); ABORT_IF(slice.begin == slice.end, "Empty slices are not allowed"); // @TODO: Or are they? std::vector indices; indices.reserve((slice.end - slice.begin - 1) / slice.stride + 1); for (int i = slice.begin; i < slice.end; i += slice.stride) indices.push_back((IndexType)i); - return gather(a, a->graph()->indices(indices, a, axis), axis); + return gather(a, axis, a->graph()->indices(indices, a, axis)); } -static Expr sliceView(Expr a, const Slice& slice, int axis) { // view a slice (must be memory-consecutive) - return Expression(a, slice, axis); +static Expr sliceView(Expr a, int axis, const Slice& slice) { // view a slice (must be memory-consecutive) + return Expression(a, axis, slice); } // slice() -- gather a slice along an axis (step size > 1 allowed) -Expr slice(Expr a, Slice slice, int axis) { // numpy __getslice__ semantics, but with axis parameter +Expr slice(Expr a, int axis, Slice slice) { // numpy __getslice__ semantics, but with axis parameter const auto& shape = a->shape(); axis = shape.axis(axis); // normalize negative axis slice = shape.slice(slice, axis); // normalize negative slice values @@ -296,13 +296,13 @@ Expr slice(Expr a, Slice slice, int axis) { // numpy __getslice__ semantics, but return a; // it's a no-op #if 1 // until strided views are supported, non-consecutive slices are implemented via gather() if (slice.stride != 1) - return sliceCopy(a, slice, axis); + return sliceCopy(a, axis, slice); for (int i = 0; i < axis; ++i) { if (shape[i] != 1) // this makes it non-consecutive - return sliceCopy(a, slice, axis); + return sliceCopy(a, axis, slice); } #endif - return sliceView(a, slice, axis); + return sliceView(a, axis, slice); } Expr sum(Expr a, int ax) { diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 5902f2af2..58149bdeb 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -139,34 +139,35 @@ Expr flatten_2d(Expr a); Expr stopGradient(Expr a); -Expr gather(Expr a, Expr indices, int axis); +Expr gather(Expr a, int axis, Expr indices); -Expr index_select(Expr a, Expr indices, int axis); +// Warning: Don't try to pass a scalar literal 0 as indices; it will compile but pass nullptr... +Expr index_select(Expr a, int axis, Expr indices); // convenience wrappers for index_select() -Expr index_select(Expr a, const std::vector& indices, int axis); +Expr index_select(Expr a, int axis, const std::vector& indices); static inline Expr rows(Expr a, Expr indices) { - return index_select(a, indices, 0); + return index_select(a, 0, indices); } static inline Expr rows(Expr a, const std::vector& indexVector) { - return index_select(a, indexVector, 0); + return index_select(a, 0, indexVector); } static inline Expr cols(Expr a, Expr indices) { - return index_select(a, indices, -1); + return index_select(a, -1, indices); } static inline Expr cols(Expr a, const std::vector& indexVector) { - return index_select(a, indexVector, -1); + return index_select(a, -1, indexVector); } -Expr slice(Expr a, Slice slice, int axis); +Expr slice(Expr a, int axis, Slice slice); // convenience wrappers for slice() -static inline Expr step(Expr a, int step, int axis) { // @TODO: name is too narrow - return slice(a, Slice(step), axis); +static inline Expr slice(Expr a, int axis, int index) { // single index @NOTE: This was formerlly called step() + return slice(a, axis, Slice(index)); } -static inline Expr narrow(Expr a, size_t start, size_t length, int axis) { // PyTorch name - return slice(a, Slice((int)start, (int)(start + length)), axis); +static inline Expr narrow(Expr a, int axis, size_t start, size_t length) { // PyTorch name + return slice(a, axis, Slice((int)start, (int)(start + length))); } /*********************************************************/ diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 22109fbc6..10e2ca763 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -576,8 +576,8 @@ struct RowsNodeOp : public NaryNodeOp { // @TODO: The current implementation does not support batched indices (third scenario above). // I.e. all axes of 'indices' except 'axis' must have dimension 1. struct GatherNodeOp : public NaryNodeOp { - GatherNodeOp(Expr a, Expr indices, int axis) - : NaryNodeOp({a, indices}, newShape(a, indices, axis), a->value_type()), + GatherNodeOp(Expr a, int axis, Expr indices) + : NaryNodeOp({a, indices}, newShape(a, axis, indices), a->value_type()), axis_(a->shape().axis(axis)) { matchOrAbort(indices->value_type()); } @@ -592,7 +592,7 @@ struct GatherNodeOp : public NaryNodeOp { Insert(child(0)->grad(), adj_, child(1)->val(), axis_))}; } - Shape newShape(Expr a, Expr indices, int axis) { + Shape newShape(Expr a, int axis, Expr indices) { Shape shape = a->shape(); axis = shape.axis(axis); auto rank = shape.size(); diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 51433f933..7dbaec468 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -745,15 +745,15 @@ class SliceViewNodeOp : public UnaryNodeOp { size_t byteOffset_, byteSize_; // viewed segment in bytes (memory-consecutive) public: - SliceViewNodeOp(Expr a, Slice slice, int axis) - : UnaryNodeOp(a, newShape(a, slice, axis), a->value_type()), viewedNode_(a), slice_(slice), axis_(axis) { + SliceViewNodeOp(Expr a, int axis, Slice slice) + : UnaryNodeOp(a, newShape(a, axis, slice), a->value_type()), viewedNode_(a), slice_(slice), axis_(axis) { Node::destroy_ = false; auto byteStride = a->shape().stride(axis) * sizeOf(value_type()); byteOffset_ = slice.begin * byteStride; byteSize_ = shape()[axis] * byteStride; } - static Shape newShape(Expr a, Slice& slice, int& axis) { // note: normalizes slice and axis in-place + static Shape newShape(Expr a, int& axis, Slice& slice) { // note: normalizes slice and axis in-place const auto& shape = a->shape(); axis = shape.axis(axis); // normalize negative axis slice = shape.slice(slice, axis); // normalize negative slice values diff --git a/src/models/transformer.h b/src/models/transformer.h index 3c540c12a..968d481b4 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -178,7 +178,7 @@ class Transformer : public EncoderOrDecoderBase { void collectOneHead(Expr weights, int dimBeam) { // select first head, this is arbitrary as the choice does not really matter - auto head0 = index_select(weights, 0, -3); + auto head0 = slice(weights, -3, 0); int dimBatchBeam = head0->shape()[-4]; int srcWords = head0->shape()[-1]; @@ -194,7 +194,7 @@ class Transformer : public EncoderOrDecoderBase { // @TODO: make splitting obsolete alignments_.clear(); for(int i = 0; i < trgWords; ++i) { - alignments_.push_back(marian::step(head0, i, -1)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] + alignments_.push_back(slice(head0, -1, i)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] } } diff --git a/src/rnn/rnn.h b/src/rnn/rnn.h index 74a535d37..c9131d87a 100755 --- a/src/rnn/rnn.h +++ b/src/rnn/rnn.h @@ -75,11 +75,11 @@ class SingleLayerRNN : public BaseRNN { std::vector steps(xWs.size()); std::transform(xWs.begin(), xWs.end(), steps.begin(), [j](Expr e) { - return step(e, j, -3); + return slice(e, -3, j); }); if(mask) - state = cell_->applyState(steps, state, step(mask, j, -3)); + state = cell_->applyState(steps, state, slice(mask, -3, j)); else state = cell_->applyState(steps, state); diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 34d8b03c9..08fdd8ded 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -643,13 +643,13 @@ void tests(DeviceType device) { std::vector vS3({7, -8, 9, -10, 11, -12}); auto A = graph->param("4x3", {4,3}, inits::from_vector(vA)); - auto B1a = index_select(A, IndexVector({0}), 0); // always uses gather() - auto B1b = step(A, 0, 0); // memory-consecutive view - auto B2 = step(A, 0, 1); // not memory-consecutive - auto B3 = step(A, 1, -1); - auto B4a = index_select(A, IndexVector({0, 1}), 0); - auto B4b = slice(A, Slice(0, 2), 0); // this is memory-consecutive - auto B5 = slice(A, Slice(0, 4), 0); // this is a no-op + auto B1a = index_select(A, 0, IndexVector({0})); // always uses gather() + auto B1b = slice(A, 0, 0); // memory-consecutive view + auto B2 = slice(A, 1, 0); // not memory-consecutive + auto B3 = slice(A, -1, 1); + auto B4a = index_select(A, 0, IndexVector({0, 1})); + auto B4b = slice(A, 0, Slice(0, 2)); // this is memory-consecutive + auto B5 = slice(A, 0, Slice(0, 4)); // this is a no-op CHECK(B1a->type() == "rows"); // actually optimized to rows() CHECK(B1b->type() == "sliceView"); // must use view CHECK(B2->type() == "gather"); // cannot use view @@ -658,21 +658,21 @@ void tests(DeviceType device) { CHECK(B5.get() == A.get()); // must be no-op auto C = graph->param("2x3x2", {2, 3, 2}, inits::from_vector(vC)); - auto D1 = step(C, 0, 0); - auto D2 = step(C, 2, -2); - auto D3 = index_select(C, IndexVector({0, 2}), 1); // C[:,(0,2),:] + auto D1 = slice(C, 0, 0); + auto D2 = slice(C, -2, 2); + auto D3 = index_select(C, 1, IndexVector({0, 2})); // C[:,(0,2),:] CHECK(D1->type() == "sliceView"); CHECK(D2->type() == "gather"); // enable this once gather() supports batched indices: - //auto D4 = gather(C, graph->constant({2, 2, 1}, // [C[0,(2,1),:],C[1,(0,2),:]] - // inits::from_vector(std::vector{ - // 2, 1, - // 0, 2 }), - // Type::uint32), 1); + //auto D4 = gather(C, 1, graph->constant({2, 2, 1}, // [C[0,(2,1),:],C[1,(0,2),:]] + // inits::from_vector(std::vector{ + // 2, 1, + // 0, 2 }), + // Type::uint32)); - auto S1 = step(A, 2, 0); - auto S2 = narrow(A, 1, 2, 0); - auto S3 = slice(A, Slice(-2, Slice::END), 0); + auto S1 = slice(A, 0, 2); + auto S2 = narrow(A, 0, 1, 2); + auto S3 = slice(A, 0, Slice(-2, Slice::END)); graph->forward(); @@ -703,9 +703,9 @@ void tests(DeviceType device) { auto A = graph->param("4x3", {4, 3}, inits::from_vector(vA)); auto B1 = rows(A, indices); - auto B2 = gather(A, graph->indices(indices, A, 0), 0); + auto B2 = gather(A, 0, graph->indices(indices, A, 0)); auto C1 = cols(A, indices); - auto C2 = gather(A, graph->indices(indices, A, 1), 1); + auto C2 = gather(A, 1, graph->indices(indices, A, 1)); graph->forward(); CHECK(B1->shape() == B2->shape()); From 7ae9709043cdcc4f9bf38e9519f06e9eccaf58eb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 22 Jan 2019 15:23:11 -0800 Subject: [PATCH 188/838] fixed a typo --- src/graph/expression_operators.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index db820062a..826bd9f0e 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -256,7 +256,7 @@ Expr index_select(Expr a, int axis, Expr indices) { // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. auto rank = a->shape().size(); if (rank == 2) { - if (axis == 0 || axis == 2) + if (axis == 0 || axis == -2) return Expression(a, indices); else if (axis == -1 || axis == 1) return Expression(a, indices); From 73a4648ba6f388c28160d464a915ff2ad7096a21 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 22 Jan 2019 15:31:47 -0800 Subject: [PATCH 189/838] fix-up after last merge --- src/layers/generic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2941b689d..1cff35ce2 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -210,7 +210,7 @@ namespace marian { for (size_t i = range.first; i < range.second; i++) mVec[i] = 1.0f; // need to compute log denominator over y[range] and subtract it from y[range] - auto groupY = slice(y, Slice((int)range.first, (int)range.second), /*axis=*/-1); // [B... x Ug] + auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] auto Z = dot(groupZ, m); // [B... x U] From 196deea12733685a11c9266153d72b921ec0919b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 22 Jan 2019 17:37:49 -0800 Subject: [PATCH 190/838] fixed build break after last merge --- src/models/bert.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index 4bf281dbc..c642abac4 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -269,7 +269,7 @@ class BertClassifier : public ClassifierBase { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); auto context = encoderStates[0]->getContext(); - auto classEmbeddings = step(context, /*i=*/0, /*axis=*/-3); // [CLS] symbol is first symbol in each sequence + auto classEmbeddings = slice(context, /*axis=*/-3, /*i=*/0); // [CLS] symbol is first symbol in each sequence int dimModel = classEmbeddings->shape()[-1]; int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels @@ -368,4 +368,4 @@ class BertMaskedLM : public ClassifierBase { virtual void clear() override {} }; -} \ No newline at end of file +} From 92443a6b508a950585d18acee4027c2d832fc56c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 22 Jan 2019 22:02:37 -0800 Subject: [PATCH 191/838] remove debug message --- src/optimizers/optimizers.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp index e1de4068a..1580f7da5 100755 --- a/src/optimizers/optimizers.cpp +++ b/src/optimizers/optimizers.cpp @@ -148,8 +148,6 @@ void Adam::updateImpl(Tensor params, Tensor grads, size_t actualMBSize, size_t r denom1_ = (beta1 * denom1_) + (1 - beta1); // momentum smoothing denom2_ = (beta2 * denom2_) + (1 - beta2); // RMS normalization - LOG_ONCE(info, "[adam] First update: Tref = {}, T = {}, eta = {} -> {}, beta = {}, {}", Tref, T, eta_, eta, beta1, beta2); - // numerators. Divide by T to convert ce-sum gradient to avg gradient. using namespace functional; Element(_1 = ((float)beta1 * _1) + float((1 - beta1) / T ) * _2, mt_, grads); // momentum smoothing. At steady state: =smoothed avg gradient From ed5b034fe03c9228dba401d5b78537d1ced0dc54 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 23 Jan 2019 12:48:26 -0800 Subject: [PATCH 192/838] changed MAX_VOCAB_SIZE to 500k --- src/layers/generic.cpp | 3 +++ src/translator/nth_element.cu | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 1cff35ce2..d44c6bb13 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -196,6 +196,7 @@ namespace marian { auto graph = input->graph(); auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits +#if 0 // denominators (only for groups that don't normalize out naturally by the final softmax()) const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); @@ -212,6 +213,7 @@ namespace marian { // need to compute log denominator over y[range] and subtract it from y[range] auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] + //auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] auto Z = dot(groupZ, m); // [B... x U] y = y - Z; @@ -222,6 +224,7 @@ namespace marian { y = y * ((llWeight - 1) * m + 1); #endif } +#endif // sum up the unit logits across factors for each target word auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu index d5377f90c..1a7ff902f 100755 --- a/src/translator/nth_element.cu +++ b/src/translator/nth_element.cu @@ -416,7 +416,7 @@ private: DeviceId deviceId_; - const int MAX_VOCAB_SIZE = 100000; + const int MAX_VOCAB_SIZE = 500000; const int BLOCK_SIZE = 512; const int NUM_BLOCKS; From f100a8649af3b2f6f3357c9ed769a5206f41e856 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Wed, 23 Jan 2019 23:20:46 +0000 Subject: [PATCH 193/838] C++ source is should not have execute bits --- .gitignore | 0 src/3rd_party/ExceptionWithCallStack.cpp | 0 src/3rd_party/any_type.h | 0 src/3rd_party/catch.hpp | 0 src/3rd_party/cnpy/cnpy.cpp | 0 src/3rd_party/cnpy/cnpy.h | 0 src/3rd_party/spdlog/details/format.h | 0 src/3rd_party/spdlog/details/logger_impl.h | 0 src/3rd_party/spdlog/logger.h | 0 src/3rd_party/spdlog/tests/catch.hpp | 0 src/3rd_party/threadpool.h | 0 src/3rd_party/yaml-cpp/binary_renamed.cpp | 0 src/3rd_party/yaml-cpp/collectionstack.h | 0 src/3rd_party/yaml-cpp/dll.h | 0 src/3rd_party/yaml-cpp/emitterstate.cpp | 0 src/3rd_party/yaml-cpp/emitterstate.h | 0 src/3rd_party/yaml-cpp/node/convert.h | 0 src/3rd_party/yaml-cpp/node/node.h | 0 src/3rd_party/yaml-cpp/node_data.cpp | 0 src/3rd_party/yaml-cpp/scanner.cpp | 0 src/3rd_party/yaml-cpp/scantoken.cpp | 0 src/3rd_party/yaml-cpp/singledocparser.cpp | 0 src/command/marian_conv.cpp | 0 src/command/marian_main.cpp | 0 src/command/marian_train.cpp | 0 src/command/marian_vocab.cpp | 0 src/common/cli_helper.h | 0 src/common/cli_wrapper.cpp | 0 src/common/cli_wrapper.h | 0 src/common/compile_time_crc32.h | 0 src/common/config.cpp | 0 src/common/config.h | 0 src/common/config_parser.cpp | 0 src/common/config_parser.h | 0 src/common/config_validator.cpp | 0 src/common/definitions.h | 0 src/common/file_stream.h | 0 src/common/filesystem.h | 0 src/common/io.cpp | 0 src/common/logging.cpp | 0 src/common/logging.h | 0 src/common/options.h | 0 src/common/project_version.h.in | 0 src/common/shape.h | 0 src/common/timer.h | 0 src/common/types.h | 0 src/common/utils.cpp | 0 src/common/utils.h | 0 src/common/version.cpp | 0 src/common/version.h | 0 src/data/alignment.cpp | 0 src/data/alignment.h | 0 src/data/batch.h | 0 src/data/batch_generator.h | 0 src/data/batch_stats.h | 0 src/data/corpus.cpp | 0 src/data/corpus.h | 0 src/data/corpus_base.cpp | 0 src/data/corpus_base.h | 0 src/data/corpus_nbest.h | 0 src/data/corpus_sqlite.h | 0 src/data/dataset.h | 0 src/data/default_vocab.cpp | 0 src/data/rng_engine.h | 0 src/data/sentencepiece_vocab.cpp | 0 src/data/shortlist.h | 0 src/data/text_input.cpp | 0 src/data/text_input.h | 0 src/data/vocab.cpp | 0 src/data/vocab.h | 0 src/examples/mnist/dataset.h | 0 src/examples/mnist/model.h | 0 src/functional/approx.h | 0 src/functional/defs.h | 0 src/functional/functional.h | 0 src/functional/operands.h | 0 src/functional/predicates.h | 0 src/functional/shape.h | 0 src/graph/chainable.h | 0 src/graph/expression_graph.cpp | 0 src/graph/expression_graph.h | 0 src/graph/expression_operators.cpp | 0 src/graph/node.cpp | 0 src/graph/node.h | 0 src/graph/node_initializers.cpp | 0 src/graph/node_initializers.h | 0 src/graph/node_operators_binary.h | 0 src/graph/parameters.h | 0 src/layers/generic.h | 0 src/layers/guided_alignment.h | 0 src/layers/loss.cpp | 0 src/layers/weight.cpp | 0 src/layers/weight.h | 0 src/layers/word2vec_reader.h | 0 src/marian.h | 0 src/microsoft/quicksand.cpp | 0 src/microsoft/quicksand.h | 0 src/models/amun.h | 0 src/models/costs.h | 0 src/models/decoder.h | 0 src/models/encoder.h | 0 src/models/encoder_decoder.cpp | 0 src/models/nematus.h | 0 src/models/s2s.h | 0 src/models/states.h | 0 src/models/transformer.h | 0 src/models/transformer_factory.h | 0 src/optimizers/optimizers.cpp | 0 src/optimizers/optimizers.h | 0 src/rnn/constructors.h | 0 src/rnn/rnn.h | 0 src/rnn/types.h | 0 src/tensors/allocator.h | 0 src/tensors/cpu/add.h | 0 src/tensors/cpu/prod.cpp | 0 src/tensors/cpu/sharp/int_gemm.h | 0 src/tensors/cpu/tensor_operators.cpp | 0 src/tensors/device.h | 0 src/tensors/gpu/algorithm.cu | 0 src/tensors/gpu/backend.h | 0 src/tensors/gpu/cuda_helpers.h | 0 src/tensors/gpu/device.cu | 0 src/tensors/rand.cpp | 0 src/tensors/tensor.h | 0 src/tensors/tensor_allocator.h | 0 src/tensors/tensor_operators.h | 0 src/training/communicator.cpp | 0 src/training/communicator.h | 0 src/training/communicator_nccl.h | 0 src/training/exponential_smoothing.h | 0 src/training/gradient_dropping/sparse_tensor.h | 0 src/training/graph_group.h | 0 src/training/graph_group_async.cpp | 0 src/training/graph_group_async.h | 0 src/training/graph_group_async_drop.cpp | 0 src/training/graph_group_async_drop.h | 0 src/training/graph_group_multinode.cpp | 0 src/training/graph_group_multinode.h | 0 src/training/graph_group_multinode_sync.cpp | 0 src/training/graph_group_multinode_sync.h | 0 src/training/graph_group_singleton.cpp | 0 src/training/graph_group_singleton.h | 0 src/training/graph_group_sync.cpp | 0 src/training/graph_group_sync.h | 0 src/training/scheduler.h | 0 src/training/training.h | 0 src/training/training_state.h | 0 src/training/validator.h | 0 src/translator/beam_search.h | 0 src/translator/helpers.cpp | 0 src/translator/helpers.h | 0 src/translator/history.h | 0 src/translator/hypothesis.h | 0 src/translator/nth_element.cpp | 0 src/translator/nth_element.cu | 0 src/translator/nth_element.h | 0 src/translator/output_collector.cpp | 0 src/translator/output_collector.h | 0 src/translator/output_printer.cpp | 0 src/translator/output_printer.h | 0 src/translator/scorers.cpp | 0 src/translator/scorers.h | 0 src/translator/translator.h | 0 vs/Marian.sln | 0 vs/Marian.vcxproj | 0 vs/Marian.vcxproj.filters | 0 166 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 .gitignore mode change 100755 => 100644 src/3rd_party/ExceptionWithCallStack.cpp mode change 100755 => 100644 src/3rd_party/any_type.h mode change 100755 => 100644 src/3rd_party/catch.hpp mode change 100755 => 100644 src/3rd_party/cnpy/cnpy.cpp mode change 100755 => 100644 src/3rd_party/cnpy/cnpy.h mode change 100755 => 100644 src/3rd_party/spdlog/details/format.h mode change 100755 => 100644 src/3rd_party/spdlog/details/logger_impl.h mode change 100755 => 100644 src/3rd_party/spdlog/logger.h mode change 100755 => 100644 src/3rd_party/spdlog/tests/catch.hpp mode change 100755 => 100644 src/3rd_party/threadpool.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/binary_renamed.cpp mode change 100755 => 100644 src/3rd_party/yaml-cpp/collectionstack.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/dll.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/emitterstate.cpp mode change 100755 => 100644 src/3rd_party/yaml-cpp/emitterstate.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/node/convert.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/node/node.h mode change 100755 => 100644 src/3rd_party/yaml-cpp/node_data.cpp mode change 100755 => 100644 src/3rd_party/yaml-cpp/scanner.cpp mode change 100755 => 100644 src/3rd_party/yaml-cpp/scantoken.cpp mode change 100755 => 100644 src/3rd_party/yaml-cpp/singledocparser.cpp mode change 100755 => 100644 src/command/marian_conv.cpp mode change 100755 => 100644 src/command/marian_main.cpp mode change 100755 => 100644 src/command/marian_train.cpp mode change 100755 => 100644 src/command/marian_vocab.cpp mode change 100755 => 100644 src/common/cli_helper.h mode change 100755 => 100644 src/common/cli_wrapper.cpp mode change 100755 => 100644 src/common/cli_wrapper.h mode change 100755 => 100644 src/common/compile_time_crc32.h mode change 100755 => 100644 src/common/config.cpp mode change 100755 => 100644 src/common/config.h mode change 100755 => 100644 src/common/config_parser.cpp mode change 100755 => 100644 src/common/config_parser.h mode change 100755 => 100644 src/common/config_validator.cpp mode change 100755 => 100644 src/common/definitions.h mode change 100755 => 100644 src/common/file_stream.h mode change 100755 => 100644 src/common/filesystem.h mode change 100755 => 100644 src/common/io.cpp mode change 100755 => 100644 src/common/logging.cpp mode change 100755 => 100644 src/common/logging.h mode change 100755 => 100644 src/common/options.h mode change 100755 => 100644 src/common/project_version.h.in mode change 100755 => 100644 src/common/shape.h mode change 100755 => 100644 src/common/timer.h mode change 100755 => 100644 src/common/types.h mode change 100755 => 100644 src/common/utils.cpp mode change 100755 => 100644 src/common/utils.h mode change 100755 => 100644 src/common/version.cpp mode change 100755 => 100644 src/common/version.h mode change 100755 => 100644 src/data/alignment.cpp mode change 100755 => 100644 src/data/alignment.h mode change 100755 => 100644 src/data/batch.h mode change 100755 => 100644 src/data/batch_generator.h mode change 100755 => 100644 src/data/batch_stats.h mode change 100755 => 100644 src/data/corpus.cpp mode change 100755 => 100644 src/data/corpus.h mode change 100755 => 100644 src/data/corpus_base.cpp mode change 100755 => 100644 src/data/corpus_base.h mode change 100755 => 100644 src/data/corpus_nbest.h mode change 100755 => 100644 src/data/corpus_sqlite.h mode change 100755 => 100644 src/data/dataset.h mode change 100755 => 100644 src/data/default_vocab.cpp mode change 100755 => 100644 src/data/rng_engine.h mode change 100755 => 100644 src/data/sentencepiece_vocab.cpp mode change 100755 => 100644 src/data/shortlist.h mode change 100755 => 100644 src/data/text_input.cpp mode change 100755 => 100644 src/data/text_input.h mode change 100755 => 100644 src/data/vocab.cpp mode change 100755 => 100644 src/data/vocab.h mode change 100755 => 100644 src/examples/mnist/dataset.h mode change 100755 => 100644 src/examples/mnist/model.h mode change 100755 => 100644 src/functional/approx.h mode change 100755 => 100644 src/functional/defs.h mode change 100755 => 100644 src/functional/functional.h mode change 100755 => 100644 src/functional/operands.h mode change 100755 => 100644 src/functional/predicates.h mode change 100755 => 100644 src/functional/shape.h mode change 100755 => 100644 src/graph/chainable.h mode change 100755 => 100644 src/graph/expression_graph.cpp mode change 100755 => 100644 src/graph/expression_graph.h mode change 100755 => 100644 src/graph/expression_operators.cpp mode change 100755 => 100644 src/graph/node.cpp mode change 100755 => 100644 src/graph/node.h mode change 100755 => 100644 src/graph/node_initializers.cpp mode change 100755 => 100644 src/graph/node_initializers.h mode change 100755 => 100644 src/graph/node_operators_binary.h mode change 100755 => 100644 src/graph/parameters.h mode change 100755 => 100644 src/layers/generic.h mode change 100755 => 100644 src/layers/guided_alignment.h mode change 100755 => 100644 src/layers/loss.cpp mode change 100755 => 100644 src/layers/weight.cpp mode change 100755 => 100644 src/layers/weight.h mode change 100755 => 100644 src/layers/word2vec_reader.h mode change 100755 => 100644 src/marian.h mode change 100755 => 100644 src/microsoft/quicksand.cpp mode change 100755 => 100644 src/microsoft/quicksand.h mode change 100755 => 100644 src/models/amun.h mode change 100755 => 100644 src/models/costs.h mode change 100755 => 100644 src/models/decoder.h mode change 100755 => 100644 src/models/encoder.h mode change 100755 => 100644 src/models/encoder_decoder.cpp mode change 100755 => 100644 src/models/nematus.h mode change 100755 => 100644 src/models/s2s.h mode change 100755 => 100644 src/models/states.h mode change 100755 => 100644 src/models/transformer.h mode change 100755 => 100644 src/models/transformer_factory.h mode change 100755 => 100644 src/optimizers/optimizers.cpp mode change 100755 => 100644 src/optimizers/optimizers.h mode change 100755 => 100644 src/rnn/constructors.h mode change 100755 => 100644 src/rnn/rnn.h mode change 100755 => 100644 src/rnn/types.h mode change 100755 => 100644 src/tensors/allocator.h mode change 100755 => 100644 src/tensors/cpu/add.h mode change 100755 => 100644 src/tensors/cpu/prod.cpp mode change 100755 => 100644 src/tensors/cpu/sharp/int_gemm.h mode change 100755 => 100644 src/tensors/cpu/tensor_operators.cpp mode change 100755 => 100644 src/tensors/device.h mode change 100755 => 100644 src/tensors/gpu/algorithm.cu mode change 100755 => 100644 src/tensors/gpu/backend.h mode change 100755 => 100644 src/tensors/gpu/cuda_helpers.h mode change 100755 => 100644 src/tensors/gpu/device.cu mode change 100755 => 100644 src/tensors/rand.cpp mode change 100755 => 100644 src/tensors/tensor.h mode change 100755 => 100644 src/tensors/tensor_allocator.h mode change 100755 => 100644 src/tensors/tensor_operators.h mode change 100755 => 100644 src/training/communicator.cpp mode change 100755 => 100644 src/training/communicator.h mode change 100755 => 100644 src/training/communicator_nccl.h mode change 100755 => 100644 src/training/exponential_smoothing.h mode change 100755 => 100644 src/training/gradient_dropping/sparse_tensor.h mode change 100755 => 100644 src/training/graph_group.h mode change 100755 => 100644 src/training/graph_group_async.cpp mode change 100755 => 100644 src/training/graph_group_async.h mode change 100755 => 100644 src/training/graph_group_async_drop.cpp mode change 100755 => 100644 src/training/graph_group_async_drop.h mode change 100755 => 100644 src/training/graph_group_multinode.cpp mode change 100755 => 100644 src/training/graph_group_multinode.h mode change 100755 => 100644 src/training/graph_group_multinode_sync.cpp mode change 100755 => 100644 src/training/graph_group_multinode_sync.h mode change 100755 => 100644 src/training/graph_group_singleton.cpp mode change 100755 => 100644 src/training/graph_group_singleton.h mode change 100755 => 100644 src/training/graph_group_sync.cpp mode change 100755 => 100644 src/training/graph_group_sync.h mode change 100755 => 100644 src/training/scheduler.h mode change 100755 => 100644 src/training/training.h mode change 100755 => 100644 src/training/training_state.h mode change 100755 => 100644 src/training/validator.h mode change 100755 => 100644 src/translator/beam_search.h mode change 100755 => 100644 src/translator/helpers.cpp mode change 100755 => 100644 src/translator/helpers.h mode change 100755 => 100644 src/translator/history.h mode change 100755 => 100644 src/translator/hypothesis.h mode change 100755 => 100644 src/translator/nth_element.cpp mode change 100755 => 100644 src/translator/nth_element.cu mode change 100755 => 100644 src/translator/nth_element.h mode change 100755 => 100644 src/translator/output_collector.cpp mode change 100755 => 100644 src/translator/output_collector.h mode change 100755 => 100644 src/translator/output_printer.cpp mode change 100755 => 100644 src/translator/output_printer.h mode change 100755 => 100644 src/translator/scorers.cpp mode change 100755 => 100644 src/translator/scorers.h mode change 100755 => 100644 src/translator/translator.h mode change 100755 => 100644 vs/Marian.sln mode change 100755 => 100644 vs/Marian.vcxproj mode change 100755 => 100644 vs/Marian.vcxproj.filters diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/src/3rd_party/ExceptionWithCallStack.cpp b/src/3rd_party/ExceptionWithCallStack.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/any_type.h b/src/3rd_party/any_type.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/catch.hpp b/src/3rd_party/catch.hpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/cnpy/cnpy.cpp b/src/3rd_party/cnpy/cnpy.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/cnpy/cnpy.h b/src/3rd_party/cnpy/cnpy.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/details/format.h b/src/3rd_party/spdlog/details/format.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/details/logger_impl.h b/src/3rd_party/spdlog/details/logger_impl.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/logger.h b/src/3rd_party/spdlog/logger.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/tests/catch.hpp b/src/3rd_party/spdlog/tests/catch.hpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/threadpool.h b/src/3rd_party/threadpool.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/binary_renamed.cpp b/src/3rd_party/yaml-cpp/binary_renamed.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/collectionstack.h b/src/3rd_party/yaml-cpp/collectionstack.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/dll.h b/src/3rd_party/yaml-cpp/dll.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/emitterstate.cpp b/src/3rd_party/yaml-cpp/emitterstate.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/emitterstate.h b/src/3rd_party/yaml-cpp/emitterstate.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/node/convert.h b/src/3rd_party/yaml-cpp/node/convert.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/node/node.h b/src/3rd_party/yaml-cpp/node/node.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/node_data.cpp b/src/3rd_party/yaml-cpp/node_data.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/scanner.cpp b/src/3rd_party/yaml-cpp/scanner.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/scantoken.cpp b/src/3rd_party/yaml-cpp/scantoken.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/yaml-cpp/singledocparser.cpp b/src/3rd_party/yaml-cpp/singledocparser.cpp old mode 100755 new mode 100644 diff --git a/src/command/marian_conv.cpp b/src/command/marian_conv.cpp old mode 100755 new mode 100644 diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp old mode 100755 new mode 100644 diff --git a/src/command/marian_train.cpp b/src/command/marian_train.cpp old mode 100755 new mode 100644 diff --git a/src/command/marian_vocab.cpp b/src/command/marian_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/common/cli_helper.h b/src/common/cli_helper.h old mode 100755 new mode 100644 diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp old mode 100755 new mode 100644 diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h old mode 100755 new mode 100644 diff --git a/src/common/compile_time_crc32.h b/src/common/compile_time_crc32.h old mode 100755 new mode 100644 diff --git a/src/common/config.cpp b/src/common/config.cpp old mode 100755 new mode 100644 diff --git a/src/common/config.h b/src/common/config.h old mode 100755 new mode 100644 diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100755 new mode 100644 diff --git a/src/common/config_parser.h b/src/common/config_parser.h old mode 100755 new mode 100644 diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp old mode 100755 new mode 100644 diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100755 new mode 100644 diff --git a/src/common/file_stream.h b/src/common/file_stream.h old mode 100755 new mode 100644 diff --git a/src/common/filesystem.h b/src/common/filesystem.h old mode 100755 new mode 100644 diff --git a/src/common/io.cpp b/src/common/io.cpp old mode 100755 new mode 100644 diff --git a/src/common/logging.cpp b/src/common/logging.cpp old mode 100755 new mode 100644 diff --git a/src/common/logging.h b/src/common/logging.h old mode 100755 new mode 100644 diff --git a/src/common/options.h b/src/common/options.h old mode 100755 new mode 100644 diff --git a/src/common/project_version.h.in b/src/common/project_version.h.in old mode 100755 new mode 100644 diff --git a/src/common/shape.h b/src/common/shape.h old mode 100755 new mode 100644 diff --git a/src/common/timer.h b/src/common/timer.h old mode 100755 new mode 100644 diff --git a/src/common/types.h b/src/common/types.h old mode 100755 new mode 100644 diff --git a/src/common/utils.cpp b/src/common/utils.cpp old mode 100755 new mode 100644 diff --git a/src/common/utils.h b/src/common/utils.h old mode 100755 new mode 100644 diff --git a/src/common/version.cpp b/src/common/version.cpp old mode 100755 new mode 100644 diff --git a/src/common/version.h b/src/common/version.h old mode 100755 new mode 100644 diff --git a/src/data/alignment.cpp b/src/data/alignment.cpp old mode 100755 new mode 100644 diff --git a/src/data/alignment.h b/src/data/alignment.h old mode 100755 new mode 100644 diff --git a/src/data/batch.h b/src/data/batch.h old mode 100755 new mode 100644 diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h old mode 100755 new mode 100644 diff --git a/src/data/batch_stats.h b/src/data/batch_stats.h old mode 100755 new mode 100644 diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp old mode 100755 new mode 100644 diff --git a/src/data/corpus.h b/src/data/corpus.h old mode 100755 new mode 100644 diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp old mode 100755 new mode 100644 diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100755 new mode 100644 diff --git a/src/data/corpus_nbest.h b/src/data/corpus_nbest.h old mode 100755 new mode 100644 diff --git a/src/data/corpus_sqlite.h b/src/data/corpus_sqlite.h old mode 100755 new mode 100644 diff --git a/src/data/dataset.h b/src/data/dataset.h old mode 100755 new mode 100644 diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/rng_engine.h b/src/data/rng_engine.h old mode 100755 new mode 100644 diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/shortlist.h b/src/data/shortlist.h old mode 100755 new mode 100644 diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp old mode 100755 new mode 100644 diff --git a/src/data/text_input.h b/src/data/text_input.h old mode 100755 new mode 100644 diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100755 new mode 100644 diff --git a/src/functional/approx.h b/src/functional/approx.h old mode 100755 new mode 100644 diff --git a/src/functional/defs.h b/src/functional/defs.h old mode 100755 new mode 100644 diff --git a/src/functional/functional.h b/src/functional/functional.h old mode 100755 new mode 100644 diff --git a/src/functional/operands.h b/src/functional/operands.h old mode 100755 new mode 100644 diff --git a/src/functional/predicates.h b/src/functional/predicates.h old mode 100755 new mode 100644 diff --git a/src/functional/shape.h b/src/functional/shape.h old mode 100755 new mode 100644 diff --git a/src/graph/chainable.h b/src/graph/chainable.h old mode 100755 new mode 100644 diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp old mode 100755 new mode 100644 diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h old mode 100755 new mode 100644 diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100755 new mode 100644 diff --git a/src/graph/node.cpp b/src/graph/node.cpp old mode 100755 new mode 100644 diff --git a/src/graph/node.h b/src/graph/node.h old mode 100755 new mode 100644 diff --git a/src/graph/node_initializers.cpp b/src/graph/node_initializers.cpp old mode 100755 new mode 100644 diff --git a/src/graph/node_initializers.h b/src/graph/node_initializers.h old mode 100755 new mode 100644 diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h old mode 100755 new mode 100644 diff --git a/src/graph/parameters.h b/src/graph/parameters.h old mode 100755 new mode 100644 diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100755 new mode 100644 diff --git a/src/layers/guided_alignment.h b/src/layers/guided_alignment.h old mode 100755 new mode 100644 diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp old mode 100755 new mode 100644 diff --git a/src/layers/weight.cpp b/src/layers/weight.cpp old mode 100755 new mode 100644 diff --git a/src/layers/weight.h b/src/layers/weight.h old mode 100755 new mode 100644 diff --git a/src/layers/word2vec_reader.h b/src/layers/word2vec_reader.h old mode 100755 new mode 100644 diff --git a/src/marian.h b/src/marian.h old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h old mode 100755 new mode 100644 diff --git a/src/models/amun.h b/src/models/amun.h old mode 100755 new mode 100644 diff --git a/src/models/costs.h b/src/models/costs.h old mode 100755 new mode 100644 diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100755 new mode 100644 diff --git a/src/models/encoder.h b/src/models/encoder.h old mode 100755 new mode 100644 diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100755 new mode 100644 diff --git a/src/models/nematus.h b/src/models/nematus.h old mode 100755 new mode 100644 diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100755 new mode 100644 diff --git a/src/models/states.h b/src/models/states.h old mode 100755 new mode 100644 diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100755 new mode 100644 diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100755 new mode 100644 diff --git a/src/optimizers/optimizers.cpp b/src/optimizers/optimizers.cpp old mode 100755 new mode 100644 diff --git a/src/optimizers/optimizers.h b/src/optimizers/optimizers.h old mode 100755 new mode 100644 diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h old mode 100755 new mode 100644 diff --git a/src/rnn/rnn.h b/src/rnn/rnn.h old mode 100755 new mode 100644 diff --git a/src/rnn/types.h b/src/rnn/types.h old mode 100755 new mode 100644 diff --git a/src/tensors/allocator.h b/src/tensors/allocator.h old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/add.h b/src/tensors/cpu/add.h old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/sharp/int_gemm.h b/src/tensors/cpu/sharp/int_gemm.h old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/device.h b/src/tensors/device.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/algorithm.cu b/src/tensors/gpu/algorithm.cu old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/backend.h b/src/tensors/gpu/backend.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/cuda_helpers.h b/src/tensors/gpu/cuda_helpers.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/device.cu b/src/tensors/gpu/device.cu old mode 100755 new mode 100644 diff --git a/src/tensors/rand.cpp b/src/tensors/rand.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/tensor.h b/src/tensors/tensor.h old mode 100755 new mode 100644 diff --git a/src/tensors/tensor_allocator.h b/src/tensors/tensor_allocator.h old mode 100755 new mode 100644 diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h old mode 100755 new mode 100644 diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp old mode 100755 new mode 100644 diff --git a/src/training/communicator.h b/src/training/communicator.h old mode 100755 new mode 100644 diff --git a/src/training/communicator_nccl.h b/src/training/communicator_nccl.h old mode 100755 new mode 100644 diff --git a/src/training/exponential_smoothing.h b/src/training/exponential_smoothing.h old mode 100755 new mode 100644 diff --git a/src/training/gradient_dropping/sparse_tensor.h b/src/training/gradient_dropping/sparse_tensor.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async_drop.cpp b/src/training/graph_group_async_drop.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async_drop.h b/src/training/graph_group_async_drop.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h old mode 100755 new mode 100644 diff --git a/src/training/scheduler.h b/src/training/scheduler.h old mode 100755 new mode 100644 diff --git a/src/training/training.h b/src/training/training.h old mode 100755 new mode 100644 diff --git a/src/training/training_state.h b/src/training/training_state.h old mode 100755 new mode 100644 diff --git a/src/training/validator.h b/src/training/validator.h old mode 100755 new mode 100644 diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100755 new mode 100644 diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/helpers.h b/src/translator/helpers.h old mode 100755 new mode 100644 diff --git a/src/translator/history.h b/src/translator/history.h old mode 100755 new mode 100644 diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.h b/src/translator/nth_element.h old mode 100755 new mode 100644 diff --git a/src/translator/output_collector.cpp b/src/translator/output_collector.cpp old mode 100755 new mode 100644 diff --git a/src/translator/output_collector.h b/src/translator/output_collector.h old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.cpp b/src/translator/output_printer.cpp old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100755 new mode 100644 diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100755 new mode 100644 diff --git a/src/translator/translator.h b/src/translator/translator.h old mode 100755 new mode 100644 diff --git a/vs/Marian.sln b/vs/Marian.sln old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100755 new mode 100644 From 8ced995144d35f7c179a6f06b16a8fffa96117f5 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 24 Jan 2019 18:44:26 +0000 Subject: [PATCH 194/838] Add scripts generating shortlists --- scripts/shortlist/.gitignore | 3 + scripts/shortlist/README.md | 8 ++ scripts/shortlist/generate_shortlists.pl | 97 ++++++++++++++++++++++++ scripts/shortlist/install.sh | 25 ++++++ 4 files changed, 133 insertions(+) create mode 100644 scripts/shortlist/.gitignore create mode 100644 scripts/shortlist/README.md create mode 100755 scripts/shortlist/generate_shortlists.pl create mode 100755 scripts/shortlist/install.sh diff --git a/scripts/shortlist/.gitignore b/scripts/shortlist/.gitignore new file mode 100644 index 000000000..bf0d379e4 --- /dev/null +++ b/scripts/shortlist/.gitignore @@ -0,0 +1,3 @@ +bin +fast_align +extract-lex diff --git a/scripts/shortlist/README.md b/scripts/shortlist/README.md new file mode 100644 index 000000000..30bf10154 --- /dev/null +++ b/scripts/shortlist/README.md @@ -0,0 +1,8 @@ +`install.sh` is a helper script that downloads and compiles fastalign and extract-lex, and copies +required binaries into _./bin_. + +Shortlist files (_lex.s2t_ and _lex.t2s_) can be created using `generate_shortlists.pl`, for +example: + + perl generate_shortlists.pl --bindir ./bin -s corpus.bpe.src -t corpus.bpe.tgt + diff --git a/scripts/shortlist/generate_shortlists.pl b/scripts/shortlist/generate_shortlists.pl new file mode 100755 index 000000000..309eeef86 --- /dev/null +++ b/scripts/shortlist/generate_shortlists.pl @@ -0,0 +1,97 @@ +#!/usr/bin/env perl + +use strict; +use Getopt::Long; +use FindBin qw($Bin); +use File::Temp qw(tempdir tempfile); +use POSIX; + +my $PID = $$; +$SIG{TERM} = $SIG{INT} = $SIG{QUIT} = sub { die; }; + +my $BINDIR = "$Bin/bin"; +my $SRC; +my $TRG; +my $OUTPUT = "lex"; +my $THREADS = 8; +my $PARALLEL = 0; +my $HELP; + +GetOptions( + "b|bindir=s" => \$BINDIR, + "s|source=s" => \$SRC, + "t|target=s" => \$TRG, + "o|output=s" => \$OUTPUT, + "threads=i" => \$THREADS, + "parallel" => \$PARALLEL, + "h|help" => \$HELP, +); + +if($HELP) { + print "Usage: perl $0 -b bindir -s corpus.src -t corpus.tgt [-o outputprefix] [--threads 8] [--parallel]\n"; + exit 0; +} + +die "--bindir arg is required" if not defined $BINDIR; +die "-s|--source arg is required" if not defined $SRC; +die "-t|--target arg is required" if not defined $TRG; +die "-o|--output arg is required" if not defined $OUTPUT; + +for my $app (qw(fast_align atools extract_lex)) { + die "Could not find $app in $BINDIR" if not -e "$BINDIR/$app"; +} + +my $TEMPDIR = tempdir(CLEANUP => 1); + +my (undef, $CORPUS) = tempfile(DIR => $TEMPDIR); +my (undef, $ALN_S2T) = tempfile(DIR => $TEMPDIR); +my (undef, $ALN_T2S) = tempfile(DIR => $TEMPDIR); +my (undef, $ALN_GDF) = tempfile(DIR => $TEMPDIR); + +execute("paste $SRC $TRG | sed 's/\\t/ ||| /' > $CORPUS"); + +my @COMMANDS = ( + "OMP_NUM_THREADS=$THREADS $BINDIR/fast_align -vdo -i $CORPUS > $ALN_S2T", + "OMP_NUM_THREADS=$THREADS $BINDIR/fast_align -vdor -i $CORPUS > $ALN_T2S" +); + +my @PIDS; +for my $c (@COMMANDS) { + if ($PARALLEL) { + my $pid = fork(); + if (!$pid) { + execute($c); + exit(0); + } else { + push(@PIDS, $pid); + print "Forked process $pid\n"; + } + } else { + execute($c); + } +} +if ($PARALLEL) { + waitpid($_, 0) foreach(@PIDS); +} + +execute("$BINDIR/atools -c grow-diag-final -i $ALN_S2T -j $ALN_T2S > $ALN_GDF"); +execute("$BINDIR/extract_lex $TRG $SRC $ALN_GDF $OUTPUT.s2t $OUTPUT.t2s"); + +sub execute { + my $command = shift; + logMessage("Executing:\t$command"); + my $ret = system($command); + if ($ret != 0) { + logMessage("Command '$command' finished with return status $ret"); + logMessage("Aborting and killing parent process"); + kill(2, $PID); + die; + } +} + +sub logMessage { + my $message = shift; + my $time = POSIX::strftime("%m/%d/%Y %H:%M:%S", localtime()); + my $log_message = $time."\t$message\n"; + print STDERR $log_message; +} diff --git a/scripts/shortlist/install.sh b/scripts/shortlist/install.sh new file mode 100755 index 000000000..49b2171a4 --- /dev/null +++ b/scripts/shortlist/install.sh @@ -0,0 +1,25 @@ +#!/bin/bash -v + +mkdir -p bin + +# download and compile fast_align +if [ ! -e bin/fast_align ]; then + git clone https://github.com/clab/fast_align + mkdir -p fast_align/build + cd fast_align/build + cmake .. + make -j4 + cp fast_align atools ../../bin + cd ../../ +fi + +# download and compile extract-lex +if [ ! -e bin/extract_lex ]; then + git clone https://github.com/marian-nmt/extract-lex + mkdir -p extract-lex/build + cd extract-lex/build + cmake .. + make -j4 + cp extract_lex ../../bin + cd ../../ +fi From 99000874b4784981e42d42d7f3c204bd04e35050 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 24 Jan 2019 20:20:26 +0000 Subject: [PATCH 195/838] Add cblas to libs when compiling without MKL Otherwise on gentoo and CSD3 (without MKL module loaded), I get missing cblas_gemm link errors --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b54766ae9..1c96f4fa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -271,7 +271,7 @@ if(COMPILE_CPU) find_package(BLAS) if(BLAS_FOUND) include_directories(${BLAS_INCLUDE_DIR}) - set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES}) + set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES} cblas) add_definitions(-DBLAS_FOUND=1) endif(BLAS_FOUND) endif(MKL_FOUND) From 2bc18347fa68d8769b4a62652de9605ce4163e60 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 24 Jan 2019 17:22:44 -0800 Subject: [PATCH 196/838] changelog added information about first BERT attempts --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02df12175..3c0dfbfa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Automatic detection of CPU intrisics when building with -arch=native +- First version of BERT-training and BERT-classifier, currently not compatible with TF models ### Fixed - Windows build with recent changes From 960a6b373716ec239721df78c19c6e8c011ecce5 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 24 Jan 2019 18:55:01 -0800 Subject: [PATCH 197/838] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0dfbfa9..98e89ff77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Automatic detection of CPU intrisics when building with -arch=native - First version of BERT-training and BERT-classifier, currently not compatible with TF models +- New reduction operators ### Fixed - Windows build with recent changes From c9be3d0a8ca7b35ee21d9e3a90dc34b63b41f204 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 24 Jan 2019 19:14:12 -0800 Subject: [PATCH 198/838] make CUDNN files compile --- src/layers/convolution.cpp | 8 +++++--- src/models/model_factory.cpp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/layers/convolution.cpp b/src/layers/convolution.cpp index 225058e7c..9453d7f01 100644 --- a/src/layers/convolution.cpp +++ b/src/layers/convolution.cpp @@ -4,9 +4,11 @@ namespace marian { #ifdef CUDNN -Convolution::Convolution(Ptr graph) : Factory(graph) {} +Convolution::Convolution(Ptr graph) {} Expr Convolution::apply(Expr x) { + auto graph = x->graph(); + auto prefix = opt("prefix"); auto kernelDims = opt>("kernel-dims"); auto kernelNum = opt("kernel-num"); @@ -15,11 +17,11 @@ Expr Convolution::apply(Expr x) { int layerIn = x->shape()[1]; auto kernel - = graph_->param(prefix + "_conv_kernels", + = graph->param(prefix + "_conv_kernels", {layerIn, kernelNum, kernelDims.first, kernelDims.second}, inits::glorot_uniform); - auto bias = graph_->param( + auto bias = graph->param( prefix + "_conv_bias", {1, kernelNum, 1, 1}, inits::zeros); std::vector nodes = {x, kernel, bias}; diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 9eb1931ca..946b8a3ab 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -290,7 +290,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { ("original-type", type) .push_back(models::encoder()("type", "char-s2s")) .push_back(models::decoder()("type", "s2s")) - .construct(); + .construct(graph); } #endif From fbc49d0dd6b2115613cdd73aa52374d0ca984714 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Thu, 24 Jan 2019 20:29:44 -0800 Subject: [PATCH 199/838] disable CSR operator test --- src/tests/operator_tests.cpp | 122 +++++++++++++++++------------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index 2607f41ff..e17eca9dc 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -343,65 +343,65 @@ void tests(DeviceType device) { auto C = dot(A, B); // CSR dot product, tested against dense product on the same values - std::vector vS({1, 0, 0, 1, // sparse - 0, 0, 1, 1.5}); - std::vector vD({1, 2, 3, 1.2, 5.6, // dense - 4, 5, 6, 2.3, 6.7, - 7, 8, 9, 3.4, 7.8, - 1, 1, 2, 4.5, 8.9}); - auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); - auto D = graph->param("D", { 4, 5 }, inits::from_vector(vD)); - auto DT = graph->param("DT", { 5, 4 }, inits::from_vector(vD)); // example matrix with transposed dimensions - std::vector SV; // create CSR version of S - std::vector SI, SO; - SO.push_back((IndexType)SI.size()); - for (IndexType i = 0; i < S->shape()[0]; i++) { - for (IndexType j = 0; j < S->shape()[1]; j++) { - auto k = 4 * i + j; - if (vS[k] != 0) { - SV.push_back(vS[k]); - SI.push_back(j); - } - } - SO.push_back((IndexType)SI.size()); - } - - auto SxDd = dot(S, D); - auto STxSxDd = dot(S, SxDd, /*transA=*/true); - auto SxDs = csr_dot( // sparse x dense - S->shape(), - graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - D); - auto STxSxDs = csr_dot( // transpose(sparse) x dense; we use result of previous since dimensions match - S->shape(), - graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - SxDd, /*transS=*/true); - - auto DTxSTd = dot(DT, S, /*transA=*/false, /*transB=*/true); - auto DTxSTxSd = dot(DTxSTd, S); - auto DTxSTs = dot_csr( // dense x sparse - DT, - S->shape(), - graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - /*transS=*/true); - auto DTxSTxSs = dot_csr( // dense x transpose(sparse) - DTxSTd, - S->shape(), - graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32)); + // std::vector vS({1, 0, 0, 1, // sparse + // 0, 0, 1, 1.5}); + // std::vector vD({1, 2, 3, 1.2, 5.6, // dense + // 4, 5, 6, 2.3, 6.7, + // 7, 8, 9, 3.4, 7.8, + // 1, 1, 2, 4.5, 8.9}); + // auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); + // auto D = graph->param("D", { 4, 5 }, inits::from_vector(vD)); + // auto DT = graph->param("DT", { 5, 4 }, inits::from_vector(vD)); // example matrix with transposed dimensions + // std::vector SV; // create CSR version of S + // std::vector SI, SO; + // SO.push_back((IndexType)SI.size()); + // for (IndexType i = 0; i < S->shape()[0]; i++) { + // for (IndexType j = 0; j < S->shape()[1]; j++) { + // auto k = 4 * i + j; + // if (vS[k] != 0) { + // SV.push_back(vS[k]); + // SI.push_back(j); + // } + // } + // SO.push_back((IndexType)SI.size()); + // } + + // auto SxDd = dot(S, D); + // auto STxSxDd = dot(S, SxDd, /*transA=*/true); + // auto SxDs = csr_dot( // sparse x dense + // S->shape(), + // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + // D); + // auto STxSxDs = csr_dot( // transpose(sparse) x dense; we use result of previous since dimensions match + // S->shape(), + // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + // SxDd, /*transS=*/true); + + // auto DTxSTd = dot(DT, S, /*transA=*/false, /*transB=*/true); + // auto DTxSTxSd = dot(DTxSTd, S); + // auto DTxSTs = dot_csr( // dense x sparse + // DT, + // S->shape(), + // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + // /*transS=*/true); + // auto DTxSTxSs = dot_csr( // dense x transpose(sparse) + // DTxSTd, + // S->shape(), + // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32)); CHECK(C->shape() == Shape({2, 2, 2})); - CHECK(SxDs->shape() == SxDd->shape()); - CHECK(STxSxDs->shape() == STxSxDd->shape()); - CHECK(DTxSTs->shape() == DTxSTd->shape()); - CHECK(DTxSTxSs->shape() == DTxSTxSd->shape()); + // CHECK(SxDs->shape() == SxDd->shape()); + // CHECK(STxSxDs->shape() == STxSxDd->shape()); + // CHECK(DTxSTs->shape() == DTxSTd->shape()); + // CHECK(DTxSTxSs->shape() == DTxSTxSd->shape()); graph->forward(); @@ -409,10 +409,10 @@ void tests(DeviceType device) { CHECK(values == vC); // dense and sparse operation results must be the same - SxDd ->val()->get(values2); SxDs ->val()->get(values); CHECK(values == values2); - STxSxDd ->val()->get(values2); STxSxDs ->val()->get(values); CHECK(values == values2); - DTxSTd ->val()->get(values2); DTxSTs ->val()->get(values); CHECK(values == values2); - DTxSTxSd->val()->get(values2); DTxSTxSs->val()->get(values); CHECK(values == values2); + // SxDd ->val()->get(values2); SxDs ->val()->get(values); CHECK(values == values2); + // STxSxDd ->val()->get(values2); STxSxDs ->val()->get(values); CHECK(values == values2); + // DTxSTd ->val()->get(values2); DTxSTs ->val()->get(values); CHECK(values == values2); + // DTxSTxSd->val()->get(values2); DTxSTxSs->val()->get(values); CHECK(values == values2); } SECTION("affine transformation") { From a8336cbac030fba374806796d0a1d87a09d69274 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 25 Jan 2019 10:09:56 +0000 Subject: [PATCH 200/838] Update README --- README.md | 56 ++++++++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 47f79ca71..6fcc45b0b 100644 --- a/README.md +++ b/README.md @@ -8,33 +8,18 @@ Marian [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE.md) [![Twitter](https://img.shields.io/twitter/follow/marian_nmt.svg?style=social)](https://twitter.com/intent/follow?screen_name=marian_nmt) -

- Marian is an efficient Neural Machine Translation framework written - in pure C++ with minimal dependencies. - - Named in honour of Marian Rejewski, a Polish mathematician and cryptologist. - - -

- - - -

- Main features: -

    -
  • Fast multi-gpu training and translation
  • -
  • Compatible with Nematus and DL4MT
  • -
  • Efficient pure C++ implementation
  • -
  • Permissive open source license (MIT)
  • -
  • more details...
  • -
-

+*Marian* is an efficient Neural Machine Translation framework written in pure +C++ with minimal dependencies. + +Named in honour of Marian Rejewski, a Polish mathematician and cryptologist. + +Main features: + +- Efficient pure C++ implementation +- Fast multi-GPU training and GPU/CPU translation +- State-of-the-art NMT architectures: deep RNN and transformer +- Permissive open source license (MIT) +- [more detail...](https://marian-nmt.github.io/features) If you use this, please cite: @@ -59,20 +44,11 @@ Machine Translation in C++ (http://www.aclweb.org/anthology/P18-4020) url = {http://www.aclweb.org/anthology/P18-4020} } - - ## Amun -The handwritten decoder for RNN models compatible with Marian and Nematus has been superseded by the Marian decoder. The code is available in a separate repository: https://github.com/marian-nmt/amun + +The handwritten decoder for RNN models compatible with Marian and Nematus has +been superseded by the Marian decoder. The code is available in a separate +repository: https://github.com/marian-nmt/amun ## Website From a6c9b59228b28f914a7e529f9c42a8f6b8a24fba Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 25 Jan 2019 10:10:44 +0000 Subject: [PATCH 201/838] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 12751ca7b..f0b77c197 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.7.6 +v1.7.7 From c6211c9276b3fc89d8b89f3ff26163c990cf1aa2 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 25 Jan 2019 11:05:58 +0000 Subject: [PATCH 202/838] Rename --problem to --task --- src/common/config_parser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index eef2e5b8f..967305511 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -399,7 +399,7 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { // add ULR settings addSuboptionsULR(cli); - cli.add>("--problem", + cli.add>("--task", "Use predefined set of options. Possible values: transformer, transformer-big"); // clang-format on } @@ -672,7 +672,7 @@ void ConfigParser::addAliases(cli::CLIWrapper& cli) { config["skip"] = true; }); - cli.alias("problem", "transformer", [](YAML::Node& config) { + cli.alias("task", "transformer", [](YAML::Node& config) { config["type"] = "transformer"; config["enc-depth"] = 6; config["dec-depth"] = 6; @@ -686,7 +686,7 @@ void ConfigParser::addAliases(cli::CLIWrapper& cli) { config["clip-norm"] = 5; }); - cli.alias("problem", "transformer-big", [](YAML::Node& config) { + cli.alias("task", "transformer-big", [](YAML::Node& config) { config["type"] = "transformer"; config["enc-depth"] = 6; config["dec-depth"] = 6; From 8ef1caa0ec154934dd943dddc54ee72e875cac64 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sat, 26 Jan 2019 10:31:49 +0000 Subject: [PATCH 203/838] Rename nonempty to hasAndNotEmpty; fix some comments --- src/common/options.h | 2 +- src/common/utils.cpp | 9 ++++----- src/data/corpus.h | 2 +- src/data/corpus_base.cpp | 2 +- src/data/corpus_base.h | 11 ++++++----- src/data/corpus_sqlite.h | 2 +- src/layers/weight.cpp | 2 +- src/microsoft/quicksand.cpp | 4 ++-- src/models/costs.h | 19 +++++++++---------- src/models/decoder.h | 6 +++--- src/models/s2s.h | 2 +- src/models/transformer.h | 4 ++-- src/rescorer/rescorer.h | 4 ++-- src/tests/cli.cpp | 2 +- src/training/graph_group_async.cpp | 4 ++-- src/training/graph_group_multinode.h | 2 +- src/training/graph_group_multinode_sync.h | 2 +- src/training/graph_group_singleton.h | 2 +- src/training/graph_group_sync.cpp | 2 +- src/training/scheduler.h | 3 +-- src/training/training.h | 2 +- src/training/validator.h | 13 +++++++------ src/translator/beam_search.h | 2 +- src/translator/scorers.cpp | 4 ++-- src/translator/translator.h | 2 +- 25 files changed, 54 insertions(+), 55 deletions(-) diff --git a/src/common/options.h b/src/common/options.h index b413f7db3..5a3f4eb70 100755 --- a/src/common/options.h +++ b/src/common/options.h @@ -102,7 +102,7 @@ class Options { * * @return true if the option is defined and is a nonempty sequence or string */ - bool nonempty(const std::string& key) const { + bool hasAndNotEmpty(const std::string& key) const { if(!has(key)) { return false; } diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 068ce9c2c..8a726fe57 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -147,8 +147,7 @@ std::string toUpper(const std::string& s) { double parseDouble(std::string s) { double res; - char c; // dummy char -- if we succeed to parse this, then there were extraneous characters after - // the number + char c; // dummy char -- if we succeed to parse this, then there were extraneous characters after the number auto rc = sscanf(s.c_str(), "%lf%c", &res, &c); ABORT_IF(rc != 1, "Mal-formed number: {}", s); return res; @@ -160,9 +159,9 @@ double parseNumber(std::string param) { double factor = 1.; if(!param.empty() && param.back() >= 'A') { switch(param.back()) { - case 'k': factor = 1.e3; break; - case 'M': factor = 1.e6; break; - case 'G': factor = 1.e9; break; + case 'k': factor = 1.e3; break; + case 'M': factor = 1.e6; break; + case 'G': factor = 1.e9; break; case 'T': factor = 1.e12; break; default: ABORT("Invalid or unsupported unit prefix '{}' in {}", param.back(), param); } diff --git a/src/data/corpus.h b/src/data/corpus.h index 7c8a0a9cb..b459eb87a 100755 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -98,7 +98,7 @@ class Corpus : public CorpusBase { if(options_->get("guided-alignment", std::string("none")) != "none" && alignFileIdx_) addAlignmentsToBatch(batch, batchVector); - if(options_->nonempty("data-weighting") && weightFileIdx_) + if(options_->hasAndNotEmpty("data-weighting") && weightFileIdx_) addWeightsToBatch(batch, batchVector); return batch; diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index 867b540ab..94168a33a 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -178,7 +178,7 @@ CorpusBase::CorpusBase(Ptr options, bool translate) ABORT_IF(files_.back()->empty(), "File with alignments '{}' is empty", path); } - if(training && options_->nonempty("data-weighting")) { + if(training && options_->hasAndNotEmpty("data-weighting")) { auto path = options_->get("data-weighting"); ABORT_IF(!filesystem::exists(path), "Weight file does not exist"); diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index e496e50ed..81c1e6aa0 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -311,7 +311,7 @@ class CorpusBatch : public Batch { * @return Fake batch of the same size as the real batch. */ static Ptr fakeBatch(const std::vector& lengths, - const std::vector>& vocabs, + const std::vector>& vocabs, size_t batchSize, Ptr options) { std::vector> batches; @@ -322,7 +322,7 @@ class CorpusBatch : public Batch { // set word indices to different values to avoid same hashes // rand() is OK, this does not affect state in any way std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), - [&](Word) { return rand() % vocabs[batchIndex]->size(); }); + [&](Word) { return rand() % vocabs[batchIndex]->size(); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); batchIndex++; @@ -341,7 +341,7 @@ class CorpusBatch : public Batch { batch->setGuidedAlignment(std::move(alignment)); } - if(options->nonempty("data-weighting")) { + if(options->hasAndNotEmpty("data-weighting")) { auto weightsSize = batchSize; if(options->get("data-weighting-type") != "sentence") weightsSize *= lengths.back(); @@ -521,9 +521,10 @@ class CorpusBase /** * brief Determines if a EOS symbol should be added. By default this is true for any sequence, - * but should be false for instance for classifier labels. This is set per input stream, hence a vector. + * but should be false for instance for classifier labels. This is set per input stream, hence a + * vector. */ - std::vector addEOS_; + std::vector addEOS_; size_t pos_{0}; diff --git a/src/data/corpus_sqlite.h b/src/data/corpus_sqlite.h index ba40ecb49..0da2a8647 100755 --- a/src/data/corpus_sqlite.h +++ b/src/data/corpus_sqlite.h @@ -104,7 +104,7 @@ class CorpusSQLite : public CorpusBase { if(options_->has("guided-alignment") && alignFileIdx_) addAlignmentsToBatch(batch, batchVector); - if(options_->nonempty("data-weighting") && weightFileIdx_) + if(options_->hasAndNotEmpty("data-weighting") && weightFileIdx_) addWeightsToBatch(batch, batchVector); return batch; diff --git a/src/layers/weight.cpp b/src/layers/weight.cpp index 98e97e4e9..50d749a21 100755 --- a/src/layers/weight.cpp +++ b/src/layers/weight.cpp @@ -3,7 +3,7 @@ namespace marian { Ptr WeightingFactory(Ptr options) { - ABORT_IF(!options->nonempty("data-weighting"), + ABORT_IF(!options->hasAndNotEmpty("data-weighting"), "No data-weighting specified in options"); return New(options->get("data-weighting-type")); } diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 2a82263f7..959e44311 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -43,7 +43,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { const std::vector& ptrs, Word eos) : IBeamSearchDecoder(options, ptrs, eos) { - + // setting 16-bit optimization to false for now. Re-enable with better caching or pre-computation graph_ = New(/*inference=*/true, /*optimize=*/false); @@ -136,7 +136,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { auto score = std::get<2>(result); // determine alignment if present AlignmentSets alignmentSets; - if (options_->nonempty("alignment")) + if (options_->hasAndNotEmpty("alignment")) { float alignmentThreshold; auto alignment = options_->get("alignment"); // @TODO: this logic now exists three times in Marian diff --git a/src/models/costs.h b/src/models/costs.h index 42e2220dd..2aba6fe09 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -16,8 +16,8 @@ namespace models { // @TODO: inheritance and polymorphism is used here in a rather unclear way. // E.g. returns Ptr which should be Ptr? -// Other functions return RationalLoss directly without Ptr<...>, but also -// they do not need polymorphism here. +// Other functions return RationalLoss directly without Ptr<...>, but also +// they do not need polymorphism here. class CostBase { public: @@ -44,7 +44,7 @@ class EncoderDecoderCE : public CostBase { loss_ = newLoss(options_, inference_); toBeWeighted_ - = (options_->nonempty("data-weighting") && !inference_) + = (options_->hasAndNotEmpty("data-weighting") && !inference_) || (options_->has("dynamic-weighting") && options_->get("dynamic-weighting") && !inference_); if(toBeWeighted_) @@ -73,7 +73,7 @@ class EncoderDecoderCE : public CostBase { state->getTargetMask(), weights); multiLoss->push_back(partialLoss); - + if(options_->get("guided-alignment", std::string("none")) != "none" && !inference_) { auto attentionVectors = encdec->getDecoders()[0]->getAlignments(); ABORT_IF(attentionVectors.empty(), "Model does not seem to support alignments"); @@ -83,9 +83,8 @@ class EncoderDecoderCE : public CostBase { auto alignmentLoss = guidedAlignmentCost(graph, corpusBatch, options_, attention); multiLoss->push_back(alignmentLoss); } - + return multiLoss; - } }; @@ -96,7 +95,7 @@ class EncoderClassifierCE : public CostBase { bool inference_{false}; // @TODO: single loss seems wrong, especially since we support multiple objectives here, - // also not sure this needs to be a member at all. + // also not sure this needs to be a member at all. Ptr loss_; public: @@ -114,7 +113,7 @@ class EncoderClassifierCE : public CostBase { auto corpusBatch = std::static_pointer_cast(batch); auto states = enccls->apply(graph, corpusBatch, clearGraph); - + // multi-objective training Ptr multiLoss = newMultiLoss(options_); for(int i = 0; i < states.size(); ++i) { @@ -284,7 +283,7 @@ inline Ptr add_cost(Ptr encdec, else return New(encdec, New()); case usage::raw: - default: + default: return encdec; } } @@ -299,7 +298,7 @@ inline Ptr add_cost(Ptr enccls, case usage::translation: ABORT("Classifier cannot be used for translation"); case usage::raw: - default: + default: return enccls; } } diff --git a/src/models/decoder.h b/src/models/decoder.h index 079cf66fb..bc0c91191 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -52,7 +52,7 @@ class DecoderBase { if(options_->has("embedding-fix-trg")) yEmbFactory("fixed", opt("embedding-fix-trg")); - if(options_->nonempty("embedding-vectors")) { + if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); yEmbFactory("embFile", embFiles[batchIndex_]) // ("normalization", opt("embedding-normalization")); @@ -95,12 +95,12 @@ class DecoderBase { auto yEmbFactory = embedding() // ("dimVocab", dimTrgVoc) // ("dimEmb", dimTrgEmb); - + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) yEmbFactory("prefix", "Wemb"); else yEmbFactory("prefix", prefix_ + "_Wemb"); - + auto yEmb = yEmbFactory.construct(graph); selectedEmbs = yEmb->apply(embIdx, {dimBeam, 1, dimBatch, dimTrgEmb}); diff --git a/src/models/s2s.h b/src/models/s2s.h index 873227194..35846e783 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -136,7 +136,7 @@ class EncoderS2S : public EncoderBase { if(options_->has("embedding-fix-src")) embFactory("fixed", opt("embedding-fix-src")); - if(options_->nonempty("embedding-vectors")) { + if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory // ("embFile", embFiles[batchIndex_]) // diff --git a/src/models/transformer.h b/src/models/transformer.h index 87ce85cc4..f98ba1a91 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -533,7 +533,7 @@ class EncoderTransformer : public Transformer { embFactory("prefix", prefix_ + "_Wemb"); if(options_->has("embedding-fix-src")) embFactory("fixed", opt("embedding-fix-src")); - if(options_->nonempty("embedding-vectors")) { + if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory("embFile", embFiles[subBatchIndex]) ("normalization", opt("embedding-normalization")); @@ -797,7 +797,7 @@ class DecoderTransformer : public Transformer { // decoding or scoring return the attention weights of one head of the last layer. // @TODO: maybe allow to return average or max over all heads? bool saveAttentionWeights = false; - if(j == 0 && (options_->get("guided-alignment", std::string("none")) != "none" || options_->nonempty("alignment"))) { + if(j == 0 && (options_->get("guided-alignment", std::string("none")) != "none" || options_->hasAndNotEmpty("alignment"))) { size_t attLayer = decDepth - 1; std::string gaStr = options_->get("transformer-guided-alignment-layer", "last"); if(gaStr != "last") diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 25abe97d9..ee2a3c9a5 100644 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -49,10 +49,10 @@ class Rescore : public ModelTask { public: Rescore(Ptr options) : options_(options) { - ABORT_IF(options_->nonempty("summary") && options_->nonempty("alignment"), + ABORT_IF(options_->hasAndNotEmpty("summary") && options_->hasAndNotEmpty("alignment"), "Alignments can not be produced with summarized score"); - ABORT_IF(options_->nonempty("summary") && options_->get("normalize"), + ABORT_IF(options_->hasAndNotEmpty("summary") && options_->get("normalize"), "Normalization by length cannot be used with summary scores"); options_->set("inference", true); diff --git a/src/tests/cli.cpp b/src/tests/cli.cpp index c3210514b..f829041ea 100644 --- a/src/tests/cli.cpp +++ b/src/tests/cli.cpp @@ -69,7 +69,7 @@ int main(int argc, char** argv) { std::cout << emit.c_str() << std::endl; std::cout << "===" << std::endl; - std::cout << "vec/str.nonempty? " << options->nonempty("vec") << " " << options->nonempty("str") << std::endl; + std::cout << "vec/str.hasAndNotEmpty? " << options->hasAndNotEmpty("vec") << " " << options->hasAndNotEmpty("str") << std::endl; std::cout << "vec/str.has? " << options->has("vec") << " " << options->has("str") << std::endl; return 0; diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index e2fb0aad6..f1a01cdfa 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -256,7 +256,7 @@ void AsyncGraphGroup::execute(Ptr batch) { std::vector> vocabs; auto fb = data::CorpusBatch::fakeBatch(fakeLength, vocabs, num_seen_sentences, NULL); fb->front()->setWords(num_seen_words); - + scheduler_->update(loss, fb); num_seen_words = 0; @@ -325,7 +325,7 @@ void AsyncGraphGroup::load() { setFn(i, data.begin() + begin, data.begin() + end); } }); - } else if(options_->nonempty("pretrained-model")) { + } else if(options_->hasAndNotEmpty("pretrained-model")) { std::string nameInit = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_multinode.h b/src/training/graph_group_multinode.h index f98f56316..340fdcbb3 100755 --- a/src/training/graph_group_multinode.h +++ b/src/training/graph_group_multinode.h @@ -397,7 +397,7 @@ class MultiNodeGraphGroup : public MultiNodeGraphGroupBase { size_t i = 0; for(auto graph : clientGraphs_) clientBuilders_[i++]->load(graph, name); - } else if(options_->nonempty("pretrained-model")) { + } else if(options_->hasAndNotEmpty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_multinode_sync.h b/src/training/graph_group_multinode_sync.h index c94bc50c0..ff1fe4e97 100755 --- a/src/training/graph_group_multinode_sync.h +++ b/src/training/graph_group_multinode_sync.h @@ -163,7 +163,7 @@ class MultiNodeGraphGroupSync : public MultiNodeGraphGroupBase { size_t i = 0; for(auto graph : clientGraphs_) clientBuilders_[i++]->load(graph, name); - } else if(options_->nonempty("pretrained-model")) { + } else if(options_->hasAndNotEmpty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index a750d6b35..01ffb1eb2 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -70,7 +70,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { /*scatterStateFn=*/[&](const std::vector& data, const OptimizerBase::ScatterStateSetFunc& setFn) { setFn(/*localDeviceIndex=*/0, data.begin(), data.end()); }); - } else if(options_->nonempty("pretrained-model")) { + } else if(options_->hasAndNotEmpty("pretrained-model")) { std::string init = options_->get("pretrained-model"); LOG(info, "Initialize model weights with the pre-trained model {}", diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 47879ab9d..c02a34afa 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -453,7 +453,7 @@ void SyncGraphGroup::load() /*override*/ { comm_->scatterState(optimizerStateVector, setShardFn); }); LOG(info, "[training] Model reloaded from {}", name); - } else if(options_->nonempty("pretrained-model")) { + } else if(options_->hasAndNotEmpty("pretrained-model")) { std::string nameInit = options_->get("pretrained-model"); LOG(info, "[training] Initializing model weights with the pre-trained model {}", diff --git a/src/training/scheduler.h b/src/training/scheduler.h index b72d8e31e..8f5c3bd90 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -282,8 +282,7 @@ class Scheduler : public TrainingObserver { state_->updatesDisp += 1; state_->samplesDisp += batchSize; - state_->wordsDisp += batchLabels; // words at given input processed since last display, for speed display - // @TODO: this is wrong + state_->wordsDisp += batchLabels; // @TODO: this is wrong // words at given input processed since last display, for speed display state_->samplesEpoch += batchSize; // sentences processed in this epoch state_->labelsTotal += rationalLoss.count; // total labels processed diff --git a/src/training/training.h b/src/training/training.h index 485318367..8863c165d 100755 --- a/src/training/training.h +++ b/src/training/training.h @@ -51,7 +51,7 @@ class Train : public ModelTask { LOG(info, "[batching] Done. Typical MB size is {} target words", stats->estimateTypicalTrgWords()); } - if((options_->nonempty("valid-sets") || options_->nonempty("valid-script-path")) + if((options_->hasAndNotEmpty("valid-sets") || options_->hasAndNotEmpty("valid-script-path")) && SchedulingParameter::parse(options_->get("valid-freq"))) { for(auto validator : Validators(dataset->getVocabs(), options_)) scheduler->addValidator(validator); diff --git a/src/training/validator.h b/src/training/validator.h index 6c82d36ea..fea1be75f 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -420,7 +420,8 @@ class ScriptValidator : public Validator { : Validator(vocabs, options, false) { builder_ = models::from_options(options_, models::usage::raw); - ABORT_IF(!options_->nonempty("valid-script-path"), "valid-script metric but no script given"); + ABORT_IF(!options_->hasAndNotEmpty("valid-script-path"), + "valid-script metric but no script given"); } virtual float validate(const std::vector>& graphs) override { @@ -451,7 +452,7 @@ class TranslationValidator : public Validator { quiet_(options_->get("quiet-translation")) { builder_ = models::from_options(options_, models::usage::translation); - if(!options_->nonempty("valid-script-path")) + if(!options_->hasAndNotEmpty("valid-script-path")) LOG_VALID(warn, "No post-processing script given for validating translator"); createBatchGenerator(/*isTranslating=*/true); @@ -482,7 +483,7 @@ class TranslationValidator : public Validator { std::string fileName; Ptr tempFile; - if(options_->nonempty("valid-translation-output")) { + if(options_->hasAndNotEmpty("valid-translation-output")) { fileName = options_->get("valid-translation-output"); } else { tempFile.reset(new io::TemporaryFile(options_->get("tempdir"), false)); @@ -500,7 +501,7 @@ class TranslationValidator : public Validator { auto printer = New(options_, vocabs_.back()); // @TODO: This can be simplified. If there is no "valid-translation-output", fileName already // contains the name of temporary file that should be used? - auto collector = options_->nonempty("valid-translation-output") + auto collector = options_->hasAndNotEmpty("valid-translation-output") ? New(fileName) : New(*tempFile); @@ -555,7 +556,7 @@ class TranslationValidator : public Validator { float val = 0.0f; // Run post-processing script if given - if(options_->nonempty("valid-script-path")) { + if(options_->hasAndNotEmpty("valid-script-path")) { auto command = options_->get("valid-script-path") + " " + fileName; auto valStr = utils::exec(command); val = (float)std::atof(valStr.c_str()); @@ -639,7 +640,7 @@ class BleuValidator : public Validator { auto printer = New(options_, vocabs_.back()); Ptr collector; - if(options_->nonempty("valid-translation-output")) { + if(options_->hasAndNotEmpty("valid-translation-output")) { auto fileName = options_->get("valid-translation-output"); collector = New(fileName); // for debugging } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 50bf5ccbc..17ba6d573 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -42,7 +42,7 @@ class BeamSearch { Beams newBeams(beams.size()); std::vector align; - if(options_->nonempty("alignment")) + if(options_->hasAndNotEmpty("alignment")) // Use alignments from the first scorer, even if ensemble align = scorers_[0]->getAlignment(); diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp index 6c0031bff..7a07a8cd5 100755 --- a/src/translator/scorers.cpp +++ b/src/translator/scorers.cpp @@ -53,7 +53,7 @@ std::vector> createScorers(Ptr options) { auto models = options->get>("models"); std::vector weights(models.size(), 1.f); - if(options->nonempty("weights")) + if(options->hasAndNotEmpty("weights")) weights = options->get>("weights"); size_t i = 0; @@ -83,7 +83,7 @@ std::vector> createScorers(Ptr options, const std::vector> scorers; std::vector weights(ptrs.size(), 1.f); - if(options->nonempty("weights")) + if(options->hasAndNotEmpty("weights")) weights = options->get>("weights"); size_t i = 0; diff --git a/src/translator/translator.h b/src/translator/translator.h index 877312b81..18371a3de 100755 --- a/src/translator/translator.h +++ b/src/translator/translator.h @@ -41,7 +41,7 @@ class Translate : public ModelTask { trgVocab_->load(vocabs.back()); auto srcVocab = corpus_->getVocabs()[0]; - if(options_->nonempty("shortlist")) + if(options_->hasAndNotEmpty("shortlist")) shortlistGenerator_ = New( options_, srcVocab, trgVocab_, 0, 1, vocabs.front() == vocabs.back()); From 79368b121f29a0c12beada95473655220ead8827 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sat, 26 Jan 2019 14:14:57 -0800 Subject: [PATCH 204/838] add gelu activation --- src/graph/expression_operators.cpp | 9 ++++++++ src/graph/expression_operators.h | 3 +++ src/graph/node_operators_unary.h | 35 ++++++++++++++++++++++++------ src/layers/loss.h | 1 + src/models/bert.h | 26 +++++++++++----------- src/models/transformer.h | 2 ++ src/tensors/gpu/add.inc | 1 + src/tensors/gpu/element.inc | 2 ++ 8 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index c0640edbf..e558ffd08 100644 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -51,6 +51,10 @@ Expr swish(Expr a) { return Expression(a); } +Expr gelu(Expr a) { + return Expression(a, 1.702f); +} + Expr operator-(Expr a) { return Expression(a); }; @@ -529,6 +533,11 @@ Expr swish(const std::vector& nodes) { return swish(nodes[0]); } +Expr gelu(const std::vector& nodes) { + ABORT_IF(nodes.size() > 1, "Not implemented"); + return gelu(nodes[0]); +} + Expr tanh(const std::vector& nodes) { return Expression(nodes); } diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index 78aed8347..0205fe864 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -17,6 +17,9 @@ Expr sigmoid(const std::vector&); Expr swish(Expr a); Expr swish(const std::vector&); +Expr gelu(Expr a); +Expr gelu(const std::vector&); + Expr tanh(const std::vector&); template diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 63756da3a..190fa947f 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -343,31 +343,52 @@ struct PReLUNodeOp : public UnaryNodeOp { * in an expression graph. * * This node implements the activation function - * \f$ f(x) = x \cdot \sigma(x) \f$ + * \f$ f(x) = x \cdot \sigma(bx) \f$ * and its derivative - * \f$ f^\prime(x) = f(x) + \sigma(x)(1 - f(x)) \f$ . + * \f$ f^\prime(x) = bf(x) + \sigma(bx)(1 - bf(x)) \f$ . * */ struct SwishNodeOp : public UnaryNodeOp { - SwishNodeOp(Expr a) : UnaryNodeOp(a) {} + SwishNodeOp(Expr a, float b = 1.f) : UnaryNodeOp(a), b_{b} {} NodeOps forwardOps() override { using namespace functional; - return {NodeOp(Element(_1 = _2 * sigmoid(_2), val_, child(0)->val()))}; + return {NodeOp(Element(_1 = _2 * sigmoid(b_ * _2), val_, child(0)->val()))}; } NodeOps backwardOps() override { using namespace functional; - // dJ/dx += dJ/df * ( f(x) + sigma(x) * (1 - f(x)) ) - return {NodeOp(Add(_1 * (_3 + sigmoid(_2) * (1.f - _3)), + // dJ/dx += dJ/df * (b*f(x) + sigmoid(b*x) * (1 - b*f(x))) + return {NodeOp(Add(_1 * (b_ * _3 + sigmoid(b_ * _2) * (1.f - (b_ * _3))), child(0)->grad(), // dJ/dx adj_, // _1 := dJ/df child(0)->val(), // _2 := x - val_ // _3 := f(x) = x*sigma(x) + val_ // _3 := f(x) = x*sigmoid(b*x) ))}; } const std::string type() override { return "swish"; } + + virtual size_t hash() override { + if(!hash_) { + hash_ = NaryNodeOp::hash(); + util::hash_combine(hash_, b_); + } + return hash_; + } + + virtual bool equal(Expr node) override { + if(!NaryNodeOp::equal(node)) + return false; + Ptr cnode = std::dynamic_pointer_cast(node); + if(!cnode) + return false; + if(b_ != cnode->b_) + return false; + return true; + } + + float b_; }; struct SoftmaxNodeOp : public UnaryNodeOp { diff --git a/src/layers/loss.h b/src/layers/loss.h index 4a28de7b2..2ac4ae78d 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -331,6 +331,7 @@ class CrossEntropyLoss : public LabelwiseLoss { virtual Expr compute(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { + logits = atleast_3d(logits); // safeguard against 2d classifier output, adds 1 on the left, non-op. Expr ce = cross_entropy(logits, labelIndices); if(labelSmoothing_ > 0) { diff --git a/src/models/bert.h b/src/models/bert.h index c642abac4..b4d2a34c2 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -144,7 +144,7 @@ class BertBatch : public CorpusBatch { int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); - int maxSentPos = 2; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] + int maxSentPos = 1; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] // If another separator is seen do not increase position index beyond 2 but use padding. // @TODO: make this configurable, see below for NextSentencePredictions task where we also restrict to 2. @@ -231,7 +231,7 @@ class BertEncoder : public EncoderTransformer { if(learnedPosEmbeddings) { auto sentenceEmbeddings = embedding() ("prefix", "Wsent") - ("dimVocab", 3) // sentence A or sentence B plus padding, @TODO: should rather be a parameter + ("dimVocab", 2) // sentence A or sentence B plus padding, @TODO: should rather be a parameter ("dimEmb", dimEmb) .construct(graph_); signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); @@ -327,24 +327,24 @@ class BertMaskedLM : public ClassifierBase { int dimVoc = opt>("dim-vocabs")[batchIndex_]; - std::string activationType = opt("transformer-ffn-activation"); - mlp::act activation; - if(activationType == "relu") - activation = mlp::act::ReLU; - else if(activationType == "swish") - activation = mlp::act::swish; - else - ABORT("Activation function {} not supported in BERT masked LM", activationType); - auto layer1 = mlp::mlp() .push_back(mlp::dense() ("prefix", prefix_ + "_ff_logit_l1") - ("dim", dimModel) - ("activation", activation)) + ("dim", dimModel)) .construct(graph); auto intermediate = layer1->apply(maskedContext); + std::string activationType = opt("transformer-ffn-activation"); + if(activationType == "relu") + intermediate = relu(intermediate); + else if(activationType == "swish") + intermediate = swish(intermediate); + else if(activationType == "gelu") + intermediate = gelu(intermediate); + else + ABORT("Activation function {} not supported in BERT masked LM", activationType); + auto gamma = graph->param(prefix_ + "_ff_ln_scale", {1, dimModel}, inits::ones); auto beta = graph->param(prefix_ + "_ff_ln_bias", {1, dimModel}, inits::zeros); intermediate = layerNorm(intermediate, gamma, beta); diff --git a/src/models/transformer.h b/src/models/transformer.h index 01201df96..02ce2f6ed 100644 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -375,6 +375,8 @@ class Transformer : public EncoderOrDecoderBase { return (ActivationFunction*)relu; else if (actName == "swish") return (ActivationFunction*)swish; + else if (actName == "gelu") + return (ActivationFunction*)gelu; ABORT("Invalid activation name '{}'", actName); } diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc index 69244dce0..b70c59d0a 100755 --- a/src/tensors/gpu/add.inc +++ b/src/tensors/gpu/add.inc @@ -32,3 +32,4 @@ template void marian::gpu::Add, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::UnaryFunctor, marian::functional::Assignee<3> > > >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Add, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Assignee<2> >, marian::functional::Assignee<3> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Add, marian::functional::Capture>, marian::functional::Assignee<2> >, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::Capture>, marian::functional::Assignee<2> >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr); +template void marian::gpu::Add, marian::functional::BinaryFunctor >, marian::functional::BinaryFunctor > >, marian::functional::BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr >(marian::functional::BinaryFunctor, marian::functional::BinaryFunctor >, marian::functional::BinaryFunctor > >, marian::functional::BinaryFunctor > > > > >, float, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); \ No newline at end of file diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc index f3cdea282..364866fab 100755 --- a/src/tensors/gpu/element.inc +++ b/src/tensors/gpu/element.inc @@ -56,6 +56,8 @@ template void Element, BinaryFunctor, Bin template void Element, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr >(Assign, BinaryFunctor, BinaryFunctor, Capture>, BinaryFunctor, Capture> >, Capture> >, BinaryFunctor > > > > >, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Element, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr >(marian::functional::Assign, marian::functional::BinaryFunctor, marian::functional::Assignee<3> >, marian::functional::BinaryFunctor, marian::functional::Assignee<3> > >, marian::functional::Capture>, marian::functional::Capture> >, std::shared_ptr, std::shared_ptr, std::shared_ptr); template void marian::gpu::Element, marian::functional::UnaryFunctor > >>(marian::functional::Assign, marian::functional::UnaryFunctor > >, std::shared_ptr); +template void marian::gpu::Element, marian::functional::BinaryFunctor, marian::functional::UnaryFunctor > > > >, std::shared_ptr >(marian::functional::Assign, marian::functional::BinaryFunctor, marian::functional::UnaryFunctor > > > >, std::shared_ptr, std::shared_ptr); + // How to add new specializations: // When you use a new specialization, it will cause a link error of this form (example): // .../src/tensors/tensor_operators.h:41: undefined reference to `void marian::gpu::Element ( ... )' From 664e87624a9242d7fd64dad1149ecd730cc38cd5 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sat, 26 Jan 2019 14:33:27 -0800 Subject: [PATCH 205/838] first try for bert to marian conversion script --- scripts/bert/bert4marian.py | 131 ++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 scripts/bert/bert4marian.py diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py new file mode 100755 index 000000000..1a151b5e8 --- /dev/null +++ b/scripts/bert/bert4marian.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +import tensorflow as tf +import numpy as np +import sys +import yaml +import argparse + +parser = argparse.ArgumentParser(description='Convert Tensorflow BERT model to Marian weight file.') +parser.add_argument('--bert_prefix', help='Prefix for Tensorflow BERT checkpoint', required=True) +parser.add_argument('--bert_config', help='Path to Tensorflow BERT JSON config', required=True) +parser.add_argument('--marian', help='Output path for Marian weight file', required=True) +args = parser.parse_args() + +print("Loading TensorFlow config from %s" % (args.bert_config,)) +bertConfig = yaml.load(open(args.bert_config)) +bertConfigYamlStr = yaml.dump(bertConfig, default_flow_style=False) +print(bertConfigYamlStr) + +print("Loading TensorFlow model from %s" % (args.bert_prefix,)) + +# Collect tensors from TF model as numpy matrices +tfModel = dict() +with tf.Session() as sess: + preloader = tf.train.import_meta_graph(args.bert_prefix + ".meta") + preloader.restore(sess, args.bert_prefix) + vars = tf.global_variables() + for v in vars: + if len(v.shape) > 0: + if "adam" not in v.name: # ignore adam parameters + print(v.name, v.shape) + tfModel[v.name] = sess.run(v.name) # get numpy matrix + +# Prepare Marian model config +config = dict() +config["type"] = "bert" +config["input-types"] = ["sequence", "class"] +config["tied-embeddings-all"] = True +config["dim-emb"] = tfModel["bert/embeddings/word_embeddings:0"].shape[-1] +config["dim-vocabs"] = [ tfModel["bert/embeddings/word_embeddings:0"].shape[0], + tfModel["cls/seq_relationship/output_weights:0"].shape[0] ] + +config["transformer-dim-ffn"] = tfModel["bert/encoder/layer_0/intermediate/dense/kernel:0"].shape[-1] +config["transformer-ffn-activation"] = bertConfig["hidden_act"] +config["transformer-ffn-depth"] = 2 +config["transformer-heads"] = bertConfig["num_attention_heads"] +config["transformer-train-positions"] = True +config["transformer-token-types"] = tfModel["bert/embeddings/token_type_embeddings:0"].shape[0] +config["transformer-preprocess"] = "" +config["transformer-postprocess"] = "dan" +config["transformer-postprocess-emb"] = "nd" +config["version"] = "bert4marian.py conversion" + +# check number of layers +found = True +config["enc-depth"] = 0; +while found: + found = False + for key in tfModel: + if "bert/encoder/layer_" + str(config["enc-depth"]) in key: + config["enc-depth"] += 1 + found = True + break + +configYamlStr = yaml.dump(config, default_flow_style=False) +desc = list(configYamlStr) +npDesc = np.chararray((len(desc),)) +npDesc[:] = desc +npDesc.dtype = np.int8 + +marianModel = dict() +marianModel["special:model.yml"] = npDesc + +# Map model weights here # +# Embedding layers +marianModel["Wemb"] = tfModel["bert/embeddings/word_embeddings:0"] +marianModel["Wpos"] = tfModel["bert/embeddings/position_embeddings:0"] +marianModel["Wsent"] = tfModel["bert/embeddings/token_type_embeddings:0"] +marianModel["encoder_emb_ln_scale_pre"] = tfModel["bert/embeddings/LayerNorm/gamma:0"] +marianModel["encoder_emb_ln_bias_pre"] = tfModel["bert/embeddings/LayerNorm/beta:0"] + +for layer in range(config["enc-depth"]): + marianPrefix = "encoder_l%s" % (layer + 1,) + tfPrefix = "bert/encoder/layer_%s" % (layer,) + + # Attention + marianModel[marianPrefix + "_self_Wq"] = tfModel[tfPrefix + "/attention/self/query/kernel:0"] + marianModel[marianPrefix + "_self_bq"] = tfModel[tfPrefix + "/attention/self/query/bias:0"] + + marianModel[marianPrefix + "_self_Wk"] = tfModel[tfPrefix + "/attention/self/key/kernel:0"] + marianModel[marianPrefix + "_self_bk"] = tfModel[tfPrefix + "/attention/self/key/bias:0"] + + marianModel[marianPrefix + "_self_Wv"] = tfModel[tfPrefix + "/attention/self/value/kernel:0"] + marianModel[marianPrefix + "_self_bv"] = tfModel[tfPrefix + "/attention/self/value/bias:0"] + + marianModel[marianPrefix + "_self_Wo"] = tfModel[tfPrefix + "/attention/output/dense/kernel:0"] + marianModel[marianPrefix + "_self_bo"] = tfModel[tfPrefix + "/attention/output/dense/bias:0"] + + marianModel[marianPrefix + "_self_Wo_ln_scale"] = tfModel[tfPrefix + "/attention/output/LayerNorm/gamma:0"] + marianModel[marianPrefix + "_self_Wo_ln_bias"] = tfModel[tfPrefix + "/attention/output/LayerNorm/beta:0"] + + # FFN + marianModel[marianPrefix + "_ffn_W1"] = tfModel[tfPrefix + "/intermediate/dense/kernel:0"] + marianModel[marianPrefix + "_ffn_b1"] = tfModel[tfPrefix + "/intermediate/dense/bias:0"] + + marianModel[marianPrefix + "_ffn_W2"] = tfModel[tfPrefix + "/output/dense/kernel:0"] + marianModel[marianPrefix + "_ffn_b2"] = tfModel[tfPrefix + "/output/dense/bias:0"] + + marianModel[marianPrefix + "_ffn_ffn_ln_scale"] = tfModel[tfPrefix + "/output/LayerNorm/gamma:0"] + marianModel[marianPrefix + "_ffn_ffn_ln_bias"] = tfModel[tfPrefix + "/output/LayerNorm/beta:0"] + + # Masked-LM output layer + marianModel["masked-lm_ff_logit_l1_W"] = tfModel["cls/predictions/transform/dense/kernel:0"] + marianModel["masked-lm_ff_logit_l1_b"] = tfModel["cls/predictions/transform/dense/bias:0"] + + marianModel["masked-lm_ff_ln_scale"] = tfModel["cls/predictions/transform/LayerNorm/gamma:0"] + marianModel["masked-lm_ff_ln_bias"] = tfModel["cls/predictions/transform/LayerNorm/beta:0"] + + marianModel["masked-lm_ff_logit_l2_b"] = tfModel["cls/predictions/output_bias:0"] + + # Next Sentence classifier + marianModel["next-sentence_ff_logit_l1_W"] = tfModel["bert/pooler/dense/kernel:0"] + marianModel["next-sentence_ff_logit_l1_b"] = tfModel["bert/pooler/dense/bias:0"] + + marianModel["next-sentence_ff_logit_l2_W"] = np.transpose(tfModel["cls/seq_relationship/output_weights:0"]) # transpose?! + marianModel["next-sentence_ff_logit_l2_b"] = tfModel["cls/seq_relationship/output_bias:0"] + +print("\nMarian config:") +print(configYamlStr) +print("Saving Marian model to %s" % (args.marian,)) +np.savez(args.marian, **marianModel) From cb8c249ec6be4881362a1952b761334ea34cca10 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sat, 26 Jan 2019 15:24:55 -0800 Subject: [PATCH 206/838] added checking of number of layers --- scripts/bert/bert4marian.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py index 1a151b5e8..e9c5824ca 100755 --- a/scripts/bert/bert4marian.py +++ b/scripts/bert/bert4marian.py @@ -62,6 +62,9 @@ found = True break +if config["enc-depth"] != bertConfig["num_hidden_layers"]: + sys.exit("Number of layers in JSON config (%s) and number of layers found in checkpoint (%s) do not match!" % (config["enc-depth"], bertConfig["num_hidden_layers"])) + configYamlStr = yaml.dump(config, default_flow_style=False) desc = list(configYamlStr) npDesc = np.chararray((len(desc),)) From 83fbd248d00e89277bed70d00bba0316101c7d0d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 27 Jan 2019 12:43:04 -0800 Subject: [PATCH 207/838] rename bert-specific options --- scripts/bert/bert4marian.py | 7 +- src/common/config_parser.cpp | 5 +- src/models/bert.h | 33 +++-- src/models/encoder_classifier.h | 5 +- src/models/encoder_decoder.cpp | 49 ++++--- src/training/validator.h | 3 +- src/translator/classification.h | 241 ++++++++++++++++++++++++++++++++ 7 files changed, 303 insertions(+), 40 deletions(-) create mode 100644 src/translator/classification.h diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py index e9c5824ca..12b012a4c 100755 --- a/scripts/bert/bert4marian.py +++ b/scripts/bert/bert4marian.py @@ -44,11 +44,12 @@ config["transformer-ffn-activation"] = bertConfig["hidden_act"] config["transformer-ffn-depth"] = 2 config["transformer-heads"] = bertConfig["num_attention_heads"] -config["transformer-train-positions"] = True -config["transformer-token-types"] = tfModel["bert/embeddings/token_type_embeddings:0"].shape[0] +config["transformer-train-position-embeddings"] = True config["transformer-preprocess"] = "" config["transformer-postprocess"] = "dan" config["transformer-postprocess-emb"] = "nd" +config["bert-train-type-embeddings"] = True +config["bert-type-vocab-size"] = tfModel["bert/embeddings/token_type_embeddings:0"].shape[0] config["version"] = "bert4marian.py conversion" # check number of layers @@ -78,7 +79,7 @@ # Embedding layers marianModel["Wemb"] = tfModel["bert/embeddings/word_embeddings:0"] marianModel["Wpos"] = tfModel["bert/embeddings/position_embeddings:0"] -marianModel["Wsent"] = tfModel["bert/embeddings/token_type_embeddings:0"] +marianModel["Wtype"] = tfModel["bert/embeddings/token_type_embeddings:0"] marianModel["encoder_emb_ln_scale_pre"] = tfModel["bert/embeddings/LayerNorm/gamma:0"] marianModel["encoder_emb_ln_bias_pre"] = tfModel["bert/embeddings/LayerNorm/beta:0"] diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 338d4b82c..73be7cd71 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -200,13 +200,16 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.add("--transformer-postprocess", "Operation after each transformer layer: d = dropout, a = add, n = normalize", "dan"); - cli.add("--transformer-train-positions", + cli.add("--transformer-train-position-embeddings", "Train positional embeddings instead of using static sinusoidal embeddings"); cli.add("--bert-mask-symbol", "Masking symbol for BERT masked-LM training", "[MASK]"); cli.add("--bert-sep-symbol", "Sentence separator symbol for BERT next sentence prediction training", "[SEP]"); cli.add("--bert-class-symbol", "Class symbol BERT classifier training", "[CLS]"); cli.add("--bert-masking-fraction", "Fraction of masked out tokens during training", 0.15); + cli.add("--bert-train-type-embeddings", "Train bert type embeddings, set to false to use static sinusoidal embeddings", true); + cli.add("--bert-type-vocab-size", "Size of BERT type vocab (sentence A and B)", 2); + #ifdef CUDNN cli.add("--char-stride", "Width of max-pooling layer after convolution layer in char-s2s model", diff --git a/src/models/bert.h b/src/models/bert.h index b4d2a34c2..d41bde86f 100644 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -70,7 +70,8 @@ class BertBatch : public CorpusBatch { float maskFraction, const std::string& maskSymbol, const std::string& sepSymbol, - const std::string& clsSymbol) + const std::string& clsSymbol, + int dimTypeVocab) : CorpusBatch(*batch), maskSymbol_(maskSymbol), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { @@ -119,18 +120,19 @@ class BertBatch : public CorpusBatch { words[i] = maskOut(words[i], maskId, engine); // mask that position } - annotateSentenceIndices(); + annotateSentenceIndices(dimTypeVocab); } BertBatch(Ptr batch, const std::string& sepSymbol, - const std::string& clsSymbol) + const std::string& clsSymbol, + int dimTypeVocab) : CorpusBatch(*batch), maskSymbol_("dummy"), sepSymbol_(sepSymbol), clsSymbol_(clsSymbol) { - annotateSentenceIndices(); + annotateSentenceIndices(dimTypeVocab); } - void annotateSentenceIndices() { + void annotateSentenceIndices(int dimTypeVocab) { // BERT expects a textual first stream and a second stream with class labels auto subBatch = subBatches_.front(); const auto& vocab = *subBatch->vocab(); @@ -144,7 +146,7 @@ class BertBatch : public CorpusBatch { int dimBatch = subBatch->batchSize(); int dimWords = subBatch->batchWidth(); - int maxSentPos = 1; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] + int maxSentPos = dimTypeVocab; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] // If another separator is seen do not increase position index beyond 2 but use padding. // @TODO: make this configurable, see below for NextSentencePredictions task where we also restrict to 2. @@ -180,6 +182,7 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { std::string modelType = opt("type"); + int dimTypeVocab = opt("bert-type-vocab-size"); // intercept batch and annotate with BERT-specific concepts Ptr bertBatch; @@ -189,11 +192,13 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { opt("bert-masking-fraction", 0.15f), // 15% by default according to paper opt("bert-mask-symbol"), opt("bert-sep-symbol"), - opt("bert-class-symbol")); + opt("bert-class-symbol"), + dimTypeVocab); } else if(modelType == "bert-classifier") { // we are probably fine-tuning a BERT model for a classification task bertBatch = New(batch, opt("bert-sep-symbol"), - opt("bert-class-symbol")); // only annotate sentence separators + opt("bert-class-symbol"), + dimTypeVocab); // only annotate sentence separators } else { ABORT("Unknown BERT-style model: {}", modelType); } @@ -219,7 +224,6 @@ class BertEncoder : public EncoderTransformer { Expr addSentenceEmbeddings(Expr embeddings, Ptr batch, bool learnedPosEmbeddings) const { - Ptr bertBatch = std::dynamic_pointer_cast(batch); ABORT_IF(!bertBatch, "Batch must be BertBatch for BERT training or fine-tuning"); @@ -227,11 +231,13 @@ class BertEncoder : public EncoderTransformer { int dimBatch = embeddings->shape()[-2]; int dimWords = embeddings->shape()[-3]; + int dimTypeVocab = opt("bert-type-vocab-size", 2); + Expr signal; if(learnedPosEmbeddings) { auto sentenceEmbeddings = embedding() - ("prefix", "Wsent") - ("dimVocab", 2) // sentence A or sentence B plus padding, @TODO: should rather be a parameter + ("prefix", "Wtype") + ("dimVocab", dimTypeVocab) // sentence A or sentence B ("dimEmb", dimEmb) .construct(graph_); signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); @@ -247,9 +253,10 @@ class BertEncoder : public EncoderTransformer { } virtual Expr addSpecialEmbeddings(Expr input, int start = 0, Ptr batch = nullptr) const override { - bool trainPosEmbeddings = opt("transformer-train-positions", true); + bool trainPosEmbeddings = opt("transformer-train-position-embeddings", true); + bool trainTypeEmbeddings = opt("bert-train-type-embeddings", true); input = addPositionalEmbeddings(input, start, trainPosEmbeddings); - input = addSentenceEmbeddings(input, batch, trainPosEmbeddings); // @TODO: separately set learnable pos and sent embeddings + input = addSentenceEmbeddings(input, batch, trainTypeEmbeddings); return input; } }; diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index bc3d8f9f8..23977a6c5 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -92,6 +92,7 @@ class EncoderClassifier : public EncoderClassifierBase { : options_(options), prefix_(options->get("prefix", "")), inference_(options->get("inference", false)) { + modelFeatures_ = {"type", "dim-vocabs", "dim-emb", @@ -128,7 +129,9 @@ class EncoderClassifier : public EncoderClassifierBase { modelFeatures_.insert("transformer-decoder-autoreg"); modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); - modelFeatures_.insert("transformer-train-positions"); + modelFeatures_.insert("transformer-train-position-embeddings"); + modelFeatures_.insert("bert-train-type-embeddings"); + modelFeatures_.insert("bert-type-vocab-size"); } virtual Ptr getOptions() override { return options_; } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index c4b96cc30..817475bba 100644 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -8,26 +8,31 @@ EncoderDecoder::EncoderDecoder(Ptr options) : options_(options), prefix_(options->get("prefix", "")), inference_(options->get("inference", false)) { - modelFeatures_ = {"type", - "dim-vocabs", - "dim-emb", - "dim-rnn", - "enc-cell", - "enc-type", - "enc-cell-depth", - "enc-depth", - "dec-depth", - "dec-cell", - "dec-cell-base-depth", - "dec-cell-high-depth", - "skip", - "layer-normalization", - "right-left", - "input-types", - "special-vocab", - "tied-embeddings", - "tied-embeddings-src", - "tied-embeddings-all"}; + + std::vector encoderDecoderModelFeatures = + {"type", + "dim-vocabs", + "dim-emb", + "dim-rnn", + "enc-cell", + "enc-type", + "enc-cell-depth", + "enc-depth", + "dec-depth", + "dec-cell", + "dec-cell-base-depth", + "dec-cell-high-depth", + "skip", + "layer-normalization", + "right-left", + "input-types", + "special-vocab", + "tied-embeddings", + "tied-embeddings-src", + "tied-embeddings-all"}; + + for(auto feature : encoderDecoderModelFeatures) + modelFeatures_.insert(feature); modelFeatures_.insert("transformer-heads"); modelFeatures_.insert("transformer-no-projection"); @@ -44,7 +49,9 @@ EncoderDecoder::EncoderDecoder(Ptr options) modelFeatures_.insert("transformer-decoder-autoreg"); modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); - modelFeatures_.insert("transformer-train-positions"); + modelFeatures_.insert("transformer-train-position-embeddings"); + modelFeatures_.insert("bert-train-type-embeddings"); + modelFeatures_.insert("bert-type-vocab-size"); } std::vector>& EncoderDecoder::getEncoders() { diff --git a/src/training/validator.h b/src/training/validator.h index d09859b20..48ab88fea 100644 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -359,7 +359,8 @@ class BertAccuracyValidator : public Validator { options_->get("bert-masking-fraction"), options_->get("bert-mask-symbol"), options_->get("bert-sep-symbol"), - options_->get("bert-class-symbol")); + options_->get("bert-class-symbol"), + options_->get("bert-type-vocab-size")); builder->clear(graph); auto classifierStates = std::dynamic_pointer_cast(builder)->apply(graph, bertBatch, true); diff --git a/src/translator/classification.h b/src/translator/classification.h new file mode 100644 index 000000000..28a8a82a8 --- /dev/null +++ b/src/translator/classification.h @@ -0,0 +1,241 @@ +#pragma once +#include + +#include "marian.h" +#include "translator/history.h" +#include "translator/scorers.h" + +#include "translator/helpers.h" +#include "translator/nth_element.h" + +namespace marian { + +class Classification { +private: + Ptr options_; + std::vector> scorers_; + size_t topN_{1}; + +public: + Classification(Ptr options) + : options_(options), + scorers_(scorers), + topN_{options_->get("beam-size")} // misuse beam-size for topN display + {} + + Beams toHyps(const std::vector keys, + const std::vector pathScores, + size_t labelNum, + const Beams& beams, + std::vector>& states, + size_t topN, + bool first, + Ptr batch) { + Beams newBeams(beams.size()); + + std::vector align; + if(options_->has("alignment")) + // Use alignments from the first scorer, even if ensemble + align = scorers_[0]->getAlignment(); + + for(size_t i = 0; i < keys.size(); ++i) { + // Keys contains indices to vocab items in the entire beam. + // Values can be between 0 and topN * number of lables. + Word embIdx = (Word)(keys[i] % labelNum); + auto beamIdx = i / topN; + + // Retrieve short list for final softmax (based on words aligned + // to source sentences). If short list has been set, map the indices + // in the sub-selected vocabulary matrix back to their original positions. + auto shortlist = scorers_[0]->getShortlist(); + if(shortlist) + embIdx = shortlist->reverseMap(embIdx); // @TODO: should reverseMap accept a size_t or a Word? + + if(newBeams[beamIdx].size() < beams[beamIdx].size()) { + auto& beam = beams[beamIdx]; + auto& newBeam = newBeams[beamIdx]; + + auto hypIdx = (IndexType)(keys[i] / labelNum); + float pathScore = pathScores[i]; + + auto hypIdxTrans + = IndexType((hypIdx / topN) + (hypIdx % topN) * beams.size()); + if(first) + hypIdxTrans = hypIdx; + + size_t beamHypIdx = hypIdx % topN; + if(beamHypIdx >= (int)beam.size()) + beamHypIdx = beamHypIdx % beam.size(); + + if(first) + beamHypIdx = 0; + + auto hyp = New(beam[beamHypIdx], embIdx, hypIdxTrans, pathScore); + + // Set score breakdown for n-best lists + if(options_->get("n-best")) { + std::vector breakDown(states.size(), 0); + beam[beamHypIdx]->GetScoreBreakdown().resize(states.size(), 0); + for(size_t j = 0; j < states.size(); ++j) { + size_t key = embIdx + hypIdxTrans * labelNum; + breakDown[j] = states[j]->breakDown(key) + + beam[beamHypIdx]->GetScoreBreakdown()[j]; + } + hyp->GetScoreBreakdown() = breakDown; + } + + newBeam.push_back(hyp); + } + } + return newBeams; + } + + // main decoding function + Histories search(Ptr graph, Ptr batch) { + int dimBatch = (int)batch->size(); + + Histories histories; + for(int i = 0; i < dimBatch; ++i) { + size_t sentId = batch->getSentenceIds()[i]; + auto history = New(sentId, + options_->get("normalize"), + options_->get("word-penalty")); + histories.push_back(history); + } + + auto getNBestList = createGetNBestListFn(topN_, dimBatch, graph->getDeviceId()); + + Beams beams(dimBatch); // [batchIndex][beamIndex] is one sentence hypothesis + for(auto& beam : beams) + beam.resize(topN_, New()); + + bool first = true; + bool final = false; + + for(int i = 0; i < dimBatch; ++i) + histories[i]->Add(beams[i], trgEosId_); + + std::vector> states; + + for(auto scorer : scorers_) { + scorer->clear(graph); + } + + for(auto scorer : scorers_) { + states.push_back(scorer->apply(graph, batch)); + } + + // main loop over output tokens + do { + //********************************************************************** + // create constant containing previous path scores for current beam + // also create mapping of hyp indices, which are not 1:1 if sentences complete + std::vector hypIndices; // [beamIndex * activeBatchSize + batchIndex] backpointers, concatenated over beam positions. Used for reordering hypotheses + std::vector embIndices; + Expr prevPathScores; // [beam, 1, 1, 1] + if(first) { + // no scores yet + prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); + } else { + std::vector beamScores; + + dimBatch = (int)batch->size(); + + for(size_t i = 0; i < localBeamSize; ++i) { + for(size_t j = 0; j < beams.size(); ++j) { // loop over batch entries (active sentences) + auto& beam = beams[j]; + if(i < beam.size()) { + auto hyp = beam[i]; + hypIndices.push_back((IndexType)hyp->GetPrevStateIndex()); // backpointer + embIndices.push_back(hyp->GetWord()); + beamScores.push_back(hyp->GetPathScore()); + } else { // dummy hypothesis + hypIndices.push_back(0); + embIndices.push_back(0); // (unused) + beamScores.push_back(-9999); + } + } + } + + prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, + inits::from_vector(beamScores)); + } + + //********************************************************************** + // prepare scores for beam search + auto pathScores = prevPathScores; + + for(size_t i = 0; i < scorers_.size(); ++i) { + states[i] = scorers_[i]->step( + graph, states[i], hypIndices, embIndices, dimBatch, (int)localBeamSize); + + if(scorers_[i]->getWeight() != 1.f) + pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs(); + else + pathScores = pathScores + states[i]->getLogProbs(); + } + + // make beams continuous + if(dimBatch > 1 && localBeamSize > 1) + pathScores = transpose(pathScores, {2, 1, 0, 3}); // check if this is needed for classification, rather not, beamSize and topN is badly defined here + + if(first) + graph->forward(); + else + graph->forwardNext(); + + //********************************************************************** + // suppress specific symbols if not at right positions + if(trgUnkId_ != -1 && options_->has("allow-unk") + && !options_->get("allow-unk")) + suppressWord(pathScores, trgUnkId_); + for(auto state : states) + state->blacklist(pathScores, batch); + + //********************************************************************** + // perform beam search and pruning + std::vector outKeys; + std::vector outPathScores; + + std::vector beamSizes(dimBatch, localBeamSize); + getNBestList(beamSizes, pathScores->val(), outPathScores, outKeys, first); + + int dimTrgVoc = pathScores->shape()[-1]; + beams = toHyps(outKeys, + outPathScores, + dimTrgVoc, + beams, + states, + localBeamSize, + first, + batch); + + auto prunedBeams = pruneBeam(beams); + for(int i = 0; i < dimBatch; ++i) { + if(!beams[i].empty()) { + final = final + || histories[i]->size() + >= options_->get("max-length-factor") + * batch->front()->batchWidth(); + histories[i]->Add( + beams[i], trgEosId_, prunedBeams[i].empty() || final); + } + } + beams = prunedBeams; + + // determine beam size for next sentence, as max over still-active sentences + if(!first) { + size_t maxBeam = 0; + for(auto& beam : beams) + if(beam.size() > maxBeam) + maxBeam = beam.size(); + localBeamSize = maxBeam; + } + first = false; + + } while(localBeamSize != 0 && !final); // end of main loop over output tokens + + return histories; + } +}; +} // namespace marian From abe94674710ca7d4535157bb2315f5a954e3a7e3 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 27 Jan 2019 23:03:46 -0800 Subject: [PATCH 208/838] hacky Nan handling --- scripts/bert/bert4marian.py | 1 + src/tensors/cpu/tensor_operators.cpp | 9 +++-- src/tensors/gpu/algorithm.cu | 5 +-- src/tensors/gpu/tensor_operators.cu | 51 ++++++++++++++++++++++------ src/tensors/tensor_operators.h | 2 ++ src/training/graph_group_sync.cpp | 11 ++++-- 6 files changed, 62 insertions(+), 17 deletions(-) diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py index 12b012a4c..5ccc46595 100755 --- a/scripts/bert/bert4marian.py +++ b/scripts/bert/bert4marian.py @@ -113,6 +113,7 @@ marianModel[marianPrefix + "_ffn_ffn_ln_scale"] = tfModel[tfPrefix + "/output/LayerNorm/gamma:0"] marianModel[marianPrefix + "_ffn_ffn_ln_bias"] = tfModel[tfPrefix + "/output/LayerNorm/beta:0"] + # Training objectives # Masked-LM output layer marianModel["masked-lm_ff_logit_l1_W"] = tfModel["cls/predictions/transform/dense/kernel:0"] marianModel["masked-lm_ff_logit_l1_b"] = tfModel["cls/predictions/transform/dense/bias:0"] diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index e7fefdcfe..5ee1d3400 100644 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -14,6 +14,11 @@ namespace marian { namespace cpu { +void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf) { + ABORT("Not implemented"); +} + + inline float stableSigmoid(float x) { if(x >= 0) { float z = expf(-x); @@ -394,7 +399,7 @@ void CopyRows(Tensor out_, for(size_t j = 0; j < rows; ++j) { size_t dst = j; - // @TODO: consider moving type checking to this function + // @TODO: consider moving type checking to this function // instead of matchOrAbort above size_t src = (size_t)indices->data()[j]; @@ -494,7 +499,7 @@ void Select(Tensor out, functional::Array dims; int axisCPU = (int)(axis + functional::Shape::size() - out->shape().size()); - + for(int index = 0; index < length; ++index) { outShape.dims(index, dims); dims[axisCPU] = (int)indices->data()[dims[axisCPU]]; diff --git a/src/tensors/gpu/algorithm.cu b/src/tensors/gpu/algorithm.cu index bdf66bac8..40709866c 100644 --- a/src/tensors/gpu/algorithm.cu +++ b/src/tensors/gpu/algorithm.cu @@ -55,6 +55,7 @@ void fill(Ptr backend, T* begin, T* end, T value) { CUDA_CHECK(cudaStreamSynchronize(0)); } +template void fill(Ptr, bool*, bool*, bool); template void fill(Ptr, int8_t*, int8_t*, int8_t); template void fill(Ptr, int16_t*, int16_t*, int16_t); template void fill(Ptr, int32_t*, int32_t*, int32_t); @@ -84,7 +85,7 @@ __global__ void gSwap(T* d_v1, T* d_v2, int size) { if(index < size) { T temp = d_v1[index]; d_v1[index] = d_v2[index]; - d_v2[index] = temp; + d_v2[index] = temp; } } @@ -93,7 +94,7 @@ void swap_ranges(Ptr backend, T* begin, T* end, T* dest) { int size = end - begin; if (size == 0) return; - + CUDA_CHECK(cudaSetDevice(backend->getDeviceId().no)); int threadsPerBlock = std::min(MAX_THREADS, size); int blocks = (size / threadsPerBlock) + (size % threadsPerBlock != 0); // @TODO: (size+threadsPerBlock-1)/threadsPerBlock or CeilDiv(a,b) diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index 34da4c688..01ee303dd 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -27,14 +27,43 @@ __device__ inline float stableSigmoid(float x) { } } -bool IsNan(Tensor in) { - // cudaSetDevice(in->getDeviceId().no); - // thrust::device_ptr begin = thrust::device_pointer_cast(in->data()); - // thrust::device_ptr end - // = thrust::device_pointer_cast(in->data() + in->size()); - // return thrust::transform_reduce( - // begin, end, isnan_test(), 0, thrust::plus()); - return false; +template +__global__ void gIsNan(const T* in, int length, bool* isNan, bool* isInf) { + for(int bid = 0; bid < length; bid += blockDim.x * gridDim.x) { + int index = bid + blockDim.x * blockIdx.x + threadIdx.x; + if(index < length) { + if(isnan((float)in[index])) *isNan = true; + if(isinf((float)in[index])) *isInf = true; + //if(isinf2(in[index])) *isInf = true; + } + } +} + +void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf) { + cudaSetDevice(in->getDeviceId().no); + + int length = in->size(); + + int threads = std::min(MAX_THREADS, length); + int blocks = std::min(MAX_BLOCKS, length / threads + (length % threads != 0)); + + auto mem = allocator->alloc(2); + bool* dIsNan = &mem->data()[0]; + bool* dIsInf = &mem->data()[1]; + fill(in->getBackend(), dIsNan, dIsNan + 2, false); + + if(in->type() == Type::float32) { + gIsNan<<>>(in->data(), length, dIsNan, dIsInf); + } else { + ABORT("IsNan for type {} not implemented", in->type()); + } + + CudaCopy(dIsNan, dIsNan + 1, &isNan); + CudaCopy(dIsInf, dIsInf + 1, &isInf); + + allocator->free(mem); + + cudaStreamSynchronize(0); } void ConcatCont(Tensor out, const std::vector& inputs, int axis) { @@ -1176,9 +1205,9 @@ __global__ void gCrossEntropyPick(float* out, } // In each j-th row, take the corresponding j-th label index i from indices and compute: -// For each vocabulary item v, the only non-zero element in a row in the sum is the item -// that matches the label indexed by i (the picked element). -// C = sum_{v in V}(-logsoftmax(A) * delta(v, i) = -logsoftmax(A)[i] +// For each vocabulary item v, the only non-zero element in a row in the sum is the item +// that matches the label indexed by i (the picked element). +// C = sum_{v in V}(-logsoftmax(A) * delta(v, i) = -logsoftmax(A)[i] void CrossEntropyPick(Tensor out, Tensor in, Tensor indices) { matchOrAbort(indices->type()); diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index 8dd17a5ce..7f0f75943 100644 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -34,6 +34,8 @@ void copy(Ptr backend, const InIt beg, const InIt end, OutIt it) { std::copy(beg, end, it); } +DISPATCH4(IsNan, const Tensor, Ptr, bool&, bool&); + template void Element(Functor functor, marian::Tensor out, Tensors... tensors) { #ifdef CUDA_FOUND diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 1ec5c3d68..85bf13f1f 100644 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -356,12 +356,19 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num auto rationalLoss = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); - + StaticLoss tempLoss = *rationalLoss; // needed for overstuff tempLoss.loss /= (float)overstuff; // @TODO: @fseide: scale only loss? should this scale labels too? localDeviceLosses[localDeviceIndex] += tempLoss; graph->backward(/*zero=*/false); // (gradients are reset before we get here) + + bool hasNan = false, hasInf = false; + IsNan(graph->params()->grads(), graph->allocator(), hasNan, hasInf); + if(hasNan || hasInf) { + LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroing gradient", hasNan, hasInf); + graph->params()->grads()->set(0.f); + } } }); // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. @@ -521,7 +528,7 @@ void SyncGraphGroup::save(bool final) /*override*/ { return comm_->gatherState(getShardFn); }, isMainProcess()); - + barrier(); // (for better grouping of log messages) } From 0fe0f18e851b3c42abefb4db8c71a477237b6143 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 08:25:45 -0800 Subject: [PATCH 209/838] more nan handling code copied from fp16 branch --- src/graph/expression_graph.cpp | 81 ++++++++++++++++++++++++++-- src/graph/expression_graph.h | 80 +++++++++------------------ src/tensors/cpu/tensor_operators.cpp | 2 +- src/tensors/gpu/tensor_operators.cu | 17 +++--- src/tensors/tensor_operators.h | 2 +- src/training/graph_group_sync.cpp | 10 ++-- 6 files changed, 120 insertions(+), 72 deletions(-) diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index b3c237d1e..19c3fc4e9 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -24,11 +24,86 @@ Expr ExpressionGraph::dropout(float prob, const Shape& shape) { return constant(shape, inits::dropout(prob)); } -void ExpressionGraph::checkNan(Tensor t) { - ABORT_IF(throwNaN_, "Not implemented"); t; - // ABORT_IF(throwNaN_ && IsNan(t), "Tensor has NaN"); +void ExpressionGraph::checkNan(Tensor t, bool& isNan, bool& isInf, bool zero) { + IsNan(t, allocator(), isNan, isInf, zero); } +void ExpressionGraph::backward(bool zero, float clipValue) { + if(topNodes_.size() > 1) { + LOG(critical, "There are more ({}) than one top most nodes for backward pass:", topNodes_.size()); + for(auto node : topNodes_) { + LOG(critical, + "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + node->type(), + node->shape(), + node->name(), + node->getId(), + node->hash()); + } + ABORT("Aborting"); + } + + params_->allocateBackward(); + if(zero) + params_->set_zero_adjoint(); + + for(auto&& v : topNodes_) + v->init_dependent(); + + // named_.clear(); + topNodes_.clear(); + + tensors_->clearShorttermMemory(); + + while(!nodesBackward_.empty()) { + auto v = nodesBackward_.back(); + nodesBackward_.pop_back(); + + for(auto&& child : v->children()) { + if(child->trainable() && child->type() != "param") + child->set_zero_adjoint(); + } + + if(v->trainable()) { + v->backward(); + if(clipValue != 0) { + using namespace functional; + Element(_1 = clip(_1, clipValue), v->grad()); + } + } + + + if(throwNan_) { + for(auto&& child : v->children()) { + if(child->trainable()) { + bool isNan = false, isInf = false; + checkNan(child->grad(), isNan, isInf); + if(isNan || isInf) { + LOG(critical, "Detected NaN ({}) or Inf ({}) in gradient (backward pass) of child node", isNan, isInf); + LOG(critical, "Child - Type: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + child->type(), child->shape(), child->name(), child->getId(), child->hash()); + LOG(critical, "Value debug: {}", child->val()->debug()); + LOG(critical, "Grad debug: {}", child->grad()->debug()); + LOG(critical, "Parent - Type: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + v->type(), v->shape(), v->name(), v->getId(), v->hash()); + LOG(critical, "Value debug: {}", v->val()->debug()); + LOG(critical, "Grad debug: {}", v->grad()->debug()); + ABORT("Aborting"); + } + } + } + } + + if(v->trainable() && v->marked_for_debug()) { + LOG(info, "Debug Grad: {} op={}", v->debug_message(), v->type()); + LOG(info, v->grad()->debug()); + } + + v->children().clear(); + } +} + + void ExpressionGraph::save(std::vector& ioItems) { for(auto p : params()->getMap()) { std::string pName = p.first; diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index fe8361618..24eb88a6a 100644 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -136,7 +136,7 @@ class ExpressionGraph : public std::enable_shared_from_this { bool reloaded_{false}; std::string namespace_; - bool throwNaN_{false}; + bool throwNan_{false}; protected: // Delete, copy and move constructors @@ -217,7 +217,7 @@ class ExpressionGraph : public std::enable_shared_from_this { forwardNext(); } - void checkNan(Tensor t); + void checkNan(Tensor t, bool& isNan, bool& isInf, bool zero = false); void forwardNext() { // @TODO: check if allocation works properly @@ -229,12 +229,27 @@ class ExpressionGraph : public std::enable_shared_from_this { v->init(); v->forward(); - checkNan(v->val()); + if(v->trainable() && throwNan_) { + bool isNan = false, isInf = false; + checkNan(v->val(), isNan, isInf); + if(isNan || isInf) { + LOG(critical, "Detected NaN ({}) or Inf ({}) in value (forward pass)", isNan, isInf); + LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + v->type(), v->shape(), v->name(), v->getId(), v->hash()); + LOG(critical, "Value debug {}", v->val()->debug()); + LOG(critical, "Children: {}", v->children().size()); + for(auto&& child : v->children()) { + LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + child->type(), child->shape(), child->name(), child->getId(), child->hash()); + LOG(critical, "Value debug {}", child->val()->debug()); + } + ABORT("Aborting"); + } + } if(v->marked_for_debug()) { - std::cerr << "Debug: " << v->debug_message() << " op=" << v->type() - << std::endl; - std::cerr << v->val()->debug() << std::endl; + LOG(info, "Debug: {} op={}", v->debug_message(), v->type()); + LOG(info, v->val()->debug()); } if(inferenceOnly_) @@ -243,55 +258,7 @@ class ExpressionGraph : public std::enable_shared_from_this { } } - void backward(bool zero = true) { - if(topNodes_.size() > 1) { - LOG(critical, "There are more ({}) than one top most node for backward step:", topNodes_.size()); - for(auto node : topNodes_) { - LOG(critical, - "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - node->type(), - node->shape(), - node->name(), - node->getId(), - node->hash()); - } - ABORT("Aborting"); - } - - params_->allocateBackward(); - if(zero) - params_->set_zero_adjoint(); - - for(auto&& v : topNodes_) - v->init_dependent(); - - // named_.clear(); - topNodes_.clear(); - - tensors_->clearShorttermMemory(); - - while(!nodesBackward_.empty()) { - auto v = nodesBackward_.back(); - nodesBackward_.pop_back(); - - for(auto&& child : v->children()) { - if(child->trainable() && child->type() != "param") - child->set_zero_adjoint(); - } - - if(v->trainable()) - v->backward(); - - checkNan(v->grad()); - - if(v->trainable() && v->marked_for_debug()) { - std::cerr << "Debug Grad: " << v->debug_message() << std::endl; - std::cerr << v->grad()->debug() << std::endl; - } - - v->children().clear(); - } - } + void backward(bool zero = true, float clipValue = 0.f); std::string graphviz() { std::stringstream ss; @@ -460,7 +427,8 @@ class ExpressionGraph : public std::enable_shared_from_this { void setReloaded(bool reloaded) { reloaded_ = reloaded; } - void setThrowNaN(bool throwNaN) { throwNaN_ = throwNaN; } + void setThrowNan(bool throwNan) { throwNan_ = throwNan; } + bool getThrowNan() { return throwNan_; } public: // convert all parameters into an array of IoItem elements, for loading diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index 5ee1d3400..d7d019f7d 100644 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -14,7 +14,7 @@ namespace marian { namespace cpu { -void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf) { +void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf, bool zero) { ABORT("Not implemented"); } diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index 01ee303dd..8e256bc78 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -28,18 +28,23 @@ __device__ inline float stableSigmoid(float x) { } template -__global__ void gIsNan(const T* in, int length, bool* isNan, bool* isInf) { +__global__ void gIsNan(T* in, int length, bool* isNan, bool* isInf, bool zero) { for(int bid = 0; bid < length; bid += blockDim.x * gridDim.x) { int index = bid + blockDim.x * blockIdx.x + threadIdx.x; if(index < length) { - if(isnan((float)in[index])) *isNan = true; - if(isinf((float)in[index])) *isInf = true; - //if(isinf2(in[index])) *isInf = true; + if(isnan((float)in[index])) { + if(zero) in[index] = (T)0.f; + *isNan = true; + } + else if(isinf((float)in[index])) { + if(zero) in[index] = (T)0.f; + *isInf = true; + } } } } -void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf) { +void IsNan(Tensor in, Ptr allocator, bool& isNan, bool& isInf, bool zero) { cudaSetDevice(in->getDeviceId().no); int length = in->size(); @@ -53,7 +58,7 @@ void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf) fill(in->getBackend(), dIsNan, dIsNan + 2, false); if(in->type() == Type::float32) { - gIsNan<<>>(in->data(), length, dIsNan, dIsInf); + gIsNan<<>>(in->data(), length, dIsNan, dIsInf, zero); } else { ABORT("IsNan for type {} not implemented", in->type()); } diff --git a/src/tensors/tensor_operators.h b/src/tensors/tensor_operators.h index 7f0f75943..5ee4c3031 100644 --- a/src/tensors/tensor_operators.h +++ b/src/tensors/tensor_operators.h @@ -34,7 +34,7 @@ void copy(Ptr backend, const InIt beg, const InIt end, OutIt it) { std::copy(beg, end, it); } -DISPATCH4(IsNan, const Tensor, Ptr, bool&, bool&); +DISPATCH5(IsNan, const Tensor, Ptr, bool&, bool&, bool); template void Element(Functor functor, marian::Tensor out, Tensors... tensors) { diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 85bf13f1f..782ced8b3 100644 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -358,16 +358,15 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num graph->forward(); StaticLoss tempLoss = *rationalLoss; // needed for overstuff - tempLoss.loss /= (float)overstuff; // @TODO: @fseide: scale only loss? should this scale labels too? + tempLoss.loss /= (float)overstuff; localDeviceLosses[localDeviceIndex] += tempLoss; graph->backward(/*zero=*/false); // (gradients are reset before we get here) bool hasNan = false, hasInf = false; - IsNan(graph->params()->grads(), graph->allocator(), hasNan, hasInf); + IsNan(graph->params()->grads(), graph->allocator(), hasNan, hasInf, /*zero=*/true); if(hasNan || hasInf) { - LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroing gradient", hasNan, hasInf); - graph->params()->grads()->set(0.f); + LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroed offending gradient", hasNan, hasInf); } } }); @@ -407,7 +406,8 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num // cost across all local devices (scheduler will aggregate cross-process) StaticLoss localLoss; for(auto& l : localDeviceLosses) // localDeviceLosses is already summed up over delay steps - localLoss += l; + if(std::isfinite((float)l.loss)) + localLoss += l; if(scheduler_) { // track and log localLoss From ddb5472717523d891f7b495f7b8f553e7ce3096c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 08:44:26 -0800 Subject: [PATCH 210/838] more aggressive nan checking --- src/common/config_parser.cpp | 30 ++++++++++++++++-------------- src/training/graph_group_sync.cpp | 8 +++++++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 73be7cd71..58fd4090e 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -54,31 +54,33 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { // clang-format off cli.add>("--config,-c", - "Configuration file(s). If multiple, later overrides earlier"); + "Configuration file(s). If multiple, later overrides earlier"); cli.add("--workspace,-w", - "Preallocate arg MB of work space", - defaultWorkspace); + "Preallocate arg MB of work space", + defaultWorkspace); cli.add_nondefault("--log", - "Log training process information to file given by arg"); + "Log training process information to file given by arg"); cli.add("--log-level", - "Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off", - "info"); + "Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off", + "info"); cli.add_nondefault("--log-time-zone", - "Set time zone for the date shown on logging"); + "Set time zone for the date shown on logging"); cli.add("--quiet", - "Suppress all logging to stderr. Logging to files still works"); + "Suppress all logging to stderr. Logging to files still works"); cli.add("--quiet-translation", - "Suppress logging for translation"); + "Suppress logging for translation"); cli.add("--seed", - "Seed for all random number generators. 0 means initialize randomly"); + "Seed for all random number generators. 0 means initialize randomly"); + cli.add("--check-nan", + "Check for NaNs or Infs in forward and backward pass. Will abort when found."); cli.add("--clip-gemm", - "If not 0 clip GEMM input values to +/- arg"); + "If not 0 clip GEMM input values to +/- arg"); cli.add("--interpolate-env-vars", - "allow the use of environment variables in paths, of the form ${VAR_NAME}"); + "allow the use of environment variables in paths, of the form ${VAR_NAME}"); cli.add("--relative-paths", - "All paths are relative to the config file location"); + "All paths are relative to the config file location"); cli.add_nondefault("--dump-config", - "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") + "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") ->implicit_val("full"); // clang-format on } diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 782ced8b3..c26cb9ff5 100644 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -10,6 +10,10 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) for(auto device : devices_) { auto graph = New(); graph->setDevice(device); + + if(options_->get("check-nan")) + graph->setThrowNan(true); + graph->reserveWorkspaceMB(options_->get("workspace")); graph->getBackend()->setClip(options_->get("clip-gemm")); @@ -366,7 +370,7 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num bool hasNan = false, hasInf = false; IsNan(graph->params()->grads(), graph->allocator(), hasNan, hasInf, /*zero=*/true); if(hasNan || hasInf) { - LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroed offending gradient", hasNan, hasInf); + LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroed out offending gradient", hasNan, hasInf); } } }); @@ -408,6 +412,8 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num for(auto& l : localDeviceLosses) // localDeviceLosses is already summed up over delay steps if(std::isfinite((float)l.loss)) localLoss += l; + else + LOG(warn, "Seen non-finite loss, offending gradients have been zeroed out"); if(scheduler_) { // track and log localLoss From 93763c30e6c430d21e9631415916c48d19784c18 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 10:02:53 -0800 Subject: [PATCH 211/838] now always normalizes factors --- src/data/default_vocab.cpp | 3 ++- src/layers/generic.cpp | 12 +++++++----- src/layers/loss.cpp | 5 +++++ src/models/costs.h | 2 ++ vs/Marian.vcxproj | 4 ++++ vs/Marian.vcxproj.filters | 3 +++ 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index e344eadf9..500dbda99 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -107,7 +107,8 @@ class DefaultVocab : public VocabBase { ABORT_IF(line.empty(), "DefaultVocabulary file {} must not contain empty lines", vocabPath); - vocab.insert({line, (Word)vocab.size()}); + auto wasInserted = vocab.insert({line, (Word)vocab.size()}).second; + ABORT_IF(!wasInserted, "Duplicate vocabulary entry {}", line); } ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index d44c6bb13..60cbcf295 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -100,10 +100,12 @@ namespace marian { for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); - // any factor that is not referenced in all words and is not a sigmoid needs normalization - if (g == 0) // @TODO: For now we assume that the main factor is used in all words. Test this. + if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition continue; - if (groupCounts[g] == 1) // sigmoid factors have no normalizer + // any factor that is not referenced in all words and is not a sigmoid needs normalization + //if (g == 0) // @TODO: For now we assume that the main factor is used in all words. Test this. + // continue; + if (groupCounts[g] == 1) // sigmoid factors have no normalizer --@BUGBUG: This is not even possible I think. We need the counter-class. continue; groupNeedsNormalization_[g] = true; // needed ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], @@ -196,7 +198,7 @@ namespace marian { auto graph = input->graph(); auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits -#if 0 +#if 1 // denominators (only for groups that don't normalize out naturally by the final softmax()) const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); @@ -217,7 +219,7 @@ namespace marian { auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] auto Z = dot(groupZ, m); // [B... x U] y = y - Z; -#if 0 +#if 1 // and a log-linear weight auto name = options_->get("prefix"); auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index d11e4384c..5ca493f1d 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -29,6 +29,11 @@ Expr LossBase::getCrossEntropy(Expr logits, Expr ce; if(smoothing_ > 0) { + // ce = sum_i y^_i log y_i(z)_i + // with smoothing: + // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i + // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i + // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i // @TODO: add this to CE kernels instead #if 0 ce = cross_entropy(logits, indices); diff --git a/src/models/costs.h b/src/models/costs.h index d38a445e8..a38fca915 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -108,6 +108,8 @@ typedef Trainer Scorer; class CostStep { public: + // @BUGBUG: This is not a function application. Rather, it updates 'state' in-place. + // Suggest to call it updateState, and not return the state object. virtual Ptr apply(Ptr state) = 0; }; diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index a82c879e5..e96cede5e 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -1112,6 +1112,10 @@ true + + true + true +
diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index a7c2331e7..3a0fe4902 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1852,6 +1852,9 @@ tensors\gpu + + translator + From 7f159ab64aa17c28e97a9afa9cd458568e46ea9d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 10:43:34 -0800 Subject: [PATCH 212/838] dump matrices for nan --- src/graph/expression_graph.cpp | 62 ++++++++++++++++++++++++++++++++++ src/graph/expression_graph.h | 39 +-------------------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index 19c3fc4e9..4e9e9a103 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -28,6 +28,68 @@ void ExpressionGraph::checkNan(Tensor t, bool& isNan, bool& isInf, bool zero) { IsNan(t, allocator(), isNan, isInf, zero); } +io::Item itemFromTensor(Tensor t, const std::string name, Ptr backend) { + io::Item item; + item.name = name; + item.shape = t->shape(); + item.type = t->type(); + + size_t bytesWithoutPadding = t->shape().elements() * sizeOf(t->type()); + item.bytes.resize(bytesWithoutPadding); + copy(backend, + (char*)t->data(), + (char*)t->data() + bytesWithoutPadding, + item.bytes.data()); + return item; +} + +void ExpressionGraph::forwardNext() { + // @TODO: check if allocation works properly + tensors_->clearShorttermMemory(); + + while(!nodesForward_.empty()) { + auto v = nodesForward_.front(); + v->allocate(); + v->init(); + v->forward(); + + if(v->trainable() && throwNan_) { + bool isNan = false, isInf = false; + checkNan(v->val(), isNan, isInf); + if(isNan || isInf) { + std::vector ioItems; + LOG(critical, "Detected NaN ({}) or Inf ({}) in value (forward pass)", isNan, isInf); + LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + v->type(), v->shape(), v->name(), v->getId(), v->hash()); + LOG(critical, "Value debug {}", v->val()->debug()); + + ioItems.push_back(itemFromTensor(v->val(), "value", backend_)); + + LOG(critical, "Children: {}", v->children().size()); + for(auto&& child : v->children()) { + LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + child->type(), child->shape(), child->name(), child->getId(), child->hash()); + LOG(critical, "Value debug {}", child->val()->debug()); + ioItems.push_back(itemFromTensor(v->val(), "child_" + std::to_string(child->hash()), backend_)); + } + + io::saveItems("dump-for-nans.npz", ioItems); + + ABORT("Aborting"); + } + } + + if(v->marked_for_debug()) { + LOG(info, "Debug: {} op={}", v->debug_message(), v->type()); + LOG(info, v->val()->debug()); + } + + if(inferenceOnly_) + v->children().clear(); + nodesForward_.pop_front(); + } +} + void ExpressionGraph::backward(bool zero, float clipValue) { if(topNodes_.size() > 1) { LOG(critical, "There are more ({}) than one top most nodes for backward pass:", topNodes_.size()); diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index 24eb88a6a..389c6e3e4 100644 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -219,44 +219,7 @@ class ExpressionGraph : public std::enable_shared_from_this { void checkNan(Tensor t, bool& isNan, bool& isInf, bool zero = false); - void forwardNext() { - // @TODO: check if allocation works properly - tensors_->clearShorttermMemory(); - - while(!nodesForward_.empty()) { - auto v = nodesForward_.front(); - v->allocate(); - v->init(); - v->forward(); - - if(v->trainable() && throwNan_) { - bool isNan = false, isInf = false; - checkNan(v->val(), isNan, isInf); - if(isNan || isInf) { - LOG(critical, "Detected NaN ({}) or Inf ({}) in value (forward pass)", isNan, isInf); - LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - v->type(), v->shape(), v->name(), v->getId(), v->hash()); - LOG(critical, "Value debug {}", v->val()->debug()); - LOG(critical, "Children: {}", v->children().size()); - for(auto&& child : v->children()) { - LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - child->type(), child->shape(), child->name(), child->getId(), child->hash()); - LOG(critical, "Value debug {}", child->val()->debug()); - } - ABORT("Aborting"); - } - } - - if(v->marked_for_debug()) { - LOG(info, "Debug: {} op={}", v->debug_message(), v->type()); - LOG(info, v->val()->debug()); - } - - if(inferenceOnly_) - v->children().clear(); - nodesForward_.pop_front(); - } - } + void forwardNext(); void backward(bool zero = true, float clipValue = 0.f); From a327956d314e43f052ad0716c7a7419096553088 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 10:59:01 -0800 Subject: [PATCH 213/838] refactoring: Output layer, and therefore all Models, now return logits as a Logits object instead of an Exp, in preparation for optimizing CE loss for factored embeddings --- src/examples/mnist/model.h | 6 ++--- src/layers/constructors.h | 23 ++++++++++++++++-- src/layers/generic.cpp | 2 +- src/layers/generic.h | 26 ++++++++++++++++++--- src/models/costs.h | 15 ++++++------ src/models/encoder_decoder.cpp | 4 ++-- src/models/encoder_decoder.h | 8 +++---- src/models/model_base.h | 3 ++- src/models/s2s.h | 7 +++++- src/models/states.h | 9 +++---- src/models/transformer.h | 8 +++---- src/rescorer/rescorer.h | 2 +- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_multinode.cpp | 2 +- src/training/graph_group_multinode_sync.cpp | 2 +- src/training/graph_group_singleton.cpp | 2 +- src/training/graph_group_sync.cpp | 2 +- src/training/validator.h | 2 +- src/translator/scorers.h | 2 +- 19 files changed, 87 insertions(+), 40 deletions(-) mode change 100644 => 100755 src/models/encoder_decoder.h mode change 100644 => 100755 src/models/model_base.h mode change 100644 => 100755 src/rescorer/rescorer.h diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 71de15ec4..2f66f215f 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -22,7 +22,7 @@ class MNISTCrossEntropyCost : public CostBase { Ptr graph, Ptr batch, bool clearGraph = true) override { - auto top = model->build(graph, batch, clearGraph); + auto top = model->build(graph, batch, clearGraph).getLogits(); auto vfLabels = std::static_pointer_cast(batch)->labels(); @@ -43,7 +43,7 @@ class MNISTLogsoftmax : public CostBase { Ptr graph, Ptr batch, bool clearGraph = true) override { - auto top = model->build(graph, batch, clearGraph); + auto top = model->build(graph, batch, clearGraph).getLogits(); return logsoftmax(top); } }; @@ -56,7 +56,7 @@ class MnistFeedForwardNet : public ModelBase { MnistFeedForwardNet(Ptr options, Args... args) : options_(options), inference_(options->get("inference", false)) {} - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool /*clean*/ = false) override { return construct(graph, batch, inference_); diff --git a/src/layers/constructors.h b/src/layers/constructors.h index 5ed7f3f53..d1aef42e5 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -51,7 +51,26 @@ typedef Accumulator dense; /** * Factory for output layers, can be used in a multi-layer network factory. */ -class OutputFactory : public LayerFactory { +struct LogitLayerFactory : public Factory { + LogitLayerFactory() : Factory() {} + LogitLayerFactory(const LogitLayerFactory&) = default; + LogitLayerFactory(LogitLayerFactory&&) = default; + + virtual ~LogitLayerFactory() {} + + template + inline Ptr as() { + return std::dynamic_pointer_cast(shared_from_this()); + } + + template + inline bool is() { + return as() != nullptr; + } + + virtual Ptr construct(Ptr graph) = 0; +}; +class OutputFactory : public LogitLayerFactory { protected: std::string tiedTransposedName_; Ptr shortlist_; @@ -67,7 +86,7 @@ class OutputFactory : public LayerFactory { return Accumulator(*this); } - Ptr construct(Ptr graph) override { + Ptr construct(Ptr graph) override { auto output = New(graph, options_); output->tieTransposed(graph->get(tiedTransposedName_)); output->setShortlist(shortlist_); diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 60cbcf295..06b109537 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -181,7 +181,7 @@ namespace marian { b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); } - Expr Output::apply(Expr input) /*override*/ { + Logits Output::apply(Expr input) /*override*/ { lazyConstruct(input->shape()[-1]); if (shortlist_) { diff --git a/src/layers/generic.h b/src/layers/generic.h index d70c7f06e..6eb60cb37 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -54,6 +54,26 @@ struct IEmbeddingLayer { virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; }; +// @HACK: Frank's quick implementation of factored outputs. To be re-thought once it works. +// Output layer returns a Logits object, which is able to compute some things on the fly +// for factored embeddings. +class Logits { +public: + Logits(Expr logits) : logits_(logits) { + } + Expr getLogits() const { + return logits_; + } +private: + Expr logits_; +}; + +// Unary function that returns a Logits object +struct IUnaryLogitLayer { + virtual Logits apply(Expr) = 0; + virtual Logits apply(const std::vector&) = 0; +}; + class EmbeddingFactorMapping; namespace mlp { @@ -124,7 +144,7 @@ class Dense : public LayerBase, public IUnaryLayer { Expr apply(Expr input) override { return apply(std::vector({input})); } }; -class Output : public LayerBase, public IUnaryLayer { +class Output : public LayerBase, public IUnaryLogitLayer { private: Expr W_; // parameters held by this layer Expr b_; @@ -169,9 +189,9 @@ class Output : public LayerBase, public IUnaryLayer { cachedShortb_ = nullptr; } - Expr apply(Expr input) override; + Logits apply(Expr input) override; - virtual Expr apply(const std::vector& /*inputs*/) override { + virtual Logits apply(const std::vector& /*inputs*/) override { ABORT("Not implemented"); }; }; diff --git a/src/models/costs.h b/src/models/costs.h index a38fca915..5231e0ce0 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -54,7 +54,7 @@ class EncoderDecoderCE : public CostBase { weights = weighter_->getWeights(graph, corpusBatch); Expr cost; - cost = loss_->getCost(state->getLogProbs(), + cost = loss_->getCost(state->getLogProbs().getLogits(), state->getTargetIndices(), state->getTargetMask(), weights); @@ -95,7 +95,7 @@ class Trainer : public ModelBase { model_->save(graph, name, saveTranslatorConfig); } - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) override { return cost_->apply(model_, graph, batch, clearGraph); @@ -117,7 +117,8 @@ class LogSoftmaxStep : public CostStep { public: virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) - auto logits = state->getLogProbs(); + // @TODO: @HACK must know about individual parts; make it a loop + auto logits = state->getLogProbs().getLogits(); auto logprobs = logsoftmax(logits); @@ -132,7 +133,8 @@ class LogSoftmaxStep : public CostStep { class GumbelSoftmaxStep : public CostStep { public: virtual Ptr apply(Ptr state) override { - auto logits = state->getLogProbs(); + // @TODO: @HACK must know about individual parts; make it a loop + auto logits = state->getLogProbs().getLogits(); auto logprobs = logsoftmax(logits + constant_like(logits, inits::gumbel)); @@ -173,7 +175,7 @@ class Stepwise : public EncoderDecoderBase { virtual void clear(Ptr graph) override { encdec_->clear(graph); } - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) override { auto corpusBatch = std::static_pointer_cast(batch); @@ -196,11 +198,10 @@ class Stepwise : public EncoderDecoderBase { return cost_->apply(nextState); } - virtual Expr build(Ptr /*graph*/, + virtual Logits build(Ptr /*graph*/, Ptr /*batch*/, bool /*clearGraph*/ = true) override { ABORT("Wrong wrapper. Use models::Trainer or models::Scorer"); - return nullptr; } virtual Ptr getOptions() override { return encdec_->getOptions(); }; diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 3edb7b335..267943834 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -182,7 +182,7 @@ Ptr EncoderDecoder::stepAll(Ptr graph, return nextState; } -Expr EncoderDecoder::build(Ptr graph, +Logits EncoderDecoder::build(Ptr graph, Ptr batch, bool clearGraph) { auto state = stepAll(graph, batch, clearGraph); @@ -191,7 +191,7 @@ Expr EncoderDecoder::build(Ptr graph, return state->getLogProbs(); } -Expr EncoderDecoder::build(Ptr graph, +Logits EncoderDecoder::build(Ptr graph, Ptr batch, bool clearGraph) { auto corpusBatch = std::static_pointer_cast(batch); diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100644 new mode 100755 index 5818c2370..5a897750b --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -28,7 +28,7 @@ class EncoderDecoderBase : public models::ModelBase { virtual void clear(Ptr graph) override = 0; - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) override = 0; @@ -45,7 +45,7 @@ class EncoderDecoderBase : public models::ModelBase { int beamSize) = 0; - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) = 0; @@ -158,11 +158,11 @@ class EncoderDecoder : public EncoderDecoderBase { Ptr batch, bool clearGraph = true); - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) override; - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) override; }; diff --git a/src/models/model_base.h b/src/models/model_base.h old mode 100644 new mode 100755 index 47039841a..aa4ae5545 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -2,6 +2,7 @@ #include #include "marian.h" +#include "layers/generic.h" // @HACK for Frank's factored embeddings/Logits class namespace marian { namespace models { @@ -26,7 +27,7 @@ class ModelBase { bool saveTranslatorConfig = false) = 0; - virtual Expr build(Ptr graph, + virtual Logits build(Ptr graph, Ptr batch, bool clearGraph = true) = 0; diff --git a/src/models/s2s.h b/src/models/s2s.h index 2f4a45797..57cabe0fc 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -278,7 +278,7 @@ class DecoderS2S : public DecoderBase { } rnn::States startStates(opt("dec-depth"), {start, start}); - return New(startStates, nullptr, encStates, batch); + return New(startStates, Logits(nullptr), encStates, batch); } virtual Ptr step(Ptr graph, @@ -347,12 +347,17 @@ class DecoderS2S : public DecoderBase { if(shortlist_) last.setShortlist(shortlist_); +#if 1 + hidden; last; + ABORT("@TODO: adapt s2s to Logits return type"); +#else // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context output_ = mlp::mlp() // .push_back(hidden) // .push_back(last) .construct(graph); +#endif } Expr logits; diff --git a/src/models/states.h b/src/models/states.h index 964617340..26f66625d 100755 --- a/src/models/states.h +++ b/src/models/states.h @@ -1,6 +1,7 @@ #pragma once #include "marian.h" +#include "layers/generic.h" // @HACK: for factored embeddings only so far #include "rnn/types.h" namespace marian { @@ -29,7 +30,7 @@ class EncoderState { class DecoderState { protected: rnn::States states_; // states of individual decoder layers - Expr logProbs_; + Logits logProbs_; std::vector> encStates_; Ptr batch_; @@ -42,7 +43,7 @@ class DecoderState { public: DecoderState(const rnn::States& states, - Expr logProbs, + Logits logProbs, const std::vector>& encStates, Ptr batch) : states_(states), logProbs_(logProbs), encStates_(encStates), batch_(batch) {} @@ -52,8 +53,8 @@ class DecoderState { return encStates_; } - virtual Expr getLogProbs() const { return logProbs_; } - virtual void setLogProbs(Expr logProbs) { logProbs_ = logProbs; } + virtual Logits getLogProbs() const { return logProbs_; } + virtual void setLogProbs(Logits logProbs) { logProbs_ = logProbs; } // @TODO: should this be a constructor? Then derived classes can call this without the New<> in the loop virtual Ptr select(const std::vector& selIdx, diff --git a/src/models/transformer.h b/src/models/transformer.h index f6aa4ff1e..595215da1 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -589,7 +589,7 @@ class EncoderTransformer : public Transformer { class TransformerState : public DecoderState { public: TransformerState(const rnn::States& states, - Expr logProbs, + Logits logProbs, const std::vector>& encStates, Ptr batch) : DecoderState(states, logProbs, encStates, batch) {} @@ -662,11 +662,11 @@ class DecoderTransformer : public Transformer { rnn::States startStates(opt("dec-depth"), {start, start}); // don't use TransformerState for RNN layers - return New(startStates, nullptr, encStates, batch); + return New(startStates, Logits(nullptr), encStates, batch); } else { rnn::States startStates; - return New(startStates, nullptr, encStates, batch); + return New(startStates, Logits(nullptr), encStates, batch); } } @@ -831,7 +831,7 @@ class DecoderTransformer : public Transformer { // final feed-forward layer (output) if(shortlist_) output_->setShortlist(shortlist_); - Expr logits = output_->apply(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab or shortlist dim] + auto logits = output_->apply(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab or shortlist dim] // return unormalized(!) probabilities Ptr nextState; diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100644 new mode 100755 index fa456856a..d74738cc0 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -30,7 +30,7 @@ class Rescorer { } Expr build(Ptr graph, Ptr batch) { - return builder_->build(graph, batch); + return builder_->build(graph, batch).getLogits(); } data::SoftAlignment getAlignment() { diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index ca7f7b487..dddd7cddd 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -206,7 +206,7 @@ void AsyncGraphGroup::execute(Ptr batch) { builder = builders_[i++]; } - auto costNode = builder->build(graph, batch); + auto costNode = builder->build(graph, batch).getLogits(); if(t % optimizerDelay_ == 0) { fetchParams(graph->params()->vals(), params_, t_id); diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index f5cbafd36..0aed1533c 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -527,7 +527,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { builder = clientBuilders_[i++]; } - auto costNode = builder->build(graph, batch); + auto costNode = builder->build(graph, batch).getLogits(); if(t == 0) { mpi_->barrier(); diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp index 328657057..62fb2765b 100755 --- a/src/training/graph_group_multinode_sync.cpp +++ b/src/training/graph_group_multinode_sync.cpp @@ -185,7 +185,7 @@ void MultiNodeGraphGroupSync::execute(Ptr fullBatch) { auto graph = clientGraphs_[my_id]; auto builder = clientBuilders_[my_id]; - auto costNode = builder->build(graph, batch); + auto costNode = builder->build(graph, batch).getLogits(); if(t == 0) { if(my_id != 0) diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp index ac6ef5d0c..28b81eaad 100755 --- a/src/training/graph_group_singleton.cpp +++ b/src/training/graph_group_singleton.cpp @@ -10,7 +10,7 @@ void SingletonGraph::setScheduler(Ptr scheduler) { } void SingletonGraph::execute(Ptr batch) { - auto costNode = builder_->build(graph_, batch); + auto costNode = builder_->build(graph_, batch).getLogits(); graph_->forward(); float cost = costNode->scalar(); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index c00a4fc2f..85fd6d978 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -354,7 +354,7 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num if (!subBatch) break; - auto costNode = builders_[localDeviceIndex]->build(graph, subBatch); + auto costNode = builders_[localDeviceIndex]->build(graph, subBatch).getLogits(); graph->forward(); localDeviceCosts[localDeviceIndex] += costNode->scalar() / (float)overstuff; graph->backward(/*zero=*/false); // (gradients are reset before we get here) diff --git a/src/training/validator.h b/src/training/validator.h index 2b76d6578..d222443db 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -188,7 +188,7 @@ class CrossEntropyValidator : public Validator { } builder->clear(graph); - auto costNode = builder->build(graph, batch); + auto costNode = builder->build(graph, batch).getLogits(); graph->forward(); std::unique_lock lock(mutex_); diff --git a/src/translator/scorers.h b/src/translator/scorers.h index 16a066b05..befaeb8fd 100755 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -57,7 +57,7 @@ class ScorerWrapperState : public ScorerState { virtual Ptr getState() { return state_; } - virtual Expr getLogProbs() override { return state_->getLogProbs(); }; + virtual Expr getLogProbs() override { return state_->getLogProbs().getLogits(); }; virtual void blacklist(Expr totalCosts, Ptr batch) override { state_->blacklist(totalCosts, batch); From badb8a624c1f55d9d7fe22a48a59234535fba1c4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 11:25:15 -0800 Subject: [PATCH 214/838] minor fixes of last commit --- src/examples/mnist/validator.h | 2 +- src/layers/generic.h | 4 ++++ src/models/states.h | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) mode change 100644 => 100755 src/examples/mnist/validator.h diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h old mode 100644 new mode 100755 index 907994ee7..7d668bcc3 --- a/src/examples/mnist/validator.h +++ b/src/examples/mnist/validator.h @@ -31,7 +31,7 @@ class AccuracyValidator : public Validator { size_t samples = 0; for(auto batch : *batchGenerator_) { - auto probs = builder_->build(graphs[0], batch, true); + auto probs = builder_->build(graphs[0], batch, true).getLogits(); graphs[0]->forward(); std::vector scores; diff --git a/src/layers/generic.h b/src/layers/generic.h index 6eb60cb37..dd64e7bcf 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -58,12 +58,16 @@ struct IEmbeddingLayer { // Output layer returns a Logits object, which is able to compute some things on the fly // for factored embeddings. class Logits { + Logits& operator=(const Logits& other) = default; public: Logits(Expr logits) : logits_(logits) { } Expr getLogits() const { return logits_; } + void assign(const Logits& other) { // @TODO: forbid changing the number of contributions + *this = other; + } private: Expr logits_; }; diff --git a/src/models/states.h b/src/models/states.h index 26f66625d..05c0a5a93 100755 --- a/src/models/states.h +++ b/src/models/states.h @@ -54,7 +54,7 @@ class DecoderState { } virtual Logits getLogProbs() const { return logProbs_; } - virtual void setLogProbs(Logits logProbs) { logProbs_ = logProbs; } + virtual void setLogProbs(Logits logProbs) { logProbs_.assign(logProbs); } // @TODO: should this be a constructor? Then derived classes can call this without the New<> in the loop virtual Ptr select(const std::vector& selIdx, From 7d45598a0689aa7f3900d9c5a1c0aca536ee1a3b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 11:41:30 -0800 Subject: [PATCH 215/838] towards multiple factors in Logits object --- src/layers/generic.h | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index dd64e7bcf..8021dcfee 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -60,16 +60,33 @@ struct IEmbeddingLayer { class Logits { Logits& operator=(const Logits& other) = default; public: - Logits(Expr logits) : logits_(logits) { + Logits(Expr logits) { + if (logits) + logits_.push_back(logits); } Expr getLogits() const { - return logits_; + ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + // lazily compute logits + // @BUGBUG: This is completely wrong, we must use the factor information and map the dimensions + Expr res; + for (size_t i = 0; i < getNumFactors(); i++) { + auto logits = logits_[i]; + if (!weights_.empty() && weights_[i]) // log-linear weights + logits = logits * weights_[i]; + res = res ? res + logits : logits; + } + return res; } void assign(const Logits& other) { // @TODO: forbid changing the number of contributions + ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), + "Logits assignment cannot change number of factors"); *this = other; } + size_t getNumFactors() const { return logits_.size(); } + bool empty() const { return logits_.empty(); } private: - Expr logits_; + std::vector logits_; + std::vector weights_; }; // Unary function that returns a Logits object From 6c69514de3fef7bc530a663c03b32024c163fa74 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 13:07:32 -0800 Subject: [PATCH 216/838] mVec is now precomputed (in factored embeddings) --- src/layers/generic.cpp | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 06b109537..813a5eb42 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -95,22 +95,19 @@ namespace marian { groupRanges_[g].second = u + 1; groupCounts[g]++; } - // determine if a factor needs explicit softmax normalization - groupNeedsNormalization_.resize(numGroups, false); + // create the flag vectors for normalization --@TODO: maybe we won't need them anymore + mVecs_.resize(numGroups); for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition continue; - // any factor that is not referenced in all words and is not a sigmoid needs normalization - //if (g == 0) // @TODO: For now we assume that the main factor is used in all words. Test this. - // continue; - if (groupCounts[g] == 1) // sigmoid factors have no normalizer --@BUGBUG: This is not even possible I think. We need the counter-class. - continue; - groupNeedsNormalization_[g] = true; // needed ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - LOG(info, "[embedding] Factor group '{}' needs needs explicit normalization ({}..{})", groupPrefixes[g], groupRanges_[g].first, groupRanges_[g].second-1); + auto& mVec = mVecs_[g]; + mVec.resize(numFactors, 0.0f); + for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) + mVec[i] = 1.0f; } // create the factor matrix @@ -151,7 +148,7 @@ namespace marian { std::vector factorGroups_; // [u] -> group id of factor u public: // @TODO: temporarily; later factor this properly std::vector> groupRanges_; // [group id] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. - std::vector groupNeedsNormalization_; // [group id] -> true if explicit softmax normalization is necessary + std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; namespace mlp { @@ -198,35 +195,27 @@ namespace marian { auto graph = input->graph(); auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits -#if 1 // denominators (only for groups that don't normalize out naturally by the final softmax()) const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); + const auto& mVecs = embeddingFactorMapping_->mVecs_; for (size_t g = 0; g < numGroups; g++) { - if (!embeddingFactorMapping_->groupNeedsNormalization_[g]) // @TODO: if we ever need it, we can combine multiple - continue; auto range = groupRanges[g]; // y: [B... x U] // m: [1 x U] // ones at positions of group members - auto yDim = y->shape()[-1]; - std::vector mVec(yDim, 0.0f); // @TODO: This vector should be produced by embeddingFactorMapping_ - for (size_t i = range.first; i < range.second; i++) - mVec[i] = 1.0f; // need to compute log denominator over y[range] and subtract it from y[range] auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] //auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] + const auto& mVec = mVecs[g]; auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] auto Z = dot(groupZ, m); // [B... x U] y = y - Z; -#if 1 // and a log-linear weight auto name = options_->get("prefix"); auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); y = y * ((llWeight - 1) * m + 1); -#endif } -#endif // sum up the unit logits across factors for each target word auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] From 685945f7ce1cf9b08b1ebe78b4b8e9b36b8007e0 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 13:55:55 -0800 Subject: [PATCH 217/838] option to dump nodes at runtime to npz --- src/graph/chainable.h | 2 ++ src/graph/expression_graph.cpp | 2 +- src/graph/node.cpp | 19 +++++++++++++++++++ src/graph/node.h | 2 ++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/graph/chainable.h b/src/graph/chainable.h index 2679843ee..3aa81d3cc 100644 --- a/src/graph/chainable.h +++ b/src/graph/chainable.h @@ -96,6 +96,8 @@ class Chainable { virtual const std::string& name() const = 0; virtual void debug(const std::string& message) = 0; + virtual void dump(const std::string& filename) = 0; + virtual bool marked_for_debug() = 0; virtual const std::string& debug_message() = 0; diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index 4e9e9a103..4df97a82f 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -70,7 +70,7 @@ void ExpressionGraph::forwardNext() { LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", child->type(), child->shape(), child->name(), child->getId(), child->hash()); LOG(critical, "Value debug {}", child->val()->debug()); - ioItems.push_back(itemFromTensor(v->val(), "child_" + std::to_string(child->hash()), backend_)); + ioItems.push_back(itemFromTensor(child->val(), "child_" + std::to_string(child->hash()), backend_)); } io::saveItems("dump-for-nans.npz", ioItems); diff --git a/src/graph/node.cpp b/src/graph/node.cpp index c11531da7..bdb501166 100644 --- a/src/graph/node.cpp +++ b/src/graph/node.cpp @@ -2,6 +2,8 @@ #include "graph/auto_tuner.h" #include "graph/expression_graph.h" #include "tensors/backend.h" +#include "tensors/tensor_operators.h" +#include "common/io.h" namespace marian { @@ -83,4 +85,21 @@ void Node::record(Ptr recorder, recorderHash_ = recorderHash; recorderStop_ = stop; } + +void Node::dump(const std::string& filename) { + io::Item item; + item.name = "dump"; + item.shape = val_->shape(); + item.type = val_->type(); + + size_t bytesWithoutPadding = val_->shape().elements() * sizeOf(val_->type()); + item.bytes.resize(bytesWithoutPadding); + copy(graph()->getBackend(), + (char*)val_->data(), + (char*)val_->data() + bytesWithoutPadding, + item.bytes.data()); + + std::vector items({item}); + io::saveItems(filename, items); +} } // namespace marian diff --git a/src/graph/node.h b/src/graph/node.h index 1397e74b0..defefd5b5 100644 --- a/src/graph/node.h +++ b/src/graph/node.h @@ -100,6 +100,8 @@ class Node : public Chainable, virtual bool marked_for_debug() override { return markedForDebug_; } virtual const std::string& debug_message() override { return debugMessage_; } + virtual void dump(const std::string& filename) override; + virtual size_t allocate() override; virtual void free() override; From 555fb80386b1fd716f8aaf1fa56cab1e8ea8df26 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 13:59:34 -0800 Subject: [PATCH 218/838] now using explicit slices for factors --- src/layers/generic.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 813a5eb42..fe4e71e7c 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -193,11 +193,30 @@ namespace marian { } else if (embeddingFactorMapping_) { auto graph = input->graph(); - auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits - // denominators (only for groups that don't normalize out naturally by the final softmax()) + // project each factor const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); + std::vector groupYs(numGroups); +#if 1 + for (size_t g = 0; g < numGroups; g++) { + auto range = groupRanges[g]; + ABORT_IF(g > 0 && groupRanges[g].first != groupRanges[g-1].second, "Factor groups must be consecutive"); // we could sort groupYs though + // slice this group's section out of W_ + // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. + auto groupW = slice(W_, transposeW_ ? 0 : -1, Slice((int)range.first, (int)range.second)); + //LOG(info, "slice() -> {}, {}", groupW->type(), std::string(groupW->shape())); + auto groupB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix + auto groupY = affine(input, groupW, groupB, false, transposeW_); // [B... x U] factor logits + groupYs[g] = groupY; + } + auto y = concatenate(groupYs, /*axis=*/ -1); +#else + auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits +#endif + // @TODO: next, do normalization per group + + // denominators const auto& mVecs = embeddingFactorMapping_->mVecs_; for (size_t g = 0; g < numGroups; g++) { auto range = groupRanges[g]; @@ -215,6 +234,7 @@ namespace marian { auto name = options_->get("prefix"); auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); y = y * ((llWeight - 1) * m + 1); + // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. } // sum up the unit logits across factors for each target word From 8061b0fdb5db291a8a2f16a2aaaf04da1d965550 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 14:16:38 -0800 Subject: [PATCH 219/838] normalization now also done on slices --- src/layers/generic.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index fe4e71e7c..ff9884a86 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -198,6 +198,7 @@ namespace marian { const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly auto numGroups = groupRanges.size(); std::vector groupYs(numGroups); + std::vector groupLLWeights(numGroups); #if 1 for (size_t g = 0; g < numGroups; g++) { auto range = groupRanges[g]; @@ -208,6 +209,14 @@ namespace marian { //LOG(info, "slice() -> {}, {}", groupW->type(), std::string(groupW->shape())); auto groupB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix auto groupY = affine(input, groupW, groupB, false, transposeW_); // [B... x U] factor logits + // normalize + groupY = logsoftmax(groupY); + // log-linear weight --@TODO: pre-create in constructor + auto name = options_->get("prefix"); + groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + groupY = groupY * groupLLWeights[g]; + // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. + // @TODO: normalize again. Do I need the first normalization? groupYs[g] = groupY; } auto y = concatenate(groupYs, /*axis=*/ -1); @@ -216,6 +225,7 @@ namespace marian { #endif // @TODO: next, do normalization per group +#if 0 // denominators const auto& mVecs = embeddingFactorMapping_->mVecs_; for (size_t g = 0; g < numGroups; g++) { @@ -223,19 +233,20 @@ namespace marian { // y: [B... x U] // m: [1 x U] // ones at positions of group members // need to compute log denominator over y[range] and subtract it from y[range] - auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] - auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] - //auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] + //auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] + //auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] + ////auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] const auto& mVec = mVecs[g]; auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] - auto Z = dot(groupZ, m); // [B... x U] - y = y - Z; + //auto Z = dot(groupZ, m); // [B... x U] + //y = y - Z; // and a log-linear weight auto name = options_->get("prefix"); - auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - y = y * ((llWeight - 1) * m + 1); + auto groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + y = y * ((groupLLWeights[g] - 1) * m + 1); // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. } +#endif // sum up the unit logits across factors for each target word auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] From 57813fdef94bd5f3dbd1bcd40cefdc287292b5a0 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 14:42:35 -0800 Subject: [PATCH 220/838] moved factored-output combination into the Logits class --- src/layers/generic.cpp | 88 ++++++++++++++++++++++-------------------- src/layers/generic.h | 26 ++++--------- 2 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index ff9884a86..c8d370c32 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -151,6 +151,52 @@ namespace marian { std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; + Expr Logits::getLogits() const { + ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + if (!embeddingFactorMapping_) { + ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); + return logits_.front(); + } + + // lazily compute combined logits from factors + auto y = concatenate(logits_, /*axis=*/ -1); + + // sum up the unit logits across factors for each target word + auto graph = y->graph(); + auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] + y = dot_csr( + y, // [B x U] + factorMatrix.shape, + graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), + graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), + graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), + /*transB=*/ true); // -> [B x V] + + return y; +#if 0 + // denominators + const auto& mVecs = embeddingFactorMapping_->mVecs_; + for (size_t g = 0; g < numGroups; g++) { + auto range = groupRanges[g]; + // y: [B... x U] + // m: [1 x U] // ones at positions of group members + // need to compute log denominator over y[range] and subtract it from y[range] + //auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] + //auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] + ////auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] + const auto& mVec = mVecs[g]; + auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] + //auto Z = dot(groupZ, m); // [B... x U] + //y = y - Z; + // and a log-linear weight + auto name = options_->get("prefix"); + auto groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + y = y * ((groupLLWeights[g] - 1) * m + 1); + // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. + } +#endif + } + namespace mlp { /*private*/ void Output::lazyConstruct(int inputDim) { // We must construct lazily since we won't know tying nor input dim in constructor. @@ -199,7 +245,6 @@ namespace marian { auto numGroups = groupRanges.size(); std::vector groupYs(numGroups); std::vector groupLLWeights(numGroups); -#if 1 for (size_t g = 0; g < numGroups; g++) { auto range = groupRanges[g]; ABORT_IF(g > 0 && groupRanges[g].first != groupRanges[g-1].second, "Factor groups must be consecutive"); // we could sort groupYs though @@ -219,46 +264,7 @@ namespace marian { // @TODO: normalize again. Do I need the first normalization? groupYs[g] = groupY; } - auto y = concatenate(groupYs, /*axis=*/ -1); -#else - auto y = affine(input, W_, b_, false, transposeW_); // [B... x U] factor logits -#endif - // @TODO: next, do normalization per group - -#if 0 - // denominators - const auto& mVecs = embeddingFactorMapping_->mVecs_; - for (size_t g = 0; g < numGroups; g++) { - auto range = groupRanges[g]; - // y: [B... x U] - // m: [1 x U] // ones at positions of group members - // need to compute log denominator over y[range] and subtract it from y[range] - //auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] - //auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] - ////auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] - const auto& mVec = mVecs[g]; - auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] - //auto Z = dot(groupZ, m); // [B... x U] - //y = y - Z; - // and a log-linear weight - auto name = options_->get("prefix"); - auto groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - y = y * ((groupLLWeights[g] - 1) * m + 1); - // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. - } -#endif - - // sum up the unit logits across factors for each target word - auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] - y = dot_csr( - y, // [B x U] - factorMatrix.shape, - graph->constant({(int)factorMatrix.weights.size()}, inits::from_vector(factorMatrix.weights), Type::float32), - graph->constant({(int)factorMatrix.indices.size()}, inits::from_vector(factorMatrix.indices), Type::uint32), - graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), - /*transB=*/ true); // -> [B x V] - - return y; + return Logits(std::move(groupYs), embeddingFactorMapping_); } else return affine(input, W_, b_, false, transposeW_); diff --git a/src/layers/generic.h b/src/layers/generic.h index 8021dcfee..70bb75aab 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -54,30 +54,22 @@ struct IEmbeddingLayer { virtual Expr apply(const std::vector& embIdx, int dimBatch, int dimBeam) const = 0; }; +class EmbeddingFactorMapping; + // @HACK: Frank's quick implementation of factored outputs. To be re-thought once it works. // Output layer returns a Logits object, which is able to compute some things on the fly // for factored embeddings. class Logits { Logits& operator=(const Logits& other) = default; public: - Logits(Expr logits) { + Logits(Expr logits) { // single-output constructor if (logits) logits_.push_back(logits); } - Expr getLogits() const { - ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); - // lazily compute logits - // @BUGBUG: This is completely wrong, we must use the factor information and map the dimensions - Expr res; - for (size_t i = 0; i < getNumFactors(); i++) { - auto logits = logits_[i]; - if (!weights_.empty() && weights_[i]) // log-linear weights - logits = logits * weights_[i]; - res = res ? res + logits : logits; - } - return res; - } - void assign(const Logits& other) { // @TODO: forbid changing the number of contributions + Logits(std::vector&& logits, Ptr embeddingFactorMapping) // factored-output constructor + : logits_(std::move(logits)), embeddingFactorMapping_(embeddingFactorMapping) {} + Expr getLogits() const; + void assign(const Logits& other) { ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), "Logits assignment cannot change number of factors"); *this = other; @@ -86,7 +78,7 @@ class Logits { bool empty() const { return logits_.empty(); } private: std::vector logits_; - std::vector weights_; + Ptr embeddingFactorMapping_; }; // Unary function that returns a Logits object @@ -95,8 +87,6 @@ struct IUnaryLogitLayer { virtual Logits apply(const std::vector&) = 0; }; -class EmbeddingFactorMapping; - namespace mlp { class Dense : public LayerBase, public IUnaryLayer { From 20a0d26e4c607579fb7d89f99d4e7cc20261abe8 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 14:52:10 -0800 Subject: [PATCH 221/838] dump nodes recursively --- src/graph/expression_graph.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index 4df97a82f..3cf9f82fb 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -43,6 +43,12 @@ io::Item itemFromTensor(Tensor t, const std::string name, Ptr backend) return item; } +void recChildren(Expr node, std::vector& items, Ptr backend) { + items.push_back(itemFromTensor(node->val(), "node" + std::to_string(node->getId()), backend)); + for(auto&& child : node->children()) + recChildren(child, items, backend); +} + void ExpressionGraph::forwardNext() { // @TODO: check if allocation works properly tensors_->clearShorttermMemory(); @@ -57,22 +63,20 @@ void ExpressionGraph::forwardNext() { bool isNan = false, isInf = false; checkNan(v->val(), isNan, isInf); if(isNan || isInf) { - std::vector ioItems; LOG(critical, "Detected NaN ({}) or Inf ({}) in value (forward pass)", isNan, isInf); LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", v->type(), v->shape(), v->name(), v->getId(), v->hash()); LOG(critical, "Value debug {}", v->val()->debug()); - ioItems.push_back(itemFromTensor(v->val(), "value", backend_)); - LOG(critical, "Children: {}", v->children().size()); for(auto&& child : v->children()) { LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", child->type(), child->shape(), child->name(), child->getId(), child->hash()); LOG(critical, "Value debug {}", child->val()->debug()); - ioItems.push_back(itemFromTensor(child->val(), "child_" + std::to_string(child->hash()), backend_)); } + std::vector ioItems; + recChildren(v, ioItems, backend_); io::saveItems("dump-for-nans.npz", ioItems); ABORT("Aborting"); From 9ce31fa3fd96dd59aaa4b56602576dcfcbad2746 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 15:10:28 -0800 Subject: [PATCH 222/838] moved crossEntropy() to Logits class --- src/layers/generic.cpp | 29 +++++++++++++++++++++++++++++ src/layers/generic.h | 1 + src/layers/loss.cpp | 40 ++++++++-------------------------------- src/layers/loss.h | 17 +++++++++-------- src/models/costs.h | 2 +- 5 files changed, 48 insertions(+), 41 deletions(-) mode change 100644 => 100755 src/layers/loss.h diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index c8d370c32..633b0f62f 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -151,6 +151,35 @@ namespace marian { std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; + Expr Logits::crossEntropy(Expr indices, float smoothing) const { + auto logits = getLogits(); + Expr ce; + if(smoothing > 0) { + // ce = sum_i y^_i log y_i(z)_i + // with smoothing: + // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i + // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i + // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i + // @TODO: add this to CE kernels instead +#if 0 + ce = cross_entropy(logits, indices); + auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceq; +#else // alternative that is cheaper memory-wise + ce = cross_entropy(logits, indices); + auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + ce = (1 - smoothing) * ce - smoothing * ceq; + //auto ceq = mean(logits, /*axis=*/ -1) - Z; + //ce = (1 - smoothing_) * cols(logits, indices) // ce term + // - smoothing_ * mean(logits, /*axis=*/ -1) // smoothing term + // - logsumexp(logits, /*axis=*/ -1); // denominator +#endif + } + else + ce = cross_entropy(logits, indices); + return ce; + } + Expr Logits::getLogits() const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!embeddingFactorMapping_) { diff --git a/src/layers/generic.h b/src/layers/generic.h index 70bb75aab..d33ab03cf 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -69,6 +69,7 @@ class Logits { Logits(std::vector&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), embeddingFactorMapping_(embeddingFactorMapping) {} Expr getLogits() const; + Expr crossEntropy(Expr indices, float smoothing) const; void assign(const Logits& other) { ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), "Logits assignment cannot change number of factors"); diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 5ca493f1d..12e69d028 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -22,35 +22,11 @@ Ptr LossFactory(Ptr options, bool inference) { } } -Expr LossBase::getCrossEntropy(Expr logits, +Expr LossBase::getCrossEntropy(const Logits& logits1, Expr indices, Expr mask, Expr weights) { - - Expr ce; - if(smoothing_ > 0) { - // ce = sum_i y^_i log y_i(z)_i - // with smoothing: - // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i - // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i - // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i - // @TODO: add this to CE kernels instead -#if 0 - ce = cross_entropy(logits, indices); - auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; -#else // alternative that is cheaper memory-wise - ce = cross_entropy(logits, indices); - auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; - //auto ceq = mean(logits, /*axis=*/ -1) - Z; - //ce = (1 - smoothing_) * cols(logits, indices) // ce term - // - smoothing_ * mean(logits, /*axis=*/ -1) // smoothing term - // - logsumexp(logits, /*axis=*/ -1); // denominator -#endif - } - else - ce = cross_entropy(logits, indices); + auto ce = logits1.crossEntropy(indices, smoothing_); if(mask) ce = ce * mask; @@ -61,7 +37,7 @@ Expr LossBase::getCrossEntropy(Expr logits, return ce; } -Expr CrossEntropyMeanLoss::getCost(Expr logits, +Expr CrossEntropyMeanLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { @@ -77,7 +53,7 @@ Expr CrossEntropyMeanLoss::getCost(Expr logits, // } } -Expr CrossEntropyMeanWordsLoss::getCost(Expr logits, +Expr CrossEntropyMeanWordsLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { @@ -92,7 +68,7 @@ Expr CrossEntropyMeanWordsLoss::getCost(Expr logits, // } } -Expr CrossEntropySumLoss::getCost(Expr logits, +Expr CrossEntropySumLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { @@ -106,7 +82,7 @@ Expr CrossEntropySumLoss::getCost(Expr logits, // } } -Expr PerplexityLoss::getCost(Expr logits, +Expr PerplexityLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { @@ -121,7 +97,7 @@ Expr PerplexityLoss::getCost(Expr logits, // } } -Expr CrossEntropyRescoreLoss::getCost(Expr logits, +Expr CrossEntropyRescoreLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { @@ -129,7 +105,7 @@ Expr CrossEntropyRescoreLoss::getCost(Expr logits, return -sum(ce, /*axis =*/ -3); } -Expr CrossEntropyRescoreMeanLoss::getCost(Expr logits, +Expr CrossEntropyRescoreMeanLoss::getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) { diff --git a/src/layers/loss.h b/src/layers/loss.h old mode 100644 new mode 100755 index ebf711470..ccd00ed3a --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -1,6 +1,7 @@ #pragma once #include "marian.h" +#include "layers/generic.h" // @HACK for Frank's factoring hack namespace marian { class LossBase { @@ -10,8 +11,8 @@ class LossBase { public: explicit LossBase(float smoothing = 0) : smoothing_(smoothing){}; - Expr getCrossEntropy(Expr logits, Expr indices, Expr mask, Expr weights); - virtual Expr getCost(Expr logits, + Expr getCrossEntropy(const Logits& logits, Expr indices, Expr mask, Expr weights); + virtual Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights = nullptr) @@ -26,7 +27,7 @@ class LossBase { class CrossEntropyMeanLoss : public LossBase { public: explicit CrossEntropyMeanLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; /* @@ -36,7 +37,7 @@ class CrossEntropyMeanWordsLoss : public LossBase { public: explicit CrossEntropyMeanWordsLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; /* @@ -45,7 +46,7 @@ class CrossEntropyMeanWordsLoss : public LossBase { class CrossEntropySumLoss : public LossBase { public: explicit CrossEntropySumLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; /* @@ -54,7 +55,7 @@ class CrossEntropySumLoss : public LossBase { class PerplexityLoss : public LossBase { public: explicit PerplexityLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; /* @@ -63,13 +64,13 @@ class PerplexityLoss : public LossBase { class CrossEntropyRescoreLoss : public LossBase { public: explicit CrossEntropyRescoreLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; class CrossEntropyRescoreMeanLoss : public LossBase { public: explicit CrossEntropyRescoreMeanLoss(float smoothing = 0) : LossBase(smoothing){}; - Expr getCost(Expr logits, Expr indices, Expr mask, Expr weights) override; + Expr getCost(const Logits& logits, Expr indices, Expr mask, Expr weights) override; }; Ptr LossFactory(Ptr options, bool inference); diff --git a/src/models/costs.h b/src/models/costs.h index 5231e0ce0..efa3faa34 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -54,7 +54,7 @@ class EncoderDecoderCE : public CostBase { weights = weighter_->getWeights(graph, corpusBatch); Expr cost; - cost = loss_->getCost(state->getLogProbs().getLogits(), + cost = loss_->getCost(state->getLogProbs(), state->getTargetIndices(), state->getTargetMask(), weights); From 226721ffe9af43dca44a5dba663c955f03108bbf Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 15:55:29 -0800 Subject: [PATCH 223/838] full graph below nide --- src/graph/expression_graph.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index 3cf9f82fb..5b4df3395 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -43,10 +43,11 @@ io::Item itemFromTensor(Tensor t, const std::string name, Ptr backend) return item; } -void recChildren(Expr node, std::vector& items, Ptr backend) { - items.push_back(itemFromTensor(node->val(), "node" + std::to_string(node->getId()), backend)); +void recChildren(Expr node, const std::string& parent, std::vector& items, Ptr backend) { + std::string name = node->type() + "_" + std::to_string(node->getId()) + "_p:" + parent; + items.push_back(itemFromTensor(node->val(), name, backend)); for(auto&& child : node->children()) - recChildren(child, items, backend); + recChildren(child, std::to_string(node->getId()), items, backend); } void ExpressionGraph::forwardNext() { @@ -76,7 +77,7 @@ void ExpressionGraph::forwardNext() { } std::vector ioItems; - recChildren(v, ioItems, backend_); + recChildren(v, "root", ioItems, backend_); io::saveItems("dump-for-nans.npz", ioItems); ABORT("Aborting"); From c46ebb0c68e4eab546177678100b7695cc09db09 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 21:42:53 -0800 Subject: [PATCH 224/838] optimized factored loss, also moved CE back to loss.cp via; bug fix: RowsNodeOp should propagate value type --- src/graph/node_operators_binary.h | 4 +- src/layers/generic.cpp | 119 +++++++++++++++------------ src/layers/generic.h | 2 +- src/layers/loss.cpp | 16 +++- src/tensors/cpu/tensor_operators.cpp | 1 + 5 files changed, 86 insertions(+), 56 deletions(-) diff --git a/src/graph/node_operators_binary.h b/src/graph/node_operators_binary.h index 7d0908237..ab2f2efef 100755 --- a/src/graph/node_operators_binary.h +++ b/src/graph/node_operators_binary.h @@ -513,7 +513,7 @@ struct ScalarProductNodeOp : public NaryNodeOp { struct RowsNodeOp : public NaryNodeOp { RowsNodeOp(Expr a, Expr indices) - : NaryNodeOp({a, indices}, newShape(a, indices->shape().elements())) { + : NaryNodeOp({a, indices}, newShape(a, indices->shape().elements()), a->value_type()) { matchOrAbort(indices->value_type()); } @@ -641,7 +641,7 @@ struct GatherNodeOp : public NaryNodeOp { struct ColsNodeOp : public NaryNodeOp { ColsNodeOp(Expr a, Expr indices) - : NaryNodeOp({a, indices}, newShape(a, indices->shape().elements())) { + : NaryNodeOp({a, indices}, newShape(a, indices->shape().elements()), a->value_type()) { matchOrAbort(indices->value_type()); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 633b0f62f..2a3d3db7c 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -44,11 +44,12 @@ namespace marian { // Specifically, it means that the factorVocab_ must contain and "". Vocab vocab(New(), 0); vocab.load(vocabPath); + auto vocabSize = vocab.size(); factorVocab_.load(factorVocabPath); Word numFactors = (Word)factorVocab_.size(); // load and parse factorMap - factorMap_.resize(vocab.size()); + factorMap_.resize(vocabSize); factorRefCounts_.resize(numFactors); std::vector tokens; io::InputFileStream in(mapPath); @@ -60,13 +61,12 @@ namespace marian { ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[v], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); for (size_t i = 1; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; - auto& m = factorMap_[v]; - m.push_back(u); + factorMap_[v].push_back(u); factorRefCounts_[u]++; } numTotalFactors += tokens.size() - 1; } - LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocab.size()); + LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); // form groups // @TODO: hard-coded for these initial experiments @@ -85,6 +85,7 @@ namespace marian { factorGroups_[u] = g; } } + // determine group index ranges groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); std::vector groupCounts(numGroups); // number of group members for (Word u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts @@ -95,8 +96,18 @@ namespace marian { groupRanges_[g].second = u + 1; groupCounts[g]++; } - // create the flag vectors for normalization --@TODO: maybe we won't need them anymore - mVecs_.resize(numGroups); + // create mappings needed for normalization in factored outputs + factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g + factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) + for (Word v = 0; v < vocabSize; v++) { + for (auto u : factorMap_[v]) { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); + factorMasks_[g][v] = 1.0f; + } + } + //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); @@ -104,16 +115,16 @@ namespace marian { continue; ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - auto& mVec = mVecs_[g]; - mVec.resize(numFactors, 0.0f); - for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) - mVec[i] = 1.0f; + //auto& mVec = mVecs_[g]; + //mVec.resize(numFactors, 0.0f); + //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) + // mVec[i] = 1.0f; } - // create the factor matrix - std::vector data(vocab.size()); + // create the global factor matrix, which is used for factored embeddings + std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); - factorMatrix_ = csr_rows(data); // [V x U] + globalFactorMatrix_ = csr_rows(data); // [V x U] } size_t factorVocabSize() const { return factorVocab_.size(); } @@ -139,45 +150,52 @@ namespace marian { return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; } - const CSRData& getFactorMatrix() const { return factorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v + const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v + size_t getNumGroups() const { return groupRanges_.size(); } + std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) + const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g + const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor private: Vocab factorVocab_; // [factor name] -> factor index = row of E_ - std::vector> factorMap_; // [word index] -> set of factor indices - std::vector factorRefCounts_; // [factor index] -> how often this factor is referenced in factorMap_ - CSRData factorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v + std::vector> factorMap_; // [word index v] -> set of factor indices u + std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ + CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v std::vector factorGroups_; // [u] -> group id of factor u - public: // @TODO: temporarily; later factor this properly - std::vector> groupRanges_; // [group id] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. - std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group + std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. + std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g + std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) + //public: // @TODO: temporarily; later factor this properly + //std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; - Expr Logits::crossEntropy(Expr indices, float smoothing) const { - auto logits = getLogits(); - Expr ce; - if(smoothing > 0) { - // ce = sum_i y^_i log y_i(z)_i - // with smoothing: - // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i - // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i - // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i - // @TODO: add this to CE kernels instead -#if 0 - ce = cross_entropy(logits, indices); - auto ceq = mean(logsoftmax(logits), /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; -#else // alternative that is cheaper memory-wise - ce = cross_entropy(logits, indices); - auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); - ce = (1 - smoothing) * ce - smoothing * ceq; - //auto ceq = mean(logits, /*axis=*/ -1) - Z; - //ce = (1 - smoothing_) * cols(logits, indices) // ce term - // - smoothing_ * mean(logits, /*axis=*/ -1) // smoothing term - // - logsumexp(logits, /*axis=*/ -1); // denominator -#endif + Expr Logits::getLoss(Expr indices, const std::function& lossFn) const { + ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + if (!embeddingFactorMapping_) { + ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); + return lossFn(logits_.front(), indices); } - else - ce = cross_entropy(logits, indices); - return ce; + + // accumulate all CEs for all words that have the factor + // Memory-wise, this is cheap, all temp objects below are batches of scalars or lookup vectors. + auto graph = indices->graph(); + Expr loss; + auto numGroups = embeddingFactorMapping_->getNumGroups(); + for (size_t g = 0; g < numGroups; g++) { + indices; // [B... * 1] all batch items flattened + auto factorMaskVector = embeddingFactorMapping_->getFactorMasks(g); // [v] 1.0 if v has factor of group g + auto factorIndexVector = embeddingFactorMapping_->getFactorIndices(g); // [v] index of factor for word v in group p; must be 0 if factor is not used + auto factorMaskMatrix = graph->constant({(int)factorMaskVector.size(), 1}, inits::from_vector(factorMaskVector), Type::float32); // [V x 1] + auto factorIndexMatrix = graph->constant({(int)factorIndexVector.size(), 1}, inits::from_vector(factorIndexVector), Type::uint32); // [V x 1(Ug)] + auto factorIndex = rows(factorIndexMatrix, indices); // [B... * 1(Ug)] map word indices to factor indices (indices into factorLogits) + auto factorMask = rows(factorMaskMatrix, indices); // [B... * 1] flag whether word has the factor in the first place + auto factorLogits = logits_[g]; // [B... * Ug] + // @TODO: no need to normalize factors before here + // For each location in [B...] select [indices[B...]]. If not using factor, select [0] and mask it out next. + auto factorLoss = lossFn(factorLogits, factorIndex); // [B... x 1] + factorLoss = factorLoss * reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor + loss = loss ? (loss + factorLoss) : factorLoss; // [B... x 1] + } + return loss; } Expr Logits::getLogits() const { @@ -192,7 +210,7 @@ namespace marian { // sum up the unit logits across factors for each target word auto graph = y->graph(); - auto factorMatrix = embeddingFactorMapping_->getFactorMatrix(); // [V x U] + auto factorMatrix = embeddingFactorMapping_->getGlobalFactorMatrix(); // [V x U] y = dot_csr( y, // [B x U] factorMatrix.shape, @@ -270,13 +288,12 @@ namespace marian { auto graph = input->graph(); // project each factor - const auto& groupRanges = embeddingFactorMapping_->groupRanges_; // @TODO: factor this properly - auto numGroups = groupRanges.size(); + auto numGroups = embeddingFactorMapping_->getNumGroups(); std::vector groupYs(numGroups); std::vector groupLLWeights(numGroups); for (size_t g = 0; g < numGroups; g++) { - auto range = groupRanges[g]; - ABORT_IF(g > 0 && groupRanges[g].first != groupRanges[g-1].second, "Factor groups must be consecutive"); // we could sort groupYs though + auto range = embeddingFactorMapping_->getGroupRange(g); + ABORT_IF(g > 0 && range.first != embeddingFactorMapping_->getGroupRange(g-1).second, "Factor groups must be consecutive"); // we could sort groupYs though // slice this group's section out of W_ // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. auto groupW = slice(W_, transposeW_ ? 0 : -1, Slice((int)range.first, (int)range.second)); diff --git a/src/layers/generic.h b/src/layers/generic.h index d33ab03cf..2bc958b73 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -69,7 +69,7 @@ class Logits { Logits(std::vector&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), embeddingFactorMapping_(embeddingFactorMapping) {} Expr getLogits() const; - Expr crossEntropy(Expr indices, float smoothing) const; + Expr getLoss(Expr indices, const std::function& lossFn) const; void assign(const Logits& other) { ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), "Logits assignment cannot change number of factors"); diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 12e69d028..3493e7c85 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -22,11 +22,23 @@ Ptr LossFactory(Ptr options, bool inference) { } } -Expr LossBase::getCrossEntropy(const Logits& logits1, +Expr LossBase::getCrossEntropy(const Logits& logits, Expr indices, Expr mask, Expr weights) { - auto ce = logits1.crossEntropy(indices, smoothing_); + auto ce = logits.getLoss(indices, [&](Expr logits, Expr indices) { + Expr ce = cross_entropy(logits, indices); + if (smoothing_ > 0) { + // ce = sum_i y^_i log y_i(z)_i + // with smoothing: + // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i + // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i + // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i + auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceq; + } + return ce; + }); if(mask) ce = ce * mask; diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index a3b6bf9a6..f38c1006b 100755 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -387,6 +387,7 @@ void CopyRows(Tensor out_, size_t cols = in_->shape()[-1]; size_t rows = indices->size(); + // note: may also be applied to IndexType; works by luck. Fix with fp16 float* out = out_->data(); const float* in = in_->data(); From bd98139d2140a11d8057fb529910d5cb280a3794 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 28 Jan 2019 22:46:51 -0800 Subject: [PATCH 225/838] added normalization to getLogits() --- src/layers/generic.cpp | 6 +++++- src/layers/generic.h | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2a3d3db7c..4b43f8e85 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -169,6 +169,7 @@ namespace marian { }; Expr Logits::getLoss(Expr indices, const std::function& lossFn) const { + LOG_ONCE(info, "[logits] getLoss() for {} factors", logits_.size()); ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!embeddingFactorMapping_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); @@ -206,7 +207,10 @@ namespace marian { } // lazily compute combined logits from factors - auto y = concatenate(logits_, /*axis=*/ -1); + std::vector logProbs(logits_.size()); + for (size_t g = 0; g < logits_.size(); g++) + logProbs[g] = logsoftmax(logits_[g]); + auto y = concatenate(logProbs, /*axis=*/ -1); // sum up the unit logits across factors for each target word auto graph = y->graph(); diff --git a/src/layers/generic.h b/src/layers/generic.h index 2bc958b73..fcc97b88d 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -71,8 +71,8 @@ class Logits { Expr getLogits() const; Expr getLoss(Expr indices, const std::function& lossFn) const; void assign(const Logits& other) { - ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), - "Logits assignment cannot change number of factors"); + //ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), + // "Logits assignment cannot change number of factors"); *this = other; } size_t getNumFactors() const { return logits_.size(); } From 85408e3537e7cb076a6550ccb1a5daa55eaa8699 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 28 Jan 2019 22:55:15 -0800 Subject: [PATCH 226/838] temporarily use strided gemm --- src/tensors/gpu/prod.cpp | 82 +++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index d13081728..f282cc839 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -178,53 +178,65 @@ void ProdBatched(marian::Tensor C, int strideC = n * m; int batchC = std::max(batchA, batchB); - std::vector aptr; - std::vector bptr; - std::vector cptr; - - for(int i = 0; i < batchC; i++) { - aptr.push_back(A->data() + (i % batchA) * strideA); - bptr.push_back(B->data() + (i % batchB) * strideB); - cptr.push_back(C->data() + i * strideC); - } + // std::vector aptr; + // std::vector bptr; + // std::vector cptr; + + // for(int i = 0; i < batchC; i++) { + // aptr.push_back(A->data() + (i % batchA) * strideA); + // bptr.push_back(B->data() + (i % batchB) * strideB); + // cptr.push_back(C->data() + i * strideC); + // } - auto mp_aptr = allocator->alloc(aptr.size()); - CudaCopy( - aptr.data(), aptr.data() + aptr.size(), mp_aptr->data()); + // auto mp_aptr = allocator->alloc(aptr.size()); + // CudaCopy(aptr.data(), aptr.data() + aptr.size(), mp_aptr->data()); - auto mp_bptr = allocator->alloc(bptr.size()); - CudaCopy( - bptr.data(), bptr.data() + bptr.size(), mp_bptr->data()); + // auto mp_bptr = allocator->alloc(bptr.size()); + // CudaCopy(bptr.data(), bptr.data() + bptr.size(), mp_bptr->data()); - auto mp_cptr = allocator->alloc(cptr.size()); - CudaCopy(cptr.data(), cptr.data() + cptr.size(), mp_cptr->data()); + // auto mp_cptr = allocator->alloc(cptr.size()); + // CudaCopy(cptr.data(), cptr.data() + cptr.size(), mp_cptr->data()); #if CUDA_VERSION >= 9000 setTensorMode(cublasHandle); //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, - opB, - opA, - n, - m, - k, - &alpha, - mp_bptr->data(), - ldb, - mp_aptr->data(), - lda, - &beta, - mp_cptr->data(), - ldc, - batchC)); + // CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, + // opB, + // opA, + // n, + // m, + // k, + // &alpha, + // mp_bptr->data(), + // ldb, + // mp_aptr->data(), + // lda, + // &beta, + // mp_cptr->data(), + // ldc, + // batchC)); + +cublasSgemmStridedBatched(cublasHandle, + opB, opA, + n, m, k, + &alpha, + B->data(), ldb, strideB, + A->data(), lda, strideA, + &beta, + C->data(), ldc, strideC, + batchC); + + #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif - allocator->free(mp_aptr); - allocator->free(mp_bptr); - allocator->free(mp_cptr); + // allocator->free(mp_aptr); + // allocator->free(mp_bptr); + // allocator->free(mp_cptr); + + cudaStreamSynchronize(0); } // C = op(S) x D if not swapOperands else C = D x op(S) From 6235641f5966053e9befae2d0a6abb92bcaf29e3 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 09:13:23 -0800 Subject: [PATCH 227/838] correct usage of shared memory --- src/tensors/gpu/tensor_operators.cu | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index 8e256bc78..ae1e02653 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -420,7 +420,7 @@ __global__ void gSoftmax(float* out, extern __shared__ float _share[]; - float* _max = _share + blockDim.x; + float* _max = _share; _max[threadIdx.x] = -CUDA_FLT_MAX; // mask for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; @@ -503,7 +503,7 @@ __global__ void gLogSoftmax(float* out, extern __shared__ float _share[]; - float* _max = _share + blockDim.x; + float* _max = _share; _max[threadIdx.x] = sp[threadIdx.x]; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; @@ -584,7 +584,7 @@ __global__ void gSoftmaxGrad(float* grad, int j = bid + blockIdx.x; if(j < rows) { extern __shared__ float _share[]; - float* _sum = _share + blockDim.x; + float* _sum = _share; float* gradRow = grad + j * cols; const float* adjRow = adj + j * cols; @@ -629,7 +629,7 @@ void SoftmaxGrad(Tensor grad, Tensor adj, Tensor val) { int blocks = std::min(MAX_BLOCKS, m); int threads = std::min(MAX_THREADS, k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gSoftmaxGrad<<>>( grad->data(), adj->data(), val->data(), m, k); } @@ -643,7 +643,7 @@ __global__ void gLogSoftmaxGrad(float* grad, int j = bid + blockIdx.x; if(j < rows) { extern __shared__ float _share[]; - float* _sum = _share + blockDim.x; + float* _sum = _share; float* gradRow = grad + j * cols; const float* adjRow = adj + j * cols; @@ -686,7 +686,7 @@ void LogSoftmaxGrad(Tensor grad, Tensor adj, Tensor val) { int blocks = std::min(MAX_BLOCKS, m); int threads = std::min(MAX_THREADS, k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gLogSoftmaxGrad<<>>( grad->data(), adj->data(), val->data(), m, k); } @@ -1153,7 +1153,7 @@ __global__ void gCrossEntropyPick(float* out, const float* sp = in + j * cols; extern __shared__ float _share[]; - float* _max = _share + blockDim.x; + float* _max = _share; _max[threadIdx.x] = sp[threadIdx.x]; for(int tid = 1; tid < cols; tid += blockDim.x) { @@ -1243,7 +1243,7 @@ __global__ void gCrossEntropyPickBackward(float* out, float* so = out + j * cols; extern __shared__ float _share[]; - float* _max = _share + blockDim.x; + float* _max = _share; _max[threadIdx.x] = sp[threadIdx.x]; for(int tid = 1; tid < cols; tid += blockDim.x) { @@ -1358,7 +1358,7 @@ __global__ void gAtt(float* out, const float* stateRow = state + ((j / (b * t)) * b + j % b) * cols; extern __shared__ float _share[]; - float* _sum = _share + blockDim.x; + float* _sum = _share; _sum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { @@ -1395,7 +1395,7 @@ void Att(Tensor out, Tensor va, Tensor context, Tensor state) { int blocks = std::min(MAX_BLOCKS, (int)m); int threads = std::min(MAX_THREADS, (int)k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gAtt<<>>( out->data(), va->data(), context->data(), state->data(), m, k, b, t); @@ -1485,7 +1485,7 @@ __global__ void gLNormalization(float* out, float* so = out + j * cols; const float* sp = in + j * cols; - float* _sum = _share + blockDim.x; + float* _sum = _share; _sum[threadIdx.x] = 0.0f; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; From db66fc70a5a10a917fcbd6d3c32bcc5b04b62418 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 09:22:55 -0800 Subject: [PATCH 228/838] fix shared memory --- src/tensors/gpu/add.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu index f8f93b372..9e0c8e687 100755 --- a/src/tensors/gpu/add.cu +++ b/src/tensors/gpu/add.cu @@ -84,7 +84,7 @@ __global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggF int j = bid + blockIdx.x; if(j < rows) { extern __shared__ float _share[]; - float* _sum = _share + blockDim.x; + float* _sum = _share; if(same) { _sum[threadIdx.x] = aggInit; @@ -143,7 +143,7 @@ void Aggregate(Functor functor, float aggInit, AggFunctor aggFunctor, float scal int blocks = std::min(MAX_BLOCKS, (int)m); int threads = std::min(MAX_THREADS, (int)k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gAggregateReduce<<>>(functor, aggInit, aggFunctor, full, gOut, gIns, scale); @@ -187,7 +187,7 @@ void Add(Functor functor, float scale, marian::Tensor out, Tensors... tensors) { int blocks = std::min(MAX_BLOCKS, (int)m); int threads = std::min(MAX_THREADS, (int)k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gAggregateReduce<<>>(functor, 0, addFunctor, full, gOut, gIns, scale); From 0acb9d2e72563b5cb9d2896ac1d4580f4bb66b63 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 09:26:30 -0800 Subject: [PATCH 229/838] restore prod.cpp --- src/tensors/gpu/prod.cpp | 82 +++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index f282cc839..d13081728 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -178,65 +178,53 @@ void ProdBatched(marian::Tensor C, int strideC = n * m; int batchC = std::max(batchA, batchB); - // std::vector aptr; - // std::vector bptr; - // std::vector cptr; - - // for(int i = 0; i < batchC; i++) { - // aptr.push_back(A->data() + (i % batchA) * strideA); - // bptr.push_back(B->data() + (i % batchB) * strideB); - // cptr.push_back(C->data() + i * strideC); - // } + std::vector aptr; + std::vector bptr; + std::vector cptr; + + for(int i = 0; i < batchC; i++) { + aptr.push_back(A->data() + (i % batchA) * strideA); + bptr.push_back(B->data() + (i % batchB) * strideB); + cptr.push_back(C->data() + i * strideC); + } - // auto mp_aptr = allocator->alloc(aptr.size()); - // CudaCopy(aptr.data(), aptr.data() + aptr.size(), mp_aptr->data()); + auto mp_aptr = allocator->alloc(aptr.size()); + CudaCopy( + aptr.data(), aptr.data() + aptr.size(), mp_aptr->data()); - // auto mp_bptr = allocator->alloc(bptr.size()); - // CudaCopy(bptr.data(), bptr.data() + bptr.size(), mp_bptr->data()); + auto mp_bptr = allocator->alloc(bptr.size()); + CudaCopy( + bptr.data(), bptr.data() + bptr.size(), mp_bptr->data()); - // auto mp_cptr = allocator->alloc(cptr.size()); - // CudaCopy(cptr.data(), cptr.data() + cptr.size(), mp_cptr->data()); + auto mp_cptr = allocator->alloc(cptr.size()); + CudaCopy(cptr.data(), cptr.data() + cptr.size(), mp_cptr->data()); #if CUDA_VERSION >= 9000 setTensorMode(cublasHandle); //cublasSetMathMode(cublasHandle, CUBLAS_TENSOR_OP_MATH); #endif - // CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, - // opB, - // opA, - // n, - // m, - // k, - // &alpha, - // mp_bptr->data(), - // ldb, - // mp_aptr->data(), - // lda, - // &beta, - // mp_cptr->data(), - // ldc, - // batchC)); - -cublasSgemmStridedBatched(cublasHandle, - opB, opA, - n, m, k, - &alpha, - B->data(), ldb, strideB, - A->data(), lda, strideA, - &beta, - C->data(), ldc, strideC, - batchC); - - + CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, + opB, + opA, + n, + m, + k, + &alpha, + mp_bptr->data(), + ldb, + mp_aptr->data(), + lda, + &beta, + mp_cptr->data(), + ldc, + batchC)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif - // allocator->free(mp_aptr); - // allocator->free(mp_bptr); - // allocator->free(mp_cptr); - - cudaStreamSynchronize(0); + allocator->free(mp_aptr); + allocator->free(mp_bptr); + allocator->free(mp_cptr); } // C = op(S) x D if not swapOperands else C = D x op(S) From 6a663f60baa868fa1659a0cea014df26f89623c4 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 09:27:23 -0800 Subject: [PATCH 230/838] remove classification.h --- src/translator/classification.h | 241 -------------------------------- 1 file changed, 241 deletions(-) delete mode 100644 src/translator/classification.h diff --git a/src/translator/classification.h b/src/translator/classification.h deleted file mode 100644 index 28a8a82a8..000000000 --- a/src/translator/classification.h +++ /dev/null @@ -1,241 +0,0 @@ -#pragma once -#include - -#include "marian.h" -#include "translator/history.h" -#include "translator/scorers.h" - -#include "translator/helpers.h" -#include "translator/nth_element.h" - -namespace marian { - -class Classification { -private: - Ptr options_; - std::vector> scorers_; - size_t topN_{1}; - -public: - Classification(Ptr options) - : options_(options), - scorers_(scorers), - topN_{options_->get("beam-size")} // misuse beam-size for topN display - {} - - Beams toHyps(const std::vector keys, - const std::vector pathScores, - size_t labelNum, - const Beams& beams, - std::vector>& states, - size_t topN, - bool first, - Ptr batch) { - Beams newBeams(beams.size()); - - std::vector align; - if(options_->has("alignment")) - // Use alignments from the first scorer, even if ensemble - align = scorers_[0]->getAlignment(); - - for(size_t i = 0; i < keys.size(); ++i) { - // Keys contains indices to vocab items in the entire beam. - // Values can be between 0 and topN * number of lables. - Word embIdx = (Word)(keys[i] % labelNum); - auto beamIdx = i / topN; - - // Retrieve short list for final softmax (based on words aligned - // to source sentences). If short list has been set, map the indices - // in the sub-selected vocabulary matrix back to their original positions. - auto shortlist = scorers_[0]->getShortlist(); - if(shortlist) - embIdx = shortlist->reverseMap(embIdx); // @TODO: should reverseMap accept a size_t or a Word? - - if(newBeams[beamIdx].size() < beams[beamIdx].size()) { - auto& beam = beams[beamIdx]; - auto& newBeam = newBeams[beamIdx]; - - auto hypIdx = (IndexType)(keys[i] / labelNum); - float pathScore = pathScores[i]; - - auto hypIdxTrans - = IndexType((hypIdx / topN) + (hypIdx % topN) * beams.size()); - if(first) - hypIdxTrans = hypIdx; - - size_t beamHypIdx = hypIdx % topN; - if(beamHypIdx >= (int)beam.size()) - beamHypIdx = beamHypIdx % beam.size(); - - if(first) - beamHypIdx = 0; - - auto hyp = New(beam[beamHypIdx], embIdx, hypIdxTrans, pathScore); - - // Set score breakdown for n-best lists - if(options_->get("n-best")) { - std::vector breakDown(states.size(), 0); - beam[beamHypIdx]->GetScoreBreakdown().resize(states.size(), 0); - for(size_t j = 0; j < states.size(); ++j) { - size_t key = embIdx + hypIdxTrans * labelNum; - breakDown[j] = states[j]->breakDown(key) - + beam[beamHypIdx]->GetScoreBreakdown()[j]; - } - hyp->GetScoreBreakdown() = breakDown; - } - - newBeam.push_back(hyp); - } - } - return newBeams; - } - - // main decoding function - Histories search(Ptr graph, Ptr batch) { - int dimBatch = (int)batch->size(); - - Histories histories; - for(int i = 0; i < dimBatch; ++i) { - size_t sentId = batch->getSentenceIds()[i]; - auto history = New(sentId, - options_->get("normalize"), - options_->get("word-penalty")); - histories.push_back(history); - } - - auto getNBestList = createGetNBestListFn(topN_, dimBatch, graph->getDeviceId()); - - Beams beams(dimBatch); // [batchIndex][beamIndex] is one sentence hypothesis - for(auto& beam : beams) - beam.resize(topN_, New()); - - bool first = true; - bool final = false; - - for(int i = 0; i < dimBatch; ++i) - histories[i]->Add(beams[i], trgEosId_); - - std::vector> states; - - for(auto scorer : scorers_) { - scorer->clear(graph); - } - - for(auto scorer : scorers_) { - states.push_back(scorer->apply(graph, batch)); - } - - // main loop over output tokens - do { - //********************************************************************** - // create constant containing previous path scores for current beam - // also create mapping of hyp indices, which are not 1:1 if sentences complete - std::vector hypIndices; // [beamIndex * activeBatchSize + batchIndex] backpointers, concatenated over beam positions. Used for reordering hypotheses - std::vector embIndices; - Expr prevPathScores; // [beam, 1, 1, 1] - if(first) { - // no scores yet - prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); - } else { - std::vector beamScores; - - dimBatch = (int)batch->size(); - - for(size_t i = 0; i < localBeamSize; ++i) { - for(size_t j = 0; j < beams.size(); ++j) { // loop over batch entries (active sentences) - auto& beam = beams[j]; - if(i < beam.size()) { - auto hyp = beam[i]; - hypIndices.push_back((IndexType)hyp->GetPrevStateIndex()); // backpointer - embIndices.push_back(hyp->GetWord()); - beamScores.push_back(hyp->GetPathScore()); - } else { // dummy hypothesis - hypIndices.push_back(0); - embIndices.push_back(0); // (unused) - beamScores.push_back(-9999); - } - } - } - - prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, - inits::from_vector(beamScores)); - } - - //********************************************************************** - // prepare scores for beam search - auto pathScores = prevPathScores; - - for(size_t i = 0; i < scorers_.size(); ++i) { - states[i] = scorers_[i]->step( - graph, states[i], hypIndices, embIndices, dimBatch, (int)localBeamSize); - - if(scorers_[i]->getWeight() != 1.f) - pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs(); - else - pathScores = pathScores + states[i]->getLogProbs(); - } - - // make beams continuous - if(dimBatch > 1 && localBeamSize > 1) - pathScores = transpose(pathScores, {2, 1, 0, 3}); // check if this is needed for classification, rather not, beamSize and topN is badly defined here - - if(first) - graph->forward(); - else - graph->forwardNext(); - - //********************************************************************** - // suppress specific symbols if not at right positions - if(trgUnkId_ != -1 && options_->has("allow-unk") - && !options_->get("allow-unk")) - suppressWord(pathScores, trgUnkId_); - for(auto state : states) - state->blacklist(pathScores, batch); - - //********************************************************************** - // perform beam search and pruning - std::vector outKeys; - std::vector outPathScores; - - std::vector beamSizes(dimBatch, localBeamSize); - getNBestList(beamSizes, pathScores->val(), outPathScores, outKeys, first); - - int dimTrgVoc = pathScores->shape()[-1]; - beams = toHyps(outKeys, - outPathScores, - dimTrgVoc, - beams, - states, - localBeamSize, - first, - batch); - - auto prunedBeams = pruneBeam(beams); - for(int i = 0; i < dimBatch; ++i) { - if(!beams[i].empty()) { - final = final - || histories[i]->size() - >= options_->get("max-length-factor") - * batch->front()->batchWidth(); - histories[i]->Add( - beams[i], trgEosId_, prunedBeams[i].empty() || final); - } - } - beams = prunedBeams; - - // determine beam size for next sentence, as max over still-active sentences - if(!first) { - size_t maxBeam = 0; - for(auto& beam : beams) - if(beam.size() > maxBeam) - maxBeam = beam.size(); - localBeamSize = maxBeam; - } - first = false; - - } while(localBeamSize != 0 && !final); // end of main loop over output tokens - - return histories; - } -}; -} // namespace marian From 49c139d950184c777d02417ed2586349d853e862 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 09:35:53 -0800 Subject: [PATCH 231/838] remove debug artifacts --- src/common/config_parser.cpp | 2 - src/graph/chainable.h | 2 - src/graph/expression_graph.cpp | 148 +----------------------------- src/graph/expression_graph.h | 81 ++++++++++++++-- src/graph/node.cpp | 19 ---- src/graph/node.h | 2 - src/training/graph_group_sync.cpp | 21 +---- 7 files changed, 82 insertions(+), 193 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 58fd4090e..3db06cc2b 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -71,8 +71,6 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { "Suppress logging for translation"); cli.add("--seed", "Seed for all random number generators. 0 means initialize randomly"); - cli.add("--check-nan", - "Check for NaNs or Infs in forward and backward pass. Will abort when found."); cli.add("--clip-gemm", "If not 0 clip GEMM input values to +/- arg"); cli.add("--interpolate-env-vars", diff --git a/src/graph/chainable.h b/src/graph/chainable.h index 3aa81d3cc..2679843ee 100644 --- a/src/graph/chainable.h +++ b/src/graph/chainable.h @@ -96,8 +96,6 @@ class Chainable { virtual const std::string& name() const = 0; virtual void debug(const std::string& message) = 0; - virtual void dump(const std::string& filename) = 0; - virtual bool marked_for_debug() = 0; virtual const std::string& debug_message() = 0; diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp index 5b4df3395..b3c237d1e 100644 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -24,153 +24,11 @@ Expr ExpressionGraph::dropout(float prob, const Shape& shape) { return constant(shape, inits::dropout(prob)); } -void ExpressionGraph::checkNan(Tensor t, bool& isNan, bool& isInf, bool zero) { - IsNan(t, allocator(), isNan, isInf, zero); +void ExpressionGraph::checkNan(Tensor t) { + ABORT_IF(throwNaN_, "Not implemented"); t; + // ABORT_IF(throwNaN_ && IsNan(t), "Tensor has NaN"); } -io::Item itemFromTensor(Tensor t, const std::string name, Ptr backend) { - io::Item item; - item.name = name; - item.shape = t->shape(); - item.type = t->type(); - - size_t bytesWithoutPadding = t->shape().elements() * sizeOf(t->type()); - item.bytes.resize(bytesWithoutPadding); - copy(backend, - (char*)t->data(), - (char*)t->data() + bytesWithoutPadding, - item.bytes.data()); - return item; -} - -void recChildren(Expr node, const std::string& parent, std::vector& items, Ptr backend) { - std::string name = node->type() + "_" + std::to_string(node->getId()) + "_p:" + parent; - items.push_back(itemFromTensor(node->val(), name, backend)); - for(auto&& child : node->children()) - recChildren(child, std::to_string(node->getId()), items, backend); -} - -void ExpressionGraph::forwardNext() { - // @TODO: check if allocation works properly - tensors_->clearShorttermMemory(); - - while(!nodesForward_.empty()) { - auto v = nodesForward_.front(); - v->allocate(); - v->init(); - v->forward(); - - if(v->trainable() && throwNan_) { - bool isNan = false, isInf = false; - checkNan(v->val(), isNan, isInf); - if(isNan || isInf) { - LOG(critical, "Detected NaN ({}) or Inf ({}) in value (forward pass)", isNan, isInf); - LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - v->type(), v->shape(), v->name(), v->getId(), v->hash()); - LOG(critical, "Value debug {}", v->val()->debug()); - - LOG(critical, "Children: {}", v->children().size()); - for(auto&& child : v->children()) { - LOG(critical, "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - child->type(), child->shape(), child->name(), child->getId(), child->hash()); - LOG(critical, "Value debug {}", child->val()->debug()); - } - - std::vector ioItems; - recChildren(v, "root", ioItems, backend_); - io::saveItems("dump-for-nans.npz", ioItems); - - ABORT("Aborting"); - } - } - - if(v->marked_for_debug()) { - LOG(info, "Debug: {} op={}", v->debug_message(), v->type()); - LOG(info, v->val()->debug()); - } - - if(inferenceOnly_) - v->children().clear(); - nodesForward_.pop_front(); - } -} - -void ExpressionGraph::backward(bool zero, float clipValue) { - if(topNodes_.size() > 1) { - LOG(critical, "There are more ({}) than one top most nodes for backward pass:", topNodes_.size()); - for(auto node : topNodes_) { - LOG(critical, - "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - node->type(), - node->shape(), - node->name(), - node->getId(), - node->hash()); - } - ABORT("Aborting"); - } - - params_->allocateBackward(); - if(zero) - params_->set_zero_adjoint(); - - for(auto&& v : topNodes_) - v->init_dependent(); - - // named_.clear(); - topNodes_.clear(); - - tensors_->clearShorttermMemory(); - - while(!nodesBackward_.empty()) { - auto v = nodesBackward_.back(); - nodesBackward_.pop_back(); - - for(auto&& child : v->children()) { - if(child->trainable() && child->type() != "param") - child->set_zero_adjoint(); - } - - if(v->trainable()) { - v->backward(); - if(clipValue != 0) { - using namespace functional; - Element(_1 = clip(_1, clipValue), v->grad()); - } - } - - - if(throwNan_) { - for(auto&& child : v->children()) { - if(child->trainable()) { - bool isNan = false, isInf = false; - checkNan(child->grad(), isNan, isInf); - if(isNan || isInf) { - LOG(critical, "Detected NaN ({}) or Inf ({}) in gradient (backward pass) of child node", isNan, isInf); - LOG(critical, "Child - Type: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - child->type(), child->shape(), child->name(), child->getId(), child->hash()); - LOG(critical, "Value debug: {}", child->val()->debug()); - LOG(critical, "Grad debug: {}", child->grad()->debug()); - LOG(critical, "Parent - Type: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", - v->type(), v->shape(), v->name(), v->getId(), v->hash()); - LOG(critical, "Value debug: {}", v->val()->debug()); - LOG(critical, "Grad debug: {}", v->grad()->debug()); - ABORT("Aborting"); - } - } - } - } - - if(v->trainable() && v->marked_for_debug()) { - LOG(info, "Debug Grad: {} op={}", v->debug_message(), v->type()); - LOG(info, v->grad()->debug()); - } - - v->children().clear(); - } -} - - void ExpressionGraph::save(std::vector& ioItems) { for(auto p : params()->getMap()) { std::string pName = p.first; diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index 389c6e3e4..fe8361618 100644 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -136,7 +136,7 @@ class ExpressionGraph : public std::enable_shared_from_this { bool reloaded_{false}; std::string namespace_; - bool throwNan_{false}; + bool throwNaN_{false}; protected: // Delete, copy and move constructors @@ -217,11 +217,81 @@ class ExpressionGraph : public std::enable_shared_from_this { forwardNext(); } - void checkNan(Tensor t, bool& isNan, bool& isInf, bool zero = false); + void checkNan(Tensor t); - void forwardNext(); + void forwardNext() { + // @TODO: check if allocation works properly + tensors_->clearShorttermMemory(); - void backward(bool zero = true, float clipValue = 0.f); + while(!nodesForward_.empty()) { + auto v = nodesForward_.front(); + v->allocate(); + v->init(); + v->forward(); + + checkNan(v->val()); + + if(v->marked_for_debug()) { + std::cerr << "Debug: " << v->debug_message() << " op=" << v->type() + << std::endl; + std::cerr << v->val()->debug() << std::endl; + } + + if(inferenceOnly_) + v->children().clear(); + nodesForward_.pop_front(); + } + } + + void backward(bool zero = true) { + if(topNodes_.size() > 1) { + LOG(critical, "There are more ({}) than one top most node for backward step:", topNodes_.size()); + for(auto node : topNodes_) { + LOG(critical, + "\tType: {}, Shape: {}, Name: {}, Id: {}, Hash: {}", + node->type(), + node->shape(), + node->name(), + node->getId(), + node->hash()); + } + ABORT("Aborting"); + } + + params_->allocateBackward(); + if(zero) + params_->set_zero_adjoint(); + + for(auto&& v : topNodes_) + v->init_dependent(); + + // named_.clear(); + topNodes_.clear(); + + tensors_->clearShorttermMemory(); + + while(!nodesBackward_.empty()) { + auto v = nodesBackward_.back(); + nodesBackward_.pop_back(); + + for(auto&& child : v->children()) { + if(child->trainable() && child->type() != "param") + child->set_zero_adjoint(); + } + + if(v->trainable()) + v->backward(); + + checkNan(v->grad()); + + if(v->trainable() && v->marked_for_debug()) { + std::cerr << "Debug Grad: " << v->debug_message() << std::endl; + std::cerr << v->grad()->debug() << std::endl; + } + + v->children().clear(); + } + } std::string graphviz() { std::stringstream ss; @@ -390,8 +460,7 @@ class ExpressionGraph : public std::enable_shared_from_this { void setReloaded(bool reloaded) { reloaded_ = reloaded; } - void setThrowNan(bool throwNan) { throwNan_ = throwNan; } - bool getThrowNan() { return throwNan_; } + void setThrowNaN(bool throwNaN) { throwNaN_ = throwNaN; } public: // convert all parameters into an array of IoItem elements, for loading diff --git a/src/graph/node.cpp b/src/graph/node.cpp index bdb501166..c11531da7 100644 --- a/src/graph/node.cpp +++ b/src/graph/node.cpp @@ -2,8 +2,6 @@ #include "graph/auto_tuner.h" #include "graph/expression_graph.h" #include "tensors/backend.h" -#include "tensors/tensor_operators.h" -#include "common/io.h" namespace marian { @@ -85,21 +83,4 @@ void Node::record(Ptr recorder, recorderHash_ = recorderHash; recorderStop_ = stop; } - -void Node::dump(const std::string& filename) { - io::Item item; - item.name = "dump"; - item.shape = val_->shape(); - item.type = val_->type(); - - size_t bytesWithoutPadding = val_->shape().elements() * sizeOf(val_->type()); - item.bytes.resize(bytesWithoutPadding); - copy(graph()->getBackend(), - (char*)val_->data(), - (char*)val_->data() + bytesWithoutPadding, - item.bytes.data()); - - std::vector items({item}); - io::saveItems(filename, items); -} } // namespace marian diff --git a/src/graph/node.h b/src/graph/node.h index defefd5b5..1397e74b0 100644 --- a/src/graph/node.h +++ b/src/graph/node.h @@ -100,8 +100,6 @@ class Node : public Chainable, virtual bool marked_for_debug() override { return markedForDebug_; } virtual const std::string& debug_message() override { return debugMessage_; } - virtual void dump(const std::string& filename) override; - virtual size_t allocate() override; virtual void free() override; diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index c26cb9ff5..1ec5c3d68 100644 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -10,10 +10,6 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) for(auto device : devices_) { auto graph = New(); graph->setDevice(device); - - if(options_->get("check-nan")) - graph->setThrowNan(true); - graph->reserveWorkspaceMB(options_->get("workspace")); graph->getBackend()->setClip(options_->get("clip-gemm")); @@ -360,18 +356,12 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num auto rationalLoss = builders_[localDeviceIndex]->build(graph, subBatch); graph->forward(); - + StaticLoss tempLoss = *rationalLoss; // needed for overstuff - tempLoss.loss /= (float)overstuff; + tempLoss.loss /= (float)overstuff; // @TODO: @fseide: scale only loss? should this scale labels too? localDeviceLosses[localDeviceIndex] += tempLoss; graph->backward(/*zero=*/false); // (gradients are reset before we get here) - - bool hasNan = false, hasInf = false; - IsNan(graph->params()->grads(), graph->allocator(), hasNan, hasInf, /*zero=*/true); - if(hasNan || hasInf) { - LOG(warn, "Seen Nan ({}) or Inf ({}) in gradient, zeroed out offending gradient", hasNan, hasInf); - } } }); // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches. @@ -410,10 +400,7 @@ void SyncGraphGroup::update(std::vector> subBatches, size_t num // cost across all local devices (scheduler will aggregate cross-process) StaticLoss localLoss; for(auto& l : localDeviceLosses) // localDeviceLosses is already summed up over delay steps - if(std::isfinite((float)l.loss)) - localLoss += l; - else - LOG(warn, "Seen non-finite loss, offending gradients have been zeroed out"); + localLoss += l; if(scheduler_) { // track and log localLoss @@ -534,7 +521,7 @@ void SyncGraphGroup::save(bool final) /*override*/ { return comm_->gatherState(getShardFn); }, isMainProcess()); - + barrier(); // (for better grouping of log messages) } From 2d70ecd787cb3a29ab97809d4ccc7444312e1d60 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 10:17:18 -0800 Subject: [PATCH 232/838] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98e89ff77..c409a9d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fixed handling of "dump-config: false" in YAML config - Errors due to warnings - Fixed issue concerning failed saving with single GPU training and --sync-sgd option. +- Fixed NaN problem when training with Tensor Cores on Volta GPUs ### Changed - Add zlib source to Marian's source tree, builds now as object lib From 095338af8d4f37d1443415f8517411be2598149b Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 29 Jan 2019 10:19:06 -0800 Subject: [PATCH 233/838] duplicate vocab entry handling --- src/data/default_vocab.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index f5d009b2a..38c0c323d 100644 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -107,7 +107,9 @@ class DefaultVocab : public VocabBase { ABORT_IF(line.empty(), "DefaultVocabulary file {} must not contain empty lines", vocabPath); - vocab.insert({line, (Word)vocab.size()}); + auto wasInserted = vocab.insert({line, (Word)vocab.size()}).second; + ABORT_IF(!wasInserted, "Duplicate vocabulary entry {}", line); + } ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath); } From 6068674029e39d95c89c3d6c23d31f99fb9122cf Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 29 Jan 2019 15:40:21 -0800 Subject: [PATCH 234/838] minor cleanup of factored outputs; renamed toUpper() to utf8ToUpper(), and it now handles UTF8 correctly --- src/common/config_parser.cpp | 2 ++ src/common/utils.cpp | 26 +++++++++++++++++----- src/common/utils.h | 2 +- src/data/batch_generator.h | 6 ++--- src/data/corpus.cpp | 10 +++++++-- src/data/corpus.h | 4 ++++ src/layers/generic.cpp | 37 ++++++++++++++++++------------- src/layers/loss.cpp | 12 +++++----- src/training/graph_group_sync.cpp | 2 +- 9 files changed, 67 insertions(+), 34 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 776042858..777481991 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -615,6 +615,8 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { cli.add("--shuffle-in-ram", "Keep shuffled corpus in RAM, do not write to temp file"); + cli.add("--all-caps-every", + "When forming minibatches, preprocess every Nth line on the fly to all-caps. Assumes UTF-8"); cli.add("--mini-batch-words-ref", "If given, the following hyper parameters are adjusted as-if we had this mini-batch size: " diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 252afa547..73c719e25 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -10,6 +10,8 @@ #ifdef __unix__ #include #endif +#include +#include namespace marian { namespace utils { @@ -154,12 +156,24 @@ bool endsWith(const std::string& text, const std::string& suffix) { && !text.compare(text.size() - suffix.size(), suffix.size(), suffix); } -std::string toUpper(const std::string& s) { - std::locale loc; - std::string res; res.reserve(s.capacity()); - for (auto c : s) // @BUGBUG: This won't work with UTF-8 characters. - res.push_back((char)std::toupper(c, loc)); - return res; +static std::wstring utf8ToWString(std::string const& s) { + std::wstring_convert> converter; + return converter.from_bytes(s); +} + +static std::string toUTF8String(std::wstring const& s) { + std::wstring_convert> converter; + return converter.to_bytes(s); +} + +std::locale const utf8("en_US.UTF-8"); + +// convert a UTF-8 string to all-caps +std::string utf8ToUpper(const std::string& s) { + auto ws = utf8ToWString(s); + for (auto& c : ws) + c = std::toupper(c, utf8); + return toUTF8String(ws); } double parseDouble(std::string s) { diff --git a/src/common/utils.h b/src/common/utils.h index d76d07fa7..777c97763 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -39,7 +39,7 @@ std::string withCommas(size_t n); bool beginsWith(const std::string& text, const std::string& prefix); bool endsWith(const std::string& text, const std::string& suffix); -std::string toUpper(const std::string& s); +std::string utf8ToUpper(const std::string& s); double parseDouble(std::string s); double parseNumber(std::string s); diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h index ce673cd2a..4ec03d4d4 100755 --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -130,9 +130,9 @@ class BatchGenerator : public RNGEngine { while(current_ != data_->end() && maxiBatch->size() < maxSize) { // loop over data maxiBatch->push(*current_); sets = current_->size(); - // do not consume more than required for the maxi batch as this causes - // that line-by-line translation is delayed by one sentence - bool last = maxiBatch->size() == maxSize; + // do not consume more than required for the maxi batch as this causes + // that line-by-line translation is delayed by one sentence + bool last = maxiBatch->size() == maxSize; if(!last) ++current_; // this actually reads the next line and pre-processes it } diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7f51b12ba..158635b15 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -10,12 +10,17 @@ namespace marian { namespace data { Corpus::Corpus(Ptr options, bool translate /*= false*/) - : CorpusBase(options, translate), shuffleInRAM_(options_->get("shuffle-in-ram")) {} + : CorpusBase(options, translate), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} Corpus::Corpus(std::vector paths, std::vector> vocabs, Ptr options) - : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")) {} + : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every") {} + +void Corpus::preprocessLine(std::string& line, size_t streamId) { + if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) + line = utils::toAllCapsUTF8(line); +} SentenceTuple Corpus::next() { for (;;) { // (this is a retry loop for skipping invalid sentences) @@ -55,6 +60,7 @@ SentenceTuple Corpus::next() { } else if(i > 0 && i == weightFileIdx_) { addWeightsToSentenceTuple(line, tup); } else { + preprocessLine(line, i); addWordsToSentenceTuple(line, i, tup); } } diff --git a/src/data/corpus.h b/src/data/corpus.h index 59d82ef55..3416f5b47 100755 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -27,6 +27,10 @@ class Corpus : public CorpusBase { void shuffleData(const std::vector& paths); + // for pre-processing + size_t allCapsEvery_{0}; + void preprocessLine(std::string& line, size_t streamId); + public: // @TODO: check if translate can be replaced by an option in options Corpus(Ptr options, bool translate = false); diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 4b43f8e85..2ff6d34cc 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -107,6 +107,13 @@ namespace marian { factorMasks_[g][v] = 1.0f; } } + //for (Word v = 0; v < vocabSize; v++) { + // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], + // factorMasks_[0][v], factorIndices_[0][v], + // factorMasks_[1][v], factorIndices_[1][v], + // factorMasks_[2][v], factorIndices_[2][v], + // factorMasks_[3][v], factorIndices_[3][v]); + //} //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members ({})", @@ -193,7 +200,9 @@ namespace marian { // @TODO: no need to normalize factors before here // For each location in [B...] select [indices[B...]]. If not using factor, select [0] and mask it out next. auto factorLoss = lossFn(factorLogits, factorIndex); // [B... x 1] - factorLoss = factorLoss * reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor + auto xxx = reshape(factorMask, factorLoss->shape()); + //factorLoss->debug("factorLoss"); + factorLoss = factorLoss * xxx;// reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor loss = loss ? (loss + factorLoss) : factorLoss; // [B... x 1] } return loss; @@ -293,28 +302,24 @@ namespace marian { // project each factor auto numGroups = embeddingFactorMapping_->getNumGroups(); - std::vector groupYs(numGroups); - std::vector groupLLWeights(numGroups); + std::vector allLogits(numGroups); for (size_t g = 0; g < numGroups; g++) { auto range = embeddingFactorMapping_->getGroupRange(g); ABORT_IF(g > 0 && range.first != embeddingFactorMapping_->getGroupRange(g-1).second, "Factor groups must be consecutive"); // we could sort groupYs though // slice this group's section out of W_ // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. - auto groupW = slice(W_, transposeW_ ? 0 : -1, Slice((int)range.first, (int)range.second)); - //LOG(info, "slice() -> {}, {}", groupW->type(), std::string(groupW->shape())); - auto groupB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix - auto groupY = affine(input, groupW, groupB, false, transposeW_); // [B... x U] factor logits - // normalize - groupY = logsoftmax(groupY); - // log-linear weight --@TODO: pre-create in constructor + auto factorW = slice(W_, transposeW_ ? 0 : -1, Slice((int)range.first, (int)range.second)); + auto factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix + auto factorLogits = affine(input, factorW, factorB, false, transposeW_); // [B... x U] factor logits + // log-linear weight + // @TODO: The weight should not be needed, since it could learn it into the embeddings. Maybe it doesn't. + // @TODO? If we move the weight before affine(), it would use less memory at least for the main factor. auto name = options_->get("prefix"); - groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - groupY = groupY * groupLLWeights[g]; - // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. - // @TODO: normalize again. Do I need the first normalization? - groupYs[g] = groupY; + auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + factorLogits = factorLogits * llWeight; // -a constant, which is OK for logits + allLogits[g] = factorLogits; } - return Logits(std::move(groupYs), embeddingFactorMapping_); + return Logits(std::move(allLogits), embeddingFactorMapping_); } else return affine(input, W_, b_, false, transposeW_); diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 3493e7c85..5c03ea5c1 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -26,16 +26,18 @@ Expr LossBase::getCrossEntropy(const Logits& logits, Expr indices, Expr mask, Expr weights) { + // logits may be factored; in that case, the getLoss() function computes one loss for each, and sums them up auto ce = logits.getLoss(indices, [&](Expr logits, Expr indices) { Expr ce = cross_entropy(logits, indices); if (smoothing_ > 0) { - // ce = sum_i y^_i log y_i(z)_i + // ce = -sum_i y^_i log y_i(z) // with smoothing: - // ce' = sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z)_i - // = (1-smoothing_) sum_i y^_i log y_i(z)_i + smoothing_ mean_i log y_i(z)_i - // = (1-smoothing_) ce + smoothing_ mean_i log y_i(z)_i + // ce' = -sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z) + // = -(1-smoothing_) sum_i y^_i log y_i(z) - smoothing_ mean_i log y_i(z) + // = (1-smoothing_) ce - smoothing_ mean_i log y_i(z) auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); - ce = (1 - smoothing_) * ce - smoothing_ * ceq; + //ce = (1 - smoothing_) * ce - smoothing_ * ceq; + ce = ce - smoothing_ * (ce + ceq); // writing it this way saves one op :) } return ce; }); diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index 85fd6d978..a4d616165 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -23,7 +23,7 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) // Rather, it is assumed that the communicator knows to reduce unnecessary transfers to no-ops. comm_ = createCommunicator(graphs_, /*noNccl=*/options_->get("no-nccl", false), /*mpi=*/mpi_); - auto type = utils::toUpper(devices_.front().typeAsString()) + "s"; + auto type = utils::utf8ToUpper(devices_.front().typeAsString()) + "s"; if (mpi_->numMPIProcesses() > 1) LOG(info, "[training] Using {} {}, distributed over {} MPI processes", mpi_->numMPIProcesses() * devices_.size(), type, mpi_->numMPIProcesses()); else From 1f7967b4b08ee28453bf7a4ace3e2acf68f789c3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 29 Jan 2019 15:51:17 -0800 Subject: [PATCH 235/838] fixed minor glitch in previous commit --- src/data/corpus.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 158635b15..d83067a60 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -15,11 +15,11 @@ Corpus::Corpus(Ptr options, bool translate /*= false*/) Corpus::Corpus(std::vector paths, std::vector> vocabs, Ptr options) - : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every") {} + : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} -void Corpus::preprocessLine(std::string& line, size_t streamId) { +void Corpus::preprocessLine(std::string& line, size_t /*streamId*/) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) - line = utils::toAllCapsUTF8(line); + line = utils::utf8ToUpper(line); } SentenceTuple Corpus::next() { From 86a212bf787d87704b3a50c22bf68a0b0ffc3570 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 29 Jan 2019 16:00:47 -0800 Subject: [PATCH 236/838] reverted that CE refactoring, too risky right now --- src/data/corpus.cpp | 4 +++- src/layers/loss.cpp | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index d83067a60..bd5711b24 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -18,8 +18,10 @@ Corpus::Corpus(std::vector paths, : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} void Corpus::preprocessLine(std::string& line, size_t /*streamId*/) { - if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) + if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) { line = utils::utf8ToUpper(line); + LOG_ONCE(info, "all-caps'ed line to {}", line); + } } SentenceTuple Corpus::next() { diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 5c03ea5c1..ecd92e793 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -30,14 +30,14 @@ Expr LossBase::getCrossEntropy(const Logits& logits, auto ce = logits.getLoss(indices, [&](Expr logits, Expr indices) { Expr ce = cross_entropy(logits, indices); if (smoothing_ > 0) { - // ce = -sum_i y^_i log y_i(z) + // ce = -sum_i y^_i log y_i(h) // with smoothing: - // ce' = -sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(z) - // = -(1-smoothing_) sum_i y^_i log y_i(z) - smoothing_ mean_i log y_i(z) - // = (1-smoothing_) ce - smoothing_ mean_i log y_i(z) - auto ceq = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); - //ce = (1 - smoothing_) * ce - smoothing_ * ceq; - ce = ce - smoothing_ * (ce + ceq); // writing it this way saves one op :) + // ce' = -sum_i ((1-smoothing_) y^_i + smoothing_/N) log y_i(h) + // = -(1-smoothing_) sum_i y^_i log y_i(h) - smoothing_ mean_i log y_i(h) + // = (1-smoothing_) ce - smoothing_ mean_i log y_i(h) + auto ceqNeg = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + ce = (1 - smoothing_) * ce - smoothing_ * ceqNeg; + //ce = ce - smoothing_ * (ce + ceqNeg); // writing it this way saves one op :) } return ce; }); From 37d5d3a1ce05b3a43acd459bfb68bb2cb039c880 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 29 Jan 2019 16:10:29 -0800 Subject: [PATCH 237/838] made locale a local variable --- src/common/utils.cpp | 3 +-- src/data/corpus.cpp | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 73c719e25..3914452b4 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -166,10 +166,9 @@ static std::string toUTF8String(std::wstring const& s) { return converter.to_bytes(s); } -std::locale const utf8("en_US.UTF-8"); - // convert a UTF-8 string to all-caps std::string utf8ToUpper(const std::string& s) { + std::locale const utf8("en_US.UTF-8"); auto ws = utf8ToWString(s); for (auto& c : ws) c = std::toupper(c, utf8); diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index bd5711b24..b39a7fa5a 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -10,12 +10,12 @@ namespace marian { namespace data { Corpus::Corpus(Ptr options, bool translate /*= false*/) - : CorpusBase(options, translate), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} + : CorpusBase(options, translate), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} Corpus::Corpus(std::vector paths, std::vector> vocabs, Ptr options) - : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} + : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} void Corpus::preprocessLine(std::string& line, size_t /*streamId*/) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) { From f46ffcb002f7b148a05d19cd15ab41adb5e4b1fb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 30 Jan 2019 08:08:32 -0800 Subject: [PATCH 238/838] utf8ToUpper() now works for a subset of characters without locales installed; added missing __syncthreads() to softmax variants --- src/common/utils.cpp | 46 +++++++++++++++++++++++++++-- src/data/corpus.cpp | 7 +++-- src/tensors/gpu/tensor_operators.cu | 4 +++ 3 files changed, 52 insertions(+), 5 deletions(-) mode change 100644 => 100755 src/tensors/gpu/tensor_operators.cu diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 3914452b4..59cc4cc88 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -11,7 +11,7 @@ #include #endif #include -#include +#include namespace marian { namespace utils { @@ -167,13 +167,53 @@ static std::string toUTF8String(std::wstring const& s) { } // convert a UTF-8 string to all-caps +#if 0 +// @BUGBUG: This does not work for non-ASCII characters. std::string utf8ToUpper(const std::string& s) { - std::locale const utf8("en_US.UTF-8"); auto ws = utf8ToWString(s); for (auto& c : ws) - c = std::toupper(c, utf8); + c = std::towupper(c); return toUTF8String(ws); } +#else +struct UTF8Mapper : std::map { // Hack because Philly does not have UTF-8 locale installed + UTF8Mapper() { + /* + env LC_ALL=en_US.UTF-8 sed 's/\(.\)/\1\n/g' TEXT_FILE_CONTAINING_ALL_CHARS > l + env LC_ALL=en_US.UTF-8 sed 's/\(.\)/\U\1\n/g' TEXT_FILE_CONTAINING_ALL_CHARS > u + paste l u | env LC_ALL=en_US.UTF-8 sort -u > x + cat x | awk '{if($1 != $2){print}}' > y + cat y | tr -d '\r' \ + | od -w10000 -t x1 \ + | head -1 \ + | sed -e 's/^0000000 /{{".x/g' -e 's/ 09 /",".x/g' -e 's/ 0a /"},{".x/g' -e 's/ 0a$/"}/' -e 's/ /.x/g' \ + | tr '.' '\\' \ + | xclip + */ + std::vector> map8{ {"\xc9\x92","\xe2\xb1\xb0"},{"\x61","\x41"},{"\xc3\xa1","\xc3\x81"},{"\xc3\xa0","\xc3\x80"},{"\xe1\xba\xaf","\xe1\xba\xae"},{"\xe1\xba\xb1","\xe1\xba\xb0"},{"\xe1\xba\xb5","\xe1\xba\xb4"},{"\xe1\xba\xb3","\xe1\xba\xb2"},{"\xe1\xba\xb7","\xe1\xba\xb6"},{"\xc4\x83","\xc4\x82"},{"\xe1\xba\xa5","\xe1\xba\xa4"},{"\xe1\xba\xa7","\xe1\xba\xa6"},{"\xe1\xba\xab","\xe1\xba\xaa"},{"\xe1\xba\xa9","\xe1\xba\xa8"},{"\xe1\xba\xad","\xe1\xba\xac"},{"\xc3\xa2","\xc3\x82"},{"\xc7\x8e","\xc7\x8d"},{"\xc7\xbb","\xc7\xba"},{"\xc3\xa5","\xc3\x85"},{"\xc7\x9f","\xc7\x9e"},{"\xc3\xa4","\xc3\x84"},{"\xc3\xa3","\xc3\x83"},{"\xc4\x85","\xc4\x84"},{"\xc4\x81","\xc4\x80"},{"\xe1\xba\xa3","\xe1\xba\xa2"},{"\xc8\x83","\xc8\x82"},{"\xe1\xba\xa1","\xe1\xba\xa0"},{"\xc7\xa3","\xc7\xa2"},{"\xc3\xa6","\xc3\x86"},{"\x62","\x42"},{"\xe1\xb8\x87","\xe1\xb8\x86"},{"\x63","\x43"},{"\xc4\x87","\xc4\x86"},{"\xc4\x89","\xc4\x88"},{"\xc4\x8d","\xc4\x8c"},{"\xc4\x8b","\xc4\x8a"},{"\xc3\xa7","\xc3\x87"},{"\x64","\x44"},{"\xc4\x8f","\xc4\x8e"},{"\xc4\x91","\xc4\x90"},{"\xe1\xb8\x91","\xe1\xb8\x90"},{"\xe1\xb8\x8d","\xe1\xb8\x8c"},{"\xe1\xb8\x8f","\xe1\xb8\x8e"},{"\xc3\xb0","\xc3\x90"},{"\x65","\x45"},{"\xc3\xa9","\xc3\x89"},{"\xc3\xa8","\xc3\x88"},{"\xc4\x95","\xc4\x94"},{"\xe1\xba\xbf","\xe1\xba\xbe"},{"\xe1\xbb\x81","\xe1\xbb\x80"},{"\xe1\xbb\x85","\xe1\xbb\x84"},{"\xe1\xbb\x83","\xe1\xbb\x82"},{"\xe1\xbb\x87","\xe1\xbb\x86"},{"\xc3\xaa","\xc3\x8a"},{"\xc4\x9b","\xc4\x9a"},{"\xc3\xab","\xc3\x8b"},{"\xe1\xba\xbd","\xe1\xba\xbc"},{"\xc4\x97","\xc4\x96"},{"\xc4\x99","\xc4\x98"},{"\xe1\xb8\x97","\xe1\xb8\x96"},{"\xc4\x93","\xc4\x92"},{"\xe1\xba\xbb","\xe1\xba\xba"},{"\xc8\x87","\xc8\x86"},{"\xe1\xba\xb9","\xe1\xba\xb8"},{"\xc7\x9d","\xc6\x8e"},{"\x66","\x46"},{"\x67","\x47"},{"\xc7\xb5","\xc7\xb4"},{"\xc4\x9f","\xc4\x9e"},{"\xc4\x9d","\xc4\x9c"},{"\xc7\xa7","\xc7\xa6"},{"\xc4\xa1","\xc4\xa0"},{"\xc4\xa3","\xc4\xa2"},{"\xc9\xa0","\xc6\x93"},{"\x68","\x48"},{"\xc4\xa5","\xc4\xa4"},{"\xc4\xa7","\xc4\xa6"},{"\xe1\xb8\xa9","\xe1\xb8\xa8"},{"\xe1\xb8\xa5","\xe1\xb8\xa4"},{"\xe1\xb8\xab","\xe1\xb8\xaa"},{"\x69","\x49"},{"\xc4\xb1","\x49"},{"\xc3\xad","\xc3\x8d"},{"\xc3\xac","\xc3\x8c"},{"\xc4\xad","\xc4\xac"},{"\xc3\xae","\xc3\x8e"},{"\xc7\x90","\xc7\x8f"},{"\xc3\xaf","\xc3\x8f"},{"\xc4\xa9","\xc4\xa8"},{"\xc4\xaf","\xc4\xae"},{"\xc4\xab","\xc4\xaa"},{"\xe1\xbb\x89","\xe1\xbb\x88"},{"\xc8\x8b","\xc8\x8a"},{"\xe1\xbb\x8b","\xe1\xbb\x8a"},{"\x6a","\x4a"},{"\xc4\xb5","\xc4\xb4"},{"\x6b","\x4b"},{"\xe1\xb8\xb1","\xe1\xb8\xb0"},{"\xc4\xb7","\xc4\xb6"},{"\xe1\xb8\xb3","\xe1\xb8\xb2"},{"\xc6\x99","\xc6\x98"},{"\x6c","\x4c"},{"\xc4\xba","\xc4\xb9"},{"\xc4\xbe","\xc4\xbd"},{"\xc5\x82","\xc5\x81"},{"\xc4\xbc","\xc4\xbb"},{"\xe1\xb8\xb7","\xe1\xb8\xb6"},{"\x6d","\x4d"},{"\xe1\xb8\xbf","\xe1\xb8\xbe"},{"\xe1\xb9\x83","\xe1\xb9\x82"},{"\xc5\x8b","\xc5\x8a"},{"\x6e","\x4e"},{"\xc5\x84","\xc5\x83"},{"\xc5\x88","\xc5\x87"},{"\xc3\xb1","\xc3\x91"},{"\xe1\xb9\x85","\xe1\xb9\x84"},{"\xc5\x86","\xc5\x85"},{"\xe1\xb9\x87","\xe1\xb9\x86"},{"\xe1\xb9\x89","\xe1\xb9\x88"},{"\xc5\x93","\xc5\x92"},{"\x6f","\x4f"},{"\xc3\xb3","\xc3\x93"},{"\xc3\xb2","\xc3\x92"},{"\xc5\x8f","\xc5\x8e"},{"\xe1\xbb\x91","\xe1\xbb\x90"},{"\xe1\xbb\x93","\xe1\xbb\x92"},{"\xe1\xbb\x95","\xe1\xbb\x94"},{"\xe1\xbb\x99","\xe1\xbb\x98"},{"\xc3\xb4","\xc3\x94"},{"\xc7\x92","\xc7\x91"},{"\xc3\xb6","\xc3\x96"},{"\xc5\x91","\xc5\x90"},{"\xc3\xb5","\xc3\x95"},{"\xc3\xb8","\xc3\x98"},{"\xc7\xab","\xc7\xaa"},{"\xc5\x8d","\xc5\x8c"},{"\xe1\xbb\x8f","\xe1\xbb\x8e"},{"\xc8\x8f","\xc8\x8e"},{"\xe1\xbb\x8d","\xe1\xbb\x8c"},{"\xe1\xbb\x9b","\xe1\xbb\x9a"},{"\xe1\xbb\x9d","\xe1\xbb\x9c"},{"\xe1\xbb\xa1","\xe1\xbb\xa0"},{"\xe1\xbb\x9f","\xe1\xbb\x9e"},{"\xe1\xbb\xa3","\xe1\xbb\xa2"},{"\xc6\xa1","\xc6\xa0"},{"\xc9\x94","\xc6\x86"},{"\x70","\x50"},{"\xe1\xb9\x95","\xe1\xb9\x94"},{"\x71","\x51"},{"\x72","\x52"},{"\xc5\x95","\xc5\x94"},{"\xc5\x99","\xc5\x98"},{"\xc5\x97","\xc5\x96"},{"\xe1\xb9\x9b","\xe1\xb9\x9a"},{"\xe1\xb9\x9f","\xe1\xb9\x9e"},{"\x73","\x53"},{"\xc5\x9b","\xc5\x9a"},{"\xc5\x9d","\xc5\x9c"},{"\xc5\xa1","\xc5\xa0"},{"\xc5\x9f","\xc5\x9e"},{"\xe1\xb9\xa3","\xe1\xb9\xa2"},{"\x74","\x54"},{"\xc5\xa5","\xc5\xa4"},{"\xc5\xa3","\xc5\xa2"},{"\xe1\xb9\xad","\xe1\xb9\xac"},{"\xe1\xb9\xaf","\xe1\xb9\xae"},{"\xc8\x95","\xc8\x94"},{"\x75","\x55"},{"\xc3\xba","\xc3\x9a"},{"\xc3\xb9","\xc3\x99"},{"\xc5\xad","\xc5\xac"},{"\xc3\xbb","\xc3\x9b"},{"\xc7\x94","\xc7\x93"},{"\xc5\xaf","\xc5\xae"},{"\xc7\x98","\xc7\x97"},{"\xc7\x9c","\xc7\x9b"},{"\xc3\xbc","\xc3\x9c"},{"\xc5\xb1","\xc5\xb0"},{"\xc5\xa9","\xc5\xa8"},{"\xc5\xb3","\xc5\xb2"},{"\xc5\xab","\xc5\xaa"},{"\xe1\xbb\xa7","\xe1\xbb\xa6"},{"\xe1\xbb\xa5","\xe1\xbb\xa4"},{"\xe1\xb9\xb3","\xe1\xb9\xb2"},{"\xe1\xbb\xa9","\xe1\xbb\xa8"},{"\xe1\xbb\xab","\xe1\xbb\xaa"},{"\xe1\xbb\xaf","\xe1\xbb\xae"},{"\xe1\xbb\xad","\xe1\xbb\xac"},{"\xe1\xbb\xb1","\xe1\xbb\xb0"},{"\xc6\xb0","\xc6\xaf"},{"\x76","\x56"},{"\x77","\x57"},{"\xc5\xb5","\xc5\xb4"},{"\x78","\x58"},{"\xe1\xba\x8b","\xe1\xba\x8a"},{"\x79","\x59"},{"\xc3\xbd","\xc3\x9d"},{"\xe1\xbb\xb3","\xe1\xbb\xb2"},{"\xc5\xb7","\xc5\xb6"},{"\xc3\xbf","\xc5\xb8"},{"\xe1\xbb\xb9","\xe1\xbb\xb8"},{"\x7a","\x5a"},{"\xc5\xba","\xc5\xb9"},{"\xc5\xbe","\xc5\xbd"},{"\xc5\xbc","\xc5\xbb"},{"\xc6\xb6","\xc6\xb5"},{"\xe1\xba\x93","\xe1\xba\x92"},{"\xe1\xba\x95","\xe1\xba\x94"},{"\xc8\xa5","\xc8\xa4"},{"\xc3\xbe","\xc3\x9e"},{"\xca\x92","\xc6\xb7"},{"\xce\xb1","\xce\x91"},{"\xce\xac","\xce\x86"},{"\xce\xb2","\xce\x92"},{"\xce\xb3","\xce\x93"},{"\xce\xb4","\xce\x94"},{"\xce\xb5","\xce\x95"},{"\xce\xad","\xce\x88"},{"\xce\xb6","\xce\x96"},{"\xce\xb7","\xce\x97"},{"\xce\xae","\xce\x89"},{"\xce\xb8","\xce\x98"},{"\xce\xb9","\xce\x99"},{"\xce\xaf","\xce\x8a"},{"\xcf\x8a","\xce\xaa"},{"\xce\xba","\xce\x9a"},{"\xce\xbb","\xce\x9b"},{"\xce\xbc","\xce\x9c"},{"\xce\xbd","\xce\x9d"},{"\xce\xbe","\xce\x9e"},{"\xce\xbf","\xce\x9f"},{"\xcf\x8c","\xce\x8c"},{"\xcf\x80","\xce\xa0"},{"\xcf\x83","\xce\xa3"},{"\xcf\x82","\xce\xa3"},{"\xcf\x84","\xce\xa4"},{"\xcf\x85","\xce\xa5"},{"\xcf\x8d","\xce\x8e"},{"\xcf\x8b","\xce\xab"},{"\xcf\x86","\xce\xa6"},{"\xcf\x87","\xce\xa7"},{"\xcf\x88","\xce\xa8"},{"\xcf\x89","\xce\xa9"},{"\xcf\x8e","\xce\x8f"},{"\xd0\xb0","\xd0\x90"},{"\xd3\x93","\xd3\x92"},{"\xd3\x95","\xd3\x94"},{"\xd0\xb1","\xd0\x91"},{"\xd0\xb2","\xd0\x92"},{"\xd0\xb3","\xd0\x93"},{"\xd2\x93","\xd2\x92"},{"\xd2\x91","\xd2\x90"},{"\xd0\xb4","\xd0\x94"},{"\xd1\x93","\xd0\x83"},{"\xd1\x92","\xd0\x82"},{"\xd0\xb5","\xd0\x95"},{"\xd1\x90","\xd0\x80"},{"\xd3\x99","\xd3\x98"},{"\xd1\x94","\xd0\x84"},{"\xd1\x91","\xd0\x81"},{"\xd0\xb6","\xd0\x96"},{"\xd0\xb7","\xd0\x97"},{"\xd2\x99","\xd2\x98"},{"\xd1\x95","\xd0\x85"},{"\xd0\xb8","\xd0\x98"},{"\xd3\xa3","\xd3\xa2"},{"\xd1\x96","\xd0\x86"},{"\xd1\x97","\xd0\x87"},{"\xd0\xb9","\xd0\x99"},{"\xd1\x98","\xd0\x88"},{"\xd0\xba","\xd0\x9a"},{"\xd2\x9b","\xd2\x9a"},{"\xd3\x84","\xd3\x83"},{"\xd2\xa1","\xd2\xa0"},{"\xd0\xbb","\xd0\x9b"},{"\xd1\x99","\xd0\x89"},{"\xd0\xbc","\xd0\x9c"},{"\xd0\xbd","\xd0\x9d"},{"\xd2\xa3","\xd2\xa2"},{"\xd1\x9a","\xd0\x8a"},{"\xd0\xbe","\xd0\x9e"},{"\xd3\xa7","\xd3\xa6"},{"\xd3\xa9","\xd3\xa8"},{"\xd0\xbf","\xd0\x9f"},{"\xd1\x80","\xd0\xa0"},{"\xd1\x81","\xd0\xa1"},{"\xd2\xab","\xd2\xaa"},{"\xd1\x82","\xd0\xa2"},{"\xd1\x9c","\xd0\x8c"},{"\xd1\x9b","\xd0\x8b"},{"\xd1\x83","\xd0\xa3"},{"\xd3\xb1","\xd3\xb0"},{"\xd2\xb1","\xd2\xb0"},{"\xd2\xaf","\xd2\xae"},{"\xd1\x9e","\xd0\x8e"},{"\xd1\x84","\xd0\xa4"},{"\xd1\x85","\xd0\xa5"},{"\xd2\xb3","\xd2\xb2"},{"\xd2\xbb","\xd2\xba"},{"\xd1\x86","\xd0\xa6"},{"\xd1\x87","\xd0\xa7"},{"\xd1\x9f","\xd0\x8f"},{"\xd1\x88","\xd0\xa8"},{"\xd1\x89","\xd0\xa9"},{"\xd1\x8a","\xd0\xaa"},{"\xd1\x8b","\xd0\xab"},{"\xd1\x8c","\xd0\xac"},{"\xd1\x8d","\xd0\xad"},{"\xd1\x8e","\xd0\xae"},{"\xd1\x8f","\xd0\xaf"},{"\xd5\xa1","\xd4\xb1"},{"\xd5\xa3","\xd4\xb3"},{"\xd5\xa5","\xd4\xb5"},{"\xd5\xab","\xd4\xbb"},{"\xd5\xac","\xd4\xbc"},{"\xd5\xb2","\xd5\x82"},{"\xd5\xb8","\xd5\x88"},{"\xd5\xbd","\xd5\x8d"},{"\xd5\xbe","\xd5\x8e"},{"\xd5\xbf","\xd5\x8f"},{"\xd6\x80","\xd5\x90"},{"\xd6\x81","\xd5\x91"} }; + for (auto p8 : map8) { + auto from = utf8ToWString(p8.first); + auto to = utf8ToWString(p8.second); + ABORT_IF(from.size() != 1 || to.size() != 1, "Incorrect character encoding??"); + insert(std::make_pair(from.front(), to.front())); + } + } + wchar_t towupper(wchar_t c) const { + auto iter = find(c); + if (iter == end()) + return c; + else + return iter->second; + } +}; +std::string utf8ToUpper(const std::string& s) { + static UTF8Mapper mapper; + auto ws = utf8ToWString(s); + for (auto& c : ws) + c = mapper.towupper(c); + return toUTF8String(ws); +} +#endif double parseDouble(std::string s) { double res; diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index b39a7fa5a..07993f485 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -17,10 +17,13 @@ Corpus::Corpus(std::vector paths, Ptr options) : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} -void Corpus::preprocessLine(std::string& line, size_t /*streamId*/) { +void Corpus::preprocessLine(std::string& line, size_t streamId) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) { line = utils::utf8ToUpper(line); - LOG_ONCE(info, "all-caps'ed line to {}", line); + if (streamId == 0) + LOG_ONCE(info, "[data] source all-caps'ed line to {}", line); + else + LOG_ONCE(info, "[data] target all-caps'ed line to {}", line); } } diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu old mode 100644 new mode 100755 index 34da4c688..b68d24c3e --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -438,6 +438,7 @@ __global__ void gSoftmax(float* out, so[id] = so[id] / _sum[0]; } } + __syncthreads(); } } } @@ -521,6 +522,7 @@ __global__ void gLogSoftmax(float* out, if(id < cols) so[id] -= __logf(_sum[0]); } + __syncthreads(); } } } @@ -580,6 +582,7 @@ __global__ void gSoftmaxGrad(float* grad, gradRow[id] += val; } } + __syncthreads(); } } } @@ -636,6 +639,7 @@ __global__ void gLogSoftmaxGrad(float* grad, if(id < cols) gradRow[id] += adjRow[id] - (expf(valRow[id]) * _sum[0]); } + __syncthreads(); } } } From 5d01d870537e8853f28604e36367ccb6b1904234 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 30 Jan 2019 13:48:20 -0800 Subject: [PATCH 239/838] disabled the LL weights for factors again, as they can now be learned directly --- src/layers/generic.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2ff6d34cc..fb618686d 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -314,9 +314,9 @@ namespace marian { // log-linear weight // @TODO: The weight should not be needed, since it could learn it into the embeddings. Maybe it doesn't. // @TODO? If we move the weight before affine(), it would use less memory at least for the main factor. - auto name = options_->get("prefix"); - auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - factorLogits = factorLogits * llWeight; // -a constant, which is OK for logits + //auto name = options_->get("prefix"); + //auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); + //factorLogits = factorLogits * llWeight; // -a constant, which is OK for logits allLogits[g] = factorLogits; } return Logits(std::move(allLogits), embeddingFactorMapping_); From b117996c0357b7a372756c08b4d87287dc1dd8fb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 30 Jan 2019 15:29:16 -0800 Subject: [PATCH 240/838] Output now creates its own W matrix in transposed form --- src/common/config_parser.cpp | 2 +- src/data/corpus_base.h | 2 +- src/data/default_vocab.cpp | 2 +- src/graph/expression_graph.h | 7 + src/graph/node_initializers.cpp | 2 +- src/layers/generic.h | 45 ++-- src/models/bert.h | 10 +- src/models/encoder_classifier.h | 436 ++++++++++++++++---------------- src/rnn/cells.h | 0 src/training/scheduler.h | 8 +- src/training/validator.h | 2 +- 11 files changed, 262 insertions(+), 254 deletions(-) mode change 100644 => 100755 src/models/bert.h mode change 100644 => 100755 src/models/encoder_classifier.h mode change 100644 => 100755 src/rnn/cells.h diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 338d4b82c..49d96ca1a 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -206,7 +206,7 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.add("--bert-mask-symbol", "Masking symbol for BERT masked-LM training", "[MASK]"); cli.add("--bert-sep-symbol", "Sentence separator symbol for BERT next sentence prediction training", "[SEP]"); cli.add("--bert-class-symbol", "Class symbol BERT classifier training", "[CLS]"); - cli.add("--bert-masking-fraction", "Fraction of masked out tokens during training", 0.15); + cli.add("--bert-masking-fraction", "Fraction of masked out tokens during training", 0.15f); #ifdef CUDNN cli.add("--char-stride", "Width of max-pooling layer after convolution layer in char-s2s model", diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 85e26d717..725059520 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -322,7 +322,7 @@ class CorpusBatch : public Batch { // set word indices to different values to avoid same hashes // rand() is OK, this does not affect state in any way std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), - [&](Word) { return rand() % vocabs[batchIndex]->size(); }); + [&](Word) -> Word { return rand() % vocabs[batchIndex]->size(); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); batchIndex++; diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index f5d009b2a..36a32328f 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -288,7 +288,7 @@ class DefaultVocab : public VocabBase { class ClassVocab : public DefaultVocab { private: // Do nothing. - virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override {} + virtual void addRequiredVocabulary(const std::string& vocabPath, bool isJson) override { vocabPath; isJson; } // Not adding special class labels, only seen classes. virtual void create(const std::string& vocabPath, diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index fe8361618..22908d96b 100755 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -316,6 +316,13 @@ class ExpressionGraph : public std::enable_shared_from_this { dot.close(); } + Expr tryFindParam(const std::string& pname) const { + std::string name = pname; + if(!namespace_.empty()) + name = namespace_ + "::" + name; + return params_->get(name); + } + Expr param(const std::string& pname, const Shape& shape, const NodeInitializer& init, diff --git a/src/graph/node_initializers.cpp b/src/graph/node_initializers.cpp index f405102ec..b2550c548 100755 --- a/src/graph/node_initializers.cpp +++ b/src/graph/node_initializers.cpp @@ -195,7 +195,7 @@ NodeInitializer from_item(const io::Item& item) { NodeInitializer sinusoidalPositionEmbeddings(int start) { return [start](Tensor t) { int dimEmb = t->shape()[-1]; - int dimWords = t->size() / dimEmb; + int dimWords = (int)t->size() / dimEmb; float numTimescales = (float)dimEmb / 2; float logTimescaleIncrement = std::log(10000.f) / (numTimescales - 1.f); diff --git a/src/layers/generic.h b/src/layers/generic.h index 062b59cc8..8e03efd08 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -125,14 +125,14 @@ class Dense : public LayerBase, public IUnaryLayer { class Output : public LayerBase, public IUnaryLayer { private: - Expr W_; // parameters held by this layer + // parameters held by this layer + Expr Wt_; // weight matrix is stored transposed for efficiency Expr b_; - Expr cachedShortW_; // short-listed version, cached (cleared by clear()) + Expr cachedShortWt_; // short-listed version, cached (cleared by clear()) Expr cachedShortb_; // these match the current value of shortlist_ // optional parameters set/updated after construction Expr tiedParam_; - bool transposeW_{false}; Ptr shortlist_; public: @@ -142,7 +142,7 @@ class Output : public LayerBase, public IUnaryLayer { } void tieTransposed(Expr tied) { - if (W_) + if (Wt_) ABORT_IF(tiedParam_.get() != tied.get(), "Tied output projection cannot be changed once weights have been created"); else tiedParam_ = tied; @@ -152,47 +152,46 @@ class Output : public LayerBase, public IUnaryLayer { if (shortlist_) ABORT_IF(shortlist.get() != shortlist_.get(), "Output shortlist cannot be changed except after clear()"); else { - ABORT_IF(cachedShortW_ || cachedShortb_, "No shortlist but cached parameters??"); + ABORT_IF(cachedShortWt_ || cachedShortb_, "No shortlist but cached parameters??"); shortlist_ = shortlist; } - // cachedShortW_ and cachedShortb_ will be created lazily inside apply() + // cachedShortWt_ and cachedShortb_ will be created lazily inside apply() } // this is expected to be called in sync with graph->clear(), which invalidates - // cachedShortW_ and cachedShortb_ in the graph's short-term cache + // cachedShortWt_ and cachedShortb_ in the graph's short-term cache void clear() { shortlist_ = nullptr; - cachedShortW_ = nullptr; - cachedShortb_ = nullptr; + cachedShortWt_ = nullptr; + cachedShortb_ = nullptr; } Expr apply(Expr input) override { - if(!W_) { // create lazily because we need input's dimension + if(!Wt_) { // create lazily because we need input's dimension auto name = options_->get("prefix"); auto dim = options_->get("dim"); if(tiedParam_) { - W_ = tiedParam_; - transposeW_ = true; + Wt_ = tiedParam_; } else { - W_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform); - transposeW_ = false; + auto W = graph_->tryFindParam(name + "_W"); // support of legacy models that did not transpose + if (W) + Wt_ = transpose(W); // legacy + else // this is the regular case: + Wt_ = graph_->param(name + "_Wt", {input->shape()[-1], dim}, inits::glorot_uniform); } b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); } if (shortlist_) { - if (!cachedShortW_) { // short versions of parameters are cached within one batch, then clear()ed - if(transposeW_) - cachedShortW_ = rows(W_, shortlist_->indices()); - else - cachedShortW_ = cols(W_, shortlist_->indices()); - cachedShortb_ = cols(b_, shortlist_->indices()); + if (!cachedShortWt_) { // short versions of parameters are cached within one batch, then clear()ed + cachedShortWt_ = rows(Wt_, shortlist_->indices()); + cachedShortb_ = cols(b_ , shortlist_->indices()); } - return affine(input, cachedShortW_, cachedShortb_, false, transposeW_); + return affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/true); } else - return affine(input, W_, b_, false, transposeW_); + return affine(input, Wt_, b_, false, /*transB=*/true); } virtual Expr apply(const std::vector& /*inputs*/) override { @@ -349,7 +348,7 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { } Expr apply(const std::vector& embIdx, const Shape& shape) const override final { - shape; + embIdx; shape; ABORT("not implemented"); // @TODO: implement me } }; diff --git a/src/models/bert.h b/src/models/bert.h old mode 100644 new mode 100755 index c642abac4..c56eb2977 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -79,7 +79,7 @@ class BertBatch : public CorpusBatch { const auto& vocab = *subBatch->vocab(); // Initialize to sample random vocab id - randomWord_.reset(new std::uniform_int_distribution(0, vocab.size())); + randomWord_.reset(new std::uniform_int_distribution(0, (Word)vocab.size())); // Intialize to sample random percentage randomPercent_.reset(new std::uniform_real_distribution(0.f, 1.f)); @@ -111,7 +111,7 @@ class BertBatch : public CorpusBatch { if(dontMask_.count(words[i]) == 0) // do not add indices of special words selected.push_back(i); std::shuffle(selected.begin(), selected.end(), engine); // randomize positions - selected.resize(std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices + selected.resize((size_t)std::ceil(selected.size() * maskFraction)); // select first x percent from shuffled indices for(int i : selected) { maskedPositions_.push_back(i); // where is the original word? @@ -141,10 +141,10 @@ class BertBatch : public CorpusBatch { ABORT_IF(sepId == vocab.getUnkId(), "BERT separator symbol {} not found in vocabulary", sepSymbol_); - int dimBatch = subBatch->batchSize(); - int dimWords = subBatch->batchWidth(); + int dimBatch = (int)subBatch->batchSize(); + int dimWords = (int)subBatch->batchWidth(); - int maxSentPos = 2; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] + const size_t maxSentPos = 2; // Currently only two sentences allowed A at [0] and B at [1] and padding at [2] // If another separator is seen do not increase position index beyond 2 but use padding. // @TODO: make this configurable, see below for NextSentencePredictions task where we also restrict to 2. diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h old mode 100644 new mode 100755 index bc3d8f9f8..c593ccfd3 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -1,218 +1,218 @@ -#pragma once - -#include "marian.h" - -#include "models/encoder.h" -#include "models/classifier.h" -#include "models/model_base.h" -#include "models/states.h" - -namespace marian { - -/** - * Combines sequence encoders with generic classifiers - * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. - * Already has support for multi-objective training. - * - * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier - * multi-objective training. - */ -class EncoderClassifierBase : public models::ModelBase { -public: - virtual ~EncoderClassifierBase() {} - - virtual void load(Ptr graph, - const std::string& name, - bool markedReloaded = true) override - = 0; - - virtual void mmap(Ptr graph, - const void* ptr, - bool markedReloaded = true) - = 0; - - virtual void save(Ptr graph, - const std::string& name, - bool saveTranslatorConfig = false) override - = 0; - - virtual void clear(Ptr graph) override = 0; - - virtual std::vector> apply(Ptr, Ptr, bool) = 0; - - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override = 0; - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) = 0; - - virtual Ptr getOptions() = 0; -}; - -class EncoderClassifier : public EncoderClassifierBase { -protected: - Ptr options_; - - std::string prefix_; - - std::vector> encoders_; - std::vector> classifiers_; - - bool inference_{false}; - - std::set modelFeatures_; - - Config::YamlNode getModelParameters() { - Config::YamlNode modelParams; - for(auto& key : modelFeatures_) - modelParams[key] = options_->getYaml()[key]; - - if(options_->has("original-type")) - modelParams["type"] = options_->getYaml()["original-type"]; - - modelParams["version"] = buildVersion(); - return modelParams; - } - - std::string getModelParametersAsString() { - auto yaml = getModelParameters(); - YAML::Emitter out; - cli::OutputYaml(yaml, out); - return std::string(out.c_str()); - } - -public: - typedef data::Corpus dataset_type; - - // @TODO: lots of code-duplication with EncoderDecoder - EncoderClassifier(Ptr options) - : options_(options), - prefix_(options->get("prefix", "")), - inference_(options->get("inference", false)) { - modelFeatures_ = {"type", - "dim-vocabs", - "dim-emb", - "dim-rnn", - "enc-cell", - "enc-type", - "enc-cell-depth", - "enc-depth", - "dec-depth", - "dec-cell", - "dec-cell-base-depth", - "dec-cell-high-depth", - "skip", - "layer-normalization", - "right-left", - "input-types", - "special-vocab", - "tied-embeddings", - "tied-embeddings-src", - "tied-embeddings-all"}; - - modelFeatures_.insert("transformer-heads"); - modelFeatures_.insert("transformer-no-projection"); - modelFeatures_.insert("transformer-dim-ffn"); - modelFeatures_.insert("transformer-ffn-depth"); - modelFeatures_.insert("transformer-ffn-activation"); - modelFeatures_.insert("transformer-dim-aan"); - modelFeatures_.insert("transformer-aan-depth"); - modelFeatures_.insert("transformer-aan-activation"); - modelFeatures_.insert("transformer-aan-nogate"); - modelFeatures_.insert("transformer-preprocess"); - modelFeatures_.insert("transformer-postprocess"); - modelFeatures_.insert("transformer-postprocess-emb"); - modelFeatures_.insert("transformer-decoder-autoreg"); - modelFeatures_.insert("transformer-tied-layers"); - modelFeatures_.insert("transformer-guided-alignment-layer"); - modelFeatures_.insert("transformer-train-positions"); - } - - virtual Ptr getOptions() override { return options_; } - - std::vector>& getEncoders() { return encoders_; } - std::vector>& getClassifiers() { return classifiers_; } - - void push_back(Ptr encoder) { encoders_.push_back(encoder); } - void push_back(Ptr classifier) { classifiers_.push_back(classifier); } - - void load(Ptr graph, - const std::string& name, - bool markedReloaded) override { - graph->load(name, markedReloaded && !opt("ignore-model-config", false)); - } - - void mmap(Ptr graph, - const void* ptr, - bool markedReloaded) override { - graph->mmap(ptr, markedReloaded && !opt("ignore-model-config", false)); - } - - void save(Ptr graph, - const std::string& name, - bool saveModelConfig) override { - LOG(info, "Saving model weights and runtime parameters to {}", name); - graph->save(name , getModelParametersAsString()); - } - - void clear(Ptr graph) override { - graph->clear(); - - for(auto& enc : encoders_) - enc->clear(); - for(auto& cls : classifiers_) - cls->clear(); - } - - template - T opt(const std::string& key) { - return options_->get(key); - } - - template - T opt(const std::string& key, const T& def) { - return options_->get(key, def); - } - - template - void set(std::string key, T value) { - options_->set(key, value); - } - - /*********************************************************************/ - - virtual std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { - if(clearGraph) - clear(graph); - - std::vector> encoderStates; - for(auto& encoder : encoders_) - encoderStates.push_back(encoder->build(graph, batch)); - - std::vector> classifierStates; - for(auto& classifier : classifiers_) - classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); - - return classifierStates; - } - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { - auto states = apply(graph, batch, clearGraph); - // returns raw logits - return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used - } - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { - auto corpusBatch = std::static_pointer_cast(batch); - return build(graph, corpusBatch, clearGraph); - } -}; - -} // namespace marian +#pragma once + +#include "marian.h" + +#include "models/encoder.h" +#include "models/classifier.h" +#include "models/model_base.h" +#include "models/states.h" + +namespace marian { + +/** + * Combines sequence encoders with generic classifiers + * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. + * Already has support for multi-objective training. + * + * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier + * multi-objective training. + */ +class EncoderClassifierBase : public models::ModelBase { +public: + virtual ~EncoderClassifierBase() {} + + virtual void load(Ptr graph, + const std::string& name, + bool markedReloaded = true) override + = 0; + + virtual void mmap(Ptr graph, + const void* ptr, + bool markedReloaded = true) + = 0; + + virtual void save(Ptr graph, + const std::string& name, + bool saveTranslatorConfig = false) override + = 0; + + virtual void clear(Ptr graph) override = 0; + + virtual std::vector> apply(Ptr, Ptr, bool) = 0; + + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; + + virtual Ptr getOptions() = 0; +}; + +class EncoderClassifier : public EncoderClassifierBase { +protected: + Ptr options_; + + std::string prefix_; + + std::vector> encoders_; + std::vector> classifiers_; + + bool inference_{false}; + + std::set modelFeatures_; + + Config::YamlNode getModelParameters() { + Config::YamlNode modelParams; + for(auto& key : modelFeatures_) + modelParams[key] = options_->getYaml()[key]; + + if(options_->has("original-type")) + modelParams["type"] = options_->getYaml()["original-type"]; + + modelParams["version"] = buildVersion(); + return modelParams; + } + + std::string getModelParametersAsString() { + auto yaml = getModelParameters(); + YAML::Emitter out; + cli::OutputYaml(yaml, out); + return std::string(out.c_str()); + } + +public: + typedef data::Corpus dataset_type; + + // @TODO: lots of code-duplication with EncoderDecoder + EncoderClassifier(Ptr options) + : options_(options), + prefix_(options->get("prefix", "")), + inference_(options->get("inference", false)) { + modelFeatures_ = {"type", + "dim-vocabs", + "dim-emb", + "dim-rnn", + "enc-cell", + "enc-type", + "enc-cell-depth", + "enc-depth", + "dec-depth", + "dec-cell", + "dec-cell-base-depth", + "dec-cell-high-depth", + "skip", + "layer-normalization", + "right-left", + "input-types", + "special-vocab", + "tied-embeddings", + "tied-embeddings-src", + "tied-embeddings-all"}; + + modelFeatures_.insert("transformer-heads"); + modelFeatures_.insert("transformer-no-projection"); + modelFeatures_.insert("transformer-dim-ffn"); + modelFeatures_.insert("transformer-ffn-depth"); + modelFeatures_.insert("transformer-ffn-activation"); + modelFeatures_.insert("transformer-dim-aan"); + modelFeatures_.insert("transformer-aan-depth"); + modelFeatures_.insert("transformer-aan-activation"); + modelFeatures_.insert("transformer-aan-nogate"); + modelFeatures_.insert("transformer-preprocess"); + modelFeatures_.insert("transformer-postprocess"); + modelFeatures_.insert("transformer-postprocess-emb"); + modelFeatures_.insert("transformer-decoder-autoreg"); + modelFeatures_.insert("transformer-tied-layers"); + modelFeatures_.insert("transformer-guided-alignment-layer"); + modelFeatures_.insert("transformer-train-positions"); + } + + virtual Ptr getOptions() override { return options_; } + + std::vector>& getEncoders() { return encoders_; } + std::vector>& getClassifiers() { return classifiers_; } + + void push_back(Ptr encoder) { encoders_.push_back(encoder); } + void push_back(Ptr classifier) { classifiers_.push_back(classifier); } + + void load(Ptr graph, + const std::string& name, + bool markedReloaded) override { + graph->load(name, markedReloaded && !opt("ignore-model-config", false)); + } + + void mmap(Ptr graph, + const void* ptr, + bool markedReloaded) override { + graph->mmap(ptr, markedReloaded && !opt("ignore-model-config", false)); + } + + void save(Ptr graph, + const std::string& name, + bool /*saveModelConfig*/) override { + LOG(info, "Saving model weights and runtime parameters to {}", name); + graph->save(name , getModelParametersAsString()); + } + + void clear(Ptr graph) override { + graph->clear(); + + for(auto& enc : encoders_) + enc->clear(); + for(auto& cls : classifiers_) + cls->clear(); + } + + template + T opt(const std::string& key) { + return options_->get(key); + } + + template + T opt(const std::string& key, const T& def) { + return options_->get(key, def); + } + + template + void set(std::string key, T value) { + options_->set(key, value); + } + + /*********************************************************************/ + + virtual std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + if(clearGraph) + clear(graph); + + std::vector> encoderStates; + for(auto& encoder : encoders_) + encoderStates.push_back(encoder->build(graph, batch)); + + std::vector> classifierStates; + for(auto& classifier : classifiers_) + classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); + + return classifierStates; + } + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto states = apply(graph, batch, clearGraph); + // returns raw logits + return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used + } + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto corpusBatch = std::static_pointer_cast(batch); + return build(graph, corpusBatch, clearGraph); + } +}; + +} // namespace marian diff --git a/src/rnn/cells.h b/src/rnn/cells.h old mode 100644 new mode 100755 diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 65bb88207..8cc787b3d 100755 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -273,15 +273,17 @@ class Scheduler : public TrainingObserver { if (mpi) rationalLoss.loss *= mpi->numMPIProcesses(); + // @BUGBUG: rationalLoss.count is float, not a count. Possible solution: make (costSum, costCount) a StaticLoss object as well state_->costSum += rationalLoss.loss; // aggregate sum cost since last display - state_->costCount += rationalLoss.count; // cost gets normalized w.r.t. this in display + state_->costCount += (size_t)rationalLoss.count; // cost gets normalized w.r.t. this in display state_->updatesDisp += 1; state_->samplesDisp += batchSize; state_->wordsDisp += batchLabels; //@TODO: this is wrong // words at given input processed since last display, for speed display - state_->samplesEpoch += batchSize; // sentences processed in this epoch - state_->labelsTotal += rationalLoss.count; // total labels processed + state_->samplesEpoch += batchSize; // sentences processed in this epoch + // @BUGBUG: rationalLoss.count is float, not a count + state_->labelsTotal += (size_t)rationalLoss.count; // total labels processed state_->newUpdate(numReadBatches); diff --git a/src/training/validator.h b/src/training/validator.h index d09859b20..a819968b6 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -352,7 +352,7 @@ class BertAccuracyValidator : public Validator { } if(!engine) - engine.reset(new std::mt19937(Config::seed + id)); + engine.reset(new std::mt19937((unsigned int)(Config::seed + id))); auto bertBatch = New(batch, *engine, From 7b649c00e0d08083818235533b770cc46c66716e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 31 Jan 2019 14:45:51 -0800 Subject: [PATCH 241/838] In VS Project, changed transformer.h from CPP file to a true header, and added the stub, to fix a compilation warning --- src/models/transformer_factory.h | 8 ++------ src/models/transformer_stub.cpp | 0 vs/Marian.vcxproj | 3 ++- vs/Marian.vcxproj.filters | 13 +++++++++---- 4 files changed, 13 insertions(+), 11 deletions(-) mode change 100644 => 100755 src/models/transformer_stub.cpp diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h index c2a7e13bb..fe9ee3ee6 100755 --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -4,12 +4,8 @@ #include "models/decoder.h" #include "models/encoder.h" -//#include "models/states.h" -//#include "layers/constructors.h" -//#include "layers/factory.h" namespace marian { -// @TODO: find out why static is required here to get to compile -static Ptr NewEncoderTransformer(Ptr options); -static Ptr NewDecoderTransformer(Ptr options); +Ptr NewEncoderTransformer(Ptr options); +Ptr NewDecoderTransformer(Ptr options); } // namespace marian diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100644 new mode 100755 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index a6c560d3a..7f82324be 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -580,6 +580,7 @@ false + @@ -905,7 +906,7 @@ - + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index d9e56843f..1e30aa196 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -202,9 +202,6 @@ tensors\cpu\sharp - - models - common @@ -481,6 +478,9 @@ examples\iris + + models + @@ -1390,7 +1390,6 @@ 3rd_party\sentencepiece\src - 3rd_party\nccl\src\collectives @@ -1517,6 +1516,12 @@ examples\mnist + + command + + + models + From acee183116876c26f3852e9b13c82e130c513437 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 31 Jan 2019 14:55:45 -0800 Subject: [PATCH 242/838] towards fixing the bug that bert.h includes transformer.h --- src/models/bert.h | 4 +++- src/models/transformer.h | 11 ----------- src/models/transformer_stub.cpp | 13 +++++++++++++ vs/Marian.vcxproj | 1 + vs/Marian.vcxproj.filters | 3 +++ 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/models/bert.h b/src/models/bert.h index c56eb2977..2ea725143 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -2,7 +2,7 @@ #include "data/corpus_base.h" #include "models/encoder_classifier.h" -#include "models/transformer.h" +#include "models/transformer.h" // @BUGBUG: transformer.h is large and was meant to be compiled separately #include "data/rng_engine.h" namespace marian { @@ -211,6 +211,8 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { * BERT-specific modifications to EncoderTransformer * Actually all that is needed is to intercept the creation of special embeddings, * here sentence embeddings for sentence A and B. + * @BUGBUG: transformer.h was meant to be compiled separately. I.e., one cannot derive from it. + * Is there a way to maybe instead include a reference in here, instead of deriving from it? */ class BertEncoder : public EncoderTransformer { public: diff --git a/src/models/transformer.h b/src/models/transformer.h index 01201df96..1db455c9e 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -862,17 +862,6 @@ class DecoderTransformer : public Transformer { } }; -// factory functions -Ptr NewEncoderTransformer(Ptr options) -{ - return New(options); -} - -Ptr NewDecoderTransformer(Ptr options) -{ - return New(options); -} - // clang-format on } // namespace marian diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp index 420b77810..c30921914 100755 --- a/src/models/transformer_stub.cpp +++ b/src/models/transformer_stub.cpp @@ -2,3 +2,16 @@ // This is meant to speed-up builds, and to support Ctrl-F7 to rebuild. #include "models/transformer.h" + +namespace marian { +// factory functions +Ptr NewEncoderTransformer(Ptr options) +{ + return New(options); +} + +Ptr NewDecoderTransformer(Ptr options) +{ + return New(options); +} +} // namespace marian diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 7f82324be..c899c242f 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -895,6 +895,7 @@ + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 1e30aa196..947bcece8 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1522,6 +1522,9 @@ models + + models + From 3312ed91116bf6ad7aeb0ed1b8516d888aa8b840 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 31 Jan 2019 20:42:42 -0800 Subject: [PATCH 243/838] abstracted type Word into a class, in prep for factored decoding --- src/common/definitions.h | 2 +- src/data/corpus_base.h | 4 +- src/data/default_vocab.cpp | 20 ++++---- src/data/shortlist.h | 86 +++++++++++++++++----------------- src/data/types.h | 36 ++++++++++++-- src/data/vocab.cpp | 4 ++ src/data/vocab.h | 4 +- src/layers/generic.h | 26 ++++++---- src/layers/word2vec_reader.h | 6 +-- src/microsoft/quicksand.cpp | 4 +- src/models/bert.h | 14 +++--- src/models/costs.h | 4 +- src/models/decoder.h | 8 ++-- src/models/encoder_decoder.cpp | 4 +- src/models/encoder_decoder.h | 4 +- src/models/transformer.h | 4 +- src/tests/attention_tests.cpp | 2 +- src/tests/rnn_tests.cpp | 2 +- src/training/validator.h | 4 +- src/translator/beam_search.h | 14 +++--- src/translator/scorers.h | 6 +-- 21 files changed, 150 insertions(+), 108 deletions(-) mode change 100644 => 100755 src/data/types.h mode change 100644 => 100755 src/models/encoder_decoder.h diff --git a/src/common/definitions.h b/src/common/definitions.h index 101200fff..21ee816fc 100755 --- a/src/common/definitions.h +++ b/src/common/definitions.h @@ -1,7 +1,7 @@ #pragma once #include "common/logging.h" -#include "shape.h" +#include "shape.h" // @TODO: why not common/shape.h? #include #include diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 725059520..6614d7d2c 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -110,7 +110,7 @@ class SentenceTuple { */ class SubBatch { private: - std::vector indices_; + Words indices_; std::vector mask_; size_t size_; @@ -142,7 +142,7 @@ class SubBatch { * idx_{w,0},idx_{w,1},\dots,idx_{w,s}\f$, where \f$w\f$ is the number of * words (width) and \f$s\f$ is the number of sentences (size). */ - std::vector& data() { return indices_; } + Words& data() { return indices_; } /** * @brief Flat masking vector; 0 is used for masked words. * diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 36a32328f..b791ec52a 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -23,8 +23,8 @@ class DefaultVocab : public VocabBase { typedef std::vector Id2Str; Id2Str id2str_; - Word eosId_ = (Word)-1; - Word unkId_ = (Word)-1; + Word eosId_ = Word::NONE; + Word unkId_ = Word::NONE; std::vector suffixes_ = { ".yml", ".yaml", ".json" }; @@ -97,7 +97,7 @@ class DefaultVocab : public VocabBase { if(isJson) { YAML::Node vocabNode = YAML::Load(io::InputFileStream(vocabPath)); for(auto&& pair : vocabNode) - vocab.insert({pair.first.as(), pair.second.as()}); + vocab.insert({pair.first.as(), Word(pair.second.as())}); } // read from flat text file else { @@ -107,7 +107,7 @@ class DefaultVocab : public VocabBase { ABORT_IF(line.empty(), "DefaultVocabulary file {} must not contain empty lines", vocabPath); - vocab.insert({line, (Word)vocab.size()}); + vocab.insert({line, Word(vocab.size())}); } ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath); } @@ -131,8 +131,8 @@ class DefaultVocab : public VocabBase { // for fakeBatch() virtual void createFake() override { - eosId_ = insertWord(DEFAULT_EOS_ID, DEFAULT_EOS_STR); - unkId_ = insertWord(DEFAULT_UNK_ID, DEFAULT_UNK_STR); + eosId_ = insertWord(Word::DEFAULT_EOS_ID, DEFAULT_EOS_STR); + unkId_ = insertWord(Word::DEFAULT_UNK_ID, DEFAULT_UNK_STR); } virtual void create(const std::string& vocabPath, @@ -195,8 +195,8 @@ class DefaultVocab : public VocabBase { str); return iter->second; }; - eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, DEFAULT_EOS_ID); - unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, DEFAULT_UNK_ID); + eosId_ = getRequiredWordId(DEFAULT_EOS_STR, NEMATUS_EOS_STR, Word::DEFAULT_EOS_ID); + unkId_ = getRequiredWordId(DEFAULT_UNK_STR, NEMATUS_UNK_STR, Word::DEFAULT_UNK_ID); } void addCounts(std::unordered_map& counter, @@ -232,8 +232,8 @@ class DefaultVocab : public VocabBase { std::sort(vocabVec.begin(), vocabVec.end(), VocabFreqOrderer(counter)); YAML::Node vocabYaml; - vocabYaml.force_insert(DEFAULT_EOS_STR, DEFAULT_EOS_ID); - vocabYaml.force_insert(DEFAULT_UNK_STR, DEFAULT_UNK_ID); + vocabYaml.force_insert(DEFAULT_EOS_STR, Word::DEFAULT_EOS_ID.getWordIndex()); + vocabYaml.force_insert(DEFAULT_UNK_STR, Word::DEFAULT_UNK_ID.getWordIndex()); Word maxSpec = 1; auto vocabSize = vocabVec.size(); diff --git a/src/data/shortlist.h b/src/data/shortlist.h index 55fe4f5bc..50dac8d54 100755 --- a/src/data/shortlist.h +++ b/src/data/shortlist.h @@ -14,21 +14,21 @@ namespace data { class Shortlist { private: - std::vector indices_; - std::vector mappedIndices_; - std::vector reverseMap_; + std::vector indices_; + std::vector mappedIndices_; + std::vector reverseMap_; public: - Shortlist(const std::vector& indices, - const std::vector& mappedIndices, - const std::vector& reverseMap) + Shortlist(const std::vector& indices, + const std::vector& mappedIndices, + const std::vector& reverseMap) : indices_(indices), mappedIndices_(mappedIndices), reverseMap_(reverseMap) {} - std::vector& indices() { return indices_; } - std::vector& mappedIndices() { return mappedIndices_; } - Word reverseMap(Word idx) { return reverseMap_[idx]; } + std::vector& indices() { return indices_; } + std::vector& mappedIndices() { return mappedIndices_; } + WordIndex reverseMap(WordIndex idx) { return reverseMap_[idx]; } }; class ShortlistGenerator { @@ -73,37 +73,37 @@ class SampledShortlistGenerator : public ShortlistGenerator { auto trgBatch = (*batch)[trgIdx_]; // add firstNum most frequent words - std::unordered_set idxSet; - for(Word i = 0; i < firstNum_ && i < maxVocab_; ++i) + std::unordered_set idxSet; + for(WordIndex i = 0; i < firstNum_ && i < maxVocab_; ++i) idxSet.insert(i); // add all words from ground truth for(auto i : trgBatch->data()) - idxSet.insert(i); + idxSet.insert(i.getWordIndex()); // add all words from source if(shared_) for(auto i : srcBatch->data()) - idxSet.insert(i); + idxSet.insert(i.getWordIndex()); std::uniform_int_distribution<> dis((int)firstNum_, (int)maxVocab_); while(idxSet.size() < total_ && idxSet.size() < maxVocab_) idxSet.insert(dis(gen_)); // turn into vector and sort (selected indices) - std::vector idx(idxSet.begin(), idxSet.end()); + std::vector idx(idxSet.begin(), idxSet.end()); std::sort(idx.begin(), idx.end()); // assign new shifted position - std::unordered_map pos; - std::vector reverseMap; + std::unordered_map pos; + std::vector reverseMap; - for(Word i = 0; i < idx.size(); ++i) { + for(WordIndex i = 0; i < idx.size(); ++i) { pos[idx[i]] = i; reverseMap.push_back(idx[i]); } - std::vector mapped; + std::vector mapped; for(auto i : trgBatch->data()) { // mapped postions for cross-entropy mapped.push_back(pos[i]); @@ -126,7 +126,7 @@ class LexicalShortlistGenerator : public ShortlistGenerator { size_t firstNum_{100}; size_t bestNum_{100}; - std::vector> data_; + std::vector> data_; // [WordIndex src] -> [WordIndex tgt] -> P_trans(tgt|src) --@TODO: rename data_ accordingly void load(const std::string& fname) { io::InputFileStream in(fname); @@ -138,8 +138,8 @@ class LexicalShortlistGenerator : public ShortlistGenerator { if(src == "NULL" || trg == "NULL") continue; - Word sId = (*srcVocab_)[src]; - Word tId = (*trgVocab_)[trg]; + auto sId = (*srcVocab_)[src].getWordIndex(); + auto tId = (*trgVocab_)[trg].getWordIndex(); if(data_.size() <= sId) data_.resize(sId + 1); @@ -150,12 +150,12 @@ class LexicalShortlistGenerator : public ShortlistGenerator { void prune(float threshold = 0.f) { size_t i = 0; for(auto& probs : data_) { - std::vector> sorter; + std::vector> sorter; for(auto& it : probs) - sorter.emplace_back(it.second, (Word)it.first); + sorter.emplace_back(it.second, it.first); std::sort( - sorter.begin(), sorter.end(), std::greater>()); + sorter.begin(), sorter.end(), std::greater>()); // sort by prob probs.clear(); for(auto& it : sorter) { @@ -211,14 +211,14 @@ class LexicalShortlistGenerator : public ShortlistGenerator { // Dump top most frequent words from target vocabulary LOG(info, "[data] Saving shortlist dump to {}", prefix + ".{top,dic}"); io::OutputFileStream outTop(prefix + ".top"); - for(Word i = 0; i < firstNum_ && i < trgVocab_->size(); ++i) + for(WordIndex i = 0; i < firstNum_ && i < trgVocab_->size(); ++i) outTop << (*trgVocab_)[i] << std::endl; // Dump translation pairs from dictionary io::OutputFileStream outDic(prefix + ".dic"); - for(Word srcId = 0; srcId < data_.size(); srcId++) { - for(auto& it : data_[srcId]) { // @TODO: change data_.first from size_t to Word - Word trgId = (Word)it.first; + for(WordIndex srcId = 0; srcId < data_.size(); srcId++) { + for(auto& it : data_[srcId]) { + auto trgId = it.first; outDic << (*srcVocab_)[srcId] << "\t" << (*trgVocab_)[trgId] << std::endl; } } @@ -229,16 +229,16 @@ class LexicalShortlistGenerator : public ShortlistGenerator { // auto trgBatch = (*batch)[trgIdx_]; // add firstNum most frequent words - std::unordered_set idxSet; - for(Word i = 0; i < firstNum_ && i < trgVocab_->size(); ++i) + std::unordered_set idxSet; + for(WordIndex i = 0; i < firstNum_ && i < trgVocab_->size(); ++i) idxSet.insert(i); // add all words from ground truth // for(auto i : trgBatch->data()) - // idxSet.insert(i); + // idxSet.insert(i.getWordIndex()); // collect unique words form source - std::unordered_set srcSet; + std::unordered_set srcSet; for(auto i : srcBatch->data()) srcSet.insert(i); @@ -247,23 +247,23 @@ class LexicalShortlistGenerator : public ShortlistGenerator { if(shared_) idxSet.insert(i); for(auto& it : data_[i]) - idxSet.insert((Word)it.first); // @TODO: change it.first to Word + idxSet.insert(it.first); } // turn into vector and sort (selected indices) - std::vector idx(idxSet.begin(), idxSet.end()); + std::vector idx(idxSet.begin(), idxSet.end()); std::sort(idx.begin(), idx.end()); // assign new shifted position - // std::unordered_map pos; - std::vector reverseMap; + // std::unordered_map pos; + std::vector reverseMap; - for(Word i = 0; i < idx.size(); ++i) { + for(WordIndex i = 0; i < idx.size(); ++i) { // pos[idx[i]] = i; reverseMap.push_back(idx[i]); } - std::vector mapped; + std::vector mapped; // for(auto i : trgBatch->data()) { // mapped postions for cross-entropy // mapped.push_back(pos[i]); @@ -275,21 +275,21 @@ class LexicalShortlistGenerator : public ShortlistGenerator { class FakeShortlistGenerator : public ShortlistGenerator { private: - std::vector idx_; - std::vector reverseIdx_; + std::vector idx_; + std::vector reverseIdx_; public: - FakeShortlistGenerator(const std::unordered_set& idxSet) + FakeShortlistGenerator(const std::unordered_set& idxSet) : idx_(idxSet.begin(), idxSet.end()) { std::sort(idx_.begin(), idx_.end()); // assign new shifted position - for(Word i = 0; i < idx_.size(); ++i) { + for(WordIndex i = 0; i < idx_.size(); ++i) { reverseIdx_.push_back(idx_[i]); } } Ptr generate(Ptr /*batch*/) override { - std::vector tmp; + std::vector tmp; return New(idx_, tmp, reverseIdx_); } }; diff --git a/src/data/types.h b/src/data/types.h old mode 100644 new mode 100755 index 2bda6ecea..549b439ea --- a/src/data/types.h +++ b/src/data/types.h @@ -11,14 +11,36 @@ namespace marian { // Type for all vocabulary items, based on IndexType -typedef IndexType Word; +typedef IndexType WordIndex; // WordIndex is used for words or tokens arranged in consecutive order +class Word { // Word is an abstraction of a unique id, not necessarily consecutive + WordIndex wordId_; +public: + // back compat with WordIndex + Word(std::size_t wordId) : wordId_((WordIndex)wordId) {} // @TODO: make explicit, or make private + operator WordIndex() const { return getWordIndex(); } + + // needed for STL containers + Word() : wordId_((WordIndex)-1) {} + bool operator==(const Word& other) const { return wordId_ == other.wordId_; } + std::size_t hash() const { return std::hash{}(wordId_); } + + // main methods and constants + static Word From(std::size_t wordId) { return Word(wordId); } + const WordIndex& getWordIndex() const { return wordId_; } + + static Word NONE; // @TODO: decide whether we need this, in additional Word() + // EOS and UNK are placed in these positions in Marian-generated vocabs + static Word DEFAULT_EOS_ID; + static Word DEFAULT_UNK_ID; +}; // Sequence of vocabulary items typedef std::vector Words; -// EOS and UNK are placed in these positions in Marian-generated vocabs -const Word DEFAULT_EOS_ID = 0; -const Word DEFAULT_UNK_ID = 1; +// Helper to map a Word vector to a WordIndex vector +static inline std::vector toWordIndexVector(const Words& words) { + return std::vector(words.begin(), words.end()); +} // names of EOS and UNK symbols const std::string DEFAULT_EOS_STR = ""; @@ -29,3 +51,9 @@ const std::string NEMATUS_EOS_STR = "eos"; const std::string NEMATUS_UNK_STR = "UNK"; } // namespace marian + +namespace std { + template<> struct hash { + std::size_t operator()(const marian::Word& s) const noexcept { return s.hash(); } + }; +} diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 266b14322..8118ea24f 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -3,6 +3,10 @@ namespace marian { +Word Word::NONE = Word(); +Word Word::DEFAULT_EOS_ID = Word(0); +Word Word::DEFAULT_UNK_ID = Word(1); + Ptr createDefaultVocab(); Ptr createClassVocab(); Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); diff --git a/src/data/vocab.h b/src/data/vocab.h index af4ea71fe..e82673ecc 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -42,8 +42,8 @@ class Vocab { // string token to token id Word operator[](const std::string& word) const; - // token id to string token - const std::string& operator[](Word id) const; + // token index to string token + const std::string& operator[](Word word) const; // line of text to list of token ids, can perform tokenization Words encode(const std::string& line, diff --git a/src/layers/generic.h b/src/layers/generic.h index 8e03efd08..55d42f555 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -51,8 +51,10 @@ struct IUnaryLayer { struct IEmbeddingLayer { virtual std::tuple apply(Ptr subBatch) const = 0; - // alternative version from index vector, and with batch dims/shape - virtual Expr apply(const std::vector& embIdx, const Shape& shape) const = 0; + virtual Expr apply(const Words& embIdx, const Shape& shape) const = 0; + + // alternative from indices directly + virtual Expr applyIndices(const std::vector& embIdx, const Shape& shape) const = 0; }; namespace mlp { @@ -237,8 +239,11 @@ class Embedding : public LayerBase, public IEmbeddingLayer { return std::make_tuple(batchEmbeddings, batchMask); } - // special version used in decoding or for learned positional embeddings - Expr apply(const std::vector& embIdx, const Shape& shape) const override final { + Expr apply(const Words& words, const Shape& shape) const override final { + return applyIndices(toWordIndexVector(words), shape); + } + + Expr applyIndices(const std::vector& embIdx, const Shape& shape) const override final { auto selectedEmbs = rows(E_, embIdx); return reshape(selectedEmbs, shape); } @@ -324,9 +329,10 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { // note all above can be precombuted and serialized if A is not trainiable and during decoding (TBD) // here we need to handle the mini-batch // extract raws corresponding to Xs in this minibatch from Q - auto queryEmbeddings = rows(queryEmbed, subBatch->data()); - auto srcEmbeddings = rows(srcEmbed, subBatch->data()); // extract trainable src embeddings - auto alpha = rows(ulrSharable, subBatch->data()); // extract sharable flags + auto embIdx = toWordIndexVector(subBatch->data()); + auto queryEmbeddings = rows(queryEmbed, embIdx); + auto srcEmbeddings = rows(srcEmbed, embIdx); // extract trainable src embeddings + auto alpha = rows(ulrSharable, embIdx); // extract sharable flags auto qt = dot(queryEmbeddings, ulrTransform, false, false); //A: transform embeddings based on similarity A : dimUlrEmb*dimUlrEmb auto sqrtDim=std::sqrt((float)queryEmbeddings->shape()[-1]); qt = qt/sqrtDim; // normalize accordin to embed size to avoid dot prodcut growing large in magnitude with larger embeds sizes @@ -347,7 +353,11 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { return std::make_tuple(batchEmbeddings, batchMask); } - Expr apply(const std::vector& embIdx, const Shape& shape) const override final { + Expr apply(const Words& words, const Shape& shape) const override final { + return applyIndices(toWordIndexVector(words), shape); + } + + Expr applyIndices(const std::vector& embIdx, const Shape& shape) const override final { embIdx; shape; ABORT("not implemented"); // @TODO: implement me } diff --git a/src/layers/word2vec_reader.h b/src/layers/word2vec_reader.h index a7e855927..88c695aa9 100755 --- a/src/layers/word2vec_reader.h +++ b/src/layers/word2vec_reader.h @@ -33,12 +33,12 @@ class Word2VecReader { "Unexpected length of embedding vectors"); // Read embedding vectors into a map - std::unordered_map> word2vec; + std::unordered_map> word2vec; while(io::getline(embFile, line)) { values.clear(); utils::split(line, values); - Word word = std::stoi(values.front()); + WordIndex word = std::stoi(values.front()); if(word >= (size_t)dimVoc) continue; @@ -54,7 +54,7 @@ class Word2VecReader { embs.reserve(dimVoc * dimEmb); // Populate output vector with embedding - for(Word word = 0; word < (Word)dimVoc; ++word) { + for(WordIndex word = 0; word < (WordIndex)dimVoc; ++word) { // For words not occuring in the file use uniform distribution if(word2vec.find(word) == word2vec.end()) { auto randVals = randomEmbeddings(dimVoc, dimEmb); diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 962bda182..3d3c8a508 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -151,10 +151,10 @@ class BeamSearchDecoder : public IBeamSearchDecoder { // convert to QuickSAND format alignmentSets.resize(words.size()); for (const auto& p : align) // @TODO: Does the feature_model param max_alignment_links apply here? - alignmentSets[p.tgtPos].insert({p.srcPos, p.prob}); + alignmentSets[p.tgtPos].insert({p.srcPos, p.prob}); } // form hypothesis to return - qsNbest.emplace_back(words, std::move(alignmentSets), score); + qsNbest.emplace_back(toWordIndexVector(words), std::move(alignmentSets), score); } qsNbestBatch.push_back(qsNbest); } diff --git a/src/models/bert.h b/src/models/bert.h index 2ea725143..e40b858ab 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -25,7 +25,7 @@ namespace data { class BertBatch : public CorpusBatch { private: std::vector maskedPositions_; - std::vector maskedWords_; + Words maskedWords_; std::vector sentenceIndices_; std::string maskSymbol_; @@ -33,7 +33,7 @@ class BertBatch : public CorpusBatch { std::string clsSymbol_; // Selects a random word from the vocabulary - std::unique_ptr> randomWord_; + std::unique_ptr> randomWord_; // Selects a random integer between 0 and 99 std::unique_ptr> randomPercent_; @@ -79,7 +79,7 @@ class BertBatch : public CorpusBatch { const auto& vocab = *subBatch->vocab(); // Initialize to sample random vocab id - randomWord_.reset(new std::uniform_int_distribution(0, (Word)vocab.size())); + randomWord_.reset(new std::uniform_int_distribution(0, (WordIndex)vocab.size())); // Intialize to sample random percentage randomPercent_.reset(new std::uniform_real_distribution(0.f, 1.f)); @@ -163,7 +163,7 @@ class BertBatch : public CorpusBatch { } const std::vector& bertMaskedPositions() { return maskedPositions_; } - const std::vector& bertMaskedWords() { return maskedWords_; } + const Words& bertMaskedWords() { return maskedWords_; } const std::vector& bertSentenceIndices() { return sentenceIndices_; } }; @@ -236,7 +236,7 @@ class BertEncoder : public EncoderTransformer { ("dimVocab", 3) // sentence A or sentence B plus padding, @TODO: should rather be a parameter ("dimEmb", dimEmb) .construct(graph_); - signal = sentenceEmbeddings->apply(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); + signal = sentenceEmbeddings->applyIndices(bertBatch->bertSentenceIndices(), {dimWords, dimBatch, dimEmb}); } else { // @TODO: factory for positional embeddings? // constant sinusoidal position embeddings, no backprob @@ -293,7 +293,7 @@ class BertClassifier : public ClassifierBase { // Filled externally, for BERT these are NextSentence prediction labels const auto& classLabels = (*batch)[batchIndex_]->data(); - state->setTargetIndices(graph->indices(classLabels)); + state->setTargetIndices(graph->indices(toWordIndexVector(classLabels))); return state; } @@ -319,7 +319,7 @@ class BertMaskedLM : public ClassifierBase { auto context = encoderStates[0]->getContext(); auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries - auto bertMaskedWords = graph->indices(bertBatch->bertMaskedWords()); // vocab ids of entries that have been masked + auto bertMaskedWords = graph->indices(toWordIndexVector(bertBatch->bertMaskedWords())); // vocab ids of entries that have been masked int dimModel = context->shape()[-1]; int dimBatch = context->shape()[-2]; diff --git a/src/models/costs.h b/src/models/costs.h index bfd62da7b..d186db4dc 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -242,11 +242,11 @@ class Stepwise : public EncoderDecoderBase { virtual Ptr step(Ptr graph, Ptr state, const std::vector& hypIndices, - const std::vector& embIndices, + const Words& words, int dimBatch, int beamSize) override { auto nextState = encdec_->step( - graph, state, hypIndices, embIndices, dimBatch, beamSize); + graph, state, hypIndices, words, dimBatch, beamSize); return cost_->apply(nextState); } diff --git a/src/models/decoder.h b/src/models/decoder.h index 53e383797..c5295621a 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -69,7 +69,7 @@ class DecoderBase { if(shortlist_) { yData = graph->indices(shortlist_->mappedIndices()); } else { - yData = graph->indices(subBatch->data()); + yData = graph->indices(toWordIndexVector(subBatch->data())); } auto yShifted = shift(y, {1, 0, 0}); @@ -81,14 +81,14 @@ class DecoderBase { virtual void embeddingsFromPrediction(Ptr graph, Ptr state, - const std::vector& embIdx, + const Words& words, int dimBatch, int dimBeam) { int dimTrgEmb = opt("dim-emb"); int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; Expr selectedEmbs; - if(embIdx.empty()) { + if(words.empty()) { selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); } else { // embeddings are loaded from model during translation, no fixing required @@ -103,7 +103,7 @@ class DecoderBase { auto yEmb = yEmbFactory.construct(graph); - selectedEmbs = yEmb->apply(embIdx, {dimBeam, 1, dimBatch, dimTrgEmb}); + selectedEmbs = yEmb->apply(words, {dimBeam, 1, dimBatch, dimTrgEmb}); } state->setTargetEmbeddings(selectedEmbs); } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index c4b96cc30..ca9669fde 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -152,7 +152,7 @@ Ptr EncoderDecoder::startState(Ptr graph, Ptr EncoderDecoder::step(Ptr graph, Ptr state, const std::vector& hypIndices, // [beamIndex * activeBatchSize + batchIndex] - const std::vector& embIndices, // [beamIndex * activeBatchSize + batchIndex] + const Words& words, // [beamIndex * activeBatchSize + batchIndex] int dimBatch, int beamSize) { // create updated state that reflects reordering and dropping of hypotheses @@ -160,7 +160,7 @@ Ptr EncoderDecoder::step(Ptr graph, // Fill stte with embeddings based on last prediction decoders_[0]->embeddingsFromPrediction( - graph, state, embIndices, dimBatch, beamSize); + graph, state, words, dimBatch, beamSize); auto nextState = decoders_[0]->step(graph, state); return nextState; diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100644 new mode 100755 index b55e8bd1d..12c58a356 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -42,7 +42,7 @@ class EncoderDecoderBase : public models::ModelBase { virtual Ptr step(Ptr graph, Ptr state, const std::vector& hypIndices, - const std::vector& embIndices, + const Words& words, int dimBatch, int beamSize) = 0; @@ -148,7 +148,7 @@ class EncoderDecoder : public EncoderDecoderBase { virtual Ptr step(Ptr graph, Ptr state, const std::vector& hypIndices, - const std::vector& embIndices, + const Words& words, int dimBatch, int beamSize) override; diff --git a/src/models/transformer.h b/src/models/transformer.h index 1db455c9e..52b07241d 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -73,9 +73,9 @@ class Transformer : public EncoderOrDecoderBase { // fill with increasing numbers until current length or maxPos std::vector positions(dimWords, numPos - 1); for(int i = 0; i < std::min(dimWords, numPos); ++i) - positions[i] = i; + positions[i] = i; // @TODO: use std::iota()? - auto signal = embeddingLayer->apply(positions, {dimWords, 1, dimEmb}); + auto signal = embeddingLayer->applyIndices(positions, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; } else { // @TODO : test if embeddings should be scaled when trainable diff --git a/src/tests/attention_tests.cpp b/src/tests/attention_tests.cpp index 9b1b61859..8dfd1844c 100755 --- a/src/tests/attention_tests.cpp +++ b/src/tests/attention_tests.cpp @@ -49,7 +49,7 @@ void tests(DeviceType type) { {128, dimEmb}, inits::glorot_uniform); - auto input = reshape(rows(emb, vWords), {dimTime, dimBatch, dimEmb}); + auto input = reshape(rows(emb, toWordIndexVector(vWords)), {dimTime, dimBatch, dimEmb}); auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); diff --git a/src/tests/rnn_tests.cpp b/src/tests/rnn_tests.cpp index 726f5ab54..97a7848c2 100755 --- a/src/tests/rnn_tests.cpp +++ b/src/tests/rnn_tests.cpp @@ -202,7 +202,7 @@ void tests(DeviceType type) { {128, dimEmb}, inits::glorot_uniform); - auto input = reshape(rows(emb, vWords), {dimTime, dimBatch, dimEmb}); + auto input = reshape(rows(emb, toWordIndexVector(vWords)), {dimTime, dimBatch, dimEmb}); auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); diff --git a/src/training/validator.h b/src/training/validator.h index a819968b6..657c1fbcb 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -371,7 +371,7 @@ class BertAccuracyValidator : public Validator { auto sentenceLogits = classifierStates[1]->getLogProbs(); const auto& sentenceLabels = bertBatch->back()->data(); - auto count = [=, &correct, &totalLabels](Expr logits, const std::vector& labels) { + auto count = [=, &correct, &totalLabels](Expr logits, const Words& labels) { IndexType cols = logits->shape()[-1]; size_t thisCorrect = 0; size_t thisLabels = labels.size(); @@ -390,7 +390,7 @@ class BertAccuracyValidator : public Validator { bestIndex = j; } } - thisCorrect += (size_t)(bestIndex == labels[i]); + thisCorrect += (size_t)(bestIndex == labels[i].getWordIndex()); } std::unique_lock lock(mutex_); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 798312fed..5643d96d6 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -15,14 +15,14 @@ class BeamSearch { Ptr options_; std::vector> scorers_; size_t beamSize_; - Word trgEosId_ = (Word)-1; - Word trgUnkId_ = (Word)-1; + Word trgEosId_{Word::NONE}; + Word trgUnkId_{Word::NONE}; public: BeamSearch(Ptr options, const std::vector>& scorers, Word trgEosId, - Word trgUnkId = -1) + Word trgUnkId = Word::NONE) : options_(options), scorers_(scorers), beamSize_(options_->has("beam-size") @@ -197,7 +197,7 @@ class BeamSearch { // create constant containing previous path scores for current beam // also create mapping of hyp indices, which are not 1:1 if sentences complete std::vector hypIndices; // [beamIndex * activeBatchSize + batchIndex] backpointers, concatenated over beam positions. Used for reordering hypotheses - std::vector embIndices; + Words predWords; Expr prevPathScores; // [beam, 1, 1, 1] if(first) { // no scores yet @@ -213,11 +213,11 @@ class BeamSearch { if(i < beam.size()) { auto hyp = beam[i]; hypIndices.push_back((IndexType)hyp->GetPrevStateIndex()); // backpointer - embIndices.push_back(hyp->GetWord()); + predWords.push_back(hyp->GetWord()); beamScores.push_back(hyp->GetPathScore()); } else { // dummy hypothesis hypIndices.push_back(0); - embIndices.push_back(0); // (unused) + predWords.push_back(Word::NONE); // (unused) beamScores.push_back(-9999); } } @@ -233,7 +233,7 @@ class BeamSearch { for(size_t i = 0; i < scorers_.size(); ++i) { states[i] = scorers_[i]->step( - graph, states[i], hypIndices, embIndices, dimBatch, (int)localBeamSize); + graph, states[i], hypIndices, predWords, dimBatch, (int)localBeamSize); if(scorers_[i]->getWeight() != 1.f) pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs(); diff --git a/src/translator/scorers.h b/src/translator/scorers.h index 16a066b05..1795c1661 100755 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -35,7 +35,7 @@ class Scorer { virtual Ptr step(Ptr, Ptr, const std::vector&, - const std::vector&, + const Words&, int dimBatch, int beamSize) = 0; @@ -111,12 +111,12 @@ class ScorerWrapper : public Scorer { virtual Ptr step(Ptr graph, Ptr state, const std::vector& hypIndices, - const std::vector& embIndices, + const Words& words, int dimBatch, int beamSize) override { graph->switchParams(getName()); auto wrapperState = std::dynamic_pointer_cast(state); - auto newState = encdec_->step(graph, wrapperState->getState(), hypIndices, embIndices, dimBatch, beamSize); + auto newState = encdec_->step(graph, wrapperState->getState(), hypIndices, words, dimBatch, beamSize); return New(newState); } From f9e63da391c0542af9ed2c87a038060aff178525 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 31 Jan 2019 22:24:26 -0800 Subject: [PATCH 244/838] removed auto-conversion between Word and WordIndex, and dealt with fallout --- src/data/corpus_base.cpp | 4 ++-- src/data/corpus_base.h | 6 +++--- src/data/default_vocab.cpp | 30 ++++++++++++++++-------------- src/data/shortlist.h | 18 +++++++++--------- src/data/text_input.cpp | 2 +- src/data/types.h | 19 +++++++++++-------- src/microsoft/quicksand.cpp | 4 ++-- src/models/bert.h | 2 +- src/tests/attention_tests.cpp | 4 ++-- src/tests/rnn_tests.cpp | 4 ++-- src/training/validator.h | 8 ++++---- src/translator/beam_search.h | 8 ++++---- src/translator/helpers.cpp | 2 +- src/translator/helpers.cu | 2 +- src/translator/hypothesis.h | 2 +- vs/Marian.vcxproj | 2 ++ vs/Marian.vcxproj.filters | 6 ++++++ 17 files changed, 68 insertions(+), 55 deletions(-) mode change 100644 => 100755 src/translator/helpers.cu diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index 74987882f..6e291a289 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -201,11 +201,11 @@ void CorpusBase::addWordsToSentenceTuple(const std::string& line, Words words = vocabs_[batchIndex]->encode(line, /*addEOS =*/ addEOS_[batchIndex], inference_); if(words.empty()) - words.push_back(0); + words.push_back(Word::NONE); if(maxLengthCrop_ && words.size() > maxLength_) { words.resize(maxLength_); - words.back() = 0; + words.back() = Word::NONE; } if(rightLeft_) diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 6614d7d2c..79ac12532 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -128,7 +128,7 @@ class SubBatch { * @param width Number of words in the longest sentence */ SubBatch(size_t size, size_t width, const Ptr& vocab) - : indices_(size * width, 0), + : indices_(size * width, Word::NONE), mask_(size * width, 0), size_(size), width_(width), @@ -322,7 +322,7 @@ class CorpusBatch : public Batch { // set word indices to different values to avoid same hashes // rand() is OK, this does not affect state in any way std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), - [&](Word) -> Word { return rand() % vocabs[batchIndex]->size(); }); + [&](Word) -> Word { return Word::fromWordIndex(rand() % vocabs[batchIndex]->size()); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); batchIndex++; @@ -484,7 +484,7 @@ class CorpusBatch : public Batch { if (vocab) std::cerr << (*vocab)[w] << " "; else - std::cerr << w << " "; // if not loaded then print numeric id instead + std::cerr << w.toString() << " "; // if not loaded then print numeric id instead } std::cerr << std::endl; } diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index b791ec52a..8c02014cd 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -72,8 +72,8 @@ class DefaultVocab : public VocabBase { virtual Word getEosId() const override { return eosId_; } virtual Word getUnkId() const override { return unkId_; } - - const std::string& operator[](Word id) const override { + const std::string& operator[](Word word) const override { + auto id = word.toWordIndex(); ABORT_IF(id >= id2str_.size(), "Unknown word id: {}", id); return id2str_[id]; } @@ -97,7 +97,7 @@ class DefaultVocab : public VocabBase { if(isJson) { YAML::Node vocabNode = YAML::Load(io::InputFileStream(vocabPath)); for(auto&& pair : vocabNode) - vocab.insert({pair.first.as(), Word(pair.second.as())}); + vocab.insert({pair.first.as(), Word::fromWordIndex(pair.second.as())}); } // read from flat text file else { @@ -107,7 +107,7 @@ class DefaultVocab : public VocabBase { ABORT_IF(line.empty(), "DefaultVocabulary file {} must not contain empty lines", vocabPath); - vocab.insert({line, Word(vocab.size())}); + vocab.insert({line, Word::fromWordIndex(vocab.size())}); } ABORT_IF(in.bad(), "DefaultVocabulary file {} could not be read", vocabPath); } @@ -118,7 +118,7 @@ class DefaultVocab : public VocabBase { auto id = pair.second; // note: this requires ids to be sorted by frequency - if(!maxSize || id < (Word)maxSize) { + if(!maxSize || id.toWordIndex() < maxSize) { insertWord(id, str); } } @@ -171,12 +171,13 @@ class DefaultVocab : public VocabBase { // The name backCompatStr is alternatively accepted for Yaml vocabs if id // equals backCompatId. auto getRequiredWordId = [&](const std::string& str, - const std::string& backCompatStr, - Word backCompatId) { + const std::string& backCompatStr, + Word backCompatWord) -> Word { // back compat with Nematus Yaml dicts if(isJson) { // if word id 0 or 1 is either empty or has the Nematus-convention string, // then use it + auto backCompatId = backCompatWord.toWordIndex(); if(backCompatId < id2str_.size() && (id2str_[backCompatId].empty() || id2str_[backCompatId] == backCompatStr)) { @@ -185,7 +186,7 @@ class DefaultVocab : public VocabBase { backCompatStr, backCompatId, str); - return backCompatId; + return backCompatWord; } } auto iter = str2id_.find(str); @@ -232,10 +233,10 @@ class DefaultVocab : public VocabBase { std::sort(vocabVec.begin(), vocabVec.end(), VocabFreqOrderer(counter)); YAML::Node vocabYaml; - vocabYaml.force_insert(DEFAULT_EOS_STR, Word::DEFAULT_EOS_ID.getWordIndex()); - vocabYaml.force_insert(DEFAULT_UNK_STR, Word::DEFAULT_UNK_ID.getWordIndex()); + vocabYaml.force_insert(DEFAULT_EOS_STR, Word::DEFAULT_EOS_ID.toWordIndex()); + vocabYaml.force_insert(DEFAULT_UNK_STR, Word::DEFAULT_UNK_ID.toWordIndex()); - Word maxSpec = 1; + WordIndex maxSpec = 1; auto vocabSize = vocabVec.size(); if(maxSize > maxSpec) vocabSize = std::min(maxSize - maxSpec - 1, vocabVec.size()); @@ -274,12 +275,13 @@ class DefaultVocab : public VocabBase { } // helper to insert a word into str2id_[] and id2str_[] - Word insertWord(Word id, const std::string& str) { - str2id_[str] = id; + Word insertWord(Word word, const std::string& str) { + str2id_[str] = word; + auto id = word.toWordIndex(); if(id >= id2str_.size()) id2str_.resize(id + 1); id2str_[id] = str; - return id; + return word; }; }; diff --git a/src/data/shortlist.h b/src/data/shortlist.h index 50dac8d54..ded1ed5e1 100755 --- a/src/data/shortlist.h +++ b/src/data/shortlist.h @@ -79,12 +79,12 @@ class SampledShortlistGenerator : public ShortlistGenerator { // add all words from ground truth for(auto i : trgBatch->data()) - idxSet.insert(i.getWordIndex()); + idxSet.insert(i.toWordIndex()); // add all words from source if(shared_) for(auto i : srcBatch->data()) - idxSet.insert(i.getWordIndex()); + idxSet.insert(i.toWordIndex()); std::uniform_int_distribution<> dis((int)firstNum_, (int)maxVocab_); while(idxSet.size() < total_ && idxSet.size() < maxVocab_) @@ -106,7 +106,7 @@ class SampledShortlistGenerator : public ShortlistGenerator { std::vector mapped; for(auto i : trgBatch->data()) { // mapped postions for cross-entropy - mapped.push_back(pos[i]); + mapped.push_back(pos[i.toWordIndex()]); } return New(idx, mapped, reverseMap); @@ -138,8 +138,8 @@ class LexicalShortlistGenerator : public ShortlistGenerator { if(src == "NULL" || trg == "NULL") continue; - auto sId = (*srcVocab_)[src].getWordIndex(); - auto tId = (*trgVocab_)[trg].getWordIndex(); + auto sId = (*srcVocab_)[src].toWordIndex(); + auto tId = (*trgVocab_)[trg].toWordIndex(); if(data_.size() <= sId) data_.resize(sId + 1); @@ -212,14 +212,14 @@ class LexicalShortlistGenerator : public ShortlistGenerator { LOG(info, "[data] Saving shortlist dump to {}", prefix + ".{top,dic}"); io::OutputFileStream outTop(prefix + ".top"); for(WordIndex i = 0; i < firstNum_ && i < trgVocab_->size(); ++i) - outTop << (*trgVocab_)[i] << std::endl; + outTop << (*trgVocab_)[Word::fromWordIndex(i)] << std::endl; // Dump translation pairs from dictionary io::OutputFileStream outDic(prefix + ".dic"); for(WordIndex srcId = 0; srcId < data_.size(); srcId++) { for(auto& it : data_[srcId]) { auto trgId = it.first; - outDic << (*srcVocab_)[srcId] << "\t" << (*trgVocab_)[trgId] << std::endl; + outDic << (*srcVocab_)[Word::fromWordIndex(srcId)] << "\t" << (*trgVocab_)[Word::fromWordIndex(trgId)] << std::endl; } } } @@ -235,12 +235,12 @@ class LexicalShortlistGenerator : public ShortlistGenerator { // add all words from ground truth // for(auto i : trgBatch->data()) - // idxSet.insert(i.getWordIndex()); + // idxSet.insert(i.toWordIndex()); // collect unique words form source std::unordered_set srcSet; for(auto i : srcBatch->data()) - srcSet.insert(i); + srcSet.insert(i.toWordIndex()); // add aligned target words for(auto i : srcSet) { diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp index 78fbc7af0..9c694c836 100755 --- a/src/data/text_input.cpp +++ b/src/data/text_input.cpp @@ -48,7 +48,7 @@ SentenceTuple TextInput::next() { if(io::getline(dummyStream, line)) { Words words = vocabs_[i]->encode(line, /*addEOS =*/ true, /*inference =*/ inference_); if(words.empty()) - words.push_back(0); + words.push_back(Word::NONE); tup.push_back(words); } } diff --git a/src/data/types.h b/src/data/types.h index 549b439ea..dedbf11ce 100755 --- a/src/data/types.h +++ b/src/data/types.h @@ -14,20 +14,20 @@ namespace marian { typedef IndexType WordIndex; // WordIndex is used for words or tokens arranged in consecutive order class Word { // Word is an abstraction of a unique id, not necessarily consecutive WordIndex wordId_; + explicit Word(std::size_t wordId) : wordId_((WordIndex)wordId) {} public: - // back compat with WordIndex - Word(std::size_t wordId) : wordId_((WordIndex)wordId) {} // @TODO: make explicit, or make private - operator WordIndex() const { return getWordIndex(); } + static Word fromWordIndex(std::size_t wordId) { return Word(wordId); } + const WordIndex& toWordIndex() const { return wordId_; } + std::string toString() const { return std::to_string(wordId_); } // needed for STL containers Word() : wordId_((WordIndex)-1) {} bool operator==(const Word& other) const { return wordId_ == other.wordId_; } + bool operator!=(const Word& other) const { return !(*this == other); } + bool operator<(const Word& other) const { return wordId_ < other.wordId_; } std::size_t hash() const { return std::hash{}(wordId_); } - // main methods and constants - static Word From(std::size_t wordId) { return Word(wordId); } - const WordIndex& getWordIndex() const { return wordId_; } - + // constants static Word NONE; // @TODO: decide whether we need this, in additional Word() // EOS and UNK are placed in these positions in Marian-generated vocabs static Word DEFAULT_EOS_ID; @@ -39,7 +39,10 @@ typedef std::vector Words; // Helper to map a Word vector to a WordIndex vector static inline std::vector toWordIndexVector(const Words& words) { - return std::vector(words.begin(), words.end()); + std::vector res; + std::transform(words.begin(), words.end(), std::back_inserter(res), + [](const Word& word) -> WordIndex { return word.toWordIndex(); }); + return res; } // names of EOS and UNK symbols diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 3d3c8a508..63f05e738 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -109,7 +109,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { const auto& sent = qsBatch[j]; if(i < sent.size()) { size_t idx = i * batchSize + j; - subBatch->data()[idx] = (unsigned int)sent[i]; + subBatch->data()[idx] = marian::Word::fromWordIndex(sent[i]); subBatch->mask()[idx] = 1; } } @@ -122,7 +122,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { batch->setSentenceIds(sentIds); // decode - auto search = New(options_, scorers_, eos_); + auto search = New(options_, scorers_, marian::Word::fromWordIndex(eos_)); Histories histories = search->search(graph_, batch); // convert to QuickSAND format diff --git a/src/models/bert.h b/src/models/bert.h index e40b858ab..84b74cc33 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -51,7 +51,7 @@ class BertBatch : public CorpusBatch { if (r < 0.1f) { // for 10% of cases return same word return word; } else if (r < 0.2f) { // for 10% return random word - Word randWord = (*randomWord_)(engine); + Word randWord = Word::fromWordIndex((*randomWord_)(engine)); if(dontMask_.count(randWord) > 0) // some words, e.g. [CLS] or , may not be used as random words return mask; // for those, return the mask symbol instead else diff --git a/src/tests/attention_tests.cpp b/src/tests/attention_tests.cpp index 8dfd1844c..6bd31069d 100755 --- a/src/tests/attention_tests.cpp +++ b/src/tests/attention_tests.cpp @@ -12,7 +12,7 @@ void tests(DeviceType type) { Config::seed = 1234; - Words vWords = { + std::vector vWords = { 43, 2, 83, 78, 6, 38, 80, 40, 40, 70, 26, 60, @@ -49,7 +49,7 @@ void tests(DeviceType type) { {128, dimEmb}, inits::glorot_uniform); - auto input = reshape(rows(emb, toWordIndexVector(vWords)), {dimTime, dimBatch, dimEmb}); + auto input = reshape(rows(emb, vWords), {dimTime, dimBatch, dimEmb}); auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); diff --git a/src/tests/rnn_tests.cpp b/src/tests/rnn_tests.cpp index 97a7848c2..2b4fda0b9 100755 --- a/src/tests/rnn_tests.cpp +++ b/src/tests/rnn_tests.cpp @@ -9,7 +9,7 @@ using namespace marian; void tests(DeviceType type) { auto floatApprox = [](float x, float y) { return x == Approx(y).epsilon(0.01); }; - Words vWords = { + std::vector vWords = { 43, 2, 83, 78, 6, 38, 80, 40, 40, 70, 26, 60, @@ -202,7 +202,7 @@ void tests(DeviceType type) { {128, dimEmb}, inits::glorot_uniform); - auto input = reshape(rows(emb, toWordIndexVector(vWords)), {dimTime, dimBatch, dimEmb}); + auto input = reshape(rows(emb, vWords), {dimTime, dimBatch, dimEmb}); auto mask = graph->constant({dimTime, dimBatch, 1}, inits::from_vector(vMask)); diff --git a/src/training/validator.h b/src/training/validator.h index 657c1fbcb..3abb8a9c9 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -279,16 +279,16 @@ class AccuracyValidator : public Validator { for(int i = 0; i < thisLabels; ++i) { // CPU-side Argmax - IndexType bestIndex = 0; + Word bestWord = Word::NONE; float bestValue = std::numeric_limits::lowest(); for(IndexType j = 0; j < cols; ++j) { float currValue = vLogits[i * cols + j]; if(currValue > bestValue) { bestValue = currValue; - bestIndex = j; + bestWord = Word::fromWordIndex(j); } } - thisCorrect += (size_t)(bestIndex == groundTruth[i]); + thisCorrect += (size_t)(bestWord == groundTruth[i]); } std::unique_lock lock(mutex_); @@ -390,7 +390,7 @@ class BertAccuracyValidator : public Validator { bestIndex = j; } } - thisCorrect += (size_t)(bestIndex == labels[i].getWordIndex()); + thisCorrect += (size_t)(bestIndex == labels[i].toWordIndex()); } std::unique_lock lock(mutex_); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 5643d96d6..09d382bfd 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -49,7 +49,7 @@ class BeamSearch { for(size_t i = 0; i < keys.size(); ++i) { // Keys contains indices to vocab items in the entire beam. // Values can be between 0 and beamSize * vocabSize. - Word embIdx = (Word)(keys[i] % vocabSize); + Word embIdx = Word::fromWordIndex(keys[i] % vocabSize); auto beamIdx = i / beamSize; // Retrieve short list for final softmax (based on words aligned @@ -57,7 +57,7 @@ class BeamSearch { // in the sub-selected vocabulary matrix back to their original positions. auto shortlist = scorers_[0]->getShortlist(); if(shortlist) - embIdx = shortlist->reverseMap(embIdx); // @TODO: should reverseMap accept a size_t or a Word? + embIdx = Word::fromWordIndex(shortlist->reverseMap(embIdx.toWordIndex())); // @TODO: should reverseMap accept a size_t or a Word? if(newBeams[beamIdx].size() < beams[beamIdx].size()) { auto& beam = beams[beamIdx]; @@ -85,7 +85,7 @@ class BeamSearch { std::vector breakDown(states.size(), 0); beam[beamHypIdx]->GetScoreBreakdown().resize(states.size(), 0); for(size_t j = 0; j < states.size(); ++j) { - size_t key = embIdx + hypIdxTrans * vocabSize; + size_t key = embIdx.toWordIndex() + hypIdxTrans * vocabSize; breakDown[j] = states[j]->breakDown(key) + beam[beamHypIdx]->GetScoreBreakdown()[j]; } @@ -252,7 +252,7 @@ class BeamSearch { //********************************************************************** // suppress specific symbols if not at right positions - if(trgUnkId_ != -1 && options_->has("allow-unk") + if(trgUnkId_ != Word::NONE && options_->has("allow-unk") && !options_->get("allow-unk")) suppressWord(pathScores, trgUnkId_); for(auto state : states) diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp index ccc450401..f131398c2 100755 --- a/src/translator/helpers.cpp +++ b/src/translator/helpers.cpp @@ -25,7 +25,7 @@ void SetColumn(Tensor in_, size_t col, float value) { } void suppressWord(Expr logProbs, Word id) { - SetColumn(logProbs->val(), id, std::numeric_limits::lowest()); + SetColumn(logProbs->val(), id.toWordIndex(), std::numeric_limits::lowest()); } } // namespace cpu diff --git a/src/translator/helpers.cu b/src/translator/helpers.cu old mode 100644 new mode 100755 index 8b40d8db7..d23c7500f --- a/src/translator/helpers.cu +++ b/src/translator/helpers.cu @@ -38,7 +38,7 @@ void SetColumn(Tensor in_, size_t col, float value) { } void suppressWord(Expr probs, Word id) { - SetColumn(probs->val(), id, std::numeric_limits::lowest()); + SetColumn(probs->val(), id.toWordIndex(), std::numeric_limits::lowest()); } } // namespace gpu } // namespace marian diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index c13ac716b..eaeeaa497 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -8,7 +8,7 @@ namespace marian { class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(0), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(Word::NONE), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index c899c242f..9e161fefe 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -1113,6 +1113,8 @@ true + + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 947bcece8..f690d6cc6 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1857,6 +1857,12 @@ tensors\gpu + + translator + + + translator + From 72c1a3b985448301de522e6e62b8a497efc77f7e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 31 Jan 2019 22:32:56 -0800 Subject: [PATCH 245/838] now distinguishing between Word::NONE and Word::ZERO which is a valid index meant to be masked out --- src/data/corpus_base.cpp | 4 ++-- src/data/corpus_base.h | 2 +- src/data/text_input.cpp | 2 +- src/data/types.h | 1 + src/data/vocab.cpp | 1 + src/translator/beam_search.h | 2 +- src/translator/hypothesis.h | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index 6e291a289..8253e5100 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -201,11 +201,11 @@ void CorpusBase::addWordsToSentenceTuple(const std::string& line, Words words = vocabs_[batchIndex]->encode(line, /*addEOS =*/ addEOS_[batchIndex], inference_); if(words.empty()) - words.push_back(Word::NONE); + words.push_back(Word::ZERO); if(maxLengthCrop_ && words.size() > maxLength_) { words.resize(maxLength_); - words.back() = Word::NONE; + words.back() = Word::ZERO; // @TODO: What does this do? Meant as sent-end token?? } if(rightLeft_) diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 79ac12532..683f41937 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -128,7 +128,7 @@ class SubBatch { * @param width Number of words in the longest sentence */ SubBatch(size_t size, size_t width, const Ptr& vocab) - : indices_(size * width, Word::NONE), + : indices_(size * width, Word::ZERO), // note: for gaps, we must use a valid index mask_(size * width, 0), size_(size), width_(width), diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp index 9c694c836..59f16f1bf 100755 --- a/src/data/text_input.cpp +++ b/src/data/text_input.cpp @@ -48,7 +48,7 @@ SentenceTuple TextInput::next() { if(io::getline(dummyStream, line)) { Words words = vocabs_[i]->encode(line, /*addEOS =*/ true, /*inference =*/ inference_); if(words.empty()) - words.push_back(Word::NONE); + words.push_back(Word::ZERO); // @TODO: What is this for? @BUGBUG: addEOS=true, so this can never happen, right? tup.push_back(words); } } diff --git a/src/data/types.h b/src/data/types.h index dedbf11ce..d204beb64 100755 --- a/src/data/types.h +++ b/src/data/types.h @@ -29,6 +29,7 @@ class Word { // Word is an abstraction of a unique id, not ne // constants static Word NONE; // @TODO: decide whether we need this, in additional Word() + static Word ZERO; // an invalid word that nevertheless can safely be looked up (and then masked out) // EOS and UNK are placed in these positions in Marian-generated vocabs static Word DEFAULT_EOS_ID; static Word DEFAULT_UNK_ID; diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 8118ea24f..7844ecfa4 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -4,6 +4,7 @@ namespace marian { Word Word::NONE = Word(); +Word Word::ZERO = Word(0); Word Word::DEFAULT_EOS_ID = Word(0); Word Word::DEFAULT_UNK_ID = Word(1); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 09d382bfd..c80865f73 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -217,7 +217,7 @@ class BeamSearch { beamScores.push_back(hyp->GetPathScore()); } else { // dummy hypothesis hypIndices.push_back(0); - predWords.push_back(Word::NONE); // (unused) + predWords.push_back(Word::ZERO); // (unused) beamScores.push_back(-9999); } } diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index eaeeaa497..ab114470e 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -8,7 +8,7 @@ namespace marian { class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(Word::NONE), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(Word::ZERO), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, From 75604a140130b35653be65a64fff043c92e65569 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 1 Feb 2019 15:41:26 -0800 Subject: [PATCH 246/838] addressed PR feedback --- src/graph/expression_graph.h | 9 +-------- src/layers/generic.h | 17 ++++++++++------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h index 22908d96b..dfd00a9a5 100755 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -316,13 +316,6 @@ class ExpressionGraph : public std::enable_shared_from_this { dot.close(); } - Expr tryFindParam(const std::string& pname) const { - std::string name = pname; - if(!namespace_.empty()) - name = namespace_ + "::" + name; - return params_->get(name); - } - Expr param(const std::string& pname, const Shape& shape, const NodeInitializer& init, @@ -407,7 +400,7 @@ class ExpressionGraph : public std::enable_shared_from_this { auto e = params_->get(name); if(e) return e; - return Expr(); + return Expr(); // @TODO: how is this different from just returning 'e'? } Ptr& params() { return params_; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 8e03efd08..d050a55f0 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -128,6 +128,7 @@ class Output : public LayerBase, public IUnaryLayer { // parameters held by this layer Expr Wt_; // weight matrix is stored transposed for efficiency Expr b_; + bool isLegacyUntransposedW{false}; // legacy-model emulation: W is stored in non-transposed form Expr cachedShortWt_; // short-listed version, cached (cleared by clear()) Expr cachedShortb_; // these match the current value of shortlist_ @@ -174,9 +175,11 @@ class Output : public LayerBase, public IUnaryLayer { if(tiedParam_) { Wt_ = tiedParam_; } else { - auto W = graph_->tryFindParam(name + "_W"); // support of legacy models that did not transpose - if (W) - Wt_ = transpose(W); // legacy + auto W = graph_->get(name + "_W"); // support of legacy models that did not transpose + if (W) { + Wt_ = W; + isLegacyUntransposedW = true; + } else // this is the regular case: Wt_ = graph_->param(name + "_Wt", {input->shape()[-1], dim}, inits::glorot_uniform); } @@ -185,13 +188,13 @@ class Output : public LayerBase, public IUnaryLayer { if (shortlist_) { if (!cachedShortWt_) { // short versions of parameters are cached within one batch, then clear()ed - cachedShortWt_ = rows(Wt_, shortlist_->indices()); - cachedShortb_ = cols(b_ , shortlist_->indices()); + cachedShortWt_ = index_select(Wt_, isLegacyUntransposedW ? -1 : 0, shortlist_->indices()); + cachedShortb_ = index_select(b_ , -1, shortlist_->indices()); } - return affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/true); + return affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true); } else - return affine(input, Wt_, b_, false, /*transB=*/true); + return affine(input, Wt_, b_, false, /*transB=*/isLegacyUntransposedW ? false : true); } virtual Expr apply(const std::vector& /*inputs*/) override { From f07af897d9586f37fc72b9b374198837dc646022 Mon Sep 17 00:00:00 2001 From: Alham Fikri Aji Date: Sat, 2 Feb 2019 08:25:38 +0000 Subject: [PATCH 247/838] reset gradients in default comm. --- src/training/communicator.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/training/communicator.h b/src/training/communicator.h index 4190401a8..6ccce204f 100644 --- a/src/training/communicator.h +++ b/src/training/communicator.h @@ -175,7 +175,17 @@ class DefaultCommunicator : public ICommunicator { } }; + // reset gradients outside current shard + auto reset = [this, shardSize](size_t idx, size_t begin, size_t end) { + auto grad = graphs_[idx]->params()->grads(); + if (begin > 0) + grad->subtensor(0, begin)->set(0); + if (end < grad->size()) + grad->subtensor(end, grad->size()-end)->set(0); + }; + foreach(scatter); + foreach(reset); } void allGatherParams() const override { From 3e9c10e95c3bb41b3ca35a55c5489c058f0f3a61 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 3 Feb 2019 14:50:40 -0800 Subject: [PATCH 248/838] legacy mode for Output must still call param() to get it into the graph --- src/layers/generic.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index d050a55f0..5841b8047 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -175,9 +175,8 @@ class Output : public LayerBase, public IUnaryLayer { if(tiedParam_) { Wt_ = tiedParam_; } else { - auto W = graph_->get(name + "_W"); // support of legacy models that did not transpose - if (W) { - Wt_ = W; + if (graph_->get(name + "_W")) { // support of legacy models that did not transpose + Wt_ = graph_->param(name + "_Wt", {dim, input->shape()[-1]}, inits::glorot_uniform); isLegacyUntransposedW = true; } else // this is the regular case: From 7832b3be911ea6a77692f592422cb11b4c35a1a4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 3 Feb 2019 14:50:57 -0800 Subject: [PATCH 249/838] legacy mode for Output must still call param() to get it into the graph --- src/layers/generic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index 5841b8047..6e12bc073 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -176,7 +176,7 @@ class Output : public LayerBase, public IUnaryLayer { Wt_ = tiedParam_; } else { if (graph_->get(name + "_W")) { // support of legacy models that did not transpose - Wt_ = graph_->param(name + "_Wt", {dim, input->shape()[-1]}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_W", {dim, input->shape()[-1]}, inits::glorot_uniform); isLegacyUntransposedW = true; } else // this is the regular case: From 0c14096d7f0a44185db2ba6b327b041f33be23c1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 3 Feb 2019 15:51:41 -0800 Subject: [PATCH 250/838] (comments) --- src/graph/expression_operators.cpp | 7 ++++++- src/layers/generic.cpp | 18 +++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 6a07611dc..b74b822ed 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -250,7 +250,12 @@ Expr gather(Expr a, int axis, Expr indices) { return Expression(a, axis, indices); } -// index_select() -- gather arbitrary elements along an axis; unbatched (indices are specified as a 1D vector) +// index_select() -- gather arbitrary elements along an axis from an unbatched +// input 'a'. Indices are specified as a 1D vector. +// This is used e.g. for embedding lookup. +// Note: To use a batch of index vectors, reshape them into a single vector, +// call index_select(), then reshape the result back. Reshapes are cheap. +// This function has the same semantics as PyTorch operation of the same name. Expr index_select(Expr a, int axis, Expr indices) { ABORT_IF(indices->shape().size() != 1, "Indices must be a 1D tensor"); // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index fb618686d..5fa5d3bf0 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -197,12 +197,10 @@ namespace marian { auto factorIndex = rows(factorIndexMatrix, indices); // [B... * 1(Ug)] map word indices to factor indices (indices into factorLogits) auto factorMask = rows(factorMaskMatrix, indices); // [B... * 1] flag whether word has the factor in the first place auto factorLogits = logits_[g]; // [B... * Ug] - // @TODO: no need to normalize factors before here // For each location in [B...] select [indices[B...]]. If not using factor, select [0] and mask it out next. auto factorLoss = lossFn(factorLogits, factorIndex); // [B... x 1] - auto xxx = reshape(factorMask, factorLoss->shape()); //factorLoss->debug("factorLoss"); - factorLoss = factorLoss * xxx;// reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor + factorLoss = factorLoss * reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor loss = loss ? (loss + factorLoss) : factorLoss; // [B... x 1] } return loss; @@ -215,10 +213,10 @@ namespace marian { return logits_.front(); } - // lazily compute combined logits from factors + // compute normalized factor log probs std::vector logProbs(logits_.size()); for (size_t g = 0; g < logits_.size(); g++) - logProbs[g] = logsoftmax(logits_[g]); + logProbs[g] = logsoftmax(logits_[g]); auto y = concatenate(logProbs, /*axis=*/ -1); // sum up the unit logits across factors for each target word @@ -300,7 +298,7 @@ namespace marian { else if (embeddingFactorMapping_) { auto graph = input->graph(); - // project each factor + // project each factor separately auto numGroups = embeddingFactorMapping_->getNumGroups(); std::vector allLogits(numGroups); for (size_t g = 0; g < numGroups; g++) { @@ -309,14 +307,8 @@ namespace marian { // slice this group's section out of W_ // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. auto factorW = slice(W_, transposeW_ ? 0 : -1, Slice((int)range.first, (int)range.second)); - auto factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix + auto factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix; but shotlists use cols() in, which requires a matrix auto factorLogits = affine(input, factorW, factorB, false, transposeW_); // [B... x U] factor logits - // log-linear weight - // @TODO: The weight should not be needed, since it could learn it into the embeddings. Maybe it doesn't. - // @TODO? If we move the weight before affine(), it would use less memory at least for the main factor. - //auto name = options_->get("prefix"); - //auto llWeight = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - //factorLogits = factorLogits * llWeight; // -a constant, which is OK for logits allLogits[g] = factorLogits; } return Logits(std::move(allLogits), embeddingFactorMapping_); From 37e6e680e12797c80947cfc4f1515cfafc8c3b56 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 12:49:10 -0800 Subject: [PATCH 251/838] invert dims when parameter tying is not used --- src/layers/generic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index 6e12bc073..d13a920c3 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -176,11 +176,11 @@ class Output : public LayerBase, public IUnaryLayer { Wt_ = tiedParam_; } else { if (graph_->get(name + "_W")) { // support of legacy models that did not transpose - Wt_ = graph_->param(name + "_W", {dim, input->shape()[-1]}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform); isLegacyUntransposedW = true; } else // this is the regular case: - Wt_ = graph_->param(name + "_Wt", {input->shape()[-1], dim}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_Wt", {dim, input->shape()[-1]}, inits::glorot_uniform); } b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); } From 4634c5c660a6bc5690103c42936f39308d7568ce Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 4 Feb 2019 13:15:46 -0800 Subject: [PATCH 252/838] brought back loss-function aggregation over factors --- src/layers/generic.cpp | 45 ++++++++++++++++++------------------------ src/layers/generic.h | 13 ++++-------- src/layers/loss.h | 39 ++++++++++++++++++------------------ 3 files changed, 43 insertions(+), 54 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 94b40d19e..5a614621c 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -1,6 +1,7 @@ #include "marian.h" #include "layers/generic.h" +#include "layers/loss.h" using std::size_t; // not sure why this is needed @@ -175,12 +176,16 @@ namespace marian { //std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; - Ptr Logits::applyLossFunction(Expr indices, const std::function(Ptr/*logits*/, Expr/*indices*/)>& lossFn) const { + Logits::Logits(Expr logits) : Logits(New(logits, nullptr)) {} // single-output constructor from Expr only (RationalLoss has no count) + + // This function assumes that the object holds one or more factor logits. + // It applies the supplied loss function to each, and then returns the aggregate loss over all factors. + Expr Logits::applyLossFunction(Expr indices, const std::function& lossFn) const { LOG_ONCE(info, "[logits] applyLossFunction() for {} factors", logits_.size()); ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!embeddingFactorMapping_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); - return lossFn(logits_.front(), indices); + return lossFn(logits_.front()->loss(), indices); } // accumulate all CEs for all words that have the factor @@ -198,13 +203,14 @@ namespace marian { auto factorMask = rows(factorMaskMatrix, indices); // [B... * 1] flag whether word has the factor in the first place auto factorLogits = logits_[g]; // [B... * Ug] // For each location in [B...] select [indices[B...]]. If not using factor, select [0] and mask it out next. - auto factorLoss = lossFn(factorLogits, factorIndex)->loss(); // [B... x 1] + auto factorLoss = lossFn(factorLogits->loss(), factorIndex); // [B... x 1] factorLoss = factorLoss * reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor loss = loss ? (loss + factorLoss) : factorLoss; // [B... x 1] } - return New(loss, logits_.front()->count()); + return loss; } + // This function assumes this object holds a single factor that represents a rational loss (with count). Ptr Logits::getRationalLoss() const { //return New(getLogits(), logits_.front()->count()); ABORT_IF(logits_.size() != 1 || embeddingFactorMapping_, "getRationalLoss() cannot be used on multi-factor outputs"); @@ -212,6 +218,8 @@ namespace marian { return logits_.front(); } + // This function assumes that the object holds one or more factor logits, which are summed up + // into output-vocab logits according to the factored model (with correct normalization of factors). Expr Logits::getLogits() const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!embeddingFactorMapping_) { @@ -237,28 +245,13 @@ namespace marian { /*transB=*/ true); // -> [B x V] return y; -#if 0 - // denominators - const auto& mVecs = embeddingFactorMapping_->mVecs_; - for (size_t g = 0; g < numGroups; g++) { - auto range = groupRanges[g]; - // y: [B... x U] - // m: [1 x U] // ones at positions of group members - // need to compute log denominator over y[range] and subtract it from y[range] - //auto groupY = slice(y, /*axis=*/-1, Slice((int)range.first, (int)range.second)); // [B... x Ug] - //auto groupZ = logsumexp(groupY, /*axis=*/-1); // [B... x 1] - ////auto groupZ = slice(groupY - logsoftmax(groupY), /*axis=*/-1, 0); // [B... x 1] - const auto& mVec = mVecs[g]; - auto m = graph->constant({ 1, (int)mVec.size() }, inits::from_vector(mVec)); // [1 x U] - //auto Z = dot(groupZ, m); // [B... x U] - //y = y - Z; - // and a log-linear weight - auto name = options_->get("prefix"); - auto groupLLWeights[g] = graph->param(name + "_llWeight_" + std::to_string(g), {}, inits::from_value(1.0f)); - y = y * ((groupLLWeights[g] - 1) * m + 1); - // @BUGBUG: Global softmax no longer normalizes, due to words that lack some factors. - } -#endif + } + + Logits Logits::withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ + std::vector> newLogits; + for (const auto& l : logits_) + newLogits.emplace_back(New(l->loss(), count)); + return Logits(std::move(newLogits), embeddingFactorMapping_); } namespace mlp { diff --git a/src/layers/generic.h b/src/layers/generic.h index 651eeca5b..f335cbf99 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -4,7 +4,6 @@ #include "data/shortlist.h" #include "layers/factory.h" -#include "layers/loss.h" namespace marian { namespace mlp { @@ -63,6 +62,7 @@ class EmbeddingFactorMapping; // @HACK: Frank's quick implementation of factored outputs. To be re-thought once it works. // Output layer returns a Logits object, which is able to compute some things on the fly // for factored embeddings. +class RationalLoss; class Logits { Logits& operator=(const Logits& other) = default; public: @@ -70,12 +70,12 @@ class Logits { Logits(Ptr logits) { // single-output constructor logits_.push_back(logits); } - Logits(Expr logits) : Logits(New(logits, nullptr)) {} // single-output constructor from Expr only (RationalLoss has no count) + Logits(Expr logits); // single-output constructor from Expr only (RationalLoss has no count) Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), embeddingFactorMapping_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors Ptr getRationalLoss() const; // assume it holds a loss: get that - Ptr applyLossFunction(Expr indices, const std::function(Ptr/*logits*/,Expr/*indices*/)>& lossFn) const; + Expr applyLossFunction(Expr indices, const std::function& lossFn) const; void assign(const Logits& other) { //ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), // "Logits assignment cannot change number of factors"); @@ -83,12 +83,7 @@ class Logits { } size_t getNumFactors() const { return logits_.size(); } bool empty() const { return logits_.empty(); } - Logits withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ - std::vector> newLogits; - for (const auto& l : logits_) - newLogits.emplace_back(New(l->loss(), count)); - return Logits(std::move(newLogits), embeddingFactorMapping_); - } + Logits withCounts(const Expr& count) const; // create new Logits with 'count' implanted into all logits_ private: // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. std::vector> logits_; diff --git a/src/layers/loss.h b/src/layers/loss.h index ea126c2ad..66a702c7d 100755 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -1,6 +1,7 @@ #pragma once #include "graph/expression_operators.h" +#include "layers/generic.h" // for Logits (Frank's factor hack) namespace marian { @@ -269,7 +270,7 @@ class LabelwiseLoss { protected: std::vector axes_; - virtual Expr compute(Expr logits, Expr labelIndices, + virtual Expr compute(Logits logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) = 0; // label counts are available, reduce together with loss to obtain counts @@ -304,7 +305,7 @@ class LabelwiseLoss { LabelwiseLoss(const std::vector& axes) : axes_(axes) { } - virtual RationalLoss apply(Expr logits, Expr labelIndices, + virtual RationalLoss apply(Logits logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) { Expr loss = compute(logits, labelIndices, mask, labelWeights); @@ -331,23 +332,23 @@ class CrossEntropyLoss : public LabelwiseLoss { protected: float labelSmoothing_; // interpolation factor for label smoothing, see below - virtual Expr compute(Expr logits, Expr labelIndices, + virtual Expr compute(Logits logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { - Expr ce = cross_entropy(logits, labelIndices); - - if(labelSmoothing_ > 0) { - // @TODO: add this to CE kernels instead - - // Label smoothing (see https://arxiv.org/pdf/1512.00567.pdf, section 7) - // We compute smoothed H(q',p) = (1 - eps) * H(q,p) + eps * H(u,p) where H(q,p) is the normal cross-entropy - // and H(u,p) penalizes deviation of p from u, u being uniform distribution over vocab V => u_v = 1/|V|. - // H(u,p) = - \sum_{v \in V} u_v * \log p_v = - 1/|V| \sum_{v \in V} \log \softmax_v => -mean(logsoftmax(logits)) - // ceq = -H(u,p) - avoid one kernel call by negating in the interpolation below - Expr ceq = mean(logsoftmax(logits), /*axis=*/ -1); - - // H(q',p) = (1 - eps) * H(q,p) - eps * -H(u,p) - ce = (1 - labelSmoothing_) * ce - labelSmoothing_ * ceq; - } + // logits may be factored; in that case, the getLoss() function computes one loss for each, and sums them up + auto ce = logits.applyLossFunction(labelIndices, [&](Expr logits, Expr indices) { + Expr ce = cross_entropy(logits, indices); + if (labelSmoothing_ > 0) { + // ce = -sum_i y^_i log y_i(h) + // with smoothing: + // ce' = -sum_i ((1-labelSmoothing_) y^_i + labelSmoothing_/N) log y_i(h) + // = -(1-labelSmoothing_) sum_i y^_i log y_i(h) - labelSmoothing_ mean_i log y_i(h) + // = (1-labelSmoothing_) ce - labelSmoothing_ mean_i log y_i(h) + auto ceqNeg = mean(logits, /*axis=*/ -1) - logsumexp(logits, /*axis=*/ -1); + ce = (1 - labelSmoothing_) * ce - labelSmoothing_ * ceqNeg; + //ce = ce - labelSmoothing_ * (ce + ceqNeg); // writing it this way saves one op :) + } + return ce; + }); if(mask) ce = ce * mask; @@ -367,7 +368,7 @@ class RescorerLoss : public CrossEntropyLoss { // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} - virtual RationalLoss apply(Expr logits, Expr labelIndices, + virtual RationalLoss apply(Logits logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { auto ce = CrossEntropyLoss::apply(logits, labelIndices, mask, labelWeights); return RationalLoss(ce.loss(), ce.count()); From 5da979a893faacc2b44f300e51f70b05352522e2 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 13:27:10 -0800 Subject: [PATCH 253/838] initialize output weights taking into account fanIn and fanOut for glorot_uniform --- src/layers/generic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index d13a920c3..2aa047865 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -176,11 +176,11 @@ class Output : public LayerBase, public IUnaryLayer { Wt_ = tiedParam_; } else { if (graph_->get(name + "_W")) { // support of legacy models that did not transpose - Wt_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_W", {input->shape()[-1], dim}, inits::glorot_uniform2(/*fanIn=*/true, /*fanOut=*/false)); // @TODO: unify initializers, already done in other branch isLegacyUntransposedW = true; } else // this is the regular case: - Wt_ = graph_->param(name + "_Wt", {dim, input->shape()[-1]}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_Wt", {dim, input->shape()[-1]}, inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true)); // @TODO: unify initializers, already done in other branch } b_ = graph_->param(name + "_b", {1, dim}, inits::zeros); } From ed0be0026f80768c8272ff44651662a750d58a80 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 4 Feb 2019 14:01:07 -0800 Subject: [PATCH 254/838] rolled back to original initialization of Embedding matrices, for comparability --- src/layers/generic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5a614621c..2c85e8b40 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -332,7 +332,8 @@ namespace marian { } // Embedding layer initialization should depend only on embedding size, hence fanIn=false - NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); + //NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); + NodeInitializer initFunc = inits::glorot_uniform; if (options_->has("embFile")) { std::string file = opt("embFile"); if (!file.empty()) { From 57e3d782e0b65703ba7743377ca2ce6c351071a9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 4 Feb 2019 16:10:47 -0800 Subject: [PATCH 255/838] renamed VocabBase to IVocab --- src/data/default_vocab.cpp | 6 +++--- src/data/sentencepiece_vocab.cpp | 4 ++-- src/data/vocab.cpp | 8 ++++---- src/data/vocab.h | 4 ++-- src/data/vocab_base.h | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/data/vocab_base.h diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index a0db64edd..bb91fab36 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -15,7 +15,7 @@ namespace marian { -class DefaultVocab : public VocabBase { +class DefaultVocab : public IVocab { protected: typedef std::map Str2Id; Str2Id str2id_; @@ -319,11 +319,11 @@ class ClassVocab : public DefaultVocab { } }; -Ptr createDefaultVocab() { +Ptr createDefaultVocab() { return New(); } -Ptr createClassVocab() { +Ptr createClassVocab() { return New(); } diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp index 4457dc064..9af727565 100755 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -19,7 +19,7 @@ namespace marian { #ifdef USE_SENTENCEPIECE // Wrapper around https://github.com/google/sentencepiece -class SentencePieceVocab : public VocabBase { +class SentencePieceVocab : public IVocab { private: // Actual SentencePiece processor object UPtr spm_; @@ -252,7 +252,7 @@ class SentencePieceVocab : public VocabBase { }; #endif // USE_SENTENCEPIECE -Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { +Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { bool isSentencePiece = regex::regex_search(vocabPath, regex::regex("\\.(spm)$")); if(isSentencePiece) { #ifdef USE_SENTENCEPIECE diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 7844ecfa4..8094c699c 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -8,12 +8,12 @@ Word Word::ZERO = Word(0); Word Word::DEFAULT_EOS_ID = Word(0); Word Word::DEFAULT_UNK_ID = Word(1); -Ptr createDefaultVocab(); -Ptr createClassVocab(); -Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); +Ptr createDefaultVocab(); +Ptr createClassVocab(); +Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); // @TODO: make each vocab peek on type -Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { +Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); if(vocab) { return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it diff --git a/src/data/vocab.h b/src/data/vocab.h index e82673ecc..afe79d1f0 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -7,7 +7,7 @@ namespace marian { -class VocabBase; +class IVocab; // Wrapper around vocabulary types. Can choose underlying // vocabulary implementation (vImpl_) based on speficied path @@ -17,7 +17,7 @@ class VocabBase; // * SentencePiece with suffix *.spm (works, but has to be created outside Marian) class Vocab { private: - Ptr vImpl_; + Ptr vImpl_; Ptr options_; size_t batchIndex_; diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h old mode 100644 new mode 100755 index 2050792f3..9a82131e8 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -7,7 +7,7 @@ namespace marian { -class VocabBase { +class IVocab { public: virtual size_t load(const std::string& vocabPath, size_t maxSize = 0) = 0; From 41117fb4326d60d18f2fbc48a26f446d66ee4543 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 4 Feb 2019 16:34:26 -0800 Subject: [PATCH 256/838] abstracted Expr State::targetIndices_ to Words targetWords_, in prep for factored word representations --- src/layers/loss.h | 14 ++++++++------ src/models/bert.h | 8 ++++---- src/models/costs.h | 4 ++-- src/models/decoder.h | 14 +++++++------- src/models/encoder_decoder.cpp | 2 +- src/models/states.h | 16 ++++++---------- vs/Marian.vcxproj | 1 + vs/Marian.vcxproj.filters | 3 +++ 8 files changed, 32 insertions(+), 30 deletions(-) mode change 100644 => 100755 src/layers/loss.h diff --git a/src/layers/loss.h b/src/layers/loss.h old mode 100644 new mode 100755 index 4a28de7b2..112d0b52d --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -1,6 +1,7 @@ #pragma once #include "graph/expression_operators.h" +#include "data/types.h" namespace marian { @@ -267,7 +268,7 @@ class LabelwiseLoss { protected: std::vector axes_; - virtual Expr compute(Expr logits, Expr labelIndices, + virtual Expr compute(Expr logits, const Words& labels, Expr mask = nullptr, Expr labelWeights = nullptr) = 0; // label counts are available, reduce together with loss to obtain counts @@ -302,9 +303,9 @@ class LabelwiseLoss { LabelwiseLoss(const std::vector& axes) : axes_(axes) { } - virtual RationalLoss apply(Expr logits, Expr labelIndices, + virtual RationalLoss apply(Expr logits, const Words& labels, Expr mask = nullptr, Expr labelWeights = nullptr) { - Expr loss = compute(logits, labelIndices, mask, labelWeights); + Expr loss = compute(logits, labels, mask, labelWeights); if(mask) return reduce(loss, mask); // mask can be used as element-wise label count with broadcasting @@ -329,8 +330,9 @@ class CrossEntropyLoss : public LabelwiseLoss { protected: float labelSmoothing_; // interpolation factor for label smoothing, see below - virtual Expr compute(Expr logits, Expr labelIndices, + virtual Expr compute(Expr logits, const Words& labels, Expr mask = nullptr, Expr labelWeights = nullptr) override { + auto labelIndices = logits->graph()->indices(labels); Expr ce = cross_entropy(logits, labelIndices); if(labelSmoothing_ > 0) { @@ -365,9 +367,9 @@ class RescorerLoss : public CrossEntropyLoss { // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} - virtual RationalLoss apply(Expr logits, Expr labelIndices, + virtual RationalLoss apply(Expr logits, const Words& labels, Expr mask = nullptr, Expr labelWeights = nullptr) override { - auto ce = CrossEntropyLoss::apply(logits, labelIndices, mask, labelWeights); + auto ce = CrossEntropyLoss::apply(logits, labels, mask, labelWeights); return RationalLoss(ce.loss(), ce.count()); } }; diff --git a/src/models/bert.h b/src/models/bert.h index c56eb2977..0c54717df 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -291,7 +291,7 @@ class BertClassifier : public ClassifierBase { // Filled externally, for BERT these are NextSentence prediction labels const auto& classLabels = (*batch)[batchIndex_]->data(); - state->setTargetIndices(graph->indices(classLabels)); + state->setTargetWords(classLabels); return state; } @@ -316,8 +316,8 @@ class BertMaskedLM : public ClassifierBase { auto context = encoderStates[0]->getContext(); - auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries - auto bertMaskedWords = graph->indices(bertBatch->bertMaskedWords()); // vocab ids of entries that have been masked + auto bertMaskedPositions = graph->indices(bertBatch->bertMaskedPositions()); // positions in batch of masked entries + const auto& bertMaskedWords = bertBatch->bertMaskedWords(); // vocab ids of entries that have been masked int dimModel = context->shape()[-1]; int dimBatch = context->shape()[-2]; @@ -360,7 +360,7 @@ class BertMaskedLM : public ClassifierBase { auto state = New(); state->setLogProbs(logits); - state->setTargetIndices(bertMaskedWords); + state->setTargetWords(bertMaskedWords); return state; } diff --git a/src/models/costs.h b/src/models/costs.h index bfd62da7b..528bf838f 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -69,7 +69,7 @@ class EncoderDecoderCE : public CostBase { // @TODO: adapt to multi-objective training with multiple decoders auto partialLoss = loss_->apply(state->getLogProbs(), - state->getTargetIndices(), + state->getTargetWords(), state->getTargetMask(), weights); multiLoss->push_back(partialLoss); @@ -119,7 +119,7 @@ class EncoderClassifierCE : public CostBase { Ptr multiLoss = newMultiLoss(options_); for(int i = 0; i < states.size(); ++i) { auto partialLoss = loss_->apply(states[i]->getLogProbs(), - states[i]->getTargetIndices(), + states[i]->getTargetWords(), /*mask=*/nullptr, /*weights=*/nullptr); multiLoss->push_back(partialLoss); diff --git a/src/models/decoder.h b/src/models/decoder.h index 53e383797..c8181fddb 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -65,18 +65,18 @@ class DecoderBase { Expr y, yMask; std::tie (y, yMask) = yEmb->apply(subBatch); - Expr yData; - if(shortlist_) { - yData = graph->indices(shortlist_->mappedIndices()); - } else { - yData = graph->indices(subBatch->data()); - } + const Words& data = + /*if*/ (shortlist_) ? + shortlist_->mappedIndices() + /*else*/ : + subBatch->data(); + Expr yData = graph->indices(data); auto yShifted = shift(y, {1, 0, 0}); state->setTargetEmbeddings(yShifted); state->setTargetMask(yMask); - state->setTargetIndices(yData); + state->setTargetWords(data); } virtual void embeddingsFromPrediction(Ptr graph, diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index c4b96cc30..af73ee55f 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -179,7 +179,7 @@ Ptr EncoderDecoder::stepAll(Ptr graph, decoders_[0]->embeddingsFromBatch(graph, state, batch); auto nextState = decoders_[0]->step(graph, state); nextState->setTargetMask(state->getTargetMask()); - nextState->setTargetIndices(state->getTargetIndices()); + nextState->setTargetWords(state->getTargetWords()); return nextState; } diff --git a/src/models/states.h b/src/models/states.h index 8e3972210..81ca693d5 100755 --- a/src/models/states.h +++ b/src/models/states.h @@ -35,7 +35,7 @@ class DecoderState { Expr targetEmbeddings_; Expr targetMask_; - Expr targetIndices_; + Words targetWords_; // Keep track of current target token position during translation size_t position_{0}; @@ -75,11 +75,8 @@ class DecoderState { targetEmbeddings_ = targetEmbeddings; } - virtual Expr getTargetIndices() const { return targetIndices_; }; - - virtual void setTargetIndices(Expr targetIndices) { - targetIndices_ = targetIndices; - } + virtual const Words& getTargetWords() const { return targetWords_; }; + virtual void setTargetWords(const Words& targetWords) { targetWords_ = targetWords; } virtual Expr getTargetMask() const { return targetMask_; }; @@ -111,15 +108,14 @@ class ClassifierState { Ptr batch_; Expr targetMask_; - Expr targetIndices_; + Words targetWords_; public: virtual Expr getLogProbs() const { return logProbs_; } virtual void setLogProbs(Expr logProbs) { logProbs_ = logProbs; } - virtual Expr getTargetIndices() const { return targetIndices_; }; - - virtual void setTargetIndices(Expr targetIndices) { targetIndices_ = targetIndices; } + virtual const Words& getTargetWords() const { return targetWords_; }; + virtual void setTargetWords(const Words& targetWords) { targetWords_ = targetWords; } virtual Expr getTargetMask() const { return targetMask_; }; diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index a6c560d3a..3cb727a33 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -894,6 +894,7 @@ + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index d9e56843f..a5c6985c3 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1517,6 +1517,9 @@ examples\mnist + + models + From 1b9dad403d59ce3bbc72f6268c68c0711fc08880 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 4 Feb 2019 17:09:34 -0800 Subject: [PATCH 257/838] renamed VocabBase to IVocab, since it is a pure interface --- src/data/default_vocab.cpp | 6 +++--- src/data/sentencepiece_vocab.cpp | 4 ++-- src/data/vocab.cpp | 8 ++++---- src/data/vocab.h | 4 ++-- src/data/vocab_base.h | 2 +- vs/Marian.vcxproj | 1 + vs/Marian.vcxproj.filters | 5 ++++- 7 files changed, 17 insertions(+), 13 deletions(-) mode change 100644 => 100755 src/data/vocab_base.h diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 8c02014cd..8eb0c6571 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -15,7 +15,7 @@ namespace marian { -class DefaultVocab : public VocabBase { +class DefaultVocab : public IVocab { protected: typedef std::map Str2Id; Str2Id str2id_; @@ -318,11 +318,11 @@ class ClassVocab : public DefaultVocab { } }; -Ptr createDefaultVocab() { +Ptr createDefaultVocab() { return New(); } -Ptr createClassVocab() { +Ptr createClassVocab() { return New(); } diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp index 4457dc064..9af727565 100755 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -19,7 +19,7 @@ namespace marian { #ifdef USE_SENTENCEPIECE // Wrapper around https://github.com/google/sentencepiece -class SentencePieceVocab : public VocabBase { +class SentencePieceVocab : public IVocab { private: // Actual SentencePiece processor object UPtr spm_; @@ -252,7 +252,7 @@ class SentencePieceVocab : public VocabBase { }; #endif // USE_SENTENCEPIECE -Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { +Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { bool isSentencePiece = regex::regex_search(vocabPath, regex::regex("\\.(spm)$")); if(isSentencePiece) { #ifdef USE_SENTENCEPIECE diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 7844ecfa4..8094c699c 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -8,12 +8,12 @@ Word Word::ZERO = Word(0); Word Word::DEFAULT_EOS_ID = Word(0); Word Word::DEFAULT_UNK_ID = Word(1); -Ptr createDefaultVocab(); -Ptr createClassVocab(); -Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); +Ptr createDefaultVocab(); +Ptr createClassVocab(); +Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); // @TODO: make each vocab peek on type -Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { +Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); if(vocab) { return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it diff --git a/src/data/vocab.h b/src/data/vocab.h index e82673ecc..afe79d1f0 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -7,7 +7,7 @@ namespace marian { -class VocabBase; +class IVocab; // Wrapper around vocabulary types. Can choose underlying // vocabulary implementation (vImpl_) based on speficied path @@ -17,7 +17,7 @@ class VocabBase; // * SentencePiece with suffix *.spm (works, but has to be created outside Marian) class Vocab { private: - Ptr vImpl_; + Ptr vImpl_; Ptr options_; size_t batchIndex_; diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h old mode 100644 new mode 100755 index 2050792f3..9a82131e8 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -7,7 +7,7 @@ namespace marian { -class VocabBase { +class IVocab { public: virtual size_t load(const std::string& vocabPath, size_t maxSize = 0) = 0; diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 9e161fefe..0c12f5d34 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -700,6 +700,7 @@ + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index f26b0bf70..85cb3736e 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1525,6 +1525,9 @@ models + + data + @@ -1878,4 +1881,4 @@ examples - + \ No newline at end of file From 79db5b174992129f7f40d5f3e7fd9af448d688e6 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 18:25:44 -0800 Subject: [PATCH 258/838] add ULR parameters to model saving --- src/models/encoder_classifier.h | 4 ++++ src/models/encoder_decoder.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index c593ccfd3..738bae029 100755 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -129,6 +129,10 @@ class EncoderClassifier : public EncoderClassifierBase { modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); modelFeatures_.insert("transformer-train-positions"); + + modelFeatures_.insert("ulr"); + modelFeatures_.insert("ulr-trainable-transformation"); + modelFeatures_.insert("ulr-dim-emb"); } virtual Ptr getOptions() override { return options_; } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index c4b96cc30..d39acaf2e 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -45,6 +45,10 @@ EncoderDecoder::EncoderDecoder(Ptr options) modelFeatures_.insert("transformer-tied-layers"); modelFeatures_.insert("transformer-guided-alignment-layer"); modelFeatures_.insert("transformer-train-positions"); + + modelFeatures_.insert("ulr"); + modelFeatures_.insert("ulr-trainable-transformation"); + modelFeatures_.insert("ulr-dim-emb"); } std::vector>& EncoderDecoder::getEncoders() { From 5fe528256181f5970c1415e81ee936c5082bcfd2 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 18:38:18 -0800 Subject: [PATCH 259/838] rm crlr --- src/models/encoder_classifier.h | 442 ++++++++++++++++---------------- 1 file changed, 221 insertions(+), 221 deletions(-) diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 23977a6c5..1f5855af1 100644 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -1,221 +1,221 @@ -#pragma once - -#include "marian.h" - -#include "models/encoder.h" -#include "models/classifier.h" -#include "models/model_base.h" -#include "models/states.h" - -namespace marian { - -/** - * Combines sequence encoders with generic classifiers - * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. - * Already has support for multi-objective training. - * - * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier - * multi-objective training. - */ -class EncoderClassifierBase : public models::ModelBase { -public: - virtual ~EncoderClassifierBase() {} - - virtual void load(Ptr graph, - const std::string& name, - bool markedReloaded = true) override - = 0; - - virtual void mmap(Ptr graph, - const void* ptr, - bool markedReloaded = true) - = 0; - - virtual void save(Ptr graph, - const std::string& name, - bool saveTranslatorConfig = false) override - = 0; - - virtual void clear(Ptr graph) override = 0; - - virtual std::vector> apply(Ptr, Ptr, bool) = 0; - - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override = 0; - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) = 0; - - virtual Ptr getOptions() = 0; -}; - -class EncoderClassifier : public EncoderClassifierBase { -protected: - Ptr options_; - - std::string prefix_; - - std::vector> encoders_; - std::vector> classifiers_; - - bool inference_{false}; - - std::set modelFeatures_; - - Config::YamlNode getModelParameters() { - Config::YamlNode modelParams; - for(auto& key : modelFeatures_) - modelParams[key] = options_->getYaml()[key]; - - if(options_->has("original-type")) - modelParams["type"] = options_->getYaml()["original-type"]; - - modelParams["version"] = buildVersion(); - return modelParams; - } - - std::string getModelParametersAsString() { - auto yaml = getModelParameters(); - YAML::Emitter out; - cli::OutputYaml(yaml, out); - return std::string(out.c_str()); - } - -public: - typedef data::Corpus dataset_type; - - // @TODO: lots of code-duplication with EncoderDecoder - EncoderClassifier(Ptr options) - : options_(options), - prefix_(options->get("prefix", "")), - inference_(options->get("inference", false)) { - - modelFeatures_ = {"type", - "dim-vocabs", - "dim-emb", - "dim-rnn", - "enc-cell", - "enc-type", - "enc-cell-depth", - "enc-depth", - "dec-depth", - "dec-cell", - "dec-cell-base-depth", - "dec-cell-high-depth", - "skip", - "layer-normalization", - "right-left", - "input-types", - "special-vocab", - "tied-embeddings", - "tied-embeddings-src", - "tied-embeddings-all"}; - - modelFeatures_.insert("transformer-heads"); - modelFeatures_.insert("transformer-no-projection"); - modelFeatures_.insert("transformer-dim-ffn"); - modelFeatures_.insert("transformer-ffn-depth"); - modelFeatures_.insert("transformer-ffn-activation"); - modelFeatures_.insert("transformer-dim-aan"); - modelFeatures_.insert("transformer-aan-depth"); - modelFeatures_.insert("transformer-aan-activation"); - modelFeatures_.insert("transformer-aan-nogate"); - modelFeatures_.insert("transformer-preprocess"); - modelFeatures_.insert("transformer-postprocess"); - modelFeatures_.insert("transformer-postprocess-emb"); - modelFeatures_.insert("transformer-decoder-autoreg"); - modelFeatures_.insert("transformer-tied-layers"); - modelFeatures_.insert("transformer-guided-alignment-layer"); - modelFeatures_.insert("transformer-train-position-embeddings"); - modelFeatures_.insert("bert-train-type-embeddings"); - modelFeatures_.insert("bert-type-vocab-size"); - } - - virtual Ptr getOptions() override { return options_; } - - std::vector>& getEncoders() { return encoders_; } - std::vector>& getClassifiers() { return classifiers_; } - - void push_back(Ptr encoder) { encoders_.push_back(encoder); } - void push_back(Ptr classifier) { classifiers_.push_back(classifier); } - - void load(Ptr graph, - const std::string& name, - bool markedReloaded) override { - graph->load(name, markedReloaded && !opt("ignore-model-config", false)); - } - - void mmap(Ptr graph, - const void* ptr, - bool markedReloaded) override { - graph->mmap(ptr, markedReloaded && !opt("ignore-model-config", false)); - } - - void save(Ptr graph, - const std::string& name, - bool saveModelConfig) override { - LOG(info, "Saving model weights and runtime parameters to {}", name); - graph->save(name , getModelParametersAsString()); - } - - void clear(Ptr graph) override { - graph->clear(); - - for(auto& enc : encoders_) - enc->clear(); - for(auto& cls : classifiers_) - cls->clear(); - } - - template - T opt(const std::string& key) { - return options_->get(key); - } - - template - T opt(const std::string& key, const T& def) { - return options_->get(key, def); - } - - template - void set(std::string key, T value) { - options_->set(key, value); - } - - /*********************************************************************/ - - virtual std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { - if(clearGraph) - clear(graph); - - std::vector> encoderStates; - for(auto& encoder : encoders_) - encoderStates.push_back(encoder->build(graph, batch)); - - std::vector> classifierStates; - for(auto& classifier : classifiers_) - classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); - - return classifierStates; - } - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { - auto states = apply(graph, batch, clearGraph); - // returns raw logits - return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used - } - - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { - auto corpusBatch = std::static_pointer_cast(batch); - return build(graph, corpusBatch, clearGraph); - } -}; - -} // namespace marian +#pragma once + +#include "marian.h" + +#include "models/encoder.h" +#include "models/classifier.h" +#include "models/model_base.h" +#include "models/states.h" + +namespace marian { + +/** + * Combines sequence encoders with generic classifiers + * Can be used to train sequence classifiers like language detection, BERT-next-sentence-prediction etc. + * Already has support for multi-objective training. + * + * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier + * multi-objective training. + */ +class EncoderClassifierBase : public models::ModelBase { +public: + virtual ~EncoderClassifierBase() {} + + virtual void load(Ptr graph, + const std::string& name, + bool markedReloaded = true) override + = 0; + + virtual void mmap(Ptr graph, + const void* ptr, + bool markedReloaded = true) + = 0; + + virtual void save(Ptr graph, + const std::string& name, + bool saveTranslatorConfig = false) override + = 0; + + virtual void clear(Ptr graph) override = 0; + + virtual std::vector> apply(Ptr, Ptr, bool) = 0; + + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; + + virtual Ptr getOptions() = 0; +}; + +class EncoderClassifier : public EncoderClassifierBase { +protected: + Ptr options_; + + std::string prefix_; + + std::vector> encoders_; + std::vector> classifiers_; + + bool inference_{false}; + + std::set modelFeatures_; + + Config::YamlNode getModelParameters() { + Config::YamlNode modelParams; + for(auto& key : modelFeatures_) + modelParams[key] = options_->getYaml()[key]; + + if(options_->has("original-type")) + modelParams["type"] = options_->getYaml()["original-type"]; + + modelParams["version"] = buildVersion(); + return modelParams; + } + + std::string getModelParametersAsString() { + auto yaml = getModelParameters(); + YAML::Emitter out; + cli::OutputYaml(yaml, out); + return std::string(out.c_str()); + } + +public: + typedef data::Corpus dataset_type; + + // @TODO: lots of code-duplication with EncoderDecoder + EncoderClassifier(Ptr options) + : options_(options), + prefix_(options->get("prefix", "")), + inference_(options->get("inference", false)) { + + modelFeatures_ = {"type", + "dim-vocabs", + "dim-emb", + "dim-rnn", + "enc-cell", + "enc-type", + "enc-cell-depth", + "enc-depth", + "dec-depth", + "dec-cell", + "dec-cell-base-depth", + "dec-cell-high-depth", + "skip", + "layer-normalization", + "right-left", + "input-types", + "special-vocab", + "tied-embeddings", + "tied-embeddings-src", + "tied-embeddings-all"}; + + modelFeatures_.insert("transformer-heads"); + modelFeatures_.insert("transformer-no-projection"); + modelFeatures_.insert("transformer-dim-ffn"); + modelFeatures_.insert("transformer-ffn-depth"); + modelFeatures_.insert("transformer-ffn-activation"); + modelFeatures_.insert("transformer-dim-aan"); + modelFeatures_.insert("transformer-aan-depth"); + modelFeatures_.insert("transformer-aan-activation"); + modelFeatures_.insert("transformer-aan-nogate"); + modelFeatures_.insert("transformer-preprocess"); + modelFeatures_.insert("transformer-postprocess"); + modelFeatures_.insert("transformer-postprocess-emb"); + modelFeatures_.insert("transformer-decoder-autoreg"); + modelFeatures_.insert("transformer-tied-layers"); + modelFeatures_.insert("transformer-guided-alignment-layer"); + modelFeatures_.insert("transformer-train-position-embeddings"); + modelFeatures_.insert("bert-train-type-embeddings"); + modelFeatures_.insert("bert-type-vocab-size"); + } + + virtual Ptr getOptions() override { return options_; } + + std::vector>& getEncoders() { return encoders_; } + std::vector>& getClassifiers() { return classifiers_; } + + void push_back(Ptr encoder) { encoders_.push_back(encoder); } + void push_back(Ptr classifier) { classifiers_.push_back(classifier); } + + void load(Ptr graph, + const std::string& name, + bool markedReloaded) override { + graph->load(name, markedReloaded && !opt("ignore-model-config", false)); + } + + void mmap(Ptr graph, + const void* ptr, + bool markedReloaded) override { + graph->mmap(ptr, markedReloaded && !opt("ignore-model-config", false)); + } + + void save(Ptr graph, + const std::string& name, + bool saveModelConfig) override { + LOG(info, "Saving model weights and runtime parameters to {}", name); + graph->save(name , getModelParametersAsString()); + } + + void clear(Ptr graph) override { + graph->clear(); + + for(auto& enc : encoders_) + enc->clear(); + for(auto& cls : classifiers_) + cls->clear(); + } + + template + T opt(const std::string& key) { + return options_->get(key); + } + + template + T opt(const std::string& key, const T& def) { + return options_->get(key, def); + } + + template + void set(std::string key, T value) { + options_->set(key, value); + } + + /*********************************************************************/ + + virtual std::vector> apply(Ptr graph, Ptr batch, bool clearGraph) override { + if(clearGraph) + clear(graph); + + std::vector> encoderStates; + for(auto& encoder : encoders_) + encoderStates.push_back(encoder->build(graph, batch)); + + std::vector> classifierStates; + for(auto& classifier : classifiers_) + classifierStates.push_back(classifier->apply(graph, batch, encoderStates)); + + return classifierStates; + } + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto states = apply(graph, batch, clearGraph); + // returns raw logits + return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used + } + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + auto corpusBatch = std::static_pointer_cast(batch); + return build(graph, corpusBatch, clearGraph); + } +}; + +} // namespace marian From d121ba4726e843ab50bf35b81150586d5a581c7f Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 20:26:46 -0800 Subject: [PATCH 260/838] address code review comments --- scripts/bert/bert4marian.py | 18 ++++ src/layers/loss.h | 13 ++- src/tensors/cpu/tensor_operators.cpp | 1 - src/tensors/gpu/tensor_operators.cu | 92 +++++++++++------- src/tests/operator_tests.cpp | 137 ++++++++++++++------------- 5 files changed, 158 insertions(+), 103 deletions(-) diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py index 5ccc46595..8070c0fe9 100755 --- a/scripts/bert/bert4marian.py +++ b/scripts/bert/bert4marian.py @@ -1,4 +1,22 @@ #!/usr/bin/env python3 +""" +This script takes a Tensorflow BERT checkpoint and a model description in a JSON file and converts +it to a Marian weight file with numpy weights and an internal YAML description. + +This works with checkpoints from https://github.com/google-research/bert + +Assmung a BERT checkpoint like this: +drwxr-xr-x 2 marcinjd marcinjd 4.0K Nov 23 16:39 . +-rw-r--r-- 1 marcinjd marcinjd 521 Nov 23 16:38 bert_config.json +-rw-r--r-- 1 marcinjd marcinjd 682M Nov 23 16:39 bert_model.ckpt.data-00000-of-00001 +-rw-r--r-- 1 marcinjd marcinjd 8.5K Nov 23 16:39 bert_model.ckpt.index +-rw-r--r-- 1 marcinjd marcinjd 888K Nov 23 16:39 bert_model.ckpt.meta +-rw-r--r-- 1 marcinjd marcinjd 973K Nov 23 16:37 vocab.txt + +usage: + +./bert.py --bert_prefix bert_model.ckpt --bert_config bert_config.json --marian bert.npz +""" import tensorflow as tf import numpy as np diff --git a/src/layers/loss.h b/src/layers/loss.h index 2ac4ae78d..32be5618e 100644 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -110,8 +110,8 @@ class MultiRationalLoss : public RationalLoss { /** * @brief Accumulation rule for losses - * In the default case this would just be a sum, see SumMultiRationalLoss, but there are - * special cases like ScaledMultiRationalLoss (scale other loses according to first label count) + * In the default case this would just be a sum, see SumMultiRationalLoss, but there are + * special cases like ScaledMultiRationalLoss (scale other loses according to first label count) * or MeanMultiRationalLoss (sum of means) where the accumulation is more complex. */ virtual Expr accumulateLoss(const RationalLoss& current) = 0; @@ -294,7 +294,7 @@ class LabelwiseLoss { lossSum = sum(lossSum, axes_[i]); // reduction factor tells how over how many labels we reduced in total. - float reducedLabels = (float)loss->shape().elements() / (float)lossSum->shape().elements(); + float reducedLabels = (float)loss->shape().elements() / (float)lossSum->shape().elements(); return RationalLoss(lossSum, reducedLabels); } @@ -331,12 +331,15 @@ class CrossEntropyLoss : public LabelwiseLoss { virtual Expr compute(Expr logits, Expr labelIndices, Expr mask = nullptr, Expr labelWeights = nullptr) override { - logits = atleast_3d(logits); // safeguard against 2d classifier output, adds 1 on the left, non-op. + logits = atleast_3d(logits); // we always assuma a time and batch dimension exists. + // for bert training or classification the time dimension is lot. + // Here safeguard against 2d classifier output, adds 1 on the left, non-op. + Expr ce = cross_entropy(logits, labelIndices); if(labelSmoothing_ > 0) { // @TODO: add this to CE kernels instead - + // Label smoothing (see https://arxiv.org/pdf/1512.00567.pdf, section 7) // We compute smoothed H(q',p) = (1 - eps) * H(q,p) + eps * H(u,p) where H(q,p) is the normal cross-entropy // and H(u,p) penalizes deviation of p from u, u being uniform distribution over vocab V => u_v = 1/|V|. diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index d7d019f7d..72dbd131c 100644 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -18,7 +18,6 @@ void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf, ABORT("Not implemented"); } - inline float stableSigmoid(float x) { if(x >= 0) { float z = expf(-x); diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index ae1e02653..28f57f762 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -34,9 +34,9 @@ __global__ void gIsNan(T* in, int length, bool* isNan, bool* isInf, bool zero) { if(index < length) { if(isnan((float)in[index])) { if(zero) in[index] = (T)0.f; - *isNan = true; + *isNan = true; } - else if(isinf((float)in[index])) { + else if(isinf((float)in[index])) { if(zero) in[index] = (T)0.f; *isInf = true; } @@ -406,30 +406,41 @@ void TransposeNDGrad(Tensor out, Tensor in, const std::vector& vAxis) { } } +// Computes the softmax +// in - input tensor +// out - output tensor +// we compute the softmax over the the cols (last dimension) +// rows are time, batch or beam dimensions +// number of threads is number of cols or MAX_THREADS +// number of blocks is number of rows or MAX_BLOCKS __global__ void gSoftmax(float* out, functional::Shape outShape, const float* in) { int rows = outShape.elements() / outShape.back(); int cols = outShape.back(); - for(int bid = 0; bid < rows; bid += gridDim.x) { - int j = bid + blockIdx.x; - if(j < rows) { - float* so = out + j * cols; + for(int bid = 0; bid < rows; bid += gridDim.x) { // loop over blocks of rows + int j = bid + blockIdx.x; // blockIdx.x - row index (within block of rows) + if(j < rows) { // compute softmax over one row, row elements distributed over threads + float* so = out + j * cols; // pointer to row input data const float* sp = in + j * cols; extern __shared__ float _share[]; + // determine max (used below to improve numeric stability) float* _max = _share; - _max[threadIdx.x] = -CUDA_FLT_MAX; // mask - for(int tid = 0; tid < cols; tid += blockDim.x) { - int id = tid + threadIdx.x; - if(id < cols) { - if(sp[id] > _max[threadIdx.x]) - _max[threadIdx.x] = sp[id]; + _max[threadIdx.x] = -CUDA_FLT_MAX; // [threadIdx.x = relative column index within a block of columns] + // find max over column indices that have the same relative column index (=threadIdx.x) across all blocks of columns + for(int tid = 0; tid < cols; tid += blockDim.x) { // loop over blocks of columns, blockDim.x = index of block of columns + // threadIdx.x = column index within block of columns; we reduce over columns within a block, then over blocks + int i = tid + threadIdx.x; + if(i < cols) { + if(sp[i] > _max[threadIdx.x]) + _max[threadIdx.x] = sp[i]; } } __syncthreads(); + // max over columns within a column block via tree reduction int len = blockDim.x; while(len != 1) { __syncthreads(); @@ -443,20 +454,22 @@ __global__ void gSoftmax(float* out, } __syncthreads(); float max = _max[0]; - __syncthreads(); - - float* _sum = _share + blockDim.x; + __syncthreads(); // @TODO: do we need this? + // compute denominator + float* _sum = _share; _sum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { - int id = tid + threadIdx.x; - if(id < cols) { - float ex = __expf(sp[id] - max); - so[id] = ex; + int i = tid + threadIdx.x; + if(i < cols) { + // @TODO: is it faster to cache the result of expf() in GPU RAM, or would it be faster to recompute it below? + float ex = __expf(sp[i] - max); + so[i] = ex; _sum[threadIdx.x] += ex; } } __syncthreads(); + // now reduce over all columns within the block len = blockDim.x; while(len != 1) { __syncthreads(); @@ -466,13 +479,17 @@ __global__ void gSoftmax(float* out, len = (len + 1) >> 1; } __syncthreads(); + + // produce final output data + float sum = _sum[0]; for(int tid = 0; tid < cols; tid += blockDim.x) { - int id = tid + threadIdx.x; - if(id < cols) { - so[id] = so[id] / _sum[0]; + int i = tid + threadIdx.x; + if(i < cols) { + so[i] = so[i] / sum; } } } + __syncthreads(); } } @@ -484,11 +501,12 @@ void Softmax(Tensor out, Tensor in) { int blocks = std::min(MAX_BLOCKS, (int)m); int threads = std::min(MAX_THREADS, (int)k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gSoftmax<<>>(out->data(), out->shape(), in->data()); } +// @TODO: refactor to reuse code from softmax, add comments __global__ void gLogSoftmax(float* out, const functional::Shape outShape, const float* in) { @@ -528,7 +546,7 @@ __global__ void gLogSoftmax(float* out, float max = _max[0]; __syncthreads(); - float* _sum = _share + blockDim.x; + float* _sum = _share; _sum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { @@ -553,9 +571,10 @@ __global__ void gLogSoftmax(float* out, for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; if(id < cols) - so[id] -= __logf(_sum[0]); + so[id] = __logf(_sum[0]); } } + __syncthreads(); } } @@ -567,7 +586,7 @@ void LogSoftmax(Tensor out, Tensor in) { int blocks = std::min(MAX_BLOCKS, (int)m); int threads = std::min(MAX_THREADS, (int)k); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gLogSoftmax<<>>( out->data(), out->shape(), in->data()); @@ -615,9 +634,11 @@ __global__ void gSoftmaxGrad(float* grad, } } } + __syncthreads(); } } +// @TODO: refactor with logsoftmax, add math void SoftmaxGrad(Tensor grad, Tensor adj, Tensor val) { cudaSetDevice(adj->getDeviceId().no); // grad and val are both m-by-k matrices, passed as input. @@ -671,6 +692,7 @@ __global__ void gLogSoftmaxGrad(float* grad, gradRow[id] += adjRow[id] - (expf(valRow[id]) * _sum[0]); } } + __syncthreads(); } } @@ -1179,7 +1201,7 @@ __global__ void gCrossEntropyPick(float* out, float max = _max[0]; __syncthreads(); - float* _sum = _share + blockDim.x; + float* _sum = _share; _sum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; @@ -1206,6 +1228,7 @@ __global__ void gCrossEntropyPick(float* out, } } } + __syncthreads(); } } @@ -1223,7 +1246,7 @@ void CrossEntropyPick(Tensor out, Tensor in, Tensor indices) { int blocks = std::min(MAX_BLOCKS, (int)rows); int threads = std::min(MAX_THREADS, (int)cols); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gCrossEntropyPick<<>>( out->data(), out->shape(), in->data(), in->shape(), indices->data()); @@ -1269,7 +1292,7 @@ __global__ void gCrossEntropyPickBackward(float* out, float max = _max[0]; __syncthreads(); - float* _sum = _share + blockDim.x; + float* _sum = _share; _sum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; @@ -1298,6 +1321,7 @@ __global__ void gCrossEntropyPickBackward(float* out, } } } + __syncthreads(); } } @@ -1311,7 +1335,7 @@ void CrossEntropyPickBackward(Tensor out, Tensor adj, Tensor a, Tensor indices) int blocks = std::min(MAX_BLOCKS, (int)rows); int threads = std::min(MAX_THREADS, (int)cols); - int shared = sizeof(float) * threads * 2; + int shared = sizeof(float) * threads; gCrossEntropyPickBackward<<>>( out->data(), out->shape(), adj->data(), a->data(), indices->data()); @@ -1380,8 +1404,8 @@ __global__ void gAtt(float* out, } __syncthreads(); out[j] = _sum[0]; - __syncthreads(); } + __syncthreads(); } } @@ -1507,7 +1531,7 @@ __global__ void gLNormalization(float* out, float mean = _sum[0] / cols; __syncthreads(); - float* _sqSum = _share + blockDim.x; + float* _sqSum = _share; _sqSum[threadIdx.x] = 0.0; for(int tid = 0; tid < cols; tid += blockDim.x) { @@ -1540,6 +1564,7 @@ __global__ void gLNormalization(float* out, } } } + __syncthreads(); } } @@ -1555,7 +1580,7 @@ void LayerNormalization(Tensor out, int blocks = std::min(MAX_BLOCKS, (int)rows); int threads = std::min(MAX_THREADS, (int)cols); - int shared = 2 * threads * sizeof(float); + int shared = threads * sizeof(float); gLNormalization<<>>(out->data(), in->data(), @@ -1665,6 +1690,7 @@ __global__ void gLayerNormalizationGrad(float* gradX, } } } + __syncthreads(); } } diff --git a/src/tests/operator_tests.cpp b/src/tests/operator_tests.cpp index e17eca9dc..360da61b3 100755 --- a/src/tests/operator_tests.cpp +++ b/src/tests/operator_tests.cpp @@ -342,77 +342,86 @@ void tests(DeviceType device) { auto B = graph->param("B", {3, 2}, inits::from_vector(vB)); auto C = dot(A, B); - // CSR dot product, tested against dense product on the same values - // std::vector vS({1, 0, 0, 1, // sparse - // 0, 0, 1, 1.5}); - // std::vector vD({1, 2, 3, 1.2, 5.6, // dense - // 4, 5, 6, 2.3, 6.7, - // 7, 8, 9, 3.4, 7.8, - // 1, 1, 2, 4.5, 8.9}); - // auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); - // auto D = graph->param("D", { 4, 5 }, inits::from_vector(vD)); - // auto DT = graph->param("DT", { 5, 4 }, inits::from_vector(vD)); // example matrix with transposed dimensions - // std::vector SV; // create CSR version of S - // std::vector SI, SO; - // SO.push_back((IndexType)SI.size()); - // for (IndexType i = 0; i < S->shape()[0]; i++) { - // for (IndexType j = 0; j < S->shape()[1]; j++) { - // auto k = 4 * i + j; - // if (vS[k] != 0) { - // SV.push_back(vS[k]); - // SI.push_back(j); - // } - // } - // SO.push_back((IndexType)SI.size()); - // } - - // auto SxDd = dot(S, D); - // auto STxSxDd = dot(S, SxDd, /*transA=*/true); - // auto SxDs = csr_dot( // sparse x dense - // S->shape(), - // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - // D); - // auto STxSxDs = csr_dot( // transpose(sparse) x dense; we use result of previous since dimensions match - // S->shape(), - // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - // SxDd, /*transS=*/true); - - // auto DTxSTd = dot(DT, S, /*transA=*/false, /*transB=*/true); - // auto DTxSTxSd = dot(DTxSTd, S); - // auto DTxSTs = dot_csr( // dense x sparse - // DT, - // S->shape(), - // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), - // /*transS=*/true); - // auto DTxSTxSs = dot_csr( // dense x transpose(sparse) - // DTxSTd, - // S->shape(), - // graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), - // graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), - // graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32)); - CHECK(C->shape() == Shape({2, 2, 2})); - // CHECK(SxDs->shape() == SxDd->shape()); - // CHECK(STxSxDs->shape() == STxSxDd->shape()); - // CHECK(DTxSTs->shape() == DTxSTd->shape()); - // CHECK(DTxSTxSs->shape() == DTxSTxSd->shape()); graph->forward(); C->val()->get(values); CHECK(values == vC); + } - // dense and sparse operation results must be the same - // SxDd ->val()->get(values2); SxDs ->val()->get(values); CHECK(values == values2); - // STxSxDd ->val()->get(values2); STxSxDs ->val()->get(values); CHECK(values == values2); - // DTxSTd ->val()->get(values2); DTxSTs ->val()->get(values); CHECK(values == values2); - // DTxSTxSd->val()->get(values2); DTxSTxSs->val()->get(values); CHECK(values == values2); + if(device == DeviceType::gpu) { + SECTION("csr-dot product") { + graph->clear(); + values.clear(); + // CSR dot product, tested against dense product on the same values + std::vector vS({1, 0, 0, 1, // sparse + 0, 0, 1, 1.5}); + std::vector vD({1, 2, 3, 1.2, 5.6, // dense + 4, 5, 6, 2.3, 6.7, + 7, 8, 9, 3.4, 7.8, + 1, 1, 2, 4.5, 8.9}); + auto S = graph->param("S", { 2, 4 }, inits::from_vector(vS)); + auto D = graph->param("D", { 4, 5 }, inits::from_vector(vD)); + auto DT = graph->param("DT", { 5, 4 }, inits::from_vector(vD)); // example matrix with transposed dimensions + std::vector SV; // create CSR version of S + std::vector SI, SO; + SO.push_back((IndexType)SI.size()); + for (IndexType i = 0; i < S->shape()[0]; i++) { + for (IndexType j = 0; j < S->shape()[1]; j++) { + auto k = 4 * i + j; + if (vS[k] != 0) { + SV.push_back(vS[k]); + SI.push_back(j); + } + } + SO.push_back((IndexType)SI.size()); + } + + auto SxDd = dot(S, D); + auto STxSxDd = dot(S, SxDd, /*transA=*/true); + auto SxDs = csr_dot( // sparse x dense + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + D); + auto STxSxDs = csr_dot( // transpose(sparse) x dense; we use result of previous since dimensions match + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + SxDd, /*transS=*/true); + + auto DTxSTd = dot(DT, S, /*transA=*/false, /*transB=*/true); + auto DTxSTxSd = dot(DTxSTd, S); + auto DTxSTs = dot_csr( // dense x sparse + DT, + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32), + /*transS=*/true); + auto DTxSTxSs = dot_csr( // dense x transpose(sparse) + DTxSTd, + S->shape(), + graph->constant({(int)SV.size()}, inits::from_vector(SV), Type::float32), + graph->constant({(int)SI.size()}, inits::from_vector(SI), Type::uint32), + graph->constant({(int)SO.size()}, inits::from_vector(SO), Type::uint32)); + + CHECK(SxDs->shape() == SxDd->shape()); + CHECK(STxSxDs->shape() == STxSxDd->shape()); + CHECK(DTxSTs->shape() == DTxSTd->shape()); + CHECK(DTxSTxSs->shape() == DTxSTxSd->shape()); + + graph->forward(); + + // dense and sparse operation results must be the same + SxDd ->val()->get(values2); SxDs ->val()->get(values); CHECK(values == values2); + STxSxDd ->val()->get(values2); STxSxDs ->val()->get(values); CHECK(values == values2); + DTxSTd ->val()->get(values2); DTxSTs ->val()->get(values); CHECK(values == values2); + DTxSTxSd->val()->get(values2); DTxSTxSs->val()->get(values); CHECK(values == values2); + } } SECTION("affine transformation") { From 9f6e756e360a89e1e76343772df074468d1f5a59 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 20:56:13 -0800 Subject: [PATCH 261/838] fix merge --- src/common/config_parser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 144c3e159..0d1863562 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -59,12 +59,12 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { cli.add("--workspace,-w", "Preallocate arg MB of work space", defaultWorkspace); - cli.add_nondefault("--log", + cli.add("--log", "Log training process information to file given by arg"); cli.add("--log-level", "Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off", "info"); - cli.add_nondefault("--log-time-zone", + cli.add("--log-time-zone", "Set time zone for the date shown on logging"); cli.add("--quiet", "Suppress all logging to stderr. Logging to files still works"); @@ -78,7 +78,7 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { "allow the use of environment variables in paths, of the form ${VAR_NAME}"); cli.add("--relative-paths", "All paths are relative to the config file location"); - cli.add_nondefault("--dump-config", + cli.add("--dump-config", "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") ->implicit_val("full"); // clang-format on From 160568effaafc8410b217d3ec1a68d04fdbc3725 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 21:18:58 -0800 Subject: [PATCH 262/838] add missing - --- src/tensors/gpu/add.cu | 1 + src/tensors/gpu/tensor_operators.cu | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu index 9e0c8e687..e9474d048 100755 --- a/src/tensors/gpu/add.cu +++ b/src/tensors/gpu/add.cu @@ -121,6 +121,7 @@ __global__ void gAggregateReduce(Functor functor, float aggInit, AggFunctor aggF __syncthreads(); out[j] = aggFunctor(out[j], _sum[0] * scale); } + __syncthreads(); } } diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index 28f57f762..a7edcdf59 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -571,7 +571,7 @@ __global__ void gLogSoftmax(float* out, for(int tid = 0; tid < cols; tid += blockDim.x) { int id = tid + threadIdx.x; if(id < cols) - so[id] = __logf(_sum[0]); + so[id] -= __logf(_sum[0]); } } __syncthreads(); From d7db8d8534e3a6bceb4f594c417e04011d85a1f5 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 22:12:17 -0800 Subject: [PATCH 263/838] update regression tests --- regression-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-tests b/regression-tests index febdc3f56..466d85a2e 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit febdc3f56f75929b1f7b5b38a4b9b96ea8f648e7 +Subproject commit 466d85a2e0504a981508c7f9c8d1639c87c63b90 From 10361f81d3d5c337f7554c17f4b6fb9beaf8b882 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 22:39:48 -0800 Subject: [PATCH 264/838] update nccl version to 4 --- src/3rd_party/nccl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rd_party/nccl b/src/3rd_party/nccl index d6297d250..8e3a3f7c5 160000 --- a/src/3rd_party/nccl +++ b/src/3rd_party/nccl @@ -1 +1 @@ -Subproject commit d6297d250433715c283d17f1969cfcb50d2b6531 +Subproject commit 8e3a3f7c5b520babff49cec54a866fa3eda3a3b6 From 4ab941b90992ed19ad1392d48efde28db552d20d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 23:07:00 -0800 Subject: [PATCH 265/838] use ExternalProject for installation --- CMakeLists.txt | 37 +++++------------------------------- src/3rd_party/CMakeLists.txt | 32 +++++++++++++++++++++++++++++++ src/CMakeLists.txt | 5 +++++ 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b54766ae9..340c15864 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}") execute_process(COMMAND git submodule update --init --recursive --no-fetch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - + # Set compilation flags if(MSVC) # These are used in src/CMakeLists.txt on a per-target basis @@ -42,7 +42,7 @@ if(MSVC) # C4310: cast truncates constant value # C4324: 'marian::cpu::int16::`anonymous-namespace'::ScatterPut': structure was padded due to alignment specifier set(DISABLE_GLOBALLY "/wd\"4310\" /wd\"4324\"") - + set(INTRINSICS "/arch:AVX") # Or maybe use these? @@ -57,7 +57,7 @@ if(MSVC) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG /LTCG:incremental /INCREMENTAL:NO /NODEFAULTLIB:MSVCRT /ignore:4049") set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /LTCG:incremental") - find_library(SHLWAPI Shlwapi.lib) + find_library(SHLWAPI Shlwapi.lib) set(EXT_LIBS ${EXT_LIBS} SHLWAPI) else() @@ -172,37 +172,10 @@ endif(USE_STATIC_LIBS) # Cmake. This is also fairly untested, let's hope it does not explode. # @TODO: Make sure it does not use pre-installed NCCL headers if(USE_NCCL) - # define and set the include dir for the generated nccl.h header - set(NCCL_HEADER_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/nccl/include") - include_directories(${NCCL_HEADER_LOCATION}) - - # set the path for the generated static lib - set(NCCL_LIB_STATIC "${CMAKE_CURRENT_BINARY_DIR}/nccl/lib/libnccl_static.a") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NCCL") - - LIST(APPEND CUDA_NVCC_FLAGS -DUSE_NCCL; ) - - # disables compilation for sm_30 to avoid ptxas warning... that's general Kepler support. But K80s are supported for instance by sm_35 - set(GENCODE "-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61") - - # We build using NVidia's custom makefile, for that we pass a number of variables from CMake. - # Sets output to the chosen build folder, i.e. where the binaries and objects are generated. - # Also passes CUDA location from FindCUDA, sets c++ compiler to the same one CMake uses. - add_custom_command(OUTPUT ${NCCL_LIB_STATIC} - COMMAND ${CMAKE_MAKE_PROGRAM} src.build - BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/nccl - CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR} - CUDA8_GENCODE=${GENCODE} - CXX=${CMAKE_CXX_COMPILER} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/3rd_party/nccl) - add_custom_target(nccl_target DEPENDS ${NCCL_LIB_STATIC}) add_library(nccl STATIC IMPORTED) - set_target_properties(nccl PROPERTIES IMPORTED_LOCATION ${NCCL_LIB_STATIC}) - add_dependencies(nccl nccl_target) set(EXT_LIBS ${EXT_LIBS} nccl) - - # adds the resulting files to be removed by `make clean` - set_directory_properties(PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/nccl) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NCCL") + LIST(APPEND CUDA_NVCC_FLAGS -DUSE_NCCL; ) endif(USE_NCCL) if(USE_STATIC_LIBS) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index 8548d9b8b..87f79a438 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -38,3 +38,35 @@ include_directories(./pathie-cpp/include) include_directories(./zlib) +include(ExternalProject) + +set(INSTALLS "") # this will contain a list of 3rd part dependencies that we install locally +if(CUDA_FOUND) + if(USE_NCCL) + + # disables compilation for sm_30 to avoid ptxas warning... that's general Kepler support. But K80s are supported for instance by sm_35 + set(GENCODE "-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61") + + # install nccl in ${CMAKE_BINARY_DIR}/local similar to /usr/local linux installation + ExternalProject_Add(nccl_install + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl + BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl + CONFIGURE_COMMAND "" + BUILD_COMMAND + $(MAKE) -f ${CMAKE_CURRENT_SOURCE_DIR}/nccl/Makefile src.build + BUILDDIR=${CMAKE_BINARY_DIR}/local CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR} + CUDA8_GENCODE=${GENCODE} CXX=${CMAKE_CXX_COMPILER} + INSTALL_COMMAND "") + + set_target_properties(nccl PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/local/lib/libnccl_static.a) + add_dependencies(nccl nccl_install) + set(INSTALLS ${INSTALLS} nccl_install) + + endif(USE_NCCL) +endif(CUDA_FOUND) + +# @TODO: do the same for SentencePiece, Protobuf etc. +# make clean will clean "${CMAKE_BINARY_DIR}/local" +set_directory_properties(PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/local) + +add_custom_target(3rd_party_installs DEPENDS ${INSTALLS}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c757f100..fd489e3e1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ include_directories(.) include_directories(3rd_party) include_directories(3rd_party/SQLiteCpp/include) include_directories(3rd_party/sentencepiece) +include_directories(${CMAKE_BINARY_DIR}/local/include) add_library(marian STATIC common/version.cpp @@ -98,6 +99,8 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h ) add_custom_target(marian_version DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h) add_dependencies(marian marian_version) # marian must depend on it so that it gets created first +# make sure all local dependencies are installed first before this is built +add_dependencies(marian 3rd_party_installs) if(CUDA_FOUND) cuda_add_library(marian_cuda @@ -115,6 +118,8 @@ cuda_add_library(marian_cuda STATIC) target_compile_options(marian_cuda PUBLIC ${ALL_WARNINGS}) + # make sure all local dependencies are installed first before this is built + add_dependencies(marian_cuda 3rd_party_installs) endif(CUDA_FOUND) set_target_properties(marian PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") From 31ca4ac876f2a44e70fb1208238eaeb206deb2db Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 4 Feb 2019 23:14:10 -0800 Subject: [PATCH 266/838] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c409a9d2b..ebe04c36a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Automatic detection of CPU intrisics when building with -arch=native - First version of BERT-training and BERT-classifier, currently not compatible with TF models - New reduction operators +- Use Cmake's ExternalProject to build NCCL and potentially other external libs. ### Fixed - Windows build with recent changes @@ -21,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fixed NaN problem when training with Tensor Cores on Volta GPUs ### Changed +- Update NCCL to 2.4.2 - Add zlib source to Marian's source tree, builds now as object lib - -DUSE_STATIC_LIBS=on now also looks for static versions of CUDA libraries - Include NCCL build from github.com/marian-nmt/nccl and compile within source tree From f91ee5351c21f9a29d1be7ef6ae2da62a6e37936 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 5 Feb 2019 08:04:43 -0800 Subject: [PATCH 267/838] move pointer for reg tests --- regression-tests | 2 +- src/3rd_party/nccl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-tests b/regression-tests index 466d85a2e..142eadddb 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit 466d85a2e0504a981508c7f9c8d1639c87c63b90 +Subproject commit 142eadddbe04493c1024b42586030b72e9cb7ea2 diff --git a/src/3rd_party/nccl b/src/3rd_party/nccl index d6297d250..8e3a3f7c5 160000 --- a/src/3rd_party/nccl +++ b/src/3rd_party/nccl @@ -1 +1 @@ -Subproject commit d6297d250433715c283d17f1969cfcb50d2b6531 +Subproject commit 8e3a3f7c5b520babff49cec54a866fa3eda3a3b6 From 8d7a0284090d758abf79d6a917eb84d68cf2a9b7 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 5 Feb 2019 08:09:32 -0800 Subject: [PATCH 268/838] move nccl pointer back --- src/3rd_party/nccl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rd_party/nccl b/src/3rd_party/nccl index 8e3a3f7c5..d6297d250 160000 --- a/src/3rd_party/nccl +++ b/src/3rd_party/nccl @@ -1 +1 @@ -Subproject commit 8e3a3f7c5b520babff49cec54a866fa3eda3a3b6 +Subproject commit d6297d250433715c283d17f1969cfcb50d2b6531 From b7d245945fb6b74201ab449bc3f73ee9db847bc9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 11:45:55 -0800 Subject: [PATCH 269/838] added factored_vocab stubs --- src/CMakeLists.txt | 1 + src/data/factored_vocab.cpp | 131 ++++++++++++++++++++++++++++++++++++ src/data/factored_vocab.h | 73 ++++++++++++++++++++ vs/Marian.vcxproj | 2 + vs/Marian.vcxproj.filters | 6 ++ 5 files changed, 213 insertions(+) create mode 100755 src/data/factored_vocab.cpp create mode 100755 src/data/factored_vocab.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c16aa1da..90dde706a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(marian STATIC data/vocab.cpp data/default_vocab.cpp data/sentencepiece_vocab.cpp + data/factored_vocab.cpp data/corpus_base.cpp data/corpus.cpp data/corpus_sqlite.cpp diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp new file mode 100755 index 000000000..df203cbcd --- /dev/null +++ b/src/data/factored_vocab.cpp @@ -0,0 +1,131 @@ +#if 0 +#include "data/vocab.h" +#include "data/vocab_base.h" + +namespace marian { + +Word Word::NONE = Word(); +Word Word::ZERO = Word(0); +Word Word::DEFAULT_EOS_ID = Word(0); +Word Word::DEFAULT_UNK_ID = Word(1); + +Ptr createDefaultVocab(); +Ptr createClassVocab(); +Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); + +// @TODO: make each vocab peek on type +Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { + auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); + if(vocab) { + return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it + } else { + // check type of input, if not given, assume "sequence" + auto inputTypes = options->get>("input-types", {}); + std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; + return inputType == "class" ? createClassVocab() : createDefaultVocab(); + } +} + +size_t Vocab::loadOrCreate(const std::string& vocabPath, + const std::vector& trainPaths, + size_t maxSize) { + size_t size = 0; + if(vocabPath.empty()) { + // No vocabulary path was given, attempt to first find a vocabulary + // for trainPaths[0] + possible suffixes. If not found attempt to create + // as trainPaths[0] + canonical suffix. + // Only search based on first path, maybe disable this at all? + + LOG(info, + "No vocabulary path given; " + "trying to find default vocabulary based on data path {}", + trainPaths[0]); + + vImpl_ = createDefaultVocab(); + size = vImpl_->findAndLoad(trainPaths[0], maxSize); + + if(size == 0) { + auto newVocabPath = trainPaths[0] + vImpl_->canonicalExtension(); + LOG(info, + "No vocabulary path given; " + "trying to create vocabulary based on data paths {}", + utils::join(trainPaths, ", ")); + create(newVocabPath, trainPaths, maxSize); + size = load(newVocabPath, maxSize); + } + } else { + if(!filesystem::exists(vocabPath)) { + // Vocabulary path was given, but no vocabulary present, + // attempt to create in specified location. + create(vocabPath, trainPaths, maxSize); + } + // Vocabulary path exists, attempting to load + size = load(vocabPath, maxSize); + } + LOG(info, "[data] Setting vocabulary size for input {} to {}", batchIndex_, size); + return size; +} + +size_t Vocab::load(const std::string& vocabPath, size_t maxSize) { + if(!vImpl_) + vImpl_ = createVocab(vocabPath, options_, batchIndex_); + return vImpl_->load(vocabPath, (int)maxSize); +} + +void Vocab::create(const std::string& vocabPath, + const std::vector& trainPaths, + size_t maxSize) { + if(!vImpl_) + vImpl_ = createVocab(vocabPath, options_, batchIndex_); + vImpl_->create(vocabPath, trainPaths, maxSize); +} + +void Vocab::create(const std::string& vocabPath, + const std::string& trainPath, + size_t maxSize) { + create(vocabPath, std::vector({trainPath}), maxSize); +} + +void Vocab::createFake() { + if(!vImpl_) + vImpl_ = createDefaultVocab(); // DefaultVocab is OK here + vImpl_->createFake(); +} + +// string token to token id +Word Vocab::operator[](const std::string& word) const { + return vImpl_->operator[](word); +} + +// token id to string token +const std::string& Vocab::operator[](Word id) const { + return vImpl_->operator[](id); +} + +// line of text to list of token ids, can perform tokenization +Words Vocab::encode(const std::string& line, + bool addEOS, + bool inference) const { + return vImpl_->encode(line, addEOS, inference); +} + +// list of token ids to single line, can perform detokenization +std::string Vocab::decode(const Words& sentence, + bool ignoreEOS) const { + return vImpl_->decode(sentence, ignoreEOS); +} + +// number of vocabulary items +size_t Vocab::size() const { return vImpl_->size(); } + +// number of vocabulary items +std::string Vocab::type() const { return vImpl_->type(); } + +// return EOS symbol id +Word Vocab::getEosId() const { return vImpl_->getEosId(); } + +// return UNK symbol id +Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } + +} // namespace marian +#endif diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h new file mode 100755 index 000000000..afe79d1f0 --- /dev/null +++ b/src/data/factored_vocab.h @@ -0,0 +1,73 @@ +#pragma once + +#include "common/definitions.h" +#include "data/types.h" +#include "common/options.h" +#include "common/file_stream.h" + +namespace marian { + +class IVocab; + +// Wrapper around vocabulary types. Can choose underlying +// vocabulary implementation (vImpl_) based on speficied path +// and suffix. +// Vocabulary implementations can currently be: +// * DefaultVocabulary for YAML (*.yml and *.yaml) and TXT (any other non-specific ending) +// * SentencePiece with suffix *.spm (works, but has to be created outside Marian) +class Vocab { +private: + Ptr vImpl_; + Ptr options_; + size_t batchIndex_; + +public: + Vocab(Ptr options, size_t batchIndex) + : options_(options), batchIndex_(batchIndex) {} + + size_t loadOrCreate(const std::string& vocabPath, + const std::vector& trainPaths, + size_t maxSize = 0); + + size_t load(const std::string& vocabPath, size_t maxSize = 0); + + void create(const std::string& vocabPath, + const std::vector& trainPaths, + size_t maxSize); + + void create(const std::string& vocabPath, + const std::string& trainPath, + size_t maxSize); + + // string token to token id + Word operator[](const std::string& word) const; + + // token index to string token + const std::string& operator[](Word word) const; + + // line of text to list of token ids, can perform tokenization + Words encode(const std::string& line, + bool addEOS = true, + bool inference = false) const; + + // list of token ids to single line, can perform detokenization + std::string decode(const Words& sentence, + bool ignoreEOS = true) const; + + // number of vocabulary items + size_t size() const; + + // number of vocabulary items + std::string type() const; + + // return EOS symbol id + Word getEosId() const; + + // return UNK symbol id + Word getUnkId() const; + + // create fake vocabulary for collecting batch statistics + void createFake(); +}; + +} // namespace marian diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 5c037f321..6bc733022 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -556,6 +556,7 @@ + @@ -701,6 +702,7 @@ + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 307424729..168c3480f 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -484,6 +484,9 @@ layers + + data + @@ -1531,6 +1534,9 @@ data + + data + From 37655127b4c3684cbbc44f483d0804416423dead Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 11:52:32 -0800 Subject: [PATCH 270/838] moved CSRSparseTensor and EmbeddingFactorMapping from generic.cpp to the new factored_vocab.h --- src/data/factored_vocab.h | 226 ++++++++++++++++++++++++++++---------- src/layers/generic.cpp | 176 ++--------------------------- 2 files changed, 172 insertions(+), 230 deletions(-) diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index afe79d1f0..e05d9b73b 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -1,73 +1,177 @@ +// Implementation of an IVocab that represents a factored representation. +// This is accessed via the IVocab interface, and also by Embedding and Output +// layers directly. + #pragma once #include "common/definitions.h" #include "data/types.h" -#include "common/options.h" -#include "common/file_stream.h" namespace marian { class IVocab; - -// Wrapper around vocabulary types. Can choose underlying -// vocabulary implementation (vImpl_) based on speficied path -// and suffix. -// Vocabulary implementations can currently be: -// * DefaultVocabulary for YAML (*.yml and *.yaml) and TXT (any other non-specific ending) -// * SentencePiece with suffix *.spm (works, but has to be created outside Marian) -class Vocab { -private: - Ptr vImpl_; - Ptr options_; - size_t batchIndex_; - + +class EmbeddingFactorMapping { public: - Vocab(Ptr options, size_t batchIndex) - : options_(options), batchIndex_(batchIndex) {} - - size_t loadOrCreate(const std::string& vocabPath, - const std::vector& trainPaths, - size_t maxSize = 0); - - size_t load(const std::string& vocabPath, size_t maxSize = 0); - - void create(const std::string& vocabPath, - const std::vector& trainPaths, - size_t maxSize); - - void create(const std::string& vocabPath, - const std::string& trainPath, - size_t maxSize); - - // string token to token id - Word operator[](const std::string& word) const; - - // token index to string token - const std::string& operator[](Word word) const; - - // line of text to list of token ids, can perform tokenization - Words encode(const std::string& line, - bool addEOS = true, - bool inference = false) const; - - // list of token ids to single line, can perform detokenization - std::string decode(const Words& sentence, - bool ignoreEOS = true) const; - - // number of vocabulary items - size_t size() const; - - // number of vocabulary items - std::string type() const; - - // return EOS symbol id - Word getEosId() const; - - // return UNK symbol id - Word getUnkId() const; - - // create fake vocabulary for collecting batch statistics - void createFake(); + struct CSRData { + Shape shape; + std::vector weights; + std::vector indices; + std::vector offsets; + }; + // mapPath = path to file with entries in order of vocab entries of the form + // WORD FACTOR1 FACTOR2 FACTOR3... + // listPath = path to file that lists all FACTOR names + // vocab = original vocabulary + // Note: The WORD field in the map file is redundant. It is required for consistency checking only. + // Factors are grouped + // - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group + // - factors within a group as multi-class and normalized that way + // - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) + // - one prefix must not contain another + // - all factors not matching a prefix get lumped into yet another class (the lemmas) + // - factor vocab must be sorted such that all groups are consecutive + // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries + EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { + std::vector paths = options->get>("embedding-factors"); + ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); + auto mapPath = paths[0]; + auto factorVocabPath = paths[1]; + auto vocabPath = options->get("vocab"); + + // Note: We misuse the Vocab class a little. + // Specifically, it means that the factorVocab_ must contain and "". + Vocab vocab(New(), 0); + vocab.load(vocabPath); + auto vocabSize = vocab.size(); + factorVocab_.load(factorVocabPath); + auto numFactors = factorVocab_.size(); + + // load and parse factorMap + factorMap_.resize(vocabSize); + factorRefCounts_.resize(numFactors); + std::vector tokens; + io::InputFileStream in(mapPath); + std::string line; + size_t numTotalFactors = 0; + for (WordIndex v = 0; io::getline(in, line); v++) { + tokens.clear(); // @BUGBUG: should be done in split() + utils::splitAny(line, tokens, " \t"); + ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + for (size_t i = 1; i < tokens.size(); i++) { + auto u = factorVocab_[tokens[i]].toWordIndex(); + factorMap_[v].push_back(u); + factorRefCounts_[u]++; + } + numTotalFactors += tokens.size() - 1; + } + LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); + + // form groups + // @TODO: hard-coded for these initial experiments + std::vector groupPrefixes = { + "@C", + "@GL", "@GR" + }; + groupPrefixes.insert(groupPrefixes.begin(), "(unassigned)"); // first group is fallback for normal words (the string is only used for messages) + size_t numGroups = groupPrefixes.size(); + factorGroups_.resize(numFactors, 0); + for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 + const auto& groupPrefix = groupPrefixes[g]; + for (WordIndex u = 0; u < numFactors; u++) + if (utils::beginsWith(factorVocab_[Word::fromWordIndex(u)], groupPrefix)) { + ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[Word::fromWordIndex(u)], groupPrefix); + factorGroups_[u] = g; + } + } + // determine group index ranges + groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); + std::vector groupCounts(numGroups); // number of group members + for (WordIndex u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts + auto g = factorGroups_[u]; + if (groupRanges_[g].first > u) + groupRanges_[g].first = u; + if (groupRanges_[g].second < u + 1) + groupRanges_[g].second = u + 1; + groupCounts[g]++; + } + // create mappings needed for normalization in factored outputs + factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g + factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) + for (WordIndex v = 0; v < vocabSize; v++) { + for (auto u : factorMap_[v]) { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); + factorMasks_[g][v] = 1.0f; + } + } + //for (Word v = 0; v < vocabSize; v++) { + // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], + // factorMasks_[0][v], factorIndices_[0][v], + // factorMasks_[1][v], factorIndices_[1][v], + // factorMasks_[2][v], factorIndices_[2][v], + // factorMasks_[3][v], factorIndices_[3][v]); + //} + //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon + for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups + LOG(info, "[embedding] Factor group '{}' has {} members ({})", + groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); + if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition + continue; + ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], + "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); + //auto& mVec = mVecs_[g]; + //mVec.resize(numFactors, 0.0f); + //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) + // mVec[i] = 1.0f; + } + + // create the global factor matrix, which is used for factored embeddings + std::vector data(vocabSize); + std::iota(data.begin(), data.end(), 0); + globalFactorMatrix_ = csr_rows(data); // [V x U] + } + + size_t factorVocabSize() const { return factorVocab_.size(); } + + // create a CSR matrix M[V,U] from indices[] with + // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced + CSRData csr_rows(const std::vector& words) const { + std::vector weights; + std::vector indices; + std::vector offsets; + offsets.reserve(words.size() + 1); + indices.reserve(words.size()); // (at least this many) + // loop over all input words, and select the corresponding set of unit indices into CSR format + offsets.push_back((IndexType)indices.size()); + for (auto v : words) { + const auto& m = factorMap_[v]; + for (auto u : m) { + indices.push_back(u); + weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); + } + offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset + } + return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; + } + + const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v + size_t getNumGroups() const { return groupRanges_.size(); } + std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) + const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g + const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor +private: + Vocab factorVocab_; // [factor name] -> factor index = row of E_ + std::vector> factorMap_; // [word index v] -> set of factor indices u + std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ + CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v + std::vector factorGroups_; // [u] -> group id of factor u + std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. + std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g + std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) +//public: // @TODO: temporarily; later factor this properly + //std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; } // namespace marian diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5844b3fb6..8ce29c9a9 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -2,179 +2,17 @@ #include "layers/generic.h" #include "layers/loss.h" +#include "data/factored_vocab.h" using std::size_t; // not sure why this is needed namespace marian { - struct CSRSparseTensor { // simplistic for now - Shape shape; - Expr values; // [k_i..k_{i+1}-1] -> value at [i,j] - Expr indices; // [k_i..k_{i+1}-1] -> j of non-null value - Expr offsets; // [i] -> k_i - }; - - class EmbeddingFactorMapping { - public: - struct CSRData { - Shape shape; - std::vector weights; - std::vector indices; - std::vector offsets; - }; - // mapPath = path to file with entries in order of vocab entries of the form - // WORD FACTOR1 FACTOR2 FACTOR3... - // listPath = path to file that lists all FACTOR names - // vocab = original vocabulary - // Note: The WORD field in the map file is redundant. It is required for consistency checking only. - // Factors are grouped - // - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group - // - factors within a group as multi-class and normalized that way - // - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) - // - one prefix must not contain another - // - all factors not matching a prefix get lumped into yet another class (the lemmas) - // - factor vocab must be sorted such that all groups are consecutive - // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { - std::vector paths = options->get>("embedding-factors"); - ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); - auto mapPath = paths[0]; - auto factorVocabPath = paths[1]; - auto vocabPath = options->get("vocab"); - - // Note: We misuse the Vocab class a little. - // Specifically, it means that the factorVocab_ must contain and "". - Vocab vocab(New(), 0); - vocab.load(vocabPath); - auto vocabSize = vocab.size(); - factorVocab_.load(factorVocabPath); - auto numFactors = factorVocab_.size(); - - // load and parse factorMap - factorMap_.resize(vocabSize); - factorRefCounts_.resize(numFactors); - std::vector tokens; - io::InputFileStream in(mapPath); - std::string line; - size_t numTotalFactors = 0; - for (WordIndex v = 0; io::getline(in, line); v++) { - tokens.clear(); // @BUGBUG: should be done in split() - utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); - for (size_t i = 1; i < tokens.size(); i++) { - auto u = factorVocab_[tokens[i]].toWordIndex(); - factorMap_[v].push_back(u); - factorRefCounts_[u]++; - } - numTotalFactors += tokens.size() - 1; - } - LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); - - // form groups - // @TODO: hard-coded for these initial experiments - std::vector groupPrefixes = { - "@C", - "@GL", "@GR" - }; - groupPrefixes.insert(groupPrefixes.begin(), "(unassigned)"); // first group is fallback for normal words (the string is only used for messages) - size_t numGroups = groupPrefixes.size(); - factorGroups_.resize(numFactors, 0); - for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 - const auto& groupPrefix = groupPrefixes[g]; - for (WordIndex u = 0; u < numFactors; u++) - if (utils::beginsWith(factorVocab_[Word::fromWordIndex(u)], groupPrefix)) { - ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[Word::fromWordIndex(u)], groupPrefix); - factorGroups_[u] = g; - } - } - // determine group index ranges - groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); - std::vector groupCounts(numGroups); // number of group members - for (WordIndex u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts - auto g = factorGroups_[u]; - if (groupRanges_[g].first > u) - groupRanges_[g].first = u; - if (groupRanges_[g].second < u + 1) - groupRanges_[g].second = u + 1; - groupCounts[g]++; - } - // create mappings needed for normalization in factored outputs - factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g - factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) - for (WordIndex v = 0; v < vocabSize; v++) { - for (auto u : factorMap_[v]) { - auto g = factorGroups_[u]; // convert u to relative u within factor group range - ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); - factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); - factorMasks_[g][v] = 1.0f; - } - } - //for (Word v = 0; v < vocabSize; v++) { - // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], - // factorMasks_[0][v], factorIndices_[0][v], - // factorMasks_[1][v], factorIndices_[1][v], - // factorMasks_[2][v], factorIndices_[2][v], - // factorMasks_[3][v], factorIndices_[3][v]); - //} - //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon - for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - LOG(info, "[embedding] Factor group '{}' has {} members ({})", - groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); - if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition - continue; - ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], - "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - //auto& mVec = mVecs_[g]; - //mVec.resize(numFactors, 0.0f); - //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) - // mVec[i] = 1.0f; - } - - // create the global factor matrix, which is used for factored embeddings - std::vector data(vocabSize); - std::iota(data.begin(), data.end(), 0); - globalFactorMatrix_ = csr_rows(data); // [V x U] - } - - size_t factorVocabSize() const { return factorVocab_.size(); } - - // create a CSR matrix M[V,U] from indices[] with - // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced - CSRData csr_rows(const std::vector& words) const { - std::vector weights; - std::vector indices; - std::vector offsets; - offsets.reserve(words.size() + 1); - indices.reserve(words.size()); // (at least this many) - // loop over all input words, and select the corresponding set of unit indices into CSR format - offsets.push_back((IndexType)indices.size()); - for (auto v : words) { - const auto& m = factorMap_[v]; - for (auto u : m) { - indices.push_back(u); - weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); - } - offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset - } - return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; - } - - const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v - size_t getNumGroups() const { return groupRanges_.size(); } - std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) - const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g - const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor - private: - Vocab factorVocab_; // [factor name] -> factor index = row of E_ - std::vector> factorMap_; // [word index v] -> set of factor indices u - std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ - CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v - std::vector factorGroups_; // [u] -> group id of factor u - std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. - std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g - std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) - //public: // @TODO: temporarily; later factor this properly - //std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group - }; + //struct CSRSparseTensor { // simplistic for now + // Shape shape; + // Expr values; // [k_i..k_{i+1}-1] -> value at [i,j] + // Expr indices; // [k_i..k_{i+1}-1] -> j of non-null value + // Expr offsets; // [i] -> k_i + //}; Logits::Logits(Expr logits) : Logits(New(logits, nullptr)) {} // single-output constructor from Expr only (RationalLoss has no count) From c962433c9479b66137921c481e3df688f93abbd7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 12:12:59 -0800 Subject: [PATCH 271/838] renamed EmbeddingFactorMapping to FactoredVocab --- src/data/factored_vocab.cpp | 18 ++++++++++++++--- src/data/factored_vocab.h | 7 +++++-- src/data/vocab.cpp | 4 ---- src/data/vocab_base.h | 9 ++++++++- src/layers/generic.cpp | 40 ++++++++++++++++++------------------- src/layers/generic.h | 12 +++++------ 6 files changed, 54 insertions(+), 36 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index df203cbcd..e8cdda168 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -1,9 +1,13 @@ -#if 0 -#include "data/vocab.h" #include "data/vocab_base.h" +#include "common/definitions.h" +#include "data/types.h" +#include "common/options.h" +#include "common/regex.h" +#include "data/factored_vocab.h" namespace marian { +#if 0 Word Word::NONE = Word(); Word Word::ZERO = Word(0); Word Word::DEFAULT_EOS_ID = Word(0); @@ -126,6 +130,14 @@ Word Vocab::getEosId() const { return vImpl_->getEosId(); } // return UNK symbol id Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } +#endif + +Ptr createFactoredPieceVocab(const std::string& vocabPath, Ptr options) { + bool isSentencePiece = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); + if(isSentencePiece) + return New(options); + else + return nullptr; +} } // namespace marian -#endif diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index e05d9b73b..59712f6e4 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -6,12 +6,15 @@ #include "common/definitions.h" #include "data/types.h" +#include "data/vocab.h" + +#include // for std::iota() namespace marian { class IVocab; -class EmbeddingFactorMapping { +class FactoredVocab { public: struct CSRData { Shape shape; @@ -32,7 +35,7 @@ class EmbeddingFactorMapping { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - EmbeddingFactorMapping(Ptr options) : factorVocab_(New(), 0) { + FactoredVocab(Ptr options) : factorVocab_(New(), 0) { std::vector paths = options->get>("embedding-factors"); ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); auto mapPath = paths[0]; diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 8094c699c..d2e4b046e 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -8,10 +8,6 @@ Word Word::ZERO = Word(0); Word Word::DEFAULT_EOS_ID = Word(0); Word Word::DEFAULT_UNK_ID = Word(1); -Ptr createDefaultVocab(); -Ptr createClassVocab(); -Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); - // @TODO: make each vocab peek on type Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index 9a82131e8..20f39c5ec 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -46,4 +46,11 @@ class IVocab { virtual void createFake() = 0; }; -} \ No newline at end of file +class Options; +Ptr createDefaultVocab(); +Ptr createClassVocab(); +Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr, size_t batchIndex); +class FactoredVocab; +Ptr createFactoredVocab(const std::string& vocabPath, Ptr); + +} diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 8ce29c9a9..95a7bda78 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -25,7 +25,7 @@ namespace marian { LOG_ONCE(info, "[logits] applyLossFunction() for {} factors", logits_.size()); ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); - if (!embeddingFactorMapping_) { + if (!factoredVocab_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); return lossFn(logits_.front()->loss(), indices); } @@ -33,11 +33,11 @@ namespace marian { // accumulate all CEs for all words that have the factor // Memory-wise, this is cheap, all temp objects below are batches of scalars or lookup vectors. Expr loss; - auto numGroups = embeddingFactorMapping_->getNumGroups(); + auto numGroups = factoredVocab_->getNumGroups(); for (size_t g = 0; g < numGroups; g++) { indices; // [B... * 1] all batch items flattened - auto factorMaskVector = embeddingFactorMapping_->getFactorMasks(g); // [v] 1.0 if v has factor of group g - auto factorIndexVector = embeddingFactorMapping_->getFactorIndices(g); // [v] index of factor for word v in group p; must be 0 if factor is not used + auto factorMaskVector = factoredVocab_->getFactorMasks(g); // [v] 1.0 if v has factor of group g + auto factorIndexVector = factoredVocab_->getFactorIndices(g); // [v] index of factor for word v in group p; must be 0 if factor is not used auto factorMaskMatrix = graph->constant({(int)factorMaskVector.size(), 1}, inits::from_vector(factorMaskVector), Type::float32); // [V x 1] auto factorIndexMatrix = graph->constant({(int)factorIndexVector.size(), 1}, inits::from_vector(factorIndexVector), Type::uint32); // [V x 1(Ug)] auto factorIndex = rows(factorIndexMatrix, indices); // [B... * 1(Ug)] map word indices to factor indices (indices into factorLogits) @@ -54,7 +54,7 @@ namespace marian { // This function assumes this object holds a single factor that represents a rational loss (with count). Ptr Logits::getRationalLoss() const { //return New(getLogits(), logits_.front()->count()); - ABORT_IF(logits_.size() != 1 || embeddingFactorMapping_, "getRationalLoss() cannot be used on multi-factor outputs"); + ABORT_IF(logits_.size() != 1 || factoredVocab_, "getRationalLoss() cannot be used on multi-factor outputs"); ABORT_IF(!logits_.front()->count(), "getRationalLoss() used on rational loss without count"); return logits_.front(); } @@ -63,7 +63,7 @@ namespace marian { // into output-vocab logits according to the factored model (with correct normalization of factors). Expr Logits::getLogits() const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); - if (!embeddingFactorMapping_) { + if (!factoredVocab_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); return logits_.front()->loss(); } @@ -76,7 +76,7 @@ namespace marian { // sum up the unit logits across factors for each target word auto graph = y->graph(); - auto factorMatrix = embeddingFactorMapping_->getGlobalFactorMatrix(); // [V x U] + auto factorMatrix = factoredVocab_->getGlobalFactorMatrix(); // [V x U] y = dot_csr( y, // [B x U] factorMatrix.shape, @@ -92,7 +92,7 @@ namespace marian { std::vector> newLogits; for (const auto& l : logits_) newLogits.emplace_back(New(l->loss(), count)); - return Logits(std::move(newLogits), embeddingFactorMapping_); + return Logits(std::move(newLogits), factoredVocab_); } namespace mlp { @@ -106,8 +106,8 @@ namespace marian { if (options_->has("embedding-factors")) { ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); - embeddingFactorMapping_ = New(options_); - numOutputClasses = (int)embeddingFactorMapping_->factorVocabSize(); + factoredVocab_ = New(options_); + numOutputClasses = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored outputs enabled"); } @@ -135,15 +135,15 @@ namespace marian { } return affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true); } - else if (embeddingFactorMapping_) { + else if (factoredVocab_) { auto graph = input->graph(); // project each factor separately - auto numGroups = embeddingFactorMapping_->getNumGroups(); + auto numGroups = factoredVocab_->getNumGroups(); std::vector> allLogits(numGroups); for (size_t g = 0; g < numGroups; g++) { - auto range = embeddingFactorMapping_->getGroupRange(g); - ABORT_IF(g > 0 && range.first != embeddingFactorMapping_->getGroupRange(g-1).second, "Factor groups must be consecutive"); // we could sort groupYs though + auto range = factoredVocab_->getGroupRange(g); + ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive"); // we could sort groupYs though // slice this group's section out of W_ // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. auto factorWt = slice(Wt_, isLegacyUntransposedW ? -1 : 0, Slice((int)range.first, (int)range.second)); @@ -152,7 +152,7 @@ namespace marian { auto factorLogits = affine(input, factorWt, factorB, false, /*transB=*/isLegacyUntransposedW ? false : true); // [B... x U] factor logits allLogits[g] = New(factorLogits, nullptr); } - return Logits(std::move(allLogits), embeddingFactorMapping_); + return Logits(std::move(allLogits), factoredVocab_); } else return affine(input, Wt_, b_, false, /*transB=*/isLegacyUntransposedW ? false : true); @@ -167,8 +167,8 @@ namespace marian { bool fixed = opt("fixed", false); if (options_->has("embedding-factors")) { - embeddingFactorMapping_ = New(options_); - dimVoc = (int)embeddingFactorMapping_->factorVocabSize(); + factoredVocab_ = New(options_); + dimVoc = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored embeddings enabled"); } @@ -190,7 +190,7 @@ namespace marian { /*private*/ Expr Embedding::multiRows(const std::vector& data) const { auto graph = E_->graph(); - auto factoredData = embeddingFactorMapping_->csr_rows(data); + auto factoredData = factoredVocab_->csr_rows(data); // multi-hot factor vectors are represented as a sparse CSR matrix // [row index = word position index] -> set of factor indices for word at this position ABORT_IF(factoredData.shape != Shape({(int)factoredData.offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), "shape mismatch??"); @@ -240,7 +240,7 @@ namespace marian { auto batchEmbeddings = apply(subBatch->data(), {dimWords, dimBatch, dimEmb}); #else Expr selectedEmbs; - if (embeddingFactorMapping_) + if (factoredVocab_) selectedEmbs = multiRows(subBatch->data()); else selectedEmbs = rows(E_, subBatch->data()); @@ -257,7 +257,7 @@ namespace marian { Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { Expr selectedEmbs; - if (embeddingFactorMapping_) + if (factoredVocab_) selectedEmbs = multiRows(embIdx); else selectedEmbs = rows(E_, embIdx); diff --git a/src/layers/generic.h b/src/layers/generic.h index a3d184f0f..c5a199384 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -57,7 +57,7 @@ struct IEmbeddingLayer { virtual Expr applyIndices(const std::vector& embIdx, const Shape& shape) const = 0; }; -class EmbeddingFactorMapping; +class FactoredVocab; // @HACK: Frank's quick implementation of factored outputs. To be re-thought once it works. // Output layer returns a Logits object, which is able to compute some things on the fly @@ -71,8 +71,8 @@ class Logits { logits_.push_back(logits); } Logits(Expr logits); // single-output constructor from Expr only (RationalLoss has no count) - Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor - : logits_(std::move(logits)), embeddingFactorMapping_(embeddingFactorMapping) {} + Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor + : logits_(std::move(logits)), factoredVocab_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; @@ -87,7 +87,7 @@ class Logits { private: // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. std::vector> logits_; - Ptr embeddingFactorMapping_; + Ptr factoredVocab_; }; // Unary function that returns a Logits object @@ -172,7 +172,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { bool isLegacyUntransposedW{false}; // legacy-model emulation: W is stored in non-transposed form Expr cachedShortWt_; // short-listed version, cached (cleared by clear()) Expr cachedShortb_; // these match the current value of shortlist_ - Ptr embeddingFactorMapping_; + Ptr factoredVocab_; // optional parameters set/updated after construction Expr tiedParam_; @@ -221,7 +221,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; - Ptr embeddingFactorMapping_; + Ptr factoredVocab_; Expr multiRows(const std::vector& data) const; public: Embedding(Ptr graph, Ptr options); From 69850f16d0ff1703924994fa531d31b8fcc7c7e8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 13:38:26 -0800 Subject: [PATCH 272/838] towards turning FactoredVocab into an actual IVocab --- src/data/factored_vocab.cpp | 8 ++++---- src/data/factored_vocab.h | 16 +++++++--------- src/layers/generic.cpp | 10 ++++++++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index e8cdda168..0f1bf5b5f 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -132,10 +132,10 @@ Word Vocab::getEosId() const { return vImpl_->getEosId(); } Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } #endif -Ptr createFactoredPieceVocab(const std::string& vocabPath, Ptr options) { - bool isSentencePiece = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); - if(isSentencePiece) - return New(options); +Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { + bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); + if(isFactoredVocab) + return New(vocabPath, options); else return nullptr; } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 59712f6e4..36ccb006f 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -7,14 +7,13 @@ #include "common/definitions.h" #include "data/types.h" #include "data/vocab.h" +#include "data/vocab_base.h" #include // for std::iota() namespace marian { -class IVocab; - -class FactoredVocab { +class FactoredVocab /*: public IVocab*/ { public: struct CSRData { Shape shape; @@ -35,12 +34,11 @@ class FactoredVocab { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - FactoredVocab(Ptr options) : factorVocab_(New(), 0) { - std::vector paths = options->get>("embedding-factors"); - ABORT_IF(paths.size() != 2, "--embedding-factors expects two paths"); - auto mapPath = paths[0]; - auto factorVocabPath = paths[1]; - auto vocabPath = options->get("vocab"); + FactoredVocab(const std::string& factoredVocabPath, Ptr options) : factorVocab_(New(), 0) { + auto mapPath = factoredVocabPath; + auto factorVocabPath = mapPath; + factorVocabPath.back() = 'l'; // map .fm to .fl + auto vocabPath = options->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs // Note: We misuse the Vocab class a little. // Specifically, it means that the factorVocab_ must contain and "". diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 95a7bda78..135f32219 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -105,8 +105,11 @@ namespace marian { auto numOutputClasses = options_->get("dim"); if (options_->has("embedding-factors")) { + std::vector paths = options_->get>("embedding-factors"); + factoredVocab_ = createFactoredVocab(paths[0], options_); + } + if (factoredVocab_) { ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); - factoredVocab_ = New(options_); numOutputClasses = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored outputs enabled"); } @@ -167,7 +170,10 @@ namespace marian { bool fixed = opt("fixed", false); if (options_->has("embedding-factors")) { - factoredVocab_ = New(options_); + std::vector paths = options_->get>("embedding-factors"); + factoredVocab_ = createFactoredVocab(paths[0], options_); + } + if (factoredVocab_) { dimVoc = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored embeddings enabled"); } From 299fdcc4a77d324d461778ac97aa625c97bea4a4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 14:06:43 -0800 Subject: [PATCH 273/838] FactoredVocab is now an IVocab --- src/data/corpus.cpp | 2 +- src/data/factored_vocab.cpp | 30 +++++++++++++++++++++++++++- src/data/factored_vocab.h | 34 +++++++++++++++++++++++++++++--- src/data/sentencepiece_vocab.cpp | 1 - src/data/vocab_base.h | 2 +- src/layers/generic.cpp | 4 ++++ 6 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 07993f485..77a33bf98 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -18,7 +18,7 @@ Corpus::Corpus(std::vector paths, : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} void Corpus::preprocessLine(std::string& line, size_t streamId) { - if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0) { + if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0 && !inference_) { line = utils::utf8ToUpper(line); if (streamId == 0) LOG_ONCE(info, "[data] source all-caps'ed line to {}", line); diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 0f1bf5b5f..0ad6bfa84 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -132,10 +132,38 @@ Word Vocab::getEosId() const { return vImpl_->getEosId(); } Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } #endif +/*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { + word; + return Word(); +} + +/*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool inference /*= false*/) const /*override final*/ { + line; addEOS; inference; + return {}; +} + +/*virtual*/ std::string FactoredVocab::decode(const Words& sentence, bool ignoreEos /*= true*/) const /*override final*/ { + sentence; ignoreEos; + return {}; +} + +/*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { + id; + static std::string x; + return x; +} + +/*virtual*/ size_t FactoredVocab::size() const /*override final*/ { + return 0; +} + +/*virtual*/ void FactoredVocab::createFake() /*override final*/ { +} + Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); if(isFactoredVocab) - return New(vocabPath, options); + return New(options); else return nullptr; } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 36ccb006f..e30b58483 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -13,7 +13,7 @@ namespace marian { -class FactoredVocab /*: public IVocab*/ { +class FactoredVocab : public IVocab { public: struct CSRData { Shape shape; @@ -21,6 +21,11 @@ class FactoredVocab /*: public IVocab*/ { std::vector indices; std::vector offsets; }; + + FactoredVocab(Ptr options) : options_(options), factorVocab_(New(), 0) { } + + // from IVocab: + // mapPath = path to file with entries in order of vocab entries of the form // WORD FACTOR1 FACTOR2 FACTOR3... // listPath = path to file that lists all FACTOR names @@ -34,11 +39,12 @@ class FactoredVocab /*: public IVocab*/ { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - FactoredVocab(const std::string& factoredVocabPath, Ptr options) : factorVocab_(New(), 0) { + virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final { + ABORT_IF(maxSizeUnused != 0, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size"); auto mapPath = factoredVocabPath; auto factorVocabPath = mapPath; factorVocabPath.back() = 'l'; // map .fm to .fl - auto vocabPath = options->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs + auto vocabPath = options_->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs // Note: We misuse the Vocab class a little. // Specifically, it means that the factorVocab_ must contain and "". @@ -132,8 +138,23 @@ class FactoredVocab /*: public IVocab*/ { std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] + + return factorVocabSize(); // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here } + virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } + virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } + virtual const std::vector& suffixes() const override final { const static std::vector exts{".fm"}; return exts; } + virtual Word operator[](const std::string& word) const override final; + virtual Words encode(const std::string& line, bool addEOS = true, bool inference = false) const override final; + virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; + virtual const std::string& operator[](Word id) const override final; + virtual size_t size() const override final; + virtual std::string type() const override final { return "FactoredVocab"; } + virtual Word getEosId() const override { return eosId_; } + virtual Word getUnkId() const override { return unkId_; } + virtual void createFake() override final; + size_t factorVocabSize() const { return factorVocab_.size(); } // create a CSR matrix M[V,U] from indices[] with @@ -162,7 +183,14 @@ class FactoredVocab /*: public IVocab*/ { std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor + private: + Ptr options_; + // main vocab + Word eosId_ = Word::NONE; + Word unkId_ = Word::NONE; + + // factors Vocab factorVocab_; // [factor name] -> factor index = row of E_ std::vector> factorMap_; // [word index v] -> set of factor indices u std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp index 9af727565..2f419738d 100755 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -126,7 +126,6 @@ class SentencePieceVocab : public IVocab { alpha_, batchIndex_); } - } virtual const std::string& canonicalExtension() const override { return suffixes_[0]; } diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index 20f39c5ec..b993233e6 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -19,7 +19,7 @@ class IVocab { virtual const std::string& canonicalExtension() const = 0; virtual const std::vector& suffixes() const = 0; - size_t findAndLoad(const std::string& path, size_t maxSize) { + size_t findAndLoad(const std::string& path, size_t maxSize) { // @TODO: Only used in one place; just inline it there -> true interface for(auto suffix : suffixes()) if(filesystem::exists(path + suffix)) return load(path + suffix, maxSize); diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 135f32219..57b6d8374 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -109,6 +109,8 @@ namespace marian { factoredVocab_ = createFactoredVocab(paths[0], options_); } if (factoredVocab_) { + std::vector paths = options_->get>("embedding-factors"); + factoredVocab_->load(paths[0]); ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); numOutputClasses = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored outputs enabled"); @@ -174,6 +176,8 @@ namespace marian { factoredVocab_ = createFactoredVocab(paths[0], options_); } if (factoredVocab_) { + std::vector paths = options_->get>("embedding-factors"); + factoredVocab_->load(paths[0]); dimVoc = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored embeddings enabled"); } From 2398fbbf2a5f9f9cdb981cc3c642146b5204ce6b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 14:11:42 -0800 Subject: [PATCH 274/838] moved FactoredVocab code from header to CPP --- src/data/factored_vocab.cpp | 248 +++++++++++++++++++----------------- src/data/factored_vocab.h | 146 +-------------------- 2 files changed, 136 insertions(+), 258 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 0ad6bfa84..3bae39237 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -7,131 +7,122 @@ namespace marian { -#if 0 -Word Word::NONE = Word(); -Word Word::ZERO = Word(0); -Word Word::DEFAULT_EOS_ID = Word(0); -Word Word::DEFAULT_UNK_ID = Word(1); - -Ptr createDefaultVocab(); -Ptr createClassVocab(); -Ptr createSentencePieceVocab(const std::string& /*vocabPath*/, Ptr, size_t /*batchIndex*/); - -// @TODO: make each vocab peek on type -Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { - auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); - if(vocab) { - return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it - } else { - // check type of input, if not given, assume "sequence" - auto inputTypes = options->get>("input-types", {}); - std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; - return inputType == "class" ? createClassVocab() : createDefaultVocab(); - } -} - -size_t Vocab::loadOrCreate(const std::string& vocabPath, - const std::vector& trainPaths, - size_t maxSize) { - size_t size = 0; - if(vocabPath.empty()) { - // No vocabulary path was given, attempt to first find a vocabulary - // for trainPaths[0] + possible suffixes. If not found attempt to create - // as trainPaths[0] + canonical suffix. - // Only search based on first path, maybe disable this at all? - - LOG(info, - "No vocabulary path given; " - "trying to find default vocabulary based on data path {}", - trainPaths[0]); - - vImpl_ = createDefaultVocab(); - size = vImpl_->findAndLoad(trainPaths[0], maxSize); - - if(size == 0) { - auto newVocabPath = trainPaths[0] + vImpl_->canonicalExtension(); - LOG(info, - "No vocabulary path given; " - "trying to create vocabulary based on data paths {}", - utils::join(trainPaths, ", ")); - create(newVocabPath, trainPaths, maxSize); - size = load(newVocabPath, maxSize); +// mapPath = path to file with entries in order of vocab entries of the form +// WORD FACTOR1 FACTOR2 FACTOR3... +// listPath = path to file that lists all FACTOR names +// vocab = original vocabulary +// Note: The WORD field in the map file is redundant. It is required for consistency checking only. +// Factors are grouped +// - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group +// - factors within a group as multi-class and normalized that way +// - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) +// - one prefix must not contain another +// - all factors not matching a prefix get lumped into yet another class (the lemmas) +// - factor vocab must be sorted such that all groups are consecutive +// - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries +/*virtual*/ size_t FactoredVocab::load(const std::string& factoredVocabPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { + ABORT_IF(maxSizeUnused != 0, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size"); + auto mapPath = factoredVocabPath; + auto factorVocabPath = mapPath; + factorVocabPath.back() = 'l'; // map .fm to .fl + auto vocabPath = options_->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs + + // Note: We misuse the Vocab class a little. + // Specifically, it means that the factorVocab_ must contain and "". + Vocab vocab(New(), 0); + vocab.load(vocabPath); + auto vocabSize = vocab.size(); + factorVocab_.load(factorVocabPath); + auto numFactors = factorVocab_.size(); + + // load and parse factorMap + factorMap_.resize(vocabSize); + factorRefCounts_.resize(numFactors); + std::vector tokens; + io::InputFileStream in(mapPath); + std::string line; + size_t numTotalFactors = 0; + for (WordIndex v = 0; io::getline(in, line); v++) { + tokens.clear(); // @BUGBUG: should be done in split() + utils::splitAny(line, tokens, " \t"); + ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + for (size_t i = 1; i < tokens.size(); i++) { + auto u = factorVocab_[tokens[i]].toWordIndex(); + factorMap_[v].push_back(u); + factorRefCounts_[u]++; } - } else { - if(!filesystem::exists(vocabPath)) { - // Vocabulary path was given, but no vocabulary present, - // attempt to create in specified location. - create(vocabPath, trainPaths, maxSize); + numTotalFactors += tokens.size() - 1; + } + LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); + + // form groups + // @TODO: hard-coded for these initial experiments + std::vector groupPrefixes = { + "@C", + "@GL", "@GR" + }; + groupPrefixes.insert(groupPrefixes.begin(), "(unassigned)"); // first group is fallback for normal words (the string is only used for messages) + size_t numGroups = groupPrefixes.size(); + factorGroups_.resize(numFactors, 0); + for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 + const auto& groupPrefix = groupPrefixes[g]; + for (WordIndex u = 0; u < numFactors; u++) + if (utils::beginsWith(factorVocab_[Word::fromWordIndex(u)], groupPrefix)) { + ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[Word::fromWordIndex(u)], groupPrefix); + factorGroups_[u] = g; + } + } + // determine group index ranges + groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); + std::vector groupCounts(numGroups); // number of group members + for (WordIndex u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts + auto g = factorGroups_[u]; + if (groupRanges_[g].first > u) + groupRanges_[g].first = u; + if (groupRanges_[g].second < u + 1) + groupRanges_[g].second = u + 1; + groupCounts[g]++; + } + // create mappings needed for normalization in factored outputs + factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g + factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) + for (WordIndex v = 0; v < vocabSize; v++) { + for (auto u : factorMap_[v]) { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); + factorMasks_[g][v] = 1.0f; } - // Vocabulary path exists, attempting to load - size = load(vocabPath, maxSize); } - LOG(info, "[data] Setting vocabulary size for input {} to {}", batchIndex_, size); - return size; -} - -size_t Vocab::load(const std::string& vocabPath, size_t maxSize) { - if(!vImpl_) - vImpl_ = createVocab(vocabPath, options_, batchIndex_); - return vImpl_->load(vocabPath, (int)maxSize); -} - -void Vocab::create(const std::string& vocabPath, - const std::vector& trainPaths, - size_t maxSize) { - if(!vImpl_) - vImpl_ = createVocab(vocabPath, options_, batchIndex_); - vImpl_->create(vocabPath, trainPaths, maxSize); -} - -void Vocab::create(const std::string& vocabPath, - const std::string& trainPath, - size_t maxSize) { - create(vocabPath, std::vector({trainPath}), maxSize); -} - -void Vocab::createFake() { - if(!vImpl_) - vImpl_ = createDefaultVocab(); // DefaultVocab is OK here - vImpl_->createFake(); -} - -// string token to token id -Word Vocab::operator[](const std::string& word) const { - return vImpl_->operator[](word); -} - -// token id to string token -const std::string& Vocab::operator[](Word id) const { - return vImpl_->operator[](id); -} + //for (Word v = 0; v < vocabSize; v++) { + // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], + // factorMasks_[0][v], factorIndices_[0][v], + // factorMasks_[1][v], factorIndices_[1][v], + // factorMasks_[2][v], factorIndices_[2][v], + // factorMasks_[3][v], factorIndices_[3][v]); + //} + //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon + for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups + LOG(info, "[embedding] Factor group '{}' has {} members ({})", + groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); + if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition + continue; + ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], + "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); + //auto& mVec = mVecs_[g]; + //mVec.resize(numFactors, 0.0f); + //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) + // mVec[i] = 1.0f; + } -// line of text to list of token ids, can perform tokenization -Words Vocab::encode(const std::string& line, - bool addEOS, - bool inference) const { - return vImpl_->encode(line, addEOS, inference); -} + // create the global factor matrix, which is used for factored embeddings + std::vector data(vocabSize); + std::iota(data.begin(), data.end(), 0); + globalFactorMatrix_ = csr_rows(data); // [V x U] -// list of token ids to single line, can perform detokenization -std::string Vocab::decode(const Words& sentence, - bool ignoreEOS) const { - return vImpl_->decode(sentence, ignoreEOS); + return factorVocabSize(); // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here } -// number of vocabulary items -size_t Vocab::size() const { return vImpl_->size(); } - -// number of vocabulary items -std::string Vocab::type() const { return vImpl_->type(); } - -// return EOS symbol id -Word Vocab::getEosId() const { return vImpl_->getEosId(); } - -// return UNK symbol id -Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } -#endif - /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { word; return Word(); @@ -160,6 +151,27 @@ Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } /*virtual*/ void FactoredVocab::createFake() /*override final*/ { } +// create a CSR matrix M[V,U] from indices[] with +// M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced +FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& words) const { + std::vector weights; + std::vector indices; + std::vector offsets; + offsets.reserve(words.size() + 1); + indices.reserve(words.size()); // (at least this many) + // loop over all input words, and select the corresponding set of unit indices into CSR format + offsets.push_back((IndexType)indices.size()); + for (auto v : words) { + const auto& m = factorMap_[v]; + for (auto u : m) { + indices.push_back(u); + weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); + } + offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset + } + return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; +} + Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); if(isFactoredVocab) diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index e30b58483..bb4be3600 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -25,122 +25,7 @@ class FactoredVocab : public IVocab { FactoredVocab(Ptr options) : options_(options), factorVocab_(New(), 0) { } // from IVocab: - - // mapPath = path to file with entries in order of vocab entries of the form - // WORD FACTOR1 FACTOR2 FACTOR3... - // listPath = path to file that lists all FACTOR names - // vocab = original vocabulary - // Note: The WORD field in the map file is redundant. It is required for consistency checking only. - // Factors are grouped - // - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group - // - factors within a group as multi-class and normalized that way - // - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) - // - one prefix must not contain another - // - all factors not matching a prefix get lumped into yet another class (the lemmas) - // - factor vocab must be sorted such that all groups are consecutive - // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final { - ABORT_IF(maxSizeUnused != 0, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size"); - auto mapPath = factoredVocabPath; - auto factorVocabPath = mapPath; - factorVocabPath.back() = 'l'; // map .fm to .fl - auto vocabPath = options_->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs - - // Note: We misuse the Vocab class a little. - // Specifically, it means that the factorVocab_ must contain and "". - Vocab vocab(New(), 0); - vocab.load(vocabPath); - auto vocabSize = vocab.size(); - factorVocab_.load(factorVocabPath); - auto numFactors = factorVocab_.size(); - - // load and parse factorMap - factorMap_.resize(vocabSize); - factorRefCounts_.resize(numFactors); - std::vector tokens; - io::InputFileStream in(mapPath); - std::string line; - size_t numTotalFactors = 0; - for (WordIndex v = 0; io::getline(in, line); v++) { - tokens.clear(); // @BUGBUG: should be done in split() - utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); - for (size_t i = 1; i < tokens.size(); i++) { - auto u = factorVocab_[tokens[i]].toWordIndex(); - factorMap_[v].push_back(u); - factorRefCounts_[u]++; - } - numTotalFactors += tokens.size() - 1; - } - LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); - - // form groups - // @TODO: hard-coded for these initial experiments - std::vector groupPrefixes = { - "@C", - "@GL", "@GR" - }; - groupPrefixes.insert(groupPrefixes.begin(), "(unassigned)"); // first group is fallback for normal words (the string is only used for messages) - size_t numGroups = groupPrefixes.size(); - factorGroups_.resize(numFactors, 0); - for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 - const auto& groupPrefix = groupPrefixes[g]; - for (WordIndex u = 0; u < numFactors; u++) - if (utils::beginsWith(factorVocab_[Word::fromWordIndex(u)], groupPrefix)) { - ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[Word::fromWordIndex(u)], groupPrefix); - factorGroups_[u] = g; - } - } - // determine group index ranges - groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); - std::vector groupCounts(numGroups); // number of group members - for (WordIndex u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts - auto g = factorGroups_[u]; - if (groupRanges_[g].first > u) - groupRanges_[g].first = u; - if (groupRanges_[g].second < u + 1) - groupRanges_[g].second = u + 1; - groupCounts[g]++; - } - // create mappings needed for normalization in factored outputs - factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g - factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) - for (WordIndex v = 0; v < vocabSize; v++) { - for (auto u : factorMap_[v]) { - auto g = factorGroups_[u]; // convert u to relative u within factor group range - ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); - factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); - factorMasks_[g][v] = 1.0f; - } - } - //for (Word v = 0; v < vocabSize; v++) { - // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], - // factorMasks_[0][v], factorIndices_[0][v], - // factorMasks_[1][v], factorIndices_[1][v], - // factorMasks_[2][v], factorIndices_[2][v], - // factorMasks_[3][v], factorIndices_[3][v]); - //} - //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon - for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - LOG(info, "[embedding] Factor group '{}' has {} members ({})", - groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); - if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition - continue; - ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], - "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - //auto& mVec = mVecs_[g]; - //mVec.resize(numFactors, 0.0f); - //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) - // mVec[i] = 1.0f; - } - - // create the global factor matrix, which is used for factored embeddings - std::vector data(vocabSize); - std::iota(data.begin(), data.end(), 0); - globalFactorMatrix_ = csr_rows(data); // [V x U] - - return factorVocabSize(); // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here - } + virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } @@ -155,28 +40,10 @@ class FactoredVocab : public IVocab { virtual Word getUnkId() const override { return unkId_; } virtual void createFake() override final; + // factor-specific. These methods are consumed by Output and Embedding. size_t factorVocabSize() const { return factorVocab_.size(); } - // create a CSR matrix M[V,U] from indices[] with - // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced - CSRData csr_rows(const std::vector& words) const { - std::vector weights; - std::vector indices; - std::vector offsets; - offsets.reserve(words.size() + 1); - indices.reserve(words.size()); // (at least this many) - // loop over all input words, and select the corresponding set of unit indices into CSR format - offsets.push_back((IndexType)indices.size()); - for (auto v : words) { - const auto& m = factorMap_[v]; - for (auto u : m) { - indices.push_back(u); - weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); - } - offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset - } - return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; - } + CSRData csr_rows(const std::vector& words) const; const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v size_t getNumGroups() const { return groupRanges_.size(); } @@ -186,9 +53,10 @@ class FactoredVocab : public IVocab { private: Ptr options_; + // main vocab - Word eosId_ = Word::NONE; - Word unkId_ = Word::NONE; + Word eosId_{}; + Word unkId_{}; // factors Vocab factorVocab_; // [factor name] -> factor index = row of E_ @@ -199,8 +67,6 @@ class FactoredVocab : public IVocab { std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) -//public: // @TODO: temporarily; later factor this properly - //std::vector> mVecs_; // [group id][u] -> 1 if factor is member of group }; } // namespace marian From e770dfa78572c643e0b29e2dfe76baab058a0174 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 14:21:00 -0800 Subject: [PATCH 275/838] FactoredVocab now actually implements all IVocab methods --- src/data/factored_vocab.cpp | 29 ++++++++++------------------- src/data/factored_vocab.h | 4 ++-- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 3bae39237..61e6ae142 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -29,9 +29,8 @@ namespace marian { // Note: We misuse the Vocab class a little. // Specifically, it means that the factorVocab_ must contain and "". - Vocab vocab(New(), 0); - vocab.load(vocabPath); - auto vocabSize = vocab.size(); + vocab_.load(vocabPath); + auto vocabSize = vocab_.size(); factorVocab_.load(factorVocabPath); auto numFactors = factorVocab_.size(); @@ -45,7 +44,7 @@ namespace marian { for (WordIndex v = 0; io::getline(in, line); v++) { tokens.clear(); // @BUGBUG: should be done in split() utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2 || tokens.front() != vocab[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + ABORT_IF(tokens.size() < 2 || tokens.front() != vocab_[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); for (size_t i = 1; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]].toWordIndex(); factorMap_[v].push_back(u); @@ -109,10 +108,6 @@ namespace marian { continue; ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - //auto& mVec = mVecs_[g]; - //mVec.resize(numFactors, 0.0f); - //for (size_t i = groupRanges_[g].first; i < groupRanges_[g].second; i++) - // mVec[i] = 1.0f; } // create the global factor matrix, which is used for factored embeddings @@ -120,35 +115,31 @@ namespace marian { std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] - return factorVocabSize(); // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here + return vocabSize; // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here } /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { - word; - return Word(); + return vocab_[word]; } /*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool inference /*= false*/) const /*override final*/ { - line; addEOS; inference; - return {}; + return vocab_.encode(line, addEOS, inference); } /*virtual*/ std::string FactoredVocab::decode(const Words& sentence, bool ignoreEos /*= true*/) const /*override final*/ { - sentence; ignoreEos; - return {}; + return vocab_.decode(sentence, ignoreEos); } /*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { - id; - static std::string x; - return x; + return vocab_[id]; } /*virtual*/ size_t FactoredVocab::size() const /*override final*/ { - return 0; + return vocab_.size(); } /*virtual*/ void FactoredVocab::createFake() /*override final*/ { + return vocab_.createFake(); } // create a CSR matrix M[V,U] from indices[] with diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index bb4be3600..b2fa81749 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -22,11 +22,10 @@ class FactoredVocab : public IVocab { std::vector offsets; }; - FactoredVocab(Ptr options) : options_(options), factorVocab_(New(), 0) { } + FactoredVocab(Ptr options) : options_(options), vocab_(New(), 0), factorVocab_(New(), 0) { } // from IVocab: virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; - virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } virtual const std::vector& suffixes() const override final { const static std::vector exts{".fm"}; return exts; } @@ -57,6 +56,7 @@ class FactoredVocab : public IVocab { // main vocab Word eosId_{}; Word unkId_{}; + Vocab vocab_; // factors Vocab factorVocab_; // [factor name] -> factor index = row of E_ From 14f2a7904005ddeb5c5f46c4c986c8998ade63da Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 14:44:22 -0800 Subject: [PATCH 276/838] FactoredVocab is now picked up as regular vocab --- src/data/factored_vocab.cpp | 8 +++++--- src/data/vocab.cpp | 18 +++++++++++------- src/data/vocab_base.h | 3 +-- src/layers/generic.cpp | 4 ++-- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 61e6ae142..79e28b3d3 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -21,11 +21,11 @@ namespace marian { // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries /*virtual*/ size_t FactoredVocab::load(const std::string& factoredVocabPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { - ABORT_IF(maxSizeUnused != 0, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size"); auto mapPath = factoredVocabPath; auto factorVocabPath = mapPath; factorVocabPath.back() = 'l'; // map .fm to .fl - auto vocabPath = options_->get("vocab"); // @TODO: This should go away; esp. to allow per-stream vocabs + auto vocabPath = factorVocabPath; + vocabPath[vocabPath.size() - 2] = 'w'; // map .fl to .wl --@TODO: This should go away; esp. to allow per-stream vocabs // Note: We misuse the Vocab class a little. // Specifically, it means that the factorVocab_ must contain and "". @@ -115,10 +115,12 @@ namespace marian { std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] + ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != vocabSize, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); return vocabSize; // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here } /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { + ABORT("operator[] called indeed"); return vocab_[word]; } @@ -163,7 +165,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& wor return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; } -Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { +Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); if(isFactoredVocab) return New(options); diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index d2e4b046e..fbb7eac67 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -10,15 +10,19 @@ Word Word::DEFAULT_UNK_ID = Word(1); // @TODO: make each vocab peek on type Ptr createVocab(const std::string& vocabPath, Ptr options, size_t batchIndex) { + // try SentencePiece auto vocab = createSentencePieceVocab(vocabPath, options, batchIndex); - if(vocab) { + if(vocab) return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it - } else { - // check type of input, if not given, assume "sequence" - auto inputTypes = options->get>("input-types", {}); - std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; - return inputType == "class" ? createClassVocab() : createDefaultVocab(); - } + // try factored + vocab = createFactoredVocab(vocabPath, options); + if (vocab) + return vocab; + // regular vocab + // check type of input, if not given, assume "sequence" + auto inputTypes = options->get>("input-types", {}); + std::string inputType = inputTypes.size() > batchIndex ? inputTypes[batchIndex] : "sequence"; + return inputType == "class" ? createClassVocab() : createDefaultVocab(); } size_t Vocab::loadOrCreate(const std::string& vocabPath, diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index b993233e6..aa43b96f1 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -50,7 +50,6 @@ class Options; Ptr createDefaultVocab(); Ptr createClassVocab(); Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr, size_t batchIndex); -class FactoredVocab; -Ptr createFactoredVocab(const std::string& vocabPath, Ptr); +Ptr createFactoredVocab(const std::string& vocabPath, Ptr); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 57b6d8374..5a79ada38 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -106,7 +106,7 @@ namespace marian { if (options_->has("embedding-factors")) { std::vector paths = options_->get>("embedding-factors"); - factoredVocab_ = createFactoredVocab(paths[0], options_); + factoredVocab_ = std::static_pointer_cast(createFactoredVocab(paths[0], options_)); } if (factoredVocab_) { std::vector paths = options_->get>("embedding-factors"); @@ -173,7 +173,7 @@ namespace marian { if (options_->has("embedding-factors")) { std::vector paths = options_->get>("embedding-factors"); - factoredVocab_ = createFactoredVocab(paths[0], options_); + factoredVocab_ = std::static_pointer_cast(createFactoredVocab(paths[0], options_)); } if (factoredVocab_) { std::vector paths = options_->get>("embedding-factors"); From 3a8d8ae71ce93228ea60b159431d82c7b1d30850 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 14:49:10 -0800 Subject: [PATCH 277/838] bug fix: factored vocab should forward base vocab's EOS and UNK --- src/data/factored_vocab.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 79e28b3d3..3f13c8579 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -100,7 +100,6 @@ namespace marian { // factorMasks_[2][v], factorIndices_[2][v], // factorMasks_[3][v], factorIndices_[3][v]); //} - //mVecs_.resize(numGroups); // @TODO: no longer needed, delete soon for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members ({})", groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); @@ -114,9 +113,12 @@ namespace marian { std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] + + eosId_ = vocab_.getEosId(); + unkId_ = vocab_.getUnkId(); - ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != vocabSize, "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); - return vocabSize; // @TODO: return the actual virtual unrolled vocab size, which eventually we will know here + ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); + return size(); } /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { From a0e6411a824731f40966e3c60c6c6551ec3a2cd7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 15:10:03 -0800 Subject: [PATCH 278/838] removed option --embedding-factors, instead just specify a .fm file as vocab --- src/common/config_parser.cpp | 7 ------- src/data/factored_vocab.cpp | 17 +++++++++++++++-- src/data/factored_vocab.h | 5 ++--- src/data/vocab.cpp | 2 +- src/data/vocab_base.h | 2 +- src/layers/generic.cpp | 14 ++------------ src/models/decoder.h | 5 +---- src/models/transformer.h | 17 ++--------------- 8 files changed, 24 insertions(+), 45 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 422432e9d..252810531 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -30,7 +30,6 @@ const std::set PATHS = { "train-sets", "vocabs", "embedding-vectors", - "embedding-factors", "valid-sets", "valid-script-path", "valid-log", @@ -400,8 +399,6 @@ void ConfigParser::addOptionsTraining(cli::CLIWrapper& cli) { "Fix source embeddings. Affects all encoders"); cli.add("--embedding-fix-trg", "Fix target embeddings. Affects all decoders"); - cli.add_nondefault>("--embedding-factors", - "Paths to (factor map, factor list) file for factored embeddings"); cli.add("--multi-node", "Enable asynchronous multi-node training through MPI (and legacy sync if combined with --sync-sgd)"); @@ -483,8 +480,6 @@ void ConfigParser::addOptionsTranslation(cli::CLIWrapper& cli) { "stdout"); cli.add>("--vocabs,-v", "Paths to vocabulary files have to correspond to --input"); - cli.add_nondefault>("--embedding-factors", - "Paths to (factor map, factor list) file for factored embeddings"); // decoding options cli.add("--beam-size,-b", "Beam size used during search with validating translator", @@ -547,8 +542,6 @@ void ConfigParser::addOptionsScoring(cli::CLIWrapper& cli) { "Paths to vocabulary files have to correspond to --train-sets. " "If this parameter is not supplied we look for vocabulary files source.{yml,json} and target.{yml,json}. " "If these files do not exists they are created"); - cli.add_nondefault>("--embedding-factors", - "Paths to (factor map, factor list) file for factored embeddings"); cli.add("--n-best", "Score n-best list instead of plain text corpus"); cli.add("--n-best-feature", diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 3f13c8579..2c1591cbe 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -167,10 +167,23 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& wor return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; } -Ptr createFactoredVocab(const std::string& vocabPath, Ptr options) { +// Helper to construct and load a FactordVocab from a path is given (non-empty) and if it specifies a factored vocab. +// This is used by the Embedding and Output layers. +/*static*/ Ptr FactoredVocab::tryCreateAndLoad(const std::string& path) { + Ptr res; + if (!path.empty()) { + res = std::static_pointer_cast(createFactoredVocab(path)); // this checks the file extension + if (res) + res->load(path); // or throw + } + return res; +} + +// Note: This does not actually load it, only checks the path for the type. +Ptr createFactoredVocab(const std::string& vocabPath) { bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); if(isFactoredVocab) - return New(options); + return New(); else return nullptr; } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index b2fa81749..848bf6001 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -22,7 +22,7 @@ class FactoredVocab : public IVocab { std::vector offsets; }; - FactoredVocab(Ptr options) : options_(options), vocab_(New(), 0), factorVocab_(New(), 0) { } + FactoredVocab() : vocab_(New(), 0), factorVocab_(New(), 0) { } // from IVocab: virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; @@ -50,9 +50,8 @@ class FactoredVocab : public IVocab { const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor + static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: - Ptr options_; - // main vocab Word eosId_{}; Word unkId_{}; diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index fbb7eac67..63947647d 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -15,7 +15,7 @@ Ptr createVocab(const std::string& vocabPath, Ptr options, size if(vocab) return vocab; // this is defined which means that a sentencepiece vocabulary could be created, so return it // try factored - vocab = createFactoredVocab(vocabPath, options); + vocab = createFactoredVocab(vocabPath); if (vocab) return vocab; // regular vocab diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index aa43b96f1..bca9db957 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -50,6 +50,6 @@ class Options; Ptr createDefaultVocab(); Ptr createClassVocab(); Ptr createSentencePieceVocab(const std::string& vocabPath, Ptr, size_t batchIndex); -Ptr createFactoredVocab(const std::string& vocabPath, Ptr); +Ptr createFactoredVocab(const std::string& vocabPath); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5a79ada38..41e0071e0 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -104,13 +104,8 @@ namespace marian { auto name = options_->get("prefix"); auto numOutputClasses = options_->get("dim"); - if (options_->has("embedding-factors")) { - std::vector paths = options_->get>("embedding-factors"); - factoredVocab_ = std::static_pointer_cast(createFactoredVocab(paths[0], options_)); - } + factoredVocab_ = FactoredVocab::tryCreateAndLoad(options_->get("vocab", "")); if (factoredVocab_) { - std::vector paths = options_->get>("embedding-factors"); - factoredVocab_->load(paths[0]); ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); numOutputClasses = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored outputs enabled"); @@ -171,13 +166,8 @@ namespace marian { bool fixed = opt("fixed", false); - if (options_->has("embedding-factors")) { - std::vector paths = options_->get>("embedding-factors"); - factoredVocab_ = std::static_pointer_cast(createFactoredVocab(paths[0], options_)); - } + factoredVocab_ = FactoredVocab::tryCreateAndLoad(options_->get("vocab", "")); if (factoredVocab_) { - std::vector paths = options_->get>("embedding-factors"); - factoredVocab_->load(paths[0]); dimVoc = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored embeddings enabled"); } diff --git a/src/models/decoder.h b/src/models/decoder.h index 64058344c..8f3db3d72 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -54,10 +54,7 @@ class DecoderBase { embFactory("embFile", embFiles[batchIndex_]) // ("normalization", opt("embedding-normalization")); } - if (options_->has("embedding-factors")) { - embFactory("embedding-factors", opt>("embedding-factors")); - embFactory("vocab", opt>("vocabs")[batchIndex_]); - } + embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings embedding_[batchIndex_] = embFactory.construct(graph); } } diff --git a/src/models/transformer.h b/src/models/transformer.h index a6275f258..48b22e113 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -537,10 +537,7 @@ class EncoderTransformer : public Transformer { embFactory("embFile", embFiles[subBatchIndex]) ("normalization", opt("embedding-normalization")); } - if (options_->has("embedding-factors")) { - embFactory("embedding-factors", opt>("embedding-factors")); - embFactory("vocab", opt>("vocabs")[subBatchIndex]); - } + embFactory("vocab", opt>("vocabs")[subBatchIndex]); // for factored embeddings return embFactory.construct(graph_); } @@ -655,17 +652,7 @@ class DecoderTransformer : public Transformer { outputFactory.tieTransposed(tiedPrefix); } - if (options_->has("embedding-factors")) { - // factored embeddings, simplistic version (which just adds the logits, like multiplying probs) - // z = h @ W // h:[B x D] ; W:[D x V] -> [B x V] - // with factors: - // z = h @ W @ M' // h:[B x D] ; W:[D x U] ; M':[U x V] -> [B x V] - // i.e. multiOutput(): - // output = dot_csr(output, M, transB=true) - // @BUGBUG: need to specify output factors separately if not tied-embeddings or tied-embeddings-all - outputFactory("embedding-factors", opt>("embedding-factors")); - outputFactory("vocab", opt>("vocabs")[batchIndex_]); - } + outputFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored outputs output_ = std::dynamic_pointer_cast(outputFactory.construct(graph_)); // (construct() returns only the underlying interface) } From 0380a2b3ef89a28894add3b3e9a1b06f30f24558 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 15:53:04 -0800 Subject: [PATCH 279/838] FactoredVocab no longer uses Vocab for the factors --- src/data/factored_vocab.cpp | 15 ++++++++++----- src/data/factored_vocab.h | 30 ++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 2c1591cbe..7adee5132 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -31,7 +31,13 @@ namespace marian { // Specifically, it means that the factorVocab_ must contain and "". vocab_.load(vocabPath); auto vocabSize = vocab_.size(); - factorVocab_.load(factorVocabPath); + + std::string line; + + // load factor vocabulary + io::InputFileStream fin(factorVocabPath); + for (WordIndex v = 0; io::getline(fin, line); v++) + factorVocab_.add(line); auto numFactors = factorVocab_.size(); // load and parse factorMap @@ -39,14 +45,13 @@ namespace marian { factorRefCounts_.resize(numFactors); std::vector tokens; io::InputFileStream in(mapPath); - std::string line; size_t numTotalFactors = 0; for (WordIndex v = 0; io::getline(in, line); v++) { tokens.clear(); // @BUGBUG: should be done in split() utils::splitAny(line, tokens, " \t"); ABORT_IF(tokens.size() < 2 || tokens.front() != vocab_[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); for (size_t i = 1; i < tokens.size(); i++) { - auto u = factorVocab_[tokens[i]].toWordIndex(); + auto u = factorVocab_[tokens[i]]; factorMap_[v].push_back(u); factorRefCounts_[u]++; } @@ -66,8 +71,8 @@ namespace marian { for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 const auto& groupPrefix = groupPrefixes[g]; for (WordIndex u = 0; u < numFactors; u++) - if (utils::beginsWith(factorVocab_[Word::fromWordIndex(u)], groupPrefix)) { - ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[Word::fromWordIndex(u)], groupPrefix); + if (utils::beginsWith(factorVocab_[u], groupPrefix)) { + ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[u], groupPrefix); factorGroups_[u] = g; } } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 848bf6001..d9292e429 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -22,7 +22,7 @@ class FactoredVocab : public IVocab { std::vector offsets; }; - FactoredVocab() : vocab_(New(), 0), factorVocab_(New(), 0) { } + FactoredVocab() : vocab_(New(), 0) { } // from IVocab: virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; @@ -52,13 +52,39 @@ class FactoredVocab : public IVocab { static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: + class WordLUT { // map between strings and WordIndex + std::map str2index_; + std::vector index2str_; + public: + void add(const std::string& word) { + auto index = (WordIndex)index2str_.size(); + auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; + ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); + index2str_.push_back(word); + } + const std::string& operator[](WordIndex index) const { return index2str_[index]; } + WordIndex operator[](const std::string& word) const { + auto iter = str2index_.find(word); + ABORT_IF(iter == str2index_.end(), "Token '{}' not found in vocabulary", word); + return iter->second; + } + bool tryFind(const std::string& word, WordIndex& index) const { + auto iter = str2index_.find(word); + if (iter == str2index_.end()) + return false; + index = iter->second; + return true; + } + size_t size() const { return index2str_.size(); } + }; + // main vocab Word eosId_{}; Word unkId_{}; Vocab vocab_; // factors - Vocab factorVocab_; // [factor name] -> factor index = row of E_ + WordLUT factorVocab_; // [factor name] -> factor index = row of E_ std::vector> factorMap_; // [word index v] -> set of factor indices u std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v From 1143ab97dfa1f3bd2032f6f0c7643632aa638756 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 16:15:16 -0800 Subject: [PATCH 280/838] FactoredVocab no longer uses Vocab objects under the hood --- src/data/default_vocab.cpp | 1 - src/data/factored_vocab.cpp | 58 +++++++++++++++++++++++-------------- src/data/factored_vocab.h | 19 +++++++----- 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index bb91fab36..be653fb8f 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -62,7 +62,6 @@ class DefaultVocab : public IVocab { } std::string decode(const Words& sentence, bool ignoreEOS) const override { - std::string line; auto tokens = (*this)(sentence, ignoreEOS); return utils::join(tokens, " "); } diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 7adee5132..cc7c7febd 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -27,18 +27,13 @@ namespace marian { auto vocabPath = factorVocabPath; vocabPath[vocabPath.size() - 2] = 'w'; // map .fl to .wl --@TODO: This should go away; esp. to allow per-stream vocabs - // Note: We misuse the Vocab class a little. - // Specifically, it means that the factorVocab_ must contain and "". - vocab_.load(vocabPath); - auto vocabSize = vocab_.size(); - std::string line; + // load main vocabulary --@TODO: This will go away soon. + auto vocabSize = vocab_.load(vocabPath); + // load factor vocabulary - io::InputFileStream fin(factorVocabPath); - for (WordIndex v = 0; io::getline(fin, line); v++) - factorVocab_.add(line); - auto numFactors = factorVocab_.size(); + auto numFactors = factorVocab_.load(factorVocabPath); // load and parse factorMap factorMap_.resize(vocabSize); @@ -49,7 +44,7 @@ namespace marian { for (WordIndex v = 0; io::getline(in, line); v++) { tokens.clear(); // @BUGBUG: should be done in split() utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2 || tokens.front() != vocab_[Word::fromWordIndex(v)], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + ABORT_IF(tokens.size() < 2 || tokens.front() != vocab_[v], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); for (size_t i = 1; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; factorMap_[v].push_back(u); @@ -118,37 +113,58 @@ namespace marian { std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] - - eosId_ = vocab_.getEosId(); - unkId_ = vocab_.getUnkId(); + + // and must exist in the vocabulary + eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); + unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); return size(); } /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { - ABORT("operator[] called indeed"); - return vocab_[word]; + WordIndex index; + bool found = vocab_.tryFind(word, index); + if (found) + return Word::fromWordIndex(index); + else + return getUnkId(); } -/*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool inference /*= false*/) const /*override final*/ { - return vocab_.encode(line, addEOS, inference); +/*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool /*inference*/ /*= false*/) const /*override final*/ { + std::vector lineTokens; + utils::split(line, lineTokens, " "); + Words res; res.reserve(lineTokens.size() + addEOS); + for (const auto& tok : lineTokens) + res.push_back((*this)[tok]); + if (addEOS) + res.push_back(getEosId()); + return res; } -/*virtual*/ std::string FactoredVocab::decode(const Words& sentence, bool ignoreEos /*= true*/) const /*override final*/ { - return vocab_.decode(sentence, ignoreEos); +/*virtual*/ std::string FactoredVocab::decode(const Words& sentence, bool ignoreEOS /*= true*/) const /*override final*/ { + std::vector decoded; + decoded.reserve(sentence.size()); + for(auto w : sentence) { + if((w != getEosId() || !ignoreEOS)) + decoded.push_back((*this)[w]); + } + return utils::join(decoded, " "); } /*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { - return vocab_[id]; + return vocab_[id.toWordIndex()]; } /*virtual*/ size_t FactoredVocab::size() const /*override final*/ { return vocab_.size(); } +// This creates a fake vocabulary fro use in fakeBatch(). +// @TODO: This may become more complex. /*virtual*/ void FactoredVocab::createFake() /*override final*/ { - return vocab_.createFake(); + eosId_ = Word::fromWordIndex(vocab_.add(DEFAULT_EOS_STR)); + unkId_ = Word::fromWordIndex(vocab_.add(DEFAULT_UNK_STR)); } // create a CSR matrix M[V,U] from indices[] with diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index d9292e429..09ac98c78 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -6,7 +6,6 @@ #include "common/definitions.h" #include "data/types.h" -#include "data/vocab.h" #include "data/vocab_base.h" #include // for std::iota() @@ -22,8 +21,6 @@ class FactoredVocab : public IVocab { std::vector offsets; }; - FactoredVocab() : vocab_(New(), 0) { } - // from IVocab: virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } @@ -35,8 +32,8 @@ class FactoredVocab : public IVocab { virtual const std::string& operator[](Word id) const override final; virtual size_t size() const override final; virtual std::string type() const override final { return "FactoredVocab"; } - virtual Word getEosId() const override { return eosId_; } - virtual Word getUnkId() const override { return unkId_; } + virtual Word getEosId() const override final { return eosId_; } + virtual Word getUnkId() const override final { return unkId_; } virtual void createFake() override final; // factor-specific. These methods are consumed by Output and Embedding. @@ -56,11 +53,12 @@ class FactoredVocab : public IVocab { std::map str2index_; std::vector index2str_; public: - void add(const std::string& word) { + WordIndex add(const std::string& word) { auto index = (WordIndex)index2str_.size(); auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); index2str_.push_back(word); + return index; } const std::string& operator[](WordIndex index) const { return index2str_[index]; } WordIndex operator[](const std::string& word) const { @@ -76,12 +74,19 @@ class FactoredVocab : public IVocab { return true; } size_t size() const { return index2str_.size(); } + size_t load(const std::string& path) { + std::string line; + io::InputFileStream in(path); + for (WordIndex v = 0; io::getline(in, line); v++) + add(line); + return size(); + } }; // main vocab Word eosId_{}; Word unkId_{}; - Vocab vocab_; + WordLUT vocab_; // factors WordLUT factorVocab_; // [factor name] -> factor index = row of E_ From 91b1403c155ea9698d5d128ce4d9746529946ae8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 16:22:32 -0800 Subject: [PATCH 281/838] FactorVocab no longer requires the .wl file, as it can know everything from the .fm file --- src/common/utils.cpp | 6 ++++-- src/data/factored_vocab.cpp | 22 +++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 59cc4cc88..179cd4d4d 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -30,9 +30,10 @@ void trimLeft(std::string& s) { // @TODO: use more functions from CLI instead of own implementations void split(const std::string& line, - std::vector& pieces, + /*out*/ std::vector& pieces, const std::string& del /*= " "*/, bool keepEmpty) { + pieces.clear(); size_t begin = 0; size_t pos = 0; std::string token; @@ -61,9 +62,10 @@ std::vector split(const std::string& line, // @TODO: splitAny() shares all but 2 expressions with split(). Merge them. void splitAny(const std::string& line, - std::vector& pieces, + /*out*/ std::vector& pieces, const std::string& del /*= " "*/, bool keepEmpty) { + pieces.clear(); size_t begin = 0; size_t pos = 0; std::string token; diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index cc7c7febd..f48cfe6f8 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -24,34 +24,30 @@ namespace marian { auto mapPath = factoredVocabPath; auto factorVocabPath = mapPath; factorVocabPath.back() = 'l'; // map .fm to .fl - auto vocabPath = factorVocabPath; - vocabPath[vocabPath.size() - 2] = 'w'; // map .fl to .wl --@TODO: This should go away; esp. to allow per-stream vocabs - - std::string line; - - // load main vocabulary --@TODO: This will go away soon. - auto vocabSize = vocab_.load(vocabPath); // load factor vocabulary auto numFactors = factorVocab_.load(factorVocabPath); + factorRefCounts_.resize(numFactors); // load and parse factorMap - factorMap_.resize(vocabSize); - factorRefCounts_.resize(numFactors); std::vector tokens; - io::InputFileStream in(mapPath); + std::string line; size_t numTotalFactors = 0; + io::InputFileStream in(mapPath); for (WordIndex v = 0; io::getline(in, line); v++) { - tokens.clear(); // @BUGBUG: should be done in split() utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2 || tokens.front() != vocab_[v], "Factor map must list words in same order as vocab, and have at least one factor per word", mapPath); + vocab_.add(tokens.front()); + ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); + std::vector factors; for (size_t i = 1; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; - factorMap_[v].push_back(u); + factors.push_back(u); factorRefCounts_[u]++; } + factorMap_.emplace_back(std::move(factors)); numTotalFactors += tokens.size() - 1; } + auto vocabSize = vocab_.size(); LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); // form groups From 67dd472b0abbe80c56a1e1a107236b3590fc4470 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 21:40:51 -0800 Subject: [PATCH 282/838] some refactoring of FactoredVocab --- src/data/factored_vocab.cpp | 107 ++++++++++++++++++++---------------- src/data/factored_vocab.h | 32 ++++++++--- src/layers/generic.cpp | 1 - 3 files changed, 86 insertions(+), 54 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index f48cfe6f8..d31339a11 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -26,42 +26,60 @@ namespace marian { factorVocabPath.back() = 'l'; // map .fm to .fl // load factor vocabulary - auto numFactors = factorVocab_.load(factorVocabPath); - factorRefCounts_.resize(numFactors); + factorVocab_.load(factorVocabPath); + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR" }; // @TODO: hard-coded for these initial experiments + + // construct mapping tables for factors + constructGroupInfoFromFactorVocab(); // load and parse factorMap + auto factorVocabSize = factorVocab_.size(); + factorRefCounts_.resize(factorVocabSize); std::vector tokens; std::string line; size_t numTotalFactors = 0; io::InputFileStream in(mapPath); for (WordIndex v = 0; io::getline(in, line); v++) { + // parse the line, of the form WORD FACTOR1 FACTOR2 FACTOR1 ... + // where FACTOR1 is the lemma, a factor that all words have. + // Not every word has all other factors, so the n-th item is not always the same factor. utils::splitAny(line, tokens, " \t"); - vocab_.add(tokens.front()); ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); std::vector factors; - for (size_t i = 1; i < tokens.size(); i++) { + for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; factors.push_back(u); factorRefCounts_[u]++; } factorMap_.emplace_back(std::move(factors)); + // add to vocab + WordIndex index = v; + // @TODO: map factors to non-dense integer + vocab_.add(tokens.front(), index); numTotalFactors += tokens.size() - 1; } - auto vocabSize = vocab_.size(); - LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} words", numTotalFactors, numFactors, vocabSize); + LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", + numTotalFactors, factorVocabSize, vocab_.numValid(), vocab_.size()); + + // create mappings needed for normalization in factored outputs + constructNormalizationInfoForVocab(); + + // and must exist in the vocabulary + eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); + unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); + + ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); + return size(); +} +void FactoredVocab::constructGroupInfoFromFactorVocab() { // form groups - // @TODO: hard-coded for these initial experiments - std::vector groupPrefixes = { - "@C", - "@GL", "@GR" - }; - groupPrefixes.insert(groupPrefixes.begin(), "(unassigned)"); // first group is fallback for normal words (the string is only used for messages) - size_t numGroups = groupPrefixes.size(); - factorGroups_.resize(numFactors, 0); - for (size_t g = 1; g < groupPrefixes.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 - const auto& groupPrefix = groupPrefixes[g]; - for (WordIndex u = 0; u < numFactors; u++) + size_t numGroups = groupPrefixes_.size(); + size_t factorVocabSize = factorVocab_.size(); + factorGroups_.resize(factorVocabSize, 0); + for (size_t g = 1; g < groupPrefixes_.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 + const auto& groupPrefix = groupPrefixes_[g]; + for (WordIndex u = 0; u < factorVocabSize; u++) if (utils::beginsWith(factorVocab_[u], groupPrefix)) { ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[u], groupPrefix); factorGroups_[u] = g; @@ -70,7 +88,7 @@ namespace marian { // determine group index ranges groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); std::vector groupCounts(numGroups); // number of group members - for (WordIndex u = 0; u < numFactors; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts + for (WordIndex u = 0; u < factorVocabSize; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts auto g = factorGroups_[u]; if (groupRanges_[g].first > u) groupRanges_[g].first = u; @@ -78,7 +96,19 @@ namespace marian { groupRanges_[g].second = u + 1; groupCounts[g]++; } + for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups + LOG(info, "[embedding] Factor group '{}' has {} members", groupPrefixes_[g], groupCounts[g]); + if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition + continue; + ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], + "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes_[g]); + } +} + +void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs + size_t numGroups = groupPrefixes_.size(); + size_t vocabSize = vocab_.size(); factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) for (WordIndex v = 0; v < vocabSize; v++) { @@ -96,26 +126,11 @@ namespace marian { // factorMasks_[2][v], factorIndices_[2][v], // factorMasks_[3][v], factorIndices_[3][v]); //} - for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - LOG(info, "[embedding] Factor group '{}' has {} members ({})", - groupPrefixes[g], groupCounts[g], groupCounts[g] == 1 ? "sigmoid" : "softmax"); - if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition - continue; - ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], - "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes[g]); - } - // create the global factor matrix, which is used for factored embeddings + // create the global factor matrix, which is used for getLogits() std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] - - // and must exist in the vocabulary - eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); - unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); - - ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); - return size(); } /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { @@ -127,6 +142,14 @@ namespace marian { return getUnkId(); } +/*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { + return vocab_[id.toWordIndex()]; +} + +/*virtual*/ size_t FactoredVocab::size() const /*override final*/ { + return vocab_.size(); +} + /*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool /*inference*/ /*= false*/) const /*override final*/ { std::vector lineTokens; utils::split(line, lineTokens, " "); @@ -148,19 +171,11 @@ namespace marian { return utils::join(decoded, " "); } -/*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { - return vocab_[id.toWordIndex()]; -} - -/*virtual*/ size_t FactoredVocab::size() const /*override final*/ { - return vocab_.size(); -} - // This creates a fake vocabulary fro use in fakeBatch(). // @TODO: This may become more complex. /*virtual*/ void FactoredVocab::createFake() /*override final*/ { - eosId_ = Word::fromWordIndex(vocab_.add(DEFAULT_EOS_STR)); - unkId_ = Word::fromWordIndex(vocab_.add(DEFAULT_UNK_STR)); + eosId_ = Word::fromWordIndex(vocab_.add(DEFAULT_EOS_STR, 0)); + unkId_ = Word::fromWordIndex(vocab_.add(DEFAULT_UNK_STR, 1)); } // create a CSR matrix M[V,U] from indices[] with @@ -173,8 +188,8 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& wor indices.reserve(words.size()); // (at least this many) // loop over all input words, and select the corresponding set of unit indices into CSR format offsets.push_back((IndexType)indices.size()); - for (auto v : words) { - const auto& m = factorMap_[v]; + for (auto w : words) { + const auto& m = factorMap_[w]; for (auto u : m) { indices.push_back(u); weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 09ac98c78..c941093be 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -41,31 +41,43 @@ class FactoredVocab : public IVocab { CSRData csr_rows(const std::vector& words) const; - const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v + const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v --only used in getLogits() size_t getNumGroups() const { return groupRanges_.size(); } std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab +private: + void constructGroupInfoFromFactorVocab(); + void constructNormalizationInfoForVocab(); private: class WordLUT { // map between strings and WordIndex std::map str2index_; std::vector index2str_; public: - WordIndex add(const std::string& word) { - auto index = (WordIndex)index2str_.size(); + WordIndex add(const std::string& word, WordIndex index) { + ABORT_IF(word.empty(), "Attempted to add the empty word to a dictionary"); auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); - index2str_.push_back(word); + while (index2str_.size() <= index) + index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? + if (!index2str_[index].empty()) + ABORT_IF(!wasInserted, "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); + index2str_[index] = word; return index; } - const std::string& operator[](WordIndex index) const { return index2str_[index]; } + const std::string& operator[](WordIndex index) const { + const auto& word = index2str_[index]; + ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); + return word; + } WordIndex operator[](const std::string& word) const { auto iter = str2index_.find(word); ABORT_IF(iter == str2index_.end(), "Token '{}' not found in vocabulary", word); return iter->second; } + bool isGap(WordIndex index) const { return index2str_[index].empty(); } bool tryFind(const std::string& word, WordIndex& index) const { auto iter = str2index_.find(word); if (iter == str2index_.end()) @@ -73,12 +85,17 @@ class FactoredVocab : public IVocab { index = iter->second; return true; } - size_t size() const { return index2str_.size(); } + void resize(size_t num) { + ABORT_IF(num < index2str_.size(), "Word table cannot be shrunk"); + index2str_.resize(num); // gets filled up with gap items (empty strings) + } + size_t size() const { return index2str_.size(); } // nominal size including gap items + size_t numValid() const { return str2index_.size(); } // actual non-gaps items size_t load(const std::string& path) { std::string line; io::InputFileStream in(path); for (WordIndex v = 0; io::getline(in, line); v++) - add(line); + add(line, v); return size(); } }; @@ -90,6 +107,7 @@ class FactoredVocab : public IVocab { // factors WordLUT factorVocab_; // [factor name] -> factor index = row of E_ + std::vector groupPrefixes_; // [group id g] shared prefix of factors (used for grouping) std::vector> factorMap_; // [word index v] -> set of factor indices u std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 41e0071e0..815b43ae1 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -53,7 +53,6 @@ namespace marian { // This function assumes this object holds a single factor that represents a rational loss (with count). Ptr Logits::getRationalLoss() const { - //return New(getLogits(), logits_.front()->count()); ABORT_IF(logits_.size() != 1 || factoredVocab_, "getRationalLoss() cannot be used on multi-factor outputs"); ABORT_IF(!logits_.front()->count(), "getRationalLoss() used on rational loss without count"); return logits_.front(); From 0eb5de7719e9716bf379d51d62c4c83607b61be8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 22:15:28 -0800 Subject: [PATCH 283/838] FactoredVocab with gap items are now handled correctly in getLogits() --- src/data/factored_vocab.cpp | 20 +++++++++++++++----- src/data/factored_vocab.h | 3 +++ src/layers/generic.cpp | 4 ++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index d31339a11..21a0eefa8 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -33,6 +33,9 @@ namespace marian { constructGroupInfoFromFactorVocab(); // load and parse factorMap + auto elements = factorShape_.elements(); + vocab_.resize(elements); + factorMap_.resize(elements); auto factorVocabSize = factorVocab_.size(); factorRefCounts_.resize(factorVocabSize); std::vector tokens; @@ -51,15 +54,15 @@ namespace marian { factors.push_back(u); factorRefCounts_[u]++; } - factorMap_.emplace_back(std::move(factors)); - // add to vocab WordIndex index = v; // @TODO: map factors to non-dense integer + factorMap_[index] = std::move(factors); + // add to vocab vocab_.add(tokens.front(), index); numTotalFactors += tokens.size() - 1; } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", - numTotalFactors, factorVocabSize, vocab_.numValid(), vocab_.size()); + numTotalFactors, factorVocabSize, vocab_.numValid(), size()); // create mappings needed for normalization in factored outputs constructNormalizationInfoForVocab(); @@ -68,7 +71,11 @@ namespace marian { eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); - ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (to {})", maxSizeUnused); +#if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() + if (maxSizeUnused == vocab_.numValid()) + maxSizeUnused = vocab_.size(); +#endif + ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); return size(); } @@ -87,7 +94,7 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { } // determine group index ranges groupRanges_.resize(numGroups, { SIZE_MAX, (size_t)0 }); - std::vector groupCounts(numGroups); // number of group members + std::vector groupCounts(numGroups); // number of group members for (WordIndex u = 0; u < factorVocabSize; u++) { // determine ranges; these must be non-overlapping, verified via groupCounts auto g = factorGroups_[u]; if (groupRanges_[g].first > u) @@ -103,6 +110,7 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes_[g]); } + factorShape_ = Shape(std::move(groupCounts)); } void FactoredVocab::constructNormalizationInfoForVocab() { @@ -111,12 +119,14 @@ void FactoredVocab::constructNormalizationInfoForVocab() { size_t vocabSize = vocab_.size(); factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) + gapLogMask_.resize(vocabSize, -1e8f); for (WordIndex v = 0; v < vocabSize; v++) { for (auto u : factorMap_[v]) { auto g = factorGroups_[u]; // convert u to relative u within factor group range ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); factorMasks_[g][v] = 1.0f; + gapLogMask_[v] = 0.0f; // valid entry } } //for (Word v = 0; v < vocabSize; v++) { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index c941093be..25e9bd733 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -46,6 +46,7 @@ class FactoredVocab : public IVocab { std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor + const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: @@ -113,8 +114,10 @@ class FactoredVocab : public IVocab { CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. + Shape factorShape_; // [g] number of factors in each factor group std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) + std::vector gapLogMask_; // [v] -1e8 if this is a gap, else 0 }; } // namespace marian diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 815b43ae1..c2aab49a9 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -84,6 +84,10 @@ namespace marian { graph->constant({(int)factorMatrix.offsets.size()}, inits::from_vector(factorMatrix.offsets), Type::uint32), /*transB=*/ true); // -> [B x V] + // mask out gaps + auto gapLogMask = factoredVocab_->getGapLogMask(); // [V] + y = y + graph->constant({ (int)gapLogMask.size() }, inits::from_vector(gapLogMask), Type::float32); + return y; } From 250de94e0bdc115a28973a67177754d97a78c76f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 22:35:34 -0800 Subject: [PATCH 284/838] FactoredVocab now uses the non-dense numeric id to represent the factors directly in the id --- src/data/factored_vocab.cpp | 15 +++++++++++++-- src/data/factored_vocab.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 21a0eefa8..6973a56ff 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -54,11 +54,17 @@ namespace marian { factors.push_back(u); factorRefCounts_[u]++; } - WordIndex index = v; + size_t index = 0; + for (auto u : factors) { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + auto factorIndex = u - groupRanges_[g].first; + index += factorIndex * factorStrides_[g]; + } // @TODO: map factors to non-dense integer factorMap_[index] = std::move(factors); // add to vocab - vocab_.add(tokens.front(), index); + vocab_.add(tokens.front(), (WordIndex)index); numTotalFactors += tokens.size() - 1; } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", @@ -111,6 +117,11 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes_[g]); } factorShape_ = Shape(std::move(groupCounts)); + factorStrides_.resize(factorShape_.size(), 1); + for (size_t g = factorStrides_.size() - 1; g --> 0; ) + factorStrides_[g] = factorStrides_[g + 1] * (size_t)factorShape_[g + 1]; + for (auto str : factorStrides_) + LOG(info, "stride {}", str); } void FactoredVocab::constructNormalizationInfoForVocab() { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 25e9bd733..dd35b4993 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -115,6 +115,7 @@ class FactoredVocab : public IVocab { std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. Shape factorShape_; // [g] number of factors in each factor group + std::vector factorStrides_; // [g] stride for factor dimension std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) std::vector gapLogMask_; // [v] -1e8 if this is a gap, else 0 From 7f66068bd51fe9f27bfbb9cb63ba07c4b0e68c26 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 22:45:23 -0800 Subject: [PATCH 285/838] fixed an internal check --- src/data/factored_vocab.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index dd35b4993..c7086825e 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -63,8 +63,7 @@ class FactoredVocab : public IVocab { ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); while (index2str_.size() <= index) index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? - if (!index2str_[index].empty()) - ABORT_IF(!wasInserted, "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); + ABORT_IF(!index2str_[index].empty(), "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); index2str_[index] = word; return index; } From 4744df010a5e3024f76dcf27ecacb5cb78d7ea2b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 22:54:25 -0800 Subject: [PATCH 286/838] removed FactoredVocab::createFake(), as it is not used --- src/data/factored_vocab.cpp | 9 --------- src/data/factored_vocab.h | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 6973a56ff..5afb5b989 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -120,8 +120,6 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { factorStrides_.resize(factorShape_.size(), 1); for (size_t g = factorStrides_.size() - 1; g --> 0; ) factorStrides_[g] = factorStrides_[g + 1] * (size_t)factorShape_[g + 1]; - for (auto str : factorStrides_) - LOG(info, "stride {}", str); } void FactoredVocab::constructNormalizationInfoForVocab() { @@ -192,13 +190,6 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return utils::join(decoded, " "); } -// This creates a fake vocabulary fro use in fakeBatch(). -// @TODO: This may become more complex. -/*virtual*/ void FactoredVocab::createFake() /*override final*/ { - eosId_ = Word::fromWordIndex(vocab_.add(DEFAULT_EOS_STR, 0)); - unkId_ = Word::fromWordIndex(vocab_.add(DEFAULT_UNK_STR, 1)); -} - // create a CSR matrix M[V,U] from indices[] with // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& words) const { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index c7086825e..8d41d363a 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -34,7 +34,7 @@ class FactoredVocab : public IVocab { virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } - virtual void createFake() override final; + virtual void createFake() override final { ABORT("[data] Fake FactoredVocab vocabulary not supported"); } // factor-specific. These methods are consumed by Output and Embedding. size_t factorVocabSize() const { return factorVocab_.size(); } From d263f743ab2bb81e6634848ebb80bd5e2dd495a5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 5 Feb 2019 23:03:51 -0800 Subject: [PATCH 287/838] bug fix (merge error): LabelwiseLoss should be passed the correct Logits object --- src/models/costs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/costs.h b/src/models/costs.h index ebef2d4e6..f3fc1923e 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -68,7 +68,7 @@ class EncoderDecoderCE : public CostBase { Ptr multiLoss = newMultiLoss(options_); // @TODO: adapt to multi-objective training with multiple decoders - auto partialLoss = loss_->apply(state->getLogProbs().getLogits(), + auto partialLoss = loss_->apply(state->getLogProbs(), state->getTargetWords(), state->getTargetMask(), weights); From 86c412c81d5b2947d29ad9620c6828cb15da9ede Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 08:17:37 -0800 Subject: [PATCH 288/838] ScorerWrapper::getLogProbs() now also returns Logits, in prep for factored decoding --- src/data/factored_vocab.cpp | 2 +- src/graph/expression_operators.cpp | 32 +++++++++++++++++++++++------- src/layers/generic.h | 1 + src/training/validator.h | 2 +- src/translator/beam_search.h | 5 +---- src/translator/scorers.h | 6 +++--- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 5afb5b989..622e19392 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -146,7 +146,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { // factorMasks_[3][v], factorIndices_[3][v]); //} - // create the global factor matrix, which is used for getLogits() + // create the global factor matrix, which is used for getLogits() only std::vector data(vocabSize); std::iota(data.begin(), data.end(), 0); globalFactorMatrix_ = csr_rows(data); // [V x U] diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index dd3f2741b..866c4b2c7 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -131,31 +131,49 @@ Expr le(Expr a, float b) { return Expression(a, a->graph()->constant( /*********************************************************/ Expr operator+(Expr a, float b) { - return Expression(a, b); + if (b == 0) + return a; + else + return Expression(a, b); } Expr operator+(float a, Expr b) { - return Expression(b, a); + if (a == 0) + return b; + else + return Expression(b, a); } Expr operator-(Expr a, float b) { - return Expression(a, -b); + if (b == 0) + return a; + else + return Expression(a, -b); } Expr operator-(float a, Expr b) { - return Expression(-b, a); + if (a == 0) + return -b; + else + return Expression(-b, a); } Expr operator*(float a, Expr b) { - return Expression(b, a); + if (a == 1.0f) + return b; + else + return Expression(b, a); } Expr operator*(Expr a, float b) { - return Expression(a, b); + if (b == 1.0f) + return a; + else + return Expression(a, b); } Expr operator/(Expr a, float b) { - return Expression(a, 1.f / b); + return a * (1.f / b); } // TODO: efficient version of this without constant() diff --git a/src/layers/generic.h b/src/layers/generic.h index c5a199384..35183213e 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -76,6 +76,7 @@ class Logits { Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; + float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // @TODO: avoid the fully expanded logits void assign(const Logits& other) { //ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), // "Logits assignment cannot change number of factors"); diff --git a/src/training/validator.h b/src/training/validator.h index 677eec8f9..138b98683 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -216,7 +216,7 @@ class CrossEntropyValidator : public Validator { } }; -// Used for validating with classifiers. Compute prediction accuary versus groundtruth for a set of classes +// Used for validating with classifiers. Compute prediction accuracy versus ground truth for a set of classes class AccuracyValidator : public Validator { public: AccuracyValidator(std::vector> vocabs, Ptr options) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index c80865f73..6641f1f45 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -235,10 +235,7 @@ class BeamSearch { states[i] = scorers_[i]->step( graph, states[i], hypIndices, predWords, dimBatch, (int)localBeamSize); - if(scorers_[i]->getWeight() != 1.f) - pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs(); - else - pathScores = pathScores + states[i]->getLogProbs(); + pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs().getLogits(); } // make beams continuous diff --git a/src/translator/scorers.h b/src/translator/scorers.h index 3f49a18c2..1b7c4e3c9 100755 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -9,9 +9,9 @@ namespace marian { class ScorerState { public: - virtual Expr getLogProbs() = 0; + virtual Logits getLogProbs() = 0; - virtual float breakDown(size_t i) { return getLogProbs()->val()->get(i); } + virtual float breakDown(size_t i) { return getLogProbs().getLogitAt(i); } virtual void blacklist(Expr /*totalCosts*/, Ptr /*batch*/){}; }; @@ -57,7 +57,7 @@ class ScorerWrapperState : public ScorerState { virtual Ptr getState() { return state_; } - virtual Expr getLogProbs() override { return state_->getLogProbs().getLogits(); }; + virtual Logits getLogProbs() override { return state_->getLogProbs(); }; virtual void blacklist(Expr totalCosts, Ptr batch) override { state_->blacklist(totalCosts, batch); From 492bf28a54bc7926355690706f82970c5446bacf Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 09:10:01 -0800 Subject: [PATCH 289/838] towards converting Word <-> factors --- src/data/factored_vocab.cpp | 77 +++++++++++++++++++++++++++++++------ src/data/factored_vocab.h | 9 +++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 622e19392..e9165d0b5 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -31,6 +31,7 @@ namespace marian { // construct mapping tables for factors constructGroupInfoFromFactorVocab(); + constructFactorIndexConversion(); // load and parse factorMap auto elements = factorShape_.elements(); @@ -48,23 +49,17 @@ namespace marian { // Not every word has all other factors, so the n-th item is not always the same factor. utils::splitAny(line, tokens, " \t"); ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); - std::vector factors; + std::vector factorUnits; for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; - factors.push_back(u); + factorUnits.push_back(u); factorRefCounts_[u]++; } - size_t index = 0; - for (auto u : factors) { - auto g = factorGroups_[u]; // convert u to relative u within factor group range - ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); - auto factorIndex = u - groupRanges_[g].first; - index += factorIndex * factorStrides_[g]; - } + auto index = factorUnits2wordIndex(factorUnits); // @TODO: map factors to non-dense integer - factorMap_[index] = std::move(factors); + factorMap_[index] = std::move(factorUnits); // add to vocab - vocab_.add(tokens.front(), (WordIndex)index); + vocab_.add(tokens.front(), index); numTotalFactors += tokens.size() - 1; } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", @@ -116,12 +111,70 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes_[g]); } - factorShape_ = Shape(std::move(groupCounts)); + // we map between factors and flat WordIndex like indexing a tensor + constructFactorIndexConversion(); +} + +void FactoredVocab::constructFactorIndexConversion() { + std::vector shape; + for (const auto& r : groupRanges_) + shape.push_back((int)(r.second - r.first + 1)); // +1 to reserve the last value for either "factor not used" or "factor not present" + factorShape_ = Shape(std::move(shape)); factorStrides_.resize(factorShape_.size(), 1); for (size_t g = factorStrides_.size() - 1; g --> 0; ) factorStrides_[g] = factorStrides_[g + 1] * (size_t)factorShape_[g + 1]; } +// encode factors into a Word struct +Word FactoredVocab::factors2word(const std::vector& factorIndices /* [numGroups] */) { + size_t index = 0; + size_t numGroups = getNumGroups(); + ABORT_IF(factorIndices.size() != numGroups, "Factor indices array size must be same as number of factor groups"); + for (size_t g = 0; g < numGroups; g++) { + auto factorIndex = factorIndices[g]; + ABORT_IF(factorIndex >= (size_t)factorShape_[g], "Factor index out of range"); + index += factorIndex * factorStrides_[g]; + } + return Word::fromWordIndex(index); +} + +// like factors2word, except that factors are expressed as global unit indices, and result is just the WordIndex +// @BUGBUG: need to encode factors that are not used ==> change to unroll explicitly, then call factors2Word() +WordIndex FactoredVocab::factorUnits2wordIndex(const std::vector& factorUnits) { + size_t index = 0; + for (auto u : factorUnits) { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + auto factorIndex = u - groupRanges_[g].first; + index += factorIndex * factorStrides_[g]; + } + return (WordIndex)index; +} + +void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) { + word; + size_t numGroups = getNumGroups(); + factorIndices.resize(numGroups); + ABORT("Not implemented"); +} + +size_t FactoredVocab::getFactor(Word word, size_t groupIndex) { + word; groupIndex; + //size_t factorIndex = 0; + ABORT("Not implemented"); +} + +// @TODO: or just map to sentinel values for FACTOR_NOT_APPLICABLE and FACTOR_NOT_SPECIFIED +bool FactoredVocab::hasFactor(Word word, size_t groupIndex) { + word; groupIndex; + ABORT("Not implemented"); +} + +std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) { + word; groupIndex; + ABORT("Not implemented"); +} + void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs size_t numGroups = groupPrefixes_.size(); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 8d41d363a..94bc024f8 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -48,9 +48,18 @@ class FactoredVocab : public IVocab { const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 + // convert representations + Word factors2word(const std::vector& factors); + WordIndex factorUnits2wordIndex(const std::vector& factorUnits); + void word2factors(Word word, std::vector& factors); + size_t getFactor(Word word, size_t groupIndex); + bool hasFactor(Word word, size_t groupIndex); + std::pair getFactorUnit(Word word, size_t groupIndex); + static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: void constructGroupInfoFromFactorVocab(); + void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); private: class WordLUT { // map between strings and WordIndex From 267805c7408a3ec85eaf00631c6bf3c840ebe29e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 09:22:01 -0800 Subject: [PATCH 290/838] Word representation of factors now includes a sentinel for 'unspecified' and --- src/data/factored_vocab.cpp | 21 ++++++++++----------- src/data/factored_vocab.h | 5 +++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index e9165d0b5..4fc5dccde 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -115,6 +115,7 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { constructFactorIndexConversion(); } +// create factorShape_ and factorStrides_, for mapping between flat (non-dense) ids and factor arrays void FactoredVocab::constructFactorIndexConversion() { std::vector shape; for (const auto& r : groupRanges_) @@ -132,23 +133,27 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu ABORT_IF(factorIndices.size() != numGroups, "Factor indices array size must be same as number of factor groups"); for (size_t g = 0; g < numGroups; g++) { auto factorIndex = factorIndices[g]; - ABORT_IF(factorIndex >= (size_t)factorShape_[g], "Factor index out of range"); + if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) + factorIndex = (size_t)factorShape_[g] - 1; // sentinel for "unused" or "not specified" + else + ABORT_IF(factorIndex >= (size_t)factorShape_[g] - 1, "Factor index out of range"); index += factorIndex * factorStrides_[g]; } return Word::fromWordIndex(index); } // like factors2word, except that factors are expressed as global unit indices, and result is just the WordIndex -// @BUGBUG: need to encode factors that are not used ==> change to unroll explicitly, then call factors2Word() +// This is only used during initialization, so it's OK if it is a little inefficient. WordIndex FactoredVocab::factorUnits2wordIndex(const std::vector& factorUnits) { - size_t index = 0; + // convert to fully unrolled factors representation + std::vector factorIndices(getNumGroups(), FACTOR_NOT_APPLICABLE); // default for unused factors for (auto u : factorUnits) { auto g = factorGroups_[u]; // convert u to relative u within factor group range ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); auto factorIndex = u - groupRanges_[g].first; - index += factorIndex * factorStrides_[g]; + factorIndices[g] = factorIndex; } - return (WordIndex)index; + return factors2word(factorIndices).toWordIndex(); } void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) { @@ -164,12 +169,6 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) { ABORT("Not implemented"); } -// @TODO: or just map to sentinel values for FACTOR_NOT_APPLICABLE and FACTOR_NOT_SPECIFIED -bool FactoredVocab::hasFactor(Word word, size_t groupIndex) { - word; groupIndex; - ABORT("Not implemented"); -} - std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) { word; groupIndex; ABORT("Not implemented"); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 94bc024f8..6c5358a22 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -50,17 +50,18 @@ class FactoredVocab : public IVocab { // convert representations Word factors2word(const std::vector& factors); - WordIndex factorUnits2wordIndex(const std::vector& factorUnits); void word2factors(Word word, std::vector& factors); size_t getFactor(Word word, size_t groupIndex); - bool hasFactor(Word word, size_t groupIndex); std::pair getFactorUnit(Word word, size_t groupIndex); + static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); + static constexpr size_t FACTOR_NOT_SPECIFIED = (SIZE_MAX - 2); static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); + WordIndex factorUnits2wordIndex(const std::vector& factorUnits); private: class WordLUT { // map between strings and WordIndex std::map str2index_; From 03f19da26c1b525925f537c694864688cfb09379 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 6 Feb 2019 10:24:53 -0800 Subject: [PATCH 291/838] Remove old comment --- CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 340c15864..022b97019 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,9 +168,6 @@ endif(USE_STATIC_LIBS) list(APPEND CUDA_NVCC_FLAGS -DBOOST_PP_VARIADICS=0; ) endif() - # We compile NCCL ourselves, using the NVidia Makefile rather than CMake, this requires to pass a couple of parameters from - # Cmake. This is also fairly untested, let's hope it does not explode. - # @TODO: Make sure it does not use pre-installed NCCL headers if(USE_NCCL) add_library(nccl STATIC IMPORTED) set(EXT_LIBS ${EXT_LIBS} nccl) From 0af135fe18f6c74353ca93c4f1f1f4631594c00d Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 6 Feb 2019 10:25:51 -0800 Subject: [PATCH 292/838] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f0b77c197..257d138aa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.7.7 +v1.7.8 From 46e9565c9a06e54f4f05f50f722da726ae437df0 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 10:33:41 -0800 Subject: [PATCH 293/838] bug fix: and ids should not be misinterpreted as factor indices --- src/data/factored_vocab.cpp | 48 ++++++++++++++++++++++++++++++------- src/data/factored_vocab.h | 3 ++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 4fc5dccde..b72db7196 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -55,12 +55,14 @@ namespace marian { factorUnits.push_back(u); factorRefCounts_[u]++; } - auto index = factorUnits2wordIndex(factorUnits); + auto index = factorUnits2word(factorUnits).toWordIndex(); // @TODO: map factors to non-dense integer factorMap_[index] = std::move(factorUnits); // add to vocab vocab_.add(tokens.front(), index); numTotalFactors += tokens.size() - 1; + if (v % 5000 == 0) + LOG(info, "{} -> {}", tokens.front(), word2string(Word::fromWordIndex(index))); } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", numTotalFactors, factorVocabSize, vocab_.numValid(), size()); @@ -71,6 +73,7 @@ namespace marian { // and must exist in the vocabulary eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); + LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); #if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() if (maxSizeUnused == vocab_.numValid()) @@ -133,7 +136,7 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu ABORT_IF(factorIndices.size() != numGroups, "Factor indices array size must be same as number of factor groups"); for (size_t g = 0; g < numGroups; g++) { auto factorIndex = factorIndices[g]; - if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) + if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) // @TODO: check validity. If word has the factor, then N/A is invalid factorIndex = (size_t)factorShape_[g] - 1; // sentinel for "unused" or "not specified" else ABORT_IF(factorIndex >= (size_t)factorShape_[g] - 1, "Factor index out of range"); @@ -144,7 +147,7 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu // like factors2word, except that factors are expressed as global unit indices, and result is just the WordIndex // This is only used during initialization, so it's OK if it is a little inefficient. -WordIndex FactoredVocab::factorUnits2wordIndex(const std::vector& factorUnits) { +Word FactoredVocab::factorUnits2word(const std::vector& factorUnits) { // convert to fully unrolled factors representation std::vector factorIndices(getNumGroups(), FACTOR_NOT_APPLICABLE); // default for unused factors for (auto u : factorUnits) { @@ -153,20 +156,47 @@ WordIndex FactoredVocab::factorUnits2wordIndex(const std::vector& fac auto factorIndex = u - groupRanges_[g].first; factorIndices[g] = factorIndex; } - return factors2word(factorIndices).toWordIndex(); + return factors2word(factorIndices); } void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) { - word; size_t numGroups = getNumGroups(); factorIndices.resize(numGroups); - ABORT("Not implemented"); + for (size_t g = 0; g < numGroups; g++) { + auto factorIndex = getFactor(word, g); + factorIndices[g] = factorIndex; + } +#if 1 + auto test = factors2word(factorIndices); + ABORT_IF(test != word, "Word <-> factor conversion broken??"); +#endif +} + +std::string FactoredVocab::word2string(Word word) { + std::vector factorIndices; + word2factors(word, factorIndices); + std::string res; + size_t numGroups = getNumGroups(); + for (size_t g = 0; g < numGroups; g++) { + res.append(res.empty() ? "(" : ", "); + auto factorIndex = factorIndices[g]; + switch (factorIndex) { + case FACTOR_NOT_APPLICABLE: res.append("n/a"); break; + case FACTOR_NOT_SPECIFIED: res.append("?"); break; + default: res.append(factorVocab_[(WordIndex)(factorIndex + groupRanges_[g].first)]); break; + } + } + return res + ")"; } size_t FactoredVocab::getFactor(Word word, size_t groupIndex) { - word; groupIndex; - //size_t factorIndex = 0; - ABORT("Not implemented"); + size_t index = word.toWordIndex(); + index = index / factorStrides_[groupIndex]; + index = index % (size_t)factorShape_[groupIndex]; + if (index == (size_t)factorShape_[groupIndex] - 1) { + index = FACTOR_NOT_APPLICABLE; // @BUGBUG: We should check here which one it is. + } + return index; } std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 6c5358a22..e2ae02785 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -61,7 +61,8 @@ class FactoredVocab : public IVocab { void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); - WordIndex factorUnits2wordIndex(const std::vector& factorUnits); + Word factorUnits2word(const std::vector& factorUnits); + std::string word2string(Word word); private: class WordLUT { // map between strings and WordIndex std::map str2index_; From f88eb0d3687575cfe55f908ddad4617c9dc68ee2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 20:25:43 -0800 Subject: [PATCH 294/838] commenting and minor refactoring of beam search --- src/common/options.h | 2 +- src/graph/expression_operators.cpp | 39 ++++-- src/microsoft/quicksand.cpp | 2 +- src/models/transformer.h | 11 -- src/models/transformer_factory.h | 5 +- src/models/transformer_stub.cpp | 16 ++- src/tensors/cpu/tensor_operators.cpp | 1 + src/training/validator.h | 6 +- src/translator/beam_search.h | 174 ++++++++++++++++----------- src/translator/history.h | 23 ++-- src/translator/hypothesis.h | 15 ++- src/translator/output_printer.h | 8 +- src/translator/translator.h | 4 +- vs/Marian.vcxproj | 4 +- vs/Marian.vcxproj.filters | 12 +- 15 files changed, 194 insertions(+), 128 deletions(-) mode change 100644 => 100755 src/common/options.h mode change 100644 => 100755 src/graph/expression_operators.cpp mode change 100644 => 100755 src/microsoft/quicksand.cpp mode change 100644 => 100755 src/models/transformer.h mode change 100644 => 100755 src/models/transformer_factory.h mode change 100644 => 100755 src/models/transformer_stub.cpp mode change 100644 => 100755 src/tensors/cpu/tensor_operators.cpp mode change 100644 => 100755 src/training/validator.h mode change 100644 => 100755 src/translator/beam_search.h mode change 100644 => 100755 src/translator/history.h mode change 100644 => 100755 src/translator/hypothesis.h mode change 100644 => 100755 src/translator/output_printer.h mode change 100644 => 100755 src/translator/translator.h mode change 100644 => 100755 vs/Marian.vcxproj mode change 100644 => 100755 vs/Marian.vcxproj.filters diff --git a/src/common/options.h b/src/common/options.h old mode 100644 new mode 100755 index 5a3f4eb70..643456c93 --- a/src/common/options.h +++ b/src/common/options.h @@ -111,7 +111,7 @@ class Options { } try { return !options_[key].as().empty(); - } catch(const YAML::BadConversion& e) { + } catch(const YAML::BadConversion&) { ABORT("Option '{}' is neither a sequence nor a text"); } return false; diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100644 new mode 100755 index e558ffd08..8a79cbe0c --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -135,31 +135,49 @@ Expr le(Expr a, float b) { return Expression(a, a->graph()->constant( /*********************************************************/ Expr operator+(Expr a, float b) { - return Expression(a, b); + if (b == 0) + return a; + else + return Expression(a, b); } Expr operator+(float a, Expr b) { - return Expression(b, a); + if (a == 0) + return b; + else + return Expression(b, a); } Expr operator-(Expr a, float b) { - return Expression(a, -b); + if (b == 0) + return a; + else + return Expression(a, -b); } Expr operator-(float a, Expr b) { - return Expression(-b, a); + if (a == 0) + return -b; + else + return Expression(-b, a); } Expr operator*(float a, Expr b) { - return Expression(b, a); + if (a == 1.0f) + return b; + else + return Expression(b, a); } Expr operator*(Expr a, float b) { - return Expression(a, b); + if (b == 1.0f) + return a; + else + return Expression(a, b); } Expr operator/(Expr a, float b) { - return Expression(a, 1.f / b); + return a * (1.f / b); } // TODO: efficient version of this without constant() @@ -254,7 +272,12 @@ Expr gather(Expr a, int axis, Expr indices) { return Expression(a, axis, indices); } -// index_select() -- gather arbitrary elements along an axis; unbatched (indices are specified as a 1D vector) +// index_select() -- gather arbitrary elements along an axis from an unbatched +// input 'a'. Indices are specified as a 1D vector. +// This is used e.g. for embedding lookup. +// Note: To use a batch of index vectors, reshape them into a single vector, +// call index_select(), then reshape the result back. Reshapes are cheap. +// This function has the same semantics as PyTorch operation of the same name. Expr index_select(Expr a, int axis, Expr indices) { ABORT_IF(indices->shape().size() != 1, "Indices must be a 1D tensor"); // We have specialized kernels for non-batched indexing of first or last axis of a 2D tensor. diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100644 new mode 100755 index 959e44311..2c9fc2a3f --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -129,7 +129,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { QSNBestBatch qsNbestBatch; for(const auto& history : histories) { // loop over batch entries QSNBest qsNbest; - NBestList nbestHyps = history->NBest(SIZE_MAX); // request as many N as we have + NBestList nbestHyps = history->nBest(SIZE_MAX); // request as many N as we have for (const Result& result : nbestHyps) { // loop over N-best entries // get hypothesis word sequence and normalized sentence score auto words = std::get<0>(result); diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100644 new mode 100755 index 1083efe29..97faf13ce --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -864,17 +864,6 @@ class DecoderTransformer : public Transformer { } }; -// factory functions -Ptr NewEncoderTransformer(Ptr options) -{ - return New(options); -} - -Ptr NewDecoderTransformer(Ptr options) -{ - return New(options); -} - // clang-format on } // namespace marian diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100644 new mode 100755 index c2a7e13bb..aa31e4d15 --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -9,7 +9,6 @@ //#include "layers/factory.h" namespace marian { -// @TODO: find out why static is required here to get to compile -static Ptr NewEncoderTransformer(Ptr options); -static Ptr NewDecoderTransformer(Ptr options); +Ptr NewEncoderTransformer(Ptr options); +Ptr NewDecoderTransformer(Ptr options); } // namespace marian diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100644 new mode 100755 index 420b77810..1e7b19f68 --- a/src/models/transformer_stub.cpp +++ b/src/models/transformer_stub.cpp @@ -1,4 +1,14 @@ -// TODO: This is a wrapper around transformer.h. We kept the .H name to minimize confusing git, until this is code-reviewed. -// This is meant to speed-up builds, and to support Ctrl-F7 to rebuild. - #include "models/transformer.h" + +namespace marian { +// factory functions +Ptr NewEncoderTransformer(Ptr options) +{ + return New(options); +} + +Ptr NewDecoderTransformer(Ptr options) +{ + return New(options); +} +} // namespace marian diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100644 new mode 100755 index 72dbd131c..840d132ca --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -15,6 +15,7 @@ namespace marian { namespace cpu { void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf, bool zero) { + isNan; isInf; zero; ABORT("Not implemented"); } diff --git a/src/training/validator.h b/src/training/validator.h old mode 100644 new mode 100755 index 1bbebb91c..cf67299ee --- a/src/training/validator.h +++ b/src/training/validator.h @@ -535,7 +535,7 @@ class TranslationValidator : public Validator { std::stringstream best1; std::stringstream bestn; printer->print(history, best1, bestn); - collector->Write((long)history->GetLineNum(), + collector->Write((long)history->getLineNum(), best1.str(), bestn.str(), options_->get("n-best")); @@ -677,14 +677,14 @@ class BleuValidator : public Validator { size_t no = 0; std::lock_guard statsLock(mutex_); for(auto history : histories) { - auto result = history->Top(); + auto result = history->top(); const auto& words = std::get<0>(result); updateStats(stats, words, batch, no, vocabs_.back()->getEosId()); std::stringstream best1; std::stringstream bestn; printer->print(history, best1, bestn); - collector->Write((long)history->GetLineNum(), + collector->Write((long)history->getLineNum(), best1.str(), bestn.str(), /*nbest=*/ false); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100644 new mode 100755 index 17ba6d573..5272c217a --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -35,10 +35,10 @@ class BeamSearch { const std::vector pathScores, size_t vocabSize, const Beams& beams, - std::vector>& states, + const std::vector>& states, size_t beamSize, bool first, - Ptr batch) { + Ptr batch) const { Beams newBeams(beams.size()); std::vector align; @@ -46,46 +46,49 @@ class BeamSearch { // Use alignments from the first scorer, even if ensemble align = scorers_[0]->getAlignment(); - for(size_t i = 0; i < keys.size(); ++i) { + for(size_t i = 0; i < keys.size(); ++i) { // keys: [beamSize, ?] (flattened) // Keys contains indices to vocab items in the entire beam. // Values can be between 0 and beamSize * vocabSize. - Word embIdx = (Word)(keys[i] % vocabSize); auto beamIdx = i / beamSize; - // Retrieve short list for final softmax (based on words aligned - // to source sentences). If short list has been set, map the indices - // in the sub-selected vocabulary matrix back to their original positions. - auto shortlist = scorers_[0]->getShortlist(); - if(shortlist) - embIdx = shortlist->reverseMap(embIdx); // @TODO: should reverseMap accept a size_t or a Word? - if(newBeams[beamIdx].size() < beams[beamIdx].size()) { - auto& beam = beams[beamIdx]; + Word wordIdx = (Word)(keys[i] % vocabSize); + // Retrieve short list for final softmax (based on words aligned + // to source sentences). If short list has been set, map the indices + // in the sub-selected vocabulary matrix back to their original positions. + auto shortlist = scorers_[0]->getShortlist(); + if(shortlist) + wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? + + const auto& beam = beams[beamIdx]; auto& newBeam = newBeams[beamIdx]; - auto hypIdx = (IndexType)(keys[i] / vocabSize); - float pathScore = pathScores[i]; + const float pathScore = pathScores[i]; + + // keys[i] = offset into row-major cube of dims [whatIsThis, beamSize, vocabSize] + // deconstruct into individual indices + const auto hypIdx = (IndexType)(keys[i] / vocabSize); + const auto whatIsThis = (hypIdx / beamSize); // @TODO: is this batchIdx? + size_t beamHypIdx = hypIdx % beamSize; - auto hypIdxTrans - = IndexType((hypIdx / beamSize) + (hypIdx % beamSize) * beams.size()); + auto hypIdxTrans = IndexType(whatIsThis + beamHypIdx * beams.size()); if(first) hypIdxTrans = hypIdx; - size_t beamHypIdx = hypIdx % beamSize; - if(beamHypIdx >= (int)beam.size()) + if(beamHypIdx >= (int)beam.size()) // @TODO: What is this condition? Cf. beamHypIdx = hypIdx % beamSize beamHypIdx = beamHypIdx % beam.size(); if(first) beamHypIdx = 0; - auto hyp = New(beam[beamHypIdx], embIdx, hypIdxTrans, pathScore); + auto hyp = New(beam[beamHypIdx], wordIdx, hypIdxTrans, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { std::vector breakDown(states.size(), 0); beam[beamHypIdx]->GetScoreBreakdown().resize(states.size(), 0); for(size_t j = 0; j < states.size(); ++j) { - size_t key = embIdx + hypIdxTrans * vocabSize; + size_t key = wordIdx + hypIdxTrans * vocabSize; breakDown[j] = states[j]->breakDown(key) + beam[beamHypIdx]->GetScoreBreakdown()[j]; } @@ -108,7 +111,7 @@ class BeamSearch { const std::vector alignAll, Ptr batch, int beamHypIdx, - int beamIdx) { + int beamIdx) const { // Let's B be the beam size, N be the number of batched sentences, // and L the number of words in the longest sentence in the batch. // The alignment vector: @@ -140,12 +143,13 @@ class BeamSearch { return align; } - Beams pruneBeam(const Beams& beams) { + // remove all beam entries that have reached EOS + Beams purgeBeams(const Beams& beams) { Beams newBeams; for(auto beam : beams) { Beam newBeam; for(auto hyp : beam) { - if(hyp->GetWord() != trgEosId_) { + if(hyp->getWord() != trgEosId_) { newBeam.push_back(hyp); } } @@ -154,32 +158,29 @@ class BeamSearch { return newBeams; } + //********************************************************************** // main decoding function Histories search(Ptr graph, Ptr batch) { int dimBatch = (int)batch->size(); - Histories histories; + Histories histories(dimBatch); for(int i = 0; i < dimBatch; ++i) { size_t sentId = batch->getSentenceIds()[i]; - auto history = New(sentId, + histories[i] = New(sentId, options_->get("normalize"), options_->get("word-penalty")); - histories.push_back(history); } size_t localBeamSize = beamSize_; // max over beam sizes of active sentence hypotheses auto getNBestList = createGetNBestListFn(localBeamSize, dimBatch, graph->getDeviceId()); - Beams beams(dimBatch); // [batchIndex][beamIndex] is one sentence hypothesis + Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis for(auto& beam : beams) beam.resize(localBeamSize, New()); - bool first = true; - bool final = false; - for(int i = 0; i < dimBatch; ++i) - histories[i]->Add(beams[i], trgEosId_); + histories[i]->add(beams[i], trgEosId_); std::vector> states; @@ -191,33 +192,48 @@ class BeamSearch { states.push_back(scorer->startState(graph, batch)); } - // main loop over output tokens - do { + // the decoder maintains the following state: + // - histories : array [dimBatch] of History + // with History : vector [t] of array [localBeamSize] of Hypothesis + // with Hypothesis : (last word, aggregate score, prev Hypothesis) + // - search grid + // - stores traceback information + // - gets added to in each output time step + // - the final version is the return value of this function + // - beams : array [dimBatch] of array [localBeamSize] of Hypothesis + // - current output time step's set of active hypotheses, aka active search space + // - gets replaced at the end of each output time step + // - states[.] : ScorerState + // - NN state + // - one per scorer, e.g. 2 for ensemble of 2 + + // main loop over output time steps + for(bool first = true; ; first = false) { //********************************************************************** // create constant containing previous path scores for current beam - // also create mapping of hyp indices, which are not 1:1 if sentences complete - std::vector hypIndices; // [beamIndex * activeBatchSize + batchIndex] backpointers, concatenated over beam positions. Used for reordering hypotheses - std::vector embIndices; - Expr prevPathScores; // [beam, 1, 1, 1] + // Also create mapping of hyp indices, which are not 1:1 if sentences complete. + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of beam index that each of the new top N originated from + std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word + Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding pathScores if(first) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { - std::vector beamScores; - dimBatch = (int)batch->size(); + ABORT_IF(dimBatch != beams.size(), "Dimensions mismatch??"); - for(size_t i = 0; i < localBeamSize; ++i) { - for(size_t j = 0; j < beams.size(); ++j) { // loop over batch entries (active sentences) - auto& beam = beams[j]; - if(i < beam.size()) { - auto hyp = beam[i]; - hypIndices.push_back((IndexType)hyp->GetPrevStateIndex()); // backpointer - embIndices.push_back(hyp->GetWord()); - beamScores.push_back(hyp->GetPathScore()); + std::vector beamScores; + for(size_t beamIndex = 0; beamIndex < localBeamSize; ++beamIndex) { + for(int batchIndex = 0; batchIndex < dimBatch; ++batchIndex) { // loop over batch entries (active sentences) + auto& beam = beams[batchIndex]; + if(beamIndex < beam.size()) { + auto hyp = beam[beamIndex]; + hypIndices.push_back((IndexType)hyp->getPrevStateIndex()); // backpointer + prevWords .push_back(hyp->getWord()); + beamScores.push_back(hyp->getPathScore()); } else { // dummy hypothesis hypIndices.push_back(0); - embIndices.push_back(0); // (unused) + prevWords .push_back(Word{}); // (unused) beamScores.push_back(-9999); } } @@ -232,18 +248,23 @@ class BeamSearch { auto pathScores = prevPathScores; for(size_t i = 0; i < scorers_.size(); ++i) { - states[i] = scorers_[i]->step( - graph, states[i], hypIndices, embIndices, dimBatch, (int)localBeamSize); - - if(scorers_[i]->getWeight() != 1.f) - pathScores = pathScores + scorers_[i]->getWeight() * states[i]->getLogProbs(); - else - pathScores = pathScores + states[i]->getLogProbs(); + // compute output probabilities for current output time step + // - uses hypIndices[index in beam, 1, batch index, 1] and embIndices[index in beam, 1, batch index, 1] to reorder hypotheses + // - returns new NN state for use in next output time step + // - returns vector of prediction probabilities over output vocab via newState + auto newState = scorers_[i]->step( + graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); + + // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] + pathScores = pathScores + scorers_[i]->getWeight() * newState->getLogProbs(); + + // update state in-place for next output time step + states[i] = newState; } // make beams continuous if(dimBatch > 1 && localBeamSize > 1) - pathScores = transpose(pathScores, {2, 1, 0, 3}); + pathScores = transpose(pathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] if(first) graph->forward(); @@ -260,12 +281,15 @@ class BeamSearch { //********************************************************************** // perform beam search and pruning + + // find N best amongst the (localBeamSize * dimVocab) hypotheses + const std::vector beamSizes(dimBatch, localBeamSize); std::vector outKeys; std::vector outPathScores; - - std::vector beamSizes(dimBatch, localBeamSize); getNBestList(beamSizes, pathScores->val(), outPathScores, outKeys, first); + // outPathScores and outKeys contain pathScores and their original indices in N-best order + // convert N-best sets to updated search space int dimTrgVoc = pathScores->shape()[-1]; beams = toHyps(outKeys, outPathScores, @@ -276,20 +300,28 @@ class BeamSearch { first, batch); - auto prunedBeams = pruneBeam(beams); + // remove all hyps that end in EOS + auto purgedBeams = purgeBeams(beams); // @TODO: rename; this is not pruning + + // add updated search space to search grid for traceback + bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { + // if this batch entry has surviving hyps then add them to the traceback grid if(!beams[i].empty()) { - final = final - || histories[i]->size() - >= options_->get("max-length-factor") - * batch->front()->batchWidth(); - histories[i]->Add( - beams[i], trgEosId_, prunedBeams[i].empty() || final); + if (histories[i]->size() >= options_->get("max-length-factor") * batch->front()->batchWidth()) + maxLengthReached = true; + histories[i]->add(beams[i], trgEosId_, purgedBeams[i].empty() || maxLengthReached); } } - beams = prunedBeams; + if (maxLengthReached) // early exit if max length limit was reached + break; + + // this is the search space for the next output time step + beams = purgedBeams; - // determine beam size for next sentence, as max over still-active sentences + // determine beam size for next output time step, as max over still-active sentences + // E.g. if all batch entries are down from beam 5 to no more than 4 surviving hyps, then + // switch to beam of 4 for all. If all are done, then beam ends up being 0, and we are done. if(!first) { size_t maxBeam = 0; for(auto& beam : beams) @@ -297,11 +329,11 @@ class BeamSearch { maxBeam = beam.size(); localBeamSize = maxBeam; } - first = false; - - } while(localBeamSize != 0 && !final); // end of main loop over output tokens + if (localBeamSize == 0) // done if all batch entries have reached EOS on all beam entries + break; + } // end of main loop over output tokens - return histories; + return histories; // [dimBatch][t][N best hyps] } }; } // namespace marian diff --git a/src/translator/history.h b/src/translator/history.h old mode 100644 new mode 100755 index e70b80c6f..dcf5e5854 --- a/src/translator/history.h +++ b/src/translator/history.h @@ -19,18 +19,17 @@ class History { float normalizedPathScore; // length-normalized sentence score }; + float lengthPenalty(size_t length) { return std::pow((float)length, alpha_); } + float wordPenalty(size_t length) { return wp_ * (float)length; } public: History(size_t lineNo, float alpha = 1.f, float wp_ = 0.f); - float LengthPenalty(size_t length) { return std::pow((float)length, alpha_); } - float WordPenalty(size_t length) { return wp_ * (float)length; } - - void Add(const Beam& beam, Word trgEosId, bool last = false) { + void add(const Beam& beam, Word trgEosId, bool last = false) { if(beam.back()->GetPrevHyp() != nullptr) { for(size_t j = 0; j < beam.size(); ++j) - if(beam[j]->GetWord() == trgEosId || last) { - float pathScore = (beam[j]->GetPathScore() - WordPenalty(history_.size())) - / LengthPenalty(history_.size()); + if(beam[j]->getWord() == trgEosId || last) { + float pathScore = + (beam[j]->getPathScore() - wordPenalty(history_.size())) / lengthPenalty(history_.size()); topHyps_.push({history_.size(), j, pathScore}); // std::cerr << "Add " << history_.size() << " " << j << " " << pathScore // << std::endl; @@ -41,7 +40,7 @@ class History { size_t size() const { return history_.size(); } // number of time steps - NBestList NBest(size_t n) const { + NBestList nBest(size_t n) const { NBestList nbest; for (auto topHypsCopy = topHyps_; nbest.size() < n && !topHypsCopy.empty(); topHypsCopy.pop()) { auto bestHypCoord = topHypsCopy.top(); @@ -55,15 +54,15 @@ class History { // trace back best path Words targetWords = bestHyp->TracebackWords(); - // note: bestHyp->GetPathScore() is not normalized, while bestHypCoord.normalizedPathScore is + // note: bestHyp->getPathScore() is not normalized, while bestHypCoord.normalizedPathScore is nbest.emplace_back(targetWords, bestHyp, bestHypCoord.normalizedPathScore); } return nbest; } - Result Top() const { return NBest(1)[0]; } + Result top() const { return nBest(1)[0]; } - size_t GetLineNum() const { return lineNo_; } + size_t getLineNum() const { return lineNo_; } private: std::vector history_; // [time step][index into beam] search grid @@ -73,5 +72,5 @@ class History { float wp_; }; -typedef std::vector> Histories; +typedef std::vector> Histories; // [batchDim] } // namespace marian diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h old mode 100644 new mode 100755 index c13ac716b..ddb475143 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -6,6 +6,11 @@ namespace marian { +// one single (possibly partial) hypothesis in beam search +// key elements: +// - the word that this hyp ends with +// - the aggregate score up to and including the word +// - back pointer to previous hypothesis for traceback class Hypothesis { public: Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(0), pathScore_(0.0) {} @@ -18,11 +23,11 @@ class Hypothesis { const Ptr GetPrevHyp() const { return prevHyp_; } - Word GetWord() const { return word_; } + Word getWord() const { return word_; } - IndexType GetPrevStateIndex() const { return prevIndex_; } + IndexType getPrevStateIndex() const { return prevIndex_; } - float GetPathScore() const { return pathScore_; } + float getPathScore() const { return pathScore_; } std::vector& GetScoreBreakdown() { return scoreBreakdown_; } std::vector& GetAlignment() { return alignment_; } @@ -34,8 +39,8 @@ class Hypothesis { { Words targetWords; for (auto hyp = this; hyp->GetPrevHyp(); hyp = hyp->GetPrevHyp().get()) { - targetWords.push_back(hyp->GetWord()); - // std::cerr << hyp->GetWord() << " " << hyp << std::endl; + targetWords.push_back(hyp->getWord()); + // std::cerr << hyp->getWord() << " " << hyp << std::endl; } std::reverse(targetWords.begin(), targetWords.end()); return targetWords; diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100644 new mode 100755 index b27b591cf..38ed7b9c6 --- a/src/translator/output_printer.h +++ b/src/translator/output_printer.h @@ -24,7 +24,7 @@ class OutputPrinter { template void print(Ptr history, OStream& best1, OStream& bestn) { - const auto& nbl = history->NBest(nbest_); + const auto& nbl = history->nBest(nbest_); for(size_t i = 0; i < nbl.size(); ++i) { const auto& result = nbl[i]; @@ -35,14 +35,14 @@ class OutputPrinter { std::reverse(words.begin(), words.end()); std::string translation = vocab_->decode(words); - bestn << history->GetLineNum() << " ||| " << translation; + bestn << history->getLineNum() << " ||| " << translation; if(!alignment_.empty()) bestn << " ||| " << getAlignment(hypo); bestn << " |||"; if(hypo->GetScoreBreakdown().empty()) { - bestn << " F0=" << hypo->GetPathScore(); + bestn << " F0=" << hypo->getPathScore(); } else { for(size_t j = 0; j < hypo->GetScoreBreakdown().size(); ++j) { bestn << " F" << j << "= " << hypo->GetScoreBreakdown()[j]; @@ -58,7 +58,7 @@ class OutputPrinter { bestn << std::flush; } - auto result = history->Top(); + auto result = history->top(); auto words = std::get<0>(result); if(reverse_) diff --git a/src/translator/translator.h b/src/translator/translator.h old mode 100644 new mode 100755 index 18371a3de..e79b52716 --- a/src/translator/translator.h +++ b/src/translator/translator.h @@ -106,7 +106,7 @@ class Translate : public ModelTask { std::stringstream best1; std::stringstream bestn; printer->print(history, best1, bestn); - collector->Write((long)history->GetLineNum(), + collector->Write((long)history->getLineNum(), best1.str(), bestn.str(), options_->get("n-best")); @@ -211,7 +211,7 @@ class TranslateService : public ModelServiceTask { std::stringstream best1; std::stringstream bestn; printer->print(history, best1, bestn); - collector->add((long)history->GetLineNum(), best1.str(), bestn.str()); + collector->add((long)history->getLineNum(), best1.str(), bestn.str()); } }; diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100644 new mode 100755 index a6c560d3a..c899c242f --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -580,6 +580,7 @@ false + @@ -894,6 +895,7 @@ + @@ -905,7 +907,7 @@ - + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100644 new mode 100755 index d9e56843f..b9036c1fd --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -202,9 +202,6 @@ tensors\cpu\sharp - - models - common @@ -481,6 +478,9 @@ examples\iris + + models + @@ -1517,6 +1517,12 @@ examples\mnist + + models + + + models + From 9b54e7f1caa86b16da84638a20566e8d5451c170 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 6 Feb 2019 23:50:33 -0800 Subject: [PATCH 295/838] further simplification/commenting of beam search --- src/graph/expression_operators.cpp | 2 +- src/microsoft/quicksand.cpp | 2 +- src/translator/beam_search.h | 241 +++++++++++++++-------------- src/translator/history.h | 4 +- src/translator/hypothesis.h | 27 ++-- src/translator/output_printer.cpp | 6 +- src/translator/output_printer.h | 6 +- src/translator/scorers.h | 6 +- 8 files changed, 151 insertions(+), 143 deletions(-) mode change 100644 => 100755 src/translator/output_printer.cpp mode change 100644 => 100755 src/translator/scorers.h diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 8a79cbe0c..5c37beef3 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -536,7 +536,7 @@ Expr swapAxes(Expr x, int axis1, int axis2) return x; // TODO: This is code dup from transpose(x). Implement transpose(x) as swapAxes(x, 0, 1) std::vector axes(x->shape().size()); - for (int i = 0; i < axes.size(); ++i) + for (int i = 0; i < axes.size(); ++i) // @TODO: use std::iota() axes[i] = i; std::swap(axes[axis1], axes[axis2]); return transpose(x, axes); diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 2c9fc2a3f..98b9fdd59 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -147,7 +147,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { else alignmentThreshold = std::max(std::stof(alignment), 0.f); auto hyp = std::get<1>(result); - data::WordAlignment align = data::ConvertSoftAlignToHardAlign(hyp->TracebackAlignment(), alignmentThreshold); + data::WordAlignment align = data::ConvertSoftAlignToHardAlign(hyp->tracebackAlignment(), alignmentThreshold); // convert to QuickSAND format alignmentSets.resize(words.size()); for (const auto& p : align) // @TODO: Does the feature_model param max_alignment_links apply here? diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 5272c217a..f825b8531 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -31,74 +31,83 @@ class BeamSearch { trgEosId_(trgEosId), trgUnkId_(trgUnkId) {} - Beams toHyps(const std::vector keys, - const std::vector pathScores, - size_t vocabSize, + // combine new expandedPathScores and previous beams into new set of beams + Beams toHyps(const std::vector nBestKeys, // [dimBatch, beamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened + const std::vector nBestPathScores, // [dimBatch, beamSize] flattened + const size_t vocabSize, const Beams& beams, - const std::vector>& states, - size_t beamSize, - bool first, - Ptr batch) const { - Beams newBeams(beams.size()); + const std::vector>& states, + const size_t beamSize, + const bool first, + Ptr batch) const { + const auto dimBatch = beams.size(); + Beams newBeams(dimBatch); std::vector align; if(options_->hasAndNotEmpty("alignment")) // Use alignments from the first scorer, even if ensemble align = scorers_[0]->getAlignment(); - for(size_t i = 0; i < keys.size(); ++i) { // keys: [beamSize, ?] (flattened) + for(size_t i = 0; i < nBestKeys.size(); ++i) { // [dimBatch, beamSize] flattened // Keys contains indices to vocab items in the entire beam. // Values can be between 0 and beamSize * vocabSize. - auto beamIdx = i / beamSize; + const auto batchIdx = i / beamSize; // and i % beamSize is the beam hyp index + const auto& beam = beams[batchIdx]; + auto& newBeam = newBeams[batchIdx]; + + if(newBeam.size() < beam.size()) { + const float pathScore = nBestPathScores[i]; + const auto key = nBestKeys[i]; // key = pathScore's location, as ((batchIdx, beamHypIdx) flattened, word idx) flattened + + // decompose key into individual indices + Word wordIdx = (Word)(key % vocabSize); + const auto hypIdx = (key / vocabSize); +#if 1 + // further decompose hypIdx, taking into account that the very first entry had beam size 1 + // and compose a new hypIdx that assumes actual beamSize + const auto keyBatchIdx = hypIdx / (first ? 1 : beamSize); + const auto keyBeamHypIdx = hypIdx % (first ? 1 : beamSize); + const auto hypIdxTrans = keyBeamHypIdx * dimBatch + keyBatchIdx; + ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); // not possible, as beamSize = max(beams[.].size()) +#else + const auto keyBatchIdx = hypIdx / beamSize; // @REVIEW: is this actually keyBatchIdx? + size_t keyBeamHypIdx = hypIdx % beamSize; + + auto hypIdxTrans = keyBatchIdx + keyBeamHypIdx * dimBatch; + if(first) + hypIdxTrans = hypIdx; // == keyBeamHypIdx + keyBatchIdx * beamSize? or was beamSize=1, and keyBeamHypIdx = 0? + + ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); + //if(keyBeamHypIdx >= (int)beam.size()) // @TODO: What is this condition? Cf. keyBeamHypIdx = hypIdx % beamSize; beamSize = max(beams[.].size()) + // keyBeamHypIdx = keyBeamHypIdx % beam.size(); - if(newBeams[beamIdx].size() < beams[beamIdx].size()) { - Word wordIdx = (Word)(keys[i] % vocabSize); + if(first) + keyBeamHypIdx = 0; +#endif // Retrieve short list for final softmax (based on words aligned // to source sentences). If short list has been set, map the indices // in the sub-selected vocabulary matrix back to their original positions. auto shortlist = scorers_[0]->getShortlist(); if(shortlist) wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? + // now wordIdx is a regular Word again - const auto& beam = beams[beamIdx]; - auto& newBeam = newBeams[beamIdx]; - - const float pathScore = pathScores[i]; - - // keys[i] = offset into row-major cube of dims [whatIsThis, beamSize, vocabSize] - // deconstruct into individual indices - const auto hypIdx = (IndexType)(keys[i] / vocabSize); - const auto whatIsThis = (hypIdx / beamSize); // @TODO: is this batchIdx? - size_t beamHypIdx = hypIdx % beamSize; - - auto hypIdxTrans = IndexType(whatIsThis + beamHypIdx * beams.size()); - if(first) - hypIdxTrans = hypIdx; - - if(beamHypIdx >= (int)beam.size()) // @TODO: What is this condition? Cf. beamHypIdx = hypIdx % beamSize - beamHypIdx = beamHypIdx % beam.size(); - - if(first) - beamHypIdx = 0; - - auto hyp = New(beam[beamHypIdx], wordIdx, hypIdxTrans, pathScore); + auto hyp = New(beam[keyBeamHypIdx], wordIdx, (IndexType)hypIdxTrans, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { std::vector breakDown(states.size(), 0); - beam[beamHypIdx]->GetScoreBreakdown().resize(states.size(), 0); + beam[keyBeamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? for(size_t j = 0; j < states.size(); ++j) { - size_t key = wordIdx + hypIdxTrans * vocabSize; - breakDown[j] = states[j]->breakDown(key) - + beam[beamHypIdx]->GetScoreBreakdown()[j]; + size_t key1 = hypIdxTrans * vocabSize + wordIdx; + breakDown[j] = states[j]->breakDown(key1) + beam[keyBeamHypIdx]->getScoreBreakdown()[j]; } - hyp->GetScoreBreakdown() = breakDown; + hyp->setScoreBreakdown(breakDown); } // Set alignments if(!align.empty()) { - hyp->SetAlignment( - getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)beamIdx)); + hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)keyBeamHypIdx, (int)batchIdx)); } newBeam.push_back(hyp); @@ -161,7 +170,13 @@ class BeamSearch { //********************************************************************** // main decoding function Histories search(Ptr graph, Ptr batch) { - int dimBatch = (int)batch->size(); + const int dimBatch = (int)batch->size(); + + auto getNBestList = createGetNBestListFn(beamSize_, dimBatch, graph->getDeviceId()); + + for(auto scorer : scorers_) { + scorer->clear(graph); + } Histories histories(dimBatch); for(int i = 0; i < dimBatch; ++i) { @@ -171,27 +186,20 @@ class BeamSearch { options_->get("word-penalty")); } - size_t localBeamSize = beamSize_; // max over beam sizes of active sentence hypotheses - - auto getNBestList = createGetNBestListFn(localBeamSize, dimBatch, graph->getDeviceId()); - - Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis - for(auto& beam : beams) - beam.resize(localBeamSize, New()); - - for(int i = 0; i < dimBatch; ++i) - histories[i]->add(beams[i], trgEosId_); - + // start states std::vector> states; - - for(auto scorer : scorers_) { - scorer->clear(graph); - } - for(auto scorer : scorers_) { states.push_back(scorer->startState(graph, batch)); } + Beams beams(dimBatch, Beam(beamSize_, New())); // array [dimBatch] of array [localBeamSize] of Hypothesis + //Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis + //for(auto& beam : beams) + // beam.resize(beamSize_, New()); + + for(int i = 0; i < dimBatch; ++i) + histories[i]->add(beams[i], trgEosId_); + // the decoder maintains the following state: // - histories : array [dimBatch] of History // with History : vector [t] of array [localBeamSize] of Hypothesis @@ -206,104 +214,116 @@ class BeamSearch { // - states[.] : ScorerState // - NN state // - one per scorer, e.g. 2 for ensemble of 2 + // - gets replaced at the end of each output time step // main loop over output time steps - for(bool first = true; ; first = false) { + for (size_t t = 0; ; t++) { + ABORT_IF(dimBatch != beams.size(), "Lost a batch entry??"); + // determine beam size for next output time step, as max over still-active sentences + // E.g. if all batch entries are down from beam 5 to no more than 4 surviving hyps, then + // switch to beam of 4 for all. If all are done, then beam ends up being 0, and we are done. + size_t localBeamSize = 0; // @TODO: is there some std::algorithm for this? + for(auto& beam : beams) + if(beam.size() > localBeamSize) + localBeamSize = beam.size(); + + // done if all batch entries have reached EOS on all beam entries + if (localBeamSize == 0) + break; + //********************************************************************** // create constant containing previous path scores for current beam // Also create mapping of hyp indices, which are not 1:1 if sentences complete. - std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of beam index that each of the new top N originated from - std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word - Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding pathScores - if(first) { - // no scores yet + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of hyp that the new top N originated from + std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word + Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding expandedPathScores + if(t == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { - dimBatch = (int)batch->size(); - ABORT_IF(dimBatch != beams.size(), "Dimensions mismatch??"); - - std::vector beamScores; + std::vector prevScores; for(size_t beamIndex = 0; beamIndex < localBeamSize; ++beamIndex) { for(int batchIndex = 0; batchIndex < dimBatch; ++batchIndex) { // loop over batch entries (active sentences) auto& beam = beams[batchIndex]; if(beamIndex < beam.size()) { auto hyp = beam[beamIndex]; - hypIndices.push_back((IndexType)hyp->getPrevStateIndex()); // backpointer + hypIndices.push_back((IndexType)hyp->getPrevStateIndex()); // index where to find prev hyp (beamHypIdx, batchIdx), =beamHypIdx * dimBatch + batchIdx prevWords .push_back(hyp->getWord()); - beamScores.push_back(hyp->getPathScore()); - } else { // dummy hypothesis + prevScores.push_back(hyp->getPathScore()); + } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); prevWords .push_back(Word{}); // (unused) - beamScores.push_back(-9999); + prevScores.push_back(-9999); } } } - prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, - inits::from_vector(beamScores)); + inits::from_vector(prevScores)); } //********************************************************************** - // prepare scores for beam search - auto pathScores = prevPathScores; - + // compute expanded path scores with word prediction probs from all scorers + auto expandedPathScores = prevPathScores; // will become [localBeamSize, 1, dimBatch, dimVocab] for(size_t i = 0; i < scorers_.size(); ++i) { // compute output probabilities for current output time step - // - uses hypIndices[index in beam, 1, batch index, 1] and embIndices[index in beam, 1, batch index, 1] to reorder hypotheses + // - uses hypIndices[index in beam, 1, batch index, 1] to reorder hypotheses + // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history + // - performs one step of the decoder model // - returns new NN state for use in next output time step // - returns vector of prediction probabilities over output vocab via newState auto newState = scorers_[i]->step( graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] - pathScores = pathScores + scorers_[i]->getWeight() * newState->getLogProbs(); + expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * newState->getLogProbs(); // update state in-place for next output time step states[i] = newState; } // make beams continuous - if(dimBatch > 1 && localBeamSize > 1) - pathScores = transpose(pathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] + expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] + //if(dimBatch > 1 && localBeamSize > 1) + // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] - if(first) + // perform NN computation + if(t == 0) graph->forward(); else graph->forwardNext(); //********************************************************************** // suppress specific symbols if not at right positions - if(trgUnkId_ != -1 && options_->has("allow-unk") - && !options_->get("allow-unk")) - suppressWord(pathScores, trgUnkId_); + if(trgUnkId_ != -1 && options_->has("allow-unk") && !options_->get("allow-unk")) + suppressWord(expandedPathScores, trgUnkId_); for(auto state : states) - state->blacklist(pathScores, batch); + state->blacklist(expandedPathScores, batch); //********************************************************************** - // perform beam search and pruning + // perform beam search // find N best amongst the (localBeamSize * dimVocab) hypotheses - const std::vector beamSizes(dimBatch, localBeamSize); - std::vector outKeys; - std::vector outPathScores; - getNBestList(beamSizes, pathScores->val(), outPathScores, outKeys, first); - // outPathScores and outKeys contain pathScores and their original indices in N-best order - - // convert N-best sets to updated search space - int dimTrgVoc = pathScores->shape()[-1]; - beams = toHyps(outKeys, - outPathScores, - dimTrgVoc, + std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened + std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened + getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? + /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] + /*out*/ nBestPathScores, /*out*/ nBestKeys, + /*first=*/t == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? + // Now, nBestPathScores contain N-best expandedPathScores, and nBestKeys for each their original location (batchIdx, beamHypIdx, word). + + // combine N-best sets with existing search space (beams) to updated search space + beams = toHyps(nBestKeys, nBestPathScores, + /*dimTrgVoc=*/expandedPathScores->shape()[-1], beams, - states, - localBeamSize, - first, + states, // used for keeping track of per-ensemble-member path score + localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples + /*first=*/t == 0, // used to indicate originating beamSize of 1 batch); // remove all hyps that end in EOS - auto purgedBeams = purgeBeams(beams); // @TODO: rename; this is not pruning + // The position of a hyp in the beam may change. + const auto purgedBeams = purgeBeams(beams); - // add updated search space to search grid for traceback + // add updated search space (beams) to search grid (histories) for traceback bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { // if this batch entry has surviving hyps then add them to the traceback grid @@ -318,20 +338,7 @@ class BeamSearch { // this is the search space for the next output time step beams = purgedBeams; - - // determine beam size for next output time step, as max over still-active sentences - // E.g. if all batch entries are down from beam 5 to no more than 4 surviving hyps, then - // switch to beam of 4 for all. If all are done, then beam ends up being 0, and we are done. - if(!first) { - size_t maxBeam = 0; - for(auto& beam : beams) - if(beam.size() > maxBeam) - maxBeam = beam.size(); - localBeamSize = maxBeam; - } - if (localBeamSize == 0) // done if all batch entries have reached EOS on all beam entries - break; - } // end of main loop over output tokens + } // end of main loop over output time steps return histories; // [dimBatch][t][N best hyps] } diff --git a/src/translator/history.h b/src/translator/history.h index dcf5e5854..ff37fb6e2 100755 --- a/src/translator/history.h +++ b/src/translator/history.h @@ -25,7 +25,7 @@ class History { History(size_t lineNo, float alpha = 1.f, float wp_ = 0.f); void add(const Beam& beam, Word trgEosId, bool last = false) { - if(beam.back()->GetPrevHyp() != nullptr) { + if(beam.back()->getPrevHyp() != nullptr) { for(size_t j = 0; j < beam.size(); ++j) if(beam[j]->getWord() == trgEosId || last) { float pathScore = @@ -52,7 +52,7 @@ class History { // std::cerr << "h: " << start << " " << j << " " << c << std::endl; // trace back best path - Words targetWords = bestHyp->TracebackWords(); + Words targetWords = bestHyp->tracebackWords(); // note: bestHyp->getPathScore() is not normalized, while bestHypCoord.normalizedPathScore is nbest.emplace_back(targetWords, bestHyp, bestHypCoord.normalizedPathScore); diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index ddb475143..452b3349b 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -17,11 +17,11 @@ class Hypothesis { Hypothesis(const Ptr prevHyp, Word word, - IndexType prevIndex, + IndexType prevIndex, // (beamHypIdx, batchIdx) flattened as beamHypIdx * dimBatch + batchIdx float pathScore) : prevHyp_(prevHyp), prevIndex_(prevIndex), word_(word), pathScore_(pathScore) {} - const Ptr GetPrevHyp() const { return prevHyp_; } + const Ptr getPrevHyp() const { return prevHyp_; } Word getWord() const { return word_; } @@ -29,16 +29,17 @@ class Hypothesis { float getPathScore() const { return pathScore_; } - std::vector& GetScoreBreakdown() { return scoreBreakdown_; } - std::vector& GetAlignment() { return alignment_; } + std::vector& getScoreBreakdown() { return scoreBreakdown_; } + void setScoreBreakdown(const std::vector& scoreBreaddown) { scoreBreakdown_ = scoreBreaddown; } - void SetAlignment(const std::vector& align) { alignment_ = align; }; + const std::vector& getAlignment() { return alignment_; } + void setAlignment(const std::vector& align) { alignment_ = align; }; // helpers to trace back paths referenced from this hypothesis - Words TracebackWords() + Words tracebackWords() { Words targetWords; - for (auto hyp = this; hyp->GetPrevHyp(); hyp = hyp->GetPrevHyp().get()) { + for (auto hyp = this; hyp->getPrevHyp(); hyp = hyp->getPrevHyp().get()) { targetWords.push_back(hyp->getWord()); // std::cerr << hyp->getWord() << " " << hyp << std::endl; } @@ -48,11 +49,11 @@ class Hypothesis { // get soft alignments for each target word starting from the hyp one typedef data::SoftAlignment SoftAlignment; - SoftAlignment TracebackAlignment() + SoftAlignment tracebackAlignment() { SoftAlignment align; - for (auto hyp = this; hyp->GetPrevHyp(); hyp = hyp->GetPrevHyp().get()) { - align.push_back(hyp->GetAlignment()); + for (auto hyp = this; hyp->getPrevHyp(); hyp = hyp->getPrevHyp().get()) { + align.push_back(hyp->getAlignment()); } std::reverse(align.begin(), align.end()); return align; @@ -64,12 +65,12 @@ class Hypothesis { const Word word_; const float pathScore_; - std::vector scoreBreakdown_; + std::vector scoreBreakdown_; // [num scorers] std::vector alignment_; }; -typedef std::vector> Beam; // Beam = vector of hypotheses -typedef std::vector Beams; // Beams = vector of vector of hypotheses +typedef std::vector> Beam; // Beam = vector [beamSize] of hypotheses +typedef std::vector Beams; // Beams = vector [batchDim] of vector [beamSize] of hypotheses typedef std::tuple, float> Result; // (word ids for hyp, hyp, normalized sentence score for hyp) typedef std::vector NBestList; // sorted vector of (word ids, hyp, sent score) tuples } // namespace marian diff --git a/src/translator/output_printer.cpp b/src/translator/output_printer.cpp old mode 100644 new mode 100755 index 000af7892..05167898d --- a/src/translator/output_printer.cpp +++ b/src/translator/output_printer.cpp @@ -6,9 +6,9 @@ std::string OutputPrinter::getAlignment(const Ptr& hyp) { data::SoftAlignment align; auto last = hyp; // get soft alignments for each target word starting from the last one - while(last->GetPrevHyp().get() != nullptr) { - align.push_back(last->GetAlignment()); - last = last->GetPrevHyp(); + while(last->getPrevHyp().get() != nullptr) { + align.push_back(last->getAlignment()); + last = last->getPrevHyp(); } // reverse alignments diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h index 38ed7b9c6..83ef9c0ca 100755 --- a/src/translator/output_printer.h +++ b/src/translator/output_printer.h @@ -41,11 +41,11 @@ class OutputPrinter { bestn << " ||| " << getAlignment(hypo); bestn << " |||"; - if(hypo->GetScoreBreakdown().empty()) { + if(hypo->getScoreBreakdown().empty()) { bestn << " F0=" << hypo->getPathScore(); } else { - for(size_t j = 0; j < hypo->GetScoreBreakdown().size(); ++j) { - bestn << " F" << j << "= " << hypo->GetScoreBreakdown()[j]; + for(size_t j = 0; j < hypo->getScoreBreakdown().size(); ++j) { + bestn << " F" << j << "= " << hypo->getScoreBreakdown()[j]; } } diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100644 new mode 100755 index 16a066b05..36591fe83 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -9,9 +9,9 @@ namespace marian { class ScorerState { public: - virtual Expr getLogProbs() = 0; + virtual Expr getLogProbs() const = 0; - virtual float breakDown(size_t i) { return getLogProbs()->val()->get(i); } + float breakDown(size_t i) const { return getLogProbs()->val()->get(i); } virtual void blacklist(Expr /*totalCosts*/, Ptr /*batch*/){}; }; @@ -57,7 +57,7 @@ class ScorerWrapperState : public ScorerState { virtual Ptr getState() { return state_; } - virtual Expr getLogProbs() override { return state_->getLogProbs(); }; + virtual Expr getLogProbs() const override { return state_->getLogProbs(); }; virtual void blacklist(Expr totalCosts, Ptr batch) override { state_->blacklist(totalCosts, batch); From 0462847998bb8039d3674c2c251a12fc0cc7cbb4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 7 Feb 2019 10:02:26 -0800 Subject: [PATCH 296/838] minor refactoring --- src/data/factored_vocab.cpp | 40 +++++++++++++++++++++++++++++ src/data/factored_vocab.h | 49 +++++++----------------------------- src/translator/beam_search.h | 15 ++++++++--- 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index b72db7196..21711422f 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -305,6 +305,46 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& wor return res; } +// WordLUT +WordIndex FactoredVocab::WordLUT::add(const std::string& word, WordIndex index) { + ABORT_IF(word.empty(), "Attempted to add the empty word to a dictionary"); + auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; + ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); + while (index2str_.size() <= index) + index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? + ABORT_IF(!index2str_[index].empty(), "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); + index2str_[index] = word; + return index; +} +const std::string& FactoredVocab::WordLUT::operator[](WordIndex index) const { + const auto& word = index2str_[index]; + ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); + return word; +} +WordIndex FactoredVocab::WordLUT::operator[](const std::string& word) const { + auto iter = str2index_.find(word); + ABORT_IF(iter == str2index_.end(), "Token '{}' not found in vocabulary", word); + return iter->second; +} +bool FactoredVocab::WordLUT::tryFind(const std::string& word, WordIndex& index) const { + auto iter = str2index_.find(word); + if (iter == str2index_.end()) + return false; + index = iter->second; + return true; +} +void FactoredVocab::WordLUT::resize(size_t num) { + ABORT_IF(num < index2str_.size(), "Word table cannot be shrunk"); + index2str_.resize(num); // gets filled up with gap items (empty strings) +} +size_t FactoredVocab::WordLUT::load(const std::string& path) { + std::string line; + io::InputFileStream in(path); + for (WordIndex v = 0; io::getline(in, line); v++) + add(line, v); + return size(); +} + // Note: This does not actually load it, only checks the path for the type. Ptr createFactoredVocab(const std::string& vocabPath) { bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index e2ae02785..ea37101b2 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -1,6 +1,7 @@ // Implementation of an IVocab that represents a factored representation. -// This is accessed via the IVocab interface, and also by Embedding and Output -// layers directly. +// This is accessed via the IVocab interface for the base vocab functionality, +// and via dynamic_cast to FactoredVocab for factored-specific things used by +// the Embedding and Output layers. #pragma once @@ -68,47 +69,15 @@ class FactoredVocab : public IVocab { std::map str2index_; std::vector index2str_; public: - WordIndex add(const std::string& word, WordIndex index) { - ABORT_IF(word.empty(), "Attempted to add the empty word to a dictionary"); - auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; - ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); - while (index2str_.size() <= index) - index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? - ABORT_IF(!index2str_[index].empty(), "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); - index2str_[index] = word; - return index; - } - const std::string& operator[](WordIndex index) const { - const auto& word = index2str_[index]; - ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); - return word; - } - WordIndex operator[](const std::string& word) const { - auto iter = str2index_.find(word); - ABORT_IF(iter == str2index_.end(), "Token '{}' not found in vocabulary", word); - return iter->second; - } + WordIndex add(const std::string& word, WordIndex index); + const std::string& operator[](WordIndex index) const; + WordIndex operator[](const std::string& word) const; bool isGap(WordIndex index) const { return index2str_[index].empty(); } - bool tryFind(const std::string& word, WordIndex& index) const { - auto iter = str2index_.find(word); - if (iter == str2index_.end()) - return false; - index = iter->second; - return true; - } - void resize(size_t num) { - ABORT_IF(num < index2str_.size(), "Word table cannot be shrunk"); - index2str_.resize(num); // gets filled up with gap items (empty strings) - } + bool tryFind(const std::string& word, WordIndex& index) const; + void resize(size_t num); size_t size() const { return index2str_.size(); } // nominal size including gap items size_t numValid() const { return str2index_.size(); } // actual non-gaps items - size_t load(const std::string& path) { - std::string line; - io::InputFileStream in(path); - for (WordIndex v = 0; io::getline(in, line); v++) - add(line, v); - return size(); - } + size_t load(const std::string& path); }; // main vocab diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 0520ed0cd..df38a7e61 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -4,6 +4,7 @@ #include "marian.h" #include "translator/history.h" #include "translator/scorers.h" +#include "data/factored_vocab.h" #include "translator/helpers.h" #include "translator/nth_element.h" @@ -68,7 +69,7 @@ class BeamSearch { const auto keyBatchIdx = hypIdx / (first ? 1 : beamSize); const auto keyBeamHypIdx = hypIdx % (first ? 1 : beamSize); const auto hypIdxTrans = keyBeamHypIdx * dimBatch + keyBatchIdx; - ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); // not possible, as beamSize = max(beams[.].size()) + ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); // @TODO: is this possible? Should be, but does not seem to trigger. #else const auto keyBatchIdx = hypIdx / beamSize; // @REVIEW: is this actually keyBatchIdx? size_t keyBeamHypIdx = hypIdx % beamSize; @@ -92,7 +93,7 @@ class BeamSearch { wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? // now wordIdx is a regular Word again - auto hyp = New(beam[keyBeamHypIdx], wordIdx, (IndexType)hypIdxTrans, pathScore); + auto hyp = New(beam[keyBeamHypIdx], Word::fromWordIndex(wordIdx), (IndexType)hypIdxTrans, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { @@ -170,6 +171,12 @@ class BeamSearch { //********************************************************************** // main decoding function Histories search(Ptr graph, Ptr batch) { + // @TODO: EOS id does not need to be stored in this object, since it is available from vocab() + ABORT_IF(batch->back()->vocab()->getEosId() == trgEosId_); + + auto factoredVocab = std::dynamic_pointer_cast(batch->back()->vocab()); + size_t numFactors = factoredVocab ? factoredVocab->getNumGroups() : 1; + const int dimBatch = (int)batch->size(); auto getNBestList = createGetNBestListFn(beamSize_, dimBatch, graph->getDeviceId()); @@ -281,8 +288,8 @@ class BeamSearch { } // make beams continuous - expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] - //if(dimBatch > 1 && localBeamSize > 1) + if(dimBatch > 1 && localBeamSize > 1) + expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] // perform NN computation From 15158deef0ae34ac4347fde416a545f049178792 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 7 Feb 2019 18:16:52 +0000 Subject: [PATCH 297/838] Add comments and refactorize --- src/common/cli_wrapper.cpp | 83 +++++++++++++++++++++--------------- src/common/cli_wrapper.h | 40 +++++++++++------ src/common/config_parser.cpp | 7 ++- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 1bf822196..161a61d8f 100644 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -129,37 +129,50 @@ void CLIWrapper::parse(int argc, char **argv) { } void CLIWrapper::parseAliases() { + // Exit if no aliases defined if(aliases_.empty()) return; - std::set aliasKeys; - for(const auto& alias : aliases_) { + // Names of all triggered/expanded aliases will be collected and removed from the config at the + // end. We keep them in a set as multiple aliases can be defined for one option name (aka `key`). + std::set parsedAliases; + + // Iterate all known aliases, each alias has a key, value, and config + for(const auto &alias : aliases_) { + // Check if the alias option exists in the config (it may come from command line or a config + // file) if(config_[alias.key]) { + // Check if the option in the config stores the value required to expand the alias. If so, + // expand the alias. + // Two cases: + // * the option is a sequence: extract it as a vector of strings and look for the value + // * otherwise: compare values as strings bool expand = false; if(config_[alias.key].IsSequence()) { - // Note: options values are always extracted as vectors and compared as strings auto aliasOpts = config_[alias.key].as>(); expand = std::find(aliasOpts.begin(), aliasOpts.end(), alias.value) != aliasOpts.end(); } else { - // Note: options values are always compared as strings expand = config_[alias.key].as() == alias.value; } if(expand) { + // Update global config options with the config associated with the alias. Abort if the + // alias contains an undefined option updateConfig(alias.config, "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); } - aliasKeys.insert(alias.key); + + // Collect the alias option as they will be removed at the end + parsedAliases.insert(alias.key); } } - // Remove aliases from the config - for(const auto& key : aliasKeys) { + // Remove aliases from the global config to avoid redundancy when dumping/reading config files + for(const auto &key : parsedAliases) { config_.remove(key); } } - std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) { std::string header = "Error: " + std::string(e.what()) + "\n"; if(app->get_help_ptr() != nullptr) @@ -167,48 +180,50 @@ std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) return header; } -void CLIWrapper::updateConfig(const YAML::Node &config, const std::string& errorMsg) { - std::vector invalidKeys; +void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &errorMsg) { auto cmdOptions = getParsedOptionNames(); + // Keep track of unrecognized options from the provided config + std::vector unknownOpts; + // Iterate incoming options: they need to be merged into the global config for(auto it : config) { auto key = it.first.as(); - // skip options specified via command-line to allow overwriting them + + // Skip options specified via command-line to allow overwriting them if(cmdOptions.count(key)) continue; - // skip options that might exist in config files generated by older versions of Marian + // Skip options that might exist in config files generated by older versions of Marian if(DEPRECIATED_OPTIONS.count(key)) continue; + // Check if an incoming option has been defined in CLI if(options_.count(key)) { - if(config_[key]) { // it exists, so this is a default value, hence it has a node type - if(config_[key].Type() != it.second.Type()) { // types don't match, handle this - // default value is a sequence and incoming node is a scalar, hence we can upcast to - // single element sequence - if(config_[key].Type() == YAML::NodeType::Sequence - && it.second.Type() == YAML::NodeType::Scalar) { - // create single element sequence - YAML::Node sequence; - sequence.push_back(YAML::Clone(it.second)); - config_[key] = sequence; // overwrite to replace default values - options_[key].modified = true; - } else { // Cannot convert other non-matching types, e.g. scalar <- list should fail - ABORT("Cannot convert values for the option: " + key); - } - } else { // types match, go ahead - config_[key] = YAML::Clone(it.second); - options_[key].modified = true; - } - } else { + // Check if the option exists in the global config and types match + if(config_[key] && config_[key].Type() == it.second.Type()) { config_[key] = YAML::Clone(it.second); options_[key].modified = true; + // If types doesn't match, try to convert + } else { + // Default value is a sequence and incoming node is a scalar, hence we can upcast to + // single element sequence + if(config_[key].Type() == YAML::NodeType::Sequence + && it.second.Type() == YAML::NodeType::Scalar) { + // create single element sequence + YAML::Node sequence; + sequence.push_back(YAML::Clone(it.second)); + config_[key] = sequence; // overwrite to replace default values + options_[key].modified = true; + } else { + // Cannot convert other non-matching types, e.g. scalar <- list should fail + ABORT("Cannot convert values for the option: " + key); + } } - } else { - invalidKeys.push_back(key); + } else { // an unknown option + unknownOpts.push_back(key); } } - ABORT_IF(!invalidKeys.empty(), errorMsg + ": " + utils::join(invalidKeys, ", ")); + ABORT_IF(!unknownOpts.empty(), errorMsg + ": " + utils::join(unknownOpts, ", ")); } std::string CLIWrapper::dumpConfig(bool skipDefault /*= false*/) const { diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 95eb3590d..f0ca51c34 100644 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -51,7 +51,7 @@ class CLIFormatter : public CLI::Formatter { // @TODO: in this file review the use of naked pointers. We use Ptr anywhere else, // what's up with that? -// The helper structure storing an option object, the associated variable and creation index +// Helper structure storing an option object, the associated variable and creation index struct CLIOptionTuple { CLI::Option *opt; Ptr var; @@ -59,11 +59,12 @@ struct CLIOptionTuple { bool modified{false}; }; -// Helper structure used for aliases and storing an option key, value, and YAML node +// Helper structure used for aliases storing an option key, value, and options to be expanded in the +// form of a YAML config struct CLIAliasTuple { - std::string key; - std::string value; - YAML::Node config; + std::string key; // alias option name + std::string value; // value for the alias option indicating that it should be expanded + YAML::Node config; // config with options that the alias adds }; /** @@ -200,13 +201,24 @@ class CLIWrapper { } /** - * @brief Define an alias that is a shortcut for a set of options + * @brief Transform the option into an alias that is a shortcut for a set of options * - * Option values are compared as std::string. + * An alias sets one or more options to predefined values. The options expanded by the alias are + * provided as a function setting a temporary YAML config. * - * @param key Option name + * The alias option has to be first defined using `add()`. Otherwise, the program will abort. + * + * Defining more than one alias for the same `key` but different `value` is allowed. + * + * Option values are compared as std::string. If the alias option is a vector, the alias will be + * triggered if `value` exists in that vector at least once. + * + * Options set directly via command line have precedence over options defined in an alias, i.e. an + * option added via alias can be overwritten by setting a specific option via command line. + * + * @param key Alias option name * @param value Option value that trigger the alias - * @param fun Function initializing options + * @param fun Function setting a temporary YAML config with options expanded by alias */ void alias(const std::string &key, const std::string &value, @@ -231,7 +243,8 @@ class CLIWrapper { /** * @brief Expand aliases based on arguments parsed with parse(int, char**) * - * Should be called after parse(int, char**) to take an effect. + * Should be called after parse(int, char**) to take an effect. If any alias tries to expand an + * undefined option, the method will abort. * * All options defined as aliases are removed from the config object. */ @@ -240,10 +253,9 @@ class CLIWrapper { /* * @brief Overwrite values for unparsed options * - * Default values are overwritten with the options from the config provided, while parsed - * command-line options remain unchanged. - * This should be a preferred way of updating config options as the class keeps track of options, - * which values have changed. + * Default values are overwritten with the options from the provided config, while parsed + * command-line options remain unchanged. This should be a preferred way of updating config + * options as the class keeps track of options, which values have changed. * * @param config YAML config with new default values for options * @param errorMsg error message printed if config contains undefined keys. The message is diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index b50dbcd02..cd8cc2db9 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -694,8 +694,9 @@ void ConfigParser::addSuboptionsULR(cli::CLIWrapper& cli) { } void ConfigParser::addAliases(cli::CLIWrapper& cli) { - // The order of aliases does matter as later options overwrite earlier + // The order of aliases does matter as later options in the command line overwrite earlier + // Options reconstructing the BiDeep architecture proposed in http://www.aclweb.org/anthology/W17-4710 cli.alias("best-deep", "true", [](YAML::Node& config) { config["layer-normalization"] = true; config["tied-embeddings"] = true; @@ -708,6 +709,8 @@ void ConfigParser::addAliases(cli::CLIWrapper& cli) { config["skip"] = true; }); + // Architecture and proposed training settings for a Transformer BASE model introduced in + // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf cli.alias("task", "transformer", [](YAML::Node& config) { config["type"] = "transformer"; config["enc-depth"] = 6; @@ -722,6 +725,8 @@ void ConfigParser::addAliases(cli::CLIWrapper& cli) { config["clip-norm"] = 5; }); + // Architecture and proposed training settings for a Transformer BIG model introduced in + // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf cli.alias("task", "transformer-big", [](YAML::Node& config) { config["type"] = "transformer"; config["enc-depth"] = 6; From 86189dd66da78e54c10fb57295c20f75ef321fc5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 7 Feb 2019 10:54:27 -0800 Subject: [PATCH 298/838] new option --english-title-case-every --- src/common/config_parser.cpp | 2 ++ src/common/utils.cpp | 30 ++++++++++++++++++++++++++++++ src/common/utils.h | 2 ++ src/data/corpus.cpp | 21 +++++++++++++++++++-- src/data/corpus.h | 1 + src/data/vocab.h | 4 ++++ src/translator/beam_search.h | 5 +++-- 7 files changed, 61 insertions(+), 4 deletions(-) mode change 100644 => 100755 src/common/config_parser.cpp mode change 100644 => 100755 src/common/utils.h mode change 100644 => 100755 src/data/corpus.cpp mode change 100644 => 100755 src/data/corpus.h mode change 100644 => 100755 src/data/vocab.h diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100644 new mode 100755 index d346e94b4..54c1f6a66 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -632,6 +632,8 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { "Keep shuffled corpus in RAM, do not write to temp file"); cli.add("--all-caps-every", "When forming minibatches, preprocess every Nth line on the fly to all-caps. Assumes UTF-8"); + cli.add("--english-title-case-every", + "When forming minibatches, preprocess every Nth line on the fly to title-case. Assumes English (ASCII only)"); cli.add("--mini-batch-words-ref", "If given, the following hyper parameters are adjusted as-if we had this mini-batch size: " diff --git a/src/common/utils.cpp b/src/common/utils.cpp index cd10be230..e12022362 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef __unix__ #include #endif @@ -202,6 +203,35 @@ std::string utf8ToUpper(const std::string& s) { } #endif +// convert an English sentence to title case +// Since title case is an English thing, we only consider ASCII characters. +std::string toEnglishTitleCase(const std::string& s) { + auto res = s; + // process token by token + const std::string wordStartChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const std::string wordInternalChars = wordStartChars + "'"; // don't title-case letters after word-internal apostrophe + const std::set exceptions = { // from moses-scripts/scripts/recaser/detruecase.perl + "a","after","against","al-.+","and","any","as","at","be","because","between","by","during","el-.+","for","from","his","in","is","its","last","not","of","off","on","than","the","their","this","to","was","were","which","will","with" + }; + // These are tokenization heuristics, which may be incomplete. + size_t epos = 0; + for(size_t pos = epos; pos < res.size(); pos = epos) { + // locate the next word + pos = res.find_first_of(wordStartChars, pos); // find first letter + if (pos == std::string::npos) + break; + epos = res.find_first_not_of(wordInternalChars, pos + 1); // find first non-letter + if (epos == std::string::npos) + epos = res.size(); + auto word = res.substr(pos, epos - pos); + // upper-case the word, unless it is in the exception list + if (res[pos] >= 'a' && res[pos] <= 'z' && exceptions.find(word) == exceptions.end()) { + res[pos] -= 'A' - 'a'; + } + } + return res; +} + double parseDouble(std::string s) { double res; char c; // dummy char -- if we succeed to parse this, then there were extraneous characters after the number diff --git a/src/common/utils.h b/src/common/utils.h old mode 100644 new mode 100755 index ffd9454c3..23d6ffed4 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -39,6 +39,8 @@ bool beginsWith(const std::string& text, const std::string& prefix); bool endsWith(const std::string& text, const std::string& suffix); std::string utf8ToUpper(const std::string& s); +std::string toEnglishTitleCase(const std::string& s); + double parseDouble(std::string s); double parseNumber(std::string s); diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp old mode 100644 new mode 100755 index 53a50614b..7e8a5aa0f --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -10,12 +10,18 @@ namespace marian { namespace data { Corpus::Corpus(Ptr options, bool translate /*= false*/) - : CorpusBase(options, translate), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} + : CorpusBase(options, translate), + shuffleInRAM_(options_->get("shuffle-in-ram")), + allCapsEvery_(options_->get("all-caps-every")), + titleCaseEvery_(options_->get("english-title-case-every")) {} Corpus::Corpus(std::vector paths, std::vector> vocabs, Ptr options) - : CorpusBase(paths, vocabs, options), shuffleInRAM_(options_->get("shuffle-in-ram")), allCapsEvery_(options_->get("all-caps-every")) {} + : CorpusBase(paths, vocabs, options), + shuffleInRAM_(options_->get("shuffle-in-ram")), + allCapsEvery_(options_->get("all-caps-every")), + titleCaseEvery_(options_->get("english-title-case-every")) {} void Corpus::preprocessLine(std::string& line, size_t streamId) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0 && !inference_) { @@ -25,6 +31,17 @@ void Corpus::preprocessLine(std::string& line, size_t streamId) { else LOG_ONCE(info, "[data] target all-caps'ed line to {}", line); } + else if (titleCaseEvery_ != 0 && pos_ % titleCaseEvery_ == 1 && !inference_ + + && streamId == 0 // @HACK: Hard-coding EN-X direction for now; needs an option in the future + + ) { + line = utils::toEnglishTitleCase(line); + if (streamId == 0) + LOG_ONCE(info, "[data] source English-title-case'd line to {}", line); + else + LOG_ONCE(info, "[data] target English-title-case'd line to {}", line); + } } SentenceTuple Corpus::next() { diff --git a/src/data/corpus.h b/src/data/corpus.h old mode 100644 new mode 100755 index 7edced620..c5af7220b --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -29,6 +29,7 @@ class Corpus : public CorpusBase { // for pre-processing size_t allCapsEvery_{0}; + size_t titleCaseEvery_{0}; void preprocessLine(std::string& line, size_t streamId); public: diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100644 new mode 100755 index afe79d1f0..dd746ba9a --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -68,6 +68,10 @@ class Vocab { // create fake vocabulary for collecting batch statistics void createFake(); + + // give access to base implementation. Returns null if not the requested type. + template // e.g. FactoredVocab + Ptr tryAs() const { return std::dynamic_pointer_cast(vImpl_); } }; } // namespace marian diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index df38a7e61..1dc05d193 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -172,10 +172,11 @@ class BeamSearch { // main decoding function Histories search(Ptr graph, Ptr batch) { // @TODO: EOS id does not need to be stored in this object, since it is available from vocab() - ABORT_IF(batch->back()->vocab()->getEosId() == trgEosId_); + ABORT_IF(batch->back()->vocab()->getEosId() != trgEosId_, "Batch uses different EOS token than was passed to BeamSearch originally"); - auto factoredVocab = std::dynamic_pointer_cast(batch->back()->vocab()); + auto factoredVocab = batch->back()->vocab()->tryAs(); size_t numFactors = factoredVocab ? factoredVocab->getNumGroups() : 1; + numFactors; const int dimBatch = (int)batch->size(); From 16a35b12eacda00a0055a93eb6d1696e71b1b744 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 7 Feb 2019 13:44:33 -0800 Subject: [PATCH 299/838] bug fix: change to has(embedding-vectors option) check regressed in last merge; some fixes to toEnglishTitleCase() --- src/common/utils.cpp | 14 ++++++++++---- src/data/corpus.cpp | 8 ++++---- src/models/decoder.h | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index e12022362..e4f6aa904 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -213,6 +213,7 @@ std::string toEnglishTitleCase(const std::string& s) { const std::set exceptions = { // from moses-scripts/scripts/recaser/detruecase.perl "a","after","against","al-.+","and","any","as","at","be","because","between","by","during","el-.+","for","from","his","in","is","its","last","not","of","off","on","than","the","their","this","to","was","were","which","will","with" }; + const std::set wordPredChars = {' ', '"', '\'', '-'}; // only capitalize words if following these characters (to avoid upper-casing word-internal SPM units) // These are tokenization heuristics, which may be incomplete. size_t epos = 0; for(size_t pos = epos; pos < res.size(); pos = epos) { @@ -224,10 +225,15 @@ std::string toEnglishTitleCase(const std::string& s) { if (epos == std::string::npos) epos = res.size(); auto word = res.substr(pos, epos - pos); - // upper-case the word, unless it is in the exception list - if (res[pos] >= 'a' && res[pos] <= 'z' && exceptions.find(word) == exceptions.end()) { - res[pos] -= 'A' - 'a'; - } + // further checks of the word + if (res[pos] < 'a' || res[pos] > 'z') // skip if already upper-case + continue; + if (pos > 0 && wordPredChars.find(res[pos-1]) == wordPredChars.end()) // skip if unexpected char before the word + continue; + if (exceptions.find(word) != exceptions.end()) // skip if in the exception list + continue; + // upper-case it + res[pos] -= 'a' - 'A'; } return res; } diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 7e8a5aa0f..44fa064d8 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -27,9 +27,9 @@ void Corpus::preprocessLine(std::string& line, size_t streamId) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0 && !inference_) { line = utils::utf8ToUpper(line); if (streamId == 0) - LOG_ONCE(info, "[data] source all-caps'ed line to {}", line); + LOG_ONCE(info, "[data] Source all-caps'ed line to: {}", line); else - LOG_ONCE(info, "[data] target all-caps'ed line to {}", line); + LOG_ONCE(info, "[data] Target all-caps'ed line to: {}", line); } else if (titleCaseEvery_ != 0 && pos_ % titleCaseEvery_ == 1 && !inference_ @@ -38,9 +38,9 @@ void Corpus::preprocessLine(std::string& line, size_t streamId) { ) { line = utils::toEnglishTitleCase(line); if (streamId == 0) - LOG_ONCE(info, "[data] source English-title-case'd line to {}", line); + LOG_ONCE(info, "[data] Source English-title-case'd line to: {}", line); else - LOG_ONCE(info, "[data] target English-title-case'd line to {}", line); + LOG_ONCE(info, "[data] Target English-title-case'd line to: {}", line); } } diff --git a/src/models/decoder.h b/src/models/decoder.h index 8f3db3d72..9258cf541 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -49,7 +49,7 @@ class DecoderBase { embFactory("prefix", prefix_ + "_Wemb"); if(options_->has("embedding-fix-trg")) embFactory("fixed", opt("embedding-fix-trg")); - if(options_->has("embedding-vectors")) { + if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory("embFile", embFiles[batchIndex_]) // ("normalization", opt("embedding-normalization")); From e994eeca5523ed85a484ffbaf4cef0620adacbf9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 7 Feb 2019 16:40:18 -0800 Subject: [PATCH 300/838] reimplemented Logits::applyLossFunction() to directly use factored labels --- src/data/factored_vocab.h | 1 + src/layers/generic.cpp | 65 +++++++++++++++++++++++++++++++-------- src/layers/generic.h | 21 ++++++++++++- 3 files changed, 74 insertions(+), 13 deletions(-) mode change 100644 => 100755 src/layers/generic.h diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index ea37101b2..762e21d52 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -56,6 +56,7 @@ class FactoredVocab : public IVocab { std::pair getFactorUnit(Word word, size_t groupIndex); static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); static constexpr size_t FACTOR_NOT_SPECIFIED = (SIZE_MAX - 2); + static bool isFactorValid(size_t factorIndex) { return factorIndex < FACTOR_NOT_SPECIFIED; } static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab private: diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index c2aab49a9..18f095875 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -16,35 +16,54 @@ namespace marian { Logits::Logits(Expr logits) : Logits(New(logits, nullptr)) {} // single-output constructor from Expr only (RationalLoss has no count) + Ptr Logits::graph() const { + ABORT_IF(logits_.empty(), "Empty logits object??"); + return logits_.front()->loss()->graph(); + } + // This function assumes that the object holds one or more factor logits. // It applies the supplied loss function to each, and then returns the aggregate loss over all factors. Expr Logits::applyLossFunction(const Words& labels, const std::function& lossFn) const { - ABORT_IF(logits_.empty(), "Empty logits object??"); - auto graph = logits_.front()->loss()->graph(); - Expr indices = graph->indices(toWordIndexVector(labels)); - LOG_ONCE(info, "[logits] applyLossFunction() for {} factors", logits_.size()); ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + + auto firstLogits = logits_.front()->loss(); + ABORT_IF(labels.size() * firstLogits->shape()[-1] != firstLogits->shape().elements(), "Labels not matching logits shape??"); + + // base case (no factors) if (!factoredVocab_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); - return lossFn(logits_.front()->loss(), indices); + return lossFn(firstLogits, indices(toWordIndexVector(labels))); } + auto numGroups = factoredVocab_->getNumGroups(); + + // split labels into individual factor labels + // @TODO: Is there value in having factorize() return Exprs? That would allow to move some stuff to GPU. + auto allMaskedFactoredLabels = factorize(labels); // [numGroups][labels.size()] = [numGroups][B... flattened] + + //Expr indices = this->indices(toWordIndexVector(labels)); // accumulate all CEs for all words that have the factor // Memory-wise, this is cheap, all temp objects below are batches of scalars or lookup vectors. Expr loss; - auto numGroups = factoredVocab_->getNumGroups(); for (size_t g = 0; g < numGroups; g++) { + const auto& maskedFactoredLabels = allMaskedFactoredLabels[g]; // array of (word index, mask) +#if 1 + auto factorIndices = indices (maskedFactoredLabels.indices); // [B... flattened] factor-label indices, or 0 if factor does not apply + auto factorMask = constant(maskedFactoredLabels.masks); // [B... flattened] loss values get multiplied with 0 for labels that don't have this factor +#else // @TODO: if this^^ works, we can remove the stuff below (quite a bit code) + maskedFactoredLabels; indices; // [B... * 1] all batch items flattened auto factorMaskVector = factoredVocab_->getFactorMasks(g); // [v] 1.0 if v has factor of group g auto factorIndexVector = factoredVocab_->getFactorIndices(g); // [v] index of factor for word v in group p; must be 0 if factor is not used - auto factorMaskMatrix = graph->constant({(int)factorMaskVector.size(), 1}, inits::from_vector(factorMaskVector), Type::float32); // [V x 1] - auto factorIndexMatrix = graph->constant({(int)factorIndexVector.size(), 1}, inits::from_vector(factorIndexVector), Type::uint32); // [V x 1(Ug)] - auto factorIndex = rows(factorIndexMatrix, indices); // [B... * 1(Ug)] map word indices to factor indices (indices into factorLogits) - auto factorMask = rows(factorMaskMatrix, indices); // [B... * 1] flag whether word has the factor in the first place - auto factorLogits = logits_[g]; // [B... * Ug] + auto factorMaskMatrix = constant({(int)factorMaskVector.size(), 1}, factorMaskVector); // [V x 1] + auto factorIndexMatrix = constant({(int)factorIndexVector.size(), 1}, factorIndexVector); // [V x 1(Ug)] + auto factorIndices = rows(factorIndexMatrix, indices); // [B... * 1(Ug)] map word indices to factor indices (indices into factorLogits) + auto factorMask = rows(factorMaskMatrix, indices); // [B... * 1] flag whether word has the factor in the first place +#endif + auto factorLogits = logits_[g]; // [B... * Ug] label-wise loss values (not aggregated yet) // For each location in [B...] select [indices[B...]]. If not using factor, select [0] and mask it out next. - auto factorLoss = lossFn(factorLogits->loss(), factorIndex); // [B... x 1] + auto factorLoss = lossFn(factorLogits->loss(), factorIndices); // [B... x 1] factorLoss = factorLoss * reshape(factorMask, factorLoss->shape()); // mask out factor for words that do not have that factor loss = loss ? (loss + factorLoss) : factorLoss; // [B... x 1] } @@ -91,6 +110,28 @@ namespace marian { return y; } + void Logits::MaskedFactorIndices::push_back(size_t factorIndex) { + bool isValid = FactoredVocab::isFactorValid(factorIndex); + indices.push_back(isValid ? (WordIndex)factorIndex : 0); + masks.push_back((float)isValid); + } + + std::vector Logits::factorize(const Words& words) const { // [numGroups][words.size()] -> breaks encoded Word into individual factor indices + if (!factoredVocab_) { + ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); + return {MaskedFactorIndices(words)}; + } + auto numGroups = factoredVocab_->getNumGroups(); + std::vector res(numGroups); + for (size_t g = 0; g < numGroups; g++) { + auto& resg = res[g]; + resg.reserve(words.size()); + for (const auto& word : words) + resg.push_back(factoredVocab_->getFactor(word, g)); + } + return res; + } + Logits Logits::withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ std::vector> newLogits; for (const auto& l : logits_) diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100644 new mode 100755 index 35183213e..2958162f2 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -76,8 +76,19 @@ class Logits { Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; + + struct MaskedFactorIndices { + std::vector indices; // factor index, or 0 if masked + std::vector masks; + void reserve(size_t n) { indices.reserve(n); masks.reserve(n); } + void push_back(size_t factorIndex); // push back into both arrays, setting mask and index to 0 for invalid entries + MaskedFactorIndices() {} + MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case + }; + std::vector factorize(const Words& words) const; // breaks encoded Word into individual factor indices float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // @TODO: avoid the fully expanded logits - void assign(const Logits& other) { + + void assign(const Logits& other) { // @TODO: we can remove this //ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), // "Logits assignment cannot change number of factors"); *this = other; @@ -86,6 +97,14 @@ class Logits { bool empty() const { return logits_.empty(); } Logits withCounts(const Expr& count) const; // create new Logits with 'count' implanted into all logits_ private: + // helper functions + Ptr graph() const; + Expr constant(const Shape& shape, const std::vector& data) const { return graph()->constant(shape, inits::from_vector(data), Type::float32); } + Expr constant(const Shape& shape, const std::vector& data) const { return graph()->constant(shape, inits::from_vector(data), Type::uint32); } + template Expr constant(const std::vector& data) const { return constant(Shape{(int)data.size()}, data); } // same as constant() but assuming vector + Expr indices(const std::vector& data) const { return graph()->indices(data); } // actually the same as constant(data) for this data type +private: + // members // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. std::vector> logits_; Ptr factoredVocab_; From 518f3c33c6db779078f96da89563737ff7032b0b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 7 Feb 2019 16:48:57 -0800 Subject: [PATCH 301/838] removed getFactorIndices() and getFactorMasks(), since they are no longer needed after last change --- src/data/factored_vocab.cpp | 10 +++++----- src/data/factored_vocab.h | 8 ++++---- src/layers/generic.cpp | 5 ++--- src/layers/generic.h | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 21711422f..153741e35 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -206,17 +206,17 @@ std::pair FactoredVocab::getFactorUnit(Word word, size_t groupI void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs - size_t numGroups = groupPrefixes_.size(); + //size_t numGroups = groupPrefixes_.size(); size_t vocabSize = vocab_.size(); - factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g - factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) + //factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g + //factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) gapLogMask_.resize(vocabSize, -1e8f); for (WordIndex v = 0; v < vocabSize; v++) { for (auto u : factorMap_[v]) { auto g = factorGroups_[u]; // convert u to relative u within factor group range ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); - factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); - factorMasks_[g][v] = 1.0f; + //factorIndices_[g][v] = (IndexType)(u - groupRanges_[g].first); + //factorMasks_[g][v] = 1.0f; gapLogMask_[v] = 0.0f; // valid entry } } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 762e21d52..de2bd34f1 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -45,8 +45,8 @@ class FactoredVocab : public IVocab { const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v --only used in getLogits() size_t getNumGroups() const { return groupRanges_.size(); } std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) - const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g - const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor + //const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g + //const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 // convert representations @@ -96,8 +96,8 @@ class FactoredVocab : public IVocab { std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. Shape factorShape_; // [g] number of factors in each factor group std::vector factorStrides_; // [g] stride for factor dimension - std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g - std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) + //std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g + //std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) std::vector gapLogMask_; // [v] -1e8 if this is a gap, else 0 }; diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 18f095875..80398ee08 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -39,8 +39,7 @@ namespace marian { auto numGroups = factoredVocab_->getNumGroups(); // split labels into individual factor labels - // @TODO: Is there value in having factorize() return Exprs? That would allow to move some stuff to GPU. - auto allMaskedFactoredLabels = factorize(labels); // [numGroups][labels.size()] = [numGroups][B... flattened] + auto allMaskedFactoredLabels = factorizeWords(labels); // [numGroups][labels.size()] = [numGroups][B... flattened] //Expr indices = this->indices(toWordIndexVector(labels)); // accumulate all CEs for all words that have the factor @@ -116,7 +115,7 @@ namespace marian { masks.push_back((float)isValid); } - std::vector Logits::factorize(const Words& words) const { // [numGroups][words.size()] -> breaks encoded Word into individual factor indices + std::vector Logits::factorizeWords(const Words& words) const { // [numGroups][words.size()] -> breaks encoded Word into individual factor indices if (!factoredVocab_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); return {MaskedFactorIndices(words)}; diff --git a/src/layers/generic.h b/src/layers/generic.h index 2958162f2..e6fd5c30b 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -85,7 +85,7 @@ class Logits { MaskedFactorIndices() {} MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case }; - std::vector factorize(const Words& words) const; // breaks encoded Word into individual factor indices + std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // @TODO: avoid the fully expanded logits void assign(const Logits& other) { // @TODO: we can remove this From 5d46e103cd11a20ab060ad52ac793d75cc2cf34c Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 14:24:21 +0000 Subject: [PATCH 302/838] Add and clean comments --- src/common/cli_wrapper.cpp | 25 ++++---- src/common/cli_wrapper.h | 118 ++++++++++++++----------------------- 2 files changed, 59 insertions(+), 84 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 161a61d8f..863dc02ad 100644 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -167,19 +167,12 @@ void CLIWrapper::parseAliases() { } } - // Remove aliases from the global config to avoid redundancy when dumping/reading config files + // Remove aliases from the global config to avoid redundancy when writing/reading config files for(const auto &key : parsedAliases) { config_.remove(key); } } -std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) { - std::string header = "Error: " + std::string(e.what()) + "\n"; - if(app->get_help_ptr() != nullptr) - header += "Run with " + app->get_help_ptr()->get_name() + " for more information.\n"; - return header; -} - void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &errorMsg) { auto cmdOptions = getParsedOptionNames(); // Keep track of unrecognized options from the provided config @@ -226,18 +219,21 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &error ABORT_IF(!unknownOpts.empty(), errorMsg + ": " + utils::join(unknownOpts, ", ")); } -std::string CLIWrapper::dumpConfig(bool skipDefault /*= false*/) const { +std::string CLIWrapper::dumpConfig(bool skipUnmodified /*= false*/) const { YAML::Emitter out; out << YAML::Comment("Marian configuration file generated at " + timer::currentDate() + " with version " + buildVersion()); out << YAML::BeginMap; std::string comment; + // Iterate option names in the same order as they have been created for(const auto &key : getOrderedOptionNames()) { - // do not proceed keys that are removed from config_ + // Do not dump options that were removed from config_ if(!config_[key]) continue; - if(skipDefault && !options_.at(key).modified) + // Do not dump options that were not passed via the command line + if(skipUnmodified && !options_.at(key).modified) continue; + // Put the group name as a comment before the first option in the group auto group = options_.at(key).opt->get_group(); if(comment != group) { if(!comment.empty()) @@ -274,5 +270,12 @@ std::vector CLIWrapper::getOrderedOptionNames() const { return keys; } +std::string CLIWrapper::failureMessage(const CLI::App *app, const CLI::Error &e) { + std::string header = "Error: " + std::string(e.what()) + "\n"; + if(app->get_help_ptr() != nullptr) + header += "Run with " + app->get_help_ptr()->get_name() + " for more information.\n"; + return header; +} + } // namespace cli } // namespace marian diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index f0ca51c34..851abfe8d 100644 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -16,29 +16,7 @@ namespace marian { class Options; namespace cli { - -// Try to determine the width of the terminal -// -// TODO: make use of it in the current CLI or remove. This is an old code used -// for boost::program_options and might not be needed anymore. -//static uint16_t guess_terminal_width(uint16_t max_width = 0, -// uint16_t default_width = 180); - -// TODO: use validators in ConfigParser -namespace validators { -const CLI::detail::ExistingFileValidator file_exists; -const CLI::detail::ExistingDirectoryValidator dir_exists; -const CLI::detail::ExistingPathValidator path_exists; - -const CLI::detail::NonexistentPathValidator path_not_exists; - -typedef CLI::Range range; -} - -/** - * The helper class for cli::CLIWrapper handling formatting of options and their - * descriptions. - */ +// The helper class for cli::CLIWrapper handling formatting of options and their descriptions. class CLIFormatter : public CLI::Formatter { public: CLIFormatter(size_t columnWidth, size_t screenWidth); @@ -48,65 +26,61 @@ class CLIFormatter : public CLI::Formatter { size_t screenWidth_{0}; }; -// @TODO: in this file review the use of naked pointers. We use Ptr anywhere else, -// what's up with that? -// Helper structure storing an option object, the associated variable and creation index +/** + * Helper tuple storing an option object, the associated variable and creation index + * + * Note: a bare pointer is used for the CLI::Option as this comes from the CLI11 library. Removing + * it would need deep modifications in the 3rd party library, that is not desirable. + */ struct CLIOptionTuple { - CLI::Option *opt; - Ptr var; - size_t idx{0}; - bool modified{false}; + CLI::Option *opt; // a pointer to an option object from CLI11 + Ptr var; // value assigned to the option via command-line + size_t idx{0}; // order in which the option was created + bool modified{false}; // whether the option occurred as a command-line argument or not }; -// Helper structure used for aliases storing an option key, value, and options to be expanded in the -// form of a YAML config +// Helper tuple for aliases storing an option key, value, and options to be expanded struct CLIAliasTuple { std::string key; // alias option name std::string value; // value for the alias option indicating that it should be expanded YAML::Node config; // config with options that the alias adds }; + /** * @brief The class used to define and parse command-line arguments. * - * It is a wrapper around https://github.com/CLIUtils/CLI11 that stores defined - * command-line arguments in a YAML object. + * It is a wrapper around https://github.com/CLIUtils/CLI11 that stores defined command-line + * arguments in a YAML object. * - * Usage outline: first call add() methods to create all the options; then call - * parse(argv, argc) to parse command line and get defined options and their - * values in a YAML object. The object can be also obtained later by calling + * Usage outline: first call add() methods to create all the options; then call parse(argv, argc) to + * parse command line and get defined options and their values in a YAML object; finally call + * parseAliases() to expand alias options. The config object can be also obtained later by calling * getConfig(). * - * Options are organized in option groups. Each option group has a header that - * preceeds all options in the group. The header for the default option group - * can be set from the class constructor. + * Options are organized in option groups. Each option group has a header that preceeds all options + * in the group. The header for the default option group can be set from the class constructor. */ class CLIWrapper { private: // Map with option names and option tuples std::unordered_map options_; - // Counter for created options + // Counter for created options to keep track of order in which options were created size_t counter_{0}; - // List of alias tuples - std::vector aliases_; - // Command-line argument parser - Ptr app_; + std::vector aliases_; // List of alias tuples + + Ptr app_; // Command-line argument parser from CLI11 - // Name of the default option group - std::string defaultGroup_{""}; - // Name of the current option group - std::string currentGroup_{""}; + std::string defaultGroup_{""}; // Name of the default option group + std::string currentGroup_{""}; // Name of the current option group - // Reference to the main config object - YAML::Node &config_; + YAML::Node &config_; // Reference to the main config object // Option for --version flag. This is a special flag and similarly to --help, // the key "version" will be not added into the YAML config CLI::Option *optVersion_; - static std::string failureMessage(const CLI::App *app, const CLI::Error &e); - // Extract option name from a comma-separated list of long and short options, e.g. 'help' from // '--help,-h' std::string keyName(const std::string &args) const { @@ -116,6 +90,13 @@ class CLIWrapper { .front(); // get first long name } + // Get names of options passed via command-line + std::unordered_set getParsedOptionNames() const; + // Get option names in the same order as they are created + std::vector getOrderedOptionNames() const; + + static std::string failureMessage(const CLI::App *app, const CLI::Error &e); + public: /** * @brief Create an instance of the command-line argument parser @@ -127,8 +108,7 @@ class CLIWrapper { * @param header Header text for the main option group * @param footer Text displayed after the list of options * @param columnWidth Width of the column with option names - * @param screenWidth Maximum allowed width for help messages, 0 means no - * limit + * @param screenWidth Maximum allowed width for help messages, 0 means no limit */ CLIWrapper(YAML::Node &config, const std::string &description = "", @@ -138,8 +118,7 @@ class CLIWrapper { size_t screenWidth = 0); /** - * @brief Create an instance of the command-line argument parser, - * short-cuft for Options object. + * @brief Create an instance of the command-line argument parser, short-cut for Options object. * * @see Other constructor */ @@ -173,13 +152,11 @@ class CLIWrapper { } /** - * @brief Define an option without an explicit default value. The implicit - * default value is T() + * @brief Define an option without an explicit default value. The implicit default value is T() * - * The option will be defined in the config file even if not given as a - * command-line argument. The implicit default value for a boolean or numeric - * option is 0, for a string is an empty string, and for a vector is an empty - * vector. + * The option will be defined in the config file even if not given as a command-line argument. The + * implicit default value for a boolean or numeric option is 0, for a string is an empty string, + * and for a vector is an empty vector. * * Implicit default values will *NOT* appear in help messages. * @@ -188,8 +165,7 @@ class CLIWrapper { * * @return Option object * - * TODO: require to always state the default value creating the parser as this - * will be clearer + * @TODO: require to always state the default value creating the parser as this will be clearer */ template CLI::Option *add(const std::string &args, const std::string &help) { @@ -244,9 +220,10 @@ class CLIWrapper { * @brief Expand aliases based on arguments parsed with parse(int, char**) * * Should be called after parse(int, char**) to take an effect. If any alias tries to expand an - * undefined option, the method will abort. + * undefined option, the method will abort the program. * - * All options defined as aliases are removed from the config object. + * All options defined as aliases are removed from the global config object to avoid redundancy + * when options are dumped (explicitly or implicitly) to a config file. */ void parseAliases(); @@ -264,14 +241,9 @@ class CLIWrapper { void updateConfig(const YAML::Node &config, const std::string &errorMsg); // Get textual YAML representation of the config - std::string dumpConfig(bool skipDefault = false) const; + std::string dumpConfig(bool skipUnmodified = false) const; private: - // Get names of options passed via command-line - std::unordered_set getParsedOptionNames() const; - // Get option names in the same order as they are created - std::vector getOrderedOptionNames() const; - template ::value && !CLI::is_vector::value, From 3c0e38382c2b8241f4dd57e3dd0baa9fd03f10b5 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 15:59:23 +0000 Subject: [PATCH 303/838] Introduce option priority --- src/common/cli_wrapper.cpp | 27 +++++++++++++-------------- src/common/cli_wrapper.h | 24 ++++++++++++++---------- src/common/config_parser.cpp | 2 +- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 863dc02ad..320edeca5 100644 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -133,10 +133,6 @@ void CLIWrapper::parseAliases() { if(aliases_.empty()) return; - // Names of all triggered/expanded aliases will be collected and removed from the config at the - // end. We keep them in a set as multiple aliases can be defined for one option name (aka `key`). - std::set parsedAliases; - // Iterate all known aliases, each alias has a key, value, and config for(const auto &alias : aliases_) { // Check if the alias option exists in the config (it may come from command line or a config @@ -157,23 +153,22 @@ void CLIWrapper::parseAliases() { if(expand) { // Update global config options with the config associated with the alias. Abort if the - // alias contains an undefined option + // alias contains an undefined option. updateConfig(alias.config, + // Priority of each expanded option is the same as the priority of the alias + options_[alias.key].priority, "Unknown option(s) in alias '" + alias.key + ": " + alias.value + "'"); } - - // Collect the alias option as they will be removed at the end - parsedAliases.insert(alias.key); } } // Remove aliases from the global config to avoid redundancy when writing/reading config files - for(const auto &key : parsedAliases) { - config_.remove(key); + for(const auto &alias : aliases_) { + config_.remove(alias.key); } } -void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &errorMsg) { +void CLIWrapper::updateConfig(const YAML::Node &config, int priority, const std::string &errorMsg) { auto cmdOptions = getParsedOptionNames(); // Keep track of unrecognized options from the provided config std::vector unknownOpts; @@ -191,10 +186,14 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &error // Check if an incoming option has been defined in CLI if(options_.count(key)) { + // Do not proceed if the priority of incoming option is not greater than the existing option + if(priority <= options_[key].priority) { + continue; + } // Check if the option exists in the global config and types match if(config_[key] && config_[key].Type() == it.second.Type()) { config_[key] = YAML::Clone(it.second); - options_[key].modified = true; + options_[key].priority = priority; // If types doesn't match, try to convert } else { // Default value is a sequence and incoming node is a scalar, hence we can upcast to @@ -205,7 +204,7 @@ void CLIWrapper::updateConfig(const YAML::Node &config, const std::string &error YAML::Node sequence; sequence.push_back(YAML::Clone(it.second)); config_[key] = sequence; // overwrite to replace default values - options_[key].modified = true; + options_[key].priority = priority; } else { // Cannot convert other non-matching types, e.g. scalar <- list should fail ABORT("Cannot convert values for the option: " + key); @@ -231,7 +230,7 @@ std::string CLIWrapper::dumpConfig(bool skipUnmodified /*= false*/) const { if(!config_[key]) continue; // Do not dump options that were not passed via the command line - if(skipUnmodified && !options_.at(key).modified) + if(skipUnmodified && options_.at(key).priority == 0) continue; // Put the group name as a comment before the first option in the group auto group = options_.at(key).opt->get_group(); diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index 851abfe8d..ec3cd47b4 100644 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -37,7 +37,7 @@ struct CLIOptionTuple { CLI::Option *opt; // a pointer to an option object from CLI11 Ptr var; // value assigned to the option via command-line size_t idx{0}; // order in which the option was created - bool modified{false}; // whether the option occurred as a command-line argument or not + int priority{0}; // priority: 0 - default value, 1 - from config, 2 - from command line }; // Helper tuple for aliases storing an option key, value, and options to be expanded @@ -227,18 +227,22 @@ class CLIWrapper { */ void parseAliases(); - /* - * @brief Overwrite values for unparsed options + /** + * @brief Overwrite options with lower priority + * + * Values for options with lower priority than the provided priority remain unchanged. This allows + * for overwritting default options by options from config files, or both by options provided in + * the command line. * - * Default values are overwritten with the options from the provided config, while parsed - * command-line options remain unchanged. This should be a preferred way of updating config - * options as the class keeps track of options, which values have changed. + * This should be a preferred way of updating config options as the class keeps track of options, + * which values have changed. * * @param config YAML config with new default values for options + * @param priority priority of incoming options * @param errorMsg error message printed if config contains undefined keys. The message is * appended with ": " */ - void updateConfig(const YAML::Node &config, const std::string &errorMsg); + void updateConfig(const YAML::Node &config, int priority, const std::string &errorMsg); // Get textual YAML representation of the config std::string dumpConfig(bool skipUnmodified = false) const; @@ -263,7 +267,7 @@ class CLIWrapper { // callback function collecting a command-line argument CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].modified = true; + options_[key].priority = 2; // get variable associated with the option auto &var = options_[key].var->as(); // store parser result in var @@ -310,7 +314,7 @@ class CLIWrapper { // callback function collecting command-line arguments CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].modified = true; + options_[key].priority = 2; // get vector variable associated with the option auto &vec = options_[key].var->as(); vec.clear(); @@ -367,7 +371,7 @@ class CLIWrapper { // callback function setting the flag CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].modified = true; + options_[key].priority = 2; // get parser result, it is safe as boolean options have an implicit value auto val = res[0]; auto ret = true; diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index cd8cc2db9..cf6951b6f 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -787,7 +787,7 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { auto configPaths = findConfigPaths(); if(!configPaths.empty()) { auto config = loadConfigFiles(configPaths); - cli.updateConfig(config, "There are option(s) in a config file that are not expected"); + cli.updateConfig(config, 1, "There are option(s) in a config file that are not expected"); } if(get("interpolate-env-vars")) { From 20948750cb5e3cb6d46b2c681f446ed75a43bc12 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 16:10:46 +0000 Subject: [PATCH 304/838] Add enum cli::Priority --- src/common/cli_wrapper.cpp | 4 ++-- src/common/cli_wrapper.h | 36 +++++++++++++++++++----------------- src/common/config_parser.cpp | 4 +++- src/common/config_parser.h | 1 + 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/common/cli_wrapper.cpp b/src/common/cli_wrapper.cpp index 320edeca5..3bae8c01b 100644 --- a/src/common/cli_wrapper.cpp +++ b/src/common/cli_wrapper.cpp @@ -168,7 +168,7 @@ void CLIWrapper::parseAliases() { } } -void CLIWrapper::updateConfig(const YAML::Node &config, int priority, const std::string &errorMsg) { +void CLIWrapper::updateConfig(const YAML::Node &config, cli::Priority priority, const std::string &errorMsg) { auto cmdOptions = getParsedOptionNames(); // Keep track of unrecognized options from the provided config std::vector unknownOpts; @@ -230,7 +230,7 @@ std::string CLIWrapper::dumpConfig(bool skipUnmodified /*= false*/) const { if(!config_[key]) continue; // Do not dump options that were not passed via the command line - if(skipUnmodified && options_.at(key).priority == 0) + if(skipUnmodified && options_.at(key).priority == cli::Priority::DefaultValue) continue; // Put the group name as a comment before the first option in the group auto group = options_.at(key).opt->get_group(); diff --git a/src/common/cli_wrapper.h b/src/common/cli_wrapper.h index ec3cd47b4..4347cb329 100644 --- a/src/common/cli_wrapper.h +++ b/src/common/cli_wrapper.h @@ -16,37 +16,39 @@ namespace marian { class Options; namespace cli { -// The helper class for cli::CLIWrapper handling formatting of options and their descriptions. -class CLIFormatter : public CLI::Formatter { -public: - CLIFormatter(size_t columnWidth, size_t screenWidth); - virtual std::string make_option_desc(const CLI::Option *) const override; - -private: - size_t screenWidth_{0}; -}; +// Option priority +enum struct Priority { DefaultValue = 0, ConfigFile = 1, CommandLine = 2 }; /** * Helper tuple storing an option object, the associated variable and creation index * - * Note: a bare pointer is used for the CLI::Option as this comes from the CLI11 library. Removing - * it would need deep modifications in the 3rd party library, that is not desirable. + * Note: bare pointers are used for CLI::Option objects as this comes from the CLI11 library. + * Removing it would require deep modifications in the 3rd party library, what we want to avoid. */ struct CLIOptionTuple { CLI::Option *opt; // a pointer to an option object from CLI11 Ptr var; // value assigned to the option via command-line size_t idx{0}; // order in which the option was created - int priority{0}; // priority: 0 - default value, 1 - from config, 2 - from command line + Priority priority{cli::Priority::DefaultValue}; }; -// Helper tuple for aliases storing an option key, value, and options to be expanded +// Helper tuple for aliases storing the alias name, value, and options to be expanded struct CLIAliasTuple { std::string key; // alias option name std::string value; // value for the alias option indicating that it should be expanded YAML::Node config; // config with options that the alias adds }; +// The helper class for cli::CLIWrapper handling formatting of options and their descriptions. +class CLIFormatter : public CLI::Formatter { +public: + CLIFormatter(size_t columnWidth, size_t screenWidth); + virtual std::string make_option_desc(const CLI::Option *) const override; + +private: + size_t screenWidth_{0}; +}; /** * @brief The class used to define and parse command-line arguments. @@ -242,7 +244,7 @@ class CLIWrapper { * @param errorMsg error message printed if config contains undefined keys. The message is * appended with ": " */ - void updateConfig(const YAML::Node &config, int priority, const std::string &errorMsg); + void updateConfig(const YAML::Node &config, cli::Priority priority, const std::string &errorMsg); // Get textual YAML representation of the config std::string dumpConfig(bool skipUnmodified = false) const; @@ -267,7 +269,7 @@ class CLIWrapper { // callback function collecting a command-line argument CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].priority = 2; + options_[key].priority = cli::Priority::CommandLine; // get variable associated with the option auto &var = options_[key].var->as(); // store parser result in var @@ -314,7 +316,7 @@ class CLIWrapper { // callback function collecting command-line arguments CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].priority = 2; + options_[key].priority = cli::Priority::CommandLine; // get vector variable associated with the option auto &vec = options_[key].var->as(); vec.clear(); @@ -371,7 +373,7 @@ class CLIWrapper { // callback function setting the flag CLI::callback_t fun = [this, key](CLI::results_t res) { - options_[key].priority = 2; + options_[key].priority = cli::Priority::CommandLine; // get parser result, it is safe as boolean options have an implicit value auto val = res[0]; auto ret = true; diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index cf6951b6f..6fce1bef1 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -787,7 +787,9 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { auto configPaths = findConfigPaths(); if(!configPaths.empty()) { auto config = loadConfigFiles(configPaths); - cli.updateConfig(config, 1, "There are option(s) in a config file that are not expected"); + cli.updateConfig(config, + cli::Priority::ConfigFile, + "There are option(s) in a config file that are not expected"); } if(get("interpolate-env-vars")) { diff --git a/src/common/config_parser.h b/src/common/config_parser.h index d6822cc1a..ef2aee950 100644 --- a/src/common/config_parser.h +++ b/src/common/config_parser.h @@ -14,6 +14,7 @@ namespace marian { namespace cli { +// CLI mode enum struct mode { training, translation, scoring, server }; } // namespace cli From 614876be4159331f7cfc6dc146d6770d652c0405 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 16:17:25 +0000 Subject: [PATCH 305/838] Rename --dump-config explain to --dump-config expand --- src/common/config_parser.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 6fce1bef1..766b5e0a7 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -79,7 +79,7 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) { cli.add("--relative-paths", "All paths are relative to the config file location"); cli.add("--dump-config", - "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal") + "Dump current (modified) configuration to stdout and exit. Possible values: full, minimal, expand") ->implicit_val("full"); // clang-format on } @@ -809,11 +809,11 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { auto dumpMode = get("dump-config"); config_.remove("dump-config"); - if(dumpMode == "explain") { + if(dumpMode == "expand") { cli.parseAliases(); } - bool minimal = (dumpMode == "minimal" || dumpMode == "explain"); + bool minimal = (dumpMode == "minimal" || dumpMode == "expand"); std::cout << cli.dumpConfig(minimal) << std::endl; exit(0); } From a848e2dce5ee39e8132f218b932f552c994208f7 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 16:41:10 +0000 Subject: [PATCH 306/838] Rename dump_ --- src/common/config_parser.cpp | 2 -- src/common/config_validator.cpp | 6 +++--- src/common/config_validator.h | 6 ++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 766b5e0a7..de6b9b6aa 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -797,8 +797,6 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { } if(doValidate) { - // TODO: Do not check some constraints if --dump-config, e.g. -t - // this aborts the program on first validation error ConfigValidator(config_).validateOptions(mode_); } diff --git a/src/common/config_validator.cpp b/src/common/config_validator.cpp index 1566e1cfc..61365c39c 100644 --- a/src/common/config_validator.cpp +++ b/src/common/config_validator.cpp @@ -12,8 +12,8 @@ bool ConfigValidator::has(const std::string& key) const { ConfigValidator::ConfigValidator(const YAML::Node& config) : config_(config), - dump_(config["dump-config"] && !config["dump-config"].as().empty() - && config["dump-config"].as() != "false") {} + dumpConfigOnly_(config["dump-config"] && !config["dump-config"].as().empty() + && config["dump-config"].as() != "false") {} ConfigValidator::~ConfigValidator() {} @@ -59,7 +59,7 @@ void ConfigValidator::validateOptionsTranslation() const { void ConfigValidator::validateOptionsParallelData() const { // Do not check these constraints if only goal is to dump config - if(dump_) + if(dumpConfigOnly_) return; auto trainSets = get>("train-sets"); diff --git a/src/common/config_validator.h b/src/common/config_validator.h index fb40ea6c6..0e73a9e39 100644 --- a/src/common/config_validator.h +++ b/src/common/config_validator.h @@ -8,15 +8,17 @@ namespace marian { class ConfigValidator { private: const YAML::Node& config_; - bool dump_{false}; bool has(const std::string& key) const; - template T get(const std::string& key) const { return config_[key].as(); } + // The option --dump-config is used, so alleviate some constraints, e.g. we don't want to require + // --train-sets or --vocabs + bool dumpConfigOnly_{false}; + void validateOptionsTranslation() const; void validateOptionsParallelData() const; void validateOptionsScoring() const; From cf5578b66e36456ac918410071238141d1fbb827 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 09:00:51 -0800 Subject: [PATCH 307/838] (minor cleanup) --- src/layers/generic.cpp | 8 +++++++- src/layers/generic.h | 12 +++--------- src/models/states.h | 2 +- src/translator/beam_search.h | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/models/states.h diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 80398ee08..e36d3e213 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -76,13 +76,19 @@ namespace marian { return logits_.front(); } + // get logits for one factor group + Expr Logits::getFactoredLogits(size_t groupIndex) const { + ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + return logits_[groupIndex]->loss(); + } + // This function assumes that the object holds one or more factor logits, which are summed up // into output-vocab logits according to the factored model (with correct normalization of factors). Expr Logits::getLogits() const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!factoredVocab_) { ABORT_IF(logits_.size() != 1, "Factors without factor mappings??"); - return logits_.front()->loss(); + return getFactoredLogits(0); } // compute normalized factor log probs diff --git a/src/layers/generic.h b/src/layers/generic.h index e6fd5c30b..6befea76e 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -64,7 +64,6 @@ class FactoredVocab; // for factored embeddings. class RationalLoss; class Logits { - Logits& operator=(const Logits& other) = default; public: Logits() {} Logits(Ptr logits) { // single-output constructor @@ -74,6 +73,7 @@ class Logits { Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), factoredVocab_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors + Expr getFactoredLogits(size_t groupIndex) const; // get logits for only one factor group Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; @@ -87,13 +87,7 @@ class Logits { }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // @TODO: avoid the fully expanded logits - - void assign(const Logits& other) { // @TODO: we can remove this - //ABORT_IF(!empty() && getNumFactors() != other.getNumFactors(), - // "Logits assignment cannot change number of factors"); - *this = other; - } - size_t getNumFactors() const { return logits_.size(); } + size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } Logits withCounts(const Expr& count) const; // create new Logits with 'count' implanted into all logits_ private: @@ -106,7 +100,7 @@ class Logits { private: // members // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. - std::vector> logits_; + std::vector> logits_; // [group id][B..., num factors in group] Ptr factoredVocab_; }; diff --git a/src/models/states.h b/src/models/states.h old mode 100644 new mode 100755 index c33cddc3c..f17038cca --- a/src/models/states.h +++ b/src/models/states.h @@ -54,7 +54,7 @@ class DecoderState { } virtual Logits getLogProbs() const { return logProbs_; } - virtual void setLogProbs(Logits logProbs) { logProbs_.assign(logProbs); } + virtual void setLogProbs(Logits logProbs) { logProbs_ = logProbs; } // @TODO: should this be a constructor? Then derived classes can call this without the New<> in the loop virtual Ptr select(const std::vector& selIdx, diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 1dc05d193..336441f58 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -242,7 +242,7 @@ class BeamSearch { //********************************************************************** // create constant containing previous path scores for current beam // Also create mapping of hyp indices, which are not 1:1 if sentences complete. - std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of hyp that the new top N originated from + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of prev hyp that the current hyp originated from std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding expandedPathScores if(t == 0) { // no scores yet From fc60cf8c38849a996d5f0439fb068186d32a9dd0 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Fri, 8 Feb 2019 17:25:11 +0000 Subject: [PATCH 308/838] Move addAliases() to separate file --- src/CMakeLists.txt | 1 + src/common/aliases.cpp | 75 ++++++++++++++++++++++++++++++++++++ src/common/config_parser.cpp | 56 --------------------------- 3 files changed, 76 insertions(+), 56 deletions(-) create mode 100644 src/common/aliases.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd489e3e1..4549b020c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,6 +14,7 @@ add_library(marian STATIC common/cli_wrapper.cpp common/config.cpp common/config_parser.cpp + common/aliases.cpp common/config_validator.cpp common/binary.cpp common/io.cpp diff --git a/src/common/aliases.cpp b/src/common/aliases.cpp new file mode 100644 index 000000000..11ca27fde --- /dev/null +++ b/src/common/aliases.cpp @@ -0,0 +1,75 @@ +#include "common/config_parser.h" +#include "common/definitions.h" + +namespace marian { + +/** + * Add all aliases + * + * An alias is a shortcut option for a predefined set of options. It is triggered if the option has + * the requested value. The alias option has to be first defined using cli.add(). Defining + * multiple aliases for the same option name but with different value is allowed. + * + * Values are compared as std::string. If the alias option is a vector, the alias will be triggered + * if the requested value exists in that vector at least once. + * + * @see CLIWrapper::alias() + * + * The order of alias definitions *does* matter: options from later aliases override earlier + * regardless of its order in the command line or config file. + */ +void ConfigParser::addAliases(cli::CLIWrapper& cli) { + // Options setting the BiDeep architecture proposed in http://www.aclweb.org/anthology/W17-4710 + cli.alias("best-deep", "true", [](YAML::Node& config) { + config["layer-normalization"] = true; + config["tied-embeddings"] = true; + config["enc-type"] = "alternating"; + config["enc-cell-depth"] = 2; + config["enc-depth"] = 4; + config["dec-cell-base-depth"] = 4; + config["dec-cell-high-depth"] = 2; + config["dec-depth"] = 4; + config["skip"] = true; + }); + + // Architecture and proposed training settings for a Transformer BASE model introduced in + // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf + cli.alias("task", "transformer", [](YAML::Node& config) { + config["type"] = "transformer"; + config["enc-depth"] = 6; + config["dec-depth"] = 6; + config["transformer-heads"] = 8; + config["learn-rate"] = 0.0003; + config["cost-type"] = "ce-mean-words"; + config["lr-warmup"] = 16000; + config["lr-decay-inv-sqrt"] = 16000; + config["transformer-dropout"] = 0.1; + config["label-smoothing"] = 0.1; + config["clip-norm"] = 5; + }); + + // Architecture and proposed training settings for a Transformer BIG model introduced in + // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf + cli.alias("task", "transformer-big", [](YAML::Node& config) { + config["type"] = "transformer"; + config["enc-depth"] = 6; + config["dec-depth"] = 6; + config["dim-emb"] = 1024; + config["transformer-dim-ffn"] = 4096; + config["transformer-heads"] = 16; + config["transformer-postprocess"] = "dan"; + config["transformer-preprocess"] = "d"; + config["transformer-ffn-activation"] = "relu"; + config["learn-rate"] = 0.0002; + config["cost-type"] = "ce-mean-words"; + config["lr-warmup"] = 8000; + config["lr-decay-inv-sqrt"] = 8000; + config["transformer-dropout"] = 0.1; + config["transformer-dropout-attention"] = 0.1; + config["transformer-dropout-ffn"] = 0.1; + config["label-smoothing"] = 0.1; + config["clip-norm"] = 5; + }); +} + +} // namespace marian diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index de6b9b6aa..fdf86da8b 100644 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -693,62 +693,6 @@ void ConfigParser::addSuboptionsULR(cli::CLIWrapper& cli) { // clang-format on } -void ConfigParser::addAliases(cli::CLIWrapper& cli) { - // The order of aliases does matter as later options in the command line overwrite earlier - - // Options reconstructing the BiDeep architecture proposed in http://www.aclweb.org/anthology/W17-4710 - cli.alias("best-deep", "true", [](YAML::Node& config) { - config["layer-normalization"] = true; - config["tied-embeddings"] = true; - config["enc-type"] = "alternating"; - config["enc-cell-depth"] = 2; - config["enc-depth"] = 4; - config["dec-cell-base-depth"] = 4; - config["dec-cell-high-depth"] = 2; - config["dec-depth"] = 4; - config["skip"] = true; - }); - - // Architecture and proposed training settings for a Transformer BASE model introduced in - // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf - cli.alias("task", "transformer", [](YAML::Node& config) { - config["type"] = "transformer"; - config["enc-depth"] = 6; - config["dec-depth"] = 6; - config["transformer-heads"] = 8; - config["learn-rate"] = 0.0003; - config["cost-type"] = "ce-mean-words"; - config["lr-warmup"] = 16000; - config["lr-decay-inv-sqrt"] = 16000; - config["transformer-dropout"] = 0.1; - config["label-smoothing"] = 0.1; - config["clip-norm"] = 5; - }); - - // Architecture and proposed training settings for a Transformer BIG model introduced in - // https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf - cli.alias("task", "transformer-big", [](YAML::Node& config) { - config["type"] = "transformer"; - config["enc-depth"] = 6; - config["dec-depth"] = 6; - config["dim-emb"] = 1024; - config["transformer-dim-ffn"] = 4096; - config["transformer-heads"] = 16; - config["transformer-postprocess"] = "dan"; - config["transformer-preprocess"] = "d"; - config["transformer-ffn-activation"] = "relu"; - config["learn-rate"] = 0.0002; - config["cost-type"] = "ce-mean-words"; - config["lr-warmup"] = 8000; - config["lr-decay-inv-sqrt"] = 8000; - config["transformer-dropout"] = 0.1; - config["transformer-dropout-attention"] = 0.1; - config["transformer-dropout-ffn"] = 0.1; - config["label-smoothing"] = 0.1; - config["clip-norm"] = 5; - }); -} - void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) { cli::CLIWrapper cli(config_, "Marian: Fast Neural Machine Translation in C++", From 785a660fe08ecfefeda76efbf170ebbec84f4208 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 09:36:54 -0800 Subject: [PATCH 309/838] Hypothesis::prexIndex_ changed to no longer contain the batch index (it does not belong here) --- src/translator/beam_search.h | 47 ++++++++++++------------------------ src/translator/hypothesis.h | 10 ++++---- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 336441f58..6cc220d47 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -63,28 +63,13 @@ class BeamSearch { // decompose key into individual indices WordIndex wordIdx = (WordIndex)(key % vocabSize); const auto hypIdx = (key / vocabSize); -#if 1 // further decompose hypIdx, taking into account that the very first entry had beam size 1 - // and compose a new hypIdx that assumes actual beamSize - const auto keyBatchIdx = hypIdx / (first ? 1 : beamSize); - const auto keyBeamHypIdx = hypIdx % (first ? 1 : beamSize); - const auto hypIdxTrans = keyBeamHypIdx * dimBatch + keyBatchIdx; - ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); // @TODO: is this possible? Should be, but does not seem to trigger. -#else - const auto keyBatchIdx = hypIdx / beamSize; // @REVIEW: is this actually keyBatchIdx? - size_t keyBeamHypIdx = hypIdx % beamSize; - - auto hypIdxTrans = keyBatchIdx + keyBeamHypIdx * dimBatch; - if(first) - hypIdxTrans = hypIdx; // == keyBeamHypIdx + keyBatchIdx * beamSize? or was beamSize=1, and keyBeamHypIdx = 0? - - ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); - //if(keyBeamHypIdx >= (int)beam.size()) // @TODO: What is this condition? Cf. keyBeamHypIdx = hypIdx % beamSize; beamSize = max(beams[.].size()) - // keyBeamHypIdx = keyBeamHypIdx % beam.size(); - - if(first) - keyBeamHypIdx = 0; -#endif + // and compose a new hypIdx that assumes actual beamSize and refers to a transposed object + const auto beamHypIdx = hypIdx % (first ? 1 : beamSize); + const auto batchIdx1 = hypIdx / (first ? 1 : beamSize); // (only for checking; must be same as batchIdx) + ABORT_IF(batchIdx1 != batchIdx || beamHypIdx >= (int)beam.size(), "Inconsistent (beamHypIdx, batchIdx) value in key??"); + const auto hypIdxTrans = beamHypIdx * dimBatch + batchIdx; + // Retrieve short list for final softmax (based on words aligned // to source sentences). If short list has been set, map the indices // in the sub-selected vocabulary matrix back to their original positions. @@ -93,22 +78,22 @@ class BeamSearch { wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? // now wordIdx is a regular Word again - auto hyp = New(beam[keyBeamHypIdx], Word::fromWordIndex(wordIdx), (IndexType)hypIdxTrans, pathScore); + auto hyp = New(beam[beamHypIdx], Word::fromWordIndex(wordIdx), beamHypIdx, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { std::vector breakDown(states.size(), 0); - beam[keyBeamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? + beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? for(size_t j = 0; j < states.size(); ++j) { size_t key1 = hypIdxTrans * vocabSize + wordIdx; - breakDown[j] = states[j]->breakDown(key1) + beam[keyBeamHypIdx]->getScoreBreakdown()[j]; + breakDown[j] = states[j]->breakDown(key1) + beam[beamHypIdx]->getScoreBreakdown()[j]; } hyp->setScoreBreakdown(breakDown); } // Set alignments if(!align.empty()) { - hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)keyBeamHypIdx, (int)batchIdx)); + hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); } newBeam.push_back(hyp); @@ -249,12 +234,12 @@ class BeamSearch { prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { std::vector prevScores; - for(size_t beamIndex = 0; beamIndex < localBeamSize; ++beamIndex) { - for(int batchIndex = 0; batchIndex < dimBatch; ++batchIndex) { // loop over batch entries (active sentences) - auto& beam = beams[batchIndex]; - if(beamIndex < beam.size()) { - auto hyp = beam[beamIndex]; - hypIndices.push_back((IndexType)hyp->getPrevStateIndex()); // index where to find prev hyp (beamHypIdx, batchIdx), =beamHypIdx * dimBatch + batchIdx + for(size_t beamHypIdx = 0; beamHypIdx < localBeamSize; ++beamHypIdx) { + for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { // loop over batch entries (active sentences) + auto& beam = beams[batchIdx]; + if(beamHypIdx < beam.size()) { + auto hyp = beam[beamHypIdx]; + hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // flattened index for index_select() operation prevWords .push_back(hyp->getWord()); prevScores.push_back(hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index 90d5543cd..5b6ff6b00 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -13,19 +13,19 @@ namespace marian { // - back pointer to previous hypothesis for traceback class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(Word::ZERO), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), prevBeamHypIdx_(0), word_(Word::ZERO), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, - IndexType prevIndex, // (beamHypIdx, batchIdx) flattened as beamHypIdx * dimBatch + batchIdx + size_t prevBeamHypIdx, // beam-hyp index that this hypothesis originated from float pathScore) - : prevHyp_(prevHyp), prevIndex_(prevIndex), word_(word), pathScore_(pathScore) {} + : prevHyp_(prevHyp), prevBeamHypIdx_(prevBeamHypIdx), word_(word), pathScore_(pathScore) {} const Ptr getPrevHyp() const { return prevHyp_; } Word getWord() const { return word_; } - IndexType getPrevStateIndex() const { return prevIndex_; } + size_t getPrevStateIndex() const { return prevBeamHypIdx_; } float getPathScore() const { return pathScore_; } @@ -61,7 +61,7 @@ class Hypothesis { private: const Ptr prevHyp_; - const IndexType prevIndex_; + const size_t prevBeamHypIdx_; const Word word_; const float pathScore_; From 46576dade4dc3f70238c06e9ee0ca801d25db662 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 10:23:54 -0800 Subject: [PATCH 310/838] bug fix (?): breakDown() call in toHyp() should use wordIdx before de-shortlisting --- src/translator/beam_search.h | 88 ++++++++++++++++++------------------ src/translator/hypothesis.h | 10 ++-- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 6cc220d47..4a68ba6bc 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -41,63 +41,65 @@ class BeamSearch { const size_t beamSize, const bool first, Ptr batch) const { - const auto dimBatch = beams.size(); - Beams newBeams(dimBatch); - std::vector align; if(options_->hasAndNotEmpty("alignment")) // Use alignments from the first scorer, even if ensemble align = scorers_[0]->getAlignment(); + const auto dimBatch = beams.size(); + Beams newBeams(dimBatch); + for(size_t i = 0; i < nBestKeys.size(); ++i) { // [dimBatch, beamSize] flattened // Keys contains indices to vocab items in the entire beam. // Values can be between 0 and beamSize * vocabSize. - const auto batchIdx = i / beamSize; // and i % beamSize is the beam hyp index + const float pathScore = nBestPathScores[i]; + const auto key = nBestKeys[i]; // key = pathScore's location, as (batchIdx, beamHypIdx, word idx) flattened + + // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) + const auto wordIdx = (WordIndex)(key % vocabSize); + const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); + const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); + + ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); + const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; - if(newBeam.size() < beam.size()) { - const float pathScore = nBestPathScores[i]; - const auto key = nBestKeys[i]; // key = pathScore's location, as ((batchIdx, beamHypIdx) flattened, word idx) flattened - - // decompose key into individual indices - WordIndex wordIdx = (WordIndex)(key % vocabSize); - const auto hypIdx = (key / vocabSize); - // further decompose hypIdx, taking into account that the very first entry had beam size 1 - // and compose a new hypIdx that assumes actual beamSize and refers to a transposed object - const auto beamHypIdx = hypIdx % (first ? 1 : beamSize); - const auto batchIdx1 = hypIdx / (first ? 1 : beamSize); // (only for checking; must be same as batchIdx) - ABORT_IF(batchIdx1 != batchIdx || beamHypIdx >= (int)beam.size(), "Inconsistent (beamHypIdx, batchIdx) value in key??"); - const auto hypIdxTrans = beamHypIdx * dimBatch + batchIdx; - - // Retrieve short list for final softmax (based on words aligned - // to source sentences). If short list has been set, map the indices - // in the sub-selected vocabulary matrix back to their original positions. - auto shortlist = scorers_[0]->getShortlist(); - if(shortlist) - wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? - // now wordIdx is a regular Word again - - auto hyp = New(beam[beamHypIdx], Word::fromWordIndex(wordIdx), beamHypIdx, pathScore); - - // Set score breakdown for n-best lists - if(options_->get("n-best")) { - std::vector breakDown(states.size(), 0); - beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? - for(size_t j = 0; j < states.size(); ++j) { - size_t key1 = hypIdxTrans * vocabSize + wordIdx; - breakDown[j] = states[j]->breakDown(key1) + beam[beamHypIdx]->getScoreBreakdown()[j]; - } - hyp->setScoreBreakdown(breakDown); - } - - // Set alignments - if(!align.empty()) { - hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); + if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? + continue; + + ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value in key??"); + + // map wordIdx to word + // If short list has been set, then wordIdx is an index into the short-listed word set, + // rather than the true word index. + auto shortlist = scorers_[0]->getShortlist(); + Word word = + /*if*/(shortlist) ? + Word::fromWordIndex(shortlist->reverseMap(wordIdx)) + /*else*/: + Word::fromWordIndex(wordIdx); + + auto hyp = New(beam[beamHypIdx], word, /*experimental dummy*/0, beamHypIdx, pathScore); + + // Set score breakdown for n-best lists + if(options_->get("n-best")) { + std::vector breakDown(states.size(), 0); + beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? + for(size_t j = 0; j < states.size(); ++j) { + size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx) + // @BUGBUG: This ^^ used to use wordIdx after de-shortlisting, which seems wrong. If it was right though, change wordIdx to word.toWordIndex(). + breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; } + hyp->setScoreBreakdown(breakDown); + } - newBeam.push_back(hyp); + // Set alignments + if(!align.empty()) { + hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); } + + newBeam.push_back(hyp); } return newBeams; } diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index 5b6ff6b00..755a5b890 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -13,13 +13,14 @@ namespace marian { // - back pointer to previous hypothesis for traceback class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), prevBeamHypIdx_(0), word_(Word::ZERO), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), beamHypIdx_(0), prevBeamHypIdx_(0), word_(Word::ZERO), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, + size_t beamHypIdx, // EXPERIMENTAL, UNUSED. which beam does this come from? (beamHypIdx, word) indexes the logit tensor size_t prevBeamHypIdx, // beam-hyp index that this hypothesis originated from float pathScore) - : prevHyp_(prevHyp), prevBeamHypIdx_(prevBeamHypIdx), word_(word), pathScore_(pathScore) {} + : prevHyp_(prevHyp), beamHypIdx_(beamHypIdx), prevBeamHypIdx_(prevBeamHypIdx), word_(word), pathScore_(pathScore) {} const Ptr getPrevHyp() const { return prevHyp_; } @@ -40,8 +41,8 @@ class Hypothesis { { Words targetWords; for (auto hyp = this; hyp->getPrevHyp(); hyp = hyp->getPrevHyp().get()) { - targetWords.push_back(hyp->getWord()); - // std::cerr << hyp->getWord() << " " << hyp << std::endl; + targetWords.push_back(hyp->getWord()); + // std::cerr << hyp->getWord() << " " << hyp << std::endl; } std::reverse(targetWords.begin(), targetWords.end()); return targetWords; @@ -61,6 +62,7 @@ class Hypothesis { private: const Ptr prevHyp_; + const size_t beamHypIdx_; // EXPERIMENTAL, UNUSED const size_t prevBeamHypIdx_; const Word word_; const float pathScore_; From 18a86e5f460256a475ab6575c07c6eda3ab41fc6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 10:57:37 -0800 Subject: [PATCH 311/838] removed some experimental code --- src/translator/beam_search.h | 20 ++++++++++---------- src/translator/hypothesis.h | 6 ++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 4a68ba6bc..1d2a3f520 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -43,8 +43,7 @@ class BeamSearch { Ptr batch) const { std::vector align; if(options_->hasAndNotEmpty("alignment")) - // Use alignments from the first scorer, even if ensemble - align = scorers_[0]->getAlignment(); + align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble const auto dimBatch = beams.size(); Beams newBeams(dimBatch); @@ -53,7 +52,7 @@ class BeamSearch { // Keys contains indices to vocab items in the entire beam. // Values can be between 0 and beamSize * vocabSize. const float pathScore = nBestPathScores[i]; - const auto key = nBestKeys[i]; // key = pathScore's location, as (batchIdx, beamHypIdx, word idx) flattened + const auto key = nBestKeys[i]; // key = pathScore's tensor location, as (batchIdx, beamHypIdx, word idx) flattened // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) const auto wordIdx = (WordIndex)(key % vocabSize); @@ -80,16 +79,17 @@ class BeamSearch { /*else*/: Word::fromWordIndex(wordIdx); - auto hyp = New(beam[beamHypIdx], word, /*experimental dummy*/0, beamHypIdx, pathScore); + auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { std::vector breakDown(states.size(), 0); beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? for(size_t j = 0; j < states.size(); ++j) { - size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx) + size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' // @BUGBUG: This ^^ used to use wordIdx after de-shortlisting, which seems wrong. If it was right though, change wordIdx to word.toWordIndex(). breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; + // @TODO: pass those 3 indices directly into breakDown } hyp->setScoreBreakdown(breakDown); } @@ -229,9 +229,9 @@ class BeamSearch { //********************************************************************** // create constant containing previous path scores for current beam // Also create mapping of hyp indices, which are not 1:1 if sentences complete. - std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of prev hyp that the current hyp originated from - std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word - Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding expandedPathScores + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from, for reordering + std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in + Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) if(t == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { @@ -241,7 +241,7 @@ class BeamSearch { auto& beam = beams[batchIdx]; if(beamHypIdx < beam.size()) { auto hyp = beam[beamHypIdx]; - hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // flattened index for index_select() operation + hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(hyp->getWord()); prevScores.push_back(hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) @@ -260,7 +260,7 @@ class BeamSearch { auto expandedPathScores = prevPathScores; // will become [localBeamSize, 1, dimBatch, dimVocab] for(size_t i = 0; i < scorers_.size(); ++i) { // compute output probabilities for current output time step - // - uses hypIndices[index in beam, 1, batch index, 1] to reorder hypotheses + // - uses hypIndices[index in beam, 1, batch index, 1] to reorder decoder state to reflect the top-N in beams[][] // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history // - performs one step of the decoder model // - returns new NN state for use in next output time step diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index 755a5b890..28cac2765 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -13,14 +13,13 @@ namespace marian { // - back pointer to previous hypothesis for traceback class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), beamHypIdx_(0), prevBeamHypIdx_(0), word_(Word::ZERO), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), prevBeamHypIdx_(0), word_(Word::ZERO), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, - size_t beamHypIdx, // EXPERIMENTAL, UNUSED. which beam does this come from? (beamHypIdx, word) indexes the logit tensor size_t prevBeamHypIdx, // beam-hyp index that this hypothesis originated from float pathScore) - : prevHyp_(prevHyp), beamHypIdx_(beamHypIdx), prevBeamHypIdx_(prevBeamHypIdx), word_(word), pathScore_(pathScore) {} + : prevHyp_(prevHyp), prevBeamHypIdx_(prevBeamHypIdx), word_(word), pathScore_(pathScore) {} const Ptr getPrevHyp() const { return prevHyp_; } @@ -62,7 +61,6 @@ class Hypothesis { private: const Ptr prevHyp_; - const size_t beamHypIdx_; // EXPERIMENTAL, UNUSED const size_t prevBeamHypIdx_; const Word word_; const float pathScore_; From 7ff1aab91f97162e159b0833158ccb8a708318aa Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 11:00:01 -0800 Subject: [PATCH 312/838] removed some experimental code --- src/translator/beam_search.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 1d2a3f520..e206832ec 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -87,9 +87,8 @@ class BeamSearch { beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? for(size_t j = 0; j < states.size(); ++j) { size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' - // @BUGBUG: This ^^ used to use wordIdx after de-shortlisting, which seems wrong. If it was right though, change wordIdx to word.toWordIndex(). breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; - // @TODO: pass those 3 indices directly into breakDown + // @TODO: pass those 3 indices directly into breakDown (state knows the dimensions) } hyp->setScoreBreakdown(breakDown); } From 791e2c7af561a388c65d557ecb77ab5494595170 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 11:35:57 -0800 Subject: [PATCH 313/838] towards factored decoding --- src/translator/beam_search.h | 98 +++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index e206832ec..13fa84227 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -40,7 +40,8 @@ class BeamSearch { const std::vector>& states, const size_t beamSize, const bool first, - Ptr batch) const { + Ptr batch, // for alignments only + Ptr factoredVocab) const { std::vector align; if(options_->hasAndNotEmpty("alignment")) align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble @@ -70,14 +71,14 @@ class BeamSearch { ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value in key??"); // map wordIdx to word + Word word; // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. auto shortlist = scorers_[0]->getShortlist(); - Word word = - /*if*/(shortlist) ? - Word::fromWordIndex(shortlist->reverseMap(wordIdx)) - /*else*/: - Word::fromWordIndex(wordIdx); + if (shortlist) + word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); + else + word = Word::fromWordIndex(wordIdx); auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); @@ -161,8 +162,10 @@ class BeamSearch { ABORT_IF(batch->back()->vocab()->getEosId() != trgEosId_, "Batch uses different EOS token than was passed to BeamSearch originally"); auto factoredVocab = batch->back()->vocab()->tryAs(); - size_t numFactors = factoredVocab ? factoredVocab->getNumGroups() : 1; - numFactors; +#if 1 // use '1' here to disable factored decoding, e.g. for comparisons + factoredVocab.reset(); +#endif + size_t numFactorGroups = factoredVocab ? factoredVocab->getNumGroups() : 1; const int dimBatch = (int)batch->size(); @@ -213,6 +216,10 @@ class BeamSearch { // main loop over output time steps for (size_t t = 0; ; t++) { ABORT_IF(dimBatch != beams.size(), "Lost a batch entry??"); + + // for factored vocabs, we do one factor at a time, but without updating the decoder model for secondary factors + auto factorGroup = t % numFactorGroups; + // determine beam size for next output time step, as max over still-active sentences // E.g. if all batch entries are down from beam 5 to no more than 4 surviving hyps, then // switch to beam of 4 for all. If all are done, then beam ends up being 0, and we are done. @@ -227,9 +234,9 @@ class BeamSearch { //********************************************************************** // create constant containing previous path scores for current beam - // Also create mapping of hyp indices, which are not 1:1 if sentences complete. - std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from, for reordering - std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in + // Also create mapping of hyp indices, for reordering the decoder-state tensors. + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from + std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) if(t == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); @@ -258,20 +265,39 @@ class BeamSearch { // compute expanded path scores with word prediction probs from all scorers auto expandedPathScores = prevPathScores; // will become [localBeamSize, 1, dimBatch, dimVocab] for(size_t i = 0; i < scorers_.size(); ++i) { - // compute output probabilities for current output time step - // - uses hypIndices[index in beam, 1, batch index, 1] to reorder decoder state to reflect the top-N in beams[][] - // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history - // - performs one step of the decoder model - // - returns new NN state for use in next output time step - // - returns vector of prediction probabilities over output vocab via newState - auto newState = scorers_[i]->step( - graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); - + Expr logProbs, factorMasks; + if (factorGroup == 0) { + // compute output probabilities for current output time step + // - uses hypIndices[index in beam, 1, batch index, 1] to reorder decoder state to reflect the top-N in beams[][] + // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history + // - performs one step of the decoder model + // - returns new NN state for use in next output time step + // - returns vector of prediction probabilities over output vocab via newState + // update state in-place for next output time step + states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); + } + else { + // add secondary factors + // For those, we don't update the decoder-model state in any way. + // Instead, we just keep expanding with the factors. + // Considerations: + // - not all scores should get a factor + // We need a [localBeamSize, 1, dimBatch, 1] tensor that knows whether a factor is applicable + // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. + // - factors are incorporated one step at a time; so we will have temporary Word entries + // in hyps with some factors set to FACTOR_NOT_SPECIFIED. + // - we did not rearrange the tensors in the decoder model's state + + // ... + } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] - expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * newState->getLogProbs().getLogits(); - - // update state in-place for next output time step - states[i] = newState; + if (numFactorGroups == 1) + logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] + else + logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] + if (factorMasks) + logProbs = logProbs * factorMasks; // those hyps that don't have a factor get multiplied with 0 + expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; } // make beams continuous @@ -305,33 +331,33 @@ class BeamSearch { // Now, nBestPathScores contain N-best expandedPathScores, and nBestKeys for each their original location (batchIdx, beamHypIdx, word). // combine N-best sets with existing search space (beams) to updated search space - beams = toHyps(nBestKeys, nBestPathScores, - /*dimTrgVoc=*/expandedPathScores->shape()[-1], - beams, - states, // used for keeping track of per-ensemble-member path score - localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples - /*first=*/t == 0, // used to indicate originating beamSize of 1 - batch); + auto newBeams = toHyps(nBestKeys, nBestPathScores, + /*dimTrgVoc=*/expandedPathScores->shape()[-1], + beams, + states, // used for keeping track of per-ensemble-member path score + localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples + /*first=*/t == 0, // used to indicate originating beamSize of 1 + batch, factoredVocab); // remove all hyps that end in EOS // The position of a hyp in the beam may change. - const auto purgedBeams = purgeBeams(beams); + const auto purgedNewBeams = purgeBeams(newBeams); - // add updated search space (beams) to search grid (histories) for traceback + // add updated search space (newBeams) to search grid (histories) for traceback bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { // if this batch entry has surviving hyps then add them to the traceback grid - if(!beams[i].empty()) { + if(!newBeams[i].empty()) { if (histories[i]->size() >= options_->get("max-length-factor") * batch->front()->batchWidth()) maxLengthReached = true; - histories[i]->add(beams[i], trgEosId_, purgedBeams[i].empty() || maxLengthReached); + histories[i]->add(newBeams[i], trgEosId_, purgedNewBeams[i].empty() || maxLengthReached); } } if (maxLengthReached) // early exit if max length limit was reached break; // this is the search space for the next output time step - beams = purgedBeams; + beams = purgedNewBeams; } // end of main loop over output time steps return histories; // [dimBatch][t][N best hyps] From d2db12a3ce88878eeb86b79149a631c1126e605a Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Fri, 8 Feb 2019 19:41:22 +0000 Subject: [PATCH 314/838] Allow build dir to be outside the source tree Currently, the call to git to record the git revision fails if the build directory is outside the source tree. This fix allows the build directory to be outside the source tree. However, compilation still works only if the source is obtained via git and not from a .zip file. --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd489e3e1..fbafc3b99 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -93,6 +93,7 @@ target_compile_options(marian PUBLIC ${ALL_WARNINGS}) # [https://stackoverflow.com/questions/1435953/how-can-i-pass-git-sha1-to-compiler-as-definition-using-cmake] # Git updates .git/logs/HEAD file whenever you pull or commit something. add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} COMMAND git log -1 --pretty=format:\#define\ GIT_REVISION\ \"\%h\ \%ai\" > ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h DEPENDS ${CMAKE_SOURCE_DIR}/.git/logs/HEAD VERBATIM From 82dc0ed93f56773d09f515d2372a7f511b4153cf Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 11:59:45 -0800 Subject: [PATCH 315/838] towwrds factored decoding --- src/layers/generic.cpp | 7 +++++++ src/layers/generic.h | 3 ++- src/translator/beam_search.h | 10 ++++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index e36d3e213..1ebd87f5a 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -137,6 +137,13 @@ namespace marian { return res; } + // use first factor of each word to determine whether it has a specific factor + std::vector Logits::getFactorMasks(const Words& words, size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 + std::vector res; + ABORT("FINISH THIS"); + return res; + } + Logits Logits::withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ std::vector> newLogits; for (const auto& l : logits_) diff --git a/src/layers/generic.h b/src/layers/generic.h index 6befea76e..85dec5692 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -86,7 +86,8 @@ class Logits { MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices - float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // @TODO: avoid the fully expanded logits + std::vector getFactorMasks(const Words& words, size_t factorGroup) const; + float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } Logits withCounts(const Expr& count) const; // create new Logits with 'count' implanted into all logits_ diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 13fa84227..36836acc7 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -257,8 +257,7 @@ class BeamSearch { } } } - prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, - inits::from_vector(prevScores)); + prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(prevScores)); } //********************************************************************** @@ -286,9 +285,12 @@ class BeamSearch { // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. // - factors are incorporated one step at a time; so we will have temporary Word entries // in hyps with some factors set to FACTOR_NOT_SPECIFIED. + // TODO: // - we did not rearrange the tensors in the decoder model's state - - // ... + // - initial word should set lemma by all other factors as unspecified + // - toHyp() should implant factors + auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); + factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] if (numFactorGroups == 1) From b50830cc0c4e1e687328f8569f6977bc3c0837d2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 20:22:51 -0800 Subject: [PATCH 316/838] added lots of checks to word/factor conversions; new methods lemma2Word(), addFactor() --- src/data/factored_vocab.cpp | 102 ++++++++++++++++++++++++++--------- src/data/factored_vocab.h | 14 +++-- src/layers/generic.cpp | 1 + src/translator/beam_search.h | 14 +++-- 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 153741e35..2390adf63 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -39,6 +39,7 @@ namespace marian { factorMap_.resize(elements); auto factorVocabSize = factorVocab_.size(); factorRefCounts_.resize(factorVocabSize); + lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); std::vector tokens; std::string line; size_t numTotalFactors = 0; @@ -46,7 +47,7 @@ namespace marian { for (WordIndex v = 0; io::getline(in, line); v++) { // parse the line, of the form WORD FACTOR1 FACTOR2 FACTOR1 ... // where FACTOR1 is the lemma, a factor that all words have. - // Not every word has all other factors, so the n-th item is not always the same factor. + // Not every word has all other factors, so the n-th item is not always in the same factor group. utils::splitAny(line, tokens, " \t"); ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); std::vector factorUnits; @@ -55,14 +56,29 @@ namespace marian { factorUnits.push_back(u); factorRefCounts_[u]++; } - auto index = factorUnits2word(factorUnits).toWordIndex(); - // @TODO: map factors to non-dense integer - factorMap_[index] = std::move(factorUnits); - // add to vocab - vocab_.add(tokens.front(), index); + // convert to fully unrolled factors representation + std::vector factorIndices(groupRanges_.size(), FACTOR_NOT_APPLICABLE); // default for unused factors + std::vector hasFactorGroupFlags(groupRanges_.size(), false); + for (auto u : factorUnits) { + factorIndices[factorGroups_[u]] = factorUnit2FactorIndex(u); + hasFactorGroupFlags[factorGroups_[u]] = true; + } + // record which lemma has what factor groups + ABORT_IF(!hasFactorGroupFlags[0], "Factor map does not specify a lemma (factor of first group) for word {}", tokens.front()); + auto& lemmaFlags = lemmaHasFactorGroup_[factorIndices[0]]; + if (lemmaFlags.empty()) + lemmaFlags = std::move(hasFactorGroupFlags); + else + ABORT_IF(lemmaFlags != hasFactorGroupFlags, "Inconsistent factor groups used for word {}", tokens.front()); + // map factors to non-dense integer + auto word = factors2word(factorIndices); + auto wordIndex = word.toWordIndex(); + factorMap_[wordIndex] = std::move(factorUnits); + // add to vocab (the wordIndex are not dense, so the vocab will have holes) + vocab_.add(tokens.front(), wordIndex); numTotalFactors += tokens.size() - 1; if (v % 5000 == 0) - LOG(info, "{} -> {}", tokens.front(), word2string(Word::fromWordIndex(index))); + LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", numTotalFactors, factorVocabSize, vocab_.numValid(), size()); @@ -130,13 +146,18 @@ void FactoredVocab::constructFactorIndexConversion() { } // encode factors into a Word struct -Word FactoredVocab::factors2word(const std::vector& factorIndices /* [numGroups] */) { +Word FactoredVocab::factors2word(const std::vector& factorIndices /* [numGroups] */) const { size_t index = 0; size_t numGroups = getNumGroups(); ABORT_IF(factorIndices.size() != numGroups, "Factor indices array size must be same as number of factor groups"); for (size_t g = 0; g < numGroups; g++) { auto factorIndex = factorIndices[g]; - if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) // @TODO: check validity. If word has the factor, then N/A is invalid + if (factorIndex != FACTOR_NOT_SPECIFIED) { // check validity + auto factor0Index = factorIndices[0]; // lemma + ABORT_IF(factor0Index == FACTOR_NOT_SPECIFIED, "Without lemma, no other factor may be specified"); + ABORT_IF(lemmaHasFactorGroup(factor0Index, g) == (factorIndex == FACTOR_NOT_APPLICABLE), "Lemma {} does not have factor group {}", factor0Index, g); + } + if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) factorIndex = (size_t)factorShape_[g] - 1; // sentinel for "unused" or "not specified" else ABORT_IF(factorIndex >= (size_t)factorShape_[g] - 1, "Factor index out of range"); @@ -145,21 +166,41 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu return Word::fromWordIndex(index); } -// like factors2word, except that factors are expressed as global unit indices, and result is just the WordIndex -// This is only used during initialization, so it's OK if it is a little inefficient. -Word FactoredVocab::factorUnits2word(const std::vector& factorUnits) { - // convert to fully unrolled factors representation - std::vector factorIndices(getNumGroups(), FACTOR_NOT_APPLICABLE); // default for unused factors - for (auto u : factorUnits) { - auto g = factorGroups_[u]; // convert u to relative u within factor group range - ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); - auto factorIndex = u - groupRanges_[g].first; - factorIndices[g] = factorIndex; +Word FactoredVocab::lemma2Word(size_t factor0Index) const { + size_t numGroups = getNumGroups(); + std::vector factorIndices; + factorIndices.reserve(numGroups); + factorIndices.push_back(factor0Index); + for (size_t g = 1; g < numGroups; g++) { + auto index = lemmaHasFactorGroup(factor0Index, g) ? FACTOR_NOT_SPECIFIED : FACTOR_NOT_APPLICABLE; + factorIndices.push_back(index); } return factors2word(factorIndices); } -void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) { +// replace a factor that is FACTOR_NOT_SPECIFIED by a specified one +// This is used in beam search, where factors are searched one after another. +Word FactoredVocab::addFactor(Word word, size_t groupIndex, size_t factorIndex) const { + ABORT_IF(groupIndex == 0, "Cannot add or change lemma in a partial Word"); + ABORT_IF(!isFactorValid(factorIndex), "Cannot add unspecified or n/a factor to a partial Word"); + std::vector factorIndices; + word2factors(word, factorIndices); + auto factor0Index = factorIndices[0]; + ABORT_IF(!isFactorValid(factor0Index), "Cannot add factor to a partial Word without lemma"); + ABORT_IF(factorIndices[groupIndex] == FACTOR_NOT_APPLICABLE, "Cannot add a factor that the lemma does not have"); + ABORT_IF(factorIndices[groupIndex] != FACTOR_NOT_SPECIFIED, "Cannot modify a specified factor in a partial Word"); + factorIndices[groupIndex] = factorIndex; + return factors2word(factorIndices); +} + +size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { + auto g = factorGroups_[u]; // convert u to relative u within factor group range + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + return u - groupRanges_[g].first; +} + + +void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) const { size_t numGroups = getNumGroups(); factorIndices.resize(numGroups); for (size_t g = 0; g < numGroups; g++) { @@ -189,17 +230,30 @@ std::string FactoredVocab::word2string(Word word) { return res + ")"; } -size_t FactoredVocab::getFactor(Word word, size_t groupIndex) { +size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { size_t index = word.toWordIndex(); index = index / factorStrides_[groupIndex]; index = index % (size_t)factorShape_[groupIndex]; - if (index == (size_t)factorShape_[groupIndex] - 1) { - index = FACTOR_NOT_APPLICABLE; // @BUGBUG: We should check here which one it is. + if (index == (size_t)factorShape_[groupIndex] - 1) { // special sentinel value for unspecified or not-applicable + if (groupIndex == 0) // lemma itself is always applicable, hence 'not specified' + index = FACTOR_NOT_SPECIFIED; + else { // not lemma: check whether lemma of word has this factor group + size_t factor0Index = word.toWordIndex() / factorStrides_[0]; + if (lemmaHasFactorGroup(factor0Index, groupIndex)) + index = FACTOR_NOT_SPECIFIED; + else + index = FACTOR_NOT_APPLICABLE; + } + } + else { // regular value: consistency check if lemma really has this factor group + size_t factor0Index = word.toWordIndex() / factorStrides_[0]; + ABORT_IF(factor0Index == (size_t)factorShape_[0] - 1, "Word has specified factor but no lemma??"); + ABORT_IF(!lemmaHasFactorGroup(factor0Index, groupIndex), "Word has a specified factor for a lemma that does not have that factor group??"); } return index; } -std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) { +std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) const { word; groupIndex; ABORT("Not implemented"); } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index de2bd34f1..a5aefc726 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -50,10 +50,13 @@ class FactoredVocab : public IVocab { const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 // convert representations - Word factors2word(const std::vector& factors); - void word2factors(Word word, std::vector& factors); - size_t getFactor(Word word, size_t groupIndex); - std::pair getFactorUnit(Word word, size_t groupIndex); + Word factors2word(const std::vector& factors) const; + void word2factors(Word word, std::vector& factors) const; + Word lemma2Word(size_t factor0Index) const; + Word addFactor(Word word, size_t groupIndex, size_t factor0Index) const; + size_t getFactor(Word word, size_t groupIndex) const; + std::pair getFactorUnit(Word word, size_t groupIndex) const; + bool lemmaHasFactorGroup(size_t factor0Index, size_t g) const { return lemmaHasFactorGroup_[factor0Index][g]; } static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); static constexpr size_t FACTOR_NOT_SPECIFIED = (SIZE_MAX - 2); static bool isFactorValid(size_t factorIndex) { return factorIndex < FACTOR_NOT_SPECIFIED; } @@ -63,7 +66,7 @@ class FactoredVocab : public IVocab { void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); - Word factorUnits2word(const std::vector& factorUnits); + size_t factorUnit2FactorIndex(WordIndex u) const; std::string word2string(Word word); private: class WordLUT { // map between strings and WordIndex @@ -94,6 +97,7 @@ class FactoredVocab : public IVocab { CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. + std::vector>lemmaHasFactorGroup_; // [factor 0 index][g] -> true if lemma has factor group Shape factorShape_; // [g] number of factors in each factor group std::vector factorStrides_; // [g] stride for factor dimension //std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 1ebd87f5a..4157df3ad 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -139,6 +139,7 @@ namespace marian { // use first factor of each word to determine whether it has a specific factor std::vector Logits::getFactorMasks(const Words& words, size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 + words; factorGroup; std::vector res; ABORT("FINISH THIS"); return res; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 36836acc7..ef5fa32f9 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -41,7 +41,7 @@ class BeamSearch { const size_t beamSize, const bool first, Ptr batch, // for alignments only - Ptr factoredVocab) const { + Ptr factoredVocab, size_t factorGroup) const { std::vector align; if(options_->hasAndNotEmpty("alignment")) align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble @@ -78,7 +78,12 @@ class BeamSearch { if (shortlist) word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); else + // @TODO: implant factor here into existing one--what is the wordIdx? + // - factoredVocab->expandFactoredWord(beam[beamHypIdx]->getWord(), factorIndex, groupIndex) + // - if groupIndex = 0 then create a new partially set factor tuple with a lemma, and all others unspecified or not applicable + // - if additional factor, then add it in if applicable word = Word::fromWordIndex(wordIdx); + factoredVocab; factorGroup; auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); @@ -287,8 +292,9 @@ class BeamSearch { // in hyps with some factors set to FACTOR_NOT_SPECIFIED. // TODO: // - we did not rearrange the tensors in the decoder model's state - // - initial word should set lemma by all other factors as unspecified - // - toHyp() should implant factors + // - toHyps(): + // - initial word should set lemma by all other factors as unspecified + // - implant factors for subsequent words auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); } @@ -339,7 +345,7 @@ class BeamSearch { states, // used for keeping track of per-ensemble-member path score localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples /*first=*/t == 0, // used to indicate originating beamSize of 1 - batch, factoredVocab); + batch, factoredVocab, factorGroup); // remove all hyps that end in EOS // The position of a hyp in the beam may change. From ff6b158ede57319670002a8325bba6dfb4eb0444 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 20:35:08 -0800 Subject: [PATCH 317/838] toHyp() now builds factored words --- src/data/factored_vocab.cpp | 2 +- src/data/factored_vocab.h | 2 +- src/translator/beam_search.h | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 2390adf63..6bafb559e 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -180,7 +180,7 @@ Word FactoredVocab::lemma2Word(size_t factor0Index) const { // replace a factor that is FACTOR_NOT_SPECIFIED by a specified one // This is used in beam search, where factors are searched one after another. -Word FactoredVocab::addFactor(Word word, size_t groupIndex, size_t factorIndex) const { +Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const { ABORT_IF(groupIndex == 0, "Cannot add or change lemma in a partial Word"); ABORT_IF(!isFactorValid(factorIndex), "Cannot add unspecified or n/a factor to a partial Word"); std::vector factorIndices; diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index a5aefc726..67a6c3be5 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -53,7 +53,7 @@ class FactoredVocab : public IVocab { Word factors2word(const std::vector& factors) const; void word2factors(Word word, std::vector& factors) const; Word lemma2Word(size_t factor0Index) const; - Word addFactor(Word word, size_t groupIndex, size_t factor0Index) const; + Word expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const; size_t getFactor(Word word, size_t groupIndex) const; std::pair getFactorUnit(Word word, size_t groupIndex) const; bool lemmaHasFactorGroup(size_t factor0Index, size_t g) const { return lemmaHasFactorGroup_[factor0Index][g]; } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index ef5fa32f9..ed152b9c0 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -77,13 +77,16 @@ class BeamSearch { auto shortlist = scorers_[0]->getShortlist(); if (shortlist) word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); + else if (factoredVocab) { + // For factored decoding, the word is built over multiple decoding steps, + // starting with the lemma, then adding factors one by one. + if (factorGroup == 0) + word = factoredVocab->lemma2Word(wordIdx); + else + word = factoredVocab->expandFactoredWord(beam[beamHypIdx]->getPrevHyp()->getWord(), factorGroup, wordIdx); + } else - // @TODO: implant factor here into existing one--what is the wordIdx? - // - factoredVocab->expandFactoredWord(beam[beamHypIdx]->getWord(), factorIndex, groupIndex) - // - if groupIndex = 0 then create a new partially set factor tuple with a lemma, and all others unspecified or not applicable - // - if additional factor, then add it in if applicable word = Word::fromWordIndex(wordIdx); - factoredVocab; factorGroup; auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); @@ -171,6 +174,8 @@ class BeamSearch { factoredVocab.reset(); #endif size_t numFactorGroups = factoredVocab ? factoredVocab->getNumGroups() : 1; + if (numFactorGroups == 1) // if no factors then reset + factoredVocab.reset(); const int dimBatch = (int)batch->size(); @@ -292,9 +297,6 @@ class BeamSearch { // in hyps with some factors set to FACTOR_NOT_SPECIFIED. // TODO: // - we did not rearrange the tensors in the decoder model's state - // - toHyps(): - // - initial word should set lemma by all other factors as unspecified - // - implant factors for subsequent words auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); } From 2f96d49b5969aa1d8a6fc4f1f8ba1b44d8a2235d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 8 Feb 2019 22:08:25 -0800 Subject: [PATCH 318/838] added lots of checks to word/factor conversions; new methods lemma2Word(), addFactor() --- src/data/factored_vocab.cpp | 9 +++++---- src/data/factored_vocab.h | 2 +- src/layers/generic.cpp | 15 +++++++++++++-- src/layers/generic.h | 1 + src/models/costs.h | 7 +------ src/translator/beam_search.h | 5 +++-- 6 files changed, 24 insertions(+), 15 deletions(-) mode change 100644 => 100755 src/models/costs.h diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 6bafb559e..fa4fca2e8 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -181,6 +181,7 @@ Word FactoredVocab::lemma2Word(size_t factor0Index) const { // replace a factor that is FACTOR_NOT_SPECIFIED by a specified one // This is used in beam search, where factors are searched one after another. Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const { + LOG(info, "expand {} + [{}]={}", word2string(word), groupIndex, factorIndex); ABORT_IF(groupIndex == 0, "Cannot add or change lemma in a partial Word"); ABORT_IF(!isFactorValid(factorIndex), "Cannot add unspecified or n/a factor to a partial Word"); std::vector factorIndices; @@ -190,7 +191,8 @@ Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t fact ABORT_IF(factorIndices[groupIndex] == FACTOR_NOT_APPLICABLE, "Cannot add a factor that the lemma does not have"); ABORT_IF(factorIndices[groupIndex] != FACTOR_NOT_SPECIFIED, "Cannot modify a specified factor in a partial Word"); factorIndices[groupIndex] = factorIndex; - return factors2word(factorIndices); + word = factors2word(factorIndices); + LOG(info, "to {}", word2string(word)); } size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { @@ -213,7 +215,7 @@ void FactoredVocab::word2factors(Word word, std::vector& factorIndices / #endif } -std::string FactoredVocab::word2string(Word word) { +std::string FactoredVocab::word2string(Word word) const { std::vector factorIndices; word2factors(word, factorIndices); std::string res; @@ -232,13 +234,13 @@ std::string FactoredVocab::word2string(Word word) { size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { size_t index = word.toWordIndex(); + size_t factor0Index = index / factorStrides_[0]; index = index / factorStrides_[groupIndex]; index = index % (size_t)factorShape_[groupIndex]; if (index == (size_t)factorShape_[groupIndex] - 1) { // special sentinel value for unspecified or not-applicable if (groupIndex == 0) // lemma itself is always applicable, hence 'not specified' index = FACTOR_NOT_SPECIFIED; else { // not lemma: check whether lemma of word has this factor group - size_t factor0Index = word.toWordIndex() / factorStrides_[0]; if (lemmaHasFactorGroup(factor0Index, groupIndex)) index = FACTOR_NOT_SPECIFIED; else @@ -246,7 +248,6 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { } } else { // regular value: consistency check if lemma really has this factor group - size_t factor0Index = word.toWordIndex() / factorStrides_[0]; ABORT_IF(factor0Index == (size_t)factorShape_[0] - 1, "Word has specified factor but no lemma??"); ABORT_IF(!lemmaHasFactorGroup(factor0Index, groupIndex), "Word has a specified factor for a lemma that does not have that factor group??"); } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 67a6c3be5..0d10f7278 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -67,7 +67,7 @@ class FactoredVocab : public IVocab { void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); size_t factorUnit2FactorIndex(WordIndex u) const; - std::string word2string(Word word); + std::string word2string(Word word) const; private: class WordLUT { // map between strings and WordIndex std::map str2index_; diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 4157df3ad..2686de20f 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -139,12 +139,23 @@ namespace marian { // use first factor of each word to determine whether it has a specific factor std::vector Logits::getFactorMasks(const Words& words, size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 - words; factorGroup; std::vector res; - ABORT("FINISH THIS"); + res.reserve(words.size()); + for (const auto& word : words) { + auto lemma = factoredVocab_->getFactor(word, 0); + res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); + } return res; } + Logits Logits::applyUnaryFunction(const std::function& f) const { // clone this but apply f to all loss values + std::vector> newLogits; + for (const auto& l : logits_) + newLogits.emplace_back(New(f(l->loss()), l->count())); + return Logits(std::move(newLogits), factoredVocab_); + } + + // @TODO: code dup with above; we can merge it into applyToRationalLoss() Logits Logits::withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ std::vector> newLogits; for (const auto& l : logits_) diff --git a/src/layers/generic.h b/src/layers/generic.h index 85dec5692..4e25be784 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -76,6 +76,7 @@ class Logits { Expr getFactoredLogits(size_t groupIndex) const; // get logits for only one factor group Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; + Logits applyUnaryFunction(const std::function& f) const; // clone this but apply f to all loss values struct MaskedFactorIndices { std::vector indices; // factor index, or 0 if masked diff --git a/src/models/costs.h b/src/models/costs.h old mode 100644 new mode 100755 index 98f9300fc..2819e13a6 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -172,12 +172,7 @@ class LogSoftmaxStep : public CostStep { public: virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) - // @TODO: @HACK must know about individual parts; make it a loop - auto logits = state->getLogProbs().getLogits(); - - auto logprobs = logsoftmax(logits); - - state->setLogProbs(logprobs); + state->setLogProbs(state->getLogProbs().applyUnaryFunction(logsoftmax)); return state; } }; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index ed152b9c0..ee86cda11 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -59,6 +59,7 @@ class BeamSearch { const auto wordIdx = (WordIndex)(key % vocabSize); const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); + LOG(info, "key = (batch {}, beam {}, word {})", batchIdx, beamHypIdx, wordIdx); ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); @@ -170,7 +171,7 @@ class BeamSearch { ABORT_IF(batch->back()->vocab()->getEosId() != trgEosId_, "Batch uses different EOS token than was passed to BeamSearch originally"); auto factoredVocab = batch->back()->vocab()->tryAs(); -#if 1 // use '1' here to disable factored decoding, e.g. for comparisons +#if 0 // use '1' here to disable factored decoding, e.g. for comparisons factoredVocab.reset(); #endif size_t numFactorGroups = factoredVocab ? factoredVocab->getNumGroups() : 1; @@ -332,7 +333,7 @@ class BeamSearch { // perform beam search // find N best amongst the (localBeamSize * dimVocab) hypotheses - std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened + std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] From 53c1dbe28732e5acb699b1c30367550d668a6df2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 9 Feb 2019 09:27:09 -0800 Subject: [PATCH 319/838] first step to actually include factor scores; word2string() can now print invalid items. Please igonore last commit, was am accident. --- src/data/factored_vocab.cpp | 25 +++++++++++++++++-------- src/data/factored_vocab.h | 4 +++- src/translator/beam_search.h | 16 +++++++++++++--- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index fa4fca2e8..85818ac71 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -193,6 +193,7 @@ Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t fact factorIndices[groupIndex] = factorIndex; word = factors2word(factorIndices); LOG(info, "to {}", word2string(word)); + return word; } size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { @@ -216,18 +217,25 @@ void FactoredVocab::word2factors(Word word, std::vector& factorIndices / } std::string FactoredVocab::word2string(Word word) const { - std::vector factorIndices; - word2factors(word, factorIndices); - std::string res; + // this function has some code dup, so that we can bypass some checks for debugging size_t numGroups = getNumGroups(); + size_t factor0Index = word.toWordIndex() / factorStrides_[0]; + std::string res; for (size_t g = 0; g < numGroups; g++) { res.append(res.empty() ? "(" : ", "); - auto factorIndex = factorIndices[g]; - switch (factorIndex) { - case FACTOR_NOT_APPLICABLE: res.append("n/a"); break; - case FACTOR_NOT_SPECIFIED: res.append("?"); break; - default: res.append(factorVocab_[(WordIndex)(factorIndex + groupRanges_[g].first)]); break; + size_t index = word.toWordIndex(); + index = index / factorStrides_[g]; + index = index % (size_t)factorShape_[g]; + if (index == (size_t)factorShape_[g] - 1) { // special sentinel value for unspecified or not-applicable + if (factor0Index >= (size_t)factorShape_[0]) + res.append("(lemma oob)"); + else if (lemmaHasFactorGroup(factor0Index, g)) + res.append("?"); + else + res.append("n/a"); } + else + res.append(factorVocab_[(WordIndex)(index + groupRanges_[g].first)]); } return res + ")"; } @@ -299,6 +307,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { } /*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { + LOG(info, "Looking up Word {}={}", id.toWordIndex(), word2string(id)); return vocab_[id.toWordIndex()]; } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 0d10f7278..c5581932a 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -54,20 +54,22 @@ class FactoredVocab : public IVocab { void word2factors(Word word, std::vector& factors) const; Word lemma2Word(size_t factor0Index) const; Word expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const; + bool canExpandFactoredWord(Word word, size_t groupIndex) const { return lemmaHasFactorGroup(getFactor(word, 0), groupIndex); } size_t getFactor(Word word, size_t groupIndex) const; std::pair getFactorUnit(Word word, size_t groupIndex) const; bool lemmaHasFactorGroup(size_t factor0Index, size_t g) const { return lemmaHasFactorGroup_[factor0Index][g]; } + static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); static constexpr size_t FACTOR_NOT_SPECIFIED = (SIZE_MAX - 2); static bool isFactorValid(size_t factorIndex) { return factorIndex < FACTOR_NOT_SPECIFIED; } static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab + std::string word2string(Word word) const; // (diagnostics only) private: void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); void constructNormalizationInfoForVocab(); size_t factorUnit2FactorIndex(WordIndex u) const; - std::string word2string(Word word) const; private: class WordLUT { // map between strings and WordIndex std::map str2index_; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index ee86cda11..aa74757b1 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -81,10 +81,18 @@ class BeamSearch { else if (factoredVocab) { // For factored decoding, the word is built over multiple decoding steps, // starting with the lemma, then adding factors one by one. - if (factorGroup == 0) + if (factorGroup == 0) { word = factoredVocab->lemma2Word(wordIdx); - else - word = factoredVocab->expandFactoredWord(beam[beamHypIdx]->getPrevHyp()->getWord(), factorGroup, wordIdx); + LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + } + else { + LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + word = beam[beamHypIdx]->getWord(); + if (factoredVocab->canExpandFactoredWord(word, factorGroup)) + word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); + // @TODO: maybe factor the two above into a single function; for now, I want the extra checks + } } else word = Word::fromWordIndex(wordIdx); @@ -298,6 +306,8 @@ class BeamSearch { // in hyps with some factors set to FACTOR_NOT_SPECIFIED. // TODO: // - we did not rearrange the tensors in the decoder model's state + for (auto word : prevWords) + LOG(info, "prevWords[]={}", factoredVocab->word2string(word)); auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); } From bed1f902a165b052544216d8abec21add29ab1e1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 9 Feb 2019 09:36:27 -0800 Subject: [PATCH 320/838] (comments) --- src/translator/beam_search.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index aa74757b1..a9fbd668c 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -217,6 +217,14 @@ class BeamSearch { histories[i]->add(beams[i], trgEosId_); // the decoder maintains the following state: + // - beams : array [dimBatch] of array [localBeamSize] of Hypothesis + // - current output time step's set of active hypotheses, aka active search space + // - gets replaced at the end of each output time step + // - states[.] : ScorerState + // - NN state + // - one per scorer, e.g. 2 for ensemble of 2 + // - gets replaced in each output time step + // and it forms the following return value // - histories : array [dimBatch] of History // with History : vector [t] of array [localBeamSize] of Hypothesis // with Hypothesis : (last word, aggregate score, prev Hypothesis) @@ -224,19 +232,12 @@ class BeamSearch { // - stores traceback information // - gets added to in each output time step // - the final version is the return value of this function - // - beams : array [dimBatch] of array [localBeamSize] of Hypothesis - // - current output time step's set of active hypotheses, aka active search space - // - gets replaced at the end of each output time step - // - states[.] : ScorerState - // - NN state - // - one per scorer, e.g. 2 for ensemble of 2 - // - gets replaced at the end of each output time step // main loop over output time steps for (size_t t = 0; ; t++) { ABORT_IF(dimBatch != beams.size(), "Lost a batch entry??"); - // for factored vocabs, we do one factor at a time, but without updating the decoder model for secondary factors + // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors auto factorGroup = t % numFactorGroups; // determine beam size for next output time step, as max over still-active sentences @@ -286,9 +287,9 @@ class BeamSearch { Expr logProbs, factorMasks; if (factorGroup == 0) { // compute output probabilities for current output time step - // - uses hypIndices[index in beam, 1, batch index, 1] to reorder decoder state to reflect the top-N in beams[][] - // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history - // - performs one step of the decoder model + // - uses hypIndices[index in beam, 1, batch index, 1] to reorder scorer state to reflect the top-N in beams[][] + // - adds prevWords [index in beam, 1, batch index, 1] to the scorer's target history + // - performs one step of the scorer // - returns new NN state for use in next output time step // - returns vector of prediction probabilities over output vocab via newState // update state in-place for next output time step @@ -305,7 +306,7 @@ class BeamSearch { // - factors are incorporated one step at a time; so we will have temporary Word entries // in hyps with some factors set to FACTOR_NOT_SPECIFIED. // TODO: - // - we did not rearrange the tensors in the decoder model's state + // - we did not rearrange the tensors in the scorer's state for (auto word : prevWords) LOG(info, "prevWords[]={}", factoredVocab->word2string(word)); auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); From 675a60875a8fcbc70862df352a0353948ec7d100 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 9 Feb 2019 14:35:19 -0800 Subject: [PATCH 321/838] towards seleccting the correct pred hyp in factored beam search --- src/data/factored_vocab.cpp | 14 +++++--- src/layers/generic.cpp | 9 ++++-- src/layers/generic.h | 2 +- src/rnn/types.h | 2 +- src/translator/beam_search.h | 62 +++++++++++++++++++++--------------- 5 files changed, 55 insertions(+), 34 deletions(-) mode change 100644 => 100755 src/rnn/types.h diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 85818ac71..c23c4d1dc 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -306,9 +306,15 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return getUnkId(); } -/*virtual*/ const std::string& FactoredVocab::operator[](Word id) const /*override final*/ { - LOG(info, "Looking up Word {}={}", id.toWordIndex(), word2string(id)); - return vocab_[id.toWordIndex()]; +/*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*overrworde final*/ { + LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); +#if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor + if (vocab_.isGap(word.toWordIndex())) { + LOG(info, "Factor combination {} missing in external dict, generating fake entry", word2string(word)); + const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); + } +#endif + return vocab_[word.toWordIndex()]; } /*virtual*/ size_t FactoredVocab::size() const /*override final*/ { @@ -382,7 +388,7 @@ WordIndex FactoredVocab::WordLUT::add(const std::string& word, WordIndex index) } const std::string& FactoredVocab::WordLUT::operator[](WordIndex index) const { const auto& word = index2str_[index]; - ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); + //ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); return word; } WordIndex FactoredVocab::WordLUT::operator[](const std::string& word) const { diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2686de20f..19ea9895d 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -3,6 +3,7 @@ #include "layers/generic.h" #include "layers/loss.h" #include "data/factored_vocab.h" +#include "rnn/types.h" // for State::select() using std::size_t; // not sure why this is needed @@ -77,9 +78,13 @@ namespace marian { } // get logits for one factor group - Expr Logits::getFactoredLogits(size_t groupIndex) const { + Expr Logits::getFactoredLogits(size_t groupIndex, const std::vector& selIdx /*= {}*/, size_t beamSize /*= 0*/) const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); - return logits_[groupIndex]->loss(); + auto sel = logits_[groupIndex]->loss(); // [localBeamSize, 1, dimBatch, dimFactorVocab] + // if selIdx are given, then we must reshuffle accordingly + if (!selIdx.empty()) // use the same function that shuffles decoder state + sel = rnn::State::select(sel, selIdx, (int)beamSize, /*isBatchMajor=*/false); + return sel; } // This function assumes that the object holds one or more factor logits, which are summed up diff --git a/src/layers/generic.h b/src/layers/generic.h index 4e25be784..08d5cbbb9 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -73,7 +73,7 @@ class Logits { Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), factoredVocab_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors - Expr getFactoredLogits(size_t groupIndex) const; // get logits for only one factor group + Expr getFactoredLogits(size_t groupIndex, const std::vector& hypIndices = {}, size_t beamSize = 0) const; // get logits for only one factor group, with optional reshuffle Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; Logits applyUnaryFunction(const std::function& f) const; // clone this but apply f to all loss values diff --git a/src/rnn/types.h b/src/rnn/types.h old mode 100644 new mode 100755 index bd44b98bb..47424b759 --- a/src/rnn/types.h +++ b/src/rnn/types.h @@ -18,7 +18,7 @@ struct State { select(cell, selIdx, beamSize, isBatchMajor) }; } -private: + // this function is also called by Logits static Expr select(Expr sel, // [beamSize, dimTime, dimBatch, dimDepth] or [beamSize, dimBatch, dimTime, dimDepth] (dimTime = 1 for RNN) const std::vector& selIdx, // [beamIndex * activeBatchSize + batchIndex] int beamSize, bool isBatchMajor) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index a9fbd668c..2b241e7f3 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -72,6 +72,7 @@ class BeamSearch { ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value in key??"); // map wordIdx to word + auto prevHyp = beam[beamHypIdx]; Word word; // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. @@ -92,12 +93,13 @@ class BeamSearch { if (factoredVocab->canExpandFactoredWord(word, factorGroup)) word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); // @TODO: maybe factor the two above into a single function; for now, I want the extra checks + prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } } else word = Word::fromWordIndex(wordIdx); - auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); + auto hyp = New(prevHyp, word, beamHypIdx, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { @@ -237,9 +239,6 @@ class BeamSearch { for (size_t t = 0; ; t++) { ABORT_IF(dimBatch != beams.size(), "Lost a batch entry??"); - // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors - auto factorGroup = t % numFactorGroups; - // determine beam size for next output time step, as max over still-active sentences // E.g. if all batch entries are down from beam 5 to no more than 4 surviving hyps, then // switch to beam of 4 for all. If all are done, then beam ends up being 0, and we are done. @@ -252,6 +251,11 @@ class BeamSearch { if (localBeamSize == 0) break; + // BEGIN FOR factorGroup = 0 .. numFactorGroups-1 + // @TODO: use an explicit nested loop here for factors + // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors + auto factorGroup = t % numFactorGroups; + //********************************************************************** // create constant containing previous path scores for current beam // Also create mapping of hyp indices, for reordering the decoder-state tensors. @@ -294,31 +298,31 @@ class BeamSearch { // - returns vector of prediction probabilities over output vocab via newState // update state in-place for next output time step states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); + if (numFactorGroups == 1) + logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] + else + logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] } else { - // add secondary factors - // For those, we don't update the decoder-model state in any way. - // Instead, we just keep expanding with the factors. - // Considerations: - // - not all scores should get a factor - // We need a [localBeamSize, 1, dimBatch, 1] tensor that knows whether a factor is applicable - // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. - // - factors are incorporated one step at a time; so we will have temporary Word entries - // in hyps with some factors set to FACTOR_NOT_SPECIFIED. - // TODO: - // - we did not rearrange the tensors in the scorer's state - for (auto word : prevWords) - LOG(info, "prevWords[]={}", factoredVocab->word2string(word)); - auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); - factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); + // add secondary factors + // For those, we don't update the decoder-model state in any way. + // Instead, we just keep expanding with the factors. + // Considerations: + // - not all scores should get a factor + // We need a [localBeamSize, 1, dimBatch, 1] tensor that knows whether a factor is applicable + // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. + // - factors are incorporated one step at a time; so we will have temporary Word entries + // in hyps with some factors set to FACTOR_NOT_SPECIFIED. + // TODO: + // - we did not rearrange the tensors in the scorer's state + logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] + for (auto word : prevWords) + LOG(info, "prevWords[]={}", factoredVocab->word2string(word)); + auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); + factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); + logProbs = logProbs * factorMasks; // those hyps that don't have a factor get multiplied with 0 } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] - if (numFactorGroups == 1) - logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] - else - logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] - if (factorMasks) - logProbs = logProbs * factorMasks; // those hyps that don't have a factor get multiplied with 0 expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; } @@ -361,11 +365,17 @@ class BeamSearch { /*first=*/t == 0, // used to indicate originating beamSize of 1 batch, factoredVocab, factorGroup); + if (factorGroup != numFactorGroups - 1) { // skip adding to history and skip purging for partially factored words + beams = newBeams; + continue; + } + // END FOR factorGroup = 0 .. numFactorGroups-1 + // remove all hyps that end in EOS // The position of a hyp in the beam may change. const auto purgedNewBeams = purgeBeams(newBeams); - // add updated search space (newBeams) to search grid (histories) for traceback + // add updated search space (newBeams) to our return value bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { // if this batch entry has surviving hyps then add them to the traceback grid From 5b7334d9049af88b36b44baeb79ff96cdf5b6ac7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 9 Feb 2019 22:29:10 -0800 Subject: [PATCH 322/838] towards correct hyp index in Hypothesis --- src/translator/beam_search.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 2b241e7f3..3dcdad616 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -73,6 +73,7 @@ class BeamSearch { // map wordIdx to word auto prevHyp = beam[beamHypIdx]; + auto prevBeamHypIdx = beamHypIdx; Word word; // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. @@ -93,13 +94,14 @@ class BeamSearch { if (factoredVocab->canExpandFactoredWord(word, factorGroup)) word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); // @TODO: maybe factor the two above into a single function; for now, I want the extra checks + prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } } else word = Word::fromWordIndex(wordIdx); - auto hyp = New(prevHyp, word, beamHypIdx, pathScore); + auto hyp = New(prevHyp, word, prevBeamHypIdx, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { From dd0a691974b13e5b391eb127954a9955f93f84ba Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 10 Feb 2019 13:44:28 -0800 Subject: [PATCH 323/838] two minor optimizations; temporarily prints scores in output now --- src/graph/expression_operators.cpp | 2 ++ src/translator/beam_search.h | 4 ++++ src/translator/output_printer.h | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 5c37beef3..731220489 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -78,6 +78,8 @@ Expr softmax(Expr a, Expr zeroOneMask, int axis /*=-1*/) { } Expr logsoftmax(Expr a) { + if (a->type() == "logsoftmax") // logsoftmax(logsoftmax(x)) == logsoftmax(x) + return a; return Expression(a); } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 3dcdad616..1df16b0e3 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -94,6 +94,8 @@ class BeamSearch { if (factoredVocab->canExpandFactoredWord(word, factorGroup)) word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); // @TODO: maybe factor the two above into a single function; for now, I want the extra checks + else + ABORT_IF(prevHyp->getPathScore() != pathScore, "Score changed despite factor not applicable??"); prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } @@ -331,6 +333,8 @@ class BeamSearch { // make beams continuous if(dimBatch > 1 && localBeamSize > 1) expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] + else // (avoid copy if we can) + expandedPathScores = reshape(expandedPathScores, {dimBatch, 1, (int)localBeamSize, expandedPathScores->shape()[-1]}); // -> [dimBatch, 1, localBeamSize, dimVocab] // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] // perform NN computation diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h index 83ef9c0ca..3b8827f9a 100755 --- a/src/translator/output_printer.h +++ b/src/translator/output_printer.h @@ -71,6 +71,9 @@ class OutputPrinter { const auto& hypo = std::get<1>(result); best1 << " ||| " << getAlignment(hypo); } + + best1 << " |sc=" << std::get<1>(result)->getPathScore(); + best1 << std::flush; } From 3a347bf12ab5f3e552b8db73b831b1ddefc72c0f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 10 Feb 2019 14:22:21 -0800 Subject: [PATCH 324/838] added lookahead for lemma score in decoding --- src/layers/generic.cpp | 29 +++++++++++++++++++++++++++++ src/layers/generic.h | 1 + src/translator/beam_search.h | 1 + 3 files changed, 31 insertions(+) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 19ea9895d..9bd91876c 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -81,6 +81,23 @@ namespace marian { Expr Logits::getFactoredLogits(size_t groupIndex, const std::vector& selIdx /*= {}*/, size_t beamSize /*= 0*/) const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); auto sel = logits_[groupIndex]->loss(); // [localBeamSize, 1, dimBatch, dimFactorVocab] + + + // normalize for decoding: + // - all secondary factors: subtract their max + // - lemma: add all maxes of applicable factors + if (groupIndex > 0) { + sel = sel - max(sel, -1); + } + else { + auto numGroups = getNumFactorGroups(); + for (size_t g = 1; g < numGroups; g++) { + auto factorMaxima = max(logits_[g]->loss(), -1); + auto factorMasks = constant(getFactorMasks(g)); + sel = sel + factorMaxima * factorMasks; // those lemmas that don't have a factor get multiplied with 0 + } + } + // if selIdx are given, then we must reshuffle accordingly if (!selIdx.empty()) // use the same function that shuffles decoder state sel = rnn::State::select(sel, selIdx, (int)beamSize, /*isBatchMajor=*/false); @@ -153,6 +170,18 @@ namespace marian { return res; } + // same but for lemma + std::vector Logits::getFactorMasks(size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 + size_t numLemmas = factoredVocab_->getGroupRange(0).second - factoredVocab_->getGroupRange(0).first; + std::vector res; + res.reserve(numLemmas); + // @TODO: we should rearange lemmaHasFactorGroup as vector[groups[lemma] of float; then move this into FactoredVocab + for (size_t lemma = 0; lemma < numLemmas; lemma++) { + res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); + } + return res; + } + Logits Logits::applyUnaryFunction(const std::function& f) const { // clone this but apply f to all loss values std::vector> newLogits; for (const auto& l : logits_) diff --git a/src/layers/generic.h b/src/layers/generic.h index 08d5cbbb9..9158fbd82 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -88,6 +88,7 @@ class Logits { }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices std::vector getFactorMasks(const Words& words, size_t factorGroup) const; + std::vector getFactorMasks(size_t factorGroup) const; float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 1df16b0e3..f366a284c 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -356,6 +356,7 @@ class BeamSearch { // find N best amongst the (localBeamSize * dimVocab) hypotheses std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened + // @TODO: getNBestList() API is redundant; input dimensions are known from expandedPathScores(); but no way to specify target N different from input N getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] /*out*/ nBestPathScores, /*out*/ nBestKeys, From d1bf38ebf4678da686336b0057b84b4df0b5ac3f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 17:31:30 -0800 Subject: [PATCH 325/838] no longer expanding non-applicable factors, as that blows stuff out of the beam --- src/common/logging.cpp | 3 ++- src/data/factored_vocab.cpp | 2 +- src/translator/beam_search.h | 49 ++++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 9 deletions(-) mode change 100644 => 100755 src/common/logging.cpp diff --git a/src/common/logging.cpp b/src/common/logging.cpp old mode 100644 new mode 100755 index 57b55de96..9c4b2951f --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -77,7 +77,8 @@ void createLoggers(const marian::Config* config) { } bool quiet = config && config->get("quiet"); - Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; + //Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; + Logger general{createStderrLogger("general", "%v", generalLogs, quiet)}; Logger valid{createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; if(config && config->has("log-level")) { diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index c23c4d1dc..1279b5fc4 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -307,7 +307,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { } /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*overrworde final*/ { - LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); + //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); #if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { LOG(info, "Factor combination {} missing in external dict, generating fake entry", word2string(word)); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index f366a284c..fbbc8ec2d 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -95,7 +95,8 @@ class BeamSearch { word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); // @TODO: maybe factor the two above into a single function; for now, I want the extra checks else - ABORT_IF(prevHyp->getPathScore() != pathScore, "Score changed despite factor not applicable??"); + continue; // skip if word does not have this factor + //ABORT_IF(prevHyp->getPathScore() != pathScore, "Score changed despite factor not applicable??"); prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } @@ -124,6 +125,30 @@ class BeamSearch { newBeam.push_back(hyp); } + + // also propagate factored hypotheses that do not get expanded in this step as they don't have this factor + if (factorGroup > 0) { + for (size_t batchIdx = 0; batchIdx < beams.size(); batchIdx++) { + const auto& beam = beams[batchIdx]; + auto& newBeam = newBeams[batchIdx]; + for (const auto& beamHyp : beam) { + auto word = beamHyp->getWord(); + LOG(info, "Checking {}", factoredVocab->word2string(word)); + if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above + continue; + LOG(info, "Expanded {}", factoredVocab->word2string(word)); + newBeam.push_back(beamHyp); + } + if (newBeam.size() > beamSize) { + LOG(info, "Size {}, sorting...", newBeam.size()); + std::nth_element(newBeam.begin(), newBeam.begin() + beamSize, newBeam.end(), [](Ptr a, Ptr b) { + return a->getPathScore() > b->getPathScore(); + }); + LOG(info, "Size {}, sorted...", newBeam.size()); + newBeam.resize(beamSize); // @TODO: needed? + } + } + } return newBeams; } @@ -266,10 +291,11 @@ class BeamSearch { std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) + std::vector prevScores; // @TODO: remove here again if(t == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { - std::vector prevScores; + //std::vector prevScores; for(size_t beamHypIdx = 0; beamHypIdx < localBeamSize; ++beamHypIdx) { for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { // loop over batch entries (active sentences) auto& beam = beams[batchIdx]; @@ -291,8 +317,8 @@ class BeamSearch { //********************************************************************** // compute expanded path scores with word prediction probs from all scorers auto expandedPathScores = prevPathScores; // will become [localBeamSize, 1, dimBatch, dimVocab] + Expr logProbs; for(size_t i = 0; i < scorers_.size(); ++i) { - Expr logProbs, factorMasks; if (factorGroup == 0) { // compute output probabilities for current output time step // - uses hypIndices[index in beam, 1, batch index, 1] to reorder scorer state to reflect the top-N in beams[][] @@ -301,11 +327,16 @@ class BeamSearch { // - returns new NN state for use in next output time step // - returns vector of prediction probabilities over output vocab via newState // update state in-place for next output time step + if (t > 0) for (size_t kk = 0; kk < prevWords.size(); kk++) + LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, + factoredVocab ? factoredVocab->word2string(prevWords[kk]) : (*batch->back()->vocab())[prevWords[kk]], + prevScores[kk]); states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); if (numFactorGroups == 1) logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] else logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] + logProbs->debug("logProbs"); } else { // add secondary factors @@ -320,14 +351,17 @@ class BeamSearch { // TODO: // - we did not rearrange the tensors in the scorer's state logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] - for (auto word : prevWords) - LOG(info, "prevWords[]={}", factoredVocab->word2string(word)); + for (size_t kk = 0; kk < prevWords.size(); kk++) + LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); - factorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); - logProbs = logProbs * factorMasks; // those hyps that don't have a factor get multiplied with 0 + for (auto& m : factorMaskVector) + m = m ? 0.f : -9999.f; // block hyps that do not have the factor; these are short-circuited directly + auto logFactorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); + logProbs = logProbs + logFactorMasks; // those hyps that don't have a factor get multiplied with 0 } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; + logProbs->debug("logProbs"); } // make beams continuous @@ -336,6 +370,7 @@ class BeamSearch { else // (avoid copy if we can) expandedPathScores = reshape(expandedPathScores, {dimBatch, 1, (int)localBeamSize, expandedPathScores->shape()[-1]}); // -> [dimBatch, 1, localBeamSize, dimVocab] // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] + expandedPathScores->debug("expandedPathScores"); // perform NN computation if(t == 0) From 462820ab2ddea751fbdb9ba2935e5f64155f29da Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 19:03:03 -0800 Subject: [PATCH 326/838] fixed a bounds check --- src/translator/beam_search.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index fbbc8ec2d..f553d3bf3 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -19,6 +19,8 @@ class BeamSearch { Word trgEosId_{Word::NONE}; Word trgUnkId_{Word::NONE}; + static constexpr auto INVALID_PATH_SCORE = -9999;// .0f; + public: BeamSearch(Ptr options, const std::vector>& scorers, @@ -69,7 +71,9 @@ class BeamSearch { if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? continue; - ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value in key??"); + if (beamHypIdx >= (int)beam.size() && pathScore <= INVALID_PATH_SCORE) // (dummy slot) + continue; + ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value {} in key?? word={}, batch={}, pathScore={}", beamHypIdx, wordIdx, batchIdx, pathScore); // map wordIdx to word auto prevHyp = beam[beamHypIdx]; @@ -136,7 +140,7 @@ class BeamSearch { LOG(info, "Checking {}", factoredVocab->word2string(word)); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above continue; - LOG(info, "Expanded {}", factoredVocab->word2string(word)); + LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } if (newBeam.size() > beamSize) { @@ -306,8 +310,8 @@ class BeamSearch { prevScores.push_back(hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); - prevWords.push_back(Word::ZERO); // (unused) - prevScores.push_back(-9999); + prevWords.push_back(trgEosId_); // (unused, but must be valid) + prevScores.push_back(INVALID_PATH_SCORE); } } } @@ -355,7 +359,7 @@ class BeamSearch { LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); for (auto& m : factorMaskVector) - m = m ? 0.f : -9999.f; // block hyps that do not have the factor; these are short-circuited directly + m = m ? 0.f : INVALID_PATH_SCORE; // block hyps that do not have the factor; these are short-circuited directly auto logFactorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); logProbs = logProbs + logFactorMasks; // those hyps that don't have a factor get multiplied with 0 } From 202b610f6f5f1bf9f33bbe9c122dc63f6ba097a0 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 19:53:16 -0800 Subject: [PATCH 327/838] commented out lots of debyug logging --- src/data/factored_vocab.cpp | 4 ++-- src/translator/beam_search.h | 34 ++++++++++++++++----------------- src/translator/output_printer.h | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 1279b5fc4..ea076af6a 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -181,7 +181,7 @@ Word FactoredVocab::lemma2Word(size_t factor0Index) const { // replace a factor that is FACTOR_NOT_SPECIFIED by a specified one // This is used in beam search, where factors are searched one after another. Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const { - LOG(info, "expand {} + [{}]={}", word2string(word), groupIndex, factorIndex); + //LOG(info, "expand {} + [{}]={}", word2string(word), groupIndex, factorIndex); ABORT_IF(groupIndex == 0, "Cannot add or change lemma in a partial Word"); ABORT_IF(!isFactorValid(factorIndex), "Cannot add unspecified or n/a factor to a partial Word"); std::vector factorIndices; @@ -192,7 +192,7 @@ Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t fact ABORT_IF(factorIndices[groupIndex] != FACTOR_NOT_SPECIFIED, "Cannot modify a specified factor in a partial Word"); factorIndices[groupIndex] = factorIndex; word = factors2word(factorIndices); - LOG(info, "to {}", word2string(word)); + //LOG(info, "to {}", word2string(word)); return word; } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index f553d3bf3..e82a910a2 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -61,7 +61,7 @@ class BeamSearch { const auto wordIdx = (WordIndex)(key % vocabSize); const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); - LOG(info, "key = (batch {}, beam {}, word {})", batchIdx, beamHypIdx, wordIdx); + //LOG(info, "key = (batch {}, beam {}, word {})", batchIdx, beamHypIdx, wordIdx); ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); @@ -89,11 +89,11 @@ class BeamSearch { // starting with the lemma, then adding factors one by one. if (factorGroup == 0) { word = factoredVocab->lemma2Word(wordIdx); - LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); } else { - LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), - factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); @@ -137,18 +137,18 @@ class BeamSearch { auto& newBeam = newBeams[batchIdx]; for (const auto& beamHyp : beam) { auto word = beamHyp->getWord(); - LOG(info, "Checking {}", factoredVocab->word2string(word)); + //LOG(info, "Checking {}", factoredVocab->word2string(word)); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above continue; LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } if (newBeam.size() > beamSize) { - LOG(info, "Size {}, sorting...", newBeam.size()); + //LOG(info, "Size {}, sorting...", newBeam.size()); std::nth_element(newBeam.begin(), newBeam.begin() + beamSize, newBeam.end(), [](Ptr a, Ptr b) { - return a->getPathScore() > b->getPathScore(); + return a->getPathScore() > b->getPathScore(); // (sort highest score first) }); - LOG(info, "Size {}, sorted...", newBeam.size()); + //LOG(info, "Size {}, sorted...", newBeam.size()); newBeam.resize(beamSize); // @TODO: needed? } } @@ -331,16 +331,16 @@ class BeamSearch { // - returns new NN state for use in next output time step // - returns vector of prediction probabilities over output vocab via newState // update state in-place for next output time step - if (t > 0) for (size_t kk = 0; kk < prevWords.size(); kk++) - LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, - factoredVocab ? factoredVocab->word2string(prevWords[kk]) : (*batch->back()->vocab())[prevWords[kk]], - prevScores[kk]); + //if (t > 0) for (size_t kk = 0; kk < prevWords.size(); kk++) + // LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, + // factoredVocab ? factoredVocab->word2string(prevWords[kk]) : (*batch->back()->vocab())[prevWords[kk]], + // prevScores[kk]); states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); if (numFactorGroups == 1) logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] else logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] - logProbs->debug("logProbs"); + //logProbs->debug("logProbs"); } else { // add secondary factors @@ -355,8 +355,8 @@ class BeamSearch { // TODO: // - we did not rearrange the tensors in the scorer's state logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] - for (size_t kk = 0; kk < prevWords.size(); kk++) - LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); + //for (size_t kk = 0; kk < prevWords.size(); kk++) + // LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); for (auto& m : factorMaskVector) m = m ? 0.f : INVALID_PATH_SCORE; // block hyps that do not have the factor; these are short-circuited directly @@ -365,7 +365,7 @@ class BeamSearch { } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; - logProbs->debug("logProbs"); + //logProbs->debug("logProbs"); } // make beams continuous @@ -374,7 +374,7 @@ class BeamSearch { else // (avoid copy if we can) expandedPathScores = reshape(expandedPathScores, {dimBatch, 1, (int)localBeamSize, expandedPathScores->shape()[-1]}); // -> [dimBatch, 1, localBeamSize, dimVocab] // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] - expandedPathScores->debug("expandedPathScores"); + //expandedPathScores->debug("expandedPathScores"); // perform NN computation if(t == 0) diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h index 3b8827f9a..e2fc97fee 100755 --- a/src/translator/output_printer.h +++ b/src/translator/output_printer.h @@ -72,7 +72,7 @@ class OutputPrinter { best1 << " ||| " << getAlignment(hypo); } - best1 << " |sc=" << std::get<1>(result)->getPathScore(); + //best1 << " |sc=" << std::get<1>(result)->getPathScore(); best1 << std::flush; } From c2232e9046c581ebc06dd4c2db8b44d648cd406e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 20:12:21 -0800 Subject: [PATCH 328/838] factored decoder now uses an explicit inner for loop over factors --- src/translator/beam_search.h | 40 ++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index e82a910a2..496fbb27e 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -140,7 +140,7 @@ class BeamSearch { //LOG(info, "Checking {}", factoredVocab->word2string(word)); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above continue; - LOG(info, "Forwarded {}", factoredVocab->word2string(word)); + //LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } if (newBeam.size() > beamSize) { @@ -284,10 +284,11 @@ class BeamSearch { if (localBeamSize == 0) break; + for (size_t factorGroup = 0; factorGroup < numFactorGroups; factorGroup++) { // BEGIN FOR factorGroup = 0 .. numFactorGroups-1 // @TODO: use an explicit nested loop here for factors // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors - auto factorGroup = t % numFactorGroups; + //auto factorGroup = t % numFactorGroups; //********************************************************************** // create constant containing previous path scores for current beam @@ -296,7 +297,7 @@ class BeamSearch { std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) std::vector prevScores; // @TODO: remove here again - if(t == 0) { // no scores yet + if(t == 0 && factorGroup == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { //std::vector prevScores; @@ -377,7 +378,7 @@ class BeamSearch { //expandedPathScores->debug("expandedPathScores"); // perform NN computation - if(t == 0) + if(t == 0 && factorGroup == 0) graph->forward(); else graph->forwardNext(); @@ -399,36 +400,31 @@ class BeamSearch { getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] /*out*/ nBestPathScores, /*out*/ nBestKeys, - /*first=*/t == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? + /*first=*/t == 0 && factorGroup == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? // Now, nBestPathScores contain N-best expandedPathScores, and nBestKeys for each their original location (batchIdx, beamHypIdx, word). // combine N-best sets with existing search space (beams) to updated search space - auto newBeams = toHyps(nBestKeys, nBestPathScores, - /*dimTrgVoc=*/expandedPathScores->shape()[-1], - beams, - states, // used for keeping track of per-ensemble-member path score - localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples - /*first=*/t == 0, // used to indicate originating beamSize of 1 - batch, factoredVocab, factorGroup); - - if (factorGroup != numFactorGroups - 1) { // skip adding to history and skip purging for partially factored words - beams = newBeams; - continue; - } - // END FOR factorGroup = 0 .. numFactorGroups-1 + beams = toHyps(nBestKeys, nBestPathScores, + /*dimTrgVoc=*/expandedPathScores->shape()[-1], + beams, + states, // used for keeping track of per-ensemble-member path score + localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples + /*first=*/t == 0 && factorGroup == 0, // used to indicate originating beamSize of 1 + batch, factoredVocab, factorGroup); + } // END FOR factorGroup = 0 .. numFactorGroups-1 // remove all hyps that end in EOS // The position of a hyp in the beam may change. - const auto purgedNewBeams = purgeBeams(newBeams); + const auto purgedNewBeams = purgeBeams(beams); - // add updated search space (newBeams) to our return value + // add updated search space (beams) to our return value bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { // if this batch entry has surviving hyps then add them to the traceback grid - if(!newBeams[i].empty()) { + if(!beams[i].empty()) { if (histories[i]->size() >= options_->get("max-length-factor") * batch->front()->batchWidth()) maxLengthReached = true; - histories[i]->add(newBeams[i], trgEosId_, purgedNewBeams[i].empty() || maxLengthReached); + histories[i]->add(beams[i], trgEosId_, purgedNewBeams[i].empty() || maxLengthReached); } } if (maxLengthReached) // early exit if max length limit was reached From 4f11b0100ed82626627b1cc1de1c8756ea1e7c68 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 20:23:14 -0800 Subject: [PATCH 329/838] no longer need a log mask for logProbs, done with INVALID_PATH_SCORE instead --- src/layers/generic.cpp | 26 +++++++++++++------------- src/layers/generic.h | 2 +- src/translator/beam_search.h | 15 +++++++-------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 9bd91876c..df06398b0 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -159,19 +159,19 @@ namespace marian { return res; } - // use first factor of each word to determine whether it has a specific factor - std::vector Logits::getFactorMasks(const Words& words, size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 - std::vector res; - res.reserve(words.size()); - for (const auto& word : words) { - auto lemma = factoredVocab_->getFactor(word, 0); - res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); - } - return res; - } - - // same but for lemma - std::vector Logits::getFactorMasks(size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 + //// use first factor of each word to determine whether it has a specific factor + //std::vector Logits::getFactorMasks(const Words& words, size_t factorGroup) const { // 1.0 for words that do have this factor; else 0 + // std::vector res; + // res.reserve(words.size()); + // for (const auto& word : words) { + // auto lemma = factoredVocab_->getFactor(word, 0); + // res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); + // } + // return res; + //} + + // return a vector of 1 or 0 indicating for each lemma whether it has a specific factor + std::vector Logits::getFactorMasks(size_t factorGroup) const { // [lemmaIndex] -> 1.0 for words that do have this factor; else 0 size_t numLemmas = factoredVocab_->getGroupRange(0).second - factoredVocab_->getGroupRange(0).first; std::vector res; res.reserve(numLemmas); diff --git a/src/layers/generic.h b/src/layers/generic.h index 9158fbd82..532324641 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -87,7 +87,7 @@ class Logits { MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices - std::vector getFactorMasks(const Words& words, size_t factorGroup) const; + //std::vector getFactorMasks(const Words& words, size_t factorGroup) const; std::vector getFactorMasks(size_t factorGroup) const; float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' size_t getNumFactorGroups() const { return logits_.size(); } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 496fbb27e..1caa9821e 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -308,7 +308,7 @@ class BeamSearch { auto hyp = beam[beamHypIdx]; hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(hyp->getWord()); - prevScores.push_back(hyp->getPathScore()); + prevScores.push_back((factoredVocab && !factoredVocab->canExpandFactoredWord(hyp->getWord(), factorGroup)) ? INVALID_PATH_SCORE : hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); prevWords.push_back(trgEosId_); // (unused, but must be valid) @@ -353,16 +353,15 @@ class BeamSearch { // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. // - factors are incorporated one step at a time; so we will have temporary Word entries // in hyps with some factors set to FACTOR_NOT_SPECIFIED. - // TODO: - // - we did not rearrange the tensors in the scorer's state logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] + // Note: pathScores was set to INVALID_PATH_SCORE if this path cannot expanded by this factor; toHyps() handles that special case on the side //for (size_t kk = 0; kk < prevWords.size(); kk++) // LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); - auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); - for (auto& m : factorMaskVector) - m = m ? 0.f : INVALID_PATH_SCORE; // block hyps that do not have the factor; these are short-circuited directly - auto logFactorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); - logProbs = logProbs + logFactorMasks; // those hyps that don't have a factor get multiplied with 0 + //auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); + //for (auto& m : factorMaskVector) + // m = m ? 0.f : INVALID_PATH_SCORE; // block hyps that do not have the factor; these are short-circuited directly + //auto logFactorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); + //logProbs = logProbs + logFactorMasks; // those hyps that don't have a factor get multiplied with 0 } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; From b9a0856df8828ef5b5fea0e3dd9d729257d370e2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 20:36:13 -0800 Subject: [PATCH 330/838] removed a redundant check for applicability of a factor in toHyps() --- src/translator/beam_search.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 1caa9821e..6dc378399 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -71,8 +71,10 @@ class BeamSearch { if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? continue; - if (beamHypIdx >= (int)beam.size() && pathScore <= INVALID_PATH_SCORE) // (dummy slot) + if (pathScore <= INVALID_PATH_SCORE) // (dummy slot or word that cannot be expanded by current factor) continue; + //if (beamHypIdx >= (int)beam.size() && pathScore <= INVALID_PATH_SCORE) // (dummy slot) + // continue; ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value {} in key?? word={}, batch={}, pathScore={}", beamHypIdx, wordIdx, batchIdx, pathScore); // map wordIdx to word @@ -95,11 +97,12 @@ class BeamSearch { //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); - if (factoredVocab->canExpandFactoredWord(word, factorGroup)) + ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); + //if (factoredVocab->canExpandFactoredWord(word, factorGroup)) word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); // @TODO: maybe factor the two above into a single function; for now, I want the extra checks - else - continue; // skip if word does not have this factor + //else + // continue; // skip if word does not have this factor //ABORT_IF(prevHyp->getPathScore() != pathScore, "Score changed despite factor not applicable??"); prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words From a48f1daf0175f3c5757b539750bcf2d0084d43bb Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 11 Feb 2019 21:03:28 -0800 Subject: [PATCH 331/838] add comment --- src/tensors/allocator.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tensors/allocator.h b/src/tensors/allocator.h index 1e402ba67..6a5f883e9 100644 --- a/src/tensors/allocator.h +++ b/src/tensors/allocator.h @@ -126,6 +126,7 @@ class Allocator { throw AllocationException(available_, size); } + // @TODO: compact memory before re-allocation attempt, maybe by left shifting memory over currently largest gap while(it == gaps_.end()) { grow(step_); it = std::lower_bound(gaps_.begin(), gaps_.end(), Gap(nullptr, size)); From 9fb1e795ef65f227bffbeebabefed1034e9f5eb8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Feb 2019 21:41:58 -0800 Subject: [PATCH 332/838] now skips if no factor expansion is possible in one step --- src/data/factored_vocab.h | 2 +- src/translator/beam_search.h | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index c5581932a..3a562fadb 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -44,7 +44,7 @@ class FactoredVocab : public IVocab { const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v --only used in getLogits() size_t getNumGroups() const { return groupRanges_.size(); } - std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) + std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) //const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g //const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 6dc378399..3ff0b695b 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -299,9 +299,11 @@ class BeamSearch { std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) + bool anyCanExpand = false; // stays false if all hyps are invalid factor expansions std::vector prevScores; // @TODO: remove here again if(t == 0 && factorGroup == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); + anyCanExpand = true; } else { //std::vector prevScores; for(size_t beamHypIdx = 0; beamHypIdx < localBeamSize; ++beamHypIdx) { @@ -309,9 +311,12 @@ class BeamSearch { auto& beam = beams[batchIdx]; if(beamHypIdx < beam.size()) { auto hyp = beam[beamHypIdx]; + auto word = hyp->getWord(); + auto canExpand = (!factoredVocab || factoredVocab->canExpandFactoredWord(hyp->getWord(), factorGroup)); + anyCanExpand |= canExpand; hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation - prevWords .push_back(hyp->getWord()); - prevScores.push_back((factoredVocab && !factoredVocab->canExpandFactoredWord(hyp->getWord(), factorGroup)) ? INVALID_PATH_SCORE : hyp->getPathScore()); + prevWords .push_back(word); + prevScores.push_back(canExpand ? hyp->getPathScore() : INVALID_PATH_SCORE); } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); prevWords.push_back(trgEosId_); // (unused, but must be valid) @@ -321,6 +326,8 @@ class BeamSearch { } prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(prevScores)); } + if (!anyCanExpand) // all words cannot expand this factor: skip + continue; //********************************************************************** // compute expanded path scores with word prediction probs from all scorers @@ -340,7 +347,7 @@ class BeamSearch { // factoredVocab ? factoredVocab->word2string(prevWords[kk]) : (*batch->back()->vocab())[prevWords[kk]], // prevScores[kk]); states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); - if (numFactorGroups == 1) + if (numFactorGroups == 1) // @TODO: this branch can go away logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] else logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] From d36b6f572aac5e8ffece84f76511848c809abb00 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 11 Feb 2019 23:04:17 -0800 Subject: [PATCH 333/838] move regression test pointer --- regression-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-tests b/regression-tests index febdc3f56..142eadddb 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit febdc3f56f75929b1f7b5b38a4b9b96ea8f648e7 +Subproject commit 142eadddbe04493c1024b42586030b72e9cb7ea2 From 4cbbfaa4867d85568405b772d0493c6b8307efc0 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 11:27:27 -0800 Subject: [PATCH 334/838] pulled some further changes to beam_search.h from fseide/factoredembeddings --- src/translator/beam_search.h | 202 ++++++++++++++++------------------- 1 file changed, 94 insertions(+), 108 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index f825b8531..86da1e56a 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -18,6 +18,8 @@ class BeamSearch { Word trgEosId_ = (Word)-1; Word trgUnkId_ = (Word)-1; + static constexpr auto INVALID_PATH_SCORE = -9999; + public: BeamSearch(Ptr options, const std::vector>& scorers, @@ -40,78 +42,66 @@ class BeamSearch { const size_t beamSize, const bool first, Ptr batch) const { - const auto dimBatch = beams.size(); - Beams newBeams(dimBatch); - std::vector align; if(options_->hasAndNotEmpty("alignment")) - // Use alignments from the first scorer, even if ensemble - align = scorers_[0]->getAlignment(); + align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble + + const auto dimBatch = beams.size(); + Beams newBeams(dimBatch); for(size_t i = 0; i < nBestKeys.size(); ++i) { // [dimBatch, beamSize] flattened - // Keys contains indices to vocab items in the entire beam. - // Values can be between 0 and beamSize * vocabSize. - const auto batchIdx = i / beamSize; // and i % beamSize is the beam hyp index + // Keys encode batchIdx, beamHypIdx, and word index in the entire beam. + // They can be between 0 and beamSize * vocabSize-1. + const auto key = nBestKeys[i]; + const float pathScore = nBestPathScores[i]; // expanded path score for (batchIdx, beamHypIdx, word) + + // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) + const auto wordIdx = (Word)(key % vocabSize); + const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); + const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); + + ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); + const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; - if(newBeam.size() < beam.size()) { - const float pathScore = nBestPathScores[i]; - const auto key = nBestKeys[i]; // key = pathScore's location, as ((batchIdx, beamHypIdx) flattened, word idx) flattened - - // decompose key into individual indices - Word wordIdx = (Word)(key % vocabSize); - const auto hypIdx = (key / vocabSize); -#if 1 - // further decompose hypIdx, taking into account that the very first entry had beam size 1 - // and compose a new hypIdx that assumes actual beamSize - const auto keyBatchIdx = hypIdx / (first ? 1 : beamSize); - const auto keyBeamHypIdx = hypIdx % (first ? 1 : beamSize); - const auto hypIdxTrans = keyBeamHypIdx * dimBatch + keyBatchIdx; - ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); // not possible, as beamSize = max(beams[.].size()) -#else - const auto keyBatchIdx = hypIdx / beamSize; // @REVIEW: is this actually keyBatchIdx? - size_t keyBeamHypIdx = hypIdx % beamSize; - - auto hypIdxTrans = keyBatchIdx + keyBeamHypIdx * dimBatch; - if(first) - hypIdxTrans = hypIdx; // == keyBeamHypIdx + keyBatchIdx * beamSize? or was beamSize=1, and keyBeamHypIdx = 0? - - ABORT_IF(keyBeamHypIdx >= (int)beam.size(), "Beam hyp index exceeds beam size??"); - //if(keyBeamHypIdx >= (int)beam.size()) // @TODO: What is this condition? Cf. keyBeamHypIdx = hypIdx % beamSize; beamSize = max(beams[.].size()) - // keyBeamHypIdx = keyBeamHypIdx % beam.size(); - - if(first) - keyBeamHypIdx = 0; -#endif - // Retrieve short list for final softmax (based on words aligned - // to source sentences). If short list has been set, map the indices - // in the sub-selected vocabulary matrix back to their original positions. - auto shortlist = scorers_[0]->getShortlist(); - if(shortlist) - wordIdx = shortlist->reverseMap(wordIdx); // @TODO: should reverseMap accept a size_t or a Word? - // now wordIdx is a regular Word again - - auto hyp = New(beam[keyBeamHypIdx], wordIdx, (IndexType)hypIdxTrans, pathScore); - - // Set score breakdown for n-best lists - if(options_->get("n-best")) { - std::vector breakDown(states.size(), 0); - beam[keyBeamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? - for(size_t j = 0; j < states.size(); ++j) { - size_t key1 = hypIdxTrans * vocabSize + wordIdx; - breakDown[j] = states[j]->breakDown(key1) + beam[keyBeamHypIdx]->getScoreBreakdown()[j]; - } - hyp->setScoreBreakdown(breakDown); - } + if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? + continue; + if (pathScore <= INVALID_PATH_SCORE) // (unused slot) + continue; + + ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx??"); - // Set alignments - if(!align.empty()) { - hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)keyBeamHypIdx, (int)batchIdx)); + // Map wordIdx to word + Word word; + // If short list has been set, then wordIdx is an index into the short-listed word set, + // rather than the true word index. + auto shortlist = scorers_[0]->getShortlist(); + if (shortlist) + word = shortlist->reverseMap(wordIdx); + else + word = wordIdx; + + auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); + + // Set score breakdown for n-best lists + if(options_->get("n-best")) { + std::vector breakDown(states.size(), 0); + beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? + for(size_t j = 0; j < states.size(); ++j) { + size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' + breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; + // @TODO: pass those 3 indices directly into breakDown (state knows the dimensions) } + hyp->setScoreBreakdown(breakDown); + } - newBeam.push_back(hyp); + // Set alignments + if(!align.empty()) { + hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); } + + newBeam.push_back(hyp); } return newBeams; } @@ -170,6 +160,9 @@ class BeamSearch { //********************************************************************** // main decoding function Histories search(Ptr graph, Ptr batch) { + ABORT_IF(batch->back()->vocab() && batch->back()->vocab()->getEosId() != trgEosId_, + "Batch uses different EOS token than was passed to BeamSearch originally"); + const int dimBatch = (int)batch->size(); auto getNBestList = createGetNBestListFn(beamSize_, dimBatch, graph->getDeviceId()); @@ -200,21 +193,15 @@ class BeamSearch { for(int i = 0; i < dimBatch; ++i) histories[i]->add(beams[i], trgEosId_); - // the decoder maintains the following state: - // - histories : array [dimBatch] of History - // with History : vector [t] of array [localBeamSize] of Hypothesis - // with Hypothesis : (last word, aggregate score, prev Hypothesis) - // - search grid - // - stores traceback information - // - gets added to in each output time step - // - the final version is the return value of this function - // - beams : array [dimBatch] of array [localBeamSize] of Hypothesis + // the decoder updates the following state information in each output time step: + // - beams: array [dimBatch] of array [localBeamSize] of Hypothesis // - current output time step's set of active hypotheses, aka active search space - // - gets replaced at the end of each output time step - // - states[.] : ScorerState - // - NN state - // - one per scorer, e.g. 2 for ensemble of 2 - // - gets replaced at the end of each output time step + // - states[.]: ScorerState + // - NN state; one per scorer, e.g. 2 for ensemble of 2 + // and it forms the following return value + // - histories: array [dimBatch] of History + // with History: vector [t] of array [localBeamSize] of Hypothesis + // with Hypothesis: (last word, aggregate score, prev Hypothesis) // main loop over output time steps for (size_t t = 0; ; t++) { @@ -233,57 +220,54 @@ class BeamSearch { //********************************************************************** // create constant containing previous path scores for current beam - // Also create mapping of hyp indices, which are not 1:1 if sentences complete. - std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) index of hyp that the new top N originated from - std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) predecessor word - Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], where the last axis broadcasts into vocab size when adding expandedPathScores + // Also create mapping of hyp indices, for reordering the decoder-state tensors. + std::vector hypIndices; // [localBeamsize, 1, dimBatch, 1] (flattened) tensor index ((beamHypIdx, batchIdx), flattened) of prev hyp that a hyp originated from + std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history + Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) if(t == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); } else { std::vector prevScores; - for(size_t beamIndex = 0; beamIndex < localBeamSize; ++beamIndex) { - for(int batchIndex = 0; batchIndex < dimBatch; ++batchIndex) { // loop over batch entries (active sentences) - auto& beam = beams[batchIndex]; - if(beamIndex < beam.size()) { - auto hyp = beam[beamIndex]; - hypIndices.push_back((IndexType)hyp->getPrevStateIndex()); // index where to find prev hyp (beamHypIdx, batchIdx), =beamHypIdx * dimBatch + batchIdx + for(size_t beamHypIdx = 0; beamHypIdx < localBeamSize; ++beamHypIdx) { + for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { // loop over batch entries (active sentences) + auto& beam = beams[batchIdx]; + if(beamHypIdx < beam.size()) { + auto hyp = beam[beamHypIdx]; + hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(hyp->getWord()); prevScores.push_back(hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); - prevWords .push_back(Word{}); // (unused) - prevScores.push_back(-9999); + prevWords.push_back(trgEosId_); // (unused, but let's use a valid value) + prevScores.push_back(INVALID_PATH_SCORE); } } } - prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, - inits::from_vector(prevScores)); + prevPathScores = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(prevScores)); } //********************************************************************** // compute expanded path scores with word prediction probs from all scorers auto expandedPathScores = prevPathScores; // will become [localBeamSize, 1, dimBatch, dimVocab] + Expr logProbs; for(size_t i = 0; i < scorers_.size(); ++i) { // compute output probabilities for current output time step - // - uses hypIndices[index in beam, 1, batch index, 1] to reorder hypotheses - // - adds prevWords [index in beam, 1, batch index, 1] to the decoder model's target history - // - performs one step of the decoder model + // - uses hypIndices[index in beam, 1, batch index, 1] to reorder scorer state to reflect the top-N in beams[][] + // - adds prevWords [index in beam, 1, batch index, 1] to the scorer's target history + // - performs one step of the scorer // - returns new NN state for use in next output time step // - returns vector of prediction probabilities over output vocab via newState - auto newState = scorers_[i]->step( - graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); - + states[i] = scorers_[i]->step(graph, states[i], hypIndices, prevWords, dimBatch, (int)localBeamSize); + logProbs = states[i]->getLogProbs(); // [localBeamSize, 1, dimBatch, dimVocab] // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] - expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * newState->getLogProbs(); - - // update state in-place for next output time step - states[i] = newState; + expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; } // make beams continuous - expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] - //if(dimBatch > 1 && localBeamSize > 1) - // expandedPathScores = transpose(expandedPathScores, {2, 1, 0, 3}); // -> [dimBatch, 1, localBeamSize, dimVocab] + if(dimBatch > 1 && localBeamSize > 1) + expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] + else // (avoid copy if we can) + expandedPathScores = reshape(expandedPathScores, {dimBatch, 1, (int)localBeamSize, expandedPathScores->shape()[-1]}); // perform NN computation if(t == 0) @@ -302,13 +286,15 @@ class BeamSearch { // perform beam search // find N best amongst the (localBeamSize * dimVocab) hypotheses - std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened + std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened + // @TODO: getNBestList() API is redundant; input dimensions are known from expandedPathScores(); but no way to specify target N different from input N getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] /*out*/ nBestPathScores, /*out*/ nBestKeys, /*first=*/t == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? - // Now, nBestPathScores contain N-best expandedPathScores, and nBestKeys for each their original location (batchIdx, beamHypIdx, word). + // Now, nBestPathScores contain N-best expandedPathScores for each batch and beam, + // and nBestKeys for each their original location (batchIdx, beamHypIdx, word). // combine N-best sets with existing search space (beams) to updated search space beams = toHyps(nBestKeys, nBestPathScores, @@ -321,23 +307,23 @@ class BeamSearch { // remove all hyps that end in EOS // The position of a hyp in the beam may change. - const auto purgedBeams = purgeBeams(beams); + const auto purgedNewBeams = purgeBeams(beams); - // add updated search space (beams) to search grid (histories) for traceback + // add updated search space (beams) to our return value bool maxLengthReached = false; for(int i = 0; i < dimBatch; ++i) { // if this batch entry has surviving hyps then add them to the traceback grid if(!beams[i].empty()) { if (histories[i]->size() >= options_->get("max-length-factor") * batch->front()->batchWidth()) maxLengthReached = true; - histories[i]->add(beams[i], trgEosId_, purgedBeams[i].empty() || maxLengthReached); + histories[i]->add(beams[i], trgEosId_, purgedNewBeams[i].empty() || maxLengthReached); } } if (maxLengthReached) // early exit if max length limit was reached break; // this is the search space for the next output time step - beams = purgedBeams; + beams = purgedNewBeams; } // end of main loop over output time steps return histories; // [dimBatch][t][N best hyps] From 168300c5d9b77b029d363b1117b79f56d071f680 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 12:06:59 -0800 Subject: [PATCH 335/838] towards cleanup of getNBestList() --- src/translator/beam_search.h | 15 +++---- src/translator/nth_element.cpp | 79 ++++++++++++++++++---------------- src/translator/nth_element.cu | 54 +++++++++++++---------- src/translator/nth_element.h | 4 +- 4 files changed, 84 insertions(+), 68 deletions(-) mode change 100644 => 100755 src/translator/nth_element.cpp mode change 100644 => 100755 src/translator/nth_element.cu mode change 100644 => 100755 src/translator/nth_element.h diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 86da1e56a..60659a22a 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -34,8 +34,8 @@ class BeamSearch { trgUnkId_(trgUnkId) {} // combine new expandedPathScores and previous beams into new set of beams - Beams toHyps(const std::vector nBestKeys, // [dimBatch, beamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened - const std::vector nBestPathScores, // [dimBatch, beamSize] flattened + Beams toHyps(const std::vector& nBestKeys, // [dimBatch, beamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened + const std::vector& nBestPathScores, // [dimBatch, beamSize] flattened const size_t vocabSize, const Beams& beams, const std::vector>& states, @@ -70,7 +70,7 @@ class BeamSearch { if (pathScore <= INVALID_PATH_SCORE) // (unused slot) continue; - ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx??"); + ABORT_IF(beamHypIdx >= beam.size(), "Out of bounds beamHypIdx??"); // Map wordIdx to word Word word; @@ -82,7 +82,7 @@ class BeamSearch { else word = wordIdx; - auto hyp = New(beam[beamHypIdx], word, beamHypIdx, pathScore); + auto hyp = New(beam[beamHypIdx], word, (IndexType)beamHypIdx, pathScore); // Set score breakdown for n-best lists if(options_->get("n-best")) { @@ -239,7 +239,7 @@ class BeamSearch { } else { // pad to localBeamSize (dummy hypothesis) hypIndices.push_back(0); prevWords.push_back(trgEosId_); // (unused, but let's use a valid value) - prevScores.push_back(INVALID_PATH_SCORE); + prevScores.push_back((float)INVALID_PATH_SCORE); } } } @@ -288,9 +288,8 @@ class BeamSearch { // find N best amongst the (localBeamSize * dimVocab) hypotheses std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened - // @TODO: getNBestList() API is redundant; input dimensions are known from expandedPathScores(); but no way to specify target N different from input N - getNBestList(/*beamSizes=*/std::vector(dimBatch, localBeamSize), // output layout of (nBestPathScores, nBestKeys) --@REVIEW: correct? - /*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] + getNBestList(/*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] + /*N=*/localBeamSize, /*out*/ nBestPathScores, /*out*/ nBestKeys, /*first=*/t == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? // Now, nBestPathScores contain N-best expandedPathScores for each batch and beam, diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp old mode 100644 new mode 100755 index 549833230..dbfc0a70d --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -14,22 +14,18 @@ namespace marian { class NthElementCPU { std::vector h_res_idx; std::vector h_res; - size_t lastN; + size_t maxBeamSize_; + size_t maxBatchSize_; + //size_t lastN_; public: - NthElementCPU() = delete; + NthElementCPU() {} NthElementCPU(const NthElementCPU& copy) = delete; - NthElementCPU(size_t maxBeamSize, size_t maxBatchSize) { - size_t maxSize = maxBeamSize * maxBatchSize; - h_res.resize(maxSize); - h_res_idx.resize(maxSize); - } - private: - void getNBestList(float* scores, - const std::vector& batchFirstElementIdxs, - const std::vector& cumulativeBeamSizes) { + void selectNBest(float* scores, + const std::vector& batchFirstElementIdxs, + const std::vector& cumulativeBeamSizes) { /* For each batch, select the max N elements, where N is the beam size for * this batch. Locally record these elements (their current value and index * in 'scores') before updating each element to a large negative value, such @@ -62,23 +58,38 @@ class NthElementCPU { } public: - void getNBestList(const std::vector& beamSizes, - Tensor scores, - std::vector& outPathScores, - std::vector& outKeys, - const bool isFirst) { + // @BUGBUG: This API mixes input and output beam size. + void getNBestList(Tensor scores, // [dimBatch, 1, beamSize, dimVocab or dimShortlist] + size_t N, + std::vector& outPathScores, + std::vector& outKeys, + const bool isFirst) { + const auto vocabSize = scores->shape()[-1]; + const auto inputN = scores->shape()[-2]; + const auto dimBatch = scores->shape()[-4]; + ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); + ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size exceeds initialization parameter"); + ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); // @TODO: or inputN? + + const std::vector beamSizes(dimBatch, N); std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); std::vector batchFirstElementIdxs(beamSizes.size() + 1, 0); - auto vocabSize = scores->shape()[-1]; - for(int i = 0; i < beamSizes.size(); ++i) { - cumulativeBeamSizes[i + 1] = cumulativeBeamSizes[i] + (int)beamSizes[i]; - batchFirstElementIdxs[i + 1] - += (isFirst ? i + 1 : cumulativeBeamSizes[i + 1]) * vocabSize; + for(int batchIdx = 0; batchIdx < beamSizes.size(); ++batchIdx) { + cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + (int)beamSizes[batchIdx]; + ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); + batchFirstElementIdxs[batchIdx + 1] + += (isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; + ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); } - getNBestList(scores->data(), batchFirstElementIdxs, cumulativeBeamSizes); + size_t maxSize = N * dimBatch; + h_res.resize(maxSize); + h_res_idx.resize(maxSize); + + selectNBest(scores->data(), batchFirstElementIdxs, cumulativeBeamSizes); getPairs(cumulativeBeamSizes.back(), outKeys, outPathScores); + ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); } private: @@ -87,14 +98,14 @@ class NthElementCPU { std::vector& outValues) { std::copy(h_res_idx.begin(), h_res_idx.begin() + number, std::back_inserter(outKeys)); std::copy(h_res .begin(), h_res .begin() + number, std::back_inserter(outValues)); - lastN = number; + //lastN_ = number; } - void getValueByKey(std::vector& out, float* d_in) { - for(size_t i = 0; i < lastN; ++i) { - out[i] = d_in[h_res_idx[i]]; - } - } + //void getValueByKey(std::vector& out, float* d_in) { + // for(size_t i = 0; i < lastN_; ++i) { + // out[i] = d_in[h_res_idx[i]]; + // } + //} }; #ifdef CUDA_FOUND @@ -108,15 +119,11 @@ GetNBestListFn createGetNBestListFn(size_t beamSize, size_t dimBatch, DeviceId d if(deviceId.type == DeviceType::gpu) return createGetNBestListGPUFn(beamSize, dimBatch, deviceId); #else - deviceId; // (unused) + deviceId; beamSize; dimBatch; // (unused) #endif - auto nth = New(beamSize, dimBatch); - return [nth](const std::vector& beamSizes, - Tensor logProbs, - std::vector& outCosts, - std::vector& outKeys, - const bool isFirst) { - return nth->getNBestList(beamSizes, logProbs, outCosts, outKeys, isFirst); + auto nth = New(); + return [nth](Tensor logProbs, size_t N, std::vector& outCosts, std::vector& outKeys, const bool isFirst) { + return nth->getNBestList(logProbs, N, outCosts, outKeys, isFirst); }; } diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu old mode 100644 new mode 100755 index d5377f90c..ecf9daf5c --- a/src/translator/nth_element.cu +++ b/src/translator/nth_element.cu @@ -279,6 +279,7 @@ public: size_t maxBatchSize, DeviceId deviceId) : deviceId_(deviceId), + maxBeamSize_(maxBeamSize), maxBatchSize_(maxBatchSize), NUM_BLOCKS(std::min( 500, int(maxBeamSize* MAX_VOCAB_SIZE / (2 * BLOCK_SIZE)) @@ -316,9 +317,9 @@ public: } private: - void getNBestList(float* probs, - const std::vector& batchFirstElementIdxs, - const std::vector& cummulatedBeamSizes) { + void selectNBest(float* probs, + const std::vector& batchFirstElementIdxs, + const std::vector& cumulativeBeamSizes) { cudaSetDevice(deviceId_.no); CUDA_CHECK(cudaMemcpyAsync(d_batchPosition, batchFirstElementIdxs.data(), @@ -326,8 +327,8 @@ private: cudaMemcpyHostToDevice, /* stream_ */ 0)); CUDA_CHECK(cudaMemcpyAsync(d_cumBeamSizes, - cummulatedBeamSizes.data(), - cummulatedBeamSizes.size() * sizeof(int), + cumulativeBeamSizes.data(), + cumulativeBeamSizes.size() * sizeof(int), cudaMemcpyHostToDevice, /* stream_ */ 0)); @@ -353,26 +354,37 @@ private: } public: - void getNBestList(const std::vector& beamSizes, - Tensor Probs, + // @BUGBUG: This API mixes input and output beam size. + void getNBestList(Tensor scores, + size_t N, std::vector& outCosts, std::vector& outKeys, const bool isFirst) { cudaSetDevice(deviceId_.no); - std::vector cummulatedBeamSizes(beamSizes.size() + 1, 0); - std::vector batchFirstElementIdxs(beamSizes.size() + 1, 0); + const auto vocabSize = scores->shape()[-1]; + const auto inputN = scores->shape()[-2]; + const auto dimBatch = scores->shape()[-4]; + ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); // @TODO: Remove isFirst argument altogether + ABORT_IF(vocabSize > MAX_VOCAB_SIZE, "GetNBestList(): actual vocab size exceeds MAX_VOCAB_SIZE"); + ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size exceeds initialization parameter"); + ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); // @TODO: or inputN? - const size_t vocabSize = Probs->shape()[-1]; + const std::vector beamSizes(dimBatch, N); + std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); + std::vector batchFirstElementIdxs(beamSizes.size() + 1, 0); - for(size_t i = 0; i < beamSizes.size(); ++i) { - cummulatedBeamSizes[i + 1] = cummulatedBeamSizes[i] + beamSizes[i]; - batchFirstElementIdxs[i + 1] - += ((isFirst) ? (i + 1) : cummulatedBeamSizes[i + 1]) * vocabSize; + for(size_t batchIdx = 0; batchIdx < beamSizes.size(); ++batchIdx) { + cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + beamSizes[batchIdx]; + ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); + batchFirstElementIdxs[batchIdx + 1] + += ((isFirst) ? (batchIdx + 1) : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; + ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); } - getNBestList(Probs->data(), batchFirstElementIdxs, cummulatedBeamSizes); - getPairs(cummulatedBeamSizes.back(), outKeys, outCosts); + selectNBest(scores->data(), batchFirstElementIdxs, cumulativeBeamSizes); + getPairs(cumulativeBeamSizes.back(), outKeys, outCosts); + ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); } private: @@ -417,6 +429,8 @@ private: DeviceId deviceId_; const int MAX_VOCAB_SIZE = 100000; + size_t maxBeamSize_; + size_t maxBatchSize_; const int BLOCK_SIZE = 512; const int NUM_BLOCKS; @@ -440,12 +454,8 @@ private: // Returns a lambda with the same signature as the getNBestList() function. GetNBestListFn createGetNBestListGPUFn(size_t beamSize, size_t dimBatch, DeviceId deviceId) { auto nth = New(beamSize, dimBatch, deviceId); - return [nth](const std::vector& beamSizes, - Tensor logProbs, - std::vector& outCosts, - std::vector& outKeys, - const bool isFirst) { - return nth->getNBestList(beamSizes, logProbs, outCosts, outKeys, isFirst); + return [nth](Tensor logProbs, size_t N, std::vector& outCosts, std::vector& outKeys, const bool isFirst) { + return nth->getNBestList(logProbs, N, outCosts, outKeys, isFirst); }; } diff --git a/src/translator/nth_element.h b/src/translator/nth_element.h old mode 100644 new mode 100755 index 91ea3792b..ca325ed0d --- a/src/translator/nth_element.h +++ b/src/translator/nth_element.h @@ -10,8 +10,8 @@ namespace marian { -typedef std::function& beamSizes, - Tensor logProbs, +typedef std::function& outCosts, std::vector& outKeys, const bool isFirst)> GetNBestListFn; From 699e48700d43104ce62894aadd211838fb648e62 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 12:07:45 -0800 Subject: [PATCH 336/838] minor fix --- src/translator/nth_element.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp index dbfc0a70d..f99b0be42 100755 --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -14,8 +14,6 @@ namespace marian { class NthElementCPU { std::vector h_res_idx; std::vector h_res; - size_t maxBeamSize_; - size_t maxBatchSize_; //size_t lastN_; public: @@ -68,8 +66,6 @@ class NthElementCPU { const auto inputN = scores->shape()[-2]; const auto dimBatch = scores->shape()[-4]; ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); - ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size exceeds initialization parameter"); - ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); // @TODO: or inputN? const std::vector beamSizes(dimBatch, N); std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); From 43e389d7c703b7c393ff12827121f869b1a1e472 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 13:55:54 -0800 Subject: [PATCH 337/838] bug fix: reshape() must verify that #elements does not change; bug fix: beam search must reshape first step correctly --- src/graph/node_operators_unary.h | 2 ++ src/models/encoder_decoder.cpp | 5 ++--- src/translator/beam_search.h | 2 +- src/translator/nth_element.cpp | 21 ++++++++++----------- 4 files changed, 15 insertions(+), 15 deletions(-) mode change 100644 => 100755 src/models/encoder_decoder.cpp diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 190fa947f..c5141092b 100755 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -717,6 +717,8 @@ class ReshapeNodeOp : public UnaryNodeOp { public: ReshapeNodeOp(Expr a, Shape shape) : UnaryNodeOp(a, shape, a->value_type()), reshapee_(a) { + ABORT_IF(a->shape().elements() != shape.elements(), + "Reshape must not change the number of elements (from {} to {})", a->shape().toString(), shape.toString()); Node::destroy_ = false; } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100644 new mode 100755 index 4b072c73a..33af5d016 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -170,9 +170,8 @@ Ptr EncoderDecoder::step(Ptr graph, // create updated state that reflects reordering and dropping of hypotheses state = hypIndices.empty() ? state : state->select(hypIndices, beamSize); - // Fill stte with embeddings based on last prediction - decoders_[0]->embeddingsFromPrediction( - graph, state, embIndices, dimBatch, beamSize); + // Fill state with embeddings based on last prediction + decoders_[0]->embeddingsFromPrediction(graph, state, embIndices, dimBatch, beamSize); auto nextState = decoders_[0]->step(graph, state); return nextState; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 60659a22a..7e3cb385d 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -267,7 +267,7 @@ class BeamSearch { if(dimBatch > 1 && localBeamSize > 1) expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] else // (avoid copy if we can) - expandedPathScores = reshape(expandedPathScores, {dimBatch, 1, (int)localBeamSize, expandedPathScores->shape()[-1]}); + expandedPathScores = reshape(expandedPathScores, {expandedPathScores->shape()[-2], 1, expandedPathScores->shape()[-4], expandedPathScores->shape()[-1]}); // perform NN computation if(t == 0) diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp index f99b0be42..febf0739b 100755 --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -43,7 +43,7 @@ class NthElementCPU { std::vector::iterator middle = begin + beamSize; std::vector::iterator end = idxs.begin() + batchFirstElementIdxs[batchIdx + 1]; std::partial_sort( - begin, middle, end, [=](int a, int b) { return scores[a] > scores[b]; }); + begin, middle, end, [&](int a, int b) { return scores[a] > scores[b]; }); while(begin != middle) { int idx = *begin++; @@ -67,33 +67,32 @@ class NthElementCPU { const auto dimBatch = scores->shape()[-4]; ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); - const std::vector beamSizes(dimBatch, N); - std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); - std::vector batchFirstElementIdxs(beamSizes.size() + 1, 0); + std::vector cumulativeBeamSizes(dimBatch + 1, 0); + std::vector batchFirstElementIdxs(dimBatch + 1, 0); - for(int batchIdx = 0; batchIdx < beamSizes.size(); ++batchIdx) { - cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + (int)beamSizes[batchIdx]; + for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { + cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + (int)N; ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); batchFirstElementIdxs[batchIdx + 1] += (isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); } + ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); size_t maxSize = N * dimBatch; h_res.resize(maxSize); h_res_idx.resize(maxSize); selectNBest(scores->data(), batchFirstElementIdxs, cumulativeBeamSizes); - getPairs(cumulativeBeamSizes.back(), outKeys, outPathScores); - ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); + getPairs(/*cumulativeBeamSizes.back(),*/ outKeys, outPathScores); } private: - void getPairs(size_t number, + void getPairs(/*size_t number,*/ std::vector& outKeys, std::vector& outValues) { - std::copy(h_res_idx.begin(), h_res_idx.begin() + number, std::back_inserter(outKeys)); - std::copy(h_res .begin(), h_res .begin() + number, std::back_inserter(outValues)); + std::copy(h_res_idx.begin(), h_res_idx.end(), std::back_inserter(outKeys)); + std::copy(h_res .begin(), h_res .end(), std::back_inserter(outValues)); //lastN_ = number; } From 94791594eb647f3e72fbe81d1e15020da20dd11f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 14:09:04 -0800 Subject: [PATCH 338/838] swapAxes() now optimizes for case where it can reshape() instead --- src/graph/expression_operators.cpp | 25 +++++++++++++++++++++---- src/translator/beam_search.h | 5 +---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 5c37beef3..43e6b1a2e 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -213,6 +213,8 @@ Expr repeat(Expr a, size_t repeats, int ax) { } Expr reshape(Expr a, Shape shape) { + if (a->shape() == shape) + return a; return Expression(a, shape); } @@ -256,7 +258,7 @@ Expr flatten_2d(Expr a) { Expr stopGradient(Expr a) { // implemented as a dummy reshape that is not trainable - auto res = reshape(a, a->shape()); + auto res = Expression(a, a->shape()); res->setTrainable(false); return res; } @@ -530,12 +532,27 @@ Expr transpose(Expr a, const std::vector& axes) { Expr swapAxes(Expr x, int axis1, int axis2) { - axis1 = x->shape().axis(axis1); - axis2 = x->shape().axis(axis2); + const auto& shape = x->shape(); + axis1 = shape.axis(axis1); + axis2 = shape.axis(axis2); if (axis1 == axis2) return x; + if (shape[axis1] == 1 || shape[axis2] == 1) { // can we use a reshape instead? + if (axis1 > axis2) + std::swap(axis1, axis2); + bool canReshape = true; + for (int ax = axis1 + 1; ax < axis2 && canReshape; ax++) + canReshape &= (shape[ax] == 1); + if (canReshape) { + auto newShape = shape; + newShape.set(axis1, shape[axis2]); + newShape.set(axis2, shape[axis1]); + //LOG(info, "SwapAxes() did a reshape from {} to {}", shape.toString(), newShape.toString()); + return reshape(x, newShape); + } + } // TODO: This is code dup from transpose(x). Implement transpose(x) as swapAxes(x, 0, 1) - std::vector axes(x->shape().size()); + std::vector axes(shape.size()); for (int i = 0; i < axes.size(); ++i) // @TODO: use std::iota() axes[i] = i; std::swap(axes[axis1], axes[axis2]); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 7e3cb385d..2f529a0fd 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -264,10 +264,7 @@ class BeamSearch { } // make beams continuous - if(dimBatch > 1 && localBeamSize > 1) - expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] - else // (avoid copy if we can) - expandedPathScores = reshape(expandedPathScores, {expandedPathScores->shape()[-2], 1, expandedPathScores->shape()[-4], expandedPathScores->shape()[-1]}); + expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] // perform NN computation if(t == 0) From 9f6f0f12529ffc393fbf922051d4aaf834a83233 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 14:32:04 -0800 Subject: [PATCH 339/838] removed 'first' parameter from toHyps(); towards removing it from getNBestList(); renamed target embeddings to target-history embeddings --- src/models/decoder.h | 4 ++-- src/models/s2s.h | 2 +- src/models/states.h | 17 +++++---------- src/models/transformer.h | 4 ++-- src/translator/beam_search.h | 28 ++++++++++-------------- src/translator/nth_element.cpp | 10 +++++++-- src/translator/nth_element.cu | 40 +++++++++++++++++++--------------- 7 files changed, 53 insertions(+), 52 deletions(-) mode change 100644 => 100755 src/models/decoder.h mode change 100644 => 100755 src/models/s2s.h mode change 100644 => 100755 src/models/states.h diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100644 new mode 100755 index bc0c91191..d0414d86b --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -74,7 +74,7 @@ class DecoderBase { auto yShifted = shift(y, {1, 0, 0}); - state->setTargetEmbeddings(yShifted); + state->setTargetHistoryEmbeddings(yShifted); state->setTargetMask(yMask); state->setTargetIndices(yData); } @@ -105,7 +105,7 @@ class DecoderBase { selectedEmbs = yEmb->apply(embIdx, {dimBeam, 1, dimBatch, dimTrgEmb}); } - state->setTargetEmbeddings(selectedEmbs); + state->setTargetHistoryEmbeddings(selectedEmbs); } virtual const std::vector getAlignments(int /*i*/ = 0) { return {}; }; diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100644 new mode 100755 index 35846e783..bc3626ac7 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -283,7 +283,7 @@ class DecoderS2S : public DecoderBase { virtual Ptr step(Ptr graph, Ptr state) override { - auto embeddings = state->getTargetEmbeddings(); + auto embeddings = state->getTargetHistoryEmbeddings(); // dropout target words float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); diff --git a/src/models/states.h b/src/models/states.h old mode 100644 new mode 100755 index 8e3972210..297c42eb9 --- a/src/models/states.h +++ b/src/models/states.h @@ -33,9 +33,9 @@ class DecoderState { std::vector> encStates_; Ptr batch_; - Expr targetEmbeddings_; + Expr targetHistoryEmbeddings_; // decoder history (teacher-forced or from decoding), embedded Expr targetMask_; - Expr targetIndices_; + Expr targetIndices_; // target labels // Keep track of current target token position during translation size_t position_{0}; @@ -69,20 +69,13 @@ class DecoderState { virtual const rnn::States& getStates() const { return states_; } - virtual Expr getTargetEmbeddings() const { return targetEmbeddings_; }; - - virtual void setTargetEmbeddings(Expr targetEmbeddings) { - targetEmbeddings_ = targetEmbeddings; - } + virtual Expr getTargetHistoryEmbeddings() const { return targetHistoryEmbeddings_; }; + virtual void setTargetHistoryEmbeddings(Expr targetEmbeddings) { targetHistoryEmbeddings_ = targetEmbeddings; } virtual Expr getTargetIndices() const { return targetIndices_; }; - - virtual void setTargetIndices(Expr targetIndices) { - targetIndices_ = targetIndices; - } + virtual void setTargetIndices(Expr targetIndices) { targetIndices_ = targetIndices; } virtual Expr getTargetMask() const { return targetMask_; }; - virtual void setTargetMask(Expr targetMask) { targetMask_ = targetMask; } virtual const Words& getSourceWords() const { diff --git a/src/models/transformer.h b/src/models/transformer.h index 97faf13ce..c65ba5962 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -688,8 +688,8 @@ class DecoderTransformer : public Transformer { } Ptr step(Ptr state) { - auto embeddings = state->getTargetEmbeddings(); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vector dim] - auto decoderMask = state->getTargetMask(); // [max length, batch size, 1] --this is a hypothesis + auto embeddings = state->getTargetHistoryEmbeddings(); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vector dim] + auto decoderMask = state->getTargetMask(); // [max length, batch size, 1] --this is a hypothesis // dropout target words float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 2f529a0fd..2954f06d7 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -36,11 +36,10 @@ class BeamSearch { // combine new expandedPathScores and previous beams into new set of beams Beams toHyps(const std::vector& nBestKeys, // [dimBatch, beamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened const std::vector& nBestPathScores, // [dimBatch, beamSize] flattened - const size_t vocabSize, + const size_t inputBeamSize, // for interpretation of nBestKeys + const size_t vocabSize, // ditto. const Beams& beams, const std::vector>& states, - const size_t beamSize, - const bool first, Ptr batch) const { std::vector align; if(options_->hasAndNotEmpty("alignment")) @@ -49,7 +48,7 @@ class BeamSearch { const auto dimBatch = beams.size(); Beams newBeams(dimBatch); - for(size_t i = 0; i < nBestKeys.size(); ++i) { // [dimBatch, beamSize] flattened + for(size_t i = 0; i < nBestKeys.size(); ++i) { // Keys encode batchIdx, beamHypIdx, and word index in the entire beam. // They can be between 0 and beamSize * vocabSize-1. const auto key = nBestKeys[i]; @@ -57,10 +56,8 @@ class BeamSearch { // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) const auto wordIdx = (Word)(key % vocabSize); - const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); - const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); - - ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); + const auto beamHypIdx = (key / vocabSize) % inputBeamSize; + const auto batchIdx = (key / vocabSize) / inputBeamSize; const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; @@ -285,21 +282,20 @@ class BeamSearch { // find N best amongst the (localBeamSize * dimVocab) hypotheses std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened - getNBestList(/*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] - /*N=*/localBeamSize, + getNBestList(/*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] + /*N=*/localBeamSize, // desired beam size /*out*/ nBestPathScores, /*out*/ nBestKeys, - /*first=*/t == 0); // @TODO: Why is this passed? To know that the beam size is 1 for first step, for flattened hyp index? + /*first=*/t == 0); // @TODO: this is only used for checking presently, and should be removed altogether // Now, nBestPathScores contain N-best expandedPathScores for each batch and beam, // and nBestKeys for each their original location (batchIdx, beamHypIdx, word). // combine N-best sets with existing search space (beams) to updated search space beams = toHyps(nBestKeys, nBestPathScores, - /*dimTrgVoc=*/expandedPathScores->shape()[-1], + /*inputBeamSize*/expandedPathScores->shape()[-2], // used for interpretation of keys + /*vocabSize=*/expandedPathScores->shape()[-1], // used for interpretation of keys beams, - states, // used for keeping track of per-ensemble-member path score - localBeamSize, // used in the encoding of the (batchIdx, beamHypIdx, word) tuples - /*first=*/t == 0, // used to indicate originating beamSize of 1 - batch); + states, // only used for keeping track of per-ensemble-member path score + batch); // only used for propagating alignment info // remove all hyps that end in EOS // The position of a hyp in the beam may change. diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp index febf0739b..7d18555d8 100755 --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -56,7 +56,6 @@ class NthElementCPU { } public: - // @BUGBUG: This API mixes input and output beam size. void getNBestList(Tensor scores, // [dimBatch, 1, beamSize, dimVocab or dimShortlist] size_t N, std::vector& outPathScores, @@ -65,17 +64,24 @@ class NthElementCPU { const auto vocabSize = scores->shape()[-1]; const auto inputN = scores->shape()[-2]; const auto dimBatch = scores->shape()[-4]; - ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); + ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); // @TODO: Remove isFirst argument altogether std::vector cumulativeBeamSizes(dimBatch + 1, 0); std::vector batchFirstElementIdxs(dimBatch + 1, 0); for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { +#if 1 + cumulativeBeamSizes[batchIdx + 1] = (batchIdx + 1) * (int)N; + batchFirstElementIdxs[batchIdx + 1] += (batchIdx + 1) * inputN * vocabSize; + ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != cumulativeBeamSizes[batchIdx] + (int)N, "cumulativeBeamSizes wrong??"); + ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); +#else cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + (int)N; ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); batchFirstElementIdxs[batchIdx + 1] += (isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); +#endif } ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu index ecf9daf5c..b2f23d8c5 100755 --- a/src/translator/nth_element.cu +++ b/src/translator/nth_element.cu @@ -354,7 +354,6 @@ private: } public: - // @BUGBUG: This API mixes input and output beam size. void getNBestList(Tensor scores, size_t N, std::vector& outCosts, @@ -375,15 +374,22 @@ public: std::vector batchFirstElementIdxs(beamSizes.size() + 1, 0); for(size_t batchIdx = 0; batchIdx < beamSizes.size(); ++batchIdx) { +#if 1 + cumulativeBeamSizes[batchIdx + 1] = (batchIdx + 1) * (int)N; + batchFirstElementIdxs[batchIdx + 1] += (batchIdx + 1) * inputN * vocabSize; + ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != cumulativeBeamSizes[batchIdx] + (int)N, "cumulativeBeamSizes wrong??"); + ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); +#else cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + beamSizes[batchIdx]; ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); batchFirstElementIdxs[batchIdx + 1] += ((isFirst) ? (batchIdx + 1) : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); +#endif } selectNBest(scores->data(), batchFirstElementIdxs, cumulativeBeamSizes); - getPairs(cumulativeBeamSizes.back(), outKeys, outCosts); + getPairs(dimBatch * N, outKeys, outCosts); ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); } @@ -409,22 +415,22 @@ private: outValues.push_back(h_res[i]); } - lastN = number; + //lastN = number; } - void getValueByKey(std::vector& out, float* d_in) { - cudaSetDevice(deviceId_.no); - - gGetValueByKey<<<1, lastN, 0, /* stream_ */ 0>>>( - d_in, d_breakdown, h_res_idx, lastN); - - CUDA_CHECK(cudaMemcpyAsync(out.data(), - d_breakdown, - lastN * sizeof(float), - cudaMemcpyDeviceToHost, - /* stream_ */ 0)); - CUDA_CHECK(cudaStreamSynchronize(/* stream_ */ 0)); - } + //void getValueByKey(std::vector& out, float* d_in) { + // cudaSetDevice(deviceId_.no); + // + // gGetValueByKey<<<1, lastN, 0, /* stream_ */ 0>>>( + // d_in, d_breakdown, h_res_idx, lastN); + // + // CUDA_CHECK(cudaMemcpyAsync(out.data(), + // d_breakdown, + // lastN * sizeof(float), + // cudaMemcpyDeviceToHost, + // /* stream_ */ 0)); + // CUDA_CHECK(cudaStreamSynchronize(/* stream_ */ 0)); + //} DeviceId deviceId_; @@ -447,7 +453,7 @@ private: float* d_breakdown; int* d_batchPosition; int* d_cumBeamSizes; - size_t lastN; + //size_t lastN; }; // factory function From ceab6036b1227f01f589c379859bc75ee9acbbc8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 12 Feb 2019 17:01:51 -0800 Subject: [PATCH 340/838] fixed a merge issue --- src/translator/beam_search.h | 59 +++++++++++------------------------- 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index b752538ae..fb762d70b 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -41,8 +41,6 @@ class BeamSearch { const size_t vocabSize, // ditto. const Beams& beams, const std::vector>& states, - const size_t beamSize, - const bool first, Ptr batch, // for alignments only Ptr factoredVocab, size_t factorGroup) const { std::vector align; @@ -60,22 +58,18 @@ class BeamSearch { // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) const auto wordIdx = (WordIndex)(key % vocabSize); - const auto beamHypIdx = (key / vocabSize) % (first ? 1 : beamSize); - const auto batchIdx = (key / vocabSize) / (first ? 1 : beamSize); + const auto beamHypIdx = (key / vocabSize) % inputBeamSize; + const auto batchIdx = (key / vocabSize) / inputBeamSize; //LOG(info, "key = (batch {}, beam {}, word {})", batchIdx, beamHypIdx, wordIdx); - ABORT_IF(i / beamSize != batchIdx, "Inconsistent batchIdx value in key??"); - const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; - if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? + if (newBeam.size() >= beam.size()) // getNBestList() generates N for all batch entries incl. those that already have a narrower beam continue; - if (pathScore <= INVALID_PATH_SCORE) // (dummy slot or word that cannot be expanded by current factor) continue; - //if (beamHypIdx >= (int)beam.size() && pathScore <= INVALID_PATH_SCORE) // (dummy slot) - // continue; + ABORT_IF(beamHypIdx >= (int)beam.size(), "Out of bounds beamHypIdx value {} in key?? word={}, batch={}, pathScore={}", beamHypIdx, wordIdx, batchIdx, pathScore); // map wordIdx to word @@ -99,12 +93,7 @@ class BeamSearch { // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); - //if (factoredVocab->canExpandFactoredWord(word, factorGroup)) - word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); - // @TODO: maybe factor the two above into a single function; for now, I want the extra checks - //else - // continue; // skip if word does not have this factor - //ABORT_IF(prevHyp->getPathScore() != pathScore, "Score changed despite factor not applicable??"); + word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } @@ -147,13 +136,13 @@ class BeamSearch { //LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } - if (newBeam.size() > beamSize) { + if (newBeam.size() > beam.size()) { //LOG(info, "Size {}, sorting...", newBeam.size()); - std::nth_element(newBeam.begin(), newBeam.begin() + beamSize, newBeam.end(), [](Ptr a, Ptr b) { + std::nth_element(newBeam.begin(), newBeam.begin() + beam.size(), newBeam.end(), [](Ptr a, Ptr b) { return a->getPathScore() > b->getPathScore(); // (sort highest score first) }); //LOG(info, "Size {}, sorted...", newBeam.size()); - newBeam.resize(beamSize); // @TODO: needed? + newBeam.resize(beam.size()); } } } @@ -248,9 +237,6 @@ class BeamSearch { } Beams beams(dimBatch, Beam(beamSize_, New())); // array [dimBatch] of array [localBeamSize] of Hypothesis - //Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis - //for(auto& beam : beams) - // beam.resize(beamSize_, New()); for(int i = 0; i < dimBatch; ++i) histories[i]->add(beams[i], trgEosId_); @@ -282,10 +268,8 @@ class BeamSearch { break; for (size_t factorGroup = 0; factorGroup < numFactorGroups; factorGroup++) { - // BEGIN FOR factorGroup = 0 .. numFactorGroups-1 - // @TODO: use an explicit nested loop here for factors + // Note: not indenting the block, for easier merging // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors - //auto factorGroup = t % numFactorGroups; //********************************************************************** // create constant containing previous path scores for current beam @@ -294,12 +278,11 @@ class BeamSearch { std::vector prevWords; // [localBeamsize, 1, dimBatch, 1] (flattened) word that a hyp ended in, for advancing the decoder-model's history Expr prevPathScores; // [localBeamSize, 1, dimBatch, 1], path score that a hyp ended in (last axis will broadcast into vocab size when adding expandedPathScores) bool anyCanExpand = false; // stays false if all hyps are invalid factor expansions - std::vector prevScores; // @TODO: remove here again if(t == 0 && factorGroup == 0) { // no scores yet prevPathScores = graph->constant({1, 1, 1, 1}, inits::from_value(0)); anyCanExpand = true; } else { - //std::vector prevScores; + std::vector prevScores; for(size_t beamHypIdx = 0; beamHypIdx < localBeamSize; ++beamHypIdx) { for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { // loop over batch entries (active sentences) auto& beam = beams[batchIdx]; @@ -351,21 +334,13 @@ class BeamSearch { // add secondary factors // For those, we don't update the decoder-model state in any way. // Instead, we just keep expanding with the factors. - // Considerations: - // - not all scores should get a factor - // We need a [localBeamSize, 1, dimBatch, 1] tensor that knows whether a factor is applicable - // by considering the lemma at each (beamHypIdx, batchIdx). prevWords is already in the right order. - // - factors are incorporated one step at a time; so we will have temporary Word entries - // in hyps with some factors set to FACTOR_NOT_SPECIFIED. + // We will have temporary Word entries in hyps with some factors set to FACTOR_NOT_SPECIFIED. + // For some lemmas, a factor is not applicable. For those, the factor score is the same (zero) + // for all factor values. This would thus unnecessarily pollute the beam with identical copies, + // and push out other hypotheses. Hence, we exclude those here by setting the path score to + // INVALID_PATH_SCORE. Instead, toHyps() explicitly propagates those hyps by simply copying the + // previous hypothesis. logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] - // Note: pathScores was set to INVALID_PATH_SCORE if this path cannot expanded by this factor; toHyps() handles that special case on the side - //for (size_t kk = 0; kk < prevWords.size(); kk++) - // LOG(info, "prevWords[{},{}]={} -> {}", t/numFactorGroups, factorGroup, factoredVocab->word2string(prevWords[kk]), prevScores[kk]); - //auto factorMaskVector = states[i]->getLogProbs().getFactorMasks(prevWords, factorGroup); - //for (auto& m : factorMaskVector) - // m = m ? 0.f : INVALID_PATH_SCORE; // block hyps that do not have the factor; these are short-circuited directly - //auto logFactorMasks = graph->constant({(int)localBeamSize, 1, dimBatch, 1}, inits::from_vector(factorMaskVector)); - //logProbs = logProbs + logFactorMasks; // those hyps that don't have a factor get multiplied with 0 } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; @@ -407,8 +382,8 @@ class BeamSearch { beams, states, // used for keeping track of per-ensemble-member path score batch, // only used for propagating alignment info - factoredVocab, factorGroup); + } // END FOR factorGroup = 0 .. numFactorGroups-1 // remove all hyps that end in EOS From febe660bb9c57e9e6ae9327f96dddcf1f4c23e3c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Feb 2019 18:19:12 -0800 Subject: [PATCH 341/838] SubBatch() now fills gaps with EOS, not Word::ZERO; FactoredVocab::csr_rows() now takes Words and directly breaks each out Word's factors without needing the factorMap_[] --- src/data/corpus_base.h | 2 +- src/data/factored_vocab.cpp | 45 ++++++++++++++++++++++++++++--------- src/data/factored_vocab.h | 2 +- src/layers/generic.cpp | 15 +++++++------ src/layers/generic.h | 2 +- 5 files changed, 46 insertions(+), 20 deletions(-) mode change 100644 => 100755 src/data/corpus_base.h diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100644 new mode 100755 index 5034c3404..f47de0bb4 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -128,7 +128,7 @@ class SubBatch { * @param width Number of words in the longest sentence */ SubBatch(size_t size, size_t width, const Ptr& vocab) - : indices_(size * width, Word::ZERO), // note: for gaps, we must use a valid index + : indices_(size * width, vocab ? vocab->getEosId() : Word::ZERO), // note: for gaps, we must use a valid index mask_(size * width, 0), size_(size), width_(width), diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index ea076af6a..d7ccb892f 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -48,6 +48,7 @@ namespace marian { // parse the line, of the form WORD FACTOR1 FACTOR2 FACTOR1 ... // where FACTOR1 is the lemma, a factor that all words have. // Not every word has all other factors, so the n-th item is not always in the same factor group. + // @TODO: change to just use the .wl file, and manually split at @ utils::splitAny(line, tokens, " \t"); ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); std::vector factorUnits; @@ -270,7 +271,7 @@ std::pair FactoredVocab::getFactorUnit(Word word, size_t groupI void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs //size_t numGroups = groupPrefixes_.size(); - size_t vocabSize = vocab_.size(); + size_t vocabSize = size(); //factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g //factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) gapLogMask_.resize(vocabSize, -1e8f); @@ -292,8 +293,10 @@ void FactoredVocab::constructNormalizationInfoForVocab() { //} // create the global factor matrix, which is used for getLogits() only - std::vector data(vocabSize); - std::iota(data.begin(), data.end(), 0); + // For invalid words, this leaves empty matrix rows, which are later masked by adding gapLogMask. + Words data; + for (size_t v = 0; v < vocabSize; v++) // note: this loops over the entire vocab space, incl. gaps + data.push_back(Word::fromWordIndex(v)); globalFactorMatrix_ = csr_rows(data); // [V x U] } @@ -342,9 +345,10 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return utils::join(decoded, " "); } -// create a CSR matrix M[V,U] from indices[] with +// create a CSR matrix M[V,U] from words[] with // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced -FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& words) const { +FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { + auto numGroups = getNumGroups(); std::vector weights; std::vector indices; std::vector offsets; @@ -352,11 +356,32 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const std::vector& wor indices.reserve(words.size()); // (at least this many) // loop over all input words, and select the corresponding set of unit indices into CSR format offsets.push_back((IndexType)indices.size()); - for (auto w : words) { - const auto& m = factorMap_[w]; - for (auto u : m) { - indices.push_back(u); - weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); + std::vector factorIndices; + for (auto word : words) { + if (!vocab_.isGap(word.toWordIndex())) { // skip invalid combinations in the space (can only happen during initialization) --@TODO: add a check? + word2factors(word, factorIndices); +#if 0 // original code; enable this to try + numGroups; + const auto& m = factorMap_[word.toWordIndex()]; + for (auto u : m) { + indices.push_back(u); +#else +#if 1 // special handling of the missing single capitalized letters + // @TODO: remove this once we use the factor-spec file + const auto& lemma = factorVocab_[(WordIndex)(factorIndices[0] + groupRanges_[0].first)]; + if (lemma.size() == 1 && factorIndices[1]/*@C*/ == 1/*@CI*/) // skip one-letter factors with + LOG(info, "Suppressing embedding for word {}", word2string(word)); + else +#endif + for (size_t g = 0; g < numGroups; g++) { // @TODO: make this faster by having a list of all factors to consider for a lemma? + auto factorIndex = factorIndices[g]; + ABORT_IF(factorIndex == FACTOR_NOT_SPECIFIED, "Attempted to embed a word with a factor not specified"); + if (factorIndex == FACTOR_NOT_APPLICABLE) + continue; + indices.push_back((IndexType)(factorIndex + groupRanges_[g].first)); // map to unit index +#endif + weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); + } } offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 3a562fadb..7b0e84d83 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -40,7 +40,7 @@ class FactoredVocab : public IVocab { // factor-specific. These methods are consumed by Output and Embedding. size_t factorVocabSize() const { return factorVocab_.size(); } - CSRData csr_rows(const std::vector& words) const; + CSRData csr_rows(const Words& words) const; const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v --only used in getLogits() size_t getNumGroups() const { return groupRanges_.size(); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index df06398b0..488747814 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -289,7 +289,7 @@ namespace marian { } // helper to embed a sequence of words (given as indices) via factored embeddings - /*private*/ Expr Embedding::multiRows(const std::vector& data) const + /*private*/ Expr Embedding::multiRows(const Words& data) const { auto graph = E_->graph(); auto factoredData = factoredVocab_->csr_rows(data); @@ -354,15 +354,16 @@ namespace marian { } Expr Embedding::apply(const Words& words, const Shape& shape) const /*override final*/ { - return applyIndices(toWordIndexVector(words), shape); - } - - Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { Expr selectedEmbs; if (factoredVocab_) - selectedEmbs = multiRows(embIdx); + selectedEmbs = multiRows(words); else - selectedEmbs = rows(E_, embIdx); + selectedEmbs = rows(E_, toWordIndexVector(words)); return reshape(selectedEmbs, shape); } + + Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { + ABORT_IF(factoredVocab_ /*&& factoredVocab_->getNumGroups() > 1*/, "Embedding: applyIndices must not be used with a factored vocabulary"); + return reshape(rows(E_, embIdx), shape); + } } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h index 532324641..1ce773dcb 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -239,7 +239,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; Ptr factoredVocab_; - Expr multiRows(const std::vector& data) const; + Expr multiRows(const Words& data) const; public: Embedding(Ptr graph, Ptr options); From 1f4dc6e608af91e1225bfb66f668cf56d721a48d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Feb 2019 18:35:37 -0800 Subject: [PATCH 342/838] (refined an error msg) --- src/translator/nth_element.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu index 23fa2ff95..fb3d86095 100755 --- a/src/translator/nth_element.cu +++ b/src/translator/nth_element.cu @@ -365,9 +365,9 @@ public: const auto inputN = scores->shape()[-2]; const auto dimBatch = scores->shape()[-4]; ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); // @TODO: Remove isFirst argument altogether - ABORT_IF(vocabSize > MAX_VOCAB_SIZE, "GetNBestList(): actual vocab size exceeds MAX_VOCAB_SIZE"); - ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size exceeds initialization parameter"); - ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); // @TODO: or inputN? + ABORT_IF(vocabSize > MAX_VOCAB_SIZE, "GetNBestList(): actual vocab size {} exceeds MAX_VOCAB_SIZE of {}", vocabSize, MAX_VOCAB_SIZE); + ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size {} exceeds initialization parameter {}", dimBatch, maxBatchSize_); + ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size {} exceeds initialization parameter {}", N, maxBeamSize_); // @TODO: or inputN? const std::vector beamSizes(dimBatch, N); std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); From a93de15370e84864d60d0b7153e402b464314a03 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Feb 2019 19:40:54 -0800 Subject: [PATCH 343/838] now explicitly unrolls all factor combinations; had to temporarily disable a few checks for this --- src/data/factored_vocab.cpp | 59 +++++++++++++++++++++++++++---------- src/data/factored_vocab.h | 5 ++-- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index d7ccb892f..6b521332d 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -32,13 +32,13 @@ namespace marian { // construct mapping tables for factors constructGroupInfoFromFactorVocab(); constructFactorIndexConversion(); + auto numGroups = getNumGroups(); // load and parse factorMap - auto elements = factorShape_.elements(); - vocab_.resize(elements); - factorMap_.resize(elements); + auto vocabSize = factorShape_.elements(); // size of vocab space including gaps + vocab_.resize(vocabSize); + //factorMap_.resize(vocabSize); auto factorVocabSize = factorVocab_.size(); - factorRefCounts_.resize(factorVocabSize); lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); std::vector tokens; std::string line; @@ -55,7 +55,6 @@ namespace marian { for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; factorUnits.push_back(u); - factorRefCounts_[u]++; } // convert to fully unrolled factors representation std::vector factorIndices(groupRanges_.size(), FACTOR_NOT_APPLICABLE); // default for unused factors @@ -74,7 +73,7 @@ namespace marian { // map factors to non-dense integer auto word = factors2word(factorIndices); auto wordIndex = word.toWordIndex(); - factorMap_[wordIndex] = std::move(factorUnits); + //factorMap_[wordIndex] = std::move(factorUnits); // add to vocab (the wordIndex are not dense, so the vocab will have holes) vocab_.add(tokens.front(), wordIndex); numTotalFactors += tokens.size() - 1; @@ -84,6 +83,24 @@ namespace marian { LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", numTotalFactors, factorVocabSize, vocab_.numValid(), size()); + // enumerate all combinations of factors for each lemma + // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. + for (size_t v = 0; v < vocabSize; v++) { + auto word = Word::fromWordIndex(v); + bool isValid = true; + for (size_t g = 0; isValid && g < numGroups; g++) { + auto factorIndex = getFactor(word, g); + // @TODO: we have a hack in getFactor() to return not-specified if factor is specified but not applicable, making it invalid + isValid = factorIndex != FACTOR_NOT_SPECIFIED; // FACTOR_NOT_APPLICABLE is a valid value + } + if (isValid != !vocab_.isGap((WordIndex)v)) + { + //LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap((WordIndex)v)); + if (isValid) // add the missing word (albeit with a poor grapheme) + (*this)[word]; + } + } + // create mappings needed for normalization in factored outputs constructNormalizationInfoForVocab(); @@ -96,7 +113,8 @@ namespace marian { if (maxSizeUnused == vocab_.numValid()) maxSizeUnused = vocab_.size(); #endif - ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); + //ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); + // @TODO: ^^ disabled now that we are generating the full combination of factors; reenable once we have consistent setups again return size(); } @@ -258,15 +276,18 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { } else { // regular value: consistency check if lemma really has this factor group ABORT_IF(factor0Index == (size_t)factorShape_[0] - 1, "Word has specified factor but no lemma??"); - ABORT_IF(!lemmaHasFactorGroup(factor0Index, groupIndex), "Word has a specified factor for a lemma that does not have that factor group??"); + //ABORT_IF(!lemmaHasFactorGroup(factor0Index, groupIndex), "Word has a specified factor for a lemma that does not have that factor group??"); + if (!lemmaHasFactorGroup(factor0Index, groupIndex)) + index = FACTOR_NOT_SPECIFIED; + // @TODO: ^^ needed for determining all valid vocab entries; can we pass a flag in to allow this? } return index; } -std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) const { - word; groupIndex; - ABORT("Not implemented"); -} +//std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) const { +// word; groupIndex; +// ABORT("Not implemented"); +//} void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs @@ -276,6 +297,10 @@ void FactoredVocab::constructNormalizationInfoForVocab() { //factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) gapLogMask_.resize(vocabSize, -1e8f); for (WordIndex v = 0; v < vocabSize; v++) { +#if 1 // @TODO: TEST THIS again by disabling factored decoding in beam_search.h + if (!vocab_.isGap(v)) + gapLogMask_[v] = 0.0f; // valid entry +#else for (auto u : factorMap_[v]) { auto g = factorGroups_[u]; // convert u to relative u within factor group range ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); @@ -283,6 +308,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { //factorMasks_[g][v] = 1.0f; gapLogMask_[v] = 0.0f; // valid entry } +#endif } //for (Word v = 0; v < vocabSize; v++) { // LOG(info, "'{}': {}*{} {}*{} {}*{} {}*{}", vocab[v], @@ -296,7 +322,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { // For invalid words, this leaves empty matrix rows, which are later masked by adding gapLogMask. Words data; for (size_t v = 0; v < vocabSize; v++) // note: this loops over the entire vocab space, incl. gaps - data.push_back(Word::fromWordIndex(v)); + data.push_back(Word::fromWordIndex(v)); globalFactorMatrix_ = csr_rows(data); // [V x U] } @@ -313,7 +339,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); #if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { - LOG(info, "Factor combination {} missing in external dict, generating fake entry", word2string(word)); + LOG_ONCE(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); } #endif @@ -367,10 +393,11 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { indices.push_back(u); #else #if 1 // special handling of the missing single capitalized letters + // costs about 0.1 BLEU when original model never saw this combination (which is quite nicely low) // @TODO: remove this once we use the factor-spec file const auto& lemma = factorVocab_[(WordIndex)(factorIndices[0] + groupRanges_[0].first)]; if (lemma.size() == 1 && factorIndices[1]/*@C*/ == 1/*@CI*/) // skip one-letter factors with - LOG(info, "Suppressing embedding for word {}", word2string(word)); + LOG_ONCE(info, "Suppressing embedding for word {} (only showing this warning once)", word2string(word)); else #endif for (size_t g = 0; g < numGroups; g++) { // @TODO: make this faster by having a list of all factors to consider for a lemma? @@ -380,7 +407,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { continue; indices.push_back((IndexType)(factorIndex + groupRanges_[g].first)); // map to unit index #endif - weights.push_back(1.0f/*/(float)factorRefCounts_[u]*/); + weights.push_back(1.0f); } } offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 7b0e84d83..b17b184ae 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -56,7 +56,7 @@ class FactoredVocab : public IVocab { Word expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const; bool canExpandFactoredWord(Word word, size_t groupIndex) const { return lemmaHasFactorGroup(getFactor(word, 0), groupIndex); } size_t getFactor(Word word, size_t groupIndex) const; - std::pair getFactorUnit(Word word, size_t groupIndex) const; + //std::pair getFactorUnit(Word word, size_t groupIndex) const; bool lemmaHasFactorGroup(size_t factor0Index, size_t g) const { return lemmaHasFactorGroup_[factor0Index][g]; } static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); @@ -94,8 +94,7 @@ class FactoredVocab : public IVocab { // factors WordLUT factorVocab_; // [factor name] -> factor index = row of E_ std::vector groupPrefixes_; // [group id g] shared prefix of factors (used for grouping) - std::vector> factorMap_; // [word index v] -> set of factor indices u - std::vector factorRefCounts_; // [factor index u] -> how often factor u is referenced in factorMap_ + //std::vector> factorMap_; // [word index v] -> set of factor indices u CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. From 76d280580ed251cfa1e3f6ebbc9e2236b4911359 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Feb 2019 20:07:26 -0800 Subject: [PATCH 344/838] added a message if new units are added by expanding all factors --- src/data/factored_vocab.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 6b521332d..89d92ff0a 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -85,6 +85,7 @@ namespace marian { // enumerate all combinations of factors for each lemma // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. + size_t numMissing = 0; for (size_t v = 0; v < vocabSize; v++) { auto word = Word::fromWordIndex(v); bool isValid = true; @@ -93,13 +94,17 @@ namespace marian { // @TODO: we have a hack in getFactor() to return not-specified if factor is specified but not applicable, making it invalid isValid = factorIndex != FACTOR_NOT_SPECIFIED; // FACTOR_NOT_APPLICABLE is a valid value } - if (isValid != !vocab_.isGap((WordIndex)v)) - { + if (isValid != !vocab_.isGap((WordIndex)v)) { //LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap((WordIndex)v)); - if (isValid) // add the missing word (albeit with a poor grapheme) + if (isValid) { // add the missing word (albeit with a poor grapheme) (*this)[word]; + // @TODO: ^^disabled to test getLogits(), which no longer works with this enabled (I guess since model has not seen the new units) + numMissing++; + } } } + if (numMissing > 0) + LOG(info, "[embedding] completed {} factor combinations missing from the original vocab file", numMissing); // create mappings needed for normalization in factored outputs constructNormalizationInfoForVocab(); From 63214994d3e30b5b7f79de67ae48f628df16fda2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Feb 2019 12:55:52 -0800 Subject: [PATCH 345/838] changed the canonical form of factored words to match the FactoredSegmenter tool --- src/data/factored_vocab.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 89d92ff0a..d9fc22989 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -75,7 +75,11 @@ namespace marian { auto wordIndex = word.toWordIndex(); //factorMap_[wordIndex] = std::move(factorUnits); // add to vocab (the wordIndex are not dense, so the vocab will have holes) + //if (tokens.front().front() == '<') // all others are auto-expanded + // for now add what we get, and then expand more below vocab_.add(tokens.front(), wordIndex); + if (tokens.front() != word2string(word)) + LOG_ONCE(info, "[embedding] Word name in .wl file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), word2string(word)); numTotalFactors += tokens.size() - 1; if (v % 5000 == 0) LOG(info, "{} -> {}", tokens.front(), word2string(word)); @@ -246,7 +250,7 @@ std::string FactoredVocab::word2string(Word word) const { size_t factor0Index = word.toWordIndex() / factorStrides_[0]; std::string res; for (size_t g = 0; g < numGroups; g++) { - res.append(res.empty() ? "(" : ", "); + //res.append(res.empty() ? "(" : ", "); size_t index = word.toWordIndex(); index = index / factorStrides_[g]; index = index % (size_t)factorShape_[g]; @@ -255,13 +259,14 @@ std::string FactoredVocab::word2string(Word word) const { res.append("(lemma oob)"); else if (lemmaHasFactorGroup(factor0Index, g)) res.append("?"); - else - res.append("n/a"); + //else + // res.append("n/a"); } else res.append(factorVocab_[(WordIndex)(index + groupRanges_[g].first)]); } - return res + ")"; + //res.append(")"); + return res; } size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { @@ -345,7 +350,8 @@ void FactoredVocab::constructNormalizationInfoForVocab() { #if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { LOG_ONCE(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); - const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); + //const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); + const_cast(vocab_).add(word2string(word), word.toWordIndex()); } #endif return vocab_[word.toWordIndex()]; From f19def2803b836e6fbc2df5f217e4c245969c4be Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Feb 2019 14:31:20 -0800 Subject: [PATCH 346/838] removed suppression of single-letter all-caps --- src/data/factored_vocab.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index d9fc22989..fce755c03 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -403,7 +403,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { for (auto u : m) { indices.push_back(u); #else -#if 1 // special handling of the missing single capitalized letters +#if 0 // special handling of the missing single capitalized letters // costs about 0.1 BLEU when original model never saw this combination (which is quite nicely low) // @TODO: remove this once we use the factor-spec file const auto& lemma = factorVocab_[(WordIndex)(factorIndices[0] + groupRanges_[0].first)]; From e7a4e5c16d029419c47a4bca956d61b7e6034647 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sat, 16 Feb 2019 22:47:09 +0000 Subject: [PATCH 347/838] Fix latex generation in doxygen. --- Doxyfile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile.in b/Doxyfile.in index 1761a2283..ba2fec096 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -1592,7 +1592,7 @@ PAPER_TYPE = a4 # If left blank no extra packages will be included. # This tag requires that the tag GENERATE_LATEX is set to YES. -EXTRA_PACKAGES = +EXTRA_PACKAGES = amsmath # The LATEX_HEADER tag can be used to specify a personal LaTeX header for the # generated LaTeX document. The header should contain everything until the first From 980cc933aa332c7870d13aacc877bdf5045f3ed5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Feb 2019 15:04:51 -0800 Subject: [PATCH 348/838] moved gen of rand word from fakeBatch() to IVocab::randWord(), where factored vocab can override --- src/data/corpus_base.h | 2 +- src/data/factored_vocab.cpp | 15 +++++++++++++++ src/data/factored_vocab.h | 1 + src/data/vocab.cpp | 4 ++++ src/data/vocab.h | 3 +++ src/data/vocab_base.h | 4 ++++ src/training/graph_group.h | 1 - 7 files changed, 28 insertions(+), 2 deletions(-) mode change 100644 => 100755 src/data/vocab.cpp mode change 100644 => 100755 src/training/graph_group.h diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index f47de0bb4..652f4beb5 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -322,7 +322,7 @@ class CorpusBatch : public Batch { // set word indices to different values to avoid same hashes // rand() is OK, this does not affect state in any way std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), - [&](Word) -> Word { return Word::fromWordIndex(rand() % vocabs[batchIndex]->size()); }); + [&](Word) -> Word { return vocabs[batchIndex]->randWord(); }); // mask: no items ask being masked out std::fill(sb->mask().begin(), sb->mask().end(), 1.f); batchIndex++; diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index fce755c03..d887fde92 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -361,6 +361,21 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return vocab_.size(); } +// generate a valid random factored word (used by collectStats()) +/*virtual*/ Word FactoredVocab::randWord() const /*override final*/ { + auto numGroups = getNumGroups(); + std::vector factorIndices; factorIndices.reserve(numGroups); + for (size_t g = 0; g < numGroups; g++) { + size_t factorIndex; + if (g == 0 || lemmaHasFactorGroup(factorIndices[0], g)) + factorIndex = rand() % (factorShape_[g] - 1); + else + factorIndex = FACTOR_NOT_APPLICABLE; + factorIndices.push_back(factorIndex); + } + return factors2word(factorIndices); +} + /*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool /*inference*/ /*= false*/) const /*override final*/ { std::vector lineTokens; utils::split(line, lineTokens, " "); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index b17b184ae..aff256512 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -36,6 +36,7 @@ class FactoredVocab : public IVocab { virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } virtual void createFake() override final { ABORT("[data] Fake FactoredVocab vocabulary not supported"); } + virtual Word randWord() const override final; // factor-specific. These methods are consumed by Output and Embedding. size_t factorVocabSize() const { return factorVocab_.size(); } diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp old mode 100644 new mode 100755 index 63947647d..1bf113e86 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -91,6 +91,10 @@ void Vocab::createFake() { vImpl_->createFake(); } +Word Vocab::randWord() { + return vImpl_->randWord(); +} + // string token to token id Word Vocab::operator[](const std::string& word) const { return vImpl_->operator[](word); diff --git a/src/data/vocab.h b/src/data/vocab.h index dd746ba9a..c6f92c90a 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -69,6 +69,9 @@ class Vocab { // create fake vocabulary for collecting batch statistics void createFake(); + // generate a fake word (using rand()) + Word randWord(); + // give access to base implementation. Returns null if not the requested type. template // e.g. FactoredVocab Ptr tryAs() const { return std::dynamic_pointer_cast(vImpl_); } diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index bca9db957..42d871bb5 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -44,6 +44,10 @@ class IVocab { virtual Word getUnkId() const = 0; virtual void createFake() = 0; + + virtual Word randWord() const { + return Word::fromWordIndex(rand() % size()); + } }; class Options; diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100644 new mode 100755 index f4617d782..60b81a831 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -77,7 +77,6 @@ class GraphGroup { for(int i = 0; i < inputTypes.size(); ++i) if(inputTypes[i] == "class") localMaxes[i] = 1; - size_t maxBatch = 512; bool fits = true; From e87db82b153a766be940b042f78e3725276321c6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 18 Feb 2019 17:11:16 -0800 Subject: [PATCH 349/838] a few things I am trying to track down why it does not work --- src/common/logging.cpp | 4 ++-- src/data/corpus_base.h | 3 +-- src/data/factored_vocab.cpp | 17 +++++++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 9c4b2951f..85dade236 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -77,8 +77,8 @@ void createLoggers(const marian::Config* config) { } bool quiet = config && config->get("quiet"); - //Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; - Logger general{createStderrLogger("general", "%v", generalLogs, quiet)}; + Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; + //Logger general{createStderrLogger("general", "%v", generalLogs, quiet)}; Logger valid{createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; if(config && config->has("log-level")) { diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h index 652f4beb5..8f836f357 100755 --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -319,8 +319,7 @@ class CorpusBatch : public Batch { size_t batchIndex = 0; for(auto len : lengths) { auto sb = New(batchSize, len, vocabs[batchIndex]); - // set word indices to different values to avoid same hashes - // rand() is OK, this does not affect state in any way + // set word indices to random values (not actually needed with current version --@marcinjd: please confirm) std::transform(sb->data().begin(), sb->data().end(), sb->data().begin(), [&](Word) -> Word { return vocabs[batchIndex]->randWord(); }); // mask: no items ask being masked out diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index d887fde92..4e81d7140 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -81,8 +81,8 @@ namespace marian { if (tokens.front() != word2string(word)) LOG_ONCE(info, "[embedding] Word name in .wl file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), word2string(word)); numTotalFactors += tokens.size() - 1; - if (v % 5000 == 0) - LOG(info, "{} -> {}", tokens.front(), word2string(word)); + //if (v % 5000 == 0) + // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", numTotalFactors, factorVocabSize, vocab_.numValid(), size()); @@ -101,7 +101,8 @@ namespace marian { if (isValid != !vocab_.isGap((WordIndex)v)) { //LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap((WordIndex)v)); if (isValid) { // add the missing word (albeit with a poor grapheme) - (*this)[word]; + vocab_.add(word2string(word), word.toWordIndex()); + //(*this)[word]; // @TODO: ^^disabled to test getLogits(), which no longer works with this enabled (I guess since model has not seen the new units) numMissing++; } @@ -342,7 +343,8 @@ void FactoredVocab::constructNormalizationInfoForVocab() { if (found) return Word::fromWordIndex(index); else - return getUnkId(); + ABORT("Unknown word {}", word); + //return getUnkId(); } /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*overrworde final*/ { @@ -436,6 +438,13 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { weights.push_back(1.0f); } } +#if 1 + else { + // push a dummy entry. Not sure if this is needed. + indices.push_back(0); + weights.push_back(0.0f); + } +#endif offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset } return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; From 7ece433b55ee618db9f374b3ea03842301ddd1c7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 19 Feb 2019 17:59:09 -0800 Subject: [PATCH 350/838] bug fix: suppressWord() should use the Unk lemma index; CPU selectNBest() can use a const* --- src/data/batch_generator.h | 2 +- src/data/factored_vocab.h | 1 + src/translator/beam_search.h | 35 +++++++++++++++++++++++++--------- src/translator/helpers.cpp | 10 +++++----- src/translator/helpers.cu | 4 ++-- src/translator/helpers.h | 6 +++--- src/translator/nth_element.cpp | 22 ++++----------------- 7 files changed, 42 insertions(+), 38 deletions(-) mode change 100644 => 100755 src/data/batch_generator.h mode change 100644 => 100755 src/translator/helpers.cpp mode change 100644 => 100755 src/translator/helpers.h diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h old mode 100644 new mode 100755 index 4ec03d4d4..950a71b1f --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -96,7 +96,7 @@ class BatchGenerator : public RNGEngine { a.rbegin(), a.rend(), b.rbegin(), b.rend(), itemCmp); }; - auto cmpNone = [](const Sample& a, const Sample& b) { return &a < &b; }; // instead sort by address, so we have something to work with + auto cmpNone = [](const Sample& a, const Sample& b) { return a.getId() > b.getId(); }; // sort in order of original ids = original data order unless shuffling typedef std::function cmp_type; typedef std::priority_queue sample_queue; diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index aff256512..846821bc6 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -35,6 +35,7 @@ class FactoredVocab : public IVocab { virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } + WordIndex getUnkIndex() const { return (WordIndex)getFactor(getUnkId(), 0); } // used in decoding virtual void createFake() override final { ABORT("[data] Fake FactoredVocab vocabulary not supported"); } virtual Word randWord() const override final; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index fb762d70b..ba366fff2 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -60,7 +60,7 @@ class BeamSearch { const auto wordIdx = (WordIndex)(key % vocabSize); const auto beamHypIdx = (key / vocabSize) % inputBeamSize; const auto batchIdx = (key / vocabSize) / inputBeamSize; - //LOG(info, "key = (batch {}, beam {}, word {})", batchIdx, beamHypIdx, wordIdx); + LOG(info, "key = (batch {}, beam {}, word {}) -> {}", batchIdx, beamHypIdx, wordIdx, pathScore); const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; @@ -86,11 +86,11 @@ class BeamSearch { // starting with the lemma, then adding factors one by one. if (factorGroup == 0) { word = factoredVocab->lemma2Word(wordIdx); - //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); } else { - //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), - // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); @@ -130,10 +130,10 @@ class BeamSearch { auto& newBeam = newBeams[batchIdx]; for (const auto& beamHyp : beam) { auto word = beamHyp->getWord(); - //LOG(info, "Checking {}", factoredVocab->word2string(word)); + LOG(info, "Checking {}", factoredVocab->word2string(word)); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above continue; - //LOG(info, "Forwarded {}", factoredVocab->word2string(word)); + LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } if (newBeam.size() > beam.size()) { @@ -141,7 +141,7 @@ class BeamSearch { std::nth_element(newBeam.begin(), newBeam.begin() + beam.size(), newBeam.end(), [](Ptr a, Ptr b) { return a->getPathScore() > b->getPathScore(); // (sort highest score first) }); - //LOG(info, "Size {}, sorted...", newBeam.size()); + LOG(info, "Size {}, sorted...", newBeam.size()); newBeam.resize(beam.size()); } } @@ -217,6 +217,7 @@ class BeamSearch { const int dimBatch = (int)batch->size(); auto getNBestList = createGetNBestListFn(beamSize_, dimBatch, graph->getDeviceId()); + auto getNBestListCPU = createGetNBestListFn(beamSize_, dimBatch, DeviceId(0, DeviceType::cpu)); for(auto scorer : scorers_) { scorer->clear(graph); @@ -290,6 +291,7 @@ class BeamSearch { auto hyp = beam[beamHypIdx]; auto word = hyp->getWord(); auto canExpand = (!factoredVocab || factoredVocab->canExpandFactoredWord(hyp->getWord(), factorGroup)); + LOG(info, "[{}, {}] Can expand {} with {} -> {}", batchIdx, beamHypIdx, (*batch->back()->vocab())[hyp->getWord()], factorGroup, canExpand); anyCanExpand |= canExpand; hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(word); @@ -348,6 +350,7 @@ class BeamSearch { // make beams continuous expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] + expandedPathScores->debug("expandedPathScores"); // perform NN computation if(t == 0 && factorGroup == 0) @@ -357,14 +360,21 @@ class BeamSearch { //********************************************************************** // suppress specific symbols if not at right positions - if(trgUnkId_ != Word::NONE && options_->has("allow-unk") && !options_->get("allow-unk")) - suppressWord(expandedPathScores, trgUnkId_); + if(trgUnkId_ != Word::NONE && options_->has("allow-unk") && !options_->get("allow-unk") && factorGroup == 0) + suppressWord(expandedPathScores, factoredVocab ? factoredVocab->getUnkIndex() : trgUnkId_.toWordIndex()); for(auto state : states) state->blacklist(expandedPathScores, batch); //********************************************************************** // perform beam search + std::vector nBestKeysCPU; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened + std::vector nBestPathScoresCPU; // [dimBatch, localBeamSize] flattened + getNBestListCPU(/*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] + /*N=*/localBeamSize, // desired beam size + /*out*/ nBestPathScoresCPU, /*out*/ nBestKeysCPU, + /*first=*/t == 0 && factorGroup == 0); // @TODO: this is only used for checking presently, and should be removed altogether + // find N best amongst the (localBeamSize * dimVocab) hypotheses std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened @@ -374,6 +384,13 @@ class BeamSearch { /*first=*/t == 0 && factorGroup == 0); // @TODO: this is only used for checking presently, and should be removed altogether // Now, nBestPathScores contain N-best expandedPathScores for each batch and beam, // and nBestKeys for each their original location (batchIdx, beamHypIdx, word). + ABORT_IF(nBestKeys.size() != nBestKeysCPU.size(), "Inconsistent getNth size"); + for (size_t u = 0; u < nBestKeys.size(); u++) { + //if (nBestKeysCPU[u] != nBestKeys[u] || nBestPathScoresCPU[u] != nBestPathScores[u]) + LOG(info, "Inconsistent result [{}]: {} vs {} , {} vs {}", u, nBestKeysCPU[u], nBestKeys[u], nBestPathScoresCPU[u], nBestPathScores[u]); + } + //nBestKeys = nBestKeysCPU; + //nBestPathScores = nBestPathScoresCPU; // combine N-best sets with existing search space (beams) to updated search space beams = toHyps(nBestKeys, nBestPathScores, diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp old mode 100644 new mode 100755 index f131398c2..f4b75da0a --- a/src/translator/helpers.cpp +++ b/src/translator/helpers.cpp @@ -24,18 +24,18 @@ void SetColumn(Tensor in_, size_t col, float value) { } } -void suppressWord(Expr logProbs, Word id) { - SetColumn(logProbs->val(), id.toWordIndex(), std::numeric_limits::lowest()); +void suppressWord(Expr logProbs, WordIndex wordIndex) { + SetColumn(logProbs->val(), wordIndex, std::numeric_limits::lowest()); } } // namespace cpu -void suppressWord(Expr logProbs, Word id) { +void suppressWord(Expr logProbs, WordIndex wordIndex) { if(logProbs->val()->getBackend()->getDeviceId().type == DeviceType::cpu) { - cpu::suppressWord(logProbs, id); + cpu::suppressWord(logProbs, wordIndex); } #ifdef CUDA_FOUND else { - gpu::suppressWord(logProbs, id); + gpu::suppressWord(logProbs, wordIndex); } #endif } diff --git a/src/translator/helpers.cu b/src/translator/helpers.cu index d23c7500f..a5dfd2e89 100755 --- a/src/translator/helpers.cu +++ b/src/translator/helpers.cu @@ -37,8 +37,8 @@ void SetColumn(Tensor in_, size_t col, float value) { gSetColumn<<>>(in_->data(), nColumns, nRows, col, value); } -void suppressWord(Expr probs, Word id) { - SetColumn(probs->val(), id.toWordIndex(), std::numeric_limits::lowest()); +void suppressWord(Expr probs, WordIndex wordIndex) { + SetColumn(probs->val(), wordIndex, std::numeric_limits::lowest()); } } // namespace gpu } // namespace marian diff --git a/src/translator/helpers.h b/src/translator/helpers.h old mode 100644 new mode 100755 index d4ff3a946..71b1eb202 --- a/src/translator/helpers.h +++ b/src/translator/helpers.h @@ -11,13 +11,13 @@ namespace marian { namespace cpu { -void suppressWord(Expr logProbs, Word id); +void suppressWord(Expr logProbs, WordIndex wordIndex); } namespace gpu { -void suppressWord(Expr logProbs, Word id); +void suppressWord(Expr logProbs, WordIndex wordIndex); } -void suppressWord(Expr logProbs, Word id); +void suppressWord(Expr logProbs, WordIndex wordIndex); } // namespace marian diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp index 7d18555d8..4692a5057 100755 --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -21,15 +21,10 @@ class NthElementCPU { NthElementCPU(const NthElementCPU& copy) = delete; private: - void selectNBest(float* scores, - const std::vector& batchFirstElementIdxs, - const std::vector& cumulativeBeamSizes) { - /* For each batch, select the max N elements, where N is the beam size for - * this batch. Locally record these elements (their current value and index - * in 'scores') before updating each element to a large negative value, such - * that they won't be a maximum if we're called again on the same input. - */ - + // for each batch, select the max N elements, where N is the beam size for this batch. + void selectNBest(const float* scores, + const std::vector& batchFirstElementIdxs, + const std::vector& cumulativeBeamSizes) { int numProbs = batchFirstElementIdxs.back(); std::vector idxs(numProbs); std::iota(idxs.begin(), idxs.end(), 0); @@ -49,7 +44,6 @@ class NthElementCPU { int idx = *begin++; h_res_idx[pos] = idx; h_res[pos] = scores[idx]; - scores[idx] = std::numeric_limits::lowest(); ++pos; } } @@ -70,18 +64,10 @@ class NthElementCPU { std::vector batchFirstElementIdxs(dimBatch + 1, 0); for(int batchIdx = 0; batchIdx < dimBatch; ++batchIdx) { -#if 1 cumulativeBeamSizes[batchIdx + 1] = (batchIdx + 1) * (int)N; batchFirstElementIdxs[batchIdx + 1] += (batchIdx + 1) * inputN * vocabSize; ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != cumulativeBeamSizes[batchIdx] + (int)N, "cumulativeBeamSizes wrong??"); ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); -#else - cumulativeBeamSizes[batchIdx + 1] = cumulativeBeamSizes[batchIdx] + (int)N; - ABORT_IF(cumulativeBeamSizes[batchIdx + 1] != (batchIdx + 1) * N, "cumulativeBeamSizes wrong??"); - batchFirstElementIdxs[batchIdx + 1] - += (isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) * vocabSize; - ABORT_IF((isFirst ? batchIdx + 1 : cumulativeBeamSizes[batchIdx + 1]) != (batchIdx + 1) * inputN, "inputN wrong??"); -#endif } ABORT_IF(cumulativeBeamSizes.back() != dimBatch * N, "cumulativeBeamSizes.back() wrong??"); From 8314387f9e52ac62d8d015cedc73589b377980bd Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 19 Feb 2019 18:39:48 -0800 Subject: [PATCH 351/838] removed CPU comparison --- src/data/factored_vocab.cpp | 4 ++-- src/translator/beam_search.h | 32 ++++++++------------------------ 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 4e81d7140..1949d3d90 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -351,7 +351,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); #if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { - LOG_ONCE(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); + LOG/*_ONCE*/(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); //const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); const_cast(vocab_).add(word2string(word), word.toWordIndex()); } @@ -466,7 +466,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { WordIndex FactoredVocab::WordLUT::add(const std::string& word, WordIndex index) { ABORT_IF(word.empty(), "Attempted to add the empty word to a dictionary"); auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; - ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}'", word); + ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}', new index {} vs. existing index {}", word, index, str2index_[word]); while (index2str_.size() <= index) index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? ABORT_IF(!index2str_[index].empty(), "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index ba366fff2..a22fd5dfa 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -60,7 +60,7 @@ class BeamSearch { const auto wordIdx = (WordIndex)(key % vocabSize); const auto beamHypIdx = (key / vocabSize) % inputBeamSize; const auto batchIdx = (key / vocabSize) / inputBeamSize; - LOG(info, "key = (batch {}, beam {}, word {}) -> {}", batchIdx, beamHypIdx, wordIdx, pathScore); + //LOG(info, "key = (batch {}, beam {}, word {}) -> {}", batchIdx, beamHypIdx, wordIdx, pathScore); const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; @@ -86,11 +86,11 @@ class BeamSearch { // starting with the lemma, then adding factors one by one. if (factorGroup == 0) { word = factoredVocab->lemma2Word(wordIdx); - LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); } else { - LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), - factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); @@ -130,10 +130,10 @@ class BeamSearch { auto& newBeam = newBeams[batchIdx]; for (const auto& beamHyp : beam) { auto word = beamHyp->getWord(); - LOG(info, "Checking {}", factoredVocab->word2string(word)); + //LOG(info, "Checking {}", factoredVocab->word2string(word)); if (factoredVocab->canExpandFactoredWord(word, factorGroup)) // handled above continue; - LOG(info, "Forwarded {}", factoredVocab->word2string(word)); + //LOG(info, "Forwarded {}", factoredVocab->word2string(word)); newBeam.push_back(beamHyp); } if (newBeam.size() > beam.size()) { @@ -141,7 +141,7 @@ class BeamSearch { std::nth_element(newBeam.begin(), newBeam.begin() + beam.size(), newBeam.end(), [](Ptr a, Ptr b) { return a->getPathScore() > b->getPathScore(); // (sort highest score first) }); - LOG(info, "Size {}, sorted...", newBeam.size()); + //LOG(info, "Size {}, sorted...", newBeam.size()); newBeam.resize(beam.size()); } } @@ -217,7 +217,6 @@ class BeamSearch { const int dimBatch = (int)batch->size(); auto getNBestList = createGetNBestListFn(beamSize_, dimBatch, graph->getDeviceId()); - auto getNBestListCPU = createGetNBestListFn(beamSize_, dimBatch, DeviceId(0, DeviceType::cpu)); for(auto scorer : scorers_) { scorer->clear(graph); @@ -291,7 +290,7 @@ class BeamSearch { auto hyp = beam[beamHypIdx]; auto word = hyp->getWord(); auto canExpand = (!factoredVocab || factoredVocab->canExpandFactoredWord(hyp->getWord(), factorGroup)); - LOG(info, "[{}, {}] Can expand {} with {} -> {}", batchIdx, beamHypIdx, (*batch->back()->vocab())[hyp->getWord()], factorGroup, canExpand); + //LOG(info, "[{}, {}] Can expand {} with {} -> {}", batchIdx, beamHypIdx, (*batch->back()->vocab())[hyp->getWord()], factorGroup, canExpand); anyCanExpand |= canExpand; hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(word); @@ -350,7 +349,6 @@ class BeamSearch { // make beams continuous expandedPathScores = swapAxes(expandedPathScores, 0, 2); // -> [dimBatch, 1, localBeamSize, dimVocab] - expandedPathScores->debug("expandedPathScores"); // perform NN computation if(t == 0 && factorGroup == 0) @@ -368,13 +366,6 @@ class BeamSearch { //********************************************************************** // perform beam search - std::vector nBestKeysCPU; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened - std::vector nBestPathScoresCPU; // [dimBatch, localBeamSize] flattened - getNBestListCPU(/*in*/ expandedPathScores->val(), // [dimBatch, 1, localBeamSize, dimVocab or dimShortlist] - /*N=*/localBeamSize, // desired beam size - /*out*/ nBestPathScoresCPU, /*out*/ nBestKeysCPU, - /*first=*/t == 0 && factorGroup == 0); // @TODO: this is only used for checking presently, and should be removed altogether - // find N best amongst the (localBeamSize * dimVocab) hypotheses std::vector nBestKeys; // [dimBatch, localBeamSize] flattened -> (batchIdx, beamHypIdx, word idx) flattened std::vector nBestPathScores; // [dimBatch, localBeamSize] flattened @@ -384,13 +375,6 @@ class BeamSearch { /*first=*/t == 0 && factorGroup == 0); // @TODO: this is only used for checking presently, and should be removed altogether // Now, nBestPathScores contain N-best expandedPathScores for each batch and beam, // and nBestKeys for each their original location (batchIdx, beamHypIdx, word). - ABORT_IF(nBestKeys.size() != nBestKeysCPU.size(), "Inconsistent getNth size"); - for (size_t u = 0; u < nBestKeys.size(); u++) { - //if (nBestKeysCPU[u] != nBestKeys[u] || nBestPathScoresCPU[u] != nBestPathScores[u]) - LOG(info, "Inconsistent result [{}]: {} vs {} , {} vs {}", u, nBestKeysCPU[u], nBestKeys[u], nBestPathScoresCPU[u], nBestPathScores[u]); - } - //nBestKeys = nBestKeysCPU; - //nBestPathScores = nBestPathScoresCPU; // combine N-best sets with existing search space (beams) to updated search space beams = toHyps(nBestKeys, nBestPathScores, From 8d3b3cc96a84f42f7fdbfe82ca8c6cc5d7076b4f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 19 Feb 2019 19:44:27 -0800 Subject: [PATCH 352/838] implemented all-caps'ing and title-casing for factored vocabs --- src/common/utils.cpp | 12 +++++++++++- src/common/utils.h | 2 ++ src/data/corpus.cpp | 4 ++-- src/data/factored_vocab.cpp | 11 ++++++++++- src/data/factored_vocab.h | 2 ++ src/data/vocab.cpp | 6 ++++++ src/data/vocab.h | 6 ++++++ src/data/vocab_base.h | 3 +++ 8 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index e4f6aa904..eb464878e 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -211,7 +211,7 @@ std::string toEnglishTitleCase(const std::string& s) { const std::string wordStartChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; const std::string wordInternalChars = wordStartChars + "'"; // don't title-case letters after word-internal apostrophe const std::set exceptions = { // from moses-scripts/scripts/recaser/detruecase.perl - "a","after","against","al-.+","and","any","as","at","be","because","between","by","during","el-.+","for","from","his","in","is","its","last","not","of","off","on","than","the","their","this","to","was","were","which","will","with" + "a","after","against","al-.+","and","any","as","at","be","because","between","by","during","el-.+","for","from","his","in","is","its","last","not","of","off","on","than","the","their","this","to","was","were","which","will","with" }; const std::set wordPredChars = {' ', '"', '\'', '-'}; // only capitalize words if following these characters (to avoid upper-casing word-internal SPM units) // These are tokenization heuristics, which may be incomplete. @@ -238,6 +238,16 @@ std::string toEnglishTitleCase(const std::string& s) { return res; } +std::string findReplace(const std::string& in, const std::string& what, const std::string& withWhat, bool all /*= false*/) { + std::string res = in; + for(size_t pos = res.find(what); pos != std::string::npos; pos = res.find(what, pos + withWhat.length())) { + res.replace(pos, what.length(), withWhat); + if (!all) + break; + } + return res; +} + double parseDouble(std::string s) { double res; char c; // dummy char -- if we succeed to parse this, then there were extraneous characters after the number diff --git a/src/common/utils.h b/src/common/utils.h index 23d6ffed4..8aa40f835 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -41,6 +41,8 @@ bool endsWith(const std::string& text, const std::string& suffix); std::string utf8ToUpper(const std::string& s); std::string toEnglishTitleCase(const std::string& s); +std::string findReplace(const std::string& in, const std::string& what, const std::string& withWhat, bool all = false); + double parseDouble(std::string s); double parseNumber(std::string s); diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 44fa064d8..61d359c91 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -25,7 +25,7 @@ Corpus::Corpus(std::vector paths, void Corpus::preprocessLine(std::string& line, size_t streamId) { if (allCapsEvery_ != 0 && pos_ % allCapsEvery_ == 0 && !inference_) { - line = utils::utf8ToUpper(line); + line = vocabs_[streamId]->toUpper(line); if (streamId == 0) LOG_ONCE(info, "[data] Source all-caps'ed line to: {}", line); else @@ -36,7 +36,7 @@ void Corpus::preprocessLine(std::string& line, size_t streamId) { && streamId == 0 // @HACK: Hard-coding EN-X direction for now; needs an option in the future ) { - line = utils::toEnglishTitleCase(line); + line = vocabs_[streamId]->toEnglishTitleCase(line); if (streamId == 0) LOG_ONCE(info, "[data] Source English-title-case'd line to: {}", line); else diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 1949d3d90..4c5b5acea 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -117,7 +117,7 @@ namespace marian { // and must exist in the vocabulary eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); - LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); + //LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); #if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() if (maxSizeUnused == vocab_.numValid()) @@ -363,6 +363,15 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return vocab_.size(); } +/*virtual*/ std::string FactoredVocab::toUpper(const std::string& line) const /*override final*/ { + return utils::findReplace(utils::findReplace(line, "@CI", "@CA", /*all=*/true), "@CN", "@CA", /*all=*/true); +} + +/*virtual*/ std::string FactoredVocab::toEnglishTitleCase(const std::string& line) const /*override final*/ { + // @BUGBUG: does not handle the special words that should remain lower-case + return utils::findReplace(line, "@CN@GL-", "@CI@GL-", /*all=*/true); +} + // generate a valid random factored word (used by collectStats()) /*virtual*/ Word FactoredVocab::randWord() const /*override final*/ { auto numGroups = getNumGroups(); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 846821bc6..433a9d61a 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -35,6 +35,8 @@ class FactoredVocab : public IVocab { virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } + virtual std::string toUpper(const std::string& line) const override final; + virtual std::string toEnglishTitleCase(const std::string& line) const override final; WordIndex getUnkIndex() const { return (WordIndex)getFactor(getUnkId(), 0); } // used in decoding virtual void createFake() override final { ABORT("[data] Fake FactoredVocab vocabulary not supported"); } virtual Word randWord() const override final; diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 1bf113e86..9c1039d81 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -130,4 +130,10 @@ Word Vocab::getEosId() const { return vImpl_->getEosId(); } // return UNK symbol id Word Vocab::getUnkId() const { return vImpl_->getUnkId(); } +// for corpus augmentation: convert string to all-caps +std::string Vocab::toUpper(const std::string& line) const { return vImpl_->toUpper(line); } + +// for corpus augmentation: convert string to title case +std::string Vocab::toEnglishTitleCase(const std::string& line) const { return vImpl_->toEnglishTitleCase(line); } + } // namespace marian diff --git a/src/data/vocab.h b/src/data/vocab.h index c6f92c90a..81f97a618 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -66,6 +66,12 @@ class Vocab { // return UNK symbol id Word getUnkId() const; + // for corpus augmentation: convert string to all-caps + std::string toUpper(const std::string& line) const; + + // for corpus augmentation: convert string to title case + std::string toEnglishTitleCase(const std::string& line) const; + // create fake vocabulary for collecting batch statistics void createFake(); diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index 42d871bb5..240626655 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -43,6 +43,9 @@ class IVocab { virtual Word getEosId() const = 0; virtual Word getUnkId() const = 0; + virtual std::string toUpper(const std::string& line) const { return utils::utf8ToUpper(line); } + virtual std::string toEnglishTitleCase(const std::string& line) const { return utils::toEnglishTitleCase(line); } + virtual void createFake() = 0; virtual Word randWord() const { From 03806ff60f17734fcded3ab7d517c8d5634f9423 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 22 Feb 2019 12:38:23 -0800 Subject: [PATCH 353/838] split ModelBase into ModelBase and CriterionBase, which have different signatures --- src/common/options.h | 2 +- src/examples/mnist/validator.h | 2 +- src/microsoft/quicksand.cpp | 2 +- src/models/costs.h | 70 ++++++++++----------- src/models/model_base.h | 21 +++++++ src/models/model_factory.cpp | 84 +++++++++++++++++--------- src/models/model_factory.h | 6 +- src/models/transformer.h | 4 +- src/rescorer/rescorer.h | 2 +- src/tensors/cpu/tensor_operators.cpp | 1 + src/training/graph_group.h | 6 +- src/training/graph_group_async.cpp | 4 +- src/training/graph_group_async.h | 2 +- src/training/graph_group_multinode.cpp | 2 +- src/training/graph_group_singleton.h | 4 +- src/training/graph_group_sync.cpp | 2 +- src/training/graph_group_sync.h | 2 +- src/training/validator.h | 22 +++---- src/translator/beam_search.h | 0 src/translator/scorers.cpp | 4 +- 20 files changed, 146 insertions(+), 96 deletions(-) mode change 100644 => 100755 src/common/options.h mode change 100644 => 100755 src/examples/mnist/validator.h mode change 100644 => 100755 src/microsoft/quicksand.cpp mode change 100644 => 100755 src/models/costs.h mode change 100644 => 100755 src/models/model_base.h mode change 100644 => 100755 src/models/transformer.h mode change 100644 => 100755 src/rescorer/rescorer.h mode change 100644 => 100755 src/tensors/cpu/tensor_operators.cpp mode change 100644 => 100755 src/training/graph_group.h mode change 100644 => 100755 src/training/graph_group_async.cpp mode change 100644 => 100755 src/training/graph_group_async.h mode change 100644 => 100755 src/training/graph_group_multinode.cpp mode change 100644 => 100755 src/training/graph_group_singleton.h mode change 100644 => 100755 src/training/graph_group_sync.cpp mode change 100644 => 100755 src/training/graph_group_sync.h mode change 100644 => 100755 src/training/validator.h mode change 100644 => 100755 src/translator/beam_search.h mode change 100644 => 100755 src/translator/scorers.cpp diff --git a/src/common/options.h b/src/common/options.h old mode 100644 new mode 100755 index 5a3f4eb70..643456c93 --- a/src/common/options.h +++ b/src/common/options.h @@ -111,7 +111,7 @@ class Options { } try { return !options_[key].as().empty(); - } catch(const YAML::BadConversion& e) { + } catch(const YAML::BadConversion&) { ABORT("Option '{}' is neither a sequence nor a text"); } return false; diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h old mode 100644 new mode 100755 index f38a95cec..8a85915a1 --- a/src/examples/mnist/validator.h +++ b/src/examples/mnist/validator.h @@ -16,7 +16,7 @@ class MNISTAccuracyValidator : public Validator { public: MNISTAccuracyValidator(Ptr options) : Validator(std::vector>(), options, false) { createBatchGenerator(/*isTranslating=*/false); - builder_ = models::from_options(options, models::usage::scoring); + builder_ = models::createModelFromOptions(options, models::usage::scoring); } virtual void keepBest(const std::vector>& graphs) override { diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100644 new mode 100755 index 959e44311..022b200d4 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -72,7 +72,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { std::cerr << modelOpts->str() << std::flush; - auto encdec = models::from_options(modelOpts, models::usage::translation); + auto encdec = models::createModelFromOptions(modelOpts, models::usage::translation); if(io::isBin(models[i]) && ptrs_[i] != nullptr) { // if file ends in *.bin and has been mapped by QuickSAND diff --git a/src/models/costs.h b/src/models/costs.h old mode 100644 new mode 100755 index 2aba6fe09..eb9af6b0b --- a/src/models/costs.h +++ b/src/models/costs.h @@ -127,7 +127,7 @@ class EncoderClassifierCE : public CostBase { } }; -class Trainer : public ModelBase { +class Trainer : public CriterionBase { protected: Ptr model_; Ptr cost_; @@ -159,7 +159,40 @@ class Trainer : public ModelBase { virtual void clear(Ptr graph) override { model_->clear(graph); }; }; -typedef Trainer Scorer; +// @TODO: Name 'scorer' is ambiguous: Does it compute scores for all classes, or the loss value for the ground truth? +// Beam search uses it for the former meaning, while 'marian score' and validation in the latter. +// This class is for the former use. The latter is done using Trainer. +class Scorer : public ModelBase { +protected: + Ptr model_; + Ptr cost_; + +public: + Scorer(Ptr model, Ptr cost) + : model_(model), cost_(cost) {} + + Ptr getModel() { return model_; } + + virtual void load(Ptr graph, + const std::string& name, + bool markedReloaded = true) override { + model_->load(graph, name, markedReloaded); + }; + + virtual void save(Ptr graph, + const std::string& name, + bool saveTranslatorConfig = false) override { + model_->save(graph, name, saveTranslatorConfig); + } + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + return cost_->apply(model_, graph, batch, clearGraph); + }; + + virtual void clear(Ptr graph) override { model_->clear(graph); }; +}; class CostStep { public: @@ -270,38 +303,5 @@ class Stepwise : public EncoderDecoderBase { virtual data::SoftAlignment getAlignment() override { return encdec_->getAlignment(); } }; -inline Ptr add_cost(Ptr encdec, - Ptr options) { - switch(options->get("usage", usage::raw)) { - case usage::training: - return New(encdec, New(options)); - case usage::scoring: - return New(encdec, New(options)); - case usage::translation: - if(options->get("output-sampling", false)) - return New(encdec, New()); - else - return New(encdec, New()); - case usage::raw: - default: - return encdec; - } -} - -inline Ptr add_cost(Ptr enccls, - Ptr options) { - switch(options->get("usage", usage::raw)) { - case usage::training: - return New(enccls, New(options)); - case usage::scoring: - return New(enccls, New(options)); - case usage::translation: - ABORT("Classifier cannot be used for translation"); - case usage::raw: - default: - return enccls; - } -} - } // namespace models } // namespace marian diff --git a/src/models/model_base.h b/src/models/model_base.h old mode 100644 new mode 100755 index 0701a8b4b..bbc7aca33 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -16,6 +16,7 @@ YAML_REGISTER_TYPE(marian::models::usage, int) namespace marian { namespace models { +// model = input -> predictions class ModelBase { public: virtual void load(Ptr, @@ -35,5 +36,25 @@ class ModelBase { virtual void clear(Ptr graph) = 0; }; +// criterion = (input, reference) -> loss +class CriterionBase { +public: + virtual void load(Ptr, + const std::string&, + bool markReloaded = true) + = 0; + virtual void save(Ptr, + const std::string&, + bool saveTranslatorConfig = false) + = 0; + + virtual Ptr build(Ptr graph, + Ptr batch, + bool clearGraph = true) + = 0; + + virtual void clear(Ptr graph) = 0; +}; + } // namespace models } // namespace marian diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 946b8a3ab..d7bd8d854 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -77,7 +77,7 @@ Ptr EncoderDecoderFactory::construct(Ptr graph) { for(auto& df : decoders_) encdec->push_back(df(options_).construct(graph)); - return add_cost(encdec, options_); + return encdec; } Ptr EncoderClassifierFactory::construct(Ptr graph) { @@ -96,10 +96,10 @@ Ptr EncoderClassifierFactory::construct(Ptr graph) { for(auto& cf : classifiers_) enccls->push_back(cf(options_).construct(graph)); - return add_cost(enccls, options_); + return enccls; } -Ptr by_type(std::string type, usage use, Ptr options) { +Ptr createBaseModelByType(std::string type, usage use, Ptr options) { Ptr graph = nullptr; // graph unknown at this stage // clang-format off if(type == "s2s" || type == "amun" || type == "nematus") { @@ -259,31 +259,7 @@ Ptr by_type(std::string type, usage use, Ptr options) { .construct(graph); } -#ifdef COMPILE_EXAMPLES - // @TODO: examples should be compiled optionally - if(type == "mnist-ffnn") { - auto mnist = New(options); - if(use == usage::scoring) - return New(mnist, New()); - else if(use == usage::training) - return New(mnist, New()); - else - return mnist; - } -#endif - #ifdef CUDNN -#ifdef COMPILE_EXAMPLES - if(type == "mnist-lenet") { - auto mnist = New(options); - if(use == usage::scoring) - return New(mnist, New()); - else if(use == usage::training) - return New(mnist, New()); - else - return mnist; - } -#endif if(type == "char-s2s") { return models::encoder_decoder()(options) ("usage", use) @@ -298,9 +274,59 @@ Ptr by_type(std::string type, usage use, Ptr options) { ABORT("Unknown model type: {}", type); } -Ptr from_options(Ptr options, usage use) { +Ptr createModelFromOptions(Ptr options, usage use) { std::string type = options->get("type"); - return by_type(type, use, options); + auto baseModel = createBaseModelByType(type, use, options); + + // add (log)softmax if requested + if (use == usage::translation) { + if(std::dynamic_pointer_cast(baseModel)) { + if(options->get("output-sampling", false)) + return New(std::dynamic_pointer_cast(baseModel), New()); + else + return New(std::dynamic_pointer_cast(baseModel), New()); + } +#ifdef COMPILE_EXAMPLES + // note: 'usage::translation' here means 'inference' + else if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New()); +#ifdef CUDNN + else if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New()); +#endif +#endif + else + ABORT("'usage' parameter 'translation' cannot be applied to model type: {}", type); + } + else if (use == usage::raw) + return baseModel; + else + ABORT("'Usage' parameter must be 'translation' or 'raw'"); +} + +Ptr createCriterionFromOptions(Ptr options, usage use) { + std::string type = options->get("type"); + auto baseModel = createBaseModelByType(type, use, options); + + // add cost function + ABORT_IF(use != usage::training && use != usage::scoring, "'Usage' parameter must be 'training' or 'scoring'"); + // note: usage::scoring means "score the loss function", hence it uses a Trainer (not Scorer, which is for decoding) + // @TODO: Should we define a new class that does not compute gradients? + if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New(options)); + else if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New(options)); +#ifdef COMPILE_EXAMPLES + // @TODO: examples should be compiled optionally + else if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New()); +#ifdef CUDNN + else if (std::dynamic_pointer_cast(baseModel)) + return New(baseModel, New()); +#endif +#endif + else + ABORT("Criterion function unknown for model type: {}", type); } } // namespace models diff --git a/src/models/model_factory.h b/src/models/model_factory.h index be14dc6d9..9d089e294 100755 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -85,8 +85,10 @@ class EncoderClassifierFactory : public Factory { typedef Accumulator encoder_classifier; -Ptr by_type(std::string type, usage, Ptr options); +Ptr createBaseModelByType(std::string type, usage, Ptr options); -Ptr from_options(Ptr options, usage); +Ptr createModelFromOptions(Ptr options, usage); + +Ptr createCriterionFromOptions(Ptr options, usage); } // namespace models } // namespace marian diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100644 new mode 100755 index 1083efe29..9f5410f36 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -865,12 +865,12 @@ class DecoderTransformer : public Transformer { }; // factory functions -Ptr NewEncoderTransformer(Ptr options) +static inline Ptr NewEncoderTransformer(Ptr options) { return New(options); } -Ptr NewDecoderTransformer(Ptr options) +static inline Ptr NewDecoderTransformer(Ptr options) { return New(options); } diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100644 new mode 100755 index ee2a3c9a5..372fc17f6 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -23,7 +23,7 @@ class Rescorer { public: Rescorer(Ptr options) - : builder_(models::from_options(options, models::usage::scoring)) {} + : builder_(models::createModelFromOptions(options, models::usage::scoring)) {} void load(Ptr graph, const std::string& modelFile) { builder_->load(graph, modelFile); diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100644 new mode 100755 index 72dbd131c..580be94ce --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -15,6 +15,7 @@ namespace marian { namespace cpu { void IsNan(const Tensor in, Ptr allocator, bool& isNan, bool& isInf, bool zero) { + zero; isInf; isNan; ABORT("Not implemented"); } diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100644 new mode 100755 index f4617d782..8e67ad724 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -55,7 +55,7 @@ class GraphGroup { */ // @TODO: Can this be made const? It seems wrong to have a stateful method that still returns a result. virtual Ptr collectStats(Ptr graph, - Ptr model, + Ptr model, const std::vector>& vocabs, double multiplier = 1.) { auto stats = New(); @@ -141,7 +141,7 @@ class MultiNodeGraphGroupBase : public GraphGroup { std::vector devices_; // [num local GPUs] /** Graph builders for clients (which run forward and backward passes). */ - std::vector> clientBuilders_; + std::vector> clientBuilders_; /** Graphs of clients. One entry per GPU on this node. */ std::vector> clientGraphs_; // [num local GPUs] @@ -161,7 +161,7 @@ class MultiNodeGraphGroupBase : public GraphGroup { clientGraphs_.push_back(New()); clientGraphs_[i]->setDevice({ devices_[i], DeviceType::gpu }); clientGraphs_[i]->reserveWorkspaceMB(options_->get("workspace")); - clientBuilders_.push_back(models::from_options(options_, models::usage::training)); + clientBuilders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); } } diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp old mode 100644 new mode 100755 index f1a01cdfa..780108f17 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -23,7 +23,7 @@ AsyncGraphGroup::AsyncGraphGroup(Ptr config, Ptr mpi) graphs_.push_back(graph); shardOpt_.push_back(Optimizer(options_)); - builders_.push_back(models::from_options(options_, models::usage::training)); + builders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); } } @@ -189,7 +189,7 @@ void AsyncGraphGroup::execute(Ptr batch) { auto task = [this](Ptr batch) { static size_t i = 0; thread_local Ptr graph; - thread_local Ptr builder; + thread_local Ptr builder; thread_local size_t t = 0; thread_local size_t num_seen_words = 0; thread_local size_t num_seen_sentences = 0; diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h old mode 100644 new mode 100755 index cd85cc501..bb19bde40 --- a/src/training/graph_group_async.h +++ b/src/training/graph_group_async.h @@ -16,7 +16,7 @@ class AsyncGraphGroup : public GraphGroup, public ExponentialSmoothing { protected: bool first_{true}; - std::vector> builders_; + std::vector> builders_; std::vector> graphs_; std::vector devices_; diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp old mode 100644 new mode 100755 index 255b5a2a5..4e7d931cf --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -512,7 +512,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { auto task = [this](Ptr batch) { static size_t i = 0; thread_local Ptr graph; - thread_local Ptr builder; + thread_local Ptr builder; thread_local size_t my_id = 0; thread_local size_t t = 0; // only for scheduler statistic diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h old mode 100644 new mode 100755 index 01ffb1eb2..8aede537a --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -16,7 +16,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { virtual void setScheduler(Ptr scheduler) override; private: - Ptr builder_; + Ptr builder_; Ptr graph_; Ptr graphAvg_; @@ -37,7 +37,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { graph_->getBackend()->setClip(options_->get("clip-gemm")); graph_->reserveWorkspaceMB(options_->get("workspace")); opt_ = Optimizer(options_); - builder_ = models::from_options(options_, models::usage::training); + builder_ = models::createCriterionFromOptions(options_, models::usage::training); } void update(Ptr batch) override { diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100644 new mode 100755 index c02a34afa..b97ac65a1 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -15,7 +15,7 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) graphs_.push_back(graph); shardOpt_.push_back(Optimizer(options_)); - builders_.push_back(models::from_options(options_, models::usage::training)); + builders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); } // Note: We may well end up with only one MPI process or only one graph per worker. diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h old mode 100644 new mode 100755 index af10d5863..a34bb114d --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -14,7 +14,7 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { Ptr mpi_; // [not null] all MPI-like communication goes through this (this is a dummy implementation if no MPI run) std::vector devices_; // [deviceIndex] - std::vector> builders_; // [deviceIndex] + std::vector> builders_; // [deviceIndex] std::vector> graphs_; // [deviceIndex] std::vector> shardOpt_; // [deviceIndex] diff --git a/src/training/validator.h b/src/training/validator.h old mode 100644 new mode 100755 index 1bbebb91c..44a4f6f70 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -159,7 +159,7 @@ class CrossEntropyValidator : public Validator { opts->merge(options); opts->set("inference", true); opts->set("cost-type", "ce-sum"); - builder_ = models::from_options(opts, models::usage::scoring); + builder_ = models::createModelFromOptions(opts, models::usage::scoring); } std::string type() override { return options_->get("cost-type"); } @@ -180,7 +180,7 @@ class CrossEntropyValidator : public Validator { for(auto batch : *batchGenerator_) { auto task = [=, &loss, &samples](size_t id) { thread_local Ptr graph; - thread_local auto builder = models::from_options(options_, models::usage::scoring); + thread_local auto builder = models::createModelFromOptions(options_, models::usage::scoring); if(!graph) { graph = graphs[id % graphs.size()]; @@ -226,7 +226,7 @@ class AccuracyValidator : public Validator { Ptr opts = New(); opts->merge(options); opts->set("inference", true); - builder_ = models::from_options(opts, models::usage::raw); + builder_ = models::createModelFromOptions(opts, models::usage::raw); } std::string type() override { return "accuracy"; } @@ -245,7 +245,7 @@ class AccuracyValidator : public Validator { for(auto batch : *batchGenerator_) { auto task = [=, &correct, &totalLabels](size_t id) { thread_local Ptr graph; - thread_local auto builder = models::from_options(options_, models::usage::raw); + thread_local auto builder = models::createModelFromOptions(options_, models::usage::raw); if(!graph) { graph = graphs[id % graphs.size()]; @@ -319,7 +319,7 @@ class BertAccuracyValidator : public Validator { Ptr opts = New(); opts->merge(options); opts->set("inference", true); - builder_ = models::from_options(opts, models::usage::raw); + builder_ = models::createModelFromOptions(opts, models::usage::raw); } std::string type() override { @@ -343,7 +343,7 @@ class BertAccuracyValidator : public Validator { for(auto batch : *batchGenerator_) { auto task = [=, &correct, &totalLabels](size_t id) { thread_local Ptr graph; - thread_local auto builder = models::from_options(options_, models::usage::raw); + thread_local auto builder = models::createModelFromOptions(options_, models::usage::raw); thread_local std::unique_ptr engine; if(!graph) { @@ -419,7 +419,7 @@ class ScriptValidator : public Validator { public: ScriptValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false) { - builder_ = models::from_options(options_, models::usage::raw); + builder_ = models::createModelFromOptions(options_, models::usage::raw); ABORT_IF(!options_->hasAndNotEmpty("valid-script-path"), "valid-script metric but no script given"); @@ -451,7 +451,7 @@ class TranslationValidator : public Validator { TranslationValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false), quiet_(options_->get("quiet-translation")) { - builder_ = models::from_options(options_, models::usage::translation); + builder_ = models::createModelFromOptions(options_, models::usage::translation); if(!options_->hasAndNotEmpty("valid-script-path")) LOG_VALID(warn, "No post-processing script given for validating translator"); @@ -475,7 +475,7 @@ class TranslationValidator : public Validator { std::vector> scorers; for(auto graph : graphs) { - auto builder = models::from_options(options_, models::usage::translation); + auto builder = models::createModelFromOptions(options_, models::usage::translation); Ptr scorer = New(builder, "", 1.0f, model); scorers.push_back(scorer); } @@ -587,7 +587,7 @@ class BleuValidator : public Validator { : Validator(vocabs, options, false), detok_(detok), quiet_(options_->get("quiet-translation")) { - builder_ = models::from_options(options_, models::usage::translation); + builder_ = models::createModelFromOptions(options_, models::usage::translation); #ifdef USE_SENTENCEPIECE auto vocab = vocabs_.back(); @@ -619,7 +619,7 @@ class BleuValidator : public Validator { std::vector> scorers; for(auto graph : graphs) { - auto builder = models::from_options(options_, models::usage::translation); + auto builder = models::createModelFromOptions(options_, models::usage::translation); Ptr scorer = New(builder, "", 1.0f, model); scorers.push_back(scorer); } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100644 new mode 100755 diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp old mode 100644 new mode 100755 index 7a07a8cd5..f49c64fbd --- a/src/translator/scorers.cpp +++ b/src/translator/scorers.cpp @@ -17,7 +17,7 @@ Ptr scorerByType(const std::string& fname, } bool skipCost = options->get("skip-cost"); - auto encdec = models::from_options( + auto encdec = models::createModelFromOptions( options, skipCost ? models::usage::raw : models::usage::translation); LOG(info, "Loading scorer of type {} as feature {}", type, fname); @@ -39,7 +39,7 @@ Ptr scorerByType(const std::string& fname, } bool skipCost = options->get("skip-cost"); - auto encdec = models::from_options( + auto encdec = models::createModelFromOptions( options, skipCost ? models::usage::raw : models::usage::translation); LOG(info, "Loading scorer of type {} as feature {}", type, fname); From 23ece0040a491ff09e4181b5ab857beeb239607b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 22 Feb 2019 13:21:45 -0800 Subject: [PATCH 354/838] ModelBase::build() now returns an Expr, not a RationalLoss --- src/examples/mnist/model.h | 28 ++++++++++----------------- src/examples/mnist/validator.h | 6 +++--- src/models/costs.h | 34 ++++++++++++++++++++------------- src/models/encoder_classifier.h | 26 ++++++++++++------------- src/models/encoder_decoder.cpp | 10 +++++----- src/models/encoder_decoder.h | 24 +++++++++++------------ src/models/model_base.h | 6 +++--- src/rescorer/rescorer.h | 6 +++--- src/training/validator.cpp | 4 ++-- src/training/validator.h | 26 ++++++++++++------------- vs/Marian.vcxproj | 3 +++ vs/Marian.vcxproj.filters | 9 +++++++++ 12 files changed, 97 insertions(+), 85 deletions(-) mode change 100644 => 100755 src/examples/mnist/model.h mode change 100644 => 100755 src/models/encoder_decoder.cpp mode change 100644 => 100755 src/models/encoder_decoder.h mode change 100644 => 100755 src/training/validator.cpp mode change 100644 => 100755 vs/Marian.vcxproj mode change 100644 => 100755 vs/Marian.vcxproj.filters diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100644 new mode 100755 index 0b07f99a0..2a0e0e6a8 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -36,27 +36,23 @@ class MNISTCrossEntropyCost : public CostBase { // Define a top-level node for training // use CE loss - auto loss = sum(cross_entropy(top->loss(), labels), /*axis =*/ 0); + auto loss = sum(cross_entropy(top, labels), /*axis =*/ 0); auto multiLoss = New(); multiLoss->push_back({loss, (float)vLabels.size()}); return multiLoss; } }; -class MNISTLogsoftmax : public CostBase { +class MNISTLogsoftmax : public LogProbBase { public: MNISTLogsoftmax() {} - Ptr apply(Ptr model, + Expr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { auto top = model->build(graph, batch, clearGraph); - - // @TODO: simplify this - auto multiLoss = New(); - multiLoss->push_back({logsoftmax(top->loss()), top->count()}); - return multiLoss; + return logsoftmax(top); } }; @@ -68,14 +64,11 @@ class MnistFeedForwardNet : public ModelBase { MnistFeedForwardNet(Ptr options, Args... args) : options_(options), inference_(options->get("inference", false)) {} - virtual Ptr build(Ptr graph, + virtual Expr build(Ptr graph, Ptr batch, bool /*clean*/ = false) override { - auto loss = construct(graph, batch, inference_); // @TODO: unify nomenclature, e.g. rather use apply - auto count = graph->constant({(int)batch->size(), 1}, inits::from_value(1.f)); - - return New(loss, count); + return apply(graph, batch, inference_); } void load(Ptr /*graph*/, const std::string& /*name*/, bool) override { @@ -103,8 +96,7 @@ class MnistFeedForwardNet : public ModelBase { bool inference_{false}; /** - * @brief Constructs an expression graph representing a feed-forward - * classifier. + * @brief Builds an expression graph representing a feed-forward classifier. * * @param dims number of nodes in each layer of the feed-forward classifier * @param batch a batch of training or testing examples @@ -112,9 +104,9 @@ class MnistFeedForwardNet : public ModelBase { * * @return a shared pointer to the newly constructed expression graph */ - virtual Expr construct(Ptr g, - Ptr batch, - bool /*inference*/ = false) { + virtual Expr apply(Ptr g, + Ptr batch, + bool /*inference*/ = false) { const std::vector dims = {784, 2048, 2048, 10}; // Start with an empty expression graph diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h index 8a85915a1..b95a83dca 100755 --- a/src/examples/mnist/validator.h +++ b/src/examples/mnist/validator.h @@ -12,11 +12,11 @@ using namespace marian; namespace marian { -class MNISTAccuracyValidator : public Validator { +class MNISTAccuracyValidator : public Validator { public: MNISTAccuracyValidator(Ptr options) : Validator(std::vector>(), options, false) { createBatchGenerator(/*isTranslating=*/false); - builder_ = models::createModelFromOptions(options, models::usage::scoring); + builder_ = models::createModelFromOptions(options, models::usage::translation); } virtual void keepBest(const std::vector>& graphs) override { @@ -35,7 +35,7 @@ class MNISTAccuracyValidator : public Validator { graphs[0]->forward(); std::vector scores; - probs->loss(scores); + probs->val()->get(scores); correct += countCorrect(scores, batch->labels()); samples += batch->size(); diff --git a/src/models/costs.h b/src/models/costs.h index eb9af6b0b..8a057034b 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -159,17 +159,25 @@ class Trainer : public CriterionBase { virtual void clear(Ptr graph) override { model_->clear(graph); }; }; +class LogProbBase { +public: + virtual Expr apply(Ptr model, + Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; +}; + // @TODO: Name 'scorer' is ambiguous: Does it compute scores for all classes, or the loss value for the ground truth? // Beam search uses it for the former meaning, while 'marian score' and validation in the latter. // This class is for the former use. The latter is done using Trainer. class Scorer : public ModelBase { protected: Ptr model_; - Ptr cost_; + Ptr logProb_; public: - Scorer(Ptr model, Ptr cost) - : model_(model), cost_(cost) {} + Scorer(Ptr model, Ptr cost) + : model_(model), logProb_(cost) {} Ptr getModel() { return model_; } @@ -185,10 +193,10 @@ class Scorer : public ModelBase { model_->save(graph, name, saveTranslatorConfig); } - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { - return cost_->apply(model_, graph, batch, clearGraph); + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { + return logProb_->apply(model_, graph, batch, clearGraph); }; virtual void clear(Ptr graph) override { model_->clear(graph); }; @@ -259,9 +267,9 @@ class Stepwise : public EncoderDecoderBase { virtual void clear(Ptr graph) override { encdec_->clear(graph); } - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto corpusBatch = std::static_pointer_cast(batch); return build(graph, corpusBatch, clearGraph); } @@ -282,9 +290,9 @@ class Stepwise : public EncoderDecoderBase { return cost_->apply(nextState); } - virtual Ptr build(Ptr /*graph*/, - Ptr /*batch*/, - bool /*clearGraph*/ = true) override { + virtual Expr build(Ptr /*graph*/, + Ptr /*batch*/, + bool /*clearGraph*/ = true) override { ABORT("Wrong wrapper. Use models::Trainer or models::Scorer"); return nullptr; } diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index f3e3645d0..15cbb7d7d 100755 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -41,13 +41,13 @@ class EncoderClassifierBase : public models::ModelBase { virtual std::vector> apply(Ptr, Ptr, bool) = 0; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override = 0; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) = 0; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual Ptr getOptions() = 0; }; @@ -206,17 +206,17 @@ class EncoderClassifier : public EncoderClassifierBase { return classifierStates; } - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto states = apply(graph, batch, clearGraph); // returns raw logits - return New(states[0]->getLogProbs(), nullptr); // @TODO: Check if this is actually used + return states[0]->getLogProbs(); } - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override { + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override { auto corpusBatch = std::static_pointer_cast(batch); return build(graph, corpusBatch, clearGraph); } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100644 new mode 100755 index 4b072c73a..926cae943 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -196,16 +196,16 @@ Ptr EncoderDecoder::stepAll(Ptr graph, return nextState; } -Ptr EncoderDecoder::build(Ptr graph, - Ptr batch, - bool clearGraph) { +Expr EncoderDecoder::build(Ptr graph, + Ptr batch, + bool clearGraph) { auto state = stepAll(graph, batch, clearGraph); // returns raw logits - return New(state->getLogProbs(), state->getTargetMask()); // @TODO: hacky hack hack + return state->getLogProbs(); // , state->getTargetMask()); // @TODO: hacky hack hack } -Ptr EncoderDecoder::build(Ptr graph, +Expr EncoderDecoder::build(Ptr graph, Ptr batch, bool clearGraph) { auto corpusBatch = std::static_pointer_cast(batch); diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100644 new mode 100755 index b55e8bd1d..b0f5b3145 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -28,13 +28,13 @@ class EncoderDecoderBase : public models::ModelBase { virtual void clear(Ptr graph) override = 0; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override = 0; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override = 0; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) = 0; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual Ptr startState(Ptr graph, Ptr batch) = 0; @@ -156,13 +156,13 @@ class EncoderDecoder : public EncoderDecoderBase { Ptr batch, bool clearGraph = true); - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) override; + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) override; }; } // namespace marian diff --git a/src/models/model_base.h b/src/models/model_base.h index bbc7aca33..0113a7ec0 100755 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -28,9 +28,9 @@ class ModelBase { bool saveTranslatorConfig = false) = 0; - virtual Ptr build(Ptr graph, - Ptr batch, - bool clearGraph = true) + virtual Expr build(Ptr graph, + Ptr batch, + bool clearGraph = true) = 0; virtual void clear(Ptr graph) = 0; diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 372fc17f6..011b6b58c 100755 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -19,11 +19,11 @@ using namespace data; class Rescorer { private: - Ptr builder_; + Ptr builder_; public: Rescorer(Ptr options) - : builder_(models::createModelFromOptions(options, models::usage::scoring)) {} + : builder_(models::createCriterionFromOptions(options, models::usage::scoring)) {} void load(Ptr graph, const std::string& modelFile) { builder_->load(graph, modelFile); @@ -34,7 +34,7 @@ class Rescorer { } data::SoftAlignment getAlignment() { - auto model = std::static_pointer_cast(builder_)->getModel(); + auto model = std::static_pointer_cast(builder_)->getModel(); return std::static_pointer_cast(model)->getAlignment(); } }; diff --git a/src/training/validator.cpp b/src/training/validator.cpp old mode 100644 new mode 100755 index 14b854ff3..f6b1c86a1 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -2,10 +2,10 @@ namespace marian { -std::vector>> Validators( +std::vector*/>> Validators( std::vector> vocabs, Ptr config) { - std::vector>> validators; + std::vector*/>> validators; auto validMetrics = config->get>("valid-metrics"); diff --git a/src/training/validator.h b/src/training/validator.h index 44a4f6f70..d49ded8f6 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -56,7 +56,7 @@ class ValidatorBase : public TrainingObserver { } }; -template +template class Validator : public ValidatorBase { public: Validator(std::vector> vocabs, Ptr options, bool lowerIsBetter = true) @@ -123,7 +123,7 @@ class Validator : public ValidatorBase { protected: std::vector> vocabs_; Ptr options_; - Ptr builder_; + Ptr builder_; Ptr> batchGenerator_; virtual float validateBG(const std::vector>&) @@ -148,7 +148,7 @@ class Validator : public ValidatorBase { } }; -class CrossEntropyValidator : public Validator { +class CrossEntropyValidator : public Validator { public: CrossEntropyValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options) { @@ -159,7 +159,7 @@ class CrossEntropyValidator : public Validator { opts->merge(options); opts->set("inference", true); opts->set("cost-type", "ce-sum"); - builder_ = models::createModelFromOptions(opts, models::usage::scoring); + builder_ = models::createCriterionFromOptions(opts, models::usage::scoring); } std::string type() override { return options_->get("cost-type"); } @@ -180,7 +180,7 @@ class CrossEntropyValidator : public Validator { for(auto batch : *batchGenerator_) { auto task = [=, &loss, &samples](size_t id) { thread_local Ptr graph; - thread_local auto builder = models::createModelFromOptions(options_, models::usage::scoring); + thread_local auto builder = models::createCriterionFromOptions(options_, models::usage::scoring); if(!graph) { graph = graphs[id % graphs.size()]; @@ -215,8 +215,8 @@ class CrossEntropyValidator : public Validator { } }; -// Used for validating with classifiers. Compute prediction accuary versus groundtruth for a set of classes -class AccuracyValidator : public Validator { +// Used for validating with classifiers. Compute prediction accuracy versus ground truth for a set of classes +class AccuracyValidator : public Validator { public: AccuracyValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, /*lowerIsBetter=*/false) { @@ -263,7 +263,7 @@ class AccuracyValidator : public Validator { // correct += correct->scalar(); builder->clear(graph); - Expr logits = builder->build(graph, batch)->loss(); + Expr logits = builder->build(graph, batch); graph->forward(); std::vector vLogits; @@ -305,7 +305,7 @@ class AccuracyValidator : public Validator { } }; -class BertAccuracyValidator : public Validator { +class BertAccuracyValidator : public Validator { private: bool evalMaskedLM_{true}; @@ -415,7 +415,7 @@ class BertAccuracyValidator : public Validator { }; -class ScriptValidator : public Validator { +class ScriptValidator : public Validator { public: ScriptValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false) { @@ -446,7 +446,7 @@ class ScriptValidator : public Validator { } }; -class TranslationValidator : public Validator { +class TranslationValidator : public Validator { public: TranslationValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false), @@ -578,7 +578,7 @@ class TranslationValidator : public Validator { }; // @TODO: combine with TranslationValidator (above) to avoid code duplication -class BleuValidator : public Validator { +class BleuValidator : public Validator { private: bool detok_{false}; @@ -844,7 +844,7 @@ class BleuValidator : public Validator { * * @return Vector of validator objects */ -std::vector>> Validators( +std::vector*/>> Validators( std::vector> vocabs, Ptr config); } // namespace marian diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100644 new mode 100755 index a6c560d3a..8edada060 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -894,10 +894,13 @@ + + + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100644 new mode 100755 index d9e56843f..81d6d69d4 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1517,6 +1517,15 @@ examples\mnist + + models + + + models + + + models + From 359398dffb21271466abeae27f18c721956e8eb5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 22 Feb 2019 20:02:31 -0800 Subject: [PATCH 355/838] bug fix: cusparse sgemmi can't handle matrices wider than 65535; made Logits() constructor explicit; FactoredVocab::csr_rows() no longer inserts dummy zeros --- src/common/definitions.h | 7 ++++ src/common/types.h | 4 +++ src/data/factored_vocab.cpp | 2 +- src/examples/mnist/model.h | 2 +- src/layers/generic.cpp | 4 +-- src/layers/generic.h | 4 +-- src/models/costs.h | 2 +- src/models/encoder_classifier.h | 2 +- src/models/s2s.h | 2 +- src/tensors/gpu/prod.cpp | 59 +++++++++++++++++++++++++++++++-- 10 files changed, 76 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/common/definitions.h mode change 100644 => 100755 src/common/types.h diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100644 new mode 100755 index 21ee816fc..393a8b7fe --- a/src/common/definitions.h +++ b/src/common/definitions.h @@ -13,6 +13,13 @@ #define THREAD_GUARD(body) [&]() { body; }() // test if THREAD_GUARD is neccessary, remove if no problems occur. #define NodeOp(op) [=]() { op; } +// helper macro to disable optimization (gcc only) +#ifdef _GNUC_ +#define DONT_OPTIMIZE __attribute__((optimize("O0"))) +#else +#define DONT_OPTIMIZE // silently ignore on Visual Studio, where this is less of a problem +#endif + namespace marian { // Type to be used for all index types, e.g. for integer tensors for rows operator. diff --git a/src/common/types.h b/src/common/types.h old mode 100644 new mode 100755 index f25520f02..3e9adf0e8 --- a/src/common/types.h +++ b/src/common/types.h @@ -73,6 +73,10 @@ template <> inline bool matchType(Type type) { return type == Type::floa template <> inline bool matchType(Type type) { return type == Type::float64; } // clang-format on +template inline Type getType(); +template <> inline Type getType() { return Type::float32; } +template <> inline Type getType() { return Type::uint32; } + static inline std::ostream& operator<<(std::ostream& out, Type type) { switch(type) { case Type::int8: out << "int8"; break; diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 4c5b5acea..bbae31127 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -447,7 +447,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { weights.push_back(1.0f); } } -#if 1 +#if 0 // @BUGBUG: No, this is wrong! The vector must be 1s, since we use it in backprop transposition. else { // push a dummy entry. Not sure if this is needed. indices.push_back(0); diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 38a5d12c4..70122a0f0 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -68,7 +68,7 @@ class MnistFeedForwardNet : public ModelBase { Ptr batch, bool /*clean*/ = false) override { - return apply(graph, batch, inference_); + return Logits(apply(graph, batch, inference_)); } void load(Ptr /*graph*/, const std::string& /*name*/, bool) override { diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 6e4544cf0..6b0ef93e2 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -235,7 +235,7 @@ namespace marian { cachedShortWt_ = index_select(Wt_, isLegacyUntransposedW ? -1 : 0, shortlist_->indices()); cachedShortb_ = index_select(b_ , -1, shortlist_->indices()); } - return affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true); + return Logits(affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true)); } else if (factoredVocab_) { auto graph = input->graph(); @@ -257,7 +257,7 @@ namespace marian { return Logits(std::move(allLogits), factoredVocab_); } else - return affine(input, Wt_, b_, false, /*transB=*/isLegacyUntransposedW ? false : true); + return Logits(affine(input, Wt_, b_, false, /*transB=*/isLegacyUntransposedW ? false : true)); } } diff --git a/src/layers/generic.h b/src/layers/generic.h index 838ee8754..e83801aa2 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -66,10 +66,10 @@ class RationalLoss; class Logits { public: Logits() {} - Logits(Ptr logits) { // single-output constructor + explicit Logits(Ptr logits) { // single-output constructor logits_.push_back(logits); } - Logits(Expr logits); // single-output constructor from Expr only (RationalLoss has no count) + explicit Logits(Expr logits); // single-output constructor from Expr only (RationalLoss has no count) Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), factoredVocab_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors diff --git a/src/models/costs.h b/src/models/costs.h index 2e4054bf5..306b29138 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -117,7 +117,7 @@ class EncoderClassifierCE : public CostBase { // multi-objective training Ptr multiLoss = newMultiLoss(options_); for(int i = 0; i < states.size(); ++i) { - auto partialLoss = loss_->apply(states[i]->getLogProbs(), + auto partialLoss = loss_->apply(Logits(states[i]->getLogProbs()), states[i]->getTargetWords(), /*mask=*/nullptr, /*weights=*/nullptr); diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index d32160d01..f6ffbebfc 100755 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -211,7 +211,7 @@ class EncoderClassifier : public EncoderClassifierBase { bool clearGraph = true) override { auto states = apply(graph, batch, clearGraph); // returns raw logits - return states[0]->getLogProbs(); + return Logits(states[0]->getLogProbs()); } virtual Logits build(Ptr graph, diff --git a/src/models/s2s.h b/src/models/s2s.h index 8a18e346b..a63c98418 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -368,7 +368,7 @@ class DecoderS2S : public DecoderBase { // return unormalized(!) probabilities auto nextState = New( - decoderStates, logits, state->getEncoderStates(), state->getBatch()); + decoderStates, Logits(logits), state->getEncoderStates(), state->getBatch()); // Advance current target token position by one nextState->setPosition(state->getPosition() + 1); diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index d13081728..f259f1c47 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -227,6 +227,40 @@ void ProdBatched(marian::Tensor C, allocator->free(mp_cptr); } +// debugging helper --@TODO: move it to some shared place? +template +static std::vector get(Ptr memory, Ptr backend) { + size_t n = memory->size() / sizeof(T); + TensorBase t(memory, Shape({(int)n}), getType(), backend); + std::vector res; + t.get(res); + return res; +} + +// bug in cuSparse: sparse matrix is limited to 65535 columns +// This function is a drop-in replacement that handles it (by slicing). +cusparseStatus_t +static cusparseSgemmiEx(cusparseHandle_t handle, int m, + int n, // the offending number of columns of matrices B and C + int k, int nnz, const float *alpha, const float *A, int lda, + const float *cscValB, const int *cscColPtrB, const int *cscRowIndB, const float *beta, + float *C, int ldc) +{ + const int nMax = 65535; // max. number of columns allowed by cuSparse 10 implementation + for (int j0 = 0; j0 < n; j0 += 65535) { // loop over column slices, j0 = index of first column + // Call original function on a column slice. + // Replace all parameters that relate to the column slice. + // nnz does not need to be corrected. + auto n1 = std::min(n - j0, nMax); // width of column slice is limited to max + auto C1 = C + j0 * ldc; // column slice into result matrix C + auto cscColPtrB1 = cscColPtrB + j0; // column slice into sparse factor B + auto rc = cusparseSgemmi(handle, m, n1, k, nnz, alpha, A, lda, cscValB, cscColPtrB1, cscRowIndB, beta, C1, ldc); + if (rc != CUSPARSE_STATUS_SUCCESS) + return rc; + } + return CUSPARSE_STATUS_SUCCESS; +} + // C = op(S) x D if not swapOperands else C = D x op(S) // op(S) = S if not transA else S^T void CSRProd(marian::Tensor C, @@ -315,7 +349,23 @@ void CSRProd(marian::Tensor C, else { // C = S x D for row-major matrices // Implemented via cusparse as C' = D' x S' ("gemmi") where C' and D' are column-major. - CUSPARSE_CHECK(cusparseSgemmi(cusparseHandle, + //if (St_values) { + // auto vals = get(St_values, C->getBackend()); vals.resize(numValues); + // auto inds = get(St_indices, C->getBackend()); inds.resize(numValues); + // auto offs = get(St_offsets, C->getBackend()); offs.resize(rowsS + 1); + // LOG(info, "[{} x {}] = [{} x {}] * [{} x {}]", colsC, rowsC, colsD, rowsD, -1, offs.size() - 1); + // for (auto v : vals) + // ABORT_IF(v != 1, "v={}", v); + // for (auto i : inds) + // ABORT_IF(i >= rowsC, "i={}", i); + // for (auto o : offs) + // ABORT_IF(o > inds.size(), "o={}", o); + // ABORT_IF(colsC != colsD, "0"); + // std::vector dData; D->get(dData); ABORT_IF(dData.size() != colsD * rowsD, "1"); + // std::vector cData; C->get(cData); ABORT_IF(cData.size() != colsC * rowsC, "2"); + // ABORT_IF(rowsC != rowsS, "3: {} != {}, asz={}", rowsC, rowsS, St_offsets->size()/4); + //} + CUSPARSE_CHECK(cusparseSgemmiEx(cusparseHandle, /*m=*/ colsD, // #rows of first (col-major) factor = #cols of row-major D /*n=*/ rowsC, // #cols of second (CSC) factor and (col-major) result = #rows of row-major C /*k=*/ rowsD, // #cols of first (col-major) factor = #rows of row-major D @@ -324,11 +374,14 @@ void CSRProd(marian::Tensor C, /*A=*/ D->data(), /*lda=*/ colsD, // stride /*cscValB=*/ St_values ? St_values ->data() : S_values ->data(), - /*cscRowPtrB=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), - /*cscColIndB=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), + /*cscColPtrB=*/ St_offsets ? St_offsets->data() : (int*)S_offsets->data(), + /*cscRowIndB=*/ St_indices ? St_indices->data() : (int*)S_indices->data(), &beta, C->data(), /*ldc=*/ colsC)); // stride + // Note: cuSparse 10 docs says this about cscColPtrB: + // "integer array of k + 1 elements that contains the start of every row and the end of the last row plus one." + // This is wrong. It should be col instead of row, and n instead of k. } if(St_values ) allocator->free(St_values ); if(St_indices) allocator->free(St_indices); From a2d381d65c9330dadbab520b9d4f58a351c525fa Mon Sep 17 00:00:00 2001 From: Nikolay Bogoychev Date: Sun, 24 Feb 2019 19:59:26 +0000 Subject: [PATCH 356/838] A more intelligent way to detect cblas if it's not included in openblas --- CMakeLists.txt | 2 +- cmake/FindCBLAS.cmake | 181 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 cmake/FindCBLAS.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b8537d20f..022b97019 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,7 +241,7 @@ if(COMPILE_CPU) find_package(BLAS) if(BLAS_FOUND) include_directories(${BLAS_INCLUDE_DIR}) - set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES} cblas) + set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES}) add_definitions(-DBLAS_FOUND=1) endif(BLAS_FOUND) endif(MKL_FOUND) diff --git a/cmake/FindCBLAS.cmake b/cmake/FindCBLAS.cmake new file mode 100644 index 000000000..e7426014e --- /dev/null +++ b/cmake/FindCBLAS.cmake @@ -0,0 +1,181 @@ +# - Find CBLAS library +# +# This module finds an installed fortran library that implements the CBLAS +# linear-algebra interface (see http://www.netlib.org/blas/), with CBLAS +# interface. +# +# This module sets the following variables: +# CBLAS_FOUND - set to true if a library implementing the CBLAS interface +# is found +# CBLAS_LINKER_FLAGS - uncached list of required linker flags (excluding -l +# and -L). +# CBLAS_LIBRARIES - uncached list of libraries (using full path name) to +# link against to use CBLAS +# CBLAS_INCLUDE_DIR - path to includes +# CBLAS_INCLUDE_FILE - the file to be included to use CBLAS +# + +## Based on https://github.com/Eyescale/CMake/blob/master/FindCBLAS.cmake + +INCLUDE(CheckFunctionExists) +INCLUDE(CheckIncludeFile) + +MACRO(CHECK_ALL_LIBRARIES LIBRARIES _prefix _name _flags _list _include _search_include) + # This macro checks for the existence of the combination of fortran libraries + # given by _list. If the combination is found, this macro checks (using the + # Check_Fortran_Function_Exists macro) whether can link against that library + # combination using the name of a routine given by _name using the linker + # flags given by _flags. If the combination of libraries is found and passes + # the link test, LIBRARIES is set to the list of complete library paths that + # have been found. Otherwise, LIBRARIES is set to FALSE. + + # N.B. _prefix is the prefix applied to the names of all cached variables that + # are generated internally and marked advanced by this macro. + + SET(__list) + FOREACH(_elem ${_list}) + IF(__list) + SET(__list "${__list} - ${_elem}") + ELSE(__list) + SET(__list "${_elem}") + ENDIF(__list) + ENDFOREACH(_elem) + MESSAGE(STATUS "Checking for [${__list}]") + SET(_libraries_work TRUE) + SET(${LIBRARIES}) + SET(_combined_name) + SET(_paths) + FOREACH(_library ${_list}) + SET(_combined_name ${_combined_name}_${_library}) + + # did we find all the libraries in the _list until now? + # (we stop at the first unfound one) + IF(_libraries_work) + IF(APPLE) + FIND_LIBRARY(${_prefix}_${_library}_LIBRARY + NAMES ${_library} + PATHS /usr/local/lib /usr/lib /usr/local/lib64 /usr/lib64 ENV + DYLD_LIBRARY_PATH + ) + ELSE(APPLE) + FIND_LIBRARY(${_prefix}_${_library}_LIBRARY + NAMES ${_library} + PATHS /usr/local/lib /usr/lib /usr/local/lib64 /usr/lib64 ENV + LD_LIBRARY_PATH + ) + ENDIF(APPLE) + MARK_AS_ADVANCED(${_prefix}_${_library}_LIBRARY) + IF(${_prefix}_${_library}_LIBRARY) + GET_FILENAME_COMPONENT(_path ${${_prefix}_${_library}_LIBRARY} PATH) + LIST(APPEND _paths ${_path}/../include ${_path}/../../include) + ENDIF(${_prefix}_${_library}_LIBRARY) + SET(${LIBRARIES} ${${LIBRARIES}} ${${_prefix}_${_library}_LIBRARY}) + SET(_libraries_work ${${_prefix}_${_library}_LIBRARY}) + ENDIF(_libraries_work) + ENDFOREACH(_library ${_list}) + + # Test include + SET(_bug_search_include ${_search_include}) #CMAKE BUG!!! SHOULD NOT BE THAT + IF(_bug_search_include) + FIND_PATH(${_prefix}${_combined_name}_INCLUDE ${_include} ${_paths}) + MARK_AS_ADVANCED(${_prefix}${_combined_name}_INCLUDE) + IF(${_prefix}${_combined_name}_INCLUDE) + MESSAGE(STATUS "Checking for [${__list}] -- includes found") + SET(${_prefix}_INCLUDE_DIR ${${_prefix}${_combined_name}_INCLUDE}) + SET(${_prefix}_INCLUDE_FILE ${_include}) + ELSE(${_prefix}${_combined_name}_INCLUDE) + MESSAGE(STATUS "Checking for [${__list}] -- includes not found") + SET(_libraries_work FALSE) + ENDIF(${_prefix}${_combined_name}_INCLUDE) + ELSE(_bug_search_include) + SET(${_prefix}_INCLUDE_DIR) + SET(${_prefix}_INCLUDE_FILE ${_include}) + ENDIF(_bug_search_include) + + IF(_libraries_work) + # Test this combination of libraries. + SET(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}}) + CHECK_FUNCTION_EXISTS(${_name} ${_prefix}${_combined_name}_WORKS) + SET(CMAKE_REQUIRED_LIBRARIES) + MARK_AS_ADVANCED(${_prefix}${_combined_name}_WORKS) + SET(_libraries_work ${${_prefix}${_combined_name}_WORKS}) + + IF(_libraries_work) + MESSAGE(STATUS "Checking for [${__list}] -- libraries found") + ENDIF(_libraries_work) + + ENDIF(_libraries_work) + + + IF(NOT _libraries_work) + SET(${LIBRARIES} FALSE) + ENDIF(NOT _libraries_work) + +ENDMACRO(CHECK_ALL_LIBRARIES) + +SET(CBLAS_LINKER_FLAGS) +SET(CBLAS_LIBRARIES) + +# CBLAS in openBLAS +IF(NOT CBLAS_LIBRARIES) + CHECK_ALL_LIBRARIES( + CBLAS_LIBRARIES + cblas + cblas_sgemm + "" + "openblas" + "cblas.h" + TRUE + ) +ENDIF(NOT CBLAS_LIBRARIES) + +#MESSAGE(STATUS ${openblas_INCLUDE_DIR}) + +# CBLAS in CBLAS +IF(NOT CBLAS_LIBRARIES) + CHECK_ALL_LIBRARIES( + CBLAS_LIBRARIES + cblas + cblas_sgemm + "" + "cblas" + "cblas.h" + TRUE + ) +ENDIF(NOT CBLAS_LIBRARIES) + +#MESSAGE(STATUS ${cblas_INCLUDE_DIR}) + +# CBLAS in lapacke +IF(NOT CBLAS_LIBRARIES) + CHECK_ALL_LIBRARIES( + CBLAS_LIBRARIES + cblas + cblas_sgemm + "" + "lapacke" + "cblas.h" + TRUE + ) +ENDIF(NOT CBLAS_LIBRARIES) + +#MESSAGE(STATUS ${lapacke_INCLUDE_DIR}) + +IF(CBLAS_LIBRARIES) + SET(CBLAS_FOUND TRUE) +ELSE(CBLAS_LIBRARIES) + SET(CBLAS_FOUND FALSE) +ENDIF(CBLAS_LIBRARIES) + +IF(NOT CBLAS_FOUND AND CBLAS_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "CBLAS library not found. Please specify library location") +ENDIF(NOT CBLAS_FOUND AND CBLAS_FIND_REQUIRED) + +IF(NOT CBLAS_FIND_QUIETLY) + IF(CBLAS_FOUND) + MESSAGE(STATUS "CBLAS library found: " ${CBLAS_LIBRARIES}) + #MESSAGE(STATUS ) + ELSE(CBLAS_FOUND) + MESSAGE(STATUS "CBLAS library not found. Please specify library location") + ENDIF(CBLAS_FOUND) +ENDIF(NOT CBLAS_FIND_QUIETLY) From f8943a78d14cee9bec67b6ebb669b328194120f7 Mon Sep 17 00:00:00 2001 From: Nikolay Bogoychev Date: Sun, 24 Feb 2019 20:29:54 +0000 Subject: [PATCH 357/838] Add the changes to CMakeLists.txt and add also support for finding the include directory --- CMakeLists.txt | 9 ++++++--- cmake/FindCBLAS.cmake | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 022b97019..f9011cd6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,9 +240,12 @@ if(COMPILE_CPU) set(BLA_VENDOR "OpenBLAS") find_package(BLAS) if(BLAS_FOUND) - include_directories(${BLAS_INCLUDE_DIR}) - set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES}) - add_definitions(-DBLAS_FOUND=1) + include(FindCBLAS) + if(CBLAS_FOUND) + include_directories(${BLAS_INCLUDE_DIR} ${CBLAS_INCLUDE_DIR}) + set(EXT_LIBS ${EXT_LIBS} ${BLAS_LIBRARIES} ${CBLAS_LIBRARIES}) + add_definitions(-DBLAS_FOUND=1) + endif(CBLAS_FOUND) endif(BLAS_FOUND) endif(MKL_FOUND) endif(COMPILE_CPU) diff --git a/cmake/FindCBLAS.cmake b/cmake/FindCBLAS.cmake index e7426014e..631da99b4 100644 --- a/cmake/FindCBLAS.cmake +++ b/cmake/FindCBLAS.cmake @@ -20,7 +20,7 @@ INCLUDE(CheckFunctionExists) INCLUDE(CheckIncludeFile) -MACRO(CHECK_ALL_LIBRARIES LIBRARIES _prefix _name _flags _list _include _search_include) +MACRO(CHECK_ALL_LIBRARIES LIBRARIES INCLUDE _prefix _name _flags _list _include _search_include) # This macro checks for the existence of the combination of fortran libraries # given by _list. If the combination is found, this macro checks (using the # Check_Fortran_Function_Exists macro) whether can link against that library @@ -83,6 +83,7 @@ MACRO(CHECK_ALL_LIBRARIES LIBRARIES _prefix _name _flags _list _include _search_ MESSAGE(STATUS "Checking for [${__list}] -- includes found") SET(${_prefix}_INCLUDE_DIR ${${_prefix}${_combined_name}_INCLUDE}) SET(${_prefix}_INCLUDE_FILE ${_include}) + SET(${INCLUDE} ${${_prefix}_INCLUDE_DIR}) ELSE(${_prefix}${_combined_name}_INCLUDE) MESSAGE(STATUS "Checking for [${__list}] -- includes not found") SET(_libraries_work FALSE) @@ -115,11 +116,13 @@ ENDMACRO(CHECK_ALL_LIBRARIES) SET(CBLAS_LINKER_FLAGS) SET(CBLAS_LIBRARIES) +SET(CBLAS_INCLUDE_DIR) # CBLAS in openBLAS IF(NOT CBLAS_LIBRARIES) CHECK_ALL_LIBRARIES( CBLAS_LIBRARIES + CBLAS_INCLUDE_DIR cblas cblas_sgemm "" @@ -135,6 +138,7 @@ ENDIF(NOT CBLAS_LIBRARIES) IF(NOT CBLAS_LIBRARIES) CHECK_ALL_LIBRARIES( CBLAS_LIBRARIES + CBLAS_INCLUDE_DIR cblas cblas_sgemm "" @@ -150,6 +154,7 @@ ENDIF(NOT CBLAS_LIBRARIES) IF(NOT CBLAS_LIBRARIES) CHECK_ALL_LIBRARIES( CBLAS_LIBRARIES + CBLAS_INCLUDE_DIR cblas cblas_sgemm "" @@ -174,7 +179,7 @@ ENDIF(NOT CBLAS_FOUND AND CBLAS_FIND_REQUIRED) IF(NOT CBLAS_FIND_QUIETLY) IF(CBLAS_FOUND) MESSAGE(STATUS "CBLAS library found: " ${CBLAS_LIBRARIES}) - #MESSAGE(STATUS ) + MESSAGE(STATUS "cblas.h include directory: " ${CBLAS_INCLUDE_DIR}) ELSE(CBLAS_FOUND) MESSAGE(STATUS "CBLAS library not found. Please specify library location") ENDIF(CBLAS_FOUND) From 4834983770ea1771f2a6a665c912f89cd1de7357 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 24 Feb 2019 18:26:29 -0800 Subject: [PATCH 358/838] added @WB and @WE factors --- src/data/factored_vocab.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index bbae31127..00af9a1b5 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -27,7 +27,7 @@ namespace marian { // load factor vocabulary factorVocab_.load(factorVocabPath); - groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR" }; // @TODO: hard-coded for these initial experiments + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB", "@WE" }; // @TODO: hard-coded for these initial experiments // construct mapping tables for factors constructGroupInfoFromFactorVocab(); @@ -369,7 +369,8 @@ void FactoredVocab::constructNormalizationInfoForVocab() { /*virtual*/ std::string FactoredVocab::toEnglishTitleCase(const std::string& line) const /*override final*/ { // @BUGBUG: does not handle the special words that should remain lower-case - return utils::findReplace(line, "@CN@GL-", "@CI@GL-", /*all=*/true); + // note: this presently supports both @WB and @GL- (legacy) + return utils::findReplace(utils::findReplace(line, "@CN@WB", "@CI@WB", /*all=*/true), "@CN@GL-", "@CI@GL-", /*all=*/true); } // generate a valid random factored word (used by collectStats()) From ae6fc3c19c9cb064033dbc093c9c1fc2410b79db Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 25 Feb 2019 13:27:50 -0800 Subject: [PATCH 359/838] added @CB and @CE factor types --- src/data/factored_vocab.cpp | 6 ++++-- vs/Marian.sln | 0 2 files changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 vs/Marian.sln diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 00af9a1b5..7f2138daf 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -27,7 +27,7 @@ namespace marian { // load factor vocabulary factorVocab_.load(factorVocabPath); - groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB", "@WE" }; // @TODO: hard-coded for these initial experiments + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB", "@WE", "@CB", "@CE" }; // @TODO: hard-coded for these initial experiments // construct mapping tables for factors constructGroupInfoFromFactorVocab(); @@ -137,7 +137,9 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { const auto& groupPrefix = groupPrefixes_[g]; for (WordIndex u = 0; u < factorVocabSize; u++) if (utils::beginsWith(factorVocab_[u], groupPrefix)) { - ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[u], groupPrefix); + //ABORT_IF(factorGroups_[u] != 0, "Factor {} matches multiple groups, incl. {}", factorVocab_[u], groupPrefix); + if(factorGroups_[u] != 0) + LOG(info, "Factor {} matches multiple groups, incl. {}, using {}", factorVocab_[u], groupPrefixes_[factorGroups_[u]], groupPrefix); factorGroups_[u] = g; } } diff --git a/vs/Marian.sln b/vs/Marian.sln old mode 100644 new mode 100755 From 842887cd661d089810bbcec2d135f0ea9ac75028 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 25 Feb 2019 14:08:51 -0800 Subject: [PATCH 360/838] removed @WE and @CE since empty factors currently cause it to fail --- src/data/factored_vocab.cpp | 11 ++++++++++- src/layers/generic.cpp | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 7f2138daf..2a759448a 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -27,7 +27,8 @@ namespace marian { // load factor vocabulary factorVocab_.load(factorVocabPath); - groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB", "@WE", "@CB", "@CE" }; // @TODO: hard-coded for these initial experiments + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB",/* "@WE",*/ "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments + // @TODO: add checks for empty factor groups until it stops crashing // construct mapping tables for factors constructGroupInfoFromFactorVocab(); @@ -154,6 +155,14 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { groupRanges_[g].second = u + 1; groupCounts[g]++; } + //for (size_t g = 0; g < numGroups; g++) { // fix up empty entries + // LOG(info, "GROUP {}: {} {}", groupPrefixes_[g], groupRanges_[g].first, groupRanges_[g].second); + // //if (groupCounts[g] == 0) { + // // ABORT("Group {} has no members", groupPrefixes_[g]); + // // //groupRanges_[g].first = g > 0 ? groupRanges_[g-1].second : 0; + // // //groupRanges_[g].second = groupRanges_[g].first; + // //} + //} for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members", groupPrefixes_[g], groupCounts[g]); if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 6b0ef93e2..1c9a011a5 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -245,7 +245,9 @@ namespace marian { std::vector> allLogits(numGroups); for (size_t g = 0; g < numGroups; g++) { auto range = factoredVocab_->getGroupRange(g); - ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive"); // we could sort groupYs though + //if (range.first == SIZE_MAX) + // continue; + ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive (group {} vs predecessor)", g); // we could sort groupYs though // slice this group's section out of W_ // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. auto factorWt = slice(Wt_, isLegacyUntransposedW ? -1 : 0, Slice((int)range.first, (int)range.second)); From 4d57f87adf170c09fedbae87f46047748815699f Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Fri, 1 Mar 2019 18:41:40 +0000 Subject: [PATCH 361/838] Add switch fallthroughs for marian-nmt/marian#190 --- src/tensors/cpu/sharp/avx_gemm.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tensors/cpu/sharp/avx_gemm.cpp b/src/tensors/cpu/sharp/avx_gemm.cpp index c41b73eb0..a1018c7df 100644 --- a/src/tensors/cpu/sharp/avx_gemm.cpp +++ b/src/tensors/cpu/sharp/avx_gemm.cpp @@ -495,6 +495,7 @@ void AVX_MatrixMult8(const __m512i *A, put.Write(C + (i + 4) * num_B_rows + j, Reduce16to32(sum5, sum6)); put.Write(C + (i + 6) * num_B_rows + j, Reduce16to32(sum7)); } + [[fallthrough]]; case 6: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; @@ -518,6 +519,7 @@ void AVX_MatrixMult8(const __m512i *A, put.Write(C + i * num_B_rows + j, Reduce16to32(sum1, sum2, sum3, sum4)); put.Write(C + (i + 4) * num_B_rows + j, Reduce16to32(sum5, sum6)); } + [[fallthrough]]; case 5: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; @@ -539,6 +541,7 @@ void AVX_MatrixMult8(const __m512i *A, put.Write(C + i * num_B_rows + j, Reduce16to32(sum1, sum2, sum3, sum4)); put.Write(C + (i + 4) * num_B_rows + j, Reduce16to32(sum5)); } + [[fallthrough]]; case 4: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; @@ -557,6 +560,7 @@ void AVX_MatrixMult8(const __m512i *A, } put.Write(C + i * num_B_rows + j, Reduce16to32(sum1, sum2, sum3, sum4)); } + [[fallthrough]]; case 3: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; @@ -574,6 +578,7 @@ void AVX_MatrixMult8(const __m512i *A, put.Write(C + i * num_B_rows + j, Reduce16to32(sum1, sum2)); put.Write(C + (i + 2) * num_B_rows + j, Reduce16to32(sum3)); } + [[fallthrough]]; case 2: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; @@ -588,6 +593,7 @@ void AVX_MatrixMult8(const __m512i *A, } put.Write(C + i * num_B_rows + j, Reduce16to32(sum1, sum2)); } + [[fallthrough]]; case 1: for(int j = 0; j < num_B_rows; j++) { const __m512i *B_row = B + j * sse_width; From f9132495319ec76cf3065d29272ed9ce9c627acf Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 08:51:30 -0700 Subject: [PATCH 362/838] new parameter lemma-dim-emb (simple conditional for factors); some more robustness against unused factors --- src/common/config_parser.cpp | 3 +++ src/data/factored_vocab.cpp | 22 +++++++--------- src/layers/generic.cpp | 46 ++++++++++++++++++++++++++++----- src/models/encoder_classifier.h | 1 + src/models/encoder_decoder.cpp | 1 + src/models/transformer.h | 1 + src/translator/beam_search.h | 3 +++ 7 files changed, 58 insertions(+), 19 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 54c1f6a66..ee49b448f 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -121,6 +121,9 @@ void ConfigParser::addOptionsModel(cli::CLIWrapper& cli) { cli.add("--dim-emb", "Size of embedding vector", 512); + cli.add("--lemma-dim-emb", + "Re-embedding dimension of lemma in factors", + 0); cli.add("--dim-rnn", "Size of rnn hidden state", 1024); cli.add("--enc-type", diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 2a759448a..2b0695fd2 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -27,8 +27,8 @@ namespace marian { // load factor vocabulary factorVocab_.load(factorVocabPath); - groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB",/* "@WE",*/ "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments - // @TODO: add checks for empty factor groups until it stops crashing + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB"/*, "@WE"*/, "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments + // @TODO: add checks for empty factor groups until it stops crashing (training already works; decoder still crashes) // construct mapping tables for factors constructGroupInfoFromFactorVocab(); @@ -155,18 +155,13 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { groupRanges_[g].second = u + 1; groupCounts[g]++; } - //for (size_t g = 0; g < numGroups; g++) { // fix up empty entries - // LOG(info, "GROUP {}: {} {}", groupPrefixes_[g], groupRanges_[g].first, groupRanges_[g].second); - // //if (groupCounts[g] == 0) { - // // ABORT("Group {} has no members", groupPrefixes_[g]); - // // //groupRanges_[g].first = g > 0 ? groupRanges_[g-1].second : 0; - // // //groupRanges_[g].second = groupRanges_[g].first; - // //} - //} for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups LOG(info, "[embedding] Factor group '{}' has {} members", groupPrefixes_[g], groupCounts[g]); - if (groupCounts[g] == 0) // factor group is unused --@TODO: once this is not hard-coded, this is an error condition + if (groupCounts[g] == 0) { // factor group is unused --@TODO: once this is not hard-coded, this is an error condition + groupRanges_[g].first = g > 0 ? groupRanges_[g-1].second : 0; // fix up the entry + groupRanges_[g].second = groupRanges_[g].first; continue; + } ABORT_IF(groupRanges_[g].second - groupRanges_[g].first != groupCounts[g], "Factor group '{}' members should be consecutive in the factor vocabulary", groupPrefixes_[g]); } @@ -354,8 +349,9 @@ void FactoredVocab::constructNormalizationInfoForVocab() { if (found) return Word::fromWordIndex(index); else - ABORT("Unknown word {}", word); - //return getUnkId(); + //ABORT("Unknown word {} mapped to {}", word, word2string(getUnkId())); + LOG(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); + return getUnkId(); } /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*overrworde final*/ { diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 1c9a011a5..f59105932 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -47,6 +47,8 @@ namespace marian { // Memory-wise, this is cheap, all temp objects below are batches of scalars or lookup vectors. Expr loss; for (size_t g = 0; g < numGroups; g++) { + if (!logits_[g]) + continue; // empty factor --@TODO: handle this more nicely const auto& maskedFactoredLabels = allMaskedFactoredLabels[g]; // array of (word index, mask) #if 1 auto factorIndices = indices (maskedFactoredLabels.indices); // [B... flattened] factor-label indices, or 0 if factor does not apply @@ -242,19 +244,51 @@ namespace marian { // project each factor separately auto numGroups = factoredVocab_->getNumGroups(); - std::vector> allLogits(numGroups); + std::vector> allLogits(numGroups, nullptr); // (note: null entries for absent factors) + Expr input1 = input; + Expr Plemma = nullptr; for (size_t g = 0; g < numGroups; g++) { auto range = factoredVocab_->getGroupRange(g); - //if (range.first == SIZE_MAX) - // continue; - ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive (group {} vs predecessor)", g); // we could sort groupYs though + if (g > 0 && range.first == range.second) // empty entry + continue; + ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive (group {} vs predecessor)", g); // slice this group's section out of W_ - // @TODO: This is highly inefficient if not tied. We should always transpose Output's matrix. auto factorWt = slice(Wt_, isLegacyUntransposedW ? -1 : 0, Slice((int)range.first, (int)range.second)); auto factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); // @TODO: b_ should be a vector, not a matrix; but shotlists use cols() in, which requires a matrix - auto factorLogits = affine(input, factorWt, factorB, false, /*transB=*/isLegacyUntransposedW ? false : true); // [B... x U] factor logits + auto factorLogits = affine(input1, factorWt, factorB, false, /*transB=*/isLegacyUntransposedW ? false : true); // [B... x U] factor logits + // optionally add lemma-dependent bias + if (Plemma) { // [B... x U0] + int lemmaVocabDim = Plemma->shape()[-1]; + int factorVocabDim = factorLogits->shape()[-1]; + auto name = options_->get("prefix"); + Expr lemmaBt = graph_->param(name + "_lemmaBt_" + std::to_string(g), {factorVocabDim, lemmaVocabDim}, inits::zeros/*glorot_uniform*/); // [U x U0] U0=#lemmas one bias per class per lemma + auto b = dot(Plemma, lemmaBt, false, true); // [B... x U] + factorLogits = factorLogits + b; + } allLogits[g] = New(factorLogits, nullptr); + // optionally add a soft embedding of lemma back to create some lemma dependency + // @TODO: if this works, move it into lazyConstruct + const int lemmaDimEmb = options_->get("lemma-dim-emb", 0); + if (lemmaDimEmb < 0 && g == 0) { + LOG_ONCE(info, "[embedding] using lemma-dependent bias"); + factorLogits = logsoftmax(factorLogits); // explicitly, since we do that again later + auto z = /*stopGradient*/(factorLogits); + Plemma = exp(z); // [B... x U] + } + if (lemmaDimEmb > 0 && g == 0) { + LOG_ONCE(info, "[embedding] enabled re-embedding of lemma, at dim {}", lemmaDimEmb); + int lemmaVocabDim = factorLogits->shape()[-1]; + int inputDim = input1->shape()[-1]; + auto name = options_->get("prefix"); + factorLogits = logsoftmax(factorLogits); // explicitly, since we do that again later + Expr lemmaEt = graph_->param(name + "_lemmaEt", { lemmaDimEmb, lemmaVocabDim }, inits::glorot_uniform); // [L x U] L=lemmaDimEmb; transposed for speed + auto e = dot(exp(factorLogits), lemmaEt, false, true); // [B... x L] + //e = tanh(e); // make it non-scale-preserving + Expr lemmaWt = inputDim == lemmaDimEmb ? nullptr : graph_->param(name + "_lemmaWt", { inputDim, lemmaDimEmb }, inits::glorot_uniform); // [D x L] D=hidden-vector dimension + auto f = lemmaWt ? dot(e, lemmaWt, false, true) : e; // [B... x D] + input1 = input1 + f; // augment the original hidden vector with this additional information + } } return Logits(std::move(allLogits), factoredVocab_); } diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index f6ffbebfc..04a099861 100755 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -136,6 +136,7 @@ class EncoderClassifier : public EncoderClassifierBase { modelFeatures_.insert("ulr"); modelFeatures_.insert("ulr-trainable-transformation"); modelFeatures_.insert("ulr-dim-emb"); + modelFeatures_.insert("lemma-dim-emb"); } virtual Ptr getOptions() override { return options_; } diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 62c60ddb8..5c7053f46 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -57,6 +57,7 @@ EncoderDecoder::EncoderDecoder(Ptr options) modelFeatures_.insert("ulr"); modelFeatures_.insert("ulr-trainable-transformation"); modelFeatures_.insert("ulr-dim-emb"); + modelFeatures_.insert("lemma-dim-emb"); } std::vector>& EncoderDecoder::getEncoders() { diff --git a/src/models/transformer.h b/src/models/transformer.h index 87bda0a89..0318674f3 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -655,6 +655,7 @@ class DecoderTransformer : public Transformer { } outputFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored outputs + outputFactory("lemma-dim-emb", opt("lemma-dim-emb", 0)); // for factored outputs output_ = std::dynamic_pointer_cast(outputFactory.construct(graph_)); // (construct() returns only the underlying interface) } diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index a22fd5dfa..43dd5f12a 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -109,7 +109,10 @@ class BeamSearch { beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? for(size_t j = 0; j < states.size(); ++j) { size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' + flattenedLogitIndex; +#if 0 // @BUGBUG: This currently segfaults with factors. breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; +#endif // @TODO: pass those 3 indices directly into breakDown (state knows the dimensions) } hyp->setScoreBreakdown(breakDown); From 129253cbc60cf41e71a953cae59f3b8f36c210eb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 11:06:10 -0700 Subject: [PATCH 363/838] fixed build errors in SentencePiece --- src/data/sentencepiece_vocab.cpp | 18 +++++++++++------- src/models/transformer.h | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) mode change 100644 => 100755 src/data/sentencepiece_vocab.cpp diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100644 new mode 100755 index 2f419738d..b2b829019 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -135,8 +135,8 @@ class SentencePieceVocab : public IVocab { virtual std::string type() const override { return "SentencePieceVocab"; } - virtual Word getEosId() const override { return (Word)spm_->eos_id(); } - virtual Word getUnkId() const override { return (Word)spm_->unk_id(); } + virtual Word getEosId() const override { return Word::fromWordIndex(spm_->eos_id()); } + virtual Word getUnkId() const override { return Word::fromWordIndex(spm_->unk_id()); } void create(const std::string& vocabPath, const std::vector& trainPaths, @@ -197,12 +197,12 @@ class SentencePieceVocab : public IVocab { } Word operator[](const std::string& token) const override { - return (Word)spm_->PieceToId(token); + return Word::fromWordIndex(spm_->PieceToId(token)); } const std::string& operator[](Word id) const override { - ABORT_IF(id >= size(), "Unknown word id: ", id); - return spm_->IdToPiece(id); + ABORT_IF(id.toWordIndex() >= size(), "Unknown word id: ", id.toWordIndex()); + return spm_->IdToPiece(id.toWordIndex()); } Words encode(const std::string& line, bool addEOS, bool inference) const override { @@ -212,7 +212,9 @@ class SentencePieceVocab : public IVocab { else spm_->SampleEncode(line, -1, alpha_, &spmIds); - Words words(spmIds.begin(), spmIds.end()); + Words words; words.reserve(spmIds.size() + addEOS); + for (auto&& spmId : spmIds) + words.push_back(Word::fromWordIndex(spmId)); if(addEOS) words.push_back(getEosId()); @@ -222,7 +224,9 @@ class SentencePieceVocab : public IVocab { std::string decode(const Words& sentence, bool /*ignoreEOS*/) const override { std::string line; // convert vector of Word to vector of int - std::vector spmSentence(sentence.begin(), sentence.end()); + std::vector spmSentence; spmSentence.reserve(sentence.size()); + for (auto&& word : sentence) + spmSentence.push_back(word.toWordIndex()); spm_->Decode(spmSentence, &line); return line; } diff --git a/src/models/transformer.h b/src/models/transformer.h index 0318674f3..1817947ea 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -72,7 +72,7 @@ class Transformer : public EncoderOrDecoderBase { // fill with increasing numbers until current length or maxPos std::vector positions(dimWords, numPos - 1); for(int i = 0; i < std::min(dimWords, numPos); ++i) - positions[i] = i; // @TODO: use std::iota()? + positions[i] = i; auto signal = embeddingLayer->applyIndices(positions, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; From 2c80015144edc95c1c82a694c3c6ddf086c589be Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 11:08:15 -0700 Subject: [PATCH 364/838] fixed build errors in SentencePiece --- src/data/sentencepiece_vocab.cpp | 19 +++++++++++-------- src/models/transformer.h | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) mode change 100644 => 100755 src/data/sentencepiece_vocab.cpp mode change 100644 => 100755 src/models/transformer.h diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100644 new mode 100755 index 9af727565..b2b829019 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -126,7 +126,6 @@ class SentencePieceVocab : public IVocab { alpha_, batchIndex_); } - } virtual const std::string& canonicalExtension() const override { return suffixes_[0]; } @@ -136,8 +135,8 @@ class SentencePieceVocab : public IVocab { virtual std::string type() const override { return "SentencePieceVocab"; } - virtual Word getEosId() const override { return (Word)spm_->eos_id(); } - virtual Word getUnkId() const override { return (Word)spm_->unk_id(); } + virtual Word getEosId() const override { return Word::fromWordIndex(spm_->eos_id()); } + virtual Word getUnkId() const override { return Word::fromWordIndex(spm_->unk_id()); } void create(const std::string& vocabPath, const std::vector& trainPaths, @@ -198,12 +197,12 @@ class SentencePieceVocab : public IVocab { } Word operator[](const std::string& token) const override { - return (Word)spm_->PieceToId(token); + return Word::fromWordIndex(spm_->PieceToId(token)); } const std::string& operator[](Word id) const override { - ABORT_IF(id >= size(), "Unknown word id: ", id); - return spm_->IdToPiece(id); + ABORT_IF(id.toWordIndex() >= size(), "Unknown word id: ", id.toWordIndex()); + return spm_->IdToPiece(id.toWordIndex()); } Words encode(const std::string& line, bool addEOS, bool inference) const override { @@ -213,7 +212,9 @@ class SentencePieceVocab : public IVocab { else spm_->SampleEncode(line, -1, alpha_, &spmIds); - Words words(spmIds.begin(), spmIds.end()); + Words words; words.reserve(spmIds.size() + addEOS); + for (auto&& spmId : spmIds) + words.push_back(Word::fromWordIndex(spmId)); if(addEOS) words.push_back(getEosId()); @@ -223,7 +224,9 @@ class SentencePieceVocab : public IVocab { std::string decode(const Words& sentence, bool /*ignoreEOS*/) const override { std::string line; // convert vector of Word to vector of int - std::vector spmSentence(sentence.begin(), sentence.end()); + std::vector spmSentence; spmSentence.reserve(sentence.size()); + for (auto&& word : sentence) + spmSentence.push_back(word.toWordIndex()); spm_->Decode(spmSentence, &line); return line; } diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100644 new mode 100755 index fcd2442aa..91d6c35f8 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -73,7 +73,7 @@ class Transformer : public EncoderOrDecoderBase { // fill with increasing numbers until current length or maxPos std::vector positions(dimWords, numPos - 1); for(int i = 0; i < std::min(dimWords, numPos); ++i) - positions[i] = i; // @TODO: use std::iota()? + positions[i] = i; auto signal = embeddingLayer->applyIndices(positions, {dimWords, 1, dimEmb}); embeddings = embeddings + signal; From 95387b1eb65850a5074eccf52ab4851a5e41cea5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 11:51:50 -0700 Subject: [PATCH 365/838] bug fix: MPI should only fail with unsupported threading mode if we actually run distributed --- src/training/communicator.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) mode change 100644 => 100755 src/training/communicator.cpp diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp old mode 100644 new mode 100755 index 35835a0ed..6307cc8b1 --- a/src/training/communicator.cpp +++ b/src/training/communicator.cpp @@ -79,13 +79,12 @@ class MPIWrapper : public IMPIWrapper HANDLE_MPI_ERROR(MPI_Init_thread(&argc, &argvp, MPI_THREAD_MULTIPLE, &providedThreadingMode)); MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); // have errors reported as return codes - ABORT_IF( - providedThreadingMode < requiredThreadingMode, - "Your version of MPI does not support multi-threaded communication."); - MPI_Comm_size(MPI_COMM_WORLD, &comm_world_size_); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank_); + ABORT_IF(comm_world_size_ > 1 && providedThreadingMode < requiredThreadingMode, + "Your version of MPI does not support multi-threaded communication."); + // patch logging pattern to include the MPI rank, so that we can associate error messages with nodes if (numMPIProcesses() > 1) { std::string rankStr = std::to_string(MPIWrapper::myMPIRank()); From c91059774431ded8978cd683d2d2ce196f6b1ea7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 13:33:42 -0700 Subject: [PATCH 366/838] bug fix: MNIST model creation somehow got lost in last refactoring --- src/models/model_factory.cpp | 91 ++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index d7bd8d854..cee855ab6 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -106,12 +106,12 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o return models::encoder_decoder()(options) ("usage", use) ("original-type", type) - .push_back(models::encoder()("type", "s2s")) - .push_back(models::decoder()("type", "s2s")) - .construct(graph); + .push_back(models::encoder()("type", "s2s")) + .push_back(models::decoder()("type", "s2s")) + .construct(graph); } - if(type == "transformer") { + else if(type == "transformer") { return models::encoder_decoder()(options) ("usage", use) .push_back(models::encoder()("type", "transformer")) @@ -119,16 +119,16 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o .construct(graph); } - if(type == "transformer_s2s") { + else if(type == "transformer_s2s") { return models::encoder_decoder()(options) ("usage", use) ("original-type", type) - .push_back(models::encoder()("type", "transformer")) - .push_back(models::decoder()("type", "s2s")) - .construct(graph); + .push_back(models::encoder()("type", "transformer")) + .push_back(models::decoder()("type", "s2s")) + .construct(graph); } - if(type == "lm") { + else if(type == "lm") { auto idx = options->has("index") ? options->get("index") : 0; std::vector dimVocabs = options->get>("dim-vocabs"); int vocab = dimVocabs[0]; @@ -139,13 +139,13 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o ("usage", use) ("type", "s2s") ("original-type", type) - .push_back(models::decoder() - ("index", idx) - ("dim-vocabs", dimVocabs)) - .construct(graph); + .push_back(models::decoder() + ("index", idx) + ("dim-vocabs", dimVocabs)) + .construct(graph); } - if(type == "multi-s2s") { + else if(type == "multi-s2s") { size_t numEncoders = 2; auto ms2sFactory = models::encoder_decoder()(options) ("usage", use) @@ -162,7 +162,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o return ms2sFactory.construct(graph); } - if(type == "shared-multi-s2s") { + else if(type == "shared-multi-s2s") { size_t numEncoders = 2; auto ms2sFactory = models::encoder_decoder()(options) ("usage", use) @@ -179,7 +179,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o return ms2sFactory.construct(graph); } - if(type == "multi-transformer") { + else if(type == "multi-transformer") { size_t numEncoders = 2; auto mtransFactory = models::encoder_decoder()(options) ("usage", use) @@ -195,7 +195,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o return mtransFactory.construct(graph); } - if(type == "shared-multi-transformer") { + else if(type == "shared-multi-transformer") { size_t numEncoders = 2; auto mtransFactory = models::encoder_decoder()(options) ("usage", use) @@ -211,7 +211,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o return mtransFactory.construct(graph); } - if(type == "lm-transformer") { + else if(type == "lm-transformer") { auto idx = options->has("index") ? options->get("index") : 0; std::vector dimVocabs = options->get>("dim-vocabs"); int vocab = dimVocabs[0]; @@ -222,56 +222,65 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o ("usage", use) ("type", "transformer") ("original-type", type) - .push_back(models::decoder() - ("index", idx) - ("dim-vocabs", dimVocabs)) - .construct(graph); + .push_back(models::decoder() + ("index", idx) + ("dim-vocabs", dimVocabs)) + .construct(graph); } - if(type == "bert") { // for full BERT training + else if(type == "bert") { // for full BERT training return models::encoder_classifier()(options) // ("original-type", "bert") // so we can query this ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // close to original transformer encoder - ("index", 0)) // + ("type", "bert-encoder") // close to original transformer encoder + ("index", 0)) // .push_back(models::classifier() // - ("prefix", "masked-lm") // prefix for parameter names - ("type", "bert-masked-lm") // - ("index", 0)) // multi-task learning with MaskedLM + ("prefix", "masked-lm") // prefix for parameter names + ("type", "bert-masked-lm") // + ("index", 0)) // multi-task learning with MaskedLM .push_back(models::classifier() // - ("prefix", "next-sentence") // prefix for parameter names - ("type", "bert-classifier") // - ("index", 1)) // next sentence prediction + ("prefix", "next-sentence") // prefix for parameter names + ("type", "bert-classifier") // + ("index", 1)) // next sentence prediction .construct(graph); } - if(type == "bert-classifier") { // for BERT fine-tuning on non-BERT classification task + else if(type == "bert-classifier") { // for BERT fine-tuning on non-BERT classification task return models::encoder_classifier()(options) // ("original-type", "bert-classifier") // so we can query this if needed ("usage", use) // .push_back(models::encoder() // - ("type", "bert-encoder") // - ("index", 0)) // close to original transformer encoder + ("type", "bert-encoder") // + ("index", 0)) // close to original transformer encoder .push_back(models::classifier() // - ("type", "bert-classifier") // - ("index", 1)) // next sentence prediction + ("type", "bert-classifier") // + ("index", 1)) // next sentence prediction .construct(graph); } +#ifdef COMPILE_EXAMPLES + else if(type == "mnist-ffnn") + return New(options); +#endif #ifdef CUDNN - if(type == "char-s2s") { +#ifdef COMPILE_EXAMPLES + else if(type == "mnist-lenet") + return New(options); +#endif + else if(type == "char-s2s") { return models::encoder_decoder()(options) ("usage", use) ("original-type", type) - .push_back(models::encoder()("type", "char-s2s")) - .push_back(models::decoder()("type", "s2s")) - .construct(graph); + .push_back(models::encoder()("type", "char-s2s")) + .push_back(models::decoder()("type", "s2s")) + .construct(graph); } #endif // clang-format on - ABORT("Unknown model type: {}", type); + else + ABORT("Unknown model type: {}", type); } Ptr createModelFromOptions(Ptr options, usage use) { From 43d39d35e344d23c018ce570c7ce5712d83186e4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 13:46:29 -0700 Subject: [PATCH 367/838] some renaming --- src/examples/mnist/model.h | 10 +++--- src/examples/mnist/validator.h | 2 +- src/models/costs.h | 48 +++++++++++++------------- src/models/encoder_classifier.h | 2 +- src/models/encoder_decoder.cpp | 2 +- src/models/encoder_decoder.h | 2 +- src/models/model_base.h | 5 +-- src/models/model_factory.cpp | 10 +++--- src/models/model_factory.h | 10 +++--- src/rescorer/rescorer.h | 2 +- src/training/graph_group.h | 4 +-- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_async.h | 2 +- src/training/graph_group_multinode.cpp | 2 +- src/training/graph_group_singleton.h | 2 +- src/training/graph_group_sync.h | 7 ++-- src/training/validator.h | 12 +++---- src/translator/scorers.h | 4 +-- 18 files changed, 64 insertions(+), 64 deletions(-) mode change 100644 => 100755 src/translator/scorers.h diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h index 2a0e0e6a8..ffc04876f 100755 --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -17,11 +17,11 @@ namespace models { // @TODO: looking at this file, simplify the new RationalLoss idea. Here it gets too complicated -class MNISTCrossEntropyCost : public CostBase { +class MNISTCrossEntropyCost : public ICost { public: MNISTCrossEntropyCost() {} - Ptr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -43,11 +43,11 @@ class MNISTCrossEntropyCost : public CostBase { } }; -class MNISTLogsoftmax : public LogProbBase { +class MNISTLogsoftmax : public ILogProb { public: MNISTLogsoftmax() {} - Expr apply(Ptr model, + Expr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -56,7 +56,7 @@ class MNISTLogsoftmax : public LogProbBase { } }; -class MnistFeedForwardNet : public ModelBase { +class MnistFeedForwardNet : public IModel { public: typedef data::MNISTData dataset_type; diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h index b95a83dca..fa9eb3342 100755 --- a/src/examples/mnist/validator.h +++ b/src/examples/mnist/validator.h @@ -12,7 +12,7 @@ using namespace marian; namespace marian { -class MNISTAccuracyValidator : public Validator { +class MNISTAccuracyValidator : public Validator { public: MNISTAccuracyValidator(Ptr options) : Validator(std::vector>(), options, false) { createBatchGenerator(/*isTranslating=*/false); diff --git a/src/models/costs.h b/src/models/costs.h index 28191aa2c..70a23952f 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -19,15 +19,15 @@ namespace models { // Other functions return RationalLoss directly without Ptr<...>, but also // they do not need polymorphism here. -class CostBase { +class ICost { public: - virtual Ptr apply(Ptr model, + virtual Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) = 0; }; -class EncoderDecoderCE : public CostBase { +class EncoderDecoderCE : public ICost { protected: Ptr options_; @@ -51,7 +51,7 @@ class EncoderDecoderCE : public CostBase { weighter_ = WeightingFactory(options_); } - Ptr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -89,7 +89,7 @@ class EncoderDecoderCE : public CostBase { }; // Wraps an EncoderClassifier so it can produce a cost from raw logits. @TODO: Needs refactoring -class EncoderClassifierCE : public CostBase { +class EncoderClassifierCE : public ICost { protected: Ptr options_; bool inference_{false}; @@ -104,7 +104,7 @@ class EncoderClassifierCE : public CostBase { loss_ = newLoss(options_, inference_); } - Ptr apply(Ptr model, + Ptr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) override { @@ -127,16 +127,16 @@ class EncoderClassifierCE : public CostBase { } }; -class Trainer : public CriterionBase { +class Trainer : public ICriterionFunction { protected: - Ptr model_; - Ptr cost_; + Ptr model_; + Ptr cost_; public: - Trainer(Ptr model, Ptr cost) + Trainer(Ptr model, Ptr cost) : model_(model), cost_(cost) {} - Ptr getModel() { return model_; } + Ptr getModel() { return model_; } virtual void load(Ptr graph, const std::string& name, @@ -159,9 +159,9 @@ class Trainer : public CriterionBase { virtual void clear(Ptr graph) override { model_->clear(graph); }; }; -class LogProbBase { +class ILogProb { public: - virtual Expr apply(Ptr model, + virtual Expr apply(Ptr model, Ptr graph, Ptr batch, bool clearGraph = true) = 0; @@ -170,16 +170,16 @@ class LogProbBase { // @TODO: Name 'scorer' is ambiguous: Does it compute scores for all classes, or the loss value for the ground truth? // Beam search uses it for the former meaning, while 'marian score' and validation in the latter. // This class is for the former use. The latter is done using Trainer. -class Scorer : public ModelBase { +class Scorer : public IModel { protected: - Ptr model_; - Ptr logProb_; + Ptr model_; + Ptr logProb_; public: - Scorer(Ptr model, Ptr cost) + Scorer(Ptr model, Ptr cost) : model_(model), logProb_(cost) {} - Ptr getModel() { return model_; } + Ptr getModel() { return model_; } virtual void load(Ptr graph, const std::string& name, @@ -202,12 +202,12 @@ class Scorer : public ModelBase { virtual void clear(Ptr graph) override { model_->clear(graph); }; }; -class CostStep { +class LogProbStep { public: virtual Ptr apply(Ptr state) = 0; }; -class LogSoftmaxStep : public CostStep { +class LogSoftmaxStep : public LogProbStep { public: virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) @@ -223,7 +223,7 @@ class LogSoftmaxStep : public CostStep { // Gumbel-max noising for sampling during beam-search // Seems to work well enough with beam-size=1. Turn on // with --output-sampling during translation with marian-decoder -class GumbelSoftmaxStep : public CostStep { +class GumbelSoftmaxStep : public LogProbStep { public: virtual Ptr apply(Ptr state) override { auto logits = state->getLogProbs(); @@ -235,16 +235,16 @@ class GumbelSoftmaxStep : public CostStep { } }; -// class to wrap an EncoderDecoderBase and a CostStep that are executed in sequence, +// class to wrap an EncoderDecoderBase and a LogProbStep that are executed in sequence, // wrapped again in the EncoderDecoderBase interface // @TODO: seems we are conflating an interface defition with its implementation? class Stepwise : public EncoderDecoderBase { protected: Ptr encdec_; - Ptr cost_; + Ptr cost_; public: - Stepwise(Ptr encdec, Ptr cost) + Stepwise(Ptr encdec, Ptr cost) : encdec_(encdec), cost_(cost) {} virtual void load(Ptr graph, diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h index 15cbb7d7d..2477c330c 100755 --- a/src/models/encoder_classifier.h +++ b/src/models/encoder_classifier.h @@ -17,7 +17,7 @@ namespace marian { * @TODO: this should probably be unified somehow with EncoderDecoder which could allow for deocder/classifier * multi-objective training. */ -class EncoderClassifierBase : public models::ModelBase { +class EncoderClassifierBase : public models::IModel { public: virtual ~EncoderClassifierBase() {} diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp index 398eb6d70..6eed8ab62 100755 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -202,7 +202,7 @@ Expr EncoderDecoder::build(Ptr graph, auto state = stepAll(graph, batch, clearGraph); // returns raw logits - return state->getLogProbs(); // , state->getTargetMask()); // @TODO: hacky hack hack + return state->getLogProbs(); } Expr EncoderDecoder::build(Ptr graph, diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index b0f5b3145..e3b6bc40b 100755 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -9,7 +9,7 @@ namespace marian { -class EncoderDecoderBase : public models::ModelBase { +class EncoderDecoderBase : public models::IModel { public: virtual void load(Ptr graph, const std::string& name, diff --git a/src/models/model_base.h b/src/models/model_base.h index 0113a7ec0..c11869cc6 100755 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -17,7 +17,7 @@ namespace marian { namespace models { // model = input -> predictions -class ModelBase { +class IModel { public: virtual void load(Ptr, const std::string&, @@ -37,7 +37,8 @@ class ModelBase { }; // criterion = (input, reference) -> loss -class CriterionBase { +// @TODO: Is there a better name? +class ICriterionFunction { public: virtual void load(Ptr, const std::string&, diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index cee855ab6..9d660510b 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -60,7 +60,7 @@ Ptr ClassifierFactory::construct(Ptr /*graph*/) ABORT("Unknown classifier type"); } -Ptr EncoderDecoderFactory::construct(Ptr graph) { +Ptr EncoderDecoderFactory::construct(Ptr graph) { Ptr encdec; if(options_->get("type") == "amun") @@ -80,7 +80,7 @@ Ptr EncoderDecoderFactory::construct(Ptr graph) { return encdec; } -Ptr EncoderClassifierFactory::construct(Ptr graph) { +Ptr EncoderClassifierFactory::construct(Ptr graph) { Ptr enccls; if(options_->get("type") == "bert") { enccls = New(options_); @@ -99,7 +99,7 @@ Ptr EncoderClassifierFactory::construct(Ptr graph) { return enccls; } -Ptr createBaseModelByType(std::string type, usage use, Ptr options) { +Ptr createBaseModelByType(std::string type, usage use, Ptr options) { Ptr graph = nullptr; // graph unknown at this stage // clang-format off if(type == "s2s" || type == "amun" || type == "nematus") { @@ -283,7 +283,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr o ABORT("Unknown model type: {}", type); } -Ptr createModelFromOptions(Ptr options, usage use) { +Ptr createModelFromOptions(Ptr options, usage use) { std::string type = options->get("type"); auto baseModel = createBaseModelByType(type, use, options); @@ -313,7 +313,7 @@ Ptr createModelFromOptions(Ptr options, usage use) { ABORT("'Usage' parameter must be 'translation' or 'raw'"); } -Ptr createCriterionFromOptions(Ptr options, usage use) { +Ptr createCriterionFromOptions(Ptr options, usage use) { std::string type = options->get("type"); auto baseModel = createBaseModelByType(type, use, options); diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 9d089e294..9fcbe091a 100755 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -56,7 +56,7 @@ class EncoderDecoderFactory : public Factory { return Accumulator(*this); } - virtual Ptr construct(Ptr graph); + virtual Ptr construct(Ptr graph); }; typedef Accumulator encoder_decoder; @@ -80,15 +80,15 @@ class EncoderClassifierFactory : public Factory { return Accumulator(*this); } - virtual Ptr construct(Ptr graph); + virtual Ptr construct(Ptr graph); }; typedef Accumulator encoder_classifier; -Ptr createBaseModelByType(std::string type, usage, Ptr options); +Ptr createBaseModelByType(std::string type, usage, Ptr options); -Ptr createModelFromOptions(Ptr options, usage); +Ptr createModelFromOptions(Ptr options, usage); -Ptr createCriterionFromOptions(Ptr options, usage); +Ptr createCriterionFromOptions(Ptr options, usage); } // namespace models } // namespace marian diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 011b6b58c..14d2eae14 100755 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -19,7 +19,7 @@ using namespace data; class Rescorer { private: - Ptr builder_; + Ptr builder_; public: Rescorer(Ptr options) diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 8e67ad724..4ec6cdd11 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -55,7 +55,7 @@ class GraphGroup { */ // @TODO: Can this be made const? It seems wrong to have a stateful method that still returns a result. virtual Ptr collectStats(Ptr graph, - Ptr model, + Ptr model, const std::vector>& vocabs, double multiplier = 1.) { auto stats = New(); @@ -141,7 +141,7 @@ class MultiNodeGraphGroupBase : public GraphGroup { std::vector devices_; // [num local GPUs] /** Graph builders for clients (which run forward and backward passes). */ - std::vector> clientBuilders_; + std::vector> clientBuilders_; /** Graphs of clients. One entry per GPU on this node. */ std::vector> clientGraphs_; // [num local GPUs] diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 780108f17..5679d81b2 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -189,7 +189,7 @@ void AsyncGraphGroup::execute(Ptr batch) { auto task = [this](Ptr batch) { static size_t i = 0; thread_local Ptr graph; - thread_local Ptr builder; + thread_local Ptr builder; thread_local size_t t = 0; thread_local size_t num_seen_words = 0; thread_local size_t num_seen_sentences = 0; diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h index bb19bde40..0e27a6657 100755 --- a/src/training/graph_group_async.h +++ b/src/training/graph_group_async.h @@ -16,7 +16,7 @@ class AsyncGraphGroup : public GraphGroup, public ExponentialSmoothing { protected: bool first_{true}; - std::vector> builders_; + std::vector> builders_; std::vector> graphs_; std::vector devices_; diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp index 4e7d931cf..28c256414 100755 --- a/src/training/graph_group_multinode.cpp +++ b/src/training/graph_group_multinode.cpp @@ -512,7 +512,7 @@ void MultiNodeGraphGroup::execute(Ptr batch) { auto task = [this](Ptr batch) { static size_t i = 0; thread_local Ptr graph; - thread_local Ptr builder; + thread_local Ptr builder; thread_local size_t my_id = 0; thread_local size_t t = 0; // only for scheduler statistic diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 8aede537a..4587695c6 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -16,7 +16,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { virtual void setScheduler(Ptr scheduler) override; private: - Ptr builder_; + Ptr builder_; Ptr graph_; Ptr graphAvg_; diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h index a34bb114d..b127296da 100755 --- a/src/training/graph_group_sync.h +++ b/src/training/graph_group_sync.h @@ -13,12 +13,11 @@ class SyncGraphGroup : public GraphGroup, public ExponentialSmoothing { Ptr comm_; // [not null] communicator, e.g. NCCLCommunicator Ptr mpi_; // [not null] all MPI-like communication goes through this (this is a dummy implementation if no MPI run) - std::vector devices_; // [deviceIndex] - std::vector> builders_; // [deviceIndex] - std::vector> graphs_; // [deviceIndex] + std::vector devices_; // [deviceIndex] + std::vector> builders_; // [deviceIndex] + std::vector> graphs_; // [deviceIndex] std::vector> shardOpt_; // [deviceIndex] - std::vector paramsAvg_; // [deviceIndex] exponentially smoothed parameters, sharded // @TODO: instead, create an array of ExponentialSmoothing objects, and don't use ExponentialSmoothing as a base class std::vector> paramsAllocs_; // [deviceIndex] we must hold a reference to the memory until this class dies diff --git a/src/training/validator.h b/src/training/validator.h index d49ded8f6..7c9b5e9a6 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -148,7 +148,7 @@ class Validator : public ValidatorBase { } }; -class CrossEntropyValidator : public Validator { +class CrossEntropyValidator : public Validator { public: CrossEntropyValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options) { @@ -216,7 +216,7 @@ class CrossEntropyValidator : public Validator { +class AccuracyValidator : public Validator { public: AccuracyValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, /*lowerIsBetter=*/false) { @@ -305,7 +305,7 @@ class AccuracyValidator : public Validator { } }; -class BertAccuracyValidator : public Validator { +class BertAccuracyValidator : public Validator { private: bool evalMaskedLM_{true}; @@ -415,7 +415,7 @@ class BertAccuracyValidator : public Validator }; -class ScriptValidator : public Validator { +class ScriptValidator : public Validator { public: ScriptValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false) { @@ -446,7 +446,7 @@ class ScriptValidator : public Validator { } }; -class TranslationValidator : public Validator { +class TranslationValidator : public Validator { public: TranslationValidator(std::vector> vocabs, Ptr options) : Validator(vocabs, options, false), @@ -578,7 +578,7 @@ class TranslationValidator : public Validator { }; // @TODO: combine with TranslationValidator (above) to avoid code duplication -class BleuValidator : public Validator { +class BleuValidator : public Validator { private: bool detok_{false}; diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100644 new mode 100755 index 16a066b05..82e484d0e --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -72,7 +72,7 @@ class ScorerWrapper : public Scorer { const void* ptr_; public: - ScorerWrapper(Ptr encdec, + ScorerWrapper(Ptr encdec, const std::string& name, float weight, const std::string& fname) @@ -81,7 +81,7 @@ class ScorerWrapper : public Scorer { fname_(fname), ptr_{0} {} - ScorerWrapper(Ptr encdec, + ScorerWrapper(Ptr encdec, const std::string& name, float weight, const void* ptr) From 980f2698c590bfb3974343a3ad490572ba307efd Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 14:02:36 -0700 Subject: [PATCH 368/838] some renaming --- src/models/costs.h | 22 +++++++++++----------- src/models/model_factory.cpp | 6 +++--- src/models/model_factory.h | 2 +- src/rescorer/rescorer.h | 2 +- src/training/graph_group.h | 2 +- src/training/graph_group_async.cpp | 2 +- src/training/graph_group_singleton.h | 2 +- src/training/graph_group_sync.cpp | 2 +- src/training/validator.h | 4 ++-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/models/costs.h b/src/models/costs.h index 70a23952f..35ee24444 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -22,12 +22,12 @@ namespace models { class ICost { public: virtual Ptr apply(Ptr model, - Ptr graph, + Ptr graph, // @TODO: why needed? Can it be gotten from model? Ptr batch, bool clearGraph = true) = 0; }; -class EncoderDecoderCE : public ICost { +class EncoderDecoderCECost : public ICost { protected: Ptr options_; @@ -39,7 +39,7 @@ class EncoderDecoderCE : public ICost { Ptr weighter_; public: - EncoderDecoderCE(Ptr options) + EncoderDecoderCECost(Ptr options) : options_(options), inference_(options->get("inference", false)) { loss_ = newLoss(options_, inference_); @@ -89,7 +89,7 @@ class EncoderDecoderCE : public ICost { }; // Wraps an EncoderClassifier so it can produce a cost from raw logits. @TODO: Needs refactoring -class EncoderClassifierCE : public ICost { +class EncoderClassifierCECost : public ICost { protected: Ptr options_; bool inference_{false}; @@ -99,7 +99,7 @@ class EncoderClassifierCE : public ICost { Ptr loss_; public: - EncoderClassifierCE(Ptr options) + EncoderClassifierCECost(Ptr options) : options_(options), inference_(options->get("inference", false)) { loss_ = newLoss(options_, inference_); } @@ -202,12 +202,12 @@ class Scorer : public IModel { virtual void clear(Ptr graph) override { model_->clear(graph); }; }; -class LogProbStep { +class ILogProbStep { public: virtual Ptr apply(Ptr state) = 0; }; -class LogSoftmaxStep : public LogProbStep { +class LogSoftmaxStep : public ILogProbStep { public: virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) @@ -223,7 +223,7 @@ class LogSoftmaxStep : public LogProbStep { // Gumbel-max noising for sampling during beam-search // Seems to work well enough with beam-size=1. Turn on // with --output-sampling during translation with marian-decoder -class GumbelSoftmaxStep : public LogProbStep { +class GumbelSoftmaxStep : public ILogProbStep { public: virtual Ptr apply(Ptr state) override { auto logits = state->getLogProbs(); @@ -235,16 +235,16 @@ class GumbelSoftmaxStep : public LogProbStep { } }; -// class to wrap an EncoderDecoderBase and a LogProbStep that are executed in sequence, +// class to wrap an EncoderDecoderBase and a ILogProbStep that are executed in sequence, // wrapped again in the EncoderDecoderBase interface // @TODO: seems we are conflating an interface defition with its implementation? class Stepwise : public EncoderDecoderBase { protected: Ptr encdec_; - Ptr cost_; + Ptr cost_; public: - Stepwise(Ptr encdec, Ptr cost) + Stepwise(Ptr encdec, Ptr cost) : encdec_(encdec), cost_(cost) {} virtual void load(Ptr graph, diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 9d660510b..b0cd02e0b 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -313,7 +313,7 @@ Ptr createModelFromOptions(Ptr options, usage use) { ABORT("'Usage' parameter must be 'translation' or 'raw'"); } -Ptr createCriterionFromOptions(Ptr options, usage use) { +Ptr createCriterionFunctionFromOptions(Ptr options, usage use) { std::string type = options->get("type"); auto baseModel = createBaseModelByType(type, use, options); @@ -322,9 +322,9 @@ Ptr createCriterionFromOptions(Ptr options, usage u // note: usage::scoring means "score the loss function", hence it uses a Trainer (not Scorer, which is for decoding) // @TODO: Should we define a new class that does not compute gradients? if (std::dynamic_pointer_cast(baseModel)) - return New(baseModel, New(options)); + return New(baseModel, New(options)); else if (std::dynamic_pointer_cast(baseModel)) - return New(baseModel, New(options)); + return New(baseModel, New(options)); #ifdef COMPILE_EXAMPLES // @TODO: examples should be compiled optionally else if (std::dynamic_pointer_cast(baseModel)) diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 9fcbe091a..8f3f07abe 100755 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -89,6 +89,6 @@ Ptr createBaseModelByType(std::string type, usage, Ptr options) Ptr createModelFromOptions(Ptr options, usage); -Ptr createCriterionFromOptions(Ptr options, usage); +Ptr createCriterionFunctionFromOptions(Ptr options, usage); } // namespace models } // namespace marian diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 14d2eae14..3917a901c 100755 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -23,7 +23,7 @@ class Rescorer { public: Rescorer(Ptr options) - : builder_(models::createCriterionFromOptions(options, models::usage::scoring)) {} + : builder_(models::createCriterionFunctionFromOptions(options, models::usage::scoring)) {} void load(Ptr graph, const std::string& modelFile) { builder_->load(graph, modelFile); diff --git a/src/training/graph_group.h b/src/training/graph_group.h index 4ec6cdd11..33070f469 100755 --- a/src/training/graph_group.h +++ b/src/training/graph_group.h @@ -161,7 +161,7 @@ class MultiNodeGraphGroupBase : public GraphGroup { clientGraphs_.push_back(New()); clientGraphs_[i]->setDevice({ devices_[i], DeviceType::gpu }); clientGraphs_[i]->reserveWorkspaceMB(options_->get("workspace")); - clientBuilders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); + clientBuilders_.push_back(models::createCriterionFunctionFromOptions(options_, models::usage::training)); } } diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp index 5679d81b2..6ecaa1d9c 100755 --- a/src/training/graph_group_async.cpp +++ b/src/training/graph_group_async.cpp @@ -23,7 +23,7 @@ AsyncGraphGroup::AsyncGraphGroup(Ptr config, Ptr mpi) graphs_.push_back(graph); shardOpt_.push_back(Optimizer(options_)); - builders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); + builders_.push_back(models::createCriterionFunctionFromOptions(options_, models::usage::training)); } } diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h index 4587695c6..e2b1818cc 100755 --- a/src/training/graph_group_singleton.h +++ b/src/training/graph_group_singleton.h @@ -37,7 +37,7 @@ class SingletonGraph : public GraphGroup, public ExponentialSmoothing { graph_->getBackend()->setClip(options_->get("clip-gemm")); graph_->reserveWorkspaceMB(options_->get("workspace")); opt_ = Optimizer(options_); - builder_ = models::createCriterionFromOptions(options_, models::usage::training); + builder_ = models::createCriterionFunctionFromOptions(options_, models::usage::training); } void update(Ptr batch) override { diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index b97ac65a1..ba5751691 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -15,7 +15,7 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) graphs_.push_back(graph); shardOpt_.push_back(Optimizer(options_)); - builders_.push_back(models::createCriterionFromOptions(options_, models::usage::training)); + builders_.push_back(models::createCriterionFunctionFromOptions(options_, models::usage::training)); } // Note: We may well end up with only one MPI process or only one graph per worker. diff --git a/src/training/validator.h b/src/training/validator.h index 7c9b5e9a6..1323b6758 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -159,7 +159,7 @@ class CrossEntropyValidator : public Validatormerge(options); opts->set("inference", true); opts->set("cost-type", "ce-sum"); - builder_ = models::createCriterionFromOptions(opts, models::usage::scoring); + builder_ = models::createCriterionFunctionFromOptions(opts, models::usage::scoring); } std::string type() override { return options_->get("cost-type"); } @@ -180,7 +180,7 @@ class CrossEntropyValidator : public Validator graph; - thread_local auto builder = models::createCriterionFromOptions(options_, models::usage::scoring); + thread_local auto builder = models::createCriterionFunctionFromOptions(options_, models::usage::scoring); if(!graph) { graph = graphs[id % graphs.size()]; From 0d1c824342726b34935204c8d1cecbd7b37564f6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 11 Mar 2019 14:23:39 -0700 Subject: [PATCH 369/838] minor comment edits and a few renamings --- src/translator/beam_search.h | 28 +++++++++++++--------------- src/translator/hypothesis.h | 12 ++++++------ src/translator/nth_element.cpp | 12 +++++------- src/translator/nth_element.cu | 20 ++++++++++---------- 4 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 2954f06d7..184fded0e 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -18,7 +18,7 @@ class BeamSearch { Word trgEosId_ = (Word)-1; Word trgUnkId_ = (Word)-1; - static constexpr auto INVALID_PATH_SCORE = -9999; + static constexpr auto INVALID_PATH_SCORE = -9999; // (@TODO: change to -9999.0 once C++ allows that) public: BeamSearch(Ptr options, @@ -36,7 +36,7 @@ class BeamSearch { // combine new expandedPathScores and previous beams into new set of beams Beams toHyps(const std::vector& nBestKeys, // [dimBatch, beamSize] flattened -> ((batchIdx, beamHypIdx) flattened, word idx) flattened const std::vector& nBestPathScores, // [dimBatch, beamSize] flattened - const size_t inputBeamSize, // for interpretation of nBestKeys + const size_t nBestBeamSize, // for interpretation of nBestKeys const size_t vocabSize, // ditto. const Beams& beams, const std::vector>& states, @@ -46,23 +46,24 @@ class BeamSearch { align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble const auto dimBatch = beams.size(); - Beams newBeams(dimBatch); + Beams newBeams(dimBatch); // return value of this function goes here for(size_t i = 0; i < nBestKeys.size(); ++i) { // Keys encode batchIdx, beamHypIdx, and word index in the entire beam. - // They can be between 0 and beamSize * vocabSize-1. + // They can be between 0 and (vocabSize * nBestBeamSize * batchSize)-1. + // (beamHypIdx refers to the GPU tensors, *not* the beams[] array; they are not the same in case of purging) const auto key = nBestKeys[i]; const float pathScore = nBestPathScores[i]; // expanded path score for (batchIdx, beamHypIdx, word) // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) const auto wordIdx = (Word)(key % vocabSize); - const auto beamHypIdx = (key / vocabSize) % inputBeamSize; - const auto batchIdx = (key / vocabSize) / inputBeamSize; + const auto beamHypIdx = (key / vocabSize) % nBestBeamSize; + const auto batchIdx = (key / vocabSize) / nBestBeamSize; const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; - if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? + if (newBeam.size() >= beam.size()) // getNBestList() generates N for all batch entries incl. those that already have a narrower beam continue; if (pathScore <= INVALID_PATH_SCORE) // (unused slot) continue; @@ -183,14 +184,11 @@ class BeamSearch { } Beams beams(dimBatch, Beam(beamSize_, New())); // array [dimBatch] of array [localBeamSize] of Hypothesis - //Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis - //for(auto& beam : beams) - // beam.resize(beamSize_, New()); for(int i = 0; i < dimBatch; ++i) histories[i]->add(beams[i], trgEosId_); - // the decoder updates the following state information in each output time step: + // the decoding process updates the following state information in each output time step: // - beams: array [dimBatch] of array [localBeamSize] of Hypothesis // - current output time step's set of active hypotheses, aka active search space // - states[.]: ScorerState @@ -230,7 +228,7 @@ class BeamSearch { auto& beam = beams[batchIdx]; if(beamHypIdx < beam.size()) { auto hyp = beam[beamHypIdx]; - hypIndices.push_back((IndexType)(hyp->getPrevStateIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation + hypIndices.push_back((IndexType)(hyp->getPrevBeamHypIndex() * dimBatch + batchIdx)); // (beamHypIdx, batchIdx), flattened, for index_select() operation prevWords .push_back(hyp->getWord()); prevScores.push_back(hyp->getPathScore()); } else { // pad to localBeamSize (dummy hypothesis) @@ -291,11 +289,11 @@ class BeamSearch { // combine N-best sets with existing search space (beams) to updated search space beams = toHyps(nBestKeys, nBestPathScores, - /*inputBeamSize*/expandedPathScores->shape()[-2], // used for interpretation of keys + /*nBestBeamSize*/expandedPathScores->shape()[-2], // used for interpretation of keys /*vocabSize=*/expandedPathScores->shape()[-1], // used for interpretation of keys beams, - states, // only used for keeping track of per-ensemble-member path score - batch); // only used for propagating alignment info + states, // only used with nbest, for keeping track of per-ensemble-member path score + batch); // only used when propagating alignment info // remove all hyps that end in EOS // The position of a hyp in the beam may change. diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index 452b3349b..aa94cbe65 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -6,30 +6,30 @@ namespace marian { -// one single (possibly partial) hypothesis in beam search +// one single (partial or full) hypothesis in beam search // key elements: // - the word that this hyp ends with // - the aggregate score up to and including the word // - back pointer to previous hypothesis for traceback class Hypothesis { public: - Hypothesis() : prevHyp_(nullptr), prevIndex_(0), word_(0), pathScore_(0.0) {} + Hypothesis() : prevHyp_(nullptr), prevBeamHypIdx_(0), word_(0), pathScore_(0.0) {} Hypothesis(const Ptr prevHyp, Word word, IndexType prevIndex, // (beamHypIdx, batchIdx) flattened as beamHypIdx * dimBatch + batchIdx float pathScore) - : prevHyp_(prevHyp), prevIndex_(prevIndex), word_(word), pathScore_(pathScore) {} + : prevHyp_(prevHyp), prevBeamHypIdx_(prevIndex), word_(word), pathScore_(pathScore) {} const Ptr getPrevHyp() const { return prevHyp_; } Word getWord() const { return word_; } - IndexType getPrevStateIndex() const { return prevIndex_; } + IndexType getPrevBeamHypIndex() const { return prevBeamHypIdx_; } float getPathScore() const { return pathScore_; } - std::vector& getScoreBreakdown() { return scoreBreakdown_; } + std::vector& getScoreBreakdown() { return scoreBreakdown_; } // @TODO: make this const void setScoreBreakdown(const std::vector& scoreBreaddown) { scoreBreakdown_ = scoreBreaddown; } const std::vector& getAlignment() { return alignment_; } @@ -61,7 +61,7 @@ class Hypothesis { private: const Ptr prevHyp_; - const IndexType prevIndex_; + const IndexType prevBeamHypIdx_; const Word word_; const float pathScore_; diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp index 7d18555d8..8dc1a44c8 100755 --- a/src/translator/nth_element.cpp +++ b/src/translator/nth_element.cpp @@ -21,14 +21,12 @@ class NthElementCPU { NthElementCPU(const NthElementCPU& copy) = delete; private: - void selectNBest(float* scores, + void selectNBest(const float* scores, const std::vector& batchFirstElementIdxs, const std::vector& cumulativeBeamSizes) { - /* For each batch, select the max N elements, where N is the beam size for - * this batch. Locally record these elements (their current value and index - * in 'scores') before updating each element to a large negative value, such - * that they won't be a maximum if we're called again on the same input. - */ + // For each batch, select the max N elements, where N is the beam size for + // this batch. Locally record these elements (their current value and index + // in 'scores'). int numProbs = batchFirstElementIdxs.back(); std::vector idxs(numProbs); @@ -49,7 +47,7 @@ class NthElementCPU { int idx = *begin++; h_res_idx[pos] = idx; h_res[pos] = scores[idx]; - scores[idx] = std::numeric_limits::lowest(); + //scores[idx] = std::numeric_limits::lowest(); ++pos; } } diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu index b2f23d8c5..82aa6dd12 100755 --- a/src/translator/nth_element.cu +++ b/src/translator/nth_element.cu @@ -367,7 +367,7 @@ public: ABORT_IF(inputN != (isFirst ? 1 : N), "Input tensor has wrong beam dim??"); // @TODO: Remove isFirst argument altogether ABORT_IF(vocabSize > MAX_VOCAB_SIZE, "GetNBestList(): actual vocab size exceeds MAX_VOCAB_SIZE"); ABORT_IF(dimBatch > maxBatchSize_, "GetNBestList(): actual batch size exceeds initialization parameter"); - ABORT_IF(N > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); // @TODO: or inputN? + ABORT_IF(std::max(N, (size_t)inputN) > maxBeamSize_, "GetNBestList(): actual beam size exceeds initialization parameter"); const std::vector beamSizes(dimBatch, N); std::vector cumulativeBeamSizes(beamSizes.size() + 1, 0); @@ -440,19 +440,19 @@ private: const int BLOCK_SIZE = 512; const int NUM_BLOCKS; - int* d_ind; - float* d_out; + int* d_ind; // [maxBatchSize * NUM_BLOCKS] + float* d_out; // [maxBatchSize * NUM_BLOCKS] - int* d_res_idx; - float* d_res; + int* d_res_idx; // [maxBatchSize * maxBeamSize] + float* d_res; // [maxBatchSize * maxBeamSize] - int* h_res_idx; - float* h_res; + int* h_res_idx; // [maxBeamSize * maxBatchSize] + float* h_res; // [maxBeamSize * maxBatchSize] - float* d_breakdown; - int* d_batchPosition; - int* d_cumBeamSizes; + float* d_breakdown; // [maxBeamSize] + int* d_batchPosition; // [maxBatchSize + 1] + int* d_cumBeamSizes; // [maxBatchSize + 1] //size_t lastN; }; From ec1c885f8e458a9aaf61cc544fb7e4f1eb993c53 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 13 Mar 2019 09:26:25 -0700 Subject: [PATCH 370/838] fixed merge error --- src/translator/beam_search.h | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 708541d70..9a9ea50b0 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -48,23 +48,23 @@ class BeamSearch { const auto dimBatch = beams.size(); Beams newBeams(dimBatch); - for(size_t i = 0; i < nBestKeys.size(); ++i) { + for(size_t i = 0; i < nBestKeys.size(); ++i) { // [dimBatch, beamSize] flattened // Keys encode batchIdx, beamHypIdx, and word index in the entire beam. // They can be between 0 and beamSize * vocabSize-1. - const auto key = nBestKeys[i]; - const float pathScore = nBestPathScores[i]; // expanded path score for (batchIdx, beamHypIdx, word) + const float pathScore = nBestPathScores[i]; + const auto key = nBestKeys[i]; // key = pathScore's tensor location, as (batchIdx, beamHypIdx, word idx) flattened // decompose key into individual indices (batchIdx, beamHypIdx, wordIdx) - const auto wordIdx = (Word)(key % vocabSize); - const auto beamHypIdx = (key / vocabSize) % inputBeamSize; - const auto batchIdx = (key / vocabSize) / inputBeamSize; + const auto wordIdx = (WordIndex)(key % vocabSize); + const auto beamHypIdx = (key / vocabSize) % inputBeamSize; + const auto batchIdx = (key / vocabSize) / inputBeamSize; const auto& beam = beams[batchIdx]; auto& newBeam = newBeams[batchIdx]; - if (newBeam.size() >= beam.size()) // @TODO: Why this condition? It does happen. Why? + if (newBeam.size() >= beam.size()) // getNBestList() generates N for all batch entries incl. those that already have a narrower beam continue; - if (pathScore <= INVALID_PATH_SCORE) // (unused slot) + if (pathScore <= INVALID_PATH_SCORE) // (dummy slot or word that cannot be expanded by current factor) continue; ABORT_IF(beamHypIdx >= beam.size(), "Out of bounds beamHypIdx??"); @@ -75,9 +75,9 @@ class BeamSearch { // rather than the true word index. auto shortlist = scorers_[0]->getShortlist(); if (shortlist) - word = shortlist->reverseMap(wordIdx); + word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); else - word = wordIdx; + word = Word::fromWordIndex(wordIdx); auto hyp = New(beam[beamHypIdx], word, (IndexType)beamHypIdx, pathScore); @@ -183,9 +183,6 @@ class BeamSearch { } Beams beams(dimBatch, Beam(beamSize_, New())); // array [dimBatch] of array [localBeamSize] of Hypothesis - //Beams beams(dimBatch); // array [dimBatch] of array [localBeamSize] of Hypothesis - //for(auto& beam : beams) - // beam.resize(beamSize_, New()); for(int i = 0; i < dimBatch; ++i) histories[i]->add(beams[i], trgEosId_); From 2069b0e909722b72e1b1cdf97ad3b24f7779bfd4 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 14 Mar 2019 18:00:17 -0700 Subject: [PATCH 371/838] fixed a merge error in MNIST example --- src/examples/mnist/dataset.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 src/examples/mnist/dataset.h diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h old mode 100644 new mode 100755 index 4a4f5ae45..0e73840bf --- a/src/examples/mnist/dataset.h +++ b/src/examples/mnist/dataset.h @@ -19,7 +19,12 @@ namespace data { typedef std::vector Data; typedef std::vector Labels; -typedef std::vector Example; +struct Example : public std::vector { // a std::vector with a getId() method + size_t id; + size_t getId() const { return id; } + Example(std::vector&& data, size_t id) : std::vector(std::move(data)), id(id) {} + Example() : id(SIZE_MAX) {} +}; typedef std::vector Examples; typedef Examples::const_iterator ExampleIterator; @@ -147,12 +152,11 @@ class MNISTData : public Dataset { ABORT_IF(features.size() != labels.size(), "Features do not match labels"); for(size_t i = 0; i < features.size(); ++i) { - Example ex = {features[i], labels[i]}; - examples_.emplace_back(ex); + examples_.emplace_back(std::vector{ features[i], labels[i] }, i); } } - Example next() override { return{ }; } //@TODO: this return was added to fix a warning. Is it correct? + Example next() override { return Example(); } //@TODO: this return was added to fix a warning. Is it correct? private: typedef unsigned char uchar; From 93316a211927dfc6ca611c2681a043f3cedfa207 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Mar 2019 10:02:35 -0700 Subject: [PATCH 372/838] MLP can now handle a IUnaryLogitsLayer as its last layer; therefore, s2s model type now supports factored Logits --- src/layers/constructors.h | 114 ++++++++++++++++++++++++++------------ src/layers/factory.h | 10 ++++ src/layers/generic.cpp | 2 +- src/layers/generic.h | 23 +++++--- src/models/bert.h | 4 +- src/models/s2s.h | 13 ++--- src/models/transformer.h | 6 +- 7 files changed, 112 insertions(+), 60 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index d1aef42e5..f19a0922b 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -10,21 +10,21 @@ namespace mlp { * Base class for layer factories, can be used in a multi-layer network factory. */ struct LayerFactory : public Factory { - LayerFactory() : Factory() {} - LayerFactory(const LayerFactory&) = default; - LayerFactory(LayerFactory&&) = default; - - virtual ~LayerFactory() {} - - template - inline Ptr as() { - return std::dynamic_pointer_cast(shared_from_this()); - } - - template - inline bool is() { - return as() != nullptr; - } + //LayerFactory() : Factory() {} + //LayerFactory(const LayerFactory&) = default; + //LayerFactory(LayerFactory&&) = default; + // + //virtual ~LayerFactory() {} + + //template + //inline Ptr as() { + // return std::dynamic_pointer_cast(shared_from_this()); + //} + // + //template + //inline bool is() { + // return as() != nullptr; + //} virtual Ptr construct(Ptr graph) = 0; }; @@ -52,21 +52,21 @@ typedef Accumulator dense; * Factory for output layers, can be used in a multi-layer network factory. */ struct LogitLayerFactory : public Factory { - LogitLayerFactory() : Factory() {} - LogitLayerFactory(const LogitLayerFactory&) = default; - LogitLayerFactory(LogitLayerFactory&&) = default; - - virtual ~LogitLayerFactory() {} - - template - inline Ptr as() { - return std::dynamic_pointer_cast(shared_from_this()); - } - - template - inline bool is() { - return as() != nullptr; - } + //LogitLayerFactory() : Factory() {} + //LogitLayerFactory(const LogitLayerFactory&) = default; + //LogitLayerFactory(LogitLayerFactory&&) = default; + // + //virtual ~LogitLayerFactory() {} + // + //template + //inline Ptr as() { + // return std::dynamic_pointer_cast(shared_from_this()); + //} + // + //template + //inline bool is() { + // return as() != nullptr; + //} virtual Ptr construct(Ptr graph) = 0; }; @@ -108,7 +108,7 @@ typedef Accumulator output; /** * Multi-layer network, holds and applies layers. */ -class MLP { +class MLP : public IUnaryLogitLayer { protected: Ptr graph_; Ptr options_; @@ -119,10 +119,7 @@ class MLP { MLP(Ptr graph, Ptr options) : graph_(graph), options_(options) {} - template - Expr apply(Args... args) { - std::vector av = {args...}; - + Expr apply(const std::vector& av) { Expr output; if(av.size() == 1) output = layers_[0]->apply(av[0]); @@ -135,7 +132,32 @@ class MLP { return output; } + Logits applyAsLogits(const std::vector& av) { + auto lastLayer = std::dynamic_pointer_cast(layers_.back()); + ABORT_IF(!lastLayer, "MLP::applyAsLogits() applied but last MLP layer is not IUnaryLogitLayer"); + if (layers_.size() == 1) { + if (av.size() == 1) + return lastLayer->applyAsLogits(av[0]); + else + return lastLayer->applyAsLogits(av); + } + else { + Expr output; + if (av.size() == 1) + output = layers_[0]->apply(av[0]); + else + output = layers_[0]->apply(av); + for (size_t i = 1; i < layers_.size() - 1; ++i) + output = layers_[i]->apply(output); + return lastLayer->applyAsLogits(output); + } + } + + Expr apply(Expr e) { return apply(std::vector{ e }); } + Logits applyAsLogits(Expr e) { return applyAsLogits(std::vector{ e }); } + void push_back(Ptr layer) { layers_.push_back(layer); } + void push_back(Ptr layer) { layers_.push_back(layer); } }; /** @@ -161,6 +183,28 @@ class MLPFactory : public Factory { layers_.push_back(New(lf)); return Accumulator(*this); } + + // special case for last layer, which may be a IUnaryLogitLayer. Requires some hackery +private: + template + class AsLayerFactory : public LayerFactory { + WrappedFactory us; + public: + AsLayerFactory(const WrappedFactory& wrapped) : us(wrapped) {} + Ptr construct(Ptr graph) override final { + auto p = std::static_pointer_cast(us.construct(graph)); + ABORT_IF(!p, "Attempted to cast a Factory to LayerFactory that isn't one"); + return p; + } + }; + template + static inline AsLayerFactory asLayerFactory(const WrappedFactory& wrapped) { return wrapped; } +public: + Accumulator push_back(const Accumulator& lf) { + push_back(AsLayerFactory(lf)); + //layers_.push_back(New>(asLayerFactory((OutputFactory&)lf))); + return Accumulator(*this); + } }; // @TODO: change naming convention. diff --git a/src/layers/factory.h b/src/layers/factory.h index e3b742442..0aaf86a92 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -29,6 +29,16 @@ class Factory : public std::enable_shared_from_this { T opt(const std::string& key, T defaultValue) { return options_->get(key, defaultValue); } + + template + inline Ptr as() { + return std::dynamic_pointer_cast(shared_from_this()); + } + + template + inline bool is() { + return as() != nullptr; + } }; // simplest form of Factory that just passes on options to the constructor of a layer type diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index f59105932..2f302ef12 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -229,7 +229,7 @@ namespace marian { b_ = graph_->param(name + "_b", {1, numOutputClasses}, inits::zeros); } - Logits Output::apply(Expr input) /*override*/ { + Logits Output::applyAsLogits(Expr input) /*override final*/ { lazyConstruct(input->shape()[-1]); if (shortlist_) { diff --git a/src/layers/generic.h b/src/layers/generic.h index e83801aa2..5b3e5ceab 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -44,7 +44,10 @@ class LayerBase { // Simplest layer interface: Unary function struct IUnaryLayer { virtual Expr apply(Expr) = 0; - virtual Expr apply(const std::vector&) = 0; + virtual Expr apply(const std::vector& es) { + ABORT_IF(es.size() > 1, "Not implemented"); // simple stub + return apply(es.front()); + } }; // Embedding from corpus sub-batch to (emb, mask) @@ -108,9 +111,15 @@ class Logits { }; // Unary function that returns a Logits object -struct IUnaryLogitLayer { - virtual Logits apply(Expr) = 0; - virtual Logits apply(const std::vector&) = 0; +// Also implements IUnaryLayer, since Logits can be cast to Expr. +struct IUnaryLogitLayer : public IUnaryLayer { + virtual Logits applyAsLogits(Expr) = 0; + virtual Logits applyAsLogits(const std::vector& es) { + ABORT_IF(es.size() > 1, "Not implemented"); // simple stub + return applyAsLogits(es.front()); + } + virtual Expr apply(Expr e) override { return applyAsLogits(e).getLogits(); } + virtual Expr apply(const std::vector& es) override { return applyAsLogits(es).getLogits(); } }; namespace mlp { @@ -227,11 +236,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { cachedShortb_ = nullptr; } - Logits apply(Expr input) override; - - virtual Logits apply(const std::vector& /*inputs*/) override { - ABORT("Not implemented"); - }; + Logits applyAsLogits(Expr input) override final; }; } // namespace mlp diff --git a/src/models/bert.h b/src/models/bert.h index b335b5eaf..fb2dab846 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -334,7 +334,7 @@ class BertMaskedLM : public ClassifierBase { auto maskedContext = rows(reshape(context, {dimBatch * dimTime, dimModel}), bertMaskedPositions); // subselect stuff that has actually been masked out -// int dimVoc = opt>("dim-vocabs")[batchIndex_]; + int dimVoc = opt>("dim-vocabs")[batchIndex_]; auto layer1 = mlp::mlp() .push_back(mlp::dense() @@ -359,12 +359,10 @@ class BertMaskedLM : public ClassifierBase { intermediate = layerNorm(intermediate, gamma, beta); auto layer2 = mlp::mlp() -#if 0 // @TODO: Not supported presently since Output has a different signature now .push_back(mlp::output() ("prefix", prefix_ + "_ff_logit_l2") ("dim", dimVoc) .tieTransposed("Wemb")) -#endif .construct(graph); auto logits = layer2->apply(intermediate); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab dim] diff --git a/src/models/s2s.h b/src/models/s2s.h index a63c98418..03baff564 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -347,28 +347,23 @@ class DecoderS2S : public DecoderBase { if(shortlist_) last.setShortlist(shortlist_); -#if 1 - hidden; last; - ABORT("@TODO: adapt s2s to Logits return type"); -#else // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context output_ = mlp::mlp() // .push_back(hidden) // .push_back(last) .construct(graph); -#endif } - Expr logits; + Logits logits; if(alignedContext) - logits = output_->apply(embeddings, decoderContext, alignedContext); + logits = output_->applyAsLogits({embeddings, decoderContext, alignedContext}); else - logits = output_->apply(embeddings, decoderContext); + logits = output_->applyAsLogits({embeddings, decoderContext}); // return unormalized(!) probabilities auto nextState = New( - decoderStates, Logits(logits), state->getEncoderStates(), state->getBatch()); + decoderStates, logits, state->getEncoderStates(), state->getBatch()); // Advance current target token position by one nextState->setPosition(state->getPosition() + 1); diff --git a/src/models/transformer.h b/src/models/transformer.h index 1817947ea..588a79b41 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -841,16 +841,16 @@ class DecoderTransformer : public Transformer { // final feed-forward layer (output) if(shortlist_) output_->setShortlist(shortlist_); - auto logits = output_->apply(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab or shortlist dim] + auto logits = output_->applyAsLogits(decoderContext); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vocab or shortlist dim] // return unormalized(!) probabilities Ptr nextState; if (opt("transformer-decoder-autoreg", "self-attention") == "rnn") { nextState = New( - decoderStates, logits, state->getEncoderStates(), state->getBatch()); + decoderStates, logits, state->getEncoderStates(), state->getBatch()); } else { nextState = New( - decoderStates, logits, state->getEncoderStates(), state->getBatch()); + decoderStates, logits, state->getEncoderStates(), state->getBatch()); } nextState->setPosition(state->getPosition() + 1); return nextState; From f309bbc799156f5de508595f9e155156ea9182f1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 15 Mar 2019 11:36:09 -0700 Subject: [PATCH 373/838] fixed gcc warnings --- src/layers/constructors.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index f19a0922b..953d01aa3 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -119,7 +119,7 @@ class MLP : public IUnaryLogitLayer { MLP(Ptr graph, Ptr options) : graph_(graph), options_(options) {} - Expr apply(const std::vector& av) { + Expr apply(const std::vector& av) override { Expr output; if(av.size() == 1) output = layers_[0]->apply(av[0]); @@ -132,7 +132,8 @@ class MLP : public IUnaryLogitLayer { return output; } - Logits applyAsLogits(const std::vector& av) { + Logits applyAsLogits(const std::vector& av) override { + // same as apply() except for the last layer, we invoke applyAsLogits(), which has a different return type auto lastLayer = std::dynamic_pointer_cast(layers_.back()); ABORT_IF(!lastLayer, "MLP::applyAsLogits() applied but last MLP layer is not IUnaryLogitLayer"); if (layers_.size() == 1) { @@ -153,8 +154,8 @@ class MLP : public IUnaryLogitLayer { } } - Expr apply(Expr e) { return apply(std::vector{ e }); } - Logits applyAsLogits(Expr e) { return applyAsLogits(std::vector{ e }); } + Expr apply(Expr e) override { return apply(std::vector{ e }); } + Logits applyAsLogits(Expr e) override { return applyAsLogits(std::vector{ e }); } void push_back(Ptr layer) { layers_.push_back(layer); } void push_back(Ptr layer) { layers_.push_back(layer); } From 213c9d14390cb67cb694cdf670189e44267fbe99 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 15:41:33 -0700 Subject: [PATCH 374/838] minor refactoring towards supporting .fsv --- src/data/factored_vocab.cpp | 42 ++++++++++++++++++------------------- src/data/factored_vocab.h | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 2b0695fd2..513903827 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -7,25 +7,10 @@ namespace marian { -// mapPath = path to file with entries in order of vocab entries of the form -// WORD FACTOR1 FACTOR2 FACTOR3... -// listPath = path to file that lists all FACTOR names -// vocab = original vocabulary -// Note: The WORD field in the map file is redundant. It is required for consistency checking only. -// Factors are grouped -// - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group -// - factors within a group as multi-class and normalized that way -// - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) -// - one prefix must not contain another -// - all factors not matching a prefix get lumped into yet another class (the lemmas) -// - factor vocab must be sorted such that all groups are consecutive -// - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries -/*virtual*/ size_t FactoredVocab::load(const std::string& factoredVocabPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { - auto mapPath = factoredVocabPath; - auto factorVocabPath = mapPath; - factorVocabPath.back() = 'l'; // map .fm to .fl - +/*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { // load factor vocabulary + auto factorVocabPath = modelPath; + factorVocabPath.back() = 'l'; // map .fm to .fl factorVocab_.load(factorVocabPath); groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB"/*, "@WE"*/, "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments // @TODO: add checks for empty factor groups until it stops crashing (training already works; decoder still crashes) @@ -36,6 +21,16 @@ namespace marian { auto numGroups = getNumGroups(); // load and parse factorMap + // modelPath = path to file with entries in order of vocab entries of the form + // WORD FACTOR1 FACTOR2 FACTOR3... + // Factors are grouped + // - user specifies list-factor prefixes; all factors beginning with that prefix are in the same group + // - factors within a group as multi-class and normalized that way + // - groups of size 1 are interpreted as sigmoids, multiply with P(u) / P(u-1) + // - one prefix must not contain another + // - all factors not matching a prefix get lumped into yet another class (the lemmas) + // - factor vocab must be sorted such that all groups are consecutive + // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries auto vocabSize = factorShape_.elements(); // size of vocab space including gaps vocab_.resize(vocabSize); //factorMap_.resize(vocabSize); @@ -44,14 +39,14 @@ namespace marian { std::vector tokens; std::string line; size_t numTotalFactors = 0; - io::InputFileStream in(mapPath); + io::InputFileStream in(modelPath); for (WordIndex v = 0; io::getline(in, line); v++) { // parse the line, of the form WORD FACTOR1 FACTOR2 FACTOR1 ... // where FACTOR1 is the lemma, a factor that all words have. // Not every word has all other factors, so the n-th item is not always in the same factor group. // @TODO: change to just use the .wl file, and manually split at @ utils::splitAny(line, tokens, " \t"); - ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", mapPath); + ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", modelPath); std::vector factorUnits; for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; @@ -519,13 +514,18 @@ size_t FactoredVocab::WordLUT::load(const std::string& path) { return size(); } +const static std::vector exts{ ".fsv", ".fm"/*legacy*/ }; + // Note: This does not actually load it, only checks the path for the type. Ptr createFactoredVocab(const std::string& vocabPath) { - bool isFactoredVocab = regex::regex_search(vocabPath, regex::regex("\\.(fm)$")); + bool isFactoredVocab = std::any_of(exts.begin(), exts.end(), [&](const std::string& ext) { return utils::endsWith(vocabPath, ext); }); if(isFactoredVocab) return New(); else return nullptr; } +/*virtual*/ const std::vector& FactoredVocab::suffixes() const /*override final*/ { + return exts; +} } // namespace marian diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 433a9d61a..c27efd74d 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -26,7 +26,7 @@ class FactoredVocab : public IVocab { virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } - virtual const std::vector& suffixes() const override final { const static std::vector exts{".fm"}; return exts; } + virtual const std::vector& suffixes() const override final; virtual Word operator[](const std::string& word) const override final; virtual Words encode(const std::string& line, bool addEOS = true, bool inference = false) const override final; virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; From ff65f6b741bb015f2db6e645ced42ab5a11a7d66 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 15:57:54 -0700 Subject: [PATCH 375/838] towards splitting off .fsv reading --- src/data/factored_vocab.cpp | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 513903827..2e2e71702 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -8,12 +8,26 @@ namespace marian { /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { - // load factor vocabulary - auto factorVocabPath = modelPath; - factorVocabPath.back() = 'l'; // map .fm to .fl - factorVocab_.load(factorVocabPath); - groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB"/*, "@WE"*/, "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments - // @TODO: add checks for empty factor groups until it stops crashing (training already works; decoder still crashes) + std::vector> factorMapTokenized; + + if (utils::endsWith(modelPath, ".fsv")) { + ABORT("Not implemented yet"); + } else { // legacy for old configs + // load factor vocabulary + auto factorVocabPath = modelPath; + factorVocabPath.back() = 'l'; // map .fm to .fl + factorVocab_.load(factorVocabPath); + groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB"/*, "@WE"*/, "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments + // @TODO: add checks for empty factor groups until it stops crashing (training already works; decoder still crashes) + + std::vector tokBuf; + std::string line; + io::InputFileStream in(modelPath); + for (WordIndex v = 0; io::getline(in, line); v++) { + utils::splitAny(line, tokBuf, " \t"); + factorMapTokenized.push_back(tokBuf); + } + } // construct mapping tables for factors constructGroupInfoFromFactorVocab(); @@ -36,16 +50,13 @@ namespace marian { //factorMap_.resize(vocabSize); auto factorVocabSize = factorVocab_.size(); lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); - std::vector tokens; - std::string line; size_t numTotalFactors = 0; - io::InputFileStream in(modelPath); - for (WordIndex v = 0; io::getline(in, line); v++) { + for (WordIndex v = 0; v < factorMapTokenized.size(); v++) { + const auto& tokens = factorMapTokenized[v]; // parse the line, of the form WORD FACTOR1 FACTOR2 FACTOR1 ... // where FACTOR1 is the lemma, a factor that all words have. // Not every word has all other factors, so the n-th item is not always in the same factor group. // @TODO: change to just use the .wl file, and manually split at @ - utils::splitAny(line, tokens, " \t"); ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", modelPath); std::vector factorUnits; for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { From 6c5bb6e54b7b82946108037c37cf4f05057d2d74 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 16:56:03 -0700 Subject: [PATCH 376/838] re-added a piece of code that got lost in an incorrect resolution of a merge conflict, works again --- src/translator/beam_search.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 9673b99e2..569e7756f 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -81,6 +81,23 @@ class BeamSearch { auto shortlist = scorers_[0]->getShortlist(); if (shortlist) word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); + else if (factoredVocab) { + // For factored decoding, the word is built over multiple decoding steps, + // starting with the lemma, then adding factors one by one. + if (factorGroup == 0) { + word = factoredVocab->lemma2Word(wordIdx); + //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + } + else { + //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + word = beam[beamHypIdx]->getWord(); + ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); + word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); + prevBeamHypIdx = prevHyp->getPrevStateIndex(); + prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words + } + } else word = Word::fromWordIndex(wordIdx); From 87d7ae47597e07401220367b9f259d0f9ad21bad Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 18:54:29 -0700 Subject: [PATCH 377/838] now parses .fsv files (for now by converting them internally to the legacy form) --- src/data/factored_vocab.cpp | 89 ++++++++++++++++++++++++++++++++++--- src/data/factored_vocab.h | 1 + 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 2e2e71702..a235cef37 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -9,19 +9,98 @@ namespace marian { /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { std::vector> factorMapTokenized; - + std::string line; + std::vector tokBuf; if (utils::endsWith(modelPath, ".fsv")) { - ABORT("Not implemented yet"); + // this is a fake parser for the generic factor spec, which makes a few hard assumptions: + // - all types starting with _ except _has_* are factor names + // - X : _x makes surface form X part of prob distribution _x except for _has_* + // - X : _has_x adds factor "x" to lemma X + // - _x <-> form only allows "_x <->" or "_x <-> _has_x" (same x), and is otherwise unused + // - _lemma is special + // The current version of the code just converts it internally to the legacy form. + // Once the legacy form is no longer needed, must of this can be simplified a lot. + io::InputFileStream in(modelPath); + WordIndex v = 0; + std::map> factorTypeMap; // [type name] -> {factor-type names} + std::vector deferredFactorVocab; // factor surface forms are presently expected to be at the end of factorVocab_, so collect them here first + while(io::getline(in, line)) { + utils::splitAny(line, tokBuf, " \t"); + if (tokBuf.empty() || tokBuf[0][0] == '#') // skip comments and blank lines + continue; + const auto& lhs = tokBuf[0]; + const auto& op = tokBuf.size() > 1 ? tokBuf[1] : ""; + if (lhs[0] == '_') { // factor name + if (utils::beginsWith(lhs, "_has_")) { + const auto fName = lhs.substr(5); // skip _has_ + ABORT_IF(factorTypeMap.find(fName) == factorTypeMap.end(), "Factor trait '{}' requires a factor named '{}' to exist", lhs, fName); + ABORT_IF(tokBuf.size() != 1, "Extraneous characters after factor trait: '{}'", line); + continue; + } + else if (op == "<->") { + ABORT_IF(lhs == "_lemma" && tokBuf.size() != 2, "Lemma factor distribution cannot be conditioned: '{}'", line); + ABORT_IF(lhs != "_lemma" && (tokBuf.size() != 3 || tokBuf[2] != "_has" + lhs), "Factor distribution can only be conditioned on nothing or on _has{}: '{}'", lhs, line); + continue; + } + else { // this declares a new factor + ABORT_IF(tokBuf.size() != 1, "Extraneous characters after factor declaration: '{}'", line); + const auto& fName = lhs.substr(1); // skip _ + ABORT_IF(factorTypeMap.empty() && fName != "lemma", "First factor must be _lemma"); + auto rv = factorTypeMap.insert(std::make_pair(fName, std::set())); // create new factor + ABORT_IF(!rv.second, "Factor declared twice: '{}'", line); + groupPrefixes_.push_back(fName == "lemma" ? "(lemma)" : ("|" + fName)); + continue; + } + } + else { // if not _ then it is a surface form + ABORT_IF(op != ":" || 2 >= tokBuf.size(), "Factor-lemma declaration should have the form LEMMA : _FACTOR, _has_FACTOR, _has_FACTOR... in '{}'", line); + ABORT_IF(tokBuf[2][0] != '_', "Factor name should begin with _ in '{}'", line); + ABORT_IF(utils::beginsWith(tokBuf[2], "_has_"), "The first factor after : must not begin with _has_ in '{}'", line); + // add to surface-form dictionary + const auto& fName = tokBuf[2].substr(1); // skip _ + auto isLemma = fName == "lemma"; + if (isLemma) + factorVocab_.add(lhs, v++); // note: each item can only be declared once + else + deferredFactorVocab.push_back(lhs); // add surface form to its declared factor type + auto surfaceFormSet = factorTypeMap.find(fName); // set of surface forms for this factor + ABORT_IF(surfaceFormSet == factorTypeMap.end(), "Unknown factor name in '{}'", line); + auto rv = surfaceFormSet->second.insert(lhs); // insert surface form into its declared factor type + ABORT_IF(!rv.second, "Factor declared twice: '{}'", line); + auto tokenizedMapLine = isLemma ? std::vector{ lhs, lhs } : std::vector(); + // associated factors + for (size_t i = 3; i < tokBuf.size(); i++) { + const auto& has = tokBuf[i]; + ABORT_IF(!utils::beginsWith(has, "_has_"), "Factor associations must use the form _has_X in '{}'", line); + ABORT_IF(!isLemma, "Factor associations are only allowed when factor type is _lemma: '{}', line"); + const auto& faName = has.substr(5); // skip _has_ and prepend | + // for tokenized map, we pick one example of the factor names + auto iter = factorTypeMap.find(faName); + ABORT_IF(iter == factorTypeMap.end(), "Invalid factor association {}, no such factor: '{}'", has, line); + const auto& factorNames = iter->second; + ABORT_IF(factorNames.empty(), "Factor association {} refers to empty factor type: '{}'", has, line); + const auto& oneFactorName = "|" + *factorNames.begin(); // pick the first entry as one example + tokenizedMapLine[0] += oneFactorName; + tokenizedMapLine.push_back(oneFactorName); + } + if (isLemma) + factorMapTokenized.push_back(std::move(tokenizedMapLine)); + continue; + } + ABORT("Malformed .fsv input line {}", line); // we only get here for lines we could not process + } + for (auto factorTypeName : deferredFactorVocab) + factorVocab_.add("|" + factorTypeName, v++); } else { // legacy for old configs + // legacy format: one factor map, one flat list of factor surface forms // load factor vocabulary + factorSeparator_ = '@'; auto factorVocabPath = modelPath; factorVocabPath.back() = 'l'; // map .fm to .fl factorVocab_.load(factorVocabPath); groupPrefixes_ = { "(lemma)", "@C", "@GL", "@GR", "@WB"/*, "@WE"*/, "@CB"/*, "@CE"*/ }; // @TODO: hard-coded for these initial experiments // @TODO: add checks for empty factor groups until it stops crashing (training already works; decoder still crashes) - std::vector tokBuf; - std::string line; io::InputFileStream in(modelPath); for (WordIndex v = 0; io::getline(in, line); v++) { utils::splitAny(line, tokBuf, " \t"); @@ -360,7 +439,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return getUnkId(); } -/*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*overrworde final*/ { +/*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*override final*/ { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); #if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index c27efd74d..220979e81 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -96,6 +96,7 @@ class FactoredVocab : public IVocab { WordLUT vocab_; // factors + char factorSeparator_ = '|'; // separator symbol for parsing factored words WordLUT factorVocab_; // [factor name] -> factor index = row of E_ std::vector groupPrefixes_; // [group id g] shared prefix of factors (used for grouping) //std::vector> factorMap_; // [word index v] -> set of factor indices u From ef5d086a546233ac1407fd8156151347f77671b1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 19:41:52 -0700 Subject: [PATCH 378/838] bug fix: Output layer in s2s also should pass the extra vocab args to support factored embeddings --- src/models/s2s.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/models/s2s.h b/src/models/s2s.h index 03baff564..cd9c663f2 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -343,6 +343,8 @@ class DecoderS2S : public DecoderBase { tiedPrefix = "Wemb"; last.tieTransposed(tiedPrefix); } + last("vocab", opt>("vocabs")[batchIndex_]); // for factored outputs + last("lemma-dim-emb", opt("lemma-dim-emb", 0)); // for factored outputs if(shortlist_) last.setShortlist(shortlist_); From 90a9670a403700587161e2318e0cba107dcf7994 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 20:10:04 -0700 Subject: [PATCH 379/838] bug fix: S2SDecoder should not delete output_ in clear(), but instead call output_->clear() --- src/layers/constructors.h | 16 +++++++++++++++- src/layers/generic.h | 11 ++++++++--- src/models/s2s.h | 12 +++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/layers/constructors.h b/src/layers/constructors.h index 953d01aa3..fc751dfc9 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -108,7 +108,7 @@ typedef Accumulator output; /** * Multi-layer network, holds and applies layers. */ -class MLP : public IUnaryLogitLayer { +class MLP : public IUnaryLogitLayer, public IHasShortList { protected: Ptr graph_; Ptr options_; @@ -159,6 +159,20 @@ class MLP : public IUnaryLogitLayer { void push_back(Ptr layer) { layers_.push_back(layer); } void push_back(Ptr layer) { layers_.push_back(layer); } + + void setShortlist(Ptr shortlist) override final { + auto p = tryAsHasShortlist(); + ABORT_IF(!p, "setShortlist() called on an MLP with an output layer that does not support short lists"); + p->setShortlist(shortlist); + } + + void clear() override final { + auto p = tryAsHasShortlist(); + if (p) + p->clear(); + } +private: + Ptr tryAsHasShortlist() const { return std::dynamic_pointer_cast(layers_.back()); } }; /** diff --git a/src/layers/generic.h b/src/layers/generic.h index 5b3e5ceab..40b8bb61a 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -50,6 +50,11 @@ struct IUnaryLayer { } }; +struct IHasShortList { + virtual void setShortlist(Ptr shortlist) = 0; + virtual void clear() = 0; +}; + // Embedding from corpus sub-batch to (emb, mask) struct IEmbeddingLayer { virtual std::tuple apply(Ptr subBatch) const = 0; @@ -190,7 +195,7 @@ class Dense : public LayerBase, public IUnaryLayer { Expr apply(Expr input) override { return apply(std::vector({input})); } }; -class Output : public LayerBase, public IUnaryLogitLayer { +class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList { private: // parameters held by this layer Expr Wt_; // weight matrix is stored transposed for efficiency @@ -218,7 +223,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { tiedParam_ = tied; } - void setShortlist(Ptr shortlist) { + void setShortlist(Ptr shortlist) override final { if (shortlist_) ABORT_IF(shortlist.get() != shortlist_.get(), "Output shortlist cannot be changed except after clear()"); else { @@ -230,7 +235,7 @@ class Output : public LayerBase, public IUnaryLogitLayer { // this is expected to be called in sync with graph->clear(), which invalidates // cachedShortWt_ and cachedShortb_ in the graph's short-term cache - void clear() { + void clear() override final { shortlist_ = nullptr; cachedShortWt_ = nullptr; cachedShortb_ = nullptr; diff --git a/src/models/s2s.h b/src/models/s2s.h index cd9c663f2..cfd0d7e00 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -239,7 +239,8 @@ class DecoderS2S : public DecoderBase { } public: - DecoderS2S(Ptr options) : DecoderBase(options) {} + DecoderS2S(Ptr options) : DecoderBase(options) { + } virtual Ptr startState( Ptr graph, @@ -346,9 +347,6 @@ class DecoderS2S : public DecoderBase { last("vocab", opt>("vocabs")[batchIndex_]); // for factored outputs last("lemma-dim-emb", opt("lemma-dim-emb", 0)); // for factored outputs - if(shortlist_) - last.setShortlist(shortlist_); - // assemble layers into MLP and apply to embeddings, decoder context and // aligned source context output_ = mlp::mlp() // @@ -357,6 +355,9 @@ class DecoderS2S : public DecoderBase { .construct(graph); } + if (shortlist_) + output_->setShortlist(shortlist_); + Logits logits; if(alignedContext) logits = output_->applyAsLogits({embeddings, decoderContext, alignedContext}); @@ -381,7 +382,8 @@ class DecoderS2S : public DecoderBase { void clear() override { rnn_ = nullptr; - output_ = nullptr; + if (output_) + output_->clear(); } }; } // namespace marian From 51c277b72d1a6b9be0be42ae4af35a8e48893152 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 16 Mar 2019 21:02:29 -0700 Subject: [PATCH 380/838] removed an overly cautious check --- src/layers/loss.cpp | 1 - 1 file changed, 1 deletion(-) mode change 100644 => 100755 src/layers/loss.cpp diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp old mode 100644 new mode 100755 index 86cd99d19..bd7ee0562 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -10,7 +10,6 @@ Ptr newLoss(Ptr options, bool inference) { if(costType == "ce-rescore") { return New(); } else if(costType == "ce-rescore-mean") { - ABORT("Check me"); return New(); } else { // same as ce-mean return New(smoothing); From 1a5d9b0ad17a78666370dd013538efaa91910d58 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 17 Mar 2019 16:56:35 -0700 Subject: [PATCH 381/838] bug fix: 'normalize' option in Rescorer must now explicitly divide by sentence length --- src/layers/loss.cpp | 6 ++---- src/layers/loss.h | 20 ++++++++++++-------- src/rescorer/rescorer.h | 24 +++++++++++++++++------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index bd7ee0562..61186fc1e 100755 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -7,11 +7,9 @@ Ptr newLoss(Ptr options, bool inference) { float smoothing = inference ? 0.f : options->get("label-smoothing"); std::string costType = options->get("cost-type", "ce-mean"); - if(costType == "ce-rescore") { + if(costType == "ce-rescore") { // returns per-batch-item scores (while ce-mean reduces over batch) return New(); - } else if(costType == "ce-rescore-mean") { - return New(); - } else { // same as ce-mean + } else { // same as ce-mean --@TODO: better check all allowed values, and fail for invalid ones. E.g. what about ce-sum? return New(smoothing); } } diff --git a/src/layers/loss.h b/src/layers/loss.h index bcf9ca4ca..8607fc5d7 100755 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -42,6 +42,7 @@ class RationalLoss { Expr loss() const { return loss_; } + // @TODO: remove this function, as it does not add too much value over loss()->get(...) template void loss(std::vector& losses) const { ABORT_IF(!loss_, "Loss has not been defined"); @@ -56,6 +57,7 @@ class RationalLoss { Expr count() const { return count_; } + // @TODO: remove this function, as it does not add too much value over count()->get(...) template void count(std::vector& labels) const { ABORT_IF(!count_, "Labels have not been defined"); @@ -369,14 +371,16 @@ class CrossEntropyLoss : public LabelwiseLoss { */ class RescorerLoss : public CrossEntropyLoss { public: - // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) - RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} - - virtual RationalLoss apply(Logits logits, const Words& labels, - Expr mask = nullptr, Expr labelWeights = nullptr) override { - auto ce = CrossEntropyLoss::apply(logits, labels, mask, labelWeights); - return RationalLoss(ce.loss(), ce.count()); - } + // sentence-wise CE, hence reduce only over time axis + // This class differs from CrossEntropy in the different 'axes' setting, and that label smoothing is disabled. + RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3} /*time axis*/, /*smoothing=*/0.f) {} + + // @TODO: this adds nothing; remove + //virtual RationalLoss apply(Logits logits, const Words& labels, + // Expr mask = nullptr, Expr labelWeights = nullptr) override { + // auto ce = CrossEntropyLoss::apply(logits, labels, mask, labelWeights); + // return RationalLoss(ce.loss(), ce.count()); + //} }; /** diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 3917a901c..8925391bd 100755 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -56,8 +56,7 @@ class Rescore : public ModelTask { "Normalization by length cannot be used with summary scores"); options_->set("inference", true); - // @TODO: make normalize here a float and pass into loss to compute the same way as in decoding - options_->set("cost-type", options_->get("normalize") ? "ce-rescore-mean" : "ce-rescore"); + options_->set("cost-type", "ce-rescore"); // indicates that to keep separate per-batch-item scoresForSummary if(options_->get("n-best")) corpus_ = New(options_); @@ -102,6 +101,7 @@ class Rescore : public ModelTask { auto alignment = options_->get("alignment", ""); auto summary = options_->get("summary", ""); bool summarize = !summary.empty(); + // @TODO: make normalize here a float and pass into loss to compute the same way as in decoding bool normalize = options_->get("normalize"); float sumLoss = 0; @@ -130,17 +130,27 @@ class Rescore : public ModelTask { graph->forward(); - std::vector scores; - dynamicLoss->loss(scores); + // get loss + std::vector scoresForSummary; + dynamicLoss->loss(scoresForSummary); + std::vector sentScores(scoresForSummary); // if '--normalize' then report scoresForSummary length-normalized + if (normalize) { + std::vector sentLengths; + dynamicLoss->count(sentLengths); + for (size_t i = 0; i < scoresForSummary.size(); i++) { + if (sentScores[i] != 0) // (avoid 0/0) + sentScores[i] /= (sentLengths.size() == 1 ? sentLengths[0] : sentLengths[i]); // emulate broadcasting semantics + } + } // soft alignments for each sentence in the batch - std::vector aligns(batch->size()); + std::vector aligns(batch->size()); // @TODO: do this resize inside getAlignmentsForBatch() if(!alignment.empty()) { getAlignmentsForBatch(builder->getAlignment(), batch, aligns); } std::unique_lock lock(smutex); - for(auto s : scores) + for(auto s : scoresForSummary) sumLoss += s; sumWords += batch->back()->batchWords(); sumSamples += batch->size(); @@ -148,7 +158,7 @@ class Rescore : public ModelTask { if(!summarize) { for(size_t i = 0; i < batch->size(); ++i) { output->Write((long)batch->getSentenceIds()[i], - -1.f * scores[i], // report logProb while score is CE, hence negate + -1.f * sentScores[i], // report logProb while score is CE, hence negate aligns[i]); } } From 45ba9477ba1a3c70a55e83a64b9d53358d1077c5 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 17 Mar 2019 17:28:22 -0700 Subject: [PATCH 382/838] updated --all-caps-every and --english-title-case-every to new syntax --- src/data/factored_vocab.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index a235cef37..69172edea 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -456,13 +456,13 @@ void FactoredVocab::constructNormalizationInfoForVocab() { } /*virtual*/ std::string FactoredVocab::toUpper(const std::string& line) const /*override final*/ { - return utils::findReplace(utils::findReplace(line, "@CI", "@CA", /*all=*/true), "@CN", "@CA", /*all=*/true); + return utils::findReplace(utils::findReplace(utils::findReplace(utils::findReplace(line, "|ci", "|ca", /*all=*/true), "|cn", "|ca", /*all=*/true), "@CI", "@CA", /*all=*/true), "@CN", "@CA", /*all=*/true); } /*virtual*/ std::string FactoredVocab::toEnglishTitleCase(const std::string& line) const /*override final*/ { // @BUGBUG: does not handle the special words that should remain lower-case // note: this presently supports both @WB and @GL- (legacy) - return utils::findReplace(utils::findReplace(line, "@CN@WB", "@CI@WB", /*all=*/true), "@CN@GL-", "@CI@GL-", /*all=*/true); + return utils::findReplace(utils::findReplace(utils::findReplace(utils::findReplace(line, "|cn|wb", "|ci|wb", /*all=*/true), "|cn|gl-", "|ci|gl-", /*all=*/true), "@CN@WB", "@CI@WB", /*all=*/true), "@CN@GL-", "@CI@GL-", /*all=*/true); } // generate a valid random factored word (used by collectStats()) From 58feb01050b64af3189f146a138082bb90f95952 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 18 Mar 2019 12:07:42 +0000 Subject: [PATCH 383/838] Add comments --- src/3rd_party/CLI/App.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/3rd_party/CLI/App.hpp b/src/3rd_party/CLI/App.hpp index fe430c16c..14ddd1e7f 100644 --- a/src/3rd_party/CLI/App.hpp +++ b/src/3rd_party/CLI/App.hpp @@ -1590,7 +1590,11 @@ class App { } // Unlimited vector parser + // RG: A negative number for the total number of expected values means that the option is a + // vector and accepts an unlimited number of values if(num < 0) { + // RG: We need to keep track if the vector option is empty and handle this separately as + // otherwise the parser will mark the command-line option as not set bool emptyVectorArgs = true; while(!args.empty() && _recognize(args.back()) == detail::Classifer::NONE) { if(collected >= -num) { @@ -1611,12 +1615,16 @@ class App { if(!args.empty() && _recognize(args.back()) == detail::Classifer::POSITIONAL_MARK) args.pop_back(); + // RG: Handle empty vector-like options if(emptyVectorArgs) { + // RG: Set implicit value(s) if the option has it (them) if(op->get_implicit()) { for(const auto& ival : detail::split_up(op->get_implicitval())) { op->add_result(ival); parse_order_.push_back(op.get()); } + // RG: Abort if there is a minimum number of values expected. Note: get_expected() + // equals to -N means at least N values are expected } else if (op->get_expected() < 0) { parse_order_.push_back(op.get()); throw ArgumentMismatch(op->get_name(), op->get_expected(), 0); From 86033cb5c42061437a9575a0d0c5c7a1abadd974 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 19 Mar 2019 10:16:04 -0700 Subject: [PATCH 384/838] new parameter --valid-script-args --- src/common/config_parser.cpp | 4 ++++ src/common/utils.cpp | 17 ++++++++++++++--- src/common/utils.h | 2 +- src/layers/loss.h | 2 -- src/training/validator.h | 11 +++++++---- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index ee49b448f..46402e38e 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -33,6 +33,7 @@ const std::set PATHS = { "embedding-vectors", "valid-sets", "valid-script-path", + "valid-script-args", "valid-log", "valid-translation-output", "input", // except: stdin @@ -471,6 +472,9 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { " It should print a single score to stdout." " If the option is used with validating translation, the output" " translation file will be passed as a first argument"); + cli.add>("--valid-script-args", + "Additional args passed to --valid-script-path. These are inserted" + " between the script path and the output translation-file path"); cli.add("--valid-translation-output", "Path to store the translation"); diff --git a/src/common/utils.cpp b/src/common/utils.cpp index eb464878e..46870531a 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -93,19 +93,30 @@ std::string join(const std::vector& words, const std::string& del / return ss.str(); } -std::string exec(const std::string& cmd) { +// escapes a string for passing to popen, which uses /bin/sh to parse its argument string +static std::string escapeForPOpen(const std::string& arg) { + // e.g. abc -> 'abc'; my file.txt -> 'my file.txt'; $10 -> '$10'; it's -> 'it'\''s' + return "'" + findReplace(arg, "'", "'\\''", /*all=*/ true) + "'"; +} + +std::string exec(const std::string& cmd, const std::vector& args /*= {}*/, const std::string& arg /*= ""*/) { std::array buffer; std::string result; #ifdef _WIN32 #define popen _popen #define pclose _pclose #endif - std::shared_ptr pipe(popen(cmd.c_str(), "r"), pclose); + auto cmdLine = escapeForPOpen(cmd); + for (const auto& a : args) // @TODO: proper escaping + cmdLine += " " + escapeForPOpen(a); + if (!arg.empty()) + cmdLine += " " + escapeForPOpen(arg); + std::shared_ptr pipe(popen(cmdLine.c_str(), "r"), pclose); if(!pipe) ABORT("popen() failed!"); while(!std::feof(pipe.get())) { - if(std::fgets(buffer.data(), 128, pipe.get()) != NULL) + if(std::fgets(buffer.data(), buffer.size(), pipe.get()) != NULL) result += buffer.data(); } return result; diff --git a/src/common/utils.h b/src/common/utils.h index 8aa40f835..6226ed992 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -30,7 +30,7 @@ std::vector splitAny(const std::string& line, std::string join(const std::vector& words, const std::string& del = " "); -std::string exec(const std::string& cmd); +std::string exec(const std::string& cmd, const std::vector& args = {}, const std::string& arg = ""); std::pair hostnameAndProcessId(); diff --git a/src/layers/loss.h b/src/layers/loss.h index 8607fc5d7..22b97b1af 100755 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -42,7 +42,6 @@ class RationalLoss { Expr loss() const { return loss_; } - // @TODO: remove this function, as it does not add too much value over loss()->get(...) template void loss(std::vector& losses) const { ABORT_IF(!loss_, "Loss has not been defined"); @@ -57,7 +56,6 @@ class RationalLoss { Expr count() const { return count_; } - // @TODO: remove this function, as it does not add too much value over count()->get(...) template void count(std::vector& labels) const { ABORT_IF(!count_, "Labels have not been defined"); diff --git a/src/training/validator.h b/src/training/validator.h index 0e423008a..f7f586f9d 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -430,8 +430,8 @@ class ScriptValidator : public Validator { auto model = options_->get("model"); builder_->save(graphs[0], model + ".dev.npz", true); - auto command = options_->get("valid-script-path"); - auto valStr = utils::exec(command); + auto valStr = utils::exec(options_->get("valid-script-path"), + options_->get>("valid-script-args")); float val = (float)std::atof(valStr.c_str()); updateStalled(graphs, val); @@ -558,8 +558,11 @@ class TranslationValidator : public Validator { // Run post-processing script if given if(options_->hasAndNotEmpty("valid-script-path")) { - auto command = options_->get("valid-script-path") + " " + fileName; - auto valStr = utils::exec(command); + //auto command = options_->get("valid-script-path") + " " + fileName; + //auto valStr = utils::exec(command); + auto valStr = utils::exec(options_->get("valid-script-path"), + options_->get>("valid-script-args"), + fileName); val = (float)std::atof(valStr.c_str()); updateStalled(graphs, val); } From a38ee501745d1f836fb9b0f74aa299243e95313b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 17 Mar 2019 16:56:35 -0700 Subject: [PATCH 385/838] apply Frank's fix to rescorer --- src/layers/loss.cpp | 7 ++----- src/layers/loss.h | 13 +++++-------- src/rescorer/rescorer.h | 24 +++++++++++++++++------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp index 86cd99d19..61186fc1e 100644 --- a/src/layers/loss.cpp +++ b/src/layers/loss.cpp @@ -7,12 +7,9 @@ Ptr newLoss(Ptr options, bool inference) { float smoothing = inference ? 0.f : options->get("label-smoothing"); std::string costType = options->get("cost-type", "ce-mean"); - if(costType == "ce-rescore") { + if(costType == "ce-rescore") { // returns per-batch-item scores (while ce-mean reduces over batch) return New(); - } else if(costType == "ce-rescore-mean") { - ABORT("Check me"); - return New(); - } else { // same as ce-mean + } else { // same as ce-mean --@TODO: better check all allowed values, and fail for invalid ones. E.g. what about ce-sum? return New(smoothing); } } diff --git a/src/layers/loss.h b/src/layers/loss.h index 3f2ec5446..0087188e0 100755 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -39,6 +39,7 @@ class RationalLoss { Expr loss() const { return loss_; } + // @TODO: remove this function, as it does not add too much value over loss()->get(...) template void loss(std::vector& losses) const { ABORT_IF(!loss_, "Loss has not been defined"); @@ -53,6 +54,7 @@ class RationalLoss { Expr count() const { return count_; } + // @TODO: remove this function, as it does not add too much value over count()->get(...) template void count(std::vector& labels) const { ABORT_IF(!count_, "Labels have not been defined"); @@ -368,14 +370,9 @@ class CrossEntropyLoss : public LabelwiseLoss { */ class RescorerLoss : public CrossEntropyLoss { public: - // sentence-wise CE, hence reduce only over time axis. CE reduces over last axis (-1) - RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3}, /*smoothing=*/0.f) {} - - virtual RationalLoss apply(Expr logits, const Words& labels, - Expr mask = nullptr, Expr labelWeights = nullptr) override { - auto ce = CrossEntropyLoss::apply(logits, labels, mask, labelWeights); - return RationalLoss(ce.loss(), ce.count()); - } + // sentence-wise CE, hence reduce only over time axis + // This class differs from CrossEntropy in the different 'axes' setting, and that label smoothing is disabled. + RescorerLoss() : CrossEntropyLoss(/*axes=*/{-3} /*time axis*/, /*smoothing=*/0.f) {} }; /** diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h index 3917a901c..8925391bd 100755 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -56,8 +56,7 @@ class Rescore : public ModelTask { "Normalization by length cannot be used with summary scores"); options_->set("inference", true); - // @TODO: make normalize here a float and pass into loss to compute the same way as in decoding - options_->set("cost-type", options_->get("normalize") ? "ce-rescore-mean" : "ce-rescore"); + options_->set("cost-type", "ce-rescore"); // indicates that to keep separate per-batch-item scoresForSummary if(options_->get("n-best")) corpus_ = New(options_); @@ -102,6 +101,7 @@ class Rescore : public ModelTask { auto alignment = options_->get("alignment", ""); auto summary = options_->get("summary", ""); bool summarize = !summary.empty(); + // @TODO: make normalize here a float and pass into loss to compute the same way as in decoding bool normalize = options_->get("normalize"); float sumLoss = 0; @@ -130,17 +130,27 @@ class Rescore : public ModelTask { graph->forward(); - std::vector scores; - dynamicLoss->loss(scores); + // get loss + std::vector scoresForSummary; + dynamicLoss->loss(scoresForSummary); + std::vector sentScores(scoresForSummary); // if '--normalize' then report scoresForSummary length-normalized + if (normalize) { + std::vector sentLengths; + dynamicLoss->count(sentLengths); + for (size_t i = 0; i < scoresForSummary.size(); i++) { + if (sentScores[i] != 0) // (avoid 0/0) + sentScores[i] /= (sentLengths.size() == 1 ? sentLengths[0] : sentLengths[i]); // emulate broadcasting semantics + } + } // soft alignments for each sentence in the batch - std::vector aligns(batch->size()); + std::vector aligns(batch->size()); // @TODO: do this resize inside getAlignmentsForBatch() if(!alignment.empty()) { getAlignmentsForBatch(builder->getAlignment(), batch, aligns); } std::unique_lock lock(smutex); - for(auto s : scores) + for(auto s : scoresForSummary) sumLoss += s; sumWords += batch->back()->batchWords(); sumSamples += batch->size(); @@ -148,7 +158,7 @@ class Rescore : public ModelTask { if(!summarize) { for(size_t i = 0; i < batch->size(); ++i) { output->Write((long)batch->getSentenceIds()[i], - -1.f * scores[i], // report logProb while score is CE, hence negate + -1.f * sentScores[i], // report logProb while score is CE, hence negate aligns[i]); } } From 8e8c4efc3bc86e5f18626949fd1fed3c924188ce Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Fri, 22 Mar 2019 21:07:07 +0000 Subject: [PATCH 386/838] Don't close named pipes during corpus reset. Corpus::reset() closed and reopened input files for corpora. So files were opened once during the construction of the Corpus instance, then closed and repopened during data->reset() in BatchGenerator::prepare(false). With normal files that's not a problem, but if the "file" is a named pipe, the closing triggers a SIGPIPE (broken pipe) on the writing end of the pipe. With this commit, Corpus::reset() leaves open pipes alone. --- src/common/filesystem.cpp | 17 +++++++++++++++++ src/common/filesystem.h | 4 +++- src/data/corpus.cpp | 23 ++++++++++++++++------- 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 src/common/filesystem.cpp diff --git a/src/common/filesystem.cpp b/src/common/filesystem.cpp new file mode 100644 index 000000000..d5196e850 --- /dev/null +++ b/src/common/filesystem.cpp @@ -0,0 +1,17 @@ +#include "filesystem.h" + +#include +#include +#include + +namespace marian { +namespace filesystem { + +bool is_fifo(char const* path) { + struct stat buf; + stat(path, &buf); + return S_ISFIFO(buf.st_mode); +} + +} // end of namespace marian::filesystem +} // end of namespace marian diff --git a/src/common/filesystem.h b/src/common/filesystem.h index 08d5995f1..2c700aa09 100644 --- a/src/common/filesystem.h +++ b/src/common/filesystem.h @@ -22,6 +22,8 @@ namespace marian { namespace filesystem { + bool is_fifo(char const* path); + class Path { private: Pathie::Path path; @@ -97,4 +99,4 @@ namespace filesystem { using FilesystemError = Pathie::PathieError; } -} \ No newline at end of file +} diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index fda8df95f..c851c3c70 100644 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -4,6 +4,8 @@ #include #include "common/utils.h" +#include "common/filesystem.h" + #include "data/corpus.h" namespace marian { @@ -44,6 +46,7 @@ SentenceTuple Corpus::next() { } else { bool gotLine = io::getline(*files_[i], line); + // LOG(debug,"[{}][{}] {}", i, pos_ - 1, line); if(!gotLine) { eofsHit++; continue; @@ -85,16 +88,22 @@ void Corpus::shuffle() { // Call either reset() or shuffle(). // @TODO: make shuffle() private, instad pass a shuffle() flag to reset(), to clarify mutual exclusiveness with shuffle() void Corpus::reset() { - files_.clear(); corpusInRAM_.clear(); ids_.clear(); pos_ = 0; - for(auto& path : paths_) { - if(path == "stdin") - files_.emplace_back(new io::InputFileStream(std::cin)); - else - files_.emplace_back(new io::InputFileStream(path)); - } + for (size_t i = 0; i < paths_.size(); ++i) + { + if(paths_[i] == "stdin") { + files_[i].reset(new io::InputFileStream(std::cin)); + // Probably not necessary, unless there are some buffers + // that we want flushed. + } + else if (!filesystem::is_fifo(paths_[i].c_str())) { + // Do NOT reset named pipes; that closes them and triggers a SIGPIPE + // (lost pipe) at the writing end. + files_[i].reset(new io::InputFileStream(paths_[i])); + } + } } void Corpus::restore(Ptr ts) { From e56685a21466817e9ed776979c819bbfd48a4693 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sat, 23 Mar 2019 02:01:03 +0000 Subject: [PATCH 387/838] Add common/filesystem.cpp to CMakeLists.txt --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fbafc3b99..f7ab0a1c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,6 +17,7 @@ add_library(marian STATIC common/config_validator.cpp common/binary.cpp common/io.cpp + common/filesystem.cpp data/alignment.cpp data/vocab.cpp From c098dd7e52f4dcc58699e1cc3cf595bdff55015b Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Tue, 26 Mar 2019 18:10:21 +0000 Subject: [PATCH 388/838] Updated fix after discussions with @frankseide. --- src/common/filesystem.cpp | 4 ++++ src/common/filesystem.h | 1 + src/data/corpus.cpp | 9 +++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/common/filesystem.cpp b/src/common/filesystem.cpp index d5196e850..81500a5e7 100644 --- a/src/common/filesystem.cpp +++ b/src/common/filesystem.cpp @@ -13,5 +13,9 @@ bool is_fifo(char const* path) { return S_ISFIFO(buf.st_mode); } +bool is_fifo(std::string const& path) { + return is_fifo(path.c_str()); +} + } // end of namespace marian::filesystem } // end of namespace marian diff --git a/src/common/filesystem.h b/src/common/filesystem.h index 2c700aa09..434c23ba9 100644 --- a/src/common/filesystem.h +++ b/src/common/filesystem.h @@ -23,6 +23,7 @@ namespace marian { namespace filesystem { bool is_fifo(char const* path); + bool is_fifo(std::string const& path); class Path { private: diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index c851c3c70..1218383c9 100644 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -90,6 +90,8 @@ void Corpus::shuffle() { void Corpus::reset() { corpusInRAM_.clear(); ids_.clear(); + if (pos_ == 0) // no data read yet + return; pos_ = 0; for (size_t i = 0; i < paths_.size(); ++i) { @@ -98,9 +100,12 @@ void Corpus::reset() { // Probably not necessary, unless there are some buffers // that we want flushed. } - else if (!filesystem::is_fifo(paths_[i].c_str())) { + else { + ABORT_IF(files_[i] and filesystem::is_fifo(paths_[i]), + "File '", paths_[i], "' is a pipe and cannot be re-opened."); // Do NOT reset named pipes; that closes them and triggers a SIGPIPE - // (lost pipe) at the writing end. + // (lost pipe) at the writing end, which may do whatever it wants + // in this situation. files_[i].reset(new io::InputFileStream(paths_[i])); } } From d7f7d29258df0a2c463b086bdb930e73cf669035 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 27 Mar 2019 10:08:29 -0700 Subject: [PATCH 389/838] new validator bleu-detok-ms that treats CJT continuous-script characters as individual tokens --- src/command/marian_main.cpp | 0 src/common/config_parser.cpp | 2 +- src/common/utils.cpp | 62 +++++++++++++++++++++----------- src/common/utils.h | 4 +++ src/training/validator.cpp | 7 ++-- src/training/validator.h | 69 ++++++++++++++++++++++++++---------- 6 files changed, 101 insertions(+), 43 deletions(-) mode change 100644 => 100755 src/command/marian_main.cpp diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp old mode 100644 new mode 100755 diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index 46402e38e..ae09f2aa2 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -435,7 +435,7 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { "10000u"); cli.add>("--valid-metrics", "Metric to use during validation: cross-entropy, ce-mean-words, perplexity, valid-script, " - "translation, bleu, bleu-detok. Multiple metrics can be specified", + "translation, bleu, bleu-detok, bleu-detok-ms. Multiple metrics can be specified", {"cross-entropy"}); cli.add("--early-stopping", "Stop if the first validation metric does not improve for arg consecutive validation steps", diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 46870531a..958dfc77a 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -116,7 +116,7 @@ std::string exec(const std::string& cmd, const std::vector& args /* ABORT("popen() failed!"); while(!std::feof(pipe.get())) { - if(std::fgets(buffer.data(), buffer.size(), pipe.get()) != NULL) + if(std::fgets(buffer.data(), (int)buffer.size(), pipe.get()) != NULL) result += buffer.data(); } return result; @@ -155,27 +155,46 @@ bool endsWith(const std::string& text, const std::string& suffix) { && !text.compare(text.size() - suffix.size(), suffix.size(), suffix); } -static std::wstring utf8ToWString(std::string const& s) { - std::wstring_convert> converter; +std::u32string utf8ToUnicodeString(std::string const& s) { +#ifdef _MSC_VER // workaround for a known bug in VS CRT + std::wstring_convert, unsigned int/*char32_t*/> converter; + auto res = converter.from_bytes(s); + return std::u32string(res.begin(), res.end()); +#else + std::wstring_convert, char32_t> converter; return converter.from_bytes(s); +#endif } -static std::string toUTF8String(std::wstring const& s) { - std::wstring_convert> converter; +std::string utf8FromUnicodeString(const std::u32string& s) { +#ifdef _MSC_VER // workaround for a known bug in VS CRT + std::wstring_convert, unsigned int/*char32_t*/> converter; + std::basic_string si(s.begin(), s.end()); + return converter.to_bytes(si); +#else + std::wstring_convert, char32_t> converter; return converter.to_bytes(s); +#endif } -// convert a UTF-8 string to all-caps -#if 0 -// @BUGBUG: This does not work for non-ASCII characters. -std::string utf8ToUpper(const std::string& s) { - auto ws = utf8ToWString(s); - for (auto& c : ws) - c = std::towupper(c); - return toUTF8String(ws); +bool isContinuousScript(char32_t c) { + // currently, this table is hand-coded, and may need to be extended when the standard grows + auto in = [c](char32_t minVal, char32_t maxVal) { return c >= minVal && c <= maxVal; }; + bool isHan = in(0x2E80, 0x2E99) || in(0x2E9B, 0x2EF3) || in(0x2F00, 0x2FD5) || in(0x3005, 0x3005) || + in(0x3007, 0x3007) || in(0x3021, 0x3029) || in(0x3038, 0x303A) || in(0x303B, 0x303b) || + in(0x3400, 0x4DB5) || in(0x4E00, 0x9FEF) || in(0xF900, 0xFA6D) || in(0xFA70, 0xFAD9) || + in(0x20000, 0x2A6D6) || in(0x2A700, 0x2B734) || in(0x2B740, 0x2B81D) || in(0x2B820, 0x2CEA1) || + in(0x2CEB0, 0x2EBE0) || in(0x2F800, 0x2FA1D) || + in(0x3200, 0x32FF); // Enclosed CJK Letters and Months, https://en.wikipedia.org/wiki/Enclosed_CJK_Letters_and_Months + bool isKana = in(0x3040, 0x30FF) || // Hiragana, Katakana + in(0x1B000, 0x1B0FF) || // Kana supplement, https://en.wikipedia.org/wiki/Kana_Supplement + in(0x1B130, 0x1B16F); // small Kana, https://en.wikipedia.org/wiki/Small_Kana_Extension + bool isThai = in(0x0E00, 0x0E7F); // https://en.wikipedia.org/wiki/Thai_(Unicode_block) + return isHan || isKana || isThai; } -#else -struct UTF8Mapper : std::map { // Hack because Philly does not have UTF-8 locale installed + +// convert a UTF-8 string to all-caps +struct UTF8Mapper : std::map { // Hack because MS-internal Philly servers do not have UTF-8 locale installed UTF8Mapper() { /* env LC_ALL=en_US.UTF-8 sed 's/\(.\)/\1\n/g' TEXT_FILE_CONTAINING_ALL_CHARS > l @@ -191,13 +210,13 @@ struct UTF8Mapper : std::map { // Hack because Philly does not */ std::vector> map8{ {"\xc9\x92","\xe2\xb1\xb0"},{"\x61","\x41"},{"\xc3\xa1","\xc3\x81"},{"\xc3\xa0","\xc3\x80"},{"\xe1\xba\xaf","\xe1\xba\xae"},{"\xe1\xba\xb1","\xe1\xba\xb0"},{"\xe1\xba\xb5","\xe1\xba\xb4"},{"\xe1\xba\xb3","\xe1\xba\xb2"},{"\xe1\xba\xb7","\xe1\xba\xb6"},{"\xc4\x83","\xc4\x82"},{"\xe1\xba\xa5","\xe1\xba\xa4"},{"\xe1\xba\xa7","\xe1\xba\xa6"},{"\xe1\xba\xab","\xe1\xba\xaa"},{"\xe1\xba\xa9","\xe1\xba\xa8"},{"\xe1\xba\xad","\xe1\xba\xac"},{"\xc3\xa2","\xc3\x82"},{"\xc7\x8e","\xc7\x8d"},{"\xc7\xbb","\xc7\xba"},{"\xc3\xa5","\xc3\x85"},{"\xc7\x9f","\xc7\x9e"},{"\xc3\xa4","\xc3\x84"},{"\xc3\xa3","\xc3\x83"},{"\xc4\x85","\xc4\x84"},{"\xc4\x81","\xc4\x80"},{"\xe1\xba\xa3","\xe1\xba\xa2"},{"\xc8\x83","\xc8\x82"},{"\xe1\xba\xa1","\xe1\xba\xa0"},{"\xc7\xa3","\xc7\xa2"},{"\xc3\xa6","\xc3\x86"},{"\x62","\x42"},{"\xe1\xb8\x87","\xe1\xb8\x86"},{"\x63","\x43"},{"\xc4\x87","\xc4\x86"},{"\xc4\x89","\xc4\x88"},{"\xc4\x8d","\xc4\x8c"},{"\xc4\x8b","\xc4\x8a"},{"\xc3\xa7","\xc3\x87"},{"\x64","\x44"},{"\xc4\x8f","\xc4\x8e"},{"\xc4\x91","\xc4\x90"},{"\xe1\xb8\x91","\xe1\xb8\x90"},{"\xe1\xb8\x8d","\xe1\xb8\x8c"},{"\xe1\xb8\x8f","\xe1\xb8\x8e"},{"\xc3\xb0","\xc3\x90"},{"\x65","\x45"},{"\xc3\xa9","\xc3\x89"},{"\xc3\xa8","\xc3\x88"},{"\xc4\x95","\xc4\x94"},{"\xe1\xba\xbf","\xe1\xba\xbe"},{"\xe1\xbb\x81","\xe1\xbb\x80"},{"\xe1\xbb\x85","\xe1\xbb\x84"},{"\xe1\xbb\x83","\xe1\xbb\x82"},{"\xe1\xbb\x87","\xe1\xbb\x86"},{"\xc3\xaa","\xc3\x8a"},{"\xc4\x9b","\xc4\x9a"},{"\xc3\xab","\xc3\x8b"},{"\xe1\xba\xbd","\xe1\xba\xbc"},{"\xc4\x97","\xc4\x96"},{"\xc4\x99","\xc4\x98"},{"\xe1\xb8\x97","\xe1\xb8\x96"},{"\xc4\x93","\xc4\x92"},{"\xe1\xba\xbb","\xe1\xba\xba"},{"\xc8\x87","\xc8\x86"},{"\xe1\xba\xb9","\xe1\xba\xb8"},{"\xc7\x9d","\xc6\x8e"},{"\x66","\x46"},{"\x67","\x47"},{"\xc7\xb5","\xc7\xb4"},{"\xc4\x9f","\xc4\x9e"},{"\xc4\x9d","\xc4\x9c"},{"\xc7\xa7","\xc7\xa6"},{"\xc4\xa1","\xc4\xa0"},{"\xc4\xa3","\xc4\xa2"},{"\xc9\xa0","\xc6\x93"},{"\x68","\x48"},{"\xc4\xa5","\xc4\xa4"},{"\xc4\xa7","\xc4\xa6"},{"\xe1\xb8\xa9","\xe1\xb8\xa8"},{"\xe1\xb8\xa5","\xe1\xb8\xa4"},{"\xe1\xb8\xab","\xe1\xb8\xaa"},{"\x69","\x49"},{"\xc4\xb1","\x49"},{"\xc3\xad","\xc3\x8d"},{"\xc3\xac","\xc3\x8c"},{"\xc4\xad","\xc4\xac"},{"\xc3\xae","\xc3\x8e"},{"\xc7\x90","\xc7\x8f"},{"\xc3\xaf","\xc3\x8f"},{"\xc4\xa9","\xc4\xa8"},{"\xc4\xaf","\xc4\xae"},{"\xc4\xab","\xc4\xaa"},{"\xe1\xbb\x89","\xe1\xbb\x88"},{"\xc8\x8b","\xc8\x8a"},{"\xe1\xbb\x8b","\xe1\xbb\x8a"},{"\x6a","\x4a"},{"\xc4\xb5","\xc4\xb4"},{"\x6b","\x4b"},{"\xe1\xb8\xb1","\xe1\xb8\xb0"},{"\xc4\xb7","\xc4\xb6"},{"\xe1\xb8\xb3","\xe1\xb8\xb2"},{"\xc6\x99","\xc6\x98"},{"\x6c","\x4c"},{"\xc4\xba","\xc4\xb9"},{"\xc4\xbe","\xc4\xbd"},{"\xc5\x82","\xc5\x81"},{"\xc4\xbc","\xc4\xbb"},{"\xe1\xb8\xb7","\xe1\xb8\xb6"},{"\x6d","\x4d"},{"\xe1\xb8\xbf","\xe1\xb8\xbe"},{"\xe1\xb9\x83","\xe1\xb9\x82"},{"\xc5\x8b","\xc5\x8a"},{"\x6e","\x4e"},{"\xc5\x84","\xc5\x83"},{"\xc5\x88","\xc5\x87"},{"\xc3\xb1","\xc3\x91"},{"\xe1\xb9\x85","\xe1\xb9\x84"},{"\xc5\x86","\xc5\x85"},{"\xe1\xb9\x87","\xe1\xb9\x86"},{"\xe1\xb9\x89","\xe1\xb9\x88"},{"\xc5\x93","\xc5\x92"},{"\x6f","\x4f"},{"\xc3\xb3","\xc3\x93"},{"\xc3\xb2","\xc3\x92"},{"\xc5\x8f","\xc5\x8e"},{"\xe1\xbb\x91","\xe1\xbb\x90"},{"\xe1\xbb\x93","\xe1\xbb\x92"},{"\xe1\xbb\x95","\xe1\xbb\x94"},{"\xe1\xbb\x99","\xe1\xbb\x98"},{"\xc3\xb4","\xc3\x94"},{"\xc7\x92","\xc7\x91"},{"\xc3\xb6","\xc3\x96"},{"\xc5\x91","\xc5\x90"},{"\xc3\xb5","\xc3\x95"},{"\xc3\xb8","\xc3\x98"},{"\xc7\xab","\xc7\xaa"},{"\xc5\x8d","\xc5\x8c"},{"\xe1\xbb\x8f","\xe1\xbb\x8e"},{"\xc8\x8f","\xc8\x8e"},{"\xe1\xbb\x8d","\xe1\xbb\x8c"},{"\xe1\xbb\x9b","\xe1\xbb\x9a"},{"\xe1\xbb\x9d","\xe1\xbb\x9c"},{"\xe1\xbb\xa1","\xe1\xbb\xa0"},{"\xe1\xbb\x9f","\xe1\xbb\x9e"},{"\xe1\xbb\xa3","\xe1\xbb\xa2"},{"\xc6\xa1","\xc6\xa0"},{"\xc9\x94","\xc6\x86"},{"\x70","\x50"},{"\xe1\xb9\x95","\xe1\xb9\x94"},{"\x71","\x51"},{"\x72","\x52"},{"\xc5\x95","\xc5\x94"},{"\xc5\x99","\xc5\x98"},{"\xc5\x97","\xc5\x96"},{"\xe1\xb9\x9b","\xe1\xb9\x9a"},{"\xe1\xb9\x9f","\xe1\xb9\x9e"},{"\x73","\x53"},{"\xc5\x9b","\xc5\x9a"},{"\xc5\x9d","\xc5\x9c"},{"\xc5\xa1","\xc5\xa0"},{"\xc5\x9f","\xc5\x9e"},{"\xe1\xb9\xa3","\xe1\xb9\xa2"},{"\x74","\x54"},{"\xc5\xa5","\xc5\xa4"},{"\xc5\xa3","\xc5\xa2"},{"\xe1\xb9\xad","\xe1\xb9\xac"},{"\xe1\xb9\xaf","\xe1\xb9\xae"},{"\xc8\x95","\xc8\x94"},{"\x75","\x55"},{"\xc3\xba","\xc3\x9a"},{"\xc3\xb9","\xc3\x99"},{"\xc5\xad","\xc5\xac"},{"\xc3\xbb","\xc3\x9b"},{"\xc7\x94","\xc7\x93"},{"\xc5\xaf","\xc5\xae"},{"\xc7\x98","\xc7\x97"},{"\xc7\x9c","\xc7\x9b"},{"\xc3\xbc","\xc3\x9c"},{"\xc5\xb1","\xc5\xb0"},{"\xc5\xa9","\xc5\xa8"},{"\xc5\xb3","\xc5\xb2"},{"\xc5\xab","\xc5\xaa"},{"\xe1\xbb\xa7","\xe1\xbb\xa6"},{"\xe1\xbb\xa5","\xe1\xbb\xa4"},{"\xe1\xb9\xb3","\xe1\xb9\xb2"},{"\xe1\xbb\xa9","\xe1\xbb\xa8"},{"\xe1\xbb\xab","\xe1\xbb\xaa"},{"\xe1\xbb\xaf","\xe1\xbb\xae"},{"\xe1\xbb\xad","\xe1\xbb\xac"},{"\xe1\xbb\xb1","\xe1\xbb\xb0"},{"\xc6\xb0","\xc6\xaf"},{"\x76","\x56"},{"\x77","\x57"},{"\xc5\xb5","\xc5\xb4"},{"\x78","\x58"},{"\xe1\xba\x8b","\xe1\xba\x8a"},{"\x79","\x59"},{"\xc3\xbd","\xc3\x9d"},{"\xe1\xbb\xb3","\xe1\xbb\xb2"},{"\xc5\xb7","\xc5\xb6"},{"\xc3\xbf","\xc5\xb8"},{"\xe1\xbb\xb9","\xe1\xbb\xb8"},{"\x7a","\x5a"},{"\xc5\xba","\xc5\xb9"},{"\xc5\xbe","\xc5\xbd"},{"\xc5\xbc","\xc5\xbb"},{"\xc6\xb6","\xc6\xb5"},{"\xe1\xba\x93","\xe1\xba\x92"},{"\xe1\xba\x95","\xe1\xba\x94"},{"\xc8\xa5","\xc8\xa4"},{"\xc3\xbe","\xc3\x9e"},{"\xca\x92","\xc6\xb7"},{"\xce\xb1","\xce\x91"},{"\xce\xac","\xce\x86"},{"\xce\xb2","\xce\x92"},{"\xce\xb3","\xce\x93"},{"\xce\xb4","\xce\x94"},{"\xce\xb5","\xce\x95"},{"\xce\xad","\xce\x88"},{"\xce\xb6","\xce\x96"},{"\xce\xb7","\xce\x97"},{"\xce\xae","\xce\x89"},{"\xce\xb8","\xce\x98"},{"\xce\xb9","\xce\x99"},{"\xce\xaf","\xce\x8a"},{"\xcf\x8a","\xce\xaa"},{"\xce\xba","\xce\x9a"},{"\xce\xbb","\xce\x9b"},{"\xce\xbc","\xce\x9c"},{"\xce\xbd","\xce\x9d"},{"\xce\xbe","\xce\x9e"},{"\xce\xbf","\xce\x9f"},{"\xcf\x8c","\xce\x8c"},{"\xcf\x80","\xce\xa0"},{"\xcf\x83","\xce\xa3"},{"\xcf\x82","\xce\xa3"},{"\xcf\x84","\xce\xa4"},{"\xcf\x85","\xce\xa5"},{"\xcf\x8d","\xce\x8e"},{"\xcf\x8b","\xce\xab"},{"\xcf\x86","\xce\xa6"},{"\xcf\x87","\xce\xa7"},{"\xcf\x88","\xce\xa8"},{"\xcf\x89","\xce\xa9"},{"\xcf\x8e","\xce\x8f"},{"\xd0\xb0","\xd0\x90"},{"\xd3\x93","\xd3\x92"},{"\xd3\x95","\xd3\x94"},{"\xd0\xb1","\xd0\x91"},{"\xd0\xb2","\xd0\x92"},{"\xd0\xb3","\xd0\x93"},{"\xd2\x93","\xd2\x92"},{"\xd2\x91","\xd2\x90"},{"\xd0\xb4","\xd0\x94"},{"\xd1\x93","\xd0\x83"},{"\xd1\x92","\xd0\x82"},{"\xd0\xb5","\xd0\x95"},{"\xd1\x90","\xd0\x80"},{"\xd3\x99","\xd3\x98"},{"\xd1\x94","\xd0\x84"},{"\xd1\x91","\xd0\x81"},{"\xd0\xb6","\xd0\x96"},{"\xd0\xb7","\xd0\x97"},{"\xd2\x99","\xd2\x98"},{"\xd1\x95","\xd0\x85"},{"\xd0\xb8","\xd0\x98"},{"\xd3\xa3","\xd3\xa2"},{"\xd1\x96","\xd0\x86"},{"\xd1\x97","\xd0\x87"},{"\xd0\xb9","\xd0\x99"},{"\xd1\x98","\xd0\x88"},{"\xd0\xba","\xd0\x9a"},{"\xd2\x9b","\xd2\x9a"},{"\xd3\x84","\xd3\x83"},{"\xd2\xa1","\xd2\xa0"},{"\xd0\xbb","\xd0\x9b"},{"\xd1\x99","\xd0\x89"},{"\xd0\xbc","\xd0\x9c"},{"\xd0\xbd","\xd0\x9d"},{"\xd2\xa3","\xd2\xa2"},{"\xd1\x9a","\xd0\x8a"},{"\xd0\xbe","\xd0\x9e"},{"\xd3\xa7","\xd3\xa6"},{"\xd3\xa9","\xd3\xa8"},{"\xd0\xbf","\xd0\x9f"},{"\xd1\x80","\xd0\xa0"},{"\xd1\x81","\xd0\xa1"},{"\xd2\xab","\xd2\xaa"},{"\xd1\x82","\xd0\xa2"},{"\xd1\x9c","\xd0\x8c"},{"\xd1\x9b","\xd0\x8b"},{"\xd1\x83","\xd0\xa3"},{"\xd3\xb1","\xd3\xb0"},{"\xd2\xb1","\xd2\xb0"},{"\xd2\xaf","\xd2\xae"},{"\xd1\x9e","\xd0\x8e"},{"\xd1\x84","\xd0\xa4"},{"\xd1\x85","\xd0\xa5"},{"\xd2\xb3","\xd2\xb2"},{"\xd2\xbb","\xd2\xba"},{"\xd1\x86","\xd0\xa6"},{"\xd1\x87","\xd0\xa7"},{"\xd1\x9f","\xd0\x8f"},{"\xd1\x88","\xd0\xa8"},{"\xd1\x89","\xd0\xa9"},{"\xd1\x8a","\xd0\xaa"},{"\xd1\x8b","\xd0\xab"},{"\xd1\x8c","\xd0\xac"},{"\xd1\x8d","\xd0\xad"},{"\xd1\x8e","\xd0\xae"},{"\xd1\x8f","\xd0\xaf"},{"\xd5\xa1","\xd4\xb1"},{"\xd5\xa3","\xd4\xb3"},{"\xd5\xa5","\xd4\xb5"},{"\xd5\xab","\xd4\xbb"},{"\xd5\xac","\xd4\xbc"},{"\xd5\xb2","\xd5\x82"},{"\xd5\xb8","\xd5\x88"},{"\xd5\xbd","\xd5\x8d"},{"\xd5\xbe","\xd5\x8e"},{"\xd5\xbf","\xd5\x8f"},{"\xd6\x80","\xd5\x90"},{"\xd6\x81","\xd5\x91"} }; for (auto p8 : map8) { - auto from = utf8ToWString(p8.first); - auto to = utf8ToWString(p8.second); + auto from = utf8ToUnicodeString(p8.first); + auto to = utf8ToUnicodeString(p8.second); ABORT_IF(from.size() != 1 || to.size() != 1, "Incorrect character encoding??"); insert(std::make_pair(from.front(), to.front())); } } - wchar_t towupper(wchar_t c) const { + char32_t towupper(char32_t c) const { auto iter = find(c); if (iter == end()) return c; @@ -205,17 +224,18 @@ struct UTF8Mapper : std::map { // Hack because Philly does not return iter->second; } }; + std::string utf8ToUpper(const std::string& s) { static UTF8Mapper mapper; - auto ws = utf8ToWString(s); + auto ws = utf8ToUnicodeString(s); for (auto& c : ws) c = mapper.towupper(c); - return toUTF8String(ws); + return utf8FromUnicodeString(ws); } -#endif // convert an English sentence to title case // Since title case is an English thing, we only consider ASCII characters. +// @BUGBUG: This only works for untokenized text, and is therefore useless. std::string toEnglishTitleCase(const std::string& s) { auto res = s; // process token by token diff --git a/src/common/utils.h b/src/common/utils.h index 6226ed992..651e91717 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -41,6 +41,10 @@ bool endsWith(const std::string& text, const std::string& suffix); std::string utf8ToUpper(const std::string& s); std::string toEnglishTitleCase(const std::string& s); +std::u32string utf8ToUnicodeString(const std::string& s); +std::string utf8FromUnicodeString(const std::u32string& s); +bool isContinuousScript(char32_t c); + std::string findReplace(const std::string& in, const std::string& what, const std::string& withWhat, bool all = false); double parseDouble(std::string s); diff --git a/src/training/validator.cpp b/src/training/validator.cpp index f6b1c86a1..4ce5a60fb 100755 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -26,10 +26,13 @@ std::vector*/>> Validators( auto validator = New(vocabs, config); validators.push_back(validator); } else if(metric == "bleu") { - auto validator = New(vocabs, config, false); + auto validator = New(vocabs, config, BleuValidator::NoDetok); validators.push_back(validator); } else if(metric == "bleu-detok") { - auto validator = New(vocabs, config, true); + auto validator = New(vocabs, config, BleuValidator::DetokSacreBLEUWestern); + validators.push_back(validator); + } else if(metric == "bleu-detok-ms") { + auto validator = New(vocabs, config, BleuValidator::DetokMS); validators.push_back(validator); } else if(metric == "accuracy") { auto validator = New(vocabs, config); diff --git a/src/training/validator.h b/src/training/validator.h index f7f586f9d..c17b56070 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -446,6 +446,7 @@ class ScriptValidator : public Validator { } }; +// validator that translates and computes BLEU (or any metric) with an external script class TranslationValidator : public Validator { public: TranslationValidator(std::vector> vocabs, Ptr options) @@ -580,30 +581,26 @@ class TranslationValidator : public Validator { } }; +// validator that translates and computes BLEU internally, with or without decoding // @TODO: combine with TranslationValidator (above) to avoid code duplication class BleuValidator : public Validator { -private: - bool detok_{false}; - public: - BleuValidator(std::vector> vocabs, Ptr options, bool detok = false) + enum DetokType { + NoDetok, // tokens are raw tokens as used for decoding + DetokSacreBLEUWestern, // tokens are SacreBleu-compatible, assuming Western languages and no text normalization + DetokMS // MS-internal: tokens are DetokLatin1 except for continuous-script chars, which are counted as individual tokens + }; + + BleuValidator(std::vector> vocabs, Ptr options, DetokType detok = NoDetok) : Validator(vocabs, options, false), detok_(detok), quiet_(options_->get("quiet-translation")) { builder_ = models::createModelFromOptions(options_, models::usage::translation); -#ifdef USE_SENTENCEPIECE auto vocab = vocabs_.back(); - ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab", - "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab. " + ABORT_IF(detok_ != NoDetok && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", + "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " "Current vocabulary type is {}", vocab->type()); -#else - ABORT_IF(detok_, - "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab. " - "Marian has not been compiled with SentencePieceVocab support"); -#endif - - createBatchGenerator(/*isTranslating=*/true); } virtual float validate(const std::vector>& graphs) override { @@ -713,13 +710,19 @@ class BleuValidator : public Validator { return val; }; - std::string type() override { return detok_ ? "bleu-detok" : "bleu"; } + // @TODO: why do we return this string, but not pass it to the constructor? + std::string type() override { + switch (detok_) { + case NoDetok: return "bleu"; + case DetokSacreBLEUWestern: return "bleu-detok"; + case DetokMS: return "bleu-detok-ms"; + default: ABORT("Unexpected DetokType??"); + } + } protected: - bool quiet_{false}; - // Tokenizer function adapted from multi-bleu-detok.pl, corresponds to sacreBLEU.py - std::string tokenize(const std::string& text) { + static std::string tokenizeSacreBLEUWestern(const std::string& text) { std::string normText = text; // language-independent part: @@ -743,10 +746,34 @@ class BleuValidator : public Validator { return normText; } +public: + static std::string tokenizeContinuousScript(const std::string& sUTF8) { + // We want BLEU-like scores that are comparable across different tokenization schemes. + // For continuous scripts )Chinese, Japanese, Thai), we would need a language-specific + // statistical word segmenter, which is outside the scope of Marian. As a practical + // compromise, we segment continuous-script sequences into individual characters, while + // leaving Western scripts as words. This way we can use the same settings for Western + // languages, where Marian would report SacreBLEU scores, and Asian languages, where + // scores are not standard but internally comparable across tokenization schemes. + auto in = utils::utf8ToUnicodeString(sUTF8); + auto out = in.substr(0, 0); // (out should be same type as in, don't want to bother with exact type) + for (auto c : in) { + auto isCS = utils::isContinuousScript(c); + if (isCS) // surround continuous-script chars by spaces on each side + out.push_back(' '); // (duplicate spaces are ignored when splitting later) + out.push_back(c); + if (isCS) + out.push_back(' '); + } + return utils::utf8FromUnicodeString(out); + } std::vector decode(const Words& words, bool addEOS = false) { auto vocab = vocabs_.back(); - auto tokens = utils::splitAny(tokenize(vocab->decode(words)), " "); + auto tokenString = tokenizeSacreBLEUWestern(vocab->decode(words)); + if (detok_ == DetokMS) // score continuous-script sequences as individual characters + tokenString = tokenizeContinuousScript(tokenString); + auto tokens = utils::splitAny(tokenString, " "); if(addEOS) tokens.push_back(""); return tokens; @@ -834,6 +861,10 @@ class BleuValidator : public Validator { virtual float validateBG(const std::vector>& /*graphs*/) override { return 0; } + +private: + DetokType detok_; + bool quiet_{ false }; }; /** From 5a10abd5b7b1dbcbcf363f636e5448019ef2057d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 27 Mar 2019 10:25:53 -0700 Subject: [PATCH 390/838] temporarily interpreting 'bleu' validator as 'bleu-detok-ms'. so that it is used without changing Flo --- src/training/validator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/validator.cpp b/src/training/validator.cpp index 4ce5a60fb..ed32964a7 100755 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -26,7 +26,7 @@ std::vector*/>> Validators( auto validator = New(vocabs, config); validators.push_back(validator); } else if(metric == "bleu") { - auto validator = New(vocabs, config, BleuValidator::NoDetok); + auto validator = New(vocabs, config, BleuValidator::DetokMS); // TEMPORARILY until Flo is updated. Then change back to NoDetok validators.push_back(validator); } else if(metric == "bleu-detok") { auto validator = New(vocabs, config, BleuValidator::DetokSacreBLEUWestern); From 4895cdb28fb0dbdedce25622ec3060a930ac0c75 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 27 Mar 2019 10:28:01 -0700 Subject: [PATCH 391/838] (minor fix) --- src/training/validator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/validator.h b/src/training/validator.h index c17b56070..155a3721b 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -838,7 +838,7 @@ class BleuValidator : public Validator { ref.push_back(w); } - if(detok_) + if(detok_ != NoDetok) updateStats(stats, decode(cand, /*addEOS=*/ true), decode(ref)); else updateStats(stats, cand, ref); From a9e2447a2d7ed1f0c9cf208c4ed1ecfa5fddc3d4 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Wed, 27 Mar 2019 13:27:25 -0700 Subject: [PATCH 392/838] write *.bin suffix whenever a model is saved and --model *.bin is specified --- src/training/graph_group_sync.cpp | 23 +++++++++++++++++------ src/training/validator.h | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index ba5751691..ba451a22f 100755 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -75,7 +75,10 @@ void SyncGraphGroup::initialize(const Ptr& exampleBatch) { void SyncGraphGroup::initializeAvg() { Ptr graphAvg; // CPU-side temp std::string name = options_->get("model"); - if(filesystem::exists(name + ".orig.npz")) { + std::string suffix = name.substr(name.size() - 4); + ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix); + + if(filesystem::exists(name + ".orig" + suffix)) { // Load the averaged parameters into a temporary graph graphAvg = New(); graphAvg->setDevice({0, DeviceType::cpu}); @@ -436,9 +439,12 @@ void SyncGraphGroup::load() /*override*/ { scheduler_->load(name); std::string nameGraph = name; - if(mvAvg_ && filesystem::exists(name + ".orig.npz")) + std::string suffix = name.substr(name.size() - 4); + ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix); + + if(mvAvg_ && filesystem::exists(name + ".orig" + suffix)) // Load the original parameters from model.npz.orig.npz - nameGraph += ".orig.npz"; + nameGraph += ".orig" + suffix; size_t i = 0; for(auto graph : graphs_) @@ -448,7 +454,7 @@ void SyncGraphGroup::load() /*override*/ { std::vector> backends; for(auto graph : graphs_) backends.push_back(graph->getBackend()); - shardOpt_[0]->load(name + ".optimizer.npz", shardOpt_, backends, + shardOpt_[0]->load(name + ".optimizer.npz", shardOpt_, backends, // keep npz suffix for optimize checkpoint [&](const std::vector& optimizerStateVector, const OptimizerBase::ScatterStateSetFunc& setShardFn) { comm_->scatterState(optimizerStateVector, setShardFn); }); @@ -480,13 +486,18 @@ void SyncGraphGroup::save(bool final) /*override*/ { swapParamsAvg(); } + // @TODO: put all this in one place, in new branch this is already localized in one place and class, this is a quick hack which will be + // done better after the next merge. Not doing this in other graph_groups as this would only make the merge harder. + // Determine model suffix *.npz or *.bin, then use the same suffix for all following models saved. std::string name = options_->get("model"); + std::string suffix = name.substr(name.size() - 4); + ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix); barrier(); // (for better grouping of log messages) // if smoothing then save original (unsmoothed) parameters as well if(mvAvg_ && paramsAvg_.size() > 0 && isMainProcess()) // only save from one MPI process // Save the original parameters in model.npz.orig.npz - builders_[0]->save(graphs_[0], name + ".orig.npz", true); + builders_[0]->save(graphs_[0], name + ".orig" + suffix, true); // Temporarily switch to the averaged parameters // Note: the smoothed model is sharded across GPUs, and across MPI processes if applicable. This brings it into MPI process[*].device[*] @@ -500,7 +511,7 @@ void SyncGraphGroup::save(bool final) /*override*/ { = scheduler_ ? std::to_string(scheduler_->numberOfBatches()) : "unknown"; std::string nameOverwrite = name; - nameOverwrite.replace(name.size() - 4, 4, ".iter" + numberOfBatches + ".npz"); // @TODO: use insert? + nameOverwrite.replace(name.size() - 4, 4, ".iter" + numberOfBatches + suffix); // @TODO: use insert? builders_[0]->save(graphs_[0], nameOverwrite); } // save main model file diff --git a/src/training/validator.h b/src/training/validator.h index 46c58825d..582c232f6 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -144,7 +144,10 @@ class Validator : public ValidatorBase { virtual void keepBest(const std::vector>& graphs) { auto model = options_->get("model"); - builder_->save(graphs[0], model + ".best-" + type() + ".npz", true); + std::string suffix = model.substr(model.size() - 4); + ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix); + + builder_->save(graphs[0], model + ".best-" + type() + suffix, true); } }; @@ -428,7 +431,10 @@ class ScriptValidator : public Validator { virtual float validate(const std::vector>& graphs) override { using namespace data; auto model = options_->get("model"); - builder_->save(graphs[0], model + ".dev.npz", true); + std::string suffix = model.substr(model.size() - 4); + ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix); + + builder_->save(graphs[0], model + ".dev" + suffix, true); auto command = options_->get("valid-script-path"); auto valStr = utils::exec(command); From ff0bd5f9396cdedae9a4913c89c4e8e1fd94d202 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 28 Mar 2019 17:13:25 -0700 Subject: [PATCH 393/838] bug fix: _GNUC_ should be __GNUC__; SentencePieceVocab now supports all-caps and title-casing; bleu-detok now supports FactoredVocab via the new IVocab::surfaceForm() method --- src/common/definitions.h | 2 +- src/common/timer.h | 4 ++-- src/common/utils.cpp | 33 ++++++++++++++++++--------- src/common/utils.h | 2 ++ src/data/default_vocab.cpp | 5 ++++ src/data/factored_vocab.cpp | 39 +++++++++++++++++++++++++++++++- src/data/factored_vocab.h | 2 ++ src/data/sentencepiece_vocab.cpp | 7 ++++++ src/data/vocab.cpp | 9 +++++++- src/data/vocab.h | 6 ++++- src/data/vocab_base.h | 7 ++++-- src/training/scheduler.h | 0 src/training/validator.cpp | 7 ++---- src/training/validator.h | 32 ++++++++------------------ 14 files changed, 109 insertions(+), 46 deletions(-) mode change 100644 => 100755 src/common/timer.h mode change 100644 => 100755 src/training/scheduler.h diff --git a/src/common/definitions.h b/src/common/definitions.h index 393a8b7fe..3fdf6659f 100755 --- a/src/common/definitions.h +++ b/src/common/definitions.h @@ -14,7 +14,7 @@ #define NodeOp(op) [=]() { op; } // helper macro to disable optimization (gcc only) -#ifdef _GNUC_ +#ifdef __GNUC__ #define DONT_OPTIMIZE __attribute__((optimize("O0"))) #else #define DONT_OPTIMIZE // silently ignore on Visual Studio, where this is less of a problem diff --git a/src/common/timer.h b/src/common/timer.h old mode 100644 new mode 100755 index dfb91bf01..fd9307dfc --- a/src/common/timer.h +++ b/src/common/timer.h @@ -1,11 +1,11 @@ #pragma once -#ifdef _GNUC_ +#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsuggest-override" #endif #include -#ifdef _GNUC_ +#ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 958dfc77a..aacc0c602 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -193,8 +193,9 @@ bool isContinuousScript(char32_t c) { return isHan || isKana || isThai; } -// convert a UTF-8 string to all-caps -struct UTF8Mapper : std::map { // Hack because MS-internal Philly servers do not have UTF-8 locale installed +// convert UTF-8 characters to lower or upper case +struct UTF8Mapper { // can't use the standard lib functions because MS-internal Philly servers do not have UTF-8 locale installed + std::map toUpperMap, toLowerMap; UTF8Mapper() { /* env LC_ALL=en_US.UTF-8 sed 's/\(.\)/\1\n/g' TEXT_FILE_CONTAINING_ALL_CHARS > l @@ -213,29 +214,39 @@ struct UTF8Mapper : std::map { // Hack because MS-internal P auto from = utf8ToUnicodeString(p8.first); auto to = utf8ToUnicodeString(p8.second); ABORT_IF(from.size() != 1 || to.size() != 1, "Incorrect character encoding??"); - insert(std::make_pair(from.front(), to.front())); + toUpperMap.insert(std::make_pair(from.front(), to.front())); + toLowerMap.insert(std::make_pair(to.front(), from.front())); } } - char32_t towupper(char32_t c) const { - auto iter = find(c); - if (iter == end()) + char32_t toUpperOrLower(char32_t c, bool toLower) const { return mapChar(toLower ? toLowerMap : toUpperMap, c); } +private: + static char32_t mapChar(const std::map& map, char32_t c) { + auto iter = map.find(c); + if (iter == map.end()) return c; else return iter->second; } }; -std::string utf8ToUpper(const std::string& s) { - static UTF8Mapper mapper; +// shared implementation of toUpper, toLower, and toCapitalized +static std::string utf8ToUpperOrLower(const std::string& s, bool toLower, bool toInitCap) { + static UTF8Mapper utf8Mapper; auto ws = utf8ToUnicodeString(s); - for (auto& c : ws) - c = mapper.towupper(c); + for (auto& c : ws) { + c = utf8Mapper.toUpperOrLower(c, toLower); + if (toInitCap) + toLower = true; + } return utf8FromUnicodeString(ws); } +std::string utf8ToUpper(const std::string& s) { return utf8ToUpperOrLower(s, /*toLower=*/false, /*toInitCap=*/false); } +std::string utf8ToLower(const std::string& s) { return utf8ToUpperOrLower(s, /*toLower=*/true , /*toInitCap=*/false); } +std::string utf8Capitalized(const std::string& s) { return utf8ToUpperOrLower(s, /*toLower=*/false, /*toInitCap=*/true ); } + // convert an English sentence to title case // Since title case is an English thing, we only consider ASCII characters. -// @BUGBUG: This only works for untokenized text, and is therefore useless. std::string toEnglishTitleCase(const std::string& s) { auto res = s; // process token by token diff --git a/src/common/utils.h b/src/common/utils.h index 651e91717..50ed8a580 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -39,6 +39,8 @@ bool beginsWith(const std::string& text, const std::string& prefix); bool endsWith(const std::string& text, const std::string& suffix); std::string utf8ToUpper(const std::string& s); +std::string utf8ToLower(const std::string& s); +std::string utf8Capitalized(const std::string& word); // capitalize the first character only std::string toEnglishTitleCase(const std::string& s); std::u32string utf8ToUnicodeString(const std::string& s); diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index 25aaff95f..b40c0fd92 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -44,6 +44,7 @@ class DefaultVocab : public IVocab { }; public: + // @TODO: choose between 'virtual' and 'final'. Can we derive from this class? virtual const std::string& canonicalExtension() const override { return suffixes_[0]; } virtual const std::vector& suffixes() const override { return suffixes_; } @@ -65,6 +66,10 @@ class DefaultVocab : public IVocab { return utils::join(tokens, " "); } + std::string surfaceForm(const Words& sentence) const override { + ABORT("surfaceForm() not supported by this vocabulary type"); + } + virtual std::string type() const override { return "DefaultVocab"; } virtual Word getEosId() const override { return eosId_; } diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 69172edea..6b1906c88 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -435,7 +435,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return Word::fromWordIndex(index); else //ABORT("Unknown word {} mapped to {}", word, word2string(getUnkId())); - LOG(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); + LOG_ONCE(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); return getUnkId(); } @@ -501,6 +501,43 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return utils::join(decoded, " "); } +// interpret the capitalization and glue factors +// This assumes a specific notation of factors, emulating our C# code for generating these factors: +// - | as separator symbol +// - capitalization factors are cn, ci, and ca +// - glue factors are gl+, gr+, wbn, wen, cbn, cen +std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override final*/ { + std::string res; + res.reserve(sentence.size() * 10); + bool prevHadGlueRight = true; // no space at sentence start + for(auto w : sentence) { + auto token = (*this)[w]; + if (w == getEosId()) + break; + auto tokens = utils::split(token, "|"); + //std::cerr << token << " "; + const auto& lemma = tokens[0]; + std::set tokenSet(tokens.begin() + 1, tokens.end()); + auto has = [&](const char* factor) { return tokenSet.find(factor) != tokenSet.end(); }; + // spacing + bool hasGlueRight = has("gr+") || has("wen") || has("cen"); + bool hasGlueLeft = has("gl+") || has("wbn") || has("cbn"); + bool insertSpaceBefore = !prevHadGlueRight && !hasGlueLeft; + if (insertSpaceBefore) + res.push_back(' '); + prevHadGlueRight = hasGlueRight; + // capitalization + std::string surfaceForm; + if (has("ci")) surfaceForm = utils::utf8Capitalized(lemma); + else if (has("ca")) surfaceForm = utils::utf8ToUpper (lemma); + else if (has("cn")) surfaceForm = utils::utf8ToLower (lemma); + else surfaceForm = lemma ; + res.append(surfaceForm); + } + //std::cerr << "\n" << res << "\n"; + return res; +} + // create a CSR matrix M[V,U] from words[] with // M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 220979e81..d6bb8a21f 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -23,6 +23,7 @@ class FactoredVocab : public IVocab { }; // from IVocab: + // @TODO: Why are these virtual and final at the same time? Seems we should remove all the 'virtual' here virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } @@ -30,6 +31,7 @@ class FactoredVocab : public IVocab { virtual Word operator[](const std::string& word) const override final; virtual Words encode(const std::string& line, bool addEOS = true, bool inference = false) const override final; virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; + virtual std::string surfaceForm(const Words& sentence) const override final; virtual const std::string& operator[](Word id) const override final; virtual size_t size() const override final; virtual std::string type() const override final { return "FactoredVocab"; } diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp index b2b829019..0e8b48034 100755 --- a/src/data/sentencepiece_vocab.cpp +++ b/src/data/sentencepiece_vocab.cpp @@ -231,6 +231,11 @@ class SentencePieceVocab : public IVocab { return line; } + std::string surfaceForm(const Words& sentence) const override { + // with SentencePiece, decoded form and surface form are identical + return decode(sentence, /*ignoreEOS=*/true); + } + size_t size() const override { return spm_->GetPieceSize(); } @@ -252,6 +257,8 @@ class SentencePieceVocab : public IVocab { return spm_->GetPieceSize(); } + std::string toUpper(const std::string& line) const override { return utils::utf8ToUpper(line); } + std::string toEnglishTitleCase(const std::string& line) const override { return utils::toEnglishTitleCase(line); } }; #endif // USE_SENTENCEPIECE diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 9c1039d81..2cafb3313 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -112,12 +112,19 @@ Words Vocab::encode(const std::string& line, return vImpl_->encode(line, addEOS, inference); } -// list of token ids to single line, can perform detokenization +// convert sequence of token ids to single line, can perform detokenization std::string Vocab::decode(const Words& sentence, bool ignoreEOS) const { return vImpl_->decode(sentence, ignoreEOS); } +// convert sequence of token its to surface form (incl. removng spaces, applying factors) +// for in-process BLEU validation +std::string Vocab::surfaceForm(const Words& sentence) const { + return vImpl_->surfaceForm(sentence); +} + + // number of vocabulary items size_t Vocab::size() const { return vImpl_->size(); } diff --git a/src/data/vocab.h b/src/data/vocab.h index 81f97a618..1de93f5b4 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -50,10 +50,14 @@ class Vocab { bool addEOS = true, bool inference = false) const; - // list of token ids to single line, can perform detokenization + // convert sequence of token ids to single line, can perform detokenization std::string decode(const Words& sentence, bool ignoreEOS = true) const; + // convert sequence of token its to surface form (incl. removng spaces, applying factors) + // for in-process BLEU validation + std::string surfaceForm(const Words& sentence) const; + // number of vocabulary items size_t size() const; diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index 240626655..bf276f5de 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -34,6 +34,7 @@ class IVocab { virtual std::string decode(const Words& sentence, bool ignoreEos = true) const = 0; + virtual std::string surfaceForm(const Words& sentence) const = 0; virtual const std::string& operator[](Word id) const = 0; @@ -43,8 +44,10 @@ class IVocab { virtual Word getEosId() const = 0; virtual Word getUnkId() const = 0; - virtual std::string toUpper(const std::string& line) const { return utils::utf8ToUpper(line); } - virtual std::string toEnglishTitleCase(const std::string& line) const { return utils::toEnglishTitleCase(line); } + // without specific knowledge of tokenization, these two functions can do nothing + // Both SentencePieceVocab and FactoredSegmenterVocab + virtual std::string toUpper(const std::string& line) const { return line; } + virtual std::string toEnglishTitleCase(const std::string& line) const { return line; } virtual void createFake() = 0; diff --git a/src/training/scheduler.h b/src/training/scheduler.h old mode 100644 new mode 100755 diff --git a/src/training/validator.cpp b/src/training/validator.cpp index ed32964a7..f6b1c86a1 100755 --- a/src/training/validator.cpp +++ b/src/training/validator.cpp @@ -26,13 +26,10 @@ std::vector*/>> Validators( auto validator = New(vocabs, config); validators.push_back(validator); } else if(metric == "bleu") { - auto validator = New(vocabs, config, BleuValidator::DetokMS); // TEMPORARILY until Flo is updated. Then change back to NoDetok + auto validator = New(vocabs, config, false); validators.push_back(validator); } else if(metric == "bleu-detok") { - auto validator = New(vocabs, config, BleuValidator::DetokSacreBLEUWestern); - validators.push_back(validator); - } else if(metric == "bleu-detok-ms") { - auto validator = New(vocabs, config, BleuValidator::DetokMS); + auto validator = New(vocabs, config, true); validators.push_back(validator); } else if(metric == "accuracy") { auto validator = New(vocabs, config); diff --git a/src/training/validator.h b/src/training/validator.h index 155a3721b..37a583b32 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -585,22 +585,18 @@ class TranslationValidator : public Validator { // @TODO: combine with TranslationValidator (above) to avoid code duplication class BleuValidator : public Validator { public: - enum DetokType { - NoDetok, // tokens are raw tokens as used for decoding - DetokSacreBLEUWestern, // tokens are SacreBleu-compatible, assuming Western languages and no text normalization - DetokMS // MS-internal: tokens are DetokLatin1 except for continuous-script chars, which are counted as individual tokens - }; - - BleuValidator(std::vector> vocabs, Ptr options, DetokType detok = NoDetok) + BleuValidator(std::vector> vocabs, Ptr options, bool detok = false) : Validator(vocabs, options, false), detok_(detok), quiet_(options_->get("quiet-translation")) { builder_ = models::createModelFromOptions(options_, models::usage::translation); auto vocab = vocabs_.back(); - ABORT_IF(detok_ != NoDetok && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", + ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " "Current vocabulary type is {}", vocab->type()); + + createBatchGenerator(/*isTranslating=*/true); } virtual float validate(const std::vector>& graphs) override { @@ -711,18 +707,11 @@ class BleuValidator : public Validator { }; // @TODO: why do we return this string, but not pass it to the constructor? - std::string type() override { - switch (detok_) { - case NoDetok: return "bleu"; - case DetokSacreBLEUWestern: return "bleu-detok"; - case DetokMS: return "bleu-detok-ms"; - default: ABORT("Unexpected DetokType??"); - } - } + std::string type() override { return detok_ ? "bleu-detok" : "bleu"; } protected: // Tokenizer function adapted from multi-bleu-detok.pl, corresponds to sacreBLEU.py - static std::string tokenizeSacreBLEUWestern(const std::string& text) { + static std::string tokenize(const std::string& text) { std::string normText = text; // language-independent part: @@ -770,9 +759,8 @@ class BleuValidator : public Validator { std::vector decode(const Words& words, bool addEOS = false) { auto vocab = vocabs_.back(); - auto tokenString = tokenizeSacreBLEUWestern(vocab->decode(words)); - if (detok_ == DetokMS) // score continuous-script sequences as individual characters - tokenString = tokenizeContinuousScript(tokenString); + auto tokenString = tokenize(vocab->surfaceForm(words)); + tokenString = tokenizeContinuousScript(tokenString); auto tokens = utils::splitAny(tokenString, " "); if(addEOS) tokens.push_back(""); @@ -838,7 +826,7 @@ class BleuValidator : public Validator { ref.push_back(w); } - if(detok_ != NoDetok) + if(detok_) updateStats(stats, decode(cand, /*addEOS=*/ true), decode(ref)); else updateStats(stats, cand, ref); @@ -863,7 +851,7 @@ class BleuValidator : public Validator { } private: - DetokType detok_; + bool detok_; bool quiet_{ false }; }; From aece1e3f91a23291249658c05e961a2b4f65dbed Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 28 Mar 2019 18:55:07 -0700 Subject: [PATCH 394/838] bug fix: surfaceForm() should also unescape \x and \u sequences --- src/common/utils.cpp | 22 ++++++++++++++++++++++ src/common/utils.h | 2 ++ src/data/default_vocab.cpp | 1 + src/data/factored_vocab.cpp | 35 +++++++++++++++++++++++++++-------- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index aacc0c602..3fd0bdead 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -177,6 +177,28 @@ std::string utf8FromUnicodeString(const std::u32string& s) { #endif } +std::u16string utf8ToUtf16String(std::string const& s) { +#ifdef _MSC_VER // workaround for a known bug in VS CRT + std::wstring_convert, wchar_t/*char16_t*/> converter; + auto res = converter.from_bytes(s); + return std::u16string(res.begin(), res.end()); +#else + std::wstring_convert, char16_t> converter; + return converter.from_bytes(s); +#endif +} + +std::string utf8FromUtf16String(const std::u16string& s) { +#ifdef _MSC_VER // workaround for a known bug in VS CRT + std::wstring_convert, wchar_t/*char16_t*/> converter; + std::basic_string si(s.begin(), s.end()); + return converter.to_bytes(si); +#else + std::wstring_convert, char16_t> converter; + return converter.to_bytes(s); +#endif +} + bool isContinuousScript(char32_t c) { // currently, this table is hand-coded, and may need to be extended when the standard grows auto in = [c](char32_t minVal, char32_t maxVal) { return c >= minVal && c <= maxVal; }; diff --git a/src/common/utils.h b/src/common/utils.h index 50ed8a580..c3266bbf4 100755 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -45,6 +45,8 @@ std::string toEnglishTitleCase(const std::string& s); std::u32string utf8ToUnicodeString(const std::string& s); std::string utf8FromUnicodeString(const std::u32string& s); +std::u16string utf8ToUtf16String(const std::string& s); +std::string utf8FromUtf16String(const std::u16string& s); bool isContinuousScript(char32_t c); std::string findReplace(const std::string& in, const std::string& what, const std::string& withWhat, bool all = false); diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp index b40c0fd92..60bd8b10f 100755 --- a/src/data/default_vocab.cpp +++ b/src/data/default_vocab.cpp @@ -67,6 +67,7 @@ class DefaultVocab : public IVocab { } std::string surfaceForm(const Words& sentence) const override { + sentence; ABORT("surfaceForm() not supported by this vocabulary type"); } diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 6b1906c88..04f625389 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -501,6 +501,25 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return utils::join(decoded, " "); } +// helper to unescape \x.. and \u.... +static void unescapeHexEscapes(std::string& utf8Lemma) { + if (utf8Lemma.find('\\') == std::string::npos) + return; // nothing to do + auto lemma = utils::utf8ToUtf16String(utf8Lemma); // \u.... implies we must operate on UTF-16 level + auto pos = lemma.find('\\'); + while (pos != std::string::npos) { + ABORT_IF(pos + 1 >= lemma.size() || (lemma[pos+1] != 'x' && lemma[pos + 1] != 'u'), "Malformed escape in factored encoding: {}", utf8Lemma); + int numDigits = 2 + (lemma[pos + 1] == 'u'); + ABORT_IF(pos + 2 + numDigits > lemma.size(), "Malformed escape in factored encoding: {}", utf8Lemma); + auto digits = utils::utf8FromUtf16String(lemma.substr(pos + 2, numDigits)); + auto c = std::strtoul(digits.c_str(), nullptr, 16); + lemma[pos] = (char16_t)c; + lemma.erase(pos + 1, 1 + numDigits); + pos = lemma.find('\\', pos+1); + } + utf8Lemma = utils::utf8FromUtf16String(lemma); +} + // interpret the capitalization and glue factors // This assumes a specific notation of factors, emulating our C# code for generating these factors: // - | as separator symbol @@ -511,12 +530,12 @@ std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override f res.reserve(sentence.size() * 10); bool prevHadGlueRight = true; // no space at sentence start for(auto w : sentence) { - auto token = (*this)[w]; if (w == getEosId()) break; + auto token = (*this)[w]; auto tokens = utils::split(token, "|"); //std::cerr << token << " "; - const auto& lemma = tokens[0]; + auto lemma = tokens[0]; std::set tokenSet(tokens.begin() + 1, tokens.end()); auto has = [&](const char* factor) { return tokenSet.find(factor) != tokenSet.end(); }; // spacing @@ -527,12 +546,12 @@ std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override f res.push_back(' '); prevHadGlueRight = hasGlueRight; // capitalization - std::string surfaceForm; - if (has("ci")) surfaceForm = utils::utf8Capitalized(lemma); - else if (has("ca")) surfaceForm = utils::utf8ToUpper (lemma); - else if (has("cn")) surfaceForm = utils::utf8ToLower (lemma); - else surfaceForm = lemma ; - res.append(surfaceForm); + unescapeHexEscapes(lemma); // unescape \x.. and \u.... + if (has("ci")) lemma = utils::utf8Capitalized(lemma); + else if (has("ca")) lemma = utils::utf8ToUpper (lemma); + else if (has("cn")) lemma = utils::utf8ToLower (lemma); + else lemma = lemma ; + res.append(lemma); } //std::cerr << "\n" << res << "\n"; return res; From db3a73212eefaa0e261505f1a10500098878f378 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 28 Mar 2019 20:09:26 -0700 Subject: [PATCH 395/838] temporarily setting detok_ to true for FactoredSegmenter vocabs --- src/training/validator.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/training/validator.h b/src/training/validator.h index 37a583b32..e06f4b57e 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -592,6 +592,10 @@ class BleuValidator : public Validator { builder_ = models::createModelFromOptions(options_, models::usage::translation); auto vocab = vocabs_.back(); +#if 1 // hack for now, to get this feature when running under Flo + if (vocab->type() == "FactoredVocab") + detok_ = true; // always use bleu-detok +#endif ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " "Current vocabulary type is {}", vocab->type()); From 9f3337b1023ac84e8070ec2917ccebe50493b2d9 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Mon, 1 Apr 2019 17:02:46 +0100 Subject: [PATCH 396/838] Added explanatory comment on validate(). --- src/training/graph_group_sync.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp index c02a34afa..d6a1c3217 100644 --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -527,6 +527,10 @@ void SyncGraphGroup::save(bool final) /*override*/ { void SyncGraphGroup::finalize() /*override*/ { validate(); + // Note to the uninitiated: validate() has nothing to do with + // validation on the validation set. It's just a sanity check. [UG] + // @TODO: rename validate() to avoid confusion with model + // validation on withheld data. [UG] Base::finalize(); } From 3e6fb9921db0fcf95f23fd5415f73b7cbff08105 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 5 Apr 2019 12:44:10 -0700 Subject: [PATCH 397/838] bug fix: GumbelSoftmaxStep should not apply sampling to factors --- src/layers/generic.cpp | 10 ++++++++++ src/layers/generic.h | 1 + src/models/costs.h | 11 +++++------ src/training/validator.h | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 2f302ef12..5bed72ec7 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -191,6 +191,16 @@ namespace marian { return Logits(std::move(newLogits), factoredVocab_); } + Logits Logits::applyUnaryFunctions(const std::function& f1, const std::function& fother) const { + std::vector> newLogits; + bool first = true; + for (const auto& l : logits_) { + newLogits.emplace_back(New((first?f1:fother)(l->loss()), l->count())); // f1 for first, fother for all others + first = false; + } + return Logits(std::move(newLogits), factoredVocab_); + } + // @TODO: code dup with above; we can merge it into applyToRationalLoss() Logits Logits::withCounts(const Expr& count) const { // create new Logits with 'count' implanted into all logits_ std::vector> newLogits; diff --git a/src/layers/generic.h b/src/layers/generic.h index 40b8bb61a..01c1c1721 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -85,6 +85,7 @@ class Logits { //Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; Logits applyUnaryFunction(const std::function& f) const; // clone this but apply f to all loss values + Logits applyUnaryFunctions(const std::function& f1, const std::function& fother) const; // clone this but apply f1 to first and fother to to all other values struct MaskedFactorIndices { std::vector indices; // factor index, or 0 if masked diff --git a/src/models/costs.h b/src/models/costs.h index c0d7ebf03..f56f6e05f 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -224,12 +224,11 @@ class LogSoftmaxStep : public ILogProbStep { class GumbelSoftmaxStep : public ILogProbStep { public: virtual Ptr apply(Ptr state) override { - // @TODO: @HACK must know about individual parts; make it a loop - state->setLogProbs(state->getLogProbs().applyUnaryFunction([](Expr logits){ - //auto logits = state->getLogProbs().getLogits(); - return logsoftmax(logits + constant_like(logits, inits::gumbel)); - })); - //state->setLogProbs(logprobs); + state->setLogProbs(state->getLogProbs().applyUnaryFunctions( + [](Expr logits){ // lemma gets gumbelled + return logsoftmax(logits + constant_like(logits, inits::gumbel)); + }, + logsoftmax)); // factors don't return state; } }; diff --git a/src/training/validator.h b/src/training/validator.h index e06f4b57e..a01f2fb17 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -594,7 +594,7 @@ class BleuValidator : public Validator { auto vocab = vocabs_.back(); #if 1 // hack for now, to get this feature when running under Flo if (vocab->type() == "FactoredVocab") - detok_ = true; // always use bleu-detok + detok_ = true; // always use bleu-detok #endif ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " From 5f9c7c00a316acc60beacf9bc7faa3913a329ead Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 11 Apr 2019 20:47:08 -0700 Subject: [PATCH 398/838] extended quicksand API to use our native vocabulary class --- src/3rd_party/pathie-cpp/src/path.cpp | 5 ++- src/microsoft/quicksand.cpp | 54 +++++++++++++++++++++++---- src/microsoft/quicksand.h | 27 ++++++++++---- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp index 12aa02703..6d9e8b63a 100755 --- a/src/3rd_party/pathie-cpp/src/path.cpp +++ b/src/3rd_party/pathie-cpp/src/path.cpp @@ -149,6 +149,8 @@ Path::Path(const std::vector& components) */ void Path::sanitize() { + bool isWindowsUNCPath = m_path.c_str()[0] == '\\' && m_path.c_str()[1] == '\\'; // UNC path + // Replace any backslashes \ with forward slashes /. size_t cur = string::npos; while ((cur = m_path.find("\\")) != string::npos) { // assignment intended @@ -156,8 +158,9 @@ void Path::sanitize() } // Replace all double slashes // with a single one + // [fseide] except for the first position, which would be a Windows UNC path cur = string::npos; - while ((cur = m_path.find("//")) != string::npos) { // assignment intended + while ((cur = m_path.find("//", isWindowsUNCPath ? 1 : 0)) != string::npos) { // assignment intended m_path.replace(cur, 2, "/"); } diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 2d932f0b7..a213efe12 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -9,6 +9,7 @@ #include "translator/beam_search.h" #include "translator/scorers.h" #include "data/alignment.h" +#include "data/vocab_base.h" namespace marian { @@ -31,6 +32,20 @@ Ptr newOptions() { return New(); } +class VocabWrapper : public IVocabWrapper { + Ptr pImpl_; +public: + VocabWrapper(Ptr vocab) : pImpl_(vocab) {} + WordIndex encode(const std::string& word) const { + auto enc = pImpl_->encode(word, /*addEOS=*/false, /*inference=*/true); + ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); + return enc[0].toWordIndex(); + } + size_t size() const { return pImpl_->size(); } + std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } + Ptr getVocab() const { return pImpl_; } +}; + class BeamSearchDecoder : public IBeamSearchDecoder { private: Ptr graph_; @@ -38,12 +53,19 @@ class BeamSearchDecoder : public IBeamSearchDecoder { std::vector> scorers_; + std::vector> vocabs_; + public: BeamSearchDecoder(Ptr options, const std::vector& ptrs, - Word eos) + const std::vector>& vocabs, + WordIndex eos) : IBeamSearchDecoder(options, ptrs, eos) { + // copy the vocabs + for (auto vi : vocabs) + vocabs_.push_back(std::dynamic_pointer_cast(vi)->getVocab()); + // setting 16-bit optimization to false for now. Re-enable with better caching or pre-computation graph_ = New(/*inference=*/true, /*optimize=*/false); @@ -94,7 +116,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { QSNBestBatch decode(const QSBatch& qsBatch, size_t maxLength, - const std::unordered_set& shortlist) override { + const std::unordered_set& shortlist) override { if(shortlist.size() > 0) { auto shortListGen = New(shortlist); for(auto scorer : scorers_) @@ -103,7 +125,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { // form source batch, by interleaving the words over sentences in the batch, and setting the mask size_t batchSize = qsBatch.size(); - auto subBatch = New(batchSize, maxLength, nullptr); + auto subBatch = New(batchSize, maxLength, vocabs_[0]); for(size_t i = 0; i < maxLength; ++i) { for(size_t j = 0; j < batchSize; ++j) { const auto& sent = qsBatch[j]; @@ -114,8 +136,8 @@ class BeamSearchDecoder : public IBeamSearchDecoder { } } } - std::vector> subBatches; - subBatches.push_back(subBatch); + auto tgtSubBatch = New(batchSize, 0, vocabs_[1]); // only holds a vocab, but data is dummy + std::vector> subBatches{ subBatch, tgtSubBatch }; std::vector sentIds(batchSize, 0); auto batch = New(subBatches); @@ -165,8 +187,26 @@ class BeamSearchDecoder : public IBeamSearchDecoder { Ptr newDecoder(Ptr options, const std::vector& ptrs, - Word eos) { - return New(options, ptrs, eos); + const std::vector>& vocabs, + WordIndex eos) { + return New(options, ptrs, vocabs, eos); +} + +std::vector> loadVocabs(const std::vector& vocabPaths) { + std::vector> res(vocabPaths.size()); + for (size_t i = 0; i < vocabPaths.size(); i++) { + if (i > 0 && vocabPaths[i] == vocabPaths[i-1]) { + res[i] = res[i-1]; + LOG(info, "[data] Input {} sharing vocabulary with input {}", i, i-1); + } + else { + auto vocab = New(New(), i); // (empty options, since they are only used for creating vocabs) + auto size = vocab->load(vocabPaths[i]); + LOG(info, "[data] Loaded vocabulary size for input {} of size {}", i, size); + res[i] = New(vocab); + } + } + return res; } } // namespace quicksand diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index 93308fe98..77d13efa4 100644 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -16,12 +16,12 @@ class Options; namespace quicksand { typedef uint32_t IndexType; -typedef IndexType Word; -typedef std::vector Words; -typedef std::vector QSBatch; +typedef IndexType WordIndex; +typedef std::vector WordIndices; +typedef std::vector QSBatch; typedef std::vector>> AlignmentSets; // [tgtPos] -> set of (srcPos, score) -typedef std::tuple QSSentenceWithProb; +typedef std::tuple QSSentenceWithProb; typedef std::vector QSNBest; typedef std::vector QSNBestBatch; @@ -30,21 +30,28 @@ Ptr newOptions(); template void set(Ptr options, const std::string& key, const T& value); +class IVocabWrapper { +public: + virtual WordIndex encode(const std::string& word) const = 0; + virtual size_t size() const = 0; + virtual std::string decode(WordIndex id) const = 0; +}; + class IBeamSearchDecoder { protected: Ptr options_; std::vector ptrs_; - Word eos_; + WordIndex eos_; public: IBeamSearchDecoder(Ptr options, const std::vector& ptrs, - Word eos) + WordIndex eos) : options_(options), ptrs_(ptrs), eos_(eos) {} virtual QSNBestBatch decode(const QSBatch& qsBatch, size_t maxLength, - const std::unordered_set& shortlist) + const std::unordered_set& shortlist) = 0; virtual void setWorkspace(uint8_t* data, size_t size) = 0; @@ -52,7 +59,11 @@ class IBeamSearchDecoder { Ptr newDecoder(Ptr options, const std::vector& ptrs, - Word eos); + const std::vector>& vocabs, + WordIndex eos); + +// load src and tgt vocabs +std::vector> loadVocabs(const std::vector& vocabPaths); } // namespace quicksand } // namespace marian From b9419a39b7728c1aaeb4f288df7e48d259db0006 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 11 Apr 2019 20:47:08 -0700 Subject: [PATCH 399/838] extended quicksand API to use our native vocabulary class --- src/3rd_party/pathie-cpp/src/path.cpp | 5 ++- src/microsoft/quicksand.cpp | 54 +++++++++++++++++++++++---- src/microsoft/quicksand.h | 27 ++++++++++---- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp index 12aa02703..6d9e8b63a 100755 --- a/src/3rd_party/pathie-cpp/src/path.cpp +++ b/src/3rd_party/pathie-cpp/src/path.cpp @@ -149,6 +149,8 @@ Path::Path(const std::vector& components) */ void Path::sanitize() { + bool isWindowsUNCPath = m_path.c_str()[0] == '\\' && m_path.c_str()[1] == '\\'; // UNC path + // Replace any backslashes \ with forward slashes /. size_t cur = string::npos; while ((cur = m_path.find("\\")) != string::npos) { // assignment intended @@ -156,8 +158,9 @@ void Path::sanitize() } // Replace all double slashes // with a single one + // [fseide] except for the first position, which would be a Windows UNC path cur = string::npos; - while ((cur = m_path.find("//")) != string::npos) { // assignment intended + while ((cur = m_path.find("//", isWindowsUNCPath ? 1 : 0)) != string::npos) { // assignment intended m_path.replace(cur, 2, "/"); } diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 2d932f0b7..a213efe12 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -9,6 +9,7 @@ #include "translator/beam_search.h" #include "translator/scorers.h" #include "data/alignment.h" +#include "data/vocab_base.h" namespace marian { @@ -31,6 +32,20 @@ Ptr newOptions() { return New(); } +class VocabWrapper : public IVocabWrapper { + Ptr pImpl_; +public: + VocabWrapper(Ptr vocab) : pImpl_(vocab) {} + WordIndex encode(const std::string& word) const { + auto enc = pImpl_->encode(word, /*addEOS=*/false, /*inference=*/true); + ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); + return enc[0].toWordIndex(); + } + size_t size() const { return pImpl_->size(); } + std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } + Ptr getVocab() const { return pImpl_; } +}; + class BeamSearchDecoder : public IBeamSearchDecoder { private: Ptr graph_; @@ -38,12 +53,19 @@ class BeamSearchDecoder : public IBeamSearchDecoder { std::vector> scorers_; + std::vector> vocabs_; + public: BeamSearchDecoder(Ptr options, const std::vector& ptrs, - Word eos) + const std::vector>& vocabs, + WordIndex eos) : IBeamSearchDecoder(options, ptrs, eos) { + // copy the vocabs + for (auto vi : vocabs) + vocabs_.push_back(std::dynamic_pointer_cast(vi)->getVocab()); + // setting 16-bit optimization to false for now. Re-enable with better caching or pre-computation graph_ = New(/*inference=*/true, /*optimize=*/false); @@ -94,7 +116,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { QSNBestBatch decode(const QSBatch& qsBatch, size_t maxLength, - const std::unordered_set& shortlist) override { + const std::unordered_set& shortlist) override { if(shortlist.size() > 0) { auto shortListGen = New(shortlist); for(auto scorer : scorers_) @@ -103,7 +125,7 @@ class BeamSearchDecoder : public IBeamSearchDecoder { // form source batch, by interleaving the words over sentences in the batch, and setting the mask size_t batchSize = qsBatch.size(); - auto subBatch = New(batchSize, maxLength, nullptr); + auto subBatch = New(batchSize, maxLength, vocabs_[0]); for(size_t i = 0; i < maxLength; ++i) { for(size_t j = 0; j < batchSize; ++j) { const auto& sent = qsBatch[j]; @@ -114,8 +136,8 @@ class BeamSearchDecoder : public IBeamSearchDecoder { } } } - std::vector> subBatches; - subBatches.push_back(subBatch); + auto tgtSubBatch = New(batchSize, 0, vocabs_[1]); // only holds a vocab, but data is dummy + std::vector> subBatches{ subBatch, tgtSubBatch }; std::vector sentIds(batchSize, 0); auto batch = New(subBatches); @@ -165,8 +187,26 @@ class BeamSearchDecoder : public IBeamSearchDecoder { Ptr newDecoder(Ptr options, const std::vector& ptrs, - Word eos) { - return New(options, ptrs, eos); + const std::vector>& vocabs, + WordIndex eos) { + return New(options, ptrs, vocabs, eos); +} + +std::vector> loadVocabs(const std::vector& vocabPaths) { + std::vector> res(vocabPaths.size()); + for (size_t i = 0; i < vocabPaths.size(); i++) { + if (i > 0 && vocabPaths[i] == vocabPaths[i-1]) { + res[i] = res[i-1]; + LOG(info, "[data] Input {} sharing vocabulary with input {}", i, i-1); + } + else { + auto vocab = New(New(), i); // (empty options, since they are only used for creating vocabs) + auto size = vocab->load(vocabPaths[i]); + LOG(info, "[data] Loaded vocabulary size for input {} of size {}", i, size); + res[i] = New(vocab); + } + } + return res; } } // namespace quicksand diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index 93308fe98..77d13efa4 100644 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -16,12 +16,12 @@ class Options; namespace quicksand { typedef uint32_t IndexType; -typedef IndexType Word; -typedef std::vector Words; -typedef std::vector QSBatch; +typedef IndexType WordIndex; +typedef std::vector WordIndices; +typedef std::vector QSBatch; typedef std::vector>> AlignmentSets; // [tgtPos] -> set of (srcPos, score) -typedef std::tuple QSSentenceWithProb; +typedef std::tuple QSSentenceWithProb; typedef std::vector QSNBest; typedef std::vector QSNBestBatch; @@ -30,21 +30,28 @@ Ptr newOptions(); template void set(Ptr options, const std::string& key, const T& value); +class IVocabWrapper { +public: + virtual WordIndex encode(const std::string& word) const = 0; + virtual size_t size() const = 0; + virtual std::string decode(WordIndex id) const = 0; +}; + class IBeamSearchDecoder { protected: Ptr options_; std::vector ptrs_; - Word eos_; + WordIndex eos_; public: IBeamSearchDecoder(Ptr options, const std::vector& ptrs, - Word eos) + WordIndex eos) : options_(options), ptrs_(ptrs), eos_(eos) {} virtual QSNBestBatch decode(const QSBatch& qsBatch, size_t maxLength, - const std::unordered_set& shortlist) + const std::unordered_set& shortlist) = 0; virtual void setWorkspace(uint8_t* data, size_t size) = 0; @@ -52,7 +59,11 @@ class IBeamSearchDecoder { Ptr newDecoder(Ptr options, const std::vector& ptrs, - Word eos); + const std::vector>& vocabs, + WordIndex eos); + +// load src and tgt vocabs +std::vector> loadVocabs(const std::vector& vocabPaths); } // namespace quicksand } // namespace marian From abbbc7f109b50136eb7b1ca6ac1b8d660b3fb8ca Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 11 Apr 2019 20:55:01 -0700 Subject: [PATCH 400/838] added stubs for compilation under Flo --- src/data/factored_vocab.cpp | 1 + src/layers/generic.cpp | 1 + 2 files changed, 2 insertions(+) create mode 100755 src/data/factored_vocab.cpp create mode 100755 src/layers/generic.cpp diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp new file mode 100755 index 000000000..056696600 --- /dev/null +++ b/src/data/factored_vocab.cpp @@ -0,0 +1 @@ +// will be used in the future \ No newline at end of file diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp new file mode 100755 index 000000000..056696600 --- /dev/null +++ b/src/layers/generic.cpp @@ -0,0 +1 @@ +// will be used in the future \ No newline at end of file From 4c841cb8a23a9e8b2141f1e341ba7d0d3dae6132 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 11 Apr 2019 21:03:19 -0700 Subject: [PATCH 401/838] (reordered two methods) --- src/microsoft/quicksand.cpp | 2 +- src/microsoft/quicksand.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index a213efe12..1560ccda5 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -41,8 +41,8 @@ class VocabWrapper : public IVocabWrapper { ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); return enc[0].toWordIndex(); } - size_t size() const { return pImpl_->size(); } std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } + size_t size() const { return pImpl_->size(); } Ptr getVocab() const { return pImpl_; } }; diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index 77d13efa4..22fe97b67 100644 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -33,8 +33,8 @@ void set(Ptr options, const std::string& key, const T& value); class IVocabWrapper { public: virtual WordIndex encode(const std::string& word) const = 0; - virtual size_t size() const = 0; virtual std::string decode(WordIndex id) const = 0; + virtual size_t size() const = 0; }; class IBeamSearchDecoder { From ada3d1f0ed5ef6e6fa8b6266b4f87e9779a346f7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 11 Apr 2019 21:09:08 -0700 Subject: [PATCH 402/838] (minor prettification) --- src/3rd_party/pathie-cpp/src/path.cpp | 2 +- src/microsoft/quicksand.cpp | 2 +- src/microsoft/quicksand.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp index 6d9e8b63a..07fba2711 100755 --- a/src/3rd_party/pathie-cpp/src/path.cpp +++ b/src/3rd_party/pathie-cpp/src/path.cpp @@ -149,7 +149,7 @@ Path::Path(const std::vector& components) */ void Path::sanitize() { - bool isWindowsUNCPath = m_path.c_str()[0] == '\\' && m_path.c_str()[1] == '\\'; // UNC path + bool isWindowsUNCPath = m_path.size() >= 2 && (m_path[0] == '\\' && m_path[1] == '\\'); // UNC path // Replace any backslashes \ with forward slashes /. size_t cur = string::npos; diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index a213efe12..1560ccda5 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -41,8 +41,8 @@ class VocabWrapper : public IVocabWrapper { ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); return enc[0].toWordIndex(); } - size_t size() const { return pImpl_->size(); } std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } + size_t size() const { return pImpl_->size(); } Ptr getVocab() const { return pImpl_; } }; diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index 77d13efa4..22fe97b67 100644 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -33,8 +33,8 @@ void set(Ptr options, const std::string& key, const T& value); class IVocabWrapper { public: virtual WordIndex encode(const std::string& word) const = 0; - virtual size_t size() const = 0; virtual std::string decode(WordIndex id) const = 0; + virtual size_t size() const = 0; }; class IBeamSearchDecoder { From 4351229899811c322bbb61800f499b37cc23a9fe Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 09:41:11 -0700 Subject: [PATCH 403/838] (updated the VS project) --- vs/Marian.vcxproj | 1 - vs/Marian.vcxproj.filters | 3 --- 2 files changed, 4 deletions(-) diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index e860f7e1d..e103ec45f 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -944,7 +944,6 @@ - diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index 8da3d3d60..c8af656de 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -1117,9 +1117,6 @@ tensors - - tensors - tensors\cpu From 01723b97b7cc89ca847ed98e3098b28601618681 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 09:47:16 -0700 Subject: [PATCH 404/838] made gcc happy --- src/microsoft/quicksand.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index a213efe12..6df9748bb 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -36,13 +36,13 @@ class VocabWrapper : public IVocabWrapper { Ptr pImpl_; public: VocabWrapper(Ptr vocab) : pImpl_(vocab) {} - WordIndex encode(const std::string& word) const { + WordIndex encode(const std::string& word) const override { auto enc = pImpl_->encode(word, /*addEOS=*/false, /*inference=*/true); ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); return enc[0].toWordIndex(); } - size_t size() const { return pImpl_->size(); } - std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } + size_t size() const override { return pImpl_->size(); } + std::string decode(WordIndex id) const override { return pImpl_->decode({ Word::fromWordIndex(id) }); } Ptr getVocab() const { return pImpl_; } }; From a4e56c63382fcc55c45cc9c744ef374904ab3a98 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 09:49:52 -0700 Subject: [PATCH 405/838] added missing override' --- src/microsoft/quicksand.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 1560ccda5..f3ae8ad83 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -36,13 +36,13 @@ class VocabWrapper : public IVocabWrapper { Ptr pImpl_; public: VocabWrapper(Ptr vocab) : pImpl_(vocab) {} - WordIndex encode(const std::string& word) const { + WordIndex encode(const std::string& word) const override { auto enc = pImpl_->encode(word, /*addEOS=*/false, /*inference=*/true); ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); return enc[0].toWordIndex(); } - std::string decode(WordIndex id) const { return pImpl_->decode({ Word::fromWordIndex(id) }); } - size_t size() const { return pImpl_->size(); } + std::string decode(WordIndex id) const override { return pImpl_->decode({ Word::fromWordIndex(id) }); } + size_t size() const override { return pImpl_->size(); } Ptr getVocab() const { return pImpl_; } }; From cb5a11ffef9ee095b02705e3940ab0248a8cff38 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 12:57:40 -0700 Subject: [PATCH 406/838] replaced the index2str_, which enumerated all factor combinations incl. invalid ones, with a map --- src/data/factored_vocab.cpp | 63 ++++++++++++++++++------------------- src/data/factored_vocab.h | 30 +++++++++++++----- src/layers/generic.cpp | 7 ++++- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 04f625389..589d17bc3 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -124,10 +124,9 @@ namespace marian { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - auto vocabSize = factorShape_.elements(); // size of vocab space including gaps - vocab_.resize(vocabSize); + //vocab_.resize(vocabSize); //factorMap_.resize(vocabSize); - auto factorVocabSize = factorVocab_.size(); + //auto factorVocabSize = this->factorVocabSize(); lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); size_t numTotalFactors = 0; for (WordIndex v = 0; v < factorMapTokenized.size(); v++) { @@ -171,12 +170,13 @@ namespace marian { // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", - numTotalFactors, factorVocabSize, vocab_.numValid(), size()); + numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, size()); // enumerate all combinations of factors for each lemma // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. size_t numMissing = 0; - for (size_t v = 0; v < vocabSize; v++) { + auto vocabSize = size(); // size of vocab space including gaps + for (size_t v = 0; v < vocabSize; v++) { // @BUGBUG:L This will run forever auto word = Word::fromWordIndex(v); bool isValid = true; for (size_t g = 0; isValid && g < numGroups; g++) { @@ -197,8 +197,10 @@ namespace marian { if (numMissing > 0) LOG(info, "[embedding] completed {} factor combinations missing from the original vocab file", numMissing); +#ifdef FACTOR_FULL_EXPANSION // create mappings needed for normalization in factored outputs constructNormalizationInfoForVocab(); +#endif // and must exist in the vocabulary eosId_ = Word::fromWordIndex(vocab_[DEFAULT_EOS_STR]); @@ -206,8 +208,8 @@ namespace marian { //LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); #if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() - if (maxSizeUnused == vocab_.numValid()) - maxSizeUnused = vocab_.size(); + if (maxSizeUnused == vocab_.size()/*numValid()*/) + maxSizeUnused = vocabSize; #endif //ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); // @TODO: ^^ disabled now that we are generating the full combination of factors; reenable once we have consistent setups again @@ -217,7 +219,7 @@ namespace marian { void FactoredVocab::constructGroupInfoFromFactorVocab() { // form groups size_t numGroups = groupPrefixes_.size(); - size_t factorVocabSize = factorVocab_.size(); + size_t factorVocabSize = this->factorVocabSize(); factorGroups_.resize(factorVocabSize, 0); for (size_t g = 1; g < groupPrefixes_.size(); g++) { // set group labels; what does not match any prefix will stay in group 0 const auto& groupPrefix = groupPrefixes_[g]; @@ -391,6 +393,7 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { // ABORT("Not implemented"); //} +#ifdef FACTOR_FULL_EXPANSION void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs //size_t numGroups = groupPrefixes_.size(); @@ -427,6 +430,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { data.push_back(Word::fromWordIndex(v)); globalFactorMatrix_ = csr_rows(data); // [V x U] } +#endif /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { WordIndex index; @@ -441,7 +445,9 @@ void FactoredVocab::constructNormalizationInfoForVocab() { /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*override final*/ { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); -#if 1 // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor +#if 1 // @TODO: remove this + ABORT_IF(vocab_.isGap(word.toWordIndex()), "Invalid factor combination {}", word2string(word)); +#else // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor if (vocab_.isGap(word.toWordIndex())) { LOG/*_ONCE*/(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); //const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); @@ -451,10 +457,6 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return vocab_[word.toWordIndex()]; } -/*virtual*/ size_t FactoredVocab::size() const /*override final*/ { - return vocab_.size(); -} - /*virtual*/ std::string FactoredVocab::toUpper(const std::string& line) const /*override final*/ { return utils::findReplace(utils::findReplace(utils::findReplace(utils::findReplace(line, "|ci", "|ca", /*all=*/true), "|cn", "|ca", /*all=*/true), "@CI", "@CA", /*all=*/true), "@CN", "@CA", /*all=*/true); } @@ -596,16 +598,9 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { weights.push_back(1.0f); } } -#if 0 // @BUGBUG: No, this is wrong! The vector must be 1s, since we use it in backprop transposition. - else { - // push a dummy entry. Not sure if this is needed. - indices.push_back(0); - weights.push_back(0.0f); - } -#endif offsets.push_back((IndexType)indices.size()); // next matrix row begins at this offset } - return { Shape({(int)words.size(), (int)factorVocab_.size()}), weights, indices, offsets }; + return { Shape({(int)words.size(), (int)factorVocabSize()}), weights, indices, offsets }; } // Helper to construct and load a FactordVocab from a path is given (non-empty) and if it specifies a factored vocab. @@ -625,16 +620,20 @@ WordIndex FactoredVocab::WordLUT::add(const std::string& word, WordIndex index) ABORT_IF(word.empty(), "Attempted to add the empty word to a dictionary"); auto wasInserted = str2index_.insert(std::make_pair(word, index)).second; ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}', new index {} vs. existing index {}", word, index, str2index_[word]); - while (index2str_.size() <= index) - index2str_.emplace_back(); // @TODO: what's the right way to get linear complexity in steps? - ABORT_IF(!index2str_[index].empty(), "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); - index2str_[index] = word; + wasInserted = index2str_.insert(std::make_pair(index, word)).second; + ABORT_IF(!wasInserted, "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); + //if (vocabSize_ < index2str_.size()) + // vocabSize_ = index2str_.size(); return index; } +static const std::string g_emptyString; const std::string& FactoredVocab::WordLUT::operator[](WordIndex index) const { - const auto& word = index2str_[index]; - //ABORT_IF(word.empty(), "Invalid access to dictionary gap item"); - return word; + auto iter = index2str_.find(index); + //ABORT_IF(iter == index2str_.end(), "Invalid access to dictionary gap item"); + if (iter == index2str_.end()) + return g_emptyString; // (using a global since we return a reference) + else + return iter->second; } WordIndex FactoredVocab::WordLUT::operator[](const std::string& word) const { auto iter = str2index_.find(word); @@ -648,10 +647,10 @@ bool FactoredVocab::WordLUT::tryFind(const std::string& word, WordIndex& index) index = iter->second; return true; } -void FactoredVocab::WordLUT::resize(size_t num) { - ABORT_IF(num < index2str_.size(), "Word table cannot be shrunk"); - index2str_.resize(num); // gets filled up with gap items (empty strings) -} +//void FactoredVocab::WordLUT::resize(size_t num) { +// ABORT_IF(num < size(), "Word table cannot be shrunk"); +// vocabSize_ = num; +//} size_t FactoredVocab::WordLUT::load(const std::string& path) { std::string line; io::InputFileStream in(path); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index d6bb8a21f..c5bfee7f3 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -11,6 +11,8 @@ #include // for std::iota() +#undef FACTOR_FULL_EXPANSION // define this to get full expansion. @TODO: infeasible for many factors; just delete this + namespace marian { class FactoredVocab : public IVocab { @@ -33,7 +35,7 @@ class FactoredVocab : public IVocab { virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; virtual std::string surfaceForm(const Words& sentence) const override final; virtual const std::string& operator[](Word id) const override final; - virtual size_t size() const override final; + virtual size_t size() const override final { return factorShape_.elements(); } // virtual vocab size with all factor combinations including gaps virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } @@ -44,16 +46,20 @@ class FactoredVocab : public IVocab { virtual Word randWord() const override final; // factor-specific. These methods are consumed by Output and Embedding. - size_t factorVocabSize() const { return factorVocab_.size(); } + size_t factorVocabSize() const { return factorVocab_.size(); } // total number of factors across all types - CSRData csr_rows(const Words& words) const; + CSRData csr_rows(const Words& words) const; // sparse matrix for summing up factors from the concatenated embedding matrix for each word +#ifdef FACTOR_FULL_EXPANSION const CSRData& getGlobalFactorMatrix() const { return globalFactorMatrix_; } // [v,u] (sparse) -> =1 if u is factor of v --only used in getLogits() +#endif size_t getNumGroups() const { return groupRanges_.size(); } std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) //const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g //const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor +#ifdef FACTOR_FULL_EXPANSION const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 +#endif // convert representations Word factors2word(const std::vector& factors) const; @@ -74,21 +80,25 @@ class FactoredVocab : public IVocab { private: void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); +#ifdef FACTOR_FULL_EXPANSION void constructNormalizationInfoForVocab(); +#endif size_t factorUnit2FactorIndex(WordIndex u) const; private: class WordLUT { // map between strings and WordIndex std::map str2index_; - std::vector index2str_; + std::map index2str_; + //size_t vocabSize_; // total number of vocab items as set by user public: WordIndex add(const std::string& word, WordIndex index); const std::string& operator[](WordIndex index) const; WordIndex operator[](const std::string& word) const; - bool isGap(WordIndex index) const { return index2str_[index].empty(); } + bool isGap(WordIndex index) const { return index2str_.find(index) == index2str_.end(); } bool tryFind(const std::string& word, WordIndex& index) const; - void resize(size_t num); - size_t size() const { return index2str_.size(); } // nominal size including gap items - size_t numValid() const { return str2index_.size(); } // actual non-gaps items + //void resize(size_t num); // @TODO: remove this, and remove the distinction of size() and numValid() + //size_t size() const { return vocabSize_; } // nominal size including gap items + //size_t numValid() const { return str2index_.size(); } // actual non-gaps items + size_t size() const { return str2index_.size(); } size_t load(const std::string& path); }; @@ -102,7 +112,9 @@ class FactoredVocab : public IVocab { WordLUT factorVocab_; // [factor name] -> factor index = row of E_ std::vector groupPrefixes_; // [group id g] shared prefix of factors (used for grouping) //std::vector> factorMap_; // [word index v] -> set of factor indices u +#ifdef FACTOR_FULL_EXPANSION CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v +#endif std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. std::vector>lemmaHasFactorGroup_; // [factor 0 index][g] -> true if lemma has factor group @@ -110,7 +122,9 @@ class FactoredVocab : public IVocab { std::vector factorStrides_; // [g] stride for factor dimension //std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g //std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) +#ifdef FACTOR_FULL_EXPANSION std::vector gapLogMask_; // [v] -1e8 if this is a gap, else 0 +#endif }; } // namespace marian diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5bed72ec7..366b05f35 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -84,7 +84,6 @@ namespace marian { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); auto sel = logits_[groupIndex]->loss(); // [localBeamSize, 1, dimBatch, dimFactorVocab] - // normalize for decoding: // - all secondary factors: subtract their max // - lemma: add all maxes of applicable factors @@ -108,6 +107,8 @@ namespace marian { // This function assumes that the object holds one or more factor logits, which are summed up // into output-vocab logits according to the factored model (with correct normalization of factors). + // This is infeasible for realistic factor sets, and therefore only implemented for 1 factor. + // @TODO: remove altogether Expr Logits::getLogits() const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); if (!factoredVocab_) { @@ -115,6 +116,7 @@ namespace marian { return getFactoredLogits(0); } +#ifdef FACTOR_FULL_EXPANSION // compute normalized factor log probs std::vector logProbs(logits_.size()); for (size_t g = 0; g < logits_.size(); g++) @@ -137,6 +139,9 @@ namespace marian { y = y + graph->constant({ (int)gapLogMask.size() }, inits::from_vector(gapLogMask), Type::float32); return y; +#else + ABORT("getLogits() no longer supported for actual factored vocab"); // because it is infeasible +#endif } void Logits::MaskedFactorIndices::push_back(size_t factorIndex) { From 3b927cd50756dd84bdd53880acc1f071b586b60b Mon Sep 17 00:00:00 2001 From: Vishal Chowdhary Date: Fri, 12 Apr 2019 15:59:43 -0700 Subject: [PATCH 407/838] added a missing #include for some VS version --- src/data/types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data/types.h b/src/data/types.h index d204beb64..05cdf96cf 100755 --- a/src/data/types.h +++ b/src/data/types.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace marian { From 0685248cfec3657225daaccbfd83106eddb079b6 Mon Sep 17 00:00:00 2001 From: Vishal Chowdhary Date: Fri, 12 Apr 2019 15:59:43 -0700 Subject: [PATCH 408/838] added a missing #include for some VS version --- src/data/types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data/types.h b/src/data/types.h index d204beb64..05cdf96cf 100755 --- a/src/data/types.h +++ b/src/data/types.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace marian { From 6d3b6d3fb0654356576e772f037206815ca902d1 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 18:10:07 -0700 Subject: [PATCH 409/838] added caching of FactoredVocab instances (keyed by pathname); Shape::elements() can now run as size_t --- src/common/shape.h | 7 ++-- src/data/factored_vocab.cpp | 81 +++++++++++++++++++++++++------------ src/data/factored_vocab.h | 3 +- 3 files changed, 62 insertions(+), 29 deletions(-) mode change 100644 => 100755 src/common/shape.h diff --git a/src/common/shape.h b/src/common/shape.h old mode 100644 new mode 100755 index c8a4bdd3d..138871396 --- a/src/common/shape.h +++ b/src/common/shape.h @@ -100,10 +100,11 @@ struct Shape { return stride[size() + i]; } - inline int elements() const { - int el = 1; + template // using a template so that FactoredSegmenter, which uses this as well, can pass size_t + inline T elements() const { + T el = 1; for(auto s : shape_) - el *= s; + el *= (T)s; return el; } diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 589d17bc3..56382bcec 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -7,7 +7,17 @@ namespace marian { + DONT_OPTIMIZE /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { + // If model has already been loaded, then assume this is a shared object, and skip loading it again. + // This can be multi-threaded, so must run under lock. + static std::mutex s_mtx; + std::lock_guard criticalSection(s_mtx); + if (vocab_.size() != 0) { + LOG(info, "[vocab] Attempting to load model a second time; skipping (assuming shared vocab)"); + return size(); + } + std::vector> factorMapTokenized; std::string line; std::vector tokBuf; @@ -142,7 +152,8 @@ namespace marian { factorUnits.push_back(u); } // convert to fully unrolled factors representation - std::vector factorIndices(groupRanges_.size(), FACTOR_NOT_APPLICABLE); // default for unused factors + auto na = FACTOR_NOT_APPLICABLE; // (gcc compiler bug: sometimes it cannot find this if passed directly) + std::vector factorIndices(groupRanges_.size(), na); // default for unused factors std::vector hasFactorGroupFlags(groupRanges_.size(), false); for (auto u : factorUnits) { factorIndices[factorGroups_[u]] = factorUnit2FactorIndex(u); @@ -164,38 +175,37 @@ namespace marian { // for now add what we get, and then expand more below vocab_.add(tokens.front(), wordIndex); if (tokens.front() != word2string(word)) - LOG_ONCE(info, "[embedding] Word name in .wl file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), word2string(word)); + LOG_ONCE(info, "[vocab] Word name in vocab file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), word2string(word)); numTotalFactors += tokens.size() - 1; //if (v % 5000 == 0) // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } - LOG(info, "[embedding] Factored-embedding map read with total/unique of {}/{} factors for {} valid words (in space of {})", - numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, size()); + LOG(info, "[vocab] Factored-embedding map read with total/unique of {}/{} factors from {} example words (in space of {})", + numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, utils::withCommas(size())); + vocab_.dumpToFile(modelPath + "_examples"); // enumerate all combinations of factors for each lemma // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. - size_t numMissing = 0; - auto vocabSize = size(); // size of vocab space including gaps - for (size_t v = 0; v < vocabSize; v++) { // @BUGBUG:L This will run forever - auto word = Word::fromWordIndex(v); + // This enumerates all possible combinations (incl. invalid ones), and stores all valid ones + auto virtualVocabSize = (WordIndex)size(); // size of vocab space including gaps + ABORT_IF((size_t)virtualVocabSize != size(), "Too many factors, virtual index space {} exceeds the bit limit of WordIndex type", utils::withCommas(size())); + LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(size())); + for (WordIndex v = 0; v < virtualVocabSize; v++) { // @BUGBUG: This is SLOOOW. Need further changes to remove this altogether + // determine whether this bit combination is a thing bool isValid = true; + auto word = Word::fromWordIndex(v); for (size_t g = 0; isValid && g < numGroups; g++) { auto factorIndex = getFactor(word, g); // @TODO: we have a hack in getFactor() to return not-specified if factor is specified but not applicable, making it invalid isValid = factorIndex != FACTOR_NOT_SPECIFIED; // FACTOR_NOT_APPLICABLE is a valid value } - if (isValid != !vocab_.isGap((WordIndex)v)) { - //LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap((WordIndex)v)); - if (isValid) { // add the missing word (albeit with a poor grapheme) - vocab_.add(word2string(word), word.toWordIndex()); - //(*this)[word]; - // @TODO: ^^disabled to test getLogits(), which no longer works with this enabled (I guess since model has not seen the new units) - numMissing++; - } - } + if (isValid && vocab_.isGap(v)) // add if missing + vocab_.add(word2string(word), word.toWordIndex()); + else if (!isValid && !vocab_.isGap(v)) + LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap(v)); } - if (numMissing > 0) - LOG(info, "[embedding] completed {} factor combinations missing from the original vocab file", numMissing); + LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); + vocab_.dumpToFile(modelPath + "_expanded"); #ifdef FACTOR_FULL_EXPANSION // create mappings needed for normalization in factored outputs @@ -209,7 +219,7 @@ namespace marian { #if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() if (maxSizeUnused == vocab_.size()/*numValid()*/) - maxSizeUnused = vocabSize; + maxSizeUnused = virtualVocabSize; #endif //ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); // @TODO: ^^ disabled now that we are generating the full combination of factors; reenable once we have consistent setups again @@ -243,7 +253,7 @@ void FactoredVocab::constructGroupInfoFromFactorVocab() { groupCounts[g]++; } for (size_t g = 0; g < numGroups; g++) { // detect non-overlapping groups - LOG(info, "[embedding] Factor group '{}' has {} members", groupPrefixes_[g], groupCounts[g]); + LOG(info, "[vocab] Factor group '{}' has {} members", groupPrefixes_[g], groupCounts[g]); if (groupCounts[g] == 0) { // factor group is unused --@TODO: once this is not hard-coded, this is an error condition groupRanges_[g].first = g > 0 ? groupRanges_[g-1].second : 0; // fix up the entry groupRanges_[g].second = groupRanges_[g].first; @@ -324,7 +334,7 @@ size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { return u - groupRanges_[g].first; } - +DONT_OPTIMIZE void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) const { size_t numGroups = getNumGroups(); factorIndices.resize(numGroups); @@ -334,7 +344,8 @@ void FactoredVocab::word2factors(Word word, std::vector& factorIndices / } #if 1 auto test = factors2word(factorIndices); - ABORT_IF(test != word, "Word <-> factor conversion broken??"); + ABORT_IF(test != word, "Word <-> factor conversion broken?? {} vs{}, '{}' vs. '{}'", + test.toWordIndex(), word.toWordIndex(), word2string(test), word2string(word)); #endif } @@ -659,13 +670,33 @@ size_t FactoredVocab::WordLUT::load(const std::string& path) { return size(); } +void FactoredVocab::WordLUT::dumpToFile(const std::string& path) { + io::OutputFileStream out(path); + for (auto kvp : index2str_) + out << kvp.second << "\t" << utils::withCommas(kvp.first) << "\n"; +} + const static std::vector exts{ ".fsv", ".fm"/*legacy*/ }; // Note: This does not actually load it, only checks the path for the type. +// Since loading takes a while, we cache instances. Ptr createFactoredVocab(const std::string& vocabPath) { + // this can be multi-threaded, so must run under lock + static std::mutex s_mtx; + std::lock_guard criticalSection(s_mtx); + bool isFactoredVocab = std::any_of(exts.begin(), exts.end(), [&](const std::string& ext) { return utils::endsWith(vocabPath, ext); }); - if(isFactoredVocab) - return New(); + if (isFactoredVocab) { + static std::map> s_cache; + auto iter = s_cache.find(vocabPath); + if (iter != s_cache.end()) { + LOG(info, "[vocab] Reusing existing vocabulary object in memory"); + return iter->second; + } + auto vocab = New(); + s_cache.insert(std::make_pair(vocabPath, vocab)); + return vocab; + } else return nullptr; } diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index c5bfee7f3..43bd40b1b 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -35,7 +35,7 @@ class FactoredVocab : public IVocab { virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; virtual std::string surfaceForm(const Words& sentence) const override final; virtual const std::string& operator[](Word id) const override final; - virtual size_t size() const override final { return factorShape_.elements(); } // virtual vocab size with all factor combinations including gaps + virtual size_t size() const override final { return factorShape_.elements(); } // valid WordIndex range (representing all factor combinations including gaps); virtual and huge virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } @@ -100,6 +100,7 @@ class FactoredVocab : public IVocab { //size_t numValid() const { return str2index_.size(); } // actual non-gaps items size_t size() const { return str2index_.size(); } size_t load(const std::string& path); + void dumpToFile(const std::string& path); }; // main vocab From c0e7c6998ea2a40cbaa8ac4cf0666f224ec29b53 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 18:58:53 -0700 Subject: [PATCH 410/838] vocab_ is now recursively created, without looping through the full sparse index space --- src/data/factored_vocab.cpp | 33 ++++++++++++++++++++++++++++++++- src/data/factored_vocab.h | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 56382bcec..baac5448f 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -121,7 +121,6 @@ namespace marian { // construct mapping tables for factors constructGroupInfoFromFactorVocab(); constructFactorIndexConversion(); - auto numGroups = getNumGroups(); // load and parse factorMap // modelPath = path to file with entries in order of vocab entries of the form @@ -190,6 +189,11 @@ namespace marian { auto virtualVocabSize = (WordIndex)size(); // size of vocab space including gaps ABORT_IF((size_t)virtualVocabSize != size(), "Too many factors, virtual index space {} exceeds the bit limit of WordIndex type", utils::withCommas(size())); LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(size())); + std::vector factorIndices(getNumGroups()); +#if 1 + rCompleteVocab(factorIndices, /*g=*/0); +#else + auto numGroups = getNumGroups(); for (WordIndex v = 0; v < virtualVocabSize; v++) { // @BUGBUG: This is SLOOOW. Need further changes to remove this altogether // determine whether this bit combination is a thing bool isValid = true; @@ -204,6 +208,7 @@ namespace marian { else if (!isValid && !vocab_.isGap(v)) LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap(v)); } +#endif LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); vocab_.dumpToFile(modelPath + "_expanded"); @@ -226,6 +231,32 @@ namespace marian { return size(); } +// helper to add missing words to vocab_ +// factorIndices has been formed up to *ex*cluding position [g]. +void FactoredVocab::rCompleteVocab(std::vector& factorIndices, size_t g) { + // reached the end + if (g == getNumGroups()) { + auto word = factors2word(factorIndices); + auto v = word.toWordIndex(); // by design, we only generate those that are still missing + //auto ws = word2string(word); + //ABORT_IF(!vocab_.isGap(v), "Incorrect vocab_ construction sequence?? {}", word2string(word)); + if (vocab_.isGap(v)) // add if missing + vocab_.add(word2string(word), v); + return; + } + // try next factor + if (g == 0 || lemmaHasFactorGroup(factorIndices[0], g)) { + for (size_t g1 = 0; g1 < factorShape_[g] - 1; g1++) { + factorIndices[g] = g1; + rCompleteVocab(factorIndices, g + 1); + } + } + else { + factorIndices[g] = FACTOR_NOT_APPLICABLE; + rCompleteVocab(factorIndices, g + 1); + } +} + void FactoredVocab::constructGroupInfoFromFactorVocab() { // form groups size_t numGroups = groupPrefixes_.size(); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 43bd40b1b..b51a3e725 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -80,6 +80,7 @@ class FactoredVocab : public IVocab { private: void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); + void rCompleteVocab(std::vector& factorIndices, size_t g); #ifdef FACTOR_FULL_EXPANSION void constructNormalizationInfoForVocab(); #endif From 4be44d30f1b684e77196cec0a1a8711b18cdb4ff Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 19:11:38 -0700 Subject: [PATCH 411/838] renamed isGap() to (not) contains() --- src/data/factored_vocab.cpp | 20 +++++++++----------- src/data/factored_vocab.h | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index baac5448f..1ccba6088 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -203,10 +203,10 @@ namespace marian { // @TODO: we have a hack in getFactor() to return not-specified if factor is specified but not applicable, making it invalid isValid = factorIndex != FACTOR_NOT_SPECIFIED; // FACTOR_NOT_APPLICABLE is a valid value } - if (isValid && vocab_.isGap(v)) // add if missing + if (isValid && !vocab_.contains(v)) // add if missing vocab_.add(word2string(word), word.toWordIndex()); - else if (!isValid && !vocab_.isGap(v)) - LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, isGap={}", word2string(word), isValid, vocab_.isGap(v)); + else if (!isValid && vocab_.contains(v)) + LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, contains={}", word2string(word), isValid, vocab_.contains(v)); } #endif LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); @@ -237,10 +237,8 @@ void FactoredVocab::rCompleteVocab(std::vector& factorIndices, size_t g) // reached the end if (g == getNumGroups()) { auto word = factors2word(factorIndices); - auto v = word.toWordIndex(); // by design, we only generate those that are still missing - //auto ws = word2string(word); - //ABORT_IF(!vocab_.isGap(v), "Incorrect vocab_ construction sequence?? {}", word2string(word)); - if (vocab_.isGap(v)) // add if missing + auto v = word.toWordIndex(); + if (!vocab_.contains(v)) // add if missing vocab_.add(word2string(word), v); return; } @@ -445,7 +443,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { gapLogMask_.resize(vocabSize, -1e8f); for (WordIndex v = 0; v < vocabSize; v++) { #if 1 // @TODO: TEST THIS again by disabling factored decoding in beam_search.h - if (!vocab_.isGap(v)) + if (vocab_.contains(v)) gapLogMask_[v] = 0.0f; // valid entry #else for (auto u : factorMap_[v]) { @@ -488,9 +486,9 @@ void FactoredVocab::constructNormalizationInfoForVocab() { /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*override final*/ { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); #if 1 // @TODO: remove this - ABORT_IF(vocab_.isGap(word.toWordIndex()), "Invalid factor combination {}", word2string(word)); + ABORT_IF(!vocab_.contains(word.toWordIndex()), "Invalid factor combination {}", word2string(word)); #else // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor - if (vocab_.isGap(word.toWordIndex())) { + if (!vocab_.contains(word.toWordIndex())) { LOG/*_ONCE*/(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); //const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); const_cast(vocab_).add(word2string(word), word.toWordIndex()); @@ -614,7 +612,7 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { offsets.push_back((IndexType)indices.size()); std::vector factorIndices; for (auto word : words) { - if (!vocab_.isGap(word.toWordIndex())) { // skip invalid combinations in the space (can only happen during initialization) --@TODO: add a check? + if (vocab_.contains(word.toWordIndex())) { // skip invalid combinations in the space (can only happen during initialization) --@TODO: add a check? word2factors(word, factorIndices); #if 0 // original code; enable this to try numGroups; diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index b51a3e725..03c199c45 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -94,7 +94,7 @@ class FactoredVocab : public IVocab { WordIndex add(const std::string& word, WordIndex index); const std::string& operator[](WordIndex index) const; WordIndex operator[](const std::string& word) const; - bool isGap(WordIndex index) const { return index2str_.find(index) == index2str_.end(); } + bool contains(WordIndex index) const { return index2str_.find(index) != index2str_.end(); } bool tryFind(const std::string& word, WordIndex& index) const; //void resize(size_t num); // @TODO: remove this, and remove the distinction of size() and numValid() //size_t size() const { return vocabSize_; } // nominal size including gap items From 78ad43dbf5d9adb93777b6d9a58cef19ea4edc22 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 19:13:45 -0700 Subject: [PATCH 412/838] removed dead code --- src/data/factored_vocab.cpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 1ccba6088..81a318501 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -7,7 +7,6 @@ namespace marian { - DONT_OPTIMIZE /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { // If model has already been loaded, then assume this is a shared object, and skip loading it again. // This can be multi-threaded, so must run under lock. @@ -190,25 +189,7 @@ namespace marian { ABORT_IF((size_t)virtualVocabSize != size(), "Too many factors, virtual index space {} exceeds the bit limit of WordIndex type", utils::withCommas(size())); LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(size())); std::vector factorIndices(getNumGroups()); -#if 1 rCompleteVocab(factorIndices, /*g=*/0); -#else - auto numGroups = getNumGroups(); - for (WordIndex v = 0; v < virtualVocabSize; v++) { // @BUGBUG: This is SLOOOW. Need further changes to remove this altogether - // determine whether this bit combination is a thing - bool isValid = true; - auto word = Word::fromWordIndex(v); - for (size_t g = 0; isValid && g < numGroups; g++) { - auto factorIndex = getFactor(word, g); - // @TODO: we have a hack in getFactor() to return not-specified if factor is specified but not applicable, making it invalid - isValid = factorIndex != FACTOR_NOT_SPECIFIED; // FACTOR_NOT_APPLICABLE is a valid value - } - if (isValid && !vocab_.contains(v)) // add if missing - vocab_.add(word2string(word), word.toWordIndex()); - else if (!isValid && vocab_.contains(v)) - LOG(info, "WARNING: Factored vocab mismatch for {}: isValid={}, contains={}", word2string(word), isValid, vocab_.contains(v)); - } -#endif LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); vocab_.dumpToFile(modelPath + "_expanded"); @@ -363,7 +344,6 @@ size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { return u - groupRanges_[g].first; } -DONT_OPTIMIZE void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) const { size_t numGroups = getNumGroups(); factorIndices.resize(numGroups); From 2e8bac26e334fd1e559a13fcef2ef5209152146d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 12 Apr 2019 19:41:40 -0700 Subject: [PATCH 413/838] FactoredVocab::size() now returns the active vocabulary (excluding gaps), not the virtual one (which may exceed int range) --- src/data/factored_vocab.cpp | 25 +++++++++++-------------- src/data/factored_vocab.h | 3 ++- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 81a318501..242857a29 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -8,11 +8,12 @@ namespace marian { /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { + maxSizeUnused; // If model has already been loaded, then assume this is a shared object, and skip loading it again. // This can be multi-threaded, so must run under lock. static std::mutex s_mtx; std::lock_guard criticalSection(s_mtx); - if (vocab_.size() != 0) { + if (size() != 0) { LOG(info, "[vocab] Attempting to load model a second time; skipping (assuming shared vocab)"); return size(); } @@ -179,15 +180,13 @@ namespace marian { // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[vocab] Factored-embedding map read with total/unique of {}/{} factors from {} example words (in space of {})", - numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, utils::withCommas(size())); + numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, utils::withCommas(virtualVocabSize())); vocab_.dumpToFile(modelPath + "_examples"); // enumerate all combinations of factors for each lemma // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. // This enumerates all possible combinations (incl. invalid ones), and stores all valid ones - auto virtualVocabSize = (WordIndex)size(); // size of vocab space including gaps - ABORT_IF((size_t)virtualVocabSize != size(), "Too many factors, virtual index space {} exceeds the bit limit of WordIndex type", utils::withCommas(size())); - LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(size())); + LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(virtualVocabSize())); std::vector factorIndices(getNumGroups()); rCompleteVocab(factorIndices, /*g=*/0); LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); @@ -203,10 +202,10 @@ namespace marian { unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); //LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); -#if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() - if (maxSizeUnused == vocab_.size()/*numValid()*/) - maxSizeUnused = virtualVocabSize; -#endif +//#if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() +// if (maxSizeUnused == vocab_.size()/*numValid()*/) +// maxSizeUnused = virtualVocabSize; +//#endif //ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); // @TODO: ^^ disabled now that we are generating the full combination of factors; reenable once we have consistent setups again return size(); @@ -285,6 +284,8 @@ void FactoredVocab::constructFactorIndexConversion() { factorStrides_.resize(factorShape_.size(), 1); for (size_t g = factorStrides_.size() - 1; g --> 0; ) factorStrides_[g] = factorStrides_[g + 1] * (size_t)factorShape_[g + 1]; + ABORT_IF((WordIndex)virtualVocabSize() != virtualVocabSize(), + "Too many factors, virtual index space {} exceeds the bit limit of WordIndex type", utils::withCommas(virtualVocabSize())); } // encode factors into a Word struct @@ -417,7 +418,7 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs //size_t numGroups = groupPrefixes_.size(); - size_t vocabSize = size(); + size_t vocabSize = virtualVocabSize(); //factorMasks_ .resize(numGroups, std::vector(vocabSize, 0)); // [g][v] 1.0 if word v has factor g //factorIndices_.resize(numGroups, std::vector(vocabSize, 0)); // [g][v] index of factor (or any valid index if it does not have it; we use 0) gapLogMask_.resize(vocabSize, -1e8f); @@ -667,10 +668,6 @@ bool FactoredVocab::WordLUT::tryFind(const std::string& word, WordIndex& index) index = iter->second; return true; } -//void FactoredVocab::WordLUT::resize(size_t num) { -// ABORT_IF(num < size(), "Word table cannot be shrunk"); -// vocabSize_ = num; -//} size_t FactoredVocab::WordLUT::load(const std::string& path) { std::string line; io::InputFileStream in(path); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 03c199c45..33ff75ae5 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -35,7 +35,7 @@ class FactoredVocab : public IVocab { virtual std::string decode(const Words& sentence, bool ignoreEos = true) const override final; virtual std::string surfaceForm(const Words& sentence) const override final; virtual const std::string& operator[](Word id) const override final; - virtual size_t size() const override final { return factorShape_.elements(); } // valid WordIndex range (representing all factor combinations including gaps); virtual and huge + virtual size_t size() const override final { return vocab_.size(); } // active factored vocabulary size (counting all valid combinations but not gaps) virtual std::string type() const override final { return "FactoredVocab"; } virtual Word getEosId() const override final { return eosId_; } virtual Word getUnkId() const override final { return unkId_; } @@ -47,6 +47,7 @@ class FactoredVocab : public IVocab { // factor-specific. These methods are consumed by Output and Embedding. size_t factorVocabSize() const { return factorVocab_.size(); } // total number of factors across all types + size_t virtualVocabSize() const { return factorShape_.elements(); } // valid WordIndex range (representing all factor combinations including gaps); virtual and huge CSRData csr_rows(const Words& words) const; // sparse matrix for summing up factors from the concatenated embedding matrix for each word From 621a7a0daf075a82b6c7a90cc1f9fccb48309e1b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 13 Apr 2019 01:45:58 -0700 Subject: [PATCH 414/838] s2s encoder now supports FactoredVocab --- src/models/s2s.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/models/s2s.h b/src/models/s2s.h index cfd0d7e00..67b0817a4 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -119,12 +119,12 @@ class EncoderS2S : public EncoderBase { return context; } - Ptr createSourceEmbedding(Ptr graph) { + Ptr createSourceEmbeddingLayer(Ptr graph) { // create source embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - // @TODO: code dup with Decider and EncoderTransformer; actually diverged by now. Unify this. + // @TODO: code dup with Decoder and EncoderTransformer; actually diverged by now. Unify this. auto embFactory = embedding() // ("dimVocab", dimVoc) // ("dimEmb", dimEmb); @@ -144,14 +144,21 @@ class EncoderS2S : public EncoderBase { ("normalization", opt("embedding-normalization")); } + embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings return embFactory.construct(graph); } EncoderS2S(Ptr options) : EncoderBase(options) {} + std::vector> embedding_; // @TODO: move away, also rename virtual Ptr build(Ptr graph, Ptr batch) override { - auto embedding = createSourceEmbedding(graph); + // lazily create embedding layer + if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy + embedding_.resize(batch->sets()); + embedding_[batchIndex_] = createSourceEmbeddingLayer(graph); + } + auto embedding = embedding_[batchIndex_]; // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie From 5fceec0d5ef5ccfdad2a9ce8e9defd12e1bec0bb Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 13 Apr 2019 14:03:40 -0700 Subject: [PATCH 415/838] temp hack: validator will write 'bleu' file not 'bleu-detok', for Flo compat --- src/training/validator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/validator.h b/src/training/validator.h index a01f2fb17..1341834dc 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -711,7 +711,7 @@ class BleuValidator : public Validator { }; // @TODO: why do we return this string, but not pass it to the constructor? - std::string type() override { return detok_ ? "bleu-detok" : "bleu"; } + std::string type() override { return /*detok_ ? "bleu-detok" :*/ "bleu"; } protected: // Tokenizer function adapted from multi-bleu-detok.pl, corresponds to sacreBLEU.py From 87a88fb4d095402af87330fdbfe60a8607f02ad9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 13 Apr 2019 14:48:26 -0700 Subject: [PATCH 416/838] BleuValidator now always uses detokenized mode with FactoredSegmenter, even if not requested () --- src/training/validator.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/training/validator.h b/src/training/validator.h index 1341834dc..97c20181d 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -592,10 +592,6 @@ class BleuValidator : public Validator { builder_ = models::createModelFromOptions(options_, models::usage::translation); auto vocab = vocabs_.back(); -#if 1 // hack for now, to get this feature when running under Flo - if (vocab->type() == "FactoredVocab") - detok_ = true; // always use bleu-detok -#endif ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " "Current vocabulary type is {}", vocab->type()); @@ -711,7 +707,7 @@ class BleuValidator : public Validator { }; // @TODO: why do we return this string, but not pass it to the constructor? - std::string type() override { return /*detok_ ? "bleu-detok" :*/ "bleu"; } + std::string type() override { return detok_ ? "bleu-detok" : "bleu"; } protected: // Tokenizer function adapted from multi-bleu-detok.pl, corresponds to sacreBLEU.py @@ -830,7 +826,14 @@ class BleuValidator : public Validator { ref.push_back(w); } - if(detok_) + bool detok = detok_; +#if 1 // hack for now, to get this feature when running under Flo + if (vocabs_.back()->type() == "FactoredVocab") { + LOG_ONCE(info, "[valid] FactoredVocab implies using detokenized BLEU"); + detok = true; // always use bleu-detok + } +#endif + if(detok) updateStats(stats, decode(cand, /*addEOS=*/ true), decode(ref)); else updateStats(stats, cand, ref); From 59acad39637c1995c886efef5901fc701e6695e9 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 13 Apr 2019 18:32:32 -0700 Subject: [PATCH 417/838] FactoredVocab::operator[string] now parses the token, to allow for varying factor order; quicksand encode/decode now call operator[] --- src/data/factored_vocab.cpp | 45 ++++++++++++++++++++++++++++--------- src/data/factored_vocab.h | 5 +++-- src/microsoft/quicksand.cpp | 8 ++----- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 242857a29..890ae0415 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -172,25 +172,27 @@ namespace marian { // add to vocab (the wordIndex are not dense, so the vocab will have holes) //if (tokens.front().front() == '<') // all others are auto-expanded // for now add what we get, and then expand more below - vocab_.add(tokens.front(), wordIndex); - if (tokens.front() != word2string(word)) - LOG_ONCE(info, "[vocab] Word name in vocab file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), word2string(word)); + auto wordString = word2string(word); + if (tokens.front() != wordString) // order may differ, since we formed the input based on the factors in the user file, which may be in any order + LOG_ONCE(info, "[vocab] Word name in vocab file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), wordString); + vocab_.add(wordString, wordIndex); numTotalFactors += tokens.size() - 1; //if (v % 5000 == 0) // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[vocab] Factored-embedding map read with total/unique of {}/{} factors from {} example words (in space of {})", numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, utils::withCommas(virtualVocabSize())); - vocab_.dumpToFile(modelPath + "_examples"); + //vocab_.dumpToFile(modelPath + "_examples"); - // enumerate all combinations of factors for each lemma - // @TODO: switch to factor-spec, which no longer enumerates all combinations. Then don't set vocab string here. - // This enumerates all possible combinations (incl. invalid ones), and stores all valid ones + // enumerate all valid combinations of factors for each lemma and add them to vocab_ + // Having vocab_ makes life easier, although it is not strictly needed. Typical expanded valid vocabs + // are on the order of 200k entries. If we ever go much larger, we'd want to elimimate vocab_ + // and fully virtualize its function. LOG(info, "[vocab] Expanding all valid vocab entries out of {}...", utils::withCommas(virtualVocabSize())); std::vector factorIndices(getNumGroups()); rCompleteVocab(factorIndices, /*g=*/0); LOG(info, "[vocab] Completed, total {} valid combinations", vocab_.size()/*numValid()*/); - vocab_.dumpToFile(modelPath + "_expanded"); + //vocab_.dumpToFile(modelPath + "_expanded"); #ifdef FACTOR_FULL_EXPANSION // create mappings needed for normalization in factored outputs @@ -384,6 +386,28 @@ std::string FactoredVocab::word2string(Word word) const { return res; } +Word FactoredVocab::string2word(const std::string& w) const { + auto sep = std::string(1, factorSeparator_); + auto parts = utils::splitAny(w, sep); + auto na = FACTOR_NOT_APPLICABLE; // (gcc compiler bug: sometimes it cannot find this if passed directly) + std::vector factorIndices(groupRanges_.size(), na); // default for unused factors + for (size_t i = 0; i < parts.size(); i++) { + WordIndex u; + bool found = factorVocab_.tryFind(i == 0 ? parts[i] : sep + parts[i], u); + if (!found) { + LOG_ONCE(info, "WARNING: Unknown factor '{}' in '{}'; mapping to '{}'", parts[i], w, word2string(getUnkId())); + return getUnkId(); + } + // convert u to relative u within factor group range + auto g = factorGroups_[u]; + ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); + factorIndices[g] = u - groupRanges_[g].first; + } + auto word = factors2word(factorIndices); + //auto w2 = word2string(word); + return word; +} + size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { size_t index = word.toWordIndex(); size_t factor0Index = index / factorStrides_[0]; @@ -459,9 +483,10 @@ void FactoredVocab::constructNormalizationInfoForVocab() { if (found) return Word::fromWordIndex(index); else + return string2word(word); //ABORT("Unknown word {} mapped to {}", word, word2string(getUnkId())); - LOG_ONCE(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); - return getUnkId(); + //LOG_ONCE(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); + //return getUnkId(); } /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*override final*/ { diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 33ff75ae5..9a916aa82 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -77,7 +77,8 @@ class FactoredVocab : public IVocab { static bool isFactorValid(size_t factorIndex) { return factorIndex < FACTOR_NOT_SPECIFIED; } static Ptr tryCreateAndLoad(const std::string& path); // load from "vocab" option if it specifies a factored vocab - std::string word2string(Word word) const; // (diagnostics only) + std::string word2string(Word word) const; + Word string2word(const std::string& w) const; private: void constructGroupInfoFromFactorVocab(); void constructFactorIndexConversion(); @@ -120,7 +121,7 @@ class FactoredVocab : public IVocab { #endif std::vector factorGroups_; // [u] -> group id of factor u std::vector> groupRanges_; // [group id g] -> (u_begin,u_end) index range of factors u for this group. These don't overlap. - std::vector>lemmaHasFactorGroup_; // [factor 0 index][g] -> true if lemma has factor group + std::vector> lemmaHasFactorGroup_; // [factor 0 index][g] -> true if lemma has factor group Shape factorShape_; // [g] number of factors in each factor group std::vector factorStrides_; // [g] stride for factor dimension //std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 6df9748bb..51f857b52 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -36,13 +36,9 @@ class VocabWrapper : public IVocabWrapper { Ptr pImpl_; public: VocabWrapper(Ptr vocab) : pImpl_(vocab) {} - WordIndex encode(const std::string& word) const override { - auto enc = pImpl_->encode(word, /*addEOS=*/false, /*inference=*/true); - ABORT_IF(enc.size() != 1, "QuickSAND passed an invalid word '{}' to Marian (empty or contains a space)", word); - return enc[0].toWordIndex(); - } + WordIndex encode(const std::string& word) const override { return (*pImpl_)[word].toWordIndex(); } + std::string decode(WordIndex id) const override { return (*pImpl_)[Word::fromWordIndex(id)]; } size_t size() const override { return pImpl_->size(); } - std::string decode(WordIndex id) const override { return pImpl_->decode({ Word::fromWordIndex(id) }); } Ptr getVocab() const { return pImpl_; } }; From ee8c14cfbc685f4a25967a33ddbba533925b098f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sat, 13 Apr 2019 23:52:30 -0700 Subject: [PATCH 418/838] detokenizing validator now logs the detokenized sequence once --- src/data/factored_vocab.cpp | 2 +- src/training/validator.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 890ae0415..619fe9841 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -14,7 +14,7 @@ namespace marian { static std::mutex s_mtx; std::lock_guard criticalSection(s_mtx); if (size() != 0) { - LOG(info, "[vocab] Attempting to load model a second time; skipping (assuming shared vocab)"); + //LOG(info, "[vocab] Attempting to load model a second time; skipping (assuming shared vocab)"); return size(); } diff --git a/src/training/validator.h b/src/training/validator.h index f831f723c..78b9f9d33 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -834,11 +834,18 @@ class BleuValidator : public Validator { bool detok = detok_; #if 1 // hack for now, to get this feature when running under Flo + // Problem is that Flo pieces that pass 'bleu' do not know whether vocab is factored, + // hence cannot select 'bleu-detok'. if (vocabs_.back()->type() == "FactoredVocab") { LOG_ONCE(info, "[valid] FactoredVocab implies using detokenized BLEU"); detok = true; // always use bleu-detok } #endif + if(detok) { // log the first detokenized string + LOG_ONCE(info, "[valid] First sentence's tokens after detokenization, as scored:"); + LOG_ONCE(info, "[valid] Hyp: {}", utils::join(decode(cand, /*addEOS=*/ true))); + LOG_ONCE(info, "[valid] Ref: {}", utils::join(decode(ref))); + } if(detok) updateStats(stats, decode(cand, /*addEOS=*/ true), decode(ref)); else From 9f7d4fd530628845b3f91c5eacc2a64151899374 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 15 Apr 2019 15:46:41 -0700 Subject: [PATCH 419/838] implemented cpu::CSRProd() for decoding --- src/tensors/cpu/prod.cpp | 73 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 408e47913..fd18ea1c2 100644 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -169,15 +169,74 @@ void ProdWithBias(marian::Tensor C, void CSRProd(marian::Tensor C, Ptr /*allocator*/, - const marian::Tensor& A_values, - const marian::Tensor& A_indices, - const marian::Tensor& A_offsets, - const marian::Tensor& B, - bool transA, + const marian::Tensor& S_values, + const marian::Tensor& S_indices, + const marian::Tensor& S_offsets, + const marian::Tensor& D, + bool transS, bool swapOperands, float beta) { - C, A_values, A_indices, A_offsets, B, transA, swapOperands, beta; - ABORT("CSRProd is not yet implemented for CPU"); + C, S_values, S_indices, S_offsets, D; + + // Note: The CPU implementation currently only implements what's needed for decoding. + + // interpret tensor dimensions as matrix dimensions + const auto& shapeC = C->shape(); + const auto& shapeD = D->shape(); + // If swapOperands, S and D are swapped (C = D x S instead of C = S x D). + // In that case, in the next 6 lines, please read all dimensions as if they were reversed in order. + auto rowsC = shapeC[-(int)swapOperands]; + auto colsC = shapeC.elements() / rowsC; + auto rowsD = shapeD[-(int)swapOperands]; + auto colsD = shapeD.elements() / rowsD; + auto rowsS = transS ? rowsD : rowsC; + auto colsS = transS ? rowsC : rowsD; + ABORT_IF(colsD != colsC, "Inconsistent outer dimensions in CSR product"); + if (swapOperands) { // make rowsX actual row dimensions again, likewise colsX + std::swap(rowsC, colsC); + std::swap(rowsD, colsD); + std::swap(rowsS, colsS); + } + // sparse arrays + auto numValues = S_values->shape().elements(); + auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); + ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); + if (!transS && !swapOperands) { + // C = S * D, where D = CSR matrix + const auto* offsets = S_offsets->data(); + const auto* indices = S_indices->data(); + const auto* values = S_values->data(); + const auto* dataD = D->data(); + auto* dataC = C->data(); + ABORT_IF(beta != 0 && beta != 1, "cpu::CSRProd only supports beta = 0 or 1"); + for (size_t i = 0; i < rowsC; i++) { + auto add = (beta == 1); // first element: overwrite or add according to beta; subsequent elements: add + for (size_t kk = offsets[i]; kk < offsets[i + 1]; kk++) { + auto k = indices[kk]; // fetch the non-zero row + auto valS = values[kk]; // and the value from that row + // This code is written with the hope for good vectorization, and the hope + // that adding to memory will be done efficiently by the caching system. + if (valS == 1) + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = dataD[k * colsC/*==colsD*/ + j]; // this is a memcpy() + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += dataD[k * colsC/*==colsD*/ + j]; // this is a contiguous-vector addition + else + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = valS * dataD[k * colsC/*==colsD*/ + j]; + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += valS * dataD[k * colsC/*==colsD*/ + j]; // notice the += + add = true; // next iteration will add to existing result + } + } + } + else + ABORT("CSRProd for transS={}, swapOperands={} is not yet implemented for CPU", transS, swapOperands); } } // namespace cpu From 1f10a700e32a7826187fefd397e456b0ce208814 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 15 Apr 2019 15:46:41 -0700 Subject: [PATCH 420/838] implemented cpu::CSRProd() for decoding --- src/tensors/cpu/prod.cpp | 73 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 408e47913..fd18ea1c2 100644 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -169,15 +169,74 @@ void ProdWithBias(marian::Tensor C, void CSRProd(marian::Tensor C, Ptr /*allocator*/, - const marian::Tensor& A_values, - const marian::Tensor& A_indices, - const marian::Tensor& A_offsets, - const marian::Tensor& B, - bool transA, + const marian::Tensor& S_values, + const marian::Tensor& S_indices, + const marian::Tensor& S_offsets, + const marian::Tensor& D, + bool transS, bool swapOperands, float beta) { - C, A_values, A_indices, A_offsets, B, transA, swapOperands, beta; - ABORT("CSRProd is not yet implemented for CPU"); + C, S_values, S_indices, S_offsets, D; + + // Note: The CPU implementation currently only implements what's needed for decoding. + + // interpret tensor dimensions as matrix dimensions + const auto& shapeC = C->shape(); + const auto& shapeD = D->shape(); + // If swapOperands, S and D are swapped (C = D x S instead of C = S x D). + // In that case, in the next 6 lines, please read all dimensions as if they were reversed in order. + auto rowsC = shapeC[-(int)swapOperands]; + auto colsC = shapeC.elements() / rowsC; + auto rowsD = shapeD[-(int)swapOperands]; + auto colsD = shapeD.elements() / rowsD; + auto rowsS = transS ? rowsD : rowsC; + auto colsS = transS ? rowsC : rowsD; + ABORT_IF(colsD != colsC, "Inconsistent outer dimensions in CSR product"); + if (swapOperands) { // make rowsX actual row dimensions again, likewise colsX + std::swap(rowsC, colsC); + std::swap(rowsD, colsD); + std::swap(rowsS, colsS); + } + // sparse arrays + auto numValues = S_values->shape().elements(); + auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); + ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); + if (!transS && !swapOperands) { + // C = S * D, where D = CSR matrix + const auto* offsets = S_offsets->data(); + const auto* indices = S_indices->data(); + const auto* values = S_values->data(); + const auto* dataD = D->data(); + auto* dataC = C->data(); + ABORT_IF(beta != 0 && beta != 1, "cpu::CSRProd only supports beta = 0 or 1"); + for (size_t i = 0; i < rowsC; i++) { + auto add = (beta == 1); // first element: overwrite or add according to beta; subsequent elements: add + for (size_t kk = offsets[i]; kk < offsets[i + 1]; kk++) { + auto k = indices[kk]; // fetch the non-zero row + auto valS = values[kk]; // and the value from that row + // This code is written with the hope for good vectorization, and the hope + // that adding to memory will be done efficiently by the caching system. + if (valS == 1) + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = dataD[k * colsC/*==colsD*/ + j]; // this is a memcpy() + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += dataD[k * colsC/*==colsD*/ + j]; // this is a contiguous-vector addition + else + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = valS * dataD[k * colsC/*==colsD*/ + j]; + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += valS * dataD[k * colsC/*==colsD*/ + j]; // notice the += + add = true; // next iteration will add to existing result + } + } + } + else + ABORT("CSRProd for transS={}, swapOperands={} is not yet implemented for CPU", transS, swapOperands); } } // namespace cpu From 569cd1be5f3125869bf32c785f48ea1c2690519c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 16 Apr 2019 14:47:55 -0700 Subject: [PATCH 421/838] increased logging for invalid factrors --- src/data/factored_vocab.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 619fe9841..d168cb4bf 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -395,7 +395,11 @@ Word FactoredVocab::string2word(const std::string& w) const { WordIndex u; bool found = factorVocab_.tryFind(i == 0 ? parts[i] : sep + parts[i], u); if (!found) { - LOG_ONCE(info, "WARNING: Unknown factor '{}' in '{}'; mapping to '{}'", parts[i], w, word2string(getUnkId())); + static int logs = 100; + if (logs > 0) { + logs--; + LOG(info, "WARNING: Unknown factor '{}' in '{}'; mapping to '{}'", parts[i], w, word2string(getUnkId())); + } return getUnkId(); } // convert u to relative u within factor group range From 44518f6a5bec0ef49955f5c8adb27e48a46a3d3e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 16 Apr 2019 21:34:44 -0700 Subject: [PATCH 422/838] fixed a gcc warning --- src/tensors/cpu/prod.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index fd18ea1c2..ac13ccee8 100644 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -198,9 +198,8 @@ void CSRProd(marian::Tensor C, std::swap(rowsS, colsS); } // sparse arrays - auto numValues = S_values->shape().elements(); auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length - ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); numOffsets; ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); if (!transS && !swapOperands) { // C = S * D, where D = CSR matrix From b96d9b33ebf2593d63a971e3ab50ab3792999037 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 16 Apr 2019 21:34:44 -0700 Subject: [PATCH 423/838] fixed a build breal --- src/tensors/cpu/prod.cpp | 62 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index 408e47913..ea8b1359d 100644 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -176,8 +176,66 @@ void CSRProd(marian::Tensor C, bool transA, bool swapOperands, float beta) { - C, A_values, A_indices, A_offsets, B, transA, swapOperands, beta; - ABORT("CSRProd is not yet implemented for CPU"); + C, S_values, S_indices, S_offsets, D; + + // Note: The CPU implementation currently only implements what's needed for decoding. + + // interpret tensor dimensions as matrix dimensions + const auto& shapeC = C->shape(); + const auto& shapeD = D->shape(); + // If swapOperands, S and D are swapped (C = D x S instead of C = S x D). + // In that case, in the next 6 lines, please read all dimensions as if they were reversed in order. + auto rowsC = shapeC[-(int)swapOperands]; + auto colsC = shapeC.elements() / rowsC; + auto rowsD = shapeD[-(int)swapOperands]; + auto colsD = shapeD.elements() / rowsD; + auto rowsS = transS ? rowsD : rowsC; + auto colsS = transS ? rowsC : rowsD; + ABORT_IF(colsD != colsC, "Inconsistent outer dimensions in CSR product"); + if (swapOperands) { // make rowsX actual row dimensions again, likewise colsX + std::swap(rowsC, colsC); + std::swap(rowsD, colsD); + std::swap(rowsS, colsS); + } + // sparse arrays + auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); numOffsets; + ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); + if (!transS && !swapOperands) { + // C = S * D, where D = CSR matrix + const auto* offsets = S_offsets->data(); + const auto* indices = S_indices->data(); + const auto* values = S_values->data(); + const auto* dataD = D->data(); + auto* dataC = C->data(); + ABORT_IF(beta != 0 && beta != 1, "cpu::CSRProd only supports beta = 0 or 1"); + for (size_t i = 0; i < rowsC; i++) { + auto add = (beta == 1); // first element: overwrite or add according to beta; subsequent elements: add + for (size_t kk = offsets[i]; kk < offsets[i + 1]; kk++) { + auto k = indices[kk]; // fetch the non-zero row + auto valS = values[kk]; // and the value from that row + // This code is written with the hope for good vectorization, and the hope + // that adding to memory will be done efficiently by the caching system. + if (valS == 1) + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = dataD[k * colsC/*==colsD*/ + j]; // this is a memcpy() + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += dataD[k * colsC/*==colsD*/ + j]; // this is a contiguous-vector addition + else + if (!add) + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] = valS * dataD[k * colsC/*==colsD*/ + j]; + else + for (size_t j = 0; j < colsC; j++) + dataC[i * colsC/*==colsD*/ + j] += valS * dataD[k * colsC/*==colsD*/ + j]; // notice the += + add = true; // next iteration will add to existing result + } + } + } + else + ABORT("CSRProd for transS={}, swapOperands={} is not yet implemented for CPU", transS, swapOperands); } } // namespace cpu From a38d09f25d66eac90b817755ccd9e70feb3afb8a Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 16 Apr 2019 21:34:44 -0700 Subject: [PATCH 424/838] fixed a gcc warning --- src/tensors/cpu/prod.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp index fd18ea1c2..ac13ccee8 100644 --- a/src/tensors/cpu/prod.cpp +++ b/src/tensors/cpu/prod.cpp @@ -198,9 +198,8 @@ void CSRProd(marian::Tensor C, std::swap(rowsS, colsS); } // sparse arrays - auto numValues = S_values->shape().elements(); auto numOffsets = S_offsets->shape().elements() - 1; // -1 since last value is length - ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); + ABORT_IF(numOffsets != rowsS, "Unexpected number of rows in CSR argument"); numOffsets; ABORT_IF(S_values->shape() != S_indices->shape(), "CSR values and indices must have the same size"); if (!transS && !swapOperands) { // C = S * D, where D = CSR matrix From 1a0849c72c47d811921568a21bafbc13c60a89a3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 17 Apr 2019 16:07:56 -0700 Subject: [PATCH 425/838] bug fix: target embedding matrix must be created before output layer uses it (if tied) --- src/models/decoder.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/models/decoder.h b/src/models/decoder.h index 697aef3b6..62f52eabe 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -88,21 +88,22 @@ class DecoderBase { int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; Expr selectedEmbs; - if(words.empty()) { - selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); - } else { - // embeddings are loaded from model during translation, no fixing required - auto yEmbFactory = embedding() // - ("dimVocab", dimTrgVoc) // - ("dimEmb", dimTrgEmb); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - yEmbFactory("prefix", "Wemb"); - else - yEmbFactory("prefix", prefix_ + "_Wemb"); + // embeddings are loaded from model during translation, no fixing required + auto yEmbFactory = embedding() // + ("dimVocab", dimTrgVoc) // + ("dimEmb", dimTrgEmb); - auto yEmb = yEmbFactory.construct(graph); + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + yEmbFactory("prefix", "Wemb"); + else + yEmbFactory("prefix", prefix_ + "_Wemb"); + + auto yEmb = yEmbFactory.construct(graph); + if(words.empty()) { + selectedEmbs = graph->constant({1, 1, dimBatch, dimTrgEmb}, inits::zeros); + } else { selectedEmbs = yEmb->apply(words, {dimBeam, 1, dimBatch, dimTrgEmb}); } state->setTargetHistoryEmbeddings(selectedEmbs); From 4ed735dc7f808cb1532d501958861d53ada38e2e Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Thu, 18 Apr 2019 20:53:51 +0100 Subject: [PATCH 426/838] Install signal handler for SIGTERM to save model before exiting. --- src/CMakeLists.txt | 1 + src/training/scheduler.cpp | 32 +++++++++++++++++++++++++++++ src/training/scheduler.h | 42 ++++++++++++++++++++++++++++++-------- 3 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 src/training/scheduler.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fbafc3b99..c287169dc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -78,6 +78,7 @@ add_library(marian STATIC training/graph_group_multinode_sync.cpp training/validator.cpp training/communicator.cpp + training/scheduler.cpp # this is only compiled to catch build errors, but not linked microsoft/quicksand.cpp diff --git a/src/training/scheduler.cpp b/src/training/scheduler.cpp new file mode 100644 index 000000000..f508f3069 --- /dev/null +++ b/src/training/scheduler.cpp @@ -0,0 +1,32 @@ +#include "scheduler.h" +#include + +namespace marian { +bool Scheduler::sigterm_{false}; +bool Scheduler::sigusr1_{false}; +bool Scheduler::sigusr2_{false}; + +void +Scheduler:: +signalHandler_(int sig) { + switch (sig) { + case SIGTERM: Scheduler::sigterm_ = true; break; + case SIGUSR1: Scheduler::sigusr1_ = true; break; + case SIGUSR2: Scheduler::sigusr2_ = true; break; + default: + ABORT("This signal handler should not have been installed for signal ", + strsignal(sig)); + } +} + + +void +Scheduler:: +installSignalHandlers_() { + // TODO: use sigaction instead of signal + signal(SIGTERM, Scheduler::signalHandler_); + signal(SIGUSR1, Scheduler::signalHandler_); + signal(SIGUSR2, Scheduler::signalHandler_); +} + +} diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 9c3cb7e2d..68ebf395a 100644 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -19,6 +19,13 @@ class Scheduler : public TrainingObserver { timer::Timer timer_; timer::Timer heartBeatTimer_; + std::string validFreq_; + std::string saveFreq_; + + static bool sigterm_, sigusr1_, sigusr2_; + void installSignalHandlers_(); + static void signalHandler_(int sig); + // determine scheduled LR decay factor (--lr-decay-inv-sqrt option) float getScheduledLRDecayFactor(const TrainingState& state) const { auto args = options_->get>("lr-decay-inv-sqrt"); @@ -105,6 +112,10 @@ class Scheduler : public TrainingObserver { } public: + bool gotSigTerm() const { return sigterm_; } + // bool gotSigUsr1() const { return sigusr1_; } // currently has no effect + // bool gotSigUsr2() const { return sigusr2_; } // currently has no effect + // test if any parameters specify dynamic MB scaling bool isDynamicMBSizeScaling() const { auto mbWarmup = SchedulingParameter::parse(options_->get("mini-batch-warmup")); @@ -144,11 +155,17 @@ class Scheduler : public TrainingObserver { Scheduler(Ptr options, Ptr state) : options_(options), state_(state) { + validFreq_ = options_->get("valid-freq"); + saveFreq_ = options_->get("save-freq"); ABORT_IF(state_->factor != 1, "state.factor unexpectedly not 1 at this point??"); updateLearningRate(*state); } - bool keepGoing() { + bool keepGoing(bool checkForSigTerm=true) { + + if (checkForSigTerm && sigterm_) // received signam SIGERM => exit gracefully + return false; + // stop if it reached the maximum number of epochs size_t stopAfterEpochs = options_->get("after-epochs"); if(stopAfterEpochs > 0 && state_->epochs > stopAfterEpochs) @@ -175,7 +192,12 @@ class Scheduler : public TrainingObserver { } void started() { LOG(info, "Training started"); } - void finished() { LOG(info, "Training finished"); } + void finished() { + if (keepGoing(false)) // false means: ignore sigterm flag + LOG(info, "Training finished"); + else + LOG(info, "Training interrupted (SIGTERM)."); + } void addValidator(Ptr validator) { validators_.push_back(validator); @@ -192,20 +214,22 @@ class Scheduler : public TrainingObserver { bool validating() { return (!validators_.empty() - && state_->enteredNewPeriodOf(options_->get("valid-freq")) + && state_->enteredNewPeriodOf(validFreq_) && keepGoing()); } bool saving() { - return state_->enteredNewPeriodOf(options_->get("save-freq")); + return state_->enteredNewPeriodOf(saveFreq_); } void validate(const std::vector>& graphs, bool final = false) { - // Do not validate if already validated (for instance, after the model is - // loaded) or if validation is scheduled for another update - if(state_->validated - || (!state_->enteredNewPeriodOf(options_->get("valid-freq")) && !final)) + // SKIP VALIDATION IF + // - it's not time to validate anyway + // - we received SIGTERM + // - the current state has been validated (e.g., model was just loaded) + if((!state_->enteredNewPeriodOf(validFreq_) && !final) + || state_->validated || sigterm_) return; bool firstValidator = true; @@ -236,7 +260,7 @@ class Scheduler : public TrainingObserver { } state_->validators[validator->type()]["last-best"] - = validator->lastBest(); + = validator->lastBest(); state_->validators[validator->type()]["stalled"] = validator->stalled(); // notify training observers if the first validator did not improve From 16917363598165a9fe5174cb318ca4aa1bd73470 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sun, 21 Apr 2019 22:27:40 +0100 Subject: [PATCH 427/838] BatchGenerator now remembers if it has just restored the corpus ... ... in accordance with a saved training progress state. --- src/data/batch_generator.h | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h index ce673cd2a..d013cdd56 100644 --- a/src/data/batch_generator.h +++ b/src/data/batch_generator.h @@ -84,7 +84,10 @@ class BatchGenerator : public RNGEngine { // this runs on a bg thread; sequencing is handled by caller, but locking is done in here std::deque fetchBatches() { typedef typename Sample::value_type Item; - auto itemCmp = [](const Item& sa, const Item& sb) { return sa.size() < sb.size(); }; // sort by element length, not content + auto itemCmp = [](const Item& sa, const Item& sb) { + // sort by element length, not content + return sa.size() < sb.size(); + }; auto cmpSrc = [itemCmp](const Sample& a, const Sample& b) { return std::lexicographical_compare( @@ -96,7 +99,10 @@ class BatchGenerator : public RNGEngine { a.rbegin(), a.rend(), b.rbegin(), b.rend(), itemCmp); }; - auto cmpNone = [](const Sample& a, const Sample& b) { return &a < &b; }; // instead sort by address, so we have something to work with + auto cmpNone = [](const Sample& a, const Sample& b) { + // sort by address, so we have something to work with + return &a < &b; + }; typedef std::function cmp_type; typedef std::priority_queue sample_queue; @@ -252,7 +258,7 @@ class BatchGenerator : public RNGEngine { BatchGenerator(Ptr data, Ptr options, Ptr stats = nullptr) - : data_(data), options_(options), stats_(stats), threadPool_(1) {} + : data_(data), options_(options), stats_(stats), threadPool_(1) { } ~BatchGenerator() { if (futureBufferedBatches_.valid()) // bg thread holds a reference to 'this', @@ -268,7 +274,11 @@ class BatchGenerator : public RNGEngine { } // @TODO: get rid of this function, begin() or constructor should figure this out - void prepare(bool shuffle = true) { + void prepare(bool shuffle) { + if(restored_) { // state was just restored, restore() calls prepare() + restored_ = false; + return; + } if(shuffle) data_->shuffle(); else @@ -276,6 +286,12 @@ class BatchGenerator : public RNGEngine { newlyPrepared_ = true; // @TODO: solve this better, maybe use options + // => for this to work best, we need to replace --no-shuffle by --shuffle + // which is true by default for train, false otherwise, or explicitly set + // --no-shuffle=true by default for translate, validate, score etc. [UG] + // for the time begin, let's stick with the explicit function parameter + // (I've disabled the default value, because it's prone to cause problems + // sooner or later; callers should know if they want to shuffle or not). shuffle_ = shuffle; // start the background pre-fetch operation @@ -287,6 +303,8 @@ class BatchGenerator : public RNGEngine { bool restore(Ptr state, bool shuffle) { if(state->epochs == 1 && state->batchesEpoch == 0) return false; + if (options_->get("no-restore-corpus")) + return false; LOG(info, "[data] Restoring the corpus state to epoch {}, batch {}", @@ -302,6 +320,7 @@ class BatchGenerator : public RNGEngine { for(size_t i = 0; i < state->batchesEpoch; ++i) next(); + restored_ = true; return true; } From d5b7af42ef539e156354e039c022dadb2a5cc43c Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sun, 21 Apr 2019 22:31:37 +0100 Subject: [PATCH 428/838] New factory function to open corpus with sqlite or not, depending on spec in Options. --- src/data/corpus.cpp | 19 +++++++++++++++++++ src/data/corpus.h | 2 ++ 2 files changed, 21 insertions(+) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index fda8df95f..0bde62322 100644 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -171,5 +171,24 @@ void Corpus::shuffleData(const std::vector& paths) { } pos_ = 0; } + +Ptr prepareTrainingData(Ptr options) { + // factory function to set up the training corpus for training + // code moved here from Train::run() in training.h + Ptr dataset; +#ifndef _MSC_VER // @TODO: include SqLite in Visual Studio project + if(!options->get("sqlite").empty()) + ABORT("SqLite presently not supported on Windows"); +#else + if(!options->get("sqlite").empty()) + dataset = New(options); +#endif + else + dataset = New(options); + dataset->prepare(); + return dataset; +} + + } // namespace data } // namespace marian diff --git a/src/data/corpus.h b/src/data/corpus.h index b459eb87a..165b0b238 100644 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -16,6 +16,8 @@ namespace marian { namespace data { +Ptr prepareTrainingData(Ptr options); + class Corpus : public CorpusBase { private: std::vector> tempFiles_; From e5027a2272710c029cf64164dc7d4b6989f3b3c4 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sun, 21 Apr 2019 22:33:59 +0100 Subject: [PATCH 429/838] New function to set up all validators in one go. --- src/training/scheduler.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 68ebf395a..4ffcbce95 100644 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -4,6 +4,7 @@ #include "training/training_state.h" #include "training/validator.h" #include "training/communicator.h" +#include "data/vocab.h" #include "layers/loss.h" namespace marian { @@ -199,6 +200,19 @@ class Scheduler : public TrainingObserver { LOG(info, "Training interrupted (SIGTERM)."); } + void setupValidators(std::vector>& vocabs) { + // setup validators if specified + if (!SchedulingParameter::parse(options_->get("valid-freq"))) + return; // no validation interval given + + if(!options_->hasAndNotEmpty("valid-sets") && + !options_->hasAndNotEmpty("valid-script-path")) + return; // no validators specified + + for(auto validator : Validators(vocabs, options_)) + addValidator(validator); + } + void addValidator(Ptr validator) { validators_.push_back(validator); From b0d041b0eac87d732c6714683f9370189b1addc6 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sun, 21 Apr 2019 22:36:59 +0100 Subject: [PATCH 430/838] Refactoring for better readability. --- src/training/training.h | 80 +++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/src/training/training.h b/src/training/training.h index 8863c165d..fb674ba52 100644 --- a/src/training/training.h +++ b/src/training/training.h @@ -11,6 +11,27 @@ namespace marian { +template +Ptr +miniBatchFit(Ptr options, Ptr data, + Ptr scheduler, Ptr mpi) { + // collect stats for fitting minibatches to available memory + Ptr stats; + if(options->get("mini-batch-fit")) { + LOG(info, + "[batching] Collecting statistics for batch fitting with step size {}", + options->get("mini-batch-fit-step")); + // @TODO this should receive a function object that can generate a fake batch; + // that way vocabs would not be exposed. + auto model = New(options, mpi); + model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling + stats = model->collectStats(data->getVocabs()); + LOG(info, "[batching] Done. Typical MB size is {} target words", + stats->estimateTypicalTrgWords()); + } + return stats; +} + template class Train : public ModelTask { private: @@ -22,64 +43,35 @@ class Train : public ModelTask { void run() override { using namespace data; - Ptr dataset; - if(!options_->get("sqlite").empty()) -#ifndef _MSC_VER // @TODO: include SqLite in Visual Studio project - dataset = New(options_); -#else - ABORT("SqLite presently not supported on Windows"); -#endif - else - dataset = New(options_); - - dataset->prepare(); - + // TRAINING RUN SETUP + Ptr dataset = prepareTrainingData(options_); auto trainState = New(options_->get("learn-rate")); auto scheduler = New(options_, trainState); - auto mpi = initMPI(/*multiThreaded=*/!options_->get("sync-sgd")); // @TODO: do we need the multiThreaded distinction at all? - - Ptr stats; - if(options_->get("mini-batch-fit")) { - LOG(info, - "[batching] Collecting statistics for batch fitting with step size {}", - options_->get("mini-batch-fit-step")); - // @TODO this should receive a function object that can generate a fake batch; - // that way vocabs would not be exposed. - auto model = New(options_, mpi); - model->setScheduler(scheduler); // collectStats() needs to know about dynamic MB scaling - stats = model->collectStats(dataset->getVocabs()); - LOG(info, "[batching] Done. Typical MB size is {} target words", stats->estimateTypicalTrgWords()); - } - - if((options_->hasAndNotEmpty("valid-sets") || options_->hasAndNotEmpty("valid-script-path")) - && SchedulingParameter::parse(options_->get("valid-freq"))) { - for(auto validator : Validators(dataset->getVocabs(), options_)) - scheduler->addValidator(validator); - } + auto mpi = initMPI(/*multiThreaded=*/!options_->get("sync-sgd")); + // @TODO: do we need the multiThreaded distinction in initMPI at all? + Ptr stats + = miniBatchFit(options_, dataset, scheduler, mpi); auto batchGenerator = New(dataset, options_, stats); - + scheduler->setupValidators(dataset->getVocabs()); scheduler->registerTrainingObserver(batchGenerator); auto model = New(options_, mpi); model->setScheduler(scheduler); - model->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); // needed for dynamic MB scaling - model->load(); - - // @TODO: shuffle_ as a private attribute in BG - auto shuffle = !options_->get("no-shuffle"); - bool restored = !options_->get("no-restore-corpus") - && batchGenerator->restore(trainState, shuffle); + model->setTypicalTrgBatchWords(batchGenerator->estimateTypicalTrgBatchWords()); + // typical no. of trg wrds in batch is needed for dynamic MiniBatch scaling + model->load(); //!! ALSO CHANGES scheduler AND trainState ON A RESUMED RUN! // -- main training loop scheduler->started(); + bool shuffle = !options_->get("no-shuffle"); + batchGenerator->restore(trainState,shuffle); while(scheduler->keepGoing()) { - if(!restored) - batchGenerator->prepare(shuffle); - restored = false; + batchGenerator->prepare(shuffle); // main training loop for one epoch - for(auto batchIt = std::begin(*batchGenerator); // @TODO: try to use for(auto ...) + // @TODO: try to use for(auto ...) + for(auto batchIt = std::begin(*batchGenerator); batchIt != std::end(*batchGenerator); batchIt++) { if (!scheduler->keepGoing()) From a12f357de4b690b8bde854d216fc4b67292918b7 Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Mon, 22 Apr 2019 03:06:54 +0100 Subject: [PATCH 431/838] CMakeList.txt: More informative error message when appropriate CUDA library cannot be found. --- CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 022b97019..6dc13c0ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,7 +180,15 @@ if(USE_STATIC_LIBS) endif() else(CUDA_FOUND) - message(FATAL_ERROR "CUDA has not been found, set -DCOMPILE_CUDA=off to avoid this check and to compile the CPU version only") + message(" +Cannot find suitable CUDA libraries. Specify the path explicitly with + -DCUDA_TOOLKIT_ROOT_DIR=/path/to/appropriate/cuda/installation + (hint: try /usr/local/$(readlink /usr/local/cuda)) +OR compile the CPU-only version of Marian with + -DCOMPILE_CUDA=off +") + message(FATAL_ERROR + "FATAL ERROR: No suitable CUDA library found.") endif(CUDA_FOUND) else(COMPILE_CUDA) From f37958a43100957f1262919a2102e83f10eedd75 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 22 Apr 2019 17:41:39 -0700 Subject: [PATCH 432/838] added shortlists to lemma factor --- src/common/binary.cpp | 17 ++++++++----- src/common/file_stream.h | 33 +++++++++++--------------- src/data/factored_vocab.cpp | 12 ++++++++++ src/data/factored_vocab.h | 1 + src/data/vocab.cpp | 3 +++ src/data/vocab.h | 3 +++ src/data/vocab_base.h | 3 +++ src/layers/generic.cpp | 46 ++++++++++++++++++++++-------------- src/layers/generic.h | 4 ++-- src/microsoft/quicksand.cpp | 1 + src/microsoft/quicksand.h | 1 + src/translator/beam_search.h | 15 +++++++----- 12 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/common/binary.cpp b/src/common/binary.cpp index 983c15b58..d4f443985 100644 --- a/src/common/binary.cpp +++ b/src/common/binary.cpp @@ -68,15 +68,20 @@ void loadItems(const void* current, std::vector& items, bool mapped) { void loadItems(const std::string& fileName, std::vector& items) { // Read file into buffer size_t fileSize = filesystem::fileSize(fileName); - char* ptr = new char[fileSize]; + std::vector buf(fileSize); +#if 1 // for some reason, the #else branch fails with "file not found" in the *read* operation (open succeeds) + FILE *f = fopen(fileName.c_str(), "rb"); + ABORT_IF(f == nullptr, "Error {} ('{}') opening file '{}'", errno, strerror(errno), fileName); + auto rc = fread(buf.data(), sizeof(*buf.data()), buf.size(), f); + ABORT_IF(rc != buf.size(), "Error {} ('{}') reading file '{}'", errno, strerror(errno), fileName); + fclose(f); +#else io::InputFileStream in(fileName); - in.read(ptr, fileSize); + in.read(buf.data(), buf.size()); +#endif // Load items from buffer without mapping - loadItems(ptr, items, false); - - // Delete buffer - delete[] ptr; + loadItems(buf.data(), items, false); } io::Item getItem(const void* current, const std::string& varName) { diff --git a/src/common/file_stream.h b/src/common/file_stream.h index 1fe6d91d1..ec03b1565 100644 --- a/src/common/file_stream.h +++ b/src/common/file_stream.h @@ -139,10 +139,10 @@ class InputFileStream { ABORT_IF(!marian::filesystem::exists(file_), "File '{}' does not exist", file); if(file_.extension() == marian::filesystem::Path(".gz")) - // @TODO: consider make_unique for next refactoring - istream_.reset(new zstr::ifstream(file_.string())); + istream_ = std::make_unique(file_.string()); else - istream_.reset(new std::ifstream(file_.string())); + istream_ = std::make_unique(file_.string()); + ABORT_IF(fail(), "Error {} ({}) opening file '{}'", errno, strerror(errno), path()); } InputFileStream(TemporaryFile& tempfile) @@ -150,8 +150,8 @@ class InputFileStream { lseek(tempfile.getFileDescriptor(), 0, SEEK_SET); namespace bio = boost::iostreams; - fdsBuffer_.reset(new bio::stream_buffer(fds_)); - istream_.reset(new std::istream(fdsBuffer_.get())); + fdsBuffer_ = std::make_unique>(fds_); + istream_ = std::make_unique(fdsBuffer_.get()); } InputFileStream(std::istream& strm) @@ -187,7 +187,7 @@ class InputFileStream { friend InputFileStream& operator>>(InputFileStream& stream, T& t) { *stream.istream_ >> t; // bad() seems to be correct here. Should not abort on EOF. - ABORT_IF(stream.bad(), "Error reading from file '{}'", stream.path()); + ABORT_IF(stream.bad(), "Error {} ({}) reading from file '{}'", errno, strerror(errno), stream.path()); return stream; } @@ -195,7 +195,7 @@ class InputFileStream { size_t read(T* ptr, size_t num = 1) { istream_->read((char*)ptr, num * sizeof(T)); // fail() seems to be correct here. Failure to read should abort. - ABORT_IF(fail(), "Error reading from file '{}'", path()); + ABORT_IF(fail(), "Error {} ({}) reading from file '{}'", errno, strerror(errno), path()); return num * sizeof(T); } @@ -236,10 +236,9 @@ class OutputFileStream { public: OutputFileStream(const std::string& file) : file_(file) { if(file_.extension() == marian::filesystem::Path(".gz")) - ostream_.reset(new zstr::ofstream(file_.string())); + ostream_ = std::make_unique(file_.string()); else - ostream_.reset(new std::ofstream(file_.string())); - + ostream_ = std::make_unique(file_.string()); ABORT_IF(!marian::filesystem::exists(file_), "File '{}' could not be opened", file); } @@ -248,25 +247,21 @@ class OutputFileStream { lseek(tempfile.getFileDescriptor(), 0, SEEK_SET); namespace bio = boost::iostreams; - fdsBuffer_.reset(new bio::stream_buffer(fds_)); - ostream_.reset(new std::ostream(fdsBuffer_.get())); + fdsBuffer_ = std::make_unique>(fds_); + ostream_ = std::make_unique(fdsBuffer_.get()); } OutputFileStream(std::ostream& strm) { - ostream_.reset(new std::ostream(strm.rdbuf())); + ostream_ = std::make_unique(strm.rdbuf()); } operator std::ostream&() { return *ostream_; } operator bool() { return (bool)*ostream_; } - bool bad() const { - return ostream_->bad(); - } + bool bad() const { return ostream_->bad(); } - bool fail() const { - return ostream_->fail(); - } + bool fail() const { return ostream_->fail(); } template friend OutputFileStream& operator<<(OutputFileStream& stream, const T& t) { diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index d168cb4bf..509815f4e 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -517,6 +517,18 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return utils::findReplace(utils::findReplace(utils::findReplace(utils::findReplace(line, "|cn|wb", "|ci|wb", /*all=*/true), "|cn|gl-", "|ci|gl-", /*all=*/true), "@CN@WB", "@CI@WB", /*all=*/true), "@CN@GL-", "@CI@GL-", /*all=*/true); } +// convert word indices to indices of shortlist items +// We only shortlist the lemmas, hence return the lemma index (offset to correctly index into the concatenated W matrix). +// This strange pointer-based interface is for ease of interaction with our production environment. +/*virtual*/ void FactoredVocab::transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const { + for (; num-- > 0; ptr++) { + auto word = Word::fromWordIndex(*ptr); + auto wordString = word2string(word); + auto lemmaIndex = getFactor(word, 0) + groupRanges_[0].first; + *ptr = lemmaIndex; + } +} + // generate a valid random factored word (used by collectStats()) /*virtual*/ Word FactoredVocab::randWord() const /*override final*/ { auto numGroups = getNumGroups(); diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 9a916aa82..3ff338e23 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -41,6 +41,7 @@ class FactoredVocab : public IVocab { virtual Word getUnkId() const override final { return unkId_; } virtual std::string toUpper(const std::string& line) const override final; virtual std::string toEnglishTitleCase(const std::string& line) const override final; + virtual void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const override final; WordIndex getUnkIndex() const { return (WordIndex)getFactor(getUnkId(), 0); } // used in decoding virtual void createFake() override final { ABORT("[data] Fake FactoredVocab vocabulary not supported"); } virtual Word randWord() const override final; diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp index 2cafb3313..b48d5eca0 100755 --- a/src/data/vocab.cpp +++ b/src/data/vocab.cpp @@ -143,4 +143,7 @@ std::string Vocab::toUpper(const std::string& line) const { return vImpl_->toUpp // for corpus augmentation: convert string to title case std::string Vocab::toEnglishTitleCase(const std::string& line) const { return vImpl_->toEnglishTitleCase(line); } +// for short-list generation +void Vocab::transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const { vImpl_->transcodeToShortlistInPlace(ptr, num); } + } // namespace marian diff --git a/src/data/vocab.h b/src/data/vocab.h index 1de93f5b4..a15e202f0 100755 --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -76,6 +76,9 @@ class Vocab { // for corpus augmentation: convert string to title case std::string toEnglishTitleCase(const std::string& line) const; + // for short-list generation + void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const; + // create fake vocabulary for collecting batch statistics void createFake(); diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index bf276f5de..238f9146b 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -49,6 +49,9 @@ class IVocab { virtual std::string toUpper(const std::string& line) const { return line; } virtual std::string toEnglishTitleCase(const std::string& line) const { return line; } + // this function is an identity mapping for default vocabularies, hence do nothing + virtual void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const { } + virtual void createFake() = 0; virtual Word randWord() const { diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 366b05f35..1724f2038 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -80,7 +80,8 @@ namespace marian { //} // get logits for one factor group - Expr Logits::getFactoredLogits(size_t groupIndex, const std::vector& selIdx /*= {}*/, size_t beamSize /*= 0*/) const { + // For groupIndex == 0, the function also requires the shortlist if there is one. + Expr Logits::getFactoredLogits(size_t groupIndex, Ptr shortlist /*= nullptr*/, const std::vector& selIdx /*= {}*/, size_t beamSize /*= 0*/) const { ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); auto sel = logits_[groupIndex]->loss(); // [localBeamSize, 1, dimBatch, dimFactorVocab] @@ -94,7 +95,7 @@ namespace marian { auto numGroups = getNumFactorGroups(); for (size_t g = 1; g < numGroups; g++) { auto factorMaxima = max(logits_[g]->loss(), -1); - auto factorMasks = constant(getFactorMasks(g)); + auto factorMasks = constant(getFactorMasks(g, shortlist ? shortlist->indices() : std::vector())); sel = sel + factorMaxima * factorMasks; // those lemmas that don't have a factor get multiplied with 0 } } @@ -178,13 +179,15 @@ namespace marian { //} // return a vector of 1 or 0 indicating for each lemma whether it has a specific factor - std::vector Logits::getFactorMasks(size_t factorGroup) const { // [lemmaIndex] -> 1.0 for words that do have this factor; else 0 - size_t numLemmas = factoredVocab_->getGroupRange(0).second - factoredVocab_->getGroupRange(0).first; + // If 'indices' is given, then return the masks for the indices; otherwise for all lemmas + std::vector Logits::getFactorMasks(size_t factorGroup, const std::vector& indices) const { // [lemmaIndex] -> 1.0 for words that do have this factor; else 0 + size_t n = indices.empty() ? (factoredVocab_->getGroupRange(0).second - factoredVocab_->getGroupRange(0).first) : indices.size(); std::vector res; - res.reserve(numLemmas); - // @TODO: we should rearange lemmaHasFactorGroup as vector[groups[lemma] of float; then move this into FactoredVocab - for (size_t lemma = 0; lemma < numLemmas; lemma++) { - res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); + res.reserve(n); + // @TODO: we should rearrange lemmaHasFactorGroup as vector[groups[i] of float; then move this into FactoredVocab + for (size_t i = 0; i < n; i++) { + auto lemma = indices.empty() ? i : (indices[i] - factoredVocab_->getGroupRange(0).first); + res.push_back((float)factoredVocab_->lemmaHasFactorGroup(i, factorGroup)); } return res; } @@ -225,7 +228,6 @@ namespace marian { factoredVocab_ = FactoredVocab::tryCreateAndLoad(options_->get("vocab", "")); if (factoredVocab_) { - ABORT_IF(shortlist_, "Shortlists are presently not compatible with factored embeddings"); numOutputClasses = (int)factoredVocab_->factorVocabSize(); LOG(info, "[embedding] Factored outputs enabled"); } @@ -247,14 +249,12 @@ namespace marian { Logits Output::applyAsLogits(Expr input) /*override final*/ { lazyConstruct(input->shape()[-1]); - if (shortlist_) { - if (!cachedShortWt_) { // short versions of parameters are cached within one batch, then clear()ed - cachedShortWt_ = index_select(Wt_, isLegacyUntransposedW ? -1 : 0, shortlist_->indices()); - cachedShortb_ = index_select(b_ , -1, shortlist_->indices()); - } - return Logits(affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true)); + if (shortlist_ && !cachedShortWt_) { // shortlisted versions of parameters are cached within one batch, then clear()ed + cachedShortWt_ = index_select(Wt_, isLegacyUntransposedW ? -1 : 0, shortlist_->indices()); + cachedShortb_ = index_select(b_ , -1, shortlist_->indices()); } - else if (factoredVocab_) { + + if (factoredVocab_) { auto graph = input->graph(); // project each factor separately @@ -268,8 +268,15 @@ namespace marian { continue; ABORT_IF(g > 0 && range.first != factoredVocab_->getGroupRange(g-1).second, "Factor groups must be consecutive (group {} vs predecessor)", g); // slice this group's section out of W_ - auto factorWt = slice(Wt_, isLegacyUntransposedW ? -1 : 0, Slice((int)range.first, (int)range.second)); - auto factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); + Expr factorWt, factorB; + if (g == 0 && shortlist_) { + factorWt = cachedShortWt_; + factorB = cachedShortb_; + } + else { + factorWt = slice(Wt_, isLegacyUntransposedW ? -1 : 0, Slice((int)range.first, (int)range.second)); + factorB = slice(b_, -1, Slice((int)range.first, (int)range.second)); + } // @TODO: b_ should be a vector, not a matrix; but shotlists use cols() in, which requires a matrix auto factorLogits = affine(input1, factorWt, factorB, false, /*transB=*/isLegacyUntransposedW ? false : true); // [B... x U] factor logits // optionally add lemma-dependent bias @@ -285,6 +292,7 @@ namespace marian { // optionally add a soft embedding of lemma back to create some lemma dependency // @TODO: if this works, move it into lazyConstruct const int lemmaDimEmb = options_->get("lemma-dim-emb", 0); + ABORT_IF(shortlist_ && lemmaDimEmb != 0, "Lemma re-embedding with short list is not yet implemented"); if (lemmaDimEmb < 0 && g == 0) { LOG_ONCE(info, "[embedding] using lemma-dependent bias"); factorLogits = logsoftmax(factorLogits); // explicitly, since we do that again later @@ -307,6 +315,8 @@ namespace marian { } return Logits(std::move(allLogits), factoredVocab_); } + else if (shortlist_) + return Logits(affine(input, cachedShortWt_, cachedShortb_, false, /*transB=*/isLegacyUntransposedW ? false : true)); else return Logits(affine(input, Wt_, b_, false, /*transB=*/isLegacyUntransposedW ? false : true)); } diff --git a/src/layers/generic.h b/src/layers/generic.h index 01c1c1721..775fbbe01 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -81,7 +81,7 @@ class Logits { Logits(std::vector>&& logits, Ptr embeddingFactorMapping) // factored-output constructor : logits_(std::move(logits)), factoredVocab_(embeddingFactorMapping) {} Expr getLogits() const; // assume it holds logits: get them, possibly aggregating over factors - Expr getFactoredLogits(size_t groupIndex, const std::vector& hypIndices = {}, size_t beamSize = 0) const; // get logits for only one factor group, with optional reshuffle + Expr getFactoredLogits(size_t groupIndex, Ptr shortlist = nullptr, const std::vector& hypIndices = {}, size_t beamSize = 0) const; // get logits for only one factor group, with optional reshuffle //Ptr getRationalLoss() const; // assume it holds a loss: get that Expr applyLossFunction(const Words& labels, const std::function& lossFn) const; Logits applyUnaryFunction(const std::function& f) const; // clone this but apply f to all loss values @@ -97,7 +97,6 @@ class Logits { }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices //std::vector getFactorMasks(const Words& words, size_t factorGroup) const; - std::vector getFactorMasks(size_t factorGroup) const; float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } @@ -109,6 +108,7 @@ class Logits { Expr constant(const Shape& shape, const std::vector& data) const { return graph()->constant(shape, inits::from_vector(data), Type::uint32); } template Expr constant(const std::vector& data) const { return constant(Shape{(int)data.size()}, data); } // same as constant() but assuming vector Expr indices(const std::vector& data) const { return graph()->indices(data); } // actually the same as constant(data) for this data type + std::vector getFactorMasks(size_t factorGroup, const std::vector& indices) const; private: // members // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 51f857b52..98ff2978c 100755 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -39,6 +39,7 @@ class VocabWrapper : public IVocabWrapper { WordIndex encode(const std::string& word) const override { return (*pImpl_)[word].toWordIndex(); } std::string decode(WordIndex id) const override { return (*pImpl_)[Word::fromWordIndex(id)]; } size_t size() const override { return pImpl_->size(); } + void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const override { pImpl_->transcodeToShortlistInPlace(ptr, num); } Ptr getVocab() const { return pImpl_; } }; diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index 22fe97b67..bc31e1515 100755 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -35,6 +35,7 @@ class IVocabWrapper { virtual WordIndex encode(const std::string& word) const = 0; virtual std::string decode(WordIndex id) const = 0; virtual size_t size() const = 0; + virtual void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const = 0; }; class IBeamSearchDecoder { diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 569e7756f..5445e079b 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -79,13 +79,11 @@ class BeamSearch { // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. auto shortlist = scorers_[0]->getShortlist(); - if (shortlist) - word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); - else if (factoredVocab) { + if (factoredVocab) { // For factored decoding, the word is built over multiple decoding steps, // starting with the lemma, then adding factors one by one. if (factorGroup == 0) { - word = factoredVocab->lemma2Word(wordIdx); + word = factoredVocab->lemma2Word(shortlist ? shortlist->reverseMap(wordIdx) : wordIdx); // @BUGBUG: reverseMap is only correct if factoredVocab_->getGroupRange(0).first == 0 //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); } else { @@ -98,6 +96,8 @@ class BeamSearch { prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words } } + else if (shortlist) + word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); else word = Word::fromWordIndex(wordIdx); @@ -331,7 +331,10 @@ class BeamSearch { if (numFactorGroups == 1) // @TODO: this branch can go away logProbs = states[i]->getLogProbs().getLogits(); // [localBeamSize, 1, dimBatch, dimVocab] else - logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup); // [localBeamSize, 1, dimBatch, dimVocab] + { + auto shortlist = scorers_[i]->getShortlist(); + logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, shortlist); // [localBeamSize, 1, dimBatch, dimVocab] + } //logProbs->debug("logProbs"); } else { @@ -344,7 +347,7 @@ class BeamSearch { // and push out other hypotheses. Hence, we exclude those here by setting the path score to // INVALID_PATH_SCORE. Instead, toHyps() explicitly propagates those hyps by simply copying the // previous hypothesis. - logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] + logProbs = states[i]->getLogProbs().getFactoredLogits(factorGroup, /*shortlist=*/ nullptr, hypIndices, localBeamSize); // [localBeamSize, 1, dimBatch, dimVocab] } // expand all hypotheses, [localBeamSize, 1, dimBatch, 1] -> [localBeamSize, 1, dimBatch, dimVocab] expandedPathScores = expandedPathScores + scorers_[i]->getWeight() * logProbs; From 1da8d5281f660637a118fd806d932e2aa42268bc Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 23 Apr 2019 13:58:29 -0700 Subject: [PATCH 433/838] (fixed newlines) --- src/translator/beam_search.h | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 5445e079b..35c2d741a 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -79,23 +79,23 @@ class BeamSearch { // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. auto shortlist = scorers_[0]->getShortlist(); - if (factoredVocab) { - // For factored decoding, the word is built over multiple decoding steps, - // starting with the lemma, then adding factors one by one. - if (factorGroup == 0) { - word = factoredVocab->lemma2Word(shortlist ? shortlist->reverseMap(wordIdx) : wordIdx); // @BUGBUG: reverseMap is only correct if factoredVocab_->getGroupRange(0).first == 0 - //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); - } - else { - //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), - // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); - word = beam[beamHypIdx]->getWord(); - ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); - word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); - prevBeamHypIdx = prevHyp->getPrevStateIndex(); - prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words - } - } + if (factoredVocab) { + // For factored decoding, the word is built over multiple decoding steps, + // starting with the lemma, then adding factors one by one. + if (factorGroup == 0) { + word = factoredVocab->lemma2Word(shortlist ? shortlist->reverseMap(wordIdx) : wordIdx); // @BUGBUG: reverseMap is only correct if factoredVocab_->getGroupRange(0).first == 0 + //LOG(info, "new lemma {}={}", word.toWordIndex(), factoredVocab->word2string(word)); + } + else { + //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), + // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); + word = beam[beamHypIdx]->getWord(); + ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); + word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); + prevBeamHypIdx = prevHyp->getPrevStateIndex(); + prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words + } + } else if (shortlist) word = Word::fromWordIndex(shortlist->reverseMap(wordIdx)); else From 9825133b74a99ead8443ec68443bd902595a0a71 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 23 Apr 2019 14:12:09 -0700 Subject: [PATCH 434/838] made gcc happy; bug fix: should use a correct lemma index --- src/common/file_stream.h | 9 +++++++++ src/layers/generic.cpp | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/common/file_stream.h b/src/common/file_stream.h index ec03b1565..a950b0897 100644 --- a/src/common/file_stream.h +++ b/src/common/file_stream.h @@ -18,6 +18,15 @@ #include #include +#ifdef __GNUC__ // not supported; maybe we just need to increment a standard flag in gcc/cmake? +namespace std { + template + std::unique_ptr make_unique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); + } +} +#endif + #ifdef _MSC_VER #include #include diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 1724f2038..e5d35e719 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -187,7 +187,7 @@ namespace marian { // @TODO: we should rearrange lemmaHasFactorGroup as vector[groups[i] of float; then move this into FactoredVocab for (size_t i = 0; i < n; i++) { auto lemma = indices.empty() ? i : (indices[i] - factoredVocab_->getGroupRange(0).first); - res.push_back((float)factoredVocab_->lemmaHasFactorGroup(i, factorGroup)); + res.push_back((float)factoredVocab_->lemmaHasFactorGroup(lemma, factorGroup)); } return res; } From d9f99641be6288201bfed90e5ab248f0c2138e77 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 25 Apr 2019 09:42:23 +0100 Subject: [PATCH 435/838] Fix compilation with CUDA 8.0 --- src/tensors/gpu/prod.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index d13081728..32f577f75 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -12,6 +12,7 @@ namespace marian { namespace gpu { +#if CUDA_VERSION >= 9000 static void setTensorMode(cublasHandle_t cublasHandle) { static int mode = 0; // 1: use TC; -1: do not use TC; 0: not set yet if (mode == 0) { // multi-thread note: this is sort-of thread-safe, since multiple threads would determine the same value @@ -37,6 +38,7 @@ static void setTensorMode(cublasHandle_t cublasHandle) { } CUBLAS_CHECK(cublasSetMathMode(cublasHandle, mode > 0 ? CUBLAS_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH)); } +#endif void Prod(marian::Tensor C, const marian::Tensor& A, From 68c0aaff9edafa294ecb6ae3a63888ae257037f0 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 25 Apr 2019 10:04:28 +0100 Subject: [PATCH 436/838] Add status badges for different CUDA versions --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6fcc45b0b..6f7ab5d85 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ Marian ====== -[![Build Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev.svg?label=CUDA)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/) -[![CPU Build Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cpu.svg?label=CPU)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cpu/) +[![Build Status CUDA 8.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-8.0.svg?label=CUDA 8.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-8.0/) +[![Build Status CUDA 9.2](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev.svg?label=CUDA 9.2)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/) +[![Build Status CUDA 10.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-10.0.svg?label=CUDA 10.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-10.0/) +[![Build Status CPU](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cpu.svg?label=CPU)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cpu/) [![Tests Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-regression-tests.svg?label=tests)](http://vali.inf.ed.ac.uk/jenkins/job/marian-regression-tests/) [![Latest release](https://img.shields.io/github/release/marian-nmt/marian.svg?label=release)](https://github.com/marian-nmt/marian/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE.md) From c1451ab02558d3c4563426f98737020af0fd6b38 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 25 Apr 2019 10:23:18 +0100 Subject: [PATCH 437/838] Add acknowledgement --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6f7ab5d85..0047a0ca3 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ _Horizon 2020 Research and Innovation Programme_ under grant agreements 645487 ([Modern MT](http://www.modernmt.eu); 2015-2017), 644333 ([TraMOOC](http://tramooc.eu/); 2015-2017), 644402 ([HiML](http://www.himl.eu/); 2015-2017), +825303 ([Bergamot](https://browser.mt/); 2019-2021), the Amazon Academic Research Awards program, the World Intellectual Property Organization, and is based upon work supported in part by the Office of the Director of From 34717b40d8aec3bd487eb083354f7824c0476836 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Thu, 25 Apr 2019 11:35:56 +0100 Subject: [PATCH 438/838] Fix badges --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0047a0ca3..d96f7c898 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ Marian ====== -[![Build Status CUDA 8.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-8.0.svg?label=CUDA 8.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-8.0/) -[![Build Status CUDA 9.2](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev.svg?label=CUDA 9.2)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/) -[![Build Status CUDA 10.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-10.0.svg?label=CUDA 10.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-10.0/) +[![Build Status CUDA 8.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-8.0.svg?label=CUDA%208.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-8.0/) +[![Build Status CUDA 9.2](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev.svg?label=CUDA%209.2)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/) +[![Build Status CUDA 10.0](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cuda-10.0.svg?label=CUDA%2010.0)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cuda-10.0/) [![Build Status CPU](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cpu.svg?label=CPU)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cpu/) [![Tests Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-regression-tests.svg?label=tests)](http://vali.inf.ed.ac.uk/jenkins/job/marian-regression-tests/) [![Latest release](https://img.shields.io/github/release/marian-nmt/marian.svg?label=release)](https://github.com/marian-nmt/marian/releases) From 403ef88c77db5f1d9acd3a69c2e88f761015fa28 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 25 Apr 2019 12:38:11 -0700 Subject: [PATCH 439/838] some cleanup; addressed some PR feedback --- src/common/config_parser.cpp | 2 +- src/common/logging.cpp | 1 - src/common/utils.cpp | 8 ++ src/data/corpus.cpp | 9 +- src/data/corpus.h | 4 +- src/data/factored_vocab.cpp | 136 +++++++++++++---------------- src/data/factored_vocab.h | 14 +-- src/data/vocab_base.h | 2 +- src/graph/expression_operators.cpp | 1 + src/layers/constructors.h | 2 +- src/layers/generic.h | 15 ++-- src/layers/loss.h | 2 +- src/models/bert.h | 4 +- src/models/decoder.h | 8 +- src/models/model_base.h | 2 +- src/tensors/gpu/prod.cpp | 16 ---- src/training/validator.h | 10 ++- 17 files changed, 104 insertions(+), 132 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index ae09f2aa2..46402e38e 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -435,7 +435,7 @@ void ConfigParser::addOptionsValidation(cli::CLIWrapper& cli) { "10000u"); cli.add>("--valid-metrics", "Metric to use during validation: cross-entropy, ce-mean-words, perplexity, valid-script, " - "translation, bleu, bleu-detok, bleu-detok-ms. Multiple metrics can be specified", + "translation, bleu, bleu-detok. Multiple metrics can be specified", {"cross-entropy"}); cli.add("--early-stopping", "Stop if the first validation metric does not improve for arg consecutive validation steps", diff --git a/src/common/logging.cpp b/src/common/logging.cpp index 85dade236..57b55de96 100755 --- a/src/common/logging.cpp +++ b/src/common/logging.cpp @@ -78,7 +78,6 @@ void createLoggers(const marian::Config* config) { bool quiet = config && config->get("quiet"); Logger general{createStderrLogger("general", "[%Y-%m-%d %T] %v", generalLogs, quiet)}; - //Logger general{createStderrLogger("general", "%v", generalLogs, quiet)}; Logger valid{createStderrLogger("valid", "[%Y-%m-%d %T] [valid] %v", validLogs, quiet)}; if(config && config->has("log-level")) { diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 3fd0bdead..4cb872ae2 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -99,6 +99,12 @@ static std::string escapeForPOpen(const std::string& arg) { return "'" + findReplace(arg, "'", "'\\''", /*all=*/ true) + "'"; } +// execute an external command +// The command is composed of three pieces: +// - the executable path, e.g. --valid-script-path +// - an optional array of arguments. Meant for options. E.g. --valid-script-args. Options with leading - can only be passed via Yaml/Json. +// - one more optional single argument. Meant as the main filename argument. +// Each item will be escaped for shell syntax. std::string exec(const std::string& cmd, const std::vector& args /*= {}*/, const std::string& arg /*= ""*/) { std::array buffer; std::string result; @@ -199,6 +205,8 @@ std::string utf8FromUtf16String(const std::u16string& s) { #endif } +// test whether a Unicode code point is in continuous script (e.g. Chinese or Thai) +// This is used for detok bleu scoring where we have CJT characters. bool isContinuousScript(char32_t c) { // currently, this table is hand-coded, and may need to be extended when the standard grows auto in = [c](char32_t minVal, char32_t maxVal) { return c >= minVal && c <= maxVal; }; diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 61d359c91..1679e1a28 100755 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -31,11 +31,10 @@ void Corpus::preprocessLine(std::string& line, size_t streamId) { else LOG_ONCE(info, "[data] Target all-caps'ed line to: {}", line); } - else if (titleCaseEvery_ != 0 && pos_ % titleCaseEvery_ == 1 && !inference_ - - && streamId == 0 // @HACK: Hard-coding EN-X direction for now; needs an option in the future - - ) { + else if (titleCaseEvery_ != 0 && pos_ % titleCaseEvery_ == 1 && !inference_ && streamId == 0) { + // Only applied to stream 0 (source) since this feature is aimed at robustness against + // title case in the source (and not at translating into title case). + // Note: It is user's responsibility to not enable this if the source language is not English. line = vocabs_[streamId]->toEnglishTitleCase(line); if (streamId == 0) LOG_ONCE(info, "[data] Source English-title-case'd line to: {}", line); diff --git a/src/data/corpus.h b/src/data/corpus.h index c5af7220b..41272a811 100755 --- a/src/data/corpus.h +++ b/src/data/corpus.h @@ -28,8 +28,8 @@ class Corpus : public CorpusBase { void shuffleData(const std::vector& paths); // for pre-processing - size_t allCapsEvery_{0}; - size_t titleCaseEvery_{0}; + size_t allCapsEvery_{0}; // if set, convert every N-th input sentence (after randomization) to all-caps (source and target) + size_t titleCaseEvery_{0}; // ditto for title case (source only) void preprocessLine(std::string& line, size_t streamId); public: diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 509815f4e..f6208fece 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -1,3 +1,6 @@ +// This is the main implementation of factored models, which are driven by the vocabulary. +// Decoding, embedding, and output layer call into the vocab to drive their behavior. + #include "data/vocab_base.h" #include "common/definitions.h" #include "data/types.h" @@ -5,6 +8,12 @@ #include "common/regex.h" #include "data/factored_vocab.h" +// @TODO: review all comments and clarify nomenclature: +// * factor type (e.g. caps: |c* ); currently called a "group" +// * factor name (e.g. all-caps: |ca ) +// * factor index (e.g. |ca is index 0 inside |ca |ci |cn) +// * factor unit index (|ca is unit 41324 in joint factor vocab) + namespace marian { /*virtual*/ size_t FactoredVocab::load(const std::string& modelPath, size_t maxSizeUnused /*= 0*/) /*override final*/ { @@ -18,18 +27,19 @@ namespace marian { return size(); } + // load factor-vocab file and parse it std::vector> factorMapTokenized; std::string line; std::vector tokBuf; - if (utils::endsWith(modelPath, ".fsv")) { + if (utils::endsWith(modelPath, ".fsv")) { // @TODO: this extension check is only for backcompat; can be removed once we no longer support the old format // this is a fake parser for the generic factor spec, which makes a few hard assumptions: - // - all types starting with _ except _has_* are factor names - // - X : _x makes surface form X part of prob distribution _x except for _has_* - // - X : _has_x adds factor "x" to lemma X - // - _x <-> form only allows "_x <->" or "_x <-> _has_x" (same x), and is otherwise unused - // - _lemma is special - // The current version of the code just converts it internally to the legacy form. - // Once the legacy form is no longer needed, must of this can be simplified a lot. + // - all types starting with _ except _has_* are factor names + // - X : _x makes surface form X part of prob distribution _x except for _has_* + // - X : _has_x adds factor "x" to lemma X + // - _x <-> form only allows "_x <->" or "_x <-> _has_x" (same x), and is otherwise unused + // - _lemma is special + // The current version of the code just converts it internally to the legacy form. + // Once the legacy form is no longer needed, must of this can be simplified a lot. io::InputFileStream in(modelPath); WordIndex v = 0; std::map> factorTypeMap; // [type name] -> {factor-type names} @@ -122,7 +132,7 @@ namespace marian { constructGroupInfoFromFactorVocab(); constructFactorIndexConversion(); - // load and parse factorMap + // parse factorMap // modelPath = path to file with entries in order of vocab entries of the form // WORD FACTOR1 FACTOR2 FACTOR3... // Factors are grouped @@ -133,10 +143,9 @@ namespace marian { // - all factors not matching a prefix get lumped into yet another class (the lemmas) // - factor vocab must be sorted such that all groups are consecutive // - result of Output layer is nevertheless logits, not a normalized probability, due to the sigmoid entries - //vocab_.resize(vocabSize); - //factorMap_.resize(vocabSize); - //auto factorVocabSize = this->factorVocabSize(); - lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); + // For every lemma, the factor map contains one example. At the end of this loop, we have a vocabulary + // vocab_ that contains those examples, but not all possible combinations + lemmaHasFactorGroup_.resize(groupRanges_[0].second - groupRanges_[0].first); // group 0 is the lemmas; this difference is the number of lemma symbols size_t numTotalFactors = 0; for (WordIndex v = 0; v < factorMapTokenized.size(); v++) { const auto& tokens = factorMapTokenized[v]; @@ -145,7 +154,7 @@ namespace marian { // Not every word has all other factors, so the n-th item is not always in the same factor group. // @TODO: change to just use the .wl file, and manually split at @ ABORT_IF(tokens.size() < 2, "Factor map must have at least one factor per word", modelPath); - std::vector factorUnits; + std::vector factorUnits; // units in the joint factor vocab that belong to a specific factor type for (size_t i = 1/*first factor*/; i < tokens.size(); i++) { auto u = factorVocab_[tokens[i]]; factorUnits.push_back(u); @@ -167,18 +176,13 @@ namespace marian { ABORT_IF(lemmaFlags != hasFactorGroupFlags, "Inconsistent factor groups used for word {}", tokens.front()); // map factors to non-dense integer auto word = factors2word(factorIndices); - auto wordIndex = word.toWordIndex(); - //factorMap_[wordIndex] = std::move(factorUnits); // add to vocab (the wordIndex are not dense, so the vocab will have holes) - //if (tokens.front().front() == '<') // all others are auto-expanded // for now add what we get, and then expand more below auto wordString = word2string(word); if (tokens.front() != wordString) // order may differ, since we formed the input based on the factors in the user file, which may be in any order LOG_ONCE(info, "[vocab] Word name in vocab file {} differs from canonical form {} (this warning is only shown once)", tokens.front(), wordString); - vocab_.add(wordString, wordIndex); + vocab_.add(wordString, word.toWordIndex()); numTotalFactors += tokens.size() - 1; - //if (v % 5000 == 0) - // LOG(info, "{} -> {}", tokens.front(), word2string(word)); } LOG(info, "[vocab] Factored-embedding map read with total/unique of {}/{} factors from {} example words (in space of {})", numTotalFactors, factorVocabSize(), vocab_.size()/*numValid()*/, utils::withCommas(virtualVocabSize())); @@ -204,12 +208,6 @@ namespace marian { unkId_ = Word::fromWordIndex(vocab_[DEFAULT_UNK_STR]); //LOG(info, "eos: {}; unk: {}", word2string(eosId_), word2string(unkId_)); -//#if 1 // dim-vocabs stores numValid() in legacy model files, and would now have been size() -// if (maxSizeUnused == vocab_.size()/*numValid()*/) -// maxSizeUnused = virtualVocabSize; -//#endif - //ABORT_IF(maxSizeUnused != 0 && maxSizeUnused != size(), "Factored vocabulary does not allow on-the-fly clipping to a maximum vocab size (from {} to {})", size(), maxSizeUnused); - // @TODO: ^^ disabled now that we are generating the full combination of factors; reenable once we have consistent setups again return size(); } @@ -291,6 +289,10 @@ void FactoredVocab::constructFactorIndexConversion() { } // encode factors into a Word struct +// inputs: +// - factorIndices[factorType] = factorIndex (e.g. 0 for |ca ) +// output: +// - representation as 'Word' (which is, in fact, a single big integer) Word FactoredVocab::factors2word(const std::vector& factorIndices /* [numGroups] */) const { size_t index = 0; size_t numGroups = getNumGroups(); @@ -311,6 +313,10 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu return Word::fromWordIndex(index); } +// encode only a lemma into a 'Word' +// The result is incomplete, in that the lemma likely has additional factors that are not yet specified. +// Those are encoded as the value FACTOR_NOT_SPECIFIED. This function is used during beam search, +// which starts with lemma scores, and then adds factors one by one to the path score. Word FactoredVocab::lemma2Word(size_t factor0Index) const { size_t numGroups = getNumGroups(); std::vector factorIndices; @@ -341,12 +347,16 @@ Word FactoredVocab::expandFactoredWord(Word word, size_t groupIndex, size_t fact return word; } +// factor unit: index of factor name in the joint factor vocabulary +// factor index: relative index within factor type, e.g. 0 for |ca size_t FactoredVocab::factorUnit2FactorIndex(WordIndex u) const { auto g = factorGroups_[u]; // convert u to relative u within factor group range ABORT_IF(u < groupRanges_[g].first || u >= groupRanges_[g].second, "Invalid factorGroups_ entry??"); return u - groupRanges_[g].first; } +// split the 'Word' representation, which is really a single big integer, into the individual +// factor indices for all factor types void FactoredVocab::word2factors(Word word, std::vector& factorIndices /* [numGroups] */) const { size_t numGroups = getNumGroups(); factorIndices.resize(numGroups); @@ -361,13 +371,13 @@ void FactoredVocab::word2factors(Word word, std::vector& factorIndices / #endif } +// serialize 'Word' representation into its string form std::string FactoredVocab::word2string(Word word) const { // this function has some code dup, so that we can bypass some checks for debugging size_t numGroups = getNumGroups(); size_t factor0Index = word.toWordIndex() / factorStrides_[0]; std::string res; for (size_t g = 0; g < numGroups; g++) { - //res.append(res.empty() ? "(" : ", "); size_t index = word.toWordIndex(); index = index / factorStrides_[g]; index = index % (size_t)factorShape_[g]; @@ -376,16 +386,14 @@ std::string FactoredVocab::word2string(Word word) const { res.append("(lemma oob)"); else if (lemmaHasFactorGroup(factor0Index, g)) res.append("?"); - //else - // res.append("n/a"); } else res.append(factorVocab_[(WordIndex)(index + groupRanges_[g].first)]); } - //res.append(")"); return res; } +// deserialize factored string form (e.g. HELLO|ci|wb) into its internal binary 'Word' representation Word FactoredVocab::string2word(const std::string& w) const { auto sep = std::string(1, factorSeparator_); auto parts = utils::splitAny(w, sep); @@ -408,10 +416,10 @@ Word FactoredVocab::string2word(const std::string& w) const { factorIndices[g] = u - groupRanges_[g].first; } auto word = factors2word(factorIndices); - //auto w2 = word2string(word); return word; } +// extract the factor index of a given factor type from the 'Word' representation size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { size_t index = word.toWordIndex(); size_t factor0Index = index / factorStrides_[0]; @@ -437,11 +445,6 @@ size_t FactoredVocab::getFactor(Word word, size_t groupIndex) const { return index; } -//std::pair FactoredVocab::getFactorUnit(Word word, size_t groupIndex) const { -// word; groupIndex; -// ABORT("Not implemented"); -//} - #ifdef FACTOR_FULL_EXPANSION void FactoredVocab::constructNormalizationInfoForVocab() { // create mappings needed for normalization in factored outputs @@ -482,35 +485,27 @@ void FactoredVocab::constructNormalizationInfoForVocab() { #endif /*virtual*/ Word FactoredVocab::operator[](const std::string& word) const /*override final*/ { + // @TODO: do away with vocab_ altogether, and just always parse. WordIndex index; bool found = vocab_.tryFind(word, index); if (found) return Word::fromWordIndex(index); else return string2word(word); - //ABORT("Unknown word {} mapped to {}", word, word2string(getUnkId())); - //LOG_ONCE(info, "WARNING: Unknown word {} mapped to {}", word, word2string(getUnkId())); - //return getUnkId(); } /*virtual*/ const std::string& FactoredVocab::operator[](Word word) const /*override final*/ { //LOG(info, "Looking up Word {}={}", word.toWordIndex(), word2string(word)); -#if 1 // @TODO: remove this ABORT_IF(!vocab_.contains(word.toWordIndex()), "Invalid factor combination {}", word2string(word)); -#else // @BUGBUG: our manually prepared dict does not contain @CI tags for single letters, but it's a valid factor - if (!vocab_.contains(word.toWordIndex())) { - LOG/*_ONCE*/(info, "Factor combination {} missing in external dict, generating fake entry (only showing this warning once)", word2string(word)); - //const_cast(vocab_).add("??" + word2string(word), word.toWordIndex()); - const_cast(vocab_).add(word2string(word), word.toWordIndex()); - } -#endif return vocab_[word.toWordIndex()]; } +// convert a string representation of a token sequence to all-caps by changing all capitalization factors to |ca /*virtual*/ std::string FactoredVocab::toUpper(const std::string& line) const /*override final*/ { return utils::findReplace(utils::findReplace(utils::findReplace(utils::findReplace(line, "|ci", "|ca", /*all=*/true), "|cn", "|ca", /*all=*/true), "@CI", "@CA", /*all=*/true), "@CN", "@CA", /*all=*/true); } +// convert a string representation of a token sequence to English title case by changing the capitalization factors to |ci /*virtual*/ std::string FactoredVocab::toEnglishTitleCase(const std::string& line) const /*override final*/ { // @BUGBUG: does not handle the special words that should remain lower-case // note: this presently supports both @WB and @GL- (legacy) @@ -525,7 +520,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { auto word = Word::fromWordIndex(*ptr); auto wordString = word2string(word); auto lemmaIndex = getFactor(word, 0) + groupRanges_[0].first; - *ptr = lemmaIndex; + *ptr = (WordIndex)lemmaIndex; } } @@ -544,6 +539,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return factors2word(factorIndices); } +// encode a string representation of an entire token sequence, as found in the corpus file, into a 'Word' array /*virtual*/ Words FactoredVocab::encode(const std::string& line, bool addEOS /*= true*/, bool /*inference*/ /*= false*/) const /*override final*/ { std::vector lineTokens; utils::split(line, lineTokens, " "); @@ -555,6 +551,7 @@ void FactoredVocab::constructNormalizationInfoForVocab() { return res; } +// decode a 'Word' array into the external string representation of that token sequence, as written to output files /*virtual*/ std::string FactoredVocab::decode(const Words& sentence, bool ignoreEOS /*= true*/) const /*override final*/ { std::vector decoded; decoded.reserve(sentence.size()); @@ -584,7 +581,8 @@ static void unescapeHexEscapes(std::string& utf8Lemma) { utf8Lemma = utils::utf8FromUtf16String(lemma); } -// interpret the capitalization and glue factors +// convert a 'Word' sequence to its final human-readable surface form +// This interprets the capitalization and glue factors. // This assumes a specific notation of factors, emulating our C# code for generating these factors: // - | as separator symbol // - capitalization factors are cn, ci, and ca @@ -592,7 +590,7 @@ static void unescapeHexEscapes(std::string& utf8Lemma) { std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override final*/ { std::string res; res.reserve(sentence.size() * 10); - bool prevHadGlueRight = true; // no space at sentence start + bool prevHadGlueRight = true; // no space at sentence start for(auto w : sentence) { if (w == getEosId()) break; @@ -603,12 +601,12 @@ std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override f std::set tokenSet(tokens.begin() + 1, tokens.end()); auto has = [&](const char* factor) { return tokenSet.find(factor) != tokenSet.end(); }; // spacing - bool hasGlueRight = has("gr+") || has("wen") || has("cen"); - bool hasGlueLeft = has("gl+") || has("wbn") || has("cbn"); - bool insertSpaceBefore = !prevHadGlueRight && !hasGlueLeft; + bool hasGlueRight = has("gr+") || has("wen") || has("cen"); + bool hasGlueLeft = has("gl+") || has("wbn") || has("cbn"); + bool insertSpaceBefore = !prevHadGlueRight && !hasGlueLeft; if (insertSpaceBefore) res.push_back(' '); - prevHadGlueRight = hasGlueRight; + prevHadGlueRight = hasGlueRight; // capitalization unescapeHexEscapes(lemma); // unescape \x.. and \u.... if (has("ci")) lemma = utils::utf8Capitalized(lemma); @@ -621,8 +619,13 @@ std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override f return res; } -// create a CSR matrix M[V,U] from words[] with -// M[v,u] = 1/c(u) if factor u is a factor of word v, and c(u) is how often u is referenced +// create a CSR matrix M[V,U] from words[] with M[v,u] = 1 if factor u is a factor of word v +// This is used to form the embedding of a multi-factor token. +// That embedding is a sum of the embeddings of the individual factors. +// Those individual embeddings are assumed to be concatenated into one joint large embedding matrix. +// The factor embeddings are summed up by multiplying the joint embedding matrix with a sparse matrix +// that contains a 1 for all positions in the joint matrix that should be summed up. +// This function creates that sparse matrix in CSR form. FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { auto numGroups = getNumGroups(); std::vector weights; @@ -636,27 +639,12 @@ FactoredVocab::CSRData FactoredVocab::csr_rows(const Words& words) const { for (auto word : words) { if (vocab_.contains(word.toWordIndex())) { // skip invalid combinations in the space (can only happen during initialization) --@TODO: add a check? word2factors(word, factorIndices); -#if 0 // original code; enable this to try - numGroups; - const auto& m = factorMap_[word.toWordIndex()]; - for (auto u : m) { - indices.push_back(u); -#else -#if 0 // special handling of the missing single capitalized letters - // costs about 0.1 BLEU when original model never saw this combination (which is quite nicely low) - // @TODO: remove this once we use the factor-spec file - const auto& lemma = factorVocab_[(WordIndex)(factorIndices[0] + groupRanges_[0].first)]; - if (lemma.size() == 1 && factorIndices[1]/*@C*/ == 1/*@CI*/) // skip one-letter factors with - LOG_ONCE(info, "Suppressing embedding for word {} (only showing this warning once)", word2string(word)); - else -#endif for (size_t g = 0; g < numGroups; g++) { // @TODO: make this faster by having a list of all factors to consider for a lemma? auto factorIndex = factorIndices[g]; ABORT_IF(factorIndex == FACTOR_NOT_SPECIFIED, "Attempted to embed a word with a factor not specified"); if (factorIndex == FACTOR_NOT_APPLICABLE) continue; indices.push_back((IndexType)(factorIndex + groupRanges_[g].first)); // map to unit index -#endif weights.push_back(1.0f); } } @@ -684,15 +672,15 @@ WordIndex FactoredVocab::WordLUT::add(const std::string& word, WordIndex index) ABORT_IF(!wasInserted, "Duplicate vocab entry for '{}', new index {} vs. existing index {}", word, index, str2index_[word]); wasInserted = index2str_.insert(std::make_pair(index, word)).second; ABORT_IF(!wasInserted, "Duplicate vocab entry for index {} (new: '{}'; existing: '{}')", index, word, index2str_[index]); - //if (vocabSize_ < index2str_.size()) - // vocabSize_ = index2str_.size(); return index; } + static const std::string g_emptyString; const std::string& FactoredVocab::WordLUT::operator[](WordIndex index) const { auto iter = index2str_.find(index); - //ABORT_IF(iter == index2str_.end(), "Invalid access to dictionary gap item"); if (iter == index2str_.end()) + // returns an empty string for unknown index values + // @TODO: is that ever used ? If so, document.If not, remove this feature and let it fail.static const std::string g_emptyString; return g_emptyString; // (using a global since we return a reference) else return iter->second; diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h index 3ff338e23..9e7734c43 100755 --- a/src/data/factored_vocab.h +++ b/src/data/factored_vocab.h @@ -9,8 +9,6 @@ #include "data/types.h" #include "data/vocab_base.h" -#include // for std::iota() - #undef FACTOR_FULL_EXPANSION // define this to get full expansion. @TODO: infeasible for many factors; just delete this namespace marian { @@ -25,7 +23,6 @@ class FactoredVocab : public IVocab { }; // from IVocab: - // @TODO: Why are these virtual and final at the same time? Seems we should remove all the 'virtual' here virtual size_t load(const std::string& factoredVocabPath, size_t maxSizeUnused = 0) override final; virtual void create(const std::string& vocabPath, const std::vector& trainPaths, size_t maxSize) override final { vocabPath, trainPaths, maxSize; ABORT("Factored vocab cannot be created on the fly"); } virtual const std::string& canonicalExtension() const override final { return suffixes()[0]; } @@ -57,8 +54,6 @@ class FactoredVocab : public IVocab { #endif size_t getNumGroups() const { return groupRanges_.size(); } std::pair getGroupRange(size_t g) const { return groupRanges_[g]; } // [g] -> (u_begin, u_end) - //const std::vector& getFactorMasks(size_t g) const { return factorMasks_[g]; } // [g][v] 1.0 if word v has factor g - //const std::vector& getFactorIndices(size_t g) const { return factorIndices_[g]; } // [g][v] local index u_g = u - u_g,begin of factor g for word v; 0 if not a factor #ifdef FACTOR_FULL_EXPANSION const std::vector& getGapLogMask() const { return gapLogMask_; } // [v] -inf if v is a gap entry, else 0 #endif @@ -70,7 +65,6 @@ class FactoredVocab : public IVocab { Word expandFactoredWord(Word word, size_t groupIndex, size_t factorIndex) const; bool canExpandFactoredWord(Word word, size_t groupIndex) const { return lemmaHasFactorGroup(getFactor(word, 0), groupIndex); } size_t getFactor(Word word, size_t groupIndex) const; - //std::pair getFactorUnit(Word word, size_t groupIndex) const; bool lemmaHasFactorGroup(size_t factor0Index, size_t g) const { return lemmaHasFactorGroup_[factor0Index][g]; } static constexpr size_t FACTOR_NOT_APPLICABLE = (SIZE_MAX - 1); @@ -89,19 +83,16 @@ class FactoredVocab : public IVocab { #endif size_t factorUnit2FactorIndex(WordIndex u) const; private: + // @TODO: Should we move WordLUT to utils? class WordLUT { // map between strings and WordIndex std::map str2index_; std::map index2str_; - //size_t vocabSize_; // total number of vocab items as set by user public: WordIndex add(const std::string& word, WordIndex index); const std::string& operator[](WordIndex index) const; WordIndex operator[](const std::string& word) const; bool contains(WordIndex index) const { return index2str_.find(index) != index2str_.end(); } bool tryFind(const std::string& word, WordIndex& index) const; - //void resize(size_t num); // @TODO: remove this, and remove the distinction of size() and numValid() - //size_t size() const { return vocabSize_; } // nominal size including gap items - //size_t numValid() const { return str2index_.size(); } // actual non-gaps items size_t size() const { return str2index_.size(); } size_t load(const std::string& path); void dumpToFile(const std::string& path); @@ -116,7 +107,6 @@ class FactoredVocab : public IVocab { char factorSeparator_ = '|'; // separator symbol for parsing factored words WordLUT factorVocab_; // [factor name] -> factor index = row of E_ std::vector groupPrefixes_; // [group id g] shared prefix of factors (used for grouping) - //std::vector> factorMap_; // [word index v] -> set of factor indices u #ifdef FACTOR_FULL_EXPANSION CSRData globalFactorMatrix_; // [v,u] (sparse) -> =1 if u is factor of v #endif @@ -125,8 +115,6 @@ class FactoredVocab : public IVocab { std::vector> lemmaHasFactorGroup_; // [factor 0 index][g] -> true if lemma has factor group Shape factorShape_; // [g] number of factors in each factor group std::vector factorStrides_; // [g] stride for factor dimension - //std::vector> factorMasks_; // [g][v] 1.0 if word v has factor g - //std::vector> factorIndices_; // [g][v] relative index u - u_begin of factor g (or any valid index if it does not have it; we use 0) #ifdef FACTOR_FULL_EXPANSION std::vector gapLogMask_; // [v] -1e8 if this is a gap, else 0 #endif diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h index 238f9146b..6b6869d88 100755 --- a/src/data/vocab_base.h +++ b/src/data/vocab_base.h @@ -50,7 +50,7 @@ class IVocab { virtual std::string toEnglishTitleCase(const std::string& line) const { return line; } // this function is an identity mapping for default vocabularies, hence do nothing - virtual void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const { } + virtual void transcodeToShortlistInPlace(WordIndex* ptr, size_t num) const { ptr; num; } virtual void createFake() = 0; diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 7247c1001..396370bd1 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -79,6 +79,7 @@ Expr softmax(Expr a, Expr zeroOneMask, int axis /*=-1*/) { Expr logsoftmax(Expr a) { if (a->type() == "logsoftmax") // logsoftmax(logsoftmax(x)) == logsoftmax(x) + // @TODO: Is this correct? logsoftmax is idempotent in forward(), but not in backward() return a; return Expression(a); } diff --git a/src/layers/constructors.h b/src/layers/constructors.h index fc751dfc9..ed2c029f8 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -135,7 +135,7 @@ class MLP : public IUnaryLogitLayer, public IHasShortList { Logits applyAsLogits(const std::vector& av) override { // same as apply() except for the last layer, we invoke applyAsLogits(), which has a different return type auto lastLayer = std::dynamic_pointer_cast(layers_.back()); - ABORT_IF(!lastLayer, "MLP::applyAsLogits() applied but last MLP layer is not IUnaryLogitLayer"); + ABORT_IF(!lastLayer, "MLP::applyAsLogits() was called on an MLP whose last layer is not an IUnaryLogitLayer"); if (layers_.size() == 1) { if (av.size() == 1) return lastLayer->applyAsLogits(av[0]); diff --git a/src/layers/generic.h b/src/layers/generic.h index 775fbbe01..7cf8ac6ce 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -67,9 +67,11 @@ struct IEmbeddingLayer { class FactoredVocab; -// @HACK: Frank's quick implementation of factored outputs. To be re-thought once it works. -// Output layer returns a Logits object, which is able to compute some things on the fly -// for factored embeddings. +// To support factors, any output projection (that is followed by a softmax) must +// retain multiple outputs, one for each factor. Such layer returns not a single Expr, +// but a Logits object that contains multiple. +// This allows to compute softmax values in a factored manner, where we never create +// a fully expanded list of all factor combinations. class RationalLoss; class Logits { public: @@ -96,7 +98,6 @@ class Logits { MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices - //std::vector getFactorMasks(const Words& words, size_t factorGroup) const; float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } @@ -111,13 +112,15 @@ class Logits { std::vector getFactorMasks(size_t factorGroup, const std::vector& indices) const; private: // members - // @HACK: The interplay between Logits and RationalLoss is weird. Here, we allow RationalLoss with count == nullptr. - std::vector> logits_; // [group id][B..., num factors in group] --@TODO: we don't use the RationalLoss component anymore, can be removed again + // @TODO: we don't use the RationalLoss component anymore, can be removed again, and replaced just by the Expr + std::vector> logits_; // [group id][B..., num factors in group] Ptr factoredVocab_; }; // Unary function that returns a Logits object // Also implements IUnaryLayer, since Logits can be cast to Expr. +// This interface is implemented by all layers that are of the form of a unary function +// that returns multiple logits, to support factors. struct IUnaryLogitLayer : public IUnaryLayer { virtual Logits applyAsLogits(Expr) = 0; virtual Logits applyAsLogits(const std::vector& es) { diff --git a/src/layers/loss.h b/src/layers/loss.h index b02aa78fa..554541cbd 100755 --- a/src/layers/loss.h +++ b/src/layers/loss.h @@ -341,7 +341,7 @@ class CrossEntropyLoss : public LabelwiseLoss { auto ce = logits.applyLossFunction(labels, [&](Expr logits, Expr indices) { logits = atleast_3d(logits); // we always assuma a time and batch dimension exists. // for bert training or classification the time dimension is lot. - // Here safeguard against 2d classifier output, adds 1 on the left, non-op. Expr ce = cross_entropy(logits, indices); + // Here safeguard against 2d classifier output, adds 1 on the left, non-op. Expr ce = cross_entropy(logits, indices); if (labelSmoothing_ > 0) { // ce = -sum_i y^_i log y_i(h) diff --git a/src/models/bert.h b/src/models/bert.h index fb2dab846..61cff43bf 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -279,18 +279,16 @@ class BertClassifier : public ClassifierBase { auto classEmbeddings = slice(context, /*axis=*/-3, /*i=*/0); // [CLS] symbol is first symbol in each sequence int dimModel = classEmbeddings->shape()[-1]; -// int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels + int dimTrgCls = opt>("dim-vocabs")[batchIndex_]; // Target vocab is used as class labels auto output = mlp::mlp() // .push_back(mlp::dense() // ("prefix", prefix_ + "_ff_logit_l1") // ("dim", dimModel) // ("activation", mlp::act::tanh)) // @TODO: do we actually need this? -#if 0 // @TODO: Not supported presently since Output has a different signature now .push_back(mlp::output() // ("dim", dimTrgCls)) // ("prefix", prefix_ + "_ff_logit_l2") // -#endif .construct(graph); auto logits = output->apply(classEmbeddings); // class logits for each batch entry diff --git a/src/models/decoder.h b/src/models/decoder.h index 962587d30..31baca9cb 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -15,7 +15,7 @@ class DecoderBase { std::string prefix_{"decoder"}; bool inference_{false}; size_t batchIndex_{1}; - std::vector> embedding_; // @TODO: find a more grammattical name + std::vector> embedding_; // @TODO: find a more grammatical name Ptr shortlist_; @@ -35,7 +35,7 @@ class DecoderBase { virtual Ptr step(Ptr, Ptr) = 0; - void lazyCreateEmbedding(Ptr graph) { + void lazyCreateEmbeddingLayer(Ptr graph) { // @TODO: code dup with EncoderTransformer if (embedding_.size() <= batchIndex_ || !embedding_[batchIndex_]) { // lazy if (embedding_.size() <= batchIndex_) @@ -64,7 +64,7 @@ class DecoderBase { Ptr batch) { auto subBatch = (*batch)[batchIndex_]; - lazyCreateEmbedding(graph); + lazyCreateEmbeddingLayer(graph); Expr y, yMask; std::tie (y, yMask) = embedding_[batchIndex_]->apply(subBatch); @@ -87,7 +87,7 @@ class DecoderBase { const Words& words, int dimBatch, int dimBeam) { - lazyCreateEmbedding(graph); + lazyCreateEmbeddingLayer(graph); Expr selectedEmbs; int dimEmb = opt("dim-emb"); if(words.empty()) { diff --git a/src/models/model_base.h b/src/models/model_base.h index 0a64a1e1d..016f3b8e3 100755 --- a/src/models/model_base.h +++ b/src/models/model_base.h @@ -3,7 +3,7 @@ #include #include "marian.h" #include "layers/loss.h" -#include "layers/generic.h" // @HACK for Frank's factored embeddings/Logits class +#include "layers/generic.h" namespace marian { namespace models { diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index f259f1c47..74abfa219 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -349,22 +349,6 @@ void CSRProd(marian::Tensor C, else { // C = S x D for row-major matrices // Implemented via cusparse as C' = D' x S' ("gemmi") where C' and D' are column-major. - //if (St_values) { - // auto vals = get(St_values, C->getBackend()); vals.resize(numValues); - // auto inds = get(St_indices, C->getBackend()); inds.resize(numValues); - // auto offs = get(St_offsets, C->getBackend()); offs.resize(rowsS + 1); - // LOG(info, "[{} x {}] = [{} x {}] * [{} x {}]", colsC, rowsC, colsD, rowsD, -1, offs.size() - 1); - // for (auto v : vals) - // ABORT_IF(v != 1, "v={}", v); - // for (auto i : inds) - // ABORT_IF(i >= rowsC, "i={}", i); - // for (auto o : offs) - // ABORT_IF(o > inds.size(), "o={}", o); - // ABORT_IF(colsC != colsD, "0"); - // std::vector dData; D->get(dData); ABORT_IF(dData.size() != colsD * rowsD, "1"); - // std::vector cData; C->get(cData); ABORT_IF(cData.size() != colsC * rowsC, "2"); - // ABORT_IF(rowsC != rowsS, "3: {} != {}, asz={}", rowsC, rowsS, St_offsets->size()/4); - //} CUSPARSE_CHECK(cusparseSgemmiEx(cusparseHandle, /*m=*/ colsD, // #rows of first (col-major) factor = #cols of row-major D /*n=*/ rowsC, // #cols of second (CSC) factor and (col-major) result = #rows of row-major C diff --git a/src/training/validator.h b/src/training/validator.h index 78b9f9d33..37833b605 100755 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -744,7 +744,7 @@ class BleuValidator : public Validator { public: static std::string tokenizeContinuousScript(const std::string& sUTF8) { // We want BLEU-like scores that are comparable across different tokenization schemes. - // For continuous scripts )Chinese, Japanese, Thai), we would need a language-specific + // For continuous scripts (Chinese, Japanese, Thai), we would need a language-specific // statistical word segmenter, which is outside the scope of Marian. As a practical // compromise, we segment continuous-script sequences into individual characters, while // leaving Western scripts as words. This way we can use the same settings for Western @@ -765,8 +765,9 @@ class BleuValidator : public Validator { std::vector decode(const Words& words, bool addEOS = false) { auto vocab = vocabs_.back(); - auto tokenString = tokenize(vocab->surfaceForm(words)); - tokenString = tokenizeContinuousScript(tokenString); + auto tokenString = vocab->surfaceForm(words); // detokenize to surface form + tokenString = tokenize(tokenString); // tokenize according to SacreBLEU rules + tokenString = tokenizeContinuousScript(tokenString); // CJT scripts only: further break into characters auto tokens = utils::splitAny(tokenString, " "); if(addEOS) tokens.push_back(""); @@ -836,6 +837,9 @@ class BleuValidator : public Validator { #if 1 // hack for now, to get this feature when running under Flo // Problem is that Flo pieces that pass 'bleu' do not know whether vocab is factored, // hence cannot select 'bleu-detok'. + // @TODO: We agreed that we will replace bleu-detok by bleu with an additional + // parameter to select the detokenization method, which will default to detok for FactoredSegmenter, + // and no-op for base vocab. if (vocabs_.back()->type() == "FactoredVocab") { LOG_ONCE(info, "[valid] FactoredVocab implies using detokenized BLEU"); detok = true; // always use bleu-detok From b570638533d145ebff0b3b5dccccfdd8ad3499f2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 26 Apr 2019 14:59:40 -0700 Subject: [PATCH 440/838] weirdo change of access permissions --- contrib/autoformat.sh | 0 scripts/bert/bert4marian.py | 0 scripts/embeddings/export_embeddings.py | 0 scripts/embeddings/prepare_corpus.py | 0 scripts/embeddings/process_word2vec.py | 0 scripts/server/client_example.py | 0 scripts/shortlist/generate_shortlists.pl | 0 scripts/shortlist/install.sh | 0 src/3rd_party/ExceptionWithCallStack.h | 0 src/3rd_party/pathie-cpp/src/entry_iterator.cpp | 0 src/3rd_party/pathie-cpp/src/path.cpp | 0 src/3rd_party/spdlog/astyle.sh | 0 src/3rd_party/spdlog/bench/latency/compare.sh | 0 src/3rd_party/spdlog/tests/install_libcxx.sh | 0 src/command/marian_main.cpp | 0 src/common/config_parser.cpp | 0 src/common/definitions.h | 0 src/common/logging.cpp | 0 src/common/options.h | 0 src/common/shape.h | 0 src/common/timer.h | 0 src/common/types.h | 0 src/common/utils.cpp | 0 src/common/utils.h | 0 src/data/batch_generator.h | 0 src/data/corpus.cpp | 0 src/data/corpus.h | 0 src/data/corpus_base.h | 0 src/data/default_vocab.cpp | 0 src/data/factored_vocab.cpp | 0 src/data/factored_vocab.h | 0 src/data/sentencepiece_vocab.cpp | 0 src/data/text_input.cpp | 0 src/data/types.h | 0 src/data/vocab.cpp | 0 src/data/vocab.h | 0 src/data/vocab_base.h | 0 src/examples/mnist/dataset.h | 0 src/examples/mnist/download.sh | 0 src/examples/mnist/mnist_ffnn.cpp | 0 src/examples/mnist/model.h | 0 src/examples/mnist/training.h | 0 src/examples/mnist/validator.h | 0 src/functional/tmp.h | 0 src/graph/expression_operators.cpp | 0 src/graph/expression_operators.h | 0 src/graph/node_operators.h | 0 src/graph/node_operators_unary.h | 0 src/layers/constructors.h | 0 src/layers/factory.h | 0 src/layers/generic.cpp | 0 src/layers/generic.h | 0 src/layers/loss.cpp | 0 src/layers/loss.h | 0 src/microsoft/quicksand.cpp | 0 src/microsoft/quicksand.h | 0 src/models/bert.h | 0 src/models/char_s2s.h | 0 src/models/costs.h | 0 src/models/decoder.h | 0 src/models/encoder_classifier.h | 0 src/models/encoder_decoder.cpp | 0 src/models/encoder_decoder.h | 0 src/models/model_base.h | 0 src/models/model_factory.cpp | 0 src/models/model_factory.h | 0 src/models/s2s.h | 0 src/models/states.h | 0 src/models/transformer.h | 0 src/models/transformer_factory.h | 0 src/models/transformer_stub.cpp | 0 src/rescorer/rescorer.h | 0 src/rnn/attention_constructors.h | 0 src/rnn/cells.h | 0 src/rnn/types.h | 0 src/tensors/cpu/tensor_operators.cpp | 0 src/tensors/gpu/add.cu | 0 src/tensors/gpu/add.h | 0 src/tensors/gpu/add.inc | 0 src/tensors/gpu/element.cu | 0 src/tensors/gpu/element.inc | 0 src/tensors/gpu/prod.cpp | 0 src/tensors/gpu/prod.h | 0 src/tensors/gpu/tensor_operators.cu | 0 src/tests/prod.cpp | 0 src/tests/units/attention_tests.cpp | 0 src/tests/units/operator_tests.cpp | 0 src/tests/units/rnn_tests.cpp | 0 src/training/communicator.cpp | 0 src/training/graph_group.h | 0 src/training/graph_group_async.cpp | 0 src/training/graph_group_async.h | 0 src/training/graph_group_multinode.cpp | 0 src/training/graph_group_multinode_sync.cpp | 0 src/training/graph_group_singleton.cpp | 0 src/training/graph_group_singleton.h | 0 src/training/graph_group_sync.cpp | 0 src/training/graph_group_sync.h | 0 src/training/scheduler.h | 0 src/training/validator.cpp | 0 src/training/validator.h | 0 src/translator/beam_search.h | 0 src/translator/helpers.cpp | 0 src/translator/helpers.cu | 0 src/translator/helpers.h | 0 src/translator/history.h | 0 src/translator/hypothesis.h | 0 src/translator/nth_element.cpp | 0 src/translator/nth_element.cu | 0 src/translator/nth_element.h | 0 src/translator/output_printer.cpp | 0 src/translator/output_printer.h | 0 src/translator/scorers.cpp | 0 src/translator/scorers.h | 0 src/translator/translator.h | 0 vs/Marian.sln | 0 vs/Marian.vcxproj | 0 vs/Marian.vcxproj.filters | 0 118 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 contrib/autoformat.sh mode change 100755 => 100644 scripts/bert/bert4marian.py mode change 100755 => 100644 scripts/embeddings/export_embeddings.py mode change 100755 => 100644 scripts/embeddings/prepare_corpus.py mode change 100755 => 100644 scripts/embeddings/process_word2vec.py mode change 100755 => 100644 scripts/server/client_example.py mode change 100755 => 100644 scripts/shortlist/generate_shortlists.pl mode change 100755 => 100644 scripts/shortlist/install.sh mode change 100755 => 100644 src/3rd_party/ExceptionWithCallStack.h mode change 100755 => 100644 src/3rd_party/pathie-cpp/src/entry_iterator.cpp mode change 100755 => 100644 src/3rd_party/pathie-cpp/src/path.cpp mode change 100755 => 100644 src/3rd_party/spdlog/astyle.sh mode change 100755 => 100644 src/3rd_party/spdlog/bench/latency/compare.sh mode change 100755 => 100644 src/3rd_party/spdlog/tests/install_libcxx.sh mode change 100755 => 100644 src/command/marian_main.cpp mode change 100755 => 100644 src/common/config_parser.cpp mode change 100755 => 100644 src/common/definitions.h mode change 100755 => 100644 src/common/logging.cpp mode change 100755 => 100644 src/common/options.h mode change 100755 => 100644 src/common/shape.h mode change 100755 => 100644 src/common/timer.h mode change 100755 => 100644 src/common/types.h mode change 100755 => 100644 src/common/utils.cpp mode change 100755 => 100644 src/common/utils.h mode change 100755 => 100644 src/data/batch_generator.h mode change 100755 => 100644 src/data/corpus.cpp mode change 100755 => 100644 src/data/corpus.h mode change 100755 => 100644 src/data/corpus_base.h mode change 100755 => 100644 src/data/default_vocab.cpp mode change 100755 => 100644 src/data/factored_vocab.cpp mode change 100755 => 100644 src/data/factored_vocab.h mode change 100755 => 100644 src/data/sentencepiece_vocab.cpp mode change 100755 => 100644 src/data/text_input.cpp mode change 100755 => 100644 src/data/types.h mode change 100755 => 100644 src/data/vocab.cpp mode change 100755 => 100644 src/data/vocab.h mode change 100755 => 100644 src/data/vocab_base.h mode change 100755 => 100644 src/examples/mnist/dataset.h mode change 100755 => 100644 src/examples/mnist/download.sh mode change 100755 => 100644 src/examples/mnist/mnist_ffnn.cpp mode change 100755 => 100644 src/examples/mnist/model.h mode change 100755 => 100644 src/examples/mnist/training.h mode change 100755 => 100644 src/examples/mnist/validator.h mode change 100755 => 100644 src/functional/tmp.h mode change 100755 => 100644 src/graph/expression_operators.cpp mode change 100755 => 100644 src/graph/expression_operators.h mode change 100755 => 100644 src/graph/node_operators.h mode change 100755 => 100644 src/graph/node_operators_unary.h mode change 100755 => 100644 src/layers/constructors.h mode change 100755 => 100644 src/layers/factory.h mode change 100755 => 100644 src/layers/generic.cpp mode change 100755 => 100644 src/layers/generic.h mode change 100755 => 100644 src/layers/loss.cpp mode change 100755 => 100644 src/layers/loss.h mode change 100755 => 100644 src/microsoft/quicksand.cpp mode change 100755 => 100644 src/microsoft/quicksand.h mode change 100755 => 100644 src/models/bert.h mode change 100755 => 100644 src/models/char_s2s.h mode change 100755 => 100644 src/models/costs.h mode change 100755 => 100644 src/models/decoder.h mode change 100755 => 100644 src/models/encoder_classifier.h mode change 100755 => 100644 src/models/encoder_decoder.cpp mode change 100755 => 100644 src/models/encoder_decoder.h mode change 100755 => 100644 src/models/model_base.h mode change 100755 => 100644 src/models/model_factory.cpp mode change 100755 => 100644 src/models/model_factory.h mode change 100755 => 100644 src/models/s2s.h mode change 100755 => 100644 src/models/states.h mode change 100755 => 100644 src/models/transformer.h mode change 100755 => 100644 src/models/transformer_factory.h mode change 100755 => 100644 src/models/transformer_stub.cpp mode change 100755 => 100644 src/rescorer/rescorer.h mode change 100755 => 100644 src/rnn/attention_constructors.h mode change 100755 => 100644 src/rnn/cells.h mode change 100755 => 100644 src/rnn/types.h mode change 100755 => 100644 src/tensors/cpu/tensor_operators.cpp mode change 100755 => 100644 src/tensors/gpu/add.cu mode change 100755 => 100644 src/tensors/gpu/add.h mode change 100755 => 100644 src/tensors/gpu/add.inc mode change 100755 => 100644 src/tensors/gpu/element.cu mode change 100755 => 100644 src/tensors/gpu/element.inc mode change 100755 => 100644 src/tensors/gpu/prod.cpp mode change 100755 => 100644 src/tensors/gpu/prod.h mode change 100755 => 100644 src/tensors/gpu/tensor_operators.cu mode change 100755 => 100644 src/tests/prod.cpp mode change 100755 => 100644 src/tests/units/attention_tests.cpp mode change 100755 => 100644 src/tests/units/operator_tests.cpp mode change 100755 => 100644 src/tests/units/rnn_tests.cpp mode change 100755 => 100644 src/training/communicator.cpp mode change 100755 => 100644 src/training/graph_group.h mode change 100755 => 100644 src/training/graph_group_async.cpp mode change 100755 => 100644 src/training/graph_group_async.h mode change 100755 => 100644 src/training/graph_group_multinode.cpp mode change 100755 => 100644 src/training/graph_group_multinode_sync.cpp mode change 100755 => 100644 src/training/graph_group_singleton.cpp mode change 100755 => 100644 src/training/graph_group_singleton.h mode change 100755 => 100644 src/training/graph_group_sync.cpp mode change 100755 => 100644 src/training/graph_group_sync.h mode change 100755 => 100644 src/training/scheduler.h mode change 100755 => 100644 src/training/validator.cpp mode change 100755 => 100644 src/training/validator.h mode change 100755 => 100644 src/translator/beam_search.h mode change 100755 => 100644 src/translator/helpers.cpp mode change 100755 => 100644 src/translator/helpers.cu mode change 100755 => 100644 src/translator/helpers.h mode change 100755 => 100644 src/translator/history.h mode change 100755 => 100644 src/translator/hypothesis.h mode change 100755 => 100644 src/translator/nth_element.cpp mode change 100755 => 100644 src/translator/nth_element.cu mode change 100755 => 100644 src/translator/nth_element.h mode change 100755 => 100644 src/translator/output_printer.cpp mode change 100755 => 100644 src/translator/output_printer.h mode change 100755 => 100644 src/translator/scorers.cpp mode change 100755 => 100644 src/translator/scorers.h mode change 100755 => 100644 src/translator/translator.h mode change 100755 => 100644 vs/Marian.sln mode change 100755 => 100644 vs/Marian.vcxproj mode change 100755 => 100644 vs/Marian.vcxproj.filters diff --git a/contrib/autoformat.sh b/contrib/autoformat.sh old mode 100755 new mode 100644 diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/prepare_corpus.py b/scripts/embeddings/prepare_corpus.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/process_word2vec.py b/scripts/embeddings/process_word2vec.py old mode 100755 new mode 100644 diff --git a/scripts/server/client_example.py b/scripts/server/client_example.py old mode 100755 new mode 100644 diff --git a/scripts/shortlist/generate_shortlists.pl b/scripts/shortlist/generate_shortlists.pl old mode 100755 new mode 100644 diff --git a/scripts/shortlist/install.sh b/scripts/shortlist/install.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/ExceptionWithCallStack.h b/src/3rd_party/ExceptionWithCallStack.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/pathie-cpp/src/entry_iterator.cpp b/src/3rd_party/pathie-cpp/src/entry_iterator.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/astyle.sh b/src/3rd_party/spdlog/astyle.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/bench/latency/compare.sh b/src/3rd_party/spdlog/bench/latency/compare.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/tests/install_libcxx.sh b/src/3rd_party/spdlog/tests/install_libcxx.sh old mode 100755 new mode 100644 diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp old mode 100755 new mode 100644 diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100755 new mode 100644 diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100755 new mode 100644 diff --git a/src/common/logging.cpp b/src/common/logging.cpp old mode 100755 new mode 100644 diff --git a/src/common/options.h b/src/common/options.h old mode 100755 new mode 100644 diff --git a/src/common/shape.h b/src/common/shape.h old mode 100755 new mode 100644 diff --git a/src/common/timer.h b/src/common/timer.h old mode 100755 new mode 100644 diff --git a/src/common/types.h b/src/common/types.h old mode 100755 new mode 100644 diff --git a/src/common/utils.cpp b/src/common/utils.cpp old mode 100755 new mode 100644 diff --git a/src/common/utils.h b/src/common/utils.h old mode 100755 new mode 100644 diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h old mode 100755 new mode 100644 diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp old mode 100755 new mode 100644 diff --git a/src/data/corpus.h b/src/data/corpus.h old mode 100755 new mode 100644 diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100755 new mode 100644 diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h old mode 100755 new mode 100644 diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp old mode 100755 new mode 100644 diff --git a/src/data/types.h b/src/data/types.h old mode 100755 new mode 100644 diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100755 new mode 100644 diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/download.sh b/src/examples/mnist/download.sh old mode 100755 new mode 100644 diff --git a/src/examples/mnist/mnist_ffnn.cpp b/src/examples/mnist/mnist_ffnn.cpp old mode 100755 new mode 100644 diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/training.h b/src/examples/mnist/training.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h old mode 100755 new mode 100644 diff --git a/src/functional/tmp.h b/src/functional/tmp.h old mode 100755 new mode 100644 diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100755 new mode 100644 diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h old mode 100755 new mode 100644 diff --git a/src/graph/node_operators.h b/src/graph/node_operators.h old mode 100755 new mode 100644 diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h old mode 100755 new mode 100644 diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100755 new mode 100644 diff --git a/src/layers/factory.h b/src/layers/factory.h old mode 100755 new mode 100644 diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp old mode 100755 new mode 100644 diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100755 new mode 100644 diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp old mode 100755 new mode 100644 diff --git a/src/layers/loss.h b/src/layers/loss.h old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h old mode 100755 new mode 100644 diff --git a/src/models/bert.h b/src/models/bert.h old mode 100755 new mode 100644 diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h old mode 100755 new mode 100644 diff --git a/src/models/costs.h b/src/models/costs.h old mode 100755 new mode 100644 diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100755 new mode 100644 diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h old mode 100755 new mode 100644 diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100755 new mode 100644 diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100755 new mode 100644 diff --git a/src/models/model_base.h b/src/models/model_base.h old mode 100755 new mode 100644 diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp old mode 100755 new mode 100644 diff --git a/src/models/model_factory.h b/src/models/model_factory.h old mode 100755 new mode 100644 diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100755 new mode 100644 diff --git a/src/models/states.h b/src/models/states.h old mode 100755 new mode 100644 diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100755 new mode 100644 diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100755 new mode 100644 diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100755 new mode 100644 diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100755 new mode 100644 diff --git a/src/rnn/attention_constructors.h b/src/rnn/attention_constructors.h old mode 100755 new mode 100644 diff --git a/src/rnn/cells.h b/src/rnn/cells.h old mode 100755 new mode 100644 diff --git a/src/rnn/types.h b/src/rnn/types.h old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.h b/src/tensors/gpu/add.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/element.cu b/src/tensors/gpu/element.cu old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu old mode 100755 new mode 100644 diff --git a/src/tests/prod.cpp b/src/tests/prod.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/attention_tests.cpp b/src/tests/units/attention_tests.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/operator_tests.cpp b/src/tests/units/operator_tests.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/rnn_tests.cpp b/src/tests/units/rnn_tests.cpp old mode 100755 new mode 100644 diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h old mode 100755 new mode 100644 diff --git a/src/training/scheduler.h b/src/training/scheduler.h old mode 100755 new mode 100644 diff --git a/src/training/validator.cpp b/src/training/validator.cpp old mode 100755 new mode 100644 diff --git a/src/training/validator.h b/src/training/validator.h old mode 100755 new mode 100644 diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100755 new mode 100644 diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/helpers.cu b/src/translator/helpers.cu old mode 100755 new mode 100644 diff --git a/src/translator/helpers.h b/src/translator/helpers.h old mode 100755 new mode 100644 diff --git a/src/translator/history.h b/src/translator/history.h old mode 100755 new mode 100644 diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.h b/src/translator/nth_element.h old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.cpp b/src/translator/output_printer.cpp old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100755 new mode 100644 diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100755 new mode 100644 diff --git a/src/translator/translator.h b/src/translator/translator.h old mode 100755 new mode 100644 diff --git a/vs/Marian.sln b/vs/Marian.sln old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100755 new mode 100644 From 9ff0025fd52d11b4f1690183066a35cecec2efe6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 26 Apr 2019 20:33:11 -0700 Subject: [PATCH 441/838] torwards breakDown --- src/translator/beam_search.h | 11 ++++------- src/translator/hypothesis.h | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 35c2d741a..274cd4079 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -105,15 +105,12 @@ class BeamSearch { // Set score breakdown for n-best lists if(options_->get("n-best")) { - std::vector breakDown(states.size(), 0); - beam[beamHypIdx]->getScoreBreakdown().resize(states.size(), 0); // @TODO: Why? Can we just guard the read-out below, then make it const? Or getScoreBreakdown(j)? + auto breakDown = beam[beamHypIdx]->getScoreBreakdown(); + breakDown.resize(states.size(), 0); // reset to 0 if at start for(size_t j = 0; j < states.size(); ++j) { size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' - flattenedLogitIndex; -#if 0 // @BUGBUG: This currently segfaults with factors. - breakDown[j] = states[j]->breakDown(flattenedLogitIndex) + beam[beamHypIdx]->getScoreBreakdown()[j]; -#endif - // @TODO: pass those 3 indices directly into breakDown (state knows the dimensions) + // @TODO: push the index through into breakDown(), which has all dimensions + breakDown[j] += states[j]->breakDown(flattenedLogitIndex); } hyp->setScoreBreakdown(breakDown); } diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index cf19f59e2..cf5d9b601 100755 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -29,7 +29,7 @@ class Hypothesis { float getPathScore() const { return pathScore_; } - std::vector& getScoreBreakdown() { return scoreBreakdown_; } // @TODO: make this const + const std::vector& getScoreBreakdown() { return scoreBreakdown_; } void setScoreBreakdown(const std::vector& scoreBreaddown) { scoreBreakdown_ = scoreBreaddown; } const std::vector& getAlignment() { return alignment_; } From ac3f1a0a4f91e76527154d2347e5b11c89d2a24f Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sat, 27 Apr 2019 22:34:25 +0000 Subject: [PATCH 442/838] Actually install signal handler for graceful shutdown; fix termination msg. 1. installSignalHandlers_() wasn't called in Scheduler::Scheduler(). Now it is. 2. The termination message in Scheduler::finished() did not log termination vs. interrupt correctly. Fixed now. --- src/training/scheduler.cpp | 22 +++++++++++++++++----- src/training/scheduler.h | 5 +++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/training/scheduler.cpp b/src/training/scheduler.cpp index f508f3069..19d19f67e 100644 --- a/src/training/scheduler.cpp +++ b/src/training/scheduler.cpp @@ -1,5 +1,6 @@ #include "scheduler.h" #include +#include namespace marian { bool Scheduler::sigterm_{false}; @@ -9,13 +10,24 @@ bool Scheduler::sigusr2_{false}; void Scheduler:: signalHandler_(int sig) { + // Note: sys_siglist[sig] or stdsignal() describe the effect (e.g., + // 'Terminated' rather than provide the signal name (which are #define(s) + // in signal.h), so we have to do custom log messages here. switch (sig) { - case SIGTERM: Scheduler::sigterm_ = true; break; - case SIGUSR1: Scheduler::sigusr1_ = true; break; - case SIGUSR2: Scheduler::sigusr2_ = true; break; + case SIGTERM: // save models and exit + LOG(info, "[training] Scheduler received signal SIGTERM"); + Scheduler::sigterm_ = true; + break; + case SIGUSR1: // currently has no effect + LOG(info, "[training] Scheduler received signal SIGUSR1"); + Scheduler::sigusr1_ = true; + break; + case SIGUSR2: // currently has no effect + LOG(info, "[training] Scheduler received signal SIGUSR2"); + Scheduler::sigusr2_ = true; + break; default: - ABORT("This signal handler should not have been installed for signal ", - strsignal(sig)); + ABORT("No action defined for signal {}", sig); } } diff --git a/src/training/scheduler.h b/src/training/scheduler.h index 4ffcbce95..b888f35b0 100644 --- a/src/training/scheduler.h +++ b/src/training/scheduler.h @@ -160,6 +160,7 @@ class Scheduler : public TrainingObserver { saveFreq_ = options_->get("save-freq"); ABORT_IF(state_->factor != 1, "state.factor unexpectedly not 1 at this point??"); updateLearningRate(*state); + installSignalHandlers_(); } bool keepGoing(bool checkForSigTerm=true) { @@ -195,9 +196,9 @@ class Scheduler : public TrainingObserver { void started() { LOG(info, "Training started"); } void finished() { if (keepGoing(false)) // false means: ignore sigterm flag - LOG(info, "Training finished"); - else LOG(info, "Training interrupted (SIGTERM)."); + else + LOG(info, "Training finished"); } void setupValidators(std::vector>& vocabs) { From d00a28e49a898bc78694a6a15ef05c5ec39b5032 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Sun, 28 Apr 2019 10:58:50 -0700 Subject: [PATCH 443/838] brought back score breakdown --- src/layers/generic.cpp | 7 +++++++ src/layers/generic.h | 2 +- src/translator/beam_search.h | 11 ++++++++--- src/translator/scorers.h | 2 -- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index e5d35e719..a7456a80b 100644 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -106,6 +106,13 @@ namespace marian { return sel; } + // used for breakDown() only + // Index is flattened + Tensor Logits::getFactoredLogitsTensor(size_t groupIndex) const { + ABORT_IF(empty(), "Attempted to read out logits on empty Logits object"); + return logits_[groupIndex]->loss()->val(); + } + // This function assumes that the object holds one or more factor logits, which are summed up // into output-vocab logits according to the factored model (with correct normalization of factors). // This is infeasible for realistic factor sets, and therefore only implemented for 1 factor. diff --git a/src/layers/generic.h b/src/layers/generic.h index 7cf8ac6ce..9d787a3c3 100644 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -98,7 +98,7 @@ class Logits { MaskedFactorIndices(const Words& words) { indices = toWordIndexVector(words); } // we can leave masks uninitialized for this special use case }; std::vector factorizeWords(const Words& words) const; // breaks encoded Word into individual factor indices - float getLogitAt(size_t i) const { return getLogits()->val()->get(i); } // used for breakDown() only; @TODO: avoid the fully expanded logits; pass separate indices instead of 'i' + Tensor getFactoredLogitsTensor(size_t factorGroup) const; // used for breakDown() only size_t getNumFactorGroups() const { return logits_.size(); } bool empty() const { return logits_.empty(); } Logits withCounts(const Expr& count) const; // create new Logits with 'count' implanted into all logits_ diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 274cd4079..baa87cd88 100644 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -90,7 +90,8 @@ class BeamSearch { //LOG(info, "expand word {}={} with factor[{}] {}", beam[beamHypIdx]->getWord().toWordIndex(), // factoredVocab->word2string(beam[beamHypIdx]->getWord()), factorGroup, wordIdx); word = beam[beamHypIdx]->getWord(); - ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); + ABORT_IF(!factoredVocab->canExpandFactoredWord(word, factorGroup), + "A word without this factor snuck through to here??"); word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); prevBeamHypIdx = prevHyp->getPrevStateIndex(); prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words @@ -106,11 +107,15 @@ class BeamSearch { // Set score breakdown for n-best lists if(options_->get("n-best")) { auto breakDown = beam[beamHypIdx]->getScoreBreakdown(); + ABORT_IF(factoredVocab && factorGroup > 0 && !factoredVocab->canExpandFactoredWord(word, factorGroup), + "A word without this factor snuck through to here??"); breakDown.resize(states.size(), 0); // reset to 0 if at start for(size_t j = 0; j < states.size(); ++j) { + auto lval = states[j]->getLogProbs().getFactoredLogitsTensor(factorGroup); // [localBeamSize, 1, dimBatch, dimFactorVocab] size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' - // @TODO: push the index through into breakDown(), which has all dimensions - breakDown[j] += states[j]->breakDown(flattenedLogitIndex); + // @TODO: use a function on shape() to index, or new method val->at({i1, i2, i3, i4}) with broadcasting + ABORT_IF(lval->shape() != Shape({(int)beams.size(), 1, (int)dimBatch, (int)vocabSize}), "Unexpected shape of logits??"); + breakDown[j] += lval->get(i); } hyp->setScoreBreakdown(breakDown); } diff --git a/src/translator/scorers.h b/src/translator/scorers.h index 389dae641..ef47f4313 100644 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -11,8 +11,6 @@ class ScorerState { public: virtual Logits getLogProbs() const = 0; - virtual float breakDown(size_t i) const { return getLogProbs().getLogitAt(i); } - virtual void blacklist(Expr /*totalCosts*/, Ptr /*batch*/){}; }; From 21269a2a85c861facc14f04784c3a75718628e76 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Sun, 28 Apr 2019 19:24:34 -0700 Subject: [PATCH 444/838] fix node initializers --- src/layers/generic.cpp | 11 ++++++----- src/tensors/gpu/tensor_operators.cu | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index e5d35e719..41160166b 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -236,11 +236,11 @@ namespace marian { Wt_ = tiedParam_; } else { if (graph_->get(name + "_W")) { // support of legacy models that did not transpose - Wt_ = graph_->param(name + "_W", {inputDim, numOutputClasses}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_W", {inputDim, numOutputClasses}, inits::glorot_uniform2(true, false)); isLegacyUntransposedW = true; } else // this is the regular case: - Wt_ = graph_->param(name + "_Wt", {numOutputClasses, inputDim}, inits::glorot_uniform); + Wt_ = graph_->param(name + "_Wt", {numOutputClasses, inputDim}, inits::glorot_uniform2(false, true)); } b_ = graph_->param(name + "_b", {1, numOutputClasses}, inits::zeros); @@ -336,9 +336,10 @@ namespace marian { } // Embedding layer initialization should depend only on embedding size, hence fanIn=false - //NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); - NodeInitializer initFunc = inits::glorot_uniform; - if (options_->has("embFile")) { + NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); + //NodeInitializer initFunc = inits::glorot_uniform; + +if (options_->has("embFile")) { std::string file = opt("embFile"); if (!file.empty()) { bool norm = opt("normalization", false); diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index 60c6f8e3f..a7edcdf59 100755 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -488,7 +488,6 @@ __global__ void gSoftmax(float* out, so[i] = so[i] / sum; } } - __syncthreads(); } __syncthreads(); } @@ -574,7 +573,6 @@ __global__ void gLogSoftmax(float* out, if(id < cols) so[id] -= __logf(_sum[0]); } - __syncthreads(); } __syncthreads(); } @@ -635,7 +633,6 @@ __global__ void gSoftmaxGrad(float* grad, gradRow[id] += val; } } - __syncthreads(); } __syncthreads(); } @@ -694,7 +691,6 @@ __global__ void gLogSoftmaxGrad(float* grad, if(id < cols) gradRow[id] += adjRow[id] - (expf(valRow[id]) * _sum[0]); } - __syncthreads(); } __syncthreads(); } From ef03692df83a829d491b156b41121a7e62a60dcc Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Mon, 29 Apr 2019 20:16:06 +0100 Subject: [PATCH 445/838] Bug fixes related to (lack of) sqlite under Windows. --- src/data/corpus.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp index 0bde62322..00195bb1c 100644 --- a/src/data/corpus.cpp +++ b/src/data/corpus.cpp @@ -5,6 +5,7 @@ #include "common/utils.h" #include "data/corpus.h" +#include "data/corpus_sqlite.h" namespace marian { namespace data { @@ -176,11 +177,10 @@ Ptr prepareTrainingData(Ptr options) { // factory function to set up the training corpus for training // code moved here from Train::run() in training.h Ptr dataset; -#ifndef _MSC_VER // @TODO: include SqLite in Visual Studio project if(!options->get("sqlite").empty()) +#ifdef _MSC_VER // @TODO: include SqLite in Visual Studio project ABORT("SqLite presently not supported on Windows"); #else - if(!options->get("sqlite").empty()) dataset = New(options); #endif else From 081b843dc52d69a1ac306284983bef8d54e90196 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 16:03:48 -0700 Subject: [PATCH 446/838] bug fix: getAlignmentsForHypothesis() should not be called for all factors for each word --- src/data/corpus_base.h | 2 +- src/translator/beam_search.h | 20 +++++++++++--------- src/translator/scorers.h | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) mode change 100644 => 100755 src/data/corpus_base.h mode change 100644 => 100755 src/translator/beam_search.h mode change 100644 => 100755 src/translator/scorers.h diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100644 new mode 100755 index 8f836f357..f17df88da --- a/src/data/corpus_base.h +++ b/src/data/corpus_base.h @@ -286,7 +286,7 @@ class CorpusBatch : public Batch { size_t sizeTrg() const override { return subBatches_.back()->batchSize(); } /** - * @brief The number of words for the longest sentence in the batch plus one. + * @brief The total number of words in the batch (not counting masked-out words). */ size_t wordsTrg() const override { return subBatches_.back()->batchWords(); }; diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100644 new mode 100755 index 274cd4079..db435b9bd --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -45,7 +45,7 @@ class BeamSearch { Ptr factoredVocab, size_t factorGroup) const { std::vector align; if(options_->hasAndNotEmpty("alignment")) - align = scorers_[0]->getAlignment(); // use alignments from the first scorer, even if ensemble + align = scorers_[0]->getAlignment(); // [beam depth, max src length, batch size, 1]; use alignments from the first scorer, even if ensemble const auto dimBatch = beams.size(); Beams newBeams(dimBatch); // return value of this function goes here @@ -116,7 +116,7 @@ class BeamSearch { } // Set alignments - if(!align.empty()) { + if(!align.empty() && factorGroup == 0) { hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); } @@ -150,10 +150,10 @@ class BeamSearch { } std::vector getAlignmentsForHypothesis( - const std::vector alignAll, + const std::vector alignAll, // [beam depth, max src length, batch size, 1] Ptr batch, int beamHypIdx, - int beamIdx) const { + int batchIdx) const { // Let's B be the beam size, N be the number of batched sentences, // and L the number of words in the longest sentence in the batch. // The alignment vector: @@ -171,13 +171,15 @@ class BeamSearch { // in a single beam, i.e.: // * [word1-batch1, word1-batch2, ..., word2-batch1, ...] // - size_t batchSize = batch->size(); - size_t batchWidth = batch->width() * batchSize; + size_t batchSize = batch->size(); // number of sentences in batch + size_t batchWidth = batch->width(); // max src length + size_t batchWidthXSize = batchWidth * batchSize; // total number of words in the batch incl. padding std::vector align; - for(size_t w = 0; w < batchWidth / batchSize; ++w) { - size_t a = ((batchWidth * beamHypIdx) + beamIdx) + (batchSize * w); - size_t m = a % batchWidth; + // loop over words of batch entry 'batchIdx' and beam entry 'beamHypIdx' + for(size_t w = 0; w < batchWidth; ++w) { + size_t a = ((batchWidthXSize * beamHypIdx) + batchIdx) + (batchSize * w); + size_t m = a % batchWidthXSize; // == batchIdx + (batchSize * w) if(batch->front()->mask()[m] != 0) align.emplace_back(alignAll[a]); } diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100644 new mode 100755 index 389dae641..952856ce8 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -130,7 +130,7 @@ class ScorerWrapper : public Scorer { }; virtual std::vector getAlignment() override { - return encdec_->getAlignment().front(); + return encdec_->getAlignment().front(); // [beam depth, max src length, batch size, 1] } }; From f3ea35cb15894f0001366361b315058f350c2a06 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 16:23:30 -0700 Subject: [PATCH 447/838] bug fix: score breakdown should use correct index and dimensions --- src/data/factored_vocab.cpp | 5 +++-- src/layers/constructors.h | 38 +++++------------------------------- src/training/validator.h | 1 + src/translator/beam_search.h | 13 ++++++------ 4 files changed, 16 insertions(+), 41 deletions(-) mode change 100644 => 100755 src/data/factored_vocab.cpp mode change 100644 => 100755 src/layers/constructors.h mode change 100644 => 100755 src/training/validator.h diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp old mode 100644 new mode 100755 index f6208fece..3c7c8bc8e --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -13,6 +13,7 @@ // * factor name (e.g. all-caps: |ca ) // * factor index (e.g. |ca is index 0 inside |ca |ci |cn) // * factor unit index (|ca is unit 41324 in joint factor vocab) +// Also remove references to older outdated versions. namespace marian { @@ -39,7 +40,7 @@ namespace marian { // - _x <-> form only allows "_x <->" or "_x <-> _has_x" (same x), and is otherwise unused // - _lemma is special // The current version of the code just converts it internally to the legacy form. - // Once the legacy form is no longer needed, must of this can be simplified a lot. + // @TODO: Once the legacy form is no longer needed, simplify this. io::InputFileStream in(modelPath); WordIndex v = 0; std::map> factorTypeMap; // [type name] -> {factor-type names} @@ -711,7 +712,7 @@ void FactoredVocab::WordLUT::dumpToFile(const std::string& path) { out << kvp.second << "\t" << utils::withCommas(kvp.first) << "\n"; } -const static std::vector exts{ ".fsv", ".fm"/*legacy*/ }; +const static std::vector exts{ ".fsv", ".fm"/*legacy*/ }; // @TODO: delete the legacy one // Note: This does not actually load it, only checks the path for the type. // Since loading takes a while, we cache instances. diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100644 new mode 100755 index ed2c029f8..54e1b3404 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -10,22 +10,6 @@ namespace mlp { * Base class for layer factories, can be used in a multi-layer network factory. */ struct LayerFactory : public Factory { - //LayerFactory() : Factory() {} - //LayerFactory(const LayerFactory&) = default; - //LayerFactory(LayerFactory&&) = default; - // - //virtual ~LayerFactory() {} - - //template - //inline Ptr as() { - // return std::dynamic_pointer_cast(shared_from_this()); - //} - // - //template - //inline bool is() { - // return as() != nullptr; - //} - virtual Ptr construct(Ptr graph) = 0; }; @@ -52,24 +36,10 @@ typedef Accumulator dense; * Factory for output layers, can be used in a multi-layer network factory. */ struct LogitLayerFactory : public Factory { - //LogitLayerFactory() : Factory() {} - //LogitLayerFactory(const LogitLayerFactory&) = default; - //LogitLayerFactory(LogitLayerFactory&&) = default; - // - //virtual ~LogitLayerFactory() {} - // - //template - //inline Ptr as() { - // return std::dynamic_pointer_cast(shared_from_this()); - //} - // - //template - //inline bool is() { - // return as() != nullptr; - //} - virtual Ptr construct(Ptr graph) = 0; }; + +// @TODO: In the long run, I hope we can get rid of the abstract factories altogether. class OutputFactory : public LogitLayerFactory { protected: std::string tiedTransposedName_; @@ -199,7 +169,9 @@ class MLPFactory : public Factory { return Accumulator(*this); } - // special case for last layer, which may be a IUnaryLogitLayer. Requires some hackery + // Special case for last layer, which may be a IUnaryLogitLayer. Requires some hackery, + // which will go away if we get rid of the abstract factories, and instead just construct + // all layers immediately, which is my long-term goal for Marian. private: template class AsLayerFactory : public LayerFactory { diff --git a/src/training/validator.h b/src/training/validator.h old mode 100644 new mode 100755 index 37833b605..968fcf4ff --- a/src/training/validator.h +++ b/src/training/validator.h @@ -597,6 +597,7 @@ class BleuValidator : public Validator { quiet_(options_->get("quiet-translation")) { builder_ = models::createModelFromOptions(options_, models::usage::translation); + // @TODO: replace bleu-detok by a separate parameter to enable (various forms of) detok auto vocab = vocabs_.back(); ABORT_IF(detok_ && vocab->type() != "SentencePieceVocab" && vocab->type() != "FactoredVocab", "Detokenizing BLEU validator expects the target vocabulary to be SentencePieceVocab or FactoredVocab. " diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 5e2808e09..3747ac90a 100755 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -109,13 +109,13 @@ class BeamSearch { auto breakDown = beam[beamHypIdx]->getScoreBreakdown(); ABORT_IF(factoredVocab && factorGroup > 0 && !factoredVocab->canExpandFactoredWord(word, factorGroup), "A word without this factor snuck through to here??"); - breakDown.resize(states.size(), 0); // reset to 0 if at start + breakDown.resize(states.size(), 0); // at start, this is empty, so this will set the initial score to 0 for(size_t j = 0; j < states.size(); ++j) { auto lval = states[j]->getLogProbs().getFactoredLogitsTensor(factorGroup); // [localBeamSize, 1, dimBatch, dimFactorVocab] size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' // @TODO: use a function on shape() to index, or new method val->at({i1, i2, i3, i4}) with broadcasting - ABORT_IF(lval->shape() != Shape({(int)beams.size(), 1, (int)dimBatch, (int)vocabSize}), "Unexpected shape of logits??"); - breakDown[j] += lval->get(i); + ABORT_IF(lval->shape() != Shape({(int)beam.size(), 1, (int)dimBatch, (int)vocabSize}), "Unexpected shape of logits??"); + breakDown[j] += lval->get(flattenedLogitIndex); } hyp->setScoreBreakdown(breakDown); } @@ -128,7 +128,8 @@ class BeamSearch { newBeam.push_back(hyp); } - // also propagate factored hypotheses that do not get expanded in this step as they don't have this factor + // if factored vocab and this is not the first factor, we need to + // also propagate factored hypotheses that do not get expanded in this step because they don't have this factor if (factorGroup > 0) { for (size_t batchIdx = 0; batchIdx < beams.size(); batchIdx++) { const auto& beam = beams[batchIdx]; @@ -218,7 +219,7 @@ class BeamSearch { factoredVocab.reset(); #endif size_t numFactorGroups = factoredVocab ? factoredVocab->getNumGroups() : 1; - if (numFactorGroups == 1) // if no factors then reset + if (numFactorGroups == 1) // if no factors then we didn't need this object in the first place factoredVocab.reset(); const int dimBatch = (int)batch->size(); @@ -275,7 +276,7 @@ class BeamSearch { break; for (size_t factorGroup = 0; factorGroup < numFactorGroups; factorGroup++) { - // Note: not indenting the block, for easier merging + // @TODO: Indent the body of this loop. Not done for this commit for easier reviewing. // for factored vocabs, we do one factor at a time, but without updating the scorer for secondary factors //********************************************************************** From 1ff9242f2b4380ccdf69ce303ac73fbf863add5f Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Mon, 29 Apr 2019 16:26:18 -0700 Subject: [PATCH 448/838] fix embeddings pre-loading --- scripts/embeddings/export_embeddings.py | 0 src/common/utils.cpp | 1 + src/layers/word2vec_reader.h | 3 ++- src/tensors/tensor.h | 2 +- src/translator/beam_search.h | 5 +++-- 5 files changed, 7 insertions(+), 4 deletions(-) mode change 100644 => 100755 scripts/embeddings/export_embeddings.py diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py old mode 100644 new mode 100755 diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 4cb872ae2..856aa640d 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -117,6 +117,7 @@ std::string exec(const std::string& cmd, const std::vector& args /* cmdLine += " " + escapeForPOpen(a); if (!arg.empty()) cmdLine += " " + escapeForPOpen(arg); + std::cerr << "###" << cmdLine << "###" << std::endl; std::shared_ptr pipe(popen(cmdLine.c_str(), "r"), pclose); if(!pipe) ABORT("popen() failed!"); diff --git a/src/layers/word2vec_reader.h b/src/layers/word2vec_reader.h index 88c695aa9..4bfc67091 100644 --- a/src/layers/word2vec_reader.h +++ b/src/layers/word2vec_reader.h @@ -64,13 +64,14 @@ class Word2VecReader { } } + embs.resize(dimVoc * dimEmb, 0); // @TODO: is it correct to zero out the remaining embeddings? return embs; } private: std::vector randomEmbeddings(int dimVoc, int dimEmb) { std::vector values; - values.reserve(dimEmb); + values.resize(dimEmb); // Glorot numal distribution float scale = sqrtf(2.0f / (dimVoc + dimEmb)); diff --git a/src/tensors/tensor.h b/src/tensors/tensor.h index f77259cb9..9ec69effa 100644 --- a/src/tensors/tensor.h +++ b/src/tensors/tensor.h @@ -141,7 +141,7 @@ class TensorBase : public std::enable_shared_from_this { template void set(const T* begin, const T* end) { ABORT_IF(end - begin != shape_.elements(), - "Vector size ({}) and underlying type ({}) do not match", + "Vector size ({}) and underlying shape ({}) do not match", end - begin, std::string(shape_)); ABORT_IF(!matchType(type_), diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index baa87cd88..5a4a4ad60 100644 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -114,8 +114,9 @@ class BeamSearch { auto lval = states[j]->getLogProbs().getFactoredLogitsTensor(factorGroup); // [localBeamSize, 1, dimBatch, dimFactorVocab] size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' // @TODO: use a function on shape() to index, or new method val->at({i1, i2, i3, i4}) with broadcasting - ABORT_IF(lval->shape() != Shape({(int)beams.size(), 1, (int)dimBatch, (int)vocabSize}), "Unexpected shape of logits??"); - breakDown[j] += lval->get(i); + ABORT_IF(lval->shape() != Shape({(int)nBestBeamSize, 1, (int)dimBatch, (int)vocabSize}), + "Unexpected shape of logits?? {} != {}", lval->shape(), Shape({(int)nBestBeamSize, 1, (int)dimBatch, (int)vocabSize})); + breakDown[j] += lval->get(flattenedLogitIndex); } hyp->setScoreBreakdown(breakDown); } From a58214ba4d3cedbe191bf7836aae3b5b1d5e352f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 16:39:26 -0700 Subject: [PATCH 449/838] minor polish to previos commit --- src/common/utils.cpp | 2 +- src/layers/generic.cpp | 3 +-- src/tensors/gpu/tensor_operators.cu | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 856aa640d..0e406958a 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -117,7 +117,7 @@ std::string exec(const std::string& cmd, const std::vector& args /* cmdLine += " " + escapeForPOpen(a); if (!arg.empty()) cmdLine += " " + escapeForPOpen(arg); - std::cerr << "###" << cmdLine << "###" << std::endl; + //std::cerr << "###" << cmdLine << "###" << std::endl; std::shared_ptr pipe(popen(cmdLine.c_str(), "r"), pclose); if(!pipe) ABORT("popen() failed!"); diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 4d04a66cb..d13d5871a 100644 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -344,9 +344,8 @@ namespace marian { // Embedding layer initialization should depend only on embedding size, hence fanIn=false NodeInitializer initFunc = inits::glorot_uniform2(/*fanIn=*/false, /*fanOut=*/true); - //NodeInitializer initFunc = inits::glorot_uniform; -if (options_->has("embFile")) { + if (options_->has("embFile")) { std::string file = opt("embFile"); if (!file.empty()) { bool norm = opt("normalization", false); diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu index a7edcdf59..30c04c2fe 100644 --- a/src/tensors/gpu/tensor_operators.cu +++ b/src/tensors/gpu/tensor_operators.cu @@ -105,7 +105,7 @@ __global__ void gInsertCols(float* out, size_t offset_in) { for(int bid = 0; bid < rows; bid += gridDim.x) { int j = bid + blockIdx.x; - if(j < rows) { + if(j < rows) { // @TODO: change to if j == rows then break, as that's what it means. In 4 functions in here. float* rowOut = out + j * cols_out + offset_out; const float* rowIn = in + j * cols_in + offset_in; From cd329be400d3e1dafb3579f5d76e6215262d2324 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 17:52:23 -0700 Subject: [PATCH 450/838] weird mode change --- contrib/autoformat.sh | 0 scripts/bert/bert4marian.py | 0 scripts/embeddings/export_embeddings.py | 0 scripts/embeddings/prepare_corpus.py | 0 scripts/embeddings/process_word2vec.py | 0 scripts/server/client_example.py | 0 scripts/shortlist/generate_shortlists.pl | 0 scripts/shortlist/install.sh | 0 src/3rd_party/ExceptionWithCallStack.h | 0 src/3rd_party/pathie-cpp/src/entry_iterator.cpp | 0 src/3rd_party/pathie-cpp/src/path.cpp | 0 src/3rd_party/spdlog/astyle.sh | 0 src/3rd_party/spdlog/bench/latency/compare.sh | 0 src/3rd_party/spdlog/tests/install_libcxx.sh | 0 src/command/marian_main.cpp | 0 src/common/config_parser.cpp | 0 src/common/definitions.h | 0 src/common/logging.cpp | 0 src/common/options.h | 0 src/common/shape.h | 0 src/common/timer.h | 0 src/common/types.h | 0 src/common/utils.cpp | 0 src/common/utils.h | 0 src/data/batch_generator.h | 0 src/data/corpus.cpp | 0 src/data/corpus.h | 0 src/data/corpus_base.h | 0 src/data/default_vocab.cpp | 0 src/data/factored_vocab.cpp | 0 src/data/factored_vocab.h | 0 src/data/sentencepiece_vocab.cpp | 0 src/data/text_input.cpp | 0 src/data/types.h | 0 src/data/vocab.cpp | 0 src/data/vocab.h | 0 src/data/vocab_base.h | 0 src/examples/mnist/dataset.h | 0 src/examples/mnist/download.sh | 0 src/examples/mnist/mnist_ffnn.cpp | 0 src/examples/mnist/model.h | 0 src/examples/mnist/training.h | 0 src/examples/mnist/validator.h | 0 src/functional/tmp.h | 0 src/graph/expression_operators.cpp | 0 src/graph/expression_operators.h | 0 src/graph/node_operators.h | 0 src/graph/node_operators_unary.h | 0 src/layers/constructors.h | 0 src/layers/factory.h | 0 src/layers/generic.cpp | 0 src/layers/generic.h | 0 src/layers/loss.cpp | 0 src/layers/loss.h | 0 src/microsoft/quicksand.cpp | 0 src/microsoft/quicksand.h | 0 src/models/bert.h | 0 src/models/char_s2s.h | 0 src/models/costs.h | 0 src/models/decoder.h | 0 src/models/encoder_classifier.h | 0 src/models/encoder_decoder.cpp | 0 src/models/encoder_decoder.h | 0 src/models/model_base.h | 0 src/models/model_factory.cpp | 0 src/models/model_factory.h | 0 src/models/s2s.h | 0 src/models/states.h | 0 src/models/transformer.h | 0 src/models/transformer_factory.h | 0 src/models/transformer_stub.cpp | 0 src/rescorer/rescorer.h | 0 src/rnn/attention_constructors.h | 0 src/rnn/cells.h | 0 src/rnn/types.h | 0 src/tensors/cpu/tensor_operators.cpp | 0 src/tensors/gpu/add.cu | 0 src/tensors/gpu/add.h | 0 src/tensors/gpu/add.inc | 0 src/tensors/gpu/element.cu | 0 src/tensors/gpu/element.inc | 0 src/tensors/gpu/prod.cpp | 0 src/tensors/gpu/prod.h | 0 src/tensors/gpu/tensor_operators.cu | 0 src/tests/prod.cpp | 0 src/tests/units/attention_tests.cpp | 0 src/tests/units/operator_tests.cpp | 0 src/tests/units/rnn_tests.cpp | 0 src/training/communicator.cpp | 0 src/training/graph_group.h | 0 src/training/graph_group_async.cpp | 0 src/training/graph_group_async.h | 0 src/training/graph_group_multinode.cpp | 0 src/training/graph_group_multinode_sync.cpp | 0 src/training/graph_group_singleton.cpp | 0 src/training/graph_group_singleton.h | 0 src/training/graph_group_sync.cpp | 0 src/training/graph_group_sync.h | 0 src/training/scheduler.h | 0 src/training/validator.cpp | 0 src/training/validator.h | 0 src/translator/beam_search.h | 0 src/translator/helpers.cpp | 0 src/translator/helpers.cu | 0 src/translator/helpers.h | 0 src/translator/history.h | 0 src/translator/hypothesis.h | 0 src/translator/nth_element.cpp | 0 src/translator/nth_element.cu | 0 src/translator/nth_element.h | 0 src/translator/output_printer.cpp | 0 src/translator/output_printer.h | 0 src/translator/scorers.cpp | 0 src/translator/scorers.h | 0 src/translator/translator.h | 0 vs/Marian.sln | 0 vs/Marian.vcxproj | 0 vs/Marian.vcxproj.filters | 0 118 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 contrib/autoformat.sh mode change 100644 => 100755 scripts/bert/bert4marian.py mode change 100644 => 100755 scripts/embeddings/export_embeddings.py mode change 100644 => 100755 scripts/embeddings/prepare_corpus.py mode change 100644 => 100755 scripts/embeddings/process_word2vec.py mode change 100644 => 100755 scripts/server/client_example.py mode change 100644 => 100755 scripts/shortlist/generate_shortlists.pl mode change 100644 => 100755 scripts/shortlist/install.sh mode change 100644 => 100755 src/3rd_party/ExceptionWithCallStack.h mode change 100644 => 100755 src/3rd_party/pathie-cpp/src/entry_iterator.cpp mode change 100644 => 100755 src/3rd_party/pathie-cpp/src/path.cpp mode change 100644 => 100755 src/3rd_party/spdlog/astyle.sh mode change 100644 => 100755 src/3rd_party/spdlog/bench/latency/compare.sh mode change 100644 => 100755 src/3rd_party/spdlog/tests/install_libcxx.sh mode change 100644 => 100755 src/command/marian_main.cpp mode change 100644 => 100755 src/common/config_parser.cpp mode change 100644 => 100755 src/common/definitions.h mode change 100644 => 100755 src/common/logging.cpp mode change 100644 => 100755 src/common/options.h mode change 100644 => 100755 src/common/shape.h mode change 100644 => 100755 src/common/timer.h mode change 100644 => 100755 src/common/types.h mode change 100644 => 100755 src/common/utils.cpp mode change 100644 => 100755 src/common/utils.h mode change 100644 => 100755 src/data/batch_generator.h mode change 100644 => 100755 src/data/corpus.cpp mode change 100644 => 100755 src/data/corpus.h mode change 100644 => 100755 src/data/corpus_base.h mode change 100644 => 100755 src/data/default_vocab.cpp mode change 100644 => 100755 src/data/factored_vocab.cpp mode change 100644 => 100755 src/data/factored_vocab.h mode change 100644 => 100755 src/data/sentencepiece_vocab.cpp mode change 100644 => 100755 src/data/text_input.cpp mode change 100644 => 100755 src/data/types.h mode change 100644 => 100755 src/data/vocab.cpp mode change 100644 => 100755 src/data/vocab.h mode change 100644 => 100755 src/data/vocab_base.h mode change 100644 => 100755 src/examples/mnist/dataset.h mode change 100644 => 100755 src/examples/mnist/download.sh mode change 100644 => 100755 src/examples/mnist/mnist_ffnn.cpp mode change 100644 => 100755 src/examples/mnist/model.h mode change 100644 => 100755 src/examples/mnist/training.h mode change 100644 => 100755 src/examples/mnist/validator.h mode change 100644 => 100755 src/functional/tmp.h mode change 100644 => 100755 src/graph/expression_operators.cpp mode change 100644 => 100755 src/graph/expression_operators.h mode change 100644 => 100755 src/graph/node_operators.h mode change 100644 => 100755 src/graph/node_operators_unary.h mode change 100644 => 100755 src/layers/constructors.h mode change 100644 => 100755 src/layers/factory.h mode change 100644 => 100755 src/layers/generic.cpp mode change 100644 => 100755 src/layers/generic.h mode change 100644 => 100755 src/layers/loss.cpp mode change 100644 => 100755 src/layers/loss.h mode change 100644 => 100755 src/microsoft/quicksand.cpp mode change 100644 => 100755 src/microsoft/quicksand.h mode change 100644 => 100755 src/models/bert.h mode change 100644 => 100755 src/models/char_s2s.h mode change 100644 => 100755 src/models/costs.h mode change 100644 => 100755 src/models/decoder.h mode change 100644 => 100755 src/models/encoder_classifier.h mode change 100644 => 100755 src/models/encoder_decoder.cpp mode change 100644 => 100755 src/models/encoder_decoder.h mode change 100644 => 100755 src/models/model_base.h mode change 100644 => 100755 src/models/model_factory.cpp mode change 100644 => 100755 src/models/model_factory.h mode change 100644 => 100755 src/models/s2s.h mode change 100644 => 100755 src/models/states.h mode change 100644 => 100755 src/models/transformer.h mode change 100644 => 100755 src/models/transformer_factory.h mode change 100644 => 100755 src/models/transformer_stub.cpp mode change 100644 => 100755 src/rescorer/rescorer.h mode change 100644 => 100755 src/rnn/attention_constructors.h mode change 100644 => 100755 src/rnn/cells.h mode change 100644 => 100755 src/rnn/types.h mode change 100644 => 100755 src/tensors/cpu/tensor_operators.cpp mode change 100644 => 100755 src/tensors/gpu/add.cu mode change 100644 => 100755 src/tensors/gpu/add.h mode change 100644 => 100755 src/tensors/gpu/add.inc mode change 100644 => 100755 src/tensors/gpu/element.cu mode change 100644 => 100755 src/tensors/gpu/element.inc mode change 100644 => 100755 src/tensors/gpu/prod.cpp mode change 100644 => 100755 src/tensors/gpu/prod.h mode change 100644 => 100755 src/tensors/gpu/tensor_operators.cu mode change 100644 => 100755 src/tests/prod.cpp mode change 100644 => 100755 src/tests/units/attention_tests.cpp mode change 100644 => 100755 src/tests/units/operator_tests.cpp mode change 100644 => 100755 src/tests/units/rnn_tests.cpp mode change 100644 => 100755 src/training/communicator.cpp mode change 100644 => 100755 src/training/graph_group.h mode change 100644 => 100755 src/training/graph_group_async.cpp mode change 100644 => 100755 src/training/graph_group_async.h mode change 100644 => 100755 src/training/graph_group_multinode.cpp mode change 100644 => 100755 src/training/graph_group_multinode_sync.cpp mode change 100644 => 100755 src/training/graph_group_singleton.cpp mode change 100644 => 100755 src/training/graph_group_singleton.h mode change 100644 => 100755 src/training/graph_group_sync.cpp mode change 100644 => 100755 src/training/graph_group_sync.h mode change 100644 => 100755 src/training/scheduler.h mode change 100644 => 100755 src/training/validator.cpp mode change 100644 => 100755 src/training/validator.h mode change 100644 => 100755 src/translator/beam_search.h mode change 100644 => 100755 src/translator/helpers.cpp mode change 100644 => 100755 src/translator/helpers.cu mode change 100644 => 100755 src/translator/helpers.h mode change 100644 => 100755 src/translator/history.h mode change 100644 => 100755 src/translator/hypothesis.h mode change 100644 => 100755 src/translator/nth_element.cpp mode change 100644 => 100755 src/translator/nth_element.cu mode change 100644 => 100755 src/translator/nth_element.h mode change 100644 => 100755 src/translator/output_printer.cpp mode change 100644 => 100755 src/translator/output_printer.h mode change 100644 => 100755 src/translator/scorers.cpp mode change 100644 => 100755 src/translator/scorers.h mode change 100644 => 100755 src/translator/translator.h mode change 100644 => 100755 vs/Marian.sln mode change 100644 => 100755 vs/Marian.vcxproj mode change 100644 => 100755 vs/Marian.vcxproj.filters diff --git a/contrib/autoformat.sh b/contrib/autoformat.sh old mode 100644 new mode 100755 diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/prepare_corpus.py b/scripts/embeddings/prepare_corpus.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/process_word2vec.py b/scripts/embeddings/process_word2vec.py old mode 100644 new mode 100755 diff --git a/scripts/server/client_example.py b/scripts/server/client_example.py old mode 100644 new mode 100755 diff --git a/scripts/shortlist/generate_shortlists.pl b/scripts/shortlist/generate_shortlists.pl old mode 100644 new mode 100755 diff --git a/scripts/shortlist/install.sh b/scripts/shortlist/install.sh old mode 100644 new mode 100755 diff --git a/src/3rd_party/ExceptionWithCallStack.h b/src/3rd_party/ExceptionWithCallStack.h old mode 100644 new mode 100755 diff --git a/src/3rd_party/pathie-cpp/src/entry_iterator.cpp b/src/3rd_party/pathie-cpp/src/entry_iterator.cpp old mode 100644 new mode 100755 diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp old mode 100644 new mode 100755 diff --git a/src/3rd_party/spdlog/astyle.sh b/src/3rd_party/spdlog/astyle.sh old mode 100644 new mode 100755 diff --git a/src/3rd_party/spdlog/bench/latency/compare.sh b/src/3rd_party/spdlog/bench/latency/compare.sh old mode 100644 new mode 100755 diff --git a/src/3rd_party/spdlog/tests/install_libcxx.sh b/src/3rd_party/spdlog/tests/install_libcxx.sh old mode 100644 new mode 100755 diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp old mode 100644 new mode 100755 diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100644 new mode 100755 diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100644 new mode 100755 diff --git a/src/common/logging.cpp b/src/common/logging.cpp old mode 100644 new mode 100755 diff --git a/src/common/options.h b/src/common/options.h old mode 100644 new mode 100755 diff --git a/src/common/shape.h b/src/common/shape.h old mode 100644 new mode 100755 diff --git a/src/common/timer.h b/src/common/timer.h old mode 100644 new mode 100755 diff --git a/src/common/types.h b/src/common/types.h old mode 100644 new mode 100755 diff --git a/src/common/utils.cpp b/src/common/utils.cpp old mode 100644 new mode 100755 diff --git a/src/common/utils.h b/src/common/utils.h old mode 100644 new mode 100755 diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h old mode 100644 new mode 100755 diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp old mode 100644 new mode 100755 diff --git a/src/data/corpus.h b/src/data/corpus.h old mode 100644 new mode 100755 diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100644 new mode 100755 diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp old mode 100644 new mode 100755 diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp old mode 100644 new mode 100755 diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h old mode 100644 new mode 100755 diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100644 new mode 100755 diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp old mode 100644 new mode 100755 diff --git a/src/data/types.h b/src/data/types.h old mode 100644 new mode 100755 diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp old mode 100644 new mode 100755 diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100644 new mode 100755 diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h old mode 100644 new mode 100755 diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h old mode 100644 new mode 100755 diff --git a/src/examples/mnist/download.sh b/src/examples/mnist/download.sh old mode 100644 new mode 100755 diff --git a/src/examples/mnist/mnist_ffnn.cpp b/src/examples/mnist/mnist_ffnn.cpp old mode 100644 new mode 100755 diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100644 new mode 100755 diff --git a/src/examples/mnist/training.h b/src/examples/mnist/training.h old mode 100644 new mode 100755 diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h old mode 100644 new mode 100755 diff --git a/src/functional/tmp.h b/src/functional/tmp.h old mode 100644 new mode 100755 diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100644 new mode 100755 diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h old mode 100644 new mode 100755 diff --git a/src/graph/node_operators.h b/src/graph/node_operators.h old mode 100644 new mode 100755 diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h old mode 100644 new mode 100755 diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100644 new mode 100755 diff --git a/src/layers/factory.h b/src/layers/factory.h old mode 100644 new mode 100755 diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp old mode 100644 new mode 100755 diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100644 new mode 100755 diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp old mode 100644 new mode 100755 diff --git a/src/layers/loss.h b/src/layers/loss.h old mode 100644 new mode 100755 diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100644 new mode 100755 diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h old mode 100644 new mode 100755 diff --git a/src/models/bert.h b/src/models/bert.h old mode 100644 new mode 100755 diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h old mode 100644 new mode 100755 diff --git a/src/models/costs.h b/src/models/costs.h old mode 100644 new mode 100755 diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100644 new mode 100755 diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h old mode 100644 new mode 100755 diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100644 new mode 100755 diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100644 new mode 100755 diff --git a/src/models/model_base.h b/src/models/model_base.h old mode 100644 new mode 100755 diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp old mode 100644 new mode 100755 diff --git a/src/models/model_factory.h b/src/models/model_factory.h old mode 100644 new mode 100755 diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100644 new mode 100755 diff --git a/src/models/states.h b/src/models/states.h old mode 100644 new mode 100755 diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100644 new mode 100755 diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100644 new mode 100755 diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100644 new mode 100755 diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100644 new mode 100755 diff --git a/src/rnn/attention_constructors.h b/src/rnn/attention_constructors.h old mode 100644 new mode 100755 diff --git a/src/rnn/cells.h b/src/rnn/cells.h old mode 100644 new mode 100755 diff --git a/src/rnn/types.h b/src/rnn/types.h old mode 100644 new mode 100755 diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/add.h b/src/tensors/gpu/add.h old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/element.cu b/src/tensors/gpu/element.cu old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h old mode 100644 new mode 100755 diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu old mode 100644 new mode 100755 diff --git a/src/tests/prod.cpp b/src/tests/prod.cpp old mode 100644 new mode 100755 diff --git a/src/tests/units/attention_tests.cpp b/src/tests/units/attention_tests.cpp old mode 100644 new mode 100755 diff --git a/src/tests/units/operator_tests.cpp b/src/tests/units/operator_tests.cpp old mode 100644 new mode 100755 diff --git a/src/tests/units/rnn_tests.cpp b/src/tests/units/rnn_tests.cpp old mode 100644 new mode 100755 diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100644 new mode 100755 diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h old mode 100644 new mode 100755 diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h old mode 100644 new mode 100755 diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100644 new mode 100755 diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h old mode 100644 new mode 100755 diff --git a/src/training/scheduler.h b/src/training/scheduler.h old mode 100644 new mode 100755 diff --git a/src/training/validator.cpp b/src/training/validator.cpp old mode 100644 new mode 100755 diff --git a/src/training/validator.h b/src/training/validator.h old mode 100644 new mode 100755 diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100644 new mode 100755 diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp old mode 100644 new mode 100755 diff --git a/src/translator/helpers.cu b/src/translator/helpers.cu old mode 100644 new mode 100755 diff --git a/src/translator/helpers.h b/src/translator/helpers.h old mode 100644 new mode 100755 diff --git a/src/translator/history.h b/src/translator/history.h old mode 100644 new mode 100755 diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h old mode 100644 new mode 100755 diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp old mode 100644 new mode 100755 diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu old mode 100644 new mode 100755 diff --git a/src/translator/nth_element.h b/src/translator/nth_element.h old mode 100644 new mode 100755 diff --git a/src/translator/output_printer.cpp b/src/translator/output_printer.cpp old mode 100644 new mode 100755 diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100644 new mode 100755 diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp old mode 100644 new mode 100755 diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100644 new mode 100755 diff --git a/src/translator/translator.h b/src/translator/translator.h old mode 100644 new mode 100755 diff --git a/vs/Marian.sln b/vs/Marian.sln old mode 100644 new mode 100755 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100644 new mode 100755 diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100644 new mode 100755 From eacfd2de114c495917d9cdf85f91f410d0caff3c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 19:01:29 -0700 Subject: [PATCH 451/838] weird mode change back --- contrib/autoformat.sh | 0 scripts/bert/bert4marian.py | 0 scripts/embeddings/export_embeddings.py | 0 scripts/embeddings/prepare_corpus.py | 0 scripts/embeddings/process_word2vec.py | 0 scripts/server/client_example.py | 0 scripts/shortlist/generate_shortlists.pl | 0 scripts/shortlist/install.sh | 0 src/3rd_party/ExceptionWithCallStack.h | 0 src/3rd_party/pathie-cpp/src/entry_iterator.cpp | 0 src/3rd_party/pathie-cpp/src/path.cpp | 0 src/3rd_party/spdlog/astyle.sh | 0 src/3rd_party/spdlog/bench/latency/compare.sh | 0 src/3rd_party/spdlog/tests/install_libcxx.sh | 0 src/command/marian_main.cpp | 0 src/common/config_parser.cpp | 0 src/common/definitions.h | 0 src/common/logging.cpp | 0 src/common/options.h | 0 src/common/shape.h | 0 src/common/timer.h | 0 src/common/types.h | 0 src/common/utils.cpp | 0 src/common/utils.h | 0 src/data/batch_generator.h | 0 src/data/corpus.cpp | 0 src/data/corpus.h | 0 src/data/corpus_base.h | 0 src/data/default_vocab.cpp | 0 src/data/factored_vocab.cpp | 0 src/data/factored_vocab.h | 0 src/data/sentencepiece_vocab.cpp | 0 src/data/text_input.cpp | 0 src/data/types.h | 0 src/data/vocab.cpp | 0 src/data/vocab.h | 0 src/data/vocab_base.h | 0 src/examples/mnist/dataset.h | 0 src/examples/mnist/download.sh | 0 src/examples/mnist/mnist_ffnn.cpp | 0 src/examples/mnist/model.h | 0 src/examples/mnist/training.h | 0 src/examples/mnist/validator.h | 0 src/functional/tmp.h | 0 src/graph/expression_operators.cpp | 0 src/graph/expression_operators.h | 0 src/graph/node_operators.h | 0 src/graph/node_operators_unary.h | 0 src/layers/constructors.h | 0 src/layers/factory.h | 0 src/layers/generic.cpp | 0 src/layers/generic.h | 0 src/layers/loss.cpp | 0 src/layers/loss.h | 0 src/microsoft/quicksand.cpp | 0 src/microsoft/quicksand.h | 0 src/models/bert.h | 0 src/models/char_s2s.h | 0 src/models/costs.h | 0 src/models/decoder.h | 0 src/models/encoder_classifier.h | 0 src/models/encoder_decoder.cpp | 0 src/models/encoder_decoder.h | 0 src/models/model_base.h | 0 src/models/model_factory.cpp | 0 src/models/model_factory.h | 0 src/models/s2s.h | 0 src/models/states.h | 0 src/models/transformer.h | 0 src/models/transformer_factory.h | 0 src/models/transformer_stub.cpp | 0 src/rescorer/rescorer.h | 0 src/rnn/attention_constructors.h | 0 src/rnn/cells.h | 0 src/rnn/types.h | 0 src/tensors/cpu/tensor_operators.cpp | 0 src/tensors/gpu/add.cu | 0 src/tensors/gpu/add.h | 0 src/tensors/gpu/add.inc | 0 src/tensors/gpu/element.cu | 0 src/tensors/gpu/element.inc | 0 src/tensors/gpu/prod.cpp | 0 src/tensors/gpu/prod.h | 0 src/tensors/gpu/tensor_operators.cu | 0 src/tests/prod.cpp | 0 src/tests/units/attention_tests.cpp | 0 src/tests/units/operator_tests.cpp | 0 src/tests/units/rnn_tests.cpp | 0 src/training/communicator.cpp | 0 src/training/graph_group.h | 0 src/training/graph_group_async.cpp | 0 src/training/graph_group_async.h | 0 src/training/graph_group_multinode.cpp | 0 src/training/graph_group_multinode_sync.cpp | 0 src/training/graph_group_singleton.cpp | 0 src/training/graph_group_singleton.h | 0 src/training/graph_group_sync.cpp | 0 src/training/graph_group_sync.h | 0 src/training/scheduler.h | 0 src/training/validator.cpp | 0 src/training/validator.h | 0 src/translator/beam_search.h | 0 src/translator/helpers.cpp | 0 src/translator/helpers.cu | 0 src/translator/helpers.h | 0 src/translator/history.h | 0 src/translator/hypothesis.h | 0 src/translator/nth_element.cpp | 0 src/translator/nth_element.cu | 0 src/translator/nth_element.h | 0 src/translator/output_printer.cpp | 0 src/translator/output_printer.h | 0 src/translator/scorers.cpp | 0 src/translator/scorers.h | 0 src/translator/translator.h | 0 vs/Marian.sln | 0 vs/Marian.vcxproj | 0 vs/Marian.vcxproj.filters | 0 118 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 contrib/autoformat.sh mode change 100755 => 100644 scripts/bert/bert4marian.py mode change 100755 => 100644 scripts/embeddings/export_embeddings.py mode change 100755 => 100644 scripts/embeddings/prepare_corpus.py mode change 100755 => 100644 scripts/embeddings/process_word2vec.py mode change 100755 => 100644 scripts/server/client_example.py mode change 100755 => 100644 scripts/shortlist/generate_shortlists.pl mode change 100755 => 100644 scripts/shortlist/install.sh mode change 100755 => 100644 src/3rd_party/ExceptionWithCallStack.h mode change 100755 => 100644 src/3rd_party/pathie-cpp/src/entry_iterator.cpp mode change 100755 => 100644 src/3rd_party/pathie-cpp/src/path.cpp mode change 100755 => 100644 src/3rd_party/spdlog/astyle.sh mode change 100755 => 100644 src/3rd_party/spdlog/bench/latency/compare.sh mode change 100755 => 100644 src/3rd_party/spdlog/tests/install_libcxx.sh mode change 100755 => 100644 src/command/marian_main.cpp mode change 100755 => 100644 src/common/config_parser.cpp mode change 100755 => 100644 src/common/definitions.h mode change 100755 => 100644 src/common/logging.cpp mode change 100755 => 100644 src/common/options.h mode change 100755 => 100644 src/common/shape.h mode change 100755 => 100644 src/common/timer.h mode change 100755 => 100644 src/common/types.h mode change 100755 => 100644 src/common/utils.cpp mode change 100755 => 100644 src/common/utils.h mode change 100755 => 100644 src/data/batch_generator.h mode change 100755 => 100644 src/data/corpus.cpp mode change 100755 => 100644 src/data/corpus.h mode change 100755 => 100644 src/data/corpus_base.h mode change 100755 => 100644 src/data/default_vocab.cpp mode change 100755 => 100644 src/data/factored_vocab.cpp mode change 100755 => 100644 src/data/factored_vocab.h mode change 100755 => 100644 src/data/sentencepiece_vocab.cpp mode change 100755 => 100644 src/data/text_input.cpp mode change 100755 => 100644 src/data/types.h mode change 100755 => 100644 src/data/vocab.cpp mode change 100755 => 100644 src/data/vocab.h mode change 100755 => 100644 src/data/vocab_base.h mode change 100755 => 100644 src/examples/mnist/dataset.h mode change 100755 => 100644 src/examples/mnist/download.sh mode change 100755 => 100644 src/examples/mnist/mnist_ffnn.cpp mode change 100755 => 100644 src/examples/mnist/model.h mode change 100755 => 100644 src/examples/mnist/training.h mode change 100755 => 100644 src/examples/mnist/validator.h mode change 100755 => 100644 src/functional/tmp.h mode change 100755 => 100644 src/graph/expression_operators.cpp mode change 100755 => 100644 src/graph/expression_operators.h mode change 100755 => 100644 src/graph/node_operators.h mode change 100755 => 100644 src/graph/node_operators_unary.h mode change 100755 => 100644 src/layers/constructors.h mode change 100755 => 100644 src/layers/factory.h mode change 100755 => 100644 src/layers/generic.cpp mode change 100755 => 100644 src/layers/generic.h mode change 100755 => 100644 src/layers/loss.cpp mode change 100755 => 100644 src/layers/loss.h mode change 100755 => 100644 src/microsoft/quicksand.cpp mode change 100755 => 100644 src/microsoft/quicksand.h mode change 100755 => 100644 src/models/bert.h mode change 100755 => 100644 src/models/char_s2s.h mode change 100755 => 100644 src/models/costs.h mode change 100755 => 100644 src/models/decoder.h mode change 100755 => 100644 src/models/encoder_classifier.h mode change 100755 => 100644 src/models/encoder_decoder.cpp mode change 100755 => 100644 src/models/encoder_decoder.h mode change 100755 => 100644 src/models/model_base.h mode change 100755 => 100644 src/models/model_factory.cpp mode change 100755 => 100644 src/models/model_factory.h mode change 100755 => 100644 src/models/s2s.h mode change 100755 => 100644 src/models/states.h mode change 100755 => 100644 src/models/transformer.h mode change 100755 => 100644 src/models/transformer_factory.h mode change 100755 => 100644 src/models/transformer_stub.cpp mode change 100755 => 100644 src/rescorer/rescorer.h mode change 100755 => 100644 src/rnn/attention_constructors.h mode change 100755 => 100644 src/rnn/cells.h mode change 100755 => 100644 src/rnn/types.h mode change 100755 => 100644 src/tensors/cpu/tensor_operators.cpp mode change 100755 => 100644 src/tensors/gpu/add.cu mode change 100755 => 100644 src/tensors/gpu/add.h mode change 100755 => 100644 src/tensors/gpu/add.inc mode change 100755 => 100644 src/tensors/gpu/element.cu mode change 100755 => 100644 src/tensors/gpu/element.inc mode change 100755 => 100644 src/tensors/gpu/prod.cpp mode change 100755 => 100644 src/tensors/gpu/prod.h mode change 100755 => 100644 src/tensors/gpu/tensor_operators.cu mode change 100755 => 100644 src/tests/prod.cpp mode change 100755 => 100644 src/tests/units/attention_tests.cpp mode change 100755 => 100644 src/tests/units/operator_tests.cpp mode change 100755 => 100644 src/tests/units/rnn_tests.cpp mode change 100755 => 100644 src/training/communicator.cpp mode change 100755 => 100644 src/training/graph_group.h mode change 100755 => 100644 src/training/graph_group_async.cpp mode change 100755 => 100644 src/training/graph_group_async.h mode change 100755 => 100644 src/training/graph_group_multinode.cpp mode change 100755 => 100644 src/training/graph_group_multinode_sync.cpp mode change 100755 => 100644 src/training/graph_group_singleton.cpp mode change 100755 => 100644 src/training/graph_group_singleton.h mode change 100755 => 100644 src/training/graph_group_sync.cpp mode change 100755 => 100644 src/training/graph_group_sync.h mode change 100755 => 100644 src/training/scheduler.h mode change 100755 => 100644 src/training/validator.cpp mode change 100755 => 100644 src/training/validator.h mode change 100755 => 100644 src/translator/beam_search.h mode change 100755 => 100644 src/translator/helpers.cpp mode change 100755 => 100644 src/translator/helpers.cu mode change 100755 => 100644 src/translator/helpers.h mode change 100755 => 100644 src/translator/history.h mode change 100755 => 100644 src/translator/hypothesis.h mode change 100755 => 100644 src/translator/nth_element.cpp mode change 100755 => 100644 src/translator/nth_element.cu mode change 100755 => 100644 src/translator/nth_element.h mode change 100755 => 100644 src/translator/output_printer.cpp mode change 100755 => 100644 src/translator/output_printer.h mode change 100755 => 100644 src/translator/scorers.cpp mode change 100755 => 100644 src/translator/scorers.h mode change 100755 => 100644 src/translator/translator.h mode change 100755 => 100644 vs/Marian.sln mode change 100755 => 100644 vs/Marian.vcxproj mode change 100755 => 100644 vs/Marian.vcxproj.filters diff --git a/contrib/autoformat.sh b/contrib/autoformat.sh old mode 100755 new mode 100644 diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/prepare_corpus.py b/scripts/embeddings/prepare_corpus.py old mode 100755 new mode 100644 diff --git a/scripts/embeddings/process_word2vec.py b/scripts/embeddings/process_word2vec.py old mode 100755 new mode 100644 diff --git a/scripts/server/client_example.py b/scripts/server/client_example.py old mode 100755 new mode 100644 diff --git a/scripts/shortlist/generate_shortlists.pl b/scripts/shortlist/generate_shortlists.pl old mode 100755 new mode 100644 diff --git a/scripts/shortlist/install.sh b/scripts/shortlist/install.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/ExceptionWithCallStack.h b/src/3rd_party/ExceptionWithCallStack.h old mode 100755 new mode 100644 diff --git a/src/3rd_party/pathie-cpp/src/entry_iterator.cpp b/src/3rd_party/pathie-cpp/src/entry_iterator.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/pathie-cpp/src/path.cpp b/src/3rd_party/pathie-cpp/src/path.cpp old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/astyle.sh b/src/3rd_party/spdlog/astyle.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/bench/latency/compare.sh b/src/3rd_party/spdlog/bench/latency/compare.sh old mode 100755 new mode 100644 diff --git a/src/3rd_party/spdlog/tests/install_libcxx.sh b/src/3rd_party/spdlog/tests/install_libcxx.sh old mode 100755 new mode 100644 diff --git a/src/command/marian_main.cpp b/src/command/marian_main.cpp old mode 100755 new mode 100644 diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100755 new mode 100644 diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100755 new mode 100644 diff --git a/src/common/logging.cpp b/src/common/logging.cpp old mode 100755 new mode 100644 diff --git a/src/common/options.h b/src/common/options.h old mode 100755 new mode 100644 diff --git a/src/common/shape.h b/src/common/shape.h old mode 100755 new mode 100644 diff --git a/src/common/timer.h b/src/common/timer.h old mode 100755 new mode 100644 diff --git a/src/common/types.h b/src/common/types.h old mode 100755 new mode 100644 diff --git a/src/common/utils.cpp b/src/common/utils.cpp old mode 100755 new mode 100644 diff --git a/src/common/utils.h b/src/common/utils.h old mode 100755 new mode 100644 diff --git a/src/data/batch_generator.h b/src/data/batch_generator.h old mode 100755 new mode 100644 diff --git a/src/data/corpus.cpp b/src/data/corpus.cpp old mode 100755 new mode 100644 diff --git a/src/data/corpus.h b/src/data/corpus.h old mode 100755 new mode 100644 diff --git a/src/data/corpus_base.h b/src/data/corpus_base.h old mode 100755 new mode 100644 diff --git a/src/data/default_vocab.cpp b/src/data/default_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/factored_vocab.h b/src/data/factored_vocab.h old mode 100755 new mode 100644 diff --git a/src/data/sentencepiece_vocab.cpp b/src/data/sentencepiece_vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/text_input.cpp b/src/data/text_input.cpp old mode 100755 new mode 100644 diff --git a/src/data/types.h b/src/data/types.h old mode 100755 new mode 100644 diff --git a/src/data/vocab.cpp b/src/data/vocab.cpp old mode 100755 new mode 100644 diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100755 new mode 100644 diff --git a/src/data/vocab_base.h b/src/data/vocab_base.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/dataset.h b/src/examples/mnist/dataset.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/download.sh b/src/examples/mnist/download.sh old mode 100755 new mode 100644 diff --git a/src/examples/mnist/mnist_ffnn.cpp b/src/examples/mnist/mnist_ffnn.cpp old mode 100755 new mode 100644 diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/training.h b/src/examples/mnist/training.h old mode 100755 new mode 100644 diff --git a/src/examples/mnist/validator.h b/src/examples/mnist/validator.h old mode 100755 new mode 100644 diff --git a/src/functional/tmp.h b/src/functional/tmp.h old mode 100755 new mode 100644 diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100755 new mode 100644 diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h old mode 100755 new mode 100644 diff --git a/src/graph/node_operators.h b/src/graph/node_operators.h old mode 100755 new mode 100644 diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h old mode 100755 new mode 100644 diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100755 new mode 100644 diff --git a/src/layers/factory.h b/src/layers/factory.h old mode 100755 new mode 100644 diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp old mode 100755 new mode 100644 diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100755 new mode 100644 diff --git a/src/layers/loss.cpp b/src/layers/loss.cpp old mode 100755 new mode 100644 diff --git a/src/layers/loss.h b/src/layers/loss.h old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp old mode 100755 new mode 100644 diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h old mode 100755 new mode 100644 diff --git a/src/models/bert.h b/src/models/bert.h old mode 100755 new mode 100644 diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h old mode 100755 new mode 100644 diff --git a/src/models/costs.h b/src/models/costs.h old mode 100755 new mode 100644 diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100755 new mode 100644 diff --git a/src/models/encoder_classifier.h b/src/models/encoder_classifier.h old mode 100755 new mode 100644 diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100755 new mode 100644 diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100755 new mode 100644 diff --git a/src/models/model_base.h b/src/models/model_base.h old mode 100755 new mode 100644 diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp old mode 100755 new mode 100644 diff --git a/src/models/model_factory.h b/src/models/model_factory.h old mode 100755 new mode 100644 diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100755 new mode 100644 diff --git a/src/models/states.h b/src/models/states.h old mode 100755 new mode 100644 diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100755 new mode 100644 diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100755 new mode 100644 diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100755 new mode 100644 diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100755 new mode 100644 diff --git a/src/rnn/attention_constructors.h b/src/rnn/attention_constructors.h old mode 100755 new mode 100644 diff --git a/src/rnn/cells.h b/src/rnn/cells.h old mode 100755 new mode 100644 diff --git a/src/rnn/types.h b/src/rnn/types.h old mode 100755 new mode 100644 diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.cu b/src/tensors/gpu/add.cu old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.h b/src/tensors/gpu/add.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/add.inc b/src/tensors/gpu/add.inc old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/element.cu b/src/tensors/gpu/element.cu old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/element.inc b/src/tensors/gpu/element.inc old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/prod.h b/src/tensors/gpu/prod.h old mode 100755 new mode 100644 diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu old mode 100755 new mode 100644 diff --git a/src/tests/prod.cpp b/src/tests/prod.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/attention_tests.cpp b/src/tests/units/attention_tests.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/operator_tests.cpp b/src/tests/units/operator_tests.cpp old mode 100755 new mode 100644 diff --git a/src/tests/units/rnn_tests.cpp b/src/tests/units/rnn_tests.cpp old mode 100755 new mode 100644 diff --git a/src/training/communicator.cpp b/src/training/communicator.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group.h b/src/training/graph_group.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.cpp b/src/training/graph_group_async.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_async.h b/src/training/graph_group_async.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode.cpp b/src/training/graph_group_multinode.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_multinode_sync.cpp b/src/training/graph_group_multinode_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.cpp b/src/training/graph_group_singleton.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_singleton.h b/src/training/graph_group_singleton.h old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100755 new mode 100644 diff --git a/src/training/graph_group_sync.h b/src/training/graph_group_sync.h old mode 100755 new mode 100644 diff --git a/src/training/scheduler.h b/src/training/scheduler.h old mode 100755 new mode 100644 diff --git a/src/training/validator.cpp b/src/training/validator.cpp old mode 100755 new mode 100644 diff --git a/src/training/validator.h b/src/training/validator.h old mode 100755 new mode 100644 diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100755 new mode 100644 diff --git a/src/translator/helpers.cpp b/src/translator/helpers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/helpers.cu b/src/translator/helpers.cu old mode 100755 new mode 100644 diff --git a/src/translator/helpers.h b/src/translator/helpers.h old mode 100755 new mode 100644 diff --git a/src/translator/history.h b/src/translator/history.h old mode 100755 new mode 100644 diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cpp b/src/translator/nth_element.cpp old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.cu b/src/translator/nth_element.cu old mode 100755 new mode 100644 diff --git a/src/translator/nth_element.h b/src/translator/nth_element.h old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.cpp b/src/translator/output_printer.cpp old mode 100755 new mode 100644 diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100755 new mode 100644 diff --git a/src/translator/scorers.cpp b/src/translator/scorers.cpp old mode 100755 new mode 100644 diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100755 new mode 100644 diff --git a/src/translator/translator.h b/src/translator/translator.h old mode 100755 new mode 100644 diff --git a/vs/Marian.sln b/vs/Marian.sln old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100755 new mode 100644 diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100755 new mode 100644 From 0d217f765f1b8166bdaaece2f018aaca3d7cb15b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 29 Apr 2019 19:42:48 -0700 Subject: [PATCH 452/838] disabled escaping --- src/common/utils.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 0e406958a..4139729c4 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -96,7 +96,10 @@ std::string join(const std::vector& words, const std::string& del / // escapes a string for passing to popen, which uses /bin/sh to parse its argument string static std::string escapeForPOpen(const std::string& arg) { // e.g. abc -> 'abc'; my file.txt -> 'my file.txt'; $10 -> '$10'; it's -> 'it'\''s' - return "'" + findReplace(arg, "'", "'\\''", /*all=*/ true) + "'"; + return arg; + // @BUGBUG: This sometimes fails with "sh: 1: Syntax error: Unterminated quoted string", + // so since this is not super-critical, we will disable it for now. + //return "'" + findReplace(arg, "'", "'\\''", /*all=*/ true) + "'"; } // execute an external command From 209c1d4a4e05f10830ce9b3f436ea2ba7a7fdab6 Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Tue, 30 Apr 2019 12:52:13 -0700 Subject: [PATCH 453/838] add back execution rights for python/perl scripts --- scripts/bert/bert4marian.py | 0 scripts/contrib/fix_hard.py | 0 scripts/contrib/inject_ctt.py | 0 scripts/contrib/inject_model_params.py | 0 scripts/contrib/model_info.py | 0 scripts/embeddings/export_embeddings.py | 0 scripts/embeddings/prepare_corpus.py | 0 scripts/embeddings/process_word2vec.py | 0 scripts/server/client_example.py | 0 scripts/shortlist/generate_shortlists.pl | 0 10 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/bert/bert4marian.py mode change 100644 => 100755 scripts/contrib/fix_hard.py mode change 100644 => 100755 scripts/contrib/inject_ctt.py mode change 100644 => 100755 scripts/contrib/inject_model_params.py mode change 100644 => 100755 scripts/contrib/model_info.py mode change 100644 => 100755 scripts/embeddings/export_embeddings.py mode change 100644 => 100755 scripts/embeddings/prepare_corpus.py mode change 100644 => 100755 scripts/embeddings/process_word2vec.py mode change 100644 => 100755 scripts/server/client_example.py mode change 100644 => 100755 scripts/shortlist/generate_shortlists.pl diff --git a/scripts/bert/bert4marian.py b/scripts/bert/bert4marian.py old mode 100644 new mode 100755 diff --git a/scripts/contrib/fix_hard.py b/scripts/contrib/fix_hard.py old mode 100644 new mode 100755 diff --git a/scripts/contrib/inject_ctt.py b/scripts/contrib/inject_ctt.py old mode 100644 new mode 100755 diff --git a/scripts/contrib/inject_model_params.py b/scripts/contrib/inject_model_params.py old mode 100644 new mode 100755 diff --git a/scripts/contrib/model_info.py b/scripts/contrib/model_info.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/prepare_corpus.py b/scripts/embeddings/prepare_corpus.py old mode 100644 new mode 100755 diff --git a/scripts/embeddings/process_word2vec.py b/scripts/embeddings/process_word2vec.py old mode 100644 new mode 100755 diff --git a/scripts/server/client_example.py b/scripts/server/client_example.py old mode 100644 new mode 100755 diff --git a/scripts/shortlist/generate_shortlists.pl b/scripts/shortlist/generate_shortlists.pl old mode 100644 new mode 100755 From 7f9e1511313d03a838b967e233420c95df91e8f6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 30 Apr 2019 18:42:54 -0700 Subject: [PATCH 454/838] addressed PR feedbacjk --- src/common/definitions.h | 2 ++ src/tensors/gpu/prod.cpp | 10 ---------- src/tensors/memory_piece.h | 1 + src/tensors/tensor.h | 11 +++++++++++ src/translator/output_printer.h | 2 -- 5 files changed, 14 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/common/definitions.h mode change 100644 => 100755 src/tensors/gpu/prod.cpp mode change 100644 => 100755 src/tensors/memory_piece.h mode change 100644 => 100755 src/tensors/tensor.h mode change 100644 => 100755 src/translator/output_printer.h diff --git a/src/common/definitions.h b/src/common/definitions.h old mode 100644 new mode 100755 index 3fdf6659f..f46da2340 --- a/src/common/definitions.h +++ b/src/common/definitions.h @@ -14,6 +14,8 @@ #define NodeOp(op) [=]() { op; } // helper macro to disable optimization (gcc only) +// To use this, just insert DONT_OPTIMIZE right before the function definition +// (e.g. where the "static" keyword would go). #ifdef __GNUC__ #define DONT_OPTIMIZE __attribute__((optimize("O0"))) #else diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp old mode 100644 new mode 100755 index 74abfa219..48524f626 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -227,16 +227,6 @@ void ProdBatched(marian::Tensor C, allocator->free(mp_cptr); } -// debugging helper --@TODO: move it to some shared place? -template -static std::vector get(Ptr memory, Ptr backend) { - size_t n = memory->size() / sizeof(T); - TensorBase t(memory, Shape({(int)n}), getType(), backend); - std::vector res; - t.get(res); - return res; -} - // bug in cuSparse: sparse matrix is limited to 65535 columns // This function is a drop-in replacement that handles it (by slicing). cusparseStatus_t diff --git a/src/tensors/memory_piece.h b/src/tensors/memory_piece.h old mode 100644 new mode 100755 index c34352633..27446a6c1 --- a/src/tensors/memory_piece.h +++ b/src/tensors/memory_piece.h @@ -40,4 +40,5 @@ class MemoryPiece { return out; } }; + } // namespace marian diff --git a/src/tensors/tensor.h b/src/tensors/tensor.h old mode 100644 new mode 100755 index 9ec69effa..e391bd75c --- a/src/tensors/tensor.h +++ b/src/tensors/tensor.h @@ -412,4 +412,15 @@ class TensorBase : public std::enable_shared_from_this { }; typedef std::shared_ptr Tensor; + +// debugging helper, to read out a memory piece into an STL vector (e.g. for inspection in debugger) +template +static inline std::vector get(Ptr memory, Ptr backend) { + size_t n = memory->size() / sizeof(T); + TensorBase t(memory, Shape({ (int)n }), getType(), backend); + std::vector res; + t.get(res); + return res; +} + } // namespace marian diff --git a/src/translator/output_printer.h b/src/translator/output_printer.h old mode 100644 new mode 100755 index e2fc97fee..85a7585f4 --- a/src/translator/output_printer.h +++ b/src/translator/output_printer.h @@ -72,8 +72,6 @@ class OutputPrinter { best1 << " ||| " << getAlignment(hypo); } - //best1 << " |sc=" << std::get<1>(result)->getPathScore(); - best1 << std::flush; } From b5e42c26d1d614eadb4be383756126e79195e908 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 1 May 2019 13:28:50 -0700 Subject: [PATCH 455/838] bug fix: the overly cautious ABORT_IF on flattenedLogitIndex should be a little less cautious --- src/translator/beam_search.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/translator/beam_search.h diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100644 new mode 100755 index 4db173000..8016e76f7 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -114,7 +114,8 @@ class BeamSearch { auto lval = states[j]->getLogProbs().getFactoredLogitsTensor(factorGroup); // [localBeamSize, 1, dimBatch, dimFactorVocab] size_t flattenedLogitIndex = (beamHypIdx * dimBatch + batchIdx) * vocabSize + wordIdx; // (beam idx, batch idx, word idx); note: beam and batch are transposed, compared to 'key' // @TODO: use a function on shape() to index, or new method val->at({i1, i2, i3, i4}) with broadcasting - ABORT_IF(lval->shape() != Shape({(int)nBestBeamSize, 1, (int)dimBatch, (int)vocabSize}), + ABORT_IF(lval->shape() != Shape({(int)nBestBeamSize, 1, (int)dimBatch, (int)vocabSize}) && + (beamHypIdx == 0 && lval->shape() != Shape({1, 1, (int)dimBatch, (int)vocabSize})), "Unexpected shape of logits?? {} != {}", lval->shape(), Shape({(int)nBestBeamSize, 1, (int)dimBatch, (int)vocabSize})); breakDown[j] += lval->get(flattenedLogitIndex); } From 8ba7d810dd11ebc7d95b5cb2c04064e6a6c89f47 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 1 May 2019 13:29:15 -0700 Subject: [PATCH 456/838] (fixed a mode change) --- src/translator/beam_search.h | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/translator/beam_search.h diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h old mode 100755 new mode 100644 From 7bd1311d87988fc5d46bccc243574da01e9ee16f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 1 May 2019 14:11:03 -0700 Subject: [PATCH 457/838] addressed PR feedback --- src/common/config_parser.cpp | 1 + src/common/utils.cpp | 1 + src/data/vocab.h | 2 ++ src/graph/expression_operators.cpp | 2 +- src/models/costs.h | 1 + src/models/decoder.h | 14 +++++++------- src/models/s2s.h | 13 ++++++------- src/models/transformer.h | 13 +++++++------ src/training/graph_group_sync.cpp | 6 +++--- src/training/validator.h | 6 ++++-- 10 files changed, 33 insertions(+), 26 deletions(-) mode change 100644 => 100755 src/common/config_parser.cpp mode change 100644 => 100755 src/common/utils.cpp mode change 100644 => 100755 src/data/vocab.h mode change 100644 => 100755 src/graph/expression_operators.cpp mode change 100644 => 100755 src/models/costs.h mode change 100644 => 100755 src/models/decoder.h mode change 100644 => 100755 src/models/s2s.h mode change 100644 => 100755 src/models/transformer.h mode change 100644 => 100755 src/training/graph_group_sync.cpp mode change 100644 => 100755 src/training/validator.h diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp old mode 100644 new mode 100755 index 46402e38e..eaa9c9912 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -637,6 +637,7 @@ void ConfigParser::addSuboptionsBatching(cli::CLIWrapper& cli) { cli.add("--shuffle-in-ram", "Keep shuffled corpus in RAM, do not write to temp file"); + // @TODO: Consider making the next two options options of the vocab instead, to make it more local in scope. cli.add("--all-caps-every", "When forming minibatches, preprocess every Nth line on the fly to all-caps. Assumes UTF-8"); cli.add("--english-title-case-every", diff --git a/src/common/utils.cpp b/src/common/utils.cpp old mode 100644 new mode 100755 index 4139729c4..de87b8a6a --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -165,6 +165,7 @@ bool endsWith(const std::string& text, const std::string& suffix) { && !text.compare(text.size() - suffix.size(), suffix.size(), suffix); } +// @TODO: sort these functions into a separate header. std::u32string utf8ToUnicodeString(std::string const& s) { #ifdef _MSC_VER // workaround for a known bug in VS CRT std::wstring_convert, unsigned int/*char32_t*/> converter; diff --git a/src/data/vocab.h b/src/data/vocab.h old mode 100644 new mode 100755 index a15e202f0..9a40ba16f --- a/src/data/vocab.h +++ b/src/data/vocab.h @@ -71,6 +71,8 @@ class Vocab { Word getUnkId() const; // for corpus augmentation: convert string to all-caps + // @TODO: Consider a different implementation where this does not show on the vocab interface, + // but instead as additional options passed to vocab instantiation. std::string toUpper(const std::string& line) const; // for corpus augmentation: convert string to title case diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp old mode 100644 new mode 100755 index 396370bd1..396fa03e2 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -79,7 +79,7 @@ Expr softmax(Expr a, Expr zeroOneMask, int axis /*=-1*/) { Expr logsoftmax(Expr a) { if (a->type() == "logsoftmax") // logsoftmax(logsoftmax(x)) == logsoftmax(x) - // @TODO: Is this correct? logsoftmax is idempotent in forward(), but not in backward() + // @TODO: Remove this. First add an ABORT() for a while to catch who relies on this, and if noone, then delete. return a; return Expression(a); } diff --git a/src/models/costs.h b/src/models/costs.h old mode 100644 new mode 100755 index f56f6e05f..7aa5a5f3e --- a/src/models/costs.h +++ b/src/models/costs.h @@ -214,6 +214,7 @@ class LogSoftmaxStep : public ILogProbStep { virtual Ptr apply(Ptr state) override { // decoder needs normalized probabilities (note: skipped if beam 1 and --skip-cost) state->setLogProbs(state->getLogProbs().applyUnaryFunction(logsoftmax)); + // @TODO: This is becoming more and more opaque ^^. Can we simplify this? return state; } }; diff --git a/src/models/decoder.h b/src/models/decoder.h old mode 100644 new mode 100755 index 31baca9cb..dd5a508a2 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -15,7 +15,7 @@ class DecoderBase { std::string prefix_{"decoder"}; bool inference_{false}; size_t batchIndex_{1}; - std::vector> embedding_; // @TODO: find a more grammatical name + std::vector> embeddingLayers_; // (lazily created) Ptr shortlist_; @@ -37,9 +37,9 @@ class DecoderBase { void lazyCreateEmbeddingLayer(Ptr graph) { // @TODO: code dup with EncoderTransformer - if (embedding_.size() <= batchIndex_ || !embedding_[batchIndex_]) { // lazy - if (embedding_.size() <= batchIndex_) - embedding_.resize(batchIndex_ + 1); + if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy + if (embeddingLayers_.size() <= batchIndex_) + embeddingLayers_.resize(batchIndex_ + 1); int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); @@ -55,7 +55,7 @@ class DecoderBase { ("normalization", opt("embedding-normalization")); } embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - embedding_[batchIndex_] = embFactory.construct(graph); + embeddingLayers_[batchIndex_] = embFactory.construct(graph); } } @@ -66,7 +66,7 @@ class DecoderBase { lazyCreateEmbeddingLayer(graph); Expr y, yMask; std::tie - (y, yMask) = embedding_[batchIndex_]->apply(subBatch); + (y, yMask) = embeddingLayers_[batchIndex_]->apply(subBatch); const Words& data = /*if*/ (shortlist_) ? @@ -93,7 +93,7 @@ class DecoderBase { if(words.empty()) { selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); } else { - selectedEmbs = embedding_[batchIndex_]->apply(words, {dimBeam, 1, dimBatch, dimEmb}); + selectedEmbs = embeddingLayers_[batchIndex_]->apply(words, {dimBeam, 1, dimBatch, dimEmb}); } state->setTargetHistoryEmbeddings(selectedEmbs); } diff --git a/src/models/s2s.h b/src/models/s2s.h old mode 100644 new mode 100755 index 67b0817a4..4829e93de --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -9,6 +9,7 @@ namespace marian { class EncoderS2S : public EncoderBase { + std::vector> embeddingLayers_; // (lazily created) public: Expr applyEncoderRNN(Ptr graph, Expr embeddings, @@ -150,15 +151,14 @@ class EncoderS2S : public EncoderBase { EncoderS2S(Ptr options) : EncoderBase(options) {} - std::vector> embedding_; // @TODO: move away, also rename virtual Ptr build(Ptr graph, Ptr batch) override { // lazily create embedding layer - if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy - embedding_.resize(batch->sets()); - embedding_[batchIndex_] = createSourceEmbeddingLayer(graph); + if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy + embeddingLayers_.resize(batch->sets()); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); } - auto embedding = embedding_[batchIndex_]; + auto embedding = embeddingLayers_[batchIndex_]; // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie @@ -246,8 +246,7 @@ class DecoderS2S : public DecoderBase { } public: - DecoderS2S(Ptr options) : DecoderBase(options) { - } + DecoderS2S(Ptr options) : DecoderBase(options) {} virtual Ptr startState( Ptr graph, diff --git a/src/models/transformer.h b/src/models/transformer.h old mode 100644 new mode 100755 index 588a79b41..f36150edb --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -502,6 +502,7 @@ class Transformer : public EncoderOrDecoderBase { }; class EncoderTransformer : public Transformer { + std::vector> embeddingLayers_; // (lazily created) public: EncoderTransformer(Ptr options) : Transformer(options) {} virtual ~EncoderTransformer() {} @@ -549,7 +550,6 @@ class EncoderTransformer : public Transformer { return apply(batch); } - std::vector> embedding_; // @TODO: move away, also rename Ptr apply(Ptr batch) { int dimBatch = (int)batch->size(); int dimSrcWords = (int)(*batch)[batchIndex_]->batchWidth(); @@ -557,14 +557,14 @@ class EncoderTransformer : public Transformer { // embed the source words in the batch Expr batchEmbeddings, batchMask; - if (embedding_.empty() || !embedding_[batchIndex_]) { // lazy - embedding_.resize(batch->sets()); + if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy + embeddingLayers_.resize(batch->sets()); if (options_->has("ulr") && options_->get("ulr") == true) - embedding_[batchIndex_] = createULREmbeddingLayer(); // embedding uses ULR + embeddingLayers_[batchIndex_] = createULREmbeddingLayer(); // embedding uses ULR else - embedding_[batchIndex_] = createSourceEmbeddingLayer(batchIndex_); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(batchIndex_); } - std::tie(batchEmbeddings, batchMask) = embedding_[batchIndex_]->apply((*batch)[batchIndex_]); + std::tie(batchEmbeddings, batchMask) = embeddingLayers_[batchIndex_]->apply((*batch)[batchIndex_]); // apply dropout over source words float dropoutSrc = inference_ ? 0 : opt("dropout-src"); if(dropoutSrc) { @@ -636,6 +636,7 @@ class DecoderTransformer : public Transformer { Ptr output_; private: + // @TODO: move this out for sharing with other models void lazyCreateOutputLayer() { if(output_) // create it lazily diff --git a/src/training/graph_group_sync.cpp b/src/training/graph_group_sync.cpp old mode 100644 new mode 100755 index 86c503fb6..6c6da17bb --- a/src/training/graph_group_sync.cpp +++ b/src/training/graph_group_sync.cpp @@ -23,11 +23,11 @@ SyncGraphGroup::SyncGraphGroup(Ptr config, Ptr mpi) // Rather, it is assumed that the communicator knows to reduce unnecessary transfers to no-ops. comm_ = createCommunicator(graphs_, /*noNccl=*/options_->get("no-nccl", false), /*mpi=*/mpi_); - auto type = utils::utf8ToUpper(devices_.front().typeAsString()) + "s"; + auto formattedDeviceType = utils::utf8ToUpper(devices_.front().typeAsString()) + "s"; if (mpi_->numMPIProcesses() > 1) - LOG(info, "[training] Using {} {}, distributed over {} MPI processes", mpi_->numMPIProcesses() * devices_.size(), type, mpi_->numMPIProcesses()); + LOG(info, "[training] Using {} {}, distributed over {} MPI processes", mpi_->numMPIProcesses() * devices_.size(), formattedDeviceType, mpi_->numMPIProcesses()); else - LOG(info, "[training] Using {} {}", devices_.size(), type); + LOG(info, "[training] Using {} {}", devices_.size(), formattedDeviceType); } void SyncGraphGroup::setScheduler(Ptr scheduler) /*override*/ { diff --git a/src/training/validator.h b/src/training/validator.h old mode 100644 new mode 100755 index 968fcf4ff..7736318b1 --- a/src/training/validator.h +++ b/src/training/validator.h @@ -742,7 +742,7 @@ class BleuValidator : public Validator { return normText; } -public: + static std::string tokenizeContinuousScript(const std::string& sUTF8) { // We want BLEU-like scores that are comparable across different tokenization schemes. // For continuous scripts (Chinese, Japanese, Thai), we would need a language-specific @@ -751,6 +751,7 @@ class BleuValidator : public Validator { // leaving Western scripts as words. This way we can use the same settings for Western // languages, where Marian would report SacreBLEU scores, and Asian languages, where // scores are not standard but internally comparable across tokenization schemes. + // @TODO: Check what sacrebleu.py is doing, and whether we can replicate that here faithfully. auto in = utils::utf8ToUnicodeString(sUTF8); auto out = in.substr(0, 0); // (out should be same type as in, don't want to bother with exact type) for (auto c : in) { @@ -842,7 +843,8 @@ class BleuValidator : public Validator { // parameter to select the detokenization method, which will default to detok for FactoredSegmenter, // and no-op for base vocab. if (vocabs_.back()->type() == "FactoredVocab") { - LOG_ONCE(info, "[valid] FactoredVocab implies using detokenized BLEU"); + if (!quiet_) + LOG_ONCE(info, "[valid] FactoredVocab implies using detokenized BLEU"); detok = true; // always use bleu-detok } #endif From cf379e9422b4ec187539b3fbd184c000b32ced19 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 9 May 2019 18:51:02 -0700 Subject: [PATCH 458/838] implemented shortlists for lemma-dim-emb --- src/layers/generic.cpp | 26 ++++++++++++++++++-------- src/layers/generic.h | 9 ++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index d13d5871a..db23c7b56 100644 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -251,6 +251,13 @@ namespace marian { } b_ = graph_->param(name + "_b", {1, numOutputClasses}, inits::zeros); + + const int lemmaDimEmb = options_->get("lemma-dim-emb", 0); + if (lemmaDimEmb > 0) { + auto range = factoredVocab_->getGroupRange(0); + auto lemmaVocabDim = (int)(range.second - range.first); + lemmaEt_ = graph_->param(name + "_lemmaEt", {lemmaDimEmb, lemmaVocabDim}, inits::glorot_uniform); // [L x U] L=lemmaDimEmb; transposed for speed + } } Logits Output::applyAsLogits(Expr input) /*override final*/ { @@ -299,8 +306,8 @@ namespace marian { // optionally add a soft embedding of lemma back to create some lemma dependency // @TODO: if this works, move it into lazyConstruct const int lemmaDimEmb = options_->get("lemma-dim-emb", 0); - ABORT_IF(shortlist_ && lemmaDimEmb != 0, "Lemma re-embedding with short list is not yet implemented"); if (lemmaDimEmb < 0 && g == 0) { + ABORT_IF(shortlist_ && lemmaDimEmb != 0, "Lemma-dependent bias with short list is not yet implemented"); LOG_ONCE(info, "[embedding] using lemma-dependent bias"); factorLogits = logsoftmax(factorLogits); // explicitly, since we do that again later auto z = /*stopGradient*/(factorLogits); @@ -308,16 +315,19 @@ namespace marian { } if (lemmaDimEmb > 0 && g == 0) { LOG_ONCE(info, "[embedding] enabled re-embedding of lemma, at dim {}", lemmaDimEmb); - int lemmaVocabDim = factorLogits->shape()[-1]; - int inputDim = input1->shape()[-1]; + // compute softmax. We compute logsoftmax() separately because this way, computation will be reused later via CSE + factorLogits = logsoftmax(factorLogits); + // re-embedding lookup, soft-indexed by softmax + if (shortlist_ && !cachedShortLemmaEt_) // short-listed version of re-embedding matrix + cachedShortLemmaEt_ = index_select(lemmaEt_, -1, shortlist_->indices()); + auto e = dot(exp(factorLogits), cachedShortLemmaEt_ ? cachedShortLemmaEt_ : lemmaEt_, false, true); // [B... x L] + // project it back to regular hidden dim + int inputDim = input1->shape()[-1]; auto name = options_->get("prefix"); - factorLogits = logsoftmax(factorLogits); // explicitly, since we do that again later - Expr lemmaEt = graph_->param(name + "_lemmaEt", { lemmaDimEmb, lemmaVocabDim }, inits::glorot_uniform); // [L x U] L=lemmaDimEmb; transposed for speed - auto e = dot(exp(factorLogits), lemmaEt, false, true); // [B... x L] - //e = tanh(e); // make it non-scale-preserving Expr lemmaWt = inputDim == lemmaDimEmb ? nullptr : graph_->param(name + "_lemmaWt", { inputDim, lemmaDimEmb }, inits::glorot_uniform); // [D x L] D=hidden-vector dimension auto f = lemmaWt ? dot(e, lemmaWt, false, true) : e; // [B... x D] - input1 = input1 + f; // augment the original hidden vector with this additional information + // augment the original hidden vector with this additional information + input1 = input1 + f; } } return Logits(std::move(allLogits), factoredVocab_); diff --git a/src/layers/generic.h b/src/layers/generic.h index 9d787a3c3..309b81c51 100644 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -204,10 +204,12 @@ class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList { // parameters held by this layer Expr Wt_; // weight matrix is stored transposed for efficiency Expr b_; + Expr lemmaEt_; // re-embedding matrix for lemmas [lemmaDimEmb x lemmaVocabSize] bool isLegacyUntransposedW{false}; // legacy-model emulation: W is stored in non-transposed form Expr cachedShortWt_; // short-listed version, cached (cleared by clear()) Expr cachedShortb_; // these match the current value of shortlist_ - Ptr factoredVocab_; + Expr cachedShortLemmaEt_; + Ptr factoredVocab_; // optional parameters set/updated after construction Expr tiedParam_; @@ -231,18 +233,19 @@ class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList { if (shortlist_) ABORT_IF(shortlist.get() != shortlist_.get(), "Output shortlist cannot be changed except after clear()"); else { - ABORT_IF(cachedShortWt_ || cachedShortb_, "No shortlist but cached parameters??"); + ABORT_IF(cachedShortWt_ || cachedShortb_ || cachedShortLemmaEt_, "No shortlist but cached parameters??"); shortlist_ = shortlist; } // cachedShortWt_ and cachedShortb_ will be created lazily inside apply() } // this is expected to be called in sync with graph->clear(), which invalidates - // cachedShortWt_ and cachedShortb_ in the graph's short-term cache + // cachedShortWt_ etc. in the graph's short-term cache void clear() override final { shortlist_ = nullptr; cachedShortWt_ = nullptr; cachedShortb_ = nullptr; + cachedShortLemmaEt_ = nullptr; } Logits applyAsLogits(Expr input) override final; From b9c06637368b38c6bc45f5cebe16b6199b4f2b04 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 13 May 2019 15:39:32 -0700 Subject: [PATCH 459/838] added comments regarding SacreBLEU char BLEU --- src/common/utils.cpp | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/common/utils.cpp b/src/common/utils.cpp index de87b8a6a..80aee431e 100755 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -217,15 +217,42 @@ bool isContinuousScript(char32_t c) { auto in = [c](char32_t minVal, char32_t maxVal) { return c >= minVal && c <= maxVal; }; bool isHan = in(0x2E80, 0x2E99) || in(0x2E9B, 0x2EF3) || in(0x2F00, 0x2FD5) || in(0x3005, 0x3005) || in(0x3007, 0x3007) || in(0x3021, 0x3029) || in(0x3038, 0x303A) || in(0x303B, 0x303b) || - in(0x3400, 0x4DB5) || in(0x4E00, 0x9FEF) || in(0xF900, 0xFA6D) || in(0xFA70, 0xFAD9) || - in(0x20000, 0x2A6D6) || in(0x2A700, 0x2B734) || in(0x2B740, 0x2B81D) || in(0x2B820, 0x2CEA1) || - in(0x2CEB0, 0x2EBE0) || in(0x2F800, 0x2FA1D) || - in(0x3200, 0x32FF); // Enclosed CJK Letters and Months, https://en.wikipedia.org/wiki/Enclosed_CJK_Letters_and_Months + in(0x3200, 0x32FF) || // Enclosed CJK Letters and Months, https://en.wikipedia.org/wiki/Enclosed_CJK_Letters_and_Months + in(0x3400, 0x4DB5) || // up to here, we have a few gaps compared to sacrebleu + in(0x4E00, 0x9FEF) || // sacrebleu: only up to 0x9fbb + in(0xF900, 0xFA6D) || in(0xFA70, 0xFAD9) || // similar to sacrebleu + in(0x20000, 0x2A6D6) || + in(0x2A700, 0x2B734) || in(0x2B740, 0x2B81D) || in(0x2B820, 0x2CEA1) || in(0x2CEB0, 0x2EBE0) || // not in sacrebleu + in(0x2F800, 0x2FA1D); bool isKana = in(0x3040, 0x30FF) || // Hiragana, Katakana in(0x1B000, 0x1B0FF) || // Kana supplement, https://en.wikipedia.org/wiki/Kana_Supplement in(0x1B130, 0x1B16F); // small Kana, https://en.wikipedia.org/wiki/Small_Kana_Extension bool isThai = in(0x0E00, 0x0E7F); // https://en.wikipedia.org/wiki/Thai_(Unicode_block) return isHan || isKana || isThai; + // Korean characters (Hangul syllables): 0xac00..0xd7a3 + // Korean subwords (Hangul Jamo): 0x1100..0x11ff [https://en.wikipedia.org/wiki/Hangul_Jamo_(Unicode_block)] + // Sacrebleu uses characters units for Chinese characters; specifically, these ranges: + /* (ranges as used in sacrebleuy.py) + uchar >= u'\u2600' and uchar <= u'\u27bf' ## missing above + uchar >= u'\u2e80' # CJK Radicals Supplement + uchar >= u'\u2f00' and uchar <= u'\u2fdf' # Kangxi Radicals + uchar >= u'\u2ff0' # Chinese character structure + uchar >= u'\u3000' and uchar <= u'\u303f' # CJK punctuation mark ## 3040..30ff = Kana + uchar >= u'\u3100' and uchar <= u'\u312f' # Phonetic symbols + uchar >= u'\u31a0' # Phonetic symbols (Taiwanese and Hakka expansion) + uchar >= u'\u31c0' and uchar <= u'\u31ef' # CJK stroke + uchar >= u'\u3200' and uchar <= u'\u4db5' # CJK Unified Ideographs Extension A, release 3.0 + uchar >= u'\u4e00' # CJK Unified Ideographs, release 1.1 + uchar >= u'\u9fa6' and uchar <= u'\u9fbb' # CJK Unified Ideographs, release 4.1 + uchar >= u'\uf900' and uchar <= u'\ufa2d' # CJK Compatibility Ideographs, release 1.1 + uchar >= u'\ufa30' and uchar <= u'\ufa6a' # CJK Compatibility Ideographs, release 3.2 + uchar >= u'\ufa70' and uchar <= u'\ufad9' # CJK Compatibility Ideographs, release 4.1 + uchar >= u'\ufe10' and uchar <= u'\ufe1f' ## missing above + uchar >= u'\ufe30' and uchar <= u'\ufe4f' ## missing above + uchar >= u'\uff00' and uchar <= u'\uffef' # Full width ASCII, full width of English punctuation, half width Katakana, half wide half width kana, Korean alphabet + uchar >= u'\u20000' and uchar <= u'\u2a6d6' # CJK Unified Ideographs Extension B, release 3.1 + uchar >= u'\u2f800' and uchar <= u'\u2fa1d' # CJK Compatibility Supplement, release 3.1 + */ } // convert UTF-8 characters to lower or upper case From aced50d97a7c28f694f3ef04493daaca4e27423e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 13:52:47 -0700 Subject: [PATCH 460/838] some minor refactoring --- src/examples/mnist/model.h | 2 +- src/graph/expression_graph.cpp | 2 +- src/graph/expression_graph.h | 2 +- src/graph/expression_operators.h | 2 +- src/layers/factory.h | 2 + src/layers/generic.cpp | 22 +++------ src/models/char_s2s.h | 1 - src/models/classifier.h | 80 ++++++++++++++++---------------- src/models/costs.h | 6 +-- src/models/decoder.h | 18 +++++-- src/models/encoder.h | 9 ++-- src/models/encoder_decoder.h | 5 +- src/models/s2s.h | 10 +--- src/models/transformer.h | 8 ---- src/rnn/attention.h | 4 +- src/rnn/cells.h | 26 +++++------ 16 files changed, 94 insertions(+), 105 deletions(-) mode change 100644 => 100755 src/examples/mnist/model.h mode change 100644 => 100755 src/graph/expression_graph.cpp mode change 100644 => 100755 src/graph/expression_graph.h mode change 100644 => 100755 src/graph/expression_operators.h mode change 100644 => 100755 src/layers/factory.h mode change 100644 => 100755 src/layers/generic.cpp mode change 100644 => 100755 src/models/char_s2s.h mode change 100644 => 100755 src/models/classifier.h mode change 100644 => 100755 src/models/encoder.h mode change 100644 => 100755 src/models/encoder_decoder.h mode change 100644 => 100755 src/rnn/attention.h mode change 100644 => 100755 src/rnn/cells.h diff --git a/src/examples/mnist/model.h b/src/examples/mnist/model.h old mode 100644 new mode 100755 index 8e18aea1a..d03800f7b --- a/src/examples/mnist/model.h +++ b/src/examples/mnist/model.h @@ -93,7 +93,7 @@ class MnistFeedForwardNet : public IModel { protected: Ptr options_; - bool inference_{false}; + const bool inference_{false}; /** * @brief Builds an expression graph representing a feed-forward classifier. diff --git a/src/graph/expression_graph.cpp b/src/graph/expression_graph.cpp old mode 100644 new mode 100755 index b3c237d1e..1df3af243 --- a/src/graph/expression_graph.cpp +++ b/src/graph/expression_graph.cpp @@ -20,7 +20,7 @@ void ExpressionGraph::setDevice(DeviceId deviceId, Ptr device) { } } -Expr ExpressionGraph::dropout(float prob, const Shape& shape) { +Expr ExpressionGraph::dropoutMask(float prob, const Shape& shape) { return constant(shape, inits::dropout(prob)); } diff --git a/src/graph/expression_graph.h b/src/graph/expression_graph.h old mode 100644 new mode 100755 index dfd00a9a5..901d97245 --- a/src/graph/expression_graph.h +++ b/src/graph/expression_graph.h @@ -391,7 +391,7 @@ class ExpressionGraph : public std::enable_shared_from_this { } // prob = dropProb, e.g. 0.1 means 90% of values are kept - Expr dropout(float dropProb, const Shape& shape); + Expr dropoutMask(float dropProb, const Shape& shape); Expr get(std::string name) { if(!namespace_.empty()) diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h old mode 100644 new mode 100755 index 0205fe864..f8c7fd3c0 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -214,7 +214,7 @@ static inline Expr dropout(Expr x, float dropProb, Shape shape) { if(dropProb == 0) return x; auto graph = x->graph(); - auto mask = graph->dropout(dropProb, shape); + auto mask = graph->dropoutMask(dropProb, shape); return dropout(x, mask); } diff --git a/src/layers/factory.h b/src/layers/factory.h old mode 100644 new mode 100755 index 0aaf86a92..d3e1e3fa2 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -70,10 +70,12 @@ class Accumulator : public BaseFactory { return *this; } +#if 0 Accumulator& operator()(Config::YamlNode yaml) { Factory::getOptions()->merge(yaml); return *this; } +#endif Accumulator& operator()(Ptr options) { Factory::getOptions()->merge(options); diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp old mode 100644 new mode 100755 index d13d5871a..ef3c21933 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -364,12 +364,13 @@ namespace marian { // multi-hot factor vectors are represented as a sparse CSR matrix // [row index = word position index] -> set of factor indices for word at this position ABORT_IF(factoredData.shape != Shape({(int)factoredData.offsets.size()-1/*=rows of CSR*/, E_->shape()[0]}), "shape mismatch??"); - return csr_dot( // the CSR matrix is passed in pieces - factoredData.shape, - graph->constant({(int)factoredData.weights.size()}, inits::from_vector(factoredData.weights), Type::float32), - graph->constant({(int)factoredData.indices.size()}, inits::from_vector(factoredData.indices), Type::uint32), - graph->constant({(int)factoredData.offsets.size()}, inits::from_vector(factoredData.offsets), Type::uint32), - E_); + // the CSR matrix is passed in pieces + auto weights = graph->constant({ (int)factoredData.weights.size() }, inits::from_vector(factoredData.weights), Type::float32); + auto indices = graph->constant({ (int)factoredData.indices.size() }, inits::from_vector(factoredData.indices), Type::uint32); + auto offsets = graph->constant({ (int)factoredData.offsets.size() }, inits::from_vector(factoredData.offsets), Type::uint32); + // apply dropout --@TODO + // perform the product + return csr_dot(factoredData.shape, weights, indices, offsets, E_); } std::tuple Embedding::apply(Ptr subBatch) const /*override final*/ { @@ -406,16 +407,7 @@ namespace marian { // - but forward pass weighs them down, so that all factors are in a similar numeric range // - if it is required to be in a different range, the embeddings can still learn that, but more slowly -#if 1 auto batchEmbeddings = apply(subBatch->data(), {dimWords, dimBatch, dimEmb}); -#else - Expr selectedEmbs; - if (factoredVocab_) - selectedEmbs = multiRows(subBatch->data()); - else - selectedEmbs = rows(E_, subBatch->data()); - auto batchEmbeddings = reshape(selectedEmbs, { dimWords, dimBatch, dimEmb }); -#endif auto batchMask = graph->constant({dimWords, dimBatch, 1}, inits::from_vector(subBatch->mask())); return std::make_tuple(batchEmbeddings, batchMask); diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h old mode 100644 new mode 100755 index 928c6a18d..f14bf8807 --- a/src/models/char_s2s.h +++ b/src/models/char_s2s.h @@ -18,7 +18,6 @@ class CharS2SEncoder : public EncoderS2S { // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie (batchEmbeddings, batchMask) = embedding->apply(batch->front()); - // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); if(dropProb) { diff --git a/src/models/classifier.h b/src/models/classifier.h old mode 100644 new mode 100755 index dae9bcbb2..773299b4b --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -1,41 +1,41 @@ -#pragma once - -#include "marian.h" -#include "models/states.h" -#include "layers/constructors.h" -#include "layers/factory.h" - -namespace marian { - -/** - * Simple base class for Classifiers to be used in EncoderClassifier framework - * Currently only implementations are in bert.h - */ -class ClassifierBase { -protected: - Ptr options_; - std::string prefix_{"classifier"}; - bool inference_{false}; - size_t batchIndex_{0}; - -public: - ClassifierBase(Ptr options) - : options_(options), - prefix_(options->get("prefix", "classifier")), - inference_(options->get("inference", false)), - batchIndex_(options->get("index", 1)) {} // assume that training input has batch index 0 and labels has 1 - - virtual ~ClassifierBase() {} - - virtual Ptr apply(Ptr, Ptr, const std::vector>&) = 0; - - template - T opt(const std::string& key) const { - return options_->get(key); - } - - // Should be used to clear any batch-wise temporary objects if present - virtual void clear() = 0; -}; - +#pragma once + +#include "marian.h" +#include "models/states.h" +#include "layers/constructors.h" +#include "layers/factory.h" + +namespace marian { + +/** + * Simple base class for Classifiers to be used in EncoderClassifier framework + * Currently only implementations are in bert.h + */ +class ClassifierBase { +protected: + Ptr options_; + const std::string prefix_{"classifier"}; + const bool inference_{false}; + const size_t batchIndex_{0}; + +public: + ClassifierBase(Ptr options) + : options_(options), + prefix_(options->get("prefix", "classifier")), + inference_(options->get("inference", false)), + batchIndex_(options->get("index", 1)) {} // assume that training input has batch index 0 and labels has 1 + + virtual ~ClassifierBase() {} + + virtual Ptr apply(Ptr, Ptr, const std::vector>&) = 0; + + template + T opt(const std::string& key) const { + return options_->get(key); + } + + // Should be used to clear any batch-wise temporary objects if present + virtual void clear() = 0; +}; + } \ No newline at end of file diff --git a/src/models/costs.h b/src/models/costs.h index 7aa5a5f3e..c47092264 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -31,8 +31,8 @@ class EncoderDecoderCECost : public ICost { protected: Ptr options_; - bool inference_{false}; - bool toBeWeighted_{false}; + const bool inference_{false}; + /*const*/ bool toBeWeighted_{false}; // @TODO: single loss seems wrong Ptr loss_; @@ -92,7 +92,7 @@ class EncoderDecoderCECost : public ICost { class EncoderClassifierCECost : public ICost { protected: Ptr options_; - bool inference_{false}; + const bool inference_{false}; // @TODO: single loss seems wrong, especially since we support multiple objectives here, // also not sure this needs to be a member at all. diff --git a/src/models/decoder.h b/src/models/decoder.h index dd5a508a2..f149c4b5b 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -12,9 +12,9 @@ namespace marian { class DecoderBase { protected: Ptr options_; - std::string prefix_{"decoder"}; - bool inference_{false}; - size_t batchIndex_{1}; + const std::string prefix_{"decoder"}; + const bool inference_{false}; + const size_t batchIndex_{1}; std::vector> embeddingLayers_; // (lazily created) Ptr shortlist_; @@ -67,6 +67,12 @@ class DecoderBase { lazyCreateEmbeddingLayer(graph); Expr y, yMask; std::tie (y, yMask) = embeddingLayers_[batchIndex_]->apply(subBatch); + // dropout target words + float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); + if (dropoutTrg) { + int trgWords = y->shape()[-3]; + y = dropout(y, dropoutTrg, {trgWords, 1, 1}); + } const Words& data = /*if*/ (shortlist_) ? @@ -94,6 +100,12 @@ class DecoderBase { selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); } else { selectedEmbs = embeddingLayers_[batchIndex_]->apply(words, {dimBeam, 1, dimBatch, dimEmb}); + // dropout target words --does not make sense here since this is always inference. Keep it regular though. + float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); + if (dropoutTrg) { + int trgWords = selectedEmbs->shape()[-3]; + selectedEmbs = dropout(selectedEmbs, dropoutTrg, { trgWords, 1, 1 }); + } } state->setTargetHistoryEmbeddings(selectedEmbs); } diff --git a/src/models/encoder.h b/src/models/encoder.h old mode 100644 new mode 100755 index 7377ef9b9..a1f6f5311 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -8,10 +8,11 @@ namespace marian { class EncoderBase { protected: Ptr options_; - std::string prefix_{"encoder"}; - bool inference_{false}; - size_t batchIndex_{0}; -public: + const std::string prefix_{"encoder"}; + const bool inference_{false}; + const size_t batchIndex_{0}; + std::vector> embeddingLayers_; // (lazily created) +public: EncoderBase(Ptr options) : options_(options), prefix_(options->get("prefix", "encoder")), diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h old mode 100644 new mode 100755 index 41702eceb..f09e4f2c3 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -63,13 +63,12 @@ class EncoderDecoder : public EncoderDecoderBase { Ptr options_; Ptr shortlistGenerator_; - std::string prefix_; + const std::string prefix_; + const bool inference_{ false }; std::vector> encoders_; std::vector> decoders_; - bool inference_{false}; - std::set modelFeatures_; Config::YamlNode getModelParameters(); diff --git a/src/models/s2s.h b/src/models/s2s.h index 4829e93de..f7fa91dd2 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -9,7 +9,6 @@ namespace marian { class EncoderS2S : public EncoderBase { - std::vector> embeddingLayers_; // (lazily created) public: Expr applyEncoderRNN(Ptr graph, Expr embeddings, @@ -163,7 +162,6 @@ class EncoderS2S : public EncoderBase { // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie (batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); - // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); if(dropProb) { @@ -188,6 +186,7 @@ class DecoderS2S : public DecoderBase { Ptr constructDecoderRNN(Ptr graph, Ptr state) { float dropoutRnn = inference_ ? 0 : opt("dropout-rnn"); + auto rnn = rnn::rnn() // ("type", opt("dec-cell")) // ("dimInput", opt("dim-emb")) // @@ -293,13 +292,6 @@ class DecoderS2S : public DecoderBase { auto embeddings = state->getTargetHistoryEmbeddings(); - // dropout target words - float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); - if(dropoutTrg) { - int trgWords = embeddings->shape()[-3]; - embeddings = dropout(embeddings, dropoutTrg, {trgWords, 1, 1}); - } - if(!rnn_) rnn_ = constructDecoderRNN(graph, state); diff --git a/src/models/transformer.h b/src/models/transformer.h index f36150edb..abfa88237 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -502,7 +502,6 @@ class Transformer : public EncoderOrDecoderBase { }; class EncoderTransformer : public Transformer { - std::vector> embeddingLayers_; // (lazily created) public: EncoderTransformer(Ptr options) : Transformer(options) {} virtual ~EncoderTransformer() {} @@ -698,13 +697,6 @@ class DecoderTransformer : public Transformer { auto embeddings = state->getTargetHistoryEmbeddings(); // [-4: beam depth=1, -3: max length, -2: batch size, -1: vector dim] auto decoderMask = state->getTargetMask(); // [max length, batch size, 1] --this is a hypothesis - // dropout target words - float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); - if(dropoutTrg) { - int trgWords = embeddings->shape()[-3]; - embeddings = dropout(embeddings, dropoutTrg, {trgWords, 1, 1}); - } - //************************************************************************// int dimBeam = 1; diff --git a/src/rnn/attention.h b/src/rnn/attention.h old mode 100644 new mode 100755 index cbb16a340..964889d8d --- a/src/rnn/attention.h +++ b/src/rnn/attention.h @@ -58,8 +58,8 @@ class GlobalAttention : public CellInput { ba_ = graph->param(prefix + "_b_att", {1, dimEncState}, inits::zeros); if(dropout_ > 0.0f) { - dropMaskContext_ = graph->dropout(dropout_, {1, dimEncState}); - dropMaskState_ = graph->dropout(dropout_, {1, dimDecState}); + dropMaskContext_ = graph->dropoutMask(dropout_, {1, dimEncState}); + dropMaskState_ = graph->dropoutMask(dropout_, {1, dimDecState}); } if(dropMaskContext_) diff --git a/src/rnn/cells.h b/src/rnn/cells.h old mode 100644 new mode 100755 index 12db6ea35..3b82c54ac --- a/src/rnn/cells.h +++ b/src/rnn/cells.h @@ -46,8 +46,8 @@ class Tanh : public Cell { if(dropout_ > 0.0f) { if(dimInput) - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); - dropMaskS_ = graph->dropout(dropout_, {1, dimState}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); + dropMaskS_ = graph->dropoutMask(dropout_, {1, dimState}); } if(layerNorm_) { @@ -139,8 +139,8 @@ class ReLU : public Cell { if(dropout_ > 0.0f) { if(dimInput) - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); - dropMaskS_ = graph->dropout(dropout_, {1, dimState}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); + dropMaskS_ = graph->dropoutMask(dropout_, {1, dimState}); } if(layerNorm_) { @@ -256,8 +256,8 @@ class GRU : public Cell { if(dropout_ > 0.0f) { if(dimInput) - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); - dropMaskS_ = graph->dropout(dropout_, {1, dimState}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); + dropMaskS_ = graph->dropoutMask(dropout_, {1, dimState}); } if(layerNorm_) { @@ -420,8 +420,8 @@ class GRUNematus : public Cell { if(dropout_ > 0.0f) { if(dimInput) - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); - dropMaskS_ = graph->dropout(dropout_, {1, dimState}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); + dropMaskS_ = graph->dropoutMask(dropout_, {1, dimState}); } if(layerNorm_) { @@ -584,8 +584,8 @@ class FastLSTM : public Cell { if(dropout_ > 0.0f) { if(dimInput) - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); - dropMaskS_ = graph->dropout(dropout_, {1, dimState}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); + dropMaskS_ = graph->dropoutMask(dropout_, {1, dimState}); } if(layerNorm_) { @@ -930,7 +930,7 @@ class SRU : public Cell { br_ = graph->param(prefix + "_br", {1, dimInput}, inits::zeros); if(dropout_ > 0.0f) { - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); } if(layerNorm_) { @@ -1021,7 +1021,7 @@ class SSRU : public Cell { bf_ = graph->param(prefix + "_bf", {1, dimInput}, inits::zeros); if(dropout_ > 0.0f) { - dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); + dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); } if(layerNorm_) { @@ -1104,7 +1104,7 @@ class SSRU : public Cell { // prefix + "_bf", {1, dimInput}, inits::zeros); // if(dropout_ > 0.0f) { -// dropMaskX_ = graph->dropout(dropout_, {1, dimInput}); +// dropMaskX_ = graph->dropoutMask(dropout_, {1, dimInput}); // } // } From 55df8440275b74e19f8961348b63111f004562b6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 14:38:56 -0700 Subject: [PATCH 461/838] refactored into EncoderDecoderLayerBase --- src/layers/generic.cpp | 68 +++++++++++++++++++++++++++++++++++----- src/layers/generic.h | 34 ++++++++++++++++++++ src/models/decoder.h | 51 ++---------------------------- src/models/encoder.h | 22 +++---------- src/models/transformer.h | 41 ++---------------------- 5 files changed, 103 insertions(+), 113 deletions(-) mode change 100644 => 100755 src/layers/generic.h diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index ef3c21933..5249f46a1 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -1,20 +1,14 @@ #include "marian.h" #include "layers/generic.h" +#include "layers/constructors.h" #include "layers/loss.h" #include "data/factored_vocab.h" #include "rnn/types.h" // for State::select() -using std::size_t; // not sure why this is needed +//using std::size_t; // not sure why this is needed namespace marian { - //struct CSRSparseTensor { // simplistic for now - // Shape shape; - // Expr values; // [k_i..k_{i+1}-1] -> value at [i,j] - // Expr indices; // [k_i..k_{i+1}-1] -> j of non-null value - // Expr offsets; // [i] -> k_i - //}; - Logits::Logits(Expr logits) : Logits(New(logits, nullptr)) {} // single-output constructor from Expr only (RationalLoss has no count) Ptr Logits::graph() const { @@ -426,4 +420,62 @@ namespace marian { ABORT_IF(factoredVocab_ /*&& factoredVocab_->getNumGroups() > 1*/, "Embedding: applyIndices must not be used with a factored vocabulary"); return reshape(rows(E_, embIdx), shape); } + + Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { + // standard encoder word embeddings + int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src + int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt + int dimEmb = opt("dim-emb"); + int dimUlrEmb = opt("ulr-dim-emb"); + auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) + ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) + ("ulrTrainTransform", opt("ulr-trainable-transformation")) + ("ulrQueryFile", opt("ulr-query-vectors")) + ("ulrKeysFile", opt("ulr-keys-vectors")); + return embFactory.construct(graph); + } + + Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex) const { + // standard encoder word embeddings + int dimVoc = opt>("dim-vocabs")[subBatchIndex]; + int dimEmb = opt("dim-emb"); + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + embFactory("prefix", "Wemb"); + else + embFactory("prefix", prefix_ + "_Wemb"); + if(options_->has("embedding-fix-src")) + embFactory("fixed", opt("embedding-fix-src")); + if(options_->hasAndNotEmpty("embedding-vectors")) { + auto embFiles = opt>("embedding-vectors"); + embFactory("embFile", embFiles[subBatchIndex]) + ("normalization", opt("embedding-normalization")); + } + embFactory("vocab", opt>("vocabs")[subBatchIndex]); // for factored embeddings + return embFactory.construct(graph); + } + + void EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { + // @TODO: code dup with above + if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy + if (embeddingLayers_.size() <= batchIndex_) + embeddingLayers_.resize(batchIndex_ + 1); + int dimVoc = opt>("dim-vocabs")[batchIndex_]; + int dimEmb = opt("dim-emb"); + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + embFactory("prefix", "Wemb"); + else + embFactory("prefix", prefix_ + "_Wemb"); + if(options_->has("embedding-fix-trg")) + embFactory("fixed", opt("embedding-fix-trg")); + if(options_->hasAndNotEmpty("embedding-vectors")) { + auto embFiles = opt>("embedding-vectors"); + embFactory("embFile", embFiles[batchIndex_]) // + ("normalization", opt("embedding-normalization")); + } + embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings + embeddingLayers_[batchIndex_] = embFactory.construct(graph); + } + } } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h old mode 100644 new mode 100755 index 9d787a3c3..e581addcd --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -65,6 +65,40 @@ struct IEmbeddingLayer { virtual Expr applyIndices(const std::vector& embIdx, const Shape& shape) const = 0; }; +// base class for Encoder and Decoder classes, which have embeddings and a batch index (=stream index) +// @TODO: also base this on LayerBase, which holds options_ +class EncoderDecoderLayerBase { +protected: + Ptr options_; + const std::string prefix_; + const bool inference_{false}; + const size_t batchIndex_{1}; + std::vector> embeddingLayers_; // (lazily created) + + EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options) : + options_(options), + prefix_(options->get("prefix", prefix)), + inference_(options->get("inference", false)), + batchIndex_(options->get("index", batchIndex)) {} + +public: + template + T opt(const std::string& key) const { + return options_->get(key); + } + + template + T opt(const std::string& key, const T& def) { + return options_->get(key, def); + } + + virtual ~EncoderDecoderLayerBase() {} + + void lazyCreateEmbeddingLayer(Ptr graph); + Ptr createULREmbeddingLayer(Ptr graph) const; + Ptr createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex) const; +}; + class FactoredVocab; // To support factors, any output projection (that is followed by a softmax) must diff --git a/src/models/decoder.h b/src/models/decoder.h index f149c4b5b..658ee73d2 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -9,24 +9,13 @@ namespace marian { -class DecoderBase { +class DecoderBase : public EncoderDecoderLayerBase { protected: - Ptr options_; - const std::string prefix_{"decoder"}; - const bool inference_{false}; - const size_t batchIndex_{1}; - std::vector> embeddingLayers_; // (lazily created) - Ptr shortlist_; public: - DecoderBase(Ptr options) - : options_(options), - prefix_(options->get("prefix", "decoder")), - inference_(options->get("inference", false)), - batchIndex_(options->get("index", 1)) {} - - virtual ~DecoderBase() {} + DecoderBase(Ptr options) : + EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options) {} virtual Ptr startState(Ptr, Ptr batch, @@ -35,30 +24,6 @@ class DecoderBase { virtual Ptr step(Ptr, Ptr) = 0; - void lazyCreateEmbeddingLayer(Ptr graph) { - // @TODO: code dup with EncoderTransformer - if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy - if (embeddingLayers_.size() <= batchIndex_) - embeddingLayers_.resize(batchIndex_ + 1); - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-trg")) - embFactory("fixed", opt("embedding-fix-trg")); - if(options_->hasAndNotEmpty("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[batchIndex_]) // - ("normalization", opt("embedding-normalization")); - } - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - embeddingLayers_[batchIndex_] = embFactory.construct(graph); - } - } - virtual void embeddingsFromBatch(Ptr graph, Ptr state, Ptr batch) { @@ -117,16 +82,6 @@ class DecoderBase { shortlist_ = shortlist; } - template - T opt(const std::string& key) const { - return options_->get(key); - } - - template - T opt(const std::string& key, const T& def) { - return options_->get(key, def); - } - virtual void clear() = 0; }; diff --git a/src/models/encoder.h b/src/models/encoder.h index a1f6f5311..c6a3487a3 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -5,29 +5,15 @@ namespace marian { -class EncoderBase { +class EncoderBase : public EncoderDecoderLayerBase { protected: - Ptr options_; - const std::string prefix_{"encoder"}; - const bool inference_{false}; - const size_t batchIndex_{0}; - std::vector> embeddingLayers_; // (lazily created) public: - EncoderBase(Ptr options) - : options_(options), - prefix_(options->get("prefix", "encoder")), - inference_(options->get("inference", false)), - batchIndex_(options->get("index", 0)) {} - - virtual ~EncoderBase() {} + EncoderBase(Ptr options) : + EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options) {} + // @TODO: turn into an interface. Also see if we can get rid of the graph parameter. virtual Ptr build(Ptr, Ptr) = 0; - template - T opt(const std::string& key) const { - return options_->get(key); - } - virtual void clear() = 0; }; diff --git a/src/models/transformer.h b/src/models/transformer.h index abfa88237..5483e9765 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -506,43 +506,6 @@ class EncoderTransformer : public Transformer { EncoderTransformer(Ptr options) : Transformer(options) {} virtual ~EncoderTransformer() {} - // returns the embedding matrix based on options - // and based on batchIndex_. - - Ptr createULREmbeddingLayer() const { - // standard encoder word embeddings - int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src - int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt - int dimEmb = opt("dim-emb"); - int dimUlrEmb = opt("ulr-dim-emb"); - auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) - ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) - ("ulrTrainTransform", opt("ulr-trainable-transformation")) - ("ulrQueryFile", opt("ulr-query-vectors")) - ("ulrKeysFile", opt("ulr-keys-vectors")); - return embFactory.construct(graph_); - } - - Ptr createSourceEmbeddingLayer(size_t subBatchIndex) const { - // standard encoder word embeddings - int dimVoc = opt>("dim-vocabs")[subBatchIndex]; - int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-src")) - embFactory("fixed", opt("embedding-fix-src")); - if(options_->hasAndNotEmpty("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[subBatchIndex]) - ("normalization", opt("embedding-normalization")); - } - embFactory("vocab", opt>("vocabs")[subBatchIndex]); // for factored embeddings - return embFactory.construct(graph_); - } - virtual Ptr build(Ptr graph, Ptr batch) override { graph_ = graph; @@ -559,9 +522,9 @@ class EncoderTransformer : public Transformer { if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy embeddingLayers_.resize(batch->sets()); if (options_->has("ulr") && options_->get("ulr") == true) - embeddingLayers_[batchIndex_] = createULREmbeddingLayer(); // embedding uses ULR + embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph_); // embedding uses ULR else - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(batchIndex_); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph_, batchIndex_); } std::tie(batchEmbeddings, batchMask) = embeddingLayers_[batchIndex_]->apply((*batch)[batchIndex_]); // apply dropout over source words From 0c280de03ff208b9d6beefd1cbdf29bea2b052be Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 14:48:36 -0700 Subject: [PATCH 462/838] merged two embedding-layer creation functions --- src/layers/generic.cpp | 25 ++++++------------------- src/layers/generic.h | 4 +++- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5249f46a1..aa31013aa 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -435,7 +435,9 @@ namespace marian { return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex) const { + Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph, + size_t subBatchIndex, + const std::string& embeddingFixParamName /*= "embedding-fix-src"*/) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[subBatchIndex]; int dimEmb = opt("dim-emb"); @@ -444,8 +446,8 @@ namespace marian { embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-src")) - embFactory("fixed", opt("embedding-fix-src")); + if(options_->has(embeddingFixParamName)) + embFactory("fixed", opt(embeddingFixParamName)); if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); embFactory("embFile", embFiles[subBatchIndex]) @@ -460,22 +462,7 @@ namespace marian { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-trg")) - embFactory("fixed", opt("embedding-fix-trg")); - if(options_->hasAndNotEmpty("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[batchIndex_]) // - ("normalization", opt("embedding-normalization")); - } - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - embeddingLayers_[batchIndex_] = embFactory.construct(graph); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph, batchIndex_, "embedding-fix-trg"); } } } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h index e581addcd..b6d7c3faa 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -96,7 +96,9 @@ class EncoderDecoderLayerBase { void lazyCreateEmbeddingLayer(Ptr graph); Ptr createULREmbeddingLayer(Ptr graph) const; - Ptr createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex) const; + Ptr createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex, + const std::string& embeddingFixParamName = "embedding-fix-src") const; + Ptr createSourceEmbeddingLayer(Ptr graph); }; class FactoredVocab; From b06870c0473254b997afa76bdbe9f0561155b780 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 14:51:14 -0700 Subject: [PATCH 463/838] moved one functioin to base --- src/layers/generic.cpp | 30 +++++++++++++++++++++++++++++- src/models/s2s.h | 29 ----------------------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index aa31013aa..66d7a1b79 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -457,8 +457,36 @@ namespace marian { return embFactory.construct(graph); } + Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph) { + // create source embeddings + int dimVoc = opt>("dim-vocabs")[batchIndex_]; + int dimEmb = opt("dim-emb"); + + // @TODO: code dup with Decoder and EncoderTransformer; actually diverged by now. Unify this. + auto embFactory = embedding() // + ("dimVocab", dimVoc) // + ("dimEmb", dimEmb); + + if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) + embFactory("prefix", "Wemb"); + else + embFactory("prefix", prefix_ + "_Wemb"); + + if(options_->has("embedding-fix-src")) + embFactory("fixed", opt("embedding-fix-src")); + + if(options_->hasAndNotEmpty("embedding-vectors")) { + auto embFiles = opt>("embedding-vectors"); + embFactory // + ("embFile", embFiles[batchIndex_]) // + ("normalization", opt("embedding-normalization")); + } + + embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings + return embFactory.construct(graph); + } + void EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { - // @TODO: code dup with above if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); diff --git a/src/models/s2s.h b/src/models/s2s.h index f7fa91dd2..50305f471 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -119,35 +119,6 @@ class EncoderS2S : public EncoderBase { return context; } - Ptr createSourceEmbeddingLayer(Ptr graph) { - // create source embeddings - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - - // @TODO: code dup with Decoder and EncoderTransformer; actually diverged by now. Unify this. - auto embFactory = embedding() // - ("dimVocab", dimVoc) // - ("dimEmb", dimEmb); - - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); - - if(options_->has("embedding-fix-src")) - embFactory("fixed", opt("embedding-fix-src")); - - if(options_->hasAndNotEmpty("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - embFactory // - ("embFile", embFiles[batchIndex_]) // - ("normalization", opt("embedding-normalization")); - } - - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - return embFactory.construct(graph); - } - EncoderS2S(Ptr options) : EncoderBase(options) {} virtual Ptr build(Ptr graph, From 4c6a809a2e4f6efec72197e40f921ce9b4cac7cd Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 14:52:50 -0700 Subject: [PATCH 464/838] reformatted --- src/layers/generic.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 66d7a1b79..ff88ce23d 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -461,27 +461,18 @@ namespace marian { // create source embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - - // @TODO: code dup with Decoder and EncoderTransformer; actually diverged by now. Unify this. - auto embFactory = embedding() // - ("dimVocab", dimVoc) // - ("dimEmb", dimEmb); - + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-src")) embFactory("fixed", opt("embedding-fix-src")); - if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); - embFactory // - ("embFile", embFiles[batchIndex_]) // - ("normalization", opt("embedding-normalization")); + embFactory("embFile", embFiles[batchIndex_]) + ("normalization", opt("embedding-normalization")); } - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings return embFactory.construct(graph); } From 44df9c1c884b7bb78a8217fd63383019b995c81c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 14:54:59 -0700 Subject: [PATCH 465/838] merged a second lazy-create function --- src/layers/generic.cpp | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index ff88ce23d..031c85dfa 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -458,23 +458,7 @@ namespace marian { } Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph) { - // create source embeddings - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has("embedding-fix-src")) - embFactory("fixed", opt("embedding-fix-src")); - if(options_->hasAndNotEmpty("embedding-vectors")) { - auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[batchIndex_]) - ("normalization", opt("embedding-normalization")); - } - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - return embFactory.construct(graph); + return createSourceEmbeddingLayer(graph, batchIndex_, "embedding-fix-src"); } void EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { From 22eb16281d96b66592736671158fc4981dd676bc Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 15:11:38 -0700 Subject: [PATCH 466/838] removed a few parameters to createSourceEmbeddingLayer() --- src/layers/generic.cpp | 22 ++++++++-------------- src/layers/generic.h | 8 ++++---- src/models/decoder.h | 2 +- src/models/encoder.h | 2 +- src/models/s2s.h | 1 + src/models/transformer.h | 2 +- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 031c85dfa..5b52059bf 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -417,7 +417,7 @@ namespace marian { } Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { - ABORT_IF(factoredVocab_ /*&& factoredVocab_->getNumGroups() > 1*/, "Embedding: applyIndices must not be used with a factored vocabulary"); + ABORT_IF(factoredVocab_, "Embedding: applyIndices must not be used with a factored vocabulary"); return reshape(rows(E_, embIdx), shape); } @@ -435,37 +435,31 @@ namespace marian { return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph, - size_t subBatchIndex, - const std::string& embeddingFixParamName /*= "embedding-fix-src"*/) const { + Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph) const { // standard encoder word embeddings - int dimVoc = opt>("dim-vocabs")[subBatchIndex]; + int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else embFactory("prefix", prefix_ + "_Wemb"); - if(options_->has(embeddingFixParamName)) - embFactory("fixed", opt(embeddingFixParamName)); + if(options_->has(embeddingFixParamName_)) + embFactory("fixed", opt(embeddingFixParamName_)); if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[subBatchIndex]) + embFactory("embFile", embFiles[batchIndex_]) ("normalization", opt("embedding-normalization")); } - embFactory("vocab", opt>("vocabs")[subBatchIndex]); // for factored embeddings + embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph) { - return createSourceEmbeddingLayer(graph, batchIndex_, "embedding-fix-src"); - } - void EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph, batchIndex_, "embedding-fix-trg"); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); } } } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h index b6d7c3faa..71d37e09e 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -71,13 +71,15 @@ class EncoderDecoderLayerBase { protected: Ptr options_; const std::string prefix_; + const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" const bool inference_{false}; const size_t batchIndex_{1}; std::vector> embeddingLayers_; // (lazily created) - EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options) : + EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, const std::string& embeddingFixParamName) : options_(options), prefix_(options->get("prefix", prefix)), + embeddingFixParamName_(embeddingFixParamName), inference_(options->get("inference", false)), batchIndex_(options->get("index", batchIndex)) {} @@ -96,9 +98,7 @@ class EncoderDecoderLayerBase { void lazyCreateEmbeddingLayer(Ptr graph); Ptr createULREmbeddingLayer(Ptr graph) const; - Ptr createSourceEmbeddingLayer(Ptr graph, size_t subBatchIndex, - const std::string& embeddingFixParamName = "embedding-fix-src") const; - Ptr createSourceEmbeddingLayer(Ptr graph); + Ptr createSourceEmbeddingLayer(Ptr graph) const; }; class FactoredVocab; diff --git a/src/models/decoder.h b/src/models/decoder.h index 658ee73d2..48404efa3 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -15,7 +15,7 @@ class DecoderBase : public EncoderDecoderLayerBase { public: DecoderBase(Ptr options) : - EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options) {} + EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options, /*embeddingFixParamName=*/"embedding-fix-trg") {} virtual Ptr startState(Ptr, Ptr batch, diff --git a/src/models/encoder.h b/src/models/encoder.h index c6a3487a3..e9fb50334 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -9,7 +9,7 @@ class EncoderBase : public EncoderDecoderLayerBase { protected: public: EncoderBase(Ptr options) : - EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options) {} + EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, /*embeddingFixParamName=*/"embedding-fix-src") {} // @TODO: turn into an interface. Also see if we can get rid of the graph parameter. virtual Ptr build(Ptr, Ptr) = 0; diff --git a/src/models/s2s.h b/src/models/s2s.h index 50305f471..924502e85 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -124,6 +124,7 @@ class EncoderS2S : public EncoderBase { virtual Ptr build(Ptr graph, Ptr batch) override { // lazily create embedding layer + // @TODO: use shared function in base class once disentangled if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy embeddingLayers_.resize(batch->sets()); embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); diff --git a/src/models/transformer.h b/src/models/transformer.h index 5483e9765..9346d0cf8 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -524,7 +524,7 @@ class EncoderTransformer : public Transformer { if (options_->has("ulr") && options_->get("ulr") == true) embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph_); // embedding uses ULR else - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph_, batchIndex_); + embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph_); } std::tie(batchEmbeddings, batchMask) = embeddingLayers_[batchIndex_]->apply((*batch)[batchIndex_]); // apply dropout over source words From 13208380578a1666b480eca374190a1c9604919d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 15:23:07 -0700 Subject: [PATCH 467/838] streamlined lazy embedding creation --- src/layers/generic.cpp | 3 ++- src/layers/generic.h | 2 +- src/models/decoder.h | 7 +++---- src/models/s2s.h | 10 +--------- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 5b52059bf..428ca5c4e 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -455,11 +455,12 @@ namespace marian { return embFactory.construct(graph); } - void EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { + Ptr EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); } + return embeddingLayers_[batchIndex_]; } } // namespace marian diff --git a/src/layers/generic.h b/src/layers/generic.h index 71d37e09e..e2834be70 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -96,7 +96,7 @@ class EncoderDecoderLayerBase { virtual ~EncoderDecoderLayerBase() {} - void lazyCreateEmbeddingLayer(Ptr graph); + Ptr lazyCreateEmbeddingLayer(Ptr graph); Ptr createULREmbeddingLayer(Ptr graph) const; Ptr createSourceEmbeddingLayer(Ptr graph) const; }; diff --git a/src/models/decoder.h b/src/models/decoder.h index 48404efa3..5bbff8936 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -29,9 +29,8 @@ class DecoderBase : public EncoderDecoderLayerBase { Ptr batch) { auto subBatch = (*batch)[batchIndex_]; - lazyCreateEmbeddingLayer(graph); Expr y, yMask; std::tie - (y, yMask) = embeddingLayers_[batchIndex_]->apply(subBatch); + (y, yMask) = lazyCreateEmbeddingLayer(graph)->apply(subBatch); // dropout target words float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); if (dropoutTrg) { @@ -58,13 +57,13 @@ class DecoderBase : public EncoderDecoderLayerBase { const Words& words, int dimBatch, int dimBeam) { - lazyCreateEmbeddingLayer(graph); + auto embeddingLayer = lazyCreateEmbeddingLayer(graph); Expr selectedEmbs; int dimEmb = opt("dim-emb"); if(words.empty()) { selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); } else { - selectedEmbs = embeddingLayers_[batchIndex_]->apply(words, {dimBeam, 1, dimBatch, dimEmb}); + selectedEmbs = embeddingLayer->apply(words, {dimBeam, 1, dimBatch, dimEmb}); // dropout target words --does not make sense here since this is always inference. Keep it regular though. float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); if (dropoutTrg) { diff --git a/src/models/s2s.h b/src/models/s2s.h index 924502e85..000177dbe 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -123,17 +123,9 @@ class EncoderS2S : public EncoderBase { virtual Ptr build(Ptr graph, Ptr batch) override { - // lazily create embedding layer - // @TODO: use shared function in base class once disentangled - if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy - embeddingLayers_.resize(batch->sets()); - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); - } - auto embedding = embeddingLayers_[batchIndex_]; - // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = embedding->apply((*batch)[batchIndex_]); + (batchEmbeddings, batchMask) = lazyCreateEmbeddingLayer(graph)->apply((*batch)[batchIndex_]); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); if(dropProb) { From 5fffeaa5a81f6b606e92d8bec882498264911d88 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 15:24:15 -0700 Subject: [PATCH 468/838] streamlined lazy embedding creation --- src/layers/generic.cpp | 2 +- src/layers/generic.h | 2 +- src/models/decoder.h | 4 ++-- src/models/s2s.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 428ca5c4e..43637228b 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -455,7 +455,7 @@ namespace marian { return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::lazyCreateEmbeddingLayer(Ptr graph) { + Ptr EncoderDecoderLayerBase::getEmbeddingLayer(Ptr graph) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); diff --git a/src/layers/generic.h b/src/layers/generic.h index e2834be70..07ca6be7f 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -96,7 +96,7 @@ class EncoderDecoderLayerBase { virtual ~EncoderDecoderLayerBase() {} - Ptr lazyCreateEmbeddingLayer(Ptr graph); + Ptr getEmbeddingLayer(Ptr graph); Ptr createULREmbeddingLayer(Ptr graph) const; Ptr createSourceEmbeddingLayer(Ptr graph) const; }; diff --git a/src/models/decoder.h b/src/models/decoder.h index 5bbff8936..9ab097123 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -30,7 +30,7 @@ class DecoderBase : public EncoderDecoderLayerBase { auto subBatch = (*batch)[batchIndex_]; Expr y, yMask; std::tie - (y, yMask) = lazyCreateEmbeddingLayer(graph)->apply(subBatch); + (y, yMask) = getEmbeddingLayer(graph)->apply(subBatch); // dropout target words float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); if (dropoutTrg) { @@ -57,7 +57,7 @@ class DecoderBase : public EncoderDecoderLayerBase { const Words& words, int dimBatch, int dimBeam) { - auto embeddingLayer = lazyCreateEmbeddingLayer(graph); + auto embeddingLayer = getEmbeddingLayer(graph); Expr selectedEmbs; int dimEmb = opt("dim-emb"); if(words.empty()) { diff --git a/src/models/s2s.h b/src/models/s2s.h index 000177dbe..dbc9a92fd 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -125,7 +125,7 @@ class EncoderS2S : public EncoderBase { Ptr batch) override { // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = lazyCreateEmbeddingLayer(graph)->apply((*batch)[batchIndex_]); + (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply((*batch)[batchIndex_]); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); if(dropProb) { From d1e1867111ed1bb7eb546065ea61495a4f8b8c7f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 15:34:22 -0700 Subject: [PATCH 469/838] EncoderDecoderLayerBase now derive from LayerBase --- src/layers/generic.h | 33 ++++++++++----------------------- src/models/transformer.h | 6 ++---- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/src/layers/generic.h b/src/layers/generic.h index 07ca6be7f..61728fb5c 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -5,14 +5,12 @@ #include "data/shortlist.h" #include "layers/factory.h" -namespace marian { -namespace mlp { -/** - * @brief Activation functions - */ -enum struct act : int { linear, tanh, sigmoid, ReLU, LeakyReLU, PReLU, swish }; -} // namespace mlp -} // namespace marian +namespace marian { namespace mlp { + /** + * @brief Activation functions + */ + enum struct act : int { linear, tanh, sigmoid, ReLU, LeakyReLU, PReLU, swish }; +}} YAML_REGISTER_TYPE(marian::mlp::act, int) @@ -31,12 +29,12 @@ class LayerBase { : graph_(graph), options_(options) {} template - T opt(const std::string key) { + T opt(const std::string key) const { return options_->get(key); } template - T opt(const std::string key, T defaultValue) { + T opt(const std::string key, const T& defaultValue) const { return options_->get(key, defaultValue); } }; @@ -67,9 +65,8 @@ struct IEmbeddingLayer { // base class for Encoder and Decoder classes, which have embeddings and a batch index (=stream index) // @TODO: also base this on LayerBase, which holds options_ -class EncoderDecoderLayerBase { +class EncoderDecoderLayerBase : public LayerBase { protected: - Ptr options_; const std::string prefix_; const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" const bool inference_{false}; @@ -77,23 +74,13 @@ class EncoderDecoderLayerBase { std::vector> embeddingLayers_; // (lazily created) EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, const std::string& embeddingFixParamName) : - options_(options), + LayerBase(/*graph=*/nullptr, options), // @BUGBUG: we really should pass the graph in here prefix_(options->get("prefix", prefix)), embeddingFixParamName_(embeddingFixParamName), inference_(options->get("inference", false)), batchIndex_(options->get("index", batchIndex)) {} public: - template - T opt(const std::string& key) const { - return options_->get(key); - } - - template - T opt(const std::string& key, const T& def) { - return options_->get(key, def); - } - virtual ~EncoderDecoderLayerBase() {} Ptr getEmbeddingLayer(Ptr graph); diff --git a/src/models/transformer.h b/src/models/transformer.h index 9346d0cf8..a609265c5 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -24,8 +24,8 @@ class Transformer : public EncoderOrDecoderBase { typedef EncoderOrDecoderBase Base; protected: - using Base::options_; using Base::inference_; using Base::batchIndex_; - std::unordered_map cache_; + using Base::options_; using Base::inference_; using Base::batchIndex_; using Base::graph_; + std::unordered_map cache_; // caching transformation of the encoder that should not be created again // attention weights produced by step() // If enabled, it is set once per batch during training, and once per step during translation. @@ -37,8 +37,6 @@ class Transformer : public EncoderOrDecoderBase { template T opt(const std::string& key, const T& def) const { Ptr options = options_; if (options->has(key)) return options->get(key); else return def; } - Ptr graph_; - public: Transformer(Ptr options) : EncoderOrDecoderBase(options) { From baf6ce2e7a1428c0977bf230476b02796820298b Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 15:53:47 -0700 Subject: [PATCH 470/838] made embedding-creation functions private --- src/layers/generic.cpp | 37 ++++++++++++++++++++----------------- src/layers/generic.h | 16 +++++++++++----- src/models/char_s2s.h | 4 +--- src/models/decoder.h | 4 +++- src/models/encoder.h | 4 +++- src/models/transformer.h | 10 ++-------- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 43637228b..a484da7e4 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -421,21 +421,7 @@ namespace marian { return reshape(rows(E_, embIdx), shape); } - Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { - // standard encoder word embeddings - int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src - int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt - int dimEmb = opt("dim-emb"); - int dimUlrEmb = opt("ulr-dim-emb"); - auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) - ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) - ("ulrTrainTransform", opt("ulr-trainable-transformation")) - ("ulrQueryFile", opt("ulr-query-vectors")) - ("ulrKeysFile", opt("ulr-keys-vectors")); - return embFactory.construct(graph); - } - - Ptr EncoderDecoderLayerBase::createSourceEmbeddingLayer(Ptr graph) const { + Ptr EncoderDecoderLayerBase::createEmbeddingLayer(Ptr graph) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); @@ -455,11 +441,28 @@ namespace marian { return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::getEmbeddingLayer(Ptr graph) { + Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { + // standard encoder word embeddings + int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src + int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt + int dimEmb = opt("dim-emb"); + int dimUlrEmb = opt("ulr-dim-emb"); + auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) + ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) + ("ulrTrainTransform", opt("ulr-trainable-transformation")) + ("ulrQueryFile", opt("ulr-query-vectors")) + ("ulrKeysFile", opt("ulr-keys-vectors")); + return embFactory.construct(graph); + } + + Ptr EncoderDecoderLayerBase::getEmbeddingLayer(Ptr graph, bool ulr) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph); + if (ulr) + embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph); // embedding uses ULR + else + embeddingLayers_[batchIndex_] = createEmbeddingLayer(graph); } return embeddingLayers_[batchIndex_]; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 61728fb5c..83bf73507 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -64,28 +64,34 @@ struct IEmbeddingLayer { }; // base class for Encoder and Decoder classes, which have embeddings and a batch index (=stream index) -// @TODO: also base this on LayerBase, which holds options_ class EncoderDecoderLayerBase : public LayerBase { protected: const std::string prefix_; + const std::string dropoutParamName_; // "dropout-src" or "dropout-trg" const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" const bool inference_{false}; const size_t batchIndex_{1}; std::vector> embeddingLayers_; // (lazily created) - EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, const std::string& embeddingFixParamName) : + EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, + const std::string& dropoutParamName, + const std::string& embeddingFixParamName) : LayerBase(/*graph=*/nullptr, options), // @BUGBUG: we really should pass the graph in here prefix_(options->get("prefix", prefix)), + dropoutParamName_(dropoutParamName), embeddingFixParamName_(embeddingFixParamName), inference_(options->get("inference", false)), batchIndex_(options->get("index", batchIndex)) {} -public: virtual ~EncoderDecoderLayerBase() {} - Ptr getEmbeddingLayer(Ptr graph); +private: + Ptr createEmbeddingLayer(Ptr graph) const; Ptr createULREmbeddingLayer(Ptr graph) const; - Ptr createSourceEmbeddingLayer(Ptr graph) const; + +public: + // get embedding layer; lazily create on first call + Ptr getEmbeddingLayer(Ptr graph, bool ulr = false); }; class FactoredVocab; diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h index f14bf8807..d759d24f5 100755 --- a/src/models/char_s2s.h +++ b/src/models/char_s2s.h @@ -13,11 +13,9 @@ class CharS2SEncoder : public EncoderS2S { virtual Ptr build(Ptr graph, Ptr batch) override { - auto embedding = createSourceEmbedding(graph); - // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = embedding->apply(batch->front()); + (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply(batch->front()); // apply dropout over source words float dropProb = inference_ ? 0 : opt("dropout-src"); if(dropProb) { diff --git a/src/models/decoder.h b/src/models/decoder.h index 9ab097123..b10f2a386 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -15,7 +15,9 @@ class DecoderBase : public EncoderDecoderLayerBase { public: DecoderBase(Ptr options) : - EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options, /*embeddingFixParamName=*/"embedding-fix-trg") {} + EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options, + /*dropoutParamName=*/"dropout-trg", + /*embeddingFixParamName=*/"embedding-fix-trg") {} virtual Ptr startState(Ptr, Ptr batch, diff --git a/src/models/encoder.h b/src/models/encoder.h index e9fb50334..3dc02237a 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -9,7 +9,9 @@ class EncoderBase : public EncoderDecoderLayerBase { protected: public: EncoderBase(Ptr options) : - EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, /*embeddingFixParamName=*/"embedding-fix-src") {} + EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, + /*dropoutParamName=*/"dropout-src", + /*embeddingFixParamName=*/"embedding-fix-src") {} // @TODO: turn into an interface. Also see if we can get rid of the graph parameter. virtual Ptr build(Ptr, Ptr) = 0; diff --git a/src/models/transformer.h b/src/models/transformer.h index a609265c5..4ea6d7283 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -517,14 +517,8 @@ class EncoderTransformer : public Transformer { // embed the source words in the batch Expr batchEmbeddings, batchMask; - if (embeddingLayers_.empty() || !embeddingLayers_[batchIndex_]) { // lazy - embeddingLayers_.resize(batch->sets()); - if (options_->has("ulr") && options_->get("ulr") == true) - embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph_); // embedding uses ULR - else - embeddingLayers_[batchIndex_] = createSourceEmbeddingLayer(graph_); - } - std::tie(batchEmbeddings, batchMask) = embeddingLayers_[batchIndex_]->apply((*batch)[batchIndex_]); + auto embeddingLayer = getEmbeddingLayer(graph_, options_->has("ulr") && options_->get("ulr")); + std::tie(batchEmbeddings, batchMask) = embeddingLayer->apply((*batch)[batchIndex_]); // apply dropout over source words float dropoutSrc = inference_ ? 0 : opt("dropout-src"); if(dropoutSrc) { From 23bf4c8f08f2a5882885a98e37b961b5c88a21b3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 16:01:08 -0700 Subject: [PATCH 471/838] EncoderDecoderLayerBase now knows its dropout value --- src/layers/generic.cpp | 5 +++-- src/layers/generic.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index a484da7e4..1dd082007 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -421,7 +421,7 @@ namespace marian { return reshape(rows(E_, embIdx), shape); } - Ptr EncoderDecoderLayerBase::createEmbeddingLayer(Ptr graph) const { + /*private*/ Ptr EncoderDecoderLayerBase::createEmbeddingLayer(Ptr graph) const { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); @@ -441,7 +441,7 @@ namespace marian { return embFactory.construct(graph); } - Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { + /*private*/ Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { // standard encoder word embeddings int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt @@ -455,6 +455,7 @@ namespace marian { return embFactory.construct(graph); } + // get embedding layer for this encoder or decoder; lazily create it if not created yet Ptr EncoderDecoderLayerBase::getEmbeddingLayer(Ptr graph, bool ulr) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) diff --git a/src/layers/generic.h b/src/layers/generic.h index 83bf73507..0cffd6a21 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -69,8 +69,9 @@ class EncoderDecoderLayerBase : public LayerBase { const std::string prefix_; const std::string dropoutParamName_; // "dropout-src" or "dropout-trg" const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" - const bool inference_{false}; - const size_t batchIndex_{1}; + const bool inference_; + const float dropout_; + const size_t batchIndex_; std::vector> embeddingLayers_; // (lazily created) EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, @@ -81,6 +82,7 @@ class EncoderDecoderLayerBase : public LayerBase { dropoutParamName_(dropoutParamName), embeddingFixParamName_(embeddingFixParamName), inference_(options->get("inference", false)), + dropout_(inference_ ? 0 : opt(dropoutParamName)), batchIndex_(options->get("index", batchIndex)) {} virtual ~EncoderDecoderLayerBase() {} From 3a978be85b07eadc139a7095029e8e366621ed50 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 18:01:00 -0700 Subject: [PATCH 472/838] embedding dropout now exclusively implemented inside Embedding layer's apply() function --- src/layers/generic.cpp | 5 ++++- src/layers/generic.h | 7 +++++-- src/models/char_s2s.h | 6 ------ src/models/decoder.h | 17 ++--------------- src/models/s2s.h | 6 ------ src/models/transformer.h | 6 ------ 6 files changed, 11 insertions(+), 36 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 1dd082007..ab2972faa 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -413,6 +413,7 @@ namespace marian { selectedEmbs = multiRows(words); else selectedEmbs = rows(E_, toWordIndexVector(words)); + selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), {selectedEmbs->shape()[-3], 1, 1}); return reshape(selectedEmbs, shape); } @@ -425,7 +426,7 @@ namespace marian { // standard encoder word embeddings int dimVoc = opt>("dim-vocabs")[batchIndex_]; int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb); + auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb)("dropout", dropout_); if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) embFactory("prefix", "Wemb"); else @@ -449,6 +450,8 @@ namespace marian { int dimUlrEmb = opt("ulr-dim-emb"); auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) + ("ulr-dropout", opt("ulr-dropout")) + ("dropout", dropout_) ("ulrTrainTransform", opt("ulr-trainable-transformation")) ("ulrQueryFile", opt("ulr-query-vectors")) ("ulrKeysFile", opt("ulr-keys-vectors")); diff --git a/src/layers/generic.h b/src/layers/generic.h index 0cffd6a21..74d761bb2 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -67,7 +67,6 @@ struct IEmbeddingLayer { class EncoderDecoderLayerBase : public LayerBase { protected: const std::string prefix_; - const std::string dropoutParamName_; // "dropout-src" or "dropout-trg" const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" const bool inference_; const float dropout_; @@ -79,7 +78,6 @@ class EncoderDecoderLayerBase : public LayerBase { const std::string& embeddingFixParamName) : LayerBase(/*graph=*/nullptr, options), // @BUGBUG: we really should pass the graph in here prefix_(options->get("prefix", prefix)), - dropoutParamName_(dropoutParamName), embeddingFixParamName_(embeddingFixParamName), inference_(options->get("inference", false)), dropout_(inference_ ? 0 : opt(dropoutParamName)), @@ -281,6 +279,10 @@ class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList { } // namespace mlp +// A regular embedding layer. +// Note that this also applies dropout if the option is passed (pass 0 when in inference mode). +// It is best to not use Embedding directly, but rather via getEmbeddingLayer() in +// EncoderDecoderLayerBase, which knows to pass on all required parameters from options. class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; Ptr factoredVocab_; @@ -396,6 +398,7 @@ class ULREmbedding : public LayerBase, public IEmbeddingLayer { auto graph = ulrEmbeddings_.front()->graph(); auto batchMask = graph->constant({ dimWords, dimBatch, 1 }, inits::from_vector(subBatch->mask())); + batchEmbeddings = dropout(batchEmbeddings, options_->get("dropout", 0.0f), {batchEmbeddings->shape()[-3], 1, 1}); return std::make_tuple(batchEmbeddings, batchMask); } diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h index d759d24f5..85df8c5db 100755 --- a/src/models/char_s2s.h +++ b/src/models/char_s2s.h @@ -16,12 +16,6 @@ class CharS2SEncoder : public EncoderS2S { // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply(batch->front()); - // apply dropout over source words - float dropProb = inference_ ? 0 : opt("dropout-src"); - if(dropProb) { - int srcWords = batchEmbeddings->shape()[-3]; - batchEmbeddings = dropout(batchEmbeddings, dropProb, {srcWords, 1, 1}); - } int dimEmb = opt("dim-emb"); auto convSizes = options_->get>("char-conv-filters-num"); diff --git a/src/models/decoder.h b/src/models/decoder.h index b10f2a386..e4abcd07b 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -33,12 +33,6 @@ class DecoderBase : public EncoderDecoderLayerBase { Expr y, yMask; std::tie (y, yMask) = getEmbeddingLayer(graph)->apply(subBatch); - // dropout target words - float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); - if (dropoutTrg) { - int trgWords = y->shape()[-3]; - y = dropout(y, dropoutTrg, {trgWords, 1, 1}); - } const Words& data = /*if*/ (shortlist_) ? @@ -62,17 +56,10 @@ class DecoderBase : public EncoderDecoderLayerBase { auto embeddingLayer = getEmbeddingLayer(graph); Expr selectedEmbs; int dimEmb = opt("dim-emb"); - if(words.empty()) { + if(words.empty()) selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); - } else { + else selectedEmbs = embeddingLayer->apply(words, {dimBeam, 1, dimBatch, dimEmb}); - // dropout target words --does not make sense here since this is always inference. Keep it regular though. - float dropoutTrg = inference_ ? 0 : opt("dropout-trg"); - if (dropoutTrg) { - int trgWords = selectedEmbs->shape()[-3]; - selectedEmbs = dropout(selectedEmbs, dropoutTrg, { trgWords, 1, 1 }); - } - } state->setTargetHistoryEmbeddings(selectedEmbs); } diff --git a/src/models/s2s.h b/src/models/s2s.h index dbc9a92fd..3d101c685 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -126,12 +126,6 @@ class EncoderS2S : public EncoderBase { // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply((*batch)[batchIndex_]); - // apply dropout over source words - float dropProb = inference_ ? 0 : opt("dropout-src"); - if(dropProb) { - int srcWords = batchEmbeddings->shape()[-3]; - batchEmbeddings = dropout(batchEmbeddings, dropProb, {srcWords, 1, 1}); - } Expr context = applyEncoderRNN( graph, batchEmbeddings, batchMask, opt("enc-type")); diff --git a/src/models/transformer.h b/src/models/transformer.h index 4ea6d7283..973410db5 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -519,12 +519,6 @@ class EncoderTransformer : public Transformer { auto embeddingLayer = getEmbeddingLayer(graph_, options_->has("ulr") && options_->get("ulr")); std::tie(batchEmbeddings, batchMask) = embeddingLayer->apply((*batch)[batchIndex_]); - // apply dropout over source words - float dropoutSrc = inference_ ? 0 : opt("dropout-src"); - if(dropoutSrc) { - int srcWords = batchEmbeddings->shape()[-3]; - batchEmbeddings = dropout(batchEmbeddings, dropoutSrc, {srcWords, 1, 1}); - } batchEmbeddings = addSpecialEmbeddings(batchEmbeddings, /*start=*/0, batch); From 6714c8b8dee6a62a403172e007a34074883e1d08 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 18:10:14 -0700 Subject: [PATCH 473/838] factored the 0/null check into the dropout() functions, reducing code dup --- src/graph/expression_operators.h | 5 ++++- src/models/transformer.h | 7 ++----- src/rnn/attention.h | 6 ++---- src/rnn/cells.h | 36 ++++++++++++-------------------- 4 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/graph/expression_operators.h b/src/graph/expression_operators.h index f8c7fd3c0..a9433d2a6 100755 --- a/src/graph/expression_operators.h +++ b/src/graph/expression_operators.h @@ -207,7 +207,10 @@ Expr highway(Expr y, Expr x, Expr t); Expr highway(const std::string prefix, Expr x); static inline Expr dropout(Expr x, Expr mask) { - return x * mask; + if (mask) + return x * mask; + else + return x; } static inline Expr dropout(Expr x, float dropProb, Shape shape) { diff --git a/src/models/transformer.h b/src/models/transformer.h index 973410db5..ef932ae68 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -148,8 +148,7 @@ class Transformer : public EncoderOrDecoderBase { x = affine(x, W, b); if (actFn) x = actFn(x); - if (dropProb) - x = dropout(x, dropProb); + x = dropout(x, dropProb); return x; } @@ -249,9 +248,7 @@ class Transformer : public EncoderOrDecoderBase { collectOneHead(weights, dimBeam); // optional dropout for attention weights - float dropProb - = inference_ ? 0 : opt("transformer-dropout-attention"); - weights = dropout(weights, dropProb); + weights = dropout(weights, inference_ ? 0 : opt("transformer-dropout-attention")); // apply attention weights to values auto output = bdot(weights, v); // [-4: beam depth * batch size, -3: num heads, -2: max tgt length, -1: split vector dim] diff --git a/src/rnn/attention.h b/src/rnn/attention.h index 964889d8d..154f63ef4 100755 --- a/src/rnn/attention.h +++ b/src/rnn/attention.h @@ -62,8 +62,7 @@ class GlobalAttention : public CellInput { dropMaskState_ = graph->dropoutMask(dropout_, {1, dimDecState}); } - if(dropMaskContext_) - contextDropped_ = dropout(contextDropped_, dropMaskContext_); + contextDropped_ = dropout(contextDropped_, dropMaskContext_); if(layerNorm_) { if(nematusNorm_) { @@ -113,8 +112,7 @@ class GlobalAttention : public CellInput { if(recState->shape().size() > 3) dimBeam = recState->shape()[-4]; - if(dropMaskState_) - recState = dropout(recState, dropMaskState_); + recState = dropout(recState, dropMaskState_); auto mappedState = dot(recState, Wa_); if(layerNorm_) { diff --git a/src/rnn/cells.h b/src/rnn/cells.h index 3b82c54ac..36c3e24d1 100755 --- a/src/rnn/cells.h +++ b/src/rnn/cells.h @@ -72,8 +72,7 @@ class Tanh : public Cell { else input = inputs.front(); - if(dropMaskX_) - input = dropout(input, dropMaskX_); + input = dropout(input, dropMaskX_); auto xW = dot(input, W_); @@ -87,8 +86,7 @@ class Tanh : public Cell { Expr recState = state.output; auto stateDropped = recState; - if(dropMaskS_) - stateDropped = dropout(recState, dropMaskS_); + stateDropped = dropout(recState, dropMaskS_); auto sU = dot(stateDropped, U_); if(layerNorm_) sU = layerNorm(sU, gamma2_); @@ -163,8 +161,7 @@ class ReLU : public Cell { else input = inputs.front(); - if(dropMaskX_) - input = dropout(input, dropMaskX_); + input = dropout(input, dropMaskX_); auto xW = dot(input, W_); @@ -178,8 +175,7 @@ class ReLU : public Cell { Expr recState = state.output; auto stateDropped = recState; - if(dropMaskS_) - stateDropped = dropout(recState, dropMaskS_); + stateDropped = dropout(recState, dropMaskS_); auto sU = dot(stateDropped, U_); if(layerNorm_) sU = layerNorm(sU, gamma2_); @@ -284,8 +280,7 @@ class GRU : public Cell { else input = inputs[0]; - if(dropMaskX_) - input = dropout(input, dropMaskX_); + input = dropout(input, dropMaskX_); auto xW = dot(input, W_); if(layerNorm_) @@ -299,8 +294,7 @@ class GRU : public Cell { Expr mask = nullptr) override { auto stateOrig = state.output; auto stateDropped = stateOrig; - if(dropMaskS_) - stateDropped = dropout(stateOrig, dropMaskS_); + stateDropped = dropout(stateOrig, dropMaskS_); auto sU = dot(stateDropped, U_); if(layerNorm_) @@ -460,8 +454,7 @@ class GRUNematus : public Cell { else input = inputs[0]; - if(dropMaskX_) - input = dropout(input, dropMaskX_); + input = dropout(input, dropMaskX_); Expr xW; if(layerNorm_) { @@ -494,8 +487,7 @@ class GRUNematus : public Cell { auto stateOrig = state.output; auto stateDropped = stateOrig; - if(dropMaskS_) - stateDropped = dropout(stateOrig, dropMaskS_); + stateDropped = dropout(stateOrig, dropMaskS_); Expr sU; if(layerNorm_) { @@ -612,8 +604,7 @@ class FastLSTM : public Cell { } else input = inputs.front(); - if(dropMaskX_) - input = dropout(input, dropMaskX_); + input = dropout(input, dropMaskX_); auto xW = dot(input, W_); @@ -630,8 +621,7 @@ class FastLSTM : public Cell { auto cellState = state.cell; auto recStateDropped = recState; - if(dropMaskS_) - recStateDropped = dropout(recState, dropMaskS_); + recStateDropped = dropout(recState, dropMaskS_); auto sU = dot(recStateDropped, U_); @@ -954,7 +944,7 @@ class SRU : public Cell { else input = inputs.front(); - auto inputDropped = dropMaskX_ ? dropout(input, dropMaskX_) : input; + auto inputDropped = dropout(input, dropMaskX_); Expr x, f, r; if(layerNorm_) { @@ -1044,7 +1034,7 @@ class SSRU : public Cell { else input = inputs.front(); - auto inputDropped = dropMaskX_ ? dropout(input, dropMaskX_) : input; + auto inputDropped = dropout(input, dropMaskX_); Expr x, f; if(layerNorm_) { @@ -1121,7 +1111,7 @@ class SSRU : public Cell { // else // input = inputs.front(); -// auto inputDropped = dropMaskX_ ? dropout(input, dropMaskX_) : input; +// auto inputDropped = dropout(input, dropMaskX_); // auto x = dot(inputDropped, W_); // auto f = affine(inputDropped, Wf_, bf_); From 9a345a1d95161dace288c13464790b0017c91813 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 19:09:56 -0700 Subject: [PATCH 474/838] moved option setters from Accumulator<> to Factory (keeping the old one only for back compat) --- src/layers/constructors.h | 2 +- src/layers/factory.h | 74 ++++++++++++++++++++++++++---------- src/layers/generic.cpp | 54 +++++++++++++------------- src/layers/generic.h | 2 +- src/models/char_s2s.h | 3 +- src/models/decoder.h | 11 ++++-- src/models/encoder.h | 1 - src/models/model_factory.cpp | 0 src/models/s2s.h | 5 ++- src/models/transformer.h | 2 +- src/rnn/constructors.h | 22 +++++------ 11 files changed, 106 insertions(+), 70 deletions(-) mode change 100644 => 100755 src/layers/constructors.h mode change 100644 => 100755 src/models/model_factory.cpp mode change 100644 => 100755 src/rnn/constructors.h diff --git a/src/layers/constructors.h b/src/layers/constructors.h old mode 100644 new mode 100755 index 54e1b3404..d5b1a881a --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -157,7 +157,7 @@ class MLPFactory : public Factory { Ptr construct(Ptr graph) { auto mlp = New(graph, options_); for(auto layer : layers_) { - layer->getOptions()->merge(options_); + layer->mergeOpts(options_); mlp->push_back(layer->construct(graph)); } return mlp; diff --git a/src/layers/factory.h b/src/layers/factory.h index d3e1e3fa2..f44f75feb 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -9,27 +9,59 @@ class Factory : public std::enable_shared_from_this { Ptr options_; public: + // construct with empty options Factory() : options_(New()) {} + // construct with options Factory(Ptr options) : Factory() { options_->merge(options); } + // construct with one or more individual parameters + // Factory("var1", val1, "var2", val2, ...) + template + Factory(const std::string& key, T value, Args&&... moreArgs) { + setOpts(key, value, std::forward(moreArgs)...); + } + // construct with options and one or more individual parameters + // Factory(options, "var1", val1, "var2", val2, ...) + template + Factory(Ptr options, Args&&... moreArgs) : Factory(options) { + setOpts(std::forward(moreArgs)...); + } virtual ~Factory() {} - Ptr getOptions() { return options_; } - std::string str() { return options_->str(); } + // retrieve an option + // auto val = opt("var"); + template + T opt(const std::string& key) { return options_->get(key); } + + template + T opt(const std::string& key, T defaultValue) { return options_->get(key, defaultValue); } + + // set a single option + // setOpt("var", val); template - T opt(const std::string& key) { - return options_->get(key); + void setOpt(const std::string& key, T value) { + options_->set(key, value); } + // set multiple options + // setOpts("var1", val1, "var2", val2, ...); template - T opt(const std::string& key, T defaultValue) { - return options_->get(key, defaultValue); + void setOpts(const std::string& key, T value) { options_->set(key, value); } + + template + void setOpts(const std::string& key, T value, Args&&... moreArgs) { + setOpt(key, value); + setOpts(std::forward(moreArgs)...); // recursively set the remaining args } + //void mergeOpts(const std::string& yaml) { options_->parse(yaml); } + + void mergeOpts(Ptr options) { options_->merge(options); } + template inline Ptr as() { return std::dynamic_pointer_cast(shared_from_this()); @@ -44,41 +76,43 @@ class Factory : public std::enable_shared_from_this { // simplest form of Factory that just passes on options to the constructor of a layer type template struct ConstructingFactory : public Factory { + template + ConstructingFactory(Args&&... moreArgs) : Factory(std::forward(moreArgs)...) {} + Ptr construct(Ptr graph) { return New(graph, options_); } }; -template +template // where BaseFactory : Factory class Accumulator : public BaseFactory { typedef BaseFactory Factory; public: Accumulator() : Factory() {} + Accumulator(Ptr options) : Factory(options) {} + template + Accumulator(Ptr options, Args&&... moreArgs) : Factory(options, std::forward(moreArgs)...) {} + template + Accumulator(const std::string& key, T value, Args&&... moreArgs) : Factory(key, value, std::forward(moreArgs)...) {} Accumulator(const Factory& factory) : Factory(factory) {} Accumulator(const Accumulator&) = default; Accumulator(Accumulator&&) = default; + // deprecated chaining syntax template Accumulator& operator()(const std::string& key, T value) { - Factory::getOptions()->set(key, value); - return *this; - } - - Accumulator& operator()(const std::string& yaml) { - Factory::getOptions()->parse(yaml); + Factory::setOpt(key, value); return *this; } -#if 0 - Accumulator& operator()(Config::YamlNode yaml) { - Factory::getOptions()->merge(yaml); - return *this; - } -#endif + //Accumulator& operator()(const std::string& yaml) { + // Factory::mergeOpts(yaml); + // return *this; + //} Accumulator& operator()(Ptr options) { - Factory::getOptions()->merge(options); + Factory::mergeOpts(options); return *this; } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index ab2972faa..853a345a5 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -422,51 +422,49 @@ namespace marian { return reshape(rows(E_, embIdx), shape); } + // standard encoder word embeddings /*private*/ Ptr EncoderDecoderLayerBase::createEmbeddingLayer(Ptr graph) const { - // standard encoder word embeddings - int dimVoc = opt>("dim-vocabs")[batchIndex_]; - int dimEmb = opt("dim-emb"); - auto embFactory = embedding()("dimVocab", dimVoc)("dimEmb", dimEmb)("dropout", dropout_); - if(opt("tied-embeddings-src") || opt("tied-embeddings-all")) - embFactory("prefix", "Wemb"); - else - embFactory("prefix", prefix_ + "_Wemb"); + auto embFactory = EmbeddingFactory( + "dimVocab", opt>("dim-vocabs")[batchIndex_], + "dimEmb", opt("dim-emb"), + "dropout", dropout_, + "prefix", (opt("tied-embeddings-src") || opt("tied-embeddings-all")) ? "Wemb" : prefix_ + "_Wemb", + "vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings if(options_->has(embeddingFixParamName_)) - embFactory("fixed", opt(embeddingFixParamName_)); + embFactory.setOpt("fixed", opt(embeddingFixParamName_)); if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); - embFactory("embFile", embFiles[batchIndex_]) - ("normalization", opt("embedding-normalization")); + embFactory.setOpts( + "embFile", embFiles[batchIndex_], + "normalization", opt("embedding-normalization")); } - embFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings return embFactory.construct(graph); } + // ULR word embeddings /*private*/ Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { - // standard encoder word embeddings - int dimSrcVoc = opt>("dim-vocabs")[0]; //ULR multi-lingual src - int dimTgtVoc = opt>("dim-vocabs")[1]; //ULR monon tgt - int dimEmb = opt("dim-emb"); - int dimUlrEmb = opt("ulr-dim-emb"); - auto embFactory = ulr_embedding()("dimSrcVoc", dimSrcVoc)("dimTgtVoc", dimTgtVoc) - ("dimUlrEmb", dimUlrEmb)("dimEmb", dimEmb) - ("ulr-dropout", opt("ulr-dropout")) - ("dropout", dropout_) - ("ulrTrainTransform", opt("ulr-trainable-transformation")) - ("ulrQueryFile", opt("ulr-query-vectors")) - ("ulrKeysFile", opt("ulr-keys-vectors")); - return embFactory.construct(graph); + return ULREmbeddingFactory( + "dimSrcVoc", opt>("dim-vocabs")[0], // ULR multi-lingual src + "dimTgtVoc", opt>("dim-vocabs")[1], // ULR monon tgt + "dimUlrEmb", opt("ulr-dim-emb"), + "dimEmb", opt("dim-emb"), + "ulr-dropout", opt("ulr-dropout"), + "dropout", dropout_, + "ulrTrainTransform", opt("ulr-trainable-transformation"), + "ulrQueryFile", opt("ulr-query-vectors"), + "ulrKeysFile", opt("ulr-keys-vectors")) + .construct(graph); } // get embedding layer for this encoder or decoder; lazily create it if not created yet - Ptr EncoderDecoderLayerBase::getEmbeddingLayer(Ptr graph, bool ulr) { + Ptr EncoderDecoderLayerBase::getEmbeddingLayer(bool ulr) { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); if (ulr) - embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph); // embedding uses ULR + embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph_); // embedding uses ULR else - embeddingLayers_[batchIndex_] = createEmbeddingLayer(graph); + embeddingLayers_[batchIndex_] = createEmbeddingLayer(graph_); } return embeddingLayers_[batchIndex_]; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 74d761bb2..2e9d1c715 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -91,7 +91,7 @@ class EncoderDecoderLayerBase : public LayerBase { public: // get embedding layer; lazily create on first call - Ptr getEmbeddingLayer(Ptr graph, bool ulr = false); + Ptr getEmbeddingLayer(bool ulr = false); }; class FactoredVocab; diff --git a/src/models/char_s2s.h b/src/models/char_s2s.h index 85df8c5db..06882dca8 100755 --- a/src/models/char_s2s.h +++ b/src/models/char_s2s.h @@ -13,9 +13,10 @@ class CharS2SEncoder : public EncoderS2S { virtual Ptr build(Ptr graph, Ptr batch) override { + graph_ = graph; // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply(batch->front()); + (batchEmbeddings, batchMask) = getEmbeddingLayer()->apply(batch->front()); int dimEmb = opt("dim-emb"); auto convSizes = options_->get>("char-conv-filters-num"); diff --git a/src/models/decoder.h b/src/models/decoder.h index e4abcd07b..2a89fadde 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -29,17 +29,19 @@ class DecoderBase : public EncoderDecoderLayerBase { virtual void embeddingsFromBatch(Ptr graph, Ptr state, Ptr batch) { + graph_ = graph; + auto subBatch = (*batch)[batchIndex_]; Expr y, yMask; std::tie - (y, yMask) = getEmbeddingLayer(graph)->apply(subBatch); + (y, yMask) = getEmbeddingLayer()->apply(subBatch); const Words& data = /*if*/ (shortlist_) ? shortlist_->mappedIndices() /*else*/ : subBatch->data(); - Expr yData = graph->indices(toWordIndexVector(data)); + Expr yData = graph_->indices(toWordIndexVector(data)); auto yShifted = shift(y, {1, 0, 0}); @@ -53,11 +55,12 @@ class DecoderBase : public EncoderDecoderLayerBase { const Words& words, int dimBatch, int dimBeam) { - auto embeddingLayer = getEmbeddingLayer(graph); + graph_ = graph; + auto embeddingLayer = getEmbeddingLayer(); Expr selectedEmbs; int dimEmb = opt("dim-emb"); if(words.empty()) - selectedEmbs = graph->constant({1, 1, dimBatch, dimEmb}, inits::zeros); + selectedEmbs = graph_->constant({1, 1, dimBatch, dimEmb}, inits::zeros); else selectedEmbs = embeddingLayer->apply(words, {dimBeam, 1, dimBatch, dimEmb}); state->setTargetHistoryEmbeddings(selectedEmbs); diff --git a/src/models/encoder.h b/src/models/encoder.h index 3dc02237a..acff064f9 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -6,7 +6,6 @@ namespace marian { class EncoderBase : public EncoderDecoderLayerBase { -protected: public: EncoderBase(Ptr options) : EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp old mode 100644 new mode 100755 diff --git a/src/models/s2s.h b/src/models/s2s.h index 3d101c685..081bf3e93 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -123,12 +123,13 @@ class EncoderS2S : public EncoderBase { virtual Ptr build(Ptr graph, Ptr batch) override { + graph_ = graph; // select embeddings that occur in the batch Expr batchEmbeddings, batchMask; std::tie - (batchEmbeddings, batchMask) = getEmbeddingLayer(graph)->apply((*batch)[batchIndex_]); + (batchEmbeddings, batchMask) = getEmbeddingLayer()->apply((*batch)[batchIndex_]); Expr context = applyEncoderRNN( - graph, batchEmbeddings, batchMask, opt("enc-type")); + graph_, batchEmbeddings, batchMask, opt("enc-type")); return New(context, batchMask, batch); } diff --git a/src/models/transformer.h b/src/models/transformer.h index ef932ae68..8f84b67eb 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -514,7 +514,7 @@ class EncoderTransformer : public Transformer { // embed the source words in the batch Expr batchEmbeddings, batchMask; - auto embeddingLayer = getEmbeddingLayer(graph_, options_->has("ulr") && options_->get("ulr")); + auto embeddingLayer = getEmbeddingLayer(options_->has("ulr") && options_->get("ulr")); std::tie(batchEmbeddings, batchMask) = embeddingLayer->apply((*batch)[batchIndex_]); batchEmbeddings = addSpecialEmbeddings(batchEmbeddings, /*start=*/0, batch); diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h old mode 100644 new mode 100755 index 3a5826d64..25b7d3544 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -110,9 +110,9 @@ class StackedCellFactory : public CellFactory { if(sf->is()) { auto cellFactory = sf->as(); - cellFactory->getOptions()->merge(options_); + cellFactory->mergeOpts(options_); - sf->getOptions()->set("dimInput", lastDimInput); + sf->setOpt("dimInput", lastDimInput); lastDimInput = 0; if(i == 0) @@ -122,7 +122,7 @@ class StackedCellFactory : public CellFactory { stacked->push_back(cellFactory->construct(graph)); } else { auto inputFactory = sf->as(); - inputFactory->getOptions()->merge(options_); + inputFactory->mergeOpts(options_); auto input = inputFactory->construct(graph); stacked->push_back(input); lastDimInput += input->dimOutput(); @@ -150,29 +150,29 @@ class RNNFactory : public Factory { for(size_t i = 0; i < layerFactories_.size(); ++i) { auto lf = layerFactories_[i]; - lf->getOptions()->merge(options_); + lf->mergeOpts(options_); if(i > 0) { int dimInput - = layerFactories_[i - 1]->getOptions()->get("dimState") - + lf->getOptions()->get("dimInputExtra", 0); + = layerFactories_[i - 1]->opt("dimState") + + lf->opt("dimInputExtra", 0); - lf->getOptions()->set("dimInput", dimInput); + lf->setOpt("dimInput", dimInput); } if((rnn::dir)opt("direction", (int)rnn::dir::forward) == rnn::dir::alternating_forward) { if(i % 2 == 0) - lf->getOptions()->set("direction", (int)rnn::dir::forward); + lf->setOpt("direction", (int)rnn::dir::forward); else - lf->getOptions()->set("direction", (int)rnn::dir::backward); + lf->setOpt("direction", (int)rnn::dir::backward); } if((rnn::dir)opt("direction", (int)rnn::dir::forward) == rnn::dir::alternating_backward) { if(i % 2 == 1) - lf->getOptions()->set("direction", (int)rnn::dir::forward); + lf->setOpt("direction", (int)rnn::dir::forward); else - lf->getOptions()->set("direction", (int)rnn::dir::backward); + lf->setOpt("direction", (int)rnn::dir::backward); } rnn->push_back(lf->construct(graph)); From 80171f3ca61d07bfce365c0b0c58ac8aa69882ce Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 14 May 2019 19:41:06 -0700 Subject: [PATCH 475/838] removed Accumulator<> at some places, and used the new syntax at some as well --- src/layers/constructors.h | 5 +++-- src/models/bert.h | 6 +++--- src/models/transformer.h | 35 +++++++++++++++-------------------- src/rnn/constructors.h | 1 + 4 files changed, 22 insertions(+), 25 deletions(-) mode change 100644 => 100755 src/models/bert.h diff --git a/src/layers/constructors.h b/src/layers/constructors.h index d5b1a881a..e6d013c3f 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -36,11 +36,13 @@ typedef Accumulator dense; * Factory for output layers, can be used in a multi-layer network factory. */ struct LogitLayerFactory : public Factory { + using Factory::Factory; virtual Ptr construct(Ptr graph) = 0; }; // @TODO: In the long run, I hope we can get rid of the abstract factories altogether. class OutputFactory : public LogitLayerFactory { + using LogitLayerFactory::LogitLayerFactory; protected: std::string tiedTransposedName_; Ptr shortlist_; @@ -51,9 +53,8 @@ class OutputFactory : public LogitLayerFactory { return Accumulator(*this); } - Accumulator setShortlist(Ptr shortlist) { + void setShortlist(Ptr shortlist) { shortlist_ = shortlist; - return Accumulator(*this); } Ptr construct(Ptr graph) override { diff --git a/src/models/bert.h b/src/models/bert.h old mode 100644 new mode 100755 index 61cff43bf..33f21fc82 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -357,9 +357,9 @@ class BertMaskedLM : public ClassifierBase { intermediate = layerNorm(intermediate, gamma, beta); auto layer2 = mlp::mlp() - .push_back(mlp::output() - ("prefix", prefix_ + "_ff_logit_l2") - ("dim", dimVoc) + .push_back(mlp::output( + "prefix", prefix_ + "_ff_logit_l2", + "dim", dimVoc) .tieTransposed("Wemb")) .construct(graph); diff --git a/src/models/transformer.h b/src/models/transformer.h index 8f84b67eb..8a6454e04 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -470,14 +470,14 @@ class Transformer : public EncoderOrDecoderBase { int /*startPos*/) const { float dropoutRnn = inference_ ? 0.f : opt("dropout-rnn"); - auto rnn = rnn::rnn() // - ("type", opt("dec-cell")) // - ("prefix", prefix) // - ("dimInput", opt("dim-emb")) // - ("dimState", opt("dim-emb")) // - ("dropout", dropoutRnn) // - ("layer-normalization", opt("layer-normalization")) // - .push_back(rnn::cell()) // + auto rnn = rnn::rnn( + "type", opt("dec-cell"), + "prefix", prefix, + "dimInput", opt("dim-emb"), + "dimState", opt("dim-emb"), + "dropout", dropoutRnn, + "layer-normalization", opt("layer-normalization")) + .push_back(rnn::cell()) .construct(graph_); float dropProb = inference_ ? 0 : opt("transformer-dropout"); @@ -589,19 +589,14 @@ class DecoderTransformer : public Transformer { int dimTrgVoc = opt>("dim-vocabs")[batchIndex_]; - auto outputFactory = mlp::output() // - ("prefix", prefix_ + "_ff_logit_out") // - ("dim", dimTrgVoc); + auto outputFactory = mlp::OutputFactory( + "prefix", prefix_ + "_ff_logit_out", + "dim", dimTrgVoc, + "vocab", opt>("vocabs")[batchIndex_], // for factored outputs + "lemma-dim-emb", opt("lemma-dim-emb", 0)); // for factored outputs - if(opt("tied-embeddings") || opt("tied-embeddings-all")) { - std::string tiedPrefix = prefix_ + "_Wemb"; - if(opt("tied-embeddings-all") || opt("tied-embeddings-src")) - tiedPrefix = "Wemb"; - outputFactory.tieTransposed(tiedPrefix); - } - - outputFactory("vocab", opt>("vocabs")[batchIndex_]); // for factored outputs - outputFactory("lemma-dim-emb", opt("lemma-dim-emb", 0)); // for factored outputs + if(opt("tied-embeddings") || opt("tied-embeddings-all")) + outputFactory.tieTransposed(opt("tied-embeddings-all") || opt("tied-embeddings-src") ? "Wemb" : prefix_ + "_Wemb"); output_ = std::dynamic_pointer_cast(outputFactory.construct(graph_)); // (construct() returns only the underlying interface) } diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index 25b7d3544..c8ddbb61c 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -141,6 +141,7 @@ class StackedCellFactory : public CellFactory { typedef Accumulator stacked_cell; class RNNFactory : public Factory { + using Factory::Factory; protected: std::vector> layerFactories_; From 356f8042932e9dbebb397e1c73b5f5df410ef6ee Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 11:43:20 -0700 Subject: [PATCH 476/838] minor bug fixes, to get last few changes to pass tests --- regression-tests | 2 +- scripts/embeddings/export_embeddings.py | 2 +- src/layers/factory.h | 2 +- src/layers/generic.cpp | 8 ++++++-- src/models/transformer.h | 8 ++++---- src/tensors/cpu/prod.cpp | 0 src/tests/dropout.cpp | 4 ++-- vs/Marian.vcxproj | 0 8 files changed, 15 insertions(+), 11 deletions(-) mode change 100644 => 100755 src/tensors/cpu/prod.cpp mode change 100644 => 100755 vs/Marian.vcxproj diff --git a/regression-tests b/regression-tests index 142eadddb..71b473f29 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit 142eadddbe04493c1024b42586030b72e9cb7ea2 +Subproject commit 71b473f29017933e68a513f7262044c46e39cccc diff --git a/scripts/embeddings/export_embeddings.py b/scripts/embeddings/export_embeddings.py index 1476e52c9..7aef07d5d 100755 --- a/scripts/embeddings/export_embeddings.py +++ b/scripts/embeddings/export_embeddings.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- from __future__ import print_function diff --git a/src/layers/factory.h b/src/layers/factory.h index f44f75feb..5ff5e8b8e 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -18,7 +18,7 @@ class Factory : public std::enable_shared_from_this { // construct with one or more individual parameters // Factory("var1", val1, "var2", val2, ...) template - Factory(const std::string& key, T value, Args&&... moreArgs) { + Factory(const std::string& key, T value, Args&&... moreArgs) : Factory() { setOpts(key, value, std::forward(moreArgs)...); } // construct with options and one or more individual parameters diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 853a345a5..c3197db2b 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -413,13 +413,17 @@ namespace marian { selectedEmbs = multiRows(words); else selectedEmbs = rows(E_, toWordIndexVector(words)); + selectedEmbs = reshape(selectedEmbs, shape); selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), {selectedEmbs->shape()[-3], 1, 1}); - return reshape(selectedEmbs, shape); + return selectedEmbs; } Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { ABORT_IF(factoredVocab_, "Embedding: applyIndices must not be used with a factored vocabulary"); - return reshape(rows(E_, embIdx), shape); + auto selectedEmbs = rows(E_, embIdx); + selectedEmbs = reshape(selectedEmbs, shape); + selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), { selectedEmbs->shape()[-3], 1, 1 }); + return selectedEmbs; } // standard encoder word embeddings diff --git a/src/models/transformer.h b/src/models/transformer.h index 8a6454e04..8b1f97900 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -61,10 +61,10 @@ class Transformer : public EncoderOrDecoderBase { Expr seenEmb = graph_->get("Wpos"); int numPos = seenEmb ? seenEmb->shape()[-2] : maxLength; - auto embeddingLayer = embedding() - ("prefix", "Wpos") // share positional embeddings across all encoders/decorders - ("dimVocab", numPos) - ("dimEmb", dimEmb) + auto embeddingLayer = embedding( + "prefix", "Wpos", // share positional embeddings across all encoders/decorders + "dimVocab", numPos, + "dimEmb", dimEmb) .construct(graph_); // fill with increasing numbers until current length or maxPos diff --git a/src/tensors/cpu/prod.cpp b/src/tensors/cpu/prod.cpp old mode 100644 new mode 100755 diff --git a/src/tests/dropout.cpp b/src/tests/dropout.cpp index c31ee6214..367029fe8 100644 --- a/src/tests/dropout.cpp +++ b/src/tests/dropout.cpp @@ -20,8 +20,8 @@ int main(int argc, char** argv) { for(int i = 0; i < 10; ++i) { g->clear(); - auto mask1 = g->dropout(0.2, {10, 3072}); - auto mask2 = g->dropout(0.3, {1, 3072}); + auto mask1 = g->dropoutMask(0.2, {10, 3072}); + auto mask2 = g->dropoutMask(0.3, {1, 3072}); auto mask = mask1 + mask2; debug(mask1, "mask1"); debug(mask2, "mask2"); diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj old mode 100644 new mode 100755 From 270fa7710f6bb2a8c3e7fec3c80bc0ce06933ebc Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 13:41:58 -0700 Subject: [PATCH 477/838] updated submodule ref to regression-tests --- regression-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-tests b/regression-tests index 71b473f29..658cf86e2 160000 --- a/regression-tests +++ b/regression-tests @@ -1 +1 @@ -Subproject commit 71b473f29017933e68a513f7262044c46e39cccc +Subproject commit 658cf86e2ab4aa76b4ab3edabd750b93d2416f1a From f3ff057537db5606ba15dbcc1adfa1b5e84e0a1c Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 15:16:44 -0700 Subject: [PATCH 478/838] Options now accepts a list of key-value pairs --- src/common/options.h | 24 +++++++++++++++++++++++ src/layers/factory.h | 5 ----- src/layers/generic.cpp | 43 +++++++++++++++++++++--------------------- src/layers/generic.h | 4 ++-- 4 files changed, 47 insertions(+), 29 deletions(-) mode change 100644 => 100755 src/common/options.h diff --git a/src/common/options.h b/src/common/options.h old mode 100644 new mode 100755 index 643456c93..c5b07951e --- a/src/common/options.h +++ b/src/common/options.h @@ -35,6 +35,22 @@ class Options { Options() {} Options(const Options& other) : options_(YAML::Clone(other.options_)) {} + // constructor with one or more key-value pairs + // New("var1", val1, "var2", val2, ...) + template + Options(const std::string& key, T value, Args&&... moreArgs) : Options() { + set(key, value, std::forward(moreArgs)...); + } + + // constructor that clones and zero or more updates + // options->with("var1", val1, "var2", val2, ...) + template + Ptr with(Args&&... args) const { + auto options = New(*this); + options->set(std::forward(args)...); + return options; + } + /** * @brief Return a copy of the object that can be safely modified. */ @@ -78,6 +94,14 @@ class Options { options_[key] = value; } + // set multiple + // options->set("var1", val1, "var2", val2, ...) + template + void set(const std::string& key, T value, Args&&... moreArgs) { + set(key, value); + set(std::forward(moreArgs)...); + } + template T get(const std::string& key) { ABORT_IF(!has(key), "Required option '{}' has not been set", key); diff --git a/src/layers/factory.h b/src/layers/factory.h index 5ff5e8b8e..8173c854e 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -106,11 +106,6 @@ class Accumulator : public BaseFactory { return *this; } - //Accumulator& operator()(const std::string& yaml) { - // Factory::mergeOpts(yaml); - // return *this; - //} - Accumulator& operator()(Ptr options) { Factory::mergeOpts(options); return *this; diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index c3197db2b..70e29a0d3 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -427,37 +427,36 @@ namespace marian { } // standard encoder word embeddings - /*private*/ Ptr EncoderDecoderLayerBase::createEmbeddingLayer(Ptr graph) const { - auto embFactory = EmbeddingFactory( + /*private*/ Ptr EncoderDecoderLayerBase::createEmbeddingLayer() const { + auto options = New( "dimVocab", opt>("dim-vocabs")[batchIndex_], - "dimEmb", opt("dim-emb"), - "dropout", dropout_, - "prefix", (opt("tied-embeddings-src") || opt("tied-embeddings-all")) ? "Wemb" : prefix_ + "_Wemb", - "vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings + "dimEmb", opt("dim-emb"), + "dropout", dropout_, + "prefix", (opt("tied-embeddings-src") || opt("tied-embeddings-all")) ? "Wemb" : prefix_ + "_Wemb", + "vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings if(options_->has(embeddingFixParamName_)) - embFactory.setOpt("fixed", opt(embeddingFixParamName_)); + options->set("fixed", opt(embeddingFixParamName_)); if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); - embFactory.setOpts( + options->set( "embFile", embFiles[batchIndex_], "normalization", opt("embedding-normalization")); } - return embFactory.construct(graph); + return New(graph_, options); } // ULR word embeddings - /*private*/ Ptr EncoderDecoderLayerBase::createULREmbeddingLayer(Ptr graph) const { - return ULREmbeddingFactory( - "dimSrcVoc", opt>("dim-vocabs")[0], // ULR multi-lingual src - "dimTgtVoc", opt>("dim-vocabs")[1], // ULR monon tgt - "dimUlrEmb", opt("ulr-dim-emb"), - "dimEmb", opt("dim-emb"), - "ulr-dropout", opt("ulr-dropout"), - "dropout", dropout_, + /*private*/ Ptr EncoderDecoderLayerBase::createULREmbeddingLayer() const { + return New(graph_, New( + "dimSrcVoc", opt>("dim-vocabs")[0], // ULR multi-lingual src + "dimTgtVoc", opt>("dim-vocabs")[1], // ULR monon tgt + "dimUlrEmb", opt("ulr-dim-emb"), + "dimEmb", opt("dim-emb"), + "ulr-dropout", opt("ulr-dropout"), + "dropout", dropout_, "ulrTrainTransform", opt("ulr-trainable-transformation"), - "ulrQueryFile", opt("ulr-query-vectors"), - "ulrKeysFile", opt("ulr-keys-vectors")) - .construct(graph); + "ulrQueryFile", opt("ulr-query-vectors"), + "ulrKeysFile", opt("ulr-keys-vectors"))); } // get embedding layer for this encoder or decoder; lazily create it if not created yet @@ -466,9 +465,9 @@ namespace marian { if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); if (ulr) - embeddingLayers_[batchIndex_] = createULREmbeddingLayer(graph_); // embedding uses ULR + embeddingLayers_[batchIndex_] = createULREmbeddingLayer(); // embedding uses ULR else - embeddingLayers_[batchIndex_] = createEmbeddingLayer(graph_); + embeddingLayers_[batchIndex_] = createEmbeddingLayer(); } return embeddingLayers_[batchIndex_]; } diff --git a/src/layers/generic.h b/src/layers/generic.h index 2e9d1c715..ea34f9379 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -86,8 +86,8 @@ class EncoderDecoderLayerBase : public LayerBase { virtual ~EncoderDecoderLayerBase() {} private: - Ptr createEmbeddingLayer(Ptr graph) const; - Ptr createULREmbeddingLayer(Ptr graph) const; + Ptr createEmbeddingLayer() const; + Ptr createULREmbeddingLayer() const; public: // get embedding layer; lazily create on first call From ef1844d3eb0b464d48d02f24e595f66b99b5f8f0 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 15:51:58 -0700 Subject: [PATCH 479/838] dropout and embedding-fix are now passed as values to EncoderDecoderLayerBase; Factory(multiple) simplified; first example of options->with(); fixed default constructors of Factory classes --- src/common/options.h | 2 +- src/layers/constructors.h | 1 + src/layers/factory.h | 18 ++++++------------ src/layers/generic.cpp | 9 +++++---- src/layers/generic.h | 16 ++++++++-------- src/models/decoder.h | 4 ++-- src/models/encoder.h | 4 ++-- src/models/model_factory.cpp | 8 ++++---- src/models/model_factory.h | 21 +++++++++++++-------- src/rnn/constructors.h | 7 ++++--- 10 files changed, 46 insertions(+), 44 deletions(-) mode change 100644 => 100755 src/models/model_factory.h diff --git a/src/common/options.h b/src/common/options.h index c5b07951e..1337b2ece 100755 --- a/src/common/options.h +++ b/src/common/options.h @@ -44,7 +44,7 @@ class Options { // constructor that clones and zero or more updates // options->with("var1", val1, "var2", val2, ...) - template + template Ptr with(Args&&... args) const { auto options = New(*this); options->set(std::forward(args)...); diff --git a/src/layers/constructors.h b/src/layers/constructors.h index e6d013c3f..a2c38197f 100755 --- a/src/layers/constructors.h +++ b/src/layers/constructors.h @@ -151,6 +151,7 @@ class MLP : public IUnaryLogitLayer, public IHasShortList { * to accumulate options for later lazy construction. */ class MLPFactory : public Factory { + using Factory::Factory; private: std::vector> layers_; diff --git a/src/layers/factory.h b/src/layers/factory.h index 8173c854e..bb493f8a0 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -15,17 +15,17 @@ class Factory : public std::enable_shared_from_this { Factory(Ptr options) : Factory() { options_->merge(options); } - // construct with one or more individual parameters + // construct with one or more individual option parameters // Factory("var1", val1, "var2", val2, ...) template Factory(const std::string& key, T value, Args&&... moreArgs) : Factory() { setOpts(key, value, std::forward(moreArgs)...); } - // construct with options and one or more individual parameters + // construct with options and one or more individual option parameters // Factory(options, "var1", val1, "var2", val2, ...) template - Factory(Ptr options, Args&&... moreArgs) : Factory(options) { - setOpts(std::forward(moreArgs)...); + Factory(Ptr options, Args&&... args) : Factory(options) { + setOpts(std::forward(args)...); } virtual ~Factory() {} @@ -47,19 +47,13 @@ class Factory : public std::enable_shared_from_this { options_->set(key, value); } - // set multiple options + // set one or more options at once // setOpts("var1", val1, "var2", val2, ...); - template - void setOpts(const std::string& key, T value) { options_->set(key, value); } - template void setOpts(const std::string& key, T value, Args&&... moreArgs) { - setOpt(key, value); - setOpts(std::forward(moreArgs)...); // recursively set the remaining args + options_->set(key, value, std::forward(moreArgs)...); } - //void mergeOpts(const std::string& yaml) { options_->parse(yaml); } - void mergeOpts(Ptr options) { options_->merge(options); } template diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 70e29a0d3..ffd9e9d8b 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -433,9 +433,8 @@ namespace marian { "dimEmb", opt("dim-emb"), "dropout", dropout_, "prefix", (opt("tied-embeddings-src") || opt("tied-embeddings-all")) ? "Wemb" : prefix_ + "_Wemb", + "fixed", embeddingFix_, "vocab", opt>("vocabs")[batchIndex_]); // for factored embeddings - if(options_->has(embeddingFixParamName_)) - options->set("fixed", opt(embeddingFixParamName_)); if(options_->hasAndNotEmpty("embedding-vectors")) { auto embFiles = opt>("embedding-vectors"); options->set( @@ -459,8 +458,10 @@ namespace marian { "ulrKeysFile", opt("ulr-keys-vectors"))); } - // get embedding layer for this encoder or decoder; lazily create it if not created yet - Ptr EncoderDecoderLayerBase::getEmbeddingLayer(bool ulr) { + // get embedding layer for this encoder or decoder + // This is lazy mostly because the constructors of the consuming objects are not + // guaranteed presently to have access to their graph. + Ptr EncoderDecoderLayerBase::getEmbeddingLayer(bool ulr) const { if (embeddingLayers_.size() <= batchIndex_ || !embeddingLayers_[batchIndex_]) { // lazy if (embeddingLayers_.size() <= batchIndex_) embeddingLayers_.resize(batchIndex_ + 1); diff --git a/src/layers/generic.h b/src/layers/generic.h index ea34f9379..15195b683 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -67,20 +67,20 @@ struct IEmbeddingLayer { class EncoderDecoderLayerBase : public LayerBase { protected: const std::string prefix_; - const std::string embeddingFixParamName_; // "embedding-fix-src" or "embedding-fix-trg" - const bool inference_; + const bool embeddingFix_; const float dropout_; + const bool inference_; const size_t batchIndex_; - std::vector> embeddingLayers_; // (lazily created) + mutable std::vector> embeddingLayers_; // (lazily created) EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, - const std::string& dropoutParamName, - const std::string& embeddingFixParamName) : + float dropout, + bool embeddingFix) : LayerBase(/*graph=*/nullptr, options), // @BUGBUG: we really should pass the graph in here prefix_(options->get("prefix", prefix)), - embeddingFixParamName_(embeddingFixParamName), + embeddingFix_(embeddingFix), + dropout_(dropout), inference_(options->get("inference", false)), - dropout_(inference_ ? 0 : opt(dropoutParamName)), batchIndex_(options->get("index", batchIndex)) {} virtual ~EncoderDecoderLayerBase() {} @@ -91,7 +91,7 @@ class EncoderDecoderLayerBase : public LayerBase { public: // get embedding layer; lazily create on first call - Ptr getEmbeddingLayer(bool ulr = false); + Ptr getEmbeddingLayer(bool ulr = false) const; }; class FactoredVocab; diff --git a/src/models/decoder.h b/src/models/decoder.h index 2a89fadde..65c856d2f 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -16,8 +16,8 @@ class DecoderBase : public EncoderDecoderLayerBase { public: DecoderBase(Ptr options) : EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options, - /*dropoutParamName=*/"dropout-trg", - /*embeddingFixParamName=*/"embedding-fix-trg") {} + options->get("dropout-trg", 0.0f), + options->get("embedding-fix-trg", false)) {} virtual Ptr startState(Ptr, Ptr batch, diff --git a/src/models/encoder.h b/src/models/encoder.h index acff064f9..baa90b460 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -9,8 +9,8 @@ class EncoderBase : public EncoderDecoderLayerBase { public: EncoderBase(Ptr options) : EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, - /*dropoutParamName=*/"dropout-src", - /*embeddingFixParamName=*/"embedding-fix-src") {} + options->get("dropout-src", 0.0f), + options->get("embedding-fix-src", false)) {} // @TODO: turn into an interface. Also see if we can get rid of the graph parameter. virtual Ptr build(Ptr, Ptr) = 0; diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index b0cd02e0b..3e50d4238 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -135,10 +135,10 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr opti dimVocabs.resize(idx + 1); std::fill(dimVocabs.begin(), dimVocabs.end(), vocab); - return models::encoder_decoder()(options) - ("usage", use) - ("type", "s2s") - ("original-type", type) + return models::encoder_decoder(options->with( + "usage", use, + "type", "s2s", + "original-type", type)) .push_back(models::decoder() ("index", idx) ("dim-vocabs", dimVocabs)) diff --git a/src/models/model_factory.h b/src/models/model_factory.h old mode 100644 new mode 100755 index 8f3f07abe..1dca104d5 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -10,8 +10,9 @@ namespace marian { namespace models { class EncoderFactory : public Factory { + using Factory::Factory; public: - EncoderFactory(Ptr graph = nullptr) : Factory() {} + //EncoderFactory(Ptr graph = nullptr) : Factory() {} virtual Ptr construct(Ptr graph); }; @@ -19,8 +20,9 @@ class EncoderFactory : public Factory { typedef Accumulator encoder; class DecoderFactory : public Factory { + using Factory::Factory; public: - DecoderFactory(Ptr graph = nullptr) : Factory() {} + //DecoderFactory(Ptr graph = nullptr) : Factory() {} virtual Ptr construct(Ptr graph); }; @@ -28,9 +30,10 @@ class DecoderFactory : public Factory { typedef Accumulator decoder; class ClassifierFactory : public Factory { + using Factory::Factory; public: - ClassifierFactory(Ptr graph = nullptr) - : Factory() {} + //ClassifierFactory(Ptr graph = nullptr) + // : Factory() {} virtual Ptr construct(Ptr graph); }; @@ -38,13 +41,14 @@ class ClassifierFactory : public Factory { typedef Accumulator classifier; class EncoderDecoderFactory : public Factory { + using Factory::Factory; private: std::vector encoders_; std::vector decoders_; public: - EncoderDecoderFactory(Ptr graph = nullptr) - : Factory() {} + //EncoderDecoderFactory(Ptr graph = nullptr) + // : Factory() {} Accumulator push_back(encoder enc) { encoders_.push_back(enc); @@ -62,13 +66,14 @@ class EncoderDecoderFactory : public Factory { typedef Accumulator encoder_decoder; class EncoderClassifierFactory : public Factory { + using Factory::Factory; private: std::vector encoders_; std::vector classifiers_; public: - EncoderClassifierFactory(Ptr graph = nullptr) - : Factory() {} + //EncoderClassifierFactory(Ptr graph = nullptr) + // : Factory() {} Accumulator push_back(encoder enc) { encoders_.push_back(enc); diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index c8ddbb61c..6bac01865 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -8,9 +8,10 @@ namespace marian { namespace rnn { struct StackableFactory : public Factory { - StackableFactory() : Factory() {} - StackableFactory(const StackableFactory&) = default; - StackableFactory(StackableFactory&&) = default; + using Factory::Factory; + //StackableFactory() : Factory() {} + //StackableFactory(const StackableFactory&) = default; + //StackableFactory(StackableFactory&&) = default; virtual ~StackableFactory() {} From dbe09c537247e59f5780e5bd20dcb843c483cb3e Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 18:21:46 -0700 Subject: [PATCH 480/838] more examples of constructing a transformer directly --- src/layers/factory.h | 19 +++++++++---------- src/models/model_factory.cpp | 18 +++++++++++++----- src/models/model_factory.h | 13 ------------- src/rnn/constructors.h | 34 ++++++++++++++++------------------ 4 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/layers/factory.h b/src/layers/factory.h index bb493f8a0..1f00210fe 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -27,6 +27,7 @@ class Factory : public std::enable_shared_from_this { Factory(Ptr options, Args&&... args) : Factory(options) { setOpts(std::forward(args)...); } + Factory(const Factory& factory) = default; virtual ~Factory() {} @@ -56,22 +57,20 @@ class Factory : public std::enable_shared_from_this { void mergeOpts(Ptr options) { options_->merge(options); } - template - inline Ptr as() { - return std::dynamic_pointer_cast(shared_from_this()); - } + template + inline Ptr as() { return std::dynamic_pointer_cast(shared_from_this()); } - template - inline bool is() { - return as() != nullptr; - } + // @TODO: this fails with 'target type must be a pointer or reference to a defined class' + //template + //inline bool is() { return dynamic_cast(this) != nullptr; } + template + inline bool is() { return std::dynamic_pointer_cast(shared_from_this()) != nullptr; } }; // simplest form of Factory that just passes on options to the constructor of a layer type template struct ConstructingFactory : public Factory { - template - ConstructingFactory(Args&&... moreArgs) : Factory(std::forward(moreArgs)...) {} + using Factory::Factory; Ptr construct(Ptr graph) { return New(graph, options_); diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index 3e50d4238..ce8fbf1fd 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -103,20 +103,28 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr opti Ptr graph = nullptr; // graph unknown at this stage // clang-format off if(type == "s2s" || type == "amun" || type == "nematus") { - return models::encoder_decoder()(options) - ("usage", use) - ("original-type", type) + return models::encoder_decoder(options->with( + "usage", use, + "original-type", type)) .push_back(models::encoder()("type", "s2s")) .push_back(models::decoder()("type", "s2s")) .construct(graph); } else if(type == "transformer") { - return models::encoder_decoder()(options) - ("usage", use) +#if 1 + auto newOptions = options->with("usage", use); + auto res = New(/*graph,*/ newOptions); // @TODO: put EncoderDecoder on top of LayerBase + res->push_back(models::encoder(newOptions->with("type", "transformer")).construct(graph)); + res->push_back(models::decoder(newOptions->with("type", "transformer")).construct(graph)); + return res; +#else + return models::encoder_decoder(options->with( + "usage", use)) .push_back(models::encoder()("type", "transformer")) .push_back(models::decoder()("type", "transformer")) .construct(graph); +#endif } else if(type == "transformer_s2s") { diff --git a/src/models/model_factory.h b/src/models/model_factory.h index 1dca104d5..1840df8f5 100755 --- a/src/models/model_factory.h +++ b/src/models/model_factory.h @@ -12,8 +12,6 @@ namespace models { class EncoderFactory : public Factory { using Factory::Factory; public: - //EncoderFactory(Ptr graph = nullptr) : Factory() {} - virtual Ptr construct(Ptr graph); }; @@ -22,8 +20,6 @@ typedef Accumulator encoder; class DecoderFactory : public Factory { using Factory::Factory; public: - //DecoderFactory(Ptr graph = nullptr) : Factory() {} - virtual Ptr construct(Ptr graph); }; @@ -32,9 +28,6 @@ typedef Accumulator decoder; class ClassifierFactory : public Factory { using Factory::Factory; public: - //ClassifierFactory(Ptr graph = nullptr) - // : Factory() {} - virtual Ptr construct(Ptr graph); }; @@ -47,9 +40,6 @@ class EncoderDecoderFactory : public Factory { std::vector decoders_; public: - //EncoderDecoderFactory(Ptr graph = nullptr) - // : Factory() {} - Accumulator push_back(encoder enc) { encoders_.push_back(enc); return Accumulator(*this); @@ -72,9 +62,6 @@ class EncoderClassifierFactory : public Factory { std::vector classifiers_; public: - //EncoderClassifierFactory(Ptr graph = nullptr) - // : Factory() {} - Accumulator push_back(encoder enc) { encoders_.push_back(enc); return Accumulator(*this); diff --git a/src/rnn/constructors.h b/src/rnn/constructors.h index 6bac01865..beb1fce11 100755 --- a/src/rnn/constructors.h +++ b/src/rnn/constructors.h @@ -7,24 +7,22 @@ namespace marian { namespace rnn { -struct StackableFactory : public Factory { - using Factory::Factory; - //StackableFactory() : Factory() {} - //StackableFactory(const StackableFactory&) = default; - //StackableFactory(StackableFactory&&) = default; - - virtual ~StackableFactory() {} - - template - inline Ptr as() { - return std::dynamic_pointer_cast(shared_from_this()); - } - - template - inline bool is() { - return as() != nullptr; - } -}; +typedef Factory StackableFactory; +//struct StackableFactory : public Factory {StackableFactory +// using Factory::Factory; +// +// virtual ~StackableFactory() {} +// +// template +// inline Ptr as() { +// return std::dynamic_pointer_cast(shared_from_this()); +// } +// +// template +// inline bool is() { +// return as() != nullptr; +// } +//}; struct InputFactory : public StackableFactory { virtual Ptr construct(Ptr graph) = 0; From d63da3db88336417de9f6e33ca4e8dca102ef278 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 18:26:48 -0700 Subject: [PATCH 481/838] further changed construction example of 'transformer' to direct creation of layer objects --- src/models/model_factory.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index ce8fbf1fd..e2d192a05 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -114,9 +114,11 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr opti else if(type == "transformer") { #if 1 auto newOptions = options->with("usage", use); - auto res = New(/*graph,*/ newOptions); // @TODO: put EncoderDecoder on top of LayerBase - res->push_back(models::encoder(newOptions->with("type", "transformer")).construct(graph)); - res->push_back(models::decoder(newOptions->with("type", "transformer")).construct(graph)); + auto res = New(/*graph,*/ newOptions); + res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); + res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); + //res->push_back(models::encoder(newOptions->with("type", "transformer")).construct(graph)); + //res->push_back(models::decoder(newOptions->with("type", "transformer")).construct(graph)); return res; #else return models::encoder_decoder(options->with( From 6d3736999a9f8e7cef63691f18594497ac9ae29f Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 18:31:39 -0700 Subject: [PATCH 482/838] renamed EncoderDecoderBase to IEncoderDecoder --- src/models/costs.h | 10 +++++----- src/models/encoder_decoder.h | 4 ++-- src/rescorer/rescorer.h | 2 +- src/translator/scorers.h | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/rescorer/rescorer.h mode change 100644 => 100755 src/translator/scorers.h diff --git a/src/models/costs.h b/src/models/costs.h index c47092264..6e0325fe1 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -234,16 +234,16 @@ class GumbelSoftmaxStep : public ILogProbStep { } }; -// class to wrap an EncoderDecoderBase and a ILogProbStep that are executed in sequence, -// wrapped again in the EncoderDecoderBase interface +// class to wrap an IEncoderDecoder and a ILogProbStep that are executed in sequence, +// wrapped again in the IEncoderDecoder interface // @TODO: seems we are conflating an interface defition with its implementation? -class Stepwise : public EncoderDecoderBase { +class Stepwise : public IEncoderDecoder { protected: - Ptr encdec_; + Ptr encdec_; Ptr cost_; public: - Stepwise(Ptr encdec, Ptr cost) + Stepwise(Ptr encdec, Ptr cost) : encdec_(encdec), cost_(cost) {} virtual void load(Ptr graph, diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index f09e4f2c3..b55649298 100755 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -9,7 +9,7 @@ namespace marian { -class EncoderDecoderBase : public models::IModel { +class IEncoderDecoder : public models::IModel { public: virtual void load(Ptr graph, const std::string& name, @@ -58,7 +58,7 @@ class EncoderDecoderBase : public models::IModel { virtual data::SoftAlignment getAlignment() = 0; }; -class EncoderDecoder : public EncoderDecoderBase { +class EncoderDecoder : public IEncoderDecoder { protected: Ptr options_; Ptr shortlistGenerator_; diff --git a/src/rescorer/rescorer.h b/src/rescorer/rescorer.h old mode 100644 new mode 100755 index 8925391bd..e4d01ec55 --- a/src/rescorer/rescorer.h +++ b/src/rescorer/rescorer.h @@ -35,7 +35,7 @@ class Rescorer { data::SoftAlignment getAlignment() { auto model = std::static_pointer_cast(builder_)->getModel(); - return std::static_pointer_cast(model)->getAlignment(); + return std::static_pointer_cast(model)->getAlignment(); } }; diff --git a/src/translator/scorers.h b/src/translator/scorers.h old mode 100644 new mode 100755 index 8da8abb0c..33bb23abd --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -62,10 +62,10 @@ class ScorerWrapperState : public ScorerState { } }; -// class to wrap EncoderDecoderBase in a Scorer interface +// class to wrap IEncoderDecoder in a Scorer interface class ScorerWrapper : public Scorer { private: - Ptr encdec_; + Ptr encdec_; std::string fname_; const void* ptr_; @@ -75,7 +75,7 @@ class ScorerWrapper : public Scorer { float weight, const std::string& fname) : Scorer(name, weight), - encdec_(std::static_pointer_cast(encdec)), + encdec_(std::static_pointer_cast(encdec)), fname_(fname), ptr_{0} {} @@ -84,7 +84,7 @@ class ScorerWrapper : public Scorer { float weight, const void* ptr) : Scorer(name, weight), - encdec_(std::static_pointer_cast(encdec)), + encdec_(std::static_pointer_cast(encdec)), ptr_{ptr} {} virtual void init(Ptr graph) override { From 40ce6007812bd25673af3d4d45c0a0c53eb738e2 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 18:44:59 -0700 Subject: [PATCH 483/838] EncoderDecoder now derives from LayerBase --- src/models/amun.h | 2 +- src/models/encoder_decoder.cpp | 4 ++-- src/models/encoder_decoder.h | 5 ++--- src/models/model_factory.cpp | 26 ++++++++++++-------------- src/models/nematus.h | 3 +-- 5 files changed, 18 insertions(+), 22 deletions(-) mode change 100644 => 100755 src/models/amun.h mode change 100644 => 100755 src/models/encoder_decoder.cpp mode change 100644 => 100755 src/models/nematus.h diff --git a/src/models/amun.h b/src/models/amun.h old mode 100644 new mode 100755 index 35b652068..6784587c8 --- a/src/models/amun.h +++ b/src/models/amun.h @@ -8,7 +8,7 @@ namespace marian { class Amun : public EncoderDecoder { public: - Amun(Ptr options) : EncoderDecoder(options) { + Amun(Ptr graph, Ptr options) : EncoderDecoder(graph, options) { ABORT_IF(opt("enc-depth") > 1, "--type amun does not currently support multiple encoder " "layers, use --type s2s"); diff --git a/src/models/encoder_decoder.cpp b/src/models/encoder_decoder.cpp old mode 100644 new mode 100755 index 862355ee2..2ac6ff6d9 --- a/src/models/encoder_decoder.cpp +++ b/src/models/encoder_decoder.cpp @@ -4,8 +4,8 @@ namespace marian { -EncoderDecoder::EncoderDecoder(Ptr options) - : options_(options), +EncoderDecoder::EncoderDecoder(Ptr graph, Ptr options) + : LayerBase(graph, options), prefix_(options->get("prefix", "")), inference_(options->get("inference", false)) { diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index b55649298..6ed069c68 100755 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -58,9 +58,8 @@ class IEncoderDecoder : public models::IModel { virtual data::SoftAlignment getAlignment() = 0; }; -class EncoderDecoder : public IEncoderDecoder { +class EncoderDecoder : public IEncoderDecoder, public LayerBase { protected: - Ptr options_; Ptr shortlistGenerator_; const std::string prefix_; @@ -79,7 +78,7 @@ class EncoderDecoder : public IEncoderDecoder { public: typedef data::Corpus dataset_type; - EncoderDecoder(Ptr options); + EncoderDecoder(Ptr graph, Ptr options); virtual Ptr getOptions() override { return options_; } diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index e2d192a05..f8f03077c 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -55,21 +55,20 @@ Ptr DecoderFactory::construct(Ptr graph) { Ptr ClassifierFactory::construct(Ptr /*graph*/) { if(options_->get("type") == "bert-masked-lm") return New(options_); - if(options_->get("type") == "bert-classifier") + else if(options_->get("type") == "bert-classifier") return New(options_); - ABORT("Unknown classifier type"); + else + ABORT("Unknown classifier type"); } Ptr EncoderDecoderFactory::construct(Ptr graph) { Ptr encdec; - if(options_->get("type") == "amun") - encdec = New(options_); - if(options_->get("type") == "nematus") - encdec = New(options_); - - if(!encdec) - encdec = New(options_); + encdec = New(graph, options_); + else if(options_->get("type") == "nematus") + encdec = New(graph, options_); + else + encdec = New(graph, options_); for(auto& ef : encoders_) encdec->push_back(ef(options_).construct(graph)); @@ -82,13 +81,12 @@ Ptr EncoderDecoderFactory::construct(Ptr graph) { Ptr EncoderClassifierFactory::construct(Ptr graph) { Ptr enccls; - if(options_->get("type") == "bert") { + if(options_->get("type") == "bert") enccls = New(options_); - } else if(options_->get("type") == "bert-classifier") { + else if(options_->get("type") == "bert-classifier") enccls = New(options_); - } else { + else enccls = New(options_); - } for(auto& ef : encoders_) enccls->push_back(ef(options_).construct(graph)); @@ -114,7 +112,7 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr opti else if(type == "transformer") { #if 1 auto newOptions = options->with("usage", use); - auto res = New(/*graph,*/ newOptions); + auto res = New(graph, newOptions); res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); //res->push_back(models::encoder(newOptions->with("type", "transformer")).construct(graph)); diff --git a/src/models/nematus.h b/src/models/nematus.h old mode 100644 new mode 100755 index 88e9854be..d8e094ead --- a/src/models/nematus.h +++ b/src/models/nematus.h @@ -8,8 +8,7 @@ namespace marian { class Nematus : public EncoderDecoder { public: - template - Nematus(Ptr options) : EncoderDecoder(options), nameMap_(createNameMap()) { + Nematus(Ptr graph, Ptr options) : EncoderDecoder(graph, options), nameMap_(createNameMap()) { ABORT_IF(options_->get("enc-type") != "bidirectional", "--type nematus does not currently support other encoder " "type than bidirectional, use --type s2s"); From 442205fd6e473fe601277f808f80c2e63b6fd1cd Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 15 May 2019 18:59:40 -0700 Subject: [PATCH 484/838] more layers are based on LayerBase and get constructed as (graph, optioons) --- src/layers/generic.h | 4 ++-- src/models/bert.h | 9 ++++++--- src/models/classifier.h | 7 ++++--- src/models/decoder.h | 4 ++-- src/models/encoder.h | 4 ++-- src/models/model_factory.cpp | 22 ++++++++++------------ src/models/s2s.h | 6 ++++-- src/models/transformer.h | 13 +++++++------ src/models/transformer_factory.h | 4 ++-- src/models/transformer_stub.cpp | 8 ++++---- 10 files changed, 43 insertions(+), 38 deletions(-) mode change 100644 => 100755 src/models/transformer_factory.h mode change 100644 => 100755 src/models/transformer_stub.cpp diff --git a/src/layers/generic.h b/src/layers/generic.h index 15195b683..14bf34e6c 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -73,10 +73,10 @@ class EncoderDecoderLayerBase : public LayerBase { const size_t batchIndex_; mutable std::vector> embeddingLayers_; // (lazily created) - EncoderDecoderLayerBase(const std::string& prefix, size_t batchIndex, Ptr options, + EncoderDecoderLayerBase(Ptr graph, Ptr options, const std::string& prefix, size_t batchIndex, float dropout, bool embeddingFix) : - LayerBase(/*graph=*/nullptr, options), // @BUGBUG: we really should pass the graph in here + LayerBase(graph, options), prefix_(options->get("prefix", prefix)), embeddingFix_(embeddingFix), dropout_(dropout), diff --git a/src/models/bert.h b/src/models/bert.h index 33f21fc82..682efa48b 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -218,8 +218,9 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { * Is there a way to maybe instead include a reference in here, instead of deriving from it? */ class BertEncoder : public EncoderTransformer { + using EncoderTransformer::EncoderTransformer; public: - BertEncoder(Ptr options) : EncoderTransformer(options) {} + //BertEncoder(Ptr options) : EncoderTransformer(options) {} Expr addSentenceEmbeddings(Expr embeddings, Ptr batch, @@ -269,8 +270,9 @@ class BertEncoder : public EncoderTransformer { * @TODO: This is in fact so generic that we might move it out of here as the typical classifier implementation */ class BertClassifier : public ClassifierBase { + using ClassifierBase::ClassifierBase; public: - BertClassifier(Ptr options) : ClassifierBase(options) {} + //BertClassifier(Ptr options) : ClassifierBase(options) {} Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); @@ -312,8 +314,9 @@ class BertClassifier : public ClassifierBase { * as this is self-generating its labels from the source. Labels are dynamically created as complements of the masking process. */ class BertMaskedLM : public ClassifierBase { + using ClassifierBase::ClassifierBase; public: - BertMaskedLM(Ptr options) : ClassifierBase(options) {} + //BertMaskedLM(Ptr options) : ClassifierBase(options) {} Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { Ptr bertBatch = std::dynamic_pointer_cast(batch); diff --git a/src/models/classifier.h b/src/models/classifier.h index 773299b4b..9faa907e1 100755 --- a/src/models/classifier.h +++ b/src/models/classifier.h @@ -11,7 +11,8 @@ namespace marian { * Simple base class for Classifiers to be used in EncoderClassifier framework * Currently only implementations are in bert.h */ -class ClassifierBase { +class ClassifierBase :public LayerBase { + using LayerBase::LayerBase; protected: Ptr options_; const std::string prefix_{"classifier"}; @@ -19,8 +20,8 @@ class ClassifierBase { const size_t batchIndex_{0}; public: - ClassifierBase(Ptr options) - : options_(options), + ClassifierBase(Ptr graph, Ptr options) + : LayerBase(graph, options), prefix_(options->get("prefix", "classifier")), inference_(options->get("inference", false)), batchIndex_(options->get("index", 1)) {} // assume that training input has batch index 0 and labels has 1 diff --git a/src/models/decoder.h b/src/models/decoder.h index 65c856d2f..a79ef3731 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -14,8 +14,8 @@ class DecoderBase : public EncoderDecoderLayerBase { Ptr shortlist_; public: - DecoderBase(Ptr options) : - EncoderDecoderLayerBase("decoder", /*batchIndex=*/1, options, + DecoderBase(Ptr graph, Ptr options) : + EncoderDecoderLayerBase(graph, options, "decoder", /*batchIndex=*/1, options->get("dropout-trg", 0.0f), options->get("embedding-fix-trg", false)) {} diff --git a/src/models/encoder.h b/src/models/encoder.h index baa90b460..61bb0d88b 100755 --- a/src/models/encoder.h +++ b/src/models/encoder.h @@ -7,8 +7,8 @@ namespace marian { class EncoderBase : public EncoderDecoderLayerBase { public: - EncoderBase(Ptr options) : - EncoderDecoderLayerBase("encoder", /*batchIndex=*/0, options, + EncoderBase(Ptr graph, Ptr options) : + EncoderDecoderLayerBase(graph, options, "encoder", /*batchIndex=*/0, options->get("dropout-src", 0.0f), options->get("embedding-fix-src", false)) {} diff --git a/src/models/model_factory.cpp b/src/models/model_factory.cpp index f8f03077c..098fc0a59 100755 --- a/src/models/model_factory.cpp +++ b/src/models/model_factory.cpp @@ -28,7 +28,7 @@ namespace models { Ptr EncoderFactory::construct(Ptr graph) { if(options_->get("type") == "s2s") - return New(options_); + return New(graph, options_); #ifdef CUDNN if(options_->get("type") == "char-s2s") @@ -36,27 +36,27 @@ Ptr EncoderFactory::construct(Ptr graph) { #endif if(options_->get("type") == "transformer") - return NewEncoderTransformer(options_); + return NewEncoderTransformer(graph, options_); if(options_->get("type") == "bert-encoder") - return New(options_); + return New(graph, options_); ABORT("Unknown encoder type"); } Ptr DecoderFactory::construct(Ptr graph) { if(options_->get("type") == "s2s") - return New(options_); + return New(graph, options_); if(options_->get("type") == "transformer") - return NewDecoderTransformer(options_); + return NewDecoderTransformer(graph, options_); ABORT("Unknown decoder type"); } -Ptr ClassifierFactory::construct(Ptr /*graph*/) { +Ptr ClassifierFactory::construct(Ptr graph) { if(options_->get("type") == "bert-masked-lm") - return New(options_); + return New(graph, options_); else if(options_->get("type") == "bert-classifier") - return New(options_); + return New(graph, options_); else ABORT("Unknown classifier type"); } @@ -113,10 +113,8 @@ Ptr createBaseModelByType(std::string type, usage use, Ptr opti #if 1 auto newOptions = options->with("usage", use); auto res = New(graph, newOptions); - res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); - res->push_back(New(/*graph,*/ newOptions->with("type", "transformer"))); - //res->push_back(models::encoder(newOptions->with("type", "transformer")).construct(graph)); - //res->push_back(models::decoder(newOptions->with("type", "transformer")).construct(graph)); + res->push_back(New(graph, newOptions->with("type", "transformer"))); + res->push_back(New(graph, newOptions->with("type", "transformer"))); return res; #else return models::encoder_decoder(options->with( diff --git a/src/models/s2s.h b/src/models/s2s.h index 081bf3e93..126580108 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -9,6 +9,7 @@ namespace marian { class EncoderS2S : public EncoderBase { + using EncoderBase::EncoderBase; public: Expr applyEncoderRNN(Ptr graph, Expr embeddings, @@ -119,7 +120,7 @@ class EncoderS2S : public EncoderBase { return context; } - EncoderS2S(Ptr options) : EncoderBase(options) {} + //EncoderS2S(Ptr options) : EncoderBase(options) {} virtual Ptr build(Ptr graph, Ptr batch) override { @@ -138,6 +139,7 @@ class EncoderS2S : public EncoderBase { }; class DecoderS2S : public DecoderBase { + using DecoderBase::DecoderBase; private: Ptr rnn_; Ptr output_; @@ -204,7 +206,7 @@ class DecoderS2S : public DecoderBase { } public: - DecoderS2S(Ptr options) : DecoderBase(options) {} + //DecoderS2S(Ptr options) : DecoderBase(options) {} virtual Ptr startState( Ptr graph, diff --git a/src/models/transformer.h b/src/models/transformer.h index 8b1f97900..dabb8fb93 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -22,6 +22,7 @@ namespace marian { template class Transformer : public EncoderOrDecoderBase { typedef EncoderOrDecoderBase Base; + using Base::Base; protected: using Base::options_; using Base::inference_; using Base::batchIndex_; using Base::graph_; @@ -38,10 +39,6 @@ class Transformer : public EncoderOrDecoderBase { template T opt(const std::string& key, const T& def) const { Ptr options = options_; if (options->has(key)) return options->get(key); else return def; } public: - Transformer(Ptr options) - : EncoderOrDecoderBase(options) { - } - static Expr transposeTimeBatch(Expr input) { return transpose(input, {0, 2, 1, 3}); } Expr addPositionalEmbeddings(Expr input, int start = 0, bool trainPosEmbeddings = false) const { @@ -497,8 +494,10 @@ class Transformer : public EncoderOrDecoderBase { }; class EncoderTransformer : public Transformer { + typedef Transformer Base; + using Base::Base; public: - EncoderTransformer(Ptr options) : Transformer(options) {} + //EncoderTransformer(Ptr options) : Transformer(options) {} virtual ~EncoderTransformer() {} virtual Ptr build(Ptr graph, @@ -577,6 +576,8 @@ class TransformerState : public DecoderState { }; class DecoderTransformer : public Transformer { + typedef Transformer Base; + using Base::Base; private: Ptr output_; @@ -602,7 +603,7 @@ class DecoderTransformer : public Transformer { } public: - DecoderTransformer(Ptr options) : Transformer(options) {} + //DecoderTransformer(Ptr graph, Ptr options) : Transformer(graph, options) {} virtual Ptr startState( Ptr graph, diff --git a/src/models/transformer_factory.h b/src/models/transformer_factory.h old mode 100644 new mode 100755 index 488c7f5bf..b282d819c --- a/src/models/transformer_factory.h +++ b/src/models/transformer_factory.h @@ -7,6 +7,6 @@ #include "models/encoder.h" namespace marian { -Ptr NewEncoderTransformer(Ptr options); -Ptr NewDecoderTransformer(Ptr options); +Ptr NewEncoderTransformer(Ptr graph, Ptr options); +Ptr NewDecoderTransformer(Ptr graph, Ptr options); } // namespace marian diff --git a/src/models/transformer_stub.cpp b/src/models/transformer_stub.cpp old mode 100644 new mode 100755 index 1e7b19f68..2e8d694fa --- a/src/models/transformer_stub.cpp +++ b/src/models/transformer_stub.cpp @@ -2,13 +2,13 @@ namespace marian { // factory functions -Ptr NewEncoderTransformer(Ptr options) +Ptr NewEncoderTransformer(Ptr graph, Ptr options) { - return New(options); + return New(graph, options); } -Ptr NewDecoderTransformer(Ptr options) +Ptr NewDecoderTransformer(Ptr graph, Ptr options) { - return New(options); + return New(graph, options); } } // namespace marian From e5af0f71c3812863636dcee836a0202314f0c1db Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 16 May 2019 09:30:28 -0700 Subject: [PATCH 485/838] bug fix: beam search must propagate alignments when expanding factors (2 lines change plus tens of comments added) --- src/data/alignment.h | 4 +++- src/microsoft/quicksand.cpp | 4 ++-- src/microsoft/quicksand.h | 2 +- src/models/costs.h | 2 +- src/models/decoder.h | 2 +- src/models/encoder_decoder.h | 12 +++++++----- src/models/transformer.h | 12 ++++++------ src/translator/beam_search.h | 32 ++++++++++++++++---------------- src/translator/hypothesis.h | 4 ++-- src/translator/scorers.h | 4 +++- 10 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/data/alignment.h b/src/data/alignment.h index 49fbde767..1c68bb39e 100644 --- a/src/data/alignment.h +++ b/src/data/alignment.h @@ -52,7 +52,9 @@ class WordAlignment { std::string toString() const; }; -typedef std::vector> SoftAlignment; +// soft alignment = P(src pos|trg pos) for each beam and batch index, stored in a flattened CPU-side array +// Also used on QuickSAND boundary where beam and batch size is 1. Then it is simply [t][s] -> P(s|t) +typedef std::vector> SoftAlignment; // [trg pos][beam depth * max src length * batch size] WordAlignment ConvertSoftAlignToHardAlign(SoftAlignment alignSoft, float threshold = 1.f); diff --git a/src/microsoft/quicksand.cpp b/src/microsoft/quicksand.cpp index 98ff2978c..440d0c116 100644 --- a/src/microsoft/quicksand.cpp +++ b/src/microsoft/quicksand.cpp @@ -169,8 +169,8 @@ class BeamSearchDecoder : public IBeamSearchDecoder { data::WordAlignment align = data::ConvertSoftAlignToHardAlign(hyp->tracebackAlignment(), alignmentThreshold); // convert to QuickSAND format alignmentSets.resize(words.size()); - for (const auto& p : align) // @TODO: Does the feature_model param max_alignment_links apply here? - alignmentSets[p.tgtPos].insert({p.srcPos, p.prob}); + for (const auto& p : align) + alignmentSets[p.tgtPos].insert({p.srcPos, p.prob}); // [trgPos] -> {(srcPos, P(srcPos|trgPos))} } // form hypothesis to return qsNbest.emplace_back(toWordIndexVector(words), std::move(alignmentSets), score); diff --git a/src/microsoft/quicksand.h b/src/microsoft/quicksand.h index bc31e1515..c4f539abd 100644 --- a/src/microsoft/quicksand.h +++ b/src/microsoft/quicksand.h @@ -19,7 +19,7 @@ typedef uint32_t IndexType; typedef IndexType WordIndex; typedef std::vector WordIndices; typedef std::vector QSBatch; -typedef std::vector>> AlignmentSets; // [tgtPos] -> set of (srcPos, score) +typedef std::vector>> AlignmentSets; // [trgPos] -> set of (srcPos, P(srcPos|trgPos)) typedef std::tuple QSSentenceWithProb; typedef std::vector QSNBest; diff --git a/src/models/costs.h b/src/models/costs.h index 7aa5a5f3e..6c54cde51 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -75,7 +75,7 @@ class EncoderDecoderCECost : public ICost { multiLoss->push_back(partialLoss); if(options_->get("guided-alignment", std::string("none")) != "none" && !inference_) { - auto attentionVectors = encdec->getDecoders()[0]->getAlignments(); + auto attentionVectors = encdec->getDecoders()[0]->getAlignments(); // [tgt index][beam depth, max src length, batch size, 1] ABORT_IF(attentionVectors.empty(), "Model does not seem to support alignments"); auto attention = concatenate(attentionVectors, /*axis =*/ -1); diff --git a/src/models/decoder.h b/src/models/decoder.h index dd5a508a2..984370e33 100755 --- a/src/models/decoder.h +++ b/src/models/decoder.h @@ -98,7 +98,7 @@ class DecoderBase { state->setTargetHistoryEmbeddings(selectedEmbs); } - virtual const std::vector getAlignments(int /*i*/ = 0) { return {}; }; + virtual const std::vector getAlignments(int /*i*/ = 0) { return {}; }; // [tgt index][beam depth, max src length, batch size, 1] virtual Ptr getShortlist() { return shortlist_; } virtual void setShortlist(Ptr shortlist) { diff --git a/src/models/encoder_decoder.h b/src/models/encoder_decoder.h index 41702eceb..4cfd17712 100644 --- a/src/models/encoder_decoder.h +++ b/src/models/encoder_decoder.h @@ -130,13 +130,15 @@ class EncoderDecoder : public EncoderDecoderBase { return decoders_[0]->getShortlist(); }; + // convert alignment tensors that live GPU-side into a CPU-side vector of vectors virtual data::SoftAlignment getAlignment() override { - data::SoftAlignment aligns; - for(auto aln : decoders_[0]->getAlignments()) { - aligns.push_back({}); - aln->val()->get(aligns.back()); + data::SoftAlignment softAlignments; + auto alignments = decoders_[0]->getAlignments(); // [tgt index][beam depth, max src length, batch size, 1] + for(auto alignment : alignments) { // [beam depth, max src length, batch size, 1] + softAlignments.push_back({}); + alignment->val()->get(softAlignments.back()); } - return aligns; + return softAlignments; // [tgt index][beam depth * max src length * batch size] }; /*********************************************************************/ diff --git a/src/models/transformer.h b/src/models/transformer.h index f36150edb..b5c0ec092 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -206,20 +206,20 @@ class Transformer : public EncoderOrDecoderBase { auto head0 = slice(weights, -3, 0); int dimBatchBeam = head0->shape()[-4]; - int srcWords = head0->shape()[-1]; - int trgWords = head0->shape()[-2]; + int srcWords = head0->shape()[-1]; // (max) length of src sequence + int trgWords = head0->shape()[-2]; // (max) length of trg sequence, or 1 in decoding int dimBatch = dimBatchBeam / dimBeam; // reshape and transpose to match the format guided_alignment expects head0 = reshape(head0, {dimBeam, dimBatch, trgWords, srcWords}); - head0 = transpose(head0, {0, 3, 1, 2}); // [-4: beam depth, -3: max src length, -2: batch size, -1: max tgt length] + head0 = transpose(head0, {0, 3, 1, 2}); // [beam depth, max src length, batch size, max tgt length] // save only last alignment set. For training this will be all alignments, // for translation only the last one. Also split alignments by target words. // @TODO: make splitting obsolete alignments_.clear(); - for(int i = 0; i < trgWords; ++i) { - alignments_.push_back(slice(head0, -1, i)); // [tgt index][-4: beam depth, -3: max src length, -2: batch size, -1: 1] + for(int i = 0; i < trgWords; ++i) { // loop over all trg positions. In decoding, there is only one. + alignments_.push_back(slice(head0, -1, i)); // [tgt index][beam depth, max src length, batch size, 1] P(src pos|trg pos, beam index, batch index) } } @@ -860,7 +860,7 @@ class DecoderTransformer : public Transformer { // helper function for guided alignment // @TODO: const vector<> seems wrong. Either make it non-const or a const& (more efficient but dangerous) virtual const std::vector getAlignments(int /*i*/ = 0) override { - return alignments_; + return alignments_; // [tgt index][beam depth, max src length, batch size, 1] } void clear() override { diff --git a/src/translator/beam_search.h b/src/translator/beam_search.h index 8016e76f7..393ff6dfe 100644 --- a/src/translator/beam_search.h +++ b/src/translator/beam_search.h @@ -44,8 +44,8 @@ class BeamSearch { Ptr batch, // for alignments only Ptr factoredVocab, size_t factorGroup) const { std::vector align; - if(options_->hasAndNotEmpty("alignment")) - align = scorers_[0]->getAlignment(); // [beam depth, max src length, batch size, 1]; use alignments from the first scorer, even if ensemble + if(options_->hasAndNotEmpty("alignment") && factorGroup == 0) + align = scorers_[0]->getAlignment(); // [beam depth * max src length * batch size] -> P(s|t); use alignments from the first scorer, even if ensemble const auto dimBatch = beams.size(); Beams newBeams(dimBatch); // return value of this function goes here @@ -73,8 +73,8 @@ class BeamSearch { ABORT_IF(beamHypIdx >= beam.size(), "Out of bounds beamHypIdx??"); // map wordIdx to word - auto prevHyp = beam[beamHypIdx]; - auto prevBeamHypIdx = beamHypIdx; + auto prevBeamHypIdx = beamHypIdx; // back pointer + auto prevHyp = beam[prevBeamHypIdx]; Word word; // If short list has been set, then wordIdx is an index into the short-listed word set, // rather than the true word index. @@ -94,7 +94,7 @@ class BeamSearch { "A word without this factor snuck through to here??"); word = factoredVocab->expandFactoredWord(word, factorGroup, wordIdx); prevBeamHypIdx = prevHyp->getPrevStateIndex(); - prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback doesnot contain partially factored words + prevHyp = prevHyp->getPrevHyp(); // short-circuit the backpointer, so that the traceback does not contain partially factored words } } else if (shortlist) @@ -123,9 +123,10 @@ class BeamSearch { } // Set alignments - if(!align.empty() && factorGroup == 0) { + if(!align.empty()) hyp->setAlignment(getAlignmentsForHypothesis(align, batch, (int)beamHypIdx, (int)batchIdx)); - } + else // not first factor: just copy + hyp->setAlignment(beam[beamHypIdx]->getAlignment()); newBeam.push_back(hyp); } @@ -157,8 +158,8 @@ class BeamSearch { return newBeams; } - std::vector getAlignmentsForHypothesis( - const std::vector alignAll, // [beam depth, max src length, batch size, 1] + std::vector getAlignmentsForHypothesis( // -> P(s|t) for current t and given beam and batch dim + const std::vector alignAll, // [beam depth, max src length, batch size, 1], flattened Ptr batch, int beamHypIdx, int batchIdx) const { @@ -179,19 +180,18 @@ class BeamSearch { // in a single beam, i.e.: // * [word1-batch1, word1-batch2, ..., word2-batch1, ...] // - size_t batchSize = batch->size(); // number of sentences in batch + size_t batchSize = batch->size(); // number of sentences in batch size_t batchWidth = batch->width(); // max src length - size_t batchWidthXSize = batchWidth * batchSize; // total number of words in the batch incl. padding - std::vector align; + size_t batchWidthXSize = batchWidth * batchSize; // total number of words in the batch incl. padding = product of last 3 tensor dimensions // loop over words of batch entry 'batchIdx' and beam entry 'beamHypIdx' - for(size_t w = 0; w < batchWidth; ++w) { - size_t a = ((batchWidthXSize * beamHypIdx) + batchIdx) + (batchSize * w); - size_t m = a % batchWidthXSize; // == batchIdx + (batchSize * w) + std::vector align; + for(size_t srcPos = 0; srcPos < batchWidth; ++srcPos) { // loop over source positions + size_t a = ((batchWidthXSize * beamHypIdx) + batchIdx) + (batchSize * srcPos); // = flatten [beam index, s, batch index, 0] + size_t m = a % batchWidthXSize; // == batchIdx + (batchSize * srcPos) = flatten [0, s, batch index, 0] if(batch->front()->mask()[m] != 0) align.emplace_back(alignAll[a]); } - return align; } diff --git a/src/translator/hypothesis.h b/src/translator/hypothesis.h index cf5d9b601..c1d570313 100644 --- a/src/translator/hypothesis.h +++ b/src/translator/hypothesis.h @@ -47,7 +47,7 @@ class Hypothesis { return targetWords; } - // get soft alignments for each target word starting from the hyp one + // get soft alignments [t][s] -> P(s|t) for each target word starting from the hyp one typedef data::SoftAlignment SoftAlignment; SoftAlignment tracebackAlignment() { @@ -56,7 +56,7 @@ class Hypothesis { align.push_back(hyp->getAlignment()); } std::reverse(align.begin(), align.end()); - return align; + return align; // [t][s] -> P(s|t) } private: diff --git a/src/translator/scorers.h b/src/translator/scorers.h index 8da8abb0c..97cc21bc0 100644 --- a/src/translator/scorers.h +++ b/src/translator/scorers.h @@ -128,7 +128,9 @@ class ScorerWrapper : public Scorer { }; virtual std::vector getAlignment() override { - return encdec_->getAlignment().front(); // [beam depth, max src length, batch size, 1] + // This is called during decoding, where alignments only exist for the last time step. Hence front(). + // This makes as copy. @TODO: It should be OK to return this as a const&. + return encdec_->getAlignment().front(); // [beam depth * max src length * batch size] } }; From cfb1be65319da68c4f3d9bd33dad1c50bd2e52d8 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 16 May 2019 18:14:24 -0700 Subject: [PATCH 486/838] bug fix: in case of truncation of input, it should append EOS not ZERO --- src/data/corpus_base.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) mode change 100644 => 100755 src/data/corpus_base.cpp diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp old mode 100644 new mode 100755 index 8391fdc1f..d901f035e --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -200,12 +200,11 @@ void CorpusBase::addWordsToSentenceTuple(const std::string& line, // is used. Words words = vocabs_[batchIndex]->encode(line, /*addEOS =*/ addEOS_[batchIndex], inference_); - if(words.empty()) - words.push_back(Word::ZERO); + ABORT_IF(words.empty(), "Empty input sequences are presently untested"); if(maxLengthCrop_ && words.size() > maxLength_) { words.resize(maxLength_); - words.back() = Word::ZERO; // @TODO: What does this do? Meant as sent-end token?? + words.back() = vocabs_[batchIndex]->getEosId(); } if(rightLeft_) From eb2e451152efdc57a120707125b75461c4f59445 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 16 May 2019 18:14:24 -0700 Subject: [PATCH 487/838] bug fix: in case of truncation of input, it should append EOS not ZERO --- src/data/corpus_base.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) mode change 100644 => 100755 src/data/corpus_base.cpp diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp old mode 100644 new mode 100755 index 8391fdc1f..d901f035e --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -200,12 +200,11 @@ void CorpusBase::addWordsToSentenceTuple(const std::string& line, // is used. Words words = vocabs_[batchIndex]->encode(line, /*addEOS =*/ addEOS_[batchIndex], inference_); - if(words.empty()) - words.push_back(Word::ZERO); + ABORT_IF(words.empty(), "Empty input sequences are presently untested"); if(maxLengthCrop_ && words.size() > maxLength_) { words.resize(maxLength_); - words.back() = Word::ZERO; // @TODO: What does this do? Meant as sent-end token?? + words.back() = vocabs_[batchIndex]->getEosId(); } if(rightLeft_) From 4b984af4d9eca69a91bf294543695252f20f6316 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Thu, 16 May 2019 18:19:23 -0700 Subject: [PATCH 488/838] bug fix: in case of truncation, EOS should only be added if addEOS_ --- src/data/corpus_base.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/data/corpus_base.cpp b/src/data/corpus_base.cpp index d901f035e..03633e828 100755 --- a/src/data/corpus_base.cpp +++ b/src/data/corpus_base.cpp @@ -204,7 +204,8 @@ void CorpusBase::addWordsToSentenceTuple(const std::string& line, if(maxLengthCrop_ && words.size() > maxLength_) { words.resize(maxLength_); - words.back() = vocabs_[batchIndex]->getEosId(); + if(addEOS_[batchIndex]) + words.back() = vocabs_[batchIndex]->getEosId(); } if(rightLeft_) From 7788d89651e27a4457f19d7a62548c58e29047de Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 21 May 2019 13:28:33 -0700 Subject: [PATCH 489/838] minor further refactoring; deleted some commented-out lines --- src/layers/factory.h | 8 ++------ src/layers/generic.cpp | 14 +++++++------- src/models/bert.h | 6 ------ src/models/costs.h | 1 + src/models/s2s.h | 2 -- src/models/transformer.h | 3 --- 6 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/layers/factory.h b/src/layers/factory.h index 1f00210fe..bcb1db0c4 100755 --- a/src/layers/factory.h +++ b/src/layers/factory.h @@ -44,16 +44,12 @@ class Factory : public std::enable_shared_from_this { // set a single option // setOpt("var", val); template - void setOpt(const std::string& key, T value) { - options_->set(key, value); - } + void setOpt(const std::string& key, T value) { options_->set(key, value); } // set one or more options at once // setOpts("var1", val1, "var2", val2, ...); template - void setOpts(const std::string& key, T value, Args&&... moreArgs) { - options_->set(key, value, std::forward(moreArgs)...); - } + void setOpts(const std::string& key, T value, Args&&... moreArgs) { options_->set(key, value, std::forward(moreArgs)...); } void mergeOpts(Ptr options) { options_->merge(options); } diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index abb2f69f8..9797667b8 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -418,14 +418,14 @@ namespace marian { } Expr Embedding::apply(const Words& words, const Shape& shape) const /*override final*/ { - Expr selectedEmbs; - if (factoredVocab_) - selectedEmbs = multiRows(words); + if (factoredVocab_) { + Expr selectedEmbs = multiRows(words); + selectedEmbs = reshape(selectedEmbs, shape); + selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), { selectedEmbs->shape()[-3], 1, 1 }); + return selectedEmbs; + } else - selectedEmbs = rows(E_, toWordIndexVector(words)); - selectedEmbs = reshape(selectedEmbs, shape); - selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), {selectedEmbs->shape()[-3], 1, 1}); - return selectedEmbs; + return applyIndices(toWordIndexVector(words), shape); } Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { diff --git a/src/models/bert.h b/src/models/bert.h index 682efa48b..80e95df07 100755 --- a/src/models/bert.h +++ b/src/models/bert.h @@ -220,8 +220,6 @@ class BertEncoderClassifier : public EncoderClassifier, public data::RNGEngine { class BertEncoder : public EncoderTransformer { using EncoderTransformer::EncoderTransformer; public: - //BertEncoder(Ptr options) : EncoderTransformer(options) {} - Expr addSentenceEmbeddings(Expr embeddings, Ptr batch, bool learnedPosEmbeddings) const { @@ -272,8 +270,6 @@ class BertEncoder : public EncoderTransformer { class BertClassifier : public ClassifierBase { using ClassifierBase::ClassifierBase; public: - //BertClassifier(Ptr options) : ClassifierBase(options) {} - Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { ABORT_IF(encoderStates.size() != 1, "Currently we only support a single encoder BERT model"); @@ -316,8 +312,6 @@ class BertClassifier : public ClassifierBase { class BertMaskedLM : public ClassifierBase { using ClassifierBase::ClassifierBase; public: - //BertMaskedLM(Ptr options) : ClassifierBase(options) {} - Ptr apply(Ptr graph, Ptr batch, const std::vector>& encoderStates) override { Ptr bertBatch = std::dynamic_pointer_cast(batch); diff --git a/src/models/costs.h b/src/models/costs.h index 17c3d0d43..8b23df905 100755 --- a/src/models/costs.h +++ b/src/models/costs.h @@ -237,6 +237,7 @@ class GumbelSoftmaxStep : public ILogProbStep { // class to wrap an IEncoderDecoder and a ILogProbStep that are executed in sequence, // wrapped again in the IEncoderDecoder interface // @TODO: seems we are conflating an interface defition with its implementation? +// @TODO: needs a better name. Stepwise is an adjective. Classes are things=nouns. StepwiseWhat? class Stepwise : public IEncoderDecoder { protected: Ptr encdec_; diff --git a/src/models/s2s.h b/src/models/s2s.h index 126580108..f10c77bea 100755 --- a/src/models/s2s.h +++ b/src/models/s2s.h @@ -206,8 +206,6 @@ class DecoderS2S : public DecoderBase { } public: - //DecoderS2S(Ptr options) : DecoderBase(options) {} - virtual Ptr startState( Ptr graph, Ptr batch, diff --git a/src/models/transformer.h b/src/models/transformer.h index 42713b521..4f0e8ea88 100755 --- a/src/models/transformer.h +++ b/src/models/transformer.h @@ -497,9 +497,6 @@ class EncoderTransformer : public Transformer { typedef Transformer Base; using Base::Base; public: - //EncoderTransformer(Ptr options) : Transformer(options) {} - virtual ~EncoderTransformer() {} - virtual Ptr build(Ptr graph, Ptr batch) override { graph_ = graph; From 31babb0fde09652fcf44970dd00c7ff213c284a7 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 21 May 2019 14:58:48 -0700 Subject: [PATCH 490/838] implemented dropout for factored embeddings --- src/layers/generic.cpp | 17 ++++++++++------- src/layers/generic.h | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/layers/generic.cpp b/src/layers/generic.cpp index 9797667b8..30060432f 100755 --- a/src/layers/generic.cpp +++ b/src/layers/generic.cpp @@ -361,7 +361,7 @@ namespace marian { } // helper to embed a sequence of words (given as indices) via factored embeddings - /*private*/ Expr Embedding::multiRows(const Words& data) const + /*private*/ Expr Embedding::multiRows(const Words& data, float dropProb) const { auto graph = E_->graph(); auto factoredData = factoredVocab_->csr_rows(data); @@ -372,7 +372,9 @@ namespace marian { auto weights = graph->constant({ (int)factoredData.weights.size() }, inits::from_vector(factoredData.weights), Type::float32); auto indices = graph->constant({ (int)factoredData.indices.size() }, inits::from_vector(factoredData.indices), Type::uint32); auto offsets = graph->constant({ (int)factoredData.offsets.size() }, inits::from_vector(factoredData.offsets), Type::uint32); - // apply dropout --@TODO + // apply dropout + // We apply it to the weights, i.e. factors get dropped out separately, but always as entire vectors. + weights = dropout(weights, dropProb); // perform the product return csr_dot(factoredData.shape, weights, indices, offsets, E_); } @@ -419,9 +421,9 @@ namespace marian { Expr Embedding::apply(const Words& words, const Shape& shape) const /*override final*/ { if (factoredVocab_) { - Expr selectedEmbs = multiRows(words); - selectedEmbs = reshape(selectedEmbs, shape); - selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), { selectedEmbs->shape()[-3], 1, 1 }); + Expr selectedEmbs = multiRows(words, options_->get("dropout", 0.0f)); // [(B*W) x E] + selectedEmbs = reshape(selectedEmbs, shape); // [W, B, E] + //selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), { selectedEmbs->shape()[-3], 1, 1 }); // @TODO: replace with factor dropout return selectedEmbs; } else @@ -430,8 +432,9 @@ namespace marian { Expr Embedding::applyIndices(const std::vector& embIdx, const Shape& shape) const /*override final*/ { ABORT_IF(factoredVocab_, "Embedding: applyIndices must not be used with a factored vocabulary"); - auto selectedEmbs = rows(E_, embIdx); - selectedEmbs = reshape(selectedEmbs, shape); + auto selectedEmbs = rows(E_, embIdx); // [(B*W) x E] + selectedEmbs = reshape(selectedEmbs, shape); // [W, B, E] + // @BUGBUG: We should not broadcast along dimBatch=[-2]. Then we can also dropout before reshape() (test that separately) selectedEmbs = dropout(selectedEmbs, options_->get("dropout", 0.0f), { selectedEmbs->shape()[-3], 1, 1 }); return selectedEmbs; } diff --git a/src/layers/generic.h b/src/layers/generic.h index e6d700ef8..741ebc88d 100755 --- a/src/layers/generic.h +++ b/src/layers/generic.h @@ -289,7 +289,7 @@ class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList { class Embedding : public LayerBase, public IEmbeddingLayer { Expr E_; Ptr factoredVocab_; - Expr multiRows(const Words& data) const; + Expr multiRows(const Words& data, float dropProb) const; public: Embedding(Ptr graph, Ptr options); From ec2d66e85275313c206f760364724108f9508204 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Mon, 27 May 2019 14:52:11 +0100 Subject: [PATCH 491/838] Add execute permission --- scripts/contrib/model_info.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/contrib/model_info.py diff --git a/scripts/contrib/model_info.py b/scripts/contrib/model_info.py old mode 100644 new mode 100755 From 865f8634854709b84bdd5ec8baab8acfc305bc3d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Fri, 31 May 2019 15:33:51 -0700 Subject: [PATCH 492/838] VS Project now builds with CUDA --- src/tensors/gpu/prod.cpp | 44 ++++----- src/tensors/gpu/tensor_operators.cu | 0 vs/Marian.vcxproj | 134 +++++++++++++++++----------- vs/Marian.vcxproj.filters | 104 +++++++++++---------- 4 files changed, 157 insertions(+), 125 deletions(-) mode change 100644 => 100755 src/tensors/gpu/tensor_operators.cu mode change 100644 => 100755 vs/Marian.vcxproj.filters diff --git a/src/tensors/gpu/prod.cpp b/src/tensors/gpu/prod.cpp index 48524f626..dfcb813f4 100755 --- a/src/tensors/gpu/prod.cpp +++ b/src/tensors/gpu/prod.cpp @@ -1,4 +1,8 @@ +#ifdef _MSC_VER +#pragma warning(disable: 4505) // warning C4505: '__float2half_rz': unreferenced local function has been removed (missing 'static inline') +#endif + #include #include @@ -45,7 +49,7 @@ void Prod(marian::Tensor C, bool transB, float beta, float scalar) { - cudaSetDevice(C->getDeviceId().no); + cudaSetDevice((int)C->getDeviceId().no); float alpha = scalar; size_t m = A->shape().elements() / A->shape().back(); @@ -79,17 +83,17 @@ void Prod(marian::Tensor C, CUBLAS_CHECK(cublasSgemm(cublasHandle, opB, opA, - n, - m, - k, + (int)n, + (int)m, + (int)k, &alpha, B->data(), - ldb, + (int)ldb, A->data(), - lda, + (int)lda, &beta, C->data(), - ldc)); + (int)ldc)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -144,7 +148,7 @@ void ProdBatched(marian::Tensor C, bool transB, float beta, float scalar) { - cudaSetDevice(C->getDeviceId().no); + cudaSetDevice((int)C->getDeviceId().no); float alpha = scalar; size_t batchA = A->shape().elements() / (A->shape()[-1] * A->shape()[-2]); @@ -173,10 +177,10 @@ void ProdBatched(marian::Tensor C, auto cublasHandle = std::static_pointer_cast(C->getBackend()) ->getCublasHandle(); - int strideA = batchA == 1 ? 0 : m * k; - int strideB = batchB == 1 ? 0 : n * k; - int strideC = n * m; - int batchC = std::max(batchA, batchB); + auto strideA = batchA == 1 ? 0 : m * k; + auto strideB = batchB == 1 ? 0 : n * k; + auto strideC = n * m; + auto batchC = std::max(batchA, batchB); std::vector aptr; std::vector bptr; @@ -206,18 +210,18 @@ void ProdBatched(marian::Tensor C, CUBLAS_CHECK(cublasSgemmBatched(cublasHandle, opB, opA, - n, - m, - k, + (int)n, + (int)m, + (int)k, &alpha, mp_bptr->data(), - ldb, + (int)ldb, mp_aptr->data(), - lda, + (int)lda, &beta, mp_cptr->data(), - ldc, - batchC)); + (int)ldc, + (int)batchC)); #if CUDA_VERSION >= 9000 cublasSetMathMode(cublasHandle, CUBLAS_DEFAULT_MATH); #endif @@ -262,7 +266,7 @@ void CSRProd(marian::Tensor C, bool transS, bool swapOperands, float beta) { - cudaSetDevice(C->getDeviceId().no); + cudaSetDevice((int)C->getDeviceId().no); auto cusparseHandle = std::static_pointer_cast(C->getBackend()) ->getCusparseHandle(); // interpret tensor dimensions as matrix dimensions diff --git a/src/tensors/gpu/tensor_operators.cu b/src/tensors/gpu/tensor_operators.cu old mode 100644 new mode 100755 diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index e103ec45f..5b8c32b75 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -32,23 +32,25 @@ Unicode - + + + true - $(Platform)\$(Configuration)\Marian\ - ..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;%MKL_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); - %BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 + $(SolutionDir)$(Platform)\$(Configuration)\Marian\ + %CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;%MKL_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); + %CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 false $(ExecutablePath) - $(Platform)\$(Configuration)\Marian\ - ..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;%MKL_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); - %BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 + $(SolutionDir)$(Platform)\$(Configuration)\Marian\ + %CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;%MKL_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); + %CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 @@ -57,6 +59,9 @@ $(OutDir);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)$(Platform)\$(Configuration);$(MSMPI_LIB64) + + 64 + @@ -64,7 +69,7 @@ Level4 Disabled - MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + CUDA_FOUND=1; MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) true true /bigobj %(AdditionalOptions) @@ -76,10 +81,20 @@ Console true - zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) + cudart_static.lib;cublas.lib;cusparse.lib;curand.lib;zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) 100000000 true + + $(SolutionDir)..\src\;$(SolutionDir)..\src\3rd_party + compute_50,sm_50 + $(CudaIntDir) + $(IntDir)%(Filename)%(Extension).obj + W2 + + + _SCL_SECURE_NO_WARNINGS + @@ -89,7 +104,7 @@ MaxSpeed true true - MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + CUDA_FOUND=1; MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) true Speed /d2Zi+ /bigobj %(AdditionalOptions) @@ -106,10 +121,18 @@ true true true - zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) + cudart_static.lib;cublas.lib;cusparse.lib;curand.lib;zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) 100000000 true + + $(SolutionDir)..\src\;$(SolutionDir)..\src\3rd_party + compute_50,sm_50 + W2 + + + _SCL_SECURE_NO_WARNINGS + @@ -973,6 +996,7 @@ + @@ -1062,6 +1086,50 @@ true + + + + + true + true + Document + + + + false + false + + + + false + false + + + true + true + Document + + + false + false + $(IntDir)%(RelativeDir) + $(IntDir)%(RelativeDir) + + + true + true + + + false + false + + + + + + + + @@ -1077,50 +1145,11 @@ - - - true - - - - true - - - true - - - true - - - true - - - - true - - - true - - - true - - - true - true - - - true - true - - - + true + Document true - - - - @@ -1133,5 +1162,6 @@ + \ No newline at end of file diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters old mode 100644 new mode 100755 index c8af656de..d77aecd2a --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -487,6 +487,9 @@ data + + tensors\gpu + @@ -1543,6 +1546,15 @@ models + + training + + + tensors\gpu + + + tensors\gpu + @@ -1712,36 +1724,11 @@ - - - training - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu - - - tensors\gpu + + 3rd_party\nccl\src - - tensors\gpu + + 3rd_party\nccl\src 3rd_party\sentencepiece\src @@ -1749,12 +1736,6 @@ 3rd_party\sentencepiece\src - - 3rd_party\nccl\src - - - 3rd_party\nccl\src - 3rd_party\nccl\src @@ -1845,18 +1826,9 @@ 3rd_party\pathie-cpp - - tests - - - tests - tests - - tests - examples\mnist @@ -1872,15 +1844,6 @@ examples - - tensors\gpu - - - translator - - - translator - @@ -1896,4 +1859,39 @@ examples + + + tensors\gpu + + + tensors\gpu + + + tensors\gpu + + + tensors\gpu + + + tensors\gpu + + + translator + + + translator + + + tensors\gpu + + + tensors\gpu + + + training\gradient_dropping\gpu + + + training\gradient_dropping\gpu + + \ No newline at end of file From 75034f1a200d92c58dd947749f617370a5121a40 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Mon, 3 Jun 2019 10:23:53 -0700 Subject: [PATCH 493/838] Now builds with VS 2017 --- vs/Marian.vcxproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 5b8c32b75..d28b467bc 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -15,19 +15,19 @@ Win32Proj Marian Marian - 8.1 + 10.0.17763.0 Application true - v140 + v141 Unicode Application false - v140 + v141 true Unicode From bdc891b9b84266b0b98fcd4f10081f3bdebee27d Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Tue, 4 Jun 2019 09:37:32 -0700 Subject: [PATCH 494/838] refined an error message --- src/data/factored_vocab.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) mode change 100644 => 100755 src/data/factored_vocab.cpp diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp old mode 100644 new mode 100755 index 3c7c8bc8e..37fbf4c72 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -301,9 +301,13 @@ Word FactoredVocab::factors2word(const std::vector& factorIndices /* [nu for (size_t g = 0; g < numGroups; g++) { auto factorIndex = factorIndices[g]; if (factorIndex != FACTOR_NOT_SPECIFIED) { // check validity - auto factor0Index = factorIndices[0]; // lemma + auto factor0Index = factorIndices[0]; // lemma ABORT_IF(factor0Index == FACTOR_NOT_SPECIFIED, "Without lemma, no other factor may be specified"); - ABORT_IF(lemmaHasFactorGroup(factor0Index, g) == (factorIndex == FACTOR_NOT_APPLICABLE), "Lemma {} does not have factor group {}", factor0Index, g); + ABORT_IF(lemmaHasFactorGroup(factor0Index, g) == (factorIndex == FACTOR_NOT_APPLICABLE), + "Lemma '{}' {} factor group '{}'", + factorVocab_[WordIndex(factor0Index + groupRanges_[0].first)], + lemmaHasFactorGroup(factor0Index, g) ? "needs" : "does not have", + groupPrefixes_[g]); } if (factorIndex == FACTOR_NOT_APPLICABLE || factorIndex == FACTOR_NOT_SPECIFIED) factorIndex = (size_t)factorShape_[g] - 1; // sentinel for "unused" or "not specified" From 56e25e386f1301bb277544d638a76ff37b397d29 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Wed, 5 Jun 2019 09:04:07 -0700 Subject: [PATCH 495/838] fix a crash at the end caused by a wrong free --- src/tensors/cpu/device.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tensors/cpu/device.cpp b/src/tensors/cpu/device.cpp index 04a79ae62..a9039b53a 100644 --- a/src/tensors/cpu/device.cpp +++ b/src/tensors/cpu/device.cpp @@ -9,12 +9,6 @@ namespace marian { namespace cpu { -Device::~Device() { - free(data_); - data_ = nullptr; - size_ = 0; -} - // allocate function for tensor reserve() below. // Needed for AVX512, while not available on all compilers. It seems clang // does not have aligned_alloc for all cstlib versions. If AVX512 is not used @@ -35,6 +29,12 @@ Device::~Device() { #define FREE(ptr) free(ptr) #endif +Device::~Device() { + FREE(data_); + data_ = nullptr; + size_ = 0; +} + void Device::reserve(size_t size) { size = align(size); ABORT_IF(size < size_ || size == 0, From 27defae8ce35b113038a5ed4973c6a82b04e7a16 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Wed, 5 Jun 2019 09:42:02 -0700 Subject: [PATCH 496/838] Some more changes - value deallocations, initializations --- src/tensors/cpu/device.cpp | 4 ++-- src/tensors/device.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tensors/cpu/device.cpp b/src/tensors/cpu/device.cpp index a9039b53a..13c3bb074 100644 --- a/src/tensors/cpu/device.cpp +++ b/src/tensors/cpu/device.cpp @@ -30,9 +30,9 @@ namespace cpu { #endif Device::~Device() { - FREE(data_); - data_ = nullptr; size_ = 0; + data_ = nullptr; + FREE(data_); } void Device::reserve(size_t size) { diff --git a/src/tensors/device.h b/src/tensors/device.h index 68a9492eb..aa57019ad 100644 --- a/src/tensors/device.h +++ b/src/tensors/device.h @@ -11,8 +11,8 @@ class Device { protected: DeviceId deviceId_; - uint8_t* data_{0}; - size_t size_{0}; + uint8_t* data_; + size_t size_; size_t alignment_; size_t align(size_t size) { From fb305877a61f6a8796370c070599ae4218530ff6 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 5 Jun 2019 14:05:27 -0700 Subject: [PATCH 497/838] bug fix: FactoredSegmenter \u escapes should use 4 digits, not 3 --- src/common/config_parser.cpp | 2 +- src/data/factored_vocab.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/config_parser.cpp b/src/common/config_parser.cpp index eaa9c9912..af0ec8cb0 100755 --- a/src/common/config_parser.cpp +++ b/src/common/config_parser.cpp @@ -668,7 +668,7 @@ void ConfigParser::addSuboptionsInputLength(cli::CLIWrapper& cli) { "Maximum length of a sentence in a training sentence pair", defaultMaxLength); cli.add("--max-length-crop", - "Crop a sentence to max-length instead of ommitting it if longer than max-length"); + "Crop a sentence to max-length instead of omitting it if longer than max-length"); // clang-format on } diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index 37fbf4c72..af40ea3ad 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -571,11 +571,11 @@ void FactoredVocab::constructNormalizationInfoForVocab() { static void unescapeHexEscapes(std::string& utf8Lemma) { if (utf8Lemma.find('\\') == std::string::npos) return; // nothing to do - auto lemma = utils::utf8ToUtf16String(utf8Lemma); // \u.... implies we must operate on UTF-16 level + auto lemma = utils::utf8ToUtf16String(utf8Lemma); // \u.... implies we must operate on UTF-16 level (not UCS-4) auto pos = lemma.find('\\'); while (pos != std::string::npos) { ABORT_IF(pos + 1 >= lemma.size() || (lemma[pos+1] != 'x' && lemma[pos + 1] != 'u'), "Malformed escape in factored encoding: {}", utf8Lemma); - int numDigits = 2 + (lemma[pos + 1] == 'u'); + int numDigits = 2 + 2 * (lemma[pos + 1] == 'u'); // 2 for \x, 4 for \u ABORT_IF(pos + 2 + numDigits > lemma.size(), "Malformed escape in factored encoding: {}", utf8Lemma); auto digits = utils::utf8FromUtf16String(lemma.substr(pos + 2, numDigits)); auto c = std::strtoul(digits.c_str(), nullptr, 16); From 3c5177ca74ce4c0557bb9bac19e4a93fd30018f3 Mon Sep 17 00:00:00 2001 From: Frank Seide Date: Wed, 5 Jun 2019 14:13:42 -0700 Subject: [PATCH 498/838] now supports FactoredSegmenter's DistinguishInitialAndInternalPieces mode --- src/data/factored_vocab.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/data/factored_vocab.cpp b/src/data/factored_vocab.cpp index af40ea3ad..68fd78f93 100755 --- a/src/data/factored_vocab.cpp +++ b/src/data/factored_vocab.cpp @@ -607,13 +607,15 @@ std::string FactoredVocab::surfaceForm(const Words& sentence) const /*override f auto has = [&](const char* factor) { return tokenSet.find(factor) != tokenSet.end(); }; // spacing bool hasGlueRight = has("gr+") || has("wen") || has("cen"); - bool hasGlueLeft = has("gl+") || has("wbn") || has("cbn"); + bool hasGlueLeft = has("gl+") || has("wbn") || has("cbn") || has("wi"); bool insertSpaceBefore = !prevHadGlueRight && !hasGlueLeft; if (insertSpaceBefore) res.push_back(' '); prevHadGlueRight = hasGlueRight; // capitalization unescapeHexEscapes(lemma); // unescape \x.. and \u.... + if (utils::beginsWith(lemma, "\xE2\x96\x81")) // remove leading _ (\u2581, for DistinguishInitialAndInternalPieces mode) + lemma = lemma.substr(3); if (has("ci")) lemma = utils::utf8Capitalized(lemma); else if (has("ca")) lemma = utils::utf8ToUpper (lemma); else if (has("cn")) lemma = utils::utf8ToLower (lemma); From 925d0f16d68eb5662b2e2196f6d4e2988f81b4fe Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Thu, 6 Jun 2019 09:17:57 -0700 Subject: [PATCH 499/838] some more fixes --- src/tensors/cpu/device.cpp | 2 -- src/tensors/device.h | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tensors/cpu/device.cpp b/src/tensors/cpu/device.cpp index 13c3bb074..941902a8f 100644 --- a/src/tensors/cpu/device.cpp +++ b/src/tensors/cpu/device.cpp @@ -30,8 +30,6 @@ namespace cpu { #endif Device::~Device() { - size_ = 0; - data_ = nullptr; FREE(data_); } diff --git a/src/tensors/device.h b/src/tensors/device.h index aa57019ad..0be6c076c 100644 --- a/src/tensors/device.h +++ b/src/tensors/device.h @@ -11,8 +11,8 @@ class Device { protected: DeviceId deviceId_; - uint8_t* data_; - size_t size_; + uint8_t* data_{0}; + size_t size_{0}; size_t alignment_; size_t align(size_t size) { @@ -21,7 +21,7 @@ class Device { public: Device(DeviceId deviceId, size_t alignment = 256) - : deviceId_(deviceId), data_(0), size_(0), alignment_(alignment) {} + : deviceId_(deviceId), alignment_(alignment) {} virtual ~Device(){}; From d2d8ec041d443f906809b912a57a3d41b2c3a07c Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Tue, 18 Jun 2019 16:59:11 -0700 Subject: [PATCH 500/838] Enable FBGEMM based packed GEMM on windows --- .gitmodules | 4 + CMakeLists.txt | 83 ++-- src/3rd_party/CMakeLists.txt | 5 + src/3rd_party/fbgemm | 1 + src/CMakeLists.txt | 2 + src/graph/auto_tuner.h | 2 +- src/graph/expression_operators.cpp | 96 +++-- src/tensors/cpu/expanded_gemm.h | 204 +++++++++ src/tensors/cpu/sharp/avx_gemm.cpp | 5 + src/tensors/cpu/sharp/packed_gemm.cpp | 585 ++++++++++++++++++++++++++ src/tensors/cpu/sharp/packed_gemm.h | 39 ++ src/tensors/cpu/tensor_operators.cpp | 63 +++ vs/Marian.vcxproj | 28 +- vs/Marian.vcxproj.filters | 57 +++ 14 files changed, 1107 insertions(+), 67 deletions(-) create mode 160000 src/3rd_party/fbgemm create mode 100644 src/tensors/cpu/expanded_gemm.h create mode 100644 src/tensors/cpu/sharp/packed_gemm.cpp create mode 100644 src/tensors/cpu/sharp/packed_gemm.h diff --git a/.gitmodules b/.gitmodules index efe6bc5ec..7f8ce6304 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,7 @@ [submodule "src/3rd_party/nccl"] path = src/3rd_party/nccl url = https://github.com/marian-nmt/nccl +[submodule "src/3rd_party/fbgemm"] + path = src/3rd_party/fbgemm + url = https://machinetranslation.visualstudio.com/DefaultCollection/MachineTranslation/_git/FBGEMM-internal + branch = master diff --git a/CMakeLists.txt b/CMakeLists.txt index b54766ae9..7cb518c56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ option(USE_MPI "Use MPI library" OFF) option(COMPILE_EXAMPLES "Compile examples" OFF) option(COMPILE_TESTS "Compile tests" OFF) option(COMPILE_SERVER "Compile marian-server" ON) +option(USE_FBGEMM "Use FBGEMM" ON) # Project versioning find_package(Git QUIET) @@ -59,44 +60,54 @@ if(MSVC) find_library(SHLWAPI Shlwapi.lib) set(EXT_LIBS ${EXT_LIBS} SHLWAPI) -else() +else(MSVC) + + # Detect support CPU instrinsics for the current platform. This will + # only by used with BUILD_ARCH=native. For overridden BUILD_ARCH we + # minimally use -msse4.1. This seems to work with MKL. + set(INTRINSICS "") + if(BUILD_ARCH STREQUAL "native") + message(STATUS "Checking support for CPU intrinsics") + include(FindSSE) + if(SSE2_FOUND) + message(STATUS "SSE2 support found") + set(INTRINSICS "${INTRINSICS} -msse2") + endif(SSE2_FOUND) + if(SSE3_FOUND) + message(STATUS "SSE3 support found") + set(INTRINSICS "${INTRINSICS} -msse3") + endif(SSE3_FOUND) + if(SSE4_1_FOUND) + message(STATUS "SSE4.1 support found") + set(INTRINSICS "${INTRINSICS} -msse4.1") + endif(SSE4_1_FOUND) + if(AVX_FOUND) + message(STATUS "AVX support found") + set(INTRINSICS "${INTRINSICS} -mavx") + endif(AVX_FOUND) + if(AVX2_FOUND) + message(STATUS "AVX2 support found") + set(INTRINSICS "${INTRINSICS} -mavx2") + endif(AVX2_FOUND) + if(AVX512_FOUND) + message(STATUS "AVX512 support found") + set(INTRINSICS "${INTRINSICS} -mavx512f") + list(APPEND INTRINSICS_NVCC -Xcompiler\ -mavx512f) + endif(AVX512_FOUND) + else() + set(INTRINSICS "-msse4.1") + endif() -# Detect support CPU instrinsics for the current platform. This will -# only by used with BUILD_ARCH=native. For overridden BUILD_ARCH we -# minimally use -msse4.1. This seems to work with MKL. -set(INTRINSICS "") -if(BUILD_ARCH STREQUAL "native") - message(STATUS "Checking support for CPU intrinsics") - include(FindSSE) - if(SSE2_FOUND) - message(STATUS "SSE2 support found") - set(INTRINSICS "${INTRINSICS} -msse2") - endif(SSE2_FOUND) - if(SSE3_FOUND) - message(STATUS "SSE3 support found") - set(INTRINSICS "${INTRINSICS} -msse3") - endif(SSE3_FOUND) - if(SSE4_1_FOUND) - message(STATUS "SSE4.1 support found") - set(INTRINSICS "${INTRINSICS} -msse4.1") - endif(SSE4_1_FOUND) - if(AVX_FOUND) - message(STATUS "AVX support found") - set(INTRINSICS "${INTRINSICS} -mavx") - endif(AVX_FOUND) - if(AVX2_FOUND) - message(STATUS "AVX2 support found") - set(INTRINSICS "${INTRINSICS} -mavx2") - endif(AVX2_FOUND) -else() - set(INTRINSICS "-msse4.1") -endif() + if(USE_FBGEMM) + set(EXT_LIBS ${EXT_LIBS} fbgemm dl) + add_definitions(-DUSE_FBGEMM=1) + endif(USE_FBGEMM) -set(DISABLE_GLOBALLY "-Wno-unused-result") + set(DISABLE_GLOBALLY "-Wno-unused-result") -# These are used in src/CMakeLists.txt on a per-target basis -list(APPEND ALL_WARNINGS -Wall; -Werror; -Wno-unused-result; -Wno-deprecated; -Wno-pragmas; -Wno-unused-parameter; -Wextra; -Wno-unused-function; - -Wno-unused-value; -Wno-unknown-pragmas; -Wno-sign-compare; -Wno-missing-field-initializers;) + # These are used in src/CMakeLists.txt on a per-target basis + list(APPEND ALL_WARNINGS -Wall; -Werror; -Wno-unused-result; -Wno-deprecated; -Wno-pragmas; -Wno-unused-parameter; -Wextra; -Wno-unused-function; + -Wno-unused-value; -Wno-unknown-pragmas; -Wno-sign-compare; -Wno-missing-field-initializers;) # This warning does not exist prior to gcc 5.0 if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.0) @@ -111,7 +122,7 @@ list(APPEND ALL_WARNINGS -Wall; -Werror; -Wno-unused-result; -Wno-deprecated; -W set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE} -pg -g -rdynamic") set(CMAKE_CXX_FLAGS_PROFGEN "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction") set(CMAKE_CXX_FLAGS_PROFUSE "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction") - endif() +endif(MSVC) # Downloading SentencePiece if requested and set to compile with it. # Requires all the dependencies imposed by SentencePiece diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index 8548d9b8b..3dca348bc 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -5,6 +5,11 @@ add_subdirectory(./yaml-cpp) add_subdirectory(./SQLiteCpp) add_subdirectory(./pathie-cpp) add_subdirectory(./zlib) +if(USE_FBGEMM) + set(FBGEMM_BUILD_TESTS OFF CACHE BOOL "Disable fbgemm tests") + set(FBGEMM_BUILD_BENCHMARKS OFF CACHE BOOL "Disable fbgemm benchmark") + add_subdirectory(./fbgemm) +endif(USE_FBGEMM) if(USE_SENTENCEPIECE) if(USE_STATIC_LIBS) diff --git a/src/3rd_party/fbgemm b/src/3rd_party/fbgemm new file mode 160000 index 000000000..25c1595ce --- /dev/null +++ b/src/3rd_party/fbgemm @@ -0,0 +1 @@ +Subproject commit 25c1595cebfc88775692660b1da7d0bf632112a8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 90dde706a..c3fcefefc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ include_directories(.) include_directories(3rd_party) include_directories(3rd_party/SQLiteCpp/include) include_directories(3rd_party/sentencepiece) +include_directories(3rd_party/fbgemm/include) add_library(marian STATIC common/version.cpp @@ -40,6 +41,7 @@ add_library(marian STATIC tensors/cpu/sharp/int_gemm.cpp tensors/cpu/sharp/avx_gemm.cpp tensors/cpu/sharp/sse_gemm.cpp + tensors/cpu/sharp/packed_gemm.cpp graph/expression_graph.cpp graph/expression_operators.cpp diff --git a/src/graph/auto_tuner.h b/src/graph/auto_tuner.h index 7bf80c799..868a800e2 100644 --- a/src/graph/auto_tuner.h +++ b/src/graph/auto_tuner.h @@ -20,7 +20,7 @@ class AutoTuner : public AutoTunerRecorder { private: typedef std::function Algorithm; - const size_t max = 100; + const size_t max = 50; UPtr timer_; diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 396fa03e2..42bb65e20 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -7,6 +7,7 @@ #include "graph/auto_tuner.h" #include "tensors/cpu/int16.h" +#include "tensors/cpu/expanded_gemm.h" namespace marian { @@ -410,7 +411,7 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { float clipValue = a->graph()->getBackend()->getClip(); if(a->graph()->isOptimized() && device == DeviceType::cpu) { - bool autotune = true; + bool autotune = false; if(autotune) { thread_local Ptr> tuner = New>(); @@ -431,60 +432,107 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { util::hash_combine(hash, transA); util::hash_combine(hash, transB); +#if USE_FBGEMM + // Use Packed GEMM only if it's memoized + if(b->memoize()) { + // add third algorithm variant (Packed GEMM) + size_t hash1 = hash; + util::hash_combine(hash1, 1); + auto rec1 = [=](Expr e, bool stop = false) { + e->record(tuner, hash1, stop); + return e; + }; + + auto alg1 = [=]() { + auto packed = cpu::pack::pack(b, cpu::pack::PackMatrix::B, transB, clipValue); + + return rec1( + cpu::pack::affine( + clip(a, clipValue), packed, + b->shape(), + bias, + transA, + transB, + scale), + true); + }; + tuner->insert({hash1, alg1}); + } +#endif // USE_FBGEMM + // add first algorithm variant (Int16) - size_t hash1 = hash; - util::hash_combine(hash1, 1); - auto rec1 = [=](Expr e, bool stop = false) { - e->record(tuner, hash1, stop); + size_t hash2 = hash; + util::hash_combine(hash2, 2); + auto rec2 = [=](Expr e, bool stop = false) { + e->record(tuner, hash2, stop); return e; }; - auto alg1 = [=]() { - return rec1( + auto alg2 = [=]() { + return rec2( cpu::int16::affine( - rec1(cpu::int16::quantize(transA ? rec1(transpose(a)) : a, + rec2(cpu::int16::quantize(transA ? rec2(transpose(a)) : a, clipValue)), cpu::int16::quantize(transB ? b : transpose(b), clipValue), bias, scale), true); }; - tuner->insert({hash1, alg1}); + tuner->insert({hash2, alg2}); // add second algorithm variant (CBlas) - size_t hash2 = hash; - util::hash_combine(hash2, 2); - auto rec2 = [=](Expr e, bool stop = false) { - e->record(tuner, hash2, stop); + size_t hash3 = hash; + util::hash_combine(hash3, 3); + auto rec3 = [=](Expr e, bool stop = false) { + e->record(tuner, hash3, stop); return e; }; - auto alg2 = [=]() { + auto alg3 = [=]() { auto ac = clip(a, clipValue); if(ac != a) - ac = rec2(ac); + ac = rec3(ac); auto bc = clip(b, clipValue); if(bc != b) - bc = rec2(bc); + bc = rec3(bc); int rows = ac->shape().elements() / ac->shape()[-1]; Expr ones = ac->graph()->ones({rows, 1}); std::vector nodes = {ac, bc, bias, ones}; - return rec2(Expression(nodes, transA, transB, scale), + return rec3(Expression(nodes, transA, transB, scale), true); }; - tuner->insert({hash2, alg2}); + tuner->insert({hash3, alg3}); // execute algorithm with autotuning return tuner->run(); } else { - // cpu int16 version - return cpu::int16::affine( - cpu::int16::quantize(transA ? transpose(a) : a, clipValue), - cpu::int16::quantize(transB ? b : transpose(b), clipValue), - bias, - scale); + if(b->memoize()) { + auto packed = cpu::pack::pack(b, cpu::pack::PackMatrix::B, transB, clipValue); + // auto packed = transB ? + // cpu::pack::pack(transpose(b), cpu::pack::PackMatrix::B, false, clipValue) : + // cpu::pack::pack(b, cpu::pack::PackMatrix::B, false, clipValue); + + return cpu::pack::affine( + clip(a, clipValue), packed, + b->shape(), + bias, + transA, + transB, + scale); + // cpu int16 version + // return cpu::int16::affine( + // cpu::int16::quantize(transA ? transpose(a) : a, clipValue), + // cpu::int16::quantize(transB ? b : transpose(b), clipValue), + // bias, + // scale); + } else { + int rows = a->shape().elements() / a->shape()[-1]; + Expr ones = a->graph()->ones({rows, 1}); + std::vector nodes = {clip(a, clipValue), clip(b, clipValue), bias, ones}; + return Expression(nodes, transA, transB, scale); + } } } else { // general version, MKL, CBlas or CUDA diff --git a/src/tensors/cpu/expanded_gemm.h b/src/tensors/cpu/expanded_gemm.h new file mode 100644 index 000000000..33b677945 --- /dev/null +++ b/src/tensors/cpu/expanded_gemm.h @@ -0,0 +1,204 @@ +#pragma once + +#include "graph/node.h" +#include "tensors/cpu/sharp/packed_gemm.h" + +#if USE_FBGEMM +#include "3rd_party/fbgemm/include/fbgemm/FbgemmFP16.h" +using namespace fbgemm; +#endif // USE_FBGEMM + +namespace marian { +namespace cpu { +namespace pack { + +enum class PackMatrix : uint8_t { + A = 0x00, + B = 0x01 +}; + +struct PackNodeOp : public UnaryNodeOp { + PackMatrix packMat_; + bool transpose_; + int nrow_; + int ncol_; + int kernel_ncol_blocks_; + int brow_; + int bcol_; + int last_brow_; + int nbrow_; + int nbcol_; + uint64_t packsize_; + + PackNodeOp(Expr a, PackMatrix packMat, bool transpose, float clipValue) + : UnaryNodeOp(a, newShape(a, transpose), Type::uint8), + packMat_(packMat), + transpose_(transpose) { + if(packMat != PackMatrix::B) + ABORT("Only prepacking of B (weight matrix) is supported"); + if(clipValue != 0) + ABORT("Clipping is not supported"); + if(!memoize_) + ABORT("Only memoizeable node is supported"); + } + + NodeOps forwardOps() override { + return {NodeOp(PackFp32(val_, + child(0)->val(), + transpose_, + nrow_, + ncol_, + kernel_ncol_blocks_, + brow_, + bcol_, + last_brow_, + nbrow_, + nbcol_, + packsize_)) + }; + } + + NodeOps backwardOps() override { + ABORT("Only used for inference"); + return {NodeOp(0)}; + } + + const std::string type() override { return "packMat"; } + + Shape newShape(Expr a, bool transpose) { +#if USE_FBGEMM + auto shapeMat = a->shape(); + // Should be 2D - weight matrix + ABORT_IF(shapeMat.size() != 2, + "Weight Matrix should be 2D"); + if (true) { + // if (shapeMat[0] < 32000 && shapeMat[1] < 32000) { + nrow_ = transpose ? shapeMat[1] : shapeMat[0]; + ncol_ = transpose ? shapeMat[0] : shapeMat[1]; + kernel_ncol_blocks_ = 2; + brow_ = 512; + bcol_ = 8 * kernel_ncol_blocks_; + last_brow_ = nrow_ % brow_ == 0 ? brow_ : nrow_ % brow_; + nbrow_ = nrow_ % brow_ == 0 ? nrow_ / brow_ : (nrow_ + brow_) / brow_; + nbcol_ = ncol_ % bcol_ == 0 ? ncol_ / bcol_ : (ncol_ + bcol_) / bcol_; + const int padding = 1024; // required by sw pipelined kernels + const int specialMem = 256; + packsize_ = ((nbrow_ * brow_) * (nbcol_ * bcol_)) * sizeof(fbgemm::float16) + padding + specialMem; + } else { + // use int 8 implementation + packsize_ = 1; + } + + Shape outShape({(int)packsize_}); + + return outShape; +#else // USE_FBGEMM + ABORT("Only FBGEMM based packed GEMM is supported"); + return Shape(); +#endif // USE_FBGEMM + } +}; + +class AffineNodeOp : public NaryNodeOp { +private: + float scalar_; + int64_t m_; + int64_t n_; + int64_t k_; + bool transA_; + bool transB_; + size_t idx_; + +public: + AffineNodeOp(const std::vector& nodes, Shape bShape, bool transA, bool transB, float scalar) + : NaryNodeOp(nodes, newShape(nodes[0], bShape, transA, transB), Type::float32), + scalar_(scalar) { + transA_ = transA; + transB_ = transB; + m_ = nodes[0]->shape().elements() / nodes[0]->shape()[-1]; + k_ = nodes[0]->shape().back(); + if(transA) + std::swap(m_, k_); + + int64_t l = bShape.elements() / bShape[-1]; + n_ = bShape[-1]; + if(transB) + std::swap(l, n_); + } + + Shape newShape(Expr a, Shape bShape, bool transA, bool transB) { + auto shapeA = a->shape(); + if(transA) { + shapeA.set(shapeA.size() - 2, a->shape()[shapeA.size() - 1]); + shapeA.set(shapeA.size() - 1, a->shape()[shapeA.size() - 2]); + } + + auto shapeB = bShape; + if(transB) { + shapeB.set(shapeB.size() - 2, bShape[shapeB.size() - 1]); + shapeB.set(shapeB.size() - 1, bShape[shapeB.size() - 2]); + } + + Shape outShape = shapeA; + outShape.set(outShape.size() - 1, shapeB[shapeB.size() - 1]); + ABORT_IF(shapeA[shapeA.size() - 1] != shapeB[shapeB.size() - 2], + "Matrix product requires inner dimensions to match"); + return outShape; + } + + NodeOps forwardOps() override { + // if (n_ < 32000) { + if (true) { + return { + NodeOp(GemmPackFp32(val_, + child(0)->val(), + child(1)->val(), + child(2)->val(), + m_, + n_, + //k_, + //1, + //0, + transA_)) + //transB_, + //idx_)) + }; + } else { + return { + NodeOp(GemmPackFp32(val_, + child(0)->val(), + child(1)->val(), + child(2)->val(), + m_, + n_, + //k_, + //1, + //0, + transA_); + //transB_, + //idx_); + AddBias(val_, child(2)->val())) + }; + } + } + + NodeOps backwardOps() override { + ABORT("Only used for inference"); + return {NodeOp(0)}; + } + + const std::string type() override { return "affinePacked"; } +}; + +static inline Expr affine(Expr a, Expr b, Shape bShape, Expr c, bool transA, bool transB, float scalar) { + std::vector nodes = {a, b, c}; + return Expression(nodes, bShape, transA, transB, scalar); +} + +static inline Expr pack(Expr a, PackMatrix packMat, bool transpose, float clipValue) { + return Expression(a, packMat, transpose, clipValue); +} + +} // namespace pack +} // namespace cpu +} // namespace marian diff --git a/src/tensors/cpu/sharp/avx_gemm.cpp b/src/tensors/cpu/sharp/avx_gemm.cpp index c41b73eb0..572ac391a 100644 --- a/src/tensors/cpu/sharp/avx_gemm.cpp +++ b/src/tensors/cpu/sharp/avx_gemm.cpp @@ -10,6 +10,11 @@ #include #include +#ifdef _MSC_VER +#pragma warning(disable : 4310) +#pragma warning(disable : 4324) +#endif + #ifdef __AVX512F__ namespace marian { diff --git a/src/tensors/cpu/sharp/packed_gemm.cpp b/src/tensors/cpu/sharp/packed_gemm.cpp new file mode 100644 index 000000000..b6b7937f5 --- /dev/null +++ b/src/tensors/cpu/sharp/packed_gemm.cpp @@ -0,0 +1,585 @@ +#include "packed_gemm.h" +#include "tensors/tensor_allocator.h" +#include "tensors/tensor_operators.h" + +#include +#include +#include +#include +#include +#include +#include +//#include + +#ifdef _MSC_VER +#pragma warning(disable: 4505) // warning C4505: 'fbgemmAlignedAlloc' in fbgemm.h: unreferenced local function has been removed (missing 'static inline') +#endif + +#if (USE_FBGEMM && MKL_FOUND) +#include "3rd_party/fbgemm/include/fbgemm/FbgemmFP16.h" +#include "3rd_party/fbgemm/include/fbgemm/QuantUtils.h" +#include "3rd_party/fbgemm/include/fbgemm/Fbgemm.h" + +#include +#include +//#include "mkl_vsl.h" + +#ifdef _OPENMP +#include +#endif + +using namespace fbgemm; +#endif // USE_FBGEMM && MKL_FOUND + +namespace marian { +namespace cpu { +namespace pack { + +//static float packingTime = 0; + +#if (USE_FBGEMM && MKL_FOUND) +// initialize with a dummy +static PackedGemmMatrixFP16 packedPlaceholder(1, 1, 1, 1, 1, 1, 1, 1); + +// temporary variable for int 8 +//static PackBMatrix* packedBint8 = nullptr; +// transformer base wmt 2017 +// static float bqScale = 0.39/128; +// static int32_t bqZeropoint = 0; +// old student de-en +//static float* bqScale; +//static int32_t* bqZeropoint; +// static float bqScale = 0.683/106; +// static int32_t bqZeropoint = 21; +// static float bqScale = 0.9672/128; +// static int32_t bqZeropoint = 0; +//static std::vector* col_offsets = nullptr; + +//inline void col_offsets_with_zero_pt_s8acc32_ref( +// bool transpose, +// int K, +// int N, +// const int8_t* Bint8, +// const int32_t* B_zero_point, +// int32_t* col_offsets, +// int ncols_per_quant_group) { +// for (int n = 0; n < N; ++n) { +// int32_t sum = 0; +// for (int k = 0; k < K; ++k) { +// sum += transpose ? Bint8[k + n * K] : Bint8[k * N + n]; +// } +// col_offsets[n] = sum - B_zero_point[n / ncols_per_quant_group] * K; +// } +//} + +// This is copied from FBGEMM code +// A better way? +// blocked row-major format address arithmetic +inline uint64_t addr(const int r_, + const int c_, + const int brow_, + const int bcol_, + const int nbrow_, + const int nbcol_, + const int last_brow_) { + uint64_t r = (uint64_t)r_; + uint64_t c = (uint64_t)c_; + + uint64_t block_row_id = r / brow_; + uint64_t brow_offset = (block_row_id * nbcol_) * (brow_ * bcol_); + uint64_t block_col_id = c / bcol_; + uint64_t bcol_offset + = block_col_id * ((block_row_id != nbrow_ - 1) ? (brow_ * bcol_) : (last_brow_ * bcol_)); + uint64_t block_offset = brow_offset + bcol_offset; + uint64_t inblock_offset = r % brow_ * bcol_ + c % bcol_; + + uint64_t index = block_offset + inblock_offset; + return index; +} + +void PackFp32(marian::Tensor out, + const marian::Tensor in, + bool transpose, + int nrow, + int ncol, + int kernel_ncol_blocks, + int brow, + int bcol, + int last_brow, + int nbrow, + int nbcol, + uint64_t packsize) { + //auto t_start = std::chrono::high_resolution_clock::now(); + // for the last embedding layer, pack it into int8 + // if (in->shape().size() == 2 && (in->shape()[0] >= 32000 || in->shape()[1] >= 32000)) { + if (false) { +#if 0 + if (packedBint8 == nullptr) { + int k = transpose ? in->shape()[1] : in->shape()[0]; + int n = transpose ? in->shape()[0] : in->shape()[1]; + // std::cout << "transpose: " << transpose << ", k: " << k << ", n: " << n << std::endl; + // std::cout << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + // two steps + // 0. quantize --> this should be done outside + int len = in->shape()[0]*in->shape()[1]; + + // 0-1. collect stats for each class + bqScale = new float[n]; + bqZeropoint = new int32_t[n]; + + // int numBin = 20; + // float denum = 2/(float)numBin; + + // int hist[numBin] = { 0, }; + + // Transposed only + float* data = in->data(); + float val = 0; + for (int jj = 0; jj < n; jj++) { + float min = 1000000, max = -10000000; + for (int ii = 0; ii < k; ii++) { + val = data[jj*k + ii]; + if (val < min) min = val; + if (val > max) max = val; + // hist[(int)((val + 1)/denum)]++; + } + bqScale[jj] = (max - min)/255; + bqZeropoint[jj] = (int32_t)(127 - max / bqScale[jj]); + // bqScale[jj] = (0.3 + 0.4)/255; + // bqZeropoint[jj] = (int32_t)(127 - 0.3 / bqScale[jj]); + } + + // std::cout << "hist: "; + // for (int ii = 0; ii < numBin; ii++) { + // std::cout << hist[ii] << ", "; + // } + // std::cout << std::endl; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + //int8_t quantized[len]; // aligned malloc? + int8_t* quantized = (int8_t*)aligned_alloc(256, len); + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + for (int ii = 0; ii < n; ii++) { + TensorQuantizationParams bQuantParam; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + bQuantParam.scale = bqScale[ii]; + bQuantParam.zero_point = bqZeropoint[ii]; + bQuantParam.precision = 8; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + + fbgemm::Quantize(in->data() + ii * k, quantized + ii * k, k, bQuantParam); + } + // std::cout << "original" << std::endl; + // for (int ii = 0; ii < n; ii++) { + // for (int jj = 0; jj < 1; jj++) { + // std::cout << in->data()[ii * k + jj] << ","; + // } + // std::cout << std::endl; + // } + // std::cout << "quantized" << std::endl; + // for (int ii = 0; ii < 1; ii++) { + // for (int jj = 0; jj < k; jj++) { + // std::cout << (int32_t)quantized[ii * k + jj] << ","; + // } + // std::cout << std::endl; + // } + // 1. compute column offsets + col_offsets = new std::vector(n); + col_offsets_with_zero_pt_s8acc32_ref(transpose, k, n, quantized, bqZeropoint, col_offsets->data(), 1); + // for (int ii = 0; ii < n; ii++) { + // std::cout << (int32_t)col_offsets->data()[ii] << ","; + // } + // std::cout << std::endl; + // std::cout << "calc offset done" << std::endl; + // 2. packing + // uint8_t* packedmem = aligned_alloc(256, len); + packedBint8 = new PackBMatrix(transpose ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + k, // in->shape()[0], + n, // in->shape()[1], + quantized, + in->shape()[1]); + // std::cout << "packing B done" << std::endl; + // int k = transpose ? in->shape()[1] : in->shape()[0]; + // int n = transpose ? in->shape()[0] : in->shape()[1]; + // std::cout << "transpose: " << transpose << ", k: " << k << ", n: " << n << std::endl; + // std::cout << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + // // two steps + // // 0. quantize --> this should be done outside + // int len = in->shape()[0]*in->shape()[1]; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + // //int8_t quantized[len]; // aligned malloc? + // int8_t* quantized = (int8_t*)aligned_alloc(256, len); + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + // TensorQuantizationParams bQuantParam; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + // bQuantParam.scale = bqScale; + // bQuantParam.zero_point = bqZeropoint; + // bQuantParam.precision = 8; + // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; + + // fbgemm::Quantize(in->data(), quantized, len, bQuantParam); + // std::cout << "original" << std::endl; + // // for (int ii = 0; ii < n; ii++) { + // // for (int jj = 0; jj < 1; jj++) { + // // std::cout << in->data()[ii * k + jj] << ","; + // // } + // // std::cout << std::endl; + // // } + // std::cout << "quantized" << std::endl; + // // for (int ii = 0; ii < 1; ii++) { + // // for (int jj = 0; jj < k; jj++) { + // // std::cout << (int32_t)quantized[ii * k + jj] << ","; + // // } + // // std::cout << std::endl; + // // } + // // 1. compute column offsets + // col_offsets = new std::vector(n); + // col_offsets_with_zero_pt_s8acc32_ref(k, n, n, quantized, &bqZeropoint, col_offsets->data(), n); + // // for (int ii = 0; ii < n; ii++) { + // // std::cout << (int32_t)col_offsets->data()[ii] << ","; + // // } + // std::cout << std::endl; + // std::cout << "calc offset done" << std::endl; + // 2. packing + // uint8_t* packedmem = aligned_alloc(256, len); + // packedBint8 = new PackBMatrix(transpose ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + // in->shape()[0], + // in->shape()[1], + // quantized, + // transpose ? in->shape()[1] : in->shape()[0]); + // std::cout << "packing B done" << std::endl; + } +#endif + } else { + // initialize memory + uint8_t* outmemorg = out->data(); + for(auto i = 0; i < packsize; i++) { + outmemorg[i] = 0; + } + // save the other auxiliary variables + uint64_t* auxmemsize = (uint64_t*)outmemorg; + auxmemsize[0] = packsize; + int* auxmem = (int*)(auxmemsize + 1); + auxmem[0] = nrow; + auxmem[1] = ncol; + auxmem[2] = kernel_ncol_blocks; + auxmem[3] = brow; + auxmem[4] = bcol; + auxmem[5] = last_brow; + auxmem[6] = nbrow; + auxmem[7] = nbcol; + // cast to float16 + fbgemm::float16* outmem = (fbgemm::float16*)(outmemorg + 256); + fbgemm::float16* dummy = new fbgemm::float16; + // pack the matrix + float* inmem = in->data(); + for(int i = 0; i < nrow; i++) { + for(int j = 0; j < ncol; j++) { + outmem[addr(i, j, brow, bcol, nbrow, nbcol, last_brow)] + = tconv(!transpose ? inmem[i * ncol + j] : inmem[i + nrow * j], *dummy); + } + } + delete dummy; + + //auto t_end = std::chrono::high_resolution_clock::now(); + //packingTime += (float) std::chrono::duration(t_end-t_start).count(); + //std::cout << "Packing time: " << packingTime << std::endl; + } + + // std::cout << "B transposed: " << transpose << std::endl; + // for (int i = 0; i < in->shape().size(); i++) { + // std::cout << "size " << i << ": " << in->shape()[i] << std::endl; + // } + // // compute statistics for quantization + // if (in->shape().size() == 2 && (in->shape()[0] >= 32000 || in->shape()[1] >= 32000)) { + // float mins[ncol] = {0}; + // float maxs[ncol] = {0}; + // float means[ncol] = {0}; + // float stds[ncol] = {0}; + // for(int i = 0; i < nrow; i++) { + // for(int j = 0; j < ncol; j++) { + // float val = !transpose ? in->data()[i * ncol + j] : in->data()[i + nrow * j]; + // if (val < mins[j]) + // mins[j] = val; + // if (val > maxs[j]) + // maxs[j] = val; + + // means[j] += val; + // stds[j] += val*val; + // } + // } + // for(int j = 0; j < ncol; j++) { + // std::cout << mins[j] << ", " << maxs[j] << ", " << means[j] << ", " << stds[j] << std::endl; + // } + // } +} + +void GemmPackFp32(marian::Tensor C, + const marian::Tensor A, + const marian::Tensor B, + const marian::Tensor bias, + const int64_t m, + const int64_t n, + int transA) { + // use int 8 packed gemm + // if (A->shape().size() == 4 && B->shape().size() == 1 && B->shape()[0] == 1) { + if (false) { + // quantize & pack A + // transformer base wmt 2017 + // float ascale = 7.8/104; + // int32_t azeropoint = 151; + // old student de-en + // float ascale = 14.85/117; + // int32_t azeropoint = 138; + +#if 0 + // compute range + float min_est=1000000, max_est=-10000000; + // VSLSSTaskPtr task; + // MKL_INT task_p, task_n, xstorage; + + // /* Parameters of the task and initialization */ + // task_p = 1; + // task_n = A->shape().elements(); + // xstorage = VSL_SS_MATRIX_STORAGE_ROWS; + // min_est = max_est = A->data()[0]; + // /* Create a task */ + // vslsSSNewTask( &task, &task_p, &task_n, &xstorage, (float*)A->data(), 0, 0 ); + // /* Initialize the task parameters */ + // vslsSSEditTask( task, VSL_SS_ED_MIN, &min_est ); + // vslsSSEditTask( task, VSL_SS_ED_MAX, &max_est ); + // /* Compute the minimum and maximum values in observations */ + // vslsSSCompute( task, VSL_SS_MIN|VSL_SS_MAX, VSL_SS_METHOD_FAST ); + // /* Deallocate the task resources */ + + // vslSSDeleteTask( &task ); + + int elem = A->shape().elements(); + float* data = A->data(); + for (int ii = 0; ii < elem; ii++) { + if (data[ii] < min_est) min_est = data[ii]; + if (data[ii] > max_est) max_est = data[ii]; + } + + std::vector row_offset_buf(PackAWithQuantRowOffset::rowOffsetBufferSize()); + + float ascale = (max_est - min_est)/255; + int32_t azeropoint = (int32_t)(255 - max_est / ascale); + PackAWithQuantRowOffset packAN( + transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + transA ? k : m, + transA ? m : k, + A->data(), + transA ? m : k, + nullptr, /*buffer for packed matrix*/ + ascale, + azeropoint, + 1, /*groups*/ + row_offset_buf.data()); + + DoNothing doNothingObj{}; + ReQuantizeForFloat outputProcObj( + doNothingObj, + ascale, + bqScale, + azeropoint, + bqZeropoint, + packAN.getRowOffsetBuffer(), + col_offsets->data(), + nullptr, + n); + + // gemm + fbgemmPacked( + packAN, + *packedBint8, + C->data(), + (int32_t*)C->data(), + n, + outputProcObj, + 0, + 1); + + // std::cout << "lowp gemm: " << std::endl; + // for (int ii = 0; ii < n; ii++) { + // std::cout << C->data()[ii] << std::endl; + // } + // std::cout << std::endl; + +#endif + } else { + // row major + // keep the original mem + fbgemm::float16* pmat = packedPlaceholder.pmat_; + // retreive aux fields from the memory + uint64_t* packedmemSize = (uint64_t*)B->data(); + packedPlaceholder.size_ = packedmemSize[0]; + int* packedmemAux = (int*)(packedmemSize + 1); + packedPlaceholder.nrow_ = packedmemAux[0]; + packedPlaceholder.ncol_ = packedmemAux[1]; + packedPlaceholder.kernel_ncol_blocks_ = packedmemAux[2]; + packedPlaceholder.brow_ = packedmemAux[3]; + packedPlaceholder.bcol_ = packedmemAux[4]; + packedPlaceholder.last_brow_ = packedmemAux[5]; + packedPlaceholder.nbrow_ = packedmemAux[6]; + packedPlaceholder.nbcol_ = packedmemAux[7]; + + // packed matrix + packedPlaceholder.pmat_ = (fbgemm::float16*)(B->data() + 256); + + for(int i = 0; i < m; ++i) { + mkl_somatcopy('R', 'N', 1, n, 1, bias->data(), n, C->data() + n * i, n); + } + + if (true) { + // if (A->shape().size() == 4 && B->shape()[0] > 20480000) { +#ifdef _OPENMP +#pragma omp parallel +#endif + { +#ifdef _OPENMP + int num_threads = omp_get_num_threads(); + int tid = omp_get_thread_num(); +#else + int num_threads = 1; + int tid = 0; +#endif + fbgemm::cblas_gemm_compute(transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + (int)m, + A->data(), + packedPlaceholder, + 1, + C->data(), + tid, + num_threads); + } + } else { + fbgemm::cblas_gemm_compute(transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + (int)m, + A->data(), + packedPlaceholder, + 1, + C->data(), + 0, + 1); + } + + // if (B->shape().size() == 1 && B->shape()[0] >= 32000 * 512) { + // std::cout << "packed gemm: " << std::endl; + // for (int ii = 0; ii < n; ii++) { + // std::cout << C->data()[ii] << std::endl; + // } + // std::cout << std::endl; + // } + + // return back the original mem + packedPlaceholder.pmat_ = pmat; + } + + // std::cout << "A transposed: " << transA << std::endl; + // for (int i = 0; i < A->shape().size(); i++) { + // std::cout << "size " << i << ": " << A->shape()[i] << std::endl; + // } + // compute statistics for quantization + // if (A->shape().size() == 4 && B->shape()[0] > 20480000) { + // int bsize = A->shape().elements() / A->shape()[-1]; + // int hsize = A->shape()[3]; + // float mins[bsize] = {0}; + // float maxs[bsize] = {0}; + // float means[bsize] = {0}; + // float stds[bsize] = {0}; + // float* inmem = A->data(); + // for(int i = 0; i < bsize; i++) { + // for(int j = 0; j < hsize; j++) { + // float val = !transA ? inmem[i * hsize + j] : inmem[i + bsize * j]; + // if (val < mins[i]) + // mins[i] = val; + // if (val > maxs[i]) + // maxs[i] = val; + + // means[i] += val; + // stds[i] += val*val; + // } + // } + // for(int i = 0; i < bsize; i++) { + // std::cout << mins[i] << ", " << maxs[i] << ", " << means[i] << ", " << stds[i] << std::endl; + // } + // } +} +#else // USE_FBGEMM && MKL_FOUND +void PackFp32(marian::Tensor out, + const marian::Tensor in, + bool tranpose, + int nrow, + int ncol, + int kernel_ncol_blocks, + int brow, + int bcol, + int last_brow, + int nbrow, + int nbcol, + uint64_t packsize) { + // does nothing. supports only FBGEMM based packed gemm at this moment. +} +void GemmPackFp32(marian::Tensor C, + const marian::Tensor A, + const marian::Tensor B, + const marian::Tensor bias, + const int64_t m, + const int64_t n, + const int64_t k, + const float beta, + const int layout, + const int transA, + const int transB, + size_t idx) { + // does nothing. supports only FBGEMM based packed gemm at this moment. +} +#endif // USE_FBGEMM && MKL_FOUND + +// This operates on floats after processing so doesn't care about int8_t vs +// int16_t. +void AddBias(marian::Tensor C, const marian::Tensor Bias) { + float* y = C->data(); + const float* x = C->data(); + const float* bias = Bias->data(); + + int m = C->shape().elements() / C->shape()[-1]; + int n = C->shape()[-1]; +#ifdef __AVX512F__ + int n16 = n & ~15; +#else + int n4 = (n / 4) * 4; +#endif + + for(int j = 0; j < m; ++j) { + int i = 0; +#ifdef __AVX512F__ + for(; i < n16; i += 16) { + __m512 ai = _mm512_loadu_ps(x + j * n + i); + __m512 bi = _mm512_loadu_ps(bias + i); + __m512 yi = _mm512_add_ps(ai, bi); + _mm512_storeu_ps(y + j * n + i, yi); + } +#else + for(; i < n4; i += 4) { + __m128 ai = _mm_loadu_ps(x + j * n + i); + __m128 bi = _mm_loadu_ps(bias + i); + __m128 yi = _mm_add_ps(ai, bi); + _mm_storeu_ps(y + j * n + i, yi); + } +#endif + for(; i < n; i++) { + y[j * n + i] = x[j * n + i] + bias[i]; + } + } + + // std::cout << "Output: " << std::endl; + // for (int ii = 0; ii < n; ii++) { + // std::cout << y[ii] << ","; + // } + // std::cout << std::endl; +} + +} // namespace pack +} // namespace cpu +} // namespace marian diff --git a/src/tensors/cpu/sharp/packed_gemm.h b/src/tensors/cpu/sharp/packed_gemm.h new file mode 100644 index 000000000..428190254 --- /dev/null +++ b/src/tensors/cpu/sharp/packed_gemm.h @@ -0,0 +1,39 @@ +#pragma once + +#include "tensors/tensor.h" + +namespace marian { +namespace cpu { +namespace pack { + +void PackFp32(marian::Tensor out, + const marian::Tensor in, + bool transpose, + int nrow, + int ncol, + int kernel_ncol_blocks, + int brow, + int bcol, + int last_brow, + int nbrow, + int nbcol, + uint64_t packsize); + +void GemmPackFp32(marian::Tensor C, + const marian::Tensor A, + const marian::Tensor B, + const marian::Tensor bias, + const int64_t m, + const int64_t n, + // const int64_t k, + // const float beta = 1, + // const int layout = 0, + const int transA = 0); + //const int transB = 0, + //const size_t idx = 0); + +void AddBias(marian::Tensor C, const marian::Tensor Bias); + +} // namespace pack +} // namespace cpu +} // namespace marian diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index 0428ba8df..48885a15d 100644 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -10,6 +10,10 @@ #include "functional/functional.h" #include "functional/tensor.h" +#if MKL_FOUND +#include +#endif + namespace marian { namespace cpu { @@ -186,6 +190,61 @@ void Transpose0213(Tensor out, Tensor in) { } } +template +void Transposexxx3(Tensor out, Tensor in, const std::vector& vAxis) { +#if MKL_FOUND + int innermost = in->shape()[-1]; + + int l1 = in->shape()[vAxis[0]]; + int l2 = in->shape()[vAxis[1]]; + int l3 = in->shape()[vAxis[2]]; + + int oi = 0, oj = 0, ok = 0; +#pragma omp parallel for + for(int k = 0; k < l1; ++k) { + int shift = k * l2 * l3; + for(int j = 0; j < l2; ++j) { + for(int i = 0; i < l3; ++i) { + if(vAxis[0] == 0) { + if(vAxis[1] == 1) { + oi = i; oj = j; ok = k; + } else { + oi = j; oj = i; ok = k; + } + } else if(vAxis[0] == 1) { + if(vAxis[1] == 0) { + oi = i; oj = k; ok = j; + } else { + oi = j; oj = k; ok = i; + } + } else { + if(vAxis[1] == 0) { + oi = k; oj = i; ok = j; + } else { + oi = k; oj = j; ok = i; + } + } + int src = ok * in->shape()[1] * in->shape()[2] + oj * in->shape()[2] + oi; + int dst = l3 * j + shift + i; + + const float* inRow = in->data() + src * innermost; + float* outRow = out->data() + dst * innermost; + + if(!add) { + mkl_somatcopy('R', 'N', 1, innermost, 1.0f, inRow, innermost, outRow, innermost); + } else { + for(int ii = 0; ii < innermost; ++ii) { + outRow[ii] += inRow[ii]; + } + } + } + } + } +#else + // it shouldn't come into here. This function is called only when MKL is available. +#endif // MKL_FOUND +} + inline void transpose4x4_SSE(const float* A, float* B, const int lda, @@ -262,6 +321,10 @@ void TransposeGeneric(Tensor out, Tensor in, const std::vector& vAxis) { void TransposeND(Tensor out, Tensor in, const std::vector& vAxis) { if(vAxis == std::vector({0, 2, 1, 3})) Transpose0213(out, in); +#if MKL_FOUND + else if(vAxis.size() == 4 && vAxis[3] == 3) + Transposexxx3(out, in, vAxis); +#endif // MKL_FOUND else if(vAxis == std::vector({1, 0}) && in->shape()[-1] % 16 == 0 && in->shape()[-2] % 16 == 0) Transpose10(out, in); diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index d28b467bc..8106f488c 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -49,8 +49,8 @@ false $(ExecutablePath) $(SolutionDir)$(Platform)\$(Configuration)\Marian\ - %CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;%MKL_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); - %CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 + %MKL_PATH%\include;..\src\3rd_party\fbgemm\include;%CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); + deps;%CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 @@ -104,12 +104,12 @@ MaxSpeed true true - CUDA_FOUND=1; MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) + FBGEMM_STATIC;USE_FBGEMM=1;CUDA_FOUND=1; MKL_FOUND=1; MPI_FOUND=1; BLAS_FOUND=1; MKL_ILP64; WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions) true Speed - /d2Zi+ /bigobj %(AdditionalOptions) + /d2Zi+ /bigobj %(AdditionalOptions) /arch:AVX2 true - MultiThreadedDLL + MultiThreaded MultiThreaded AnySuitable true @@ -121,7 +121,7 @@ true true true - cudart_static.lib;cublas.lib;cusparse.lib;curand.lib;zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies) + cudart_static.lib;cublas.lib;cusparse.lib;curand.lib;zlib.lib;msmpi.lib;mkl_intel_ilp64.lib;mkl_sequential.lib;mkl_core.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;shlwapi.lib;%(AdditionalDependencies);fbgemm.lib;cpuinfo.lib;clog.lib 100000000 true @@ -346,6 +346,19 @@ + + + + + + + + + + + + + true @@ -612,6 +625,7 @@ + @@ -951,7 +965,9 @@ + + diff --git a/vs/Marian.vcxproj.filters b/vs/Marian.vcxproj.filters index d77aecd2a..eaa6cfe25 100755 --- a/vs/Marian.vcxproj.filters +++ b/vs/Marian.vcxproj.filters @@ -490,6 +490,9 @@ tensors\gpu + + tensors\cpu\sharp + @@ -1555,6 +1558,51 @@ tensors\gpu + + tensors\cpu + + + tensors\cpu\sharp + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + + + 3rd_party\fbgemm\include\fbgemm + @@ -1722,6 +1770,15 @@ {a86d650a-2268-43d9-9d74-cb17cd6b534b} + + {4bb88f6d-7ddf-41e0-91be-a43dbcd0e9b0} + + + {6c2bef00-97a0-4881-a6f0-ded54b8520bf} + + + {95f7ce7c-c649-4d57-8d2a-d724bd75fe84} + From e8ca9a37561efba2c86806518fd52d908e5ff1b7 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Tue, 18 Jun 2019 17:09:14 -0700 Subject: [PATCH 501/838] Update some fixed --- cmake/FindSSE.cmake | 23 ++++++++++++++++++++++- src/tensors/cpu/tensor_operators.cpp | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/cmake/FindSSE.cmake b/cmake/FindSSE.cmake index c152dd74f..e6d3e9caf 100644 --- a/cmake/FindSSE.cmake +++ b/cmake/FindSSE.cmake @@ -56,6 +56,14 @@ IF(CMAKE_SYSTEM_NAME MATCHES "Linux") ELSE (AVX2_TRUE) set(AVX2_FOUND false CACHE BOOL "AVX2 available on host") ENDIF (AVX2_TRUE) + + STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE ${CPUINFO}) + STRING(COMPARE EQUAL "avx512" "${SSE_THERE}" AVX512_TRUE) + IF (AVX512_TRUE) + set(AVX512_FOUND true CACHE BOOL "AVX512 available on host") + ELSE (AVX512_TRUE) + set(AVX512_FOUND false CACHE BOOL "AVX512 available on host") + ENDIF (AVX512_TRUE) ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Darwin") EXEC_PROGRAM("/usr/sbin/sysctl -n machdep.cpu.features" OUTPUT_VARIABLE @@ -108,6 +116,14 @@ ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Darwin") ELSE (AVX2_TRUE) set(AVX2_FOUND false CACHE BOOL "AVX2 available on host") ENDIF (AVX2_TRUE) + + STRING(REGEX REPLACE "^.*(avx512).*$" "\\1" SSE_THERE ${CPUINFO}) + STRING(COMPARE EQUAL "avx512" "${SSE_THERE}" AVX512_TRUE) + IF (AVX512_TRUE) + set(AVX512_FOUND true CACHE BOOL "AVX512 available on host") + ELSE (AVX512_TRUE) + set(AVX512_FOUND false CACHE BOOL "AVX512 available on host") + ENDIF (AVX512_TRUE) ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Windows") # TODO @@ -117,6 +133,7 @@ ELSEIF(CMAKE_SYSTEM_NAME MATCHES "Windows") set(SSE4_1_FOUND false CACHE BOOL "SSE4.1 available on host") set(AVX_FOUND false CACHE BOOL "AVX available on host") set(AVX2_FOUND false CACHE BOOL "AVX2 available on host") + set(AVX512_FOUND false CACHE BOOL "AVX512 available on host") ELSE(CMAKE_SYSTEM_NAME MATCHES "Linux") set(SSE2_FOUND true CACHE BOOL "SSE2 available on host") set(SSE3_FOUND false CACHE BOOL "SSE3 available on host") @@ -124,6 +141,7 @@ ELSE(CMAKE_SYSTEM_NAME MATCHES "Linux") set(SSE4_1_FOUND false CACHE BOOL "SSE4.1 available on host") set(AVX_FOUND false CACHE BOOL "AVX available on host") set(AVX2_FOUND false CACHE BOOL "AVX2 available on host") + set(AVX512_FOUND false CACHE BOOL "AVX512 available on host") ENDIF(CMAKE_SYSTEM_NAME MATCHES "Linux") if(NOT SSE2_FOUND) @@ -144,5 +162,8 @@ endif(NOT AVX_FOUND) if(NOT AVX2_FOUND) MESSAGE(STATUS "Could not find hardware support for AVX2 on this machine.") endif(NOT AVX2_FOUND) +if(NOT AVX512_FOUND) + MESSAGE(STATUS "Could not find hardware support for AVX512 on this machine.") +endif(NOT AVX512_FOUND) -mark_as_advanced(SSE2_FOUND SSE3_FOUND SSSE3_FOUND SSE4_1_FOUND, AVX_FOUND, AVX2_FOUND) +mark_as_advanced(SSE2_FOUND SSE3_FOUND SSSE3_FOUND SSE4_1_FOUND, AVX_FOUND, AVX2_FOUND, AVX512_FOUND) diff --git a/src/tensors/cpu/tensor_operators.cpp b/src/tensors/cpu/tensor_operators.cpp index 48885a15d..e6be8add0 100644 --- a/src/tensors/cpu/tensor_operators.cpp +++ b/src/tensors/cpu/tensor_operators.cpp @@ -191,7 +191,7 @@ void Transpose0213(Tensor out, Tensor in) { } template -void Transposexxx3(Tensor out, Tensor in, const std::vector& vAxis) { +void TransposeFirst3In4(Tensor out, Tensor in, const std::vector& vAxis) { #if MKL_FOUND int innermost = in->shape()[-1]; @@ -323,7 +323,7 @@ void TransposeND(Tensor out, Tensor in, const std::vector& vAxis) { Transpose0213(out, in); #if MKL_FOUND else if(vAxis.size() == 4 && vAxis[3] == 3) - Transposexxx3(out, in, vAxis); + TransposeFirst3In4(out, in, vAxis); #endif // MKL_FOUND else if(vAxis == std::vector({1, 0}) && in->shape()[-1] % 16 == 0 && in->shape()[-2] % 16 == 0) From a71bcb5f7c9ae3b0514fbc53d85ad4e6349a8242 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Tue, 18 Jun 2019 17:34:06 -0700 Subject: [PATCH 502/838] clean up code --- src/graph/expression_operators.cpp | 12 +- src/tensors/cpu/expanded_gemm.h | 78 ++--- src/tensors/cpu/sharp/packed_gemm.cpp | 476 ++++---------------------- src/tensors/cpu/sharp/packed_gemm.h | 9 +- 4 files changed, 101 insertions(+), 474 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index 42bb65e20..d7bfd19c2 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -444,10 +444,10 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { }; auto alg1 = [=]() { - auto packed = cpu::pack::pack(b, cpu::pack::PackMatrix::B, transB, clipValue); + auto packed = cpu::variant::pack(b, cpu::variant::PackMatrix::B, transB, clipValue); return rec1( - cpu::pack::affine( + cpu::variant::affine( clip(a, clipValue), packed, b->shape(), bias, @@ -509,12 +509,12 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { } else { if(b->memoize()) { - auto packed = cpu::pack::pack(b, cpu::pack::PackMatrix::B, transB, clipValue); + auto packed = cpu::variant::pack(b, cpu::variant::PackMatrix::B, transB, clipValue); // auto packed = transB ? - // cpu::pack::pack(transpose(b), cpu::pack::PackMatrix::B, false, clipValue) : - // cpu::pack::pack(b, cpu::pack::PackMatrix::B, false, clipValue); + // cpu::variant::pack(transpose(b), cpu::pack::PackMatrix::B, false, clipValue) : + // cpu::variant::pack(b, cpu::pack::PackMatrix::B, false, clipValue); - return cpu::pack::affine( + return cpu::variant::affine( clip(a, clipValue), packed, b->shape(), bias, diff --git a/src/tensors/cpu/expanded_gemm.h b/src/tensors/cpu/expanded_gemm.h index 33b677945..03e58ae43 100644 --- a/src/tensors/cpu/expanded_gemm.h +++ b/src/tensors/cpu/expanded_gemm.h @@ -10,7 +10,7 @@ using namespace fbgemm; namespace marian { namespace cpu { -namespace pack { +namespace variant { enum class PackMatrix : uint8_t { A = 0x00, @@ -71,23 +71,17 @@ struct PackNodeOp : public UnaryNodeOp { // Should be 2D - weight matrix ABORT_IF(shapeMat.size() != 2, "Weight Matrix should be 2D"); - if (true) { - // if (shapeMat[0] < 32000 && shapeMat[1] < 32000) { - nrow_ = transpose ? shapeMat[1] : shapeMat[0]; - ncol_ = transpose ? shapeMat[0] : shapeMat[1]; - kernel_ncol_blocks_ = 2; - brow_ = 512; - bcol_ = 8 * kernel_ncol_blocks_; - last_brow_ = nrow_ % brow_ == 0 ? brow_ : nrow_ % brow_; - nbrow_ = nrow_ % brow_ == 0 ? nrow_ / brow_ : (nrow_ + brow_) / brow_; - nbcol_ = ncol_ % bcol_ == 0 ? ncol_ / bcol_ : (ncol_ + bcol_) / bcol_; - const int padding = 1024; // required by sw pipelined kernels - const int specialMem = 256; - packsize_ = ((nbrow_ * brow_) * (nbcol_ * bcol_)) * sizeof(fbgemm::float16) + padding + specialMem; - } else { - // use int 8 implementation - packsize_ = 1; - } + nrow_ = transpose ? shapeMat[1] : shapeMat[0]; + ncol_ = transpose ? shapeMat[0] : shapeMat[1]; + kernel_ncol_blocks_ = 2; + brow_ = 512; + bcol_ = 8 * kernel_ncol_blocks_; + last_brow_ = nrow_ % brow_ == 0 ? brow_ : nrow_ % brow_; + nbrow_ = nrow_ % brow_ == 0 ? nrow_ / brow_ : (nrow_ + brow_) / brow_; + nbcol_ = ncol_ % bcol_ == 0 ? ncol_ / bcol_ : (ncol_ + bcol_) / bcol_; + const int padding = 1024; // required by sw pipelined kernels + const int specialMem = 256; + packsize_ = ((nbrow_ * brow_) * (nbcol_ * bcol_)) * sizeof(fbgemm::float16) + padding + specialMem; Shape outShape({(int)packsize_}); @@ -147,39 +141,15 @@ class AffineNodeOp : public NaryNodeOp { } NodeOps forwardOps() override { - // if (n_ < 32000) { - if (true) { - return { - NodeOp(GemmPackFp32(val_, - child(0)->val(), - child(1)->val(), - child(2)->val(), - m_, - n_, - //k_, - //1, - //0, - transA_)) - //transB_, - //idx_)) - }; - } else { - return { - NodeOp(GemmPackFp32(val_, - child(0)->val(), - child(1)->val(), - child(2)->val(), - m_, - n_, - //k_, - //1, - //0, - transA_); - //transB_, - //idx_); - AddBias(val_, child(2)->val())) - }; - } + return { + NodeOp(GemmPackFp32(val_, + child(0)->val(), + child(1)->val(), + child(2)->val(), + m_, + n_, + transA_)) + }; } NodeOps backwardOps() override { @@ -192,13 +162,13 @@ class AffineNodeOp : public NaryNodeOp { static inline Expr affine(Expr a, Expr b, Shape bShape, Expr c, bool transA, bool transB, float scalar) { std::vector nodes = {a, b, c}; - return Expression(nodes, bShape, transA, transB, scalar); + return Expression(nodes, bShape, transA, transB, scalar); } static inline Expr pack(Expr a, PackMatrix packMat, bool transpose, float clipValue) { - return Expression(a, packMat, transpose, clipValue); + return Expression(a, packMat, transpose, clipValue); } -} // namespace pack +} // namespace variant } // namespace cpu } // namespace marian diff --git a/src/tensors/cpu/sharp/packed_gemm.cpp b/src/tensors/cpu/sharp/packed_gemm.cpp index b6b7937f5..e3840b8c7 100644 --- a/src/tensors/cpu/sharp/packed_gemm.cpp +++ b/src/tensors/cpu/sharp/packed_gemm.cpp @@ -33,45 +33,12 @@ using namespace fbgemm; namespace marian { namespace cpu { -namespace pack { - -//static float packingTime = 0; +namespace variant { #if (USE_FBGEMM && MKL_FOUND) // initialize with a dummy static PackedGemmMatrixFP16 packedPlaceholder(1, 1, 1, 1, 1, 1, 1, 1); -// temporary variable for int 8 -//static PackBMatrix* packedBint8 = nullptr; -// transformer base wmt 2017 -// static float bqScale = 0.39/128; -// static int32_t bqZeropoint = 0; -// old student de-en -//static float* bqScale; -//static int32_t* bqZeropoint; -// static float bqScale = 0.683/106; -// static int32_t bqZeropoint = 21; -// static float bqScale = 0.9672/128; -// static int32_t bqZeropoint = 0; -//static std::vector* col_offsets = nullptr; - -//inline void col_offsets_with_zero_pt_s8acc32_ref( -// bool transpose, -// int K, -// int N, -// const int8_t* Bint8, -// const int32_t* B_zero_point, -// int32_t* col_offsets, -// int ncols_per_quant_group) { -// for (int n = 0; n < N; ++n) { -// int32_t sum = 0; -// for (int k = 0; k < K; ++k) { -// sum += transpose ? Bint8[k + n * K] : Bint8[k * N + n]; -// } -// col_offsets[n] = sum - B_zero_point[n / ncols_per_quant_group] * K; -// } -//} - // This is copied from FBGEMM code // A better way? // blocked row-major format address arithmetic @@ -111,206 +78,39 @@ void PackFp32(marian::Tensor out, uint64_t packsize) { //auto t_start = std::chrono::high_resolution_clock::now(); // for the last embedding layer, pack it into int8 - // if (in->shape().size() == 2 && (in->shape()[0] >= 32000 || in->shape()[1] >= 32000)) { - if (false) { -#if 0 - if (packedBint8 == nullptr) { - int k = transpose ? in->shape()[1] : in->shape()[0]; - int n = transpose ? in->shape()[0] : in->shape()[1]; - // std::cout << "transpose: " << transpose << ", k: " << k << ", n: " << n << std::endl; - // std::cout << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - // two steps - // 0. quantize --> this should be done outside - int len = in->shape()[0]*in->shape()[1]; - - // 0-1. collect stats for each class - bqScale = new float[n]; - bqZeropoint = new int32_t[n]; - - // int numBin = 20; - // float denum = 2/(float)numBin; - - // int hist[numBin] = { 0, }; - - // Transposed only - float* data = in->data(); - float val = 0; - for (int jj = 0; jj < n; jj++) { - float min = 1000000, max = -10000000; - for (int ii = 0; ii < k; ii++) { - val = data[jj*k + ii]; - if (val < min) min = val; - if (val > max) max = val; - // hist[(int)((val + 1)/denum)]++; - } - bqScale[jj] = (max - min)/255; - bqZeropoint[jj] = (int32_t)(127 - max / bqScale[jj]); - // bqScale[jj] = (0.3 + 0.4)/255; - // bqZeropoint[jj] = (int32_t)(127 - 0.3 / bqScale[jj]); - } - - // std::cout << "hist: "; - // for (int ii = 0; ii < numBin; ii++) { - // std::cout << hist[ii] << ", "; - // } - // std::cout << std::endl; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - //int8_t quantized[len]; // aligned malloc? - int8_t* quantized = (int8_t*)aligned_alloc(256, len); - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - for (int ii = 0; ii < n; ii++) { - TensorQuantizationParams bQuantParam; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - bQuantParam.scale = bqScale[ii]; - bQuantParam.zero_point = bqZeropoint[ii]; - bQuantParam.precision = 8; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - - fbgemm::Quantize(in->data() + ii * k, quantized + ii * k, k, bQuantParam); - } - // std::cout << "original" << std::endl; - // for (int ii = 0; ii < n; ii++) { - // for (int jj = 0; jj < 1; jj++) { - // std::cout << in->data()[ii * k + jj] << ","; - // } - // std::cout << std::endl; - // } - // std::cout << "quantized" << std::endl; - // for (int ii = 0; ii < 1; ii++) { - // for (int jj = 0; jj < k; jj++) { - // std::cout << (int32_t)quantized[ii * k + jj] << ","; - // } - // std::cout << std::endl; - // } - // 1. compute column offsets - col_offsets = new std::vector(n); - col_offsets_with_zero_pt_s8acc32_ref(transpose, k, n, quantized, bqZeropoint, col_offsets->data(), 1); - // for (int ii = 0; ii < n; ii++) { - // std::cout << (int32_t)col_offsets->data()[ii] << ","; - // } - // std::cout << std::endl; - // std::cout << "calc offset done" << std::endl; - // 2. packing - // uint8_t* packedmem = aligned_alloc(256, len); - packedBint8 = new PackBMatrix(transpose ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, - k, // in->shape()[0], - n, // in->shape()[1], - quantized, - in->shape()[1]); - // std::cout << "packing B done" << std::endl; - // int k = transpose ? in->shape()[1] : in->shape()[0]; - // int n = transpose ? in->shape()[0] : in->shape()[1]; - // std::cout << "transpose: " << transpose << ", k: " << k << ", n: " << n << std::endl; - // std::cout << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - // // two steps - // // 0. quantize --> this should be done outside - // int len = in->shape()[0]*in->shape()[1]; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - // //int8_t quantized[len]; // aligned malloc? - // int8_t* quantized = (int8_t*)aligned_alloc(256, len); - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - // TensorQuantizationParams bQuantParam; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - // bQuantParam.scale = bqScale; - // bQuantParam.zero_point = bqZeropoint; - // bQuantParam.precision = 8; - // std::cout << "len: " << len << ", bqScale: " << bqScale << ", bqZeropoint: " << bqZeropoint << std::endl; - - // fbgemm::Quantize(in->data(), quantized, len, bQuantParam); - // std::cout << "original" << std::endl; - // // for (int ii = 0; ii < n; ii++) { - // // for (int jj = 0; jj < 1; jj++) { - // // std::cout << in->data()[ii * k + jj] << ","; - // // } - // // std::cout << std::endl; - // // } - // std::cout << "quantized" << std::endl; - // // for (int ii = 0; ii < 1; ii++) { - // // for (int jj = 0; jj < k; jj++) { - // // std::cout << (int32_t)quantized[ii * k + jj] << ","; - // // } - // // std::cout << std::endl; - // // } - // // 1. compute column offsets - // col_offsets = new std::vector(n); - // col_offsets_with_zero_pt_s8acc32_ref(k, n, n, quantized, &bqZeropoint, col_offsets->data(), n); - // // for (int ii = 0; ii < n; ii++) { - // // std::cout << (int32_t)col_offsets->data()[ii] << ","; - // // } - // std::cout << std::endl; - // std::cout << "calc offset done" << std::endl; - // 2. packing - // uint8_t* packedmem = aligned_alloc(256, len); - // packedBint8 = new PackBMatrix(transpose ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, - // in->shape()[0], - // in->shape()[1], - // quantized, - // transpose ? in->shape()[1] : in->shape()[0]); - // std::cout << "packing B done" << std::endl; - } -#endif - } else { - // initialize memory - uint8_t* outmemorg = out->data(); - for(auto i = 0; i < packsize; i++) { - outmemorg[i] = 0; - } - // save the other auxiliary variables - uint64_t* auxmemsize = (uint64_t*)outmemorg; - auxmemsize[0] = packsize; - int* auxmem = (int*)(auxmemsize + 1); - auxmem[0] = nrow; - auxmem[1] = ncol; - auxmem[2] = kernel_ncol_blocks; - auxmem[3] = brow; - auxmem[4] = bcol; - auxmem[5] = last_brow; - auxmem[6] = nbrow; - auxmem[7] = nbcol; - // cast to float16 - fbgemm::float16* outmem = (fbgemm::float16*)(outmemorg + 256); - fbgemm::float16* dummy = new fbgemm::float16; - // pack the matrix - float* inmem = in->data(); - for(int i = 0; i < nrow; i++) { - for(int j = 0; j < ncol; j++) { - outmem[addr(i, j, brow, bcol, nbrow, nbcol, last_brow)] - = tconv(!transpose ? inmem[i * ncol + j] : inmem[i + nrow * j], *dummy); - } + // initialize memory + uint8_t* outmemorg = out->data(); + for(auto i = 0; i < packsize; i++) { + outmemorg[i] = 0; + } + // save the other auxiliary variables + uint64_t* auxmemsize = (uint64_t*)outmemorg; + auxmemsize[0] = packsize; + int* auxmem = (int*)(auxmemsize + 1); + auxmem[0] = nrow; + auxmem[1] = ncol; + auxmem[2] = kernel_ncol_blocks; + auxmem[3] = brow; + auxmem[4] = bcol; + auxmem[5] = last_brow; + auxmem[6] = nbrow; + auxmem[7] = nbcol; + // cast to float16 + fbgemm::float16* outmem = (fbgemm::float16*)(outmemorg + 256); + fbgemm::float16* dummy = new fbgemm::float16; + // pack the matrix + float* inmem = in->data(); + for(int i = 0; i < nrow; i++) { + for(int j = 0; j < ncol; j++) { + outmem[addr(i, j, brow, bcol, nbrow, nbcol, last_brow)] + = tconv(!transpose ? inmem[i * ncol + j] : inmem[i + nrow * j], *dummy); } - delete dummy; - - //auto t_end = std::chrono::high_resolution_clock::now(); - //packingTime += (float) std::chrono::duration(t_end-t_start).count(); - //std::cout << "Packing time: " << packingTime << std::endl; } + delete dummy; - // std::cout << "B transposed: " << transpose << std::endl; - // for (int i = 0; i < in->shape().size(); i++) { - // std::cout << "size " << i << ": " << in->shape()[i] << std::endl; - // } - // // compute statistics for quantization - // if (in->shape().size() == 2 && (in->shape()[0] >= 32000 || in->shape()[1] >= 32000)) { - // float mins[ncol] = {0}; - // float maxs[ncol] = {0}; - // float means[ncol] = {0}; - // float stds[ncol] = {0}; - // for(int i = 0; i < nrow; i++) { - // for(int j = 0; j < ncol; j++) { - // float val = !transpose ? in->data()[i * ncol + j] : in->data()[i + nrow * j]; - // if (val < mins[j]) - // mins[j] = val; - // if (val > maxs[j]) - // maxs[j] = val; - - // means[j] += val; - // stds[j] += val*val; - // } - // } - // for(int j = 0; j < ncol; j++) { - // std::cout << mins[j] << ", " << maxs[j] << ", " << means[j] << ", " << stds[j] << std::endl; - // } - // } + //auto t_end = std::chrono::high_resolution_clock::now(); + //packingTime += (float) std::chrono::duration(t_end-t_start).count(); + //std::cout << "Packing time: " << packingTime << std::endl; } void GemmPackFp32(marian::Tensor C, @@ -320,190 +120,52 @@ void GemmPackFp32(marian::Tensor C, const int64_t m, const int64_t n, int transA) { - // use int 8 packed gemm - // if (A->shape().size() == 4 && B->shape().size() == 1 && B->shape()[0] == 1) { - if (false) { - // quantize & pack A - // transformer base wmt 2017 - // float ascale = 7.8/104; - // int32_t azeropoint = 151; - // old student de-en - // float ascale = 14.85/117; - // int32_t azeropoint = 138; - -#if 0 - // compute range - float min_est=1000000, max_est=-10000000; - // VSLSSTaskPtr task; - // MKL_INT task_p, task_n, xstorage; - - // /* Parameters of the task and initialization */ - // task_p = 1; - // task_n = A->shape().elements(); - // xstorage = VSL_SS_MATRIX_STORAGE_ROWS; - // min_est = max_est = A->data()[0]; - // /* Create a task */ - // vslsSSNewTask( &task, &task_p, &task_n, &xstorage, (float*)A->data(), 0, 0 ); - // /* Initialize the task parameters */ - // vslsSSEditTask( task, VSL_SS_ED_MIN, &min_est ); - // vslsSSEditTask( task, VSL_SS_ED_MAX, &max_est ); - // /* Compute the minimum and maximum values in observations */ - // vslsSSCompute( task, VSL_SS_MIN|VSL_SS_MAX, VSL_SS_METHOD_FAST ); - // /* Deallocate the task resources */ - - // vslSSDeleteTask( &task ); - - int elem = A->shape().elements(); - float* data = A->data(); - for (int ii = 0; ii < elem; ii++) { - if (data[ii] < min_est) min_est = data[ii]; - if (data[ii] > max_est) max_est = data[ii]; - } - - std::vector row_offset_buf(PackAWithQuantRowOffset::rowOffsetBufferSize()); - - float ascale = (max_est - min_est)/255; - int32_t azeropoint = (int32_t)(255 - max_est / ascale); - PackAWithQuantRowOffset packAN( - transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, - transA ? k : m, - transA ? m : k, - A->data(), - transA ? m : k, - nullptr, /*buffer for packed matrix*/ - ascale, - azeropoint, - 1, /*groups*/ - row_offset_buf.data()); - - DoNothing doNothingObj{}; - ReQuantizeForFloat outputProcObj( - doNothingObj, - ascale, - bqScale, - azeropoint, - bqZeropoint, - packAN.getRowOffsetBuffer(), - col_offsets->data(), - nullptr, - n); - - // gemm - fbgemmPacked( - packAN, - *packedBint8, - C->data(), - (int32_t*)C->data(), - n, - outputProcObj, - 0, - 1); - - // std::cout << "lowp gemm: " << std::endl; - // for (int ii = 0; ii < n; ii++) { - // std::cout << C->data()[ii] << std::endl; - // } - // std::cout << std::endl; - -#endif - } else { - // row major - // keep the original mem - fbgemm::float16* pmat = packedPlaceholder.pmat_; - // retreive aux fields from the memory - uint64_t* packedmemSize = (uint64_t*)B->data(); - packedPlaceholder.size_ = packedmemSize[0]; - int* packedmemAux = (int*)(packedmemSize + 1); - packedPlaceholder.nrow_ = packedmemAux[0]; - packedPlaceholder.ncol_ = packedmemAux[1]; - packedPlaceholder.kernel_ncol_blocks_ = packedmemAux[2]; - packedPlaceholder.brow_ = packedmemAux[3]; - packedPlaceholder.bcol_ = packedmemAux[4]; - packedPlaceholder.last_brow_ = packedmemAux[5]; - packedPlaceholder.nbrow_ = packedmemAux[6]; - packedPlaceholder.nbcol_ = packedmemAux[7]; - - // packed matrix - packedPlaceholder.pmat_ = (fbgemm::float16*)(B->data() + 256); - - for(int i = 0; i < m; ++i) { - mkl_somatcopy('R', 'N', 1, n, 1, bias->data(), n, C->data() + n * i, n); - } + // row major + // keep the original mem + fbgemm::float16* pmat = packedPlaceholder.pmat_; + // retreive aux fields from the memory + uint64_t* packedmemSize = (uint64_t*)B->data(); + packedPlaceholder.size_ = packedmemSize[0]; + int* packedmemAux = (int*)(packedmemSize + 1); + packedPlaceholder.nrow_ = packedmemAux[0]; + packedPlaceholder.ncol_ = packedmemAux[1]; + packedPlaceholder.kernel_ncol_blocks_ = packedmemAux[2]; + packedPlaceholder.brow_ = packedmemAux[3]; + packedPlaceholder.bcol_ = packedmemAux[4]; + packedPlaceholder.last_brow_ = packedmemAux[5]; + packedPlaceholder.nbrow_ = packedmemAux[6]; + packedPlaceholder.nbcol_ = packedmemAux[7]; + + // packed matrix + packedPlaceholder.pmat_ = (fbgemm::float16*)(B->data() + 256); + + for(int i = 0; i < m; ++i) { + mkl_somatcopy('R', 'N', 1, n, 1, bias->data(), n, C->data() + n * i, n); + } - if (true) { - // if (A->shape().size() == 4 && B->shape()[0] > 20480000) { #ifdef _OPENMP #pragma omp parallel #endif - { + { #ifdef _OPENMP - int num_threads = omp_get_num_threads(); - int tid = omp_get_thread_num(); + int num_threads = omp_get_num_threads(); + int tid = omp_get_thread_num(); #else - int num_threads = 1; - int tid = 0; + int num_threads = 1; + int tid = 0; #endif - fbgemm::cblas_gemm_compute(transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, - (int)m, - A->data(), - packedPlaceholder, - 1, - C->data(), - tid, - num_threads); - } - } else { - fbgemm::cblas_gemm_compute(transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, - (int)m, - A->data(), - packedPlaceholder, - 1, - C->data(), - 0, - 1); - } - - // if (B->shape().size() == 1 && B->shape()[0] >= 32000 * 512) { - // std::cout << "packed gemm: " << std::endl; - // for (int ii = 0; ii < n; ii++) { - // std::cout << C->data()[ii] << std::endl; - // } - // std::cout << std::endl; - // } - - // return back the original mem - packedPlaceholder.pmat_ = pmat; + fbgemm::cblas_gemm_compute(transA ? matrix_op_t::Transpose : matrix_op_t::NoTranspose, + (int)m, + A->data(), + packedPlaceholder, + 1, + C->data(), + tid, + num_threads); } - // std::cout << "A transposed: " << transA << std::endl; - // for (int i = 0; i < A->shape().size(); i++) { - // std::cout << "size " << i << ": " << A->shape()[i] << std::endl; - // } - // compute statistics for quantization - // if (A->shape().size() == 4 && B->shape()[0] > 20480000) { - // int bsize = A->shape().elements() / A->shape()[-1]; - // int hsize = A->shape()[3]; - // float mins[bsize] = {0}; - // float maxs[bsize] = {0}; - // float means[bsize] = {0}; - // float stds[bsize] = {0}; - // float* inmem = A->data(); - // for(int i = 0; i < bsize; i++) { - // for(int j = 0; j < hsize; j++) { - // float val = !transA ? inmem[i * hsize + j] : inmem[i + bsize * j]; - // if (val < mins[i]) - // mins[i] = val; - // if (val > maxs[i]) - // maxs[i] = val; - - // means[i] += val; - // stds[i] += val*val; - // } - // } - // for(int i = 0; i < bsize; i++) { - // std::cout << mins[i] << ", " << maxs[i] << ", " << means[i] << ", " << stds[i] << std::endl; - // } - // } + // return back the original mem + packedPlaceholder.pmat_ = pmat; } #else // USE_FBGEMM && MKL_FOUND void PackFp32(marian::Tensor out, @@ -580,6 +242,6 @@ void AddBias(marian::Tensor C, const marian::Tensor Bias) { // std::cout << std::endl; } -} // namespace pack +} // namespace variant } // namespace cpu } // namespace marian diff --git a/src/tensors/cpu/sharp/packed_gemm.h b/src/tensors/cpu/sharp/packed_gemm.h index 428190254..4aef905d5 100644 --- a/src/tensors/cpu/sharp/packed_gemm.h +++ b/src/tensors/cpu/sharp/packed_gemm.h @@ -4,7 +4,7 @@ namespace marian { namespace cpu { -namespace pack { +namespace variant { void PackFp32(marian::Tensor out, const marian::Tensor in, @@ -25,15 +25,10 @@ void GemmPackFp32(marian::Tensor C, const marian::Tensor bias, const int64_t m, const int64_t n, - // const int64_t k, - // const float beta = 1, - // const int layout = 0, const int transA = 0); - //const int transB = 0, - //const size_t idx = 0); void AddBias(marian::Tensor C, const marian::Tensor Bias); -} // namespace pack +} // namespace variant } // namespace cpu } // namespace marian From 93b752c34968db0cd989899f131ccf4426002542 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Tue, 18 Jun 2019 17:52:21 -0700 Subject: [PATCH 503/838] fix for the case FBGEMM is not used --- src/graph/expression_operators.cpp | 9 +++++++++ src/tensors/cpu/sharp/packed_gemm.cpp | 9 ++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/graph/expression_operators.cpp b/src/graph/expression_operators.cpp index d7bfd19c2..88a6222de 100755 --- a/src/graph/expression_operators.cpp +++ b/src/graph/expression_operators.cpp @@ -508,6 +508,7 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { return tuner->run(); } else { +#if USE_FBGEMM if(b->memoize()) { auto packed = cpu::variant::pack(b, cpu::variant::PackMatrix::B, transB, clipValue); // auto packed = transB ? @@ -533,6 +534,14 @@ Expr affine(Expr a, Expr b, Expr bias, bool transA, bool transB, float scale) { std::vector nodes = {clip(a, clipValue), clip(b, clipValue), bias, ones}; return Expression(nodes, transA, transB, scale); } +#else // USE_FBGEMM + // cpu int16 version + return cpu::int16::affine( + cpu::int16::quantize(transA ? transpose(a) : a, clipValue), + cpu::int16::quantize(transB ? b : transpose(b), clipValue), + bias, + scale); +#endif // USE_FBGEMM } } else { // general version, MKL, CBlas or CUDA diff --git a/src/tensors/cpu/sharp/packed_gemm.cpp b/src/tensors/cpu/sharp/packed_gemm.cpp index e3840b8c7..f351a96e2 100644 --- a/src/tensors/cpu/sharp/packed_gemm.cpp +++ b/src/tensors/cpu/sharp/packed_gemm.cpp @@ -170,7 +170,7 @@ void GemmPackFp32(marian::Tensor C, #else // USE_FBGEMM && MKL_FOUND void PackFp32(marian::Tensor out, const marian::Tensor in, - bool tranpose, + bool transpose, int nrow, int ncol, int kernel_ncol_blocks, @@ -188,12 +188,7 @@ void GemmPackFp32(marian::Tensor C, const marian::Tensor bias, const int64_t m, const int64_t n, - const int64_t k, - const float beta, - const int layout, - const int transA, - const int transB, - size_t idx) { + int transA) { // does nothing. supports only FBGEMM based packed gemm at this moment. } #endif // USE_FBGEMM && MKL_FOUND From a6052e4202e74c515ad4ea34daef99ff2edb7ac0 Mon Sep 17 00:00:00 2001 From: Nikolay Bogoychev Date: Wed, 19 Jun 2019 17:54:23 +0100 Subject: [PATCH 504/838] Fix compilation on gcc 8 --- src/3rd_party/zstr/strict_fstream.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rd_party/zstr/strict_fstream.hpp b/src/3rd_party/zstr/strict_fstream.hpp index 21173c737..fbc641640 100644 --- a/src/3rd_party/zstr/strict_fstream.hpp +++ b/src/3rd_party/zstr/strict_fstream.hpp @@ -125,7 +125,7 @@ struct static_method_holder is_p->peek(); peek_failed = is_p->fail(); } - catch (std::ios_base::failure e) {} + catch (const std::ios_base::failure &e) {} if (peek_failed) { throw Exception(std::string("strict_fstream: open('") From 56092425ed56adae33d459224a39f62b8f534182 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Wed, 19 Jun 2019 10:25:42 -0700 Subject: [PATCH 505/838] Add missing FBGEMM lib files --- vs/Marian.vcxproj | 2 +- vs/libs/clog.lib | Bin 0 -> 33970 bytes vs/libs/cpuinfo.lib | Bin 0 -> 244232 bytes vs/libs/fbgemm.lib | Bin 0 -> 17549664 bytes 4 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 vs/libs/clog.lib create mode 100644 vs/libs/cpuinfo.lib create mode 100644 vs/libs/fbgemm.lib diff --git a/vs/Marian.vcxproj b/vs/Marian.vcxproj index 8106f488c..f2e3878b2 100755 --- a/vs/Marian.vcxproj +++ b/vs/Marian.vcxproj @@ -50,7 +50,7 @@ $(ExecutablePath) $(SolutionDir)$(Platform)\$(Configuration)\Marian\ %MKL_PATH%\include;..\src\3rd_party\fbgemm\include;%CUDA_PATH%\include;..\src;..\src\3rd_party;%BOOST_INCLUDE_PATH%;%ZLIB_PATH%\include;$(VC_IncludePath);$(WindowsSDK_IncludePath); - deps;%CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 + libs;%CUDA_PATH%\lib\x64;%BOOST_LIB_PATH%;%ZLIB_PATH%\lib;%MKL_PATH%\lib\intel64;$(VC_LibraryPath_x64);$(WindowsSDK_LibraryPath_x64);$(NETFXKitsDir)Lib\um\x64 diff --git a/vs/libs/clog.lib b/vs/libs/clog.lib new file mode 100644 index 0000000000000000000000000000000000000000..8bac353445f48f0750ce76741f99502c5784199e GIT binary patch literal 33970 zcmeHwd3;<|+5ed)ZPTPnQd;&UK!5@zq)nSHES>F6W}4ZUnWQZQZqj7hhNelHrBF}^ zFN@Sr0Yw%?7IEPfd6h*#MJa*;f`TB6ih?ijih{3fDz3cW?{n@wb0D=t|9*XzEwe*@yRIP|PDYnErUU+D?_hpe{j!CA<**jT0)cQM5)J#q`PD;fi+N3f z(cWzj40d+)+I{_9J%elPUXAnnm-v^c9U)Xlckfzz6aRJc0(s75m^|leIC`G* zuAVi$p3~Fxby5R8x(#ZR$-VnITUVH~K1|yCC(6WCzFqqJ_pJQ)taK;!4rk>kRk|}x zy0G5Wux;fXeM6l8t)2P4fmW`e)=;8-eLm9Fo!95Jon8H{1N|MstnH}nU45!5KiXBw zj-c(;IIpg*p`oF=x_4vjmW$; z$vtz@@=D;iwrjF3=d}e&NjKGr?cso)0lrw`U_=y5q^$ovtw ze>hmK+3L!4+vs{91by;xNr|biF!xLyBh*3j#uh|Wdc)GNXC=5XDRyZS87XG z7;w+-L|zT#{S}z~Hz0!p-P&&Z<^Z$6 z!ELX7l$QZ!(=Nyx2Iit&koOf}e!L6vo&n~?U63~c+Tl<~L+q%(b-*0!;C9sC6M;Ev z7vxPvg^&vVH77}ejY;8%>njU5;EcaL4bO#|+L@o*jM)(a6YVT7A$ikx?X+$c@6L;^AZ!;@^3hJH(9+iCfqef!>*n5}^MQ!A3D(n=Cz^UV1yIbRH`p(WMuJY$7YnXua3p-mW8!$QqknHDRoW`q8;T&CQr zY}yLhp?EeHR(m*PpK{2NSlqH!=G2}JV}i>_#1d9+RoXI~5|_gwoGMZ`>nL?O=}f8- zmF(llnrP^OP*b2$7djBk`}F)t#gegHFp#zE1SWDU7tL6KFxt*O-6oK*l?^q;l3_cU zN?K}fhwf8`9%)X7c%t@lcoPadjJ2@bVkHF@JI0Bvs-0q`1zs$Xj>kf=Tp*oJ1(&OX zf%qjF<5NL2BpOJD<1#K;J8Y$`WY|iER;m2}x>;JBj)fE1n4Qf9a?M#ZJCg{+rQu}z z)F`&K!W5k_ikHq<@mNCb>qssc#SUizk(}aSQgG=gZY;@5k+HH_xfp~@90jq$v7D)V z(kN!eYKft3Zsn6lu^~Jak?Ht^bUM2#Yp;kU=QS9bPdT*u`LcG1n;^7AASBg(Q%Z&~ zppBMX;y^f@uo6KlBNv5ipYpK77RKFj5NsF`6TFD5l@kr7Y8-CKC~i0vYED>5T$)%a zDFZ_prCt;!sg=PM+uso}(aVe_TLSS|IFPeU6DN83y0wu2)<;;)g@hmJPNWiz@t|$3 z3|VQ`Y==|HRL<5@$&LhasIMOAC#6GFCGo(jP%0UTHQL!w0CAwCWn-;dnOL&XjwN$e zCLRc(iX&0z6{%u*DrjS}TA`esw4fW%L>m%vnN-}yvJ1v7HQxbGP&)!A0xRtx=J<+O zI2Tn%Ib}+BSSFkerR`jxQPnx{$vcG07}}9|pfRhCK#7M(_aK%Uw*w0uY*7bBW3ld% z;XoYP!fr{$akbSFr&v)RMi+~uG9e3kIF_>mp%9jh9f-$Oz$seh?a$7O;tFR|k(?b0 zq(ystO*%;J;b?^jvbdC3o~dTkE0c|&38G>g5H4R5*AqB(0NO!*iI<?8{fuL3H+qnVJIEMIMXhism0qO|azGVcP!<9K%UCBg zTiKi_3ksU*#f0L4EX-w~1#4MWk2}KCyeQX17GRvu6gJ%plLom)vw%I+!OrkvWel^c zl4w5kkmxCDGs>*=;?Rd;lsg_(UX(~{PMZlpb3suvy&%nZFTV@f%?m+o9w}~Bv%C

Ql^T>gQB8#!C~(v9H1k%i=+B&%)mW1qkW_o>kJ}1 z=!h{VbBh4=X*){yI%BTJQC@x)qe|bzxoqZY)H+wc)(eJi%Eq!>4P)@GWjNZ4FBUGw zHMIiQ(4AU!Ua+W3$9Yk1_4A4lV^*T8U6pG==6mrn-Lc7mUlmIxV%daj)!b`S?}etD zgYj543f*f|zdO+ucyW%GMt`$_bT#!tFEX2rH?OplftDbYR1-Q)$B@^$p$NrYTgG-5 zW|9jnbC3i5u;gRK7R!+@x zYR66yyj-R^izttHGTJ2p((#0`x}js?{ZL1;QP@19h_#YdG9G~iY@|}vZl9flKay(B zi6ceuWC1rPvw?_Z=Tdec2+u--Gvt=(mRV2=8QdvP<;3^>2{^b>)!)U{52c-3g<@Ik zi)bA)F}WJVOfSYXW5r7^4PZtwAeuraB{;;9p@f;mfD;+6VMyi|rUg>QHOY)lk>!(O z!U`v|a)n8rP$0+BEej02q=?=iL9Sb*hGZ&iVQVaVOGEZ4AtP|WUBAS>h@POy@ZK`9 zAcjJM7G#uYMvn0?#;`~$)wB?|J-AG8i4Z!V-$mv`rvi9w7+Tm~D+3qTlGq^mle~P{ zX^=WuQ*lYS)5L^8!x2$z*0MP?M3p%erxMb5#>7aTqRDP3!?d%!n69r5H;yb+F^U<< z1R6yu-Q{lspw!Aloa4lcYP$o*3rMc7h%3Y67MxJT!c-Tr42Q8`NfFnynHGrb(qeW_ zhbPIISj0)kq?s9B>y%58D0fRMsFmRi0%kG9Kg`^4_p`!xV;P9qM@7d`g1em+1Ol8n z%vYHmbn6Ikx3U6Y=sFgN+s5;x>qV5ir4PC4!lRxr4# zRwkW^Ayn?f*9mdAu>x^hOO5<9FqO*Y`#0tL1ATqnT^;RwJW3Sf?}ZoZ?9OM`4Gnho zp4KCa>l+{=JRQiO3T#GG$yf-n&J*<&CKK^#VMa1BgahKMOet7uVL&;;ra%vJMNxgS z6av%2bjDM{uQE5tL`YhgMln$Lk%4fuFapU?6zhOPLNdb8LY6s9aqE*hZs&Jut71yQ|XXRaD#K5LaJOL?ua`DU3oJdzSDKy zsrn6y;qqpiao0j_8_L4DSEz`CWmdrrNyi-51Iycz)ldEerTO$n>{P_&PQliZZW(oA zN*s(6B02N!D+fK?WMocpn3y{{PACe3GRq4bCpKxphsO)Q|`y@n7IN;jtyfh6piI22TB z#7#1|AcA!*u~a5&YMN|#*cvcsYbToM_exmO)Kq3rTvl=1jly_XHuvVV3No;)X^O$Q zdX2Y5BmYfEsbDikcScOYErVWTB7wy1kq(PM(plmuyLy~fWRd8Z@47fGMbCJ zhXLz5N4OMlzTm#-n>VuNDRfVRgMZC|HWO$s!*QFIv*NsW=Uy`L{wQN_gN$H%SD5en z7`jB3u!%@FOK}7qvtn>wKxw6Na(C5AHYdd3kchkgLUf)Xpxq!D_x^>9N5Y$N$B=G1 z$++<^WV!Pg!JS93P~_mozi`k6T)R+I+Dr!S{0k$Lk`R?_h)AF0(BWDHvW7a@q2m4< z#+Z*#JPItf^LT0jvHQ5Rnqf@El2lG9MkE{lUr7W6GyaKKy+%{MvFw6dyr`#K~S=T4~sk2%=b+w0p0xHxP* zIX*=WCnVr}bc94a-vT1+v=i=oPDEf4iuxCo$WfzCl-81uO}fL!3_!@=(qRX(uu*qd z68LZoAvxHp7YbpkghDjXFL~=jz{9vB>_3^NGBTr0@q}y`z7J06uw1@)Hk4bbl`Vt( zD;GClTh8FA7=nXD1QZ&tSrCt%U_HS~&@FYI#o+A)qA;t+L%A`8eN~yj?D0^TIb~IJ zqzBvAx?Xf{b6+0ATqe^DbT$Cv+#)p2ebDIka9uz{msK5%+0j4Hbw=JE#AZmJhq-CA zabpE)zN~5;N~N1u@M)ttIDG}DJGytTZ>jGG} z*QQZ?#zw}MGl!;xvltTxYoM&^GSK4b6lO+T`)C~W1JTLTI@Y!KW6&#{bJ?hqxsib0s=MUrDGvH(H{L3Q@`vdrEsa`hVbOJcrXfj~PEa}2bc7bCi7oHv3)NXfo4GhBF zD67h{TK8&Y6I>FqyTo;26RXkW;3_z{oDwteWMeucQ-VA`M=WyLiW;`UJ6$d65LmjF zdNbkV%HaV+;_o3IV<~za3)FEHasMMHb#a1bu?4hqxN%ojMRzlk@9S>w$Pa{i`=g!x z$)OGT{;rO2eoa?Ten5J95X#6TMrfm~YIpX`+ajZ+*q17fN){Wu9CSWb8;Sz!__ELz zdI5TiyK7k{;-MG{{fZA)5U(E};DdS-Q^)gzP(eJco5OQqG0ty-kcYRE*qmD5_9Trs-K&P9+W^Ia`(~CZnyAkIrWOeYYnpl}E)I?W7nC)2roNUdcjX&9_ z)VS^7oRB?VN*-5&wJ(!kt-DGYPsZxxLy~$1;;v&B5AexCUk(A@$t+$8oYyjWh|U$! zm>&#w^>jwtdk}}vOPw>qRYABuT!@W&Hv58>nLRlk@a>jCZRIpr3UOp+#?(^X^cHcZAC;h8OC3LD~ z&G4~0K;Waubm16olu0i_sMR^=o3X-0tCKJdMYtx@9HS#Zm<)m)!fgw9;Shvp>};zA zJ73((h&47Ms0b^DN6|!8QN?%{q({SW!qH7ZbLgj-_Hg>@=PG0^-OTEXo{_}1sM#tu zEuzwB@@5jv3cY57@u`p%FRVijYKlnDyw=u^Q}-v}xbx1Ly7P>FbL=mtLfa?_&V}`w zMrqgKz{V+ar*lJw>_>oNOd8{^)kN013!~5J0jSdYk}GeD zo^3`m?8eE%7Gm|#pP?3+ z18%x>%S$JJsjK_#1MnuuOF}O_?$JNLkyzaSpMPF)V8golv+!~lDya1JA3kH&;}z$> zk$Cll3!3-4uoEv#3H|XipLpnO#Azp|MT#T*M7U} z&szsRiCE(V%K6*V)9-%n{%6y#H+}BTzt8<_La7ske#XozzW2S2dpAFJ{i%06fA~oa zO5GszOWxkLtuwOkub2Me#Ba^m<08Do^EaX2_>VJBTy@a&Ki-i4#bKu&F#I{CtP;w9 zXyB-Of7r0w3l008efl|voP!t1t`vGC*|GNvE8m(srElNo&tCZUJ?O8{OIHkprv3JB z#{_TxTwBjw5Tokx@P4K5cQbE1;qfQcVV&W@OMaxb-i4P&g#NSd9WrUj-_D(I?C+*N zblW!<|5T|zaoq6XaTGE)BE1eTn!btjagV0w_xh^moW8i>@Yd6y z(R)uDs9xVSIM6z;uC=SDqkE_`k8EvS?Yu>c8tQB7T8B0e*VWrG*j>Bsy-9*vZ|&&C zi+)h=rwt6^1>d#rRireoXYjp>k(Lc&LjXaYOI!Q%eZ2#%YgVt#Z`fd?_jU(fcj##y zT-Vj#Y4^4F4{jdk&4WOy*+EX-t*+rU4DP2Db^p{@PWZ~{rUFJf%d+x_tRy#ZF8 zDWgaC8Jx4lH4Ytb_6{?PH+P5GwpB>yB0U113MtK8x!Q#P>}7>I!^xGaE0NQ z-#S>6j*h$$ANh4)W5q&Pab-hmr(JkFIbK)aEj&J!$O?7mnEPC!k69VV_ zgQY$)*{a`G|M)Aj?|KKr6WyAtNgsJ%yklFmW6_dF9oC;?9S?l-kElHOgVMV4cV@GHarCq<`arm*nrEmg zJo5e6$Wxjb9nnRjBNxedN9%Q^(UGfb?&2_=b3ZRkzz&=eI62U2pK|hjdqc`rd`eXY z|F=l)WRv6j(3-Xxxr>B*A^zjlJ(Rc#|2Y>*)zkQYq~uEAwH+?GYUFf@Imqb}i;!C+ z+zfJ@bERq>aAGxC?w-vcLlKUKTaN>YN!$n`r#I;&DYNhc7eb#y@(!5z|!=t}U zWQIDvYJaeOYC~`5Pf7cb|Y)>#^xjyu1W*4I~s71Kv!=b%n#V7^>+;p=6i%swX>VkU!{772HnDJ zUJH7FZ&1Lo$v4Y@*5%v#yr=|Fje_d;%~T1aP3aF$ojCk0JVS=+PpqQWY5gf7vn=M+Qtbp_6Su$v#qP*fD&D{1TM$Ylo2oS~dbDi@ma8A9i6gB^A__znxL>o@_!RJMXc(P9obDgY4$~o78l;e%J zcZWGyPDRQFuSdEOX*bdVq#KZOa`hm+4CxTkZy`McDYiiBOr-ZB{Rq;hkfL?Ne?a;% zr0*cb{2eYwdp?eI2GUO;-5cp8NarHOx*g{J@KZ>SLHcQ=oH3UpU4`^Yq^(Fli}Sp}l5ke>XI4yIEVgI@rlosKX4c-q5HyHSQEc!&P)>-!rry7~0E*_Nt++ zfg`mPwN$Ags?R}^pHTBP&_Za_+J;EPD<4SWeHy@9VFr8n?Zr1S=0ZN(e-8d7=#u({$5+=`Um z!0kxs4cv*8-oQ4b^ak!jIv4Wm@zLG@bJ`owv^IyUy#Y<@Gc0gazVXrqLVFGzVXrqLVFGLBEFi6f9ZN@z=vTO>K|!MT!4 zxg&9+1K3MA(SeJRnT`e{h1 zMHb+r{Q>5*KcH!-LAcr<&@}o38b^OX(=IhM`U9GFy`j+`(6n0&?GZzxKcMB&AJDkx z4eeD!qd%Z=^anJK{(z=2I-qIv2Q-cTfTqzO&@_&NrqLhJH2MRYMt?xl=nrTb{Q*s* zKM+LS+8?w0ZpSnplS36G>!g%rqLhJwD->+m_&bI&++dSe3)E- z5A+7XxtJ6>B%pJ=J%nunXdlIYZN{kzwBhF>(1!nQVvTR>wUX&S77aUcK>T1 z9=rc%xgG_(PwU6Swfm<#=LNez=C%OWL>r2D4L^*o82;Un((vzrlz{-OP%-@bBTeJ{ z0HiRwsv0Sbt~wAYXBV9L;VnoHM~Y25AN3l(9qG|X??4LUJ^V0In5kjjiom9R_${O` z<`NJ%9w}X#AW{Yd!bl-oSx9O4pcTXch#}?Ow&h5ZNaILPK$=3z#Q>*Zm}?!wB8Gn+ z>d=OtxpH+9{%e};4eee-`@W(5#?W3dv^)&@v8b^^bve0m z^?T&Byw{yvx%!8p`Jioe`=&X$3Pla9X*B%0EDgV=y?=&(3Jw20ZU_L||1vaPZ~YMj zSovYH{U6x&KZhZvg)fygq;nf_J}fl;*OvTJ^~BpQpH)VZU3H_al=A`260M2 zM>w`0tJ<;sZOCcc|54s`VE#{$8Bj3)XFBJu`5*mA0M|qtWbA_v?J$q;V@YzI{17Q^KkS#- z{+}Ri!ujJ!Y5Si*+J*F|NY^8M5-F$FQ%Gqqe~A7S6!M2ella1GMGAf?5F_88_y_a8`U^Z$vIHebQUGfY^5 zlq&%N1i2Xi|3Pg3d_c7AXRe$<1%${*TkYh^<>59>>o>HE4enEhcCDe^U}#@6wA&1g z{nYXvH?&_F8b(IA*i;+Z8-|8{Ib}I2lv1t^adH*(tQ^`xLu0s9HKN zr-JTNc2Gb1#HW{Nn)FdE>X^?dRSRK}=ukemZ%TxEhkE#=SfC32Dqcq6ah#mfch$#! zC~1st=smqaKVZsh(6ewS3NWJQszlWuA#*1tC;` zmq9Z5fuU~Qq$-?8NyL*)RGfTLYO$P;{>SG{y-!Nb!V^nv0C99uK6iBZh|@Y2^-}c+ zumIc3UZQ~0OZ~8tY)icL^19rJPboeO0PwhavKr=KT#K|C>Geo?-M)a77`hyKi8F`q z{x>w_Zixwh6aV`l&!`^({)gBX?b5Yw67Z) zUEnfK^UCO!!J0j}5b9zhKa1f>k&!o|Bfph7P7t0~eKbDu1J5#e(z^_zBds<5^S4Du zx@)4cAfh99j&??rtD?@{;gaC)@90rWVl6r{3m;AkJ`$&o;ll5w?b@24a;}!L3wE?7 zyla4Kgfm{Rk-&D=1Q*-hA~-&3P2eFq=TSl|zCi!lIN$Ac7v3IHq(RQbV95C%Qpt|i z#Q`W$ypp&^>ZJ7Of7N*JlGEG|#@Kbi0o~5JNP|KIVgQhp0<>GAuuNjvM zRIIAOhsU)O^)kUrL&L24i%7Xl_|eORxpIcx9U2#%rcs{8(Y9;aCk*YAhIWmiecsTv z8QT4Z_L!l;UvT8TX=t?nx(--xr(XI1nntUq+d`|SX|!RQ_K2bRW|gT~o#kp)ZisAjI7-Z!mnYP!N1T8@W%&==Arb&e(DLfh-` z3bn_AD-b2QNL9Wz{rRbnm2WHSo7^@jt;_Op(tVL#C}`5|Mw{K^7-Y0uougbTwG#F2 z<#IK<$8jQq>Qm<^m-FdgkxM_(J(h_K%GEi_Jw<5mzP;XY2pG9KN4cH2*wfC&LtEc*q0njY|5~rIQs`HDVpgTFk;m?bL%%pnGAp?ga zFFz~lrY>=?>6&xfKDzvxVDrqU2j;D){KJ(O)wG>=!Lr~zN56bku>SEQbN6oEd~5n2 z3pPJ+aLJlC?c|kbTs5hA(^W73@TXHJ{r;rNQ?H0r-+xTabMI8`_pSWIGk#e(;{PIJIKTUl! z{latV^dHRr)g4zHx%XwyKmF18rk7s*vH#D%UY5A&%gx91Jbv?o&s~08?b`n?+xY2y zo;vL-m+rP<+irIa)1}0>(J5U> zyuS}fSE|JK+JXXC=e)m3NbOtVyKXc@d4E5UE2qTAS;$(SNbiN|PTAhNCJN?3{GWsm zk5WaNR}MKPKCU3Ta33Fd0&1KRy>ccChQC5M2RR;-#>iny3UUtZzh&N7Ia37Vl~XoG z4t4~P!Q~Spg zQ)A`K7$ax;7&$Cm;yVfdd6o8GdNVHa=pT{Ds6Q8`8qIVC>YB9?x2=KPz- z%GpgYZ$r-R$nn4{2>8FIKcAOV`Bx1I8}QvMy&W!%6ds&gJW7>IIZzYu-=)xc@t7$T zv(Evg(qKY@(Eqe#y3+2UDRiWHOfsc5X#&0v!>#Llhcewzu5c)2hVo^HQf??_85;^$ zf<7o&zcJ{g)Wnl!U2>c!DNq_*1h1f8N6#b2_#e*%GtV6jO#xZGiVyD{a2^(8G=-SA z@nMXCbF3JnDa4e5&$_t2iZPl(OeOdu_;BqMV>G4K^Sh6;fGI&#Odk*Jx7TJpH+as6 zBX|f#d4E-s+ockptSRI^|DUx#FO;fvN(n}J>*DMy@p0UEJiPp=ZatrQw#+Fdn4-G& z0Jl)rC4Y#Q6-uR@Qi3U}YfmXvga3pa{cO|KT1VkTwK=5(Q&iVp;1=p?eX#bhLMivA zmtczO;;x~@=dDZY4c2wBQe{EMuC2d3^6o;ZFFB%nBJ^>dlOd+K)WM(>*3PA;mqMMprEYXG#1xleAGM`MM%8YJFD;aM#wkTiaj8S3 zl-t5%uYB#>g;I>3@gSzS)S;l1__*qMbbO-~+RJU>z9pJUOmV5hKq;)#-|zGFmkXtq zI;DsyE;UC=O~!v78}A68P$-pmN)c0BiZNk5Gg<1al_#H7C^hVqBBr?15mL&%mM^{Z zD=aU!UpG0Wh$$|`uw99dR-Q-sw_;5 z@9?c_+xfYA_>miWd+bdEV}3Ki4+8X(vmy}j$Z^2&W%v1WeVyU!2x z@Eb&G3gMm_`ITZX!GRd(!9f(x9YCQ2@&n(>!Hpr0fy)4mL2K$5uzTZf4Bjxt6w%{5 zrhr2p1J}bp2Cm7=M!k0LUp8;qvK`GXzki8;i2^3%x9d{=Xe^Rw3a3MU*^e$;D2-TF zzigr3#RM&X6TZYSh%W>!1}*IBS+;OlO(-%@gkRJYSRS$zca*Lz{1O9XzU!s_{qK3a6K@H(>GP=Am9q@BoTqGsoPd7!Q zjrf`{&0|+$x;r(Y+nvpFRxq!CaZQ6!basivQEUj!|SrXC;Y)V)# z4i+1Q5=sc=Q34?$B&2}pv1z9F5JHP7A(W5+p(Q~4{m+?u&)!|h4Sw%^?|t7}dv)gi z=bR~LX3or?+B;XaH8-{Aq(G5bd5r++8e_SDX5>oD@piuxVCI&aJy((86Ex+2-y z*`0*Ah7{RYSW~;;6wnZ) zK;)aqhN3}l%;WUs$Qmuv({>7!)!8a-lvQ!m5%W630bewfBbZxYtA@L?vN4Cx6>-Lb zb>W@J?x1;Mj-WFbb46o2%Xe^D-Q@@cBe9SVqsXe{T~=0iI|AOI*Bx@_YALs}JIK`I z(4AhF?)HZqy&cK5UB*zijBzpy7!d2apleZ*?sMo~w_kVp^*xmfx$2^^VCWkt$nU5N zh5htDM?pKSr(xL=YinjFPHyXH={&h7(bmzHGA*4a?9koeU?Ar7=2Rr4tk$gNrsh?s zcz3d^vzv`zo9Fp62ehu2g4tUaB<9sdNU5XzOV* zyNo;H2e8V%7Byot;(77q#fN+%yWKc)E}wajTCs+fsIRI z!j&X+CngbZT{IGLI@Y2c+mk0HF|(>(yYmvI)@4(1IsE=eNcTJIzD+8+LyrZ#bpdA( zsbqDLorj=@c({RVb!$&DWfW6N;jME-!y&&n1fyUzLaENKPOKR)2P^Ai3`vkcXRBkd zRx%~brkbI9Tw$Nn>vf!5J6-ad5%9P$sRTlf)EWV+UCu}};K3Ro5_elrm#zmw?x4dN z_E=DlKM;w!oQ`^rmA5XS2R(H#jUCz5MYY%G^8^E~I;@Bqvnd2zezYAdb|Z2$i(4IZ zMLmH!SHKbQG-S~_UrY}LffYXBdgD$Txs5|0OOA4V#6ks^)2!|JEc_Xf%#~%&) z9r3n!7LA3mWO4>w4!5ObGxdPW?e_SwxanIRm7K+PPWW?{P(Zn0y@d zZ9Pc~9I6YsT$uAcVYHVomW>H`Lmu4)!?!ru*^!0oJbE-5b35EvV+lIT?ZI5^^T!;4 zj#M&RnpvJ;ohRUrMIF8*3`UmU6A1*oXmSVj6)c^YWTU8)9gYMej$?Y8T5{5>3D-qjx(DTpx2??+ zo9hb2bj+nu7R!&+eW6&u6?QsgVIt+9>#vJ?gW*ub(a_!0(F6ORU24w_M%@87#%avl zvt-4hK@1aTz{~wSFTao@--MAZRM&|UUb;u@_B>)V9bd;+t4@z;<1PEc%V{P z0;nz7-NqIn5$#H0TJMQt_Gwy~bkt7Aek(z%{`N#qs->+Hb6#6Vsx{H6_#BS8j=9>- zIC6|2Ha2UTQVHH|tV||4Fc;>;cTjvg7R`wj&{PYuq;hFJ9V%DtBnb^N-}aJgBqupz zvOXz^RaO3E%F{RxCaktvJ52-UiaEkgY%H-BadbDGTssSvhqj1Zp4wTeKs2FZ+`4J( z&aCO7VAj~vJZvppb9-xhZ_la(?FDioXpu)t2(?u;3otVrih08>v<%Iv(i!H|MRcDl z7Ku6xa2gt|>{%gYY~E$GC?HdYtS=9p;i?CnU2 zZom_A(*_C4L$fkHgYZPXZf8i(79gL+<#2_={;=+%f*2JakPfD{I=43vb!F|wAUsiL zI2iI_hL#%TGhw9{arwh2kjG*6zLm*Tf~OZ+rMI)!!1OINcS&_a4!17^rNCTgQ@6wh zvAOm7ow}ufme>f)ay006TB<0qnBqg&K!mZyrM{eB&Q!H8CGfm&CQv`)TqcVxQ$_A6er6^Ma;cYnF&VvG=vAkvceX|{9HH+ zIprphleNh*n5cB59C<-=i;#r}sfe_7UP{J@$xq3I2TREq@VN;YU|u>VI#@E*YOLla zV}NiPv-7y z&;S4I=Ee2RZsvaTy}B_0Vz>G~@NV#b3-A6n-*5i^ax?q?%56K(|K38Mm3z1Kzu7JH z|4ui>{zu);|CV>~{x2N|_<#8x`ko(kFs5A`64;~r?}penya{Vae>*qlPH(n^96R@d zezRLK>IB?>rVq9DKk;tQf2Ch`P`8!E2P>{D3zqAA+ks2i^u*>S@W0O5RKmQWl%p9F z%+;-+d5%d^u6gq*HyP6{J2*N>GXIU7wS)Li2lwAvZxfoXTdA-KHw)&Mb2!$xUEQ9}yJKq=-NBt}D@)Gd zKDm*dy`_C*o_l?lXZmYujMM$gi;aAiy2DMYlU^JI=~-$((^}fPm-ciwFNJ=W8bw%2 z6`9sNt#ic*W|!NQB-`_)IY-*QSpw$Swr?W6=f%b-+p~YWD2e6%{jUYNo5G<;5-E zou+N;C|-8hO`3KU?hessS_{sar%p<$D@{8Q0nae=%Amz%;7hLPU0E47V}3qQ({2S{ zlS?6cfjG1>m%-Oy@?C+zQEu}}QRYh}*QP4hbeJ*paLpg{_$ta`%k%Ta!#CQS;JeAA zB!_}H%8+kfdS$@>V(=(VO&ct|rxE!%__pebYe(r#N3`9mX*2PqPXWGzrS}SOKX@vA z#HHc>(T>Zv+~8qG{244gDqk1)9Dc>Mqx9Nrnsyp^{v1$zJ1Spgk*56@U`p)A9eZ60KV9OdK?YDX9w^NtcMePWjElD<>M$DJiTKE@C{6FF48$~ zH~75Zdw2lfz9?Y(G&H%oF^&q}2yTRuI-9?-G!a~=*~+Pe?s-cad+;d4(siA^tJ~BH)YaS8-qN$Qrm1OqQ*y=3Bbr)k znvbZSTitTRiq;jgkC-)kMRn_}=9#mTtyN1`5bx5pvu7SrpKMPy^(4(T@2(cJgDivR zX$#7=<0)DzKqk6s^$_!5CggI*tUx^<6s7=%9OEP_#2BI*k2?uJA}Z%EWT z8}tNHXsC~d5_-hx3hCMmgSkN48FSdVIDsb%Jd1Fm->)5E$Q15W#uInP5)DqDR%Ngc z+bR1(XDHxFctcKKTsssJkLK4P5FNAvvkcyx3gq*~8&G&>C<=`&jD{M+x;EDk%c(=Y zm^WJQ))S49KtsamcI)wY!Wjx_PD8ZFTAy8uqJ`s8Z$rZEjA>Jl#4oslc(}lW(ls>3 z6J7+-l=XT%+K7C${lJs}lcu5THqnvmn0Y9R&ZBcYB~qYS5<&Au(B4Q#+uz8;uaYQ)e=zK0~Z2nTen*oQ_%Pp;jh6 z;xYM{i8?P;YM7NNr%Yrk(2$XaI95iAxyZ^Kh(Mnqj1HfL^ALk`xRukAw}sU#X0esE zF;3|@@vNZ79gQu~CL8IMSefET5N!4OF^zh>L8~-)Mp${=A!j@u@HiJb10f!OW+RNW za)kq7y;^7&Ut*P?8t+AvX zl!YTqk45Vn%nE4-8%mjI z<#dP8(=b!;h=?=TjT4Ui=;s2 z0I9GtpeYwG@#qbP{i!fAIl#(8hBnqv?~M5LxMVZg%HfVi@rYUiqpgAGIx6RsT!wJW zi8(J84F%jw5-@jqU?C<(v+k9-%qq3USPYgo4mPt-95@&oOf}62tS-^|K|ADBwGqaH zcgR?;(=;uNQ+LP+9fo5Kn%UtF*&$;{cgFoZm6C!E-60>DUmrc_CE7mB%4;amU$1** z#yLEPp*oiE>yWRP);Pi%ABR?Hd1Ex2w8k`}^`}{xVVmNCIE{t@{N~JXq?JE6ah{)} zi_nH<*Q&BIv$-_ZKbo2S^c;=>Bhj?3nmL2au<~it1M#r40mIg8n`$d- ztkD$;#Qm_nV*1UFHq**yEc9|Mx;d51vU0}bp~l4t3|beAR2?c!YfEbD**RQ#JwCSt9udyrYUng*I2n{&vapFgdRzYmbf;-$Ulb%^Id~rU2;+=!W*GRl; z(D=C$?;bSXDe;~`<6RQ34;t^5c<-R`9*Or28n2eB)bsp<#(TxG1O|Hq@-ZfY(3@d*aQP^$89Gf-Z#umV%N3fLORWZpFa9l)Rgh7UZ#->hR3>y{~7CWmsi%%M=Ff@K_S z(7c$p8no#~?n5}Pp}sNRKsjI~qn0G#G*-eG-LSD({g92MLJegReKF0FAunb?9~o7( z>`OFYJrZqf;Dr=14`b%WNZjew6AjUX(}k4<4`zxjQYF(dPV4c&bF_iizE2}Y(;BXcS zoAMPY0n3k z3^w!Tn0heDWN6_bLn#yt%AiaZ#oGssS0;<%3kQv#$uk_q4;eImP19PeiAI)JQ8V$! ztViCXLi>X$iez$`EH{z&0Yhv613#;$O+s9JV4V3d9Yrr35N+PKMhlND|NDSw)-#rB zBg=cuQb`T7#WHSsR}9K#d61BmFfgIKvmB(UflOvWEqQ%4cySF5>U*~^M-9dB?KC>Trr`hW}L2R&XTV^`ZzNrs?3CsxW#%N9zvFgLov!%ajnLvmtb0j}oHxG2i?!Y(+q2#)i} zq)5{*!;xwuz{{{W#?7#D;xIik%z6hW?5Oze3PM15&97<>+k;3|}iP49}5btYng#d;_%Nt^x@Ga9?^Xg%9XO4{_HaBMWr-H&k7 zVWOnS(xbTO6SR1xc+*$H@rFLkeY@J~5tr#LVJ_^4&5Zz0JH%i*O0o<*7-)gZ4Mses zmqZl>Dy6|j5vEIoBLV>rZ@4JJ^nq{$c_91 zrXPg47JKU9sKskDBGNZZCkV%z4zQRL&Q%&Dl!EC4VUBvtyXddnJ4|(&{tspdz(*Ny zhTua~&LGj2=?7shjPb=w64F;_O;1^w&JPQZ^ceEm5Rd8hU>>xpULUIu#9`z-G8jzX z2eX)^r3KDQkjL%IWcNwQZf93lJ6@kh-b^-(|K_+rOM5cDsyBrfTXk@HzXfd5V$OPG z0S`trf;CSlaI9)!iYzc9z4Zb3kU~pD3d=i|f^oz`1a^={6qzNGDl{SueJYzpeS;#0 z7?ENa$o5ghP$L2_3-MzdkV*(1X29Hsp@}&B0y;eNSiDdp5=)VpB>9P$#bBUeN({pd z25S0*7h8KV02Etn#G(o+l6%M)L$<_-<(_Yrb%X)()~cRWOeu{tVR$gZPWa$d_M=QV zfO*GEqSOFAQM_0r!b69WC^KO3h*4jsh;lO#Q{#r9nciq4-l#gwM*n~qnm68gubJceHlw>k_j z9d!d@D9mHv1t(2u6gk9*#QcdJs-G9agbx)Q^9qcE&b^9ahMCDZBeWgI`-Mm<$LMkM-RCKoQBs9trgDnB~cfw1piTSaih}JLh@~(#Gl~2Kf%@7JlMXp(%P#4qBz99Sh`t%Kbb)=_bG+z{80EEl#sZa5NSjj^yZg87UW6lBzh zZ>aFO;Ne{uh}Op?r(qI@{Umw%6X81AEU_!Z-6DATELx)Jg;+)ji&_0_{f;zvN`(jSqKnBCl`!Wa+LZI{5;M^<5#VhrP1X%j zW2&ax2$XTT@H9r;iMSKKCaMEWpu0{pa~>^BKHMTDOAGVG^C;V7m~^rphD(uB9U}q_ zP8Z$-m2lNZ7xC_!EC_Lo6%KaMu@9E;ae|v&!caZAM{&~+cV2i0PrQ2#;0@_MC%oCT z=|_{0pFtXRyfCXfmS_qrzOZ~zcNTffQ=$`u59@1Ev~-}oCB`gTgC3%L#%7bTb)Yo% z5l%0B0@?n)ui)&C#3mxv$l~x;$9cke3(}g_P)F-3J<=HF35P(N0m3y*i&UC72re+- zaMaU?JJqyIC)jSle8(Ga_fcyCiYYW=PzAId9I@0O6fwkza7Wp%$r~cVhZ;DR7S6b! z!weMneWBxWr12zp;RA9f+_>Qr;sO!`Yv)dcahuzTSwE!FzD-~Qqq-0V7Z6yTCM(Uq z1jFrvj@f{lH1IZ2X$geC$DP1SfHAW%ZZgBE1Lh@^SBMKs%y8Zy;nRu(|}tV3^G)QZ_}w-cl46U}g+W-xLE&%cBZ6L;`MZoZC2u zh0gi}975`z6B>s~Pcb%)VyHucom53&0fU&Q4Il9~P6C6F%|Dw&(i+)BE-l5wvtz`g z2jD)Du6$J#1@Dd##ojVwBBU7jcd|m*DnW=Q=#Q-V!AmB~5vo5$N@f)4$J0!_VR!&B zg=ucSP~6?HSQ%TgrHgSu-B^0jl=CFcu%LIxFuR9BE~lGj0L&7!ER96#@g^oXIusXr z^0NP!Z-04T*sUvm(3~ro2f8c0t9!nE8wi z3*j&e29B1Yy(&2-X?1@eBH;^;(yq$o#1%Wl}8mT{*Fs9L$?WGz^IayRRG;oKtC#%&`N@i6ldXve7Pz zWU&*=iALg4FL%dh5sR(ol6V5^z3HbK|L&yrdZ7!W@z1 zLMMeAKg8=@io{%zjWmn}?LN#RV$R6MtDqMrPtb`cYDML?F`7m*JURZbQuozI8)KNb zy-^rP6+37oiIQbY3}&)0I*Q8wjiRNRR+?M-2E29~s)v-q$r{4wFJz1+V(?O*B}HPS z6qSDoOqB+`jMWkM2V%^AR+%RL;8gnx7@?waI=S0T?@mml(31kzt&QOFAC6v-5817}_LoeogiCpu6E9nBjD#gLJgrr0BmOAuMH_7)jXQ=M;7< zqG_L-K;^L0rS&+L6QrFvmMYK_P4t-8fxzjoR|pmg>any+PNJn?L!*HWep;#=N(9_^F>L^E2FK&~iK?oq>6mni%DaXlOY}`u znO|5dL+9eS)oCuK#W}4DFcovF9{@d^+`Ou(8?9b~bm*GBoTTG3vn9fdlgW@TVWCOD zV{X+GNCDJ!@J`s&?3qaAU;vMT!O}Gboa^HQilnK7Ec&2UR5vd6Lk!(z@*fubFaCaG*RCWO4K(-5>YCfIh459;D9itLQ8(? z!MxhWvdP%M<0wM|mw@yahhr{53#y{>cR5khps>e5`TnOMk~{?U$*y+vogSQE^tW_J zde`8svdx}kE8Z{L!^)YA1bOVTg}Tq|j^_GF)|FKKa#Ca*KDh?A8t)^0m1RyRq^?h? zqE{Q93GVvTU8v=!&qmABFggz6=3O$?m{cP&bc5PC8e>;L|LV=RWO_({wAR7I&iflO z5P!pU)`;emi_}ZeI~wa`2&W9z$$F%vSW#%HgTVR%lA?9qPEHB4^=Sm3R8!rm<1zf8 zq3Sp-Fv#nwR;(OHnpGaETpLFk zI?+TFdm^qEvAvl1T})Qj zxp}eX!-Z^(1D4`s#Yk;Q%5!(sMh0;+2|)GiK8^B;bF8+D_-46jp6Px*%rb2+lrh1I zH;$z*m*q2RSBqWZ4L(+QMdh^oG?VZpQ!e~Kg&%KbZci#3Pb1Zwv(Q&*sL=u&Wy!9v zc|?+J4w=T8+J;6g8zZ)S)Q;5A;4LgF*HM~8qPZK-htrFvlL>l;DACos0(TeK7|=}z zWMR4Az^jld$d01&8Z}Y=s_fJ*w_)Q6oRNSFgLOf67A0dtYtF58GOvt7nsi?i*}-=d z!5qg~(Immv%ve33XBdkB?uFFvI6+bft>s7|vRH%Pp{Sfzr9jcl+_lY|$!c|_Xxbyy zo`{4(DkXD5YlOAt)d2x_7~?TUsv`j!9U(P54#nkfNRw_HE`i(2=urAXxSA7KO89)Z zi-(mOo;l#TiRKimF|-af`tcM*lbf1<%)&tl zcm#qC53TNV*Y=q`sP}ZPPyL?HUmd>nJbTl5GMvi7@ zHpatr?z2}?M9zFk<)!)0Sb37Y9zo@`q-*pcY6o)&aA^+#@C7M;m)($U_F^P!D$zKW z!0quka46gt!ml`lFsKu{A6s}Rh@PP+D*qMLayOcm`Xg91HI>rfA49#9N{RtX-HtpS z)IZtX!P5;bRtGWIIWrumgmkUlklhl+%MM`EO>9kBfq($=6RSY#;kMta|<k)Z-rc?w9jjzrAvK$L&|5j4h16H~Pk7p8dUc zaEm8({u5gN?V9#W#-o+@oPBHgo9Ewt;qk9+iI4aXJV49%;k~CMo=Y6JCOB`|rs^|7 z<27xrozlO^e$@8;Z@QrG&&zITYyV>ZN=-{MUU<|~pMM;#>Hg{Giw>B*s`@*c_6Xz8 zp8DJ)+dn<;lXaQ>$N%Pg$3f@g@z59Q`}L{EE}1;))9aEiAAG|8X%zG`lyxG$^mz&` zw;UAD>1t0K;jsw+1K%ckXsmhOQp}Dk@v}JV+11mkmiA%}SxP5{Xf9ckT+`gOF3~d$zY*|l(p*D%ufWr0-!>s@ z;Z9aREbwiUTtm8|-xEl!>P|Mb^w8t^tF)GROX=64dX~1XSeab2hJIxVuLemrbu3M- zYU^&n&oOkT*2%MTgTE`{Tj{I^8@Xi5Qc_x~r{x=!$gq%0@oPKZQb|T(pad)&JG;MS zYQ1ZgqGNS-uT!f2_C2euvpLni$0{sVmS*CTimZ3bbhX z$GEIgT>O{HupD2~S|z^p!3*xwL@|@4scfa7hl5t*Tf)*5D(1FU7`2zeekgqOq7jAp z`x+|CLVcdf38B7FWrgVEGH}ev(+SlG`SJGElF)k=8r2^fwpF9U;3Q|~%r z^#n=p{3*j2@UQ}Qq9#{4ZXqc(QBqNxkrXKksgSNJI5qW$igP}`lnKr3^dVi5ZmRHA zx}i|9cD!IKj8L(5i(tPM%nywe^eP?ia&XMuNyi(4 zjMXPcPbW8ZchHYG{ik=mkCAzWQ34`VoP6DwgOc>7bVH$H9?S}B!L`C}F+wF8bv%Xn z`J&T2Q}xvNF!eN}q(v!2W&cWlQekl9R3p3FZznb$p;cRiVpoSA<) zGaoxM?>RFs>6!cV4Ud$b{Uj0&^&eZ=TDt$)2x^+1DKYx4GxH30-}P(9YN-hs|KWF1 zy3f7gobrrw{vT39JxFf*ee2g&X{qrUzuy}Bd}@d%^TzhA>(_=jvc($tdy0H-`$Oy3 zuF_HyGaJ*$H5)&_p|T=16qcuA`-AIm=Q5@C$!sL6lx1?xe}NMDc>Dd%?VAJZ?~i%r zdn&t1&#FZM{h6~XH-g*KS4Kh7e_suS$i($kQRr$4VW^&6c_W1;_%ll@*J6}bITOy~ zoy(j{6UQ&RzZFqr4GOgm{H@}UIbgS9Z*g>9N2C`A(aT^OK%Xf?fIrd^*@;iT@}yk( zJ#ZtLvxNBG%wI#voiV1ciKOJohT!cXxM$%kZ?oCB&tMv|*|mFlly@O^43eEYn#N-` zSx%r138>FUv#cMohq3U#`8pg`O%tQ9tHnYqtuiG%?;F`58>YNXS)Q0UnypG;N;EQhlRcI4|tz@m>r19AsrN+JRZ< zwPzk?sT@$88D??TpkAt>4jZV(7L%(TT(dZ-DqFE=sExBBCrgmy_P3QGZbTVfVCU3q zO4B%NW{!IY9cMI&C@G`mNH1d`kY=GW+?BWEJTYNeU5(VHK-r~O0F-K{qLW+=YE6jC zZYFdUCsfJ_?KdDHM@}N6vZY}ynQ7c*Nn@V@X)KdMSme3UP5GBuzMRXj*8ur5xB?bg z?wzJA>Wz;ozrw7FxE9q9mvi#?L(At8#cT1D68cs?oW5iS_-_WqN2Z_N0sblB#hP0F z(CUhxm)r;D4ttQ2tZ;7*7#Jv&s!)%<@@=?^)*(j@pqbM>>Ms8M1#_Q#e5l8mj z{Llbcj*ElvU0~p_!D*V?;3`bhB!syjO~VL_)AVxCBS2};Oatu(g>KWN7A#rP8$f4( zo(qb7MYDOisYD%Q>rY`tLT z3HAfQHVgKEU{ps+xkP(ju&)H80jTmFVT4NH1vRmmg4GB{eqog!^#momR=VJh5UkK{mZiuD z6>Ad&n<&_P!D(&4N8B7L4V$IPgJC-y5T7fG2``arW{=O^ADBO1|wT-{;9*)T%C0tcM=Xw}d9`}?;c;Jke^`l!E8U-b^*25N*a zz8X0IY6oE`)4v=+!zYyKUxkb95BKTI4aDA0w>t<4Wi~8D(7u8A=Wi+XZt2r2RMwu- zO&`jB>IF_#&-~ldm(&7V{!*Z49{1nzA#JO+Z!Zn>`$_}zx1MbGXSVp)Z~IUc>#%Lk z%q#x>(<&YMhR;&EKl8aK^M2{3$Bg7N_Zz8ap6(mU5+3_BfB#8^*c#1$t@{N}-x@8O zg(nl%!kNGBIH%|J%me=Z6t%~#S3^UT=VAWLL;l0Inj1tTzrL|GoSl8VzyFLvY{T^V z4|Tuk&z!#M&LRjpQHRV^{{D$+|CTokLz(+kd8r){8_Im)->|uNf+wRx<0FA+{>*cn z-n+v`AiiH;3>_`5ezy7%B?#^VRA~Jdr=<4EJZ*@vOd#{syXT^VVfWjYj6pcoAGGhL zTs`D6;n>{p;!JLJh&sxvVj9LFLd#c=;>${gD zJ`=8RX1>-l+n{@A<`a~7>~n9odgia*ujh=?{T-UwUYV_a`{vA3=!gyXPxbdZ3vlVn zOuWRODZC$5N_nFFBi;&ZdEH*Re;UbZq0+lId-^tPM$p|qVuHVaM4#XONVqTb)Pz>O zui>c*{PjFF+1ZzRs?y)@-|nBXrT4MYP44}+Z=L@|_XTLuu^D%v-~OV1{*_dr(zE8F zcT;t%-wI`JC1qj^`!n3Xpmsxbr2W2dgQdOq9!!1jUIlZM(HA>2j|c1z^@Yozwuh0V zGxHT$22vg^TAZ1^`oa|(p3T#n9Z_iDzVL+YTcG-h7+Of8Z<)VUpWnJ)>H4>k5=IpC z^_X6|spTP4U#`s98^W215A`)J4R3h4^z0kJ=gIU!ezSk|W_TA~%&;*Cm)?CpbVt>8 z_7|3A8YhJM$EGuSMPSPtg<<<2^h~f)pZ|n2^HHg1tFzSe>0wW2zL?S&a%LWyqA!2$ zjZ8n;k>|Df;6p&A*kDSbEk2 z|K7eml<8GtII|^?dD6f7#Tv`-)w@EOcW5M(A-^)CobTGu?=!t)dF1<7|GNfqq0Ckp zKxC~+X0!*E#2oH_c=t)v6&B(%CAQ^FJBx{>(2M-aOpLdR8CK$Q$ijGdFt{>TQu^r0 z;nMv*Q2h2sqy2>!2DWU2#%KViFb zrl;?is_m{qPv2E2YmkaFW5WPR z(*VhMoW3o$T1^qs5(;>8Fgj6KzKu99QFCzt!)Y|5^7f)ZCaB{QY`` zAHM#YP(PmD>erVCGJl}!s!%_d571J&#*onc6sS-=^7PP#?Ww~ve=-|!NM0kZrG_72 zc9h>!QBL?Iq?Mk{-Ku{oE-Yp3Pa_V+#WY}+Hdv(3-3c z=r~yRez_`h+51J@?ucD~54#o2QZGk()>2a!q*6bWE%gMHdLosY1YxE!{lvtj4smIq zwXXY`#O0Q2}kXPnxc`MAGQGuRu5A_V^_6Pjy@5AknBqr~kuW0hV=V=t*pV8N1 zmiPCm4U}Od`i2y7oU)aDe7g)rh?ES@2`Y~3);FYruk@~L5^Sr?1Ti_QU`7v_0U$6P zn3k8Sj57N2z9Cr61F2A%WpvGC9;*WCZ>aAmuxWj~6Zmlbh)V4tF1YP>=mwSPr_G^cD@|Sp>yyTE+$~)Ua8~|E#(g6 z5IKm*c~hu;ioZ=9qKTl2Lx-Xya91HVfY1>fiX+s*A?kLD?;_#5AEA{T`Y1VBcUrK)QJ%h=k!jOP)0(RN$4jM`ZYo^ zme`EYF&ugfp?VHIkI-Tcy(*!%CG>X*eIX&+D0L*LP%B0Vj!)FB3?aBa5gLyWoTdm( zLa32LQxS?FG(nq2leCQWa7T{_I=(A-ee5QGHgnZ{X3x=2X zsGl*zU74fH>1?-6qo4Chbapf+`Q8pWV%4VyJbHNq*JLQvX9mvYAI+!Jac8s%hdGlt zoy&6B8>3k{azdGMbf4&Xl-Q;eP)cpajEhpD5a*g&*W@q zPWv&`N?u#K!Ati-A13RF0YbM)j=}xdXVCE2|Jh5l6YXXdZUM4}VC`Qc$0Jd7^NG%s zCg^mKI&wq_kW#lY;~1>~$H~mQ$WJj=z}Ksh>1`M zHcVP(Ez+jmi5tM9EVm7 z5q}xF31;XbJ7u{@glC8Xo(5MT^EQvDfNFb5Ol$7#?oQ&6fN>5#VZAj;w?CKCNW#f8 zQuP|7N*-XGQ?R{-NzB?nEzCg;)X0^nMgxtO2KsR^Nwq*?f;7;oyaq~xgG>chD%J+V zZAQ^7HOMV2b~@{50>+5a>>;cpRi*aa8`xBf2Uw_$g{T%vXf~e(=!6L<(sJ*Zjn>?J zgl;2b@8L3y7Cmm`Yx*-|6rHJ1<3k-k9%O{v%Iu?ggqXd3Ik5W}zkcM1VI{UV(c5=) zZ=E{jvqq<+rX?Lb!yT}iJ0y)9Yll>Curyx@DXmh@8rXzZxs0_yi;6`&FyECHsbrq< zd^b4&EnURf>{H;z;}<7k3G#E$KjFKV?Mu)CAE6Uf7vf+1=UOWjQ23 z9sFbF+-jrVL>>IkJi1OBvRT_$NVY~bVBV`Lsn(>0A|68ra&gDw`en@5IOE2NtwQ1> z*%}VBHDl#LGL+#0t^z%3M`@T1KZMmX9@npM0eXf!f&x&XR-sT`oK6pJ8cvsLjzv!8 zj4v#)wv_dR1C{aRyp}5D^k*Y~SGgbYW=oN(=vhImq0C>wKtFHoR3)cjL^q!3Rvwjn zA+GP>>s~y@O4l}RBoD-f;@@f4YH6HK_@Qlq{f9#~7Si#S$t0>#BT$+NdZI^a(!g=_ zN6k>in3;|v0W!|0w!h%ca0Yjlv7)(sttU=!2VX>}LGEIaZWbZss!=w1z|hP&?r{ZA zNp{@+RmVNllytd$Jl4$jYnN%gw#&7{3NF_kE3j*?7hEj$n1~GT;0%9JNQQW!(q>ch z>ZX-=_?Fa0YJ8X*q?r437}pC~jnQF4aBui1{5u2v`d8Y)_V)^}w~<8+L$wa>GKF6*$ zL5IhIl1*LQxkyW|<4b1+da#o7Y<16(nhi~h#h zY{ImzG+D(%=Ae@Ow2QSvY#R%29pWFNIwAV+PXV)yl z#XJ!~x~Ce4xfXW~J6>xTMi?C{pj+xF6jtU7I2)P@$eBUwKeJKks)(_~$bF%W91{fd zg=G((@b6ZOFfzVNc;Jm-UC;!iX2f*i&$VUR8MaRgEUNv82YIcs#?mUw5pTAN zWi3}NovV*K&AzacpJA=07Cs|*@Z@$bHR{O3sioEVNVpYWEGS3O5tO(KPJ!-*N}NpvN<)#Sl#M%{5Pr+5ylf(GbjzCM5+xWj_N zw0B&kK`h>4E+YMzF)F+MQrx+t<|Z!6I93ChlK@xit>H(&4lmN))>>`%3?ntrZ1Xr* zL3IO~SvIrm%Q!OFhTUd8mz9pMUCn7+Hhh@&-Qh~sru~uIy-1pfhI^FxCKOv|7o5}+ z1A7AUyp6lS$!uUxkfxY7z`)7@HtKY5vjI{DmI|s2><{gSYH967Ja2HW?Yr7I7*|KZ zcQJ+1Sxwb%sIUKqGyH8KuNU*%X+;vp;FCsEk!2vC*$vz@vkEub{-jme`?(3pVoYO8 z@OvILAr$=uuI{?wRoHzzRhaD^S0mS(IgL9n zRhxPWPLAPouB}u{VvqiaC^1Gbe`buL1&*qYrKei!5@Ma~Ep<_r#b-N8dtE!wHeM2a z7zr+9-tyx71BWzqlcQDB?nb9I%~|#^;x!z%5ba{dWrx5RNL|J8e(qBAR1mh4w47E) z|Ed2H=HS|n;l`ydb)0cche&rQghO}uWQBgf*Aw}mY)&h4BXiocNFB?Kc8rat6k3!j zrMx-(P&~vMu}!l(U|O;Z&%w4av@}`7seBJAbe7Z7M0Jx_(Luh)RaJ|4Q`$HmOMcX2 zSnxs?Y?uD^xORYc;Q;;8h>0-&#WFjnjBXub&Vc(vMJhA3KDOAs>@-8W=8c@tahx&j zAkB`w80RtfLn2d1o7#Xx_<~QW%Y2 zJNLJ%aKJTf+tm`(v>-Pk?c2DOe^o-KPu0}gvnFL`PAl}sIpJ%eFlBaVU8mZ(ptxa3 zsxW(MmPQRpQjy34uV+ck7RNGy#v!fUmE3=hRu1L z-X}Q3cNXVCt6n7&u~Ba`O^M%@8F@@+{xY@>{ijbZqKDv=9jJ zELY6>h>Ly~*%ra`vy)y$M&f-=4rkw_#OCla8=m2p97YFy6O5zUS>n$(4Iwg+52avZH=t|NKE4~oPjQ+ec8ra?-DRg$0MRCFAVD|uLQa5cTo zlJuT2XE~`!NxrWg02TE?MLVeC6D+oZ#i)u(>@8F)$wke&HCMKGu4rmk%Y0)fQKR0^ z3AFF9u2%if@S!PYHt!lY6kKO}oI3+eGisrC(vZjDUa-ngucg@~#_R5J%uhqsSaTGf zVIRj@Q6t^-$x%+Ut$Br4WhdJITA)0+^k|Nfyrhskxz>~Z{ICZJlEwTR3-)7B4C4(k zSxi%K62|=_*mIw1yI7l!efD1p&SM83y+;c7uHw@np)B-NT$@r~!$W_w?O3h3;8<;p z{n)`DGN7z~#2&yOAZxW3-;=Co44QQ)FKf%=0rF@9Aa-O-a3m(*%NKg+GWmE?+N)&PYg|Of-7^x zkdgH4!?^s|k?Eg}7=l}nxsP4Y>n}!>tiPJ7F`2pJp_G^`Fq{k!p{M?}moJ$sw~r5opj5hXqf_uxy9{ZW`6FR9=~91_j>*w&9mCs{@3 zqKn1?oNI{1(hUMTEv#)N~4m|(#lBAd~l55i&q0 zN$emS#8KMZ@-<+f&l^w>UB9C)wKgc2u6-D>^jTwEe{NiN^Y!uN>Ut=~vf{5oz39_x z@Xs@@Pd2XU;J7uuJ1_pH#7tLtp~ z@8!j->umf1Jo{$l)pa)hZJuef@#;Dozju+UznQ2DAy3M!*VC@=`a0wt}{RQrBJy+5?KyC22Y# zu^RMJP{gHw28xG~(wjiLK0UEgP1KM#5$uD5|=Jf=Se#d(l47AABCBt0GU6wsqV zDW1Hyr-G7K@HEinpx*&q4|*o(<)CMQ{uv8=k=-Hr8fqobCRnR_Ax_Npo z=m=1hJ-t6@KWHWB1)wyvE(9fy#}7bt(2GFhpqGG>=L4sH(lo@b03|2Km7u4CUIm%~ zy&9C99X|%W67&X8x|{P8(0f5|1bqS&x=Q~Z^kz^Ra=!rG4tfjdFuO+2T z%7RYQDE9f(zkAtGk(!U3#bl(KU%$g==?;ke-HFC(5pc|2c%V~h8jiA^!rfDYwH*1R52+UZ0<{{R{VrL7jWTp}kvioJdMUC;*5uR-C?Ne>$apBQL4D9$sc_XWk7vh-xo<3Z_+MiXcaD2*e!CDsb+ z0)=-fO*W(plpl5>~4Y~nzDd=}Wn?TP2T?N_)dIIS8Ku-ki2R#q;`=A$ro(Fn4=mnrxf?foA zBk0AT4}o3+`V{D;pf7;p#9R7v&=BZXpkdHrm{gp9OYa351EpKz^`M7|<(BFerf<6m+uz??D;75S|0eCg&A3+_UFM&ouUj}UjeFd}~ z^iQDAf_?z{0_fjBUj%(0^koD8lYzeu`XTUlK|col4D=tMbe!`O(21a*f=&kg0(2wj zQ0Vb{ps?ZT?}HYCUSQxC8~CN5!-4+?XbI@gL1F9D_kv=ll70>pTZ{CIpjafQYewQW z0q6qI3eclLCxJQ*TsLqZ=w#qwP-@2{=u}*{fnv@`<1VK*9~5`o=xk;B0MMgBX;*@| zBuzKblAwoUzMF=xdU%&Yco}tV&EEwhpBR$&hsPtYC43^oz-V^L^ zf(^r*sLE1ego?HCg25UZvIh%xm|(D3hAga>!ABcWl^(e|R9RpH3@j}e#<78&FW41= zT`kye1*22CDn0Z)L-sYn(9aAEJ;~rJz&xtTf?i`_dkMCWU~>gKO0a-nundOmGQpYz zOAB_kVCM^VkznK!S7o_Vu!jVDR4`ftD89c7X2*Q1Fj^5%2yGDk6 zBOzF`VCM?;}P}6zo~SUK8vM!Tu)L$AZyXLX~-(5h~W`!DNLUBG@d!=n-XA zW?eAys4HxZU?&Q8sbH52_7lNw66|@wXl0_(drPo)1p7p=F9a*X8bFnKgb{+1Ua*OR zO%?2L!4?SS5UfG4#e%IB>@>kP3idt0ekj-#g54+BLxMdk*b9QaA=ul3?T7V-svE6j zlwN4fqA;&u^@1%FtXHsgf^872Pq3c}_6x!86zpEXXj`iCeNV9ASYIe?q!B9CXnU%# zNrD|Em{YJNf-Mv5B*9J*>|DXVFW6OrT`SnH1-nhKR|TWvIaDev5U>_e<)e+P!pa02 zD_DhKv|3VpwSu_?O9<91*vW#OD%d%KWdyrUu%8Gu*U@ZSg_9o8;a^F zSz2MKGVdeUeu7OCY=&UR36>CSwP2lsT_PAc!BrkN2=+6Nds*6>CQdMk_c~mRiBwf*mJVLaGBvn;>HSl%uLT=7$}G#?MyObG3+5B7OR$t+-xKV7!EP1o4#6H2>=D6U7wpf1eJvP{ z)EH&n+X$6tlLVV9*inKVFW3sfIt1$$><5BfD%f>`{Y0=w1$$DkmjruFFnW9CG+4|M zZLAS0)+z+66f7WESg<96EfZ{=V5bY#Cs@B=_X+lpVDwyzD$5Ik;duoED>g#K+7W_H z7wj0p8U;%V)+Sh=VEuyqOt4=FM!#dA%5tAzZwdB}VDt+Ditj7I=;u)sHo*uLYcmC_ z5zH?by#R|O@PjgVUu$u+@m0wneZf1barXj|BTvFdP;)$~?pf6>A3zc9>vu1e-6|34(P9 zc8Xx%5$tNgek|B6g8f#oZGyce*oT6BBG|s;%rft9go?Gp1Upi&xL}I}YZh#!VEuw! zAlQ|HT_f05!5$Or4}!fc*uMoU*vssHql{3AHd?S+!Q6sH1&a&TBiPA;og-LAu%8Ha zlVFEnIHchbQg0%>CfnXO4cD-Of73^if{v=q* z1T)`KBZU2+U{eHZ6l{rL8wBeU>@vZw6znO%o)c`lVD`Pub{k`aO0@BU1q2HV_NZV_ z3RbX>S&yMcs94)euzdtOd|!U)9Ql2WP_foO(UiTw2;qh6f?Xrn&4T?(uzLjCBG?mx zJtNqwg1s)-2ZDVh*w=#LT##W;M;f6Ltz584f*l~(5rRz@>?pyUf`tV;MzCdqH3_yx zuoDG4U9d9+>lf?-!LAhS8o_QB>{o)_BiI(fo)GLA!Cn>Yb-_Lm>?6Ux77RxbP5l`m z=ufaof*l~(5rRz@>?pyUf`tV;MzCdqH3_yxuoDG4U9d9+>lf?-!LAhS8o_QB>{o)_ zBiI(fo)GLA!Cn>Yb-_Lm>?6U7_BY4E2qRRi`AhHKazDH=d=v93(EK+d!^6QJ;-P~M z>XUBoJXxKiPqcP-u1TzJYQ^vKwWa8RRCSEjre!fYnH7ZFP<@8*(F*e_m(t7g>0|=- z$=Vcrqqb}+>MrcZK&vc5RKDf-!V9KV;!7W_bkamI)1|50=r`;Xo{6w>3R6fqg%#G0 zYlZ#R2$j$+I0M^z!6^N(RJ&H$vRa!o^8{@#ce$-aEB>%(+t8;AHy6af$PX!3fX>F3 z%A#%$a{zzl<7?NbEGk6xJDM@u1e-R&g&;oD72&@#@K;Hw5J`9#vrQ^SDN40T^?QFf ze%rXG$80X|DqCL4rDzi^t4HTm3bR;_QV4R$cq&_S;&-zu@c+Qxc@=oMe_L}?dxac} z1gf>*`Re9WvISvF1fP%Y89cJ5qOGHX8Fnb&Q=uMm$g;II;U{@oDpH*l_|;GRR$D4r zao{Pv6)V=I@IyU#9-^Wp+0)$J)|G1O?C7cJY^CB1C~uFJP3FYjWcRuXI!r>`*(&i# z6O>T>)BsYOSHXwysjRz}Hcz7LGJ$L@td$s0Vrx1BvdH-7F@9AEjhgE2<$i`9>82i8wf4ZOnzec5Myp#p zSGK{j`rcKq;mTNay=RgY@&xcV$-8USx*j#e%;zb;Nh+b~fr@7OkvwaE-~DQYrhC4X z_>6BQUHz@3XPVNfx~tYNKRtNWjOlaFJomXddw(#>+kEE{i+=IoX$P+Or1jd(mtMEu zua6jY+Lhx%$xYWsTerUXt1~w|xN-eb&y3@W3V-tI1)FDYy?FlZFN`wdFQ4$o)9#)B z_(i|D#vx;mFTAOF+~liH ze)5!_2b}wy`sW+=nlkSFm)HI4!n5yusW0FypLp?^`&@VaAFn<8&)uhOdhXV>%l~oe zsUzAS{CkBqW#9W^2hV)|)SnL9KDjBCzVFd1r~Gi!qQ}SnaVDMq;nw~O@7U+c6McVLF}L^5 z72D=b^kK5#kE6MBP1m%xmP^_SZQHlgyyR$2Hl?OjRpI7hS|QU~Fp^UzB}srLA-nBF zmbLzw#=jm@87B;P-@x){nu{Kp4%VcH0%!Xbw-K7kE^z>7WamQ|)+yMt!bo zy5R_wwlyzWBiL;Z<1g-M@TU;Z-eNIiNrz(WHgkc~_2*|l9heU8IDqGA{58`VIv^dI zW$iYa$*G1{Z5f9J3x%xZfu|fiui&qlPSJpLs7AXg4?Mhi+Y3;<7ov z{*!7SYkALRpzj(Q7bx&%w;xfy#DTfC?PKO`eQykec)4@M_ z+XVHEp;~1qGJQ^!;!()7we*v_3Kw!4v*4PflL&Hd*p|GrlsAm6~Hn zF;7ljWWDS*cs%$sZ{I)vNR1u3#33jCFi%ci@X=WEs=Vd7wye~Nh7|L#`~?O=7bwfw0T)4N%(lp#etxl+`Z>^2wv(r3Zwv8`FD zTMQ}U$(14-o7Lf^=N@)%R_d>Y6!GLr9l}zz_)DKnuWUFeD>ZePLWw6=>QEqdyw5^? zw#|R^oUBxfAw@j7QirjW*?TX$d-PwjQsx1B;>nf5LIHW1QZL=u_F-1)Wg{=*$(6!# z&>}VF!J><@QsWIHL_DfB;syeM(xHhFKW|N+Y53Z0T{gb3{7FK>`j$I>v`6;ZIZ+** z-D~j6qo?4VZ`QZaSf!O?LS7u!fj8XY7pdFUbR~SrR0uB((CNo!4akqT%Q(qlV!8Hq zCoK{&^&TW*v?QZEJw{w-cOdHy(bs0#!cA-2*7U9sHXpru%FRDdB_+z|QPI{4<-GK- znSMd?B>JOXePA*YqJCJ?ikQFnXT_A<4xNuCE4_G(A7?Uur!A52#r=5mPkXZ4y(-zf z8rAGc;x%pXSYq`90PLNs+miCYR^{|L)3wUp4k~}8HQAgBRq|40C-Hd+WUHPhX^F@a z8Bmox$pP_ZVcXQ37TdVTwMa*luGu4ya*CL8;^Qy@)Blv^C-;L~WfL@>AT#AS*?g?M zw}*aEJr6ds$b;F>VPLhj3AZCrH7Dqf)CHqHm&3@$p&2o5m%|hE)kWQPAqP1&YiH57 zw$knGt*x$|1r3w)rFNE-S4E1vE8>W_eNm^+6?B+>PgCA1q`aRZ=Jq&(u9#jIcKUZB z)>AvHt)q4p%VtIW4qu%&<_+m}I~P^{QizMF?swFM!hZViP+p|kS*gz2S*9aM75tRm z+F9&Gt)11{DYBlZBNPpKV;-k3N0yzYwX?`;T05(=RWv1ny2}v?Mq(i!RBg?4mx9PK z=5>YxzUahdutVdl&g}{}wsj`{Jo4xh&v^T%R7M~*MFb{089 zRWGhuOZ^xg(cFftb9g+isO}EPf>9l#)Y^4-r@H99E_ciwbL#pxk@fq6 zo?s*p*?n2-+@xluBWKlASU=Ne&90g|duC1bbSqj>F}7K`^V=d z0m4ln2?8Q0Tp%DIU{(kSa&Jf=Kth6mq6R|90+H1$h+6fqDe-DctqXOj)E1?#xTCc- z5i8(UT&mQi;!cDrsI{P0{@?G+oH^&*+#vP!ecyiG-+v(YdCvDd^UO1I=FFKh^UR#| zjC2r$x&F$_j~DMO@z1VBUpAyWF7ClPLhJs%?cyq-QRp@^P<1dF=|_cWQ$088;dJs+D$I$m% zycUM_cmeYfO`x zlyVXJXx1&qsPAFOHAm`mz2-xen25i3TInkZx?k*VxqIpyW)$Qn^qomM)J zvZX6VWDPr`YEe~rT~(>>d?{5EFne2PL&Ga;YD?>CD@xHZEtMU(Qud~57vsgKuD+&X zc*XFg^UlK^GA2dL*J@@JEUaso)@a8$VYkElBG+k%o%L=&CY6>JR^so`sg?@?lU!d1`;#EqL&tk zbqh2ssj051s>EDHe9B0qX!mj1`>v@g$G$)DDWhC?eQkL~m3sNW_AT*pW3h&sCG{h+ z%IaZUe9AzVuzG=>#y>8!h(Qnk=tjWHY z4?b43Abb1Nfe(K8`=8z0X|>u&KiRi!`TcVmVjs`FGimO$!hugMlna)jXWz)X(jNNT zhmV}S?vwL>xA*)vc$B%F(5@Rm+yvl~&K2U$uBKx0f%IyKR)#FQ}=l z#5;6t{e^a|JbH~36{Abhk(`eW+v2we8CT}%D$Bv>T4F&{g*W073T-Vmv`#2hMLNMJGNaQ zIuD31+a6x0_M$MQ46P5_qw7VtZn39n|8a|{+6aPRaSCQp)sp%33($J2z4(0i z(meg0IG3S9tn`UeyG|`BLp#_|SC(#MSPt|`;&wAiKJu47QJu9<6mQ;Q-x5JzsL#RA zatYJMH2ksj3pI|m#3GB~F^ehToQ1v$S07m%GDCmGqCE0tu`dK27V}7)V`)9{r#z;} z`Q-;F6n3UQ>zAJzQP@QW;{#4%PZ+GAaYkx#AwD|Rqp-0sHDJp7CP<8mC~p=8XA}7+ zzi070d~sr;;euwFLH2~$FIUFHRtEn6B>KSV1aO)lUa!Z!2glO)bg9_loLMXB9O5Bz121Q1fT)R)5@~ zd2i_coY^_TJIIDAV^eJ5=%V9|0D3Lqp; z$luAK)rWjZYq7zA2qo;mC%y6-XMPY$y05u$_TIe4qWyV|vkx>Defc@Q>vaI|HvDhQ z--8b*?MX_>E&V(xW%8oN+WMA!#P=kV6*ZaOKPZ_XlPPJeoz?O;#};1!D$lHfFshBK8r_L3aUeZT+=pLQ&^VET5U2X7#KX-qzfutdEQ$J3&%r3hY#2ddwZ@0#Oe)Ipp3la!)VBacuy|kFUsh z=yB+6=(}K9dpoI{jV*TQSX7Rl6L-pG+NVk^6`KV5v3Io}`-P}|J-I8jod&a_(!iX# z+BKi;M7fL_i?s~)>GHa|r4=>h^)<_t=d9XwQ?=>G$Ml9_-27LmVt1xSIzUQ$>CnL@ z1J3;8b5L2SPe1l@V<#o_YabsQ*FGjO1{>1b*URZP@IGKR&}<`Qqev6b8Q`vX=WrV% z`LN^zlYe~R1u51#;DHDCXWsI%bDg4hT;mN`?2|WO7eK6uyupfn>IO59!+4oHBl#|)GA+2GczsB(-(Az*~fNlb1qG+G2^`Q5H z0+YKCZ3S(?@r$6Jg1!WL0JI})`3AHzD4tgqb{Z1~@1aVz9>z2K%)fpIa)8?+g|L3l+w^D_i({RM;qkjWt-g z!Kw^)rNMq`uzL;mfWclf821TLX&g2fopKb0onN#q-8APHnFganjglL0u!#o4E-pH5 zjlnK7*d+$L+F*2%QTlE%*sl$?&0yON_KLw?H`vz(`_5o~+*Z_#4}NLRFVYQ`ZLpBR z@(p&m!LBq|JRYD*U!vx)3Ikoa9!~dfQut6_ct>dUyR!QyALOe0mlvKEhc2A#1?GpJ z3ZEUXyKu5Ycw}4nK&W|dV)Ndfp_O~s$ve{2{As(!4y!-v5caWy$37phy`9n3)7{0) zBkV37I&UespaGcp;~_xyMBr2C#1wb}PR6jS#-FGAlFolf>W(d~3qUp8pj_=^)y;+GGEIBxKU?YYU(CsJW- zfY{N(Cq4~^jj9Bj+BosG!R)T+bzshl=!$-f>c~gXTcGWe!N?G;E9x1ZRAY7=JW?MQ z{GN_!qwILK-3i?l^Qfy6x(l`laXk707%RZ5IX_}3Kdpp0ZhScgUjbYpds{`wVHaud zGK_(>N9RoK<6Z(>1@?mM&Vk&?+H_=lK;RN7R2j7+%ibJyH%ghm__%F#Hnxl7_XpUC z#iI_PPfP_KW6$C*9Ua^TUtih5iPEVZEyZv9-cBtq9iFE#<1qBlW~Y{$TaA>0TVjqE z0a>=X!)iLU?5IzcGU1rx&{22Fp>?NLb=J3{Q#;UQzwFeqKhHk2>ePPjdlX-oNkykN ztUI>c{@cD>?v8DSr)&2cc|o0y?XB{-z|-~gb_TJZ8G_x3mHFTvc`i&{;~f&~14-4f zO_j&nT6JvsDI=$=$)}~etI6jm_wHoJ)}4ziPpl@Hq|>i`LLQ$Y`?c;^*`8QuN5q;b zkFSy4T6e4wo>=IaAvkHtp#*WnH%8pux-)#?_jHE&2Exv8rBvEUc7(g5Bg}Te?gZQP zNAC>xMhAH=65@?qjGdb!Z)CMxQGT6omMj`+B({K8bP!0P#_@J2yUVDI7P|)BLD>?b zuFLN7wV;$o7u(7Lg^#8+6|^7dJkb827lNjOt^qw6^eWIHpnQIw0{RP3bh)jEL4jGk z&wm2S4&_SFcR^Q!vfKPqP;9^{H-Sw9y$r|fMlS&63-)5rsh~?i*=@cEGz@wkD97gU zGHKlmiY~p}H1=%JhjDx^C^jw>)u8McU<}dv4)iS0Sd7dx)k<+LaHhAPT zqO}2Z3@C35Dvwk@xt+0&-+oHJyZUZZ|`1y*U_3bg( zmj*jzFf7~DawlreFUA-wXs{UuV=Ykjtv1*d2ICvAvhO~Fy=t)C20LW1Zw(eeS*W;d zCR7^Z4aRP@!mvrEma8z>N`tL67(3HS-*pCi&|r@mjGbvE_ng7rG}t=^+h;Itx~FX6 z+p3D&O>=(H*I)w-Hql_*c2~vahlf<$a}CB#conwNU{@RL=LUPwVBBg?>Dy*7Zpo`K zcGHzUcGDGhx52g;%r_xkOsGr{Sy_J3sXkG3s%>9_N@DJF@+aa?t;AJaJdOhr^TDHG z6k=sT!!T;VHwJAD_KP(Hr3bjuz}M$kj^pE*M$Yjy3KV8weWY4SiHXdBd}j}Ej;xkT zc(HU6bGOx4TA8LoR-gu}b?3t?6%(6oHJLw-8tDHqtih_!ADok8ERY;oAXpjsUn`LB zQ**SOM2lPY#6K^Ydfh$yzgttfYvHBw-`L9>lUD9e`Q<R7oo>)e-~O&jvuP4^x;KkF~^^R9jMy=A3tY|(I zI`^8d5B?@E@roP2ed6lNH+22Q;D_$My?OhN-o2l&c0c&5S2q8-TRI11WcXpr1;^Xb z4Rw0~q1}M{HT{pg!R>j3dlxb{dj)Xh_7b9kSYKx?v-4*yc-~UCJ1tI-Vd)*3SKej} z|J}<4d8?xvA#Sj4zl*O0UVAWEM|)tF>TT05+UU-gU&^p^L7Rs)a95*9Jqc2VzC}TB z3~IR`IlExF*o@_&HZmaPVl#!D!$SmTv00S@V0!>f5f*@Dj?xJ2kxRld`EtiVyV`q} zP9*|~<;F=e4BOM;ZpR*WXBec6XXzAQ>YOEE)5czj4Fy>m418T?kTRaMc*UJ%T4$a3 z(0k6}uKhBO@mM9sstudvZG3 z3IaU?_C1M5aq~aMh*1_0v^DA0b|S4Zs=e&tcwSUEd3-^^gd&fdb72PksZHl&T((r; zct^uuef-qog7H(PdZWQJ|LD;s7UUOBoSNr#sxHhrYP7tfVE*``$&*S#6TGRZ)d0%2 zaoKj)2)f*c7UUL=&nwFPck`D!siZh>a`C^FKNwmvsd#cxe*Uo=dTM$`X22DUFL0U* z1Z(>OXQYoDF-o4IFL1p08+?H;oq{(feTPKz1?EeOCq(fD=8Kqg#BvD++wcYE?$Yic z~yYQGH`)1<;{Jf4n}2Qfl#VG~ZZ0ZxWGDcWBEuwkzZUT9&;C zDd%ces3VBK(?Bn$=d9yXR^haLvVIZ2Hu&G_aCs8c2VF_>;jkF~35(Q+T94v? zNmtDP@(#M_Xpr&W*~d^tVid9PhAX-{OmlCM0~+?TPC@E-$)y z@0QEe*k&|;$S)v&CH{VZhoSWEqkkzpahgVzG<0p!Fm;a~w`UUm$S|I6> z<|8}C9)PJyS4>6Mu8`YSVtQ#qDCvhMF)qYNAQqHjBIp&viMbFBx-_<-jm;ONm)QwZEK-}cZQtu) zM1ZO9rDmjj07tgX(&H+#X?wnOd!W^!q?Azn=lGq}Bh(lQm>fVqby$eI?{Vq`r;n)< zXT`odO zzBfgjZfQ886rx5V4HhLNL=1_{g)WQCCYHwb{4a4vna+|4N^47C?JQ$0UthGf)C7$Y zDzsX!rc$YC5j54iG$m6LZQAz`bAhUaW7H>*y}X55&a$l!*pfU0iG5ef2thX-K!jyF zf<+y6c7xDpU$vG>UBYcky!J0$WHHpitJAVWN5ZQeAeM8>!!{(PSZub@x7lZ8k*lk# zva`-U)81BN$5~>Zi7ecf8TJ`tjgF(McFSm6z>}Q3F0$^ooL~!hbfXY$4TtOu%ksAk z4mbj@+X4aFZ3_e)fk$kCV1z)+JN6liXYBfis4ai8&!q7T-z@dDZ7Ii@)nEG>QbMb9 z+BYPGRO5$CGHFbl_UvNE|b#9jtO`axTP&6O|own(s5_MiN`^ieO;LX5Q9S5 z?6h32N<9r23W%g-`eB(5JU-eo7-v+SmgS?a#{!ij5mbjO?6S2BbuR6Qk*P3l%fUmc z(Fhm_C4vLTw0*FHku0|a4jhS{%2KKh0!Idz#g^-#8l@&vR24zOm#GvjN759n=Rzzn zMT_$*pj+1k5|c^kd+by|de7_R5oyn)MxUvhBG~pR*wR)kT*Ed_smrLcI^4d)JS`C{ zZ7I^FhcefCSmRI}Et&Qi*}UmWA)g*6*k`0R`wVEy5q-5pz^#qxd}yDM+Hip(AfGrb z&w0+cwAsz*{kDK7IViOS*4qM}R0EE{QV8fP4V0#lv|RUOTj}V2iH-V;JT;P z?T0h1YI9IsXy^X4Tbdws4pBPiA|}&7va(9|c3^!6O1OivhB5V(XgTl?W1tPa4G4Ay zquA-W@tt+)VYTjPwSC(CkLFSZLI|vca2_v6i~*t)pmrIjo;7!qY}?-sN|E| zmQWoYcU*a^%LJqh8cBt$XOaa*XhXK}bo)#b&)Cl|ra8(!6W}w? zK9hzsee5%q`%DmL+Tl!S`^syPCJuGmp#jSPNtmJk|HFT73HV}Sf=DV+5+9WAK&AIp4HX8PCTi1pFrP!{Zj`FJbBRIA*T$jt{yi0 z)ao=0&1F?rXOE~JnNvL~VRZFrW5%9dT|F)s%B`**U!9lVX~M*U>PZu-Pn-nsv71&XK>J&n*Q_N`bl6v*)DZw`}eJd8T|`1!!dz zr=95(26sFBs@V}ZTfo5zV1Tq3v;l1m0RQgU6vdsxCxRg0XI={Gr$!{ zZWg#nk}Cx_S#op1alkZIRDvs%+yZb#l3N6Bs^pe|E0$b6xDv^&05?r?KLK}!%qaL2>fo zb&`t%S1-AEa19E>??L?ChMO~TT^GmZJK`KER-@_=&6!rs8#&{Q(%CbB2tMHg>9`kT ztHC{?j(uW2?iHrQHJxgpSdW@gfK87I|LRN^($mpW`GjkspJujbY|gP~z)x5TQMNqEqzDn+i&N0}h4kG2<)>}##ze;{{6nVEj%$*8KehUt`;!pc~*H%?m)mJRA7vHJc#w@C!;%vnM z7vXU-1P`+5avd6$Y#}V>kI3{T;!Elj;ZikiQp5=2$N$vh`}*#EhD&EOo=?% zfg&9|iR5qvo;uCu8%+FIRlRWfRC+wC1YhdDwr11USczQ9*B22KiFg*f6Uxaj*~w+wgd?x5PJWcF@50vY z9$Pch(8Exvs6zWWKH>h1?_*zSyW$Qfie>n77htw(sBT1BKb1x$8w)Ow$0vKm>1FNC zLmKkX%OirlCA(`8Rpu+P8Sa@K&g;;{BbLMElu8qyFBwbZcer!(HB4P5~+lQt{>T#CGmVFoGK?sFIm0^VHpwz(}Vus#3|u z1@6_sok-3oqv3g3jp)tqWFkGyDF#)IMxpHcc*;JVF8<1hvt%~TMouSr48e_K+nFmb zkIJcC5Sfk!RT1NH^*)w2xZMV1In{YZJC@Z%6n2KEys~*ms0%z-UQnlg7RlrOQk?tL zu*tv%m$pqS;t6&2XdCI=r7y??Uyex>k9%s=D5pkk^;5%fM&l-VJP^n3W-QGZn2%!kACK+4tAji`rqnvHeNjvUVz-y6C^d z<@lLLB#R4sm1cEOA47TH>d9M{Q!n_HXTG0!^Wv0V7nv8{-A-Pe#;KZ-MZlAOMx=>% z`#OkgybZcj{HjU}hKcdgM7w-4o%=)OqAx@`cyf{L)EadQ-P1l%T!&(GEKN^y9v!L? z8juh#QWD&`aqfYKWNrq--Cnhgwk_#w8&!#Nfhb#J_kEh~*fB}g|3X}oF&>eO$QEn` z%4Ci#6T8NaaIQ(NuCbnb6-L;XRi$zx?$s+i_i9Gum0u4-mU|4zbaJbT^)wV~Ur)(r zITuHX{2d~%ctsprR4tqq+d+(sbvHdD((odz%1jmfVLWlaX`j^L#hCXxswaS5yd$0R zQ`&rJ$Uc_W(H2QhkE{;&AosuM=)R6lomJ0~fTzx8nMb~G8{GPMtTPIYLn+)%XV7lKgpETQ^d~ssF5b;SIB0{X;}>*-ZN`_0lX$(g|&I3u^!K^%1XyK(DAOf zDm(R5)rmRCSf1yhk?q_tJ!J6=@znGY&P}_@_4Ks!J%agG^)pUhAMY(cGO~t0*uI1K zWqbE+pW`G@As?>YJauH0^Vn1M@ik=fS(!=OhEa}&k+Q5+r`23sr_LXdMK}dzkmD(X z5xlsn-nPh0I#=&yd8~SEw#Jc;MpddWAXg81bLHG9Yog`KY5N1(WKZ>Qbr;m7wUFy- z&wc7-`scha(AK&;R!&>6PoA9se}rt0gmc-|`|w!LWzTS$0ks;<+g4{d*Hx8i4$5?Z z=Z490E}Y83D`@{7^FDbTk;&4^L+#?l2cFK`$jGN)FEnaCPw#A$(?onEi&}MIOzm(k zmU_G;qknqHdzUz+rLN~S$ju5*Zs4@;z1LdzowJ;t&fBuFb1Km@@;IXF?>uEx=HEr; zU+`w$X(W_gAETbV5YeJLYSi;=F>1;=p6q2tmf@dqB_5Vn!fs|q&~|kjpRAi%E`|)q ztJ3NRf}KR78RNnEh(mme$3Dw1zMv};iHvN8jA(^qw?eX7AsMZZ^hgA6n-Qs_VU0vu zX+}RGQl_=$v`Cq>R%xZRN-M2Z-qTv;J*|~J=z&Gplh!KlX{{_wOT##s`&!_E83E)( z0wW`VtVkd;63B=E@G|O61p@~Wz=%j7I}(UY1uydvI?^M7$lT)H&Km{INdypS$H+)K z(AY;ri8KsvK@k#>hM`mHmB9TL0YnxM9!U`rk%nPlAVMP2uo01l;Z-;yN@R(lH{g}X zjOlJ&0Yut?!GZ{hNITGJkC2G8 z1FwZ%2|O7hfJi$sBTEc#4-ru!4Z~1fghZra7!HV#h%^kHTCW6};RqnoFuY$yNJLti z5osxU0ufOn4a4h{R{{-k1Q2N$UIim0A`Qc^P=rLJVQ9c3BqECtova86_9Q&5WyEm~ z<}6cE((u3w;IG){ny%p9kU0Q3B~k3tQxcR80^~GAvClOP!JV@TaKgQFRySZJz&OHR z>@(91glBnSjy)5e>xDT&&JQCmFt9UCKzvyQPBp|=MBvmz^_B;hBjkkFd12d_^YYqpnTK#V|!thfnt``;)ptrF9*e(r*(%uz6W#~j`=Rk^RI$pKG1p- z6tjHR+n|`8v)Bjc`M!AnxCqCmg02Kj2W1MWfl_V{C^nR^KGnxDv6$V&aUalm zpsAqP&B@9DT?{%BbQvgjt6Bj%8T4Y%GEi(#U{!)*a|i1}P;AI!T?+bh&|iT55_AJ7 z(|rK+AsoL5`UL1J8vha$8yHy0adM50)eZCm9H)cA6WtmC`X%TYpxDa5Itw%|Mp%nL z6G6`h?FxE1=!u|V&{WWkphH17fnpai>-V4`&|RRDLH`0uyE>u{)Z(}|=tZFYL05t1 zfL;kY7W5j>GeBZ0p;ig_1y~k2OQr6`V{DWpwEK-0rbzHPl946 zF^hwxl>a*@T;#0|7*XJHSI~nv9tips=qaGsWy~528XGIDT+js2`Jj~NKHxoYe5pSE z6)3hvu_4@cZ(ED-B@fFG+27Lm@$HlAT0O;#D&eq4LfqsDF5`BCo=m8ur z)W_$8{u9SoiYE141sWSCtQ$eG>4J4DDC&&$5NHq3$3O$1FM07sn@p zE&%Ndx(sv#=nBx&Kpy~|1WNf*(5G;GzCOML^kp1hqmLg3?bHtC3EBm;8MG(pJD`0) zzXhe9@pz}7hU00VXM&yyS_)bTS`JzRx*U{4MU=k*bR~|T(#Ow%UWMZiLGejD>l4ro zpd1mu1vDF!@p3^o{ElKpuJRSbkm$)1PzvFu%!m8GuW*L`;Eca4l8}H7>wg13X4UnOU^HbYc4@#7;Lt| z$_;jv!PXm$AMsTBo-`OYPf^%s2J3~kRbg!1Rl3|BMPazxHFlZ7RvYYggWYMcod$cs zVEYY*yI1Q=K-;R)NYb2NoNBO4gRxCla%UQ>!C)5}Y=gmWHW=G;757Ply=ky_4ECMD zV$jkuhF=WRT!Kh97~6TJFJv&>uNs?cu*(d#+F-XE>`sI6gOW<$3kKV7umcAB)?jc6 z(f0MyT!QFhFt-2NK7*ZWurh;PYB1cLTHh@O`?bNE4ED6a-Z9ww2E#a!P8S1DT3ogMDDII6POBE%BQ3i-86kVz5aDD>B#ugDo;x*kC_1 z*u4gOz+f*L>}7);G}u23))mhgWnT}?`Nc?soo29e3^vDLml$lN!8RN0Zi786 zS$%4-&kdG{Q_7ZP&H2SC20PVYrx|RV!Ok(*9D`kAu$2b8*qFRleQS7IA9=NGqwsMn)8ck;OLp2AZBU~bqL%HiB)UPFY3Tem)HfG!@Cf;GbFZ7 zbAE9hxM>pmrRGpp;7TNRr{?_PQE7puXIm)I4WL%jl*E3pll^NUU3LK6F(=KSIja6ySZ zt~tNh4vu4l3F0};`NeK*UvqxZ3*0GEu8-!>j)NN_u{6#3 z#aM8IB^J`0UrYrzNMdJb&M(Tqoh-2m&7r*lH&9~rn)8d5;08!+t>*mV8gQu+yIymC z@f&dcC3c7A{Nf>S{Uo+kbAItBaD64VQ*(at8n}SO-q4(1d}$>8`GL%;KFAZA^9x_c z7|}5!hRvs$J0QOte`>2$#o^^nTqrSnKTh9UotDix?~-Tj{W!n*`F-QIPCs|%*qqOw z>T&wbQ)9E7Ec83Y$Xn>jD!$k1la3^@pG=G>5j&_d1BuOE`&Dk@gTIBvrbMd+MJ7V zJqMx9oPjb3RwZB_(u2^#TRaFYe8_{)Le5kyZ$ zEJMy&IS4J}td)b%Le5(`2rcBym4nb8&RsbO?fJV0VTo~G%aNgloY^wSD5k}^EeBy* z97l8zmImjy9E6%VhUg&F%sDRyVct2WXb@~6i8}B1Ajo_-F9L&xdJ!1JSu&R%7{qxp z7lO=lrp$%FM9!7D5SYl>G8Y09IbY@=d^~Wb%!R;0&Xu_k*u&W}7Xo`YU*rI_!gsswG3-?}EoWPwGJ$l`@{)j&ZXf781qt8qbuAre!&?X6RT_ zM)H?wgv7fGFVl_n1$n{~!Z@;bLv-!JTOq&D?Q*v{%BH!kWtZ`*-l@l;wybQ@;_~@b zx%hs$@SmEVlO1pcvx{oF3j}NLqKc)DStI2s?xNaVyn|g-pB$*(Mw!WIyQr>VF}g#P zT~xQ=;_;&c?htHa7u5^w3#x^sT*NM_=x~YSa1XmfAh~3&yC27M$>@u)T}j8?#6Z<@ z7u5)T$f8)OWhloGP2a1)?||IJsQV18UD5QN4E#|oLpg?M`r=_9XQn@5bwp^(K6p%s z5uNa6pRDBYK#Z!7EAs0gGm@`S2*1T&v|VF+0KYXAHsX%~WY#?lv^RcGH(O};Pyc#VSs{+ToiqSA(I^`7om^YoDP|aNVzEbRzl|LNVy1oEZ4iV zjIyxp{5}Y|Cyznj+sCNyL&zPB)EBYq?iLu8#7gK6sAsqpf2>`1Uxe!Q!k5An{>Q&a zURPUD>g>;2v!tf}N7!#yZRBtc>TWNgcHE6I4X^TEEGu7Ji8t*3@%?tWYc4yR%gf6a z)zsBvtCt^Z|J_t*8ho7h0TcPO;K$tJQ~KjN^V_%Bv^NEZ{N^biAC+bDyQjyxk1t+V zWW7;MM6&PYgO3$0$lgA6;DaCj{%7}gTCE06lYMtRHK6Mqr!4*3lee52s2F$4&GPF? zXnu3LS1&xd-^XvSzHIE}uP?m}qub9zU$Sr8^84pB#6F&TXVTnhg#({jh)E1w$7nkO z^9xMLzI`L_N_*&UA3k#Sx=+sk-QM%xfO8r248k8n95Pn`%E$H~@M)lZ@pqNxIWYMJ z{@~z;QW#x|4>Hfkm+u1AHH)h10)tR92bW$@gFTurs0%E_w~|ZK(@O1~fYZ~4rwvaZ zIdVki@U+r~irV_p#pR0@Ev*>7;NRR@INCk%LsjrX{vZQ%52Cc8zGhL~zpcS86RqXn zj?W^nEx`8!5MSNHY@XYT!j#!)o#Md%r|zzd?|wGKW;Sh0`a|=PrfszJApU-fzcevH zwya=fUzNwc1r_9mK9v|LwJ$(b z`3iXnkO>_+e^2oyqj;%L6))qPk46V=9iU7<4S%!&Z%i_vn#dAF`(*J7^aEwpL6?!^ z%N*R7C3zBfRg&Z|A%b6pP1hVIfDE?5V7k1g zx~NYAa?gl%l@#6UKVVv6mU=M(z#){;^@Foi^&?caJCAzRjVfdc)3f&6y zfJ86eogY427`}QljtaxKY*pvC;QS%5dF%46!tmOS3XN}^xvsHjV_{?d-onPB1BHz< z4;41f7OM}(B(36SYcqFl#Tcwex~(yPePhvv#@W|4&b&EkQ2zSrq)VHslP=3oPI_cV zW6_#5O-XCml_(51Y)iO0OzRXq0xXX=->roWEvmQ++plr@EnCDEth+ zY~6GO7riLH(HEY%&bq{c`A?(TAgu*XwvJJ~d ziJ#a|BQ5n+74_=d#+babn|FL~xNIJVL-4VSN;F?wCAid&;S7SbrhKvYtfv=;Oi0`<%6R28IkB@v*Rk{-4ZT>3dv4De0oySX4%wch* zPhmdMgdDLS4F?sJ!|8HymQUc-rL2DGvZd%JU05cy*Wq)Ab<%v^(a$iF$d~5Nl4kMv z1xyM?*;J0-09N65@<~T;@ZU+`BVU`F%*}w%N-5ME_DA0_ZarkU?BD%uXcj2H`HGQ6 zgg=|OY4VsqYX7edka4y=GpofIonP6#L5Z8_IT6t}wvNuFR=m7y(Z)*g7iIadP7adC z+}bW3{>e))ZDaF&|g)M>T3-Y+aw;x?0etf#b zhuR`9?3?J?P8NSeMYoeGkh1*Io2MbZvpJiy;!Un+$GLZ;{7{+JG#s<`3Ss$tzCJRo zX|A-=M8K0)iInA!-gGTWT6VF;;_3QS#g{D8ntqhD=1N)qsI+zC{JObt84|g49|mf4R}QTpoAybr+wQI=)U~{82g&MA2!N<_<{RB;}VG zoma`@Pf!n@GdkBxj6X`}p(r}-yf;DW_Y@T+M(1tv_&E^}w;G)rB*q`56a7p0rJ!ZU zn~+bS@y#e}{n2X{L=PnXl8ijsv@uV~W3_|OWSqZLHUKe5ER-@CmD!TV%hB;X)fli* z#^H}LARaPolG)?MwNeS<+t7WDf={e{0!Rhmqd(js`!PE%5ziL zccPIP`tWu3odb`Q)8)_+=C;lEk?K&sxhVlib&-;XeUp3x#Orvz(S&~TiRDPkwv!#` z@ltp#y1dGSpgexW7b~t458$c(xtNN@m8Zf)J0H=?P<_>WzATdtvcl5jf%5o6WWkPG zvt(IAeOZ-$o9F?Veo|(Rw5JD(K$YvsQRA@em%u)}3?RIPC+`7dy2$h`_+A)m9cUku z7}t%N?ucW!#a7L)D=V+9Ex!<%$W7S@=v(3sj#n-OHm{nyQ}RF`+$`wvTEdueMYwi{(vm3p~1L zS#Os@4w1#KTA0vZer&!Sl*Podh!bUYonZzR7t3ae)WvU(#z!JTlIFWcvm+}Rhd(3>u|4Ny>)0H6J6oFF>@f{I3 z^-%pT4{jFE@AksBF?+nQtz@qkwinMIFtGD@p)H3ZaN9=I99=LHGz@|QJMcY3wm|@}KwX9*=#32m#{q`}pu<2r z;@oi19-ya!rh%q|20_uOve<@Zf=&U=0%fuC{D+{YgYE+z2MYf^5d_5|arRoQKA;mo z*#=GoWhZkID3@VQ1|6Y~bM^5Aeax+uc)m~{pQ(>4K?{K|1T6x!K&OE=fnxZ>dIc0? zcNP|X%i}ln@!R?si@)W0zL(8{+)(77gDVO0h9T#d3jh>$zCP=h8}%z}rNOS&T!Q$y z!G3M9O$K|}V6PeMJ%fE@u&)gEwZVp>T(y0ggQKXycy*M`vkg{mFl;EG6&V%ZHWR9R|D0V2>K?_Xhj3!Co=gHwHUmump6tmCZ?- z^NaolJK12x2Agg$E+1BLD-6cHsugyz!L}IeA%is;j0=fX-gX)6RfFv@*arqXWUy}y zHlTx@?qJPf!JWZI8El%tW*UqRfy(B~47S-|cN=W0!JaVKmj*jzFz#om^u=oqV@w7c zYOoxGjWJlM!R8u_i-wiHUmNTbgMDhSahNVra^p4U7h!|_%wT^t*xLp>4Q>y8(b6P{ zahmfBE-F>}Hfqi;`g+l+b%7x1W{zK8hu0k85JT`{L!HwsB(E%9L$wV zA7Vw{P`n6zIY^xHYMSWw^r<4Q9#3urJ9by_Dvtf+kCkJW1Axa$so$a9vqQ9W`1XNx zw1}-2H?h`BTU0z`P+LWv>}}W~nqKr6px4(fMzqVqBLrcf{8c{~$kjPk98;k;oj1nG zN?87K5<``d;uwP?X|cjy7O*M&Y4T~ek6L0c3rJeIKjoMI>|gTUcMX9%9^V_hxAVP2 ztG`<~WL&S0zIfoe50ek|zV)R)@7VnI_D;Kxbo$e*`@Y$H_pfI5wWd6_HRh)sp8aOi zPrrUBaQgUZR}UX@>C2Vp`g32{wD5~LC*3gjmU}*GKd)kGPR{46-oEjz&sSbGzWThs zpEKr(yT8aDa`3UFf=inFe)@;ltg-g-*pf3!r`~pIp3DY*{`;08(3#B}wAHK<(N=E8I zV*r^~@k?oj_IzY`>|qMAKD!>&H_lkwS`The$i%zsiEpI`H!NgWXPLs~*UiU(7Hg&3 zEiXaJ&{6HT;28Llse)awTwb9nK4*p){6Ls9dmNXXjTG`g1{=B52BeI1d(lXHsaYbJ z3Sfwr2)Wz%BZIByK^rh>JqLt6dFE8P;*ZiY3URxq@=lKRvBf|Hv2gRJ*n_~%>H9jC zl<~NyqF@k9;TZb8H(`SlD~^f~q>ON<#kv&h3-A|zgI>68loLyhbgN~;v7WS8x=vcx zf3@fuC)Na=mXz_N#TGUeL$oTb%)HXRPOJqwmOATpCrt6AwL1O&mz-Ev>R3|7lNLvt z92ehCG5`9Ts=a{4-8z<(@#Kr+R!+WNT6n@ZC)VRSmXz_N#a7;%ma5@QYrBpmWjtwh z^Q2Y0@WC&fSg-3?Qif^-`}ub8x)XN?9P8U+pEQ(j#4fiH?LO0bzRP=}0$+Dm6Jf5A z5a|HsOqNNR$|{U7E`~>5U42c3j! zsH%4Sf~tyz0P3nL8ft6mFD%37*ee#6RV-b)u%=3+Mj4z*#fp>Yp3ttjywqs!&+hXT z3(9Mjq-I7qNV_GJhNDR+9a-P2<7g5MOXx->Ra6R>)i`jULet|ui9>ghslxJ~nvsR4SvjNB8TxK_5r2d4_CuIq&C*pVn(sDW-aR3T@Ah1@D132qhhQ7N z+gI4n7%eR2Tn|3Nd|dVCQ^^&g`9|}R#l3h=($7DtZp+vDZ2K9hbw<<2N0$Y;b$pT` zFiJFiCn2HTkl}6>3@oB(`q*jaU~z~oJOax(nm#`8CUN%L6{6|ejq@`gH?)@$H2ID2 zWlqKE1(3N)%dtgi%Xj;F$lRvocnPEVZnGSkvxB@U0#defFO6?jJE?Eb*ZY7nQwIcJAWIqh~*wd0FA{YGN>& zFEhsr2jP#w_8066o)ZoXr{OP^ofL#3{C$DHb~LJTbZKqXvZZyU)$`_8EnZxPr$AZ7 zqN?&GrS%JHYAZ2?SX+OgaeXzOz(3fHm%j9}zWkdhw|3zD@w}l?46J)>PTIql_e_)% z`}6;)zP$f8yh~BQ@}0td$fMBtj1#D}7T*c_fwIEl-Ol23YB1 zyVl5UFxdaY-lZr5Pf=q?M;0|21&dvUF`#@E34yXG$Ahw{`RG@ZA>@>IslqV5BF|#C z2_si$upI_tbFSjb;xB-wVsb&_Pf~l#49y7544oM|t026;VDU^7=M?H zzXuUR;pOqcj3eP^@-vzSAl1bDj3ZF8hXYKB4W+QBQ*9zYIV!n@_W}h8 zyFy8iY)frFrhP*~(j!li2_{{;U4kEt`AC6aLP^&)g{=2`gsg9RH26dPUI~qU1=#jb zcxqy}IX~Qx7=9@{`{_`^Yx&{X@p<8@Sh3&ou9im28I);gcjEPq?RK z{czQ6-rEPV36kqA(L}5%s2{ZYu&=&j=%3FZ>rMNPgAXQDtb7+odj{~Km#`!0x+X;I z6be6wxHtnawEEka`VK6|rhV;5_T+_&b|f@I6!t>8GfJnKiJXy)qa>!@QKp$f_BhFQ zA=_25NET+dmwZ>rcaVIyAe>$ONsoVYVqM%8-@cd(TGHJH6J6l(1QS9@kH4SnLVFlA z!G)e+D>%so_9WQxt8IaLC#7EbYnqt6N9Tz)*wYpJBpdAI0(;w_+xk8>nBtNQ*kErL z*q31Z+BYwB4C-ggb#uw}mtg3t<`|dhsWupNfddG#klu9{-9UqOc4TIqc<~4*q!un zR*X)v0-VbDJtUh(w7X=}QM>ZvC7Z!plbSMRwP`2$EQ+doBU{4VBs_x3)O|9NtkRG} zR@KN+jCX>R9UTfc+jaJ|@G~#%#SQssZ=^FOXno|v$iY(`Z(`soca7%+)YijOMsHGQTR3QC}?g8gqv6I zNgR{NhGkuDZn$|s6XK3(Qpe+Q%r}>@{83OIp+rLX0ff61&)JoS`3nvCN@_x=d0*0` z@Kd4YPkW)k@`aeA=9cauT&MhX=S<9BHyJboMjSBC)mZleC--`~yg!j@!HR;_PX|L~ zp*f-1q0-sg@T{i?frt}_G0eyj|2X_py|O|eV`E8COjQ6`6e@dOVMp zWzkSxTd@ERO3H&Y#}?HIZNxD>x>;`6ex!D3MO9th(pn6&!QF(zW4aLGf`{|;Wq?ly zcA->0;6juyU!j(3vRw0F*q@}xWPBjjR+>6peTOZ8GmY|Wkx4RM)YaEERMa=r!Y{qL zs+^2UvI{=S*A>#N9tASPd*KZ>UKTB$=i0%pNn-5NpeBoU6oyNS{SmRB^ALNyKNY8z z1gT8oqkmF@Ii4UTUT=3c9A@Z!m?n>rsqFL&CttIW{-GLiOvKk7lHn28 zv16M~Y|PHB(t_?75Soew0*(&p4wpFnXewA{0Yop8Du$pu;-s1`E;W3ygemZ#mpe81*ofy?-`yz3MpT*l98&yWyO$D)QXD8Gk>#^9`eQFBb@Pg9BRzEH` zJjP?5X|Qy|gD~Cj_^Fr`;8_uheXdr5UpcYN32#e4-@J7=*ML8KU&gysgT8s|Jttj} zxt4=ocf27zUU&R5%|?K{LyLXt4y6-c0GP6^tviuY5Ykd@-8}YM%H13p)PZej5Ad__ z_q@i3pfP3p$`=JTh?rBh_;v-%;>big=vvTB&|m0d3|hz+iMK&d2mU@Ny}42o(ft4%DP_xXOK-qZz8nhVn@1Xqr z%D14Lr}zixVNeX5T3wK*&Y&lN`ayexb_YEfbO`7e(8-_^L9s-^Is+7+fU)>V0C-Ya zHJ}xs4WMYhtt&yX2*tV;v;p*CP=1!>Inb4$dqA%M{Sp)#lUW^O1st1^E+{|Ck_n2> z7gzBln_^aIctpxlh>9MEq-=Yslh|JH)iduIhGxBiD$x%{k)1v(MO zD?#yz4gp^-Yc}Xw&_$qEgRTO_)@s%@ptpc-0^J1q5a>Oi&w--sEqdu}17%;}Y0#ml zU#!pS#g&|2Zl|lTh5D>tTwvr_qm>-nNhS9igWYGa2MxB{U~d_WUXvvX!c23u}0&JikoTMUNluCZ4PhHI^{g9gL()EH|ywP0&%gK<+kg-tP7vBAzY zSed~tF&M6xj(dl}?lRc327A$9hYa?u!4gq-Ro;>{hpmeZR${Q98I1mN%4UK3rQ*hE z4%@5Z+AFM5bADmozn?M-t*=2r42@VZoB)gg!}4Z;6s^aK`BnAvmcyA?x$^r2n(wlT z`r1W&`pZrqI~OW^--`AV&^EJcq8=0;Maon6bDT`m%1CURkY|yqaWAI^+Q*sG} zq&=Lv;2-Tkw;T@&JM;z<>1@+iVr@C}a9l~nP#g;cLwms7F^JfK<3=im;uwPuCC8y> z%EbP=+yC^ouWW}N+kxj7#Z@PN`|iM4+kxk&*G}y@`N7{?X*(zU@r7^pop$=2c`yI; zv|<0)`)$u#-!_GjhA-bat?iozrzlvY`61U~BV_%=>jy4>RgBt>xAzmwnwj6d?m(UuB$11)( zM@^2n9c?(x8SCRLWfqknwCS*;PK&=iIpR5`(XEH~QLJw_e(iiYjvHt@WQZ4Znc@(xphdKmZsQh^BfRmC?KWniha_c(4O0*UqU9!H^x zm1ob~?YLm%0b!6b9>9gLTrPLy8lNEf<57k=*a)uln&iq_Am@ z^{scESL_QQZHy5dyC;at8^Uo&F=Gub=ZI)$U6FqTFN;wge+uJeksj~_+sh(74gcgB zdRd$(cgVZ5kD3l-`9$-wsJCwuRe*TFB-S$gJZ2Q{Tk5mdGMAOIi?-a-#(XwnaqUmEB$nVU!Lo|JC z2j@ZVJl?JdZP~~B;BhS@1@RM2-!97$eP65M({`M5^N#h)y{2rrPX!0%8KKJ9MrPOm3}@&!fwu$whd*?19{~5pwKaqv`pvyk4G15SXJ_rnTPcHps3-$7gsUNd_3l%giH zr}Cfb+4FyImp=QBDMYC=pp#*dqOV}_0gNkS@%0n^IE$~3$AR){p`T;1ABO&k#l8{o zp`Zc$so@TCesKnV$t7?lujJG&eG0qCU|ThZ4^$cK|ArqG+a8vn{i=*AE=#a4C`wQC z0|nN~k17?IDoJw6k4jZ?%_Z=|=o%nc{0%mOUmyj44F}8^(6K0Fm-(VidV5Z5j z?*Nuq%xjo0r<9W+;1ktLYZsT-%K=UfPpgoO+cr-QVIF1c&fF9^r^TC>s)~JDm1{GC zfYwJsbLQfYiSUhpfj2#Ga#USiUgYqVBZJM;oL}xFrQ{wmSX*Ut6C_3*rEJ zYnRk4`5!19j{gSmXNPEIgX<{E2Dh^=8*&)>#;?LEHK)o(VJsUZr~U61(f_VKHE1kIKekm{fRJbKS{9JfKV)Nb;H~W&-ubkm3$AhYJZ}9 zO|Z+}OD(91W8m~rzaX^o5U#sbI}|6$B;5x8E$&iCiM*?WzO_va>2RefxSDOb2sP#= z?<>zsdSpAjruv4i4jzF(a)dxo?hQy;9c|xmJ4$aI1+H$}^d5qtQ*x6bn~1-7JDKoH zdzoB#LVWYyZp_tRakTnh2}!GV(TealMUCgg;V&_%kJ!qm4V={ALj$|>!aJH<;zO(7 z?ZX^!+t!?d=6!*rM-GOcf%nn5*ubzpXZ0ap{c(BW?K-*eYcjQcr{{&Y!NoBU3csSy zNIcYdQDT1Ma0 zn1d@lQF2w_SbyTg3UGOny98Wc$!!1^lH4r@`y;r?68o#c;&9Av*z4^m@7)}cwTk!u zhoHk+--oWv5$)db(e_7l?}yk=wqo6(n>oNfcfO;#+|W@4y$l@!cx4KuEPzn~iTn98m+m0G%@JXH}qD32#% zH@4woGm7E&u;fh?!wK-7Vp%83>xsuqO-1=4M;~{BRAswUDi~tQ^b2|1!M6a)E{3w3 zpysbA(|&Gc8D;dfs9#W9RbHtqm9Mze$$riUQb&?ZBL!DM=}bWH#buDv5ydil6DhV7 zY-q7MT{-}9?v8K4^~gxwf|_dei0lDOZEE&{OpiNyu_CidvmU5ei>j7m*$VBWkB=H1 z+vDqG&hr&to@Ny(kkN0HSLt}Db5{v^BIz=vN|zzL#_2NVby8CeDyWK?0#h819KN_F zcAS(Lmk`zWhI@Ra!H8MRg1F7s8x`Xsn#h}C&nj6O@UjI zF@@zag*CntMPF<`I9zPisq~CUrG0FybyGsBOe)%@U-nh!%st;K)Ui*fsu?RS4@A-V zVnQmk#>amDc499}&56*hwiD~+(#CiH*p-3y?GrorVPah8sCj1RDF`hrs-Dc(qHQHc7o@omRx`^{T#y!cPDZ#O(c5Sl1e9lyYOT0?7Q4A@ zd5V3e<&pc40Zzl8+DWWp@iOcOKfk= zdl3c+txOza62c0B_63~;iuT$%1N3Cj8Ty!2lE>$Q4hEeKIs~*Dl>Kn_pJAO7Z*t@tm`+0nGuu9u(7Z)=i+JL2uXR{{%W7$84JLyt6ozln=TGbOI=wmx-W! zrWb(j2gUQwVlxAqtytuBI%pTrGeOxroCSIUDDq(Ot~>{ncPsK@aqo*#&>W3(L#H`7 zW;;*@It%oCP&OUsfvyBy47wI{31}lI@@|DeD?on@iZZb_fzAj09Vn`YMRyY1bk=>K z7l6XWh{IOWZ3<-szk4Ar17-iaKmOGF3ORgz)nN6S^9##hYYcXe!R|NM9}L!Pu)i3L z&8$kJ9nx3ncGMg;N-@|;1{-X!VFu%)RK?}Sze?X+gH;-AmBB7I7`MJsac?x(4ud^! zu-yiG%V2aVQ*lqyoL>w!*f4`FHrV+Fqf3p7yV76}8thSny=|~}4fc0~9W+>{SX&=A z9HeG^h1Xzd1{-6rpuwsQcAmkO8|)&3-EXjm4aQBT`Xe>9$rL%is4&=k&EYHB2D`vu z+$c)LZ8q3`w2lfppgDXAwrjlT3O|*uGdhW`S&5?SHOZ`!!fs>855%AH7ExS0j?qp4 zbNiR@oE=iZs2GZ4j8i4%j)6AEgaPs_OqKnqSUIf7_AOQv%$PAc5Gt88J}@aSFg7qC zEoa5hfb2>I(3J|9t`yHTM3>f`5l6O839rZq$PNS|woZawhrslr8AAiIQv#Wx0oh40 z$Kw0r)6SzPXjoFUVi|ntt16*d?dlzn=?-WJ3|d-UT~}2v(^X#w0XsOr6{WR|P_qQa zmsbbUR-~zr8>p!Z)XZOkBr9Q>U1Vu1RFNSo0izO8`MnXToHj-~V>b7>qWfRJtonAr zTH6=M-rVQo!^8TFzH$8dwr|j{ZWxwZef!FrY~P?i-SP7w*ZgAnY26B|9^F)2z5Tw# z&oV2r=H6@{pHbCx+>{TW-Wi;>^W?jC76mtalp}7x;qsrn^Xcsgul{l71LxIjT>fnN zby?r%EA)qJ&&ug{?XG9PIPTh&3r27F=&oP)oY-^C=7kp|zx~+GjNx4;7C(F6&z?Rp zxyy!eSKpOU{f=Gf{`)oq;=H{#{4n1l4l*+QU)u~QS!GKs`|#y|u=8eoukd(hdx!hp zw+p}Rd5EBVi`<(WB?493&G3A5Jsc46>PeKtxo$ly;aDGAJ4)vkW}#Dr1CJ+0l9b^b zd@eYK&QcFK`$8+vCL)w(aQvA1DgfR_0kmffY}pt(1MzxDDSs$A!J<8+;$Ov2X2gVm0VkQpS^37a5B! zI)iufoec^^hZ>qa{zu0V@yV+O&b#@}?K+l}@ubBZd3~K6Z>$G(EGgqjtDB5vr{&#z zXPb^CWrW*Lpu3DUT%Uh2`2f0tteFfiD}vW$V(wV%bI1CI>GMmk=y;D4OTE6TFUEUg z^^mcGI@Ss24|>dr1qXQ zC)WEq7G=D#ddXOMI@YZ^-|GKl5!QJ@P*tZxr~y`da2qHI^>jFQ{4-9+rM zVH=A!PUYS>C&C%bevrC1(`i%W2Rn_88&@_ySe7=vI5;`CAXHp1bxIH&m2sKlviaN{ zmpv(OT=uxsys7*hKLLG~acSeSWv^viHo7h2vV+1_F*P_jzhGi08U~y4l!GyxX z!l2W;8JBHe6>#WR*~YPOCM~UYY{(DhmrR;4Wm0G|Y;a04ZH228(UBRKt$H$APY~a< z&CJRQxPtN6^Z4;O`yyqI$k89crN>^q+!tx-B6t_*J0hCLUKX1*cZlM#7r;~E0?6IK z8xeuGO0bQ6k?ztmQqY!*@Lb~aWg>4+SCF1sQRJo^gB+iz#m6W&^BCm((06VtxnL`g zDdwZ9l^nh;8<7vZn24HVlv{EPa!j}G809WF202__aq%(8G2K0Ko9k3#N~~ zHg_@+h6xtFH{rYhxG^ab7b^W2tSS|wX$+=sJ;HYta0AmMsvAhK@G&{=Q869q+X8)W z19z#~$z$&&m_BA#)I?2_q`|_+IjKb8UY#T{p~A;rM&ab&*T?ua0Qc*Zj`cA=)=&L? zeIEdF1#l~-No=V6VEFD+F`Cw3@x27$dltA;r%P0*__8>C2u#-*5(lxMSH*9m6z+` z(K!x$!sV8@LXcqLn}&+~K|GxQULbLyp5MiAUXBUkcQ{E56DoX15x&E~d|V`P>^Y0@ z1zEBDOtHjh|DjeazX9An70Yo{uocU3;Qyv#U==2mzOE(cFXN&KrLRBSCIT}@#hGv) zO5Y-2s#IJkeLW$65ir*U;)1Mz-U`gKfw&;)_c1U(2I7L~>s5;P0Jvy63f~N1<^muz(KiT~F@d-s`ep%hdLS-{ zzBRx!2jT+sF@J9b=CMFr5Pfd}b082GppX57<0?kRICS~zT#m(jxM<9{f;}G$UotR` zKwOaUl>t*5hzkfG^|b>z;&vSm{9R$J32(g zNMz{r8wcFf-_ghRXhE3zRseTypuT|qF2j9YFns~IPQcw529Ek}4FeYm-2a4$yE{zW z{bAxB4imRIOx)vP;5s3`Plt)y5hiYT7&x}`FNKMFBMcnJAMb{V+aD(GvoLXAhKc(= z3>@pVU)$l@j~j`Lti+#>vBd?)#fFLN878iOn7G6+a9t4o(P84ohk@g`dwQ5SXPCIz zVd4tIz;O=0G7KE+=M`b%&JBk1{~%1^`{fqkPF(`UFk&2~`3BX}$U%^z9|MDGanMOi5}|IKuFn|x~kP!DkdY97U*18RbQcdK0eU3 zroId>i{u8(b@f$eRbc<+P#ZN9HwY=Z3OnA{0oMcf{30Bi6r(^E)i;zha3x)I{Bk@x zD@%&1aW(*EjiTePN0+Urx)#e2sh)@isHv!{tEt6(4DRm&BFp7=;1!z;iy9Q}4c!PV z_l8bQ0hsOt7gg1kHdbPXkmz{s;iSC?keQ9dh>rgsqKeAu8Z@7m*K!AIoK2u8-{ES* zg_;0gq_zK2%`W)AywmN;eLFwMKQilv$9^4sT@If?;?L@S)%NWx`W3u-^O>9XpL{x| zV?Gf44M*GB%Cq_(nDW)a$F03Dslrzw@F31~()v9MEQ3#XZS*er^0U&rUO{yx_;>zx z_QC~&yM2Fa#rwn09O$_L`%4S{yW59GP5z}RV%q1idmh<3=}ob75cKp(*}G~{Tx{fm z71tN^Y{{9UC_lj8b8yitM10xx+|&u%yu)4$dp52r)m{MmnW~*5?xm`|4EAc(UJd&) z)y`zEQSIl!UaQ(Kf_=GazZ&*$ah2kE{)1oL{TFtWu5T!>s>L^F|MAw;dMQ#J@7ZmQ^e(TfU~K{;$-DOi+WV_w#(rBI!SS&(w&`4lyYzsy{n6mja6B*r53Dq8MY29bb&aP5 z%feA9Ruv1n}HhxW$NGjUFuWoEjKTWQm?cL^VJqZ$u)7yWCv-$l78^2acaLWXJoI!SbUvPR~IVeES zwc+EBJE4EWRUd~xVLVz}$CuvnD=+<|an=mtb27GxCbjE> z7bCtOW}J^rtWH)Eu$Y}6ikTEF23v-`FweqPpQR{@8{0=piKv`MML#pU3}hZeEuSV&I;I%NXqf_$=&KSjMq_{Dt> z+%M#rrb-zCd=@U2(j?*5?oa&=&ISj+y&btjBMTzqoM#FZXYLi24;15=XYz}q&xdsD zfzS-|s2CR~|FxF*A?$<4e0vy_d@L;WaPs|8P)>$%pS*#fTR{0xYz1Z1p8!Q?f`?v< z1=!m`m*bvzCdHGWEuc?>vP@|E?bX399~{~&;<1oCN=x1?sx3zOAD#D-&U;PgeX8>g z={)YJC-udvHjBa~!IC#f=kW!VTarOjjd=58yw^mx~_u}uoHtat=#5&NXR9XkFgDNOH z{t7ma|H?V_sC6AmoZ>nExYF>s12q7+j+8y_7RO__o^iGodE)Bz*4h?J=$i3BjH67~R(AlwWPs0g^m3=kXoRsrwptQ^YDo-hqg~yS)gix8MQKxt}YI19O_&8U{IA zbH1XVuBp2lx_TxhG+Ld<+gwxI8oy0#J9HGu;%>Pu@oOdzKum-u6F1=w+vcy&xx?Ms z_?651&BG=;vq?dit?O7k+%5Me-bw0R_5k+ij+&OxcnWv;o&~XWdm3Zcz1|pYKfZH) zl=Jv5m-i!=_bu0`51rnZ-QFD{oYu`bcR0=8+CAqTA7j0I`{5oCZtON~jQufFj`vIJ z<=f!5>-aAB)E^r+QSVSF?Fyw`pmbLhl5>}9)Nj!0#y5Rx)Sc!Z8FRN+hRk#BX9%5) zOT$=f%WYaU!j1xz+UEAE@%82m)VIA}yOYJuen!z5r!U&!YE4gaPW`&xvMWllkNU>D z>+lO8_Fd`Y6xg3R0mu?0MTta>@~?lno3pH;>q=J1dY1L6@m=uk3cm!KX)-Cd3#Bic z%=pr<5+kx}lupV&P}}ZTKF5$?x)_i7RrI3P?uL(nMnEz}NUw~bcB8@{3we7yzi%5W z+|P>Gqs&!WOqYt!#b(3T9|b=b9}#tdOEm^n@k%zrz<{#?#qa^mQBg%rxv0?DK+rpM z3L@AdGLhQc?2)6%X#er#3D}&OZ3so5CmP8gsjR_=T;!VsUxs>-={kh@ zFsed+@{_OTbEIeG@6tINMo~Km|j@vs(k<`Q=_c5v8F*tR|9d2Kt-8eLZtZBm`u0Y z#)jpM4Mof|7JfHC*9u@K(^~)@L&0NFa3c+BMajj^)q1?EwEQB39li}hbWaEQQUHXE z^Og95LQBBqbrmbCY8zE`e0~DckG<*Zh4!lyJ0^zcBfhlI)5LfJ{qmE=;3^>&k8K(I z2{9H08JAv2oaym6Z3Tv6CHUr`P9zu}N-;BmordvISG*_({I*jD{cWVmT;(EU*Mlev zzWbYVR>8b>Ud)&%b8NKOqMh}b)GiX4;dUZG{NkDGcbKB2b_=`8Q(r^l;GMD8y3nFGLz3&OdgGFw#({3h)jRvLNpM97MPJrujt=!~JS#ocjS%FZp$}NBVx=zM3M90PHsJeO zq%ysd?OZV^*Y|v)@_V4aItceYLD}E!51Ivv&$c|ppxpm@xq8ojEKO-09B25jXR{{*v>5bM&>5z+HH}$v&+DLPgT4cbkJdbgLDz!* z3VH#k35AgRc3VNIzc*+LDAHKD7&H;o3(B3SF9GFeahHPHL5a%&-2hqwx)Bs%SN;yl zI^lXyVB|jFpjYDl3eamoM?IU*@rrSq2QytO)y4U)t?s`EIak-S4X?+cyx zlg|51=Z!%6$gqr7Z5Cyc&YPz57VEsFIuH4*%AT+Du;;qUyG!T&L+4@SsD|$Yowr}- zeXaAp(|O~~n!d@Z&7!z;UXIQy(Rmd*uTkf%(Rr;p?=qeDh|YUV=Y62__UpXwbly>& z*BhN_nSLj#Hj8qK&a>;hMxD1t=e6j(R-MOv^rb$YzAp2!P3P^@d7tRKgF5f1&ihs8 z**j_a9I6c)3hBH8op-IyyHV%et@G~Jc|YpBUvyq))aNqHF{%x-nwTk*ylB;CQPfRk zaHtz+z!F^EHmCPzj#RxzoZeU2GAPYBZfM84_Z#9csCqmB41HvU9ZDwQ_$AJ1C1J`c zp=bxEBs{nx`Gq!mVZPH-9Cf4x?BSCKgcVmNF1~P(##2#;cZzt=)=;)Y+rEd5iiu~+ zkikaZ7{r_??%C5iQOFMy(^o@a=MEnH08s#BZCXh3sG)vq9N3J3(3Pxl3FD|j@qxcKHu8V@?!M4yYr{a%M*}8(O^?cp2^T+WKxok5W`c9%Zw`#7tI;S1X3im zhv6(%Gll&anJKhc#GW3KSBQJbyG^ylVEY!0=fvYXYMZm^sCnf;&soQujeR|59Y3qF zhjsJJ-&*Zk@BFDT+VhjE!Tg|UM~abp{8T?BAob+19m^Y`oA2(VVd_=ZRjg>NsKMS5 z93-ia4u_gPLq_8X?Ju4WPBX}L`y7_}OKBr~;~^c3hcjfyoR20d1em@;HGBM0b;R#v z7rUa&F7<)(2{JR1arAXzuzY51=hH{kFl~x)Q5ha1VJ=j2@*+?MZ#`%qP|UcB$FCK1 zB`=Ij3%~A838rdA#XRQqxbC`WWle(b8Ji2{VO<_xTqeX7>1i;v;iJ z?D8RA1H(@k7N2Fc<=FkErg2#*#)sjQphz)9VhLg`%Z)_3ptk@0QZx#6xAwxoZy?JO zBvlPVjcWqT*kgwkMQoQ|9DucBTO-Nt;2jsL1PjJtP%kQ+KU>~dyZaV5re!fM-{OXe zMGsiq)yc!SoDZ6y-VX)!;5Qs}9q0(qCQueX-m|#zotP|cv{{rAomZjSWQSbh*ddp= zYjoTVI`2uH*QWDWb4yv)+)~y#_4$S#)~8Lf`mT?09^VF5;}`y==@j6{1(c=%&`Wa? zE{-~-VQKmU<*1Whj@V4f>Vv%BtBnQ(D@R&4q=!%yTy?tw6czn{rT~fke0)m~)2|qp zZwW%X>H$maFT(^uqvB!GBCkcog*@~0!#%16&xxR`K$-r$X9?mETb3Z&WWPc3*l&=$ zr8%3h$kMp)tb}q_>^kcqiLpK(ss;g&mLZeomzV$CpY}^97 z2CSHAdgJ1Zi8L%v^>}Mn5mvPnnCDo|g#Op>7bnyT?W-lNICT{v39h>1is#Q(Htos^ z)20}gZ&^V;dcfkVK@ZEyGEj?p-yL)X?t6lwSrUCtG*n`*DBiQIti~nF3T+ng?VjYJ z10-B6+#gBBv6+*&jXIBwo856W;A<1*t)#6>d~iA-7m8Vfp4 zO>{iSYNFF-5#4#oE5p6y-K*MS#Fng*r#?_gp&qD#NY22)Tp#G6;RD5$VbU{Kbc=9QS;vkUO64pi4mcP*s8U0<8dL z`tqI+)hb-_p`uN0CM$W+BwXd@Uy_IFUgce_+F}$A*QD$dI`0{s_kzxQMd$HRlKLj8 zHgz6BOd&YE?_m**bJV*|ob-+jW3}}#t&vlGCJNYyti@%lUU5KIQb3c>QqR;Fifg%1 zqL$UJ#3E4fVk?}!BkIiGZQoDOU*X_ghVcrARWj!Zq~8O@Si_hGi0f#G!gmL<207_> zW&>DjGp4=+5BcJaLy}1WE*z5Zv{M_?1cQt`lTUbk#wv8z47C^IVhKybMT5#zH1$`3 zvIBz>;TZ&a4Jgz5T2Pj-b8yKLMw^9O-Ko4{)h4%+lel#{?{*z`m(Kf#&SSML^?ji8 z_UpVtq@C2qE~La^IYx`g*s1-N`uoi3Q zR&$bPdi*gsa$ZTUJNBJc>fXUf60f~csN_hJ8uCn1gGhfq_fC4Q8iu797e|ECaM4Ue z>XIUNMPoV{vw;xCIPDxN4Ui*4Ciy1i3x(|eaGpIb~f#a5Rm4!tztQw6Y~l? zCl#Hs==2>dEY-Lo#q8ZOVT;875nNXw+r|bgV>lgrCdF)AWk#0N*U0sQc*ds3e<*UJ zD7~?=5(7i6Lj(bs1aZ%&F+to<)So_+qDo|l7qk@P)5ki7N1^lHm{&MSc=VR^D9hL^ zo%4sRVYzuEC5*?smw3a!94@{)sB*2eIUy=K)-pnSOUD(Gt8p9F? zeJTfEf0-|Ra53-rz|&mcSvEH}fhOSoW>DtaZJ@-kCohX5Z5DBKgXAq#T`kWt3&-4vNxZd*C&1;kjo>pg>LVwXPxZ$- zpq0Jtp4BM|C&qU?o`5jv6XTC3sK(j!OT@}F>(e=yczYGom37gr_Gi%bT$Nyb%yA4$ zu#QzuWu>fGFK=7d)?le@iFbMrIhzhgKr(V=UvHG3bIS?Nrd^S6j&!zW9Cu=m^5vN3 zvsf=mK}@wx%-5}@LZ^A#@%`4PqnZwiuI@!~K)6~~Mjs+nU9*ZcWm=HewON}>&8Tdq z&$Bjv!T>#;rH;|9O>g3USJcWwKY|7q{V=2pceM;I!)774%ilt{G_DDb=WDItE`Os7 zcdZvswia1zRUe78NN{?0Ij4T#(4%sLXHt^2nHy$eX3IJC*~YI#&*f#2t0MOwG&#LU zZFeiSXLor|@8j|=#mkcV5iak?_A_>b^nyjBz2}|myRwpn!X#(&aqGGURIR~)*5-x6 z<=NK!1otScXp>8eT`e_z4lSjNTzVXkji-imnHb8Sb&FXjf&|sty!Y7gQmYp(l(i|=b#rN#S$K%uir~5ygw!=7q{~ohM#d%} z9~u&ar9fjh=cuoc@^0_5hramD2vZyE!SD_J)R59qU4n==Xw4gdN4BS$QrZqZMA@Ku zH^>O-p-V%?v^J!CK}eV9Lb}`+%4OXLzaBq+C?cfuQ*ah#aUU$f<8vy7gO)3g5MG&Z z=oVO<-noe*oXxK{xKOVhmu+k85*|y0$ut)K+)kFrUd3Jd@8lp$Y-8 zUf2Zz^PGFkX>C6K3vI{GvNnGY8`h}G*C^KJf6?J5mv!BTus}hQ>ChXnRc=7mg>7a4 z$qrnknEkjw*w`liAZ$U7>(*f_X~6bGJ3+;!lw`x&YHkB<8XY~0&9QtS>caWCLLTJYY+zvO)i zTa4g+t@9#K|8^5R3v3>t`4re>EV)pGbws4X{gtIFmo4-Zyo+?+ov?9nO{DUm&f5dq z1w!Auu%Vv=*H2)RVfhxe9)fpF=f$CV?8=K@v=cU6oN8oaFUIlDi z;uNVY*LfGf##OSB%EhqBSbhfETA`1_-Sx1Ul?K>agl#o!7Yo}tuyH787Vq9J5w>;U zvCnB%*2Bgc)vR0s8+#aLWdm${RL#oeu(3yKR<4F^qp%^8Vl30UbNi`5?svp41u56% zN#hY;+VXoEpVALcUCq!|vs?D5P9zA8ddh?Qd4EXznk%+xP7u8Lktxb)<~DPG_(vTk zp@Zwn@Xk4Mfbw^! zyyZxbgP*rWxH(hAdoAt8V9Ctg=I%bP200HL8|A^WxLhnD4tQl4<-u~g+<4#fCAg?1 zHqx^(COT%MI0AzqkQp1WmId=IFnttlLH@(y{>Dy|l~JZuo}&n}~%8v5-23AXR&dA*b)kT87N**%C7f!HS3sww{F=cc>bfPSp6CxQ`Ut zeB&?DrHkjspy^^+n?inHpY-DVaBTDiCm>a@u(wN)w;{|4zNGCdO2g&m5y~0J3AwoU z8!!bcLGcK&$HVZ9@;yL1*Wm)d5qTD@c9)(m4%-`l6 zj7MNxI5Qa=+5C3QL?mT&R0qoz_|$zYe2s3GAN)?*-279_$w&}O?+$|fn^3^u$uBpI zVDIrOs`4%Nh24T@$%SZ|WmJ3%1s8dkFB}mw3^D05po5q&XU3vtIMlM66pK;(+( z8hmco53ZbhV@dmOUwkiC%;2;)m`oXl7xJu~7z&nDCJjp<%L>z-(Pl1o#mA{W3akoN zoJ)g^u_sJK_2I`DAHW#Tg=D^n@e=|VX@NRY14UBP0z^^*5h+Olh~z+}$>ULv$QdA3 zH9Q5zXni=Ryc&`6bz$qm=}Zs!zSEigz)Jv+Edjn|^lJ(9sZRE*G{g3=7&k?@B31tN zXvo=CqEG8#6L%wM2IwuIoKw0L^fXXZULL*)yd9Kp{~iE65A-2Ww#cZ=JokV;0(w8_ zqoAzdwt(VNFlQvrd}#xH3HRGTUjy9<%9#uBJW(jNFN0b@UjgNZT5p1~&xSb+PdeyF zpnMsz50ur!A<%NruR+fPMQ_}5H7Ht1&%>b5>Ddk174)B=CxCKkHTspF?x?&5fU@br zF4!I{E9a7S50;rLcrZOhpy*e6u$)^-1YH9<9F+4GBS0?)#rWED3n==Ko+m-afW8Du z`L{u_OxeR_x^_?t(j0A`ClM6Qv}Zafx(FUSXdY-5=t5AQ2vP!i0ca`c^`ICdcs7GB z1>Fu>4f-sGzxXyR?q~{ZJ;wip9jqXeH(NEDD{d9 zKtBY%67(ZbE}`8AdK2iEpm%{{bmDmg^n1`ZKz{^f^ZyIzd!RpoeyH9b21QXhN?D;7;#A6WSz(UljNZ^sbN{9 z^H73R9?FP{J6Gpjrt_}Qd3;W!z87`g7dr2V&ihU0A=Ons_&I@;t=D-RElOUK&U-}X zJ*M-}ZmYQGbY3TvX(`)9wON#2I4M4k0U;6u_y<0-lsb6h|c?7=beVK zE&V-RwP8;Jop-&?yHn@!DUp5*Lb;Q&LsgqaN!EGebsiUjbA}*BIj-6)N)*Z_>(Lm+ zqS`D8o1aWobqnjG3J%DLcitQMoJRc#jKQrI|>i%~YJHj8o-Y#issD7UIMoQ(<_M}aZQ zqpA((Bw>su(4n|BhE*nBLpv^Xls@-&bE^ z%ZBDQ9?IzyFTr>+XTDXu3Bv{vgP^4vSFqP$sKT&}Ejyk*X=X7jJ#jg4@#T&*o|S*_ zJs1am2|_ka41kM=WL1)hm)2yOrXN(ll8_)+YMfSSI zZtl&*&L3-u-a=%8xIb0*wD<463*$R=xif?I_KaP*B%v+vkISw3YRd+P_V2=&piG4z z@51uoO^!j#CR{QL_$ZDT9 z3y+Rdd3@0(dDJIyOLg9*I*$*j#G$LC;%?D-Z8~qK&U;nop);e(eyj7?`H}j}$QbEw zXVqp=*Z@c#J3$i1YFhGGO-r6U!pnjqyt?2BFC6DJJ8n4?^N2d#G{Ghh;eFt?`df+(H&igwpk1RGPMdX7Gb2TnL zN9-apU$7#_LqWEo|W`d;&I@8pRf8UTq@?N<3`ylmF zC%#wIO)$M%g*5g0rd}w$GA*c+jl5=p=|v}B4KGnz{)N(evp`cPzd_VYFul(TX=93W*n(U^PqP!AG#X8UcM`{v zDs3#OQi7u_!Ce zs`Pp|em@+=1T85II|x#Pv{A!Xg4eK2!M0jslvU%DjQj#+PF{xM%ABj@xOd=_ zv&_blYGr;!ZHhPEk&6EGZYjIaRAAGQY02tfIcYwhj}r znvG$SuEa~5qiXSz25FsJR|UUScFkh=Um?DpLzMr2^Z$VqVEN^Pl6Hc;^Hgl({}JqX)SVcP=R2w`i3ZIrMLtxp(Q zuOHi({ANVf6~_)v*N$!c@$j*CXPG{Hd`arU@zbXa|Gx5$k(Xy2yL;o7FK0&F@yTN+ z^xb#whBYg<)<*OwJt-#jl=Pc>Oo=VLW6Y1u&wtgNy)Wj?-#-2OGs!Q0ld$jS`|r<~ zv-*`?bFLbH&d`dwjpKjp_TZ;KKb`)}xP#ZEr@sEptaY2$Ui8v+8B=y|JMKDv&D%fE z>UPtW7gnd_=B|0BqGaPoyJ*VN zn+nhFH~-b9t}bVb<=A5veo-*u;lsB!G!AWexpM7_ry`#CTiXQ*qfdJ5?nBS7>vvh( zrRT)``iiKtFtej`C9hXe?Rla z=AQkB4F7CQ>)atvt+x-JeSVLFv6~yxhHh?Z%ldrb=}qg4;vO0GVQc4}A7Az9xh8;WENknLGj#g0(zM%uxz;s(_^O?IK1}X? zMe8@q`b@v6Y5N;Dt$*yzf|Shlwa2522SlH1n%r~f`yb~_exg_O@kjoZetFGZ!!7S$ zcl7Xpq+`iHJ@WS7A8%fGiaF}Ddj_nkF1fdG!O2(It9u+Bxbv%s7kiH^yyx)??jO0Z za?aiN?S8R$@#SA$JNm0LTP(32OG47L1D8&}*Ro_?dD5JHFU~l->(;NoJ@x3L5%Kr5 zjQ@M`oinc)cTeZ!UlYeZf7SczKDy_+oCj{byX=)$-?X&cx_IW69Vye3|9;M7M`5D- zjW<6!>2K!`aBTf&%+T6Ze!Xk!o!z?3?e|ha_IbbVE_zzIa#7=i zWe?vKdGOnACClHqyyBq)gC~CUh->YIAME~UQHizb<@u*xb?xA5H_fg4uJzb^L!W)` z-%q?Uqb(!9*F~R<{;{QV`nA{G`~0z`Tc>XvVEgvK!*6-JbUNXj_in#q|0L7)Syyj( zWvV-4Y2}jnv+^!{y87(urAxZJS>I<+$)2Cq4Sej>=Pvu?r8Dg;FG+?XSzR~^wDe6H@tH7=dsOqO(`@T__WiKqIvIK z^wz8=(<2A0E;|3Imwr>)zM6K?@O!sJJ-IvW_}1cI3b$rmuN#ffvtVPD*Ay54Y9ZN6J$v+XyV_)qivJ;yN@t97C@E-8-v z^EsfP6F7!cwl#t>-&5-ql%M2;4e?eD&DMS%|X;M=z+lvZR3!#&W zV2JUTVvTIZyE3@P??Xo&HbqAz9><4aT9cV)UE z#StV8G5%69;22VSmbJ|^r0%5%3^D#v>}wh!9CT~N%ZAhriog)#FU5AqkZL$%gw>FG zmm)C4_)B#I$82KS(%jy&+XO?3y)_zQ{H37U$FFC9d$!1sV*5u!45WOHMq}$K5B?E2Ec z$FpV=#}o9c)V%rj_Ii2=3=cwE2pdh$c6!*xn@xH6XPN!1+r@Xbj~6Fw%qF(}3_Z;W z?evH_4-@l<=Gmc*o7?N@3m3ELP5f)=)2E#tKAmRMSp3u2Ymu9_B7h zzjk`qi!qzn3!ypv*OM^#XAFJQnX_AF6ZEY<#Y?5;T*r0z?5r(UQde|p2 zo9^`q&wzG%5(I{+MB^FhYN>xcGwh6v0D_}9{Ta63JT z0;7fJhZpNnPy@p=RA4UgiC1DfJ;MZswIg84n_mr{*Iv(Xfw|sC&#-oSMhJ{nzM@Cp zv7miU!S3O%&pm7p+jN`lZJ zmvKp+#^G`)ny~_-&kln#QgRyVYOqnZz$m&NzMP@ysdGFkr@PMKGfy)D9Jbbqa=*$s zLFYWBas~?yKH-zwWdPep$GoIs5;ctcZWfpX9rJ;T8K86irE>b~oF7z9Kb>P{vce40 zIe5t@%xIl6K;_UkX&vXs;XdLx6+tz`fWt4x{5kj@$QQ!{u(f;Ua+Jb1{~CwoP;e$C zt1Q;of|IOqSo$P~CuwtjS(G~J!FzeRi6S`?GZ8kvgw{EDyDyjC8Jx-B@I|rC!5RU) z{`BJz!T^TN!xtL4+pyE9cO zf${Sz6|RO~8@_T!8&cdUi$-Al{9@{wu_33d8$SN1`(fF_5{-S(gb^4&znF4{U#I;2 zpK7~I_-0i~VEp`Ip2#%hleqlK3AY-4?N_A)#?LS2iQ(5n6Tcp9NX5!WnqQdV>GsX9 zNq&B9c0GETAvH;r5*RGq<%=&2sV7w_f${Tenvi1IAmrf>zj)1%;=%a>YNskCFn)fe3n|uS4B_KF*ZD|&tx5@upI;e%er+<1 zpJn*fA9V(e!1(!<39jsQ)4j*w*PoX?Vct1aDS`3x%K=v-ULTJ+`wK&g3y5h1#?P-T zaE<4y(;XH6V@O@4N(qdgU)e%ROT&wAh`Yp)YEz{I#?LROpI>+VxcQ%k)FD+$VEp`Y z`T4c>#@_cDQr*#d&@` z-WUC9Riy;R&o7R248PW0e*8p3>ULF1V5mlU5}asZQ_w5$ongoa*KFbwKvVhFJxxSN z^R23f7=Ni#!7;|vpI7wK~;(vf2l$tr4~5lmUnMlWk?-U zrHJvDTI44+`k_A88&W5z1)dmxsWXI>s#G~NY0(3Q)L2!D7=I~DuOMEU)E)bH;Hnm{ zY*mUFf2m?WskEyg-kIV8dFr+S1rHJvD zLSaR`RA;4h$&a5JQg^FtV*I5Lc^|3b-nL_g)J|237=Nh>KdJguE5@8!8tzl2i1C-I z^pncBw?1r0{j5q6<1e+?PwGHL*QX4ro}65SA;w>72{=Y-OnCK*n+>UPsuVH)QdL4q zbyi*+_9hw}EnYKKHZlHEXZlH9b?5Lm456sPuZ67#!kAaT*7^@HwA3h3rHGL{kIJE1$+^UjGuG!A%?6=) zlbKjwb0Z{X%qYsR6(yynr#l=jcg9Q`-uByc?yT&5hubwX+lFJcbnf_!%(UAcMbb{mf8(z$6)_bhjM zx+~X~nKL6XBRk*6B@oT#Y<)D*3(S9j_# z)++j*BNgHd5?@HLqo(|~WC_^$H*l}5fW4$}tM2^=o8O+2xHq;a_1P~d@C&`7qUFt*THG8*5fo)s!b1aZeP3E?C zbh&K}%k=s`EjzPEU0Lo-M|xhajjQqu_AIQz%gfKOarvCi9iKUK zR;J6DpJx*b_Y8h+Mn+a%zSC|Kt%uH^kdvE}m+s1T+2rCr!y!A%m7kB+Q?B+i_>K%` zw%g^-vdQ&-hC^<89$MUte4AVmXgK8Ab93D3Gc#;*jiBx@(V3qy%i+p*Agp55py86u zM%|g8ZA(s>G1)JI6KC3UGtx8jvTVr{0-Z8kS@vvXu1#HSXz0su=Gq-uPIMi_GQ}B5 zhEt{+L9oxVYfh=U)1>sQjEtPznYlJ?m7?L4Gb=aUkqNJ`ZZRMjlV|2-xH8ZO(pNI- zVv`-&?tE8zj@_oOZ8Y5SW}@)eU0EXc19hgm^Bp-3N1g~ufTP>y&O&ZEXE|+RQDlH~ zjx9YqYo>E%z8yU_E|hHN?wFO6nTfWY&p&yZqVOyiRsr!lW&I@c1xV!D|5PXA}>1tXH(+ReV6tS zhdBl10SsY$jllUVtDxIn^w(!$I66cf!wdy4Auhkz_fM*^;qkV{1###0!#X@k+6RW2Smw(a4omCY|O~mDSbO*H$(pjF^3L z0uRw$R&^Fm#z@GhUA7#$Iq(v3+KkBkCuh|UP&&tF#8I2py%Yf~(iSfB(0RH?ddv}V z0T^tvSXW!27?8+zkmVIMb!%#{>QvLxsU4)IVhPTCU#1M#F;VR>_2|K`!O@3RCmnBj}^3 z9j3mdxIB02?zX=x3YuGb^EYRY-Yi;>}x zA+cCvUoFbFS_VxC0cCLMB%C#=WnyG|=j0@xoaxlgy|l5qy1W+K7yFc|sCFnmapiSY zNC>Q0&^+th4p!e#QL?PAwwjNpQ5>S%;VM>_m8>kmiP%0t@6rx+7LIi^9xYQuV9ZNv zOUp_cd>|3+AZL~=#?y?|-&&Z>?I6n=mz7`vOLY81Jmgvxz*D2E>PwLD(ebtbz8r;D z)l}Bv{dcse8#FOdb6^!-bbO@QrDlX>{XRMQ;?HzHH`{MUSPP@9r&+`92?RO8Jz+ zx{Bqs^@WwCi?Q|tCrH*5m7zASDQs9$RaegQIUCk!wQyh!{0F70YRak`%k^p=r`L=6 z7cC%Gdbr~kpjBZz^S|{}d7M^kU3U!v*z}Xh+U!7*d($JE+Kx0GjkaDi0k`V@Zg%fF z4^TMotL^yyEoM`KvSSM$s~uZd0(W>LnvO1C)gMRyt|LmZdoS^jy3D(eAK1d8zQdc| z6Z>cOYdRXS(%N)1(t6QexNqUpvSSM$tsPtV8187AVm`606AR>yEiC+yi)=a?)iAQ@ zsMXqho5H0S)X@_QJjHRFygtoU|pVbun0{7-G$QMmyD_>;yI>1%N3 z&+v7GJwmnm&?*idDGw+^-wgl#=+c#Q!bS81}(i>=CF8!c@3 zd{unSV^(5eJ4x{3U>hK8Jz-;4+pP3~ZJ4n2hmARFRuW(vE^LXgB?#LH*ait3Pa7O8 zY)P<<5Vkbfh6o!^AUs*vcmm-_VY9);6gDfFu*sN5;{FJ(_rRGR@a;PX1vI3Cm^{LG zw3?soN_j;YzN{M?!C|??@ipm z1ZF-~c6HbN)>Nq!dj$pT0ra2T25bh@Zo8o5fQ@|80=N2Uw_OmH6#6So3*0U!EpT_B z_B#ws3eb@nxPMS;N+2RIq^W_vwBMyD(3g}zUs3|&k`kCeDS?qoNeu{CN?=G+0z=w< zx1_*CPM(BfB)4*BHzd!bc0jnvKFbNqSDwkIUn4gO_y$gV7B!=l?df3rDoUF8{=!3d zjDbBYwz%<(2fY$>3g~Xosi0_hJvQ~;4tgE#IS{6MHs}qY<)HU~R;m0mLGQyodpiFB zU8nM!K_ABb#p=Bm^a0#&Q134TeFFFJL+Iy#ejDy@R_|{CMLp?x1at)`!oZ`O#Fk17 z&&#++|4p2h%lkKR-yifh&}7gE#GQQZHIN1xt=@M5ore2v>OEJz&cHo4`y{>>XgVmj z=;Zy0poqICLA@UY>c;&r^?o>LKJG`W_hUd&vwOIiCgoE>vG&6=NxkQ$HifvKuHMf8 zEyaDNde1)3V%&4fC(5%1Sqhr3-g8rnWw@WG-k%2g9PSsW_pF^`aml?OXtVIN4wc7T zlRV~ gZMuGM)r>O6!&v_mnl3{yaU^bxMGy|RhvZ|>CUnhqkN8k+APZV_-DHo zBessE4X1x1oXA)3m?N~|7)98630@D?W>J{lY@1@ZA(gOUiy7G1a>a;Ef@rfS>EN-w ziczvun?-E#$<{1JVd~IkQOaRs8y2Imoutj8G{P1yc-%OMHj8p0Y<%2el#5lHMVvIw z);31D4m{c{%AK&WEshZzz0+pl7Ez)_icz?3eWc($4;$O780BTvW>MaUZG_-qQz&7x zDF23SxZr)Q+APX%u(9oo;igZ*X5ltdqGgRyxTzQ0Sc@_cwnTv&qS`FtNN={7F^bl9 zTExzHg9Xn49Bme5Hf)0gZ=PziC}prE2;O4VX5opE-xMR(4 zU>Kgud^zJ$TZ`#j9V5OFa3baOn2u*m*~;l0w3b-G?5ii4npxZHdc-#jtnUpD{bFrn za8kfwoo{eb!C|d$a2R*i`v!+`XU%VL7*p2$28Us0?Qd`xX4d}(hhb(5pmWfp!TMo8 z4#Uh=z`!ufYzK4>Jr{MI&Pihk&%`A;bY~x3Ggw38GqD{7w%J6V{SE7_key=R7YgIj z(#9wFnBRMQkdl-(QNoS42X7!0tXgUQ?AwEC6p?V>9`I;yUt{B22A>C*ZpFCrn1Vhg z*xLgRi@1~Z45lQE1y`uI2i!r-&j5_H_^aL?JP+NM{NH|iAiJ)A$?Jn7h|lA=I6KIG zu~8zWC;BUZ|K#<;iK_A#y|}B&S5}3dcKow5h@Boe^l;wC&kQVGb^rF*d>j1&c>l@e=cdPu5UbPON`2fbX8-w zwLh0(fkI={7St?a9*wmD)8kuQGXmxniov5P!Zk{iRo(@gSCCnQ4$yQl32jm;N>>NW zDir&=mPuE3p{4P2EtjtB*lMmFPcFo%1IO7##Q^2Ato3wH}vd zid74Hz~`v;EJT`Cj+>~$1v&_H7AV`49MG|#4}*>a<#@*iS_SG*`Ou-1;QlnwGEj_} z1bz|dL%3%eQ+_n4>_@Wwq7D65od?Y-Z>`Q_4@Kf`(s{S)JdR4FzF5^}^h>)99AfRC zF=?vSFWvZP;dARwuKi;FlZj`&UDNL+C=uqr*DqxQ=Wl#pjGN&UUcagmE8x2W>bjnE&IePSolF-;b6m-+pt%?S0L=y=lKS)BAjj zJucI`-Mihs?NH>lgR%Ce1J=wIS9H^UbBn#decQokJ8>=cSnu=O4#mNB+vh#4o44Cr zX7;qVr1!C#+qNBuw43+fCeq$wae24v=XGAimunm?f3g}mO%E*Bw@e_FWMIVo59ozeQSe(3_roJC#WVJi%b2N}#(%)^l@=GZ*b zK;h<@35sZW&;oP5%d-%)J1FCe&PCAHRBJU{yp5=(GJ6J-8lMhKk`mjlh-G*ULNk@Akb>?H(n+EpRkG z3~u`Z^aX2C-vq7`o`QGzw7}>sYQX58&v!4DT^|!n->YyR3f%W9Hk7_Pq+K$ewp*zf zh9sE2J5h2n@bn$#yfw^7T%yHa?ekp|cc7mi%YWker8WTgY^(FPpLR& zp}+;v_dYP+2I9m*bZU}@`fy5)auP0@Q2G+!HWirD193t0tpTPv5El?Wrq`{&JQ9ct zqVF|eJ`BVK(f125G5w$zMvq4jeJ2AmJ`fk6kCp5k6(bblFI0Y<4&35EeLkDWQGOt< z=63pQHs4M)ac#J63&63qM=kJI%^K~77yk=-epL;BQP1ysJS%;au41_uq6VYYtz0lR z6fPX-N~5{(wSQT4Wp!iylA`*mvntRqM#o=(La6~7s!P#`MvFtf>WILqu~6O1e!xxk zm8~l&t0*cf!N*)e#~}E<3SWxUK?nPqLvYWfXcN-F=mW+uA7K$PU6gzItSl+2=C=30 zv-GVmH3w5vF9Oj3Pb=!`YHHzUe?WQukQQ0`sV{@a+rH@divwI6YM0km*DhXz4u%ZB znk4bKjaIScMRgU6>qQKi;QC@a_NMpZVwDzWdUyNKsL8)HMNIoVcF!YQC!zi>5d4LA zxLR+2;HAQD=YQ3!Z`rL+$b%!|OgmOSbVj53@$|c`#izQ5ZAZ6+ReqeQPs-j^i{fG< z7p%Cxpl3_Y963iHXF5Fbo}@=V`gHSx^`Eb}>)?v_@Myy0IMeG#w&pAu|6J~{M?QVx zZ})d?5{JEjf9G#!FI+IV+xNFtyg&TRfu0-0p)BA}I-zpu@WG#b*mV9W7rs+_KDq~! z(OnKUbN2(ogNtSaEO&x3%fH3N!*!IhDTPSg#dRgi5-Pa}J7EM8{NzG@+gH1)K4EE9 zLw#X#QlZw9Pfi+}G!`S@38`a~3LDGn8VbwsT?uFr_m^6}rl=k({r=oK!>SrSb$>!w zEh&YK4OP|j{x6HtDtRQ`pNf$dC=|@UB1p?hs_|j!U!mzgh!aYNSRGpahf_o=K5D8! z9_9!xHf5a6osVwFC$O_u6{8$fZ5HL2xF?V6DA6uD)IHW) z>@AAR+h%W>AIU>gqU|kCT=}v3$nSi_WR#MHxEp2|tnAUVZ^S278V^&h96c<3p0?cx zuUiKrJf0NN!3fW$c>_xSSxYbsX}D+*KrymI&Eet5ZZIeVJsgzb#9XQvv9Ti|*L%=r zQS$IFd8h)_clDR+ysLGdvt@2#oVz76v5&jOotU7f&ow|y3`id)38&b2o$Q}Jn6SaO zfQzwa4!iFHFg-pF0Fxp=dPGYj&pCu3OdQHxrZ@#J^uRa)>)*7sP*jy?QdP+%PYUOt z6R1oz3{EjF6jiaZ14N8^M}sEdl4(PmMd6r^wivFu5jGsF_7&enqY-j}8J`PMjRoHagYiHIE_(XQ4j`0632wb==>?QnEXYwIMc*!;q_17=t2 z4fAo+f`Jh+SB$`otF?7Kd5#y6fSYn!Z|H+NZ_Yti>r>INZ98;=tF`$kljnfR)%p+m z)RMC^r}ZCuVRL#Z^3-lxTP{gtan5PIooVZAxjb?AXcTzS@n_zub+AV~MlnoWx@9xx2luIPoydcedbQC2wO}ruStxtWGzaZSI!x z#7XX!>ck|scbUi4>WC$F1X3RO%)|tS5#eZEjqoE^8uzhv@4UT*w#j)3zyMS#rcrn7p zV+;of+f3L_61H<;>o06vb;NcpQh8XnJ*C@r>$bOa+YH2brjVEiTY|8a!Zujg*!~^C zH3}A6z`P-c{zKrl5OgFnLKx9{uy$l;n*A)F9(8wAlS_a!`5dItT}hAM1j&&?+G&0x zVo^ZtIShjmdwc;>fb9i8Xt=?2I##$%7d>vA09H{_nY4ORk}$~tqb3FTNdev*(QLX} z)X(xoy7~r&8hZnfFF@Bs*i0N9q{r{ay__maG_wI@pC0SX<|K<6dSb<$F*HW|1;FHDkfir}oT$xQ}qD6!o-@Edt*el>3t(4Z``z>r7B90p2G0>ci zqKOy&x{x**@z>#=+Iaptr}xRb7PbUjrwU#d6ei>NU3Ls&V?7tfn&V=kq9S6uL@~5o zM1VNZDJrR}D_Mg`rN_5H#YS9;vOnTPl)*gZC>HVM;xyU-brG)Et}(IZp0Vam9V%nP zh33tihGcJ2m7Ua1*~r+4DE`cyVk2~Y(TZQ-GQ@jbYD?#za#O?<(`v-s5uax=$qHZa z7n&L1cJc4;c{bLZ5Yq|qk31o0keQ`zP;o%0xmcl^!2v-vUvLBp2KUD?!{*U3-KZ>h zVp7>ADEbYLDc?^^Weejm{ViUyk3;Hn5q~PKF}PB2v4g_yMVlxVl=Cx~e6!#vUOF;I zEy|Smt8v5F^H|#bu=CO6QM0w`X*VZ=k3y^PA;>cYJZ4;?&Ciu-}-lpC&Q|ayk?FGs@mG^Vi z`w~#hJbIR>_e<4#KB&YmSMNDTi@MiyAt-8N4=WMWte$5Aa&lZzAF<{g|rS&^y<8+*_NjSYe>^%5AdH3+zUF7tBIu_Tum(X zeWmlh)p;pMSBaaT+AInWE|t6{o%e>$dq?LT)_Gs)ycDFjs!z2+pU!L2c{a3{5|^pk zEK1)Yov`Y1lG6FrPDzpZNCFr>f+*i&o&>@$%t_4*fkmF0C2i5TXB&t!#CTd)!*nIG zoE>UIDi(H$rB-QUgCwP>`#D^i38tSI=${;|OkUDYN&aqv((99nIJIwv2$?2Lol=^y zlr}sk0%I0_uw-i{m_I3SUZN;R6H2d4 z7wTjUrI}!QS>xcmGjT2)G7W~Zn&HS(=9IKC^qGQJSF$Pr8@RYK6GZErAY11t34@2G zO{yQPFaKWGaOL0c8`b!rw#-ITe1Et5M^_$OI`a9xw|8B9O6i_n?uy55&#Qdyf!ME7 z%hHOkfA6Xu8}`NzyYms-X|L4Yc)ZK&*7IsdMDF|WoG)*_Y`~u9Z(MnJ{G$gx>7Aw6 zfBxr{bD*2aUVa0^0!v*N|NQfIISt?& z9*e;6t=mZ0Xu1eJv|$QGm@W#J+0$f-fBtElGum;^X~(&u9p~P5oZWsLIkO~Lo#2;z z<_@Ts&N_!v#?olvVZ&;0cvQf6^`5B_<+qVB2TEoWD<%pJIKS*2Bkp{4O(QUV8yUxf zYiwlPch$T18&X`ZP9rdWesRzz{bF23RkZ>p5Q_-(k! z0jAl+27#vjp;zb0=bd-gt5U>B>xU|buKu4Yuq2yJZFKN6?3Spr++t4%&CZdx-%CAT zhkzluuao-yuR25y;^MO~BZ#XUo%%a8U_+#KVJoi?Z2etY5E1#(P;)RQTL18#k(!Di zvk^gd%L_=HM7gb<|K5s$81Z5~v15$X{Kp&rH7ZY@wc+}B{V_XI8) z_7H-twP73gBQQNq6);Q;A$;7|D_||q81T!0*`VTBqXpB)-pF=fHqM3xhW-ZAHyZqx zf%#d*h0@1f(rEUY=Hh}0rH}EQ2F!UrxM1PygZMrU%}!F}_a|qjoksD$mn^nO`LJbrjz_ zfq6^Cg$f_z+qoDk)p5~;Du2x19wqqX1s6>yeT?r4U|gjF7E&MM+YHQeKDbc%Yc9h} z%yOx(qxjYWbG?cS6+XuIWngZs=s0|g?|Z;RRZ3haeT?rUVB!{ctdH?^19PblE>wJ< z2j*W(q`r>gJE0177A~4l;bVNS0H)$h0SlQQjPK3Byrkkn>0^A41M?1N4Pio+7sj_o zH5T&Vq9HC+d@lp$zGVW|VSV)*Fe7UuE>!p!-$GzU*GgQd@G-tifVo`7h0@3P?gHk< z#5i>NQ^f7-6fk`_{z(U411aImy^(t zYe}Qe!FwNEH1s#v+FtsTqGE`n3AHwL3UFCr=qnFXUo~*6!_aqSnEGx6?(Q)3?G97l ztH6B_hQ6P})E99s`VqKjI?9jHDn=qh=SLcFwlMUa5vIOM;OfHAw;@b@R|9uj82Yw{ zsqcB<-VQ_GcVX)L6}ag0WGZ)*AH!74@8!o>;HHG3Z+@8iih-*RL*K<=>e~q1jbZ3} zGE99tfqOL!eP4#D?+4%_&j0=VNKi4qmmed5OAABa>@f8$1g(V1$?{0 z{>JKd`ta>a0FLh#JYnF{fb)idV?XYhb~tr| zVd4&kfs2Q}ufxE_0%y7ae?G>BReyo$VugFxFme6Ez%hSEg^8OG29D`$4-+>lOk6>j zxS}v|Rbk**uIj_Yof9UmISd@j%cWuBZVCez2YnBQiE9fJ_i8Yl&vsPsQdu8~=O_S& zbutPjxBb3(YsB*WvDHM!^>POM-}NWw#_bdLYA zJ6{XW%u6rIOP}e;m|K+Vm|Ha4KG#u{k(WDncAmS)k!w$PJCqa^9MK^-$3DLZXTxQl zhJ$yU*v43T5ZR$j=A4ZDqPg~LEWVWR&K<(%+1;+pqAa&Pdk$XacaF!A&HgcPxd^=7otNtRWZLf1d3(l&CYNX738|+7TGg09CPLr+1+mJ zxhF#x?Hixp7tPBpnv<6`wZ)ap%di@#bNu}x z?l`zF69J!FFsCRBR#$HJY{#6u0t5+*?<6Qf@c|)cbCK1+i7mCL#O&)n+mY#-tpa%B z0s)b}fOJPT{K-2l*PUn2%(Tz7>w)Ry3)2wr$PX0d3v%RUYB&L6p`?eMeL;R6@~9MK zQ10A0*f&#xqkX}yTqK6u4PSkbC#%RVzDOUxeV|&xVq;MwVGHI^pFP9r$jr#gUx4-C z(&rdo&>ScPY_{Xn0>_-W;$#IzCDs>{;kM72FINv>oG%V>@JDIs5$}r< z+6waZ0>JYh=p~k$8^M$ATJ~#@9=;Ix=A%XPs;4g`e|Fy7yo@}znsQixD!uCE3vwV{ zOnXu1Fd$ffZ3JYVW40^Hh2x;wC976~di%nqPqG-f^D<;&Y2NnnMdDmb>8+OJeSKY- zItUYeb)sHzJ7!}Ao(z0HUmUAil;=#>ocz2wE@q8=?p%~Ly?meOi$;FWbz%Ej|+6gREn7~l(&)dJI1 zuQ&$!y3e+srUyuSKoWcb$jbQ(G97dOpXR;+gp0*W|3xXp2Q{-RB0uDz(;Z7G^UK znCTNomx960($b5%tk#d&D^k_Q0_W`&n2T}xwZQp%1wx0ZYypp3s&-tkS70(0$~Vgt zNd+CdR~*f66AzKI+CI*Y)kq#912miZH+~` z28;Y)*rt3U&(TnXH#5Tteta--nRhd5(1xaKE%t-OTx!3@G1=Dz5Je-=SXVqIs1&}$ zkC*NqYcFJzp+q{B$frUD3|rGS^?qo!HJnW3n_+vc={Fs1sUJrxkPLMMEjkbBVJj~4 zBlG!WYrBg<8-|f;L8XmxX$YWVIha4!3`{R~gI*p$rxWpbEb10bQ)WD`@S_`+hT-y{ zN7CcS;}LIF$^ZuQUBPyom40c+uxwFxvE`pwl3!=}Cza$cvHX)u^6M@Cl#=|VmjB+8 z{AHHEswBU`@*7L?ms@_QBww8j#r6%C+5<`yz;C=B;OMl88is$?+?#vD#E%@oAJeGDAjzaQ-75Qk+h zbU-YRbrGk)TtZY%%;X@Z6~h{435zGTDgjf1faVM;N%vNCPr$Sw;8ghcviA9nErH0G zd3Pq#C96!tgcVKarG-f@_sEHC+fqxf2+$kw6Q*%BdjucJc%^LQBOz#%wv=C z9H!gXb2fgISl27nBcRMacNze6%djD!yd*%GE%GZ$@|D>lzp^BMsmyZZk15H=FEZg5 zO{&3CUw=$Ja3(pQZ|v>t!z*mMQIAPA86=}V24sKvHiTxHRFf?N@cko0P~xPTd@&F2 z&>6<5mBMZ-Vh?n7or$AJRmv|Hu|Id?B$WKi%tjFC#)6uwGloJmRHX;3LVUXi1j2sTVmDO1ksAYF}kngj|> z7{QF^LQNvgb=J25zyofnB=tl?;TnZ;LxkM209Vvw1j_>~m`_QTW1WH(0j@1GT^F(| zgX}_7nuwe+0Zuj{)%=yEwcI1h?3f4& zYhtq)kXeZUvsvN+WH8Xll?Efm?3AdYK&2dPWSDIcVI&e!S#*(M_CbWf9Y&We%h^(G zj@nw7tq_%cA*XP%jK?_TP-JpuFGNu7(VUyf;T-R@?HgtzMEGVCEE~ecDF+EtF#8~a z$YEMWf0f>0)M@rW1dzb~ViHSN}h#7LdLZ$|0 zIhWU1O><1s6idN(;8-Y9MLSW^?k|AF!{r_MY%GFx9kzsIB#;cHDKfhZrx=<+GRh6Z zEE_Ec8uUcO+Awd*R2mM~gStzQKE~3~GPDNt3^_;KDkaEAGsW`Qnpe&(AypG!`IVNx2J>@f4ecEqXK_~ZvinBV7!QRdZ3Mf)+16|-l!gfs zkC3wxabqnmjBWZhOdNTe(>Tk6ZRhP8H^s?C-H@<3Raq3rO%gxxf<9s{cjY*(i^s*B z##@{*T)bB*`^{umTehi>n(?sFpzcN33~|z(ZUR|da7r|NQfv!1AerJsi?b=x(i>kx zTAjE_7Ki(J*sAquiT(j0lP$zl->csV8Z*UWnhThXjgR3PM}iZi-19PtPPGEEEkSRq zkkP&t?*%dA8jER7N8Ee}8-}a{Y+R(pF*#4OpeEeErYVJe5=TgkE^Ic0#wsw|D&n~1ke3}S`x9J7jfg+IHoS48!$HQ0PbL3NhvJB*qlSc*)_qEVN*`1zEXS_ zVAx-dgtkPk(24?c8Xy@~m*gxAvtd!$WWnB&1rN(`I6spEODm%aceN4`X`I_62RNuM zHvH6$Lo^OcPfnRb4m%{^R27NvmO;cstT+0fmLxC;#r%s&q}FH?rL^Q@=T7s{18}o1 zrhHGbuzROjvKy*rI&!do=Luo03n9+HulV!BE~xc4Sbs8YW@I+EBvWGe?h&y_hRddK z2tH`4HHl3V+~~r+Eag(~5=azIjr2SyP;FU~UglUjb;t{;9eL#*52_IsXB)C`K($GZ z9tjPB_ygosdb~s`i|ye=0aHEfp*&yllw?X9sct4yJde6&;Tyxeb;vaHb%5|5w?s6R zPiVto)<7%)+3?AN?ARJB3XeO$gON%mQnDvNQTW{fb~G1?7etxl!toApVLZ&8AvO~s zaK8f(JZVy}nZpGSFxin&HoDQ2g)1IlBk@e!sR#E-aG`Q_I_$YWfe;p?9eEc{L|ucWl^I(75?SvNS1vU4VJYh!BB}gQ=@+onsaPtE z8=$pFm~7Y?aaWZkg9lWA&mjobh6tD)50Q@*xGGY1m|R$z0M{&6lpjTH5(UHKp|G&R z-hWb@C^BJyJf>dY(q}bAgdy_yDjLT#DbNmY3_gMsP>D>cDH(>J&Rg=F-{2f%3d&)N zxpC}2a-*uMhD&R=)cPCdu&U;7NNc!rgIU`bwhgEktJCs{af1);uaQf;TvhWOlr&2fc5X(s76(PYC)ir(j-KIR4WIIFpa*Rq4i zMmZb;*@)74(cA0i0q_t2zD6i|?oA%xdk=tozD;~ZaIRkQENbTrJ-nFO^ZccIpVB)W;xqYOD zbeqXA`vo#U`>5S25vP?%YBNEd-KG$zjhGq^l?rOLErZ=F)(t;`wTB~+dls(O$Ux#% zEwR$KrsFE-uMcjFVWz06DFe}(mtg1Qm`?a)>}@U4&Ta~#^s*0<5!(v~xk_$&nz7FN zvKZssrBrZ%&L5hjVIp5ZWF)!K~rq@sbvV6bl)Lxv`0 zzoDaqvUr}7bT1l!q_F_h2fLt*5V1-z6u@&ykJ7$k3jN6nKXTiDMC%hDDTwsc;!!+S;Qz zv`v?$Wn+psYs~qRHMSF^j4^rwpu%O4{z6Ti1+k=(Ge)o#7C~kJ67JMJG>cw~6=YS- zEmTVcH~FzVMM7{3+*lj-d|b>AO-;C6qTCnhJv2%Fk?FHYoGR(Rqd0Uvr92!xPk*r10h_`1uk|&&mgkHI3C^G4fZ=Dpr^^0e0zT=CaeOl-* zOuPPlKlzz6uRA(?$&Z|!ci`m+LZ8{%d~$!P^0;r`ea&M(`r2XtiTh;804K3*|LKEo z@A>rW+b%ld;9EX*8gzGt&@*%I`slZ6p1tIb%ijOg-T8@M!VklR9#3~2`fU5_$B!SF z{q(uZ-uymZ79;fHlUqXNOuzVqna>El_{5m!g?@R(DQ_Qfe4Gwqf?UThBZCypz?*Fa_}T^B|Vu6Fv^)7?vR%NYvBKQ)qB3 zQ2*e8o_n%sEl#KDAKb>DuKYd6)5uR=4(MDNT_L<%3s1Xj*nHp>f@*sEyN3G?FeP2C zb)7>!!$UmRi;SW>C;>4f#W16}b^8kNnIl*~A z$Euw_BE}zcPH_ILW7SSIu>4`^1ZT32RXfKZ#+P_baE{fnYP>j}w+Ki}=vcMWjTnDy zIsq?QSFvh2yatJb_EFFnt9GtK4F5s9QOByC2NA>c1=^2vtlD`AG0vD1oL6_ZYgDECEV{}$i@e*5Z!qv#*igbiBWh!;wi9hYW1X#{yeA%m$nx1f+9jxPOgw$;c zAvKsrcqGCc!g&a@2p8z|kb)_Nz#|QRl#bd9qp9m-&l+evXEES>9t1vTjiQ4i4I55gvjN zO(N?51ma5lj8!{$X_RxF&{#{xs-=k(?s8l!oL|)w^!ZHt)-+9%8a{_1_HUwTb`quO zd58ryy#thyhrY1Wux1RGjIL?@eCXdhHk7_cYMc3v)b=K5lV#eBklH>AA!dVFW z5mJubS81EEYIzma!$RAEzl>Eo)Um>SR>y)J``s{MRGTtk6D3xMzdQG7!oCJn!-#$# z7I|gD7Vf1rxHa>?VZYdN`!ZkeMa!_|Rv~1|p*~{1A_$MfbrVAN%NRnMFJ9ApH6mn- za^a;~l(A}Ot)-ozW7W=Oh~19w1RUh2V%5%W#FXs4Iu>l@{WM>y8EL*$Gmd7y_^!9* zBbsp`XjB8k1$%GC1GZfw2NUxfIhZiB?qFGtK*%ORGl=Ozoy2t2BBbg+j;~TZW7WDq{pax_Lt+l*B^+wk{6p|PfnDbuBJ{Ht(&ZNqdi zEAhVWetHbNuPR*bpZCQ{Oqs4J605`CyY^|i{vD{3MzjWoI#N=m>%`Kn(O1`PPY?cw z`(N+GeBFT7VI5$arS-mnklxof5z>6^LP+y<6GECVUekQth>)%NulTA~Wvtriw6q=_ zt9JemV%X{d?(;gPye}pDfW`S+n&!*4?Y_M))s!?}-;|~t&3t_bsFO!DCCv!cz;M|< zn({zx*vLV}Je%(jj~+-fsh)18X?Wte?QIdcFEh4sZ!OaM&uzx8LhG>gzJ!picP&Diu`eS$ z65(|SX~rI;KV!;_VTJ(>lALQm zqyMFFG-C?qSM~laB0V$q-~pSl_wGy6|3EYLGia0=e-a@z{!4^3W4MnaX6)w(+2PrR zmBtxUX6#X+(Tp*s%$UN_geja~<1k~KqJ7>NH$o2F8{554GY0RZXagk-8f(t*q`kDr zlK(v`Ccbn#%4R++Cnq7QFsX^RqIFn@Zy{vs-GPv1>`sJ7B81TqGxi;X3-mS3*loCG zt8!$hR%NW(Im6Prb*x&Jkay#1g7Z22WlWhdB}+4=Wc@8oGiKX%f4s4qr71`A#Zfm*SmB@Xjsv-rI+7@Hj|4}2$SmYoP^;QC^OG-DSbr1}wR^)puO za2d%NcY?!OGN#O!!col%=T|k%7<AyAiVWet?i>>|TWIp${OW zH+DZlnlWC}jNOfpt@;^!RjV?l%$TBeYpyb5NEbMoF~*b`Q#hn3sTA*t*|yyuGj^*q zQz_@59`<1Znk z8G9KaJK~E7k4K2*i`F<}%8dO?Xy@TCW6F#v9L<=*`85tR#wpro#%ghS;AZT8X^{gl zW2fw;MV9`Po3V$`I&8g%5wi7ujBqAGc*kPKFujNwdmJIn7_Vu@9zn=fm zrfA)otIXJaf}JPF|5OP zuvtEfkj?UWg!INREX0gmiID2wiLX*WW6F%(A~c#Y#*`UTIGQnq^Q#(W>>c{O)G|v^ ziKm<~WDnSk;YPdu1oy`Dpn12p#@>-P_6jsgjlYU;62d(Qrz89eLYlGH5K`mVebE|c zOqnq(I6-;^D{mljbEBK8mN_b3}cE@u5KAQ4C-LjDbi)tM?unvZL*(?}D(kw6=V#ZKIF=H@g zqW+Hoqtwrs+UozZ&{#{xlo?YvnlXj*t9t*;m>NDbV_({*8T$xOqusGwyf00^D~9Zy zJ$T_lqty5r2sseC5Ym)&BSb6kR=jAOompv|F}2nIL!q&@j47{7;l6-tg=@YrIk&3$ z!ou9C$qS=%XC-l2%T{Q<=<(#v=-d^}z9(Fthm`ZkQTX#w(eZr5uEDob_HeJ37|Z$! zzIWog+=*LF;u0rSNY9awPoS#L1?E9ko}RuQzeVfAV&=UyQ_-Sj&iekrO`ZIGW2j~| z@>HLBcvw0rPoqwTbt(sOGQiXE-TO1GwMEagE_J5sXIhz6rc7t$d#JTm=f{9$Ux5;^ z5Hm=?2UQT=UyHcXJ!92!Ot7M@!?mJajo2^oo#1>$$Euyj5L37(bgbGbt1NRW!??tU zBhY*(RRK-ov_FYe;qPxGwl|5d3YQWpMu}%emAKwYs2C-lA64RORzk%n@zSUgU$+t} zMv2!)mH4KWP%%pU-%%xQvJxsr2{v5CUbVZ+N~jnm#*Qj+tCdhON=z)0;M8V5w;`fh zP{k<0A6NxTrJG)a?ywRnMhTuFzITalTL~4T#CwV){H6M?l~6HC%q^1em+E^~Ld7U? z?5GmEt%QnEV)3XF_gD!PqeR1~68BmO6{Ez-qe|RwB~*+O;UWpTiRSYFBB}*dj1tYG zN?>zIOQ;wn(xXZ|VkK0J5`|GEerzRFj1s4gDnYxW>Y-wkIAc_apI8YMqs01AC1}x9 z9x6tOzELHfv=S;tiNR4Neqkk4j1pUw1cu2h-D#JAYChK@qDrM=l=!en=t;=2zxEz- z%)G=OSR6f6+l5y*^mNy*>%m0PQ>&S^y+gJB7dG>^RF3{N#*kx|m^M|zGb~?z+Nod) zX5Ysoq**38ec4y1pVt|3yoafrK7IQeM12PdF|(fppn1l}idYFNVDTQ@l*me1g2~Pl z%4BM;P1gE)xaF$z&-jVk}bMhgF|E%ZLD-Hg|nl&e!dBw#qW{h*5d&k1;xqH@K@%4jVYnpTNd%oX$-{jC|tA2auO&e#uFm=PPmbZQA8-GjA zI=AmlQ~pTmwo7K+`|T%8`Pc3K%i)iI?CS5`cg&Xmn)=J%{^i$??zuBE@RbP$f7Gw8 zzW%JW&pcr8(N7-K^Nl}0{eO-==>E?>aQT|Ahu**BGe3O!plJ&)*zjNTU%P7074cmU zZ(nr8fB&;w&-mfQh3jU|-)+is{n(RE`|kPu5AW@IfARZDZzTUiFI{u#dmen~DwF^J z*7b#nc+Kq3oOfvb@#o&P^sGPq@}BIxrB7_XVch3VKK_56yY2^b<7a(s_?8F%eS6FJ zuZ}x?`OTlLdUn^Zj!(W2uG;>GWYy9i-~783Pd)zLOCJ4f|BWBK{H<&LnD|5TV@~@~ zhhBO6E9Gx}_nf0%oPFC>pLQ$1I_B{g%ddOuKl-*-wC)|8skbyoF5TYvlfit|==Kl9|cuYY*rn@_%Y)Y;cgKCSbwyH|WDb?6Ve{CUzj)TmVY_u`kI@D;!O`+FmCn@zyH&jul?ZZpMHM(MeF|Xl{Is2 z*t6=B-zaN*`xn2y=7FD9?)v!0Uc2wH2QS{k1-X10yZSc`Eb8sPqPMc_?YB8AG_LRI z9A30|F@D*#Ls1rW5B78opWQKR; zfwT&K%eejueip488Zs36>=k9h<}whH#}7m1=g6H1&SM^B5w4~NC@14;R)CViRc(Os zeq7BDQ2KC{4N$mBXEnUMuEbS)fN~SA)&?kCeV$<{i-vo)4$s}R83ft1t0)U2HWuIW z&M&)NY1*$))R+MqQGzQ<768|;3I2}q?l+oj1iu{Z#N@qN% z6=n4Kn9iEXy}iZhR11b)jY(&GQ9A5q`>NMO!O)*K^_ozW4p;p9s@G(}&|5U+nN*Yx z?_2GwUhFaGy}op&6s5yOyH~H*242NAM~1(>rU}O1UNuGOa0kPySMpQG?Go^3&!-r=@l(66{o}T6YPfv7o~&!FJHY% z_rpU4C|aPZPs(6l4dnJXB7 zy^bnM=V-y0dL_!9TUwmXF@o{e>*%6%<_X5sYwiudzN9#v`GWD+YhF=03j|}@t7dod znc{Sg6^y@L3yRV?PB5lk|JFJc%g>SeVQGW*@VKIMn1a_2bLU_4a&bDGZGz)#VNp6K z2quod9E0bro%Yk>bQTMy7R5C7@Pwjt>I7rzReF4Jy%(%kT~Rvqf-&t?dVHZPARWIw ztS?Fjw;O;l^%{KZ<~NG#1$O`#e|s$}N@uxXMz7Zj!T9^(@}hKB3TE_rohTT8KU`Uq z&Pjq1O(Iq|`OTk8`e8ySI zA(~QWDT_4a5KD<^%5+Ovt0_lV3U>|oIoeV#(3ImWPpT>BUPE9!pmANuUQYmSQpV{el=vu^8HoAjVQGhI4liV<{Fxza@yV6pP_p z8N^tM#cB8;V241LI8I+kKF^jm@$OR<=viZGU9F`Q3>=~#;BnU<2O6O8SLmSQneiZGU9 zG2@FcmSQoScY=9Yip9|P3t}wAV(9kp7Qi2KO6%#3cUTRgsJX*DsU;=s3 z^9knVNeyc$!36SZD#}Z(?3mYOT1qg1yqb&h@}zFoQi2KOl_<(ftxA~JqgqNZfxK21 z<>g8JSxX5fkXK7lUTUSlysBwX@excQuOz5J9ePspwUl52d8I_k-(JdlXC5&vC73{7 z>7u+msV*%gm_S~cqP&y`%e;1IDZvEt$_DcKz{9V;>`7gvr34em>wQIesa+A~^&KrG zm_S~+Kwg*JcgLBY)Dv1tFoC@CMR}>6CFb>_mJ&=LuR>8?o>Uc=H~0u9kXI|HL3^lH zH$)$$r34emt4*YgjiL@csa0A^FoC?*6y>FMDwx-)T1qg1yxIeKQHP$?RxKr%Kwce1 zd8yqV=JgpZC73{7rxxYqN!_BQ1QW>Xw4%J!P6hLNOiKwSkQetNz466*c~XDYQi2KO zb$U@=YWIM7jUUIq@+>XpDgXTSeo%vXc~Zw{DZvEtaz)BNmeq=$d4#o;U;=raAyTGq zuwI^&tEB`J$cxi+upg?`AIQ#jEhU&hUh724{BS9eifp44+%N-%-EdW!N=yRyuyLaSCVfxOli<>g5oqoo8B$ZJDUUTSB6c{OV( z!36T!Sd^D1b*7dQOdzk`qP)~DAM@I&r34em>jOo3c~ZNylwbmRomrHZ+9_pTPiiT_ z1oG-D%FB~_OG^nRkk_W7ywolq^EzTY|2mFf0(or))f->zhn`eKO9>{BSHDQhsGzso7eJn4r{$MXCmW`T26&E;NEEVS|<; zCMb0-C~8zvYWvTo!x=PEr)nu;f>Jv`sVJL*zx>Q!{QXNkscl+n2RK2g^FZ;WlDj)l z9Fy1QwG=TysgH=1W;@-Dckl9~?$Tspf>P%Pr0TD`7>g~F*RQn{F+r)1ij-;P2d;Rg z;7R?rmLeu7^|64|MIX7+m)A^Ay!a3kl=?U*6=f!`@XK$`^YW_GQp5zMJ`s>w)z|bb zPb#IQhzUwv5RiIlTX>Nt)vKk52})fkQf7P&-2J0Hp47**6fr@mPX?r3x_iS)PwHwd zMNClYqJY#J4_sUCNqt945fha9lt`IA-+sq0pY)`j)KbI*rFI6S3J?AKSWoILEk#UF z>SB?aiog8)C3EUEo>c85MI|OE^=VKl%8b-|F281$Cl%FF!~~@-2}o6b{%Ukp(|b2) zDPkbiweid$9EHFS1|jmgpi(Y;e=dTAc_6NL$PGO6Hh_~6*7v*iAvf>QIeD1P%9+R6 zEWtd`W(i$#_>IZOOHe;NW*C8I_K^5^Z?P8+&o<9Po`9x`|43j#56~VcDA)DCFeCwDM`~n zrFgokB@;D`W#ssChqJxkJeazAl(}0r<4yYAa}7sY$oz0?$eAnqrhDa<5LTnHgq&uS z5;5Jpl$7O_5Hz|kSxnWa%K>uy<6Jzl$(&e^nV$&5W_seDbtmlXAw zr9@4?C@JbOONiR>QcBdvEFmi9pj4ak`Q=j0qRnfmoGhzEKDrx}60@ulVz!w}3Hq2N zMU7=DQ9auOO4iXlI$bhTW7gMT^gjzJm5ebZqm*VOWJEH)sVP_nkFjc%8)cxZk#srjt(qH&53O3JeuA-TnfeU|t_Q5- z(o`}VPldBhc(8Y{$C6j9PBu3uW8p}n@ee&#Q#>789nG$8G`-8RR>qo=*+e)M^2%s= z%hN5*EjX1F8L6J_aaz)`rf@c$F$r2uINTU%iNq6`u$Q22D~m}rrjpUsnN%hzH5kdx zHYQiEj;2!4kjb|g5pQaaM^kZVR!^|1scja?#z<3gbtu!^+~|!2A2;2cZOW{UgupdJ zz;ai{604g-;aIrQSji!e7tdx=VVo7-Xgi|Ei{h-`=0qylXn%y_@mf|#BiUrK$t#Y> zi$A8Q5(@Ro;;|y3)!|4>GHZ+CW3|L%k!TBY)BW7C&_0<^YIU+H z+~O5wq<#Skr&7(SSeWHC3L2r}aHugE31_m&c*;{$2}~xA^D|?%kVScg8qrR%NVX+o zOKUOJCp0WwQtSKFEo)f3vSI0p`Z^yG(YnQp8yXrAE?eHPR9I(JcYl0>^V`kUr+?~1 z$C;zocmC(xNjeK(zV|ussh%2m%9p&?v1B|2OUgRzE%C!eW1}-Q?0?>!^e^Dsp)tJX zM{vF;!AYyYch3l1?bPfU{-@4}V|s4_cliol>3pXm-x29yN$z|XnBV(wrP8|txXIXI zxF27B#^T$^^nA}Vkj@?Ca_G(fX`H&a(SHYsJTCvMfjdZJSuer)p7AG+>5csCa2j{} z*W={3t+ zd)6&o*tvd1*TVXh_1z2CtzXx$a9P8;`t{4YmNxXPU%Yl5v1_+BEM1uE>FdEUU2FA} zp^}RC_Vo-I(xUF(!L>t!U2D1hu&8TM|GE$0%`bR?;u*LBh@W#$b*|_wvtg>YYp{Q) zfBkUn{M>@t%)oH(rrr_95`LTNw=AF zA(u(Iv2-YmBRZF8aQUdqlYY`%cPdd!YR(<*(`0oAn4`0t z<_9UhX?_El?uQ^ZUn(Z68GcAMmnmc-nWWYn-gcohG1Cvi>FgQmUThBabdaCh7Rx2# ziCDB)t)>SZ?1!tI(l7?eWHJ*`$}-u`@*|@$ovl&%A$~5kgEBE!b!6Oj%Vwk(!ti$!hS z=LW$lwbpDFW-||%=@Unnf)zM7YIRwyAG24as*MHC+bb{^TMeyP;QYM;p~FCpvu<+?@;TcY3;p~& zw2Cebhe8x>TVs(f*&;s}wke;$G2}Rbd?en?3@7;U!Ng_W&8$Hiny$6j4;FK&{TjS` zPAR4?fG8S?#=1IY23g|AOLvd87qZDvBArU)Q=tNet!bNjKQ!AKPA2lru)Wswn~t{B zkE0bxhB|^4-B{{neq=tMY;AWjXu~j4EvU3HE)4-xEC=)Fnt|!%ZqUmE=yW0;k44?0 zY08Y}6@GNX(lA^e^hkOfdHmK^sWgDWd{?j?XQf{nGAvuvU2OR$mgLu2{z)bIODzB7 zlKgthKcyspspY@7B!8LZuPVuJu>8i7{N4A`QeiMm6jhV$q!k6v?M=l`LUAx zh~>vi@}rjDRFd!cnaw5nacfx;CHZxWEnszNK%E7&lm;xZfMjVvJv@SOH4k8_z~U{D zbur9Z+;}V8^g}hhOz7HwkH*sQ7qC~LWXgq?j;+B+!e|nE7Hfy0%3DZE)ESLD>7C@V(pv?lW|OQ zO|+cKw{r_{Ix?*V@sw~@xNh?k<1>!M>1Z!rb4TEwZvgA zH%S4~nP|*Sq-E)8$z_g^W64;Gp2)!(U9QOR;BtvDG=;jUWGWP+fawBnFHp2fMT>Ch zgI2_%3~o7nFXlT&XOJ!=pn1%{u^imo7+%Dt_+$NiapoyCPEpZfT!^B-px}A*E0wmX zQ|`k2)Ea7vS*}?+3ztJk zMi4bYS+iyqZWc?(HJM!0W^5MAN;c$X6Ya5N-X?99$A*o?R3;~*f=Qbtv2Zha>3*b} z^{~48};y0U(eY+eeR~ttynTnlDBE1F8G+oO-4NBH5tj=k7UZ7wt|~jf}W_Z zE9819`kA_};&m8yR;)86#*;#4UX}zk8dps!R__>p(=}-#No}EAB80(Zh?7V()1Aq5 zDv60YTZkEQy+URfW;vJFSYOdJ9PCW2`66iQ!<}Py!InsJ(;Or_y)>7ctD>0>M% zEl+DhCxUa(tx|$)D2FbJ4UJ41-cd4fsl?1=J90>oJWi9v24W=T(SET!c5#&-L`aoM zlG{cC(Eyrx0>*0qO@|xDO5aW(!*=~^0R+3R;5-}Rrf+oFbny;%)Z{{_Kl{wjb ztE+G9h~eEWRP3eZI^0-$HlaOROb;=!hvb(#icfdf&BR@G45pwMI=2a5;k@1#%hC)SLmM<&-1m-UZm{(FGn=lU_+m&l=M zWY(2haS6kwA)w3x=Ryy!YEH*n}Sh)2hq##@{*T)eF&Ta#p0Tehi>no+SPRom7! z!wD8Cs~1kWrca7(;a(L}oM>@2MOu2}%SfveH_76#^N(FUpO)x<5Hi_9O!d9`ouDyO zET*}D+1~gtu5na2bIR=zlju|{AS+mUbA^od<#^kG8P`}$YdYfQL)cql9bn@kEsn`~ znguoCRwqpcJMQ?mVI?;GM(2#Vrfm29!a~ zL}ytX7GPAg?LdcEjwxCpmgK!((_}LInZ}_O8OQd3*WVAbblHByDl^+E;@F+e%Z&0I zr17KR7CPRsbZbgx6cXV}2vNH#oES)kCn4liCfbTS&U7$Hh8rQ|1ie%|ZlflBa^OY? z2USpB78`^N_z}X0WMm&xd@|C)ICI^;QMiqKkA|j1*o8$n`)y3YBy2pgZVd=>!bI4(b1e~fO5pFbyn27bB z@Y9k62BDaLF^SX~ZK9Nxd~EV)K6(J|3&oW0NftJGG)p$l^h`$%wt74vtfe8u8Tb`{ ze%LUx{s!w$rp=7Z=9Xki4BtH>7Rhi~77oD&P2u5_L;?3iaN8p)4Y^AoQ8+cy^PoVr zyG44LW9if(FQj(lm3us>Mp&F}$g1#I^hjt3#2+B9(&HskS?nYx3Yh9)59L<;Q<5od zq`L1&@jU99g>MY=)*;i(*8#$tE)&sIKA{bVS!1yTWWy&5vSVwkC_L@}4@N4PNXec9 zMd5b`*wI`lUJzxH3&%Uah4C;q5!g(G!2J$D@K8d*W)2rTz+^{8+2}@77Or@Jjl?r? zrykrV!G+4zJr-`lCYoFfurwYQfRh6K%W79lO%kRi>agej1VUKwcH~_+5p@lgR%U4R zOJpHPT)EKDho!7{h@|pIrC-2Or(&rvR=c%Gm~7Y?aaWZkg9lWA&mjobh6tD)50Q@* zxGGY1m|R$z0M{&6lpjTH5(UHKp|G&RHgQs%C^BJyJf>bi(`PkBgdy_yDjLV*3(yYl zAw7Z=P>D>cDH(>J&bzmq-{2f%3d&)NxpC}oCSe+?WKEhYB;8zV+Rbn`b2f-v&%31- z{TNzRHGe|}!<`$N?8(GZ*)n;m}mo%z#w3bq$yA^KtnGE>~y5#%blW7K9I=F>ZH3N7uFpFmA-bR%?gklRN-UbWBtp-|0)uwD6T2s1L zwCpgZ(h_q}!*jYeb`GK?nAS{GIcvq-cp9sisv4qXdy|cKIFiXmRNKtM^+UM!wN26E zRM_g{7G{+^HYI+g@{GCkrym9L7n52|W38x0DH-c{cBk9dJ2Z@eTUGNm)nytspW-|r zdva((7O@sZ40m9Nu=f9`aB?X+|lRTb_h7o-I2;dyL_xCNO1D!HeT>lx_l?CKec^ba<752m+l>KW|q ziuSDU-He~~u~m3tY!IHpxK_vvEiISF!WNNpNQ9<`_- zZe_8*OA&7$Sw-CxGN^um4547PVI|@mK}l^CSn1mo0;LwiLKRj}Eo~WWV6hIv$ggK` zaC1NENbXslltumFy@Z3*Tn4 z82QCuO=fv@P1IV`Gqm;|Ti(U3PAoq}7aWc#nT*GoNP)MgkT_O(X;?Q&m(~CR;IJ#hR0Fr|y4Qs)YjV zK1tPoj>Y9I)KUcZf3X}zx^M&BSR3|uT$ui*Cft5e?uwWvnickpqW|6w-qk65i4Zvuq_;_9K}M6()6l=J{7hTT4TWpNvtU+(C_K01|}PB+(u$a zZ;-MT-XhZ1QPteZzWx|;G#yp<^*ZT~oNC;F$)qx=Zav4dM@?jTed+4{;h$!uvQ+eQ`(EoyUKM=l%7==fnLxa{(ZQ{Fz}#!D~!-RWQL?R)cx zx#|eLsb!hDcYX9*HP2ph$7Sz->hAo+FX2UZg0DQ~(KlXAtr)!FjWu%`HrC^17QYty zPqzQ`!MFE(`t@xW9dYn2pE?aXS%3_tmc`Ruhd$f>`tjojWy^W~mj&-=g;JHF^RyM_MCdyXEv@~!jBPkv$IgSXtd;t6cglrx=QfBEU_Up^;x z*R98V;Gnkutiq2AQbM1#JWr{rD{(bbsoewrTZ=r{7+G zZt`Hqd0OaKJ+gGt|7`#L!`FZ5JH2o0973fQR51MqhZf%bP{Z_R8)l!o^}M6cJK1qQ zB=kQmzj5)cPyOz;jtgEm>xSpg`W1elA@s^MLy<{;eCwp}tzSH2^BpL@Q;i=ROf7q1 z+V$`I$dq5f^5TJ=cd~A{EK83uyJZt`n}-)|bPhb3-u|xPz5`7ObHduL zVL0>$n9vY>%AW4EL&JleeZ3ntzdNa|!_P8?hq@0iu`R443WlFxyc;R;cYd5nH}!1l z8rbFzAldD-)b&b*$RC>mVGf2h0TL zZXK(}z5>qcb!N^CcWn9U_djrLWalgHum0n_)1QZV`^W!A%5?zV z-j8tPVc9IA`9wJz%V&r2k-B@lx4mF~oQB!43s11$({^!t}4l7c&Or)#n7q z*}>RigvTSqGz@j(N@CUW1~Nt4YH2)|G*rSZ4AkBfINO2 zhdmb7dItWosVd>TVM3AE9Q-{@Vn!={xLV;f*}CHrZj92}(Fh}!W3@6uC;s|UT7Tr! z(Umo`^yMrRiaMf(Ru~LI=0y#$Jk$n1=vg~>eKlfAQA_im7s`CihniAnoEr%ZXoSjAvl<_Y=d^@6-yQ_#|5QA?zx+QI# iA#zJnyOZ+5tD;wU_!1iN<_jOkq&DJtDkAiQNdFH|HBEB> literal 0 HcmV?d00001 diff --git a/vs/libs/fbgemm.lib b/vs/libs/fbgemm.lib new file mode 100644 index 0000000000000000000000000000000000000000..d568dcc93a90cdb44b8b092703db85cf7bfc9504 GIT binary patch literal 17549664 zcmeFaJ98vUwaKcfAj3wzx_Wx z``drxU(=t#|MBc^)t}9O{OoU^ex79TWCM@h2L8$a`8UthpMUS4eR|gQpTXi^JhT4% zZ-4)6@bvQ}gC`q!vVkWXc(Q>f8>r0&{_fZR^4Zf*b22#iPyerHkMU>t@BgpQo_?NW z@MHt|HgNK9{^PS!KmYr``|RZDr%@Ta{kQ(f8~C%>K>RQN z>t}8K{KJcX{p{&Sm%+Dx^82$UfBv)o;GaKx`gxMUlMOuCz>^I;*}#(xNE`UC{)_+R z+0)OH47zLs|Lwo}|DLt+^WT5^`)7GS|HFUt|2!-5^FRH+|Ki!7<>!Cy|L4yh^Upv2 zhkx^ITR;Eupa0{tr=KSoJlR0EZQ%d@SO3nlr=KSoJlVjbw}FH8GMTLw^Hm!6_kZ;F zqseTYE@#PoG`_h@r_(6jY_V(oP68;!oo<+YuQ~LY?ZPR<&HBP?$*y|6h5M2GA z|9Ea~@5cD@$IF5}7>$4`UaeAVXBT;_E8s-tLRdT4nI=D?%V@P8q3DJnH~i_L*x54R?REcIj%)_W1)I*d&d#GDsyH7RV0@oCCTTHj5Yv3ADx575z zxg8T{s9R&ZEp<~}x1n#Y>=yJbQEy-03^(~}+d)mi-45d|1#{%Pt!U10x2Esb;yH3m zxNgb88SZurT`=E!+0Apo$=jDOqaOHZ*Azi zotVxTXMb&X9;)MR$Iy`mLPr}a=g3=Noyi_cFI(qa;%<-ec681-$G=))y{*8GEVmZf z8FC{1R;<0f*iKEYrR+^X-jcZs3R+P+$KEva*PHcXvwpjr-=?e8WOf%&S3fCGSNGJ@ z12AWp-x6vRzB#sS5p^qU+oSt7xOT+$Eun5p*RHs}4Ymo-?U*=2-HM4L?zYs;ac`e- zCUHx;-ngkr)UC1IioO}&_*XlOw-n5g;kKeV!%akQWNcfp^VZ@ywK8j(z7@ua#M?4- z!F(Gk=g7D0<4oXe>&sRd*HE{{Hld;=zNyaJK;A}7XZmg>rc2zd>ARhn&KPHZZFe53 z<8H^pkp@CX8(zkdx4=4+{nit8&N3ZP+GD&OoinLZA#B~^5N%t5UE*#@=PbN#Kf*E2 zThuwCZq3md@|LttrM~Uz6UQq5-j1)oI-1YE*^25rz1`L+c+olqhubnjmmbg_T2=&- z(6T0wgmzVd6hJ_1Qx%MaHdVn$XkQgbNZYzV8roI{($JzdSZdg!E(8UwX+luYx*l3} z3Mz(HeL|AZt}0LnZH^Tn2S!4hPQgfMUlmA5Yhwk|1=7&AGLVKAwZZJ6O~;TVw5AJ5 zLi?&fT3T}qCJ6kiZDk-0Eo*~Gqg8D%8d}u`qoD)ZKuTIy2NKb`K9GpE6vFHQOOb7< zLnEObWoRU{uL>lm9oLY8z~97Hkd$K)dr)XRc$aDI-m`tq;+*55v}V3iD*+H zv>edtxFJYrPZ@%Q_EpiYYfwS7>lu=UmbHOmXxlRw4XwHcqoD)ZKuX#hH^4BF0Q_lP zA4o)-3PJpBPdFqE?Wsf3&;e~AG3~hr69xX&x;~JIwiUwU(XK)m5$!635zz&OASvx@ z1gU6WB}hdpI$`!e$B%Y0ER%#5v|*CazABKS7Mw#91pcFKWgrbLYlF$8Rc$aDTGa-l zp#$1LN?KP364AOokcc)FBC0j*sY8&^o-zap?W>|)*Pw!E*E1vyEo%eC&}v2nl3+Bn z>KcrO4rl`@X*EzG0r=CpK9Gnu6@vI%t64GRl7{xwA!+D-s=i+ExgY zN4pAPM6|0AMno4Bf~2&s5u~Dhl^_+Z>4e!tt7C{pLreP5Xy|}8ke-$tM2Z4`Yh52m zMB55sB57A4jEHs>!ieaCLXed9HG)*MuM(u9Rh@{^O^X^4G_K|=?$(XxY3QMBwI zl8Ckyf`VwTjD^@i7L15?9fT3l1%)6f?M5u50Dsz72~yFjPLP1sCnA!F7BwP?=z>C! zniide2?PIXUnNLI>pEc)X;~+Xik5Z4sOW@Fkd_W81p%OL(kV zdFUG7fyP&r`{bGHU)g)i_0~{{x$#GXvRM_|7Ok1kIjEWv!5Q?93*Ex*1^1B-|w=&_u1bA_V*t9yU+f9&i?K( zFeZ^j|NZP>nT|HMsi87_zx;&gbhVg{rWfSZa-+>>(g@%EfS)7yIR#5Fo;!cJoPS@b z215ZuLjqk$6xl2NgAaut^oD>Ov$^PCXT4e6rxAXDKlEwwUr&ZZ@)3JkoOXc5@ds=j ze}fm?Ev@N;-?T9Yzepxacre_6yR!o*2>VFQ$N1z&a=VU`$exZL&)7cEHPIy`21jApTN#53yk4YlPFBeur=5`}0Q6ekjI|=rf;D?m1mlWU^EI zA{5d7VFXC>+N)Stk5{1>kSd5epU;Wx;+%deCZp%O9#LqR%A%LZRzMdOA?ws1Mi6$Q z6Zk98Kq#`%ekw}Zb5IOXPV+4M-D2=_nrGoZ4~5T+=Uq+*6$9x)l@6{Mtdd_(4#DgW zkI;!id#h;G{_(4H9WT=rV)w;Y_1kDb{C9+*B6%2*Io`EA>#PoD?z zc-R{s_Qpr%CCndRL1Q&rS9=dC_1>-2`+23_dzE_c7w?S;BIbXPd>+lG3-IYxIs(st zv=VGTxxb&^ChPe!I;Ob_d=|flbYONzvQ)^M;`fs&CavqAi*&`Cfv-aePfIF)PnYv} z{WYCI$K@m$O>TKY$}=vK*@7>BicgbUh%YzEDy1N7)B);II!-qC>v)vjr|UGb^dmW{ z39=Y%kj|L^B*Er41RqlSaW?tOCXE;CC1oQ^TM+$&NRgq5Op*}t!-o3y(4&1HA+wnUlbGKm<6{e6?Pt$ux4nh)AygsgHl*$VNLsA8=<^9|JF_ zy@$0CoDjN5jg|QcPB@CwMx;AUW|Kt(z$8MuXevqKh%?gw?BfPupEL*?MtDYgJRIG} z-_qq|{4-uAv%9n(;g_TVv%k+^OOP5K_dtM?**d*jCin5}e70IAvvm|-k29y|#Y6D$=0#mmzti{Rw z#}_nKp`pv<56EzCzlsFs%kUV)I6MZc&WMmUhIq-P{sn0#-r(-?0~4idfiI#!n9-4}tq0+ggzLC$cc)(?Xa@V8F!UIzV$r9+yHBP! zqa^O{zP#NXy|{(HUhX9~H!q*xz8K%WxEZH=FP^`+dGYrVvF36E5%D)1QITt?TI)-QJ7A-Wc7XAszxTQ$>87jg#5!PczC58H>qigww?N7#sjT zljAK7f+Qx=xFB0XRO0zEu5Lg~vCkU`Fa>02Kats55LYk=osx2Z4AO34mYpoq_+~R6 zr^{7zqEx>~IjDyfuuIB^+4#(8eCBI>X*9mn8$*V++&~7whAs%C=sjkF1<3)+u#AC4 z(bW5)W#H*Y08xcaY-%N zu?GAQW??2U$fKn{k+hoRA>>mcJ&xC{3bqJkG%!qXI{!wNB#l;zMvHeV)+Cruz#qk|3U zSrbUrC}Tr0~kcGHtWk^=ik~u!6Gdmg2@D-QVol~T;2X%;I zYylU$eMy!xu0G2*@o$th?JIR!L)=dLwu1W-IJ7DU(+CokfySx;(fVoxhtt-&NaNb8 z$|5GK82Uh3-0P%bkpKA2{B|-M&tdV3O}U|IH7^N6z#YNUI= z->0+n@+V!~;8MsdKTANEd-b7~%wYXotomU_WS198uA^wh;dN z3?{;`=BhWaBz>E}(zYa*N}rgxvMNQoae^sOryzsWlD+Cz!mOkh!%#8 zIs2e`-fJ?7RMeXEGhb*-FE_LGWSXi5mQ}m2QB}Kh!Iu8CBJL!r7bQJu6=2?uHmbyF zk)JpxY_@82U6^2{6mP#2JFq$*(bJ~JqJ?l;f$}WwCVkxBGm#g^?5ePDEWAAT01>7FfPN zUE_Ue^8(0hhVhk}3%n%W0n4i7Q=Vn;B#SSW4Y_WLCYmC*1L5#R#;U*$LEvJ=^3ak6 z#TUdm4$Ju#N{IA7u4dH)#z_q@-W6CpL@bwwlZcrRLIr-b|~eJtjmngcQa+?YG@kF#T}{Kht2;@?_^QT|nS z5M~!QTBGw&e*}N72dk_3`k{$(!io?2p;}`|My( z+MH;18eLzXU0)tvpNe@n$aOcJk8dXHgS{1$&y}`s&(1C`FHeuPw!mYy?W^eY^5XRD z=$!CL=JyB}vqW3tx5MMZH%G9H$%O)_iEw~u+;Jp{IFF7_--5MVpGA`yOx7kKU@nHm zd_}7tg5>My3UqgRI6RF`0d)$E*C49N4BC;sO`x5{21|}$eDov_D(N~pxi~pJKRSes z4lfS&4)&HQEWW0*+f>p2+vx20)!Wy@bFlLl)5!`}if+Fi>?I&J0CrADhQm5qM8t#{ z(@lAnr-sOA1*%0zvqdISVp^=yl{-n&8hayW26Q%pnoaabYJO`*RP%dj85NEt;+m9XBd^f{k5|FAG1wG1OTY7d>4a)|{OY60ul(whVO`ocH5ir&PqQiI z%Zxdx-}%N~wVl_9)P`C6QNGnpNBP2*@2g+yvtH$!ti5Po+6HB%fyGLcZ?gu%zU;qX z2k5NEkv%P^1)1#EHQ0g`sAdpmiy+c7rjvo@p)Eh%2&3)E55cU zPJEYD9r+55yZpmuwRWC5A;Ad_Jwvq+Gg0`9uBFOTFXnbk{k+S2R`ABcQ#@R);4DkL z{D!A**07$qf(dxV^7m(2_=eJ$=M(^4ptazSzfaa*;fM{m2>kgy`MHWyIDmb>UAm5v z``gVu9B_$oKlb`-RqRkOWhZ>%qp#`hA8=%6jMHklh#9iv2z5)QYdA&(^Ih198{gNe zIxKs$Jr-a^w=m;{QivI`*!)GS{_T>JJ@LuB4g+FLo{py|p&H^9u!UAn4(S||wUGGq zKDmqFUJy8xsGdd&J&1%08P0L055kC^qj#PR!O2c>zhG^Akn1~&EF}%1PrC!Wu;tT) zZ?YIfz|Th%V5$=@kz!u-8omSp+FT#dxwLmsDgnF+e=4=lD2C zCkU1z8YfY}U}lqJIMdpRd#W@Q+ExkR`f?DkN@0u|p^=)G4g|@PjGN$-TG%p<^@9Mi zEpU;FSZCt|5COo-ltE$u0ks_AH z!O>gr11-US4;cp7E-ZC?(njn79#{DEoaI{(WNF_T2G!yiwTVJ_cd~6?>}!rx^5Mxv%BODE)@t&nuNX5jTkW;GsjbMk)^D0M6d8GH>-o>k<~bG zM>S0)$;|N*&V$iunnH5r@M<{|$SmygC)|sppz;n(Ogdyai#k-9?&Rt-qk98)^th7_ zL3Hg{eVN8J*$lpV4_94)MUpvE9FvL_*y?C9kI9*}F`S0OP2;O5`Xzb}cN0L8biG`_ zm1$@dilVC91PMn9Y{>s+>tq5a7s!&ba%B{aT-U4d4Ej|F` zA>MC8Zl{XiCLX+n8&1-IS$&v{$SEXsw>t!{A@v;XTfXBx^R;HiWb7tR!#abYHDl4c z!`e7YhtWIAjc10tBiBGH!ds45R_D)ah#m%yyY3q|#oS zd_|3&h@xKDLc;ZfWfHCzuqk27O?YI{<5=1BE7W0ajA3cXjbzeD``l*ZVX<_9=HzLUtZkm9h1V z&cJR-S5bA&em`H}fMQhvMkdr38Y8tDR-&#+%Gy|Bjg4%NI6BxUy`4SPI)L@Y=`bP|On8==mLOLcObjFnUa!5y)`2L5IYj9$DyNQmm~AAjDXG@r4{; z0am(58#x-&QQFAq`;O9f-C=tJryz&hIx0GPoU5zEb=e{4I17?nf1Phu$!xTWSA5E< z&7{<^Vm-f~e@}5tw?tWH#hx}a%F-(=Me7YrAMzkN+t?t&v5ba?v+t7;nX9>l;A}w& zhJgmN6qG?aU630(%8jrh38TeaICp`%$=>Juet3pc!}KpOWnI$=x}zOAsz{%Wf!#9` zm^{MXp2(Gj=m#4~otNYwL0#3WC18D)p1q(&TimNiO4>z6OhXvH#0Jx@%83nq=j$w4 z;qy@M!A9Z6TjlmIsuH!)Uf(sWQtN$FLV=C63e33K{Z7v^~DxtmOWq;BqFL zhoimRk&Q-m9?-|>h;5Y`FtGy+WWrQ?*_xCWf>!%!(hBszJjx+;(>@!%8$M-9saKFE`BfjVr^$ZnPvfG#mRJXe3lXSCTaDBSdjc;VxBN zobhglW)sY^%4HK#f$E$q)isP%3dLl=;kXoCis7IVbRsjn!O}5aW~&V(`N$P7CISeX zw$7ChZw^nU+)h?_<&s$(;mb&i&CTj(@pUAuz>-me&Mc{|6&*8(3)eO^6(N$55p-K3 zU1-KJOb7_}ERtz+7YQTt(k>E4XI7o0j7+PRBfS5PM<~#BUU89T9=t>T>MYFzmLqss z0$%$O#w4N&ah4^#1%&Sg9qiPZB$3q>%T_+P2OHm0PD;-gW%ESk0KKrL+1{gLdmxz;h)Cvnk_x`UDG5(0E0sw-X z*bG`|bJX7K-?J1VKKj7l?zC(Qi(bMFtd`Hwq{-qfxrO#QJku<}7_uTDjL-D$gXd~@ z`^iND#r~B745N)sjOAn%0cH7IfdgQ4qOymu2$bx{IuJ_BZ~)QKf)KeL?!k9WAZ2bzD$2C?w^UT;r zfu8r=aHUiyi``%3Ciydtp^Om154j8*J|Mf$*lTWQ22>Jq=q#1jb(n+MvBSg@lA5`A zIm-d3 z9Bce}lg6_R6v`MHaswYe>_mnb(o|l}GNNxB$y_F$;Z37MxR&1FT9LGbh%@qX;(^+g zxF5zsL}eP7=*SC~AUw3bXF;RyqV5K zNI_GeK{sFHsO+oF^m+_1R{4PWT#s~lH~(Is3$#8T{m4gQ(C!gG(*6l&ApukvP(6l# zP~*c5Zi~tnABY();-vcL<9TpLnmwSok@1@!0RF_^xWt7M7+f%*s0ogTcFmN*ZoRRH zn?Ie-6(Yf-ltLtMZz@58$HfYfz%#98NFZ@2PZ1Up!p<7Aj^XgVXN}Jwe<#L|$z<~c zaM7p(pF%+3aaLx;c_NUuZV^FIF9{@#8FwCO(g%u?CVL9C1(H^W;drHHG3hOk#!adq zY4p@^Uf3-VaO@^s8ye201&LO?ltrJ2^5tsMAPl1g%r8oIb3Ja_!XAx7-;P=6bZt zN4Cj4IMYKf!XK6>;t?gf*CHS55yi=T9TzQBw-*yL3H>%%B)1c?7cL(<&ZX#nhT9uZ zUA)4D2XhTVe|-^(8~D;Glr}EU`uO*du1wtCPZn{1AFdS^6Ok{&Gg48)N&~%aX+fB?>8_VCkCCNxz?=D~Gp0?t-8=ImF&&c%;NRxP*(Jr&;tAW_AW%xDieA)tKl;9(0ly=w}lfc>eeTgyuGJ`iLyH>aN`rtut)^J zn>?~5zs$y`5a;3XU^>CYSTE5_ycQ0PGmSSsM@A<;LBBbJr4G7Zx=>ft9O>I*tJZKp zMZ^;?W^hgFUu>PbhyYfL=_nHlsq15L#ZiNS7Za}#DGhv3!4WcoV!NyxU*gK|#}Daj zHDCU!c=%haA+N-{r|hdVQeVS8x~$Uo?2-xbV8;p_iV}wy)Q<{Ms6L&0QjkKuuZ=>r z1*h<&Acd!j{7(x~sQ0x|sJ7q~o))AKUOf;i|E~&CsQ0x|sJ7q~zA8weYQtv*Db)Mg zC{$Z;3eUnQd?)=(wH0b3;!84$8a^hzOtnYdX~3e<>R`v+NV=1w4KhL_AiAw+ltYTD zEUKh(Ju&K9n9#v1x)4~>NNvNJC`O8^PpU@&K8Jn1r-KQjkyR6F8_q-_Qcfp7($QOT z>a?_BIbd*l3K^*aOI1)I#W-Maddk95WnEUrDhy6f0Ib{lWV(o6YZ`hj44-|g$t&}% ziuug^Hh!&*z;X6_4YlQab$mFre6JB_`CcPWe1G(H4x7d-8H5~2;F30Hp3#pkA+w6F zdN@mF;-or72GxRq<@$fjmnuG1u z*<~AX`0kzE@k_EJNB;Zf7v;Z1ey;D}5u0sbz``ADN02_p@6s`BoPdXHsLHI1b-LfmrleKUXF2 zhPXL?{B>vNgCz3fA*@>c0jh$Vli>QQ6)r!!Xgp+`A4%I*7$^5@xQXa9DfdY`KMlKN zr!0!j1<$&P17EKungr-hCg{!s%E{?``3f}}9g$UYF0)XktVHZeBA`@(lrSqOz{&u$ zP=yt_5uySsgUG=OTqdS&m5#^D`INhg%|yVQU=f_Gi29n5iLtnw@L*>Jr#aIRRGr9W zUg(E5;Cv67kEjrpTZjzlmQQpDluAq}$}Yl$(yam$3X~8g0wSq}hk#!BECc|l!hsg2 z3-F-zE6;@nM(}YmOK$GrzD(*#=%;Y%eX{C%+>qR5Pv{^ICg7IOFh|54bhvNO;i#}L z0><+6ZV7k7z`i;-0s$j?440+BU1?&2Wgfi&z7#}%035QLkMk4IzFGdHn^oga!U^L~ zwy%Z1d>;&AFCUD?U4HXMnNwY}5_>A3AO_VARak5xT+pN`b_4`YP=H?sNjw0KCh(B< zB=DB9g;Pt7{(4%xKMNfcTTF43LRVy``jTN9r>*uEE zJ?ut=NgJ%tE@HS|39e~Zl29JGZBx0>Op~C7=`lIYP#hbE!@=wXor#H=1D)n+59!SFCpR*bJn!VL_up^-# zvb`DPOcPW};ok;+Yo#xX-`fmV6K%*LGl$Xr8|)S0$I+x`NaA4E$4p5U!RB5$V^D?7v97iKi%`h?J$gP)DFGDT+#!vTRgD*%he`mkf4 zS-9dQ4u%7eLQ3fsfRzMCRksM+((;1X9@N6NL|TGvXEEEtc7Ax5%O~ud0=Z7BNHkWK zj>thzzCJ?(PXVIT=vi{y1Vdi-HW2j5QhXAx>aN z7tpf;dc_zEqL)4M(dSzt51J$lNLlB1ERS5X5j*oP^lDch0cP4xuw+;@TcN!|1^|yK z@PaZo*We;{7Ey3lf;|v*fNfH4HJ+`cgn+7rG-!2vf~H zRA?^JG*||4tMAFezxwi-8xng1+Z~e0&Jq~ObdUj7=0Fk9vmi=oJw;KtC5^u@CH(T`}5x~J8?mWP?J*v>v>3m6cPQYyg%61Sup@CUL zc23iW+6!*7vCYs}RmO#25QG5tOynadwiAIIuSLc)-2%*-x*Tq3C;>*AY6zHo`>O&) zH7N_6u&ElcHDVqlRr$p0%^=+30O+2Y5?#SGt6;`0acB-%ii7PTR0KrakBZG4>`9oGV-{ROT4?G1HjmdYsp(d--$Q;3QP(F~!pk8~=nkHV_5 zx7S&}&}g*xby6U29HZl96QL=E zv%-yE(12PbSneN2kac2DMgJiYEU#m*PD=wawU`GeRBvVBWHrSgz4DH8P*ePTPUgTd zYgj~o=%85$0m4VmFVIm6JKM3KVAltd)gfSNjqC=QA<{Tdfg6(h)&9#8P~;hH{MC`0z^;IlY}Q z&OsVrTeN~hhCfL}K)(ivk|tM-DA}((QTe85r76E#{UoXOS%@gLqmL}yZB{tn#0opx zIvl@uKnO4^g`r9irA-usn6J^A0IJ%sgS<=EI4es>zrp^l!SV5LpcVXpiqW&_%W;Mw zlr;=`!MM%+V0Gi(ZI17(1~Q!cuvg?f0Q;-ue6zqil`ASf*zYp@=TZ z6D?gifJn(s)L$W6k5 zNNVB4yPqhl6LesFovnz7nbRMVm%F_krQe{~AZH@)V!+rK&{vm6fxa1a73*uqyMVkw z3!)ZF)VU?InviZf!mw@`9)r7Cu-TqzJ*FvD$TO|1Da^m(_i&fw><*voB(J`_pG+5U zV;5E#;9++PvD!1O$KuihShX~saAgQSgPJ%Y@@)}D8I~eBJwUL0%z{%#b|FzpiKl0g z@LLxS$TQ7|4S-H=V<_bS@2>aj4P^X)Fu?)#d2HcEAUZq769{naSqx7_PT-wMDRGqn zXiunT$2ZGS#m?a3(Jyx3;-Dq;aBVAjE*q?KyA&W|x`|92y^DvV`}i9?pfDB>^Pc`7eJTMezlkYB`37XexURW?!(|51s)ojDEnR^-&eq&1O7~j?cL|xjyJl zsU3XR>UgJjcq)7h8vqSo@G~ZU+Os3AyL9!?C2RwEGf8bLQnad;eqe&(Lc@J(?1T3F05l9YjI}trcsJL+`0xJkZB0rJ8N*M177#!X`jgS0*lcL!Vpu&R*7vDMQrk<^XvK^LYJ4u$_ zu7qrg0FY=E!7uo1hHYWSRL%a6C_s$dt}QeHaNVDjxVnckB^TI zql3NCJcTB+`5Jzme4C6?_{V07Pr+&(*)By>2D&wY$s4;G&GwRbRAsh(3E7naB*{=@ zXaG{S*O^>Mu9Xd5G<`RlN=%uiC3r0eLn2q$bnH1Z370&Yo15|&wYM+9Z5cpTmbIN% zd5prwBAiZ{%_LwAT4}Zz8h?L@R8b(s4_j(%WSVMoG?RceXntxWY$dE0=r56GxM6#D zOrcAREWqJT9UI#cw9&8xM};6r#0pOY0$d*scM{_Ti`DZiGJ5qsF5&V!xNsSsD@#|a zcmk`-30$xruai5N(BYVtj7GR{Y?F`_B@T!Du>^N}$(Yjq25y%m%aDM8P$0EzkOGsXgN}#VuYK5gz)n+7zI%tQ5S4BIn8yaiKvR&NXIE=fVT@a~h@DqYqP4Oq>azW2wLD{F@hgddaQ zLnDw;wUZU!@9qx!@T%bzO5WX8idwdPyG$_z!W=zByBtGaFNuj5JE4oE$~6R>B9J5{ zwlD~g$ck53=#t}tGN>bi#Nvy=%B72H1t%BvNUR{dK>EPPYo#w3o}a#jtbV&42CD{G z1^YfRYz4Kv^cOUODf>6RRL?z9>pq zBu}G(5AqtGBnw5HSr8=;4AWPg;bAl!yFFDrwQ}ahBg0@8|C?!Ig; zV?86BBOC=SgQC`e+$fDQiuHJFIBEl$S(I^;5mG8~_g0wq)l1v}=(JcpmWN|iS-n$e zYGoo1v%9?D?x`@X&FXWsrM0cbp*p*F)0sa(clT9zBw)-Lv%4qBiRh#xY~Jbyr4S)9 zEe_L@d6WPMVNZ$pWtAg{%$tn`QW!oZa|1-0#KcOUNw_=ZOcC525gVN+L#gjPU;4&=%jSs=!U+X9L|m3n|JXf$%qQUn=1_HT%rwIhwzM>{X1PBxkq&h0= zz-@_jvZ3Vij_L)Z^<|CJSn!v^sfh!ztyYqLB!86}I4bO40F6$yH9bvf-n@*mx3<>!{wg^VJLJ+(d57oI+m0x)+0Of>7gS6%if9@ zy>mrw0g_ThiQb+e8Y}`p5u0Qq7=TN6MF1!iNOAZ8__fA*+9%3f?vhnSW2kS)fsH*E z4xLL<70m@f2+?2}K#CkCdxZhHG+G9bLXi|#2q3?P4G2mR!=WoF4n^B}6%B4Gl5$1S zd7VRaTo8sLOvz6X7%t5ggrU$OMUw-=uTR!Fr@F)SA(>Z{8B`zQ5G2`(MuQ-PsI?3r zMW&Ly!hl=~E(1uRNQx^2kYCjlg1wr?)gvX4QLw+GBs8N}d&h8f2K85DCVB92tm5}8 zFCRbjSMWC33f-cc2mYf4I44eGCm%2ZW;f4{1N z`YZUQC_MCssESp4i>Qj_N_-obqpPG)QI@}Jh`I_vPy{9w77oFsvO*9P8l*_P5d3N* zU3sC}U03!xo2Vs+!;!Qr>MQ^!M7QO^DMFTtE(y-1_442p>ZBOM!TIf>1RP0(R-0hk zO}I@sYRtt9D+h?A#Aj6wAf-$y43?rXk39vz3O1%NSPETIbQuec?-)UC<~u83g;!I4DxY|Cz-{wy=Y zR1OeHiO;GWKuVcZ7%W9&9(xLa6>LmluoSwa=t6~#@(NtMP=_ioDGzb*E#Gq_C8bOe zix8*4lH3(HjDr4RXHEc-l-Qibw^<-31REO! z?+^i7%;_hQhxK#XOsVct5yiV}R#B>ZLF^7ElvZHGxLhST07{i;ULBxS1E5n^6$v%q zN-@?5SJQ5dasapry;9Uc24schwX4H3(UOD%jV@cC_}edV;(Y&^fM53Z1w6n_?=XyoJE zuyHOd3D+xJM{kKxv9c_IMsKGMijZIckwPpBMAKSXc3fd7NG3rsCkT=f2+Aqgh*G8W zEX-1ppoES*K16ScTZt30cjlN|*0OO7aGr|e%_SOdbHnUp{3y~(8f$Pb%{q(p3d1F& zEp_EXBASOu&G3INa*0dvSsz*l97%-Ywguogby|)r1DsSqb#R&$RA<%|tqRI6C~g&m znHRyzgR5vQ!t|x}Dp4(*ECEN-qr^8A&WEq*GKIas@6u^9nT^CDZ-+NXx*Yaa5T2yZ zVQ_))47ZSxvl^KO*j#TbL2ZcFl>@6#E5%$w_b7zIURZ$SBHmaItYp8El?ktsUf}S~ zFfs)^NuSe8C#yHfYJG8tmx{3S#G<<-hiaD`>zu4UeD!sbIP*Id4+cIG9MB{|86_2f zW_YY85R?b1I93tS8+@yj`h!48O8oflo;L@07JE8*Y>SIatu&TPb)~3&%cU; zQmB$*E5Kl|F9nho;z7khN!AKmgs^WJ9u05FCr7}LbQnG&VJzq0D|<*T3`vJm9!?mB zSvyCi0G13;iY-cE>$Vs~RhWl_f>NlGVk~K0aoCLyNKwlv087!6l((vQvjAimlRzfT|)hm zBZmcf2ogXfdWQ|=U9J#pN1=cuMGjla160(W3c^!#CpDnzt_ZvkyDA7zp-+mk0DZzV zBzF$`${|bul4N*o3WAL zlN4DvO~s}b5EuM}!M9=iR*OKAd=)hXVbfAw4(D*PB|u0jY;LiTH*B(D+}vm?Nzlqt zL3eICKuVG{!IS|R<})=wR_s+u1UGyJTs>-tL8aO4Q#C+VXqF-`PjjHpl_fmi2Wx;V zz!|QuA3qNbfDO-W=U~4HyK7AjpztMY5G3tA64tNY{x#aTlr6 z+7P%ry-oxQ2T~nX5D1Hu0#S^-7RZuN#nuBLle_Fop2&qu;>ex1nU|{QeZM#;g3SkF zkPIk>Ukip12WSG3Vn$N$wIgylL=%V z9+W^6gaI~IyVHsft2UQz;Dev2Y3jphBS(XJHD<_*g!f&?2m&Lj!XmfRfpgwj(@4D%R~|UeW|4 zNt4yb9B*+H8_Hn*S^aem*RvoFQY2rRthY&u>PD`4D5LiUNb(w#mwmmMv1$eh3gzR=SrKueT^1L5h@$7Vv8!h2N7ekV(EVtk@qA!t1+- zA$NESsFH;$P}vpS0;;4HFQKK+ee=tqEsfaaaO}Mx(LmD18sIz9`=>{zuig$XP9t({ z*TJs(!u0j&)#2OW<)!n5X>cLT{kaU+`_+e`(VJQvMziblm9-sqed<752ePT~t+B@j5 zwGPvCotzZK;_l?Zgz2;IBn!kN&&zjaPZg1;rz4xSin5k~=t^1zMuYI`nc!v?AXyB! z`1qT>u|X4We6SM_NB8l!bU7LSjNyUhyR;tx!SQgpOn%0<_=dB1X*nXv9C@XV7$bYH z4t479(u;Z@4|d?`!SrsK+*7E@Y#qf{JH25P^~mGRg-IDig-ICzu5zR_A{6pgUseNN zTxGdtxL&2}cr*LHOco1xAJ(rnB3_~)rfGQc<}5(jg;6n{3Zg<~QdICF_-4AfPu6J& zM`vJ*(0V;t+)vUKyqE|JJ&eIKzd*ovGMx+3z*x z;FM83U#uf|5eVutJH6{6{7U}V-|4~s;Zynre0PT5kv|S0Pul6dKE^*zqG)_`mrkcq z^u-dpdl6W~3B(e74QY55Ur}|`LdH>ZWB|KekVX+QDodlMJoL)%Q`71fBO;ixcvVh)Fe!kZ%72FW zTv$gD>}AKB^lmbPNp*r#Ic4sKzz>H0Om{#^Wf=$x7C(dmB-KcKIvZ7@!02BXzcOz& zH7;EWN63J}gb6?@xC{Wzlu7~!q`*-jG5IkB0tRxEtR}bdY7GGHqGQYqzb4Cgy-X(S zRdmdz-(X<(aD_>-o-g4uK99ld#8oIU5gTl532{~d+e2JC$4kD#o!O{p3R`L!wmaGK zc)IN|9H8W-4uG0gY62KYAz$dQoB$#@^l#6n3An-!m{Y^Ns860)JtQ;VkjH=pRJ{^s zUp~o)%2V^~hO=y<&D%)KvW>`|c_UfPHqu<)Ktf>+c)rRTNC=L^lQ)!*jD~8)$s0?k z*`$!SkpNknOg6|{N@$KWjW?7aX+xeW@$XGQ=J#fQp3b=Xo8O!Ld7hK)Z>A%>g@iy_ zC<_z9BqbBd!i24%grzi8$5@V3g%~!t@}4RRPfypF#=#f_G#J`BMuYI`7!P(1dOJIx zAb*OF=F`P8U9Hj))MX}1ta8dh7sloI!!k|d*?g2n@GsJ2XXgm3^6SmwK7~R%>~W{# z_1pd7X9)Xn^**WDMZdF;Vg5;f6T>z@lKoR-AIuneHG)T6(X z==w4KJzdV@_1AP3e@m8=WCW>4vbffAV$fq^k|7|Mr<4rTU&&mPC%l^w#aXD~S0 zM=-jy;#L7OryM0bQhi!{4L)irA+cbi9stz~OiEhStIa=mc7CUcV42L0W>fcKDz;58?m*!&(ZQ1z#7(Yq zC8CGuYA8O&CqEKU9ZQVK;DRFvs74z=)Pcb0)7R2L<4XaM098f9=MPjpIvEsXpN}>U zm~sTm09Opo314>K^&`SPDwZ)-%z#`NXg@K!+PTPT-xPhmf(^(hy)0PHBY59avQErW zOQuM=+?s4MxnHA{Sb`)QuZUde?^~Q!1=6EL&E~b0N0UusG!M=?pCVOb?86v~1KfLv z>7~bFh)x2m3pt6Xq7@x@A=@TfDAQ%Uf=$5b>=p_;7bNWWu^40f^1U)(3ve!^gm2M` zA}!3ZMG++6w#4-%8CpPR$`JE~PNoE>^efohM`Lb$xcM1R#!yaM(~=sH4&p7NZ!+{l zX^w15{4h;m;eall8Muk$GnF~-^lFeP0+u2Wb6rS)6qCp6Hb9AhoQ^w{tBSaLZ1{0N zOuD!nd3sevD;Aq9r5O!Q6WY-bFs0P2Aqxul=rhC#_^Y6nbXcmUh|xft`DmYa=#j=1 zkBEuWAdSHY7ACn(-aHmucC7?Dt|z043&Q^Z$B6XCLzjhz5}j~jI`eQK_PNCP44dS*UQ ztF&A?uPiYy;M%0-#lnUVDLFiAf(IDmIV4k%XB?l4gBI}x+rYdNpU+n55-J*NtV?9A_TV;DTuz-IGVwg15b#)`vCbTD4F*U1Vjba&~1m@N%|ghxJ*>V+z@q z2Cls!Zwq#&ajH&D*zc8W%>o=gqN8qb_|bvjaT2mH!)@*>OvRs~S$3TxaaxUL zkYdne$T$KkK5urXMCs@UmRT-Z+4cj89Egq4MeSO5lWHTi9oADcg?glaqnAZ+T1=(q zy-Cv0oUP{*Q6Tx#fzf=OyKA9oh7O3^CyoVrJfF>rT7K4>>hsEnoS%XeNSRWE(QW6c zM9Hv{ggHb3lau{d+AB0w$JmNY*dYy}h40a?uOVhl6& zoDoIZF-G04uWGy>W=o&6#8!AsM~n+0Qqo0TKvG&quv+(?v65>*8UjlSURh!~0wJ|G zH}egg2ZY$4-GRu)??N+TVTVl!NdUJ9h< zMeA&SKT4Nk{-#3Jsmz0{wRS_tGg`rj@xKmL(Bl zpWhjU$NVaC0r2vUPF3T?g}}!R$gI?c7-Fr2+~N=jJvPyXg0e75ov08&Ius{SA`CZM z%9H|a93f%#lf|@a#HgbXScXn?)VaSxj{^# zDW@HlJWTI`g?H592~&`igoPJL;-P~EVkM7gv@Uj>juW`tkM0h_!;Q+8yXH0xD0WcW zV!L~A=;KHT;;nC7AnOeXahK;oq!wIFV-{C|kJt#eJ*|1kP;->pH0Q@yO?xibJ`IKr zKZeCYy5`uwr|itj=5*8-lqPcX-k-)fi%XoAr0m-z%{_W9nVap@y?oN2Lus>E+QYI4 z3j^?xmSI!(-=^c2lYZmYu}tCi+)wE(44D1>!^s*>vXZlq2RpCQ*%4d_dXX+?={+VB z#zhw((odgX?w?F2Yq(B8%+**9p}>7vAJ|a}*~I*31IYyLSNzjNDNr4lV@*>_o;Xz2 zjo)n>){h@~6KS3wV^hfjbx*FV(4b@_2!ASPDJ~<^BSIlqETU;iTTsGB4|J%cw#};L zG26P~N@a;l!&KdW7|@!{;+~m@v|vT6@S-fS?X4GE*=S@&C?#S;S=gPwnyGV{6uW~K zr};I+pRKPY(*%19mZx#5&g$9kTd}@;BAT_pG0WMH=U4MJ1iriIyUdbC%`5~c2l6){IE1qh2=7wzz8#$ZxZbQ68+=Ki2me38H#V&FWzawc zk-RsON2?uEe`}bA(RNVbyBu4RAEMfl01;K~5+rxJSgnlPaIO~^Aj-G_0b#e!y7lNm zRkt2JsG8BEMS@MZf{U*ve8GpSULlg_dtNQ)n+4wfgq0jjS>M7-LBFMwyRU0{ed+0Z z`HFmZq+P&#LSuH36(n-2C=~bek>D!P@MJ(UEsKtGMvER?z|6+18hFIFki-DwTG{He zm91KN^torfJQ720he@wZ*T^M_3+<>fFC^8>WlIK|7Yu9WvMrNEyj!tZJsXCIhSz;G z6Nv^a)XKC7&{fly+zcQ_3i**Mit_ydp;45!Z*8QI3{f8O$-LYk+6a%$jTDk0$|IV; z{A{a=Y|!f@N5Cbr2pNX_Y~xDp!48C-rT zz7%A@W%bg+{f&{3Lz{bf*8^4P_AOa%xFGlJ@iw9^WZAsvm%zyylkXaMMfG2XFH=Eajoj$1_3#&L&X)*gim;MaK|vOb{z zH}Ax}KdE&+b-?^lC~8~?t*>+ri#zt8SKQlLMt3%DtW&oAqO ztPp)OS)X2#sabg$EAkLDLs@;bKAkV$rpxJOovbJD?`q^LpqD9J%>$2dR^@){!dn$2XoMX!nu_Bc)Uu{WRTYT&8Z4EJ<14MVicizlFL>Wr$*=Oj$ zDiG0q;O{4QfKjbeliS+pV5uBs2Z~@Cdkd?M@NnTWnZjNZ?AC+ZcdQWA3QhrtSZ-Ay zUN4WxzVooK$WIe%xsZc!zNPanUFRSfsM}`X-e>mgVm^|AIKHeP!L?c24tzKl8Vnf4 zE~2s^UxTa=T?8_UKMNWB=DI+r=ZuJqJzNEOm=v~fCt4ULs~!X^e|2SfTzevMhg`|a+`#B7KbO*83!Zzvuw+r!kNy$;hhsh_TV?0a~_jV$hDWe zfk^wkafo9NjXZ!y&e4|KrsO-fluuuap zuUZew$h?sOhGRJCyL1HyS*rogfCDqDBmZLFzWNTgjU2PGEm=Sr9_=PQ`Dt1$q*rYN8Aq~k+<4rN=cK5Xqmi6`FnX&o|i zB-RMB1AnOPlqhWK7lT1lH_sS>fWCR!q6j}@p3K8igVbndu7(x$_i9SDpn#9?&b@jS zYr(^q*q(CTG$?fyB2HbwPreE-6J9~SOEs36ysFm7s~y5Snh;*NBB5ouU}r%AAGPTh zkj{QhZ~sU~4F(*~oaSSE4);Nj8<=jA6+AWt_cJ}JL8a63)dSKtNVPbC>gNrag9YoK z!0jwths#FRak%VAy{j~l-6tyWwXzKAKia*fX3^h@_iz<{1dp~ms{=G@9K z#BEdV@{o3mRcu(R;zbu*6TQ2)>`hFC*1R)7#ZA#L_YY`dC5$IqUOXxy2Nbo5ixv%+ z6^@eq`3N$bX|S*!H(cJWE|bMVxmb$8!?^~2f1Iw9$-T%Sm5;RA8`TqjdkTw?uxDaC z7Z+6#c(lPeY-Q2<#jmOMFH_?SAoOE=H9w!hwZz;+u7Sz~?zh(<6lo}}%?gUr_%6N6 zV2~X%-_lO7P3P6Z)6Md|=xF2p6XJn&tn;&TRtyCQo`!0(42hc30ZyiYAO-Fx^D~iY zAi91wRcm6!ZA_--q+EflOalVDpSvH6Mng+GvuexX6|Y^2EWC*rDuA;LBl{k;^dR7} zIQ1=!0@li_rh|uESFMGSE@~z(rx5149bYhW3GjeC3R{w@N$K-!(8r~ANKVwyh^@@X zXKD757l>i2PkY5`^Vy6oP^Br8MMx!K=R!9r`=yie+5%N2JP} zfv_wwfDuhZO++yb0kc`PIHxeAHFYv|#I3oB+O5#5&%1z5kW|43ANn*kYm|CTfHiTT z!(?0>a~-rCfx$>hYhnvL=(5X1Y8kH!@S(#r_0gJ7fE!oWL=)8O^785+m;)Uq)4@q6 zd;*`o!nBHRzUc~0EQurYge}md>&L0{t;;HyE~~t@v9977|GaA|tr46f(qN)bJL0{n zAh$qMj`;WpHN=}9Fsu*{7IZZcuRRvn0yWj-^H4liD0CYH=ZG}eW8e9<7?YSUpSCn- zM|9hGd=$mK!^7zO?CAR9&E-{evRuxW2fGJ*li9)E7=G|gKiEt~8=ge3udm=9sjF8J zR9z1Gupu8Vm%aP|e_S8zt>^QDz56-bg8S?s9**XVb!6^IU_=dvu-o_WGZy>f!_CjH zX)+=Y$;$}zP&ZhDY9R@%Y4hHI1e-b~q*)#i>fYPi&|AkcfR&b%#sh^1AK|0v7@AR9 zXiIovhsQ2&#!DkZL;OJ;Q9Cbv4xQ*!BhwGP>I zMB(SCaG<4JhNLl}jr&ASCrjA%`Y>a46{*;V`ic_KMBU!t-mJa+ST7TJAeWC0jc?2> zZTNN-i$gu_EI_~l7A6uWwM$X6J?iG_pEzigmZSwXe1ubnElAUXTxAUR>!UPqlu7W$(xS*&4vAt*!tL{ z%5#ASFV$}ObImsEXKqV&>nB2FYWT=}Tzyhw!KRHxOu}{9uLT`^#0OeM!jQUR8cPpX zi?hd^40-0-*~QTZAJP-k4IPO*y{w~w&s}}AbIN8ft=lSv^7jZU5oU_*&6!hmXlk3@BY= zJHSL9P1eBDQhL~7iwh&}G5fKJR(MWLk=~AJXt~ysMoa) z7$WB6{?RNtZ=+=O2*ky$>|uyYvtjStqVe-J^^}`6hG^7-kFtfO>XxQqYEyJ|S)9g@ zM|8KaqWn0bT8`FuO7bWeCRVaVhBa%-!E5HsLHD$12t4~;GKbzn^KLb3BbXf)?@>3SrK)_81U>__I9QUqSE+M#tJU3 z@$4~KRDrvGqzzRRg(l(l!idXq3W!AGggew5*pdb&A{X%3T$mpN?KAPcna=4xwnobj z0yzv7KrV>8-V^b4bD@irP|rrdS~#(QhmXXG^*75oyi+aCuGO&mEWO6UOGBs4>gU2D zHeD-KII#d{Iw1qpW%7eZ+-CLhzHSErEAq z2>-5jE^@Wf;0XU=GJ2rXbPx(DrIqGD6MLI;A>BJP~m?Jg1Le z#1ni4#L@zw;WUa*b+8$n(IGFgje7B@@93jI70n0nXxG~uuSGl9fy>%Yrjs?iyLR$p zaX-18tUpBfwmADxP%nPLGcxB`IiQ!pv+H>30)~L`O%I22^F3^f|sj^TWqv2K2>l*5^}l zze&A4nP1cP!~1)0V~M1P{}C%8#1-p^{ch4bnTtQ9N-ahaxZ{{tRknQxDK&}m($id} zz2RpIQq(wA^G)_UmE`Sxnk;#>;i1_fNkz(Yph>7*TcDhfwOBcXR~pxE;U3DP1YU)G zH~)S;9>Z;^F{t@fI)j_szFnltS$Yr0XZvT-_~tH!aXT7-Pv<>nBgV@nq$Pk01Es$V zpB;oTqdc#eB=n+>TG^LQW6(ppNpTT%lXuiwnN^) zX9lBG$t&aIbewGN*D?ESzKr2su*rIbfmnPNfwR@LgpsMpzsG3{j^@o_8$>YJ)yD$Q zX}2dx%`U0s!$(^7PCu4)PA!aUTBT)0H0xDqCF+^)NveR_ZM6RPDpwR=SuC@(N4a?9*h+^766qftovD=4vzv&j{S!UX* zOHWLc+qx-}R4JbH$9AVEhr2WH_%S!^nEtR>R9=0Wd;iG;*%n+;6Svi#D!QYF;7Ol! zTGU)<)x>SpC%Wf&@|2=2?$fAnaK(B$-#5s2{8kdErmsJ_d$@!P>z#+cr_1@<`DBJ$ zHQuGaZj#x0^4IiwvtDf0c&Q%z|Kto*b3DJAufI;?Q79C@;Kl`bO6d$U#4xD+ zMxYL(`oRuS49$RTKnO@}DJe`6+__ty3HacZ;oAqVUWUsm+({tS1);ETaXuT*qjwt^?5C-+Er^~c!j%oufR=n~3pT~Eq2LOnU)@H+poNMh3|g>F1b{1I3xVKj zWVVp*Ie8y`F473MRVAzGKfuOESjyvCU;Oo5I);3r#C8Q5Z}=|6bDZ%_rW?FnqXaP9 zp2qs1;Ur4#YOD=zLtx3P^OM=1Lx)O<{I#+dNn*4}YP)h+9?85F2x-qud?Hw2>w#Bw) zf>assk##RdU99_2(uk$9|L_*-;hgpbW^$Fd(8Sbc=~Zwu|5nR5NLFm7H38EHet`Oy zsx*EkXD$#fPpHrz7g*edL3BL__~IXQZy9I`^b>S1i*;?ApS3LrVG!ZnE$no#AgLOC zn$fB2Glgi>p=fl%ZQ?1-fLuopAOZS>8PajOP9{8qk=@wGGJj^rj%16|Wiq7`Z$~5h zX(M#lW&WPt*ORq5;fMW-z|`?%x&8^M{fgvw4kGCBBxwDX9%UN*0R4SDpRMD6p(jX;$pw0>++X(IoUub?8*+Y;2>95b*%*H9&Vs zW|7>iDtiqDPTc0(0?t_0WR3LD*|&u-^P1o?87n5ZxScN2zkIL9{5aX^a#|C?3dG7O z6w{R+mcA{%)~~iHfZW`yYL+TO{a&+N!NvaigU47k* z>k0)d79hyF7&k3;wY;xO0CY6{F<(}X;>^7W|9`#xQ7)qBHDgSE|2mq~2dX}*R<%)e zuyfF(hjZvQw|b7+!jz6ZqeM5r4o@&??ZY4Vh5jg{vK2PIBAe6MyishWY?&rH<-<+h zIJs<{s(!Sy1H{Bvn58iF#aBStnd^rMRk`$1>W8yMA@s?A60oszPIlYb`F&^S6S#(B zelA;j41nPNxR|83>G#PhjmZ|v<>q!BucJO5L!lI70OG%s4SW)~+6?LGlgl@$P@J7K zarl4}%VYwwK?}kLGKxQJpu|pY^r{ElnZ|;6unY_oV-~V+v0SHi)o1-axB5KD*xH1yQfTz&;0wXG;s`-f7>gZK z5UNtR9QeWz=3%TD25p+7AP3|>zlGQ%1VkmxSi%B;g%Ue>PEzZm5^=!JkKbhEY+Y8F z5nK#!xz-_C($Nl03y0$Ob&SN|ngYC!5E9WuOG*foA#p9>4pswJ>0dVK>=t(FUl7ah zMOQ zIE4YZAr%MZfmH;Uk^JS`j6;2w0ucC)_?;Tg7f7t_#^0m$X-MY|H$UUaIG)Yd@H0xu zl2f%zm?Vk$XRl@AYt5?dYXPzol-R*}w6p3}(suwy99)^eox>yvCV4XmaUxV2wlEgLT0Q3;ABUi~ZseS) z&yhp>dd0lCA-#fV9V+0Vo6r4xN-*2fQHe(U_!K+~)!oaOf6Z2^c!DqjVrS(|mVhKK zKz^%m`$vwEDb7aOpe`rI&G;2u?u2WW@f<=Qcs}gN6Cv}0UI7S(e?D8KuwtC7Ylw=C zSe*3FhwhXZY-_)`A;?Oo-r1jhIFi|jrYZ+}HeKX42}cZ{ta2+UKqssc2JiY$ZkKTn z$U@75VZ5=DF*`Pmu8HfJSDc_PL0Ns^qaMqB1^tkpl;wZlW~?nOlqYXCQlOqu4d@hC&U45Tzy#njMM4jK7nGOU0|du&ed$U!b$;j#6fAoi*w;lhH$%x zxlO|MoyaOj%UNAG?goBiZ4IA)Qm!Xepu=Y4)qGs!F~uAOQ*i@A?c~K5bZPiMgu$9; z?>(HdBIJIL)rvu8jH~iNmu3RwPeOWmNQ6EcaUzZQ29FdAxvVDOdTG3d)0Hdyv{ELc za6#2Vc-b0a!ultn7qL;GL2Ki-K9v=V}G#Jy5 z<|{We?H3tOTNN9!FS09jO_#ZtG23S_W?MDBkLSzp$#TT=^4c?gC9C~X@R`RPdsXHx z*=dEsR(TILX)h1Sm{%y&wQlVuldThEOhlxtoRvvFU9$TC!aI}`8xF5^J~hF2W7nKR z^dKr)FtZC-VT5afvy7mK>?rDLj1g^oU$2P(eJXcOsH$@&^>i zn4U>d-hGIMy9Zg@rhTM65HF)!#nmG;aK4f34%}jlTCL~Hv<3-D69O)&wAM;F|EXqA ztVUV@@gJ-Cs!>S&9S(4j8i_Ct35?CodIvk&wP^Y?s7K)ZXvy5^F(A-)Ne`f77p4eA zCWNepDhYTk3TL(H+PBn6w?s>}#S4YEH7$MAs)mbqzNX}r?K&TUAR}{;u<d0Qf%b zEDefa#w5<#MkUYPD6my-GdZeBrZuc^@yzdV363_5sSRH82;Zvnaq*iwF@EVZb;Ad? zuL2icZ9BQp9=MrHgJMJS5m!OWgCE~)RC1xcPY-V&b8b3f?jZW|rA3pc)-?nMR+*cV zPHAX}g=-A-WF!%^J%pS^CH%C*LKU>NY`(gx1yv7C)$j7w@>rt4hgw_UC-?mDV|+ba zrN=c!xIyIU#GP&Yq}dD1!au{L0Ru&hrN(4QZjgQqtFz}~Q*G8-*u|2tkAv(Mp+|S& z*e)}ghri327J^}1qfSuUArY8gqN&K^lN`h2ax<)=HzqthH91Pw3AeUK7NT$n=T^~l z1&&3oM=eo$16>Oe9Z!}ke5jEO47lkz9mQlK1D@gEc|%5PGy)t_)SmE}cGp17(MhR_ zY|#Ya!A2ND@lR3CpmPepkQCc}l62%ah@fbc$yk##@r_(_43YGswM^9}x5DI;wDI5* zEl)Kc>=bE3pv7>_D!*DV0>#sh>Hc++?&F}TcTozrbP(sDoJe4;(XcOGcjl7{3l?{$ zv!GsFxEz(gmy<#u!^4*(GJ`Isp9f)R&n(Kko<5|(eJyC+ePzwnVF9@IP@uaFxz>G^ zLirPKFwSHZxdQ;95Ho^5aX#`VV_$^0C$kwh zDw?@gfd!K{%lR75Tkz4o>Fh?Nl8r8;T_!)a%MpLwv*u+B>iq15()e;BoRz!}0&$Y! zYwGrvt#I}3QTo-p`JAkfJ*r^w?lA@nmcU{NY(AG2;jf`>1oUcUeCH-Q=pP*5;hqww zf}-cY9-bUNJGe>jWXO5w`2p^8@4#KNbliofEWzF?J)=rq!6m2dMyX=Mf65zZ1`%ji z42L!DX`=q533!V^;&Y>1rF6{%N2n*?E^%QjiRE;all@73dpvm(Y|Xa_4jjkf>D26g z{vDnYqSw?ulF4jm1}pnv8Y=f`@`F6`fK#mw-Fp^aY;TJN?s-+k@Hm1gY^J}!Wm343 zpuQ|+F~@hkmhpNHFvUNX$z;7^C9aZZp_DkxC#)qQVVHq?CaHD9_QY|zzWut!69bhp zHTq~gnhHLNg5L7D%;X!`NmBh)W<1zQ13gGv$_$}zszav$F*2|-q*Tk_7I1HG<=3w% z>HOL68X+=z6=^JPmh)tEo1oQKv`d%37j-=fuZyLgN8xptb-XXmAd;!nw*`c#+5P;k zCgJva6l^WemaDAeyS?;Yl|RqcB~d(yKE}VN%Xz&1n$DmSx|}4V$t{1~m)};2PnF|o zRR3OU0y7631Z;DwM;P2c*%ldP-vZn&^D(}j!G%e76@{pVUG|ux92iv(cKB6zAz+ns zfm~jdt4eYgm%K_nd-WD32(@0lrQCv*3)^}Go-Q9^#(}S?%3H89 z3y;%s8d{Mg+wG6$GdNDaxs^-exP$HSJa_z=aCx%Ci&q~>q{Ihf8{MOG@?*VB6pPr} z*x!($<7AKQ_GmYH(Xtk~hvAV2tVXT#!({Y`HkX1cV}!cBPm|?i2~fC1M!AApua6SA zbu;$O!+YSL4<;uXg06t89EOyq_4tK|&=GF^d_1F2XVyVG6Wz&WR!=&v|D{4Qhwjb@ zzFg;c|Le*6-Db9)OjB_kDtLcA>sepOQXCf0u};^_>J!D0L2RGI%5Hq$xjC=LH zD)H(%+BdnLS`%Cg)gYABkMA`tDKhv}1GFfg^%zTKzBZ4^gH^+#?<4%82IqZB^w}?;=pkW7haa0cs6e8coAvRi64h&kf zR7cagHd-*kn+rV}2UH6*^n~p~IHKzbm_&a{Z}CwAUhR3WK0M8u!7dPsd5fd+YZycYzIdV5;*hte;XnIPT;vzJ&P{Jd5Fakaxk7%&x(U05$Fqh3j zz@2K~G$DAvdHsi|1gFko$&oWx@Gk2_?AJ?zWqiVJ@QD)*4!fvrMcXW^ByiH zUZqF#`^|KwO$j2lV*%Friclh(2T=lVCWJsIhr5sHVUh#c$BTmWInDGr)69k4?!^s- z^f{w_j?>v{vi=$6%Ia`7i1G8_z5D#`BsD6V>OP9$MVB-65Wb0`j zDjvUJ3X-pPi0drEV{FNG^{AhBJ*9V-tJm0mq627^@I=f6?8qAx_)sNpoe*~u=aJxH zOtaKH5_n)zNN`?=s<{QDJrBNQQOh^7<8fgE80C2cNCv$GoPu$Z6-Km!z{`-Th@h-} z1d>)oI6j@(%^b3bH8n_@!{+h`hFya0dpH~b0~C+$RYnS8hl+uOf)1dX zgQn8HUM{{SGu#nU8|8FUMlrxabW^}=h4@=;noGuVVyt=>4=&N=CV|I&MXnK$`yrm& zWRcur>L)sS0vldQOiz1dyLl*--PnT#`EtN6kh&2M&^zVeflkCo^iDa#fKDXSbe!F0 z!z0cq@RT*n%^i)|c)}agq&$W@k|Eyjjz_!df6v(G&v(Z?^*`K-eQ*p3!CO%2;bpR3 zPJTp3@`>i1-Y^1%t-iuTD{!0sdWn-`M_WD^VNk)+fukAPf`at7baMA~O);O&m#<(` z79`APnEUc~qMbwp^tBNc&@Gh6K?NdZ-$F#nZU&L*=r~=hzy3DCndj^qQ|3Ah zU2P=h_X?!&@J&>nMzwPhD%F0JPL$dyj8bot^;evH$tp6wkpA2A`2yB8;jH8cYhT}B zf=16^SY~a!7-mA0j#U%HL0KB{MIl;-GH4u>rBOnRhok%WTe_T#f5x)h7!Q}ri~ z#^M<=aP(cOwNHK6B*qJaSTlxP!QBN4j?Nk(ge=ZnK~naLld=c6%8`Ph)WabWMPZ?^ z(xRyWFRrp&Gh8_R+swW%lf@z(c@Z&9lX%WFjRRtUvJ0bPJQYNR%A~01*n_@-9KB0_ z-9V`ueE)j0UTm;5(}Vw?;Anq5znZVVPG)z}$%%+!;ui=bszV{KV7DZGsAZlWDIdvk zlMYcU7!U~w00jnRY&1N9AAo`bY;mf3P=5r8e>}v}6_9)iTV3bNRRdgWVZ_cg1`Hqu zSmN12hU-CRxbAmS*FLK1(6tY~%+9v0Yadm0a6ODVscbJTZ_rA;$9V={t|C z%KFYj(2{%SkyQ)dz5pWm-s-@OeEZ1SBHvr}@2&9djUX*5+)Gz28uwz5oNry_Ub^x) zUnhU0qSWD{>F9aQ_t&)moSVw_WcI4N+CRjd*WW4|jxAf*zH`8v!|-*W)q#0(WdzQajllbVxQZhk*@azcAQ92^51H zZUV*OHX(-srl1gv*RVlVjgA0x)Vkd1!9Wf}IgKu0BevVaM(H@&+{1J4>3s^%Z@~Cr z?@f{kJNRO|q@7gC`_k-a2nEQ+$JyjBn-rHO#n5P_8LFKD{x*+E{L6rI3ZH$pSP}SYLyuH|a-C5PCx$)=uy2 zHFHHqrf4mG(Mb@9j+zrRPa3n) zM;{}|fh!b%D>vqOc1;$57fL7J>?I6Ls)Lbq`4e||G#ylP>3~yH%P7|};tOgIWE(bY zwSSMd80YpPDJ|&&2~&YCw>PlKEWQIng%#{6ECHsf3I6tcO193#^95`R0851l^Cd28 zTIuOWmuRe%e71eabUEfgLyWPPazfEmh;?~BH4K;nu#}*A>mVEO3Rpj-@DI2jH@%N8 z$Y1BP@my$UAj~~%)PM(OC@?re09Hj-O5rN-yf{VdYDT&gXbmRKJzmv zxD~(X!w9h}W%L1d;iA$TH0*1>y@l?Zp1)sAN7L)s`D}5x8IRK?UC1jx2?oH54B*rUFR3u1 z7$p`Ofn1Uzu@-uJg>pXUJi_+i>tuQV979JV{N$BZx2l1V3DZN;KoCEj(m>E0`yr_y zjG^x6AhaIwU~~}7P`7jtx|bXcA~vRjp>*K;~7P%quhOSIl9=&m@uWVa^Iyh=hx<(WrztMc?YPKV{`BfC|3dOfGZ0`=0} zyhIxtvPivzw9_^9z}-`{-`o{CpDiO(wc5 zQ6JsyO4K`(=(0$?gtsWuygEW>WtvBL%QE%3P^V?;BfMpqdVQ$VLiJK!hg9WKgKgAr z^l}uVcpxcxxCIc&LosRkrWWPwQ&Q5LG<{UJX`9}8MR&#NCA&3w=2a3pD$hK!Ta~BJ zaXKtdAK9(S)9X1M7O0o*<|W!_0i^l>wCw_jq}Voi`=;x@LPA^S*M$N|d5QWa6J3_5 zkM8Cj+GnHQnM9XG>Lt8Inda3IIxEvW!dsT9&xJZIQy<|i%hc;boffK>^3qcMb(75G zQ?2kkkvOty8~W?dlic{r&epe*EV;?HiOn|})s!*02`Vec*3u?7D@A4Ni$eu`-Az!F zvh@+aC3pAPtv4KXUbtTB+m>!#TGDOl=271T>H2)D`_lDM-v#M_>q*u%}X&Kyg**0!aGBXKFpN^05jH(h)`(#fQmV3gB`uR6>(Sr@qwrX%_YVylb( z;VmvdV17x-0z*X?TqKyUPHH5`yC0xNg81sBM#9|t z0jea7vF+%j`28qd5O{H_+mxoy1-mOvAJuJ2)9XFm6{nZ%*5sKtW_MJcd1SXLPoLv- zSe`zzTa~BRb2=cwEnfKS|=G>S*Eg5Y%6kLrT=*T-Yhzidr@SvpHs8&#=M6WmN#fD!z z?Wv};DNWyyTFySVAx$6EZA#NSo_1H9Ub0)0XI=)~QF-Q(-Kso&j?-az`p9loo?g%C zut2?ZH!smf6aVT1(6$qQNwJyuYH@(cEyLt2TNz}Jrkmi_}Yai!vn{hUY%i&dM~8@RnujbD>Vl)JJ&BGWGgUr-kaJJV&bigPoHf>Fs8n zUZl%edLOZIeR!Fymy;ju?=SoHWH^jI|NiXYVC$s%a(P=tvPDw8S#pbJkLGHD<{lp>{>08yHhjwq<10O1!)BfhI%6?O`b4T_Ls7Wb5+ z3rF+Wx9i#YY;m|5kJBX_@`c9s5x3`ANi3P?HB7P3!n#SZH&k?x#_tj zH^!Eth`5_)#TZyE$!n9L!&5g0;<<#65TaBjXl;=*6YIzd4Ule~C##lcE&Pd(l7+apC#>Uw4B()W$otkTOj4dgrM#tC+G}Wv; z_l45tE)E=M8}t5RWH=3}_7G4XUG>I+`sixao}7xTKO)p5vHHV8jfB;#OXcc_w(kiJ zOwb#o*p~uU*bmbn#oo-mgCrB5_beE7l;j6y7%1gVSjILMo(p-Gogkljii9(k6(2=$@9;2sdwzNG>U?+@9qc8u(ZSx~!SlQ2e6v^`>_H}Sb+8w~eGvaY zd++iaNs{J?adu|Z(><-(-Rais&Wx~HNxbN?3X+-3EMYM8h+r_u%nXv1S(%kt7nd{W z!Hh1(VID&_NJ%(CV-Pbka%C_u0eE%-zh~ z{AteWuB;>@!_7YXeNX%B;bye(ICyb=F}j$XUtr;FeDZX)PEMXCpWxFjTE=O~X>j@K z#p|<|SHVz2jSfxUK5RDYZ4%#(uivFF!+jivm*fD7{W%DF5Xe`JOs8qFuP3A7`SoNt z9SkRoOC7!FMj_sC!(zY9wfYKWcb>T*?a_Pd2 z%3$eSNu=i%ZdB>8cJ5O=+>zrJ9eN~fvjr)0({0)EsixbqCb^9^MqJF2lr`R1XJSOo zT9M;2UXItBFOb2JNicrL93*sHSm_6D=jWCTaB#9eUf;$q;?*eH#ld8JE*9p<64^B* zk0VST7puFoXqK$EJIEhN_L#3LKseNW1n0aBCquSrfgL}`{-rsM4z3VA2u$Titom38 z7d{>Yke-uV^c__R%uVcgkkI*IIwc{^ zFtyLO!|aw%X*j^EDbYQT(ESV6>QFtG ztQC6`qAk1&u$j}SoiYLiOM5Tq*v|O#EK=K0q>`m}BvKo0&my&sL;5UJD{!AeEt-hv ztx$`A`xdIri+U|o8*tx3wYpKSrD{dqiCEj5L20CC>Usu60i7M{aY z`I5^}--f&sL$rz2Hu&)8-!5XcA@?p;>&TXk3btq(qQ`p4Iyx*3;S_9N0kgyeh4@6v)xAa}aMfiInVwQo`4Qt@;X)yxi3SV?9-hZNjwWvSutx$`A`xdIri+U|o8*tx3 zwYpKSrD{dqiCEiM3Tt%u-qk9Y7PM~G3YQNwlRM-zP?3#DC+}iRvQD=3U^p%>kF~^H zC+2Ki+bL9nzc?YwzM26wBwBH$Y1ZyLdiOx`L4l>}f}}hdE5z!0Zl`CF+Pd4SmfD3# zZMZ#))Y{kfS*BLtK80G8Z1+~EMZkRv)#gRL7OD-nZ=qVr} z2?De(TQVl11^00HGsMC)v;cW0hG-M3ZPQ4<#cD(DU98rjYro}ch3;3dBn4^e29ZX` zO@(ow2)cj4+We`<7K?(lLH93Mt3&l%vR3SN(Z)Z&e2F|uh<@EJrIP{Cynq2n~a?#ker;s0+ zVW5mm+R!yVs#gs<9bJH zeN?r{lybeKeR9Hl=8k83-F zp<^1cETfO{diD8gb+Ouv9_I6S8<6ej`aazj3+iQ=Ed#e%U$22%V@rRL7sa4{BY&LM zQW(=Z7JB&*QYoD%9Xa&4Y&!Re%NhW@0W0!vufTe#%0lmKUu3;WJZwvLwzJII7m*E_ zZ64k)GV46HM`$EUxUXorjb}^6d!s%PT2B>9BSp(%Gar-4*Cq7bq-_Mygr!x*)ppFU zimUbRyAm+sn3mU$y=juTWnY1L)@6s?x810WLA78zQ2SP8L)Lb*uP(FJyYGt7NR;r} z4;l4I;zWO;wX0F(W~S`KUSscNl+VU$;A*w|u7RuFkY5#8>)m%9V7*i&O3bq^yXw9R zb!QB!ZFk?bkhR)<*Fxqw*c;W^diPyjYOQzQ^`Nz`QFB7(xBIS&tL>Oy6<6yazY?%6 zy6?)sda24zx^KJ4I_$pfK#cLegYLUJvbMYL>c~6?d!stL=)NmTt&8rv2DElGYCa!b zHw14xT$Dp&G_tm0Msx652^`JA>w~QswcA<$)FN=}D5Jl~Np!HRwe=hMi1gwj& zTNzj{RoO|{wTrC7uxkfmj9hKa)JPpdb!4qf(lmn1bFeq6lSGL&HZawf8ba%-LTRKZ zDYjqyhyPUl@8Ie2;H%>&&yT)RzaOxF9zFjG3gAEf<6r#SdwYBT`VW4&_kSGza_>KS z_~qU&{?C81_x|7gclQ4I=5X(C|F^@v|KjVv+55l#lasyw_1_!r{fj@D?ES_6KH2-H zKfm5v{dXVt{_B4k?)~d$zuEilpM2i?XaD|xy7#~Q=l|v2|M#!{+r7Vi|KIQZKmX1D zu=mgZ-~VavKRfuJ_kQso{ENMR@z4I{-mm}F|GoEL{q>*z!T3E{5kynOZX1I`4#*7=X<|^KeWs*_kOweC-4XVMa%pO%Bg>TNhlx{P7!H|KfETb z|Fg6;Y~>eF?vJIiv=)}(fBx|v|ASb52`GM*e)IFa`hR{2E&T)dgRRJ)1x}8e;>DjR z!+!))p+Cj<;uEiflKU0VxuKVn7{yhKt zul9a7{``V?FIonA9_k)tLG_D2(MsvJzlMLI7vod(cv_C@j@uahO{s7?wCrC%toXMW zW%^D2Cv6u?;fASSi-Qi1CN;l>er&hnGRy{=|P{q@cg4HpDl-CeN_Xq5Te0pcx0%%&$|w6||`*i&pKQ zXp6LcxpjewwoG#zUc&yH+Y2JWsOkRaccG_2+3D=+7m8I;Yx^^hA6gZ+DLHzhRFM|9 zCN5?A9sWyb9(y!lL9}F%W<+hI$f@&Q4*!kc|B6Hjj1>6iFUcSI8O`c2vlHL`C96s6 zQ#uqi_XAFwmf+<$2AUsZ+y4OC)5%ixQ6Wq8OTibXSMK-UPi29b86y2F`1x1By(}e+ z6hDK%`0Qs|j3#|^UAyCKMglr}Xoj*r3jD6qR;><43Fsw%2!Hu!^e3Lj{XF$wZe_d& z5Z)oY<=4n^7JQ!hDt@qobykuUbcOTlm6tj^>K;D zRNk5_4gMXrM|r_d^jm>}f0A3IpJjC7Gd>eNzMw`&qBx>M`Ebl=CqIW8w6_u23*w;~ z5&bmP1-C4U3v1#Z+Vj(q0=6M%KRmMj8E~opUjMCHT9lzJ|5B3FUoxzeZ^TV?!AC~) zH{}{Rqf+6R18=B3K~7XCv&>06M$rJw&i^<3^3a;rGM zgnxeKVE1|2_>JIB|GezCh}Yy-dRjtjJby#|@z{XeqR;R+igwPQVNERia|8D(4n3!3 zz7_ag@;yQ){ER&g_P_W9GXwgGwtzqX z0=~(Ai{H@iWiELw`8)oZ&XQv8gE1WEUw+2^;^#ktXpP_SKPM;qCxiX{_i(IxI9@L| z+jzH&Z^KQr*y73V(f$2;7A5O#Fb;OfZGhi}@8cPK6Ml~FgLm-Hw~OWGJ`R&Fn|Q~I z!O^k%g8{9A$D0T5!tdhkI!r#rE2y|#M7N7s7;W!%VGnk4K=b`!dnA)*HDH5lFT& z#9JiC2UIw%$i}I~TJjl!W^p4b#5w8__yzu@R7s4&e-3ykBG1sV(0@pX(Udk9&eD}j zav;P_xm%_P_V?i0Mqh&I{s5vW#8V~=0GN&1VEwo8JbJiK!rS;h zh8sJKwW*G^H1%siFM!vLLV20~QRgQQ%R96wA z4pdgA|L96>l`*xdy7H)W>I3&vx<>dn5i9S+6!#1-vF9}XRXSyorV5CH9M(P{|4Dw2 zghwgiJfSQmUiwhb7@ez(sNFU^Wdjpd74qgWR(KQ7!`b>_m4xe?-(dfG9nHRvZ%^Y* z^6A^fF2)1q*?n01gXigy^a>z?oDgiFjnQjyRYzefkRjZ|Fb{F(MhCQV!DvmNbrjIV zD4oq!;5#wGtd_QewP1K|B`u}GFbtUJLpH3-=N(ro7!vD8yI@RK6)=x0P=c@;D>34i z1*9~ZDW=(yJ~h1weAU*_RMT4D^(w++v%Z?8+n9a@Vd_bJ98Nalzk*Yk(;|6lf#V4u zRw5yryK^sZ3T(GjUf!X6MrECd7S(ZPoyh8Jj^AGJ`M3D47WiU)K=Q=Fp^E zmgP#>7P11%b@==H-%*)WnKUGgzS}I~S^NW}J&+TvcFFc(mXKLpfXwGw5LT{f5Ka0u zVrUqvCboBtSDkRu3n6RnvX}}bqGqquz+?h?1CdKunh@cv$VfyaKDQz`O50gSY!$<}wP z#os-|;U?MADNIvYK;}9Lo5iOgDJ&1+$*5T70nTG_0j;8FgnK}o73qe9u?JaZ5Gg`t2MNo5;l?9jTG&OV! zrEuVc7AfnfJCH9$U)zRDBc)r>x8&4RrM3+1xiwhSV$JDHoyAd-hvdHkdoD5OX=;g1 znI732GR&(&{$h(o(6bWLO=nm$7+3(wCc-#a&IYM$8B(>N?Q}Fjw>uQY#!M3|$t#4S z&EYbPV${=<=VAw}Uf1(jpU}6pMKWvRO*Xl;j#iOCGWW=r7TecWk&N2bm~eK>$d-mB z`Dd$Z_mwe8;j<0RmB*#io;-FVK_By92$t5hBQf;jvyXg(hO#@66)QKT$3-Or6I}~L z2>Ag)-0Uo9Vf%ELkn3<{djynXFac7BR2%1X-hyB&&-+1^m6A0zPO|^B6l(GSXM`84 zB)*Hc;Vy>J^lAqC6JL@3$oTNBG(=CynHnWJNK4U1OLMFVJZx^LZ(IwvqZ%*Ixu9SP za>d*WGN3KlI-1e@w@;Q4O#J*9t~W5f%+Kn?T%Y~L#dQ3*zyA%G27U|Qyc@v|=I>#; z2sWOQU2lxv|wAE(d2i#aqP{WsXv4L-eZ;Lp4{BqGKx$Qq(O4QdG-;EZ)y(#8_Na2eG)8BSLvMWND0YZn!gAR)&H>FZ-b0|~@344k{%a8lx0@E6OQ(cc|3_xg z)4Yp55^|$;a5Tf~iIL66p{6c*oYu#L`z+ce*HQ8b({^#)#mC{v@Y^TrO|n=nehcxB zB=`yGVYR6?=ORl{3{M)OcvuzVVcOAVFdkOLC@7x7JZ8$IKs1%2wzAtd{GjiT33fqF2NDLD2Y(E;q#IR)w()_ zatV&`+zsIy;b*d~P#Z}f>U=DH7zb)i42axS%!PzI8Ok6dxVyg`6j@nRNh$dPCe)e+ zCNhPzOvq(86Xl@D`lNbvWkRiKU?NjU%Y zP#Dbhoep5UOQ!C((LLSk3;i>kEkJT?o~&r;jes7E4_W#}>5VGWgUl@);ubv$B1~UG zC9|A#`u~7$zoPK%RUmh($eNArA?5dd2`HHTv;pO)Jd~rBpgb)Pco)gPgHK z1%s`cD%dRGpggY!$ZM6hs#;r>wyK&_mA0zd3vY{_U=zbgY=pP zlc+%g84FSn06&DUGE_pCV0?h8Um#+VLvW(|#oa2tEh0?q;3J;pAc&40zJ!Z8oZ6Dm zlUra`7NSs<^;ldC!h;HMTW$EwGJ*v%$`$-uTEd_x=o+7N01Py0es}YbY}HMV0jdfR zU$9C~tsoMwN?1cHz)o&MPZiB5blT94(g3vz zvL~WJ4QZNyuRg%K7yK%zC6!h+O<^J|+rRcLL`Od=przNCs05-0zZAPh%*HC}SQh5% zYtE^e651=GCu@+nJq`%;Z;{btjz=*lr;BJ*pkjjyq<%cA%-~G3kcj#*#YWs~fo{UN zXdpJ4MaiX> ztjy{_x>PF$?Lws%6Jq7I6cbXl8cax3T$m_c+tu;lP^*{)2O@Pikk@_7@SxPI&V@oI z<>NGFVUCS^#o!C3!=O76gV56$R+`u8)SuD?vNKc`Fa&eWWT#s=AKU!>oFi4uO0(}jvE6~c_#*neai`dF&PEEL-AVZWyh(7 zUnbQ_d@weef`Dzlg5^hXt)>ha=eq)jolN3dBAr9+sPj)hywP#dX;axLxPnJ?>8qf_LHga{|lV0k3_rT&}kXtbD^X;rsA{ zy+9Pk+b!G*1GmM*ckAsJ)*-MFJ%({($w5H(%ZC!-5HA{WrH*?9KMddp{$?y*tadn; z8727o67(8Xmeqk=bN!82l){eqF|E>*w-RW-p6;|FEa06p{5^Zp?KgNm42|*!u)C~L zEmYnez>Wn9E7=vWph*O(*_NKx2Uy(TxkM@ZQpf?K#&?1!4X~)7-wo2To#}wcxkN2n z(-4QRaJxf^@S_sJ_q}Nf?`U(bxreLWxza0Pdj%x*_9rT}zk-LoU*mQt7_j8Fa)$|; zFBSKk8gQ}4B0@Qv{Dix^OPDw`06661`dzJ&%7+S2+PucTMqjHK0aHkN>q|oxba^go z$5xzxQDkkjWGfbD2cbs2v@3^nZ6Nhm0+Mu3r^_j6-9JP zc!PnI-V4(3-D6kX|+FvkLtdXp5Dy2=My zGEKpVYj^Byo<*cfnP?F)vzgk@$`L!6A<{#SLr*YsKK2Yxl}R^1U*N%|xXUqFr?-%> zU3GXAOG@32Nap z!(}SX*Eetj>@6;~dVlzL`F*X!9b!t6&Zit_6I&Tx}w52z}}>9zlHNP95MVi_tEm^HVO}pp3jbMpUvRc^QY0x&GRR-XY<*!n|b{7 z*^_5C&;B+bJL0MxaZQ-U7l3h;32qj?D%UO@{+Xa3AJ3mYn}2omWPbSk_V{4%M+_Cr42 zJbiXN`|8c0BOFTHXT&#%sQFMG9-5jGW4W2!JK0i8mIykz$d4BZ#IT+Pfgaverc5KSId3GUQ z_!c%v(}RTOy>tf(7-=k)f}KFR zboM&XF7$zpTAh;b^D zNA-`xb9;3dt>-H9cgP$uuov)hWE?S&^N@3uQw+Daf!=rCJix^f%Q%R3%ik=LAixgn z0(7J;4d1^X!yD4)>-*byo94k$N?q!813P7@qytqbB6^rY%1UW7^)1Xoujgn!@|&9Y zPq2OREg4n;ccLU%NR#hvOJ)lVHbeV{xv!HKDXhPZL+ zM(T3p-iVDWu!Kr!sOW9VQy_SB2IERfqdv(hs!^!`%1HxGz=|jt!C`0d7H(i&-KB4R zYHwF2>XM+i)*}t$1y-~(6uylF$or-3zKJ5ttn~)FVAW{2R5cj|H6&kiqwTO*U(SdCBy+AO-Ck;L-N`iHCed>1D>cwtT49853MfS`v*11NVB<9TP~ zFkBD}^Okss66h5~Vo=R6} zA?2xB7~T!U0T)RuVIlxBYFysP-lqU3K!~dc3@d}0<5f-}>jsJtJ#d|!&?Z?bc`b!E z3uf13mP*s%cX&ICGan5@kVBG2DowY>W!xNH@_IXH?cEWdduTvonmz=+Vi|dNTC>7v zMRWER>uJwKyCDrab%)((u_Tm+G+DOQvf6Z~Z__Avew7gz#AL($4CE>cx-%|6J4}-= zsR&O0{*Hy)@)D=}ll9kOWY2Fm$vj(5zNHjpwBXi#W(2Q?GpR0z*nl!+nsLrLm1+RpdRno9*Cuv`DU2cmnjIKtHdJly{Mhv&zW9HYKCENT$+5-m}wg=md&c& zI_(;|67kcmhiP12WdSs!v$Cz0)tfv0tnOU1E@%OaJ3XlEcPh}8mr+$IYvM=SlGZc{ zUn%RoG%9o%vRoyuf_Im?lfN`Cd8cL@mwwq+Yv$qgj!^_czCNs#6rmD>3^w8pz00mLa({aKb zSf0YEJTt&PcVd0TAk5P+{joq1u=SHZ4slPA(B|2COB9u8yJEf4+#icN+J>0&^T)il zVv%lOI*5r?fYh)|(VTTwDu&}ZPWK0+v-Iu@@E6<$`4PWkugF26``{E5KKm4wdfYP>#N)zeQ6eVHUT(>yr)Z`nrx?>fQ^W`z(xrx)>PUbzEVjVbGG30?n=deof$@QexYF1q z7ibFyF>u+pOsmRBDnmko!}aEdLT~jh=`?T0#lHwzeN#=_hoz~;J^bYYGiH}Y+R)gj-V`gbRMTO)`^gOZJD}4Pi>G(PX_DK2Q|p0 zFO$V~JF(f^d?aqx5dpYKCp~1LR+;{+XO-gBsI;dLUzxD;xeV8yf|egnJoOZUA<82- zp7^R$9^30F1VfZZG=K43r%bl!b%G<{5?O@hQ1M-NT#}|+>nQ|Bj7v`ji(GbOvDg&; z6|8rw%u+UcwSmJRVQ<^*ETTC+wa=PnNHiD$EpxQ^H0VjTXr2%Vvc#JA=W2iqmDC%(^oerEC>X#0j zT=%hrBGiEsL;cc$o9{j*CAe|avKGp3Fz_nk@^HP#&*on}l%EZ*7x`ITqlW^t;_Y67 zMcr)QBv^#kj|6RA(MJi|@cNOU)h+rcLMz}FWvH1>?|58$ewa=x@OLa>-KS-i9kw@j zZY8N89Kh8QkJvq#ow35(WpKu-e6mwgS^~lZbuE^^4HHtLys~MNSx$MlUgT#BlRcE5 z4XzjYSz}}m1!%?Fy#$MLoxVx12(KRr+PtEV613sE`mn~kN30MxzIvyTqCDuaYQ&kZUTVb2yB}F6f*Nt+tCt#abMHr}5;w-W z(MkEKmOco)JZhiPw7Fn^rD;R$Q<_%q>907gu)WE%sL$@HJd0p^m8Z>ddMr;HY_IaP zdQOi8YQ^2YM4O#$YJ42tuSut=vPkBBV4N@WBZuJSR?FORw{LzniQ10E>$5~{xcy4h zdT3prMQR1^QKlrrupB$sTbULC_bgMJ3-wy2HsGFRYW1OB3)PCe38{+L!c0pW<);#= zNmL+dV&$SdYM;`y4K4aBO&e;T(zK2%`YTQ=Y;W=`nn~!XJd0p^m8Z>ddMr;HY_IaP zdQOi8YQ^2YL|dHzso$Mrx@xaNTxdzAy9Z!vsimiP{DeeU_*Vcl!=)vr+3v zqR%3=0{1A>qIrbg%Crc$XPMeusMj*J0rxCZs}J>Bs8-~)rK%idneFvke>X^8GE$H{ zqF02BJd~5BZD>)!K0A`84Yg0(w2mwKD^4qHZ}KdfN$9COi(q?|r_FJCEKeJ3uky5d zPLBm@#ofL{+nWGsqyy@70z^Tq>%47)%Z4XF6f9MfA@36)3KwN1Y8y=Wa%dNc+Hkk; z&^8;jjwJdlQY&zeGA)`%=&ekPfP0py&4qd`QyXy4GPU|puZ3zw-h@=mlk{CrfCwa2 z`+RbpUvAf@e9x=xmO-D=v<)q)*=HBhw4wGXP3yR#zv8sQ_9oAwnS`Fovk10VdDhR^07NwABfa`W;Z$6CeU{+2MNg6CeUh)nv$JxNgJUzWLdP>$bs! zFNb!Ks13JYiCRYzeHN(|xJQ{5%_HmSu(@ciOtr(+cJ+5_GkuFRgNy5ybXLu zCTWwcEe`cwwl?rRkgYWw^ z`x0$_t*X(Kwv%NlEn?jy2d<)MW@pH`mLl1Zj*Z0#WSuDUK8lmquUcZQeIqumZWYSk zS74CEUd?ivkg9zdyVaXcGc$GqN!5i|Pn-Fr*dchx#t!?d5w7< zUg-Emz&*><=0d%-QU#S_w7m`X_rD2t$!!pP9PaObJ%JzJ z49U;nyS{W&WJ-<|Vv_?i4WTjF{$e0Q=RezT3^aJ9aT1Nhe_T5O?_@dy%!0s9t?sUqm^5NievZ-+_c1s zsez{0A;^g}P|XZ++Gu=0d1^{K?u`_&1tt}3x6zkix<8l<;lDs zEKb6kXcucq%FhdkjmC-*kH*CE;M=_SjNXmz*R$^zt2<~uS#Njgdo(HuV=+ql-2sHH zEcJ@6C^cN30+Ec?#awA=488&)l}Z&F)tZO`Ch?vCZi%l?p#jg z0hwhLUP}55FTv>Hn@mU@NeR&i5+e9_wglu;r-IN3{|V*Dx7niLzO(h}bGV7N(K1Xh zXVygjk*yez)`fme9dRClgf3576tj@Nk9RwaM7iXSl2#iq7e-wG{Cqlw@HN$hwjJA% zDp=auEaF-G!(tbQpp9L!eV8Sn49S);0*>D4i<#+ye4=~k+ez4FCLu{C^O(@l#W_r( z1s!I8COYSVm5xDQgujeBUFk?7CKA)oB1H`JQhy=?5zP?eUHDzRU5Cl1colw*wu|U? zF$<&Z-7c#lLcmrU5onTs;JXK1EJ5!L3K31-_*%tNs)ub~JVkK25s~mF%V10nP4Z_p znr~QZonD6EXgCqJYFw14@pdB;27*N%wdBT zM>q>O8J~++u5J25+AlUl)WklB8-TQn>ATs7BmD!APMi2O^y}h>@fK_jzK*xcha^fC zF#f7%K9Xv;@dZ$?Nso>1L6-n?v`&)sa&mhY*8^!F3d^2rdYV$LV3LXe5WkM+NyB(K z$u`igVRW~;`!-(OeQH#3Z;)Waa!6^mAc$|)n}*RcicMop>5bRhnog0LEspoE8|YIx zL7Z0`YVwEd1W%gF(8bCDtYAXSPruD4Ae4Z}1q=;I`2@>8a->*NQP7}7NNSYk8gCbO zz^0f!RFX4k^%a`V&-ood3N12_M5ZXF14~gnR4HC<$J;ncxP@5zDDA+PaFB-0#70DglWa0|sb*{Fo;Zum`ZM+EGEkcaoQ!Elq1 zMRP-=pk(`W$@u|m|A3UUBRFEwUc)d=YAM%{n7_a<1>DPhc3uG5zM4n|B_eI(BJ&w} zh8-F$5`Y%M5h{m5fcS$rh((uSTR5N-+lO%ORoZ5{d#xSM{|IS-NZNtM=zJiay_BB4 zgl8|MXH)6f6rN3`XQ5m%#F}ARb+Ixx3D4x3N|W%+*d#o&HM#yAqhY7J7D%9jO$+c1 zH6>O6CohfBo3l$QTT2ojf`tdZVQrZs?-I6{Rx-^w)U46G6h2BeFKQMyZ*o%+I}EZe zu2{Ws#jEwqo0^xa+5m)a;yLWVcv$K7V5WM`mLiK#Xm9B@5^hgmtMYOp0j)?bQ-Dy? zl7TIuu;~DHhQQ=haEjw+$YsK08!cd#in&dO7|%tr4OSy%hL-R1HJKF!k%i(`wM^3v z-scL%g;XQZglE=!JrL8Fs48WmFkdQQ*VXmnb@>9nav zxAEWMl_`tg#xVRpyI)7i@biy{;JEnWar(cE27DD>Uc?c}=v~H@&&ii5NwRXv$zJ1b zV%#3d7YWj0wwB>N8orMNq~BfBbXDJ5(_YwySdJR^AaL5UFQ$Fz8o0)$y|7Jq_M>dp zv^TbWshjHBhrYS8J?L9B-Mzk9+vHc*K}~Ag596MKIr8l*nzOdO>Dya8M~(^Co*bOD z?Z?nl=6$$0Z@gpHxw1juU8`Ja+8f&hMW+fEeB&oSjQfb`jBhV7U9?R;`OvqYn9dky zzq%cfscri)bi_b#_)s}-+ym=O_FjUzXxks-ess<_$6uaU_Z8TYWp9z4HBO}8xt~jo zd(t`K+eh|>4YS5QvQ7|qQ9I|qZRW2Y5_pU2db^&%u@taG?=Y_pd$*nE)h+Ys4lU=H z@2F{=+Bg_;yH3mxc21WtZhGro-*&l#d+f$v(A+b z`tDlgO4HuhCMY^pxZoQ<`C;5gOlN$1iRq$k^2vw3{ls*}IQx~IS8q2-SKIbu=!k*f z@S$?vxChpm?7aka(Y8Ov{pg%=j=wyy?klh(%ibb8Yn({Gb3d0F_oQ>ew~y=%8)l7r zWSt=JqIS-ISIm=l)pe8V`-7uy1Ykk%%sYaSZ#|$rcvb{L@T>`h;8zuh05Tt+s$dX& zs)9lAuL=a>TNjAIw=xieM{Tgw(4#I=1l}}}BJi#UuTH@;b&ozFAox`U>cMAU0dimv zd^!b#;9nKK-2&UkzrK}$7(8kNo5#5$kDY~p;7u0-f`3&Y7H>%u69oS9tqjEASsP3m zUbVq6c-02O&;xBC67T9j5Z?8HAbcr=*@Mr1LxJE&843jdsz5k?Ttf%~KlxS$V(_dD zCJ?XMU>Lk=gJI}_HV}z-bsz}u`alpq6_QTO_)|v;!Jjfx2>w;!*EOgh{CbAK;8`0e z25%#AY#b!PFnDzhhM@=AKqOuR6#~G&-t~bXd@2On_dVbc82qV2VCaE15R5>!>>XZ2)_zpAo`#XgyLT#h{C^05QP_=Fngf=ho1~93&DdnvJm{M0ugy| z4n+|758ui_44$>YWa3pD41-r~FbqA=1|spU4g}#{9|*#yLekk9f9gmf_)|s-!M`f} zx&{@5U(XO2JZl5R;5DNHNiYmvU4vohfi@6{*Fc2;@UM4$APAod!S=n)S+T|i27l@h z7g*mL4btVe~*N2%~AW ztbe$g&v$V$dYI4S?OV8bnH(bpEgeW_Mv;q@Pxfc))#q@7Hz&iz$yg%2JlRaE`L%8*54E{T28qt#yc244ih7^L&lKK+H%guc>i-Z0B_Yl385L8lD zp#R`3ULRxEHk!Z}fCQowX!{=SZPFGF4)N>g{(e1+pi>K=>?T@lp^5Rxi8vK2kf9&2 zzdIRnDnKFpW*f)h0y!2u{X%f|uh z*OP5oiK1drq$uiEq^dOFPqI1~ZAh00k`SggJA`A)hvNe(Vv~eLC3o8ub+iq2DQPz- zj?yN?xII7Y;&2Y(AwiFxp)UoevO9bnQeWVYFzAimqvJy;={_t1mBM=k&)|9-JP%{~Sk)?QnDeNq}92-Kyc>C#MD{ zj{2sCA$66Xk~_Gu)GI}hiNl|kLpZb4J4HD8@#E82ML`Yrc)Ap!UTY{&-XH3j@O$nFlmt$0-rbWDh&sCb!8-`(AZp4 zI(LMIfb(AEQ=-xaBtT41>xI)=qWl=kXQs8v1NJ3ETlDOu5N&{c3DFuldnrXLVlQGW z%82?W#v;Vt#AtJoeu~kC*qa!wKGIJ)S|PV9NK9WGk_8WPOEWtMcvWTe?IQVfu{<2F z?}N#iyon^6(nw2rxVOp6Ql1fPLo0esGuBh3*n(F1+`UIiabm2eO0h=3N39h%&ibg9 zqD1NOs>PYJzN*E^)t&q7zQiz8ixX#kRg0UiKT5s0vDTSl%FopFL+a&mdlsq9ANwp) z8*a}cwYpNDWoiZPQ>aD#d2fYU1l+e!ZC=!Cq1u4^7OK^adM#Be@=nCs^mJ9@sc-LQ zt)|LC9K3yacE66uyPB}oe5$TqPN(Uva`)BC#zi=lHkylk$sKtohG-M(!J`$c4Y_x* z4ueN4S1WYCf+Z=);}mQWbpL|2`IEn%zA(U|U~SO-3)bpTJ(sK%dqbiX?=YKIvP947 z`tn&-u?j3rEL4@p?OCL@p-7)aYQybWq}FjrpJi$V?o+5m6A`@?Y7ubXLbZ8OuZ3y@ z?pvrO0DCNKFib!+^0~BCL($()FR-% zg=+JnUJKO*+_zAzZq#e3T9J1m);4EQ8tIw3o$-N^P~SEW;m;6VhWa+-ofx7` zthT{Nzr|`p?p>_bkw?GfYK87sutn1lJr-;cbpL|2`BTpYYlH4zuvUlaxn!-_8xpNp z$I#^jEguk*nR*ad9`y{0V56K!Z9|c&mMXBcCsG@3&sJ(3hxA#dR^UE`S~L;STcH*K z_bpVL7xh}GHsHR6YIUPtOVx_J6S1~DgHpd|@-TxUKpNTtjOs*d!#%BRf8b+WT{Q;j86W=K_q=`*u3b5mBpw#Zb< z_05pl6tE3`ciwNaU~7PS1QNF5?}3Pm@|M1fxCnnwL~L`k#~@-G{+@`~>S>Qb##RRE z2w6(Q`hCqOE_Z3cR(*x2r?-6?`OKyt(t*t-?q0P`I9EIFv?oe&Vyvg0Vx6yk)LL=l ztdD9T$wQ+Tf*L)$;|S)=Szp!Sn2EW`QUW@HakH=PKFWLh;;HU#w6=xTMvfg^72?q%ynYU#h}HGnPR}B>b+=V5wF{BjaC;W1wXf~7 zOs&9u3biQN?yXRZfcqAz&5L?1R2y*LLbbY4ucc~5-icUS9wpT8nLLaV1nBH20nfTJ zCZYwWNcuCx!Zfr1c_)Tw6RU00NWaBuL+)Ly)}d>^ZL|#WKC^=WIOl+~6b@Y|9Lf}~quKYvX_RaiKL)S`g->jf_P>3yBoD&%2AXC9 zpP&Ho$LQVYem(nsvAR2pX32WHgAyEwi!&650@)}sz>CvM1pRq<=>R{| zka)w=tQ(YUy@r%C#>@?&5!7y|U5?D}E0}F-N^EONebiA!(hOvQ`CeS0$lwRFg7OU% zSaSIWM$GfZ1<+9cB zRR2mT3(RBjkpyRF%HPI|yH5$F59_BdFr5j;`EEJe=i9nzl5NR}gs~A}oZ2o$+X(BT z9UDH4xvP#+&5Rw~Jkj zEHLxkA3V?W0@x?YhGMEWdL3A5hQsYP`VvB#y-UJH5^p0&W&@)6U2@BY2+-7d_=-1X zta2EXu5whq%2RWd>tq|=M7uc1&v5b{lvcjLfzJmv?g_Ki^e(4(LoG7G-d)I;^eXJ3 z4i(utbadXr*l2YZM)xq<31B=HvM*4~Ve(}Y?}Ag1HhE4==af>Bf0EFVN}Cb*9NklT zi{<7%C1tMm%{C@_4Zm4NFd*ATi)4p-P9C6o5MJUylbGN2@el^Pqja$Q9iYCCclyFG z^R1wVvYTkJ4fglPBPZev@FCeVlS5Bqjk$F(W(-U7hv_iF3#!tb0MRdm+2w}$5?I$y*_5lYsu*1xo~txaQNyfS z#(Uja78atlJ3ZT?Qm+kM$s>r7L*Nd1w8tnH?&OePHR6)^z!V{;9YY{Uoo%(gjj@ZV z641d)KG0dF)>+6~kzWy}9W~KaCNXSK8ahX8SgD#OT2ZW~9b<@SOSxtYl2Fx5tZ@a4 z5T%W4vbV~g2`a$v>bXwMG^A*!wmMLD860+nr-Qr(eHqY!ykTKRR;&EEtcq#fTq@7L z_7|Mn5T3i|mhcf(fSM-YtLA$+O*_mAvb|h0f~?~eC4}kJOhW>7YRe#0Sn~Z8&%Tdu zv-3IKxca;Bf=wWiS(pLjsTAl>#ru0LUo~&*0;<{DNe|SLIcQpK$n^?irboBaaLv#- z$hr&DC=<6FAOpWYwREr@muy(>s;RU(Y}X7%>bFOfP~ypM6mku)a{UyvS`O~{X-yB8DkV>wP#A$O*bJ98w*Rryw)>`}0&Of#mm)s!~f zt1;o~c>Wk|R{>k9PzjPJ;kTMfIgu(A#Zw(w@s#mmsH?LFQP(vhQt-CXlWB-pw=IqP z!B-z-n=x?i%A1qxS=)rAtHmtbk%g_`6xyAAine@3i78Jp`BOEK-QNcm-oxHIz93qc zAgmHKN$%^(=(H(lJm%D8F?^Lmn&GOU;(N${_1S(bp>XmSV3e4T9t)npvT2V_a^P{zt8k7 zQtqfLqE<(6nr%)J$z0T@L~|9TzMJI#dd<+v@<7BuUoMi7Afq9dFcFc<7^uiajAZ0O zCOS@K+zL6CDl8^#T(hYoTTII#DI}Ck=w%9iCQ}E2Wrl#%;ugq?TV?=s#qsTXF_2^; zknA;lg{agrd^9zN9a3A^try?Eg}1(N3+(;Fa+TS@%H9CO7uJT`yWKR}Y~ro@avJOa ze2>ph<0M+#rzw^6mGTlCWf-5H!K-HR1&%~C1um9L_3Q8@O_!!|<^gc{F1%b{tl&L6 z&f68Bj8?ZQg)#-jC{as_GVvz9%e5dX=AP0@AD$n-*jTD*o~IRcyuZhG4uZkO`2~Bw z5GYs*%EfXJ6&nURxdJH^m_)KOkt>i^{g72{w$+cN3#7EavsMj9KPF!Sv9(q^1;&Oa zk{40xA-sMVCv?}d2c8+bP&=7`D2QacFT9H^0QaDGRG8H*gUSV~+ zNEYjr1jbtH#~HkCpq5umKh_iwQ%t6(EJV-IIs@1{9>FGU!UoY37r`82GrXQ0EY`cM zlqd$Sa7#h7KuId7dR}XBt29j%$D5WhN%~G!wfj5-9Yg7ff(kvU4YA-L{B(V}PCmi=djY&0*H$CTcsX8gzCaV%7HdXc@mh8^ zPe5jxIo{W;Z>@@?dHbu1q=f)c2}(NwqB4}8D5#|XVHZoqGcxTSlQ{xJoH3GhVJX$e zx7@E*7pu+aVLpd>Jd)1qd)yXM{gx5iEUaRDY#FgNRP+#XQAp`2%Jp5EE%~lxO;rK47OkrSYOjy2gowDCWpSi? zr4eWA9U@<9>!Odf8;-5_XswT}4ym;`T^&+;eQZTc#OeKX9Y$*+Zr?*pEh$eqDKOi| zb~Se`k7H{9YNwB_0jQ;5T@_J#eQX^>{n8rB$d}q$?_&$qWQ>Xh_Y+mm&T2twr;n`# z$yb;*YOcLLwz{0!>tpL-YEOCQ1j$n$TNO}i(Yh+2_JVaKM6LF*l@awzYwh;2b|H1q z$J!AXBXyg7Y;{Pj^|93<`3loU&9&OcR+Lk#eQXU(4J*%lL5XfK+nV>6gJCqJ#zR1J zxSEXv&EfKDPew&H9}yaoSo2|_L0}E*Qo8l$5xmENlVB|(wxxhI_QSM@*qYh*5Hj(3 z%Yso)AwM$1Kq>daGP+oJF6CW%KsNQ3pSEiNkX3ozBP#M*KZyFJwL+h3QAn+&?9Qf) zN#5!D?Phtql(&A9HA3NYaEsg7{W`*nf2p`EYlvO+1lH8Fy+q5tkH3o-VKRACRY0wI zU{yfv#m!2HT8*2P5%o)J?Z!>Jkh+MQb_B)<)c7oi6w0eZYIcI75lFtmv{7@!=`9^w ztvsNj>JU@kbVnV(f}r3O9DK!$okjtS385%w7G%pE8p;_Cvv`-@DGJ9*C}jkld>MJ0 zj&gHwVNo$(Y%i)T-xd}vqH3O*(vy~aR?HU_7PUxBIxK{zd^vJy84DftT4?))?zsjG zl|^&XJ=jH+jTRtX)?Jc@P+~)w@Elbx;=PK*Zk5%G|67n!i8S4n`Io46E#=VZEo#Mkvb0RQRk&o(BDuv?6 z!H`@P2}4tHdZ9QEp5J_IJ)RBk0Uk&vZ}FFpq{GqHH@_)7%vl8}4px)u>eZ&Gfa0h_ zOQ^<~mP!8E>#7}KnMld1)Zt<`OcN#6zq1;8b0^h{L)ucM;{}VT>qm9bZ*}y_S@inj zwq5*)eTHB5=-?gG&> zl_IF59$*hNByNE;T#Bw3SXD+8Q)q)3rGZo~#kS_oP=Ph*nG%({Haq>5IT#%cSLVQ@ z^u_}`gUE1KgJNo#t5&$FD0hZT<$+fIS_`iRPa9WqpCW{cq3M;Lnx7&oC|e_HIw2L- zrVUtnw4ze;CaOW*z!f2iMP+XUQ6A=Rr_f~R8j$&V&iR{SOeN@KbGHIIrerIdw^byQ z&)F10539xBJ;eN?lW4n*z68_#0eJe5ckWg| zL5y{v?_fs;%N6S#BF!4fpB^nWp&sTeJ;)yBEIrDc$V}2u`LTM0vh*Z>KC%>U^oJfx zwR2|}Q~oq(2TPSIpBqex=MMuGv=4)iL+H*vgs<=?IjZ|f?IZvX{)P)P)Qca0Y^9r+ z7%_TeIS|RmQ)@goVabyz72X>ltBdyTWXlBq`c(COfu3&u;sI7pKBN2zl*%5@uPJuS zZPEFxY(|70W?s4alz2kr>X3S+%&>b0>k(v^R#XPWj!=J&6B{tY0VAH(E)jRKz31Qa znKDvnC*K3=nz}+N36VsMC`n+YeLL^rN5k+^wuKH*0i-d)g8zc=WHj<;I3m9bWFpzt zX6P~^`LhO{t`E}`BP#gFSj%2HFbD?3G{52|&@0pE098fPXa!kgrZI%%lVs#%b~&A6 zgP?TILV>o#2*TD#SK^RbA`SoyG}01A8b_RlvBXn*3Fa>EUc}4gbp)6G?9!9$gYWhS z$4c$5kB1*Xwfe$Y89@BYiVGH<6?$rs^vcszjOp|AgAjd6YkzS5G9Bm;F{Gu_4{9~J zp_)@Optw`Pcr9WCx2GF)-QSaZd&>rroK zW7n~OE{hmZf<+t5C=J*dQi_&isvyqj855P@^uQP5uhC@Tr+;Gbx1qB&Pa3JPWXz98b{RzotQz9 z`eU9?EE2dvEm@Q6aB1?{QIZM%Q9fxs2--)lI4<*XZJJzy1C?ewT(~)Nc}|Mx3U1^H zLEdFIG3PkKIgqpm78F^_r`(o3=f)5ND_e92Jg7F9vjg5MlL+L>hf1?eeH2+?Rep-- zW_j`!FQG4S)e7(giov0D7wA_{1y^bFD8S|^ zp)^RU4K($1*9@Yjwf0y7eCh&S04`9pbDV4$bZJ{~2L9%6?xW?+Z4@3HJ)a%jKAXX> z=TD=Xo99nv&*rmdH}m-EvnS7Pp8ahQ;7Z^JSh0b5db)Wct$h46K0bJQ6CXxTo*vDj z!P95Qv#*|gbv!tpKRJH(WFS`tDnJ0PamUnKP&l4HeK!B<=*j%>`R(z+;K^6dj}Pa^ zPj2VWpB~QV3We|$(opN)K>6F~`R$X#=kb$hK6o}ehF`~nIGzo@x;Z|`BRD+Fkcba% z;`z7w%+ ze48o0#RWb3e%{&j!Ew5%hlO#rJV9!kG-6gp83T(4N>XEH=|UHCqt<8(p^<1LTLe$cnz;y3VPC8eQMi=nxY}; zT|R3~qk>(V&JYlqszN|9i7G4r0SSQ+N1+VNMpOod*1F0FLLwl*J0eKUP1uk0Ql?D; zX-(0l^hTyyvv^ybTX}iIoTyRqox-Au&N~(>6B5=&1Ah^rV)50ZBa1alNqOT$0Bzu9 z1BKWI65S+VFPZWua$ih({PclsA9h;Ex(PcimEQx+AE_Il`M25K&;0vX-P_E+&TniM zPZswN+c;RPcVxOpmAq!}GOw85dCZ=SgK#hyUtV8AIxr4Mr;T47j9FM-p1Yho-U5c--&R@KI@$%(0e*Ic}ef{bMG<5YEz8=3! zzdjs6^OvtjL;U(|Bfft%d2#vratg%<5|G2!)9H&B;~@f(zn-4Hn!cR8coD=CLE`G_ z8nWK2m%-*b{qo7{tKrMhDWv1c_90Hce)@WNeRcWzMKF)!B>*1@j z;0AV$e=jkA{NnWG~#Ek1jsLgyLc6E7qgVy^U3wg*XI{6Uj$$+ zCr@Y5E)maOzkUJIdHotyoNVK0nLhgJ>ec1hWHNmj+-=tno88G%I0o!eVjrY8dHw3* z>hkmzrA)WDpFCYfOTi43dU z0BXVs(#FTZi|dQg#pL_~X>Ma+VVwZXPw;7{FmoDQfz^zzCa(gz=K^Fh{RVzqX}OvN zFRw0P#q9D$02_=>4t8;3uW%MzU7cS|M^|U5&w(V4;`RJy0f?Z!ki=PVeSZFGIz2m8 z%7T*fWnTnm(^qHb;|ro_;>$=E-&)WWz8;>AUXRbGsZ~JJ#5zEHs2u^~A{d`tgGjE< zgT)H=cPv05T+N&HZjmh3D^YqHoK7#nf5)i2+jR^^MI?Q@_`JA9u|F)w>wAr$sDO*tFJ%)mrELYqC z?^a%92@d6zB-pLI6Mb`vYiUGENJU!dR=!B6Y1UapI4xL2(J(3!^rci>&x3)2Xjsa{ zohTR$r!15VA))a)gcMV&!2^hH-A)hAqwFwGwmLq5qSN5f5 zSAZsZviMC1l@TSr<*Axe10zcs#aVri#iA(xi1&@rS`f~wRn=U6jT7R^c7K~d$5T7?Ca0Fa>}Ors(V#bzq$kil8Jym2Dt3Utxz**5e|fl!Cw@r9OHJrfydB*yZZU-E+RCzN^>ZrWuDT0q1n6 zHbLP}B}b7@tMiY>Rw6}Bc=xzf5UD_1C= zYGsS4`BwI6b`~Z3T$hyP%BY!H_HjNv%a_pQXxUPlL@iq+pRLgnhXwgp_PGdWYW+-K zm(8{~nM^U^!e(kphi5Hjqh{``@+74mM8tnP+zfw3i$2$j+uY6<&*%SHK`Ipkm{FJVrpcN@~KHo^kBnpx9)gmKHG($*}dH~AC zG7d^hlV}`T@{2=5T5)JyUSg(VUsd9T%(=s1Xi6Clb(zAUK0!D%=LLtEg(e2clm#5> zmz7v8W1oL$>gtaQy87&M;r6si{yY=ZMWwc;J0`6X8z5VQv!t6 zq)G@us$W)#sgVh@$#wNw?$e5*#avgY)LKXU2nk0h%QN5nH* zGj{byx~~36-_;+PyZR$-R}WjBXRwJ{K8>B>v32!F#;*Rz)YZd*<%WG8>`jO_*%HF> zvn4WtU3B*a1%-Fm3M!$#prDHN>Nq>%YuSN8Kn`*NGdI1q+#gC1x2X~$>20p z6-`OKm%XY{*UpV+CfzP@R&neMd~iGOS5RmTiudk|ao6 zH;^W%)RaUaeUwWF#1q>}f+%HOLJ-CBWBl})+V!VjXFLD&L8gySzt8vb(j(n`e)?F` z)2Gk!eLZ@OvrvP}7d?_Y78j_QL&laoO>0&jFeoUJLnD8h|EB$;LZ<0^nE>QWa&nzq1ABZjhg#;wV2*|az9JA#5 z%$DQf=O0C&6)K zc(M=2`F(~=}V%Um}Jdj0WVcsc?9YLom1) zE<9xy#W6S_(QULycC?MRggz&}+d=c;_A?%Rn8Yh6#BT|rubfu6w(0@H;?TmWUxZeALW9u1pjina1-1g-Im`Cgw_n|CO*)h zgn+ELg$SUT&uVJ@w&lu_Qh1==*_gD%F;m(rAoLOwl|W-kJTog_)X_FX*#dI9yMs$e z&Z(J<^w4YvzjsBGD>fn?#fJ4(m70K~MKMnxmky04N&_jWn$}d?BO1gk`T@l`62mKw-<*iQC_j$c8q5>hwJ$qPw;?_>Mc15CFB>Pyp=E9VqLDWTJpS?0hzQWA{MxXxiQJ+ zXQSDf0?JXoM|9W;b$&oy7LvY?AyY{{EpQe@D(td0Gm&&(Ovc#?8X}vuhjL_HEu$H# zc^i@GLSM1QxtSX3v6HBznu;WCYO#~7qLOM9-PFOR$}E$hI8}ycZI8*K49rn-IIOB= z{+U-jGgP%)(#WgQL$Z0R;bI=j({zt3@27Hve6_L3Io9D3jntA&PjWEQMFEy-GE$hJ zy%c=OTDpn}jaQ4+z#7!e&Tzv(jW55!KrtOc(l~)Uw8{|gC?<_d5m*(A zUa}17XB;I}mZXL@e1*MU2h!9uuCT|t-X_zfH+H0gxHw_Ifs7;k94&K{$y;GbY;qDr^q{p-w4lfQ3 zo}f%8L+A<5HV+N*0oSFT?3cL-5FF+RH<}FJFZdb%hXIHJ?|vc)x(GXGb?&IC3~ zW(D-pK^@Akk@awB8R$aH(+jPhp{2xAGTGFyte{jw&d_f~KtS|&!W%1EQ-S5`tx8n- z3Ao~mI0sH?3}A*H?TC?`H&FNvhe#Vyhr*$@v?Vuv3eb>{n2R@fO)AUDW$);;lra*_bN` zKN3k>>{;;QBx$ABXM+v2V7CE5wxi8Zb5XTYg-zlj05$=v9Tf4o-h2*~q1Gt?o0dO0T5ymd9rR7X*bm&sAp-=wPwD_qMEiwo$G3vHP;Oi|+Y$mm@q zpqV>j{YKhL{cgkQO7#<6!^yW`la- z<9dssnmeLqql^=nMmr;?7BlQSm<^5z=~3VCzDpBpwbms@&CXNvsIM_-sp@U3pVgr? zvB2!OG+kDqb)cnEb6Yhp3$2MoX{)V?jGd1};DB9HceiEYX*NCSj z>(wxsI!!kgtU&-@`I>uRVs6@!HB7cHW~V&2QBga!nDVXbP%8(Ha!t2o*-j|>A@ceb z?!;`WZ7y^pr4RxsFfswb8QsbU_r zxbpVUXK^XL4*)fU*eSJ4*RD=hS9nR#?zlA4FN*A`6%+v`RMZLUc&72l%4m>;1wG z#<|p?s2PWs*ZE|FXhweNgLZ^P%{aUro^M9pBh^?ld5z1>z$KTk4NJM%`7PWF1ug-H zEWTH6h9DP-zJJ1w4~3V_Xx#>mxGdkIaznIxb;$E@)7hiWVu;!3LrE@g$Hey0GfWJ) zLxz>kwxC_du8#%BQb%~RZm$3UAE7m&>E1BjzQzkWGriD7rOuKb2O#wlry2C7m3EOYXxlo|y7Eo(1U})3eWTjk3P%=R_%~ zbcQKa(M`f-SL{$?=o{zJJ>02~CyYlIbB*2Oo!a^`@947>f zql6Izd>RyFxM_<6u~B!v!bfe*p<1ohv$y>?wl1oapRX#4LNQW(wwtD=j+w5CRB0e3mCbUs z76iPN5Q*AQX|WU-pI$VgQu`w!t%wMK0x=1Y+VH_G=ykNcgA1AHA?fkzF1m}!8;UOb z|I*Pwcu8`r*Kq72c&2!XSSoa;V#_nMM`*6JFMe?m4GT6I*f&_*)+|C zYRN=&%yCjN*5O1$9=^icE2!>j>!irw8sNyO35BceI*F3>mcO=bt)*)$q?#r3lw07&$rfb3N^#^wUB}h>bIeY{hir|d zpknEqeNp!29yl|coPv?Q@q|0?Xbk-l)AKlr*0o&^;Ya0>bZXilWipoo)14(LyDs?y z=Cg8`9Vm0mB`Le^9{|kf6h}= z@fKRPc9~Xxw_5z&LmUDzvXhJK1k4<FzDRTR77~ZdE z(LKyr-7ePQX1iD=^Kebe1;JOrlkgQ@;d`~+e2P|>P2e!8v0Q@4LZ#h^-_sSna)h0{ zz6xO{pU)A7qbXKNv{=Q|Eu`yGSkc`ahpcoQ%&tQYPQh4apQ0^aRpTderxr=;Uaz9h z(E?SO4x{5}dGj#06TsQi@FmgN^=h%2m9H;_kLz9ZnM@^}MYA|TZvY=v_7GA(Iux5D zh^6ANeKG=*)Dmc5LAGME$|7wMjUL>Gje(eLHF*=`2EPVG2h-AYcBcIG+$omYGpAJc zxwT{ZwfwWi+3YtBbAclv%+le%;5(TI`nfJA8uIW}I-6ae8>b4>MdN{hoh)1>CVGzX+UMzyGh}Z9c4fW@-IU8g|4!}V~2MObPPw#G%-k~ff1$~#E8&L^{_>TMJL>>LUEHd!{ zO*&DuSlyXsi4>iYWx>q66Z%59p!@6_A^48X2m?SS!~*ufsYvq7Wh-g-H{-PT^GXuxiDlsuiDVDt@?+amPUodB0`O zfRyf%Pfo%f^E%sz?00ZpW8jV5#mQUfoulMo8=po=6kKgUC9B&oh<3~0z^fVoD&*qg zf{agyQE5tmALI5{-YsJ>xX60;J=|w~I|y$NbX1{^L^Umb7~URPi$^!RAow_(P7x9N z$0d0)Bf7<1pdX$ePnL@WENk-P=6*3-B;N$@#g{2x=?8o%=K{BSLZKHMh++dmxglam z7*Z=B%c$bpVK@L7P9xaB`eQIT3+6X>F%%)9hEQuNcwSWS+ePx}>xXESyjlNnHJ|U| zB$zl=lFApN!A7L=Mf69E&2bUow4j+HiY>l<0kWb*&#n)S6`6#unDpqj35=-XWB}Id zuy?T7OsEulMM-vkBP)4MqnNfNbg>O>(4}2y8p$DoHbxjA$+BrD#yfE zjSUn1Mp$u5XKJ$J-bTt2^pIp{PmMkVWe)iT4v;YLuz-wxZ-bPfM>r)hs(G@^)tV(7 zzM^oqcd*m{;N;|FaI$~)u$rYE-_zg&XghrYPV{`kVE`Rp@Z{?eu7$x{2L=MilmIhY zHl9#Am@0IL2^qocST7eqw>8}D;^obKyj9u+IC95FSK_1H4*GitjLlft{yvyoYG=*T zPVKCDvdqq!CCu%NQ`IaHlO2htATsEFs8L!A<1WBcaAdhCTJ8`maf)88D*OZNA^@*{ z6W>9)K{7U`?f20=ygqEJ$uc}zehB*yY=uQ51N@J~0>VeRidgRkx^ytc;$a1*q0lZF z7u>{mVtemndYul#Guc;9lZJHo14BCc4ZrZ49DB1{Ea7@ZGI6%UW0*?5ZYO0Fw1Wnt zz!`?>(qr+B#HH^5kf*tl4y34X%A_aDv%Dk=@&UaxMJ3RkGVV8~IPT?u10Gn~XAFuD<;;cL! zH?r{*Sc72{nH^9PpLeDsMqk;XqZ=I@9$qXD0W*wjaaM;9ZmYq{kQ1}*^~n*dO3?|v zv8SQkOrg+ZbU;o*SDBo7{9auj(r{gj!mA5VA}83agf1QP3s~8C0w=3W=?FYy@lmhP z!pQ{12JjtWtWnGR3Wx~AHeniw$foG*cW$&Kvle>F3Dtp?v&Xs7l5h#pmBCfb*DT;9 z&k&NbIhq?S!CiU7NL@n!vbmTZP)bA~G5QzCppo9niH^Wyl#G>5d9Nu$PBKs-ebqpn z$+`JX8FCUjA;yyEDtJ&KL@2-~uZ7f%PE>}RV6&37q+TFk8C}T+E2Sgw6nayM-U^!5 zd$DH4!6w!MA_BZkmC4;k!?0dfZj1KM7&1k&V>UM!| zT!~eAbJZJo&o9K)R@i|6`$gcG6+Qy|IQ;PKlOi024?6xkYryF(Hv$WF`<4H43g^2;wf@hRHt?H@nnRN9WnIQZ7wqerUZ$Ix-Z{0^%Rnxc;pMw>?4}3=3%tylH z2xgF@U;=M3BOU`R{ zl(QVQRb_^^$XZEaqG(K^){6cC=Tm~JRI8k|e8QUBIS(%ooEN$3u}t3MbPp;nE$yF$ z#Wp@oDr9mrpfJy3IG<*!73X!-c(#1$j+Uo3@2j-7ZvM#n+G@pALp1ulvl+{1R%Gja z4P+fiEjVg$Yq&Nx-r*&_?}Nq`T+^Mx!eD}?6rkY_CBBJ?uS=HAcD}Bh4~vJOq)+Gd znd$7zca;}6RC&bKNi+%AyPDp=Jrwx~b|S-L3M@^PGENah?GRDdvM{By9p>>Jx`6a4 z#YC_{F!BQqv=c@+Xb?=i09P+t#CRCB z#zsKF(VxCALBm=mZkROYq&w7-F|dTrKm;(t%x!G(gT`SEy7kUEv`C`zqGB2~wDdUn zB{;tWDGbldRQTP}eTzUL8-%x{3$4P%BL0GoL0TsoI1aroDGb(JBYUmlF(?+{X_L4l zZOhO!j6!*nu`rCnGH`r~kO0;)Mgv-58AN1>@St>s$7(GW1X`B_MY1HQb@Fdl)f89L zzq|z+fYaA5!=s+8qBe)od*YemUwcH=JUDFFhh+&Fy_-Q z1Apyac*H)^BZD#Y-Il0(AqP#fa`Q)PF?=}MHs&z4jj!%Q5;z>gZQNlTZpakV4rBA2 zhjV_=Itzzqq7!drkRv=5wGD18*L6GH-rePV7Qq|q3QuiS?L3V2nU=CzGO@sjQr6B( zT8VDpYf!8il^8QdeCDp5tFw8PWqhT!NmZkHONkH$qoVQFGp_2oh`h1qpQ6#W_5N>s z|0^c5^F=Fv&YG5R!wwO7IPCsGLlo^ChCV(O-E^P7qPZ7*I8v8Q5SYG>M%UkDb&#%f zXx@8AJWO>DnEKi$M8)udF!bH1rQw&r6kk_(^c$MylypqLFKjd{RuG(^{ zcUte6J5emPaLvJnB_6Rylmh1*i;kOvFE(_c|6b*t`z2?4>94l<1S5ib7p9sk=&!WQ z6e_V6&xN5F)wl|`M-Hv<@7(ZF2e=hjm3Km056e&mgzY!*rdxXv@xR55JQsQA@gV!5 z978ahW;BdYiMKa?o6AUfd#|n(iSvK)7kGBe;TeBz%YHsJta<&Xc{}JiUjpw<{XPyz zDzjN^5wG_G8)6tN7j=7kXGOEn@HL{1+0n;=|HbE@AD^LS&j5`z^C;Do{y%D>${I3~ zHK!hu!MBw|&awd<`ClB%dg8E&W1fv0U8OYnepvblaiz_-g^@JZ-ejec_QcTI#|J7P z%&(nTS6x;;uBM=-Zg@j^IR>~!&?Ke_F1}s{>*mKbcxd1_JtG7E`Lq1mv9+EGdZjU9 zgnU(`Utp^??<{6McR)1j|0Ihu|LZKaiwO9y31mAZjsLDtCX@fm5(b%kSfq^53x;+s z56lg+irzoPAj`-*^cDrmwSSI9hgYlx%yaD+NK^;wRRG3{+u5yZS9Kodbmux+Iqa>= z+h&12Sm@spA+J$O#5^Bc)hulal5h=bw_LG4$tgk@2?7A#>~ba=&D`xY$LfkgSc zy)wIUTLS+)KT^x{=zY*?!9TtC(awfjrM|WmukCEOW!ZB0S{|qjI=n6a?@TQL%d|rr z=!d8|5po7Jx63Nd!)oS4$S%tQ;7>CwQ2zc?4LdBOGGOj({~$YT8XiSHosE&T^&_il zi@gJ!MZWzXd?w=%mRnYqWgfPc*|Bg1L8|vw50gOSrNrTD+F=@@qNHf&VAD7AQuPeX ztI3Fu2KZHz;o8;SWq!DJwR`m)mSKmY`VQN$!!jxlN%d_VWQVZ&wl>77lFay1%PlL* z+Pkz5Kg`;_`VP~yLs5OV!L-9PLfgOk_Sx6dzxqDJgZaklyA5`=Ynk5$yV|$<4%e`D zufE%2*kKu!?O%Nd46;L2eFrqes*-Hi>bp&rwQFhLCd=Bl`VQZ;cCWtMX4+vIp@*dU zR=0Zo-0C}AyV|?V57(}CufD@F>`+wSVHM&%)?zO94o5LVyThWwe;clegIcWEEK zW$j*lhiTfOsJ`1^+F=@@?N@!Pzg}V11#j=-A~b6bd6^M8^V%hyMb5nTDLo=&U%M3{ z5w!1cj^FkxOyrLJ_{OAszwKX+80P357Au5ktgjsnw`zH|eF{K38gALkID9SVr9-R9}YeHpzC!mp$h+*sHABS+Im0ukku3hb& zSHrcd-HY9@3_BFXZrFw$mQi^~ie2j=mmR;*bTgg%Ew$E+MnuraEYwL|)rgZZCc8#C`nE_xLiK{Gc{A4-e zbgo@vs;sI)hzN^=_^_t-C6DsT8VX3hvDW`KdJJKsNAU#*<1T|JzbRdX8=(K&>Md99 z6BU8L{okfs_Gw5`_!5piFB5+d9*QkXTLm@`IxRI<70+_#%I`nln>7+&&x0rapaT9qW~UXTwh%6 z#wTJZs~NPj+N9mTP1@OQ(jL$z?VND5>C;Zm@t%6|A!Ywp7J5TXWXA$jh6sK&jB3+a zE>qH}uF6>zo+AE1bBr^LqJ0W-^L!?qnK=@KS~!mCugXJmR9CZbRQ*XRIcCB$k8fxO z)(XQ(`f^Y>jvDoJXB%oN=QOzR9n^aMk#O20Jhv!41DnO$?OK724Gzn+4}z7ss$AGo z4~a$W^X-SE?QdimA`+E9&RZ~0dNR8099dlI@)*ATZQ^PxOe^<{2m zJ}?&Qo_*A6jrq`6gZ6(2w4ua*h_vkx2%>8lWYRL4TUksS`L`|S2g?_Zzpd;GE83NW;YPS6 zVMM(gb|g2@*B@wOSJ51yI!dw4t6@F|?Y)Y6YJ~WZBnozH?`ln>H=jo6m%-B*7h{dY zAdalb!!pW`O>fDxna!sS*JhuB&0t%&;ijI_6t;neQ9^x^PuPKHVAK5=j%|h=h`rT% zo^GF(+VTP>T1lf@lc!%=(3m&SebTE}TWd+``9`U$r+^T3ebQp+x=MHRitugt%DSpb z{=u>~R}V#6Py|C8LXjB?Z77vR(7Ykm86okuM>vF9gGhrT*Ix8C7V`pA1{(q)PzId} zg5?Y)1%$#GX}^aQ^U~1TGFL%Od2Q~527Ip>U%4%IS6^7^$<3us4~?DWc>%IRnO9pi zuDsms@Q8s zkAtFh@7LPm?;PFuK%J}1JK!dW@?CYFim8=u7b$A}m-LK5vU4B?)wDzvoZDIL>~{b5 z^S5!JQsHwe*W2jxAY30sUn`%7=xSrcn>Uqxn!5`B2ecr_Ku}tsH<}d|6jW!G*3uhk zN{->Ce3K4JSk49M+d$6A{+srZ7v%^7H4QLq;B7(#&@*OM&^EcI|)SNHRSH$g^;arI7E>LJEnXroU-BnJhdZkEpgL+FHl$wg0XKEohw=G7BI z1bKbZX9U9myas#qpbjbl`9&CouR29Xz-ZH?9)UnM@aiGZHhX0OtPf`SjBACQ$|Rv~ zp-;QVhj7yTI#=!l*DTYfx>LN`q)*p7C(C!D@)f%OYfS!@)8rLp;1{EKuZqbIp}bPn z+I^N58ee>>Y-@!vygpS~Wu~LqQ&n4PGIIoNTs**MjX}PnBtRX?H;19GL6o=Le=bfJtfc|kdmI_JQQs)JJne8LE|%jlD9!_sRSs_PnR zNFQ)N&=HlmeT}_DoP`s+Il2meOd+2r!>9F zIVK}tSP)@a3VQIj3pa9MzDR(P4h&-ECTm&JiJ)`Ms;!?da@@JRu<-lLdL}QRD&2NO z3iV=4z6d!{n|*5n(rX+Okg`z~7^^ot0$_a_KY_6UdDV>+1-YStpx5Vtu~q>m>vJnY z39@4HYlUgIhGvoBvbD_$ZQVgbJ1C&GGBJc=f**+?NGFW3x1g>f^$T|gJV||a=KfF9AxD;1&?lo zAv=e_qfC0?ndL{Pr<1nR`6Sl1dsbDbh8TA0;wvV~s2x6MpH z+JLz&3~PhQb>Yz(tkr`#jJT&j2ssyqV6q|mL@Ifa$p%{8`)x{PGFRzlDspFVzL$Ku zeac)~=`JjGd&Ue$Z+f=YW|Hc*zk8bSdOBC{c%>Y>@H*bbjGBI(K`ueHkh!J8=A zblX~Ge9m@Z+eS+Zi%VQp+MuRbz?eIRuVdsh#k_NJv*bFc@i}}org06U7nD0aJtIeq zLJeob1liaId`P^C7fJ41(Ve5!k0PCq9G{sZXMRu-!(TI3p%5Fczj61=656dH@(q_E za_B+pSDXZ4hcRpVOcxH_96u^$(Gj9mRs}}rR0gZ`U<)$5$%8=`4ndk0Opy>emF~5W zQ-sX0dI{y1{R5jLIdnVxGQG}GUpr5$n$gDnKB!pI?(rcJ1wC+6-NCx$3Xr(mgcuZ5fgcfE+yPdxaac*5!kxo%KF^&^t|C_c(1p7 z91uPRReFYZCAbjL5*+?6o$o5+ti)S;{+<%dLNd3wDi=zwtN2;yxspKG5+)Rf7F7iAz- z@@=6sY{Q_+LB*w|nK|Yvq=SGR&T|UA682T=pJlGrTB{KSB_?~g>MKi~Rh5{X;?5l@ zyfx82lBTv)7n?wwqrDjCXbt3aS6)O@g`N4@E(-G1zSC1*OuRmsRhzLQX~u}8E|tW5 z0O5X!)LJQ~rlxNji7;ynky_^W8jAg3^Er=Vy&OeMPZP!&|5DZD9|n7BRMN|iMMTO+k|0g2EA(dr11k-zon zp^;)0si<}?HL6UzvM62h(M!Q^c)oKCZ_w1vozvFfna1b2l9#-*j)}RZ7;A6g6`}Cj;)xMB3QZPn_l9@u}(|sCVqLKMQk5WaZ3SgKd(k-&g zSO}B0)?&1XosdMva~xWVxSpRt1GGuWpGWvKRrnX#;m#M9v3 zY@(SED78=|8~AjehL>n$e$b=TGGi${Dv5N96d6lNVY150?Da1KDKLd@g(6(k6rpCL zpqUH_R1a-q0u7ESn5+{tlffoE$bz>iCxUEnQ7{l1*@sdAz>2ei7*@j zpZpLeL3kFZQ-3ICm?ru#2||ubnLhO-9;DM!Gh{bIoBtJ5VpdS+BoYna%B8Ho0xQGO{fmsiSw%RN+nw~{{YN0UF zoTSvP^r@ACqBP&3%x5T!CjlB+xYoHEm)Jrre681A4edDr}5C@QLsC2qINs*5)7zs!VIMD~19{FD)V&5e5e$ zo>pb4q?>Li=-LuCl3BOeP~HW>+)!Tbs-Gw(i%}arl!9R*O$)_q!^ym4!PYYPo(^9A zyA4fplnp_;pnJfO`Xl0kF`M}m;)dkY9uAon7!h|!RdBS2Lk1t|(G@X=f+|QN=9(eD z>$$Q$QiQ=FJ}d&N+|L?XD zib=QB5ZWOz1ygdgh0u_n^oR?aP*oDNh0tK39-JW)29>izW;J9iJ@gDYxGjXbXTwZ8 zGQa4i8FFMOrb#BocOh7_zvMM$A$v@pm#7QBV3 zW_idKhLgtvn+Zg4y_Cw_mzLt zQeN<6!zY2#_qVhQ$7t=#l~|))N1uws0ys}ChQH;FSgC=Tg_t}X)EXxL$=ckdiL`Hc z=nPr8v}tUAF+W|J)HeIfreKF|qc2Nj?9?Y>TdWokF&7{_12Os9Tkk5ZbW@6I{Xt~0 zn5Pbllw6+D3QtB`=aLq=ynCkBVINQrwZU9vT7OTu8zmolLOO24JaL+_8qh^O99`)c8DE5v_4i>h=;=4rWNNQcKDFxDOr8$6>&RzRl0b{{20o> zSUY=OVTv9UK5cQl@@T*E+s_nb+3M3R`x7nFi=e~8BA?Ruu!uEvf8?7R^@IoQ7h2zV#xPvw=1TG2P_@Z%1Jtee4}aMn;B{r@;D9R6$Y1 zbu%1ZZF~lne*wf^Kg(TIT!+tr8a`5kVht)&=nEnFSS(!W-BYmMLUUPk z#K?NAFy;FWDnKTmAk|64+hkcDhWE{pY0fdce5?14K(>03Y*)56@j6QVS~_?_J(Wd* z(5I~zA;QR5sBUSs#Y{u9L~jd3h)}Q50}(oF#aEN9^C<0#B#ID^6Qvw|ei?fb45z|m zM4m%8KV!=1N{|0XoAEiNdNVcMNNY8?MBiW)n|)&Et4v)nOW~dyC0(hG z$My-hs+&)@_{vX1DIcT`cl#gS3eb0Npl<=u3UqBK7^O&lWxZ!UzG{ZuiBQTY3X1~% z2_m4yK%+CO5d@P9YCJjFKBW~J50lWCk#L!A+WHk^Q4sQIg%v^22KR;EkZFVbu5oC* z!GGX5bl&y|hY&t_>}#-S5QNb3fVx&udiTHzc`F*!)uKQ!-?T-v43I16{f_)(R_Je7 zcmo48>J4?m_5sX4JhtugZV?lLPpGLRg4py`jCq{sO2NmAu4C$ujOKvx)2615UkHV= zfsVVbx~y8%8-9)AX}nb|TA#En-SCm#8jkC<+Kf19c<<(Z>jg#B$Dw|kWTu)yI zuWG2S!OnTkvBW^TK(b+ObX4ercGBMg8uzj#?chy1Qm4ITLRNQw@J{e5ioOnu@`E;<`_JHByXvgEd4(S6Of$ ztmi}T!FXCl*!o-S{*<1I+6H%VO_|&2c2SgIh{NYInlzBYH{S77(r!T_m51DA4m21f zI-#CgCtl_9rEqY{bYmi9Yq7;Hu-X?Od4A*@WqQpE_0y1Y%*<)T z7V>-&Lk+fK_1RSunYM^3)FMBf?8fX6H3p&c=Jp}y1sAw6CX9kfw`Wk}pb)vmHu=IM z&f>X^1JEcn^N-3mnm#<`MDZDpDr_c)Lvyy(9;Pb$uqketPtZ{n;S(JewgV^C%+C!FtEY2aLUc8(g_&+pY0YBj3k0Wnu=8Rwrp& zyv!}BuPrVs#Rm`E^ugUWRV(Kj_w3RqU9_7MVvjHiHG>n>d1@D5xEhhk(7(*g8q`)N zd^R@|p~iFZT(2Abvyh7;>5u6=YJ?-$8Lqgl-GV7H?807ZRUj@6NeS!?ZMR(zYV0EX zc`9lLcHxl-va2##WNcT3dtGsfyUoo`VLJaN@!Y!Nva$#jI2n;=Y@P5TVy_6@NZPA0 zn%nE9&vQheEHoJ0c1qtckhM>Q(t=4qw38Et%BL=JPAK!AbRt4=Y86mzc0%ck(&{?b zoHo;I<%pK;UG-?*MKqm*vCaFJYBo|Z>wB`!t*eN9u~!XByo--;s79F8B)LV(1>XXN zSm#AJJw%4n&(7Mp?#4(Ns)eqxNJ4WU68t_nS)^&+mZ<~(MU5p93TQL?8>*@z#O|%W z+{I-j5zZyZ;@KFnKqf_JGG=1s_zKS6SxL2akgqR?Fw3 z&R)e*`YaIUPRRj)0FTLG`q4LGyhQk-m7MRS@!0Lo(PCs84w*d1FhBn|%SEM);Z@Va z?OORTst&I>XuifpWtEYmikwyMNLl5~n>1{gb3z7&o}3fsyFIRI@wFcjb-9Cv4a+c* z)6fuqUyYSkxzn>Vn1RMGU2zIOnXh_Qwo$ztvuPUY{ii2!7V}32GFnIFYfOCuW)D?s zUobNG$wY480gbOwZu5|7E%qf5w%D@}xF!yguk8cy%-mnhIjKOYIjO)((TTm=imNaZ zU52kR@nu2f&RU+=-OP#iQW!k$UyhklIQ08Le)lDV4M(w#E9-AObh$>zS#;j)tcrs0dvlE zYsud2tYYODQ7Y$uab53ftgB-3F^lilFU{x39FUVSI465h|G_@uv<&SwGbd}HxSScE zSA@@_jjY56`(}u@G1^OwzGap5Gu-v1GbVc~tK4{Rj29h!OH2FKmR!h8b+TG%1dHbr z%e>A6t27RUK{cijmWdbJWMQzXBTHuClg0&u_^ZNsm2N&&G)TV*UHTC6s;4&nqp`K!tl^^w0BgYDH$fEhNu{-iF}iNO5_w3cA9@|om2c%Af_GCM>Jg` zwND;2anM9iWld&Ipb{!rYcg|!x5itLOr!}mp{j+JHS}^PRCj{M zk7#t2HW*_%OjWO1d_Ql6&-Q!#a4+T$gss@^G;t%6q4s^z44<%`nR-VwmTWKoK#pae8@@olUFq9BwyjU z!u+xJq7f7Q#8dLd6^%bBEe3r1@ausjw3OLWh(2$~jLB|Sy?e&|+J@PcGkEK2ctd4X znR~{d;^NF=S4s96#pQ!a&*(q6f7ux&Rt=EL!X*rFRAmqvz=K@i1D9 z_&I~crOZNl@nat{#5EU9BB#5yp&mdL3rls#5Ke_) zI1L#FzlPrncpq)Vs5r9Nhg)wN{}MxT&LX+Z!bb21c8MA71OR#z9h zXJ_WHB%;tIgIE^`QaEQdnsH^UGwyo=KvS@t9)24qAF~5^l7SzJEtYJHEq!$KfaZ|{ zF0yso^ZvE_*D=zeKTe>l-7G@RPzbD_S=H*xZmsrE_s z$@Zy^g}Za}o9|D*5B81d|408H@|Qf5z8E+5=J$%4U&wF%PxDs`e@aeT`Y*_uHx@4O zqz_uZBPPFP?%pMfxK?@638)|Y|6pJEYx05$d-Jy~7bM%8|5*QdOn&n->D1&&Iw6-i znwO@p=CtoRN}n%miMia~5_7zx^o4?!#3dt|U)hCctPr5(I4|6i9R1tEjmh>iKfH7c zv5-N{!a3O5ggg7f=aciJ_eyJ9MkSxQS%V}ktNG)PKblu^#|xWbu>%~dq1C*_p0sAO zz4@*DB~|IYPJhR7qoXD9Do5!S2L+G4`L!Y#mf!p`x5m-Df?H!h{kwwF=ON8sBEpGu z?9I<7t=Z^UB7D{SisMH3=?XAA7H-P5JMHJ&XE-l7|Gcl@D7ziMBk+r1{gAwoNO09C zue>$(M@S;0*eG6O1Hun#2=>4gDofUZ6#3C4Dz+g*ytNv#rSA+_7SGO3@c?XNMaYQu z6mni1Lt%viQA`ZI*zSHX}WTsGL22saX7l#O8+ zhd&*Ir)T0v*_B)!&#&ozAb0f~+z+H%_9LrA2|%t=d?!h3tIC{2MkZ1tQ#6_KY<<{G zoRGpn9NGN77WWIdV5FvFV>9_Z;Y+v!lmf}r+*mRdD|@--XlPQZVl0K<7vVmZvrhxl zh?H@-?!}+e9~);{jVm8hy=2t@rxT9#vU!y=_&xoxfpB=ree!;RZ92OPtr8JKj9pm5 zEjS7Q(Z(#a=?^PQxW~w&r*Z~|Z894Ph2be=ekPZVfveL~(3)W}h>woiuLnCKIhQ|g z^(U7+ox)?BQpq?QrIHlRDDKB1nWpf#p%@-#qe$rr2?eG=XfZ{w)eutXjSn+VGFg+g?(w-RXod(vTP~Km9l&(qf8}qCrQ~wQg*47T`gtHq->Rx-63Vq zOWDg(_JNdrB4yu5*>_Twh%!sqlB`|Dv))p6hLoKpWp*h$AY})o%+b8bv1D4DV@X}I zV@YH8$@VFzp)TLO1AwI7b=#AAH@=Xc)H`=A{y6T#pYb>1&$Pw(v$_#QKF;@5i1S0! z#Cfd)XGcp_dah&P=j%%5EN~=ua+Ln;Xx>*;y057D;OtMjaABW+@l$L|%&@}JZ6yx_ z7jy0CmY8cDr605;Ugju$-OElR! zr?B*;mek*bTX^GW)fJl6@XW>Z{zPAGnQCCFi6&Qa>pweGjUa+1Gy>MY?2N`Z7*2bk*3F9;Dlu z`}7ZPL)7d)l1lbfIa1%{RDW=)Xs@e1mN`oQ7~7J#tYl6PN9s=Q<@DzWnfhVL9C&0? z$-c)OsXORzw{@HlhAnWUjIfpLyN-th-9(k_d%%&poSFU$XC3N9vm;bFOrB`-3Yt#E_`l{*u?T9o>%RwidAx z_W(!F9g9m||J;#!kp6bt&Y!>r_gRjfZ!M;~l*t{qaB@_M`wU0VSA?(#cikL4U)1hm zy^Q<8xb7q2la3|(!Lmuai!Rv@_08HHE7=eA_1Yc$jL;qtQ{ukF(ewGmCHp%&dj4l| zi5sUkI3yWH(6b-K-6gSkEipN0GvPG1WdF&Io_m0%(?{Hp^E;NfJ2-m245s5?0elF5 zz#Pc-3x@NfIPel!Rn0whEf9%Mmb~eL{Q@6z4ICKW5<>+YD|vGS_wAb}J9=&q&lBl6 z40xjCP4FUI$dQ<9gjbLOxN*_z959*j9(JzPxaGxwiz;f7CAWkSDqhBIo3a<4#qF|C{D_QQPBy4AZ zaO&cHzK@~>o`(ls6#+NG21v<%^3g{?Ch6vAI-3v9D{NVrP9;@A^Lc3n&F)?W%@=3b zlX|)L;?Ko9MkMvh+lD`5*5S{@hw$gT)%a7o9Dimv;m@OWD5-FMdM3^V%{$SEyx{ze z-qzu#s3Ir(wGL@mZze+QS%9eG7susRtwU{f5ML&xy@=zDBj0u$HIBql#hWIDxUb@Z z;uxO>khkHdHxnzl0=1^7zGdtz-c0mud+E)>dXAn7bUjC}^riQ|xo#T2e-HN? zIQAHR{~_+D;fEna-WJq6ST2>f@`qjc{b#s7S}5Ddkk4^n$vw52w?@a}**Z|Y$#sTB z>47jy^1W#Y1PYX$pRYMI=?5b6S?jGZXd2oDexMO?aA*+ zcMN`v9dArUqPwvj502oT_$POW0^u(lECQdB{-dPJJTkDUIoiT8S4L46MN5_Z5c`@t-p}CcI{u7&eK+@YcYaUq zHgessCD%JV6pk<^v10y^3WEnY@hI-=|KPrgD=#+sXf_8^5Z%w2MDLU0`8N*UX$;Rf z984+uJnp57Wg`0zSH9KY1rhe-g+2UPSFTIJQO9*fA|Qt@!;i5``Ev?N(aoiP^oSHn zpt*SZ3+Fgs;CJ!fBJp?O`j6TkVmmkoaYl2zDC#J_Um_Y1tA4F)i!uf#-CHD4(9k1 zq%ZE|&%4QxKnI&m;FxvBI21z?jl7EdmbgL08Dm4n8 z5Bb9@xH7L>xbiMTCKklx>NvRHv&J#Ug7s-@d3_t4!;lGPNd=>Il`459A`Np~a6e}sga zBpjtS9v!{$qy9b}Xl*sk#Q9{Lsom5cXDS{FaHiJF7@R3Ai*Tlf!8n}Bo9Ez6xp*SZ z)IyqqGe!1PoT;dqjWZ=djE^)?@pKi=RHFYIXDVUW;Y`K|-hWTFNLz3EiKi>~|?U3h6|+OYH!Gqn5Xjoi1eqrR+>8qc)Dfjgzuj zQZ`%4=1JMbQg)S;HA~qIQg(}!Jt}2SO4&v!dtS;Y{R>;(m9o83Mq^&WZ&Yvy9OVol zqf$c1Ql%_i%6dszu9W3V*)%DeA!QX(RwZQ%q^wEGZj`c>Qbzet*mA#=t(CH8r0fML z+bU(BN!czbJ0N8Tr7Q+{QMeJOUB$CBDWjY%!X-n>vZX9n%JQY`d?_oIGU{asH|9v$ zB~o^|lwB)j*GbufQuc_HJtt)wr0jhu`&i1pma;uk_Oq1zE@eled>8f|qg}5%P zDa(Pf*_%@Ku9WSRvM;4IuO_Q=R zDXWmOMk!k$W!FjBO;YxNlsznE>!ob7lzk*+pGw)cQnpXZev>j=obY)(>#SWRuoNll zEoEm&*+?lHEoD=rY`T=WrEH#*EtImWr0g~+yHm>6NZF%O_HQZMC}pon*_%?fTgvuI z*{@PYFR?^B}Q)}quGfaqF9H|5T!W08uV1) z66F<*zX=UO(c3xiVa9zN9)eqgczJ!@*If|2#OZxoNQ!2>Jt2=~wwWiesRz^9xLqf+ zQQJ>p!(KR*4P4ucW!=}CopIx7tk>exSq&~($Ow$s*>u0ON}5M%d>H6DHBZiU8$b-TZ z7;k*6@$~xwZF#~Nrr}Z=jkeJQ*3WQ7M=a-|E3!izXBJGkypjtwNi3J77mm#w zBr(JVNwy}y9ZC8cCW%sdh_+GMpyR2xMvM@?pgZa<(7`dxI7Jqj6k3#vD0~jQJguj$ z1uyc0V|?6-hg`TtTGgL!MPtGnRVPyL;@}t`w-7J}w>nNc28|o9)&-gt$N0F#ZL?AC zCbznum~^wQb&IAY>^J(WvyWTFk7V!FwYFf<~KmZwqJtIg@uXfwyf-Z>XW! zYUu`RgLXTePlFCm6?^+t)}&{Ir!3qJQ(3s2*%p_>RBj#A;YC@iU=NEXzTbeXG~?l2 zji+4okTW<-Xwa&MWB96v29$1vTFN(84pe`Qgk~J*)uHPAbO? zhuNx!3}n7}D7o&X5(Nk4-ZYa{4;66tX<&L%YYWE&{LEH891Yv1cEiZ}(Hy4aJBYet z5iko*6u2uCd>+V`pCT~czh?IBhT$=%GPajm2slW^%)W&v3ad`%5c$W9&jbH{*9WT} zvILCcjq@2QAbQdrzw-vrm42vJWw?R6Ov6&qZni1|PO@8pdC?!o)3NRbqJn)1%zl5I zpM6y5rl96MQLBBWyC!_YfEnYD^W&rHwo=1*tx@^+65y5u!FMk(Py6Fc?0XNG&;4G0v55E;CWk?e=Fa9c|6-=eaW|n*n!a{!Qv)sjMLCS)l;jI>=Toi(; zVL?ixnOX=7QrzL|%Owc3Af+xy9+L$rv%^yLU64{?#vvD!_%7fm4h{?N`7KB>T{EO! zkYc*l#}qC@l2(71F72^ekTS;1KMDou5-mt^m?@eqNI5GApk$_8w1O*CXAQ{lUu<#! zMO_Ad)cHuZ9d-BmNk2Yu`J!8QT;trB|HZ4K@0o0Cy}INF;CtXlM;h4aI1;~;wCf^V z@5gU0enGB4p-luLxc)?}_M^Tftw1^PDAC(ph7~A{XFqU7m$+%e?|35htCeHd+o?Hq zh*qG8GFV@KV!rx>=Jqb!JrheaUVv%+KN!*fjU_Ajk`g;+=QbZ~{yM*T-@>0_l9pbK zc*B%LdIQG_v2tS?*0E%dXc_%$(g}-5hLtOCEc^|?lL5qPju9=xvH0YJ!j_>|(o->M z5|aQ6ThciP=JN4HD1F3TOALoOz&Z%?f9a{_7>ROQVlJaaB)NWaV4fv*NArU8LCxD8 zfH=JN&}x@;cxGR?A*P^oV}A4pj+WHRF%QvZU$`!Yh_F=UAyRIMS!_>Aj_H_`+zaRQ zn514GC2iTYCuz&}{G?uQCH2}kJSjP`SJIZ(^OLr0Se&%wQ#^YAqokCmq+Xkow(JIY z)8eF*9!J@eQeuuwO6hZC{+XLEZNl1@`@pURZS!O4YjKy4^Jdt76vSsgD^nn}N)rXzPg|V;|U_gBf0e z3@)QTVp$C(5q-gnwi@O_xY~;YUu?Y^hFbn7^M|of&#@ISRjjY+$7QytbXG*G68J&I zcJ$@PCG?6-j_wc}8+}Sb--PtosKl5K<}pLB+i3;|O+s-X)c10hB$L3R)lRV--OT+v zM#fQ%?H<<5QwcTfP&070TX2=j@nVA9ZMdJ#@5Sn<+YlZzA;Y?L+h)wSrBo*L#Z;!7 z@k9*Oi5WtxaDN)tKZ(14vq66Y=F95(V%8bypTzMZa&N`GNPBj!f2&eo%vd6QO1m@+ zN1;RZQ-Mnfsf)DLIjOo%%xEJ1Gx&3wj_CCZJ=g6QbG=4z_g5L*e~HoDRHIHNcXK5; z2XW4Gxz0+1&USXLpH3FnAv-gVgVnqrm zQ#ztGQmb+8!Oy#9>OOfd)=WKwd&=+nnkmX^Lc_3TY8`~R7)aLFOi?zFFuxf=6?o=7 zsH~%d#&~D&D_U+R)=XvKJQK1^ocH34QlM!ztf%{Hai-_b;hcl>KAcfVHBp&42xppO zLHE?{KNdeRGls6>5gA_DZ0#zZ&6Bc=rHq1F=-w%1YozQ^DSJiAUYD|ur0i2E`&P>K zNf~mP=0*qYDxUR~vMebZE@dO7tU}7Fq^wcO=uL(2?@}qlTNF+9xRm`%%3hJO*QM-x zDf>ank`T5cT)Jvk@vNtm(L5L7F3nmIIGVL0Wb>qqW~~U>by9Ydl-(_5G*3?8XjYcs zdr`{XmNJ^`Cvcxj*;i85A=<0kNxOZwcL#rL4D1=SrDN%4ov6 z;JZ@FmPpxkQg)M+JuYSclCrm@>^&*_T*|(ZvQB72h;ZqwU17$RlpQB!XGxh|%ILL; z(2dcq;?V}&l)uE0w!{%PQCnCo7Oo{D1)wcPNv!`5g;=^=hbwC5K9)WWSN7(gikf%i zH-Des{L{jp;*yq*gN0b=wiXNBXn9*MEpKzs^0wvL^0phbccgE%g~`D3&;WlJ}sbxRkGUo3Qcp^zH1fQrVkXaS3s zu5ZH_kbm=5uV2G>JZNZvj?uMgv!Cxs4Q2mE^h`7qjkt=Im}C9u>RMvX^Ak0;#8Bk| zpBY-}i_)7QYKghUPr8^(&-1ltQTlR-XK(E22fmSmsVG96&PdNd&{CA44UeXLTz>RQ zev8ID566P&x2a7J4m9iI3R+^8I+_d9$8Y{MIWZc4Qaf(O&c>*Oha1)l?So<@cP0L_{xwkE`n_3OoLM z#F<-?mc58x1QtbUsUO`5Dtplbp>$76`p46)NVp5PAY@o;1~+3T+K)wspQO5QEO#ftQGLb5b{o48n%z5^FYy`D*@sxPV6W9eudAdhWWG9$eg z7VF!a-{HNM*Z8V`5rL%}3Zh>TotIb97g@9mnn+e#@=yOAog>P(`OU9E5z9HbLh}~j z(FvkX3?&R~8NfR+iyW74N~U}2$MCiPV(I@&T9FgOx@iSS48sNQ$Xtx~4*ne(Duk#b z6NPinWHN>h%Z>~_*TkE8)OKjS+&_;ulWJjF9;doC`WohnqV5AlfVv}-%bQNr0m|ii zuSBFh%YNr`vh|M15Z*DNWYW50G8xh{ zIQ7tG=3C)MW@)P=!6 zJ$jaH6ravdBWB+2$sAyxZ{n zlfH?LzTMW@5T2sDM!va|=5d{C(68+pC5^6oEu0pUCl}xyckJv`&TIVd^HA zmPwPzPfiY)XaKm%XH{f~4LT@|G;qM(oUuXrX|kU}mRLJSe!7(_T*4LVrGhBsM5u{P zF~~e>PtzQmY?N@X-Hk`@j!|ma1pPAPCv@QuJ7pJl6)=1e_DWk$fxbdx&?US`bZfq+7P-$Qg)7%)ks;rlwB%iS4i0^DZ4|;Xjz7EmzHG+ zH(rynH>C`-4mI2XDWlH2(5241&^=MgPL{GvDH|YVMN)Q-lvPXF94Wg*$}X3(RZ@0` zlszbAk4V{@QueNt?Ub@Fr7R9Zhr%C;+EqO3E@eHWY><=u<*QAUVPYB)R=~e)m7I8aOUx12 zey1g7f_UTg=^niP>VZ9YUMMJCpVaGRdR5k(wB-f74BNgZsn>?2E$i|6ECrptO;Gp< zm#IzYk#|bkvT-q#a9NPFWhcpF=pN_v#d1pxs!zxJy-1>3f(|!+-WR?ox`WtKL;K(^ zZ;1Q8?-`BKN*uAg(wvBU1PYECUNw`OHl|fzRJnKM59yD`q+L&it~flt>Qu3jy56Jm z@OFgp9R!WxXLIcYj-_`9<@~;j0V^tZ!k)*U>F*0p;rB-xu;SgqEs&!Y$062fqdvqd zBRWnoVDNppx5zj1it+Xx{A*}Cwq_@%C{ZzCkl2=g{Vs0}g= zKkwm9f8>Nf#jq#N6LC(*`2w6z#+lMD;)=I9j={BP zbI?^hw^hg*aW7l^zli(G4_dZ-2C(mj0s}Ia06}%)6?*G zQhGA}_DILLU*hF_To9vyvK-uWJH`yD0Bni54x!6v3^BEt-~Uw`GJG+g#t7-bQ1M^_ zjdf!1@bx_;jA^FfL%K>|$OjRJZuvNW_J)pp7!X7#S?OVqW+SjH*NV0BZ}u^M3KO}!1Yer*t8KGT}hm%mCH*w z7Z-7)UZD0HEH@S6$|I%Og#{B1XI*|Kag%&){C-@%6e3GI2QPhN44HLX(68~KxKx$ zOQl}cFZ5M!UDhu&3_ytY&D1^qnY(=&$~}7ub*)8tp5@aelzm3(tPkP}m-_&G26XP? zfaiFrt_@*u_BtpJ#E;4+a?OM5zj3VC{zzcOU=3l5I5wGMsayFF#~xvD*l%#g#Q|;u zz;AHo1`gO}uuLC_`Hnw3iMvLF7GwB*PXpO3-+`ECIbfgx;5YEX-;Q83Qy)m6VGnIg zfI{+LoZ&8mIr<>W68=nVkwnpyjvsBrqLcXzvea+|DJgqS z%HEN(_oeI$Df?Q=I-*S=Y)R0r;#n^#qk#edAdFhKe3irT*cx3BrZGgv@Yzl7vkATYZKTp_a(9;ZtToD zFUCIljY%wq=9=U1GNs{K=%>r8UbqrWgrea{?_uU9q4ibRUS4=d<0ZRK4?&~fHUgGr5xFT*d-dxDx<9?l4zqOc>cb|==BZD2a zIGUw2#;`=JSBu#|x~TRH7NYwGEplC4k?%)xnRiY3)4SgM@$nm`=QTh0_W%7d_o|DZ zIPcoi&ul#7jjtA+`Q@_Hze}iozL)EbwbRxnbv}F2zLuH|-+$RMYwBeWbt(G#!GkX! zb?}CQy(o_FO@+otb(@#5&&f4d^(yrswQyd>qGv8P=%sNlhC9!l(;v+&pQW2e9~{K##$ zHR`kk(f4S*Mty}Z0C&c<BKR+;m|1>E{?`E zef&i)F`{kW^fvIFckqTuX#47D;~mBeq!cYTd56#W6l^rAoIb3_sqKIZoH2Ef4A7 z7@si2P?y19pM3Z;)+2k}>Za9e9OL5_O}p2_aBBAIS9GoMnij|SxJB(|Jq&A(iXN+L zHEUWNJa8~1w5!D1(VyU3nyc^ytjk2J9h>v`eoK=o=NUKolwauS zwf>go_G(N>^R^>2INvQoZ1ywB%di3$L%lV=5$E^qOR8$O`Qg$`c9kH=C;0Ov?Z%0N z8e(R?-?;-GA228(G1#YNnh&i<_69H`E3Em>2jA5f!ajP* zheMS=%E-QI8@qdsz^T#;%@cYXBYLOMYXA$reDE~^vx$bQab)9X=HDN|w{?MyZKSb5 z94hw?n z+4#7?4Oj4O1mD!PHg?R@)_l9c=K*HsI%~csz;^%>)BdqJFkj3TtQURBnvZ6fEqVpz zCaQI*E+StE(Lxo%oB5J^7n89}U-}{BC3OqoIJK zP?26fz~ecwQLK>0-EpY$?_=P*2bk$e0;lpXrRR%|h++lZtobPX?gXaau>z;EZv!5m zdVCc7DlITy-icA{wNnI66@Gum;}3dAu|?YWfXcsBus z1y04+2Yik}QOt9ez~w9W9>U`uxlwHEaBIFa@STDQ%p)Dve0SsVp3zb4Z$;L8-NE-Q zFdZgZ^WBWc4@{0?qo!H&QTcK^FfCMpkzeU&@@@8)@woHMD7L-Cnvdcmt2BzOst`C; zdiggVUynw|RJL2j+VXn6A{Iz@bVnH{tP*mq)SX zi>&$5Vc*xlbh%pKXlUFl{Dxz&<)x)jEOEIt-~AYGEC=R@8w9RQ!IurbpApG@S6TBt z1HQ$;;jZSnLbdeD@2)+u!d|}P^IUXl|6~*>_W6f6wzA?L_*wZw@8;2@A_eJ>Kb|8wqLJdJ2s`9T4 z@p0}3i_9Le~Z^?wWT zE?{=QX!hh_fusJvnSb-ZciFILwjod8Xh_eXU{Xs2ZjplT13Vr&E1DfOTi{goQAE8B%-bG;Yf|u$f8WLufbN$HoGSb_fp0A^ z3-DT!j;j@X^f9s>Ez#`fYpv~DM0|gZW<%+l5jg&$;3NMw-X6^+-YsxS`!L|LADAZ} z6u7wxzDMwQBEB8g;Yop0g$xYGh0DZ{XG>Sb{4n=3cerkIQEDb_Un-X*9ADU{N}*>wZ%up zux;rA_r8Mf3p}25MhqL0C2*?v=nlU3f%$%zz`dj3qj9OR7_9ldP~cSdJ(Gw$fx>9y zbb)(S!AC1nCX~dmz2yR@vTrH)mScO|9yHK_<0S>(8}xWV410qHSa7KLDvjh2~zvGh<*smkXI@p$huF>LyJfm7Lc0QsZ)227vWDsamc_EGrV|9T9&^c{gy#qY0(kN9_E zSnOv4r%Eqyl9lX^Vd?t>PUYVhz_$sQrPzL*jyDwct;FMXu^rfq1T-jcsO+l(-!5P# zQAY%aD!nWP-yn2;Hc+gE z=4gR?O2PLs_>P*;f&FKSz^UT*A@HrncWcJe7i(}lq2QxrTyt z?hyswU%)r=iVke+U##t`1K-tGbzndLRp1^}@FD53EjM*wH_#F<9IE*6fN$V!9oRqr zA#f`HJ`cX-0h5mS&}wYF~;^6y>1ETyme;J5=n zv-C0-d^j_MN2o;yO|ROzn@d`Z%m49yob7iu?sra(N_c*>g z^;HjnQ`tx5-@V6mWJmQDxW^RsHGuDgjE-!@0D(iWo3{TJ;XztXM|M}fz^T%U3w)Et zc4YsWC~zwOP6FS#_!`y&B?5OqVc+rK8&K7eO`KzGUpDwY0j6t{z^T#;RrdpzbYugT zS=)yXm9w{jd2gk_eW9>#3HsA--`$a2aG$`@=zv-KnWDcQn7L~N?g9lLh9=obk9K79 z9=GPB=x+w*-e&}EhJtTA_=c_P$gbI7%|{<5yAzlnUlh2R3cf1vJ^4yU)^nS{QLo0# zznhRP?7(z*SKy{A_-J@~$@?AI(;r&%Z3o|bz&yND;HD|~`h)MpFFUeHUs>}}Hme8b zioF7-3cq~t9diKj`-8yY*3`cQV-_haz=r*9lvmtoD93cinl`98;*k6uIlJ~)p3a;Ctk z;$u1Zl7_~y+lC9AD!mZjKzs==+Yy-00nGNXf%)nt#Ifp0f%z^6CgnV9z8$dd?-#_e zB{Kz16~E)*-+O`iv0UI(`HlEyT^PqkRR!jA0rPl$V7`;*VnTFdV7{KfxG%NldlL5T zygZKGyinj&@w*lE%!9yOv_#+v73q)o>X*i`-pc~>odry%6@mF|H^i~EHwNb03e1hG ztoc^LzU8;au|9VSoGN~o!oQ<|dFMWXb13{veE0k_jtzexFyBmIc0UrB@0rKr*t91D z^UVb&>RD?(y!vHt;0uu3*9)8~es>@}?*?Y;3j$Z7@GtSL-Wtd5c{MQKQ^4H(c3{4m z_u`o6gTQ=?fvNb9HQ!H&kD*`0u>rdTP8Gi^;ophCe78s7RP7DoyXE^hw&FlwzDIx= z`%7THpMQ_TXB992ghOTDYS@=z>%<=JC~zwKFqMIwpU{crb{05Q{L)S-lYm*;Rp3g4d3cgc;*?+UZ zU8vw|0^d=$bz(Q%Zq1hn|2_`P`o9ZYy@GEeJUHONPVC^r0tb52_H!@rB|g%LWj|%j zN3Z|ydk*<>y*1wg_;(^Oue~5}6wg&=WiLonp<`9efu9^S3huu0+8{yUxwZjAv2V0;h^!ijOp4rkyEp z7b^IsgKxpGc=nFnn(s5zZwG+c?hrUt|6>{W>hQAt_p#P|)SvD#E}k8jEO4s&m!`+8 zJ~y6?m?3buHH}{y7CIN0TT2D5OyS=NAh>c?JbSm&nvdEC-vHBGCvd9%M+5kNamTZ| zxz>E^z_$pP{7bC)t^nU-P4O&ep}?u)m%+c2fSGi)z|B^uV`~Td-i#2zTMz^9hhfdv$pRg@Lm2^ zJR9?_z^USQCHN|V>F|lbsp@kl_-@|`%oo;ti@UoQA=0_MF5)_gP{?&PTnEPlGc z;np;MX_)SKV9qWPI92|=6$DYU64-5()_jA&_b4#m%n>+M{Qd)c&bbM!(|l{bGVmP- z%;lF0oT@yaX^xqHNnk%M7C2S>ZbSGTwFL3IT;NpcncBOx*Cnv;Zm{Nyfqk8COkgRi z1x}S-sJ}4ht_1e`J=T2L;OlX30vo?Z;8gi{9r$)Xn!xUVLf~*~8o%q{-*vz=JtuIg z_@Mgjz=i}ice6Dg`S)sImc1fyD*w{9oJeW>i~4!%o)x&A|IzB@ti!;S=Y z(@ud?#cv(_`y?=@ej{*Iiu_CM1Lt=M?5%y)e3V|k2WG=h0;kHqRNi;_J%Lqgds?dO z+XcR6VBB$-$bmz}Hv@eCPE2ISCJ7vFP2-p9kAA>BdZfUq(lha$d2AxP;kdwj{{*Jr z$$|Nv?48KE_Oa$$4Es(6W=(%w(t z5%Jwso5)_B6PWKmzzm-snD3%X64_msS@V^_zO}&gxXPNZGwhqxoXCn>1WpydY2dpI znA=teT!q5F#JA?gME1M-$oOj|&`b zP2-o!zZZe|^jU$sRN-IZdu2l+J8pAezD!^WUJ1-M;EhCfsL z@6(x$?Hib{1emjPtohPm-}^&4v)_jboGN~)f7Iiw&TMVIz^U3B#Mh^wGaEl9FkcNY z@e>2{T{^WhyW`xzeCvRjQEbiE7vs;{T<~wDz^USQCWgz7xUe&;cMIHd#IafV+!FzI z#Qe@|zyfQ&$50*=0Q1`-fm5Zw12Aa)lFsb$rPh4gsC)tDrt1Xm3Wa?s@b3dRcV^G6 z5;#@-Cd0oU0<-&GfxBA47Yl;%4|HZz9_eo^3>6!y`y`@yevW~aR&aH{x?fq%~iX7~pJw^+fq9|V8h(V1Pj)0&U^GdBVA z+cyHIYCpdXzVr8XW?B2K`8I)XBrx&63Y@C`cnkjh0ka5Sq6thmRPjrEyMVdBv%oD< z_?MPry>&zv_Ea}(K6*d19hkV|1+GTHw-$VpPU^xEPqyY;0lw3KdF~8>TcF^>QYLnN zRu}f@0D%L&Y56=2e9r@O!r21Xq~Lo61SjNmVY^3I^I=L5BBTpDP$Y1w_V}~lTQH#u z8$8*XkNO`)z#MnJz*Q;iqX?Z`+=Wdp6F61;#=*aCU@oo_I92)bAnY49rwjYmZOwNV z>`U-;VQVfHxEh6htHAfmWnI`!S6K5kf$vFR23{?2hD zYr!hD>Z??#BBlC!p7T9(?!9yG+?k8$|MmU9!zaw#XP)`a+2=XmbLI@^CysrloAuIj zsqL}+h5uL#f0e~(v*t)Z`Xv+moH+8+B0&w*3;w(#8YcLXBXZ+kau&Od|)^GWlU z??+t%&V`={?`oI)V0r1ZtDE)F?$q|yz+UFRyIBw7Q-|Ez@Ruxq`MT_bIk{F_7vZ^u z?@rje4xHgVgy;6WK4$HN6oR-?&Y}o5IBG;NXT6kBuq#s{Leeld&>)CO_b4$Mku=iJRrcD!` z+w+LyjhD{IwN4A9wuh-^)>+{EewOgu;>+>XpETrJhlWzyqrN&3od3)h-cpzNegS)x z7w1}|+l1$qze8cK7Mzt!h38iO*q*Ljo@*7YNNukl>`ez}@r}ZBYyalM-r%3*TEF^v zYI`eS?`d$(zEgN^`GF;sz1L_*wjU#|7Q1HwHF%w+YA>GkUez3nZ= zeSZVq@P~!VUZJuVvnVG6gJ@U}YYu;NE6yt{j$MY67_WZ}yiJv z;y<_eE(Py8ANC&fX>SX7+kM!}{*A=dD}4RH8>+c(@hygtso-3%d8yKG6*%{J*y{&- z&w%rp4|~>cbFHrU zb$m-tyci3aO)*1NYmMT9cgELd}+~Ujfdh+?Dc_>CZG1M0q+(Mdo0h7fwM#NQsu{Pa5_Dn zI=;-0KHyYnUMhQ2!KwCPuT68j((g|29`>-u^n1ytz0bhQ{DXIV3pB?oeC6Ox_OMqB zBVlkJ)Vx&b_ZT>vJnZrLu^pW59p3e=7dXdio?Crm`i<5c9IDN&e#`-Ho`<~xSh&uo zy@$c;@UX}7_m)q4oz@{;@W;(9zI=WZf)mlaRQYiUI9GYtV|}|5oHu>g+YZhr9`=~O zdFv$(I8>Wkehde%Tyx#xOM7)b?Og-jEk5i$=F{GL;O+Ebulp1D&=~%>rOJ zTYOnw!rWzEAc~kRJ<;Qk#KJl=}{K(q~osBgXSoM_|GjrR)DwC zhrP8v?Y#xw`#$XDJn1ig`++x9bKT<0{Fn;P6`GeS{Z@dp(!(C}V=Xvu`>^*RIJ-RT z6~NxXPx*`Q5b#E7u3LO*Z>CRsZQw2QVef9A_MQRnRS$bZ5x$+^^xNcJo`--lQuEy6 z%lw!L&Qc%tt^wy3ANC&89GF&{Tm5(kyd56)`26Vhw7>i<0&leDy2Y3FYJJ*U0p3av zd#oR8!FgZvQq_;0w<8W|BYhEgQ6Tq3_!`_9O1Ji1A%a2>ZyU&Nc=X~1R zN#3(Q;;T7c;TsL!L=St^L(Smau6e1_?>=zWde~!ryiU$@5|32&I)iht=DFoB%imzl z@rv(E@EUyByUM4%N5EU}!`@b(_PYKt*XoHsZf@}{#DLC%G7o!9zq`SC z)rY;U;C$#|kNMmAd5HrK)#g?|27)(2bKT<0@XhdPZz*`!`LOq(PkV2H_r8Zc_CIo7 zKz|y4+*0MoRB);_&n>=e|JuM=$&xgJ1eA?>(?>P^9OuzTRIqXI6{OAKtq2{^8m*sgJIP-kiYXfJQhrM2~ zcem!?P;GAM_at~PdD!a-3p;$;%Xul+>VZFQZt*R^xq+HP9yhn}O$6_J4|{xmTm;S< z%}Z53)`Ih-hdt)UJK%JE**kv^2B){?xy6_EiZus^YIBS4Z15sJ>|Nv2-dgaU^kMHE zpZ2=Gg69YRxVgo*80Ut7Gf(qUdj`B$J?!!Mu@ju3uX>l~k>HHeJh%9={c8Yc zr4M^|fb*b-J;ryl=HO6mZuzkbyzJLfmp|rjZ_Ux>T-%q?;7#`ew|riVQ~KQ7T69IDMNKW+x^4j=Y5__Vhb zybpcY>->hl`r8-0fts7DJWl|pP4iNv-&NpT?_rPm@h~`V`mnbhoKHOLu{`H(@fY90 z;1z4GTYPEne4q9%2k&Yh_U`d%Z!>srde~$8vKyS^|15D%l^=t_Db_r<_%c7v2j>bO z_Evzi(!*W>?5))t9IDN&ery46yN5mMtL!)Z+Wj@4YZ_%eS-`?NO)ym=n>sIRUA zXOrfo%8!@8dCS8d^J5P}>_-Ll1i_f1Us8uRQk$ zuSj#<;!AteecHPSyrn+u-Qm;TGvK}IVUP7=CpgD$^)Al?!5N`>Zt-P)%m8Px4|`XG zbF&Y7ztJ3+R-0S>*b3fWke%uY-BR=fC;?v$9@N)i^I=*aQ`f832 zo~wSI3SPB`y;2xy180NgrAoi&!P(+rkM(UAIQ`!7u5UxY8L4@0<&XAeYK~WYuLEzD zhdrj><38+-b!MRoQQsu`z;QYqJ9?SEq;C$)BUe0#- zhw#VEEqnzyH&AoPwVgL8@!J^?6LfH{X29l z{@vl&H@j61+aI$=HO6m zZu#*DcSjBIq#>=U*<<&&5_W#mzQzio#SDT&yNM*Jf?Z6>c^AdyyRhz z^=$_@eLnEc-~QkfXJ`31@P=z{s`#D* z&J~)MDnC|$v(m#J%gb7D-u7YdLvVI^*eig&ga6xKe20KHQghwnOM5eY+G_)EnGbt+ z`?U8Ac&~ccoF z4||y(`^(?H;0@GVxA-!A6MWi>fY;_>kNI&6IL~Wds`~LdIPZAaV}4}rfIkd>+)~*a z3{J7;xy6_5{rQ@s4B|hx{J0RjD}30y-KV_{@SgKw?|q;4I{g##ck#!~Exw$8RS3>( z%}bSjVQ?<-u*dvZ3C;!|_MQi4iw}FdG>7rv=9V7?pP=7^KW=XErQSG0a}*K(x!Ics z-sL{*t@3H_Iq+Whu*drG8904E^)Ana;0)J1xA-zY&H?9gANH1kbEAhnwg-=B4i44k zmVR5n+wQ|&_D+ebm%U@b8?3o*@n!fX`m{F>yvsf8Q4g&G=M~LMRX^SW=Y0=*%#WOZ zVcrA&xTUgJ1kPy9bBiy_bFJnmgZR%aKURRZ(ucjZKJ9G>?-L*P@;>vIzr(>R*Ic*w z79)Oj;M}2ksnTx^IFEVQWBP3Yr_<-&`H=@sFU@lcAM>|Jb8x6OxBRFEugSw6!?(<* zz2AVh!H2!KecJ1`3-fsJ$IUIir8qYfoQpIsReoFn&NUwP*#4~nXR8l;?}PK14|_en z@K^py!JDADZt-P)H2Ji5GkACSu(!ddy&d4~@vz7AJ8U<`yYa^@RenqcXQt-4#h2yz z5^(PDVQ&pMk9pXmzS^QW3?DbQ`tccfng32*{y08ZpgGD|3VTDrEA_C)@KyV?cL{h` zdD!Fg<4$ni)Vx&ru^pUGJnS()^7df93jVmIvNsZ(ahjJZe;YJM8N`2X`Eeb1t9;md z+^4-T^u*de`Rd6!D^v;j2;Pliy zx9}Cf-Vn{fq1xQ?V=8#n9`;zC+kDzv3ErI^_E`R&^l9%S@OFFH>;63BR-g`dnWoPDDdHCbz7GH+1zvd_+{&Nf88Q@Lzu*dQo0q1tjOO+q@ zfwR`b9`oaMaOuTeA=4_-sK+lRR0{DS2QnGe!K+b`zTw)c zNR=HkXj2?cO|UnixPKB}olo98pS&x4@Cp#V>l1i06T(N{BR+YrCgau2p4&2nrBk*$ z4~JjR!L#sbmV$zUlSY?}tr$DHa_pqalF6gWi+TqtCyfeBEt`1SfKv+#dnY+Rc5>Oo zlF8#I21?6DP8k!ZD4SYVaauCw#MY)rePghqzB*hLUU*vN|irnxba?2e-*;?l{bW5)-^P8uJWHny}ZFlj>BsIjBRmQ9?b zZXIg~)<#uMc32-@Q97=qvTTwAo;qrL<%He^*+`~;;;$YRZn>3_&r};#G-FCjFx)a@ zVW@R({ftqSRda(QTk9KYTV@Qcsv1xgtR8$~Ro&2<69=6-sP@F_y6Pb(o;;*_P~FKj zgNFp`3TIT4J7d9+!6)kYiiy$n4Z)Tfb=9+ijg5h-c?$;iudNTyXbIQMIIFd)DKaHe z-_TM*xTdCmsCtf7WDU>Hn2Bll)`2MA*CT}XzOue19BK*GMGB6ccwE8wW~6ofh4oFd z3Pyz*n{k)0)!pio9|+8=2~`#p*a|iMLPkliZtM}n0@jkY!;!G zNUe1U_JQ)MmOx2qMWAJVRdb*=7^$jnK>6vD9}YGILd`9T90?Dwx&Ujaj|9V24K3>S zEx2*htboCI16U0%h%k+r=6{)tZ!=i?=ReNMn}>5nMS+p^vjSBO4WSwomrzq6GCLe> znH_4VHN*ldDY2@depXXsuqk5o(iV@=#BngwitG%u)L$61`s=d?wC&bgQgbb^RNoYA zunNHc?V)Usg*P_TQa!xBDKes{sAX<_^ZXX;NX^JhWVANbN2*)vtfMp~I|-#3`B8&9 zVfEG|d^=dBMC_erhvrw4V;jm#&S?%IWF|m%N;wQQhgz(|^}UJ{?KA}IBGwW5d|TrA zaQ&>=5ffM|BO$N|LNMG^)nNTV8?iDIIJLo==;`c))1jJ3^mIDj>rRv+8AE;q4} ztf^=2)z#OScvuAqM(XMp1Z$)5C`=&WE_Dr|s)$K+t2lvE9SSv6H8jtzGKMnSthNLi zTEE)*kae^Ub4OyLRn^XEZHd&?2T{`POt!iu7-At1l8Jv}z*VS)#j^PrO0SiekF!Fd z+UojO5KLF@(;;xQ{SXAn`e{UBGf8nfv1z0 zuZjfC)k$hjRdcf{1iU&)rHO=Fn`%^olaQn~1jdDG!a?4I2~JXyDgjbB*xXQ66I7{5 zRzYHdSJyY;S*QZL9*iVeQXQOSPAAEZn%1y6og|@xO|{lA1d^?(jw-9-EHuc0nov_q z1kae^txYXxCxf*GfdEbp8QgEc5VU0Tsv25@R$2VTsznXf2GEFwnuZVPCxvlLuqha> zuL%Uh;ZQh$RysH<6kZ4$fy$}{frelc9$OLyof|&FFw_L0BL3PmgJ7Nk0g2FPKM_L1 z5kU?&rif9cpHR8maR#f@Pp~E}Y>5O@1?o&cfpW9r4AKx7>wFN{NXu~5r1X>j2lSg4s;@k87jZrns`q`qH9I@e7cKP@nJ93*H(S;^Eg z>lAIHXF4`2M~yTW7N>LJxR?vAj5Jc|Ok7y3gPP-ZN2Li-hXQp2hB%XXt5R-Z^nT^n~x;QAB^yzg$NQu;hQUddAHYu!sFo81FApzY|RdM|!wg=+&B2E!rMF(E-;ZW97c z!77*#Egp04z&KN4T|Ihm3$4B> z4PMY%AB+UXRgJTbPsyY5!h;umVsfeSli#|&B~Zo2)EpAQ5nf%lUFo_VV|W>c{Y63~r|VXMIUN(;R9w_7 zM@~D~wyIM>*I{8yX)gL_y^BH7m=P&!Lk(A=wZ!i73?wlgJp=v=NsFnj8|73lsD zM#?a1R#`ILV$pSvfZZ98^wdFT`nv}`<`Nxfws+(w4|D<_!F4=29px25ATEFT9Hjjt zlIE|lbx`c0aDXQjpyE9o@|VCn~8L#Sz1 zTsqn9VDfa*bR8i3iIzM-aa;t%XS);)k)vJmNUFHoscd0(Cl8uQtGJ60M(={I9$B%E zUAv$s4;~AyzQj?`lZPyUk3glC0@?u|mx*@!bb8Vf9KAvZ$(91$ebEu3Q~eeC=#8RH zn%y`RC%u=+nCPWCj6-#WIWWEJ5Lml9_e^@FQjADEeH%OFI{6^Owyi;J$y^btp%+09 z?AC-D8aU)*KlzhqxgCPIj-Rtj;xHUp*c>(6mNKj~YdUu9X2rC!fq1ix=uJjAwq#3z zzP>FZE$k-EYFniIX^u9{L^GxbY0}Sjw8e)M9U*;Zh1;|e4!7}vh=Ng30zAB?FYfrN5iQX%=aM(>nM%y31 z^nDfXW>>f5Da|Ioz}!$*d%M6VPwr%w;6C=iQSvm8zbG!@?F+4>37>eOslIG?S%#9Q zzWap;tbM6l7C0`B%?Az!^R90B;C?CYYwK`bGn9WOZX}sqtA5cT6`QYHR2TjL)D#(; z<>Zi)4>7tiFbX5o=;H@w2diq~jHH8#r)}4c8g@pTY@fGBfAsmp+c}>|{RWy_I|NN& zEGI2RFq+LrKp{rqrZmlmTQM+yb`Ud%n&!drfY}ta&h9|`bpr;vU&!W44Y*0=tKn0? zc40)S(0<6&q#ioI>{e~+$gz_rSqE7;`S%`US%Fbi5vUBXC-WiF8}J!~TLOWJlP8ZI ziLj4qsA_2`4Yt&T>zffD>vV8NU`87!f`uaioD5<&uL^^GRXD)Iy(tt4goAa#aImQ+ zI9z)t3e;?Yg5yWP@12uB9XII{;60<3`{2n0&+EvAm{t}TI2aN!C%+CeMD>kuhlctV zb%UsJDkR&o>_=E*aNU?R0vHG|9u1frhBQ-?oiIy}0nLMcBXHS*WS1q(^h3=XfT?v6UKkDWXk6d2 z_tzH{>2NUS%|7k4^6?xE!bn^?cdsrjDpID949CO9>|{1LfeGdx3Dh(V;B{GS|; z*XH%yl*z%SmQYyDb*{fqP4Ash6|QRJ#N?d(?MO_UsHVZFynPlCwNd6R#(-ObwUe5w z!g$(+8d@8htncIE-am-as_}T38&Iv1G~Wl{RW;j2ICuf4TI<`74uwZ^hNlDrt^MJ! zb!!YA55jo?+zhkBBEtucP{lPipsH^)V2E`nuDdDbx_ZpE3t$=yOfs=Oocy&h}v8qXWOyz~M;FW~KRSV-D)ClNlxO_1}qhxuY zG+0;F+5ll7^R1zP^n4$6+MvV5!C5durS+W{?UTzP*)SU%(*c!mLT?M%ZER1fXH8W3 z(4q}MQ|9D}^JgOCq60%#W(Uh&*&6a+HGX4op#y zYsTpb2r8qyVqT|$RV>tzo+lYJJU`+Qyg3hiG61zJL_|{ zovN7si)o^mx-kl9jVT|#%brYERoMm8EM$eK?UMKtt1x|}HH;Y*b>UEBppKJyDUAPU zM=E(gSLN_fJAeVG;HE%05~Ra@BRJXv3h;l`Z& zADhIBHri-M>%+rUzL}ylJa+Ui49G?=g7Zi+OBXJb&S7t{$p3I1= zDO;-8+-K*Qqfn?ZLS?46X=|p9a-^S_nT&g#@O_>9=q*O)P3jyi?&-+P*ew_yz#*@-jv7jO=y$Zubqd- z4s1|*xBjxiAdVDz&*Gg#mH%~kMMxeEswQ&c zfZ%)>2V&<6u)t>7(UmExDhYmsC#9n(shT+r{EO`j!dn;W4J4K5c3~2TJ$?ObvI~hb zgKQk3NBH}}@RN2p9R5&* zpxF0-u_|+m$?&DJ*Lwoc8{jrkX;TxzYc}&DcDKGC@6j(%S&iO*B#0-=Y>;m~Qj_6; zvE^0Ln#zk+BciLRPFt61YS>aod*M$1ZFEDNzh%Hc)o|^yA7hD6&f4GzOrXxm?`LYK z-?A4I3e~#inQBnNx5ZQe7%nbFz6YV?!cXi3iB?r(2L8~T{44A#UnM;$HL>iY z!Iw-yA;!xFYU%YboU#Pv`AI{2qXF2HkQCT_dKj-{V$*Hb`zgb+1_ zO)>@4(m06|>uR)ENl(W6*ec#}tteA^?`PktHvAr!3TB!ymPI!WOCNMj{?E{7kvvKL zEYeLIyM7nP#a4T%F&qQmj9py39&Bo616mq(82(`w)arV?B8ms1Qpo}Jb|oKBvp113x-n9RglbU@%A-jsrLB#P z?6kHvVpt$2|5#f&nF461gtBf><@LAY&ze5XE|u}hHCE=T+WD|uPU1%$3YC0cYDyTS z2BnP}$5s*SV+YKcJ0fVS5riow2p$qZ9%%acpSo$ZpM889u#a7A*Xl?syaL1J5%G$N zQ@keul7QFju#0GYh!w6zNuICl+^7#>b~vJ{s>(r!>1?!7l$FtV+Y%lS^-%AYbX=L6`|*^mZYuKzPDPeh!F`U zkeC`1VDJz-32pVI#^#I?i0{>;(2lUD)Au~nSWH6MgU1bF474<3;1~{pxvD6BWIqJ> z7*~0xO8PJCry$==iS2mu5gD*g;(cwrwiSSDa~9Vt$jRT}lA)%zM2`p@=+Y3!434U+ zzcdmj&Ld(TFnHkz1EF>kW(u0^a%5p`v}y83ig!-FEhtC!k>cb@XS#A>I6b|C)V zY@z;cmwCLZqh92Slc^Lr^X#&Lx9vIWN9!kEoNYe?R8eDtjaR+q^F4CtENcoF^X<0V zt^u)4mri^>2z2YE6Wkf@Xg(m$=fmu^wgnuWmSwl9M&i>g|;U?X-v+#G=Nys@A+9BT@4lP)S7p!vf_n7*lS|HOP#ndz0fJ)JH z$yx^tK{dgUsj6EQwt2je6+v?m6^88QeQBQ!lw!ocieApvrnya_`AzEXkJ;tY(U_Yi zn~A2JD+`=n3%iCVxGm1WZFibx(ffIZTR2lzvI_A&6Sj zdWHRjh`MVqlGVUR3G$Nze=oS?e4HnMUbXpl^WpVGRJGtS)0FV}q8k!+Xb!il2OAQO z{=pT^;h$gI&3b&|N{p%Bt}@TVQJ7o8#z!8>>XBDwJL2N6mLhwOEvj_0=~~cK^~%c7 zPdJlMDq}vdqmKt>EyggaYHIl6V_X}`$3~$YOvZ11uxhUACsJc}xTLhHV zoa|>*YB^%q@}T0wnX~Mq3p`4ai7&APtzw)SVmAh9sR~<8p9~9!+V#h74qYUwo#SYd zDM&{8RN>LfY!S{UWwTvX0#VVedP=+OEDlT?6_d%f6_uAAc5>L>b5o5}?;@e-P~o0u zOS(Yv)p1n)-(-_eec%C>Rk(8U-?i(1eN!`(b6mZj4%;`_=aUx%uSQdzRWS)sa6J3u ztda$em`ZoL5NAGdG;9)cM_HE67BYRu6&}bwM&sh)39=@B6On9LeI2duk2J9 zosix&c5&zIMYI&weB2y^c$}cN0a?+9vqIyhAtlZR8C+;}1^p{L5*pB{oM*AIRWnKh zZu3cbH%!y9aI>Mf8-Jn|v$#qJes;mnS*lgTnrWP;5)mpzh~hG!c#9m)U%ag+;ckT~ zwii|*Z&S?A|sI?33SAv|$Sp;^s2 zcMUNSKbSy()B5U_V*5Mp6+qxb1fsarg#xu)DVE;WN|h>;8N>b9+N0rNY ziIFr78PhdTAL7cM?0_Gkf{|oqIavB9vyk~^p{nEk<|l6|NJg|dr{#Fr;K?0&07aux#IS_E}eSlt;;OyC>YAi zxbr_hJm>7*-Trl3@XyE0Iil@n$R&jjn%x-b|7lBb@qiy+eb~f-4VLu>g}<@>=uW5p zdr{`__q#s*&~JynVp#`f($2Nlp7PSjpRWDS&KFMKK5ppQ1^DKQ!dF*6e)qadZ~9@^ z@cmm}JEHhi%X&}YjW4Wu`Npzg{W3nE`pMys-*B&Gos~s9Pv%rDIN-1iXFb%O9hi5+ zsS_>hw+inx_|mTTpFMQcMYUfIYAmie)3W}d@RpN*cYOQcliz-0@t*7Q`hN9au!A(p z%lJ?8mZ$!8%MHH?41M9ayQ)ebv#cu=e$L0A|9kDTHz1=xNy$}Z52hi z#h*m(d+p9PeABoS`OAN|@udN!6Yu)>mG5`^;rabmSk_Gnzxs^{2adbp!uPHU9XRyF zyVt&k7gF{of5FT>N8a52${nZ8TK#52Yu0wl8l>>Iu3ogHWaD869(&gjzgRx*B82f` zg)e_@_@&2w+;H#nzo;1B^Ri7ZBRmR!>7T!?oPF|B2venocwn8Tj2t`OyPSz-|>@Pr~LJUYp=fHcV`4k=3Cas3jg^B_ng*z@`?Gs z|Md&+Px-u}!?LCwKs#f$PTY83+xwRF^LwigJZ||>mi2_f_hj!l_qNRLC+7Y4g;j-b z_ZW?D9d{-_Z_qDK8`9LN=Jh9j`;R+*_2@nLCZWQ+9C!WXe?NId);}hle$Dm2d+8y| zdP?EN7nSv#)G+#t74J^pen4Z>R?GTA;aBu(J^$c4kGZ+u>Q()I`PK(FTGrSuw0}~k zC%3*f^7GGY8aG_BxM)PBWj&Du;&e`h41vLuL=U#Il}O_}~ZXp6~rd|Koo2&f@3K zTVFE^ukC%0cFx`XpH~k(pwFP!3YK1eYiY(%lq-dwv~|ws@4xWiz$d@TI_(b|O8Q&Y zKNY?`aLiK|oHrro!rRZfw*BwBYAtIN3@b#X@zjndhojySWl8MiXf+T5k*{&i>vURHSa`LFHywCDI~Yg=#o`>|Kg{=l+IVJk1=->-Fh?5`W%obb;x zfA;8}em_OJEm!!3-EUaG{({4%Y+gO*k*&v_JH)d7s_+%NIy!1c_u5wU@i}XA4_Q`k zS*M_p%FAdtc-Y<7{Ap9qYwn%g_TmB0ePmhpDSX=U$(!~+a=?2}-Eeg8=2^p0?+!(K znU~RX;Oq0x%j=qb_5~}a^!QQbB&;QFS!ncR`xN!+MwIcvcqZ)B$6ay!DNZ-wdaI^r z20W2ifwZxpj*DLx9J^rXkmF`Rt=5L-w-jKqZOe=Sg){1#Y8qN=gE;J8*nhw&rwkd? zzi>t?8q^szSpA6&>-_pq|JiQlFg;hVhCQ#orM0S|pe53Z<#uNbEF5sk3``LShOs(a zOK`@#ni&ziwAM1Cu?2?%2KOH@r2oKE3I`4{0a$?XL*+hj3HUy|lgLoFDEUtzamKhO zj(*67P&rTi@VKzn6Qyqja8A^-^#68v0^*t?tkN@k-+TfrkO|4|;j)_$dC_A=G<4rWtD*g& z-*_p&OPqTAO!>FbE%`P|If}%$kiTdSIhW=ePs18E4&TCU&?&)JEmBk@M}cLIrf47i zx8ibR_kxjzrG+gs*va1)wzv*!sK(vWzH9&EM%BOZ;XJ1D4Taz+UvRe)IQP zfc8eY8(rD}|0P(>lyJ|~eb;oUxsh%ulSX*$J`+7<-$uCc$PxdWTiEwq5*a_A|0kn0dZh|Wm zpChZtFT}_;+)~!n)75K4TfaS$UP}D-DIXWQ#PH<^-bT|WFDlG8J<~{%)@jt zZ!yWtBS~h#n9PDPnFVVz7?W8rCbRI0r%akUtz>e^l#;0>)59Wqa^qKKbN$h(Y1YDS^LPGvi40&o?e)>yk!4% zyRu4_XLpni>$rH~#;w!-Twe2OUG5Q!8B}X*&5I@NPn4XHiHkSl#wG3RN>1n~S+p*v z{KoPuhNE!9x{J>3x@_alv!@hn{BU~L73Isb2X1^P@FMX19#=+hSkk`BdBaT?{Tu;o zZ?!(i!r8JD)|FRf}nDZU#{H05_>ODs=ala)6 zvR__^{n(n0@^xGH+xTUucE94d#2Z_)&Q7!S=Pk=#wsFt7n>M_CUcc-WcIs{3xMSMW z8+Qh_l-E2_p0$3%`t$ntx)MoL(mpb~J9Y%7RU_39{$qXu6SUWw;RinPF*W^f z1jf`yawJ44{KLO`uo2GtG#P4Ye7>XgQ5o*#-245@!fFN!P)UA$MuxRY{r))Pd)D0<-OWwIK~9(Diik|Z*|_d*#rp~7l2p3hR&GBl5HO47M7pTITZGt&-`* z>OB8zP$h#iZG@w56oKJ9q`YN~aSsjeK3tKysynyfce(nVtIEDuT@aH~C#tiZG0NCI zkF`9>c3D1>yC}aHXU8KZd5|Pqam;!&K7$39$a_FoSH@?su29PC0K)n(K11pQrOYm1 z%=_^frp)K7a%ezRexZX9ESAgh8T`6#ett17VzQ0PkYrO5NNFG`RuXKs4?A#3EhLW_ z%UYdjwXupmhW$<0^Y<$3pT~YP_8()<4Ez`Ny|E`g0Q-}&thV9U@51kq*nfdNtN*dU zOL3om*fVL#n}$8@vt1g9-^|~CnukA+I$JFU!#o0YwzIX!U{@M!wZVR2um=qGYlFRL zFwDcy_O==9U4wN(m?bQo^-*W*V1pfIu;UHZ-(VECaxL2>iO2Z{t2Wr>2D`#w*Bk6+ zgKad}(*}FPV1F^#zYO+;!45~-N?3a9BNR%5jWF0$gPmirI)lwI*wqHR#$XQ_>`{X~ zVX&tR_L{-|Y_NYD>??!yLe5K=3-nQEtHfYs20P1OQw$a|Sc}01r}wj1h&KY%v(@-_;=o|Ek1bak^r#3MnaKun9*Z2Cp^PZ8#D!_>ejK z1dc=uvNIuP=b{3|i^0QC+?`@@5m*$1GqF!!3}Ow9bj09EX^FwIG{s;)KVoo@5`&n$ zq{ZN%%=u}H!68P}r6&eU&2@#izij(J41W4ei@`A|#NZmG*P>$ZZxDmG_z;6Xc9&mK zF?gaDezq9gPyN2Y2*Klg2*H`jg&;LuSCn}RD>kN-r6L4#O-P2Si?UsW;Bo5gcUTB6 z)#Z{qwha`5bb8Ul!Yv&kcsSCSLhwZFDe#73Pa!x0dkVoZ*i#5HiFy7^>?r`*7TE%@ z9&MI5*?1%ZP%sKW!4~PGE*1r#V5<#A0Vo&+pj=ARBxU=)CYZ8I1JpkNe$!lM8b zi~>+F3P8aq00pA}6pR8;FbY7yC;$bc02GV@P%sKW!6*O)qW~0)0#GmtK*1;g1)~5I zi~>+F3P8aq00pA}6pR8;FbY7yC;$bc02GV@P%sKW!6*O)qW~0)0#GmtK*1;g1)~5I z>}7*d3JUhQ!6*d<>j~*5*HQ`!cACK`1qG`#7^R?Ka}7o*C|Ds%5|6rAl!AiU0#N*d zE0A8P{DRl@VIQob{gcx6$NI2~Ht41D_J5YOZ!2rhsce58{-B{HpNAbj!I4p);7IKg ze7bbl(+;1YrF??Fh0U_|SITnle-WCllsfQCcm_+_C*_p3ua9c7<=N|pZN7NU#@DC4 zGqz@TU7Znt$`hCcx8PICE!c6wdhrOt`<3-n$)b)NFWH^Mw&S#9c#DKzefk; zV0B%!ay`*6=#+G9AUyngOO%PIe~&+}QrO>;O1ksaIUoMLDT}bDJx;PE@$GE}Gv2rN zI*@qZ9{qn#-yS`H#*HWbY;gh_Q2X{+zWn+2Zj1>r+W<}j8EkS*~} zL|Y{ik4GZ$1f#?gY>_?^i6_`;&s;FbX@tMjC9K!6q7Pp203O*iQ}ibA$cLVD}sBcLw`|!TxHnzZr~!>k^i1 zebm`H+F-{S>|}$TYOqNLn`W?{NE5NwOCLr3dH+M?9J9EjeGlZ=7e3@%T}91n%A4m$ z&W+7l@04?U@_VF~at}DA+~=qS-(@MMymq#fOO$bN+}Sd2idA0of|78-{ypL)TuIg* zCE;eKC*j5-Z$-jQ#gRz3HiPkrArfwtIlBf&BH=pBS@wPZh~gU^x_ui3+$rIzz@mgJ z#y))s=i#k$O1Q()l5j_*DdBpgE8!0EkZ_0YrG#tvW+dEe-?W4~B!z^#)*<26LBbL9 zl5m&03#X`r6Ui1Gj`btiijzw=s;^J*XIVp2-}(C}@RV%qf+P;Vc2{S6{_l`ztmFGa zqU9r9DbbF=p3Ao+BGCk+ zL=$Y0J`#y0*lL4Oq6tQcCf8D;2}X$~7$uru+YCmDCKx4}@F>v)>uxYgG{FWMj1o;S zN;J8a5=}5lG{Gp*1fxU~j1o;SKEDN{L=%h>O)yF{!6?xLqeK&o5=}5lG{Gp*1fxU~ zj1o;SN;JVJ(FCJJ6O0l~FiJGRDA5F?L=%h>O)yF{!64x^M&FxYI}EndUN}Zlyhsko~?JL#XJIt=*cq*pgVftL| z@-LCBC76Dfdq2iI7o%M8XR0}Kxy#Aoyt#^+&y^=_!`m149F^xWbFRF7RL_lj&dy$e z12u)N$Fhwdp1oPkpgY5L23?ZD(jP-=mFg*Uv7@D|mFbO^E@?wVV!Uhqd}*uVFG`qF z*P+f|x@4=~TL@R|l({_h+f1?ZHHD6e$BA=YS11qKGNvKCXqzSF;Y;gaTI#8FA5M>% zRrgf#S#>2@U#gMR{G`h4Mg-AiT3snli8eb6N21NDa3qTC3Ul^)9El>k!|+};JbJt1 zwb?Bw?M`iWIanM?4Pc+1HXF#Pb*bHJgO#&_)9S?UG%BSwo37ZC!Jdu)S=P}W+UzXF zp9ub0ia#Jxo7pq0cz%i^od~0`F1KN7B#`asFm--J=1l8{Ff5uQ*6r2^&M3th>Z;i+ zq@vU9Mx{MYT{qBN$7hmk(UH{X)V1iewgHNCR)S`tOYCBG?g=xJ*{ptd&-|YCoU3M{ zyX->6yWL&6TmaHGMT$;q<78yc4vd}Fwn}0AlB<{NE0A1A(e?E){wyp|=Q^4h2H{vF ztA&W#uCF5XJGHLr_T8PV`th3ZRRoL@4bDz3t753Y)e;)gf zvEPP0U0-Y=c|I3=>ZR{vPhE8+_S99!V}Cesic*el(&fczZcJ+O$70X>v7NB>)qYS_ z;sWE5=qtgfuLN79k3?Syw%TCSSAsoYFzPG8UNji>m0;TpMtvn1RTYoASe^BeOmh>A zqnyI)Yj}*UU}qSNqnv`BZ?I~EQB{dOw!gw-`zshnIR#s5u*VI?Hd%OE47S~1?;ETe z(pGrP1+h2CV5b;tl)=UrjDsn1?YRc~p}{US*h+(~GT1tUZ8X@62K$r2P@eQXJ~i0G zc&5mGf1{70Zme%aOMQamN>fXjyeR9`0nedOPA z@XkjWkJnE(ql7#46Zp9W^m@YGNJ zj1F=RTc*}eea&_0>8G+3`soi2{nQEisl$hUTIH@-qWUR16HN3|bSBs;6dAs?;H{k| zCD%^$L4ANf%bJ)vB>hn^>4RdnQA_nvzdO75r1%Kp)T!_6D*>^?hOC>^PI>r?9X3o? zJ8=w{+NleYk6MVWH?`9bu%~vSuAp{e;!!&>$;mqnduk@O4Yp>w*k~pmiDnXvnn^Hf zCc!u}Ot94kqh=C}nn`%nOoCA}2}aE%*fxVvGYQrSVHO@WlVArMjG9R>#!YzCOoCA} z2}aE%7&Vh%)J%f0r4{T7gHbaHM$IHVY9_&`nFOO|5{#NjuAvQHs4@WPlDZOFsdiPes3_UC&B(`FsdiPJ}?;7lVEG`9N|$H zi|R?RZ`A+vcO+L@n(5+8weM*!H4~?4^;o7{P9L7VDfx?=N!(61yU3+{gg-sG92 zhUyUGn@Vsqt#X`qxRsW4sPpb_s6vF>>&;E4+bPlI6n%Zu!_jnXxi^1Z2ag_7qyE~ilHmzGfKmZnfT%#Tny z*h45C1EIv1CQ~_@jx>TNJ)!h{b6t8usVId|TI&!>|Hk0luYCxmpQe^4B8+5y73IlR z6cNlg(sIR~%*Fimax#@C7fN(8@xj4YCsI0@zK?>*H#u!5lgzJjb24$9s|Mu8+41(IN^4Mu?^7zL8>D3AoBKoX1sNw94Oqd*djaS9s|*4JPZNP# zu9s|Mu8+41(IMCNP@9t7mNZ)FbX8WD3AoBKoX1sNiYf|!6=Xf zqd*dj0!c6mB*7?<1fxI_i~>opZ&V=tnaP#T0NuquR0DIxkPmu5FFmJHyg>5hOS;8H z936nLrzeh3op%>Q4)Np|lk?|GN)$&??@^EV*BEW@QQC`I za1y8pAdbU{DB_@$NTG%1?2m9HQs^ebd)n|=iQ+}kg(%8S?-9M}Y-ERHpS~zc(|c5q zmMF?hQxtXcBZ@kEh@$UjPWKc=2N`jbo+!#S*QF@k2pjT7nQinhbX$iU5e0q zB%(+}5Z@bIg(Bk5TQoem2%@*>ZT#^)hm<1d5EMk}faEil_ETrS!=h&?W2ZOPJ&K-g z_`67B97*#ldOpQ96g`$)v=erv=Ik3{SU zMzJFp#g1SUJA$n?7{!iY6g$GB*b$6kM=**V!L}KUVn;BF9pO>z2u86Z7{!iY6gz@Z z>G=UMzJFp#g1SUJAzT{2u86Z7{!iY6gz@Z z>V_8H{pRu%!lDX0Uq=_JF~jHP{OVTY>go?s1(yN_S)|x{}uy zm~e!}ySOA5rxryg9C6Jq&N9MeBg{0y>RpyB-sN~bPnPUr+Ub=Ua*t?3=HpkYSxEo4 zau<>)Mkg(INSGxZ>b$$;@#FtdtDU92a#yBWxNBtg-mcs=1w}zb4lAgLoGT4>1CB)Q z{L-9#6h|U=o-$`yW#UE78z|~dk;B1fHmWnQPfz5)@8dPaNY>(-h9JpWT&Y*?a!xVo z?IChTD{sI=SeFI0PE9RxW+0Y)G0R@LOWx8uEptJ_VhyJ&hX&t{a!xTCr)(T=#EKu0 zGZg`pmAi_~b;(4I$D4f&j;v)R>rxfq*Fr*8rx*=>dSe&e`bhnbpPkg9Fh7fK%}B19=s4ndr01er{S>LIaU2a*zuh#{ z57gQ3sD?^db&GYIBFgDGvRB=@9Oo#inEez>9oVCv-^LuI=ZKyn;`9(vB=y6dB5EM^ z6jAh0b4YD4?nBQJ)0R9wwkf9Arr2WYFKByYIuegWObJFYCDE#qnHwGo53ii1f!S|9>tVk6jOp7W-v-9!TKAFVoETIDY=$nN-(z7f>BHf zMlmHA#gt$-8;p~@1!LD#cyAc&F9zeH3&Q)tVEIT}!4A?#ovkAccC^7bHB5Mw25T}{ z*kG3!jFZ|VEY}$9dV@V;u(bwz&0v2v*dBxN-8`|^RUdV+@(gyo!TK9)hrxClEIkjC zQ=DOO)vqPao=3W;CC>hD^(?b!>(>-!$$d{E%>1sNWdzw?uAX%kvR?!ltGfuYMFzVT zMGiM*dkqELU%-M5MCE^9ypHRG=f~*BB3bF~~JHj(i3EZ&2{@!knN6nPtPgHPDvriq6=S%AoE`M zN(7n5!dD{5qSKv3kVU6Ebs+6!RL^S^Z$ff0MhDg&{CO^XC1NZ(-ASg7yRDLNC~ClW zb`(#Eu~?55>ovt#H~htVwET*(Zb)E?F?y~jvW~@`VvO>HVvL?~iZP}!&kw_%B8=^b zEyAW4j~0(agb79wCfFip8H^%KFp4nYQG^Lb z5hfT#m|zrPf-#nYQG^LL&R}en1)~TP9z~d76k&oDGCMiC|$MVMd|VS-VF2}Thn7)6+16k&oaBFI^S4Bx!VUssCHSY87hN*L8>Os}-!W)B;wc zviEWYtJ9F(BDYxCMQ*XfFW6!niQKx*oV^`KBDWqkXW0vlms`I_L3YZm^T6Wr2g9*X zS8jREe3CV(xPsLwxCNb09J+B^xqhm0QcMMVs$yFna!VFhm;`>Q;vbqQx8kRR^Ft{m z`CP$jgu37`bpxlLD}?hE>U@roM^0I_103h8baccGR@a@0fj3#%-YKiN!jOlbt4VIH zOt6LL)}sD@KS5U2<9vxa*DFC*@ohTsb43@Q5?STF_>{;hkHx1xR5zB%QB**DxW_I& z)uDd-S;@dFv=^zG7(ncQD_N9U!GtT zT7uD!C)f~!jW!tDUg6C#Sd+mnF&KrG@Hp#K>^)$xbq1r*5*|gB@U|I@qDrv;7>v{S z1pA&o>TC@#*vST)YOr$*R%bB2YAyGjXRr$mcC*25G1%`6_6LJ)HP|}_<1z;lJ}z@0 z_vnUaj9}gMQD^HFgS~FB|F5JJpO1RkDF0IG|5lGm>w7?uQD1BIC|f@5T zu`-K%YBpFKjzm7KFlSfcNaWKRbC&(bc=_}*6kex%Itwhyr$X%0mrowvDyMupBrW;0 zGb6RP$|;||m#%!0nd$EGX%FTaCI7%rYWXzdn~_g%rI1f~Ddbah;U|$#-U~m8eDYZM zN#s*>;inFiB9VmTBK8-3>u7c_i2(eI)Wo zu+;{mJQ9rZNUo(k5{&XlFv=srwi%4_NHEGH;ZYt5MtLL{V<;HqkzkZZg0V#vjPgh@ z$|J!jj|8JU5{&XlFv=srD31iAJQ9rZNHEGH!6=UeW4a1Pc_bL+kzkZZf>9m`MtLL{ z<&j{NM}kov2}XG&*i{Cj&q=Uf8SH+8Z8q2|2K&fhpBRkO!{r`aeL(Elvd3P1;6219 z_qd1qY{owM>gar4apk7zu*=C*m|isjhl5_i8c&>Ag(+89=Ba(u(9c_jm$e^ezL|*G zQ`r?Y9ctaDQ9UbuRF?DU$lScaZJ*|z-IBL))03a!qluM0KjTLeTZ`2~P{+Y6=b5*^ z%(b9oQO)x9Gjs4oKP+`#vl%ag6}0zQF*5i59+$<<&D!`y)B25HG@H*betBtGL3FKi z{iQ`r5Qi=Z7Hr&;*IImW>&DH~{u{H@z1NJwQc;(L@a4e&?0yattY5@3M{jL>2OnKj z@9({m?4yfG-rc(~*?LhwK+5v1`(bvUjobe!MnI6uL0-crBigNw9Rtbfmx{qkn)qp$JZ zvtJwRCHV#;7dqImehNRggOuFx_7r|@2kD6^>P%8B5`|Q?myLQG%>I4&(IWK?#`f-* zuU^q!mXq}i3(B&;|L3c(ke&D%qbi*GYmB+a>SBD^2A1l$<)i37k=5BB#3>PH>v1IF z?0tjn#*v6K_R{6-VK@?Tc07*cEM2_u;;dr7L~-^xSe$wE9QNsmGc~izYi`##ukszco>u#V#K;QIUQDqp`~$Oji@Y*(l4J zRo7?!#%fRk!ztPt!I6(k{?gUw#a74Z^BRlm#FzG{xvo%!h5yO&QEa$U_`K%9#5L-Y zvy=(?yo&Mr28GRx5oXb^GR{%wx_N(Zz5f>z^=q6?%dW4rN{0ib^U`X!9?LjZxZuy#CH_!P!$)t$@oae@`5m3k{>bQ?Ugy$ zdRmEaCTal~c063C*>i$FpX~O#hG87D6pAvHwcX3q@BOU9a5FAhDn@%{R$})4kCAuV zGJ3g8(Pz($>+!Of@=m9z-``V3GK<|P$G0cj2&wJ4OD-hiAgrS z&OIAa`x}~FY zyjS4Q-q@Dzzz?UNk*&rZ5W1{s6uc}98?a|ZpkI-NpL6P1sOWE`%bNHC>?xEl!+sZj zbAB%c^$P4cjQeBUkLT~jo_3$c-WJmThMbbQbUYFvEf|HgV2kvT2x-Aq8;n9)FbZk8 zmO@%E3TeUE4hXi*U=-4VQ9cW=vp$lgy#%8tRd{_3?|6eTJ%q=)DLfA23RZ2f%MEsg z!LBzLhjHav%4fNj!?=RIVK7Q+!8nX7JW6W84oBJw)>|LR7c2!EVK545!6>BVTGl?n zD5M4ZFN0lcu-gpAY9&1Qq4l*KycO&-gY7mLI#`<5Ssx*t4R)BpID<~Yf|8=`admsa zxVpXEqsCyf490I+3U9u_78&eHgWYSe2Moq>HMw@3!L}LfU4vz!+b6tE`lz$j-CzeB zY?#4LH`tj58*i|44HhuiT!V!Sw%lOsgh(3v%wVexHWM03!dI)0IwyR^bDwlr@V3al z=&*brCFGm#ur%rpOLIBbx-ItlJiL8r4xB0a!O{8YU9u&VB(#C1{r^)7v{KjV~XLmoCZGnGaiYmExi>kd4zWlP^3XU6h+lSHn3h zpSp{3m$|OczE!$ilu=2#C=+9P5Z_1_WogU}oL!Vx99@)C(M5UIPZ#B3bW!RZU6h;A zMX9$gx4QZ0qTHKK7lkrix+uR@U6iOy?}jeQYe~8&r)K)-qMW1(4*lM-T@t>NOcjc$ zhN?@l920B*YUR5~_Py_tJf6Hu!tPiVYBMK;#&^dUoZH;HB4A{e_Mg0TxC7`q^X zu?r#?yC8zG3nCc1AcC z24fdQczt*i#->Y96n59RCT?ze>Sv~Zzqz!FK%=pHV zRq?-(4(QTWW9?SxV%-H3y6Bxiq(qxfzF zK68-JjrbE->_*&;eY)KU&sn_Ejo>U^8R=N;qZ=_5>=FdXvf4em5zY@~EKcl3#E)uB zR_checqgkHT%>M5A7|{_>YT+Z-G$4H?Dx}Mn1btML}P`yZZEnEKUP62jO#93udvf& z)O@tNFk7ATF`^-F@h6UGu=^BHM&`xbK)MSYSa?I7dm?$)fC(HOz32wHX@)q0jaBtc zwZX7TbaGx*hJNZ#{vV<)qx?taVEd&*{_oBhY(0@g{vYiw|C?cGj52n#k;$Xf?`$<9 z(QM@Z?u_2Z$}2JVe2bOgBA5BrQ0#~VU+K220m@QTaPQ7I6Y-^`7p$&oT2<=Nio4yr z24|v%!oBb48t?`yQg;oOs0*WAgNxN~`3OpUNLRSTb*{Q1IywP|b4w7xpud{p=dd!Jg8;343-8*vDko;D^|Ax({=a z(|xM3C$AlQ+NZFzy9lR1{74spN76+Qj9mo57U?7DA_%tHV81Xtb`j)Sb`gZfE`ng} zA_%t4VDB0%6JZt}I}5_=Zm@$5Hqc;03^v+e>@3K&OdGMs)Deu+eFS4?K`^?}1mkoc z!PXg!odvjIH2Vae4;ovrU1>kxbmgMx6EK?8H_K?iM<}s zE5hrkkN!tXf4<23ElK|>%6|0$y}gnCd-8jwmi>3f$^IO~>;G@EA9A0)C48IM9*Lpc z@8v4@5BeH%{~DyP$o*S!By#^HgT0L-5%iy!vzcf*MDBOTk(|8?NAYrhCn}P2)PQ3P zl>4_}pT6ApHEIx*`wP;N`B;@Y=DNL>`+btg{XsEm{2P(` zb7F4bl>3kFE4hEe-pT!+I^_OGko%7&k^6_b%Y7;@k@rU{QQl(Y{YOfaUuzwMIZmBi zCLAS~_a{2!y;!=^8e~0dW&4o#x46suF{qK8fFJMq=M5IS%X`Lgp}J5e8^yODDeebq>n`63%1%|l=y;C;>)#^_<~X5 z3r2}A*fxVv;tNKJFFZGiNMu{&N zCB9&k_<~X53r2}A7$v@7l=y;C;tST>V3hcRQQ`}a5??S%e8E@=1iR8;l=y;C;tP)w zUoc91!6@+sqr?}C5??S%e8DL31*60lj1pfkN_@d6@dYb0*jR&6;tP)wUwD-Gf>GiN z_7j8MWUxmJw$@;j_;M{JzFhmU!9Fz@CBE>oP)3D!q(16m9c{1y20Phcl=yNjCB9sn z0}(G+SACQy=Wj$VCx4rv7-ve_hgP)1Q_mI2UgvTc77{q?1^e^V747fiy@s69A6;w> z`yzLF1x{kVQ11Po|EM+FtPFc5yxnDZH=(Tk#ke;O)|C$1l)IdDsiNjf%%kC#g6jCi z>74f0N=`VivIZ|QJd=C>s~c8d66)HvKv@SXW+yFaNEF zZUmzzUqC2bZb3;tBkSmISbfpi*}RyaVCEYMn>Va(nBKMR%;nib&F6#`%i)tgn=BxxF>JwEbf1gDkwoQF6jFSQcc-)45llj&#Dh`3P_Bv<`ecx^e>kD=RKL z;Z=K$3l$%NGi`*lX8fmI_1v#PGH}0H?ac?Lm$b=mWS^X8vkD{T9nV$r+nnc!LG1Ys zb^g*NTlJp4J=|wWf$Y`d`Q%`e=XtsQ;zG9>tN! z_NUC**Kj1F{_p1OqbS1hqJ9vnlT*}lD6&89d=K{NiF)53yXN}`gK-PXx;~|-pP)Re zbcIh)>;)d8ewgA<0$;vUonk6b4h)Ll{)aFc`?f-}QZAfr->hyLfUq5yAogS5dXOnZ z^Oc=LaE-io`3q%-kMmK=&SjaCt$Tnae-)K&PqMcfD%G`ZuAKso5IR4K-4oXgP}kj@ zAj5mL?V*e^a(AUW)yT7Z>M%~H(P`?jHQSe$u?H+gid<>EmQa1H> z03wPM{o?i&>bKjPwv*M_XgE1|Oi%H5@#9FGmA{6;*Bjdmw%#0=xm{!Qd$S#(Se=PI zf3rqW^sd7GP3*T|4}sJ6FYG_Yo~42Q@4ncR$5~Mnz2(?b9CLyQ#qm_^4+1_G_vPmU zW@FE(GM8h|6>ymoyzVF1^ZsnhY_Xh+Hc!Mdk3=jBMzJi|B7G!cS+Lax`-S200Wa55 zEDMifSul!a!L}KUrByIWWZ`wzM>1<)Fix2f9@9dw;|<1i5Nx8sIAun#YJ*WW3&u*x zqt4b92D{2&cN^?JgFSAr^#*&}VA~D$rNJ_h%5slx`lyRVmy%#Z48{pHf^mWxkGfbv zgI#5?s||Lu!RTs|Ywt4{MYUkh8Emt`P|ox{wi=9U;mUop^$}L>HQ3Px8*Q*N3^vtZ z=NK$vumuKNY_O#UD@R=Az7_iDn-#6ah;h22^X0V5FBtrEGbM|E%iBSEUIr}RV+jyaR5~_<+s6GJ}A26%1PhY5dc%S8CnNgv7 zRa!!|Ax)vWR4EZNcO=xxH#z0|0#TtF@(`+%AolwD5UO(>LUpx6sJ23=-no}THL{mN z_39Kt^#g}corJHCcl9AuUxtjV^CwhqfKWZ%k5C<;dKi3|icpn#5S;h03MG*focFaz zU6ov<(pP&F3O;&>?jn`$*b5aYy4G*A`t9~E3bMsX_GHiJ=|3Py1%yw3Va#HnBu zr^2H+73_F}QJf0K6c8T8sbCbRf>E3b#`+`}#i?Ld8SHL@QJe~o;#7DPr-D(O3Py1% z7{#ez6sLkwoC-#9Dj3D7U=*iE3bMsX_G-3Fsf6>O8iC{qP{ z-C&fdg6%OFWvXC5&_`XAOciXT!6;J&n`kh~RKc1JMwu#Do53hk1uI2~<53rjGF33= zIx$`zRjebN;8A_H9KKchDPldUZ{#i?1W1X~PyKsT??4HS_o$A^!8$QXJgQ?H9#wh( zPgqC4YWa=I!#Bq1P#u#e4psX3lnXWXtL&btG@MG64nQ%4DupvL&W)-<9^*FM$39d# zKIso#g+z8z?qBUtwcV(3sXbPCx9voZccYHUi@H%`U8pXO(N)w@sYfqPOOFQ9)T2N2qerVf^yp}4x;H2p+~<0=H)nj4tg};Pmiw17=T4dQaVmK*kLO*K0mY7 zZ~N;KwjOnJn!dw}V^%)r_j@Bx(jp#QMs0S zR50pM!KgV73PwFD7}G*9>QTX{M+IX! z6^wdRFzQjks7D2(9up#QNgH31*0AnjCxcs>QTX{M+IXY z5{!COFzQjks7D3kb4xJlQNgH31*0AnjCxcs>QTX{M+KuE6^wdRFzQjk=r$FMdQ>p# zQNgH31*0AnjCxcs>QTX{M+KuE6^wdRFzQjkzDYfLEaI829=#1EG~QY2SC1x+@Vg39 zB}Qe0f0I8KX=^GmdK}8&H!nuD_*Cl=#tz!ALW=DJF{-?x31U=Sq^ZPcoQu>JqjZtl zVl?U$?O{7b9U}C-bVX<_5?n+m9Wo+9uQ%8oI1-t;)|`C?Mz4zFbKKC1)^=|DiTVsNjS+NLT_-0(DNWd zZ`n%`dd^;o&;=<(=uHk0It(IorwSzq=xH6;niSY!4_h^YM3`#?p0qQe^f-J4BIr z0`?S(Mc7khat_n~WA97=w5rPgf0;p}Q5(iI(bTBa+%nWM_hy7qP*73<6%7!<9S8(* zK}JX129#WK$r3bj$x_SGG;2UcQ7gfH7cj#$wQ|9&|MNND`#$%+cit7Te)?|;@427* z-1WWheZSx5oO=!;lNx$N=1`j=GIz55A~L!1#DCwWh)bSJIdOTXJ-;L_DM?(4Auh#Q z%nkmx@DNYF)&o7~)dwZ_W^xVu(w{Auh!bmtu%ZF~p@9 z;!+H8DTcTdLtKg>F2xX+Vz)Z8PdH|XOT{5B759p>*PX3la!+v$amiX+IorLquB=DT{9k9*i|xCe>O^A3K>KE-JHsY< zLMPTe<0(%u=7($gAM4CI?#TLOQcE~WxZ>t9HmRyE)iASw7cBX4n}|KntF)MHwAEJ zNpU$mtWEv=@DM$wjJQ;!4)N29i%XUt7J{-h?&{o^NBJIMd27wp)<&>cFL8rI z%p@0on}&J*qYkNgh!kq_scs)G@b#z-)L+FNSx*k}dxTB!3)N`zYV|odr74p>sQZNY z^HR%_4&JT`%Tx(ZtJs-gzPG(9^->UrY89Gv4;_Rv4FPFL6h${%^Vg&gYNSY|eNg`> z#NJa(taA6zE_z-FOZuQ*9Ol)-1KlO-cze`rV;$1vbbn8aNzP-}hJt-FW7amef4OWW zKiu1dc}7Q3AJiL%`J@l(+|KByp>!YA>xMNG(YZmGucQy^YYJyz>D@v~HtB;pIhY>KF+)(TP0 z?M;gT#52oh*p%(hv1v|3FEi;Q5lu-VS_}~_)*6>2qQz!ALqv-qqO~p}S_}~_hKLqh z;tUZjhKN>NU0jlg7DGfU4iPQJRS?_L84oKlM6}`#b#|IFq_g5kq$ut(XT0$xc8{~i zol&o+b>DQh+*yNNTdlixT(YL2&Ng<2NmSh4&c5di+oOFTq7{dT7Q4yWZ=B6_*5PcC zvxl9%=~E zP%Cb(vp+kde>m;qEobjKqc1qETNjt?9&)yxvrU|B?u;C<)}?8!_Hlr-gPpB!x<+w5 z782$nhi2nK<>?&C5DxgOreTi4NQ(p!x|NcwnVw448R9-Z+9xqjd7RKIT>_Qqa6 zd*v*=v1bF_RNr}YYzdZs1WPc(9%ixxKem!A0gptnhpi;h|GY2dK`2YGirt7>%8yLG zC0J%(qHMmIhZUB9yg>DqpiI6wYY9%M$`TB((h?kBi6t0enpz!4%L1HKbCuP2BL|PP zYpB0H%238wjay>h#Xt;7L#^~;QHBTKMD0ZY&~CHu3M;MCNztR?tcP=kpj z_?B6Mrz^1pH<%?z`g}$d+5OX{l0Khh3FcR036RB$!!GFYK|`~k&rBLE-P#SQ0h%~E2R5;P7Y^IaSM<%6izyJn-n&b^!c3L zP~k?-$e*<2SJwR#+2?a7$(;Jc>k&EPKL?|TxF}6RJ zx7ZZPf3Zz-lAlpFB>5>x@{1w)#aiQ%B)`~fXGnfAB)`^0@{1w)#gP1BOPnG3#gP1p zL-LE!C_@a%FNV+;+sYY|Uku5wI3&LqS3?ZRFNWk7L-LCu`NjBs6GQTgA^F9S{9^pl zi6QyL5MN?QelaA!*v8I~{9;Ic#c|Jzakq;h`Nfd@Vn}{5B)=GvUyQp;49PEs99E&n6ky5C#f~#0W6c{vog!Ob10n#wp1?=cbAhh8Amdb!iJ`Y63K$2~Kz z?_ISu+J1ArIWPugxZ_Tp&kdN@s{@)(y*iL+lO}Jml63lY&Tg}kl=!{A^a(3Tr@!P& zPqX_oU#H)1-;1nHztW=6>Brl&TAg0Dak6weU9~mYE~4mr73lQA7Ck&f?@*;q-@mg? z&)465gC6Hq#J*t%gTfB{?Yc1fTvG!*GblyhpA6KTila7|339J1uqq&!L*~T))ximZ>l9g)~*W4sQ zzh?iUDCubVt6@IrXjvOs)}?3E7EJ~*tH=y5bhuBVK|XJ|u( z_4v+Vjiid9G0dx@Wgbff{^DUEd6wmdfqdP{)Y4--4vIl$4O z=xkIdI($EyqQhy&LVFfIKDuAq)}}$P+Y~)c-yb^L9v4H8i?zlj>2a~y z&d}px=y9!!9v4H8i=oHGmN-L?i`CdMD-LBYwx%_jg7xi>T>l*F% zug+d@_FvZC&)W_3v1{*<)>m@)jm87@t@}-YhNqYNlzLdOJzTUpmz5!Xi{X>gF zZy#;b>h*R-dGSPV?^~7L{(6;qyKl96yI&c-{Vmtc)#+^w%)XC(6218Pg}*BG_8JBB z_WCJJoz>fWrIuy&_DvbRy{hT$n^u?ZX;LPsiU0^GQ#~=`GOA<&&js8e4Iz#w8m&+Zh^LacFGCp|Qod4q{vhF*LRq8e0sFEr!MxLt~4f zvBl8XVrXnJG`1KTTMUgYhQ<~{V~e4&#n9Md+c_KT42`WgG`8Z<*kWjGF*LRq8e0sF zEr!MxLt~4fvBl8XVrXnJG`84Urc%Xv#HG3jjV*@8Rva2z42>;D3vn?r-D1<6o$U;b ztvEEc;{M=lo-;JI;?UTNd)?W;oY_0paa`B9R2QMKwJsW4>pte}DQ8(d{g{1vg>%^C z59znKuU)JDZA-1a`nC`Bv79!Jzki#xUw?x(>NeK7<+R_Pu(JOsxu>;MuAjkomIf>JH@s(U zN=~=$2lEX6N3(Wo(Vk?mn`U%v@3T{T{frqFqtvwxVcv%$?!BosMX}^;*O_#DJfLUZ zAlyM-w>P!U&Gp#PyBA#`cb#rMAm0=Bczj4>2hCFUv@y*)o*K4@wE)JRWW8J$$J0pEFh5+=OyiX6S81{Bahc~Y9W=e* zMNT(Xv5;_WEqKeWGt*aiAs)oK=vdl!YUJ|?~=NY;-zQ7hqlwV~fiSqlLJ!vJ0@|S(-dsdPtU(Iep zEv0X#d{O>^eT}lB{7H-A<;YuYTDd3>ZP&Nlddsa#w_ooUvd;XbX}rJE?bmk*(bNL( zUT|B!{`ho*m6en!oC(dj{cWs~E_TFmYt z=G>xUorg;{Io{9K!}dhcRlZ)YFwcJ^)_F*+V+m);xu#(!jv7C9*toGrjyZ1Fq;QsG ze9sRnF|3@r!Ru-B7YEj-rbK3)RnH1x_Ol$X+Gz7UdMu-uQuBw0T{RdzG%Wi}I3})x z_4_+=>@mZ}jT%4f@bP0Oj`P3+!@^a=Nh>gEF0*r3c13CjNpqPy!mu?ljemp??>2)+!o?~QT)|L64;yDzg)}h@m5B7Sb8=bG%|PNFuzKer!(cnERGtB zq{n-N=p<#lewbGUfj%!T&DhbS`%v7u(d@y0?7J&aG6tbzX3P$KnvP2z&0k&zX=v z%JyfntmQnL($ff^4+5X$J^SN1oD=vo4v@g7B!Mr6z!z(cOA`2Evz;OE#Sr*f7lAK^ zz!yW{i!E`6z!yW-E3PgssU5o*jan49iO2B`6x-9;-p**$qIC~-hO8Gm!`Tm=(Wph+ zyW1JRh+_OAYTaj@)!20uTQx4(F*{q&83JE%2z+gCsIy_tLM3tdJWg`PXQy>(illX~ za(1<|`ukKUNzQ)g z>@sJ!IJ@21Q_dDUd&L=2U&r#kGo-#4E!8Ql7H#N^mg-_#IosRWzRnn?P;m@XsO|k< zHUhkVU#$^1+rDl6Lu2&Qt+cX@K_BgFspHmzR#m&wRy#s5hqGxAyS|AQ*U{|P}yAqv7!16%qH%cC8Wr zxfbffZ60wy+NOy6u{K5AkGB0G?)jV$^*^*JqMqk#PSlUF=deUQC5d`5M7>yRT#~34 zo9zryFNUbsx`=u)M7>`{hqF#)o)SnuC< zbFr+wy-`ZmzNu|@()LYnPs+>N_iNid&f8mS^xaD(?&nSEv1*dIf8TDfrnbi|asOm) z%$yORHIQa*r)8kzIJE>g# zm+dS{$n^VDx%p|c>;zKt|3`E6lIdJrQ{E+I`p7#&3O*;)lRW)9L)tsuA9f!j4`u{< z?K2@c-%zbSKie*|wD_e~l0Kj7Y>}0uz?b;aH>@Om-o>5@T6&$8^7Z+v_HD`P^LZA9 zKEJ}I)#~%IE!4C6d`wmP{PZ&Vd`QShlAIr6CqSd0`Z%pyLZ6Q}jjT6Gm5iTY$@mk) zhG_1szy2Yq>Owzmxt=8Fiw`nhNS_b3?J7Cn`qA|HMd9f3lk*z%aMN=7ye+JnpPZi= z*fuHooapnz!mmspR|_sU2kX`GSl-U(@vYpDWPkR~6T=R7GqSKFMq{UMc8t zuFqF5qG<~!oWCO6v<2JP>|$R@dQ>m@UhJtM=(FMI(Ca6J`EEtd zjrSYV?}|#l)0U=qGJZf<=c}LKWW1`TICJZ_$j%16{yF=1GQj@Qdi|<+{zjW-lksnw z&Th1845u{>`uiZ8qSMjq=yaYU==9;XAC{kEQ}j8{*PK2_$w;45l0FwhpNsMPFLtK0 z+0M}CibJ0(4t*|$J{Loui!E`6J{LouD-L}whCUZVpNpZ&#kO*WJ{LouD-L}whCUZV zpNpZ-#n9(s=yNgjxfuFf41F$!J{Loui=ofO(C1?4b20R}82VfceJ+MR7ek+mq0hz8 z=VIt{G4#0@`dkcsE`~lAL!XPG&&AN^V(4=*^tl-NTnv3KhCUZVpNpZ-#rAf#ue0&a zCOP||Gm`FFm!!M4N77vkeJ+MR7bEE|M$%mjeJ+MR7bEE|M$%nuS7*C9>$*xVjv>Sq zw-Pn`@{d8YKW8^x6`Fm@q`o8Ht6HY-SU{#D(bvzD=&yH{=ocZ;896+^InBCu&?O}L zC$KrqN0;bfQ1JZRdF``7nr~jMG@opjSJM0}D@mH)=&apJ65k7b>0&EMn!oN#X{?to z&6&$e^J^^%X?})HtC!|grp^adCCwSkaMuPZ(z`^59y502(LsL-EZ=gPq!{QJ;p2+RYS~=JhKs&DTvS z=Z{62ADh}iR+`_Ek><<0tP$N@fiyqAxLie^N?hlpd6($HaJ8F@$ko-t4VAPVsSoqp zgnI%huC>=JDPNI0m9>*Oqqr2`(!PRG)Li=9jtD`YEpjf1kF@kgf-ov#De1j)i21Zg zZ=Pq>NN+yf2TeNL2Oaut{Dw`D-n-fqY2DAJNN=P#()$3LBE9#q{jmJ|Hbr{#{LD%3 z>+Crz=}k$}TMX$f)*6?T8W)@GY>vkvy|pgVTX9HlF{HQH5@+0tVn}brA-%q_-H-TMX$fhV&LgdW#{w#gN`&NN+Kuw;0k} z4CyU~^cLgp7vsJcLwbuLy~U8;Vn}Z>q_-H-TMX$fhV&LgdW#{w#gN`&NN=$Xoqfd_ z(pzyzZ^a?K#gN`&q{hWajf;^Q7b7(;MrvG))VLU_aWPWkVsz6Y_IGEb#>GgDD-Llj z_LaC)7x6Nv7%zh=E-R@ou}`mXcVlCFKojOg}yV(`X?`%CXLYs8Ug3%g-++$AL!PvDC@a<&Jq#b2YcM^fM{gV0@W8G!t(`@$I3v5z z#@LRVrvJ6Frs?(5Cu!gwZ=T+?#XU{6)|S5h+->bDrRDU1f!6N3xiP<8`br)^l9uJO z>^h~h=iTi%(k;^4!}6)q7ROT)E1^YtPfbG$cN_PT;ec{uNYWg=bc);FN$|cCttE=C z?$AZFNwGdCxy12c=Cn(XTdKTrz(NfonQNr7e!kGowNQVq+TnY~*qKOapJXK|?OD!l zw33u|yDwd6B`NL2zVsTqOY@cXDEqEtm3F&Dp|p8*raGm)P5Bn;8bgv6>ff|oMA4}w zmG%Ka+)xXAK!_e$MrrRJ;z@$5;rNNAmG+>pQH@2OsOs+ovBwqvPFOy<=9uV*wqZ4r z4%zfn5`6D?8?^N1w>PV_d2$qLq@u>!N#mBR(&iadwuSm3Vdtya0Vgfg4-M>lDK&d! zxGG26!EGIu74BnQ&C{O_Cr=CUmJr)$N$7VADqU(07{>ba5Z7Uy!|YXD5zPo6u$rY zEx}(flHU^ibtBo9;O`it+9$8`6mDmSaZ7M+)pSenMq83Tg^w(@k+>SOy@$PO@6fy6p zqlkHoposAirg~ZqvMG&6Xr{BJP5Hc7kH#eIpXXmrDc5z)DP>Ah%3>&GvDUaGr7Skv z8A@3UrL1*P%3>&GF_f~{5@#r7F_g06P|9K`Wigbp*e1@<%wi~I#i5kNP|9K`Wigbp z7)n_Tr7VV07DFkEp_Ii?%3>&GF`7$?z31%nc5TJhiA#0SSDk&$*f>wej; zh-qDW>ypm@Dp$MI&Zb*j_v`fPWm~ET>&%;%7MvElc1*F@wFRBUZY9mH>!+VoROs@h z`Q!^-8=sPEJb&pZZ_AOs!fJ6HF6=yu&}*0Rg+=Guh?FzK)s1y7#9qmKUNTqq;7xY?B`aJy1m_(^2Cw4<%y%EeBblc?FjpBWYz7B7R4_N zFN;^NZp-A6v+DNyCDpC+!u%`^u>*>ta)VQ5)$LKG)$RUaqX%c2U}x3s5j96d7nM=B zLp>i{iMrj-PDtwZIA5<f6J<+=5oNo-O;NUdZp5EpQxq-FyquyvEvN^+mz1Pv#Za_jt#L_; zR&2I26s;JFR_mf@#Za_jC|a>4&QP>sC|bp#XvI*pVklZM6s;IP&SEH9F%+#BidGCo zD~6&KL(z(%XvI*pVklZMeqY4+eGx;^ilJ!5-gAbc6+_V~4n-@5q7_5YilJ!5P_$ww zS}_!@7>ZWxLTCKeiT%nM+E$EaJz5uSD`xjq%+R)Cb*3j3hqe{l!Wr6DY@jo=t=Mp9 zXj`#UoS|*S&UJ>i75kMlw5`}=y9+6;7NKp${&V?T9)s2AZ_l^8_+!Z5lDiGPiM(>) zf{qH)wkNlyUnWdaw!C`($yc(6tHZtCg`M>{?64l01}*y}%h= zi%6*c+LzvGCCS(aeJKy=@-lXgMe%cTnoX;fv1MB-XJzc@k}@`%r0rW}l6F{W89T_% z(%vU5BV!M-B<AK^PFkdV|yF*x-$XGQ%E-ho%59@rA6ST9!r=kDwj6udW*uM*7 zo|duY614BxI>^@z?b;(>ziw0H=nghTzJAB1$X8@6^7R|GKjiCyHYFj;=R-X0w2`ko z>vHmS_aGJcK2nl=6+^y?wZLA^Du#R&L%xb3U&WBG zV#rr9LA^Du#R&L%xb3U&WBGV#rsq|D1fKC&lXJ z>$!FprxUVBtg8CLOJ`)`3UX_H5TvU~*bbAgl_X+MrjFcO8>C*a+raM5PpWpc4!Gmy z29|}Dc1^Odd0h;8#}}quJq;UQ&0HEbr(JWsUyRynt-=ipKC33Yjdta7OE;O_G0d0Nqw9v4PqH3O_b$Qn13lW^{!Qm%*k3hz^lghrkD|)Z zqnp_jeMwpsJ-U-k(W8BAiXNq%GkTQIg}4zmMTheI%IVNQ*mF)zXemjDilIZrTH}&* zsMu_0=uk0qsMbY?ilIZr(4k^WoS{R-(4mS$hl-&?#W-Uzbf_3QR16&|h7J|ucR>st zDuxafLx+l?L&eacV(3sYbf_3QR16&|h7J`&hl-&?#n7Q*=uk0qs2Dm_3>_*q&KZhS zti>6MRO~0tu5|WGXDCv|J>cvSXKy)s*V#Iz0$F3V=!hEjE9QtDJk>9+PDjRFwYC6T5z%Heg zt2J!39+{)iA#<5-m^WZ$wSBFwLBRj|VxHD+U*L4h)?W5OrCYWx&MY68SssP?Pg7sa zt#(Edq@+$IId6By14WXP2a1-yY$XX&9wTvz)$YlBL3*owZL)&&K8r$-USreh1Zn7x zxpZI5>R@e%9g=!iF5MS%HiND5HbD(PFek?b$R zdPz6Rv%-A&Zj`SJab>zu=2tr1jq+E*ytFMMa_ummbfdg5Y_wQ6%4%+#trpl>X61uHnzL*w1EESClI3moK z&NJ^3Vm^^QtBq%5-poFrH;ogf2JAgEsxprmxwQ8x0{$ zB9`TSZHmC9DH1~XAe$nDN7@u2e30#r_5@RHN@A7|hxqetiXi6snG?iIO|nQ3Q<5MS zLlBF##w7`2vDwZL#9|0yt&1QQLlBE0h{cvTLlBE0h!uw*7DEt=aRtN>#9|0yF$A#~ zf>;beEQTN!LlBE0h{X`ZVwX8X5R38qr8s`S#1O<{HFh1vR*g&QLs@J+XWMujrcmqd z?QCCX-*a}TvooBX@@m(exoE6`t`$Eo&??bB;-(~tjmiXq~we&zJ z*@n2s*PK;Wd?y_28dBhjl)&;yRsW=DCEp?r#6mS>|fRHCbiEHQ8juGN~KH^x_qFSsqE=g32&31;U7DH5PT|~7QqFM}5Ew;oN zqFM}5tvE!r7@}H?uaFp`S`1MwhNu=pREr_1#SqnEh-xuJwHTsW3{fq{uaOv{S`1Mw zhNu=pREr_1#lGMSQ7wk3Rve;Q3{fqHs1`$1iy^AT_{_u*)nbThF+{Z(qFRi*Obk&i zhNu>M(b=oc5Y>u9R4Z-+ld580j!Six$IY>3XNNkYqXw;ern5F@h-z&QQLVV&IQyNm z4rli`LpE#OSDjsNcdOWqajDv@b<#JnjUd>=x+7+EK|_e|MJ#yZ`m3VD zvs1zv`N?S>6ZV5%P9Jko_(jjP9%;0>6F|`pDGi)WPESfL%j)B#8{t`DJLS6(mOd`i zjWAs)rpNS?K2~~}ZiJqe((j<%v%Ij!n_O=tOifdKsO3piR-q18qMnKhdV>W1fpSeY~MPKc$Z;Ngs=$ zkHz>xiqVI#*lcI$V=?rx;?T!p=wmVTvDgx4=wmVTvEtCjV(4Qr^syMKSZph2oVOVI zSaIlMG4!z*`dAEoEQUT7Lm!KwkHyf(V(4Qr^syNFSPXqEhCUWUAB&-n#n8uM=wmVT zu^9ST41FwyJ{ChCi=mIj(8prvV=?rx82VTYeJqAP7DFG4p^wGT$6_R%#Yj4fk#rU# z=`2RlS&XE!7)fU_lFnl2V=?rx7)fU_lFnl2V=?rx7)fU_lFnkwoqgc!_jU&>u01Y& zRL#76rD^7u?5?R^GyktIV0%D>4#^gWX)^ToM{CaU(IxUq>eF0~s@y=CB7DulE+>uqxWuha8Xx5Atz>EZg%y{^JyY^GD~%6HEz3&dq#NH^ zb}da_l(zr}Fz7n^&b{buxJHy6{b>quhQMr~5q*wnP=96xGwRUMKj0&l&8{bLA zrSSUp?J3)h?*SpGSU0|Vg{6t~-6zbKK6dpU{Hc(>Jcp{0zI?Q_!+67{NZ&8n$B*3L|~{+7#*g9orAfN7xkU%X2R$eSc`rM@e5wlD=X{U$NG>q|~t3Y-dPc zF{H27Mf!>%eZ{!x#FjWi`idca6^HZ{L;8v#eZ`QzVw|-Y(pL=WD~9wHL;8v#eZ`Qz zVn|;xq^}s#R}ASZhV&If`idca#gM*YNMA9euNcx-4CyO|^c6$;iXnZ)kiKF_UooVw z7}8e^=_`ix6+`-pJ?`vjXGmYgA$=8R_e#u=zG9??6-R1VjMT6gsbMiv!(ya{#Yhc{ zks2183f{x{5=5;I|*KQk}(xbgg7y&OkctyLt{bs$=PVCw7e)lPj z9jAhiH`*lgHTdzkumLKnX2<%=`R2WHl{Ok>zi&v@Dx zPlszdW1LNsE`hIKrNy?#C7!=@5S^vXo6=~vOsYFNe@m~|{CO6k&eE0+qC>RQ71sNC zcAXb;h3I9%qPGg8mvn8iKGNFOv>t%_w>5XGy_frH`YqPQgnj&(@v8gJP#4rq^_xz3 z!Txj7W6kxO_FmAh?YX>{_`YF>JbE>x1Wz<+@7Js(?Y)lOvSPe6A?^KbD`_c>A*H>C z_|kRk`tq~`NnzRN!Z9hOJ!a(CV}^|zHGbIP#@)>IfexVJJNB_a`21Xq_;>a5?4{n-tEC=gGf9=(=HklFhFfcJsDvzNul7rO4aE@*`@#8LjOXwx==i)53XN z*qxxs!zPRzam=X0;*Dby_&{?Frg>SgCCqObc1A-G^3;*_rM#Z)jE*|mnT`6_42ZrO z{XFUutx?mnNT(gN@=Y6(!&CF@>F6-Owq23j+4IrbWlkO4E+M9p-2Cdbj7!;Rh(8*R zUtl-2)fb0LKQZqV7I!K6;^ebo#iV18Wnq3*zcvqtc`8|&8YW~mT)3xcG+6qY+!=3y zHZfe2KZp6!M(4^9qc$;Un*x7vZ_Rv```B@Hj~W_CbLaQgjQGg!t(h8zY;Vmpf#)91 z>#do`i^I6LW*!~s-kMu%rP?hE_SQVh7UlkU?(<{Km-xaMlGf)5_cNRF#m1Yo+jPE7 z`QG7e_^#o3NaMd|Q|@ED&&4)n`Oj>Mx4FTl@7VlrY>M}}!=`wj-`L^OJciGZI8-nD zA+?+HK=u=Q7wiC62k+D;eo{PKw@|xF+7kM9!LxiB!&kPLotZqfyD4Y zVt61iJdhY3NDL1oHq_ZLXEb9{+)2*D7dw1L7dyMk<7h6b?fuEwLT8Je{nZ)TL;L6+ zm+GQ#JKM?G9?tf1HpJObXJegBaCV`yOPu}G*)`5?c6O_?`OX$N`@6G$IQ!69ja?mG z0|pb+xvlAp0RqLgb2iu+4Q#aT@ylgR5TlGKc7pK_;>pSx8 z(MfOIE6XqT4TGnj+^~9T2*&2N0z)vK8-_@?MTu41)SfaWtYY*3k51tOIDHxT+?n+v9DN34w0w4mhNUHIYjbXTFP&3zC&Ea zzMt9JnQvMY4)H@i)j7mheZ7z15N~leGTe?<4)N;B9OB8oYNM?xhd4cD;64_I_>0sI zvJUZ$Ao3Z%rri$6)UP?j_kw#s2iq-O%ppEt4)OSmL%hu#;)LkD=*ty2#6{*1cV01v z_+rK(9%K&jGk#%v+EKULKhCA73rZZ~H{A849U*TpXd}A@M>xfj1Mf#0bt&wADszN8 zXS}Aie3f;S-mRv`N^*-I7OuX+EuL$S7wa=8;}$uH2aCH!uG14?#l#&x66U*x4dq{_ z2a8;v+u}8HZ%F(h%-4l0k2_ot=D7xGFRnu?<)VKa*88+LMt%oYrenOot}Kr6N}J*Y zud^wR@kX2C7;m*Hj`2pjia16-1>$fJIL7yEnsbbF*QZ)DN^*>1I7YG7xFp9YHrv@8 zkHax)T^yt0aExL&MzJN%o_B_0R9szLvNwL6;TW~<=FYZqwx=^3qvCLk+8&Nk496&j zV-&+Nis2Z=aExL&Mll?t7>-d4$0&wl6vHu!;TXlx5@I+;F&v{9j!_K9D28Jc!!e5C z7{zdmVmL-I9HSVHQH;-9496&jV-&+Nis2Z=aExL&Mll?t7>-d4$0&wl6vHu!;TXkm zjAA%OF&v{9j!_K9DE5rArOt4Sio-D~4#y}~WB08Xj!_K9D7LG!-JBijY_zjhXZS>| zdx5iyozW;w`@knEjyIvj@QGqiIm0K4edr9IDE7IyWJA|E+t3+4QE^*2+s4^oX9qc3 z-*hNztQPf*ONAWcJ9ZD}Y8XpbE|N`)D;G(&6|G#fej0ZwRk`@#)G48I@tZ9j_Z~Q* zd&h#i-m<#D0X^QLa&e+67paR$%wpQXt=t;MQPeO_Y_z(@DOSUHsy%~>)G#)esbQpp z&%7GOhvFK>yE+~kwxFUeKQHi##Z-TvU$Kv;gkAgu)hs5xe|D-_Tso!k{z5g2BQ<1b zu5Qt46?0ehZ}E#Pf65xg{JO+H7j%mCo8G+9Vs86Qf4|W}n_6$6d<*g_5;wP7K^2LQ z+9FxPWmb|Athd`!>`PXX5&VXgwDdbxl1<#lmv*s-RK88@VqeOvP29|)wzoaMW$RUI z6U)BeGB9W{DiX<0qZ?4VWYN1sM~+c-)Lyn!f4xIA6^XqASW?C&YT%}QEq+Ldf1tEY z+&=_8oT*4uJ;xZ^$U$NGT{ZVxbyP8%xL1gIG`NC7Hj$@Pp^C)Du--CXFKZKd(3Px6 zq&83$bH8v)GTb(LR@m_DQZ{jtt@@J?Hr7{dwE3$7J0s=d5}Ww_u&jcL#NlyZp^C(j zVcA`&9b|3d--9+!Y~qn-6aP_(P5g`5#0eRjc$3-0vG%U;7Yo`%OahH#mxhD7yBdo~ zz22JkFN$sn7V$WDaod?ie9l-h>}z}2{E@{iA~lqd7&if{7S{XM)E5a2O7y%DpK|6^@n38Qmfw3z@x@tT&c)oq{2$YnW;nOH{f-fueX1v z$DB$;ARKyzo!VZ(=kJ^;&{?n#8 zOSp58k31hK$r6fT3B_9Dk}RRvY-e*k4oj$Yv4o1l5{h97#g;gG-WirqaacmdVF|^s zgkqaG!yJlX2^EJW6vGmVVF|^sgko4iF)X1NmQV~!D262z!xD;N3B|C4Vpu{kETI^d zPz*~bh9wlk5{h97#ju28SVA!@p%|7>3`;16B^1LFieU-G7CXZdieU*Ahb0ta5LYqG zp;&Kcm_xDsoM8^dj&_DQ6r1S`b0{{;8Rk&zMrW8qu_v5i4#i$_hB*{_*BRzejG! z%yFqMTF)8gP;r|(!yJn3>kM-!*26TXwud)hycrC7qe`oooB33sr&^gm$P=voy27jpY_G}$K3Js*oL`9vyxWYB>I!B0 z{!nwD)jDAjlDfi!%9y}=+{|U`3Q=aoP2l6c-bXNjSGbcKZXcUW;02YLz!AP`qpf(w7pPC6gDmpXzd<7wQ{G9?$rdVe-zgD~idx)TWrY**3-GU2RiL z9`%5jyld?0vHV_}V)CB1Y0l))-c9v>lw|V6FnMCFaY-gmY__vG9*4=(x|lr0Ve-T< zd16bPJ?{*Yr#MWW;xKt)NFOn*o!C~+FnMB_JjG%1#4vecm^?8|o){)i43j5@$rHom ziDB}@FnMB_JTXk37$#2)T_A?Z6T{?*Ve-T3XZd2>LZhbH5Tf4x*yHTswXGYZV z!MlrCWAzq;J^PNlmxJtjmd*5F6LULWeRlz0KQN7G>G2-6B`LmBXVD*;0 zW^~=&hmJlyedKRsk9=YM^nL#wUPEh}*tJjFl<4JJ28Y?=-lp1nrrlG2?jXCEHVU7e zME#KV(e+aY47C3=H#TqaNN(VgaFW2`Ibt)@TI_P9<_C;37~T|QdSh6q*8tPwoy^K# zDZCz-Uf!A6^?wUh~ThBYma$s`$fbYP-As278BdLBEmn zav#FHL#F3G1dA;HA#myEUwO!uA+lF)tUkOev};SpEA~F1y$jfPbIe*C5_U{BJoId5 z+3;MQ3?YYwxSxgXtYc;MCUINX&R9E`y~FaGY7UI%hV=?n?D4EB;9>zr{k$t zvSJU(J2Y%$bwB8yHa|SD>r%c+Z|LIEbD^-!-H$<{-WrS&SOTs;)u42N+& z6nA~P9;(rfi%Y+E7hU?AXrBf;uxqz}xj&w3_>;Bm-_0?k5{hRX{(;wMnQyTv{)6AH z2W|SSO`o!<-RsfcY>H}s&8FOZR7%mlfOxbye&kr2;#c^9@GB?T6u)wq9rM~Y#kg*0 zQ}izF7pXGb(x!ZvtdG*=czFKi{LETibAE=B{EQfWMyxe1$Bhyo#AH`$9Gh5_!%*-j2K@;vCW;~V#M}!#x)be&uHB-&bUd%&Tw{)v+JDw z(%B!J&2#pevwu3PwQH;WekLx}SqJku_El#)IimrLj)l%}v@T6y#P}+S;daD+;_OOi zcRRb!*`v;$boR0{yVK(B@m7ZRK__?G$GXnw2Skj@R>jc;p4b3qbb%-KePQ?(-~dhX}_!39jxPH+!e9SoPFKdE6!eb_8;>GW9+i8fH#frspymh}U?-^FhvPp9hrK$*ImtRLv8$`9NWWH~CLOLXWlV@Dnx zw9P;}RsCiCz#l8|1J?x|pVZT2{lFXT89ag}nf(vB|z^HZ)>!FM#*?O$gFpF9@Zdhxxi9=XP{hns|Zp z!hGTd#)kRQUf{bS<`e4$ZsnrJe>~Ih0&Cg7>tnVuyucpzLEr@rvME}grwCqvZ#`as zhM0JP;kJLgz-czc3tVASyZ|36>rqFA2jH2T^8nNB*{nJZO7Z|=cmT21xFioCHrp8< zKnxF{b@2dVcmOdxfY=gecmOdxfa34~Vt4>CJb)NqJ~2Fi7#=_j4CJb)M;KnxEch6fPC1Bl@P#P9%OcmOdxfEXS?3=bfN2N2uK86H3k51=?a zfEXS?3=bfN2N1i@86H54Ph4@!oW1O96;siQ!wo3T?xUFD2E;aYh8qy;=L|O>cC0hp zfY`ata06n$c7_`eyVDtNKx~OK+<@2{&Ts=_UF;5~v|5B45Zl-pZb0k>XRkQ>k9mMY z?Xs_c2l%~xQ!1$+Sot1+1a+tpc;&zamDdU!TuS}(KwAOzUs)|ci(RQy)j_9PfLWR4 z-R)pfS^sJ6VfoZ)i{q)Oz`R<5UN-+9tQAPt2vk-Na8tFqzs=4_x}PS{((^p!#2&Dc zl>Jg)`VT9~0o2&-tffy_DZd_|knX3p0NsDTO{>-YWnbLM>i*wVrTgbpsr!FjiSGYN z8QnkL124b|)Zn?Ekye|}2aXLbJ_sbyK+|86+Bq#j_r z>HaM$(EUqI_YbQ~_fIq3|NYXsUpimv7Qukv$PMziqICd0+(oS1I)M8N>wcOvp){gs zN^#xKxtUD0zoZkqYr}l8I)G7OX`=hj4)clbKQhdh*8PZ`DEh?eewjW_A&PjWq5J7E z@}~g%TM^yAp?whK{+n!y-lswn-OlS$=zVlQdLN^P-ap9pi{3xkrs(}SHYMlJhsS!i z+7!Ldvo@#q2b$uM-lrtJFNWS1YmH0N`(m@5q4&km`&t*hFNWS1L+^_%afaR(L+>lD zE-tC>0x|Ty;`kDZ@%0l!?~Cp0481Rg-d7xYUyNH^481RQjb`@ehKhIR)lj*e^dCH~F! zh03P(X)=q%pJ&tRC4QBu{p+ie_?J{E@qbo{#Q#AVi9gDe?)DW({8KX$|N4x?pKcQW z*DEOTul~3s{D8 zD*xk>^z#Z!dQ$m2+BYk*L7a;A=!V|8JOmY!@Z^f=o|R@h=*iQ=d?Mj#R}w{~B|PtI z5%URc#&ZovAOG>pLc-H&`{glP5ebiQMZ)tk3KE`LL!|gHno->HrCk z*h9kep%MRcnsS!yVl>SWPLe|xTa4Bd2J*2XduVZQWBJtI(4digR(J@X>Go}l zJJ_-|Kd0NAC7u5_h|i~F5TAGepW98vuY>`7?zbnJv^}p|N!#;86I;hhQtw~0l9qnk zN>cfI_|jVYzUFKD8&~P9?YFWhx@WfEvMRNG7{aIY06rR$XNa9d-*Ce}SyJ2UjbCzP zhll8SWwgCIf97XrP>6rLxVC5c(IM#0jJDr9XmWm?_72M*t+~gVR21uX>yQw0uWN5> zQWXDbARZn}g_`F#hp6KHZcVoJN;c1*YzH_lY~+_==ltvrxA}>I{jSt2*EA+LJA}>j zRU2*o+`wj~B)#PR(H5`%#pb7$N$!)dyfduVEwtp}ZD}P=PqDX**9&hN)7;7K#?Cz} z|2ZspAlffA_D)aRV5MX4AZP4Vkm$&$v6x>#{Eo9tE)Ai{*gI7DnT`v`%-B0R=EG{X zi9YYwq^CtB=dx=<(Xn^vzAKknN6fH8y3Jzec`O;(Wd_BZ$9&X{vV_ zc@-F+a)nLtHFIo=ula>d@ikOM(DaB8khrI9%Kp%_IY0A?9h>|NCHWaK{ES#@T#}y= zo9%3l$KhwRE`COFJOIS-Gh$1eJ?{)Zqd5GG;_x$K_!%+$j2M1KY)@xAV#V+?io?%{ zo#qS|BgQpU+-1)0cE;<}isN-^#qs+tR%6#uY}L4A)hEu@bGD5$UZ>W&dpq0L8A4av zJJi{k&f1*)$k`Rn?skTs(LSgW)ArtSR%4&C*s5{KzSz#b;B0ee+`Zb~KF)ZtT5OoJ zkd{TIC2l#{I2tMIzEjcNLk2Ut~CSk(-elj`(pQ+qso+B&_u zew7w`P-A^pY;e^|M$C!r*kdclj@@pbf~*&ha2YO=PGTgTWXC>lB`qb@Bs;dPFYWvm z#qaIQm#u1Pum=!!jHFR@RV~=DGF2^EJNAdF>{z+hR9QRr`%3KC&rE)+2Q1mI_L{ko zhS0=n=x^^bc1%4H7iuL{+KweQu2*hX<`Y(p`ZQEw$9~}cVYqEXcI@)X?AQcfwb53U z9cxWluB;uqI<+io$Nm*;Y+}bom=*glNcA#y><;UFHyP-L>e48>Jq&a+v&cX<2yfKm z?Sw`Tte6#hEMvv?Fe~gX=T_-0mU=7xl(62X z!kY2ye%&rYwbqO;9oB3In_|tlsj+7GXRH}k1Iu-eO|fP(ZHhIc;vZ}F1G|J+vs^2& zxi*hAd(5UeYj&6kLRm9PvSwmfGqKjVBx@!%+u0nC!{iSfHGhBXtznu%e} z#IR;!STixKnHbhg3~MHaH50>{iDAveux4UdGcl~07+*OtzHDMxGcl~0*x#LD(!@S= zhDj5%dnH~MlP0!_GfbKoW8F|%EyARUG1iS3CQXdquf#BEVi!5P)Y%+om^8)x#u+9} zjEX3250fVLrnBYF*0p0(+y-%}E}}bQv0a_*>5P7`v_1N~QXHeBinTgp)EhB6HdY*? z-iZCi+3%cnIJ?K$!_FRe_PnzfovmT|Py1+yOLfsgXAe2+Y|VH}H&=~TxX-H=i=elw zBkVrt-`2SXjn)F08Z>$;ww|x147VDzUd6|!*@ZRpYS8+&t;}k({Bdbo?a5KJ+N@kR z#Qz(arepckX=MJGHpbK8n$93?o%BWA$(F6$k!fN_wy2;Zt8P@9TkN=Hy?B7jaM2n- z?B7FCU*^}26|!Ti*h7a7i65}&YVBCrms>Q>3_1SZwhJ58 zqof@>G)OIK#|{nAyoa)DL-Ci|2Zs24Eq-8#|4nH-HaY}dk*OW)=e1+~!t!6&{2@BA zs2w{X#9-+PzqG%!V_5G_U$0k~=RdMzJj6=Yj#0&>>b*1EQw_J($FYs*qEdEj zlC64r2pi?AHrl*u%uY^OuEdV1m$5&lmL+!V*I`?c*N)8!^P|khbPZOfs2#f_1l|+v zml{w;cC4}iWquykf2N!rdmx0VW{k=z1Zx!iE*vlY^U# zuf#;D{wf(zCbw~F!lqa~Uhx5CDzjtBfHLVVS2CbX<#tS?IjKu9YSQ9f!LVaT+kI(2 zui-mFKg`KN{He4ZyD6-g*fE)>WI&nph6>xUOT!kDEB)itJhft0C1iiefHLVVma=0f zhV?#?eGOAnHj}fC|9EzDcV|bF;bDtCnq^aJ%C5F4HD!DOv0HrUa&`^ z7yG3+?3Wn!OKgd==bd4{6o>s%9QI2L`z5xCGmMxR_DgZtFEQ+w81_pH`z6Me62pFp zVZX%q^%lc^iDAFQuwP=>FEQ+w81_pH`z40`62pFpVZX$%Ut-uVG3=KZ_Dc-=C5HVH z!+wcjzr?U#V%RS+?3Wn!OAPxZhW!%5eu-hf#IRpt*e@~cml*a-4ErU9{Sw1|iDAFQ zuwP=>FEQ+w81_pH`z40`62pFpVZX$%Ut-uVG3=KZ_Dc-=C5HVH!+wcjzr?U#V%RS+ z?3Wn!OAPxZhW!%5eu-hf#IRpt*e@~cml*a-4ErU9{Sw1|iDAFQuwP;;$A0BkimkZ) zLd$-f_A5OG%#`tYHDTrKma4k`H|k3nr}goS`Vt$g*H(hTy3!uKGE={@k}T8{&R(*T zEY!Qcw9aly87v;hTKY#Tw75^l{HwiQ_Hdj>t(O+niURV4>KmqO4oPEVBKtHI~lCS45s~6 zHdu=c);-l2EKC9t)Vkme1}iZMYnj1%wt&InJW3cW*`j2ymRVurYt?)$+9A5YhE#dE z=wL0|hBHkJ!?#oO7>1p~d@@)|Zp+l1vE^5~lpd^Qjj(^g&7@ePl5Dc zEy+RLP~1>qJTD9@CWE!)R{UD@V3EOE^mzfoicS25PeHkL8!y6a$4z z&Kam#Cfj77D9J#HVW7lXI`C@~C_7zRoV10{xm62m}=O>=g(GYpjCFi?uaK#5_X#4u1|7$~vloxSMnvv#*C zZjHE97co+c*jCOkP+}M;ZEvKr!<}KE6o-LQ+(phVbvDP@waze5S{DPQbumz47$~t< zoxS0VS4R|gb6l#k`lb1U_8UB;Y5%4{O@o_;^gZ>J1ME{v*GmoW+xmQa-!@yrYNNKD z-}mfk?RHo8w=q{1SY-dUf7q}r+dj~z?WY^Hhxr$Im8&0vv+S{b)7!`I+-9SqGF2=7lv?Y z>{$L*;p?3hH+gk zZC(1M+Ig*u7dO?;>$B6s`WbK7DRjKwePU1RGhj+%|B=rOsC~L$+mK#ucW+VKRDZ+# z0VC%!Moa(GhKI3S<~D6%T{q9|8OCy%KX>AX{m&g91_lW+jUtUULd!E@q?QkcK1Z1? z?0;I%jt_?3+3~^Af9iPi)z{|FJ=WH>fjMnZ5nHfuj!JpV+v z>Kz|EGhlefhhxKxUB2xW%|G>&scZGnxy~zat_$j?Z=~)Y82M#h+uXbnV(enr*ftE~ z-hbpH18N`cc>J}$=3ZyBPpFqQnY6vFJ~MY&au=iBE`|oSiH~3XQ5=#!*FlYTirXV>=hRa$IAT^Gs4!4pRKOnNHPg+Sfm3Q96K5r!%(XU!bOTTTU z_$@hK`X?)C%a2=0TkdL)y}#JMuF=N!-+U80aXCbcv1}GcGJuBQ+Mw`VO|}-XXe6*Fv?0 z{cO2L9(|*hy>UzG?;uN?Ymk;Ol8px0P~$uThJ>iM!b*v8UFfE4VOTz;_A^m$i%D%L z49wBgaE$F}V2F9AwpeW;KMB+p+M88&isQoxQlZos)~o57Td!A`=RY#D{IslZ>&apC z9(LlW@neUL8++uK6py*j{yi*!fVK->* z_rmJS!fuZD-5goFMFi7s8ZBmah`YX)4?)$I$(XE1hh?7$)v{VPX5`pohS{|mcKG9qG;Px)Crj=Ea_Px>VRN71Z`EIBXExXNV=N{S+6TUO`Y;be8eZ)PdHswrBAQ(b$&QRCD(arm`~r$1+FU{+|jhR zaCQ83VQG?0`ehuQK9J5&AO^~@-eO8$=lQ8kbDbB2`He##J*cPxfALUT?G~LIYHOB7 zqAd35!cTeksG;E+Bl$yZ-Do5`)Yh%Wcx>;}<(R{UNd+`C@XPEl9%_qUU4E(e>B6(P zp`pcQYi&L^)Yb+tJ0L#hy}NY!m>G#Hq6UAl{mcEKt6sO6b?u*?>(-x7m#C$;&40_L zciEJAekbj#*}`we9yaA6X$g}S9#g-wDI#i-O?l$mAc!qIbDps&UsFqywD722W>X%~ zZ`<^Fo4#jLZs7(?M6G4h9c|ibQ~2dJ{f^CFY12lV{@kVmY|8CtU*eY8Ha*v-ui5lG zn|8DF;CRu87uY;+WxsCAl+dDNL)bZ+8kg!!Smff)b9Sw>UpV`nGjiP8-s8@mcJ`vP zSDk&%&Q-@^S2A9ALuX%cMjA_T+c?|T*!p!xz2v#>`G@3IeX063(j6~ zW}iuXENjQ5x@a?JUw5{@GrlM~7P`UGc^u*FIA@cck^fcPOlP+{qjF2z!-Xi0_8wyR z5wT~T{nOdo&g$)xR~&9d>u&FCXJ@1-6nB8Lan6o&*5d4RXV*Bp-q{1r9&z@Lv-h2G z-)g_B$ECVxYiHkb*5s_&*>{}{az@HZ+v9CmN_7!+{$jihD|WrJ8=di%tKuGZ_L{SQ zI-^!!an$N-dkiTo#*o5d2RJ*}*(7HtI-}B3>&|jE&)Hqh7CZZ^vsLVYt93scm+GRg zJKNgXF3y^qjdXUnvkA_Qch>6cOlQ|PyWZJd&hB;ginG_9HGC#_%xlM`y6DT!Hg>kP zvu`;Ye4(7@aQiI( zm-DPF=h^w9SD5zf-PQ7*wQ<^W_D4;7o@XDK5|I~MX<|rwUSTDrBWry z|6nDhJ*}QKPIkWKjLtcf3Z+X9r9!DKQz~>vU;0NYX&?05t$o~YCGF!~D=8`ZMfkHlVzl}u9frjVK< zg>@4Trs#Z2VI5E=Df;&nOo}GS)Lvm<{3l6@mQ871Ww#rS<{Qy;%Q*J&lqMd`svIn( zd2?kcO$|Lg$u_4n-Us0tkkZsOm}DtUSd!8_BkbaiN>iGbhVa!xvVoN5Q(^v{up3gE z+RZ7J(u5@`%~>Jts*gLRxjeOlB&B(4*!f&fX(so>?j=*2v{Hzor@QRh<gi< zNok%GRw|RyYzjfeQkqAHrNvU3UkXdpNuUBL&9NbzNK%?4wMc-LzRpT~mbuQ#r6$)| zdC*NijVVnVOUSO;VlG|$hc4pFQ!J_Zb6ZAIlV1mtn*3^z)MOo!nmnjTYSLQ-Nll*l zBsF>JkksUh`m9ZPSd-M`Ax%>AQJcPJQ*L6Cnq6&5Qgf0`;g+ZjNzIFF+GzYIHYKUa z4M;z|1oSp5Am(_H9 zmpbDeLNQuiYJ0zN_Gf2{oW14jU1zk-)4I3~ZEq`Q)}AwFyE@y=*=T1+Ih*N>J5byE zt+U&l-R10FXRkT?r?Y=MtF?Pn+pCXD_P}&TdQ<1IgEP{bV*Q*QIy7N=q1?UXTIlJH4>(2h=>u}qK;*qxKtPI=4^LoBb^=Y>}+Qs92%&{%UfYpgjw(^zx2q_O6np|NIE zy0K=@qK!3kY0TB_i<@h#*~i`zuwF-++va9lFwPAP6zAJd0l7@_fKz%}Yt7NtBC+4d zg=(uA8YQ+jZ85j0wOuVVt!>**HZ;~yT5104z(-VUv__RnbFWB?d8cs z?L_`{;GigVxC46H@jg`Ic$Ys?7q`0HFTd5LwFYfJ`3!3_8(Lg0=u>-7^R#>F&mCw# zY9af$sBDYNidtCaT2Riivrf0pq<%No%956o?V05qM|%0<%yMLIxt5pS;*x9K>B@%Y zy>`xVGt2jfE~xtAelT?g_2M+K{9?b63us4cg z_eae((zHprzEiEF4Btlf8!ooBmBdhm$}I9)yGm4fDvOvjynNZ1a*XeJX*K)8D#v)L zm9*t^t)wj@`jt<7%1Zh9#A)^<%jOellu163K}uFMpXgn(l+Gs(u}h*xmOra1pLlE0 zeByz8gweH`eByzf8(CsClLnQ8gY`Vt4rg%Kz}02)hBs7|H$>?+hV9M_NwR%xzE_y% zKS|!Ovc{C^_#|me*=XCijSWS6mTpXmiJxZw%=Xl*VjblFCOV)?nTC_c6K&Uqt3{Sk z4JZE?(tkU8mID~sI}58%{Ysi}G6x~Zr_%^SZRwzY98D%)s}Pvx)>wN~n|G7T6xtSv)OeaQxl#PK=g8j*)Q zU6|i8?-24E*0BaHZ*mP7IXz1+*xx4p$t@0B**tZKnXDAMt`94f$}@4cpAS(b@=WZ1 z{Sc%)6PxZC<{QKDa8^1VdXFvFL~&7VRZSEZ)fR1{*l0grQS{X^SMrv}>6O0MgaF#oA+lBoRMHC(#*4;?}-ZcY1_8(45n z%Y*f{dGcW$EVC)k-E^bG$gb{pjnmA6eB7-zCGW;fNFHu4yE&O3Zc`c;j<#u|O~=}l zJX{~U$OCP9kWI)lu*Qn(9+&Fq5jbXBI^!NvTwiB2E)=72q4q&TRxz3s ziv7$P^#)>pa<Z1q+2!UZbu6>uQlWfU z5BolqetWsao)Z1rF5KuWi%h@0+!E$1&4=ygZTHBDS-ZUgZTEV{ZTEN&KG$}yM84~8 zd*uG#$#>a3YKK!@%4)Mr-?<#P-rBhz5ol;(X?9NE1yao)k#8Pu({_Dl% zhrEw7UCqPOX)e@8I*rw$`Eb;>mrgGu&&n+CWyhAJ$EHqO9G2(5aBQDepH&)HR*tK{ z1+_F+qa8=5G#4$a}FOU(DS$h*X}HBsUT zJE|sa7$v2kr~=i})vctZyz8N*$5=@TuOE1cH$Ig5`jwaFSxKp{2fVb@N=khVvL8Y1 zgVB1m4;tiZALm<1Nw8mADL)C;yFe1`NsA&0Myvi6Pl8p{s4hu@O{ppgc3znzSj$IE zf}K((33f)&Bp8qB>>I_1%SOAR^kqd3r%)1%$9>7hbC^wXFE-L)oWkl~Sqdv@DEA0! zNAp5%DH)-%-cnewB!x9C#0{=Ag>^{?Pa4WS8RqW@yCH>jW0+sZQdqDgg++5`dtES< zVft87SU*VZAW31}5_Zn;Jd8AxJIF08h4pO76xPjQz4N=&mQ7)896qdaDXg7C)IlL% zKniPd*nR~mtl?qB#J-ckO{cK5p>ip#uZOktx-=;)tyLz4#q+e}yT5p-FNUBpDXi^7 zP`MPAnjEg5%4ZZxVXYIkRW60ak7#6DN?t22kiuF$1eH%=4GeL5A2lR<*tWf)m~Ow+ zIfcb5f9Vv~24Ox)VYP>?mPui~7J|y9u>KjM%A{J}4?#()g`d83Vot|XJcZRrc=)@f zk`$Ki9^$Vlp2AwoetgQOuttQa1z&qA^0Xchf|Bd{ z-7v2vXy}OofAL$jEw*;HEHM&TVrUF89=>8Eu*67UiIKn(yT#e<&i>}?C1-Crd)L{jcD=Pdx@Fb&Hgral zGBNv1;<)cRJH*)tXU97GzO!l0Xi}!_(dE4MJIC3*&i>-;Z_ZwFR%4&N*5zdtZI6az zVl*TZ<7E{wnvaPc=IjV(``aWIa}t8RF?L!MqIK4 zhco)d*7oSoO>w(9qiZ)Yx^`3CS6EG{vYx}zn-6}R{ z1ES)L1{FmO!8tfI(hM!2Xiy|-XoDaM2%y=b$llr zs|fmu4Gh&?6tN3qw${v`I5Sg=8~ye-4AHo>-HRbtg* zBgIY@naq&alQMEl)Zr3dOg=vK7MpNQ=R6gR43HS^AN1cKiI6il6feYS7q<*l{3$^ zbH7jhC)JhreLgDA3N4vb`i)&mOy2|yw3(ob*s?QQc5Zre z%#yfuXUyz)MbxZj$eSOk%O`k2<)V4opTDA0egmBrZ zYoCm;B6?4th+cW=B6^#~f>z-odbioa2C0Z%+@H4lEhwYMPO?&fS77(79AMj7iOsiT z`SnRwKd|WbTUu#($x?b}7D_7}WjC4s&a~1{4x32@iy?GSvX)ubB^5F}>n9bmz^5}y z>D{nioh=Ccoh+p{vS2AaJ14!I-pCo{cK)n=y788gyOqxh`+mVXbiE~)POG!+gA(R7 z^=eAg)T<7&OJ^63_v&sxY!kq6g~$`Yx}tyDpgPaxwkFC?*V`CHr^C#g>{uHkd)&q- zcD-O@l(Z!T-a0Hk9s*Q2UIlBQ+;`)w_bqs%WMAWE#!^z zBgTJANnD3DkMoMO7A)^LUnrAJ;o!whH{%eA5#?AH*Zwu>L{k^8(%v=jA z+fDN2Ge8Ru)D+Ra^qz}Fv>sr z?4`wy%wg7pg=4EuvF)NT^;WKD?xuwPkdCcFOy(}$5eFu$3dL68I&6w#N|>rih?E72 z(xIPk^M=gWM7YPbKFybO6fazc zf*syIaX~P~)aF+{$4HNxY zsHYBes}gb=4qRV1qU?i39~y35w9njH$mLlPVV%}3!?RQ249_^^YGB^(H5Vq|vKuNi zk*J?v^y!UC_v=G$RH$Fk8MpY{xHAypfvDQVxt}Vdh%@pLx60_}d8tJ=`7|TM@U&9k zuRzhdS!S6pugx}+FR#rpl9t!*G$ti2lGktt6b$7x+})D$TAfX%q^5nH^Y}$T#!2>qLNnWPEN>kXW0-OoeFvG8XH#G5V1&*=Meda zJWo`>A8kX_zlTh>(uR=f5W|D*$8%XE)0NveWjc&erXzz)M+TXW3^E-VWI8g)bYzg} z$RN{^L8c>vOh*Qpjtnv#8N@j<>)DXuRzik+PulGv)^0 zH%AQTJlUgS3&frkTP^mf*p_xKunYYx3=wCv8!2|O*d(!Y#I6&& zNvuJvQEaK$aE zM4XW!2b9}~98j`$CXJDG496-0M4XZJ5E~+PjMzD1lf^C-n<@5lu^Yu66PqW7B6eIa zir8`c5OGF^a&}~UitQ`bTdbehXtA+kh&bc&E)%;(>^iYW#Tvxk7JEf6R{E7%b`+gq;obyzpGuX4LOEdAdi^%dCu zOttUyLTcX+?SLlLzJf7rd7*0A0d7$5uhexwBE$Fjk` zJ5R>r*0T`2f3LjPyt|?BRbl;w6h6#YVH+RRGtw>e`28kB-Taq+VIzL{MHIeM ztpznVC?E_l!&@WOw>_*O=b~B!=N@EZoQvlJDtvd@7d2Pb7*)T&YAzggRQ=$d z<6PXQxsG4h7}s%+jd2}|ZH)Tgmo}EK|Doh&TK_|lWIQXQcEh*V|1v_{()!gExxxKjdUmWt!znxy)3i;&tPw|2-}p zuWPjKfI1$o`^ELYm(;F%svYhiB}3mP*j`TcJ>4r*?@n1a^{%ydF3#QG`nuyO*6EzY$i} z;LC4;diN7we5~G4Qj4_~E+|XA<9T_4uO8|hTV<$sxEGWRD}$P{7B)rIJIKvZ((XhX zKg6etsdw#sF7*yruNO|ov0(+&y93-POTEL}9aQbY>fKI0H%qK!vS<1=cRFEvBG+rk%8NM*uF z(az^)sdtRC%TSee@UqBU|?nZ_B6&KwzeQw+taL#9|cQf27qn{xG*~>`SuR(xUa{C&(*Js0% zNP)is!EuAlGGD!el##FA)f!2wccYC-sEgD)+`j}v^$z!@q30{|ybjROY-m^VI^0zd9@k()=yz}1@Btg*g0;|w_yI$|Lo_e+J6wI1 z+7Lfj=y$k?L%+kt9s1pGZ3vwXCo}ZBcWns$?uWLw;HTRV`W@&Dp-`W+M#=y%A=gMNqSsz|>(-}aUI9mc5NkwL#BgMLQ_{f-R!9U1gHGU#_? zkBLFQBYRT}`W+ecJGO;>M`k@6vTen75QBb4yB=akiXAP6KBgUdnd^XlM+W_l>?W~W z#9k1Cl1e*-)UjN_?rHO6vv`u!17>*^`kHpRtJ6G%yvCG8n6+;RN z*Rfda4Y9RicxI>F#^G3-08j5^2&(0_z>iPXPYmBuB^xSstQeF$wnc>(+Cj-9`>EIi zVz>~}4#9P_`;*vbVr#^pJJSvUcWjGD0hzDXG1Q>$7=`9s#kr`1RsRQ4Rpq*a-~ zwB5x9kvrOKf8!!)(f)d&v~`cFD^)7(C(ZIkQ70yuH#!rK6_H4mpFKKlj?bbl36qBW z>2)flwk*}QyKP5Oe+zuNcm&7Wq{x|Q{@>T~vWWDD}F8PQ<1 z%+R?WM>lSUeU{ehr8V9%ZnyI3RyFOPiJ{x21>dDz(L99CLg(z@^s&iZP8QabBfS~?rzjee4n&gMuuo8$9)mX!w`eZ_c%@DJMhD+IqoOHFUOB(St} zhD&r>I)fI-$ib`BuDh=b|50+o#lB^sbC-I{{Ls0}oejzkonypO7ZcxpZNYOVwp4sO z)tArExk4&fmPl=Ut-w%{e~Z(*(?KO6G6g z=5{ybTIXgZf>6?7auaQ9H?et5PM$&~ue;m6!WjAQY&lTb0Py#0n`2?vxpRX7AAUS1mDzz1K5wh~tk&p6OOt z`sVf{gESF^MOFB8EMVdJ3o)G~g%$*5*vod_s`9_qiF0nXTV+Tms3??_PEc3Kq)zP> zw`XUY&I&Mlq$Ltnoi84%c%-mIg$>`%ZU?3zLTV?d?kKHJiq(=EP1Ilm@FYs3&psda;^ChY2M)D=8Sw_;5 z)NEr?pCU;L_XWXFlEO7VDM{7YWLlEKG0T>u7AN)y_x05_jqqeVz2mQgNjbRJL(+i+ zh3Ppqgd~OQul=071la>N#N`7L6t0?(pl}U<1cjd*Bq&_&-nStxgOH$R*$`3_PS)?6 zbhNAO6{byga_I@*7_KsYm<=I4eaE)X)V{hMZ8+M7$j!4SpgKqake(1&i0#C4NhCd0 z*~=fLCyY^gB7^iq2I+|m(i0h^Co)J+WRRZ7AU%;mdLo1LL=6GILs8J=(1b~mx^ zVz_$H4%YzMp*9K`giNwYV&{lmFLtxoFU9T?Lt!#5?`1K_Ok|LmxDLp^WZQ;g_L?tN zEjCIFh5y+W`I>AyOAJN+$$lgDy4agyABe3It2Cj8Z8r(WDgtD4k|CQDV-*3!C9*xm zhKS({r?kV@Oldbo>}IiB#qJY(Q0#HBC&beeJ=J7u?_5fiEZ12V-*3)?vtVHKDRHOeffp`>IzFcxQ@rt zj$QZSP^-Tp)T-7QL_q28UM}F%{I14sd%Zmnd(iS zbyIKR?8pN9`>mv#LR)P)k)_m|i<_x8aA_yio9qawX?50FQSg(AIN1?XVT9DQIr$A| z;(;Kk*fjWb@rbB)wyg>KC_-vTRQ2m$Dc3ab+;mJ-o?=sFo3_F>KZ=P8lOF#@#Rk8# zC?v`h8~h-PC^p?}7Bv*SiZKug?dZ{dMU?}p zIKP(N+p@!x54WJNjd2SOvoUS~?q1Y$uCuXxJ?Ce31xo8VaGT&6?+P1!dp##36e_Lf zoK;fKVLcr@?)1e03q};xbEx;6=F-eyv*+KzEzxs+l+tsiO3#_GqI7`5vjZF4|92WLZODAd8er=3DI=9qO;V;BkrHReuV_mc(Gn1d6FOInSY zoR}A@F;th%mKqcF;ReB{CDoWqd>d}gZJ4RX{LL-1)R@h^Yg(YjRJ&oU#!$wI-|6r7 z)$NgZtt_O*;AJQIl}onDP-CcjWT-J8`ua1}m~vlzmKyUNw<=XV3u?^AzQQauW-DJ- zh8nZf7nQBX?CcAR!-?2?S!&Fe-YQmOR{3;%vhI5} zgcgIM4bWoN*wFUB4uKxfVo)UsS`0!qpvB;sBGO_WvA0QTF&Lv3Lk2B|3|b5sv=}mI zF=WtU$e_iLJthV%hU`r-Xfb5aV%Qd144L&{$e_iLL5rasv>37;Vn>SM;e>Y3VrT~~ zh74K^8MGKOXfb5aV#p8%MD~&x!hpyiGt&-XKxA8rK`$fQL#(gZp<+K48znYT3;{u0 z$8};ii8YEX5c`YRM`C!d&SK%+f!^`F{m=MJ6;T`3>gBuhuAO0@bt}XStJHk zhU{%Is4`@qh;=a4gA7IDF;)@aBaCDy5>Ix#7>dM`A!UztNZBJpwJkDK+ag0fEixqU zku4N^S?m+B&%`!2)q`!f3dbsfoyEF|4HO$JhL13^?Wtn3#I6$ijo9zRn#7(KyVKrl zxV&G5V}(kuE+3K2qsv)u|^neY-8w`y?kS5&vM0Gh+wdHBZi$(^hodV7?# zI&~gOT~+>J9tvVLt(njeG`=&rQ$r4EZdFA;I#?UlUq9APPs%bat$6i`UCvO7y|c+Q zldBkGQ&T@QS@NoJt`)bMTHCzxRpUG>cV%HVR?&$zR_+P}u{KLVEsEwHjAQPQsaE9a z*e9C49%F@`@JT|ehJ{ZOvTfU`XlgMmsg{W*uQjbY>CLA1me_e?oBZ|CLAAEYK7krn zi%{dLW9b@KU1AldaE&V@yAM>2D}~?G_iI6gt8n{S-Rk({z8+O&TW(dlif*~(yLBvC z-Rj&z`EJMCDVEH4TZdSj7RBBairMLI>x@*HNa`@zEmGvc&5&~+wJ|RFX&d8`k#9~x2VXPE7j!yo*j&&#(5&{e z?XR%mw-aHHt$QANeeo-0~8W;s=VD_)vmiw<39>IxRtalK<7qp z8GqZb)~0WB_Verjow?Sx7A@53yFq>MVbCjwv_1rt;6`Dmc<#91!64&%hojt-!8&K! z;{NQ@E4>#{=c>V{Pp~>yU~vG?80Rmxv{?K!H|J{3ven>S#ok7IH7FL};L|5tv3Rh! z%-3bkA55rEu`<=y=lv})kIEDh@}BZ}_qj5KLT3#=jg#SfcF>BK(`t0PKjwDN%}$Cy zsR$KZ22nJ{0&@~p={O+-?wsvbao`S*%%FtYS0HW(aXCNthSWDE+W750P2~zT$GM>7 z>eEUU#6J8~?b_l$L?q;~)rrMpR`5=EzBejSP76kdxX~U3!ZWb-V|{L?#N1^0tj<2S zM7b=i0DT47(Sb*Gq8%ALhI7~jF^Oy)#%aB4<#0jeny~-6|D1wJ3Igt_$v_IG{-FS^|U*U z|gYUXCeq{TiCys zL)gIts@^sZq3S*xV!FYG5Y%wffS`t} zVGY-42y3s{5JDQxVdUAB+YkcV={5vMC=3L)D{WY1!|QDbfeq&$^6Ze^1Yr$@*&wWa zVMA;$p6w!G4KfFXHH=YMBZII;24Rg1!WtQbH8KcmWDwTKAgqx=SR;e5Mh0Px48j^2 zgf%kj$&f)nV1y*a)$c#1K@+wm1X1Jcw;%5ZlO}5PM2&rPyk* zuf*(?Ds0;}9BUJ_6YC+ikJuq%eZ@`^8zY7QDQ+JEq_}-|i`^@>KSK|iJ{B3i$Vm3M*z00%ihUrqO02SV)OM3_tRmP; zY(KGTvHoIR?OmMf-7OsZ7mAS36)mX`Az!#@mc0N4vgaBIlBT0|f+}gTU!fRTsMr>Z z>mO&m7AlR|wYEOBOZ)#$k#Qs2^1NuVmgtPzudB{jP2sC%#3B>I+Rn;LZk*SvTSKpl z=C$rW+w#%YL1Zkj<(WF;S%q}QTD!;npU&7aow0eRw@_*<`{t>#-sPdxc>g~uHRAc7 z;vcjE#&zLImogyyeVp6F#yHm=CBwP!*->h|*TyI?zAE--8>7JZcg;oAH08%FZH#jf z<;->9Ersj2&&Id~OKgln!np2-j*7q6QTd7>}$@stob$F(hJPjh_!MW#K~6cr*VW^ojLS$xS*~NCc;f;hcVzp$%MZhhS5l?ng)=!6w_@qyN?0oxp@>jChB)dq6Vc z9G}La+_$8Rc$?cjpX;6}BcgUowv2d;TOI4?G3t8p$h1I4q?Q+p6_@&gVi}PO%90W7 z<tRNbxO0M#KvuB$rrKi?Y0+gADY!#biYLY1m&tly10ypcj!zRaWgl{mJ!Fg9RuwkBl4(a$%y;d zl_n`8;+uPrW+2S@$1Z)T88RZ?t4;ez^k?$0whiZ&mbX~@D?>&^tWu<$;c)Nd(>1=e z*ym-wwJnhmSGOb#ug({TRjw|a6p6d};dbZCmJzAm#lK7nbIG&0fQ)#VFS<~_V#Nr+ ziaG;7A5_l4Q7GNd2(ihO5wA@w8vT5|PctmrhDJ3FaWu0d~+qyY6goub+ z21G>s+8`q0Dg_a7OB+H�CEi8{(1!5fQ&qh={mSLqx}3?d>~53wV~ zAR^KZA|mY|B9cKwB!h@Z1`&}AA|e^0^T=Kjv*Qx755+oIf75QWaI7N0)rbt&AubQk z?PR#Jk|C6U>@YDDT_c+;hA&`}%@q5&*o|Vp7W<9Zb7G6cJ`h_awnpr0F;x8FdXaaF zv5H`T7|t-V)5MTRNrsCK+fEa^TkKx3-;4c0>>07=#c;lIc?gN%Iv^*Kp{zOCj$*rr zA+nEli0q@?XtA+kmxx^^hIl@Jy+)A?R#BLHp;V!QCUa{xJkZ;MhUyGq!7a6{a$+kO)p-2}QigeKqMY_mPq>Bt+ z#Uz_7cCFa;Vt0uBQtUagMPh#y`@2{xdl%+9)(^)jf_KGMi1n{;OnrqD_s^C@C|(st zD6X*x#Xj{PM5>?#E28jM<>QB1fZ`#YTo{BV7?ciBwD`r3O)|{;9%tDL7P>g7DhyDB zQz5GRRTV@LcC4H=uYN$=`guRUX!&+!^DkPqtXJ8*>YfWKXExd;9pWKmxXGO@LUEJ{ zWR_Ritt>3n)T_RsSGPEebimB=o~Y)9$iz>Fy~et0sjTQWA7%;Qr{{72J)cwL{d(E$-EErOIGpMs|@5?2&cg?G%az-y{ zmNSZXon+2vi73MsMHNO-gL7;*lTm}+Z9U1T!G_d)Y)5kbvebM$t7b(TD$0-*zw%F@ z`7W^bnf7!TT4=?-?0i|b&i(p=nRuErN@{I*K&Qgi(SHAA+uIn~1vW-y?+%T@eMpTj4mO0VkSgAX z8vBQhQRBn?nMUl$a{!J-8Z%FgNLcHA~Zj66?e;jAiH4bhos@omm^KX*= z)-_y0DcufFtW|DT=OWeNHeE=!Ygqu{4Zfr}fDlW*)!7lH0|;>fPO^XY`eHeyzYKm8 z9Fxlk798oz-O#)LOl$gQpMKosw>Vg^lc{`Ql;2=-irZb*Qk9PvR;~sr--q7(_r97q zSg^CHd|-q+{dotQ_J53gZQ`Mj5u#@gW1dzs3&qGTE??soQa zl?!s0Cf>2rMaA&COaU#KvRNotka8E#Zm4{C?X*&3rGo`=F~yl0X*M{lDEw^UIllAw z-Xa&&HMx;z4WDqrh%qBi3}e`EA{^+gMtLi!d~7wSc)-|1Yk+Hg$?E+uLiT(RY?D}0 zN~wcUe>ciYMNI|#Vb`cCmMSuakiMOLSvw|nI9(nJM`egxl}KyFW^L_8l_i7rU{~vQ zypTaBWAE*jw+Z<{dpL3|PEQpGy9a-dsaQI43w^1(CVDQV3SrgXxKSyc2YXxQi!Pz3 zVf~cmn0|>xxAR6-zCGA0ZqIjZIx8%1-n!{&^NQf4o;w zFY9g7(4+3fJR3IH5c(M|#?a5u<_H^NdC>3_2JYbTG0C+cz>OVr0+_$)JdlAs~_LAhE;5jugY0MmyZC+4d4K zC}L!fiZzI#pV<~+b+lU}_O)17>u=ia7LHlQqSygq{lx}~)rujkj_a^v6fVz>PRMQ+ zyH)Hyu?NMT62lWRm-mj?`(odSAw`FFmEo9OF~pFf!+k>t9qmpK8!3is4ejd1pqP_E zD`(pVu|~0lVlRs=6MI)|eLFL`JcNOAd7Fu$#u!;ov4h2G#ZDBPCU&V9G%v0LnirP` z&5I107un}x{}9{SG$q<~4#z5jYB79bl5J5%jCQCZMusY4WKh4zE)%<4>^iZV#85?y zZBa#xZQm37K&-+vCE9Hqj#UI(iXp6yZFd#hL+miIBgH0*A%u=?p>%N_w}>^0Ef9NI zY_ZrHF{oc$-sYw*ad}&XV->-HV!gzEEH+APqS%>YL+t&I%R43;<0MbrA&A_!T#lC%yza{@yeO& z*$E$sPIY!+EM0K^DKq$=qc_>PW}CBalGC7dlANmlrFv6=?JX@iUDQl+s&U!Z~L$$MYCuOENsrlV~{sq$-oL?|)SvU+r-r0tBw5p>rW&ga! zRG>pmW~#z^(lQfZQf6XMStv8L{`#A5`q#TY6_qDL{%U|9X4gmgjn)er@jES&zly^` z;Ro-O4C%t#4ke&ZHTNscMUDdJqM#?GrwcWPOa)3$w`lAE8>93zUt=%Y7^SB^_B-Yl z46-q90g7^R9aC+LBGj!mmM=oBwaZmng!-ddL4=xZ!*4G_wJ4}87NI7W6ruQ)JBUyx z*oN8vMMbE=KL4DQ2sLTOG89atJ zMusM>A4*6Avm~U^zRq-@*nHP^;y|%VlaRn-32CCY8`e?@X_}j-1H~Tm)x?2fo126L z7E4IXH|>>>iP93%g^78wgmj0m?-@x*b9}m+Nl0&%l#p(7yDM{LnG({b*4GFPieE+x zNl3f7)errB9TL()Uw#WDq`}@Wetky8f~17R1!YM{+juK>86+gO%8-z7oz9eyP-r?+ zLh9;9SrXC#ZdIx>+8JiCfm>xsNPD?ehJ^GLHr;;io0({`Sv!n-(r=yshgUV+6QEQ^ zp;qj@EC~sBg{1hmkx$1G(*3^F3<=31Q0y;DLVCxoGNg``ZWK#Mt9?2a#<)GjC8R2B ziT}-RiG;*XgZ=E{5>mO{H?k$9Bit(f4gAQbGbALcQ^i8EcJ@VN^((ZBk|9{Q&>rYU zaliKRX@+3oQJ}zIfy!vLW|^PKIoe2mCg%hr=}gYEjh$huI6OD@#NkmUXI&D8nVh(3 zBr`dyY%-n6iDQ`;KdRs9>BbsxNhOjEG z!;VSFju1Ou>;$oKVyB5s6GK=Pmp51J7h;cz%@cc048c@v` zWIKdo6+v&Yeqtw!jT5^<>}O)X5&NCk60xOX_QV*j7vDL=7E}Z%f5vS=`7<( z5<>_T?T!Z@A+{PQ)Hj~~`d z)!9C}mbD2+){g&;gRk06?%XtGyu~C<8E@s#ri|azuGoPMx503be=u=Y|N5ho)!Yh| zaho!}`ik17ufKf0aqgtGL#EbRO|2zO?+$y)0*jV4&cj@bpNi&ICFWW}BIa7q(U_Or zV&1q_E4Yx(2{T_v`4!PJI!;Z@TY$D!4Q|!qruR;2YMeKD$upMaxO7qD@*VeCf}(C) zmoDnIjhz=Cmi4JGRME|P%=+q{u&=@zcjz$m)ka}od7o9zyxM#0)j?&iAUY^fKF^L; zSpKbJ$;`|t<9!dGYx->Mi{bt~X&vbM*R*QYTef?R*uAH*dkY(v?{2$i8JSy`sQh-m ziDL`TaF zX%Ejr6Fk6Rh=(5x+hM6|*dGYbLfeK^+wKV)%wLw8KR7i%@M#u(gMC#vUCUwg0mh5h ztFr~6hX;*V)Ti45biMr+XXN#C&4_tOeVspf#yWSfWnbQiqi^4EoU(Pzi03Ufa92cS zl`ROPvg{05VOtQ`PyL&x>NhESrrL5a2iEs#Z9nH3hgf8m8JKF;fMx9YteXy;c|#Xm zv3gZ@J$;4!ci!SYm0f!;u>S@;V*mZ%PW$iJ*){d=LWN->mGegJ~m9n3U-D6`^${y3)=WL8h*_)dCQ@cM=6?@di@>Q{|+BR3k{%%%B z*|z@LhTldN^NMd-HQ&bh>*!Io`{U60!PIi(YT<<;7o2?R1fM&==JK!Br=g+MI;by4 zqySzfa>0+sjvq0`=hxVLyi8bCC7Yl8DB~H;Rk28(f%$3pFPP?y;&83iF6$uG520?6^`C3gC;0k5HN67c6ZPuKvXgC_ zZp=k?(Yp37t`TB~u)wQ*1zY$E&a-JAx|P0ybG3p`%1#SD4Stb}?4lJMm{r5>I^Vwy zy+84ea-OelPhZ_xHjSffXGISC1C5bq$(vC6nb+%SFqiHa-d>d&u zy@#JWa^m>m6ULu>>KVgFPB{5{`-Cx66Kpk)xH%;RtQv2~mJ8;afV@9B4wBh)r{j3Z{&$z6K9c9vs}2_`MV;{)gA?oqea)^PNP95h5!f z#$sW}YK#`PuYH&`)`JTb0DRk5dxp0}cSJ2u3(hJYQwAdjg0)W=r9K`Cqo;l8Z3?7$ zVlH0qqnIaj+*+Sr-}ef836~Re-#GU-DiVm+v)!6<7h2!o(;KMutvaH zH%eyiraxSY?%l#SWRt{(MA^Ss#>>78x)QJm;RNm) zG$hWsbb&k^SA3%{2;#H1jZg0wY#E-t{Hft~c1G@tT>oI0jm32nQ}K;h?aF2+N_6+a zTzn=ld73RtOk;E24O_=Qlyod8dVaNA@q7<8`uqr4tm-Q_;%AL;;(QUV3VVh=iuP=0 zE8n>?HzfFBs{y&}NXcK9DqvXX8dVeYEG~6-u!lj5xXn05D5aG=M#zP;@BHct6Ya@B zJ31wg%^+`>{1GRQ4Jz;@k4>+{G>%Q5uyyHUbA(%E9Gm0aXir@;hlQ)cH8cO%Oexzs z=+F+Y1qJ>J6hFiXl`omEFp@8spagQn` zng7+=bTXUh5AOyzh@I_U+mLrK#Trk)o7)i20Z_1E1H~M-X@p(kG{44%xV-+(hPYfT zvLUX5cIB(XwQ{Kqad|-6GHzAx+Yq;?&uxf%>pyIGzYW`)QjQOMy3%g@2o84%ly-dQ zrOJk=NN1{U9Zp-ME#r(BYeSq@r`iz5eu52gI8V1B4ztzUtHTL=z728W&amMbHiSZd zjt!yEUu{F|-@P`B6#6f1T~z2XMunaX3OyMVdNL^VWKihIpwN>+p(levPX>ja3<^CN z6nZi!^knF3GCLw6gDy{obZ6S_A=X3eNU@{EMv38Jifu0wyIkxhF{C@w4#$=2T`cyl z818Mf!}AR7z7WF=oUC;?RuOa(+ghx<*j{4QV*SNV6dNsO$2wfc*rry z*wbRKh`lcMo){9OxxZhFk^LH$3w*6x?>;gCsr-iUu=ljF=D9q$+i>3>clP< zL&!4iP>qq>^0*kPF_J;jq}@wmZ;8Dl_PN+U#M;>Jgl#LsG5fHGSU0i3Vn}x7dhr`% z+X-UR#b$`j7P~=go>-IEdt&&6E!WY`^meYJV>nh3AkdVoi&%HDy~L`-5Rl5{AqGEV7hGP}Mrea%&!EZo2q+WA* zhl~9{>};{~#cmXvD+a#-m-o2XAH`k}`;*vPVhBxTTZE=^TQ)Sm0og|3SVgdf*fwGZ zi5((#qS$D$sbZ-3$aP#PhKi45w}{;?wm|Gzv5&-%tj)G0!GsPYhdsyrVv8Tjd5_?Uoe8Xs)TZdy-Wnw+_cDf?dRR7u#R#K(QLJL1NuaujTJ!uW;;th&RpDb91JZ zy^!xtt9-m~=|ZA#nJuxqMZ#rfq1pdRZ`pV3xuT%AEYJCq?=Y*}t}2K>;QW8yGrT3l z#apef?7m~9nFH-gql21#Jn8?uXXat+jxG#6Z2!-A&wBh@yl4AaS2FUu%Est3>u+Q9 znc>NvbL~AYoQo%W&P6O9J!$*o`(uj~aYyoD1I?J!N+4)aKgo+k4b9;>^DW&G4w5BhOe@Z-f6@;!(Rq?YgJhK^K91R>)^nXJ?A}&|LS2iwPMIkz)luD{l8! zWn+U4bNA;WyIg>`P2>_3jcbp5R(Q3v9rqQ$X9X8v5Ui2UiYv&KpJCpdM{Q&&{#dK6hs>vWu46J=n%6ws4<*rZo!q`vwzOdVT*@v{$7xu85Z{dm@d|@Wz*k9!1 z3WB|SQ`)QCEH`CQ+0MZo?&gGljO?%GAKNT9C*dFC=H>gx9`*I@ns8sG{bOza75=eN zi50~DvE^L3{A1j3{BCY7?H~Kd8(QdF3IEv3ZgY66$8uL>`N!I@Q@zvS9~40s z(9e82!#}pnjk5e>8+!K^^N%(9GGhPOvpyaB$Dk`BG9uRa3w?y^X1B`nj|ILZ#r$Kp z`!eDVz1yebF8Q@jXZXjiccX0o*hao7v41Q&bF%$ofA^O0+4~Qlj{RfNPjX;zajtc% z{=vyv{;@ZF0mXbU&-&cBR~9CwC)&=w9JY>+Pa*%7VXJE zJ31x(WAuO(J2t!fV&h};y~H$*&E7uUF7ewbbZmOLRmQQQqb~N3)r6~x{A1A`?QGxN z{txtz1^#ugPX6P@31G#mtOHitu(SOuTJ4Ng=F?h&m2M-zT;2|S>QtTPASH+fy z{aNfUVr#`(*-_^5;4$Sox{2*8)>EvX*Z{HNVs<8k%ez4AA~8F@;X3TdhITiL-75C5 z*nf#F6VGnD&=bC~C)XvA>AHSH`wqilGVv*>>iO;yS()j#UKx z#0H2B6+2e!La}LLKNY)LY_8ZZ#9kD8Rcy7`r(%Kmow$AQm2sQfiFFi%UyOF&7duvL zxY!u6@nYwRT_E;LvHQf9h%FWSM$9S)gxj}cIMycEMGSs1Zr}c5eZ*?SP8FLdHbZQd z*bQRvm9bxbC3e3U{9;`1t74yv{X=X6(@SaBCLF5>HW%AU%z_s~JJU78^)@ehJ=^Xc z|70H7!1`Z>d1U^DJVaI@kIX-hXL)43>ig{1>+?y;OfoByJqW2}HT8Y|m8oRN2DX|- z^{-hb*{JXfdjl+wtfqbt^T=#{N+Ro1&)V6=^T<$2J1>vyb9*i-=oxFCM`k6r;SW<6J3jvPu}?I8J?2@O zS;I-IRz%IpFLz6{Kucdjy49pr3!7G*^k(C{kxS-VM%B`%kt?>Saqg50I$EySfYyEL z7ZoTa-cUJX|1ejqP$BM?=Za099G2@2%aGTUy_T#*KHoYc`dXjmgjuceFP{tdX{qId z`JOj@w(2$8g*UJZ^RNpGkPl|NaL%h8OV#I|Y1xC8CRJVbv=yqZyc{_%$OjuV;`KuL zU^VqmMEPG)Ue_EP_BcnZMLA*cST&^D{aZa+(V2#9Fi%&svyJ7s!`j>}@6qV7GK7n(cJo)_q^I@8AT{Z$X!6*ld!y5FqeuNq^+Z|kqhEY3}T z)lfTs$JwgPdC~%Z6<)n+-MU|fzv}zn#$R=C%3qbv0Ba~K56;N&SM`?PsjF`&{wwaU zI?8tFc()s~2G}d!aw}Kq;S@UA&Lh)VCR_-xLYa% zY_2(q-V4sjMQwAz0luvK46r3wT=1&*NFi_9Y2M~@Uq8HQC;0joyR$3K0K45Xz?KCU z<|4ajT`kA}JI7b>v9I7f?69w3p}c7jVk|eoGQbuGm*yh7Xa$)WU^ja6I0Fpt$w9E8 zJK5lRVg}e>e06aK*tO>M`ZyS$i|nG+<$Jwu@nvt^Qm@y4xj7whST%xLk?Vp(i$guc z_xwHIlsE(I2J>+J#quE{yJ%DLJzR~x?3IZMRI!KaCqAzQ9xi5i70v*g;tScMnTKn% zZ8S6)D^TrTqS5A>LnLTawh0mKb~S6B{Y7`wMn%0_I1fJc#v8P{EO%~}cdMIkL!ABf zL+{*A{IojL<{j$u`Ps=h`|CW*{@PLAE&e|8y<0r}8p;kVOO}~VXMZ6!{tv!}HVH+x zQ1;hKw;JTX-#Ghgxz%>MB;k23ko|R5zad9Y7#O=2?& zWq&>ImScP~;_R>G!Hsnu7EQCW;MaisKTTf+oVFQ-AQJy-Usu+*%``WFFd9Rql#w!Ol^F_Drkm%S#-m2fa z)jkQUv@VZ>L5GGA9H@r&^r}p3@#523NFvv z==_TG=z!-_XWvUcP484(*B-Z`zPSd=-s(_xSYne?*<0A;hkOCWJRo=a-1xx%Dzq-{ zjY7{p;#MU*9yss~iA7^W8hv`NR2~}lq%l_=Y*u!tM{OYW#-4S|ZFIW*a>5i*ApaZF zQg2jrO{ob!ZB^78g^)_PAd@y=`+W`1&soH=KsjpjWcW!?Fmn#aqJtapFk?eD^Xa{P4{(06vv4dr{wI2$ zke$U{L|+8~p4&0KmHm4c!2UXzFJ`n2H@6|~(eTZ{2EHfUd!y{EBW(H-oBrH}xR=3a zG}VR{kx+;DIrx(P9P%%0c(6@FtA{UX13RMdCE>IiY(pIJTFkd0d`dV!;8Vgeg-;2G z4L&6tF8GviBEqMH6A(Tn_%Gp8>S06pl<-V|?Z!v;BA?PeHcp=s#^_TbgHMU9J{+S@ ziR>;h_>{=tQ)1i4#NbmRdsFNKG5D0&7JW!N>*mhce7+!2>H%jaj zvCG6R7lTiU>%bMCZC@6{ZITQ(NZKLmiVSxUvMxv9nS6m*luE;(RL)H};vaV={ ztSd5PU6CQ{iVRs-WXQTAL)H};vaZOGbw!4(D>7tVk=2PI>xv9nSF}Uc6&bRw$dGkK zhO8?xWL=RV>xv9nS7gY#B16^{8M3a(kab0dtSd5PU6J8;MTV>^GGtwmO%OxY6&bRw zXosvTGGtwmJtj6!>}@gRU(v3ud1crZ`B!9Hiy{AtY&S9FUy&UkhWsnC!D7h2B7=7c zV{HQDUy-5YKN<3`$Q}|y{uNoH81k>kJ`qFy6#OrFmu8cH3~Q zBB&Cp7W0kbysN}!i~U~g4`PeO-Vj?Mwo+^(^U84h+J|Em!QNu~i}ew! z5gRXdy4VF`7l~aV_A{{tu|~0_V#~!o68ozdJVe~)PT^QZu)o-WVl`re#D<9-FLtRI z@~YU+kBB`ghRi9h7nxJEgNKN0gK(@O*hFk|vG0iOEH+SVu-GYL@EUO)=ZZ}cgNKOg zMK%@fn#7(KdqwPZvG>G25PR0NN-l3VIc+T1P>^a6FWF?xZ1AU49r=mk1m zb1$+ndVzkXxri#F7YH$X`Cg!D1-w8vnH9W1sFCsQy+AF>S{iDdGY-G5V5fER0_~mQ z1?u{3yg<9Byg=!!rRU8HG}V0`(5G{ORWA36E4|u}wcQx)yYW4FbGrI8{%eUB==*Bd zJ=LyM*3u9=Q-YwCI|JgZrB~!A*~A(7Z}##Yeb1FE z)D_~UTxwZM!&6yHcZWWmsBN@)`97V;eSL5JyL~!a7xd{c`Dj1+bP&au>C^epx45)V zhZ#$ow)$!A!YrRoI%CQ7EBm8A6F!|?y~AUlPSoK&gFWTb;qN2gr$f8v%LZinbfD1S zy)71D3;A@Ob*tE?GuWqF=+pVojbjCOwNJ-BooHd%KAn5LW$e?5T4wrm7J9=BpU!n| zRLrOIh|i6EI*mSE%%^jf&yA&h_AdMy#e6z<`P|s2^MFrh`E;&v zt1O=mlsmf|B-~iZ3Vl<3ZtT;69%yyB5)P6AJ{?GfmV#i{(kNrJ`R8GvnZNCJu}>#D z^4UI}NBt)ipMChS0lq$qyd3|kFD8RaO?@Add)|CHowIy8xA+2z`E;)Fx$%L&F0_t+ zUxmD7x4Tt_AK~7_qR}V#2$^M)%BK@u9Iz)Znop-oi+no6TI$n@t|>J^tKu0(3+zEI z+o!X!UuxrHvWZW}KAm_^b|~%B`P5fe>{zVux$&`B8CsV<7GJnk#<2)|i@9hVi*lci zeLBfK_>avfT8(2J{^2cJt_8#*sEeo#QrSy7qPWs$Q0u8kSWA?1K`3X#G4NIPT-ks(uv z44Fb?$P^+&rV!cp#gHjPhD;&aAybG9nL=d96e2^W5E(Ls$dD;ShD;$cWD1cXQ-}o_L10^Vr#`JOjpBF+62fM;_|i?L(ULc4>9BnksT+7oFTHQV#paH zgD(eTZ35&Bk>T^rWXKsJdqE62Lu7A>A!mpT74+B^IYVSyh#_Z)><}^J43V8AHb!ik z7;=VaceNP&J7f=v!M{WHgc$rgWGlp0ifwGZ63n%XtZ=L%*i8)I0&d)P#KSQcZ>a24E`Oq{gc>RVrAwRVcQMDG4nu(Z7z18STC_bVyKqKbsR5t zg4pF^SBm{g?0&IF#o*uJ^5Dzi_WfCG1M{DdwF$>8GevAGv7N=bi47G)E)dsoiP&Xg z$oAnnknKaeMPjdr!Iwk3_ryLC`%G+0^OewU+ifXIF( zcCFa0Vt0r=DE6?}4W=n_d2_j4l3VsND&BqOuf3R7nTB?P8yD9#1Q6PY$eKjSQKA^@n$_f3@2t{pm{v)2jSIYOA-oO6v*1hVV=+&*E*F{fswp^iJ^^KMH%^xtc<7SPw45}LP)7t*C z2bS5#sQPuX4<3yl_>^ngx4jVYDmd5rereoixT~%y{WSh_q z28(FWxoOpyH`vU+(G)GSbJKgPK921>!=U3&qs6xUNn~j(Txi=_SSA)`P3^N#C)rny zme6~&YU%M+)(>_8wP?8IllOT7c{t^}R@bBN78K~a#g#K&TI(OysjigP>+ z`kb%V{7~Vm>nG( zYm2eu(Xj(kGvbf<7Ly&5LDL!<=O4u{JUwIPyOqz14vu|^Y{5Dl8cX?!K9@FSQ~Tyq znC4Sc_DoILb1Q7q0{g~O`8QA2mzDJ{$8Wvo>dG1I&Fz!=&J(_lZf`>Vb@c8w?0;b+ zE?0S9b=uXI#;-apv>syq4s;}XZhm9tWbDdlcn16Yx|D~f!1vFiU|ZWs3sJ}487U9X{qAr; zUAp^y{to4P1wC>n=PogMp5LHJdwKY6^hxs8boF-lk6s?!lhR*%8e}`w*X^dOUH4Qw z+}&E1>w{XGi1Uv2uF9sa1qynW7p+2vPZdqD&7kHt%`*CzS$@bhxfXtg=qz786;Q~q zuljVASN%fL343qj@}2CP)6sTyHXG)K9G&nLoa~2rbDP)GmvL2M9;=^?vE@JP^Tzn+ z&&O!_JCtt_EV7Dd`Z|z4{Da*?PRT>wu5YWOg7?iG^^mU%xh425;~9BVNuqIXiPiGi z!F9_Z*wPkZpKmGK$~w1quzfJz8q5wJ56;h>lZyqebQ;G*YXt`&WZc-QkmQe7dy`Il zvyjHI43oi_(kUD`Y3eq>&>Yjv} zoC>!`{Qjffs9nM+_2C{Eq3_YZ@uzo^ap1r5xhzBm>pOj#LLWNs0iWJD(K?yR0j0%0 zpYB_Y*1P(2+}e~R`|FU{z*K4n8qId2Qf>fr^8&ZpBC#RK5BJ~(p4<=|2>2h6iG|&` z)TcL19E3ut9s}HJXu>KjUSnAeZk6Xauc9SaGyuN zm2wlnj()1@u>Yg8=Fq$y$WJjqSN_p|al3gdbkrYwIzD6hi`*@-i3QHRIo_z)x6n;e zv_I}OHuTEGH2RQkgN^exG(X({R=0Vp65pW5-rSQ|G*8!I=@bBjGf~g*p%z(=Kw7lJGejK3x;FpP$6vu&|vDv6}zxVV;#rmDuB5}X( z;?@e+53cgVQMggFAN`$^YQa z{YDz>W`1~%pOxGsXy5)3V~F&~``}JP5{4h#!Oc6cET6-3_y%43_H{P7t{>d-dSbkf zT5J2mGumn!;-h!B0oY#$Q_b<*2K9VSIE^-aZ5sDX;o~V_heA-X=gnZGN|`t zJBZ<)Le@hJ7ge$!ik%{Ms@UaXSBl}r&bGIUEf%vXZa%jnSShwzY*XuZw#AXcSVe&F z7&7y}gzPx6S~2s+gm&hK3GI+q!M4-I=7`~P$92GoOS=ZKXT=tZeJb{aSX(+=Vwa0uCw7w<-oDuOUa1I#ZD8OBzCRX^~b-D-Ja|wv0KETqI11a(P{U#7_ubDJ`qEf1X*r_sO@^; zn0>)Wtee>~XOt#FmM@E4IGr)o9Tss0ha@g3ZLX z6zeH=u-Gs$=;>VV$ztQg@I@M~17D<}-OXaRiXlsab_>K_7F#Tayan1JZ-L8eUdySI zU1v%w!+1c7m0?`xyg*a`aubNF>u>8)XCzjJ`#~9QD5wlW{{6%*8wvTB>SYn-x4*5} z6wm)tewS`zH(hjv{l;4^{w`d1@fXVB|8L4~ft}10e+L#4f6uh@CMo_78f=nlpQ`4v zZ=c$v=38fdo0uLn0RrzVhZm#-;PD$7b|H6;4|!l5Z3zK5 zEg<_BOo|D=1&?MX_~I&5MDV@eEGWSuf|23-4K_wG_G)X$xi{Jv=l;URIJeBM zHWYz7+8D)Mq@+>=uF>3KHbxQnWZ5CufOcK%!q0W=W@B8(UN**c9BX6G+rRSQLi;~o z4z9HeZ(0t%%dGHPai$Hwy&Rko@Sm20Pb?`15BAqd$iahge1gM@%E2{0|3@h~xW?p< zgn~PM5^D0Z_JZwXCeCsP|V#uA=swH+%buH zv6y?UFMqVe+^BCi2o{tSbFcDtH74f%#l+l){e*y++u8aCk3AV;?w8(tqXJ^?xC}9O zpmj>Nm`geEG~Xj2JeSB`bB~T?*Qet%mcPghF&CFY)O9I*TAk{1W2FxxnB z)2FkeKW;qEEOt#l_qzti%7#Y>Aj#&1`AAg9&-0rxQ*~gh;oxgyHj!F>4`yEk+Q9q+IzV_)@%-z6SXNkFp z^v&#d2#Lv6Asjdey1JN)hphsCp_p4|8+N!InRceF)|zF$m}??sO3Z~AmKJl*Hr7nc z#W^V$in+M;CdJ$;n@o$jIHo1Oo{sW7_uIed56@StZK9ZzJI&^GFi916Mo781V?xSp zupy+}k8KDk_bXd1BwL)JkaD})5K=DkFTj`B5K`_k8&=s6g-{^nLL#$Xs)IlVDfdv@ z9!R;vZ3rnB&wY`Udx?!x%EcI^TrxPLvx&4r8Cu$1CU&{lbz(P(-7R*n7(`tz?`1KFx@7N)t!HN;*@od* zMX;;b9%676(QcsFF=EGwjTRd#hIVWVp%-Hn!8KwKddY4VgV0O%N3j>gJ`wv&tkQ%M zv}hA-5{^{_+lhTg?7LzRdbtjWx@^!jx#C|Szqu5s0u~cli*!t^7 z>#YdKOkxzxHPxrX0cnv?h`|}H0|)|aV`%+FBwE# zvJb>6Of)4!xHRoh`i2apZ^(8P+e7Ryu_MJMi(M#olh`d{jbiwuIM?yA7>eMKtr7cL z45887zOBMBlb^-9iR~)}p_j{psLQqpk|vufc8M6mrD=DK*cqk|kewZl{hNb#gMTh~ z7f(*1dTX`vFSh%K1^IaZ>cRp6s204tja|ZB!W}eX>7cSV5xk4wM1)-q8u3gNT$%{p zo$tZB^AWsjVT&G7dudf(%r0YbEs}P~rHI*GjF{c(>4?Q$hFIKc3+ny)%V%uP>mJQJ z-6rEG-8aW9;W8PeYfI^PX&9s%F393hxuEi&B$v~6dLlA1D=_y$3(Rd(JTP}*<&4j% z%@zsDU06A@!FrZ`hQLqsTh!myf34Yn^R54igyULm{P|%xE@Pnm8v@oe8Ix-@{~PCy z9oDukv1=`g$1N)wk88VjTtbqaAI0OIVdsHMvP%la<1UJH*Z=V*vX3_WOCoTet10{A3fq{VzU-SP5q(?6 z=-a{nZ1gQ&X(@%`^OzJs@eEG}F_vtNjZt{r$SxTaUbnF^3aWilho$ms{~|n6%vbGqZx+T5H2^E4OA=*-y)@hn19D`SuIB zwFbv1KtcS2+T$H67cd|O0Yo)6f3VN*m6BUaMBl=m(YGkt(yKj{70`6_?K@UiyDZl$ zH$Fpd-AA&iZD07;lIYtSwd)==v)x-isBXW&&W(RI`u3}`Ou_as>p!HPXWAB|0S{-6a{iVSj!4Gp|N(i;MeE0Mg zH3`+EP}J?kZZ+CBCyu&3HleQ!4A>K5C#uP?cnP>(*@lOgQSURLZfa8)~u zr=n`H0d?Uboii>gJqH>4w68X!O&=hMm0J*T5+Cz{1kZk3F;E)+#e zxx1L~x?{MEXwBREba`Tz3q;Kx;YOuD=ZoE-`dl6ERunbcD{-oxJ-oTVcavR*AU?FsjZX`HDkSfd&8ec?=g- zTi?<4E-Om5wk5(U&pvcmZDC;*J%|dsxTz&h(E|P6&F;iF+oOKp)~7Rs)xeF4_47x* z;<%sx>eE>QHfvsF^z%l(;&{>RefmE)D%Kwxae(X(&rYjtxTXDj1Hk?|n1tDqsMw3l zZZkV49ktO9>WZ zlwipq!ID9OC4&S@c9$3=STabkw1Wgo1__o75-iyVVvu0THnx4Go%L|YAi3rvw1Wi8<>A?a>{Kz_mB_9XyG0BVEZZWQmUecOLbg(DwHTslX@_W9wna28*gYt>c3A0ZGHfX zhNi|!Xm2Y;U}34D5<2Xk4NFC67PZN%Y>Yvncm<|PX-}WwTs+-yE`lOB_f;FC{`tO* zQT0S-7*)@$?P9{YyV)4k(F1IZc4KXfb|2Unw*W!)Tn8ddxQ@hjzq+i(8 zG_8}KVpe#r-`9rUUMF40u+$w(>ZIwg)XJhdX$`J6!FDN~v_x1c%&C*MP3ffRu+$}{ zldhku&W*^>NjH+-*%b@-zm|lhc2>LYQL|7d#WQfG$~nhQxghv)sB)gaZYpOwEOjfX zoU?peo-~ycEUj{Wlqh(dRylVosB$tawWm~0E`N!soHI+RoD56d+EmUbP364XH3O)e zf%OgE+lqvxMk?pud^g@~6|0=XGgQv4yg!o?Mj@5+M7J8_n-hnlwm{|lg*S{-Ar(+LsWKM}H)Ym;ta2XU%gjx-+LRoGeo+qWeuXO(v`?Ds9Maw4ZTTjgwJ_tB)viCUAH z;iC`KbuZdMy-;!+{jxo%X3}L2hyQ}Q>5JQ+L z?GR>4JA|2%A?P6vE#*14~=av5kqA(vTMX{5xZRsYA5ZU6~pC+4D|#sR^f5)WI3BA+aVll zV;`r9Se00{7%H-{ExxA6wx^0s6q_P8RqR%=JH(zA!#fg}_rBPNVxNosLu?yU7`ZLm zhhr5%Z?S%2s186oR0rTXP#u5_)d9#*S(ofOu|J5-7kggpMX^uCz7T`j$>p^U$Lyvo z)=6xCu>-}1iXAJ4y1QH+>h5xzXNvt)>;bXgip>{$QtU;sSH=D=_NCa4rZ957yM$vE zL0_>$#gO$+J7oQH`_31u6Pqjc3o%pzU|UoIVA~hOUK0CE41uJyL;gS8ZX1qS*tFOl zVr@;aq+PpktdMp&+kVH9c3EAyOVxDyZ~ED$AK7r$Xbxu|W5eQwT>IBQ)TOok+^m{w zuloHgT_1V+frBrtglkaw_+jn(*Z1v&@U)=mtM(T4x8~G&NZPOb!#osyZCW#-A!vMO za;JtID!DeTwfyNOtL*9}UCUmGi@MhKTvIs{4<;sm*1u*|T|cj`|ABp4uYdhvt-D#k z+rmEe&)Awe*LR#%d0%^v8?*(^|M^ABw=0`}(XwTU74<5cU)^(|ttzT2Svj+#U7AfR z{Y@Q)U$vXu*}kP`8!~15pxX3T^%yKU$}-rG?jC*RY|seQBkff`uU9tOe8+v3Sp4b*TX#YS_Nkwr4yd)x zSx`A+8$0fZ8l9gi<%+qMmu;POPuN+LI@`o#bXFg9)<$+04x&!0oO!iP#YJC7jY?># zeY@9;SY&NG=6|KoI?>AN&TFks^gXotv}<1s_wPyTK;OTnRjb~z-D|||J&oO4*tmRm z+r7Exyt;LVFm|+eel1rkMZ4hqi~7_r?9**gb=jgm(_XB+9Jl?tIIYe$zkj_|86IyJJF_x0$A_BsjD0j&$oZa`kLh+;>KgWkS5mdM4XL)>6E;}( zvef*+sri9VU*Kh3VUPC+3~M>`GOpI67q3@m3$mX>_dz2T_35?%p}mc_OrDX~(=~R| zmMv(UKbg^?>(ardg5HRu-+$VRQ?|B>tOI72?>eyFtMSgf8Z+z+Sz%ic*mpF)d8&Ss zvS(ZlT~j%upUI&%GgX;&;LIDk;A+&Xvg_$9?7#CC_o?jKdx8Bo;1T=p2Y1?k$Ih;) zezn`suwOa8EOog)A#;8jo1cvN18zbvw*AFV5NQR-#d5+jj8>8ZRqvqaYV^l^T z)7;bSzD&h)wvFX0p0BoPu6Uy67rwkV(}v$p@wB+mth%hVF491-skK#E{mGJwClj29 zn)PXJeRNU9bCl0NC8c|(g$V3lBisphvK0jxQ}Tg>5bSPksd?7gRD=DqcR)WB%yS)g zLR4`R>l^27w(<3!YtvP}{?EL;!8uyj`(-EDHrAfFS3hPaG;B%3@aQrY3vy%PZ)dh$ca{n6^ox>i<;{D$B5T6 zZTeVW-FJO;XW2B~#w?9Ihq|w4YITdtjtSc4F3UxB(dzQQ-}e(=_JHyhmcE_q<`k!} z>RR8FU42taN_C#kB0`Fje@zw+}_%AUN9lW=roI9JXD2ep{FDAqq#Q{rPvt|KW zo9bkLNyWB+j=s>1@-*0#j*jkK<5tB!#rKfUjX#L@s86@`GY}iHz^AuP9Ef!E@EK-t zgCF?zETyyKz}CA_X`LN*U%MSY)rWe| zgME5!(ADk)3AaQlvK1YIZ+itneD;3m)BIE)&fe%Jd1x>x_qlzl?-=`NUwoPsI2RGr zhEICNVdT+`2OA10FnOXhB&M-BjbZELM(5XtMb9H#E;`?fmT`E)jrQ_Gj}vEcxGFf> z&_~gp?HF8_`!F{o7}jclea1E`@_1$`O@6cR7L4cC=o-~8IH35a`xx}wE#nxW3T5&b zA%Qsvy7+Y>{}}zS_^12O`@4I?V#j71pBtaC+b5=RY<3J=mp(S%bE^`c^23I(P{p1) zHv5OG!j&)o*j!NdpZF<16o6=F{Sf`(xgLuCcJ{A4EvfBPX_Ra)Oit_6EHS~x$hy#QmF&a@(QS=ozG~zNNGb{>%3NBFsilBlh zLBSOrf*{(AfdpI<7ZmpmTw@ZA!;Yc?Zb6NR7!@=`NkRk-`rUI+^{MKf?yiBv@B8AL z@hA1vzwT32T~%FGeeUwq;64EbAGbd!__#?z!N+|83O=4uQ1J0=`rM{?s6wfqY||Ev zsN6%r|0?8&GlhcR+NKpY-PxwSY>Mj@1tV}NoM%&9Q0LneC*7h(YH%Jev?IL5G%EFN;~*zxV|=GJBoD_J4mcbth?A*V&=;U z*9Q+Rw{eNs)nXIHZV{U}s)@Vvma=h?x$k#J&=7ZhL~$I z6uaR1P~w7YPqAKNs9;RH3&k!L8z**+*etOp#OlQ6i7gd-P3#M?uf$N|g8R2!*s>^m zu|vdAqnLImXTg0QEq0mM1hMPHZV>yW*b`z;i_H^TB=(xvaa7wM(lpEhs73(Ef)KO7{&(Vu|jM%_wN(2ZOtD+);w&L1qX>8 zDpn=dUF>|Zkz!Yhq3i{>i%|l(4V1kgdroYw*gInHiLDh|C$>qf#5^Zl-)>>6X@Ic- z$-0a66gx%ibg^^9&J(*y>=voS2 zij5JwR;*U+X0cnvo)eoZ_9wB`Vt*IgAXaKRDL+S(uw`}h#Xb|OUq|{TdwTr4y;kr< zg+-o2K>)JfyyE5m%8TNnVw1UG5uBn%-pIlRsm*d~*r24MC9hkUZ z-$~WXUVr3rG1tpZn5UoWzg{YLNBpyS9 z{jcy+b^S)XRESGu^gMoj&^Oh|TJ%lzwfh>Eo^CBJ#alaGp7Cl$+VR7jh_AW=y)kJG$d#>)Xdv0#5{v5M{=W2{ix7>4;5UZc-xjL<= z=W4ji5Aa+K$M@J$vEOE5-7sU8ILA2fu2A;BWE`)nPVRof;Z^8B8z9 z*p&qBz2BiR!F+qPx1Bc>g$D*Yj0ogm%d&&(74|BQ)gNcnVU*7cA52dO!7Hh z?Yl)s&uClqvA(&CFDVF6H_XOel+THqd((VQ%}UZSpA(N$w$G{7*L`q9eNNNeysclq z@HzGIUCH>8f&k)#y?eM{1#()Q+drGyJ(%Y1MEIe|R+t~ENy+4xABww|?T5PC`|ni5 z4|Q5>84Jb05A}-o+d+OPZWEXMFN*u2n7Lov4>iwMepI7}OMaH*huY|-ksk`-j+sjw zr?QP7uV(T?)w%t%sUHUC$q&WjlI@40U8B;g6a7%P`WBM>Q1jd>@3a4T zDf$2O`N$8ISy{3l>QV0*`JpmB6a7#oJ=$M_AL?c|D&&WH!IwrN^vgaU`JteMWxE&R zeyDSDdY|X>k<$f!fJ8r3of{?jq1L!nAwSezzBKYfJ>>I|AL?nJPx3=ul_$d9#eE%hZk6Ecc+rg_Kh&ad zQyD*0=9BFn{Ho;7=7;)GgZxlq8tRAQ>Km1I3yvu4hgxOt13{4FheCXQ{2U$b^N}B_ z4vYO9omSirMNd*D#x||6X}L||g~HVeFBC2Wc%g7P z!3(vEP2q*YS%en~XV*Lvo>y{(O>w=#3x&vhc%gdO6ke#oHiZ}JGMi?+P(QXdy-;Y; z3q^+ada`ubY8t=`MRvc~!(#A4u`j$(w1XFl3|=U*HDYVUwzFeJyOv?AEPxk^3@>`@ zi{F7{2Z)^@c9IypP_!E=hDQR~^3{PpYH^kl&`%vs-v9H7$+1X{^ zJ;PShpo7>UVx7ghiQ%1_eFuoyl@PM4#Kwu)`Sqnuab`oi+r?&x{aWmIVk^W}iQ(GB z9yAU9Dh979StGkLxqsV*t)@Xsu|3507CTJrNHP2x=lX_=A-0qZztFjjYsK(eo$O{Y z{Awq|uXgr*RBXQ3^I~s`y(5MkK=$1zh8r>&zJc5ZzI$Zw4U?hf3E6RC$a^6}-V5z+ z7Q0pKezAweek1m**z;m9iM=ECp4d9E^^8AG#TJRZD25t9+y-g@aT_I#Gi?CeP@ZCBlc4< z_@iitF*CU@@JEr|DRz$-#>}MMVzISi7&DW7o0=bocI9EKEcm_{{86;)Ahw?vd{6A# zPwZl`%f+UN-7JPo8TN%giraWn>^EX>iLDT8WZoP0#W|`VH>K6pF=FA}PvqqK>tks2=HFnVhKEwxNdQM9oHGPdq1TgT2n=bvM<|iRw9MMfuKE z7Wufa$DpMhQkCU1=XpLdXT9%Xdgr%F4_jIity`FaIvzVw|M_Ri&P<9{+s#^*!vBj=4_D5^|j~IZM-V9Ypz04a#?d zm#8kMUq@^?%!m57dyz876USS}lo@15aN4Znx#=p;VY0(@X*+))UfQPXM9a8UIRx=54(=+7m^$?hZ+ll_uTUpWj9SQE$@dp*-a0n z&I~>bE-A_MEeUq?b!BHa&Bp42xBZCZ^I#qCeNysZQH6X{9xR4tKWf=c?^%iRj9q41 z4ajaf)i>~^g^O3Vfat;Ore*SAaRX0TcGH_y#Xn=0*+61;(>VWpQFfDQAoj;3sf%rM z_^lrVbeKiiO?R57>9gScFqsN29v-`FPt!zn4O%tS({!hsN7+qPzAMfB(Bf1d=)3Zv zfAT21>0a|HeG*()lCjI|O14+&L0|W}dWE^7?52b3d6gjWzwiC3-K&JcT0Gi|ZN7hT zuM$+1V%be_S-tKytsB*rOi1!7mHB6nyhb&^+! zrzqR2l=(b8Q@h19&fM%K$nCfL7MjMa@?|%@KceiWRYA94e9T#!C%fq<){&`2 z^eRDoUFR#qiL7if(hfc%(Su-Ba7ZvMwj24zIhg5|Lv=VZJqMWkyyZN}!$Lo9L%T~J4tvbaHe{QxCu9errbwznOm&Fn)nV=Qf=`hPm+Em%SyTiyQGeb?Q%|b5_aVt-oFZ5CcpHhk>}yI(7Ldv0Y~-$w<_Xkz==m~ zoXj>fa66Nslle}JXbg;X9rsQm-lN;&<%G^`B1zQ#N0=-~wZb8sC;hZ$b zD_rPj#wVjYW)IyFqI0r)VJ{F4!9RR+g}gu?`_kxKaJ+=9?^n(^6gU@Oc&~(W(a1jm zSB)KM?DGff`iy^qZ)^6^N}R;-5AWUZ`+U#-e=TG!%^xz*+VG3uc);(2=O_F=xN{$G z(-Uk8KMo#Z@Yk5{#r-$%CL%*N5gD?H$dFA$hHN4-WD}7gn}`hAL}bV&B11M28M2AUkWEB}Y$7sb z6OkdChz!5h$*vHq5t}ZCY$Dnrn}~gpO+1+iH#GxRqPJ2hs7QhdsXZYVyI!xefg(Y6Z3?Sq0$;! zc0U!{Q4Co^v_qBi_D67ZZ zFxAOvJ*Y#OajrxSgt~r{#aEpT+LOenhg4OnL1DqoJ#% zeCHb2{MXO&?e?==@s0Rd_OSQm!udIe+IuFK+OML}_kuTK`d+rN`viS2J6VgqmufM1 zVd#5-7lupWg`w{SUKlRj$DSngy_{gJY~M?j-92;jb0(P;d@p#z+j8GagYt7C-^(sV zeJ{+H}E?#umOWn>kPHq>M$ETmKqwo8(dwIu`3DdQZ*nyr>AdVgS;Ee z&siex1~>5Q)JWUtBn!r-7bLTR!udIe`+A%Bjz{@9@A`V9{G5kVX9gbySCnMzGV5(% ze$IBD6a)EvKi^#BHDP|vpM7)in($-HFt5qF;DVUfWLxi-?KQa`hcx&qwOKXcc2<54 zega#>vb-kvxo@GcE;vQ`IoDX2@qnECocqj^^5@{9m?wq1m+eWp2dlFrXcRGIZMWPy zal7aBq%c3Ht2`-nzMdCt{>Q~VDM|S`@TAQ1m2KZ>O3An+PYPaKpzz~L8f9HktUT^X zY2(Lhogc4g@Q`cFlX8eWDLgLOo|HPQCU`N`E76leDLcuN@~rPinl>2?JBTvecK3~X_l0P3O!ISc+ zuQE&N&ErXV*{uqBQhw=6BTvc2M^*tHhI_za|jd>kTDtW~`DZ3_lQf}Zq zt$-&5!vkd89XJXm)7oFwInATq)Blp7-rpHpH4ZjiI*})(4qFO8$s6WL*`q<8lx_|6 zq;T~wrn&`<3wu(2Ywuw}kmO16V3^n)VJmz03WCU!QioapBzqM1q@esoc2Zot=H6Sr zH1edZ2(62Iwdj9II2Wv67kN_X(uq7Nb=a@)6MXwTDafB$i4Pe5;T^h|AEnOh;75t) z&%lpzv|YvUqu>z+zlr%o+>f##oL_EJ_)(CLV80k@KC&tND7ebtN5KiUJ~cS8=h+mO z1NZqaJ@~kX)~L`kJ8en@S}9JDf}qNlgao|erav`QP83v zg$(b}WbmVqAs>e9ezAwe;76ey{3x`8AB7Bl6tXp9YsKJ4p{3v92 zKPEdt3_mK!;76ey{3vAbqmaRmLIyty8T=?@3&dU!dqWKQFtqzn>|?R7#E=g|JLJP~ z8_0(tLp}@{@?pr34?~807_tFkgT$^98z+W*81}U@8}_|jY=#)}VQ7bZ7;XdC9~tst z$o?XRd>At1!_W@-Fl5MwAwxb48S-Js_7*!#4EZp$Lp}`ckPkz4sTlHM$dC_1JLJQV z-7JQD7&7F;&<^=9Wb?(44@35*81iAr){AWvLp}`G2hSw!b`e893>oraXoq|lGUUUM zAs>be`7mU+iXk6{>|rtF!;n2IhI|+@{N_iiEOAr#PsorDLxy}9GUUUMAs>be`7mV2hap2g3>ora$dC_1hI|+@Ar#RLDk(!J|Tkd>Gonqe6DC*n?s(i7gZRMC?ir5=s{}i)#)^L5Ow#L4ww#IGj zDArc2m)P-Q!^AESyIc$&6|N876K>;fu?1o;h`k~9me_}4AB!Q+h3i9}3%5}%)`V)>JgP`SXnyGoKl!r*tAl?Lon_3Y0%7+8fpu^TZ z)5o>9?34Mu2E97(r$`o>K?TU;o9Nd1J?X)pznh z%d>ck))h87&&!r>@g9vTA|vFfVC)_&wcHD=qOGN;yk;ZNe9FEySWjk+Z8K|ZH6^~n zZKq~TT1}3xE?f^rvavx}-m*KVedbLors2@p$M$(U6XCEuhxx&pn|rXT>}X92$2G4? zkFm4eugNE=s`T7=9!qWMT-entyGF6_Bjm=}DCPjnu^dE@PdFF3=iyoWx`92fI>M8S2QP};>E(6>Rd#qg_2&4u%dfx4t_mN|d~x~7bwkQI_Kr=**VzZF?y!7F zIgc99N<*=EAD#zYn%g=s?HEoGA7ysFr|1`O-j6=)M&vASo3ng!&T{<9$9e{a)0%KP zozvP`Hf6TTt2V%pC)<2=!7W?i6X0~ryL0fQRugeD&?=o+|7`RaG}ls5#=kvpjxGOJ zPXk8usly&^xcJKNS*BK5k3mZ`q7N3Vw*3mKQe#Rhzh02;R+_S}TE8av7FTuttb9^u zbI913K7RW-@I|-9m_E2`<&5c*JBZJIwptG2gO_Wjxol_M=vH~fT4Z>6B}32-T`dUJ zp)=(Qu}RjVGvyX7eaKpLwLGJxr`ih?*L<$ExaO(WqWh)JT6h(UQeRN8GS~frP?sZY z|DUw!=DS~fP@jAQ`((z1vBzX^y8Umz@7&*Uj2$5Uo#J{pei@zO;1{LKgY8QyN}Idb zdV=ZX9o_blRI8wG3ctV2X~t5CF8;wIe7RP=bnGo;E}##t*i%!hTpSMB-|BF z3fUnR#houHIm!ftQ!>?e*Pg}%AZC33r1b!WP=gBy7ex>t6NL63G9g1*{e_Qk~7b{V9+qTo?Em)VOTuMdY9N6 z6S~3>uS0_V8r7l&UUhTEV;^T(B-+UteEPSy|Sy@hS^xRU5=cIK!dL+p{( zDCpe+?vv&Af94PGz<6FG|MSHVwlp^cUMJy>c-y9!hfP?51v$pyDdPn*K+g6}oTkRkm5Wym07gKuvi74DZE zZCYW|b~ZJUy{3chFWe?q*%WRQ4B?6W#5;V(Z34NNZWFZ10=yQHjSE|4UObHK7h<=| zZid*e#W0*C*9W%=`@(HPwnhwY6S8gWv(v6w*eVOYCx#bD+8rqdrxe+7Vke0GSnN!( z>%^`XyG;yk6RvNL*a9)QO}Kw>o6v4sJN{(N!&X_)R;-;E(#>gyQU>gc3~jRhV&{qt z7aJ=!UhDy}N5tlc!70aWtQA`)h7=NR7bzsPYZkVe1}(&P7u!n=zd_j-=_2fVme@IB z7l~aWhTp2}J6Y^LG5l)f`j&~kF1A)|o!B;Z4Rd|X!j@exVmpadidBo9EjCQ7R&0vc ztzvhGEf9M_>bVpoez6q_n` zlh|Ei_leCBTOfvk@44MK#8!!YD7LlTw`tcjY?TH3itR6UtXOX`41CYN1H^`jT_A>m z?`bzp?AKzy6MJ0@Nh0V`7JMMKMr^$phMb|@4(3|mHhvJc%7UZCx{943c9Pg=vCG8n z6MIPPX|ZR-7KtHog!{5qY@JxsCYk*!4_j82Ol)Vd4r2R>!QsKr)=zA-*kxi9#I6&2 zK4R>F3ZI0ZJ`YsIfQDlwYLLWu-z>I@pxqVe)4vNemApdnwn42TlJ@H+8P5HLD zC-Quze~Wu!vK=b;0%~lURNC&EOmZWj=LgqPt@h~!~1mHeX%lUd7qr+ zfzMwxzHKbocE z%&>dRJ%PJYgWVJMVj0G^=uY^pwYU_opInL`A9Pt9WG%WZP8U1R zT69@lp`~~=q|4$KE&ZXrG;tevf#WuMSc}`Z$Xaw>Ot)6H^8z&zbDbB9%?i$oX*S(* z=S98Tk&Jv`kHOG+aY2If;==$DEVFuVS~>{t_N@RXXa79V32Z$3k?1f(*a6PbBg6rW0@08$rxZI5jxgMVJrIG95X`hdDM#v)B zLELfIg9%;s7wPO>a(Y9g$reKMxE>yHqb&V9C$j>2H?)`_DCBy$*_TE;GQsB~*TZz5 zk8dNN>){NyN^(8C>{f+b5AnfWpjyY1O-9;0J_~MXkvAqM^8%gvyHT?10kV|6arspP z*8{Jb;;x4d-e+@M55Kp2a9rMA?DLW9p^wiexE^kGqe61G1$^6IBzI4a&Eu#|4SN@t zyYFzT1i2gX8RT5J6>u^jtYOo|8|2kyf8+AF9@@E4p>y#aUmBf@?PK#e7vB$i7e5#6-74W+9PCEXx!6D4 zRIzh`2xp9z1lI#zaP$0yt_QphXIu}n%re{c@U)R^*F&9=T-U=QWAZX&Tn~7e5e!`q zcu|SF9%^hh*Y$w&Uc~hf1i8)!yu*HGD=OrCc-EG+G;tpf%&s=YgA~pO+y`#3Dek{; zKHz===L2qwa6aH>YVZ3sxPQR;fM*z-4|sOL`LK_D7&sa5&4%-#uT9~6fD;n@bDP5X z@K2jo*tFEn37ij2ZGYi>7+_O4A1<*eoDX;h&p023Tbs@YwCH>w8)th@hD;4II3LL1 zd?1@42Im7AoDZ~v^MMS`2eLI{a6XX1`9M23AIRW*AcOOP>`1Yr#c*6{cY@fD#m*GF zPV9QI+r;h^!&8>)TOfvWM254(ec9TMH(ArLRpy}%WZxG<&IVZrF*{!&>mr8l5ADtq zs}VzWD=aMw9u%7?h71h$eNJq#*b=e##Xb^4wgvmP4qGB(qH&g6+v6sZ&6njT(jTl@L+y>kbT;F%X7Du@!!zlN(!zlM; z80DVq9I^AnCW%cJ!&oBhi?Kx5_m5&L#TuEUPrJrp%Y=L}WLvQBA!41y`iY$?HdyRj zu~A}hO>lj1LvXvdi`9wE6I&|wn%LjPHi#`Zy^8DmW7x{)RJhOXmkn?#1ZLJ#IT2sj zm>kH7xGT(wSi_u%<&~BcQOT@`Af6S`!$yKf8bs+tD;SkYi-7DHyW^tkKsn7HY~LvH&(EVCp|dYi?TReaLFpK85j0fvjb)%s# zUNFmLnBDfq%qm(XdQNi8fRz>Zzd^l(dYfYt2r&tICVt=xL{hE%2JI4mu%z1vo7RiS#NUSf`e zoyhh*2EAh2%A_`Aj+SRO*fFZJV-(A5_?c%mU~rK?pV}W;4VghW`VGV=9QGmX{%bqG zX+>=frm;+oW%JB!(5lS$zAD|NURHyBhT--Zsy(Zry2D#lsW+>U*D%CRvgb9tkTyoRY;)!FXi6wJGGxUv~yC!oimIhM>|2VveETmG*c zdUF*_!!MhRt6;D0gI=zm$&l_+`t`z83gdQcD4+Dxe?$7h65A%#dHcnXNhq*e57}MT zqE3%sE&2iwjYVHTg;*bJ(HAgKOGj9XzJMAnMeR?nd1q^J&B#=x2Vj`BUb6qA^o1w# zcmQrN3wQtq+H^}j0C`64sP6$dteyt|h3Na)VLHV&9|VV#)&*aApkMJcg_EsEckh7# z+K+63AEPA!0jl`pRhEAHCD6yPOHi^FmIXy z#4V;N94^J)_A&e`oThM+^{aNPXw>+QnQrzG8wXV$yVtNZg@xX;+!UKAO<|d{`Kd%- z!DKg#d#W4s6z^t|_d zBm24Lgp>vQ&1`>}dNn0M+i-b8`odWBvV;^|3!|+|6YtjC_Z{gA-1lFm_AntZb8srG zM;hINrVaHW%yjc8eF49Zt-Egq=?mP#46A9rLBCsI9PZ1{B|+xxX<83 zU%G>=k?Mpyc_fJE_z18Ace_#i09=yeBfw!>=vIY%1i$d5k*;~8&nNi=M!8koeUmTf z{W-TPAI87dMS1;s`;huvbg>*gy*steEh(RuakO!`45 z?=!}{R851K|VS+l-}bBKY7l@7ryF3=K|5UaeoYAa7_x%+7Zh- zvqZ47VhKQBxmCis*xKI4GONb9fMyJTX)(FuGL(M*W|)#+W%KX=)P)>l7Qh4Wu1(FxN7b;A`fC1U zX9^wwcy{0cIL@ZnZ@hzNJOJ?W&;x)LJpg3m?9-6J13(5302w?0WHZEOi_I6qyCCge z7h5N`Ud(P=q1{$t%SNFU+gl8GORn!QF+27lJ4OuOSlZ#`46U+Yf*5=OWDkfvBKCVR z_yTB$kuJEt4Ps5~c#~}#w#tJ2#SRv$6ss0HUd+x^xDBKqaD7NW;C6p5hH`pjsAoqz z_)E#25<^TF84?j__ktKw63CE}Ks%%)kRc_33@Hg@NJ$_=N&*>D63CE}K!%hAva`gH zl0b%(1ll1bfedkHWJpOMLrMY}QWD6Jl0f#l7;cJWxE<0CDG6lF!WL5!$dHmiyGq%? zV@rmV1lr;EELpAC6fvYE(C!W~q$H5R7r?$qNg#tSfD9=KWJpOMLrMY}QWD6Jl0b%( z1Tv%~kRc_347xuVQWD6Z`;#FhfegAo8B!9+9us>?Y>wChv5&;y3t(U9{#;*+uvHdx z5Zh0zv)EB$XNV0JyGAT6cDLC5Vo!?wM(i!I6=ID{FX#R>4qK*7i?tT(BzBnCIb!FD zT_Sdc*hH~OVh@WwDz;c`iP+m>?~0W)$?Wr1Vax7+Vtb2q5<5(+tJpDOL&eS)yI$<) zV$X`r7F!{n0mkNAV0>CIC1;~4{Sbw4*_ zz~%eufuVN)bGy$`yD#6lBFHH5)UFHY_}kk)=hg9}YDWK=O1|BSTm=7*R`Px4zY!%L zx>}S1kfr2L_eKBzrU2w9`8Bq?-++=||IkCpuff+PtVxum`m?f(8Q~l^fDt%xphTaX49V-1&Uw)6N^k9)nf3x$oF*iU? zF`A|?~6=s2-y!PX}p~E>mRC?~@Q>N0NT~wui&HHUgQ0Y-NFIlC(*R3Lzo~N||Dm|6fNM~c2o=BzVisB-5 zK9&9o-+ZLfvsZ#j|DbnDQ0YzYvA;qp{q4RqQt7Aod?A&dk^GTLf0SD%sPxm^D1HF) zsr0{gt3oRMxNt)m+4UNqPg3bk^0L1qm42pM6;kOLbRVho5O#1iMUII)D*Xv=l&sRz z%@V2fJnO|(dT3ErhG;WXdg|KI70RIdNTuf;EkUKfz;Annl;JadX{6E*39aK2b$w;{ zLbpm#hOPR${c+V$hHHF2Qt7$(#Z~&Fy-#74z6P7{zdj97>G2kpG0z*R56ur_p(d;J z)VZT`)5hLbpo2!2S00rf#qy!M=Pwe30^peg27&jz(7O1!;F#43=K>?MW>$@Jf$>?B zRC;bo$@hGU)Vpz?oKDg9l$Xo8o(KZ%H*f+Y~xI?yJ!0 zakGI=e}+w=)8kPFogOz!==8XMylYcDgFdn;o;{!26mj6t>HWK5diy~(51oEi$e*_< zbb7>NL#M~v7j*j9?0iC}M+7r;`Xg-$ogVMd8J+%qYg4C3i#k2oIQukYcxfSnPEQ7% zo@|ELY_a)b(CKLhot}N6)007`C$pECkU^&>!@ZGqdy91vgHBI7JNBU+jxZU%v+Rr4 zGqMR{*NHtK2A!UE(COJ1Iz8E^Vt*6+S}bKpoOb15%f9nsdx{}apLTu3kat0LmKd}n zvWvtn5yP)HENvQ06hrhr*-SA+?~^?zhUk4VTrFso1&H1!`$!DY`(%jTryZjA$q>Cy zhUk4VMDLR!dY=r@`(%jTCqwi;8KU>e5WP=^=zTIo?~@^VpA6CaWQg7;L-alwesPi^ zdY=r}KG|Mk$BH3(pLX~aN;^dFlOcMa4AJ{!h~6hd^gbD)_sI~wPX@)F48L;8pxBck zdY=r@`(%jTCqwi;8KU>ejuu1oJ{h9-X@}^2vJqkzi6MHQc2MzYcemL6VvEI=h^-M@ zE4EQ=li1Ftl5-oog{`unTC9iIAhEN>E)=_1>?W~W#OlQ6i7gd-P3-St8^rc7MV$M$ zci1Wmpwp8U~l_tZe z(q!$$_7UqT)<+B@AFwY*K49Nl#HNeQ5nCXJLO|?`F%a1IbFnYQzH5p&?Y*OYA-|jNi=dP6}K3^!HQjUY60{^A9utfeq68BD3t7ezQ7# zXpi(~Rq3hi>KuG*QT5H|K}cuReDV+i6?umcSYm?|G_ipBUIVMH?PsON88W}dZdTRl z_iP-2kE*U6T$;YJN&4ONaan^0EVQuRT0GoL%;n$#Hg3SYs`TKX`?b|=Okub20v2{? z+{?xb=#@UHP4{%KHa!QuZFTPF&HHI}Mc3xuan-F9l+QGm>=6ggrgN$jbn=)40{t_$2+N|$_`I~$of$!QILNI^r3ai2&i%p+v zhqJVa9kZ(R!g>eI%I&ZG%7g3eP?c7t``O9u-yGuzEQ|-)*Oo5CFap1^_4gX|Vz1O< z53$eqKTI6%*gYIg079{CJri>82i=bH9E-V6V=#en;aCD@*T%C7YAv8XYe0c*C&xY- zMi)5yMH?icz-L3;eQBA0tg3X^oVa@oCNR(k6WCyqa)&Qd8^?cLe*NWkkyy;V?Qj@# zKiNK2MyapN9!p@lFOpI}zAk&d!Nq%4=Je}m`w%bQ133@7&^DY_gwuhX*7otKiUF}$ z&$3U~_8eqmoX-x&61Z!t8ruv`#=KSiC$*Z4Q-M~#bJ5+$62M6q|Mk2%{U=rNG?@85 zrITZ$J1h#%g{5$;^>cv%D+-(olkdmZ*p9knw|}CYy{HG}?Vo9fm-_Bw=izQuIiT{#mHjJE zAMoST`p@0l*2^&k-n8vgNdM4Ul+T|ubF!#F4_?nG(c`6z!u-+Jq8yLcE-t;yS`_QA zlihD+S7|RKTrFY2K$v9R@%xU(JEV`10{NqJEFUV+`2yG??z4_RkUZ>lD0S=gV7^9u>4J zsVR9XXqKAgL5c{o&WtgT5rAQOzuUJjMP2=&wo}LZd`;=C_VM?zDn(G<5a|CG=N&wV z9TC-I?2Mrl2-b!0vi80l|1f|bPsV%>feN>qC`Up^pT|G!hevF~0{^M*xAr|a&h}uS zt>bR2DA*?!mtSCfg8pt8RmSFU7m(N#WQ>H8Ip2)#bFYXBCx3nsYTu`!wbTt9Ue*gCrU zvYOa37TalMe7-L`)g|~=b_(izzPxm+;OT-AJc94%y4}G=>ipw(H6{4?khb{(G5Gc=Y+8HswP=P>s{M5!^r057)YL(AZR6}9~S5y^?Q_UyAQ>U)+wIEQQy=D}a z;vlZg_e~X%;9)evjS3apdD53g#daR`c?zC5co2C4+p9?JiwDj_-iR05IUuL^5kB8C z_9=1#>9K>;-6$)NHz$xDA7PnWeJ{2n@jCz4`O>J^4&E^Af#ru2A7hHoe>Zj#@)g^` zofp2`!eS-X^}JgZlI(x%OQStuV10Bz=@y8O?3^RvyH~ah`UJZKSD5oK{(-ZDxpC+r!URDF-wLkqc<~)v;Sh%cTT4ON z7`r;%jS9)eruW$&j}oj+@6$Yv`Heo`ENe%zC4E@kY>>dhg^N{6D6PZw?IpIa7~bn>cY+vh^JGYhV&C({@U-z>2w z#9kMBQ|t?|uf*DHmDxXp({p{0>ABDQh#e|+xEN%5+Wkmuh!}$HxxUe2mx=vK>~^u) zVkol1zVC{?FNPvJ-0t7Rz8165zrwz|hpnaoitKQGsP)gjD6&I#uGnxf6xpHO7_n=` zYQ-KAdra&NF@)W78y|{&ELLXHIJdD~*eVMS7poBKDb`2qbg_YAW5vdc-7Ypm3`KUh z-N(fs?2|#*=RSWRh9Wy;W%l!)cH4!mvS0_XABZ6opLYKvcAVG=Vk5;+^`G0gLkvS2 zlR?<$`XKC+trq*U*aoqGitS`lJNxbuw#ot&*&#bh48lGcgne!Uq4;D2#Kww^7rRI7 z0WpN)v+r-k-V$3O_Kayxw5tnS-@F4L46YBI0J$Ocm8p!%UtJMb`H_Ly5S1U#!MrN} zkc=Sxzf11l$=)fls?j7V{_iwg@kbfKNb%1cUuew#w*>$CpDt1J?_W>zztS$5|7!mK z4$Z&*>DWBkzj3Oo+)R@FTl}lC|3W*z6yldyi}CY+uofkGygqU%etvT)ULU#iyLQ{5 z*pJ*wx&jWi7F_}TwRE_(=n5DkJG{`*?jU>V;WiGp7PnDlEpFoiYtboiyS1{N0vIhm zH-dkmS-~lQ2#PIt3KWdskDLOm)eps|wf&#$6u8<{@dHXa#+(8x(Ji>woC23`mQ&!o z1gF5|o97f5@4Inyj#J=nIR)AbD=VL7DJlO_GP{URA@lKz{)`@wY$mGrOl{c7!^&1hT2LZ9y?NuT@GxOCS5ERyuE zbGsvAvVE?kKRUK7lJp<&erHS4=la{4r2l+TN&i0Y*Cj#H$8T=vqmgczPtw21tvH%B zB>k1X?gmKuE4^bR>A&jpk)+QRB}w`W8z1L~9FjhJB}n>@`=$~keQL~wB>fq_ib&GG z-RC0(`hK4;B6m_pgJENP`V8gB>hgll4MDLuG@9-s|J$(N^CnQF6l$i z3xds&^eN#-l0HT7=n6Nv&;Al5{mE`rNH)I8mqxNNtKvjQ?HXU6Bs&8G~ZA|B>kzIC+SZuEa@+`cNdK09^J;G@n1*#-os(h%b#bFq+j9l z36efa6c&!s$FI@2&JHCPsvP_q&@NeDKS|Qx&3h%dTj*Pe&JFzl#cS()?8oKXDCx7F zeq7YYduB0F|Ce@RTAHYi2Pi~+JXj&>ue9k6Hig7(GI9-~)FJ9)uwIDzxaJ}1SJ@Pz zKIB;NH8zE)j|wPeQ?uTtgKYXwJLVAe%?anCK1582`gogysE_yHjHo}>+7$KCqNqXSj#r`;f^w-g6r_iv33HS+RG;-WU5+>~CVodE)wZ3|nOZs+E!9 z?uAxaaHtr5U6MiGr`?%ikoUko?Kc6B{8mRSYruw7W~}KCwk&FN%F3_LUe!eXPZv z$6?C^axo0x%WWJb)dONIe_x!vVre-uO9KJET2 zwynwMWX;1?S;bW-#C|LGw%EI38^th`FF(gNCZBWvnuRT^ zVkHL2pLUSxV*RqpbjKd)4OQuf+SS?Z z(gxZ^4g7yOT0ix&1?!i3uzu%t7W7+Q`-t7xY&hMj@|j;%T|34ins=_UngRnX^8D>| z@2qhBxd|; z4#V@q6}8Q*sB~(4J=;zRL!}!Kk&lx1rHA=}nVWlHst~o_gaz-b(qrsw^|PY)RoO-F zz2?0gLA;Ok8nmQW>V+^M-;OuJzO8~wEC@esM-NBN>)s;>-wqh--Xj1U$E0*ptoHqu z1mNTBR;9b;1mOE;InUMj<#vHoc6dAWCd%FqvlH)S?-%C@!7n~u?)WQ- z&poag1F~_=M_P-z`Z?Ch*3~i2NUp9v*{qgov<@K3=Jw-+}2nmxu23;!5g zkC*)}4tkY78>C9UPN8UfWs4I`YB3es{*F(5C)lq6I(lET?(1thB(IL%%a-@><*1xr z*`klH9iT{Xy&aSQzmkGrT?(BjdSN~{xPED>XRt9CRB~^yQ=`GAeY1vnMn^|qlz3(! zeN_3TJyzv^4GEC%R#Xe$oB(--=;JrnK->CVz6XDfmBeXafV_W~f}NzM*ZGQ`Nws&D z8z8?|vYP&~pOoWFO=ek{4f|@?W4?>cfKFzI$xJCpu zo!^$8$pP|I&oiUX+K2HKMOq_Q)G4Nd>l}e9G!=_EbcWF7B2WTCXWSA6=OpeSX{G*BFGA z!b&S^(L_gQs9SH7aKs_eWaRgPs&|bWMXN@g1>8pPY2Z2F-nYzqtY~9L*84O>_3poU zs(1hVsyB}EV=3%sx2!`@Kad@2jQ`3G&kbtvB?hvi2y)?b0ZAyXdP5jOZcFqX$#?ay zcdG=In`P#sbJ586ws_$B9AEJ+-_pSKl{mKHAKv+(aevSLe=TG!OM;p%)`qqW4H)xv zHtlLtJno=z+{LVi-G?3_)MCdsFNku{C1Q!fCgSeP;G;7Pd?; z5rY;^yCcPp7Q=fP`<@^MEu0L`NA|r=?0PY1;k1JmPP;i`I3r}xy=e#Cn+)~h$(o0) zvY@S4JFy;Oy~Iux>o0b$*l@A2V&lb7FP__dMC?7W55(4q!Art@F0pHZeYXl*cAUj_ z5<~nF?S3eR5O%VDVrPgUtexAqR;*TRrr6_RuZtmYoqazL^O6mIO>SrZa=R_VR$1@^ zv0cUfN9+KxzGA0{A>o7T!*9NDpT%m$9ua#?>_xGqVt*7{DF$Vm+dwcow^3%wG1+!u zt1Q@0>_D-8VoCY#h`3+ebCl}U|_qO}`{|+sACwuByV{!xm!jYcRpz6uMZ1yKq8;8_xCi(-$8Bu1 z7PryPo@CUUkF!>`-i%Q)a`ooXW(B>ui%qv&Z!Q?eZTI}JxXTX`^yULCZo6^G2Qj^Q zrS-KPHof`a&C;89O3<74+C07aYTu2&=IG7S6}LUzm%m|p^FM9@$PqRo!IUgRrjE46sK&(ATn7%WnY8GGF{SuK9lz6YkMhidT^ z4ONR7d(A2~qiq#)eLr@WT0Gt7f1BDZ0E^V(Yn`u+l||20i}#Hw$&9^*T6~Z9>nOFD z>wm-4;-3~(i)VPhW(jJs{XnyTXBD**y{$U#ZP(1NG+y>8)mTi2W6*Xb$NV z>Mma9W2#l2aw*f?s!(k8>Ao~li+lNel3KivTP3N*wQf~NEk4kfMhEB+pHEPW+lNNQ z#vR3Bx!LWaQXVUDq=MpV@fU8>5VaUeLR>Ar$mgRg6B-6aP>!_zoa))wMTF}WRw%pr z(&*?MEb+teOMHF)4bFJ5WWlWe-$rqGLV-&kf-+}@xUTl~`_fJ9STW?lvhPpCej$dKX!e~UcCXlr zV#qw9-5`T zPhw3>h2?g)4O?YF8?hb54i~Es8zMGTY_!;AViUx!6MIl>rr7ghFNwV=_Kw)sVkuK* zdFSTU}h*2-p0PVJ=3wkV}7b$XPMS9T7!~^5Kd>KBA)Ul;SbSs2D!k)@+r)e+ui0 z`F}cfiG>NXT)4#{um56>mkakm<+sA6vk{To(_)b~4SQ|=+O>aSU&bT*8rx>p*lJ3A zh1*Wetj|K0ZJU|Zh3i={{{ycQZZ`n?$eUJ7!{M@z?(=pUI$62$E3ewISXlDKYP$WxGUTUjHeeOD&=ENtj^hG?1W>Sk+M@*aB zIxy`RPW$*&CA`NYydWOOj9=IBc;vP@%O~e7ukiV+#t#gqHQ{tRr?s@=;WkachYW z@rgT9WqJDxSKB|AFRw0d-(^vc^!r(L!S}P3vM%^LwhhX6S^y7O1?>{ztl2gkqB=!3Co(vR`?W`0Vx{^tbG;H=(($SO`yQdnr}~=CEhrry=gUXsNXN$&t^;ldB^c_R z`g^ByeU|`o2lxEkq~mQ%MwiSAe%NSuLFxD;U)NAc!0mk=|4=%{<11e+?`pTZRQ+|~N7OFhU*Mx)26I5gKzjf9urVSSeRld{>xAEhR(gmA*zJrUr(eR)jrS=WLBJr8x zaF5sq3e@sm=f`(wqkV&WgRM&X`HE3}mzP4nl1@Qc$#*RbyvQ)17x+Fg&HG|o-3FiU z=&Opx=Uin1Iank_U*vWtmNu|__)TukLowPm^EaPA*f$f6Hu{d~-C*3zK(o2k?e>mo z+SHE`9eu1XyEL{e(!1aAWj&;K<53X=%T4c|UR3XX)$R5uqIZw5y}#0r`wlU^HD9ff zN8M_;pML1wJS%--PK!L{!$(@DHQtdbA|87G@cE4Xomo+3Z1ee|qba&h@y$c;=H|x~ z*CL zQdD!Ir=h4m>_)}(a2(dx-Kx+CpR5!U?Z|ky=6Jc-kxbBcq`BvdrXJ|Mj*p$H+;~!4 zPcOMuA$=VFn|Q5}vwc1~px7I@!lFZx6J3o%LU}(qy82CDS)`Zqs>!dH?+`{;Bg(_- zaRoe&yTv;dsx^W{ z_RNKY17PBt=CKPVzS&<^J(>a8C92VDx?3e^x6lBCpp8Ba%Hd+u#kAXDZqv^CrT&IM$j!n%12UmEG= z-;K@VTzoI=9etN_za=;q5R?)l;16`8=v?d>GI*rzYVQ_4X2f#qm-)+-7HMOEb;u~2 ztL*jE;Nh3Y+5g0T~^SFv#e5 zbU{Xk=nA>Kk4;;c4BpqKkkQ`>d8V#7N=Q@~PuXX;k`*Z!wrD_#QS$f zK8Nds@;O?R&&kHw-jhK-Cxd)W2Kk(9h8W~?GRWt&gM3c*ju_-~GRWt&+r~aCS+lT3 z`J4>$Iqi;=9Ug9Ekk4s%g4mD6&J=@uPCLlw?0cKoonmvuAfMCj4Kc{)WC*wB`rzOo zYb%CuYchme(+=U*WC*t=L%1~=!mY^=ZcPR!DH+18$q;T$hHz`LIbsXM)`}tUns%GS zO6+`-H49r!gBD`Di|r-WSL_tAv&7C3yGZO3u}NZ+#qJY(NNk?iA~E90D*X(<`*jTagV)&lWZkpH~Vt0$-zCgRhVt){OTkM}= zfhknn=Uu{9(_nY8qr|$1^%gr(Y=GDxu`yy}#UQhD{~)t-yC^+Ec8Ay;u?1qQ#r`bT z*lyhHTNbv;g4SX?h#^3mc6*EU6zd~4QVfDS_iwTo3XYIX7em1jvRPtJh`lZb!JT%S z#7a!sCTkY9?7A1*U+iG9L1Jf%T_|?3*f_Ck#O@P&NNj=F3u14Gy(RXc*vDchGlIQu z8lcPwKga%J2a8pTRf|D-XJ1I~?0cxGRIMjBj zkV7C-{~GnhzisXS4(j;ratGx3O#c>lz+`(9gFB$craA6_>Aom(2h8&MQlf-9_*?JO9R$60W%Ssh`A>?bze za%Vx&IP?BECBZQX&Vm7@_nEU`yO^`!Gw-+g&Vn8Z&VqiM=PY2D_2D_rg2xnR{+RFP z`qVMzEV#&=1&1U$3yzWK-`@M;A36*0@G9&qI92^R=Jdl;Ffq=2tapl}{&#(TFW0vr z_49*#U{XI=B=t{lyIqo{e*3X)fAyt)p2!AC{Ud!_tdYAKynv{oKcuCiR0w zQvVfh*P)?O|IKdR#MQ9Twwc#`{y^Uhq<(JZZIk-JBB}p2x7#Kr=jTfOr^c2=QvcK5 zubZTPt{c7$FBW3!Ia1&8rd|c|^ z!r{?_x1U>@R&#HuXUp&ssBm0DkSyimK7iD){)eoiP(?i$$V0Oh4)I5`X6?y zLQ?;+zBD={CcW8TbU+~vB})B=yHT>#Z#tg+MN&Vnn&MJ_nfKWossC2Hea7WvC_izj zzoXA5Nc~s1Q6Z_HT6!ceU*Oi!QM)kg9i6{?@^Xz^B}o0XZWPJO*M^%aCNCfBHth;Z z{WUls{`ZrHNd0&d%a|9G`hRb)G|5swt93_rpN{sTlI>i}BlS0OqeABb`FWWu7NR|) z?xS;oti5En3#!T`O8qD}7k9g`987Vy3)Hv3l=B^fmd){Qn&&Ui$e^>#GF$3@+DNw4 zUuUGw-Z|nUgDx^ARiAMlEn46`L@<>4@s1If`fF@9SL(-kFDCUfWOx}w zy|xTuKc1Kn`|%`&*pEIp*c3Nli2b;sK{(0 zA1#XgWaDhF$sqQVLF^}k*iSY?3}QbS#D3aA>?ecRPqs!3Vm}$ge%dt)TNL}rAokPl zNZB1N2C<)ZxS!Jw?-^ugib3orgV;~I+r;h^n=;FgP4S%9c{vOkI8jz(4% zwoGsq+d=FoG5BG*zTRRdiVYAOBnH`^+qhN?vOn3)V!svpz1X{A?~82`D>31m`_e3I zncyt8lNek9wCgMeR{+_uVsHhJohAlX09lRL)nfOEJs>tuY?0V&V#~!4U5_<44d4pk zHX!?xZ5Otz*0tC{VyMPKyA#AvjfLz?vGc{?3g9+=A$FtK@5G)HTP(Ij>{GG7iS1$% zI=8!f*s|OOF^K)#F6y$-4s}_`P?v>lsMytF6UC;A-6VFG7+e8dA7p>74;5O-J{9|$ z*w?zhw>{u}j56W#^CWg8!WT?x+^*t*#TMTtsXotEiv_oAMGSp=u zYi<%cS*x&B7PJ*>C)QC6t^oFh?9aYG61!aNDzUWK&%|bmJuWuYG$gL?rm&Sy#J|t( zgAEYz12gNHekP3hUsIVrv`2b*Rr;=WjQRg7ijA9I-@AE_bT`EO2iYM` ziuvE@)yENuTbW*0KJ#<0J6@GOtx0wI)%3CP!sC^xxhP6qyS+WCEaHDug-QKm271_k z_dyGKrdG#e{=2nLFYJ)Ac>j652EE`>_!AlLKc^B!$LGX~j^8lQWAA5M;QkF27OK9~ z3OTP}7mxCHwCS|>BaAZQi!F}b!q?Y+>UGI)FywW~mtl6>8#1=7CSu+3HN?99VgA~) z-B7ZN)tYz1%DHlGhraY-iKzjK5;rQYe3Tw7kx^9V4qVK8JP2 zJ(U0YSdH=i2;-k?nF%k>U%$5BK>H9@?)ojeXWGthT2WhrY0MX~3}b$_D)YUsN_VMO zX52o*aQh6^X@v1tcX+ER^=35+jSsQY?1B6*goVbJ+KQpv*V(kX!_rs){`i%?C{ph6 zZugOf^_QZg{dy3sZl}a4ds(+{tY4f&)KS}kYh|w>f(6QBSTg!*#>J- zk4L-{759$TBCD_#756?`I?!6w<40)e*Vf{i@#BtbKFnHF;ZL(xwhDhi9uDR{2_`e#O|6+0#@k@QPkzQ|mZGTN%q>e)K?@4uZ z7U}g2&2LgzuSYfpym~j;^>5$5lKq2Y>=>f@ZVPiReQoRWsLtCwT)r70 z`=hPPKd>eHA&*T*vGlj3_HY*I_z)e;uAw^qO>WLZbg^ybOP_D%n~5?5?nreAz#<+0 zR<~PIuS5-02!`w*QcuSpYU}@l_uI*J{GoO<@K6YXr!1xP?1DNzLZe@EyA36gx_4Qg zK;DbYGO&2!$SR+Xf3I7e=4T`d+3y$g|KthT$LU_|9jT;^wDxkJ-`iI-Qm3|cZ1eea z{42cI1n(7v?4MUS3>s0r4|=B}IzEifb)$V^LHF?zHMjfHR^f%1sXkt4IM6O`y@RZe z^7$eqYOn*-+$eqkatoT{u>IPt3hDIYd}*ZfUgPs|?LVI`Pc{GO*rCs-{?7Du71HJV z`_k|0Bl9Nzo!E)V3Ax8dhPVTPR0c7?MH2A zOUN2*0m#v%9mN83$Vl9LCi@L3xha}s@=A0aIb+1e}+w=!sAH>6(09WsPMRbylYcDf}q0V z(et@Yq48!^coPGp!p{o%^EQPF53RMrrtm?aat%V2p~CNOpW|mXJ<_JwZ@fQeRQR2& zO%)z3s__f4S#r`4og&6ADurJ)JT;I`RUBw29oh3Fx>>{ygVmFIDD>hpU z^=!D^Wn!p*PPSfbH&d_4_6%EP!GU6jh;}IiB#hw$JE4D=JRk6Q|Z4hf| zdN+?j>#$|sJ2BKm=dtQ8)>CY_*iXd9h>aDyL+oy`t4$l?b|;3deCj%Unhj9b@$v_C z9S_Z(>6p5{)Pbw(5Y|0JehK9D&Kol!^4+2kd8zD$L*%cu+fznA5B_(m=XSgJ&sWb` z4Ck9r&nMf_iQ?X~)br`SC{oX7(X0+03#WMZjK{UtIgk2v%vruNXL+BT<&e{` zOKrnxMK~SEY3&G`!e5c2p4ZrFzXA2U{-K9@UW0wer=DZMYTJ__-KEsz^HjEaUS~_= z1#fV}XmEu4?bexkHk2mDmqU4=vVE4Vic1ks!KD{ji%W5w(bk8U6|{8(j%~TN-hc@A zZCtd*i#q>uwe{^wEqr*hwDr;iZN2s8Y3qz+|1zgy`Yc7VaqfG4Wti`DdVSmuSgkZuLNa0?VS>oad^`Zdl|*3$CdG`d}*YNLrKN>fRS}P zqJ1}C8Y$yD`+R~jKFW>a2Oys^exq9zQpN{`8_GzXXZw7TGG6UgNy_-o+^Uc=euOWL zl<~cNK0z5jz>SiX@n5-Jq>S^d7gxp`d!Nlw#z*+gCb~k$_{^JbKwjQ93Vh16I| z&ygD2HMEY)!r8@V@P>4vTP3Kmr@K+4j1TnrNEzqe7gxr2^*)7_@fuw8{@1Y~$~a!2 zGUjKyk7%{)t;f=VpV?Cn)2{Du&h_eM#eA>1TaubS`Fx*2T}o%Wjo$E-*M` zCior7_}e~TTp6F?HdMy(E}7@=KRY7bU!}uOh4)2#i3@4t_u4XO<+E&R`fLrJeBEq{ zKG4R$wkfo6+*qNF<8A|Oywax7#_=qJHjcX`v~k=(-nA(nLLb=_51-F%`iV`Ujr*6w zH11Y*;1Qud*{0z4+Z5XPOq)U*f7+(d#-Fg01#NsQ`y9~5e_&H+<9PqgXyZ`$sEwmV zZJcbJ?L8T^aWZJ*WYEURW{Axen=b}!oOaO0*%#V48MJXSXyati#>w^&!;vF{HcmTe z<79U1LxxvwGJI)ihgU1I31ZiYJs<{coOaO0*%#V4*{5QE6Z={$Wk;NLx)IvJwY$q=tu*pCqvXa8KTz75VcN* zsC6<#t&^=3L)1E1W4nTBhp2TjM6HwI*CH9B*2xgHPKKy;GDNMDA!?lrQR`$-$;lA4 zPKJNUZWcq-IvJwYX@{tFGDNMDZ4yJ&IvJwYX@{tFvJPVVi6Ls8c2Le~H&E;>u?b?= ziQOf3pV$*(Pm3X*lG|7z7MS8pRvNZ!G*+=)#r{X^0I?IrP8Pdd>?*Oe*w4gJ>z&(u zT(2#8vw#@e?wzn86ywk3`SYNSI#4t7}?XDENU2KNfOtHts zQ2(8M=Zd{A_L11urb5%MY1k?YI*A=7cCy%MVi+5geaDKS;0)OmvB$)o5?d$}}6UuY1_ar+Gt9%F?{8l>Ed?EOOVT zlh5Jwrc*eb-rc5!tIT(|vN3S!^-Qm-NKLSPdCOoA*d>KyCSS*F!%aeRx&+ zko2O;bVZY!Ki^pX+;Po&4yx;JC1rYETis^fonzXR*079>)uC_9axqc$`E>8B;`1BN zUlhiwe_A(h)lY&&m8qBKy+5kW^QcQ-TVv0|s?O`mYmc`BY^7!@({EIl&-|+D+A%@- z&R7`yANI}z&Wa*k`+b;$;D8Qb02OppP!t1(#eiW(2OVTkbQM<-K}Q5c!m6wh9YMUf z2V4wm4k)hf>Y8zls{_n{aRtQ$2E+s^E^7qi_dM_0{dAu*XXXs9-}l{ne>*?1s{UQ6 ztE$7Pr{5~9LSLSlSD87qb&olf)e9;zOHh&0%!bnsD7kmT3vG>ZEPX%c+DXf{D4H{A z>C(i)_eFDizm7sCb577ccIVGjd?Edi0er(=T7O-1(n;*PCS2z2NV`}-GcNNyP z|6$Nl`(|A+bEDu>cYGtia@3oiLunMd{sVjrZ+VyU2o|_cW{nl@b8_y?mF~UlC`V-R zV*OXc>+5DuSp0I`?9p#8u3Ofn*K_;~zHL*B+kb||?JF`(u56E%N1NXrw)yx{?HRx} zw*dfmqp;1b?MtTJiC}Uq{4z{eZ*dXh1nB*6Rs7D=;qP9zY~QlB)2&Ci$$))EKi58^ zpPTrMeyV*&|JApz)UW(;3C@0Q{r9%1z(O`BPJUkf55u3Yn>~KXvouHJ6{y zr<+^_e;h8h^33byJ(d)`!7u0e8DIUXqo3(t{apR(qahmk5;kxr*P`lJ^`?G}&)O3$ z1xkc%wx3DbF&DT!LwA6=Zk)h$frs&Q@_76$ihO(;9mh*^j!(}yUS?rm1fTiUgy9+P z+9LRg1@NH*p#QFHS&QP>`22Rh^Au}(|B8OjgfiSJa7Bde-?w@$zM{8J=zmVLWAfjw z*vU)I@7%Zg#eUP$?X$j|$9P)KQ?DqRkS_me9&R_8jz!OMN7brKADGHaup7$My6=Aa z2~jIE(*oQ-lac`Mr#X@U7Q*lM2>$o$KjVLcuEGDRvCMz=y7-5oOkWqDA3FLZJ*TjI zYMxsKIr|AdXUlMV_38i5;qys|noKNs*x|6}3Hp7j4K zIz4v{{A&?wFzR|3+@CY)Z!6$8y#~I2o?Y6K^I7}_;(z0{2L5;hU0e1@ZJL^e`R-^9 ze2%anxV~sZtb^awhxQ}v!$JO99AP=_fRgrWU)!BC>?fW&wtAEu?~mg?HN1({gx87x zo!9gl_%#;G6Y-Muzji5py*U!U%i0;c1iQbMq`ptx5j@=D#JnHck4C(4mUyd{{KWb& zgi*)v&&2>fVsn?ZsY1MA7Hb!!>lr1(DXtS#wC7X0c^}$;Y$>;~COI29e}VA_3xyW! zoO2Mq0RJEZ*@xlrXYBi@4+%am+8A-p))}9Y9v<{by_bs8rUL$q3WD9yf&Kw`4MLw2 z1$US}vvE86N1qT);IO>!EY0Co{<#SI)by`XEbAPVH8wpKb^0U~#f!??r?3QaKg$ea ziWe2PtdxRK<~ojXq)In3J%qGdpZU#M!{&%xpnP%&(F- zqr~pB{VbYGVpJO5w8pKp#${uVu;WMC#T%O(?4Rz9)xud68{99_F1Y*pMOuqAelC}! z*A$f|K5ovnyx=G*wG!JTV&${EIND+j!zq`yr{%>AtJm(S!NHuw1)InEf{Ki<%Hr}N z=Uq986X44@X2v^Gl-X{Y zHZn0i$d1Mn(?0$03u&<-huCpnZHZ?&%)-fO?xYs=8e&9q(QKoa6Jh4I#KOMR5OZ5* z;q4P=nX}pwPoTdQnYH0*&PKy5>Ku!;LE<#ePK?N%?Pz?Oox|81ZLHXC7H(q$ofX^9 z!rLY~E}xah4~Q8A$0lOsT6r+<(=4xKlbw^OS;~&aHCtnO#vRboj>k>L?{~SnN~0Su zu*p8ROiG%x0%5%KEwlY>x>EZeDpCtJ9seX0=e9J4Ld2(p=TztW<}4n@Sb+2 z*>5WIQronc5X@>Z3^&5)e#(adZp>kROwS3~An1mcD`Y`(Sj$Zihs*WgU0oYsge&vs zyN@0mJW$lAR(2!a)q`NKtij!|UsHG=4}z`Lulz_jx(jCa>*v@cD%uq&Pk%16gpKtl ze<~zzEH5R(?3q^+VfN?ZFt?`r^FxbOpg%uLWX=Bg+`?S3%lft9L0fjkd6y`i0g*j(4#VAZV z1nxil&9T9j7vv3+fVEOu|vg<5IbJ%L@~;7UdE+jH;LUU zHb-op*sEfT#rUZ6{3w}v{rKQ^#!=yH2eF=F`-q{Z!*~?yJT1jKPdh>kBOqkx<1k*0 z*bQPc#QrLFpV$*(wPK6J_%WQ9OG6f~3k_ME(U8R%4OyIZ5!+I1N3mVR`iS)vJ4S4< z7{xj-gJPYRL9xym#X4uVirp?o0ng)6!1H+TihUr)<@h`v7vl4HDO@|wC}TToFSe1` zc49kttc9hsLV&{uZ6r&N0r=<~$m+_3)3u14Gy(P9nY^B&5u^+_N!*j;V zXdn8DgDu6niS-jZOzgK}%GhuBMEtHr(-YYmyr>)bZ<`SMiG zxIC4&)j-8NR_qM1v&1eEnLsXRX}Pv!aDCiZ8sIb!p~UJ~O@6`ppP*oR`Di+wHD zv2|4LW}&Y**jKDvjB=sZZ;03^v9V(3id`U<5#tupUcdXr9uj*^Y>C)cVr#@&t{c^5 z-OyJYY$UdcSU0il#10WVRE+yr__=W(3$Ncqv5Up-6uU?4O|hk7pNg##`(6wW=&%fa zc<$xyBGymrFtI^mL&QdjacLN@^SNRdh}|OgC$V{A&x*Y&wpi>#v5&@BvR7-#DJ+(wICDt5WpEU`z#>cpNFTO{_n*bid(Kt61P%|c(BU<czoh6w5N38yZ#pc4J?%aVOW)2BuVXQ~ zHYZ(A4)26my)@_efSlujg(Fe>g2`oJc&NK(s+LEru0gi{uf%E&mV9D0`C5zBwMdi{ zt9iftx5Vl}$k6S`XocVs^(gpUqUJY|e)Ki?{OCLI`O!JJrMg)C5_~REe=Ejs4qd?B z6h1#nQQHOVy>+xDUYlLO-W$Gb0s9I1njm21nf0A)@;oZusu;MV6T!j8>f_@eXWIn-9kcicT3Cvxe(3AYU2WS zd!_4{la5bjPQ>~oU$#WN1S2d6BI!2>hFi1760z66p+tPMrE7*n>?T>UM9hzXgCLTA zG32bkSR(cU8%o4pV75g31hWeMURWaLSH?++c&rr~OT=DiLy6c6tw^@s$T8tG_u0tf zWy?1b=`llZ`R;21EX91!x1pHt`R-mwdj1P*jlv`8nG25K%NI${oY_iB&t6Di0R zmYyGA(qR7DNzYqaRj7u}wyH#uGW0I1%3-FCatht6LTO~9bR(f6GO--(6>o+dy}l)ET8{q0&a9vu-NUkqgYVN8^fpZw1Gj|7>8#V>xO12H1Z0`(brr4Ya>U~ zdId6?-GiwY4zqA9M@P4SX3Nq2EQ7}6%RTI9+^W43VGhLo6Jb{8z%aL_<;%k?Rss1k zzY-qH(b1V65;RYa-q;FlT#l|`!`k1@&5@(o#nFAVNjZ7~-ncBu0qUgIeIlnwj3Q`gyhQ6t-+)`N1GDyC0sC+qxpiJl%s19%$1`# zW}1+r{bSCgAkE)iY?nfUG~-Z^=DP<4X%<95`Wd*7!sX47f;1oJ6r_2Rry$My76oZO zb16vkSxiBiFAd+pWs6XdPQ#@jJpnE`&5`j$tGNbl8Qd9gDM)j;K#r`L1DAp{2gW$K z?EBN<^6Li*(j46oD{J0`I~DEK1!<4Bi5LZGXB4D8-T`71 zq@5ihMnT%yiDDF_ol%gcuQ;F}?d(=DKKGqbkoI^Kq@7WacJ`xK3-qkB^+R8qpo176 z*PfO}?jEm~7)C(IhKQlpLpD|nJsq+O#3qTMf9+@+_G}pM4l&x1dby8_Jtg+O*mAKi z#dt+~T8>vg4_?2{I*XNx?ITtuRwc%-PCV^#VnfBa+M<_nme}=TH;dgRcCXmuVo!-Z zC-$P)dtzLF$m{Zr*mq*vV1#&GIOF5-dWmuF$Js$*G;MG;Q0!Q-Q^dxIO%%IWjQ3(M z_c}2?Qk?xwY>60cP`%u5#J&^T0N1;x?HKxsgRRB372}609&doyII%Ott`@sq><+QJ z#2yxVTx`DBb7D)x-Vyt!*cW1(K<4%ObqRgN!Tw?gicyC4c(hBUuQ)hQtVV3A*mSWQ z#Ab-i6MI(dRk6ilKZzAVwDoekhQ2mIcd-M+4iY;;jMB8HJyGmrvD3tUCpKM-(zK_Y zAx3H1*?nTO#JIL2kG2Ww#JIMjvlU{rRdrSp`icXtK6i?xQ}>-n_}eZ|2ZV!g$V6B{ZvQS4$d z%Fv$QRbsb_-7fZ=*o$H-#8!%}5u-Hi=esUsUN2+4(1-mL#kz=5hW2;|ik&1jLhKB& zv&1eEnQ>X4 z-i;Vdw~LLYOS#TyL!;?_`HZH^>eqxu(?u_4+f4UfQ!;HO>d`0jugq%|*u9HOk7nk} zOd5N5LH>%(q`9A0V`kF6y;qi@bT7QV$x!9TS?R(`I(r{5m^?SS z57^bV4;U?^VVS$wjGL#F)xU2By6>8G^wqu}SQsNbG}{lsDtgo+SnaM{?kQul=rA*E z1(wLG?bR}y#_=etD390cKN$W3?SSXP68i0zXav2OM$ny`-V1C!7y>u75g24Z*+63h zwNeBol(9{luz}9t2S9&q9Y%XwY#ip+&e0BQjqDlr2N^s28N#@E(9_s8ZBhNomG9sL z>uBZuEY9-ng*0?tOhf0+P3#kP-bCyXme?R{5J!iZIX|1<%y~h=ta zXDhI@DQueC;bNcK8JmOsUovN=vf|3``S7{&TaVkGvlrlV#^2zs2G4}g^X`m5>hyFfiWil&Z^8<) ze8@S&P8&O9+}M$0&PZ50-N*bAX3LLSfjd}%Y=Bd&z%8smnk{>Q>!nW$x~3jUMe(8n zcht!ZMXnDq>tI4*O?sgfx}gPKTpfedc}9aCHbh?jd7kBmrp~p z<2o_F3gTLH=%zwu%llZg_IA?a;pdhmTmsiklF>AFd?%SLdmTS2>K#m%*|J}x zn`fCVN9S^9(Uu8ox?Hp6amd}p(ajRE@|i7fWA&;==egPPbgNf~)FHtGi3>K5*)s1w zM_b~Sc9N7PjFyOC4qCt~p$gb0#Dng@rpNIyPtXIaZ%@=A2n-YV$e9>u*)r z$3_mnO^8O$&cO>{(brY631wtNBg;}1&;fU{NW1IIwhxP9TT$tZ&TLyKSL?MnGB~%zfF{k1KeImRiu?+h882^Y zIo_n1u~%W2bl>2F#?6cm#TzgdpD56eH(9K>A7@&)tBs)Se!QS@Gh_DOMV7F!{yW2t z#{G9zBFz4yg&B>`lY_2d|4p%21^Vwwi?o~ikLGFo{lNQhcK_Yhq*?J`OSjqoQnTVc z(df}1e>3rYw=4eb5b_RCH^(8|5$?rssdjSMQN`t(0aaYyT?WFv87|(BYIwKDFO3>L z1*X8|-TqOyyn8K#%bW0d?s6uL z{6x4^cSpd*&9~;aaJil@2O09J;ouqymxJn5xEz{PdpUfl_Hsl}?d3>011_%ts=XZh zD7)q^xKw-ZgG;rSMv6TD58y^>Z*TZr?WNDvUT2fgTFz*G?2Kx!GpfDLW{S-gs~1}+ z_KMiQ#J&<^pLl+J7xOZ<5!+s@m)PE7=*uuIf5v)RUW}gK@5C+^`@PtmV)uwWDMsDb z^W&a*p5JP*H2Tw7>(Ga%jaVnKu43KA_7vmGj+Ze&>`1Yb#72l+BsN9tO0jFiZWH^n z*b`#4VjP!#9xKGY5?dqI664e3tsDAax+k`Y7|oA89?g&a+{(phi0q8tSb4lLV&{mR zFUF-oJl>^ZG(UDm^JCBN8L=0{-Vl3BY=s!-l0EGju^+@}e(Yt?{Mh4l6Qd!rvm?d0 z^PaPl#72nmy7v5dWqUkc)6RHRJG()Q=Eu%>4ST#gO=jy>K+p|3cg z(Xq2##P$^{7du((6fv$Y>}f9*yI<@fvDsqvVhhDy5qnqc1F^5fXo$?xii35brh2(F zM0U2D*dAiXhz%ASDK=W{DzWRtZWp^#>~CU^i7gd-Uu>1wmtvbhFZDWi34QqCD@NVd z&*M0;p<>*j&(n?*J5Q`ej63vsyqm=C6uU?4F|jAb-Vx)<#9kNf(C7KB8~X5dfY`QT zyNT@~cA(h7Vk5;yi_w_a%eX@9ZZR4ddtLr6He2ikv4vuvi+wG&DfCh=WAo5g9BePP zlh_er1H~>9n<93l*fnCeiTzpZ6|pzOz7YFH3|sDm^=lRSih~WrI*L)n_4A;L>+N#9 z*ok6e#7+}CUu>e-Jz@`ty(C6sVlN{F9n{Nh8Tzo6w%GP!Rbm6gsN#BBs<@sX=gyst z6Z@mswPFv7JuEg?Y`)mbVy}r&#q~0%;(EE8Lqm18Rp=`Yb`slFY(KFB#7-7FMXVC@ z^`2i<=*y?X@|pV!D6!N|Vv}D}V)Ln$^g_|2AMYx5ZVmQib^ zk|C|NWQ)Xl#J`B}T7z>lqu)K~!3xy|8VIi&;VqE{s9i$08{NLu6kfGH@_&@qY2~eE zuP!Q|mY!SDdqJqYhV7Z7ykgK=duFw3$+QgU{}|+xH zQs(9Y)9ER7zpV1g0gzXD4KKarC_Kokyly7twU&3F-JZ9oRnuizF6(+Lby+`3UDnlD-p%~z zBY2;5HI_ScWUH|)@~E*};T@D}?CXgA^VQg9*yjGlWKRB!^}q2p-E$Nh%)ssG7MQs! zyomJCNHM}v+-uTb<7(_}R^kAa*n|x(^Q*B;Z?#0R8k=i_yKDMpsIiNo#-3GBjlEl{ zYIn<<|7(s7?qf>VGbbHQ1RAr!b@?&UF>Rg)soR+zM!l2sM_4@*dS|E9GYP#D^D6jj zhTi!Q<0bUYzgWUZ$HWTN*3zX`AoWf!aDC{V-BOPx^iD4@Tkm|#@*YuG?{rl;(lLX8 zlg2^N+6txK>4kQI-q{X|6-V(>oIS!{pJeN0+2G!9$0NlVrp1J1xft9-~s=W% znDee(Eu554lG^HnmUo=`-b9$GAGUA-ZIz}p9C!(xC1=TJig|lktj4s}AW?1JAl*_m z)>hLE%E+g!K5Nx2q^-If-I%ueiscbE2}B0`d6-S|hJ_1gtMe>UVQqC^JFQq-jYdu( zZS^Y4e{HnY?syI-wN=gwbD?4?kksU&OSmfdV`@`qt2KqRRaZ}w11qVm{+QFBEv&e> z71yzFA#KhT*#g>}i@dS6x@lNcq^(A0wr%iOYJF&{=M~Ua-?plEwdXF48M(Q{>7!1^ zeKr1)iZKMdzH41!;aFRZ&bA6Y*rYbj3Gl|3 zAG=h`ZN7UlW-A8jL>R9E>Mi0#r9|C&-)83&47mW}K{gt=7+)m7f_sjl)l zKy{V(d#bCvf8oYg!`m>`Ro=X*uJV3Gb(POfs;hivegl^ed#b4RZ6DOlR}iMUT83+p z{0O*ISAPSy4DJbVsjhOEQC;Qm8VZ-g>QuNKoK#mifT*r=Oi*3rSfRSgs{m!zaP&`w zdp%sLt25wIU3~&B)zw$vM(XN=@VmN7pR23RCZUy_QC)RLb=4WwRcABBW{cH}QC;Z+%uy6Wshv5&>R7W-Cg9rUrMEe?Iyky4BwTzP&RD;}?(7}a@a+|t714Hp|FcBa_5 zVzj08wCLF|?M$&d#GV&hAoix%Qn63PR*BJ;*2|zRt=EqWlsn_I)YdL9vI$IN##&XiH09aqy1Vdt&S3h{x*?`p|D; zTZ!!?wyPMQQJ(eyu_MH2RqN%_me%8q5j$UOq8Ke`J>E@XbH(P1(Sp|FagNB#_)hF6 zu{O9bdc2a*R~+mvwx`%{#EuiYKx~rOAH=Q@yIE|e*u7#8ip>?HRjt?gH8EP%I!i&H z^g6c;eZ|4HVm-vDsd~KLVke1>5IaNcEU}BkrigKIXU~s|J9}NYxU(}Z?(FO%u}{Rf ze6z=E7y6*)iR~n|w-|L+&+ibiL&ZjjQD^mdT;|!!pw8-y%RD=qDt5Qn{bDbRy(acA zv9H8Zt)u*?vw9ir#WoV7rt0PP7NcdXvlGO)XtOg~%X)rXwAmSTR%h3X-7HosRwuSx z>{GGTV&99ELdWznx`w{ufJ&+}Dyd%XiDD;ex-b}GO#Ab`ti!BtR&gyC3 z75hMpnyQ!4CiE2tJBalZ+efTStV(Qv*m+_#Vl%{U6MIqYWw9^CR*P{7X0PAop%3F( zY$q`;(CqQJK(n83wb*d63&kdjT_#4I)zjW0_9roFs$K@IWj)@fVyncy7sFS=VZ64X zuT9WSY3h|I@hWoAvEOlg1Y=bO2qYpH?7 zuvIOEq|#}fa&G4fV^5fK#(VabXDYMoPjPrCZBG&9Y1_2dtxwB~o`p*e!beOx4fvfqIYe{ zG`>|d*jW>;1}3j@xvRg;6vhS;^VPbZjvF2i{(#=6c+ zg1YyHby||Dn=`De?c?=teY+5Lq(63klzANa7t7kQ*z+D*H+$I1Rm{wj4vmuz z^%&gzm!(%dJ~S?#X0Tz3vN%N?wX8A&TU`{0ZEfFT5w-O%42S9G;*HQJJ$epr!QTDdX)7q z%gmcJuXTA3bQpG%om+D6oXTskjVxkeyp2s~VAHyg(c3a%DSH)m)M@O}dQc~(pIyK5 zTb$^RFPjPREhin*wkB)`ZtsiHjQxF^Y=^oFvDQX z=2aBU?lZZz}*W&a`j(+Ck5^v!09{(a;3W~uS(~hJ8Qbvg3 z((LfG1xN*M_Of)_*>Sg*Wtd78(&H9l@oZ&e<}=jeP3GJO!!yEkIFn}bb%=c3ChtLKHi)qiqr z-7>`R9z)!=Ow~k>A-^5c+>)Bs*iWxruSJ^$qf^e`sBiTPmDB!Mi}UDP^lW9(i_6(G z6-A?4m;Y2(o;f&OUet<%m-nWM-b+iS4#yV3dG-h9oj0GXo4X;NT%Y|d ze6G)q#C_M<+3-2zD$1_eE{D%m*;tFEjjZdl{HDg^je^hPoeQ69wp-xh^_+Kx&vPCK z-#-u|9b5!owvM|6o>jSPEYCu$KBzy}`}%o0?yfuR()cF9rO0|1{z0kASz~#C>9}0D zb$}fo+VY>ldO137G}+e$y~L>PbU<={28q#H(I#w?%4YrxWYFOJTv| zn+7knuL}pmBEjlSitF^da+8VV~hK{%Jt}W+a8`J^33rFOh7y+c42n^?LPV_D0Nn!UuvucODGF>DAYgke_8 zcIPc^Jg&GPnA@UdP?TCPb$_ah5*(Pc!yPtmkxr2n-YJ1%j|mBo(WAK9_bOFCEw&-RO_HVbjN*Ro8VfsWjvLfYi!9Gt^roqMu}Lt#+HnQxk>zWXxgxnU3QLT z7cVI~Iy@I%&ot=qVZT=@QK>T_ekib%!Qi3v#;L~>6^! zyY7}}T)RCi9M^6K3wO_HzB~q)T`Xc@k#MlZJ186t(MeRGL8GzK(D>4yF&_nc2I~dq z1l!_fnY9M8_8avv#w%i?U8Xx&DI5jkE$qhYydwN+?~)i1d5ndRv_#3}ij$(?R(8}| zjxBjm80MXV192M*?;4ci9U|`6+#X^~>Im2@xoKgEB15EEu{d8gxUbsL&xl{9G;Bv+ zNla5@dGJ0_$k6iarWVkRZr`=Y(*Mr3M}4#kggs7JCWjxVLWAH>Sxt~_EX)4lWF`d- zrec5fu<&|K_fN4!YOH@2S;2AttPaCb|L}`DY;hYD?w|FooC5vBfwg$;w2$vsn^Gz+eFDpM@<6A)1zN~!P5eyf? z<(o{f10$Y>>bGxSiy*b*LsPCDcOgS3HMI~fH)eeV|Dr#>A5*{E5dZ!f!Vb{1_|p0- zxX-|4cqv@yLN$E69}JiGxU1k&FMI&*aJbay{)pz{@mX-G$#M9%hgSA6TxxaPPLli{ zxYX)q!7YRPceq&bwB|W9JvFvIs24T1W8tzMe1DHLwha)+H8%QOV{0 zoKa(QHdCx#jA!ccsH}NBu3YPEh1g25HDW)AZGv|2v|O*&({3lWqu8NhM~EFScB0rA zvD3t^6XR3g%b@P&LtinzJB?UpF|Hu%@eUOmCq~uU z%eYpI8*@ARi`d;_kBa?Wj9QnML9NTvrg4oqYaRNq#-teS2R+`wV*SL978@isTx^us znPTUP{ZZ^%u|J9ZMT~ZaUcX1h-WFRXmc~suJdePXj(IXqyJzUjr^}p$d+l22 zGJNr%F0&lTvUC}~0aRwPbeV(NnGO?bGZ0_0HlK{emp(&tm6v;Pvq~s0@x1$gbQdT; z{|$5(x7hoS?$WRwei7ZJ1}~Zob(a~S4Rx1V(4_9NG&?+Px`i*W9WJS}EC|E=Ilyo` zT+vD0C9rU$-}nxd2_0ry&+PcQx{d8%nMZf2L2>_GbeD#`M%|?bCzjA%7|f@;v??z; zIGs;-;jJ-gPRBdz+UYLkNa0FIU-;Zs?r`{A6X}aLOh0-!e14Q~wtn;h_*`Ro8a~%p zXe;L$OKdB*8E&l}uOECKZxDPQk6%A|8LjZ1=VkPQ&sCX|;LBEJ-okSySCzRE zvHGCjkAeI1RhdSu>q?RJF#O}znw+^f1RdaixvETA%NLteWv;hmYjRYXU0Pn=gevoe zrCD25X1AtQnaeHH7CEX+tyGy>JKj81=2naQd5$X6q2;YmW!_AvGQ8mNP;XL|;nTfw zRc60NRT;h}#F9sono0+oeWIrFsZ~7IRBA2UP*XX}c;~{J3ce1;-&$)b8OwiDtJhhm z*GCqPHI-TmH`G+FHvU7xA}d!@*(a~2GT+k0nu_ONSW~I9bPY9?T08B6nu^b)W@{=g z60D7;($fknq^WQPu4ZW}$69u=rs4|+#xux1XbWj7ZYNYgQ#s%2R6tYliTaHa)A-S3 zG-q}z!Yp_f;sg$ruB4{I>7Asea+rl1(^NQ(lhjn)ek~RSIJwg-P33WmSV&WO)ndh( zN-cA5Qd7CaQpBnSXQcw@C!E{m^zk$cH%C)RSvSX;ig#-vO=Y6x-)qxe5xwlP%o+~+@GGR3g5pYRV9NsuBy=Is*1BosGT#aD$b~?IHRiK zY^GSf*fU~ORXiS56;HcDjH-$=swy6js){qJD$b~?IHRiKjH-$=sw&Qo5IbIss*1IOF5k86UjP-WK~&jH-&qqpITZsH!+SP>eQA&VDUM z8zyJy$1p8zn4HmGiM}?$1!7#cz}cV0mWaJ0_KDa(#l92!No)g*Ar0GQMV{Z$V*FL> zY`EAcG48nJ`JF39eZ|==VspgiiG3vYiC9ZqW1g0KZ_X}E*`I1jGBwH)5WN{IJ-e?hS(gjd15b#y(;#l*lMvfbP49%CTJb{TxoGerNztL zPw@^AJ5=llu`|TZ61!aNDzRI|xO$_PLHi;vmzs;SC1UT0abG@<*DCa34L-4sVta`7 z78@xxT8yfS=SNk=%e_QwniyAQ^mq@5JukLE>`k$yVxNjpbMZ1ZhJNAsZ5sND1KJHa z+h44o*kNLW#D<8`mdMji5}PS@huBMEuZnR;Jx|LW_2|Q{Y0xU1bqRe?6UBBEJ5-Fe zM4t9kv2kMDThH^W5t}MT&BfFHO^o~PIa?z3j@Tz+{}e01T$Pu>RT(`kEqR>LlE+zP zrtYX;4?Md3$nvAg2bLdQdEuucaKFq_OY&Q<)Z(E)m4xpnm6PZER z56UyEGlydLv<5~izek<%g}m?MRob_@t|BuGCNX{5#pz+S(!D%W2a^?K^IDIMVX*>h zm3LsN>MkiBeWwZ$=}LmC9SRQejT8N?G|&H#uP%Gg67u zz)*#e%HrZe^AyH4^OQI)3{wVoN_^Tszy6ghyOhC0(UiDq^ zHL^(=W>d@g?NL^jOq-7!`&TcjDtg)N8V*8#pd+fP=O;{0N<(`S?~U1$=C_vV3AdQ5 zm=-k4_~d)o`P_19j^PPfC(H08=R{rouv4ta9GYu%f)nkVFglqZ8J$eU&@-cx*C;g9 z;2g8Wk2Rz-Kyj{-($QmM!dk$HFuXK7{NF?SxS?eYs^yj@!y-$Q(oAe=GCyHyGH2M- zsDEx9*0iY!`{&kSG&T7&+svduALVEr&%?%qKYdK=cy2kW3LBGQurZmRIVfGUjw>A) zmK>&!yM+mFZ%L(tcf}l~BiFcOXJqTfCA%TAOAPqw5(*34TH4um@VVGDpXD31{gckUR1v|_-hDIc-(R*BLhM&Q59<@7z_U8s9e8lFH zD*9NQAJD{}u4kAkGCbk4x_Jg9H(AP!tp@xlIuzkMjonh%faFAr=msPd6~41@jSMxs z2}h-SU^|QGOW#t&Io0C5lPI7O1Ck1yM-W^N1CsZH&w`tqYd~_b<-e{q`VT+3-wGG4$nho@`JibaCa|05N_}A>%{boSY31ivwKNAKdTcV2^ zHy~ML>87?gCD<_aeCjc)FAYd$*lD*dWI(dUqTQ3)45z_GojB>n$pIJ&AgdiP``9g* z0Aqmolkt28BwJf98zy4qGazZU*g>_I138-7-N(Z5)Gj6ZAlR&c0Z9)_wMo+xjJ(rR zfQe@z^B9n97M@W}IGgTwmZdN7M}v`LE!;UV$DA}6adBeJjNo0g^lsZ<`v881}lmN8{UldiHao+MJ~74v21Xkg-g^an6Sg&rinh! zH4@>;e`~SMv{YxP%tI2Xa!nAJDr2d}CTf;65^<3vu30@>apO*?9+kEG+Rm2w0d@iuGrZCE$B>EAH8c?EX?ZV&1kygJ3APMoA$EKL zot4KxWLG@2VP>Gl?PlS4#QW8p9Hw~;L|R#*#wN?(uoHKJ3Su`&uwly7K};-dJeKfiq7^~^T(v+0z#4LuIN*0<`gMe-Ym)G$l?+i2~CjE(XO z85}rO_9z{m)dUTVLteF5v2n;V7A_!U++~p(>zlLfXxw0&c#GDDWvfrHaG}1r*J2gu z8`nhQqPUtD*DB06o}<1v&f;uqh9P{n&GVN>#JCSJvqcQ{X|{+_3z92hJPp=B#NbZi7+pa>G3o6VCc?U=ad;z6&z?F#^=HrpJ~*LLes<8pk^#w zN*VlYol*wh`dA;ngGW-vVPGz0(C1Qyvq`9>GfElGC}lXCDRzh0Y_WPVF2d$%DQb9H zF2d&QD=}Q*A>$%!9&a15?ZtYD?Jb5j4b$=!-_xEZ#z(6&O5R@X#bVcqQOfXmcZyNU zaQ37aMGa@m#6A@JT8vLIk4FPU`q0jyubBN6GQMkiypdwGesy-X*aWdl#HNYeB1TgY zPfN+m%l(H~3S-1s%g|RGv=`e*Y)dgdRy@DG#P$`Vg6{e8Gdhn)0}*F55OKy|qRu9Z z@t3HxyTxXU)r&0``&8_EG1w-CWl+lS{3vC3xkrlqMvQkxk9VQiG_lLXct`bkw}^2u zG-vb0mWzEVwjQoAkH-zB=qomhKxbT}!C7y`+h2_L6OVV8*hsO_VrPp@5W7Tdn%F&J z4~Q)idtK~(vE^bdAO?6{)(L%B&{}L`v8}|m5vvrd68nwVabo9-O%%I9Y=+oh#qJY( zLabKoU9k_u)`b}0=doVs!?e2?O+~yv_7y7^8!L9Y*ac#f#Qq?5h1e{yN5ozddsS?i z*oR`Di+wHDsdaQ7okL%7u&-FTSbwp@#kfG4x4{Ww7m7_5`;*vT#1@IYF7}n!8nMkF z5O^6|gg%UWv7N;Rh#e_5MvP0Kd4HTQHc{+SvCG9~i9I654W_&yUNHnWn514&vPA`^Ae)jZ_5ii zs>t&(JeJUEQmweTnQFzect!eerB*aN*Mfq@LHPuWKVSge6COY z0Y29!_#uqz7+h%Av#3Md-yt(~Ird$a;(?{gH5`=n^xXU8vs zqHziqW4f_PMT5UEgW#4NMdPXT)Zm;{CZT97vv|LgqVXFNi}i?yE#1S?BRu~a=n>mDsYm#v z$>q=^I;LJs{oTr<9x=&IyPzJyMZ|;Pe&`WXq(^WLW$pBcBFm+a9&xkf+l&RRqE&|? zJz@*XKAxRj#rhT0BidN10>&3Z!#XvoM_|GTe;X#I$CJhvJKND%kHEAP{$f3XlP#3? zVl^YFN8D~lV?CnI!m%FlsD+c0>v{Bu>%&NiN$;FpYp6#&WU+3xAwoUEo3SxHg0n!; z*YWgEw{WaSaBiru9c*2gx z6VXd89P1Ia7A`dB{h7roFz5YEPEq{c7hi1HAfO)MXVbJEalNH!jvn!2Q#;p0w`SKJ zS&bjl{KTNaVpY^5IQ10-h4hH0ELN;XJYeAhdIV?28t)ta1dXmWN}Q7{>>@o!c(sKK z_01nGR)N0p1-as3aH}P1+Nk0Hi}Sxyk8q=kS~h?9$9Gd+<+Qn zV)bHFBs?t@2~SH!!Wk6_XSl{gMn%HeHeysHob?jpL(&=AG)zlH!Wpjt`r4S-?~ICs zvx^n)Ix#8|9*>HI$D<}#=a#rQty@%X`mr{%JV&U%Z{%)(he zu_0pBVpGI8B0RsR#O8{w(YV4HjVqkdxWXBYE1c~w#?k1E zzaKr`D6tE~XlCK@ZWp^#>@l$?#i&DgeyhYfpl_XR9QyE)x!C?<{lqw{=;iWrV^7Pw zsIv>jCX4YGtjD`k>^ZR)#a4)|6k8+qgBXA5dVbU^yxeWYdWh{IM!mx09V|w@!r7@} z}wh*J4g~vNktXgci z*yUnZi9IOxu-F@7Z;5fCDlcQD*!q~~ch({FVXG6do?`omm5Eh}QLkVw#R0Vm&yU-) zIJ;WxdNFR);_>bhdrs^{vHLK$1jC3lH&h7fzi;2C zjJ|caN6647tSqoK2%|u`_$LID^W!Z%Aqic zm;;l5SL-eu{o`@dhk9)8!tq?)>@$w3^;oovSehL(H%cj@EWlPBZ+Z@;QEZxBc;2Nv zvf*V}S+~71H|NgWJ;BS4azqv{)_*m;zHat}#V^;*9{u*>x@BE@JqHty3)=3D9;nF7 z&EAWpXl_OC*Gr~;2C1SVGXVqUsMenpF;-%ik7;Q}Lerzs?+zP%d};mn?c4~4jXo`G zbiR$M@k^%NiBSLQ7yB1IOFIni45Ri(=lASe{W7xbJRSb-b<6fGYdakyq(>|EhHnP~ zBN=pp+hwqM(^Jv@&!D|tsbBfy5}f_q`tOf<5hu@`Pq^d9Jlb8<%^tt_C7%2nb<4W- znp<}Zw)iNv)2_%IoNa7@jc9Q8<8U#RXI?Mwv83pY$xBKu9mxT<>gZ?sS3g(3`e+Dr zz8ne4?}rwUU4EC$AyAZ;B$H4HDv0HFRadJck6Nkckysp;#;v|+~r(& z*c3iLx;=a@RqU;!w0d>nfonr$3lGEBYbZR7N36l9D__rkp6~!eipKZpz=IQCZ}V@p z&FJ$NjQ@?W8T}&=9${d;mYq_Yrf$L1E*C910za5I$^w`uZ-~{(IaS!0iN)DAjdCPn zpKUpPlUI=FkK<$Pc#$pU#B0R=#x!nXfo2UZu~?tx2onFYy_H9zn*6zkJrB}11SP5O zQg;M@YjL7&48o5EZKDtSSg^ePXcYK6i@RC^L3a!De{L$l2WPIS1Z_ixS-kZcUDb(C z;O3c1ykt4~W&&K=i9enAp&w_K&q4Tj{0oBpY*wE$Q+{@Prz?Yh7IDi6ZmHpHSb7+K z?k`J4X;ZbgvVH+J6WOUs?f3N@z{jp^HdZsDzw6E zD6VYBJVuW{^^B3@P8~Vyv<7>P_{=4xnR!;+iB=rX`EV<)pA|>b5kJR4=|RDO)GMhd zUR2z+I>#~;=o;FxbTd^VS8aTr_ID_DwpC@gRfSdKXvTgYRu%3S;Z+%!9vB>zT9At3 zMOE2KRVhWPTKwZWNW4OLb>V(#mp)r_l!a3bmhs}md6AX*xG70ASn@It$5psanlgvy z6>o21nQnj#*RzE9MlFt{{ZrVr8zRn^R`9JYHbNX9;BiKYF52F*aG3ble6~gn-_q>~hTiaQ2-fe#i$NRJNv2gOs`bf@VZVy{o@$~Y8i7<0}+`_HYW}JTug6$J0 zkRxv~(;cnItW7_19R9+rY|*XZ5-Ja}aEs=(v35 zHg8!j$0lOsn%D58k7&?jk(|Yvy=!^IHFFE3xC5f)*LYoUn0;#TJ0+SdN8VzlFIl9L zCe3Xa?|7@)em3&BidQu9_6U}PMb~&klbh=;|J#C6^yDGf^yn-Ip79vZXL5529@;QR zP|pvua7%l!V9C)fpj+^9>Yx_K1&_5jJ{7C7Ib)yw-On;;m+0^0u-?Ot#;w{r5w>BS z2(wi=;mMC6k|)v79EWSpP{v`!{PuTN!=03lPGrrB4zlo`cBa`c8S_)!T1*HYY%vTs z!YISIt=ZvBzpK^wjn zYisrEn-=!FV0OQLQPgCA&as4z^(Q|&O5Ubib@TpZW%)5ube#S9SeRSW{W;HK73fd? zz=*PDf4pSjU3H(2Ms8)Wps1;Syvovb#;a?GcKn@{=P%EGFFn!wrI_W6_j}nBB-^Z~ z3?$dArxJ{h`mFt4_{J?5n)UE)I%(EZi(u~BWVJZ$rq(8-Sx;-kiT?OIi-OqZ`1gGX zJK*lZA3qeW*kllTYWT`Q@r<{ftKdEZ7hgQo@E*7jE}!CW!sWgFZMeK$VPCWwK59RL z%SSN9HNGo+2bY`UaK$q2c~5bTpS+P@0T&Olnrq;e!JPpY5=;$8$*FMXz#RjZ<6#_J zcKPXW+0|#j<)A(f?z?ay)6`HTpcqGqfMOiQ0G?NOxK|*I?Q*cGYb3_e=VF|*N#W7r zfMT38zIb{(3XINXit#UYC8u z(9tS5fH{hABU_)>;|zJVt*C8PwWY?TCqi9G@kSN(Gb+jr9qLi^+I18 zGbnP_MT`bT&barCr`<)Yj~Jz6k4J+dk2hFsqS(b^my2=7Bu{&*7$rSt6!JVj3VF`n z75hN!Ut(X0rEraS+LoaYW>aDtiE+^kkGG@PzGCHKr;2g68n4TFVl`q@#ionhAT~pc zt6F-wwAJ+do))9r?(BUr8Z9{saJ_rlbm%J%+KFu_#&y*^-o9dk#ZC~rNNkGOm15V3 z-6r;Du~}k|h&?U#yx5y!lmq=dC<1!@C;~d$L9C}3ML>^75zym}6dNsew%7!*OT?y$ zJt+3D*jzEnfnLU^Vynd3Lq_wuY!v#iVxZV=V*867D0Z0Guf>LlRf}=0H80~rF|M`d z>@u+%#cmP1TkL)@nlE`;N`&+k2dl-t7vrjH9*?W8dAx1JwioLqwzt?pVuy&GDmG4x zVxgBovCzx7Pi&T0tyrDd0x=pldD^97?~ApBROe-^8~U(+jMy$>`-+u|4HFwFMzPTI zqgd$WQY>^vvC!GWVvmb07yDFf9ms~Bwm9@b{1)3(Y#Xud#d?YDEq0vPP_YZeCW~Dr z_D8W>#Qr3Z6U2sz{Z?$e*tKFeik$~tz}uiE^yL$c z|BR>fFCZH8#hIdUCtT`TqH#L|CK^L3>cpLjAoxl*E_v#pb`7;-C}vrE7@1&fvhc6* zNXnLlvqZsvTO;NRx@p9G`-9x|kbwpoah6O8`LY|{LK7OXM9Tk@!^EnT|SB5~O2 zIU+HJF*@b^v01JEzeyyX&hw)Z3pYn3o?(Y18rwEQ6o(fWzaU36jz!|yoOC@ouGhrJ zm*yNFkaIk+@CB1gL$@q+hq`Oq5akhxYmn{#E0MScRmmq3lQ%;oep(`NEfOTx+vC0Q z-x7)YB15+!I~+cjhL3^IrQxTLy&s(qpC9GdkA5`XXowQBOT!<-=VCG?WEYMpA^TBE z$SxdHLiVFy!RNy9Hh2ln7LI>~>nYb_Z5U!vINl8I&lipxwOGRoQz#rSFCZNMkZuj( z__Bm>%t^SPBOHHJKsf%wYQk~ve+7i&sTS)EOT_Oy{BNm*M`_mUKMK<9{;5wA(k!Nb@Yf7! zmh(D65KFU&*In)L#+rPHmie&4J91{wDXCn`Or<@el`&+*ITfP+0J>Qcdq92@k zFCn6PzPlHad@p0}D4bK`)K&jx#YIwX5R_SQRaP7&UoY-BNWO=sUQI~8UR<{1d!6O| z+}cULu47Qjyx*!4Nx9I|tSX0_w8{BzuS)N9H*AJUnV-_8v!fvc4@@mg$iQB`Y#I0o ztKMIlkb(b|IOYb0lz}-VOW~E$CS~BI7Cr*kRI8?CV4tsTS_Xd4GMiaQ2Ie#)UZbts z#>3BPGdxNp1NXAy10@45P1SO$(#7LkQ6z#ws5RM z?rh;$2L6?W8fsbd*!j=l=*iAu?!rIyh1W?#`0eq8TeQ{f|D{Z7s0|3l9GX!GKgD1x8Rf1!H|I; zFC+uIu-lkyxUFR$55yjcFk6*YFwwB5Y)D&}!ZI-JWD3f_2Pd-TQ0r&mSO#9oJaDF! z!PBX2AOp`TAOo*s=^K}UYb-zhZ?opez{&e)pI}zwGVo};Di@Z47h0@X2Ijmu`<0KM z?8o4jP0PS8>^9b)`OOz&8JJc+$^N|4!i8jg7mo|bz)vQ!W`EAHa4Z8aWy#i`3!9XI zFSc|p1M`~9@OA(+3Lni7JSq7(|jeCePN%o`7d;78%&p9#XaGuEs^m_jh`?r+284U0lB zAGQ>N`4FZM%(n#!zHBZE!C%6q5Ih=ZM^1Y+3c*w0mcgA4mqIYd31q+;j=nK)IU>fv zWv5dJo&%RcFo!in!Ws_usc_$bdnw$v;G*mrT8dE!?u7G>gkW0lx>Y89E(AN9gtK-= zA=nv(U}yZn;B2NCg1Mj_bQeqslRQ3&>U zM~HDM(AkM%6oQ>m2=@GL61!E54Th)@pCOt zyOmfkF^qsP-VibLddS9#p{GN3f!HK5jx8?(JsZZmL+maw{;>0SoXPO|y)U+0jAPT| z@ly79{BXe;FI;EboxoYC*gj%qVpU=T#EugiDt3n0Sz_0V-7I#O*u7$pi#;XwoY;$E z?}>dR_KnzgV%y+~@N?tbjGu2WF@EUp>>#m2#0H8TD|U+57_o_By!Cp1SBYIG#%tf} z!mHooagN9tKLv62w%9VUR=D0hUYpS8YgRhrnw6gR2*n#HcB0tHVyB7yPHdvs#bUGz z^)jv#yHkv-S9-a$0rhxXv(nkOVn2#e{`Gk4hrZ%q53$~22aEL+J6ddz*o9(~#V!;3 zqu66&Pl|CfEw9UaVn2wbT1DrvUg*Qe6JlM&wiMe@Y!|V9Vw8lvT?UC!5_UF9Y^>P1 zVi$~>{2mG!k*tvVz-LjBldvU%VMvIeJA#lSQ|*LUcZviR~+mnwucx+Uyn!8*XuG~ z>};{gVwZ?Ll zRf~-kJ6-H%v6*6X#pa8BF7~xpM@Xq&m(4;SES|;6#VG%Jyu-x?i=7~LvDoj$ZWH^n z*eo$h!rp#$Vo!@L5_?^Y@~_wBN3ks+%{tpU^x;DevE9V>7dudFxY#JMD$L(|egi^Z zK7p6d;a@=D4UkQ26}+OHTdlobk-06j3jQ|7Dmai;aOv7u1((Sxxah@fqu}0aN~W#C zX$?7!BSQE<6wl38ZKd*L07l5Z_t z7^jv(YAk3LJg=%~d1C7~2)^Aj^LrG*1huYz^|NN6JH-ux=akbTcuvA1_-a`M!xD9| z$1SUW-waaUHEZCj%VgV&7~!GWeh5~;qZYwRb>(tT8JhuznQb^FnOEDZ^JNv~@p}CS z!(X7?>s(m=zWox7eizf|xAS&v;XbhEWn;>^+FEZx$<*~=_*#`24U=C+NtpbWo6YY6 z$Zu#-xB30P9pv0-xJQTBWArxW8Oq__pPSo zt=aOfZ7ai$_eNe)XhItD{sV@*+vPRn&GbuSGqeqgo~r302rz8mO?G$D$-S0&#-i-t>B@7OBE!x<@00 zo9SMcS8b&b?q<68G+%9H1uo(Lf4aOnBc;*F3G0_KWb?@S^4Lp?BP~^ z2a^}61$$W=rpKaA&!nPwQCa&YOb5${oHOjSu|vj<9XaNVgneYLnJ%{{!46zj;MpjQ z<0im~P%xwx>;)F5PYkwAJ(`N*MFsAtlN*XuH!^8(VPP%UjaA!Oq35AME>Vh}wnC`| zd!d`8&%}CX3sX_NsL(!zG}}v=8wQTsa|WS3=dM;9HC&g7cd+6pih6ND`ZvKYse4mV zyr{UXb&jRT)y;G>Yp2<|nJ$aw=H)aiWYuUf%sDL>`&m^eAA40=rUwT*q-Ld}cu`fj zSYEu~Yb}!o$0sycH`C=J#G~xky@_M4&z^$vT3DGAOrJePW$uKl=`oui=lm@0m}Acz zGWO(?Pak%MwIj`R+gifL^;zDyzqG_B6w+t6v#flnQtGqUS-W(xcA>UiX~%bwKI`rB ze$igR71C$@`r16pOxNS>l)kHI%Y^k&K(yw={cfi&)6&oCxzQyIMFYnDbi<{r|Yj9O65#9C?lWQ?Xy={igI)#@pAb zvX70N*fhLDYUkhuu;_!N*z}WIL$VT(LhyHKuqDbkFxVRJ3R%scv$GvjbjL%Nx*Kn8 z><(HCalbL<$k85M!gzG1Hf>QE)D%@EPBCZfvK80243YyYX=eLlPJgzr*s&hxqGeP1 z8=I$tr7EDmZElfv*O_e^7R588!Wf-dx8RA?`Yn#cgMR>STn*cnEyF_JwyJkUeud0x z7d5rQY|^YYy6+tvoYc6Ux*uL%@oJVec(cuF*^if6thgVquy9x1wW6~ue<#_|_$nBY2(ucehN%RuCz~(HV6M?r2g! z?QiKe``>Ct+Y{{>{qgq>n`LYK>l*S7cw+HK$gkk?7}Zn05>P$mqii7DKfoOZm$&(m zaC!Tm0++Y>N8$3ewGb}vzEo9t&wd*&Z&OrN`N*WI%17llaQTdF`6+u`$`OVqL8%=eZ|2xV%v-L65Cr0eHo@bM2uIVm%%qO`ig^##eOe#r`SDW zPl{1*^|W7#trp``*UM-f`tX<$>m|Yl-pc=do_+D-Jdi+eC~HPmf1)Tl$KFaxog-I^)_F9&e1; zIb!FF{a)-+F`Ce$R-#AohmXTVgB3R*J0=`$3H6wq6F!ZM}?cVl=#UcBI&E z#CQ#RT3*E-kJqm=Uct_I?K+zxMsr(dymCF>ePXl3_*I3+TOhVv>{GGTV&98xiF>4{ z?H2lqgFa$?#kjt%$K(3CUYGO4YQ(0BO&7aCY=+oJ;e49>n(Pa*fC2+xz z`ig`7#SRoZOzhWUL&U1Zri)!Ec8}NtVo!?A5&MVOOJY<{y?#_rz0R9Jr*zgO^c4r& ziR~!1m)O2yBgICGO%tQ>te5eGSgqK*VjqaLg4XDDX%qU21FEOasGfR$+luuN8z6S1 z*cdU+XnWf8#U_efDt5UT)l)B*>ZzxFL+mZF6=EyJ)`sJd10b)$8D4zLQFzdV!dc6^%WltGw5n-0bmdL! z-?BaJjSSto^g#Gr*X#$M>zdpZ+>ibdK0kURe15bVHz8LzPlwM{&+Enby`<}!R6_kI zl~7kbsf7B`Yw+&ss^?SiWviZl&!c+&h*(rT`M&(~RnKPFo?d8jB7b`N-w2zVISSzs z2L6y<3E5(8RL^rvV&gaIso=y@$5xNB*`HYTywpk@pc0#~F=Y*SgTPFICBnQ$__si+ z=K>>Zqk2w~dWkxQf39Bg>D9REd6m-jjFRCL*9j`x^E>*+Y)r4kNP^9d=_}L~F=vCn zSXcD=H`Eo6vvkeS6>nmG2^&*xcc0W1_pkzEUC|3{s4IGb*}CFPW)%!7tSjDPg+_WI zSD<6USXcBy8|sQ)XrE;3W!acsX8A_SE@qr8-%U-mqm8NO+fV`ae1BC)ubj?WqVPzs zEYBcoi)@jy?WUiCT$6@1ZF~~%+)aekT@1=n9D3(tYLbY&CoDEwS-M; zn2Rh#c-B8s^s)E7mZF zTDX}S=C*cHu^QllBCk(Zb)w7)YnU#3#TsUmvY>|9)~ZlI!<=m=)tH7ESQfF^@V&*3 z#iFzwPf8U@4fBdbY>Ez?3T>n*?+H_$#1c-foc!YnWZEDzSzc zjhsRnCa0~}PQ#?>OH#vJ&%&{W8C}B7)-XS7FtCyurVD{_D=xFx@v!_bOx?5w_PNC> zpnmA+!R4@@3in*NR55GdQpKb?LKX9V zxRHu^GyJY%(&s9svq@+jXH+qrQN?sd71P;FvDsqvVpK6b9#u?FOBK@@RZM4ifeaZ{ zOlRAOZ7 zusq(kV(XxfofU_^;$U+z+KPI9RJ=W2KQZ)q$OefG7aJvZrr5b+v=#OI(6eC~GsW%@ zdtPjT*qdTY#Xc2VB}Q9OFM|(EKaZ|r{1C|5?qVF-&ML$z#b_()`3)80ciztaON?K9 zJG)wJrq~@~4~ji3#`zFWOIuO;im~NK#NHEIA4fc1htP+96WdB`C$U|{_7gim>S%7>g)-zTCrtfABs`C^t4}#Z2@i2+18;Cc3Wb* ziR~|Tpcoeo_Ox6y*z3YYgPn2FU}v|A-7dyufju5~3H5j%h^-L&N{o7@$7>1o&(p3O z`ig_C#i(a`JT4#X@u+7ypAik&NVf!Iv3JH)t~sOQJsL_NPHV(*Ai zvGjOUEInQqu`R{87_i6VV!$5nc(D`3&J(K<)6?D{HbacsrI#^J>?5&H#J&;x zPOKF)L{Hl$^uZolthd-8F062R)}%oUuRtS*UQ)( zI-s+yLSJ#Pk64*ll^FF*PkWr$P%&zko*%72J>DH+cZoeL_PE%5vFF78DfWffBFu4n z8Lx*vsFRl**R@^cC1-T)RC&pHT}vx3sp{JLsPZGrk18KnestxePsf#PcShHt+?chr za?%G&tFWZ*(|B2_;!dcQ={}hieKM{4W2ecYXR0#G`jk91uw6y(XX<93J!VcGK@qym zKWocNp2V>+PupIrvZ?G}YJ%X)c!4O-jBCByOXZn^_AJl5QU1;E_Q^b*nRo3sYbvW( zmuJe_(&n#XTDnDfW?}iHrAsS{=E1h{t&%M+LbYgr`D%IZRVCBLBO@61jmL(ObE}Ht zT~hmI2KFq^%r5UyRC4b{U2Bn|fAyTYTd*DK#4@+y>oG4fihQ~+76^uQ-j7vN%j&;v zSMf+Y{Qt;DHiU8H8}-ZQy#x(&p6}`UY|4&zf_>vc&uplt8w!iRWtOUQ-c*WT>hZag z>SBxEw*IYA*bEdU{jl;g%e}P~X9aT~St~RBNK|CoiOiwxR8JBXnr=`{6lg_!Fscq2 z{{vgAV&~DtUC<6e{ReMj7u+9xDb~7W<9j_#gUhz9$}~aX<$Y4Jw8)k0cn)3+@YOre#J49TXQ-;sOY$ z2m-Pw?jVYihCJ7((HJytNlYYa5@Sp232}+L{?G4z)u*d#mL6jM ziSMObhpzh7xwopTtE#_q@44q%Eg9=m%6!sP{y?Vu8M_p!7M@@x4U3K(Fl|`&2pgk2 zqh?t4^!loSc2y-Nvpj0)dM2;b?Lz>Zx_#`IJ4{bR(u1R;eR|ufzi46qnynpc@9%7y z!*h>4=C}zbp1Aw&(!B?g5v|wmp}p@gTi-o^eY*WPI-!hChsCzuS-;HiP!p3z z4i6=Fw^r>vd#C$l^8GOT1p9-{exG1Jpc}IUF(f6-@t8Zbd>>3E6gbfeWI}=3umdk( zIfiJGpeJBp*v>!dFKM);bU%xEf6*?O*kFxZxj|u_e^id!ziXq*&I={Gd!tCTt$#DH zPs%!-%aF!Bf*)y%V#2$2<|L%y1~ZM6`Grjl^+h`aJ4ExM0Z|v{5S$neMR`F_wyIW! z^=YATpK(!YQ@Ew9cLYw{rM_y%OCmVdt4+OVxIv4ece|KnlXCE*|$y`+QuWm$Q2Nm;&~B|YMYDfh

8`zBOTgGdmycGtCOA!Bj)N_+r#*xo!Bs`JsdT61#72q|c3E z_umht80n+k|GeXo68N`ohC-cWVZ`X5VAQvrrYw&3(XjRNiLJR7aJG(#x8|qON{!NC z2kMR3Jn;c%r&_qF>XT5_W{Dl;Cd*^%g<OVduqzMu>fYg7o9^FQZ5SyqEnGlt{+ZL;niYHZ zUThl*v#Z@~q%gbM5+mvCYIhprwkeoot1$`V>}ou%B5R;Mplo{7EjplgNYBbXaxZ>I zxXL!NzpG}~Hi+Ph^KoU`2CC)hhT`7_BG(zNqXe z&S)eOqmf9*a*MMi&hB>B=ifI1ZYILS){U1e{Mgwx&S;#K-3Vti^oY^W z!%}5Lkh<7HXFqmEldS9>clM++yFcQxq>+~$O*yKM6!Bur^(2;Yc7(Ih&Imr29rHh_ zEb}^v-QtXSoy2Iik=>D|6~)HHOCi*=e!=0rY8TX-BURshNgJYE|)`e*%M64 z6U!vB;LOOpqu>}_W9IeQs|TzwgQ|gd447l@^>drKPBNo2M`w=8969>vqn0+?$(LjF zs4W#bMvP8KJ4V)XVuKtBp!p{_M!R)Oax?Lab5F8^ub!U#zZb2;Y z^RQbexoHu{Xun{8u-Wey>?_+kM)UmvBhx$YED_>_O-@%5%C?#dfBEpfiI0Z_kP1{z*Gg4OVXFa12{^=tObfcg>07a-e%O=YiDN z+Et+FuFjn(MEURS56`ayCyFzMe&dr7YuMic`ue(86C7xpWq1166Gu-RKRcYesb={C z#+I}pcSQCmj?f%Y^xLv7x?U#*^FIEes%?F4U{92_=T?0qY&$Ro z)Pb9IU?{jw_>N?lceColwhh|2S;yOFl$$j=xLL=WqUCb7NnOm%8r_MTg|I%s{!nXq zXRJLp>qo)rpw8VYzM6-Fq1-A|{ZnE6zY^8A=SIy9R%ayCeLDlpZw*GB1Qu}x9UF{x zNq7;t2o65%m0@c#wSCfU8W6U25*Wk|z8j1-N_?QS+r-DdJ6I+Cv!vT}M%cP(qK>rN zgadp`uo{-IO3xY2N4h;&ZIbxFNw6}U+zZj3Ae$1i?YD7vxz zT^}9D93?Qk8vr>xrMUslDSkakbcJ#W_dYI}Q^!5)Knz zJq{CR3x|n20*8q^0f&j`1NO%=A?Gl?X;M=T6H9WK#OB-4h~Y4a@$eGEVG?U}hQlPr zCy^ZvlNe`242Ma~9#ApEVG_e(lHGRBc5;?+R_BbLEtNgd*_qC`!&Tq4&Teq_GiMgB z5!?OQ*&&jcu(dKe$=NJtj3ZE4GBC4L88ta;amHL%vSUb; z?3ndR>|2&XMZFITFJ;61&ma51rZl5!?OF8P1W);vA`L zl_^*;`o6_>a>nGiVmL>#!#NT=)!CWOu5osqvt`a6a5mYVRjO}Vyp(f}`pP*PVSP?% z2Whe0KsZQMThG4Pour2BEd#pSRZ54@?CuTLA(S07Ji8{7t=l#8&aB`d-HU^isrv26 z4m>1tqB?Vu${$%!XI(|+BMtsraFNck-QgnDW@c%O4%F)G}^9)bjT06X<%n z*Z%%s-~j_8&80f->6X<`*|%z~n;$(pyuc;mD9-Lk`iz+@@Hu!H<3vUSBMOP14Y@4meXar-9 z*4RI%wn#n_mfLn)7FtE%Tdg6OVM_9C0pD zEN5UCJyCXzz4B_yUL~>k~@o_>_ZmiWwXkjP4A#a(-mbNo5P8I}-&{ z8RfWq)Isxwgb$H)n)VLWCdX~}D+gH&G=U6p)Lu6nMzd{^b=upQWplR-U+L;BgOZO8jYf4Pgl~BIyY?XWEL&X;|0N}XQz%7 zc0Uhxdjv;nF?((MiXEv|+fwLA@gyyDq>%J!M~X*a8%K&~ox|9X;yG66Na07_X@9vF zzeDVTyXck};z#k@gdbI7@Ay&taNyVcIOg18_)*+>xell6%>Vhs)`!}Ax$T*Co?SQF zJKwf7tj}3!@Ay;Pclc9W0r*p#H2f)UAp9wA9Q-N5FW4`SkeomDmZ@9$Q!G_R^K74D z=f_KxAvLJj<<9T{#poDU*@vAy>g*Y3E1j)l$D^|A$4izg$k}Gjc6PSAvwfZI?~MCK z^_}MIJI=0gM*OqvI8!?2RnA^^MiwX8eduf}J7;2FjF(LJI%{xtxHDqowGX}-E$2RS3xlFCkTMy@3>axJO8 z>z&=~>;Y$wIKw|s*>{|ko9807Uc6-fo-+n-Xdl*I5Ze)7EOxZBW1OAuEbHtBXFqVZ z)Y-kxUUv4HGrJ$+{gMYs`~8CHV;%FC;-$){##ybiW1SHxMbU}D zt>UwH=8VayqQ6T{6{DSthw~j|_HqNi*tXZIu!kxg&iCamhx0AALr*(ihuV%6&iAzr z9WUb z>047U-UyDO?JIVU=Gc}(=jZ|>h0f81M$*m^Ei�<)%7aoP@D+#IvT*IcgJ*$M3)z zv)ai1uEyM6ct`1QJQ_&wmS_~gTUui8ct_llct^|aonEDn?EPGO=c*wbk1rK(2p<4% zh)V!(h_i+_#GQjT#2te-w4=S_4e`5_^M+`-mp8~?3rcV>4ED)dV>ZpSYns$g5Ib`yL5)P6~kXu6uvo{M);V|$FEzmzSAqv`nMo%toZgkO7} z?vnd&TR%=`eu>nrxQ1LUK873n-gcvoG_Y5>#*KgR}U&@Y_K{z;gF=ddXR+w6O4+)!l~n? zqgXg}l+Bfva3+#oOGVgPBo>bF9;9!=mq-mE;$vSOthP$*DCxDR)2di3++LxMVzF?S z1gnl>;WiChi^Rh15R8h)!YvMVeM1+{G|ocXSL~%Mu`PvO3XlClFQw5)+DqZV)W%EU ziQ_Q#Qh1sacq#PYu-Dv+--I=`va$VLA47Z;ekbrrcn0H>@aV*MV9X<&8wVeS8!G9e z{MFWpg!{X_JuPUaE}xJ7W-$_VG1mJ2)HV z%m%c@c4s?d5RuB#U!#36h)C>CXAB||d)C?W&dTlTQdxR#WQSiOhF>CvUm~`zv+>T3 zcecRUB4_v|DvMvD`W|-ns53IY$nG6y_$6ZaB`Uj%v%$`ebT-BrgM(D|0%yz>A+~M2 zq@EUwZwr3OA!Jn`mqNlhnfuxtyEEu+v8)Pb6l7JnB?OOk5aec^F2jR|a&IRdN(f^! zH^ze4K0e#V@!NouMf@Zci)4$#*mg0~IE;-4rG>E_d*Rwfu(evj&vXP^=oCqMBV7+- zTN6AI4`SmtqQS=QG}?PRL2NgfHA8!NPU4VEs~X7nJ-2xd_KCqgbU@27xz3j4nEcR| z3LTSLyTsDrY&+PafspGB?fp|bCdIx3NC7lL-v)d168xiy_9U3D-xj(hhGBk>rBWDuY=*7_E0$Z|OoX+Iit9e^NF% zdd0rLe5c7df&a9t9?F@)d?T--+SV5ac2mM($R|Nif`o0{L*WwbQZU-#q0-7W9?Iv8 ztHW(oVu$ij`q;K#gmQ<4ei(w$eiqipS}@vSo%F+?`{`sRdMvNF+Y3gcVU+CN`ERrx z59K5?x-qVfuMUsp_ORZWheG!O3MPt1cI=^?Vg^Hk(XYadNHCf{Y%kj*PB49oeyE-;HiaoY(e((Jl@pJMv_9jdz~^ zZqofPd^EmA#>eHY()}-3oENM*3Pu|rwiXFS!xiS)*4d~Y&iA*2T}|+0=1`>VEB0g> zZA+mi!%uIaC&Pm)?aA-}bEhTO-_n|du_wb*DCx=I$lM8wU;Kuwv1RU|TVjSI!|w_X z%TRmAk>R0_V{uu``G#?1xbJd~454Sw+4@j>f5F~a=gD=my>o>SiZ+C9L-*fg0XTNfGud@f8 zt#S62v!3QUXunMOp#73MQ*2vjq|Ou@;_Mh_cJ;+&FL1Wd*-xGQ%vqDO7H2Ox`?Is2 zc6DjLz2c?HsGlofjp|hVmBf+KWYjw8D*~iYh znNuRW0r8Ser{-*5XC$+f-SN&QJ6q)JB4El)~ zexE*?MMEsv!SZ=8wU&Qu1A3k?N9U2j?^G??$4ZBYvs`Dx@NsWc^{-_d%J9CKY-5}> zjz=;H*{&oVT*H-pLRvN(%wuWgTGlMPKiXy)>DeA7C0xUm{Q|1L!qOoO8`m_G(Yq+w)>kUVdue{>&U`(y6!|0!7Eguymg0pmdiB6j)3$Hu7qB@xf&Q4)Cf3<^u zSGDkIb=QOuKi7x+fXV{bSpGm=`4jf#R4u$R_K5iTX}61pks0!YR@hRZ|MR3>ed&NP zo?zQrWn_B!RQ}H(%K=v;U`*X5bj)z&S{AtAv@Efu&Rm_EAb@doYA6)FU!+q;oj3=Z z{Vu_NgJQvAdj!eFkLn(wKzX;T;zL76bk)V-a&FJX*)h~v*nLg2{|de+KnbVBDY5K4Px zD7bYfh)eWDSXZYFQ|73mzKPwpm%!kXQ0kOW3!!H!b$i0oNp<3I4g4Y$DiV4&I2dh` zsHvUsv2O;Wq~DfyU-+nuoM41Y{(Pq$6#Rh+!RpY2RoX$p>Q}+4PvWpri3Rw8M+Bo@ zow^{{F(8tjTe+dRv$ykSwsU0~ha10muCKA>9`-jrX80sL>&eH)vk{+Qk-g)ia0B6^ zG}=2p2{(qR!a1$>ey;ftIJo!{8(F=S5i zzqG3FPtNcv#47Ck$gX?5R2hcMiha!)`RT+?aCWk@)0{DwMRwnG#$Xn)>zv)?j4lzj zRz@#4d&$`+&gc@6-4^jukI*F|wv98oM8rlqqf12WbZ2ylh-IDqk26BG*xDnaOGI|f z&hB&egtHaSNE4v4qzO>jN~=q3!+6P(xI5dy8IxVe?r>+PIy=)DiJfKleP=Ds?sxWQ zXMc6p%k-7X_KBA&qg|a1aaQlF!P#rh);R0LA5b4iVzh~RK5zm;4m5KJ5@Y*q^t=!5 zugi8YzTX^zPBNf9kk5ejXz&pZw@aN)kxce|YoTwon=rTpJDN-I*t{28Gv#kuCbJJM zPgg$ASzY$Q+QH9OEutL&-=NV7HH>@52DDo$v|6gWCzHKDvrUxQ=E164m$pfOW^(_K z%gJ-56lcS-bs@!B!v)<&W|rZ(oqT5IhTwykD`Hh!W;2ua{Y=`E67KLYJY><1 z-E-#=-pMS-wde5vswZ1PBcr_*bQljry*&d_lqb_x2>=a6&?{^BN zeqTi2_bI05_jEn4(ejU#-54JV=RB?F`EjS=?$)sLeZ4DUU|8oL1xK~oQhWM-drWNp zo?O?*ps>zAD#wpk*YrK{4e1nRG!;hC*~Rt!ckTQU(bY+cGPG7rFx$m~%)S=R64LT3 zfekY`v~ee$A)Keawu@KUPxM#!LhR`mT^QK8UF`^&6wC|LmYp8hmF;=~Bf_?k;W{By zi)%TGwg@6xV!wk>t-V9Rb=!CaCs+lChi!K!cBqaJyn@HWwlBF?usp2Snpg0pPP_v2 z=WgN9c1)<$v{!%?@fUP#<-N9cIzmvO=nf1y`4qe*tZ$X5KCSw3I(RgbqB=jcBc<}W z5>5$5MMAayI~Z-0s41oLscBp=>XkUCw94nu&ka@s6IN-J&$U<|tdeQp(kdUT(}Pvg z`ADgJ4y!g8_3u>WV|PJZS-)tbUUN8cZD0Scjs`mt`WpBt$h~-;ud$U)?GM$=3ti7M z9v#m!6dlh!gU%mm@96bwD5Mi(ohyJKs-5ip z0ek1H()Yj@Mc)HoD}4`~Mfx7NV(EL}d4#`!AItu^XLJ4n`b>QfEXiLG!(R}?Ul7Az z5WCYEpInS{BRl*BF&?>M_zPn6IEZZ!FI7gHIor}%owH%iMmamm*_qDp7qk!j1?_{g zAcnsnhQA<&zaWOcAcnsnhQA<2>HsnP1u;UX#D+RM(b-AP<~lpi*%D`WIBRxxpR-nH ztDMpIp#9SKpkvw0RJhod@ls{Pa1XKFoRNK549`Jj$2zmCF=qG+vcq4H9eodC^gW0@ z<%|$2u~(h_#Th*gD%&Ssvg32Mo3n2?WAMA`JKNbjXAF5)eGGY*o!tvD`<*kp?qfy@ zG1+Zu+Dv6PkC#kLIAcJE>{dB@*;#$Ic`foG(=+g{UH9$_WU_tD9T=9iXI9ix4-Z+fb)$1|4dtFC;hc>!8|fTwX_BR1vaxH8qr5F0Xq+C%`?` z_2uV=_@L&wXB=U9z1|x221_LLdTZ3HS{!4Gk882Ur2g})+B=Q*%-#}GcKzOfP2)1j z(jwx z3d)|}lFnN0m7Ex5Pfs{Wa}eL@;H2TUy*_OJp>5Cf9%1Wx)m#3aL_XoRv$oW^(yt38 zZn2y~e+`GmX-_NtoI)QMSD#bpTxIPk`<;W`FFREB++SwQH=aihg=9FI4pB1U4ff zu=DNuN<^$+G!#7-ZI==ej|i2vRrGjpdxdRRBz7o8kJ@hz+cpe}p7~yy!upP;=r=5) z=q08+}YlD|*!CJo`)L*}5>SS0}1(PtlJJRtF?>cuLXJ;i4fo7lnI* z4wt6z8Mi8%5?z!im#cxbX4BY{XdN*i$N>ouStVttu8y#0jD+2$g>!i)~-A%IB$8z3#s>NiZURcH)3| zmh)8VX@8Srj>u2v2>!ZlLtNiz?}+~=JwIlAIuy+E8cs39~w6VP-${X#Sa*x@2 zPLvNasUlI%k^+Ln&bL~`eh@G9h<@zsm(G6e?2pa}15(*noW1UhXO#A_VZ2lsZSL&z z&h~Y-zq2XMxD2$9bDYh0cBiwuojv4ixiikWjs;gm^)VV-tU6xOXl$`T&JK0EZ#bhl zUv^WRG4fiBk=HC$Mn87;Q)lF+k{!9JWcNpB&p7L5Qdf5C#!EKKtg|mT+uhlo&h~Y- zzq6UnW;^?ivum6!b#||_$DKXt>^)~6Is3eQY8~?z;-$*aXfDPqxca<@x!n=Y&Tuxz znO)y;eS`t2?0wE2a`qQzZ#W~HulCFICo0Pr2C=U@YjuV^mt7}~^YdP7lML#X0e9L} zKRmnKgLP~)LY)Qc>|0^-JDmfH@EjX`5OoxfGu`5GRwUzb_G%{{=jk{c2T$F@ad`9% zANQu^-SuFckpF3k@eo7ve>fONYQDXAoP18Ff_R)ino`eXN$7-Wk5gkqrbi@4Cj7|GzGVFEwxp2>zp|x5L4TQDR_T^|dRVu$YKcYsl!Cr! zDyMv8&Rw?0C>k6hbGTSK)A#AfoM+nA_xiD?Rz>Ccnl8lE{2i<4ABw9vG8Ek~v;m`H zZVT%}gOo+)T^H78SzHY)A6GLz@TCbkoY&tMgl%1jt2r$gb{1FjdSVCp7W5lJxe4JQ ziL22;GV#ZRPV_d~@GIej7K^Llm)=H>g_A;DO=H+!7vgG~L&1Dpje=eBaW$%_SX>Pb zRur8Uc28W5N)?H#xjYmq5?8ZDFiMKvlom(r;L+eN#JzRiUmTaW#X3QSrE%n}S__oi>#pxqMTcZ7I|&~#5uPH;=Iw`&owc= zuAL1;IRX?>&htJe%8@h@!oK8EaQDHprX*+b4=cJ`XHcb$Fc zjFxiMM@zZt+u9kCHDa`s%Z`?EF=kFPI~!o#W`xvH2=H#@TFVXE?jo*$vKq?X1ZeqFnVM%2}$6UUBxivyDyu%8nTeWk>tC z*dET9c0-K03uQOm*~!k%aW>!C#m>I%j2Ii$$FO77M~scwL(X1z_BUr6*{6}+rty+Z z9q(+gv!k4ibvDo0`OauFSA8~yB|er%oIU33@6O(Lwyx(DS(#=23x~p~Pb+HFcpu|xzuqs%%LD9S z28OcyqZpS)TWU{XlWmac>~bydgTgxhs2o2@MHTiO+o3t#9kqXxa0sPGn3ZYv}O+sR>dM)jtEvs`JZa@=CB3?qvDY*#|Jwl0~*7}ZTpIK^7V2s)m9(zrq2UrbH-xq^31eN&onD};QPlym!CpLz*Vy}}_J?NWMUywrXH+!L zKU6dqK5;Dv**j`^Uc8;7LtS%$qpp|RJ8>;9+dFYABnKj{1xK89zJKCcHnDf&S~#b~ zweU3&*TR=dTnp!rxE8Kg;#%g|J8Hez-q}BQY)-A;WKRsKb(W;o#Zc>F7spFd>tdWm z+40H6I5%Rbbum7R7;0T?ecQj-2Jw>OTEtN6vct_28|DnTB8FO*9co<+wJwHQ7elR! zq1MGv>td*NG1R))N6svbK+K41ksWa@vLmiVjJOst8qUS0J0mra7!mQZBMx4SIC!z2 zI3r1r*ptrwFQ=QFlc7?O6o!#v07H5w* zd(7FJ&Y0do`{-*|m-gE)UaE}taF%g4&e?I!7C0jzkm|eD83}>Jo^WP&LCijM7MYro z-QswuM|8cjPF1+&?3tO$*|Wl~=8mIPIuPV3DV_^SsjD8WkQ9C@x)74Wv@dZ;iqFU0 zrPvf6FkOjF;lcF(Gd9J}deGl%6P~h#ho|tEDGX0}A(q-al-eC(daRvT>212;q_=68 zEY#Z<*m0#pRGu}fZLQiSd;ipWyI6?I!4Vm8IQyLmY&A)v7}O;F-F5{ws%CGiiGW-L z%LrT7YwuzqDnngx4-F+QFFPsvARHQ}J}tO&LU~|ZeNHHIv9%|#cMNvdcPOyAvpN;n zC=()$$SYGMz=U7WD9Q(zs7bpk0Va2Zk{`G#X7$v-CUjL;OAO~DXsWHavU`q6=-j-p zo)D@Z=)#&yu1ncH_YVcz3hO?`Yr?h#i5=zxOl}I>HV?v@0F&#(`tByIH!mWrYr~-> zLnhL~nkL@^gH;y-OfXIcn4BHfiv^fSw|6p;8+W+crY{XwG(8Xcj9)1`CAu`Bqf=s= z4?iS1ct@TFjMYC=Y*8?~wG=wOxf6VhUvR)8VFs#t)@$Y9lp*k*^@1*75t zCdULjO`bL_eEDtuiN!Y0q(ZUHv$jxdHyTNcZ5|lh2L+ATJQ*Bzka-iWJ=^|rFP_6| zY^A6DO^zX2n`bjxn@1s9dk=d@<609z;C$a`Y_4@QcB8!$Uc!@s@DhGY(Z;u0{gmUT z%V}fV9&2Nk6kZ~BzDgS-yhM!f64?=6B1U+L7~v&igqMgBULr<#i5Tv`7;0aP@Dee? zOT_kdw!bre+hoT@AUisp#O6D@-PtdkJ>`rut+KB=`-?MT`Bh)vc*&B`J7d~tl|9Va z5zZz#o9gU*XIW>LJ0tUq>T7gH^tag4&RU(l>+C~k-A#q+n0v-cmC^RjnEFih9q#N% zXLFsM=j=tKB zoIUD{bpHCh?>YO(+0~|jbS$W3*`bZQ*Dsh!UduBE^c|TwBr`H|Xy&l`HYxX+M2cI! zA^QqVq87qkm+fxpUcwAwZ&FIoF8_apu)IQ(J74CUX%WBKo5|{5ehvK2Y>erW9ZkC5z z-hN6A)~`Q%RJDB``(A>K-cPva7WNKJ)f?N%Q(BfLLb@#s zFKe=yP9u;|DE(`;TZo1S{cuEVmY*97%kFs~QEpOVfO+ESrle!&hU-9U}=%)lNV zD4#+wZ*QqtiY8~#Fid_D8A4T@cFI`7kxzTLWS~=f>zO+NF%)QpNaPB!~$s}93hT>{l zI}c=e5Ia@dG~Qime*}zdY2nX>MC5H=b`ze#M%>sU1qf7x}c>K|@8 z_RF2R|smRa8by-5brW%xSkfw%0odj-RG$W z5?@Nkb8~$$Z0!+FJl5zi6k)#oJ=M_7N8C6VZPaOSH+J%I6i{-KojhF&+l657Y3!%% z>)+b&eMRDUc*d`>^=bAJst`it0Pd8G$Nd-*jh8B;-JH=JF1yjr#yLCF*<5GecSggu%98Fy z`yfBQ*h|j-?2P>QvSThU+4VC$CsrLVRYtYWNC6|eL!2G%>|JLcIx9I2CC4GEh%U5; z&%fAlC=%IHavbdUs^mD79EZ^}#*LjkVSMbop-W4SgWVb>$KimoVH_3dR-o0t+ z_xAd=j+ptZI}Yoa{E_3pk{kyy90xHR2QeH6F&qam90##RXU)!9ovm_)<6pl zIg~tylIJjM%6Rvj=q4(4IkXA?`G-9R!hb&Njzf2oKXM#cQh1OU;Xz`T#7mlvO6;f3 ze&%*f&RU$Ubhg^r-<-YeY$Ln&wBJqRrON0F&InP{e)o1p@Q~OE&Ile7JIfitLt=M0 zyVKcy&K`31H)n4o1}k=AcuHXV*Ks+1dTh9(MMUGp4OleFWR9z7L!cS|qkvyrj?~F+z(}me3-x@y^b2c8;^_ zo!#urhN8q}85*nlUUv4HGwZ2~?Fh=39W(N>)Fav|Ub1PmoDFj}(%C3yI1Va{bQ zTSl8W6e(o1S!`tZI7=dLp-4*!MOymV4Mp0L!*4eliOe*owT(tvQMGWx5RF7~8YQJk zL?it^jz$`hjz+2}h(;P`XHn5elWa+W{B+_Mb}w9O*F!oksnx9LpZ>MIf6BO|qN!=p zPQp!V6_<2zm*SERvd^&>-$8WE+Quba*X6jR#a4aCaY;9J7?;G2RWz#T6#E2g5mht} zYl~pYMHPKNtgquyMYPdbbf85Q!SYc>#|J(;5jm8OD!MkYO;J8vT$hD%14I3j%E}_u zKggnr1{R4b>JvU#GD0OC9gkd89LtH}^AJ@;dtwxIA*$$%P<=kCNG@nOFe)BZG&)p1 zEPSg(73o`5RMCh~N1K1*sG>Qxp)kV#0waYH{udfaNBE=C+jI^rPQp0CpJxI?sSYTc z9(9Wj=xu9hmCrMJjqzUg*I8WANw(b@W#@2965P+NN^t)Yd*9#UgZ^Oe#1(OBi7P@4 z6Ia9~u)^LsOT-m%=MY!K9YtKx&Gt@Q5x)_pR>I%{RFG6YOX?mFqkBM%;}D~JK#XG% zqkBNC(OI*zR%ff6aSl|L?g5phdq9lt0kN%|(LEr>eJwk>2gK+e5TkoQjP3!kxz2v# z>^5i5I9uuLb!X^()%TGz2F;3X5-;_LHgmR}vz?p`cXpt&qnwR(#$W**GlK=Rj~_cD z7o`|!Z)8W>8?hIiz3S}m&fa&nu4!kLt&Eo}f2OlR&gdwRU5&G`&g`m;_dDO&H=UvK zRTh=6<71L3u{)hHq*m+?&QSScsC<>((ymOgt>UH1h-5cnUw1av*#u|jJIgw|!5Il~ zw2z-T`=zs|oIUG|DQUG|65y!pM$SknFLtN1yPcKPd&9qIAi;mMdSA=npptq&an{TU zqfZ<=V^Si>zog#nr@5rw?N=XF#JHQbV**yPv83Lkl6p^$3iwy5cc#RVdS^-MT@3Xu zhI$u6y^EpV#Zd2Jjn0~#wK`kn4E3(EsCSh`y^EpV#Zd2JsCO~cyBO+S4D~LCdKW{z zi=p1dQ14=>cQMqv80uXN^)7~b7el>^q29$%?_#KTG1R*l>Rk-=E{1v+L%oZk-o;Sw zVyJg9)Vmn!T@3XuhI$u6y^EpV#Zd2JsCO~cyBO+S4D~LCdKW{zi=p1dQ14=>cQMqv z80uXN^)7~b7el>^q29$%?_#KTG1R*l>Rk-=E{1v+L%oZA-5Kg#4D~KM)Vmn!T@3Xu zhI$u6y^EpV#Zd2JsCO~cyBO+S4D~LCdKW{zi=p1dO6t9&-YvO;T@UF9oquZE``Sk6 zl;nF!zE4fZYLpt^!vtKV#&=U_D8x>O^!}fe@BbUx-shMuk$PuI5jJ8(*oYBfBX+s7 zpE@JLMs`ikTAZzPw%Xa>oDpH8vP9Uh)FUFoMvOiMF(Pclh_DeO!bXe;8!;km#E7sF zBf>`PR%Zn1i#_Y?d1rrd_J*^Mopm!Mtg@TNOFg2^oo(-IXJ`96JILA5&W>?*fwP6q zNLb0>g(?Is3D-cbt9TY(3M&s;@`9R2dO#BetD0Vr|51)Ooz$W1Jo9 z?0jcT(@KfT=muv$aK<#PZ0!;K(%IwAo^MLRxne6g; zFSM3FS66lGlb*zE;RII|BR@Fld(Jq*Que$ys^^84 zt!H)8&emHjPtQXg=jpMKp^8nDX?pBKg-=vF_`#}$^{E^^VXCGl!&FV5vrXja8Iz2P zDF}$+k)wbZdYucyV7_c$ZaNI+7iRTEJ1?YC_*7vqgNmo;**SDWA7Xpj+4g8|Vy9s+ z%AP}d9%av&RU`~%P^g_xJt)*ZrC1n@rgo*zcehaB!qs>2YFv^9tp)eSQFq6Vy zl*LBU!6?L-%(TCPbd%h@4&z`H?#zN<6yz{_&AoU|uCbLW`#U#=CN}2q?4&V!ki8Qs zvcTRg%(qR78$K%`D16U^pm2E;g2I(Gn zqY+z-hl3c6*kUwdi_LX*yEDE4*)c*@c8pLJTjT64XN*vl9V1j_#|Tw1MyQH0E>&zF zXQQ2sb9TD3vz^g&t+I?&Ree8m_Dg4fa`v3F3KNYg+dW>YjQTk1>ui9tfzI}I#?+_U z?>C$s?Tnyu?Sr6l*}ukr%BUnqOLBDg-KS4DY1Y^&Gbf%l;ec5)r_Gu za&)NuaI008CCO1S92G;3iXlhEkfUPAQ8DDG7;;n$IVy%66+@1SAxFiKqhiQW zG32Nia#RdCDux^tLyn3eN5zn%V#rZ392G;3iXlhEkfUPAQ8DDG7;;n$IVy%66+@1SAxFiCl@LRYiXlhEkfUPAQ8DDG z7;;n$IVy%66+@1SmE`FEIXODezT9-60yp^=?E&*oX&o&Vs8G_QcBV^uw4_H%&7-|P z{aXTAi(9B2R4nc&Rel z#Mx%fzUXXgXZtxjz}XSbMmwA1>|AFA1ZclMaQ1++N1QQVkL=!Zwy9llVw=ZH>iie0 zaYpE_7@@l?RYnV(F{`E6mCnBF3^^Lt7cX_%I$Be|X!3yS`gyBc8?v_ypxb14c6qY1 z-cnoCS!eye7477k&sl$btsS2_;~%oU#hvjs;f{C4 zhujty+O~gBXMEeSZgGEnK2b$Me|%jkOGW69e=hXL_pwd&S(`JbK09;3?0TE+XbvB1 zUUN-ucK4%}a;K|Dp58;HsGwV|uy1~~eZT3x`K!(9i&h`Qkw0bMe9;sYY4JIJt@`G_ z(WSomgY0t*vd?2Pwyka7{1IL5o9DXfxNm+^hkf(h8AVf6>~5c6Eqc}u$J#oJNl`)1 z`r@#DZ$Z!cn81c4de756>k|{(@;&SPghkOS-m}iwgKB#d_BH z`J#U~e@;5+S^s)4%=fITOFrMTeq-2Ov7UAML!)Sv_pGZ_k)Cz+VHfFHe=+R8NY6Uc z{#pNe;>@+zv;O&Tko^)??e(m`8tN+6vyLcWs+kTWP3c)@mOz_TvS`mbx+03|!r4PQ z>g=g!y@pL~UvX3BG}}_xvp(BMVbA&;Bk7*?3yjUQzk;51u0e-!&pP*2LC?C8--yA+Ied~PF^sRG&)3?soMc+DShrV^LO#0Tj zV(D8y&fdpcd7iJizI6iXq#jvPW2V@At5uA~OfecW#g;guF;lG3S+lcNXRDm?*;OBn znJPVlW2P95nPN0%iqV)UMq{QJjhSLUadw-tXPm8c#sD#uMMtVE z!@|VYiI*y)EuC%UY&U0nI2-1Sp<=3UhBLB1i(TpLyUqwA)qa;bd(7D%oIUT1p<=3! zW=)mt881~vG;50Wb4EC+m_>NRWeF!0Bb=0_%7}1MF~Uj3e(db0&Ilcq9igMLd(YWN z&Ilcq-527e%4lzA`#L++**Bb#LQQ2OQ-#_`CykUPRhSOZ{LfT{YuQC!Qibh}ohqrq zag)cMIL(vaqFhR#ZZM}s6w$u zXU)!9ovm_)DpXlip~|8P#ZZM}s6sJRp%|)A3{@zGDilK%ilGX{P=#WsLNQdK7^+YV zRVaok6hjq?p$f%Ng<_~eF;t-#s!$A7D26H&Llugl3dK-`VyHqfRG}EEPz+TlhAI?8 z6^fw>#ZZM}s6sJRp%|)A3{@zGDilK%ilGX{P=#WsLNQdK7^+YVRVaok6hjq?p$f%N zg<_~eF;t-#s!$A7D26H&Llugl3dK-`VkK3$76W%aldAAV`&QFE)&JDC!nN(GF3G}@ zEVSRKQe$Cm;ye^^sj<)}jQbRgh3TH=&r)0A-S!-oDr8AL&0_R4i_u0YMo+UCJbJkYjL*H*=lF>G^;E<%__T*U2S4CDvQzPDMtU37(LBm^fZgn(=0Z{89mKn^fb$k z;pk$7K#D!gZKF;=Yc9^pxoPFpFRY)z?gPd)yUvPN$`UTT^)i0Rc zcVy;}%*f24nZxSmzP8vdw5qMA-B-2s>^l>k$#)NU*EXc4OIV@l5{6~>>1(qk*f{XR zdNn_PX5Xmg!`^jKN29^3Ti*L?W2Wji_p~zaH#0uqAx1Kuy-)i`{dz!C3 zYih$eV?sCdJ&XyzM`Oa9n`V97VB^6*JmOKA)QB)B{8^|C_kMw8wjOd$wXH5~`Dl`j z_&r}VZvBv&6y|rNUIHL{K zSACH!74}eXzi!(eYVPMjR`s9E`cw2!GgxQ4ZFk%=q>ZFEsw`q!QMyfeQt34~tdF$y z!2#UcX%F@O!TM0M-alC1R-}izKG+{@_VvO3s$xCV8pb-=3hWgM+|%VAYA(KX5A|VI z&#s}KWgYfVbGvmdJo3^|Q+MxYuCeti13R(!=<&&mwD zjG;qEpEhCo)X~$XPMk7x^n_^>x7#_fo7ur!gVj&5-KdB3^{|^$d^bCnZ5qL}n`$d_ zW+=BH(Uq=lc3MU1!nUTwHg&V}+qf)j+a%2TaDvxA)VkR>>15Ui^k2sQ7#yD;V4WVo z&WLg@peXuAIBkq5Z)G()sPk_3ldRC9P*C0M({1@XVO_I8O!ta*P3%5BpnaB=pm~Us zqX>oE?k6UCrBegiY0Bi&k(58*b)AaX#AxSW^xuh0u(K1B9*hZF`zE%YlGxfOY)xii zN)3i*2jhcL&%_5zciD4V=-V|dp4d@xmWL5xtBtz#i!a@w&v&$au-ZLgl^!b38Nrec z{e0@%(7lsuRk{k zz5dJWo%;FB%Jup`Yr0BGo+Y*Kip{sZiP63*M*FVV5@&ZfYjoD^tku~nXPg7o_m(r- zcg1Mm)jqazM*FT9?Ypw$1{UK1#rLxPMF|nK2 z^Uhv$_OY{W_I=B4!+6P(t~lG(83|%#ceFEFf5oOao9^s#XIDA9$=Q#b{leL=oIT@g zrL)(a{mt1%rrkLLOGFkgRYtow8|-YPvr*1wI6KAJ70#}9_8VvSID6Vzt20u?>R3n> ztMkH`8nJDi?d@z|XOo;Ed{y=mXO}zszO$R0)tg?_J{sbsk~fnrc{4{Iwe;WW%@m2O zD0wsXqgV1~>?A}{$(t#8GbL}P|Yn^9T38I{GG5yP7i!0(1#G0J7I9us#wKJluRF)_!mEFiLX)&U##E7yIqkm6~C@V2y7sTjE79+|^j3_HH zqO8R3a7I24G4grHj(i?se{=S>Gh**$N9?`q20Gi;*{;rpI3u%%$~HI~L~jobBgqva@N!cNXQyFiBZ4(yW+Q_2_lty??HlY5GW&gl{jJ5q%xXgcqJ(Ngf!n(rX2zA+UYObL zp`KrK7-q(umQVUIZNl{S#x`7F2b2sFV`ouRG)U~oV5Wp91c^0;^>;l;Y)M$(#e&3O z`5>`l1HYuJL1L!{^X}o)5hS)OtiS0&V%p6R3lf9ngT&4XcBdx-!SX?3`-E+`CAKL@ zY?9S~d)T%`2ojrAmYe0|OBN)yMUfz}?W}b1Ah9EZ)luQ|5G1Cv-Gv}A1q$VZ#N?Oc zgTz!(u^=&`Z!PF3?0%N-zDSVRm{6!lkQl1TW>rbVOeKTF6rI&kkXS`1TO>&AkYH5I z4^a@_R*Az%28q>#I*J8}AqAtTSdf^yc{&ObLs%9K65BlfNSk`hT1d=O;9L^@^N?XMt6jNLkngT#2mB!k4# zZV1o&HD=Pw{*obL{00yr#*>*4E*t>XIbcG>xDN>tW1obGaf#5W!cZJS#JJlC5##$_ zVegzGLd3XD2odAtU6)`$h#O6BtiL=|B z{l?in&Q?2n#TgEX>f>9~K5$6H){mEzEJloB!Lq~o5F6p_NM|@CvYYH|nzM7AF*Bj; z7CXD%+1<_>ojv4ixwF4Jd*4|<6X!a<>UgO#+SVDfiEF!nB=O_jsu?!VM9_4bgrNbT-o2SZ5QQRhUx|*B38!k_xfpUi|+axKeU2 zY}%P3abG3(!hXq0?nQ4TQ52Qj3)2WC_rk7(l6%n$$6?}>Y37ml?%imo3o+aaG29C= z+zT<>3o+aaG29C=+zT<>3o+aaG29C=+zT<>3o+aaG29C=+zT<>3o+aaF|rPc;a-U0 zUWnmdh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmd zh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmdh~Zv{;a-U0UWnmdh~Zv{ zeJ)mDza+>3wXxcL7K?#18i;zYvhkQ7jOvl^6p=RQ9*dmO6XU*{ja}?(BVM1XHWN%6LgBti-MIy=ePdCnF%yVlta&VKEz$=Sos9(DGPvk#n+xJt*jRlHOgebw1^&JK0PES{98 zq-QNIdxEo*ot@^4kSgtCu`@!d#FjXF)Y{w?w7OJmj zy!4L@UcWS?;u@Y^5$1Jzr7rtkL-vhf*}eK&0Zm0b!Y$B(%$iu~SPw!2!5+_CX zHNP;c@BL*KJ7sC9-aVyi5%(;~sOqxIGuhL-Us;>IKBUu{_k91tsi2dTj9<~3C}H=S_=>A2n+^#?81 zSoW;vbE}qH>!Tx{YFRz0x#c6$Z=G>OE$O#L{UK+R*!;Vew~zaM%j-{VX~z<^tbVqR zR9nHXXnuay&i9dOtLMOEs;z>Fpbk#u*$NXuJsBo~>T8=C7F|7HvDKRlh3X*B7GVGi zg*wTW3Im|-vWp`f0QG`d4YI0<3jI_8P-NOFl4mOugwsgdQ^xifMb~s10JTrBKGdug zAoJ}a0ZI%y@8*F`1H(H1ltt@CwY~X0 zP6tF0iM4C6>l-404zP7jv|cLruC099p&k2no_C9@V3z$cw^WflTMD7d=hnWeY!ovwj0`nP|K#n*INSl>ES#8gYFXkhXnPbnPfz!$qmD0OToMF^Ej z4Ns(lO6A#_ZUq$o)JX^xM%#p%w@lQO%9VvteK0DRk}5SEgO8>#v&|A8rM+xeUk-JB zF=3U?hQ+s{z^hKCx8kt+g{rDL&3A>}m|!FjO$+0A!#Jl9P8;@QmsSK6O$ zUVE}yaM2)JZ!h1~a^nOuaYqo$MA>QfzMj=XaMJhfo!}&X3*L=`lQuVjB5BW(dJn|r zTWK+R55(v_5L@Dm-UG2lXU)!9ovm`laj8Cf4^+0d9h2B5@ls{9l{0z|WS4P9KZ4ju zXQP~*>WqE_*%73reO%-WK`+MGZ`nQRj1iDxkqKehRm4lmUnRzfZ`m>8TZ|FkVvP6} zW5l-@BfiBL@h!%PZ!t!Ei!tI`j1k{rjQAE~#J3nDzQq{vEyjp%F-ClgG2&Z{5#M5r z_!eWtw-_V7#fCUz#J3nDzGcUVZ!t!Ei!tI`j6MS~MtqCWXCTIiZ!t!Ei!tI`Y^5_s ze2X#STXu~27TYvlQvND2MtsZeNVgl~>>OwFo!#v07H5w*d(7FJ&famhh3PKsw|~4; z8Exb2YtEKCd&1e;XU{E3_L5{DJ7wm?(b$u5Ru z7elg(A=$-{>|#iEF(kVfl3fhRE{0?mL$Zq@*~O6TVn}u|B)b@rT@1-ChGZ8*vWp?v z#gOb`NOmzKyBLyP49PBrWEVrSiy_&?knCbeb}=No7?NEK$u5Ru7elg(A=$-{>|(n( z8|)0pE;}TBY;L_37}EgjqLIeqcdKN z02(m@XvFq%MktLKJ+-p?hBLaf#0aI4-M5@w?5x=tIf-Pq!r9Z#%1r92J|gpFMcCWJsoso%1b}u<2 z6OkC1h*XwLL}Fwj65GugnTW*5L?k;h5s8tBNQ_KGVq_u`BWy;DOhjUY&4`hSNbGKB zWFiuK-q~NAz2S@)ec633UQ#9^u_4Zma=Wq4<~cjx*$F>zFkMQ`LX2kq5`%t3wZB2P!ZEVmo8 zz33Oq925dy9<=SBwZIoUry&zimt$W>bR7G#NgVq!$Er-mzI2cVsMb!QVqfT!R~jIC ziwmP*X4?^_qhOYq)%LcNtL*(#M!^(K1EkJo8aRg99;4{&PNQIk1p&viby%=IrAQQv zqV$P^QIer!i$%d`t|y{k_6|Fk-C-2WRbhM7VH6BkYkN^JN(*#I2ol(Z zBs;jn?XSx*EcrA*jg+!1J-HZ`C1Jh7Vpw4L7?xuLAJf$smUJ4RmQcJY?1mT??Ph(8 zVS(jiSd<2+J`tyqk720^b<9s}QyL&*SZ)v7UJmsW!=m~(wHTI{6VVdw#jw!yRy>Af z+hCQSD~_+SOED}8B+19H92F|c$FQiPVlgaRhEh5e`7}U9Vps-+LPcU&dIX~)F)Rav zQLz}7A;GF&;(WIk!?IqmDi*`CW3Va`!}4MH5{k#L)P%}(9}~l(bFLVcIiZd=|NJ}C z@g%i1&)qe4bm<5dp2b~F$3q0mOJ+|5%Y3Vk2o|2>xd@hv?H69!nk6-di_sh|Msv6r z&EaB8oY5RE*66I+S*x>E&iG8KkLGZdr8!)Tds1vGXEcY4Wt`C*F2=8g%F-MzMsv6r z&EaA+hl|l1F2?v*v3s1YcE^>=^$lwv{s!wHV`HWykneF;uqL zWM_7#b_88WBjWa4dY^re-&HqjPb8xjDM9K<6p&y7!mt?yi^(0 zIIDGbtg{oGEp&FFv!6QqnKMgK74L)Gp4#s#&R%!6&@_{d<-&NWq?sGO^-l{Hw{!X* zu9=JEwkc_5u8{CjlXgin_a^SUCY+LzW_HzA(#%{DCCzLan3m&?Qdkz#%$L|Nzce#T z(#&FLW-&Cg7@AoO%`Apy7Hf3Y?5x$huD{#5yv66 zw=?=0#faljS>iavh~p3=jzf$%4lxp;h!KP>_M$Tqp@{w6+567cHRY+YBtnth=be4Q z8HrG2MtTPg!h>-|IcIP=G5sKKg&Teo_~(nMj{k35}}BZ2t|xUC}Jc+5hJ2Qj6^76M0AKPaYiB(vEMr* z5sDazP*j${WwHM8Qe~8JR_E+EXOo;Qa(0ok+noK}*;CG*bw-9o?e{OvP|IS>1*PLt zNOj-Z1@+bS3r6&+Ur^tDdfJ^;ThG3^e$nIs)%EjM zx7KB=hi6}|&t5&CyIp0OY`rBbd8l^q%G#>m9^9+B>C`Frw7fUAvE{vKjZyOpXZCF* z7m3Z|^xj_^GgZH_Z5`w(ImAv|COf&WvfT3VO=DGKRS?y=8M zF}LNT_aA7UJL}^DYf&dxY1%(EXm5#qaKAd%_50(|~upfC_ zw8MTazn`?!vb(&a!|SWVOCN;e$Lvd}>y~{L`toxJwAxwCAJZ5eQ(qku zpPZAXM^&U}_}b~CPd>?HM@E+3 z$FAR6yWkihmB|jb8_{O_vb!PkNweLMRSW<8VL_Ig5$$HT8GZCoOV6~MPcc5Y&wLk5UpeTk6&`4 zc(x>?e2AKA9niV?^i)j7ujW9rVRak$e8z(29(`K9UJ`pS*4>UR0 zT~p2p<>XTnS4VGKx<0J07eb@7ZOXW*lSkXMX`_#uK6REZD-6FI3@gI7B11ViWG>n* zYA4!Q5hh2-~X47evb~rYqB%`Nft8wq^OcQB`^FK_JEiUSN`$t} zLRK_c5bf5vdC&66f=*3b$a^Ip5+_GIS-9ePVnxyGW&2v1p5@UI(WtV6LozO%>FHBX z89nvbV`og5X&)^2Fj^Rj++4O!bXt^+&W!q$og23Cq3EZGqQUEh4fuDw6HWdQDF@GBc->~;5?ERbePE^>#_PtR*zpuHdun%ono;pkF;S!^V zOYG)&sYi5+vpb#L?d(2h4>{v=s=l|Jani(?K2H1C${D>`Vmmq8&DmII6P%srY=N^Y zoqgBYkDdM086y&PEH62G$JqzYs_h)gZa}d zc9*l?IQxUMr<^frLiI6fLi^n!Ug{C`ceabO!Or$`c7QX+PpH1*oGoy+$Qk`mvb){c z3TIC{d)e7*&faxK4g6VpSGx~X-xuPg%4n#weViTdY_c>P7oqgNccbwhm?1#>p zo!Q+IAB){5aeXVCt#bCVGm?^OzZ=9$l@Y_B#ddJ^b!S7JF$03^m;r&M%IG9#vz*O! zcAm4Foc+kzFPt&fR`osOY^AdgoPFY~hdlvxELHJRWweE}{?2Ng)jB)g*<@$4ot@$A zT4y&nd&1cYXKS3j|1BhMrT?3Wuj{fe25)dtMYl}$543*Nl`pNUx^-pkf(aE>{pZ+`n%$dqgI827 zLLm(s*W8e8t<5g0+h%DdyZqFO%%Tzfhi8v6-|~^nHdVv3BdrZ*_@Jiu*Qp=(@w{c# z!^br)s<@C0G%hvU6zUR!owO945A9p`X4Hx&c5jZvXHf`s*iF>ln-%w_{KRc;fMntp$S}S76lh;`DwiuJTf$RpoxH{~vqr z0cJ&czH!g8Ys3Y!pok=~tcnGVipGv*7j|{gMTmkW79bQwKv5xLK^H_24=5;h<6n&h zqef$lB__I*6|iE9UDtx5(TFXv;&XzllK()W9Pg#8S>LBKy#aWL{eq+)uFZP{Pu_s?m@Ro|> z)~zEX&DGnM^maE%$&!gKH zq129OgKuY7TeqjU3r`Lxg1Kw*>vn1Nz0ffxuG{ytT0E(W>vV--55>9O2W5lcly)gP zeNnqNf*WmN)ah?nrA=&Md|)e8r}Kvq*Xdk&zAR90oT(;5^YzBz<6dc+oR4c+b)VJb zR~g%?T~SR=NwGJME01u2oV0ulW-+ES4uZSz$k-#OftsBjY#elRv3%kLBsxP%)RTse z9Xx9IL{qk1P<{d{1v|C7CO8@@C+g+HfJ+c;+pa`aKhBDG^_LFSn4V&+ce{Lw{V*?*@CRDOAKCHZQ*|~(c|5j$Uz`3-Y=P<)v_3c!dGhNoRPXC!fjor~ z_(Hd!$BTR&oT6A_b^Vz3N-A~s6#vln{^Xdz$)nV%NOrL;l~SkR>0{H|#+pwmb4ojA z!@&PBO`A~WJOjE}Q=Mal63U!~uD6yt#I$u>nFrQHq|7U9x}zN@u6-w)j_dlkGC#=L zj%qVL78k!q+4OF)6-+2|uHZkcQ2bMTNoCI6wz;jyMzPI^tMEsxjc7%lu<1yJ|J|nJ z-w98ua2`{>p27tD}t@<@N!%Dt_$Z;A;rxLBbUR-CVLoW z6?iY3UK;!?*fuyg*ddsUT}G8Xo5kAxg({mr!D>9sMJk(jh+LK33X)XWytKQZ*1Oxz zoma+TsIqx|&QaNlbT<=+f4q0H>U#KhSIDTdc|WDT=KEsmYu=Bji>Y(Tx$e~4ysS`f zzk%W17=DZ4{?O5k8$)X}@r=I;L+;pXFyxWsN6>G_@b4J%G=`dJ;*sM=(0O%u9YbCf zsK-ynknPjE$mR1onbqarKpjYzb4R^|n}lX9qi@%2ql05>@U>XaD2uK4*V(w$#~U&R%dv<*oKsIctY2 zpx74SSchP1XZ@WW?Ccn4qn(}a>_TT(JG;)=0O$v8+rV(F1kHR0e*Rgl_^^IGjTFJ+!SwS3mWP|f|?Q#Ah=V!3u!#X*e}$>;Tjf-Y~SRPKbTjl-H3O@ZLN zykXeFMb|-6PNB|xlC1IaipF72_J!DPESH#`eC7!im$K9bmqE9_u?W+3mr}yMM_4zd zV$rWB)yx+j(lG3~vlmylyt1m}9O%<0Tb^CjIH#KB##~m>^8Vpp8GQfnPZrIe^d(g5 zs)2KcggO?o6=K^OuZiZ?#O7vI?g%XWh{gE#YRfmHA8GmQgF)3T9~|>?%kmd-<}v2} zDn2C>%3dhmE>yg{;8OxW3EA7JHnT5e@6l~#@8^)cb%<0~&HTOT-mY~SS~O;c&XEJ5 zd%vLWomBDdLsY$EQHoz>;aP;t6tQ@9ebrWdu>0POi{F*l^Af+`U@TYs9*0vlX%n7> z%Pl`*G7Q7j7QbmW;WJEHayPYAzXxD-^;d6?Q&h3<7&xjd9yl%>rEau(d}eO6db_8o z-&yV62}Zd3)xtX2Dl)!Ms`}-*o=^Suv6ZMRsDAmoFHQX_8%C9j!y2@FgSEb*_!n9D z3ogTfEuwz;NXmyQ>T`bFgwuS`;NwIP+?b+(k3*@mSTOi;yZeG~eGN&j;eUukm8gJ~ z=VEqRqHuxjq z8gmQIh)r|d4z}qYvD$G}%o=!=;xKgHZl~01%O`B%ZiW2j8bhJ&!vy$Xf%6C|)E#qr47{-fOHa-57 zU$Rm^iLFUoDc@sr*YGv@t4&9{iQUU+DTq#hJW5$kg5pYz)(c<)v8|mQOSl21!v2hz3c49I8v<9Az-F%)!o|JcFu6lh2{2lMhm&>{>oXCv#HK*boP5^PdIzp z*=x?;bhZX0sg|);IEL?1I@`|K9?tf5Hq04~(}zmui`~esPpdvH`#E;~RusX;q}%nEpy2=6^?zu) zelvH!cOm$p?fR=_)6bsy-@~qt3x=JkQ65gb07-T1@7Tt%|2&$>=h$BX$3El!e3pvx zaU$~H#0s&j@K zJ6mNQpHkn7uwL@5NI!0vZQMHc5YOkd{eT$y`9siH@J$J?Yel2{REU#pucSKf|76Sm zUe5bbwyN!Ie^M+zW7F?rwI!CDZF=kSE&*BOyg%Of1u}_e<=}1)cZJ#`c2qew`fWeU?XhIG^+WDBIC#&ig*Nd6ne6 zm(w|N-cPgUBjQ~_lPCpZ-bb)O}yQppj>RnKE-)|xV03I(8u3oTxLx~a{nru zjzso#HeJYhpRt`1Iq&I9jmP(I3wK&1(GzbnxE0h9>^~44rQ}-+YMq0u)bV!A`M#r> zl@LC=uX3VI@19IdE;2k3OEeCI-NlyST?QPz2w8mlh zmV?*g+_xOo`+YWX_{V$cN{lDH_q@OIHNq(v^0mfv4EegF8AJSjn;yb&B!;aR(tFQW z0QBDTJi4Q;c718d?>b z7mmq$FLtjp#`wj!Csp@RXWUM)Po41=Csq-T>HEQAjP`S^(!TjEw!5>TUYFi`m817w zLr)uX2BM_Np^>Zc({4u)kDpt#Az4KAqJ$8{ll9vmwrgJDce23}=kFcgou6Nj9vgSdI30wed=s;XE>k2a&+OV9G?nR_e5uxIJ?3b zUtOpiUtOp!JSic=3%8KH?5x#U8P-U3`4U5Qw{XUn7-DtK8k`Muc9^r_&W?4)507aX z)0{C=su<`ZFbT(woY`Nj#So$l8v){+7i1D#k5wz6vf}C{) z#4YuB9nr6oeD%=r^ZA~F{`$nr3AyGo&(kt*i3a$d;&WU9O7P92j6C!7Sgo#Sp7)v( zJ@dTe{{O}^Prn>r?%*^{*yiuB8Kv0fN9Nk*d3(xhoBw%en{P$|%I6^sXD=ETm(6SA zGAr4<9>ye_H^f-3Z2mV+@}z9O7NvGTvr{o#ZP}b+oj=GlD0feO*}M%-*C41#Sm%3L zEuLG&WpZzOVm%aX^ky@`pfvNqwe1!JH@kUYfmM=qo(G~-natluQYKTY^I3`AXQ?9@ zn(t}T{IAS5h?j?=l9)lYO>k6-g#NR}_c|2)?#B1iSt;n;?zCViL`GzMKh^kFr5fK) zw&LqT&qRXxY-1b6qVSPm9&Yn~Zbyq@f1Y^(`p&0@{rg$P4z{=Xq#HNB*W2{K*a9Qt z`=4yylP08SeCG*ByLx%>WJ=ioT-&c{#`hUfBM$3*R2Y&z1Od)ah~ z@%<4ilxBSA(=UD+u^@U}1mBWa5#9si3jJ=IPBXsqGLmk5|Jq6wGQKO?9qrKL6Kg_q zK@h}U4++uB11n=i@o+xNQ3%kt9$iur;ry)ey|XQ~t?^y%tyJT?+z^rR{b$x(TjTpU zu1?#(P~`F_SdF_;ByxFu&lS1NAW4yH0uDTv20*D%~4681;fA9#>&;`vV7J9jH> z?>vr-*k6DlBlbLr8L{VaW5k}<21e|8O`sG$1Vgqz8bdChSIMj-o`8pSNn(yk5{pg4 zT8PaH$0UixxQ|til31+O*;~%IUsR5gSmk)yh^-fn$@VTrAA`#M(#!ECD8>_AWt`-U@p>_ScU$Eybau6~ z>zpli#t-(X?wiiuamKFaYL8vdRjw)=>k#l=m>A!BsoY-920LSibCo;S*-U4bIlIc) z1I`vWV{Bf_c*j|LoHb${!!fupoKfm(xw|-Hyk2ZyXE@tKHryGduF6qrX&DopO?7sz zvzwj4*AmuclwNfirB{2Dx?+^NVwAdKl)7TJlv-Wk`)I$3@ujk*zzRpY7|JUv7 z>b;$CurI@ge-{V)O}P&C6{udDY2#o&E_AT-#Bm3E)mCS8AD7R;&L>H!QTo!PM)~xZ zt46oN`I_{yH{r_3J1_5ktF1=UyzEB?g?n=AX$Wf)L|*piI2jBvFFVi2;&L?UW&e4a z9Bpd%TyU(*Q7uefb{>>cP>%9PA3ZxFZ`jxg<3|i0JM`qyu>?w= z;vg20qjZe&H%s?o+=)KPrf)96iGB_%249u^E@+n{Nv9ZJRjLzxk`-r&I+CP{(|%f( z&xx-1?lvw-dHQ_^Cwf~+N`I)F=-aoEq}?!YA=?iwGD%7&db3Uc4U+Vt6iLb_BR;i8 zvBP{$bVCL}3?>yvFwA26Q))7ohpZ-o>6fmp>n zGL?^k)Utd@)HZV(BP~3|v-adMv1vNd``C0LC;CvE8#&Qw8x3$BU~816=HZ3k)X&s3>(YY*YI<6d_p%1j_G$;Bs zRw~Vj{)m+-d7tE!p4XX0_{YCmsKi1(^ouc{>a`g|s#jirs4jVzp%0zAj6QVk zCi>8~!mtm9+ha%{I)7wv);95soQENQCiJ265L4Ciszg=G^P8%chm@+8mlvv9-Z-dg z8Th4|AG#QQ=wkOdqYqu|b!W?+F$k@?^r5TVI#?gE zuHl$`=we$tJIKo&;*9%1<>*6Kd*?ah1x4(BXY9@`#t^RRzU}N|Xa91>j0!4O9gbBZ zLrIp=Kce>N=U2I7oSowAbY~YiyVTk5oZaM%pZigJ?BcCutZ??evkGWQm1`f4;ia9k zZqD|0*4r75e^~ceXS1BmarRqh*E(D1Y_T&YQq#IHk($=|b7xGCCAJC9E|uFn97Dj< z8P4{wTz_W=I~(k5n6pXFCOezuY>urfP&O^{4^Zv5NzaZ zYiHXz+rt@M>Z&`$*>GpmoSo+Az(opG_`+z!Ed;aFwB zaIM(h&W1W0;cSAlQ=OgTY=*Ooon7v1PiPD+qb3|HK^^l4_}{oy}s9ys#&+-*J)<+zQd3OUfJJ~ zB))xLWbW#lISZN51|ySKS7s5fxaHq3S~~SxDzc8_Y$A1J<=(OPtcn)w1Ez zD7zYOgjm;X4tQjLza^XT{PkF}^IO_;IY-{YOz+KsCqBQF(=Wk>jZE;!*#5NA#b4Tf zku57MoK4HVR7v1xAcfk5nx|ylM%wpV!nE&SAMtuTvw1ep{4GlRZd;j~_Wc&m@1EUd zT*tag9p|K2pD{$~)osntCUeho>D|y_T?@ADj&E}(ZF84sb7Ohl^KLluv~wq`7@Nv& zb4_BKJK0i`+1#wLr(^|dD~##*sC#It=9_7?Vgg?eSp1HXvNa<2e}nf{5BEfzRVu-=?&*P zX1(EA;a|AN{O(qd4=0K@PigB7=hO2j6wu$E_Q*RL)AAf2SAJJ;sd>feE9NsLS3KTb zpAW2-Bp#uaY%qH(E64v7%;rTR`PN|oSe=zq>T`zF-nZ$C3ciJCk20n8sGnjjRoWU+ z7u8^z&x_cu^c?T9n=00&1m0sOj2S;_+?Y|wmmZ~6z7g#P!D(e11&0NbgOib={-{!O zybov1SdRDCZTd!Rc)VNoTkSdrWYJp+In;j@GlEBf^INQq5}f8&S;3BWM0oVFfpV&a z{cQTQ*a8$N=M%pA$3J2#IR<9~Wex)6?aG5k+QnW7B>FW|!tseOfcA|_<$Ei!4en4S zIm&mi=Bn-7J`v3`7ZBgaNC;+6(~V+_OTMLGt!k^4lFR*XC>S8ESi!*gNhrWaTXww8 z|3*TI7NR)s9BVQj6^;kWPq3X3nY7Qc>7T_`Ao0e5D?k;H?igpXOuR9-w@OhD`PbSV zSJtn9V_dcYeTsm)tSnra=r|_(y7N86Lssg~b~vKk?61TQM>0zsYdwa4NPQ67iSfSg z)E3zv4A(?1^!RQ`1jo5sZna9)MYF=OTz5NuhuR89x!8{^{KkPlQ&xN${}7@t+d6o{ zuA4m9^(x`0vO|NH3-yW5W`)yb_gqA$&PGPK(Lsl@6WtwtyfuAeS^uCWxHA|S>|IuG z?(l24#I}Fon}=4+$$j&{dqnP=2VTUJZyxx1x6PXeUbP*DfpcDg+XT*+fMx%9FXa-t z;vd7v{Nt+!-cHc5q=}b&NW7*CG2|-;UW^z>=f#At9(a-As|U70k2u$CDqamN#e$b$ zxFwD$z2Q4z_zZ>@U^o@Sc^J;XZ~=yV<-kkM-!bH`oUa^sjQPre*ABjN;5DAF9HwK) z_2GRr`^sTAlOjE_WR59-E;h~Cyl_kbbd|f;8Mj=F`$l!?DHr3miGAvfJ6o(G9FwP9 zjGl7U-NDQ4?5y5dUuXQ0s4jn8s(Z1s%bnfe>}F^8IpgI|bw70WiL-L-Yb|5VaI7-; zxwD?m4t92^GX~XFmqB$cgF$sM2Gzy>;OtIkyuhg3!_MAt_O`PQIHoGc@VeUD(iuCM zi1l%XGcjZbJ3G|bBxjSI{m$7<&K`BP%o#o9S{HiCwOqcC5L@YN9h`AuUBj`;U{`0o zoE_rDm*XbhGUh%y3W>jwzIR{ zoz*+*>uj_$T4mXCWx(*dmT{J|Yn|QT>_KM>oju{~X=k50``py_?rfB^vCif?yUf`g&hBzX|G1V*|G1W$`bE0gxJ(qy$sUixmz0woV!U8Rc82Jg zs@-2gEWbkec0l@<%`Z<4DG3`5s;us+FeF=nB~fIFid3|`G8!K*izQ*V6$x{(+oGzoHCdzM<=G{5xzZN2 z0J-rlWNy5rTn=7>gzJ@EOo@5daY*onrbyDcs+kvhxZb{y#+!dUTptSQ0ij>Km8VtV zH0zylA(f2Y2xGZ2`U9NMNg2&|$2+3Q@ffbQj85^27fZ7q1?TlyGb+oMuwK+^RIUuh z6c5fT%{pasAFJ1x7M#DdU28DOgY&IcX%S|+Q zxv^F0iRL!Naiu;hkY+vmP`n*>8=k#_-$1hzPOna_)*0o>t@K(h)NupAvoQb4OLdr! zv*KO6iE2zY8CyRVU5|wFVK#4oJ?_(C&hvG(Qma!q?`w_rk1bHbxf}Rmn|GUs^B39l zva<5vwj$wtMynZ2qCAfTS5i2$loHF0is_XC4)dSdmPQV9K1|ap9W|d6%6xd_gJBT- zPfUU)!ujJ+;A<rbeI0l@I@TzoV4vh<{^3RtqV~dmyDP zB`*X;6Yn9EnAeA#KYvPHUKS{ITQQ_Z{AΫrE(p)@@T?Xw=QPoHD@=a|2HMK&7F zF-cRgY2n<;fYMa#R%iD*qcm0BR%dTHE!M-?FTLC#XA_*A z>Wsf!)#Yzi%U}##j6W|izFiS}&KWN=V*IJ7+-BjJQkskX(%B#{H^SK{XLFsg8?);2 zRf3km7f51@oGo$ojI-yRed7#Q_pt6y!m$p)hRzt6&@%WULd&Rk*4Nny&Q5lAv9rsa zJ>YDCGbT6JGM;kwsWZmFwcMZJOj5ZG!!f+8ch=h(C8x?!a;h$qn~RNeM#(8g$*FQz zIJ?>zGq9=LLTAgJz3YsUQ{^Z*)gC3M7$v6|g{IhE&IUUh=4_I)$V&r+lON%M5XV*&ovvB{tu@h zzZAdG6uIar$OoeA|C=euv+2j7*pNHirq7r{JYvqc+t#Mi=t=arugmsF-aJ9u> znl=0Tr5Ud^_4he8zqtBKTC+DyQ-3ES&3Io|fA3%cj0RC^@^yKs>aP>ZCWEBBk@m6n zwkoLpx}i*I>hDTxDOdg7VC>O!_1B8Uz#(2p{qgaRA(5e0>=*V_$eaJZHvO|AN|3sd zeIb=*{Nu9wf;U~tago>XKlH_2?KI=d^P0PdTJuqw@%?ZDe+P4Su{7h=dDWo8sN-D? zF0&Q58qW*ki>N_9I^Smda_yLgNUA{|0h%otDvKWE^Qb|Jjpwc4Ms}!Jk^aKsYLH^{ zZYwo3COQ&o@Dvn~rC-Uoqi>eaD!UEIxFfZaFn3dWNo#Hz(|qydT#dGDDMYTz0Gl2Z zD;-yeQ*CbKQkG(jyOI+Mk!qXgY=C6wJS(Fh>A7V$fW@?1;#1h%ebRy3$5tk8-sVfF ztZL+j9%0kHV{Il?BR7!I8G7AgOHPE>d0xm7U&yRI+uDeNxKg6sV#gt&8o7GEwMrd} zsz#QRPrNa9vL#ewRvlJbgTE}K4!OIzDO3~mg{*;Z3uZ4VdZDCC%0$c`Yh@PNA&Sz3 zUkG#dnr6=C4KB}L7$RMXS$yRmnX`FE$PJMOSVA&H%1gONL$l`WPH`B9NO?WZ36Wa< ztY*OMAMcf1jb!KILiTKK5C!Ej4C66U-f3vh=1z;J3GapJ>mV4}(S~+xo&mIDUyC7Q zqr8qVHp(;hc?@|FU%-$T1ja^rsiy#C1djrgm&B|9Wmiwxu{kC?w%By6jTr6NVzgt6 z(T*)fJGR*C&Xzl)9b4sS$5y#@us&j4!!g;h#c0P?xr4miA*J!NqWng^X#z#SU@yD`!p4raHUP8FOo?F4KZ* z8BaT7T5z#9ovnd=BDPjIrnKN<+d1R3809I)DuZFpm=;`Yva@rX(QvK07dyM%*`J(I zl&ai9XKy)M;p|gq_+^C4rQuq2Hx0)sgFT(qIBRq^z}ZR8e&g&iXID9+6xA~R;_M-3 z4?FvZvk#rEi_4zst{;wJ{hZY}V^%Dc~v?-on@R|>g-BqG*_#=x^OH# zXV&ERoACS3lg8Ve)=$!Dt?1S+=yzOWPHV(8lR3QMt!8$t&`e#4Gw%DCslTL|I_smR zApzdxN!6i`nrra`_^4aa%n$CPUV=ukuK&e8>MqdB<~1L9X_uB~M?*S)@NY9r--49v zG<9cv+AoAIZpXPkYO^vwoi#eUk6Mmqn#ZgQUVAD67+n5sak2d`#EYW0v!Jz|dKDAGH} z=9OTHo?->_TcY>3c}K(+5Lu${vU!i&O41UomHfCJKAK&`5>1VH6K*5?UNtwDCRv>Y zag%IP+f#0Jv|3xlj#nOSPaUt%M0BEOW=lt=j^zmHj4k)Zv%O~<{!30=?TWL-{6s?CH!e65gG|2m$p!XiDO zfJOQRTh(Y~*o8B|cb+4~p49a`j$hbv;)eKSG??eeAFQoHmS|@0<$K1g!XDCQHzagD zcY}P>or>yumRrM){I9HWTB3EnlwgUz%hoy968$Y#G$_duy}wn@Z;9@2^V(XXo2>aZ z|Al(L88fQ!C>rT~Ug2}~K81Bs@AFdJM(?-AVW{_c{mj+-CivNA_K)|%m6%BO$L8&T zrf6QWDYPeINTELkLq?8y`cd%nGBXlG9tE1BU%+rG1p4o>M`(&}k3&yW^tu?*6g>{Z zNf^$+a0-U(%lj~fJoSfR!m&zx0VB)!<5k@w zo$&`OHpUr`q}Wtv=Q_K_+4atrI^%Uxb@@_&V;us%6cFRF7vs?v<1?NZ&jhhv&h~M} z)Y~f8-`R=ICOEso8SV9IkFN(*_YcnS5-VhkW2+o((kl0sGrlMg7Gv~UjL~Z`Mz6&fy%uBiT8z8qiD#z%x*zwLza#js}pmN>AvGf-JsY$crp6AkYkI~rdF9c|uONmsY z6ZKSFhgePq=eej8Myz>B^Q>gw&*aRzgwkh!CzJCXsAiGq|NSOsz69q>3Y>gOP4;35 zKIaQO;%z=>-beEIoco48=Vr{#n-`l8g;KX`VNB|l*X>+&dkfCvq`KwRcn36l42G+% zZd1I@#qwfDUguu+tVY4N80*FSN(?~r6fLf1l^2_8wzt*WBQ4rJwcRp2Iq<1-6dnwD z0tbjCabY|RrK(rfORCpFSP8x|KvuPaYV~={Evi;c{otsS;8a#uy;7aZpRi5DRkPv~ z*Ysebv5uuWl~1DS>_ek(h&bSm>~TdiE!d;qPtM)!&>FmfvY z$>!bVv1wjs7<(@d?kW-bQ=G~zRw>P?{DqY&lpMpUogi{mF}I8~w?vro&Kw@`{+FKbZ|o7becp zw;NVs*b)E65a6#Cs*d+Bs!g+PVaV%0)!w-w=bBS}@}fZX$%WIW{62>CDfh(wph~2c zVLe`*vMTY9_Qcy0B{?RQC^pU6yl_k^QRTRARE{c9tkv0D&bU1)N0q2@ov}V*>xEM&7=H+=OP{hBtrlXm*@(UFY`L>4>|7#o07x>^Q6TE^&6hvj?3$=j9>a?d$?$ra15EPoKgL$9Mzx7QT>Tg z{fSZiiBbKD;SEX17%&$5owJ*q&3D$~>`7--f2#X$XJ0wn5RbK-+acH_9IFg^IorqC z0nYk6VZ*0NEtk%l09P`@#UGhRNbOC0T z;}xhL7ho02*fcG)9M8sVw;UJD3(YGvJ1!^QGKubWq%EoA$_DK5{NZTecsvr~%5GaiE9+P+M)4lCePf^}H7 z&&WFbt~Fmwo>3zzVb<`%^6V58lDkvclcT)Qvtb>+J617|by(uBkac)ZTf@jYTw~LP ztixPec2dm7)+FIltivO$k}_IHd7m?|4nGhJ2PQk5Qte8Z?HG%9#;wEqS-VjljUgk1(Xtp(5l3F{=pg zw5Q;nSTe_C92T4IjK*QH+nmujEJow7>b~x5xwDU)(KxJf>tKCUw`({ii#$!s| z&VJ>L+E8^TI-Bb3TxT~syUp2?&YpGlrL$GeHo-Huma%y_hL2l2+ud2cv%b!lKuL9( zK#60O!Pm|L!~?}jR+yf1xi+(#%oBYxgL?MM9MyiCANL*h+N=um;#J)~vpcWijl-I2GmDWnxfWeN8;31fG#QD4 zr@&=;>hX{ zjr5e3Lk)iBOfs~uFI8drQ-askuFz+H9KC$b+F|w|KOpPZ;pAa<+B$R zHT>Y!qiV8pw(qQp4LtwT8A+};}kz7gJ4U=lyt{(9ho18t?+>-Up$d7`1`O}aH<(w z2Vh!%eS#{aC*hNhWeMivk79?}0=wDM*l1MP&Cs-Vmj!>eg)`+aSDTCE!hZ-sTVw4_ zEGlSA>rLBaEwy*S$(GMCwtKtinYMr=uCaaE`TOJx=7Y&XTh0RaDc)(*UzBwY$RcyB z@&ylyiOEs6_q(C|&Q@nyY@XtPC!qa5+q}1}{S$B$)P6hoTHh`rG%01S#J?ceG$weG zK2~ZgJ{I8vN475G{U}e5X@NX(LF(9F+I~|+lp=!9NYpPprN((fyT=xvFQXB=a|OY* zR*U9aon-CHqOPflxZ#PY@H1OfTmvRVC<|R`Ef@NRE?oxMPM|EUD5qp5ciuWSf9;qW zi}z>U&DxHlj*_&|S9f~b{9nXWa>578MarMrA-02(f|R?N0g7B%7#F1MAs63_B{m(c z$m2G>b8Hz&4=m4tL#@;Wu~JD7EO*JqwkBO-yCg19&#*S4T_V96t;sZ-zg}#YBs{U) zB?nuf;&DfoQ-10L?W{S`&zhP-U0SnQeygqdH}Ryi9}*nZ2EPqzJj?HvHt!$s3;zNy zHFphe!mskbGWije=sX_v@Q?Q|KEteoe^-T!F~uD*PUUw#hW#+)g@ekI7ZfT_{?d@O zrs?k(?hB>3Eq1_g4Ethu9ELm=RGK$oNTtaWkxG;Mo=TIycPdR@KdCgSh}rHr7;?G1 zS7jAvCp>G)smd|MAjPI(>0;a_v0I(p>x?l-m0RNMQD@wK)nyD)<#3Y=8J((Pn>gFt z*$&Qjc2@6J7Rq0C3ds3+nmwergCpO`^4G5ol%9V995{=JHT0g zXGb_2>}<5N6P%sxY`U}maYj9=W!&$Ku}ZN=oIT-;u4SzseakBMGw40Bjl;3ZfC+uY zws$tr*;#5Dn|FJ7`wTOv74(H^DK!`ql&GAvrw#S zI93^K>TC;VJ38CN*#Kt)oiPiO+8gd{oU`%H*x^;>E_ZgnGj@38+)Dnw!g8-TV-_aW z{lM85&c1QBF|?w}Z5EDI22A%W#&o}027Rnz$2dFL*(uJ>b~fD^zcr+~w>n$q>=|dR z&fapiCZ54nw?jBq8Eoc^-d3&OGtQoO_T80fWB3u1zM1E#Nufs%&Z|djs~%WTpXpRv z{;;`HTfX=Z9a!6{(z;$xnlOC~KUoUX`pkQ^nQ<;mYco$#nbws*1ZDbe-Hfr2GmqEy zT3I#gEL7+_tfeutqH5zBce3`&oZP-XGrwQv^qzIwJk&RHay44mX5HFMbHB{6ZheP+ zQGZVJsTH-eYBrw#`5Lun&##_UF`(-9xu#jotGcA`u;zwgPa$5|t?vAe$gW${_`r~c z@?{O^luLmCNme&|2z1=iMe`@D8u%dkN3x`;%0=^sAKv&^%!C%SiWG2lvumQ#H7Z>} z`KQvD2@z_gtF3gkl}4jbi_oK5N;TBx*y^Yj3TYV~A3&vw+1Vv@c`#e1<6mRSvL59N zNJ|Dr7w$i+V*9MVG-=tVBlctV+e_I4&}#{g)Xd8%F4wKg;82v`s*cLvF322EL1hWn z>&dE_hm|HQSK<&#Skm^CD=hcNDVh|PlW>XTP54(BuC}mDF{aiQ5tej-@g|4cwmmRK zLi6%)y3E!wO<3|)xEGrVHctr4z0!o`ohB@4Qsq+!o-4}Akx=D_5R!aKQBvMMKY)-t z!`7}tj*y&e?CY{rA*p_CYm_P^e`W2i>&vRfv_xW;m=KgmJPunq)XHxVn{s`wf9?$K(IvJI|u!`|uXFd_g2PK%} zV%i{&=zGZu6-jH$2ec_xsF3L6>%q8MVz4CsP1z@H`o}Sa6Bm8ECTjPx={0Rzxhp05 zwu@~=9?^HNRf-!w^NGF{Rw^C~jElaDY;GjAq;xy`ip;b5Jz{b$pXfWxN)3;d$|w5x zG)ky&1IuRt$E=qpHLp$0P>DreSTwCotUarTb0e>m#| z>7=?0I;w89v+mCJb+*4Vc3oB76P(e8DmK>{Wtg*h6Gn`T6X}Om>qsSAZ$kTH1I|Y;13_!y!>vz4U-hitW?8??=x%kHC+fEOacyQn zJqgZN*jZcar* zW~S5@6r+Xa7QH!vpZ1)BIN?1*7F~Ph35_#`w0!pEf<==jtr~dO5K}#qp}zh+gvyB9 z*pR7j-==Dfs@6nRneFz)MOL*YrkKW7GsfC>+0|;GI%+_w%o;FN*0=njtdz>vM3;Nm zLdO}|2D;o8tw0c4fi%hH5x{-@Xz^I#NY(aw!F1Yi3j${91AI7U1x})G7b5~#{xFUr zV}Co9Ceb#<1xpfbdyM5uv?Fj5C*7)i7sgxY85pj%L`!q477PCEkMmOwsroS3hqbc5 zOYC8a!9T8KnnsI)f4@p-v;x7uE@*N$ThK&Xke=WO6b$TwJLerX{~2hur9YTvtFbl* z{{Wh;$y&-6{QGHo@b4G4PfONp!);kn$HQt&OSA0~(`CuP-Z8Ox(rn!8>fpE3G+S~n z7uRgFak2zKb?nJKUtmvSri4JA0(aGR_^H{pv3-)J+4!^r%UVqBJ!v2j>Sp|0^}y$JjW^mh2zzEtDeb;9s%09IvkNzI_WY zO{umNLrSk@7*cvI#}E&^P3vHPY>8>^GfFj{(v)gE)mfgWG|Tf4QK}t|;YSz_!}?H) z@ivl`VvK`Gig8R%P%%0|#cmGAIs|lriqQ$Ga&&@svOS|F`mg{mpHq^+3%gvTd#7h&e(5Q>{DmF)QD|}ea12L z9Sg^lK}L+dE>&)bmt!?&vToZaf|DQC>ArS?8|_O-K3Aa%5i&BL+E zfV~98m~l*X*-KFD2xsgsD8~MRDtE238=S$N9G0UeROJ|NQ(eZ}#9nj8w6bE~I4i?m z6k9tS>tG^7Y(r<8c)8xr5Ty?59_x%>JyG3h&dzgoiL)!5-RSK1&K`BP%-P4z{^hI_ z#1WSU(?&Q}8B{y#?u?F6m7`-+>o>^R;m(*rM&+16M&+(|#weRuv$I9c#^HHZb;pNe z-y@TY88?x_^;Xj_F?EJCF7-(8*(Qyv#2LMrKgQXY;?4Xx?#*0+UlXD~?#<-uoZ`+* zUitpdnF*(7WIO#{&dd$5AGzys>L3UinlpLZv1po0vjcNNXmr>1f;sbW(`Arobs3s7 z=XU2SH#28`95-in4Fy^==H^ZCQian+(rkTkhTt}t_VQ7mdcp-ykZx#yqh&@id`N zbnI#k3~zU8aIY<#=}WaT6}E65rBZtwE<@RTJr0#3^^k%Z?RnHojf!n%d!%H0<*#kK z+S?B1{rm=-UT7+bJ-O!E^uJ-RB#Z2o#~9x!rnDjzwv){p9-9~0D=)Qq?^ye^SE~JQ zpmpC#u~(jswM(~GK4hgN;%TqcF)BfoF>8GgME1&`*>q&DR6}X@%6o0`*W2!-y;8MO z?3G_xr4)PRDORXZr0rdsD|tg5@PSQ7_R5cKx{$q+(TJ=hVeJiUI`R_oagp6qBa>(z z)h0!iX0Ke+cIyT)`52G1U7uJHMb@Gf;gcloyOCbcr`i}y3Gg+=s8l}H_N|pFWUo9j zu_i-oI$D#FHl1Rx{Dl=tkF<5Lo$^aNYiO_3S(9q7#H%9w<+oR^Y4h6JE61@)+rR%* z#+Iaryw7As+`0BZ+7nCSnBr+-)0|P5 ziZPz1a=br@F`g#I+lCn9X=2=du}_`xP>V61rgED&V?0f42WO0@iPbyn>x{p0wKu`p z#m@MyNp){_cAK*ooxSRe_HMPuS5T_^6K9ljVm+K~>#WvUy|bg74Rv;^vq{coIGg3{ zPtNXf_JXrloW1Mp183_%IB7e&hGP)R&bDy2qqAL{(PpW-M>?a;Qf!Pfb_En;S3r(c z26sARau~5?&Yp3`)S#;SmNQ1+#Av-#-5;Y>F-jIOrj8M#^-^p*XFED$r$CisR8Hj> zffHk=K(R}mUFnReV^r=IXZJW`R8Dnivs5`o<-{0)6JxqiF{X|Y+uYd>&USWI>x|Y* z)g9^VIA`ZOyU^Ly&aQLT;%uq2*PShQHXF~+TEDsB*!K{H^_gz4OE%(JpHFzSP1cq_ zQrGJRXu{sbH6gnj$QJpJTwCPRSyx`d7TJUfvPF*c^qu`OC&?6fnoN-=!4&y)Xo~!G z5mV$ctZb?&k{3Nlp}PP5rpO#sm*b1%&Bs(-F<;~*)OGHQ7K`!G+iskay4E1JerPv5Z*)g!f>@UU5Yodj|n2~l(w3#0c+bC|AOFUd)kS- zk*9~#WwwrKnvQoeNwOa&G~KRgnr^OXI=c3xJsyJgxY6QltE=G-!b(ZQ70wz{l7{2U zQRd(%@w<;hSvrU!a@Cj`Nfdk9gcMik_$QIUDsv zoaSu2$~J9cOve`J?)I(~jGT=x+jLr#Y?hUp9#blLqGXBD@NCqK5r+Xb-8FV#^F+xMx=Pnuk6L35#iC?%HtORI z`JIhFvL@O(8^>|A+y4D0^%XB`*+1R~R$`(n{!xYSuNL|ytFKPLbRm7kAlW^b&!E@C z7*bcUGaLPkpJPZrBX=2tWIV$eB;%RQ@;xwQc^)C^ut^wFhw=2J4&xD_4&$$!I*h-5 z>ac38FZCDiTUq@@=a+(L9FzVMn}(%}%?rnrU{j2bA}Yr%7vsJWqy7@({t=`85~KbS zqy7@3{t~195~KbSqy7>b>THCwbDYg^M%AUd^fRjN!_N4qC-#Li_KOgs>QY?>qs6GT z#Hh8zsIbKPIveEdaA#*ZJKNa}&Te-0xHE>%w2W7sz2U3^%0$a;AC6T9J)CXp>_BG+ zIiqq?-EqzsL=$5WP0O9>Y_>CYT2nc?9aWB>To9w%QS4!7bUTWD=*0)3-JgVG zmBBBZ?dq(Lv;CY=SE(*_m6kzWB}QE(_CL;k=j?CJ<~w`L*^|zwtJEHKmD+0uwIcST za17s=bhe$dUpiyxOm)XPJJHz&c)nD7)#2Fp&^*~d*j0G~VSS2=o-&+Og`J4CRktsI zi7jhHZ{P?B$u21Tx9now#)8@*X9*gqkZzH{*FPz1kjO$5Kz{6T_W2V*5Aocga` z@T_)x-CoFUGwhU_VlR}Q&2KN1wry)K z9LJX1{{5$f6V@ado8rBJ+8`}9#a}*SQ@kQ%V^eYM^Z+=u(9;+`g>uvuG#GNPQ9JRZ zrgq}VPVK~#n&o+rsGau1klN@-tPiyjZv$CvbRM4iq>VTxgP|A=hGH}riqT*wMuVZ) z-<;84C`N;!%6;UF217C2iNbPggkv%os_qufcJy+)INR45Z+L2tKV;RVAwldCXEYd! z{oWbF!eTEud(BxT_K(`5g-Yc%a<-|nJ)G_BjGrb@T`FhQ9qEi8DG;0OY_7Aro&Ckx zYtG(u_Mx*+obe-cS}s31tYz%sj7cZO20I((Y?8Cd&KPr2dyF}$F4IqnUF+GpaI%64?JKxzI&hB#d zxU;96z3S`@Xa98eu`~Ai(lYpoI*#eX1!DYgf!MFT93xF)XF8kWtS6o`Rd=Uw?ECfW zLXhZlItTf##qvEq((8rQ%=h?j{74u-%6w9vndo-3j75*?%NIjSeS~y?RU0GWhG_o7l6*ZRbvDwl+UW;^()6oCw%ZF8yUm1U4(Y5H%bs2kQy-wOVr)}Oy zYsmV8)lC}`RcAJiusSI6YgqL#$zv?-q%F3(<(1JdTF0^{Dt#emD)Z2~4g5wg}X}7jPCK?4t zp@RNyz?b5FYYSr#g{#TO)E^=N#~>`D#@hQKI&)33mLdhjmZuooEWIM1N`VCkO$s()aNaOp&RF@Q47x0~JUOdaA6xs1;Umd=lu=bDl z_CJ6kd!mRU8;_a?;vX|wMggyU9loBnQWx0vM424VOjl@p(1=?N*~g)6CdatTEngCk zMYi~>ZShei$4_Gq`gs&wg)J(@YRE@1K1N0{qxcugrO+yExm0jpTItGI>39;4{1K*g zVr!YlO2}UPX8QMDO!+#1@hwl`OpS`jRrzZE#oAmic1)AjK|XgXei#pU<+Bb-8y2z- zN`yvUOohkdo1901%?LNS*rx-yx4YOzCo?hbnWo4#u;yPxhp*nj09@3rKoB_F;cGLS zw;XfU430ymuf;f&|Fuavnt9B_Ki+RB$kxNZyFykALBx9yg;55>ei-t?LO~Y)np-v# zBZb*(nEw!lt1yI_s);+RD+CsW7#>1RcpZZwe-{s9$YVtz##4+zES`uAV)588h{fMC zgIK&iQmBo^kn6*HPFAS#`&<%g98(ZWY#P==j6p0hnh3@Cf?13~EHMVL#CVqz<9-qQ z)ER}Q7=u_Uw~4dOoiTK!a=SYl>THCwbDVLHs67U;RF^?4G5(yzc*zuFQVy|qobl~~ z826~k^>Vh4vjd#r^tnE`uv#_Td_fHOoA6EbK>31oQULt+c6_XGYT%6 z51}`gN^PlSLcD-V4f1Tv;f?oDsl8yC5bv&0xtgdPGbV=RK*NlQcUP-ib+%l^1xT6r z5~SM*X5|RWW3JlmcsGkz%w_59?2h-awXl}u+;}u^2L7mH7c79M=LoYp3Nxif{42E? zEPx}B8u5C0Jv}ud7*ivj8Y;78*c%O#7d2r}JGl1f+M{ZZ9DMXq3tzyIks{-J^ISz% ziSsV0$PU7FYJ0S?D~7AB$Wm;7dz-vz#?sOi*+8uQVw7YCYDKroY#otmJSU*U&k2_B4fj&P(gopxb2UR1Nhhvbih=z{XLJVQoC-oqHMD3 z7Nb}4A{Ms&@poNXZv9hHl8**i1yqBnGl_y{%2GRjy=R+Sa{TL1Yd1F~;z(ofw=oVL zho+CVd2v&r)E4)$)Yeo}ThzSqy&UPNe8xcqj5d!wnJ3j2Ma1uI(>6_4S07rzjqQA4 z#W!raSp18U?@B9ma!i-xQBRUwDe*5#+rsMUZkrp41Idy?>PgZeD!r%Gj!TI=>gh%+ zlx7Tk(n=LdpCRL4wBVUm`lqpDlvGdL)dyLrx>%`v>WQAO!s#-G*xYEzif6?)Igfgh zU{5#xJ!_5cUQ8`9_NVyO#n3~W1bgF`I5Oz$DGe3Bx(w4BLNRR!#nj!@QWH0-?Ozzb zYQ~&uT%h}x@hc#mg3g_nfaHEZjo19-_tSW78jpYd%f<{Joc(@UYaE8(PvZq9{{1wC zubN21Ki&&gVtj44)bysU40aR_)s7g|ju_RB z7}bs#*H&x?XVg7nR68n1wIfEgBSy6&MzteGwIfEgBlfg2svR+^9hIZn5u@4>quLRp z+7YA15j)t~XlExlo9-;*>{4e}I-BpT#o0T~{^6_>^n#Y#B^;{^n1WEOyR&_rF$JO4 znJEZW?gVEv<%!L8MpK>`J3y%}O?hHWK`8dJGj@Oyd*2x}d&oNDOj2DM^;C|gJh5KR z201(28C8tRQN^e(Rg4%_jMyKY{n^NW$i=ZMrat7uAqQe7Jk7CSpygJ zL)%JCXrs(8s8)xt)kxS_oCu3+yr(9VATL9>K!QAb%#y+qBrB#`vBR=Gk_yU^Kjp}g z&rzi|DM!YkNaL)lx?h8t{uS%ms;y3M?VEWsE=(v#BuqZRSgtVn3a4w*XEyP=I$VsXWk$#3n6k;;U3xF9&vl?hi8N!YCwo{IUBOi-EBvKT&=MBy<2 z)B3ATQzp%>Oq#96pP@`>ETF_U=AAM=cq{&*!RVL>cTbZs* z7-{|vU7Dmca+S$gTUPrvaUd#_5jL+=Y+j^H7TUa7u1uP3T6?*;GSTB=JZ6woCX@~% zZPN}(S0-Ou!Kh138W1E~%CfwwEtx&o$ zkxDI?Y@L@B?&cb6ZcX&xQL?`MZ2J6QGdx~5LRT*C+_A*2xD{eVKG7DEaM`sgS` znlWVfhQDL_=~^+Q5cwyD6e8`gz^^dP6LvVJd6q53kVo?w4EZZ&n1(+jhH1E~8K&XS zk6{}AEGb-mf%W2YdAZLDmjm(WCgH*{`RT;yrxT-}PK|AFvoiSmb>R#n+fwPC4t#J0fv(AtMs!JE0)^8JM zn>*Xj*#XYRI2-4Tjybh=v9sHq{mB`_FDl3IiHqT)@^n6mNSN5RF2^nt>3oJn2|wjpfd(t z#D+UN)>$W9fz)1?a4bDsa`yYZxkwV+Z$gJ zwXwZz69!kBF*9$@Jf0X*F#)b7>NiBMN3NdvInLN*poO9R9nkCu3|CXnq~^>kmR}*V zzNwoKt#33B@Tn>rXyH1hbsM7fO)-szgq+EfUm@4}Hl^Ji!8LAu(~bX!&XKoQK{>-K zL28(UuilV{qSX8fgIOB+6}SWWcK=$NzTE@`jcc=Q`VAP@$Rgv~aN`@rL`h^^+sWqb z9-9{#*DkVoPg(mkuBrX^%F2VMQjBZcVeQh5Yk#s*7uxnjIr2&{u5D=hEy|H6H(4@A zUYc?3dRu(-31XchDaJKEh~P_wv4`zE0%vP0RLHo-=Nmr5WuK*A45uTTTU_8AY;&Wh z{X=a!9*0N>96p}kZiUiv;H|V$g}N4HY;I&+q3dYb=@LF4eg9HCH&uZ(J+4Cfgd<##x;<|AhjG zr(3Q7;tf4l05yXo1rTp3u644Zm}PMo3LsvB;sS{Jr8_waH+-NO~n=D4H&c@WbNVfZSBcm>|XU4^H|rnRtJhGV!fhP18mS3%nvj}&cd zJi-2sAx}f?aeoqgf#8)s`_|EMmNk?Jz{ofxl~Vmmn7*;zklzjQXr8I6Og%iMQrZ<4d$IlIZ( zQfH4jW9~cEeZ^TjC=an8g=4UQIb-fSEtk3PRBj7rG_Z-`EDg)iz$V7rcd9$d*<@$e zIlIwWv$I9cmN|RI**~3q>}-`Yn%cD7&BCz`>{S=CzRnmfQ91lJY^*~t+Sw`2PIvZO zXV*Gg=xnhw{C2|jmOI1mC}f{G>xw5_?W>=LV|Z`tthciRogL(Cg0oYd&2e_Ivl=|3 zY8iFmSSk7{M_Tn$N;>{ZW6~-mAsw|6rBxlm12MCidKA$L6?)C6pR$3T;iwY^dU)xG z1$wf1>0~!Mts2c(Ub@0|vt~35SLmZx_~vdFMt1>lonEmF%CxQ!rnEX8`KyZ%1o~ z&xbUc^NToD=%>I(gR&rWY48z~w_E+q_5crW>3HdhC;1r`;NjsYHNex1vdI7sUxv#M z_oXS1{+RBGa{4Q?4dM?tRS`XAD_gR0OyAOtx(!uh8B$5lwlR)%!903Qg5a~*yeK>) zwKK!i4vk~+eH1BEg)lUjA&2?VV;E`-+VX#w|Pw*YEzXMLS9 z)0^rt)0^s^<&2r$#BOkQvomITQ(b0yQ#odO6MMs11)j*o+J|Gx^d`2gvje@{LC(fF zWAsJKV5T?KWu`YVc773K=NB<%dK06mOpN&o#Q3=+vCo`+;cQ);T`ISJIEKz8&Zr_( zmnuT#s3OEpaCWk@SnstDCx;Orr1?>YOYv(9+FQr-2!vC4qa7coX()ZWX^ zTAek_II?@Ux*3h#s~ctv?B1yXW9gHB7 zmM^W(RLo@`7P}hc^LPzeG#Os4nX+TqsM)a=%^wbzm-(^sxV#45T|=7}eOoF>e^$)l zgEXz7@imTz9;7JKN*C-uEqX9)evXtJq8Re9o>LtyD^2gfGDvIOJ0l z4<4n%KWw^~beMn=Cs?UJ7nTm=P=L<`j8#SUuSDR3g=qVr%8K6W$DLixHa7}A)4Z&I)Z5+n>oE`)uktE44{%!Et97ZwK7cYRpCA-itTjpB})~=)5s;H12=hhYjUHa zpJZh@-W2(Ap+L?w22Z(f*0W+KqR+`N@0l;vb|fp1OR0qf8 z`4Xe&ON^c`F?znl==l=+n=^X8#OV1_xsROD^CgB0d{~ZpNNiJQTR5XoR(te(sa$Vo z`#R$}uW}4ns~q1Ti1FQl*zcYF!5R0J%JH7Aa?GJ7#vE#5%%LW>sWYZF5n~QDm17Py zv0pi34mC06P*b_t&gME}4mFkgi?i39F^8J!e(3BIXLNt5E_0}<9CN6N?d@!^Gv-iJ zImRwjjQQrU^_eGS_+m7oG4n?G zBg_qEDbV_LVt4VN~6n#N(Pm>8`Spk6}ST9Vn_Vp1jQnWOi zcYj9Q>3@>=2Av~T%r^fPLiY15hm{O6xdwN|QIKKS6nL`&7G z!Or^LU@KY0lzfd;%*?5e^l7nr=mazR87$+qmgOdd_gm$fMIrWr;W>(4Fb(Jc zbMi{-1yf(ORi9;vt}sXAG?EY+jj>$$bI_V?110Qx$A|sDVYnLdhdI-B`1wwI+rt&# z7?uSzDewVke{7Wg8f}`UFy4gdS7FG5=W4OQNq?(FHPqi~&4sGjyEET7#{1^eGS*=8 zuS%$%hO{0qy-npDhX#6E12?pLAed*8i=SD=Qt&4W8IMn?0Tce1k^z%JVEfoIt}Cc| zc(Ez1M~w7Gq-BPnWRcMZs7MymlCNgPaWSK>E?PP5`l)^-XO^SVth zw#A&{iz#o{IUtKXZ)aJ#V`FkH^1RjCys5Eyl5)qR{iki-d)7WZZ`zD1_!QnN;(6nP za|`}4DQ_hB@=3X_wzsF-ZlmW-C)EYTqe6U&I>sulZ51i)#@lp{a5~%2wAkYFC2RW_ z1%u!gtHoq(TKvZ`QI&`q@lfw?iz?!IW1;!ha-pcvj@E<@BX<@%7~`detp~TR)~4fK zg5pw;8ata_o3$laqwf^)*^(EpNSo%7hU?fOiba*UTltU}1RE8QhX1q$=^$`LzOw0P zMcP^IxbZOQkz=Ut0;@GWcBqn_fY|nd)>a{poMKSX4*g?dP42bnct@-}(lKL&ibtJz zZ27!Q4Hbnhdk;Qs#7QR%9(Tg1$Z`@p7 z5s@S#SR>dkI1It$>yT41n;A|4)^~6<+!gWteS&$I#~wy%p7nD6LjBnZE#`)-xO%xE zt7?#B$coqYHX*B?aTtcIb_0uttRz1t#@2=R)Rma-h<{^3&Y%_Vv*@MX)Eh&}zU#wj zZV+WCuR@fe{CU!9N4be^3Qa>Wq+DbN70#cBA?x#=ot2HuuO->YF$Jr{7_1VzFdXY( zudc-SdRpZetP*3eN{qoOu@%nVclNn6n*UVpCs(5xB{tpJoz5sKRc?VZb}15LcuRHv?(98h|8({vY@y0khGY6vtk@RLcJy-m zRIHXklbp&8aW=)-G-nq&yTlpgrRq{%s=Y^?G2|uolC#&Gl|jg7oUyZ#*feM7IlI-_9nKzh_PDbboxSSpJ!k)PR)yzut@FC! zSY@!QvtG`II2-P4oU`%Hu5|W4&hBw`zq4bZL$us6;aE0rL;-nz2QDv#w5p5I{$p|$X zD9I5fdR4IW^+gI)Ja#(U!no2<~4DJ%~j34^Qh)CQHpo-ff%l~YEH2iHsXP~t}~Bsx=QBb zlvHw$yefGB=IgJ)9>NdCv{ZAKwl+e>2G&RE-d1Cec2@^OO%?Mtt-s#Z$ohC*X*JNn zR+UGiRF%xLJE@Wn0+UMKr=UvaZ~ptK8>l%spJC^wfoz2Bu)>p>1|?Lkw+!}%?hQd zWW7cxq>}k89+&Gk+qA-(JV7zH#3#Y8(*HkP$lzrQjG_)$k<6Sm#dN~Lz60**Y-9lnHO<~ zp-Scjx~)oX0?hvL9?I27T00A=WbQAjWL|}+f?F|!?amT$P$jR4CDPi-oyFiRbvjiu zUoufG^In}*%kM&yNG)?rYFTU=S{0iYj;SM=7}c`MQ7wx-@9bq~RLd$ywXAY$VSU8b z4#z43YGkoZoNeoD2WN*nJIdK4XFQ*^3~FSxcd@g@&K_|_jjVDEvZ^jS;EA;l$8bGx zMzySRe2J%W%n~R@wJb)pEJn2~Mzt(wZo**ZANs>{x0s>{x0VpPjwFa?BcfU}dF{l*#9vg%STt1g4FVhqNLQ7wy6EsIet zi%~6$F%^dx4WeRH%VJc^Vp}-d+Svil`a2utY^*b?Wwl4OtoEpu#i*9WZgO^uGpc2k zqgqxus%0^%WwH02t#q~y^rp(O>zT?i*Pz%=&Zw5fsFqcZYFUhGS?o8?PIGpZvum9F z#n}VSsFu|p)w0^7S{9>P7W>v&IWCf7TZUsDf^D1)a5m7{5NA}&s+)0kfwP;O-Qw)9 zHM8q-WH^>p#_bzsjDvl4Vz)u=ot^xCGcGRB#Y>P&@m_|l=ixtmZg|!h*lo{V-rA7) ztBK>r%)6Au^_kbLSKXI&RSzE8sUfppH!8w_acdLC3lec_gsT_SXFAoEKg<23#c%N)(S=I4eW{-ctroK7iO-+1r#KM+uMn4LL z{sD};$Gp+R{BGPfR!gd3|}F;RSwl|B@u53s%@oBF(Vv#3^sQ4wo4t%0 zZ2_Cw0{ARso$>H2MBT+l@~vz>|5Lv>J~JhSZ$Gf!R&MPdK=?}J<|a(M##&#NE_^@3 zs?Zmm6~25>3xa*qgzpe5rtt8oXyR>~uCZs{Q}N8Z%%(4b@Fk0c?+D`^N)^7!QopXP z-9#+rb*o-)i;1Gp=R^3CMZ))7E4QMJ@MY}3$>yz$%~N=|3%J6=$6EXRy0SLo6bRpA zQ-rTXQCwK%3lBex%Z4ZU7_V%U4g)iSSeqa?Pe5tYc9xLkvKFJ%3XYa@-e5K4z zvc*&QYVk#~(KBTFge@w?8GNl3+A$VajXQ%IY;L5}q(Xmc>%pi$og(-cdaQOlJpBK# z_Z|RJlv($84Gf9{E8wDH9(>Wov|_@5VMb;UbWl`4F#(HU77Qo`6qFz~C?>>=u8}oh zPOA5Xkvn{TWnU3UQ5=7)pk++b z<`cf-J*sOgDk*%2S;Pw-RcH_*Z?i-j^g@r0^6gW7KK`BMNe`dxLB)kHHeG|qb@Pvj zn=IUG&Wr7|dD6w$n$5i7a6gu;)*9R9_@O8FA39{n34@VU#%)J@)XpBZnTg*2@nv9t z@8XQ}!)=;(3-)%wTaTU^|NYwpFHFNTV(WwRB(~8X?cenwuQpwZb0@OGIH5tO-Wtv? zwAXl67;T||U{%Vspe6i@hfXPZ*a0 zV=|YqimfBrYT*=%QIf%!%s5z&$#7LkhT{VnjtpdHh@B;dqaWiQ5Su3kdnMcAYKU<- zIg*vxHX~~mPE`cRA(L$?ww>6HV%^2+#0H2RDt5luMPk>8{Z;G^vAe~d5PMP#wq>p# zdfuyHu$>qRS#tgQiowcEHbl&<{2@C@>{PLf#ioi~FLtxoJz@`t z+3zo0zki6q-pqAw+AO==b;79@L3=R_xnvxMTymXv5~~&)BzBnC1hGkCH;MgC49^?m zGVr`HF5?NYC&gf^X511nl+q+?7fzX>T&$y5PqDqlkV9tM5n_0f8QDa!sbW`(?P!{W zaaG||Rx|$#+2mVYFc&g=Lo)`f@CmcGsp0yBW`DSPuJda&|X(r*wAXn_8BfHSS0-XFay4bWhDV?_=Lq z+?UOjO!Z|Se<_h)!qT`a4d%s3wwdoS{3-|g)__5P!WXs^JvHqFsd#(6*g+BT9HYVZ|L^JPEpi@)7lMH%N5U-rhns1#o|g?FKh z^D5p&WFub7=Odli-shJsV6&KnkSs+rn zL2!}(dZI_%mo@2btVqTgn{pff0Qr5{JNSEzecAl+`!eJI`F9C$==im1=i97~9$?q@)X|t?o|$Z% z^+E<$HW>!=kRi`ZhCDObJh7+5J`_WqnQ{2m+4g5KxU$KRac0~`Vw;KWBvvg3=MUQ+ zBQ{A4dpi5WA%}4|_LAWMNA|K9ewAeK&M@u;3#2yuE5PMhbSFt9hN!j1Z;gn@T#MTvC zUkn*%v}h4@7dukySg~`&CW=iJLl&CLy;%&NZL+7uP=N>ljX@=;O(D*eL9_#^J?%*6r#tAG0(ot=3s_Y&BYuQ7#~k z0~g14+Ap7lTJyW9tya?ySvNI~I?%+wi<=rJfD+u)3oY!w+D)BTyB0N6-;$F`{%uxG z)4%)k;F~T$qIqa2TCs0Pw6^MadWT{81|h#@Ul~O!4)zqSAKO%}XkB8vXi~JoDg@Dr z8r91zT1zrknZptp?{#BzVi8v z!})ASX*OtF+XYq!7rj7~)?+qEu|d!EMk%6{8QVgl^ltAWN?zXY^N}cJcDRrz-6YX^ zbDxj2U}v9Cu|Z$uL206ts=bgX9h+E@2|gd$pfC3MG#hkRk4m#aFYu^BqO`BS8*R`7 zd_LMF1ARV4l7o?Z_oy=!9ZT|Le^0WYw-6WX?7tp}QvN!xGaK~I#Y8EDFK%hu zql7FY+}H-as=wFR28|O|p1)9(;^3cagI;7L*9P5SBx!@jky(!C?9(`K2o?-&&^Wxt zZO{;+HhVjxPfwIE7y41(yax7X?R`i#9olzw2ZFhOHe!35pTULjH; zp&(9iUd@Wrf#q3oiYbayvT@c6+0Ee;P0(Zyh(Vl^VY9LA8)Dc<{5o zMF4S1264)`9%4Pka3RPz+@WCHXfgcm$nFxmPwa6q9H|)hk=UnVKZyMz264*eLY%TK z?vRmfAl6H)x7Yx&L&YFY*%soI{hcF*LEdE7i$Ri-)r&!rk}VW_Q4H2+yxSu9RjktF z9a)EPsv_tlwzXJ4v4h2q5IaT;;*|YCoN`^R5&Nsy9b$Kj;kg=YyGZN@v0ucH#%0`E z;Z%h`SA%R5F^E$#h*L~e1bxK%i47KmMVfICr)&#xN_M%})nfHxv&G&Ndt2-?u`k6S zPPq(-Q!b-JIMpKPD7L%Uo??fK9W6FO>~t}RQ!WGIl*_nH>`t+l#9kHqRjkPh*|sZ( zQzo9pwinw;tdAHZDVJMrQj_b_JeW}&&XuH}lNuI2sx(i^V9 z{cQzt8DCs+(&Y@Xw~xn{H6F@~YxyHAfPZ`V0^nNS!@&wAyOyyBCsihX9=m!!OI%*< zW%Hf5Ap0-f&iL_0SB3=!+b;4Ij6BB+y^~bW@u40@)dSD*T%Ygi7ntxI&+z#vrZT}I zm3fTw_e)ioqdc4{6B*W-KHuFJ12N0RTwp2_EK-?cJ?@#9lBCK+wx_$l_h#%ps!Vu} znfg4+`-kV4{hw+#2aZZfeNutMRZKq5@pM~0eiqm@pfb6&OYj^o^oHEt$J_Kje122! z2%ckhl;%063>xmOpfcGi#dCazFDmX|%`@0`v!{v4>*0G1JjdK?O7I-Nhm{CYJ;$qg|GA#ywVXws zV@h*+j;DFIjsElRQkS+O5STM!D}?jh5}W^{{kuNo`CP~6*?al2p4h?=n4j7!T*uf# za2>B}uMnKrvEEnPPLq7K>qbU>sb>Z2OJaDz=Vf__T~$Uu;9M zZelntFz!e({4mJiI%a=oiQ!O2hMyPX9uu1-_P*H1V&97WD7J!aNA_0{PF47|4cYo) zJr##XK61GSiwzVzM(jATv0@X%t`)mM40Xx4+?irei#;dyo!C!e>zMRo+xFpc_{!SHx|Cnr=82ra%GsK<~dr9nDu^+|S+TO~x?ZPRRQYJ$w zWlU8BD5Xrco!CBN@FX)1|79FJ$z;ci!IMmOsn``_kBH3_dtK};v44tvF7}fcJjq-J z{Ks6E4Z^94U^lTn#0HBUAvRnLkBek~W5wV}CYvsHyV&z$FN?h`_P$sPySC^0p_&!@ zLynZJvsgE=-NpKf9U%6zm>G$}{)(24#=*8ww)CFRO}yBK(W*Apk2avTN2Z%e*5_(l zJ+|lXQk@E8Hjz)Q&iq)LvFgz^R<#LaKk~hSRbno{psh~n$FLAdFcgi1Fkv0GAin9Ez2iI3op+ux86oN=aiE! zKVOYVnUi>J)mJxvy~4m4!(uD)8wHn}xiwS4Yi=qB8eW>RyZ zYd%Qdj@}mkDLl}(uf;JjdrZN^EbiQwR^e)@FEOeEU1jrEIa@uw4m8Fi#RvN0dI+cPXy%&k@7yP~Zz5i&S?_3kz_ZJb}!>rdE?H{gQqXck1 z(M{v+fxg)wy1BJ)h}|2>Q{f8Pj;;M`ibQv{&qt!09o5AapKqY=y}ptU_~IeD*=lxd zh-{+56*SlCzNjLpT?Ad>J@@h-FvoPp2l|$I7yKIWX>f^}`#hZZz+U`h9MGM19xe$ zE|jX}Et_b`=XsxN$3AjGG~*-VAC5PxnI9$G=BFgTb;SJ1KXf!um*4tXv7N3!suqu+ zwDH&eYl!MPKUuc5u@wLCcHZX(>%5b{_nbXfsAJH_j)T_)fBNsJ zScP*W&QfW~+8ZqfS{esIBx`Y0L9!Mb-kh4Fa5og18ryem)56$TNY>)c0<`gPTMo1_ z&c#`6+`+EMsf{tkWG&e^>x~RfVlp_1$sQ0xvX*R~7@WjpABrJaONRBtREyweF(hlr z)(NLr-^bywxQUTit8+f`p%3SAcl%nWaGpx6q_Q3idBq*e;8900V-CJ!9Prf z`p#r;iIrJ2S+j78^_|I973-|H9mEb4Lw#p11L;_{9VrH{Fc~UVF%A{0$o?jFzu3cK zZ-_OBAsx%MD}+-OL8TZ{v|I)%Rxu70tH}C_9Uz8uEaOIrT`qRD7&I^Apn2KfvtloZ zeJA#l*ovl6*%s=TZ99o=EmkE~BX*S7P_gl18L{n5BXAiQU(02LI@l^vwX;f8e2h}8 z8r2LtNaQ4ORfwuCbERvVD06FRTNA%~wC<7V8dae(>DTz_NqA^dZKip3)A<}*YlWKU z%TLWwuh@)GuP-E0yEDVo?)+90FUwV}R)Xq;D%*}l zo?>+Htat4DpmtqiQ@Pr8g*@8za64*lZyOa#cbTN%f2eDriA&R`C=Y^*h^#=XvbigPf~V38QGIQTiC198^%n$siatHa!>CUq;mG2v z5RT9J`(w*mdqLJnI1Y5aVyXIcxQ9oDsNVPao&1U!NntMLT)SEZi)8Q^k85l_#ud`( z{@y2flfu-egS~&~Q}%y?>8rsh`jiSIO`k6Ct=&)hlv{gJOez%6r=NJkNT2@T^Ba3d zQ6Z}1Vj?N2REArNOT5*2-U|AZi@zZzgYxLpXS`97q%btlXb-9`pidw2ccUEUqdwm* z79H28_?gD_DemUL)m>DdKJGy+V;dl!KII73Li+UD#DZ~kY^U59@r~jznWj%sf)>Xy9I>K8Qe$H~T>*W%k~dt_e+|&5 z{56a!+b&qMm_Ehrnk~KQS$0&a47N6HdU()A6_u$^M+a-zce=GzT0Y!1F`hd#1FPBi zFHH6}*gKVW`4m-DhNQ~15@UZ(T8SaDIFIcW*QaO;U5f3xez}>2?G<_ycTLa^=hv(r-Jm?HM=?b` zN;b}VA;WUWZWV(bCBun{af`&>5W~h}9P}vTuwG<;2&ZTsCW9Vj-1dsYS4q}WY;Q5_ z#cYdRm2o&ul1&hMSPU6j#yupvOnli#zBvgL64F_kCLq>hB2;WRbn+_L&T00 zJ4p-!CFkVrtPg{>xNSm!7gH5 z#SRvOjg@UtOPOtt6U&I5FLs64HDWLiv+ZoL_r*RID>s$LxaQ%M8M(yTiR~b^v)BN! zL&eSzJ4@_5u?xlS7DEjve%{x`-V*z#*ym!X{KR!e+SiA3nz=q1?9uSp{Xc#>r(!IDDrt5V&^ zS)rNjqS6*J21^PS#)M*&h&)d6uUkyHR4!hpP%8Y9to+V%gzn%iEaFR;_^aoInp;i655aufo1ff zGNY*Mh&X^+psTNi7_HcK9{d9JU>ABu`iZ0{#Mjz_iV#ze~Fd(g0%H#|M9 zYVj5xwu2=;JNZVz2k7jK^H3T``TX|(k)jg*;Sv5N@;^A|;Y)34|BeoMDVh=o8N2Qu zmDQ9`Y&Y0EwvS!EkAg;nwrpknLtEmUn$?zg$_aCwn4%AuY`pbKc1t+bBDhWLL9s`~ zUKRU?*hgY;HM748TOTfC<#4LP?jmFv+|G>KL2PHSJ;nAB!!E=A@B?Gpabnn8$>5V_ zT)o&ru@}W~Lyd92h`~cmhJmTH-w)lqG7WWUxjuyLA>RDtUbcn57gP4T6dNygk=UhT4~YF; ztU>HuG1L;~GQJjD!>+^0)(NL70^~Kxkk{n8^cLG+>`<}8#V{O|ZCi#@(v~$7x-+NQ zcfVLxbFv*G3j2F;_s*1M0e>%2d#D%Ot%NbDU)j$(Y2n4eGp~g=>)^e`8s$287g`f@ z@Or)AR_l$iIh1Q!Zs(o-&d6*n;H9_o!tjIbYT|PG9lhhFRpIEx?aY{FrK7iV zs9CX#$ zrS9#;bg_JC(%aiJC10BK_U@RLFHL%T2Q)b^Ts3?%$s$WxNlO>)Tau<7@rnG8(sYSU zCv&HL?W5D%+rOZ<7e|fM;jpYY8V!eS@CBzja}V*bsOZu>pT9x!I`_E{iw1_b2L21rhI-?5>cSQy!?#!L-^KlWJPn=RXra5yfd@B}m=AM{XGR0|%Gxsmv zDBYR6nlHMCf9uh6ic4_jeud2uoany>2zLG&UN5UQmg3C)ldoHTXYQu{USnr2PEdLN z{$0adzqEB36O1l!KR>c(apk(8$7qSb(sTKizxnvKBArDH1Lp9qzCHA2h@}P{v z>4R}Ui>+bnNVZNmMPDx2W@3F5*H7#Sv17z0iCrLet=J7>&x^e*_O=*CuX3HCRk<$E zs${6}ONR1PWE+WXCWgTdjH?wx5hbz{#YTyZ5t}S_vDnRGsPN0>&K8>|)*$w-*b=d? z#gP5vx}d@@`$Ha-40%wpUSg>5OLmGFiYSqx!Y|vR!Y`TapCP+S40%w-ArHzpICRNS z;g<{*e#ubbmuwB&Tgg!2mkdRe$hwH3!Y|n#VuQs{M2T%tM2T^5?UJF064}*a$b*v2 z7JF0dZ7~#4VjPMnu|MQN$&d#nLmre2c~G+b#SRif5hccwPHQQz83plEX8k&qjF)tEe;&6Oq-)nlGRJHw5OS9 zVYsd7;-*t5)2^;eZNy`dC)rH4A92!B+hDy*nU-d$tul@C|HMyQZ|m|O?x$@mAk!?h ze6nj)Eh&$s7T>?g##~EnUio)cQ!w4n*eKW(9Me*La78 z&9x2oeo`RrYb#Ri?cR1}lrQIrA8)Eae8oD7XDk
WN10Ukzy2<0}L&HP#t%IzMX zZ)}Y{()r?2GnOMgoB|Pv(fK|fS!21F#@5)gJZ^q0b4h`SiXEd4liYwGaMiUD4I&YL_jim%EBoId@7EI}v zW{o}ATcufJ@A9a)CO*|3Q;M$-`PiO`CGYL?Db`qAg_mNDh2IdnJr)+#J1W5%ixgK7 ztRsQQU&E-fZHyHWi1Ud+doJT}Z-)$r9Wwl)$R>#0C-#uo zT(JdWpNf4U_KO&9y>S_khFnG)u?@r^4H<`&CFAxNJ5&s5OvXVPGH!$z>g$q08Zr)o zkPL#541$mhf{^S*u|;C2ugf@lDp?rUA)Kyo`8_KO&XY~tODV6AYfMX;XOCSse5^$_bR)=%tUu`|TZ61!du zk4fcn9~YY|_Pm&t5)0$(_Z5DQpT*WS!Nt$EemGST>?l?xRwvd=>^L!$DdM_ZCU%wB zEn=vp%QzG#;&M@(i0o}K6elA4gB|o3*Cw2@n=)b$gp6Ax_J-KMEdF44#X-WwU%R;Y zdoo|8+~V4Ar2R_cm2zQeZLrQ|X^mIR&01QwvPQDBW|NR{Lu*>K+|muL^|mfBss5)J zTGz4l$;lLkp4P%;ao_bkVI=foWpQuK%@39&^zN{2Poa0eO;PCKDk4|t-D$g7(%rhP z9mpW`Rqlk!smfNg3b2%Kl?=?ys3Bl z{K=N11B-HWM>v0`RHb*ihe!2?Ds6tWv-=vUI9T7P;^xKPi_&~laeMnJBF)ED9AIg_ zy;IVBr};KY^QK3yq9AWsx8HvO39U?+MwJ6 z8?uHQC5;R2bnjnciTfAcu1z39$B3EtHA@ZH$J;R4DIIdgnHZM^?nZ|Z8! zB5&#*K2LAzWbd}ofBs#n&Q=7%ZbWQ_aL!s{^DXV)(IKxkL5A}fgj|iiLZF=+&SU4v zC+y$@g*z2n2<}v5(;)D^v{$spIWa5nkjtRJ!xY`AWaF(DvRlF_x>Ly>6njK*uZsOc z3?h^LVXfGEoiU&Wf()+1XvoU&p7VjGIV&BVAqV*SL95IaU}tk?vxYsGF5yHo65 zvDd}k60`G+uR)8TSvX}*Pq9CX?I5-@HWZsD_LLX}UOT!@d)dgw?#=t;I8Skj zQOJ4v_YT|H&#TtT(q!88$b8S|aL#^wWV;outj=DoyS!@orP``|^>pxtk7Pt)ojjFv z26!1AmQLf0;0C;Y^#Cu z<%HouR`AtkzOL=k&}tHH4Z)*YiRwC4-3QI>o>>BWYR}Am?J~1AS+Ug}V>;k^rDtZh zPCYYwb?iClO{*|lotay`$?WR0=T^=>Yp1H-msNG2iefsMFA>|yPqJy@Nc17Bkgm2+M?$|}!(`d!qMmAJ*aG-ppOi}ciTa&}ePUY2d&b}HJH zPv3n~`MM|;)Y`TTdnzy3lda91_l}xltMROjuYB3YS5k()_h#(?`^cVrv=X+R=h=7P zJhNNNn#^H2Wqj{p-+%l`BF|U$Jqv@jZ*`Vm&`>um7S5MFCp=FQp7^ zFV8k&12@h#b~n1C{X=C_+%wTQ!I&C-KLlK ziTOIanBKXKq$X}}BLHbG{&n+9UR*nNb8tT$<48|N%dmhbiz*f3ul?#SCQjw5{s3pFLph&sEF=Hb^vz>?JyD3Vo7@~qMG|=zsT1U=}YduBV)Hd z@{AE3W^qHjVODor(NmxA=pA8*gqRzDiPU$+2E71&x}QiHqSWG z9=^DE0yAN;#g#Pf58x3e`L#Hmt$dCtcCB*nlKFSEZs+q`#9GI7_&MJ9is9!Cf>V6H zmCvK~`98m4>{BFkIf8x*8$}-Ee7YPL-GQa{qK|P(?3ekx_YtiKZ-}g}6?rvWk!Tr7 zO^(gPgywm%uPZq|6j42WO$u3Z={ktSyCM41;Z>U4&lix>Ezh@;M}{!=3CjnF(HST z#`)MvijKl*2x9nJd+lbg5Ysj|Z&ZW5LR8}zf@~=^(HIlY*i{$UEA|u{tTqanLCfd( zBVQn{C)x54f1|ykU!3!^BKy0hS&@w?W>Lw;*;2@E4yTw!B|{dKamb>Q%@cb{44;#6 zUyEVA$;!j2ilDvN24Y)?Z6j7I)~yh<#ioi~FLtvSWHi>GMew{Btg&Q}(U__T z%1sQCK}M5pCbosxPGZ$!$ceJ;-eM<+oh){?*m$v9#b$^-C-#yU9@xTVd?ogqSW}a! zT<2EdREyw`Vw;I=A-0nkq&3_26oa%TJ6;UZnry1rm16gaJtQ_)Y=IbZr|6|cfO;_O z?`JW)vJDvoHRB+t8CNBSA)sWv#4s3?Y=9UBgOZIEn;>?Z7*b=5dqixe*kUm}RFZL% zO~+uWMR0LAl^s%AMBLZgVJ9o@bL!;ca1PZy!Ybi>VTEsdWEMf)TUng#%)VRw-3dEF z<~Ma2Uz1swdE&C_?|w8V@4l_g)@yn`S;NS01fX)!gxc~}kJ`J(&YS)AalyRV?+mvA zk@jv+D*nf9xsdgiHb#;cXBsN}wpO^v9KLT`(cD*RTHWz}&4j_hnmse*lbeyzH%Gnbv6XW^{q&eB_D~tM zsbrh-Z1-)aguSyH>6;?~!?@x_|W{)dyD})c=r!XSKA0GDSH?VC0JOeQXy_T7a*z zsLs|YOf<_Z%2O=Bd)x26rVX}Ceo?-+y@Ro(xE_l{d6l=qev(gwK3}2H|X8u!C-)k?ns#L<_ljP>*y;(5sr(sqzK1K9%Fot_jR|+y?mm)(qfXL zd|zWbd%Il=igFw)N)zR{Rx%Z4m#q|>18nX#pkPU?G*SUkKF|ZBXJl?^iw!(yT-_#8 zw6~Y-6U!r4TEq~9*)~tjZ)5mRk%!J?MLEBM3r)MPW>?i6gX2v`vs8s@`9Q(SF*SXd7CF@4 z+c;gWALN0p8p(BhQkvEGjV*w3{cwBl9dCDu%k{%el{EPLBPQ1`DI(W#K|0O9yR?VH z;_}n(F%*~QkUFdSH@Aj=aws5ZT( zJt$u1C|;ro*Us5zk>bO@?(@vvV(mZj`PE{IH!i&o^mikjiQ8n^lsH=d#pjDWmj)~N zvIiA=*ef<0?+I)iTamc*#vOFr{lVb`A7i@DxAu9g$V{J)r`?m%8>i{BJnG=sSCf?9 z==&#+DwO_4jxfF^XZU=yK_~hA>aj02A-%Cla9 z&CGRN|Kpqr5sFjI>~KXPwsD??xURBSh-;jNAdc?~d4s(s#rAM}zojkFD$I;p#a?aX z*{F@}b&9=iX0K!HeQYSqW2eRYIKoV`SA2sI=Qu7voMTU1WUtuxui7hq;EU}QhiHiQ z0rraZ!L>kEtV236BaSJGb+U1`wq!T~k=-f=u}%iD&bS7#cg3)s7zeS=xHh)FWUGc# z6#*nV*+yd9h+zw`EleDY8z_c-p9}{m#$6%?u}*f27{odmj+ShT0}dI)IvI{*WDx6Q z5bItqn?WDx6Q5bItqn?WFy5O*2y5&83(aW_L|t6VxNh9DYlM%du-c2oT>=$ z#8t9w#CnSDErxo&j6*$NE*H<2A)6w0yBI1mG44?@RAeGUrkib%=_Xs*PTgdygi{p( z#;lTI%qrtBW|a(MR>_VLJ5KCEG5ckNamZVVnkWZQ~i94i^du`=!qG1T)VgY;(HG_hO7W{CY=tX>SZWwwQF znf*1lLnB$saH=AJ^d{R_3{O`j+d&M^StY}BRxwo(pvEs5YW$L2BzCFT{bCP`9d3Gr zZI2G8N)XpL&}PMTO{}S6_X-8CT zS^?5}mWyoY>sF8THF^C*F?nqbCe#MS6!sijE*19Y+1z<`<`pE*Ynsk7%kxJy6OJi| zjIPPNj>oD@I@KEV7&NDQ(*$>UHv6>6=*4U|APO813TeDMbIw1GT4?Irg!C^5K4Jf!u`0sz?P05Zs_bd2 zR!$gqjZ9np@}}LvXjGeN^Xg1Z2YX=6K^<~Z-PqDy7MZ55$@B?YO}xt_b9JU|hnmbW zRa^u+g#4H+ER=>UG`!h7e#%2{<9*I8($U-3rzQ~(&QSm zoMtsM`nm%D30jpZyC-^huI#?R*%~ob9LetE{k=`oWp{rMtY{>=huXUH;lg{z7C_kz zN5DJY?kfK#j?y=Av&rtOipXwk%lY=t4h1psmCTa+1*k`r7nj`#WKKDH+|GLbjcyR@ zJ}JAQGM9MwtH#t>LUx~Q0aVlr{MZoP+_|sp@4Yl_sXougQ{i*m<@0e9e?p$)d*n?M++)jTxZRpDY~Y}yv?|z}n)ycA z#lEv_T;ohQbeJvHip{TLkI`8zm|*u?A_1LisvV&#T!h)H)97n8zX3hC+_Qu4(DNQTPoZ zWsdIvsvJ87R5^A)sB&y`sB--Np~~^Qhbo6l2J3r1v$I8_m>Dz=7LXE8X|xh}hl?J0JW7_NmGce&Wr zVz-N77iZkNVjqfqFZQ!od)sDQ=MBOsQ@vu_h*gW#ioqSiwg-!yB{oj%7O~sJ9u#{- z49Rr%hlDhS;}a zKZ>nj(t+z&5l&SEtBI{ChG)GpZac9WvF>8M#h}@_E~ki%6q_J6N$hU1`^BCUdr54u z*n47Mhm7p ze47v6lB<(&YqNIgz3yM+@wz`^D`Hx-sMo!<#gyuG|K2{f_4j|o>y9(-zwULnAAoPH zKj5_#yZd2Cs>|*^(QO89cmE=0cV9o$``7_JqJC}LLmM04$A==n%-1n%e8(P|Pvqma zLN|01oSDeEcgvr1k3{~wCJn(Y68R0@NjqO?}Qn+5!T>kpGShfx(y_Hz0dz(HgzWbaYa*QY=HB_i%WD|gzs&;K?yc>Iw1Zf zoB9d9Doou$_u;}W2p;f{0Nq#b^Iw}y9W1)bFv5ASm_9ztcMPQ3FZ1_Gu&GnU7qh81 zt^y4OIm;LDy00W`>h(S!G-+z7-N<-+YC<^S<^>1trrFeSr3iNbZf8Yhm`kv!(bQ)xEVXNIl=ZFmR-cmkPKb60yN!Htas8VR=m^@^doN^D z|I*)$Z0fjt#;*;V0WBNgk@D>4^N|L|ot0cqc0!416s5G1UX1*qcZ0Z|&AJLku@AHvO9huqO zO_C7f*d$0n!UR!Fj3aJ)UzOVM+sfV}EN)Y8<$X7{spC|a=PwlJIBis}5YL_C6p))c z$Bo%!?i|N%rOC6oa~z}v!`wLzz>Ra~lVfXyb0t~h3dw>0@Zz^y+U;m5!+mBA2A$4`Pnc~fNd`jyGra@F=%VXVZ1ln z&Jmk0wpi>vv5KbIwkwBImh%*Y-e!N$)?7w6v3`*aOct(qg;6$+zVq?V45t}ZCG2m>w zQ0zr9)OzN+pw=_vFdCe!i5(2dRt~4Ex`EhMV%v&Ad9y8)H`_valRoQlmnF$gJD;EYh*0`Yv-fkQZ$kBcV&556FVYiKy|}-ct9mQ*s9xN! z?rhy3Xs^qxdQ+U`{ar#f*wXT=-Ysm6gWv_fgi1QgvD4&Jxjn2+Pc#eGOsL#cXZf~G zE(-<~be1=Gr*xL%`$?)^T$LYV3!mec=V!KM7 zVcgIsRF9EQyp4bIG@tnS-u<u4qZ6FL}Tw0+85v1{vMR>6X#8v zc107>Yj|Iced5!w9*zG(t=nMlxhvSJ?8+`#S0m_BzJi z$7aGjc3HfSBP(3u`2L`baa4dZ#(rqIwo%yh$VcPnjeIl?*ig(}?G;KH2e_{l`5`xyrxA>+D;?IQLU zF*pJkH&yIPF_gJxe>22h7kf(#L%JDHh+g=tN)R^gNtj}}``tVXQ67zT2)t?eCQ ze^AO~P|92ep5RM{C-{;`<|>VyJz_ zb@{6pYM+rI3(e)iLrykJ47Jb5>~|8zy(ebBnUKLl&bTtWwr79M!YR8WA-1s?id!=d z#jV-)NU>wZP8EZPoN?!hohJsrIQzRt3`4caV7g`7uf@I>D>t3WIC#hz*H)~Z*!E(` zOfznv*kCaX)nuXdCLH1B2G3f-TRpaCV+}mZ zH1M;HH85S^u7TkfM|v3-7pb=ItmPZmDKX1;b~yF#;`nZDedlw0<81X`=J@s;Z&v<7 z@;+x)@-y3(X8OLvbZy-9jgv4;-#CHfH+>(nR7v2EtUt=(FKmi(_y?QHmBW2(=S|wb zaibGS;6e7f%yM{X?cX~$x!CGkLs-K8-C(PcwSQy7;YU?MY2`!~MBq(&}e|Hkhisfi=|clUyt7)OazO?)suSP*4~ zfAaY~U4bDpJk#eNDcd0ci!?E7Zg)u6#J|{DlwjW8+{1VUj%yV-?(EvE1oQSp1e)d5 za-~{B5Ds|$wN!090c|XEmHm)aHg_>l@Izj0e5${yi*@;{+T%l9EQB)Rurhl@#%U-gQ@zI7$ zYG~YwSnMqel^;LS-;Gx8K%b8i+1NPn1xL%sXXZwY4W%-p=Qk#6RG&pX3Y(hHlpSMr#~W^Pm_cV)z zjGk$)kkL3)#AWomHV^yuxAqEIje|VqanOaG8#^cL-1rqCv5ap4@)^4YR z!Ol&#Sge_?7a6{L#;q!bn+0SzxiD@AF&q=faI|0?ez;_(id`;twb<=qcZtCq!nW^- zeJ=Ku*l%J@ZR>HltA|roaZIdAtVRqEc;+&W5<6M!G%=){8Fzyi?A&Cqb91>LihUyX zvshq~m2vIDsTRT7Vjaac6~l0AE~Aeao)<-i=S8tUJV2OilGtTpSBcFMdqV6Zu}{S? z+?xHZ7EV>*IcOoftNU_mkH;PRcn=Lj^>_svA1%>^+EB2vS z6BDIeF3jFs?kZw1dy}m%260Wco7f&=`-vSWHd<_~*fnB*6}v+WW^XRzF|k=6+2#RirD314~ac0h9TJOZ&)~0 zg7jW!hm-u$yUyy2ZD&7MD`Q@ZGUhCF*7Ql#)T#}vka8rnVz&%m>}M_MC>fuLpI5Ua^Aa7~)qHZihw(au1$`4K?4_EK!Sj^V-~a`GV3>3bdTapkE@QU>BGDaSic|f_e!vS-|K;mt>2L3tWe%3wg9HG+gY1; zyd5p}?QAD%@cDn3T)(@BTu0LMZ@#69JsE7cMIh}G1gH83g7urb@9D8SK6z5xP;tn< zK*=XJjD37gWLat*4a3>Uo-AFbkr8zbo ztH;F!aeSOLMK$j-Ao@w~bFEmPNhyx+b)-j)j725Oo?#!^&^JLLEB86xM+|6}cP@qz!$an(pO55ran|8VYv4o^OZ z8Ryf3Oz(EHSLojB?G^g_etU&+yTM-L!>X}~#+Xjt(O$o>*E6kOw8Qx>tCzR(>*!9_ z5>re!lZ_KYx|!@&u?NJEZf4tgVo!lsv_7-YzwiS#Hz)5 zitR0Sf*2}@a~Wrgp@KLWDu{ErGsK<~dr9nbv9H8ny=Gff5NChugi|eo_F|ifp@KN$ z_7UqV2J1EZJ6&vw*yUn3iouS}{+<$p9h(f+Yxei8SZhR z{a*I6-x%G&?DP*cnd|*QTzO5}zt&6`X@$vH^?NUC)N|11*`e1xn|?NAa(7nzuHFPC zzfC^R-sF|;R`T1*eb)_IV&%T;nl8zf`?lwRRnET7iilYi@6lFP?EY-kZ!ZMqZ&X22 zst7nf`9W6e`;C6HIN99v=Z9!pbHC}}9~Zb})w|{PE-HM#WYxolTWxPFsgg@#)iNVi z(A$=f?Q!k%vmKV-AMY>Lk=OH{o41}f640+%|7Whfv6X4JqGMV4+Nh5AE1r5bD#D*fvZyJFx%k>zyP^f9F%T_-IGl!WMW_>L~ndI%d1F z&x{q%r*?4x(ADGG{|;*RHt#7{?cU{VVR|jH!#psmMTQKArMIPa-}U#;FWbXeBo&W# zzFKj$3-$Dqci6F5Eizn45Av`xi=?ly!68JeJaP@?o$a(1)*btx)YLUI-MMwgn7Yny z>mKeOcBAxE^1Xc}BVX0mJ*tpZ z`xJjS+CbFHe~f+PJl5>PywNH}hb3dvydN$)Us@Tp67T^!d(2ICdTRL}SsTo=$xwya zKb$wA{@U6<=tlfioBG7L6son>UZGlXrh_VdION#vp=kC~jqP=e z&10v-JWdxdUmt6)$R6XjjO;P?BxH}Vagja7j}qBq{1~BzaYG8r^+M4$zo{*6>+)>& z7*kZkWaGkjD}3Y%8E%j=?nbr6NF%bR#1@LZA=V%U4b1+Uhf_A3L=0za_P34L_F^~% zvTaYXeqslUjTXaUfc-%sv+ZSK4~jh^hC>14o)bgPm<*5nW`D>YlOcOd1`SMx>@is{ zG3a8l0b+-WjSxFs3>jtihm11YVwgMGQ(|9=eJi$t=}yLB5Hs6uBeuO*4>1(*W}NMh zVcR3b&J(*(Y`WO(Vsph7h`lWKnpj|a9M`WroH8S`*v4XeiR~-aUu=*V4B6}thHQS0 zOT;b{dsOUkv3JBi5QEX0{lRF>{@RGGDz>>8ighz?7qPBl7?RBXFeI6AFj|wrXiavr z7>ae1Js^f+-DGcyy)6c#HT#3nnsIne78#0llWi%6V%=o>iD4iz8A^Mx?b%{~6}w5S z&U6Ihpn=(T86IQ}+bxxEt(W6|wKikVfd1Y`8SiH0xSz4dPDAmn2uF3LnX4_)%6i=U zS!(!;V)eLF9(RR919k8FQTF#`W*bqAdoiA@WRJ6|L3;WHs~O#D5{^VY2Ypg!G8qT2 zx~Bi+``t4qTRH9pwJ68^TCyDXepZhA@ygk6o8r7W)3clvxhIuZdogZ%#I+uO4QKY+ zv0B`4WShWkInhNiWb#B*-(EQ9lYT#;yWUq*%lbR0n~itA;ybLU?e9QH*FJHYn(q)Fx3mI)q=l=8Fo?_~pz8}LHgZ&;yi+O5AyJWhI{d#*RJ2ZY;0!}{lx z*{(&I?S-(vPkhrQ@9KDVRa9a7%24y}Wd*huL-XP(R&(teZ8h<{rP92(w@J;r#HMmJ zZ&%wxlMZVfadDnTnTciAyp0{!xTe9W3#Wcm>Afe|-W4RwQ2eX&Ys(%Iy~PS_Cq*x8 zmEU9B@a-pAWj<`=z3X!UISlx-1;UW3kmp90(X^>p=>==?i~T6o!#&$F80 z?5T87d#DHUc~&EAwbt^FxvoTQgTFt?3T%T#qLv=*KNc6YxZ?iWJ4_X|NZDt{YeI(Y z>+|zdMD8lqW>b&cI7j5N_|`k6irh*5A#+9Uh0Z?C>(6Gruo6Tri)**`96s{>*|EyF zjHMrYgGM6vNPF)cj~(jY0{jCFK7XZ&+@UEVccyQR_}JN`$c4^$-lJA3E^;ASJ_)2L)s}~i%h(q=*2zIi~RoG9xk;hC7YAqky-~OFq|Nc{pW7p&3gnu}PLjR%$ z$vq(}rhgx_80g{#dyQvtv-%f05$@^L>=k+#hhxm+@OZPm;)}V>UhlP6=wSTZp@Xqw zK?h@7LkHt`3muH#D|9ewqCp4aT%XMfZ(~>ebWvl9E^4yzwsvIjPLo|Bc9R&+JdA^j znsIPZlfgwzwpgs0tq<8s;S`HvldU0!8zE$HQDdqiz=Z=DY!+nrIg-IeO$HY=8MYSL z?P7O{y({*i*!N;Ti?z3H#{M=4r&tu53?=xujB3TziuDsaSPb_}*cL^x+4dGO6z(E} zi<)t8QIjD*OopP^WZ#MXB(|X`PR5~J7vpviLwRg66zO6d%43tADR#El6=K(j-7ogA z*b`zXkInwR5c@`Kb^AuST$IOVT$NajSTC{OVke2AJT}{2A$E<}-^A_^dr{1OEn(X) z#l975V(OCX2gfv*fihfVD8oh8No;GeK4SgE28$gbcD~p}Vz-IiDfWoiOtE*wJ`h{U zlquKm58+fr@F%fN#87;Tab3g?5*r{kM(iB1o5gMw+keGue+PwAC1ikcfm+xJ4NYkt zXmdh)`ZsHXwh~;mvOyQEWBrYFP;1YPMjCmkoX)IdEvNGo>t{~V>HNK|dydn&wM8wx z(;4Rt>-+zn)7kb||Ku*Eqw3U!&CchW4$k~BYjzH+Q^V}c>eN}Y^KPMB#=ezqc3#a6 zmXyou+7yQ-!}^vhoIkeRJee&1t3^RLpJK1eD4dt6K6Rvt&-E)eTqG>Uwhc~k*$Ue; z)-h{)#(rDK_S_9&=HJAHelP2Xf2sAU|Jr16Fj^X!3umP++~;1FQOJF)eOF(L0}5(n z941mnAv2lC%3M&mFMFpGrEu$g{zWq>gGDCgLC&kv74F8qiBnC=%%}ZJCgqcSRai0z z(h1jCL2!Y81W2cPpMTa&%3zU5Kh1f^n8b`s%2)Y&C76^i@xWpx?Blx9-C))zlhBCy^Uzrak&Gh?zTp?jeLoA}~VOv>ZE_d+IRT)EJ_v1e>IGAZL!mFMr@H(vNlTbp6A zHNv?L7bCL$IVjGBUBeuQRK(WDM zW5p(jT_$#w7;ceqxwtdK<>H7&_NLg!VxNiO2*No03fLC)qseeMBZJr{>nn!g-(;vC z&A6+?ka{J93}qbZN0ZGFL;Yy7#bWP?A@$0(s2|NZ2vRZ#QnFpeY|jZ9>PIsU(v)#9 zCzFj5yH4yzG1QM{9O`4S?dxK1iJ^Wp<4`}EaX*RuCf45eLdIPM5EDR#ElRbtnQ+0C4=?VVyziajg#y4YJ{fn7**x#i&$t74IDELNkq?qa>g_7_7{ zEVe~eEKF4d)5NY9dsOUkv1i1d7yC-=JF&G^$Sz~OaLQ_*i)}8pm)O2yfyvA8^M+H) zBs5_)#ua4DYW#(HjLm6mHsh?vcv*@*<9rwM6GJ5y_7k^SZUM*-~7>oxH4%7)AbYly99h*YMpQ zH7IsBBF_j}s6QAL5<#~Y@&hM}hM z^quG_Lh*#^gZ^%0ie>t>kZX85kB(f!oB4c-YxouqN^=cA?@@({RbAolMy_FIcGFzL zfALmnuHpF}Rme4reJ}g?X6!M?_xbpypp=!S7)?lNiYZD{ zvT?R_vJ1nh7WUAzEW1$*o?*s8kg_d45g9zgWDQ~vq-3~!VO(1=T)C1#nlcX3l&pss zyhddG#Bhs+Y_u4JE7>Jt5Uyl62wghY%#cpxm=8;WgN1mWGkBBBSSf7vi}kLlNd(TG7h6^ z8HZ7|WEfRThEcU-5UykxRZE6ZwPe?eVN@*{M%6M7qiV@8s+J6+YRT5I?}7{>m25My zEyUm+X54;agTxLKnS%7h5$9)K# z%c|{xyNxWsnWYk@7`#Sd`lL-!n7&|Bxx%!K?T$$^Fb=d3rm(XtvoK9F1AkL?aj>Jy zJa~He7ny+@9MBAm?UJrcds{EBcq@*8EmfJ~B8Xd4&s9V|vBEtW@Q>sfV#x zE2PC%KL1n+R^FB#nQP^xH~0BcmFT(tA#;@|GnfnVD$&zCK2oCC|4UM$`+LL2k0CnN zJ_B!NG;gFtkF@t@dAsBNTY9PMWp%E{`g~+WTzS9qyVynZDuk z#gr)ArqeyDPhr)0vIXqpK?_{2M)~QCS$h}68s@R~B0o~t+S}FNjr_5=z=!aPbZgw& zixshFkts=fsL#h`SsrUIZXX4~3<>Re%m(Ak+B+kjkUXXTuz`b)8hmK~le~LuzK6YK zA$5u??)b{^xR++_#r1c(wfA<9DrD`wz~7Cmy=VA*d^6;+_Kx$QbZhTp9#>_XtFX0q zC68%r?VW+o*7)z=rBZE8AXMvPYlL$mPC&Q?U5dH)YKwzVt+&^B?z6$>AynVBSD1Of z4d-!y#yk#~F!$oCfw>m~41yIucnDVPS`e(*+7PVxR z1uLd#?j;*@))4C~wu2ac z5Nx}r818(Loho*@*wteA0Wb~+KQ8wzF${|$`&6iX(<&{ncz#KwwE5W7SS#gf_fF|k=dRwz6$IE(67q*>+vA^~Gw$x{Dnl zc8u65Vk5=ucM~oH#ge&DJ3&mjlW!z_Cu>O+$DhBH>+5d!7ErLIZVYn;aZ4qoM zcCgq$vD3uP6w8Rg`pad&=F9%>6?;YOb+K>6eh_PBQj=|A{bhd}i~U(_AF;k-CyAXZ z*3onU``a{}DnYQ;+s{91Y=ZSWFIn8z!Yzw9yb^BSs<31+)K{zZs)8ZW^L5R}+sO(T`%u(Md)s`z9eZZ_ zSnZ}at#EO5#)hdrQGNE@%GqbtR`tGYK=-LEUA%Vpreiu-A|qz#ojrQwuf3;YG^*7@ zI<#NiLl+lZnwj#`uKXb_L~WM~f+~+%WTsxdoUm_O+cC z)1L*(4YL+i^o-q?yu~srbqb%TO>qk{Jq~|@6k~USvkw!JJI_;R{dXW z((%M1(h=f>S|~n7FzKj;w!78-1;J6itDUIb=&<5RO>8$@SY;iwC)hMx5!oc?Fm<$h zOhF|}BjG;5rQT|kFCKBe4|PC^Qb{GW^mn5*q4Ufu818NEHQkCz()GN}mBIQJ)g!31N71h8qfP6v$i{!6B*n>~(k_aw4G$k1{CEEt3)X2)isS1DC z4q2rb9=1c)Ner%BG7K2SR7HR^CmH;?WO!;V8J=28cCXljVspgii@hZFsu=va><|82 z_UH4y54FO7eO)?*Q!QAU$5|h-e&JL_aDo`h2(#_kV&law61!9kiBVs6fW#=<&J>#? z_MF&DVi;n|wm*ukZP(vqmElxHu!GpnV!Mm&DRzL^A!4J%#)zS`Fqbi1Y_`}uu@}V_ ziM=cKp%@e=`@>*UOqn-YY#Xs^v0AagVn>Lzwd;NMhqNjCOD(2UZ-<#uv~lE{gCC_(@ubPrwsRoaIw2w zd)gzhb2rIy?HS3;{}%1p5?96nv?r#hJ;}z2L3@%xdom8%lMLFE4BC?n+LH|0lMLFE z4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE3_nLQXiqX|PcmpvGH6dSXiqX|PcmpvGH6dS zXiqX|PcmpvvQ5RHJ;|Uw83*l22JJ}(?MVjhNe1mn2JJ}(?MVjhNe1mn2JJ}(?MVjh zNe1mn2JJ}(?Mb$_7_=uDv?t@BJ;|Uw$)G*SpgqZ;J;|Uw$)G*SpgqZ;J;|Uw$)G*S zpgqZ;J;|Uw$)G*Spis!5J;|Uw$xsZI4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n z+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n+LH|0 zlMLFE4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE4BC?n+LH|0lMLFE zY`OMap7->pt^P0PJ+}<>J$JgRcEg9en)I z{==4w&(`==^`x_#pa4VFZQz7cVa(@;jyax9PPuYilB=a9x20h-a~9J zu>-~Wi;Wf=D|VCE-^AvK%@=z~>{YS%#6A+MFy+W~UOAkq2=H(jGCW*{>%6O2H?bjN zM~XppvMmORvTdsVoMwlPytz-f4c&3;wR5wkTQ8j(nr@3$&b`W-7x&eoM6P_b{Vd!Z zq`AJ@53z2AeYLpPwlu!l2J6lZvEj2^{u_L?>)1MDrERAv&033rp^>$AVu$aoom*>J z1pj3(RE;X0FM$y0OU?6*Z2q1@Ui} zCJXJ6sx;naOUAVIRmPW7YH?fa?S)OXKe;n>FN^7vU92_8x`tDW+xGHC9P@dyb#kB2 z@8uRzSYoI7{A9Dlf<>0tgPgyUZi#)`w?@g9*rDDpE7jt{<4&LN?dt@=Uxm?ai0 zvc$p)XM>(%w%4R3_U+hvktO!8-tJ`YAC}l@K7Xm5t&kVToPn z{g+^gecrbli^{@&I>+Z(bQYFac9dp`#aC)i7V*Wy63bR8me?D-QHmvYUk@r|iAAQu zYOMNlk;%Bg=OYJaw%AT&sEv=jT+^c?Ww=_Rb$g#rvBaL`L1~uQJ3XpU0otSe-DpLa z42&$X_+l`;FDf*g&l0<>N2OU}k;p+!n`o27AFxuLSd*Q6KH8w&eLlq!yQv2iFHnnb z>t2sz(Kc9Oxz`l8#Qui04;ouyr#Wx*pU@K9U~g2~NhGqw;xL?RiCtnOX^F-0Q|{Mn zS`r6J!O#+miR)9e+dSR6Tq*ef;;OtJ6UD@?IS|9xSv z&8%OTVsVbinqtqk%T@|QOffM@HqKg;-5gG}@aLhDAu-7~Bqqt`i6JpbhR?w`BqqtQ zRmiYrWcUG*Z6LOd*!E%r#RiLw6`LT2bR_%3)gPC!K>07w#NHG`I+E+$Je;ZsP^y;f z&tf}>;VCLymqB8OiJc&JvKXGH!v4mK-6D3I7*dpsdqnI_vA4yV+Fr=G6~n2D0QG># zHV}g_B!e)-R7FrNRx8$DY>?PcF{CZo7R7DZAO1^ry_gMU4%q^+XT&}f`$WusUtxbg zi#2bSWi7+0ir~*;s2jxPqL3}u1$BeSju%7SAhNT>P&bI|La`}g_lrF&_Ldl)yTZ1g zi=oUd*&ppth^ZFA>fuyHFk5V%*m7~#bm1P8%H6oyU`M*;;tt~`|J%e}ii>r*xHH8G zjk8?b*-w1AxXY;+Gu*C~mW#WX@EVaQse|n5a&dQZZk?g!;?9mE)lGv@$q|pghqxQ# z7jB(w37Dd|BO50MaYqK%DdQmS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@ z?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS z$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^> zAnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#LkS$RO^>AnwQ@?#P#9+>t@tF%II64C0Oq;*JdB zjtt_C4C0Oq;*JdBjtt_C4C0Oq;*JdBjtt_C4C0Oq;*JdBjtt_C4C0Oq;*JdBjtt_C z4C0Oq;*JdBjtt_C4C0Oq;*M;&xLYpn{-0;=?y%i0X%qcjGk0k=(dFuHxw^A^|8^yV zTlUM<-EwuuQmBw9%TsqJ3>tW}s>5QFEKl9FX=LgA-^|>NwM$QC?l475CmAfAWUzFS z!O}?vODEYQVz6|Q!P3b%SUSmI=_D(&@13k!I7Le*87!TQ>#VpP#Qq{SL~NYcxnh@# zp&TmvL+N)e<5{uS#W1*#asL!U{Vp;TL1o(&!zq@tA;XYD#vQ1*{$ePC%DACoD1u6M zrr1Sdmx|pXcDLA*V$X`bF7}ofjGSD?O5v0}Qc!Gju`Xh}h*^d=Ypw$^$ikSJOY@D`5-@wO^nMNr}Qe!X9_ubJI!vZ45I z_4{9Z#F=^SXFhZ6?CieY^LoEOVk^Wj;s^WtiAg5RmHLPuWWN+c0adbrVnf7E6FXCE zir58WbHpwcdqC_Vu|_c@5^}p3`-A)UrC3M%Jx$groa2)_GKf2_OL5n(a(Z=n_4MjC z0|yOG3`h)23`z{Hp8T)t?4#eTC9@{EqAJ^cPxX4H=F{v{GxBAteVTmSOOehRp6L46l*h}x@@akDxnpbQ(w2#W zsIySV^c9plt(-mFs*X0ILFHU4X}a=Bd*xLxggVh8>ls zIx3zU@2BnTmdct|*kReb8+tD>WMuKi$(~l&^t9-_OhJO*V~HMLm(3_%ZSKnr>?~F$ z=eqT>Cizh3Mq533U~yvg{GunV#HqbOWiuX5cVJd0%DSAk%>MW5#*E^o*W3A}1M`nI zm+rf~->#=j-{lT=Kik9En3TJmD{ zb4wV;+vtZCZRGK6p36S=`T_p>;~AdI1Dbj+>96$G_P?IKYbROG$Em+{^p>u*Ju8tt)Ig$9swjn^xHJWzH^+nJk+rY&ysrcJOnHr$Vpz@+R(T zgo|^YFAp&nCs^c~9O?Y=*an&@YJ#7cuv5p5A9IX!7gC^Rd#mPL znY;U|(Fy9|%kjY~GseciAydkg1{-P23r`MB3jp9S@U#W>d~tt+3$qc6X06XQRz_+h zM?3`zf|&ulnwtk5+FggmrvLs|?n@-)@utRo!gm%PYqhh#8ASeg4&P%#?J*s`UF;Y3 zr0Z?k*QWSf!+VL(JG_^THXUzMs3&+Yacb?C=OkPKmNqmA*Uc+7#i@MNrug_Ru_->5 z@MwN!Q}mDT%#=s7xBv3&W<6q#kMYQ+iCrAdaim7J!^1nat5rKZydzsE_AfCUIoJJG z3=j0kwg~4+ZM3Bn+exguSVF8y>|`-qNbGO6*d<~&i@|-&c2IhlD-EFZ$e{1Yz7XqV z=ZtLgaIPfiB8D!OSd8T1_)%E*x2D~2*MWOZT^iZV#E_`R zb&;sYbw3vSOzb-``ym^)+d7;p4Yn17mScZUiRD#YUe&FxnR!*`>+D8ud38TfRhPqX znOAl8OKYHg!R1w5)~YReRcF@+e!=HeU2#l%=2cx@)&1aAUA6rVIjtr`f462R{ zs*Vh*jtr`f462R{s*Vh*jtr`f462R{s*Vh*jtr`f462R{s*Vh*jtr`f462R{s*Vh* zjtr`f462R{s*Vh*jtr`f462R{s*Vh*jtr`f462R{s*Vh*jtr`f462R{s*Vh*jtr`f zEU)VFs&4I7-Jk7x%5++Oug~goIW6h?zsG+ zKdY_~)VWjD#b6EiDE z$TkV*>^Q`>7u!kfP_e;cXNgS|n=Up}>^?D+-Q>Q^6GPcevggE5c9ZODv2VrN*m=jR zX0!;li9`AMUd4PwzagZ2NOM4^JDC{(XC?Cb`Ib zm@r@_tBcKnX+BJvFug)%%7ocWiOj%#mdDzMGMCTtvy{)Wvo$s^V(7E18M&%&(aJQJ zWuxuXkJM$kh3z^H&fdz_8Z8SVqvedw@VV8P;j-4Ik+ZChF+2q?^kQfUN&0WY-KO^_MQ5N`vwoTgXreV z_#ew&;xn4F?BzgTDH;j>L0{g~4TLaE-sH=J%rFTS877Z*erImOB%-lvV3-{5{YGUk z|Ln_K`ToE#$^GeXhDorW%*Zf#tG|};{$ZG8|D)_%Hj%?H zxq}~AF2m#yZxxLMU+ddzg<%r;C_xYzCTIF`WSC?}xeSvB_*!TCg$~0c*UDj-M0g!m zw8*b`_OK>-SU>kh%^4<#`Kys1aHKCcXP8`x?b*kg2EQQqw=d^-w1(%?_+Yebn51E~ zxw4iO-bZAZ#A64$NyhrjW|(Z}t#TPA@o1v?GL{?rtI-MC%$IW*CfkOMav3Hkdb{4? z`Aog*EgL4^#@@yZlaqY8$^SybWUalBUak_~F6ree@k}VQTqW*%NOMga!M!mK!*Z3l zd8KQxOv@y`bMT0%o&C)~Yx|cl3nRXVhT0V04zM&%v1yG>@yUi|5}$ZjCTndv-lh;Z zuuS6Q!ZL|#0G3HyIj~IPWWq9u4>{yi#BLCKQ0x)0m&8y5h5fxN_Muoi6E@uEj^SKMu$vfapm5z@Vh4zg6+;aa zw)>43YM_w)PHc`C99-l@qRe;kqd{Bb+M<^3v*uk-(OhR>))9 zdB<+{Uk=}6URs3@$nw%EFRk*@Dkg)n<)Y=Kl>~KOTK#`6t!n+}UpEswn4`2JgR~-p zv?7DFB7?LdgR~-pv?7DFB7?LdYZQaDB7?MIJ4h=sNGmc(D>6tcGDs^jNGmc(D>6tc zGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^j zNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc( zD>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGDs^jNGmc(D>6tcGCbWPgR~-pv?7DF zB7?Ld8zu&6MFwfbc92$NkXB@nR%DP?WRO;5kXB@nR%DP?WRO;5kXB@nR%DP?WRO;5 zkXB@nR%DP?WRO;5kXB@nR%DP?WRO;5kXB@nR%DP?WRO;5kXB@QX_c2&YxvaaG`n~* z&4=IXQ>$F&!@Rc2YpeWYtDx3?QX#J}dTNuc&PV>SmHl=LtZ+p%2HIJ%(XsQ7t=t0u zi`d&ZPe0(tRzHTPR(1Ackj6pG(Ktv3;~*J~gJduclHo}j*_~oA4wB)y8r!`gwnPlZ zLAEmy9kyFPoGT65i@`X^b~}r~I7rq@48}n+Tosrr2`(1G25^HaQh5NT*IA>4V#8BLk`_fzNK(TRRDDKF1lf?#a4~0B|@B^Y2W1nPIr~0=hb_yE4ZC_PRe;{fw zR+ul)YcpCIM)(J_m1xW;&uIQaRA?LyXW`dpmd!lVw%2O&;BMBK=D}~yk zUcEO$F~`N>t97Y*jo+R0Lc_-&f5LTHv(X!ur+Vx-JKka2xvBoFtYfUZ2KMu2!#l@4 z-SE|7dw1o(5l+{NF@xt$T>%Cr;S5!P+Q#gJa% zqpGY+&xIMq8|PUM^b|g8bLmdPlh$kMBt+`$?$-0qZMqgtLOk;7zT+_*ep-09m6$>sI0<2L4+1&~aUNQ75+3M1!%29cuT@akTrtOL zfBnM@Ct-CiCt<(B&#Yo0b`4HK?i!thc&l3LB*f*N%}GcjWoaP{b=jPR_++=Pc;l6R zct5ohOKaY6TfWiRT`{X*GsPQ^@rEpxIKejgkuPuO-naVch5wjy*N@b9)G0f%m{p zKW;kj`tpvxU+_qBzebx!5-jpa9_8&mFW5r$?RvJ9em2FMN3|lC_;PpO46Kjb%yDLY z1dFVX7kaw|S*?#%{@Mqz*COj9Yf&ER{logm{wJFC@z55mkK^nF(9O4LOvGiY6^S2R zK``1+8)`-JY!8TeQoY>Y$k8KC9+y^X5M{T6fL2G;ihRYFQ&z}SM^&-SXVZ}Q9T)_& zeDknAa`Tg8uEq&XU5$VAm2y}gX>M%J`gomp5smnLl`qrjhke2?P`K72X&0{@+0|c- z2HM^(W9^-NIqnn8s5*it3;5xSPkJP>GoRn#v_9yqwv6paTw~Hh812Z3-g*N+oj8oC zz8v31Hmf9_eOMjJ_?~3cmc;dRpSNnRwq!qlHQJLJUye@bU|-%i_QqzUui=y=yix92 zlaF}2NN2P*)3vt%twaDd~An3C&ObtvPLmH<|DJa zVAu|i`N;4-aa}y-BkL}PVRU2LYcF?C}2a6plc9Iw#0k2au}j6S6sr@% z13#|&o)}aq*>bUO#L7+eaT}e(IddqA?IKnoRw*`0%zVhqh1dwOW5w_gko%VuLy8;O?P7lx z`8-tMAow_!L}68uu^S7Q5#^%Of;>`<}O#U_YdC3cuimOkk{EiaGjmQR+`t@_Cs-?eRbz` zc3x-abvAw!=XExMN#pEH>5$ji|6QH^wka3tY|K$-lTEW9pJdS4WYF1Uw}?S! zlR;;*9e%iyL1&XSia}?SL1(iabT%1uHW_p_8FV%obT%1uHW_p_8Ez?L(Ai|r*<{e! zWYF1U(Ai|r*<{e!WYF1U(Ai|r*<>gKN(P-x2Axd?olORvO$MD!2Axd?olORvO$MD! z2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd? zolORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORv zO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD! z2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd? zolORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!2Axd?olORvO$MD!me<+&=h^?Q z=h>s}s>}2k<2&Gcd!C)kW1JV-d7*8;vGNbJ6YcX4v@up*UTB};Pjd4@`~Ttt?H|MQ z>~~DLP-kO~-fFT7>`qMvZ#5ab)nvDd!COrRZ#CO35`(vzY>61W)nsO12-jUdoTImz z{lQz!c6fqM25&VPywzmzR+GW)Kz6YhY(8Xgwy+(%)nq8jO4ca0ObkU?*=`g2oy2xq zgmWy)O16{Oo@%#`7>cs89g4EDKP0P?ohyc9RkGiT-7EHIu{yB^u|;ApihUsVu^5Ki z;5IPa2KNQSZIEHO4YEDN_7>|WHb88&7@XH!7ejEcKMcV^cA3~!Vkp$gc5qp<-TPwy z5<{U@w)<9WYtyV`+lF)IT^8F_EFo4UcAOXzSh?LZ#eOYzmDqJ+a3OQM4PyTkTPpUc z*cW1}#fnU`a=TlEb2fOW*iK?_W3$~pVhJ%MwQ^rZiNSeIHbd+du{*^6BKDvd3c7M# z6m;cw|0T9etb-|9w%aJ2GcSSIE@Bm8m16zG28fLpJ4@_pvFpX|61!Jyk=To3(AnHS z=xiQGi#q!ryCF6IJR7zr`NpGKvT%-@)!WVEUQsrSgS@t3#R&TwR~ytXo!F_i0M2q& zmObA(&1nuFHELAZR#R=~&~{Z-k7Z>u_q6qL`@|8e{6C_rH^kAenN|F=xK+H3zk72# zFSBkkt>U+aR&l)lxgTNQXFbs>4&xZh&%!>QZW`a-F1gI|vpwzh3z3a4k+n39=PW;) zsj&+RuMIlOG`__4B4rxK8EMHhez5;0gK7L=Uu&85vTu9X{R)DkM~@xJqM?24b(+TC zSX0w@&%!0a<}!_Q*Jv8Yo7P&>IIi{PP2;QFik!_fj*oTD^0Q<7$Ro@5YredLdy--K zuJz^9&GHQvS-wwmzA(4t`!U;#xaE6^ub5LI)4JWx-yvAOU-H#{>H7uCcdajvHOn_x zWcfbc`D?AUd{6V{QMFtAuCq$Yz8P4)Ykm1dvwVX^mhYtV2V%DFsM>9nzxI0UwaD^4 z-(MT#{X-$w`tl^Rd=JWD`99RoV=l}0G;cN5Pa7=XJln0XeA9LtS-xojk1XHpD3|5? zXy5!C-#jeeTq}p=`x;*`X>&V}{ zamL!)_;L=*_j%qZZpY4M`Nr?^AlN!~7;(!to~H#tv?GW6a%A~Fy~#GRS-yAmR=F(S zH+ZY&EZ-IWYP2VN`*L(bd;4+@%Qqe#;yue%Etg(|-hS1<^1TGB1ua{?*YUoaTE6j3 zm*rn*`NlUw`MPn-_t!?!E#LO3Y-0Jw?YfEO8~0_w(DIFYZkpvgXfkpMzALdG?d-25 z)Aw9^9VYNvn;vUZ+&WE>_n7(m|!Ss!@3ez_}*)V*8TF8BQ)49!8VlNXZgnRbu$?upPo~Y&TmB52ML$7Q@46vX{gbi@huMq1eC0 zR*Ipv6SoUHJ^Mp#Cocx$b#lkjrEji`^o2hZv+W+r27=F-*uH zmoZlod?mI*Y<-iSY=`t@w!>&9WEjnatV*np*g&x%VyO4Tb*G5kEOxsX>OHX?>OFBA zZ;HJohAK~N_qo_Au|hj8uG=}BD-F7c;juK=-9v1M*pXuAik&A0k<4`=lDUmPh}|Rh zkQg3Sv)y7bJgO%9Pz;Z%$ySQtQ8n3CCPy(>8vHDrD+#K^`iLPtneC1f!>A`@81;nx zO%cPRYBD^QX1lw@o)vpu>|?Rd#4w&E*DbI+D*IbMoUl-P-4W5j+d zcClE6=?`wVGMvj{=Dx>nG~1N z9qf|Jv{!$xPkz_bUY%FJdG(ur^lO%TJQ&PB`gOl`{?TvFYF~Nv``=N&nfB-(;*($e z{HOTE9PQC$7nnpPgFTuI_Gq$O#bA#ngFTw}J(_HZ80^tx_OL%}w|+QBdoN!=#)_RL_8YM&V!socBX+CUonjc(hug)dKHToRVi?tj?B8N5#eQl+ zlIwN|=SqSd#C{>xOU!%;VSj_f4i`f?YW6o(>=v;*#88f!?NE-I`}a>V6l@~?pC{h)of@TI_nUyTtAldsqzSX>6?|cufrEX|j*R zV4f!XMhxa@G9)Bpt~BTz&Xokc#10TUM(lX8)5Ru;p=>q#gL#_UfKi(4k75hOUJ(04 z>~pbIVudD5+22pYxzeD67!s1XjornL5E~|TlGs?W--*o;gZO2C5Wn1RD1NJ_kLy%D zeL{Kl^y)SP69WY{~S>-y5+Ub(vN+_5#6q7wGP zhJOutlkL4mwxjQkSQ_DStD^mnhObVlZ}{%xPtix^rG{2MX)PCFvF)X)wjD1--|-$| ztG?Xu@o{f7yt8AGNe{1}13sDoh<4+UGo^2D!)1q?`o`@c=l+Ea{ zdiCmrSH-SL&ghI-X~I3u^NOBK6n$ISV^P_RXVc4J@6jiDT1B7aud5SfT~1qO|NC`g zW?Aeb=ih1_(dm4@&87RBx3X(1)8D+C-7ELBLs-|QYvpgoL*5)ku@CT~*!^w44!~Ip z{#9^i@U45}`rFs#SI%f6#_4Fu-`vmF8f>ll`C4yg^*8sk*ZHrnw};WRuYj z;#LrhK4w^|e)kDlfmK>Z+j6C^Km)Z_Fi#vjvv@pfp6c*lL9lL6*&f$yHcxZHTYctU z+iZ_@@ri9+A?)kCr}RSDw>kT5%(UE0A#4`3j$F~7`ig~a+l4FoMPD9JuxS7mxuWT& zUJ%Lw>*WAp(1seunkpiFT?QV}bpXrJ|(fa7^uYDSOjjrg9 z#$Wc=XzuQ47aRNEr=ZZx-BAhUvoi`8<2(+vzrYF)N3F6I!v3ANI^54TT+uw+hsE6S zSqfp}IpICNB3;q=ZTYY-@8}(!s*W~~Z9ZEe>|=bb^L;J2qPh98v3kiFg|JVv73TO# zEx4l52zLqiO|!eAFYs48h8I~7aJ<-`##%Qme7&h!Z|7?l$M!O#&^2~|#j`ewoq$Z& zGfvyJ-c@sju1^X#go0dnPsjRl+)16y<=od>?H}KAmh0nEZ`GX38PCsxU^5-^&c3`s z?2t1GS>urR@s5A3+_(Kk?XcmIV@D4gJ8G2q*i(h9+xt3a z2iw|jzddY7f}>5S<}PH7?}2nzGyKZwu4X(^%q(OLvodbh2N#5Q_YU?<7x;V&!5ey~TFj#45$0 zd3kIni=85d(py}2ve->xw~3+j7TY}_hSFPP3&dU*YZNOrg~)bg;ao|8(pzL(iFFnG zrC3j~-eSYVMvDDf>^EX4y~S<(P7IHE$?%vLbEXi*{wapiTWkl-%XS^DHCd-{t|aIz zhR3{YhsV5ZhsV5RP`YGLx@0K5MK(t4cVctI?iYJN>^U*}6o&o1CHAgZU<#4jC=TaJ zf_7pZ#V`aD+wCcKgcyck;y#}xHc@Q4*i5mz#r`DrqS&iqUx+OiD>9wQeOWJ@vx4qo zJBlH}m+g??%l#WJc8u6mF%0v>cDtCi;JSN+bMh|d6>MI?+P8&e>HhE)Yz~uhUcm-= z1>4L}PI(0zv!X)nuCef<=%K*5tf`ZK`0)?e{gG)nbkcmTa1RhRC2` z$)I4#ZV`imC4+)xJ1AH(C|I&aF(_EFz;0G-2L($81xp46O9ll?1_et71xp46O9ll? z1_et71xp46O9ll?1_et71xp46O9ll?1_evjE}Wx+C4+)xJ1AH(C|EKmSTZPBGALLw zC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKm zSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPB zGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLw zC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBGALLwC|EKm zSTZPBGALLwC|EKmSTZPBGALLwC|EKmSTZPBvivjH{4?0qt7~Rn)fwL<--m)Nw`(iY zY5cuDgU#hM&P&+5gtcoq>uAJz37eO&$ArbF@Sv~1{pBTW_)Ie|VGI6C686XV40db# zeN6?6Il7I>F0hXf8QjKXa2u1|Dh9VP+0$Zh8{VrgWH&_y%^la zWUv9S-M(V*9g`g`HcAY+7hLxuu`9$-;EwAq6njDJZL#;o);A5v{@RCgB>~DZlWi@w zlUR4LDzQFdNVVmS-lfiAwcCU*yilHns+bt8@(5_ap zO~Sd70B&P4xQ*E#+{R>Z8h4PuGpW&{wDUk*vn!|#oiMuFkQ%P ztP{?a1ntE(6x&T~FR`Iw!^O@PJ6G%?vEPc_BX+;oOJa+~z7+dftk`rW_qk0tXGO@x zb`t9^Hc)JY*s)^gi%l1Ug5@@#U@@2T8SFiF3u)d(j8LUq#3-L^PGYnDnwgjtlozE} zP|olZXRDxG?pEO=tlykgVXL29X?1^T8186mw8AhPShF9AVc4z=FOIw>p5d}I&v2u4 z(8M$RRC-L*Qg@$VDtWhMc&^D z-tG^%y}w99ihF+>eZ`#K-;v&o6_MfnMXbecOTJ%Gsp7uo{RQKGp~cDgU+~w~dVg8j zm=%%X{e@M}&bDtRD!hD%d4Ito@9!*cH#g?*&Gi1h5qmB2{?7CDcJ}_^{jK%oih{ym z=N#VO-hLi)d4JFLR!9293GZ*C_umTdFUt}~-rqaDqsaTqj&gZ_aT%JI(Kiq8FW1W9 z{XN%L%HjQGN#5q3oWgb+m^n6Os-*|HIq&aA-a7LBcJSpK-rwWBQ7-QV;_Tl|) z#5o9B_Wt5YTM#t${^G|+mVaSI<3@X_+`e-Tmh61rNV@lTnUPHIFG@ej<_pCN?=SAF zf}!^p_slf!FDn|y-M{#L!!~lce=oN-a0A!c6z*Sq^x^((v?<)bZ~?>pi>nXrU%Vx# zaJ-96;r_)39`0XUQm@z)XBqBad{UR#6rWJIe=(pe+`ss~{o0mM_f3n#0u^GWq}au3V$h&$cZL`=DA_czE5xo5gWhDjr^Qh2i3}PPb0xtCVjqiHmHn{YI^mqD zYO$@w_7vMk4CS71iJd8Sx!BcWDEGv5?-F}X>?JXjdt$qH#Zc~v?B8OY>9oU$OpTr;42+2Ia~Apgb{G5?m*Clh`9-^TeJJ!;@96`-RwYu}$okJh!n$ zI9C$vF4jY=x7dMVCySjTHb?AIv3te-ELJDhAoi}=rR7N-`))wu7Q1gQ6sB6oaB9gQ8?RC`vLYN-`)) zGAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$i zC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLY zN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`))GAK$iC`vLYN-`)) zGAK$iC`vLYO0rIN1(QKhl0i|DK~a)HQIbJXl0i|DK~a)HQIbJXl0i|DK~a)HQIbJX zl0i|DK~a)HQIbJXl0i|DK~a)HQIbJXl0i|DK~a)HQIbJXl0i|DK~a)HQIbJXl0i|D zK~a)HQIbJXl0i|DK~a)HQIbJXl0i|DK~a)HQIbJXlI0cU545O}z2iSFMfs*(Q5G9`-RxfV*8509m{rb$8ue`W69u-CA&!M3Nb{**zO-<$n+#bRbH-(r@CYx ziG3mDR_y4VD58IA=~yv7N*Y5<5ifG_flmVvmZ=7kgIh zd9g3Wz82fme#vv6w+!b>f<46c7ONJk5gQ|Rs@Pny%f#*z`-|8-vB$*T6Z=T4%zn3X z|27EcN`kG#ekRsathX2xCD(yqb{@_S+*qYItGDBJTX=#!@@U zT?$)$cf{x18MH{>kKwWXU;$y#;m@&y1Mb9Yc8b$aO~=1$N7~Mg_M_PM;rg@N72k%pY4ZoFlT~f}-Kj2m zyt-(rCkgvr;bZVbLFz2w^fbJ4(qlL;3mU#U&Wa{4-U3%y!#f!TlNZ=$qYbWkd=gUE zed?<3(O5QPukTPUc_aJmR3;||mC36*!zzalV^z^NMNi^WvHGzdpOwv6@+}i~8*MX| zu-h-ASh7`2KCre=a%o~%;;6)tiJ`-eI&%J1_F1I=8PUgd*Yi1Tn!28UVfUB{>wdMh z{*PVHsF#fYa@0&7<_}}b?O2;Snh!8~w*3Xcm_pbL5A=B~|H1uv2igxG>|+r8(tE+@ z^QZRv8jiVw;ONm~N1o*E`f0tMz6|SPPplR!Ey__RxtCev675g26?*v!bwy7GulUZK zWE)4=9^-Z5c{1KEY`irtXxxA@UC`LcgM1s4d?#Vw#t*w7_-7&9y4hUOxOum(rZOI4 zV4Er5?{HhX5-6Bcl+#x|$Qy3pyLFar^54Gvx$Rb>{aLn;jlR5J(MADSjqod_f=PvA8bXCFVf=DW76j_Et`U2lv`}4 zj{TXXHs)ZS5OX)+F%D!~%E4Ua%hBk&1AMuyzm5mKNMQ?tuCd>egRGUP%Jtc?H{dpF5kF-&`Ev9I{K=Q2Q}&=Q z?-M&^88H-`vJt*x9a=2%jCPm#uCWe23WD=`J;go@sT)euBF~rk4m{*Hpc{iTf;|dO zjAfYWPP@+A~$syj!`31)Z`K}f4 zEVBjN8|y=uFh@APeuvr=Uy*Q#-(u5VHpRWw>Lg8SwCS-nU24-&HpL;|W7Cc{wI`#K zerZ!vtdqvrbgE4eb$HUI$J-PRbwmx|P{*g`I9u;2n_5MrN%(a1GLBD$JtdulfoFeh z(@pI3KVehc6yRKc-=^3v{2nn~>dy57I~GQTFh`#O*)-crGPv={{vh^%*h69q#NasK zx;Rf<_e-(Bj*qN3oGS^o7DIrM>vk90Rjf*^j~FgN_BUP(gA$X$UBLcs7Q0>SZ(F)b--J_#krK?Zmo@^%S$KG;B9oY@C>7O@#d+1DgHaBzBwF{bCP@y(tFQ z0M{+HD~|iuCY-a>9I;Mf39%|MI0U%vK(P^G$BJDbmK1~kn(JOERx4I7_Kw&GVnr4& z;kxUEb0xtBVjGKLU}3h~PV84=a1!9vlAyoXK(P^G$BMy8!2ZUIO%^*}>=v;*#GVqv z7);zQoCNF-V=$4e6vG%yWF>Z^Wji07zQ9_yQyNR`b9QZ>@u;N z#BLM2U+e*~SH+MU%l-=1<6LR5PB>?|>0%p-?JBmX*ok6e#LgBwSL}ANyTt0m8pIZf zy(spv*k@w1OygqjOM|)L+}cObs+0G7AnI4oRV5E=(>M8kRk9oz5oNuJxK zGWl#(*MHQET%Jh2dRB2_=8&ywl1EvT;;BT}zClfLP#r4c->hR1vpys1Y@n)SRq@Q?%W6h0m|1*awFNHli=nQuuzKWz>SV7@Hf&VQoQnGM zCU&Y!wn3P2=tXVp+*HiB@XK2bZ>(HUKY9GB0rynY-(W2+Y~#&`TYzOTw=}%r?IE>T zduXz)ukqax%g{^ZMQz!OdKk_g+G1T_$+u;pog)@->&0B*>5~>UeEji5+a(-Hxo;3L zso2hxvnz{(j%_<9wJ#n2n(B4)i>%XS=(IT58HDW`p3md_P$FLEGGE5H} z{o6iVnaT1$voD_t>;6k?z1Cr)9H|Y5dCaBUuC%6MBm8(7W`7vtCOvF)fUkw?v*j=m z6XFM2JIeE?3Y&z9pvh10jbKA|=moF)#!j%G+xQIzsrP+^g=(y?YIGO8>Vb*13j^Kd zJ+(;-1KsEBl){{0pi$nCsm2Hc;WvK}+~nFl3Ipv@c$l*&hH{dx_dsj|S;9c$tTT+0 z83enu6b3>ws<0QW*-kdlTF>A*Q}~?}pE+gx2+L#`H{#S0Cl5Q}m*m8jAAwL__V)8v(fQT`QE9`(1?V|wEwuP?; zKU+x^+WVV1%p|@J>n?%t99?}kl+Y~I0>Hds>5Uz@oeZ>|6 zLTGfC{aIWr7LPzc2!{uqvx9fhJ=o4dy0~%RD2{N(iykt&zb_ZZdO#2;)dTZHb~RDX z{C)KBy!~0<-XnT=z?VD5jwU1UgB`_Sw1Jh=j^4%0zzJ~%y!001g8de z1H{at$X`Z)2dCyOZ`5*t2kp)ccc-O*&VJt4<^_k?4Po4flU#zSz%LzY!LB72wGHP=gLYz@iqJlqcoT=_J-IJ zG5bt}?Y?pCL#ioc|Aa-&3*1E)>~|t*hsOq7NlT*?ZUb5GhC5KPG~kpyEoevwE%k8s7I zTUDDI2x=}2S=h+gicxg@YfbW?Hgzl4={xeZ#LVJD%+oWo_@ZhHY82I`JU}%gJ+jbt zHsTbn+RuZWf3Ksi47kD?f5-F#y<5zFWBz>u)~G z0|)07*B?ExqOSh>fveiy3o?3EaTpF*RDb=Et7`5ow=uZCvA}{YqT32v{DNK6HdHrp z<-+>w%`kLt7^Xl(+WYIO3Ux z?@oH6;j51aR@e~VuiFsc_MtMz%6D(}RU`u8hedWBm>I0f2Kb(s7JI<^*5eg7sthbH zn{jmO;t!jJb^97^Tdj;g;5&x#hZAisJ^rwPeKIl=VeoxlfkUzBTE`!9CBn26fQWo5 z2l$V}zIKUjX@B-`vH&rGH4i{sohtxg$6jz(MgU^>?>PW*$M+C`xEp67NDn~V@2nyg zA82N5?@``x6F;Z;S+tX%(@7Zth;<7424GPDVw|rxHMW6f0uU&+8w5XZE&u^dh+pZq z`F6P<_@P$k%g?tEgn$S-(OO;MdvbW;9l-$Yp?x<8S34iv+Cao_y?J^d0`_t|{mvGM zINe{{v9*DSk`@9HSH(8KKtxCD{4L+WQhz5<3b4_aKPxBRBC7$v@7)IJGiz@{UkysoAQRzQC)CTJ*1FR}9*q=q= zTwX_!iEj$NG!_Zg3>OLn-%q^vvRKF_rvyCI4t7t!&~bsy@iyDVdWbtu zI(h%=cx#*k{F=vSEEU4o!j~f(1ri1kTZnc#BLD$`i_w`(ZuxFSp`U*j6cs!fJDHh5 z3B)@FdiTu@8GfnvTdKpo(3c0>h}k?`yd5_<`$cR!nL!C`2hWAA3PY?_W>5krzuH^H zy)khI3i8@vJj1ENsY9lMd4zT9zVPMf)UEX8xM?R#fC7)M;mg}dP7=RZ$Bh_yQd$9t zA>OWsor^=g5u8YcS&s4*hFV-<>*fLxXo^#Fyf>|} z!{6Tz7`L{Qo%QhL2{wPfFR$>oEAq(&g%<`@1sLAE5r^9J-~Y-@(j<8ZzX{ePHu10> z3t|?HHbrdWEt|ql5|2$-5%NhJ*?BqMrubUF*`|1(z;XI-uvi4FO1Igx&ZdY>%(eYM zKX=&_u?hS}NyR3L>>GeK5zH}pi0lH}I@#^vTxoz1KH1;Iu-9Zu#NHA6Oblfj*{*}_ z7v@UMB$#497u!MX5V1jGxIJ)Pm~+`5(u2r;D^@F3FNT{a+r1+8otS-Jg#E1_&Xrmu zJH>Dx7;ao|uso0ic zyNT^3HcV`!7zT4=f4>pCRP0KzTgC1aTO{_P7}9yz-#23A_I{If4(CdOZeqKL4HCl; zZdjuPmZ#MIiNW;2{xF0a+ubkrfEbEFu$@Kg!ggD}Wwzh}_*WEUp zD+y}E`iTt}J4S4r*y&=E#HNaMF&)Qk>=@3qBGyrBx3QL!n~>QAmE~zomfn;od$c0T zY%+I6ZHE8Ca+#3Y6ecg3(GWq4HBMeyk(Rs!+gOw2CG6E&B`=j*2dUJSRPIY_l9$@x zbkyaOmyj2xETz;5#EGe#E&P8gc?rK=)-ZX=?pwCbs1YAI8@<;-8&JtKc`XTi03c88Rjq5TFl9|c$Skp`<{LYUo zD(#Dx1+V$WVNt&tzMih`|j0y^TPSjxnRkS=2xRcU&VpbaMvCyY zi6^;H-Vwe6)ANqn8Oh8$!gm+$u?H87K4w@d?+9N~f??hfzHsoRb#TF{=72cZ9D$jc zNBFIP9nQ==>S+v_NBwMy(9JZP_Oj_Lo5I#oYg2@Z=G%0XP0NC2r-y_*zSBW7+=UP5xZRMcCov}7K^;p?kJuP7)GK9wbHtEuM0T&(pT+9L8pIZfy(k9L40^Q_=pW(W&&#>Ju#bBf%8z6R=*x6zjnUU>Kafba}C3c$_j5KWbfY?J~3&fC@ z#CFI*Vt=2Db+lV8S*LKWB-mMOH!)N=Wjj3VyB7yMr?}Mm15V5RhX{ib}PfVwGUAt zdkEi1@$4bX97?4Pl|6b;JG0h!NXmj$bx17o6r!T1EQ=^3bY%t{>zxv*X6T=P++DN*+CoH*DC@s_Q_R(5WZ;F zAVH{Gxb9AD)f-Y60)TV;ii)_?lR1GisMxx?t&c)f?HiIM{$?kEzt)o3R*DH=gQnZ8B7*=FXX|_1Db6?%8{hI=Bj3x)dlF8?(P0=NitGL@?1`+b&!# z@XH#ZJe+9=5i4Mny)w+<{^I5yQe#6_hKUk?{utXJBQ!|-neXd$^e_?zbw-4yizWX2tmOy|6fRve|qB2pPVr=g&r>RulE=_peJhR}vupMOG<>2lr&C%ftRgi=o;I+3&@!5PL|hRtyP1>~Epi3bF6ReqvV_ z+pQnYm3RUW*{{Tq07TYP?0B)!Vl%~Ni`^-9w-{1^*x!7yXT?x67q6BCAB)*1CS*tj z;<`u$V!QTY8;b2FwwKsYvEgEfWw1ZQGT0wdfyj^wM0SnX4PrvbN!zrP_!gfXsF~iggp~FE&u@EU}4VSBqUQ_MF&DVjqfqB8EyR+&@%8;lAu7 z)?MrsvDd{oN=yqieQWI&(}L|KiYY`XJ~?6v$;U6ZVkJ#O3QX(qkOD#rR%T-6y7pPj z9#x3)c)}zeWFp3b3h6P0Y$-fYS^q%_k4GN<|C7Q4*`F(g$Nd_qaKV~n@eFHSkYLj= ziwBotD}x04*oDI&L6yxhNPwU0=|O@e_A$uJ=fM|wg>C3Io33?`AXh$5OHqQzt#N?= ze(r1U#46h{`@K;B1I3z02`1%=61-g47~GQ)C15@e#Q66UC7AjBMhVup9SedfiW1Dj zVz6%EfzC3c1fz3D2`DRaX5(Dw+l{huuJ`4aT9J*zG@3qzzYhj0j$vPq!EE32UagH5 zB)xfi`ym`zP*`s_5fkrGwBQ_XPhHsFKGnFs*C1N3SqssE-^4b^XhBEoy52YVw!bsT z&fz=rc|l?Dc8=_vFa2D`b1brD=j`YQ(2DGwKHf0O&Kcy(xw3OM@m5DQ9|b^m&i%fp zuiKlDXpiv0JU`}d3qB8CkFA&`JLdymvAOIVJeY*UNr_qNwDjzp)4WY{**RbPt5J%{ zGGC6ebKriD*M%+iqs@_>gRfXq=e}FW&f(E6FDNK@I;QBeW#_;!kA$eGB5T~Ufinrk zlseqweK|cl=R9Y*vU6xliC;O{vU9rl-ZYn;10563&Oxz*cy`Y7zPz5_lkp}%?V!?1 zl-QIdJEyI0GIw^)p5AU>I~7f%2&wFx4Sa)Zke#!yuh6ROoN8~Ho}Gg)o*;;_a~OE| z*559;$>!nZprCMCQ0>_{wZ7csfB&lpg6&9V5CPxuYZ64b!TLZD0Y{1;0&Z=iY}#nk zR1o1oTSkd;NDx>uri4!kM-o=m!!OK?K~BQ$YmeJ}`)Y zIp*h(U0}VFAwP!<`8j0B&mlv84%rg1cf>vuLw*k1b+G;7x*LUaB?0nt$aWArL=3?x zu6u^q*eId4747Eept~{J833`h47K7o1?FNfY z6q_uD{2aEsT&!Mff!HfzuZtn?i|c+ZR%~*etW7vq5}=+48S05(uEgr_rC47vyVk;X zCy3eg7P7O%t`)mctU>GvG1L>`c2Q4++eJMQGE{#c>m1IN23^EpjbOXJVnf7^6gy9B zn%H$>H;LUNhU^_~w_a?47;<*F-FL-47en?A*DW-;&UWjDb2jdP*hXTgEW&me@|EkN zvIrT5d?iD9Rx*@lCHtM&9I{GEX#5Og}gq}--EyKBz;1^;$i|s46zt}-y zhlqVG_N`dXpu|GEWu*lrG{#U(@(*5(vMRYKk=&zCav59^qcXWXv3&gA7-Xo< z2N`ON2P~2=C#GPKp*kC6=<~|y<5?!PD!HPn$AYq%$j3C@JP`#%s%)U4$7+(xP%yP` za$uX}vwf3QR)aZF1u;;oaqZ!#o-)eY09Kq;@XcVDYZEduihKBDxUd~-q!cFRo zaUX!6W0Dhg#70;0jmHKNnqD5A3G0643ph;fOq+z&OU*ih&@8Lpx(HhsI;)s(V6kne zmhV{R$(U@T3Eki?XPk?+)3G-luZ(s8R_>4Ru>>P zybsLAXmElWaIBycZMhfP2X_`d7~JbSbE0h=;X6Fyj2D2xBmaWHoXciXM)UxWO1q$$ z|10pexTj=h+~9#xPj7RZC23?I8)y$})4H*N=6X-*HCZlqc33QhrkNa_!@Oa8KNS;f zllb8s1b=sDLR2uI&~kLZq5#P;-fm`W16gXa;4v}%@;S!_`m;3+f;$Sfu`uGS;Jjdm zg3VWLcz(`AU+&@^AwP$ogDqp5&#cG-{c@OZeu#&xqVgkO7fvV` z(sF*zp}x6z_#`9dbE@^eulL>|76ggs=fGBtUsd>Y;|gRVMTswCZRYRn5L-K5jb)Ct zh_C%SUyiH~*Z6YRSdb!1{>>ua!L%%wv#h3Ztbho1421-OARax5$8z9BkM9UXR6H?= zVaM%bZ$zd81lK{M@9z4sIALZS2iMeK-;<89QxdPT!Wuu(31vYXF5`sanX~;rc1kkh zHfZ#O@5nYSx-Ff~mUFpj~0wK{yUw z7W(qQg5E)cv(t?+{5`H{^9om0zNRd-XB5~i^XpUWZB@W69JeTY>Gh7pKaRP zrj0f=5k6_DO%bTU+j@^pk!yl18yr}7n+~z*RGZe>bh=Gnw<%mB{p{4iHG=G#<7~aZ z+7wwePutW))T9q=`h-n?Y5RwO(7iUremri|R3K;{7hBzI4=_hJ3E4C;xJk(1CSg0c zNyy+PAzL8!lo-wn+kGj9<0Qk8a^0=PwiW9xwyPK%G+eij815Tn_;jMEuSvRr0#VW)Gh~X=M>moSBbrBpQyGac9F|vP( zEfs^WgzF-4jqTtoA+xW`kZl~!l?I!M{Yq?iF*r=PE*vK84?j`J&Jmj{cA3~sVz-Gs zBleuwmttRw726fZZL|sJEO;emSAMuJXv201F@%Z8`iq?=hLM`_ss%R1FtQKXAH)zQ zB3mSe3>vae#l8?j1`S><4Uj>@cF3S1Lk11mPGa4~_7O{n^%d(ccB|3$GZk_CJn{ci)__oSk#f}j>Uu?SAU1Il& zJuFrywnXe5v5hQ*!(&63i2M9=u^q&A6WdE{gxIlSXNgS|n=Up}>}s*=#gI3{eW?!T z)-Z!cF{M<&*Hqq2V^wkwA9bg{M%@`~qwcJ*DAIpA>dqE+ z!$2&lD*35|cq*PW-{MIhR8BwH8ocHs?!098!9F7!YLf3GfE2l3&b0A%=2vx{Up?}R zs^lwY6<5tX%*NX}*v8v=Hqo_h=PC2cYm$S*A$RZD{Tm&T7hX$ep2+eaM~4*pNG@M>}9r8;rbTL6cYO&$9}) zliONoWFEIQ+~O0B=%R9t&8}R`_HAQh?$kBB!*zWnTYuNiQFh+&QFczQ z;Len@Z$xe0Vu!`?9cVi_wf&BNNo~G(PHO8NCt>5o7oj$9+P1AJhShl?HLQ+>RI&`K zbHL2vUmzxfZ_-%WOl4<#JBN18h^*MKIw);hHlxN3F&#TsCQr85N~)SkO=la9`2$2( z?EPN-xLs9cGm4w_#AJk4hS*ia(8?&AOAoF5);=tmp%r{#?Pfm&P%nM0Ln}FwWDc@B zTP4g0SwbsOD5b)GnI3Ab`r98yfN7omrMIs)z}D%F)y(nGB$R@mxKSwOi^63=z3eae zT?8wBJZ^*Eow-6M_!MP^PVfkIU+?cek4GTnf?ss}$B+wt?6xj+a-*-79y-AnPGsq!*=$EOa+_@#2udV*EK1K~2s zxBHZx;EjTj!Ir`0CJA$or^83t#im2JXZAfG8fT7ft#RgjKalh?wih`&A?9O=f-1}} zYQ=augT0fc+qCdag8mLLk=lB2yLJxvgPzxL&LJ&w$v34YDLZ-3^YkLl zCUT|8&_NcD)HF?#DYu`e3`O7_TIm}vjlGYVK@lAA^WJLnSgXvS2wGJ)ISmMk;M0v) znGLd&9cZ+5nEvr|IXd)=KnMU@baup5xRivM(6@p^D8#R8~dtFag(sZ1(R@_N62EfP0|1THpTYwyCD_2 z*w{PKc z#4!F1*S%Qm8nGM17KkDJh1-Q+h3mqvLbkD8No1Rab0xucVqL`$eqcLL3X7WN_mj|O$?d=5Psk`5Pra1N$|Q@qu5HZRbm(%nCo^3=SqTKiR~_SgcvHuavP`# z%XLu`mJCDvl3}P{vISzO4oe273EM3fyUBD4*=^xmE2Mm*-4fC~B9&#^S6`SW>Z|SZ z`Ck_Gh}Y*7^^ou>VSk;i5DEMEP`Z%z&sTMQ-lTmZ`5eZkIb>Cwfil zY#A@(*jeE7q;^?LTHvZ_O-T!=_5OB1rVn$ebvyQg8B*(kt%z|Mpw{u0v^H6# z!rEq2>(%HkxTsmRj$23T)H-OG2M(>ALR|B(Qh~N!#u^cVl?{gy9GT0 zut=>_w4NB-Kr?E6f4g#8Q0ov^kcKz-c6W8Lgx&7SHq-cMR96#EEom|d|C@Yl}Fs@RwMYxAYp8-4jX`w=z21;q{t!qHLJk7>9} z#SWE*Ux9dN9*LqXiX9F2@`h1r3+&EF>WS(rWGZ%O-A>+W!`JYFtXVk|mw+0t0H_d{!LkeREzUz(C|Xmfp; zIX5`%c%*5XE4GbHm5&qfg}0g#Yn7?;aUDT1q?<6})`wNzN0j};_ySKdPE(Pu9k;7w zD1Pkzr{0OxBUCC%_5o#f^3M9$>1;k_1<#H#Uen}Y*WghLHAEM7Giv`ZU*TA@&2875 zvPWP5r-C7k!NEE~U%#qws*m&4d-?JNoBx|HKksjO-~g$xNTWVK@TVlRlj zB8HSZu4^ff;kxUGa~yDn{UKb-c3Z36u3|_KV>=X_VLOCm$j%p=E_R6+yjX1asMvh5 z=f#j7#da2h4!5yxI9C#EB(|y8&SJZX9U%tC82dX@?AKy*#4Z)PQS4T+r^ObDq5da& zE)AB8nJ*_~<>6dOu%j4;p5eY6A~s0uOfjT5upL|p+(uFit^_h9udv-SV$X@aDTXu_ zw);d3X)I)IO?I=tcHx{=m=G%$+fl5W7+eWlcc9n^v17%OVn}1*c5fHEOYCoA4~sn^ z_O#fiVqb`DXp)`#vPn3{paB_z2Hb8>wd*Z*sMuh!31a7nT_<*vSiKlR3f#t5Vk^YT zO+#^CI)`&5!7gHZh*gSJiwzJvObjV2?9WEN3%8pydF39vJ*5RE6q7*mN+-KbWKCYN z(PXMp$t&h6sLd$I>Gp%Q&rcaOE0vtG!0rO>23V`)ly-JFme7z&wP;Op%GY*&kT5YI zpPbU^)i6Qk$Df>npD+JGa*Ewr{4(?!BN`K?h5VEwGxJm0*cI3`KV_pZKLsDgEWw$K zxQsnu3F9&c+Z@Y@47R!S2+b#UH^{V8ykM>1GQGp5YZ;-*ITj2vRdAxSXR5%NX=d7t zOqHbnN=Bv%_A!;If)8P~fK8OCGHq=$ReoD^MR30|RT{Bv)4XwSM{DdBxWQxwaPUN@ zr}vj^Hd6)n*+_;o6Mi|&J7G=I3AVyZzWj(qg%a(ffmGJ9JQFa6Un;E4G2U)mQM7?9 z;TMQ3{77oWSTL7Z)BiBfs?j>`&NzsQ&8`yoKr*ZOi`W5Iw$5u|gR9~Mgki7XM< z_-n1mGx@zYY$?y=!q^6)Jd^pp-g~~2$TO++;o1fmv;!8ev?GPLD96L^Eu2hq(4 zz(Cz^>Ag2+qgdsy^65X0;7ebQ@=Tau5O*)c127QR__>`*M;PnNQDos3y$Zp2-KX^NXYShj$X? znQY<9t;#bQ>P?&GnY`kgi1JL{@^*T7d_x&7FW@=WkGmY!!4 zSdW=`Cisbf@0zqc6MV4Gm)g`G za{JJ~$TWGvmXT>vX8VT-4Kf7~p}}{4Dnf(D-i**-j&=>QX|{DT*fq#t*C2a946!LP zM6cKmnI>d72C^^3@OF?P(}eA|7DJ{9S$DBr#qfKE>-G`DEsYGHZ?21PPqNF!@PQ*k zSeES;i@hoKq1Y#42&Zyg1X{T+#&03R_$_2T#P$##jY2-OAL2QwtHBt zP7IMj_V=pTXJTK8Ay&h7NHk=Bh}DqoC3c9|AhDyx?8*$+#rQ2;cb3?FVt)~vC-#__ zjWZOki;-NoFW-o*7Av)@klVmWE^KF?fspMghMW_&s}eg=3?sR4-QS7L5wqxKxbD4T z4~x}_JuS9K>?1Mcp>P}Dh^-bYH4(%8Lmmp-A$~)KL4?T$i5)I>y4VDmk-#>_9P$+QM~*i4|K+hpbIF z*9zOob#`k@MRyd~K?VuFWa4I%WbZvvsUu~N9^A~xE6+A9ljbCt;-mB-_A`wW8}Da#t42V;1t?N&|KnS(nzDtw?1j4 zPlh7}zj?AdQdWmSkL#>Ode9>y-qFXdDf&?Q+Z^K^Lu@WR-Z9iZDVgz(cdZqysrZIk z>v%_w^pPPRENHZ?HH~*n*6VDjvD>0sdV8F)q7X-6h zh(^JHjSB}mi-G|oe7*B?+ewbJx4i{BNy-5}&bPa!$shEMrzd}$?|U|)wN{ezym|T< zD>I!9i8(!@NXKdZ+IFtDkf?;q8a5-GX^4Yn$5`p-FE$-x1%jE$AQ#6r7$t+u_w`oE zT2kxF>)V64RWX&CEg1w)BrT{AQ@okh64(py#L*N)>{7{MErH-0=nbP}5RR{sD;eY! z--*A)93mOk61?-UHn%vMEgDVno|=ndFcX7vA3yn#=p9c6+1QtxO9p`;NEM~P+V}W! zlneryfKOB8bIGugUnkcFxdw|xv4J35B?u}X98zMb*=kq z8WkH%gQ8-bnh1%C0~+Tk-O??Mf}()40Yo4u0uDH%fC!QXrx?X4Vw}(zMWd0Z(FVH} zXK)H))CT7P#W)YhTi^PszN%AwPS@e$y~%s;_D`&;|Ji%jt~zz9>RWrQwebx>aX^Q~ z?WQdk=rSD7;ei&HN?R^aiiW1-X-Ecx9O7-{27`=nmh_-WxrX4-=Xjy)V2~O=_?G8H z+05Ho#bA*0yi#s3$Pvzp3I-YEMVkkM%)~iu{uf#on(UKY>jEAkxz+_p+_ZH8cYhfY zqS<2YUJ}F5x`11F(z=ii1;MiwJ4o9W@Vwp6w0xy4VN2*~OK5voU%~5bnYJxNX-iG1 zjqhgr-PLRYJ#C3l5O_Ht6a--ouq`}iOV}20EyL>rUn{&maEpM~2X6YXDIgTV6yo?> zZHfKkW*ONQ{%UH7wgs%wwm>%7dPg=ZT;trZWUwu;9Bd0@O=53};k>XMYzr*c!TL?M zez?X^5VEbr4pzAVVq?Y56q_oBP!R6p0kMb0o)mjl47LUC7q$h~{g>F6VjJ5zW4X=4 zHO`4bwx8G;DtC?;Y(Fe_iP(c;@cH0=pA&=62N~u>VO`{|Ap1^irC1kJ=Pb8nxW+kA z$g0JVv4RXHa<+##QOJ<4iR=ootHmA?s~3A)?0vB>#4tMw+uPW#F79LVaLxKBRxLJE zY=qePV(|H3d$1I+Jp_u7-6(de7%T-Whd>dQgQbA%B{BQGgzRlGBrIjQvT)5jR>WW_ zV0-(E9VmwI5Y|O_2+P4fKnD8&+3jL?iec6hmczWQEQjzAGFS@8?1edGYlLfM!B%2O zPRepfKgzmDKT0-HY_ixDVn{^Fa&yHR#m1PvV0)*AYeneqdi(iD`r9HtDrYng``FTc zZT}rg{C#UTQ;A0c93IsP0Uqh|-gQ4(SVx8Jp?5e_NoH6Fo<09JhjqArK%}F64w6Vr z<4IdoWXFV*j)t*6zmDE5jO@U*maC&JsH4Y%e32a;><&sry^*a^QFperTt$7KUFYc- zk2zMV(sne}maDC(vtm42nvi6FJ0Bd&Subyl2QHzOB{iol?Qi86<1r*lQqO7C6kME= z)Qs`Kms6~y#t$K%q^`BI8U!a6l+?ISWJ>Dvw58v9qo=xnLxjd$-+r%Tj{q!|)Qr&R zlP#&c+V|dqq>dsq=tv(&XdGyrN44yn3hR2w+z1VZTx?aWte%E_+4m70V5Mzw>e9{$ zl^e@yCb5iVHLmSdn2aR9tnQCCf+G_%;7PWgy=1nx7r9@q(91PHvl);<%2! z_fpBUnaPL^h=OEb3*Dw6*8_;P`Sy>Sqv;pd-{Xl}#?w5O?6H?-l3q6y|@|H|QMH5VV-X_ydf`yV{X$b9c5|A*%`ROtrxub7yP znN|*J9Y+eaj$0PgbdxP3wf=x@e;ID$8^ShDJYqI*1`)G?Uld|Ca5X@^;|BJ!E%8%; zddIg9^^Q9c)I07#Q17E{3Du5!airSsH;qEojy1+?kWH~(kRfJ+3^5yIh}j@R%m&#K zvA4w#v%zv-h~aTWwn4bYNl3`H7CS`c4i&@Ag5}NsYFE&@KQS1XTq<7$czZJuD zBxH!*V4u5$Yh?k(_sM#R4G=p*>|C*lVrJP2>&_HIo=LWchz{=eA+dU~#bSu)V7U*( z5Ya*Qt=JD@o12d2KDG$gY?fBBUy2LEW>x4q}MtAnPpFRct@8{l!L!Vd@dqy-W<+pX^bw$Hg$GDL!o*{7bBrX(F;U z!!?`OMGP4y+1}5@ka3diX|V-jneiD5>{gUxLC`S%|G?xT7;l9Q!KM|dsT-s`fl=Pa zhG-sIQW8NkS}V zrA-*p(7Wy?4?hWK-Kd?>yL3S~&q(bMpF6~7%9z?7i{4Yi`8+V0ML1i&!}ve zV3{B<_cRTb`{9MgWj`E!wPj>Hb5!-zPHY-UACLp0_l!!*>+n9C!%QSLbw#?lL2S+w zL#VkyY_5&kLHI*96>PQw=_=H01)6OgO|=pA(b8H( zeQcetkDJUuTkS4SG_}dqVb9vlK*t)Fo$#9V;_=26!=G*ZVll`3m%V<>pvYihwu)b3 z29anM4;U~WHKRq~feyoT8C0E*$z&R8FcC)=GeQ(G5yxw*G!e)7b`jA`0TXhrapHIO z6G{*FZ)&$|xHf)frB}l^v7=4Hv04*x#KZk}`YR-C3_r7fL2y}Ut0?h4M&S*e?6Z8f zia5OCIgd=}>usQ&Ri23BAFXZ=mif*x5eMD`gP@ChEWoZ%tnC87)wJybqZB>7y%!7G zF7Q@ZTz-nny`|i2lAMTRQo=YY9w*{9ugJgzNLh>xSt`izL`4nt zOYA;vcfh;!7xvFGJW6=;@OEm|wdB2oNg*}F50e7KM2mq6C`6Biwgv1C7*&iUFiOM! zDB6Z39_-t+EZ_*?PYbVt*ou)1T%a8yb{M$dY9w2O4zPKH{rsS{M&It5ILee=0$tkC zcbOcjPiKw5S8=wNN-F>4gc)!gjyi%if0Ee7VZxm_*)e8*Xeo;XYSDhMVdCpe+ayrx z6mPpwV8ci&k?h%T65FVaV@vi77mTGONu08j>i`b#WiQmSjRNJ)^R6A@-v=ge;e~%> zUYkYaRfG{;qzCS4hkZq`W3bFhudn;Y$)isnIC|KyQ-+>u_X||PZjSUS-vny~wZWl5 z-{8+?g1~3E1P1$OYm}^S$8=nM0}nBfze5{z5N-_Esy`O90QBTrU9_q|;sk zcqlaU66l_U1?IZ&*-d9Bx`JmiPS<+&@1~F=SwuPn;g9y2%{4h5#{+`{?qx7Ipzc++ z{Mwdx*s{zzaiAFp@JKPY>hbv5!aV?AB@7Jj*b)W?T<0(_;H!gy0k<3&7;p=Rf#Ee< z!oYxcg~-6LzQ62uw>@EvZUJPI#o!h|hL8i6nn|BX z4#+x-A>@E;2eF;S5OTn}y~J=Gk>QTby1y5LA&cxbF@zkD!K}c#u<((6CI$lo8GiLF z2Ll5ce4fY<{6V&tSfyBhvBSk+bYNW=9a#4cvDsn|iNXB9a!-pb5PM7PJuw&^xDOZ| zSa%b#3bAd(x{D$BfOU@)!!#FUUpVa&=-?id`cH=K_}dlUSpe1@naMy(DJ8 zx{$ps_P*E`V&91U)XV_f$7bQ0xl4#ui^0u+<=|$3wX$H87^cG@J6CL?*hONOiQOwU zM{Kd!>tbJveJ|F=OatsoyKt>6=p?p@*p_13h`~jHb>X7GK93bUQ|tmUOpL*Dw~1kj z46=WUHHjf0f&2Jg40$Wb?g-b4aAjz+TUL%CBG;9nw@toSYj=n$n@zN?Q?I&3n3JNa z^eN0+(8{Jd-QVV)c*WB~K-T7)>$I}*n@{Si${)7R3e0u-rMF+%uBvWKEJEuZtuh^2 zRrf-5>0{-0JwCODvzk`dJzKTs^783-Se@D#wNq=_^r<^}%bL1JtGks})h+B(cUGHj zFZZcCwxX)89%hJ2E7zy)cr!tKQ#)jN?T~r3rOPjP{>;+z=f!C(`#(I$(pFZTKexgr zpV+rHoNkga1ik9^?PLPF_dT!;EcK}(reVg2%IZf3VV;MExnqCm|Mx-s05ek92bD-G zX*nZ%&oOSPD<6gd;!|%028*R}y|^TbEc7B3Cgfi?AwRB?l`GKp(bHOU|3fQ#KT=^0 zm>uH8ACCUJeQ0SoAGRI7X|s~{p5ui#nHKJ^-91sIc8|n$g0OVEIkjf32~7vsV~urU zxJ}%d$97O#`Z&x6c&CWX20hx67*~R;_&0E}8OaJBI!ZxB!dvZ-7vt;Q)5;T*JZkc?<{7SgA^D{tvcXZNouEoWKBk zudnWa6O>+M}o z&{%-GNO8u3tG%UMW5M;#wo7RF0>*-4ydX!C;Z<2-|AOE*F3#a9ffvyr__(C6vsj!T z?|evN2L+4;7-)g&EYetTuD2UIK2+NF`OfxFh)_>6{t5%w7k0Z;j{ zjL@;HFb9LDT5vFcjKa$xMhRmNfV3e1!s=Tubx}%*zftl)@I$sS0E11K>JU~ED4SpT zcBjNH)8Zet6NDBA!PCCW*mm+($w$Uwe}X*Xe^YNh>8+5*=>TuTxLRYm9;J!I`}9ix zG!EzIko8(Rf{x)Xa6}ljr!#)CHHd%w=Fl2Emtg!r=m=`yv=4X2bAlG=FSBmq$npsO zz$y|uHQ4vfO=1`P9af`uXcg#{Ww`+rqZPrp; z68H%`>FifF($v{SeG^V!^S@Bk<4#p!FLQBX0X!3OMLo27TGZp_-AvS1CSfS*aR*Mi z{x=i#cwR!xrbT_FG4r(^kH-qcHN@BSvrJK6F}+uqoglWd98 zjcs&txP1;u3(-H>mJs{6dLj1l8;7Vzo(=35&&x>EZ(#C(UISR8s3)6jdnZFb$sp>< zAmz#G#oiKoPYhC?y9Vs?O404?NxI%2U7$iFDHi#i& zfb2!FMPlEGtq_CP0QUi}0oJV%>m_!E7&1Mw+=XJ(#I6;aDOPRzg8M+WM%K+7@A?Fj{*`ZTvjI|EUu|_5HjrOjcTb4xqJ{OuUUi3b zDy{cH{(UPYxX|D@w>n8b>5757}A+&8&y^jgqK{g#Dh3-RxA|)Lv-3JDl zaDCq^KvRbmD&q=})~<;kkhGo+X^k=Rv6Z2;X3ZMc)Y?@_Q$J77W-Yg+}i&WN|_;+w*N?E^N(%S~SlEdCq*8LJY zD4?u=X;&8flUG|=?~zc?X=PoT_$;lgA51ChWu~kj%uv>U^0UyAvc}sj-qGMV5J!3x z7*gNO&p|i8kRc(Ec*O$h6Ot>fq9M#4@a8#>0({qLskR`=_y(`@y%@lnDCVw>#Ub#hM8n9u1qL{X}+wPg*VGJD=bsk5Bd{ zk6K2|mYMAnrzUdQ$Elg(ZNw+{3g3=TO`UIVojNsn)bcnlXo=OOf}2mYfE{k;-Knv2 zUsx?~;cdNYqt!nNP6@i%cy>Xx91VdF_9}On3SQp^zqmoD%`T@bmF~gnxME zK@jJ2um7(IVmP4hXM(n)osQ>C&~9i;2x6R3Y;R}F3g32KRb0xYv_C zA@+%xJEeK}WbqS&9s?iQOX)+qLx z*b=dC#8!x{ZIYDxT_;?#RQO^@DayVaA$E+|M6tt+l6aIj2h3jpL&k4&2e9cy{}i@@~XPsFj`#t97Zs0 z;IG)Z1^V0do#CGTY4y~xL38)?)>hLr;kb%s(+Q;A)7x7Y3%RGm;P=1o>3dgpyvD>W zUFcl{V}!r=)F0P9eN#Kv_{!{3wrgg3>&+q$bv$mcgPtcITpni`KxSyyWU&bzaaR%eAUs%$X6ZLd?8=;{ZK5J zlM>1MWxqcx78#-8Oj`{x;+<@5?q4*KSk z%SU;~3Z=;ZSE3{6*vE-&bnHvtjyuK(^(@bF+B32|%lmoH3VD_z$|m^*_~>V1(3vL0 zxMzoY`3%qUzj~o;&+>uZwE=$N!?T>%Ys;SHYkS+P7zdJwvev1WtNc*znhnBBhFFf^W_=tp`OI_+CN(aPAc z;CLJne9Lh+gKzl)TiTG{_(itFF!Tqugl{<>2~gGe;XY|ge3?+!kJ%FH8do~hHNHHk zYut38u5lBGx~{h+)HR-)k-Em<9Mv_}I1EiT+4f3?VQ4Z8LzB%B!+1oyGjfL(qwRFz*<{(7a?0LhU^k# zABf>@K-S968SAbYu9dlMlMEveEO(^J{aWmJu~A}Yi9u(x?qyn*o|WFJZHK2 zVo!@9+?{nXfg#JaGm${nK3wAjhGZ3DNNY!iw05k!ml!3$qa>K-s%#LiD*ezm6 zaL2lT6?;JJF|jAbUKWFHXWj3_R*G$5_gwDx7vUNwH6+8NhOCQe3(1ZW`>ohTVz-Fh zE`})uSrzpa+H2^|L`sCS8kr+<{zGP4mY{Kubjip_dDNQf|Z_J{wu2w zy`MWio$&vzD!*&4JAhYeh|2Ljy|Lgz#dX6{`UBL11iFJ8P1J^@* z;M!ck&2b!$UUU407dU*t-2r@uaLgK4Sdk9k&=b_T;;8SeMtdx-{Vmj($Z0*0}oeY4Kjh>DgVG(&7g+*Wxe& z#9Dl|YasXx<7FwZJm00k)0UK@x&ywMVqL%SV=C#|(Ebd48-_TTq>XAHr~R_JvSwEo6K?h+r)`4Fyy?4pO@Hi8;&7? zrSyw;nI3Y6Y2sSn<$T{|Y?^r69JlAUtgTmg#SF)7yolmj%kQ}ThJPB5iZ1i**t?lS z&g;ASI@LZy*kf*MNyqK2{nK2>?ViqJ$L;NWJHv6CL&aH++duKsn@>+eKmX2m6gzJJ z&9~!2tM~2Lar;i+PHOn{Q1oe5;&?BW<+#0u_bfdWjk^|lHpbh?b=;;8cI>!~lMT%s zd+z5MiXP~NvK_Z+p6F>;SmB}QwY;rWa@>Y0vS){1xHzu+d^^{1`*vrs<90o&H2(`V zGwxKmni+TST+IwAoYu^^^-3N`Iom6eFx1Srf95!Db4obg5!o#~-=G!Knz_z%=NX6V_H(v`W`4_-`n;-eR_q6{GES?(z@IBb)>FSbnV8?hB)YulM%pVtZ3 z%7Tr=;MmRfb`^tTH`xJVaOWn2J2%^dJ2%;Qu}NaLh}|yMAO;1@x-W{EB`RcZiM=QG zxtRT`!g4<~HOang7Os_foHZHZthtY0s~i+C8KSFM?ldt(SCdT^yH)H@V$X@aAoiKq zS7I2EW_uWs=6-h(+e2)Bu>-~6k;=M=d}iJM1nv5+{aRaeueMxj@2A|Wudq7BxL4yI z`(wFRwTD9t06&6ow)iPV7 zR>dn#u2%h{UCn7vYuueH?a2lE^J;6=EKln~Vm0!#?rvw$5>)%UbSz#k54TIgButE9 z#zoe$aP7Xj!u4Ylt`Ggl!u8M}SGazU6J}1WcyW$}>l@}54VD(JyJiblj-nRr7hPDm zZk`aQY2mt6;@|5V=X0@e_Qz2*2oKykNFp^bcOHwnZ^o>Sb2R zFS^RBH1~_H$Kf>p`}YaXv}ZJ)QP7)Np3%kX&6lh#ct%6J!4vu`TS9N*oMQVYwmi;0 z?`KQsO?+?Ao472YH*xXW8}oR4PZ+z!{Q+aQxNks*LM=gu;+Y!h(2MQOnK~3}^o%B( zVtXfpXEYf+qsibIO$N_sGI&Ol!84i+p3!9Rj3zTfX2{kE*XS9|az7W_QRQ$KVBKTI zpti{-i~Uv%cRjZE2Ql1e$&h%Qb&+_S3us z28!W)lfi$E zO{`k1R_sKvlf*Fo%DNbTWnbCIFd4#y$q*V$hR|TLwd`G*tZlef7R(WQNNkmKV~%4p3~@De z3#*U`>udLI_C#1Ufn~z#)1Ha2JPUQcL|7G>iLfw~1uKbpy8S zGvxK&L*`f2J=QR>rn3Lll~vsyM+WZsc`|Tc$PC;UHqXHQN6Wx%$+mB>4BQtX_||h< z!#TM6!cOKfytv_p??*hW+e6(}r(xw`Wp z^8iSXks_<&F4pLywp=aU**!bY_5lUPNb!PqfSob^&GOexc=9cA&?w(!^bvC;V+ga5 z?CfLpdV0M-q-1B$tntz+e@(}*NtHKnYpa>TGrn`=*=A>c>N{WE@lzO__wjw))S?V6 zE_D{C)QSf!2U*LPd+YrZIN2(uim#eGH+nQtWm#UhSXN zB|f9r9cAr5>YoMnHB@yNrS>b#8#-vg8yaI?5JW+6bwbjlM`=;2oge;{emtk@GP*Qd z{L+hj8?OlI_-k*7(`JuRL&!)Rjg+NFY@xOBx`*&a{KVjly#$*!KpYR&SJGYSqKW2KApeBK~&6nO%JVT;3s7oVVfp3Z&3 z)7o4Bqa!p4#E0>|vBi}1+u$9$#4~ba|H`X; zU=iLuEZt$x;6}UgMme(g^UuDp?e4*G!EV8Q!FC=zT<=|J_P>9dNQOp+UvKn>=P5KY z?mHO8#9x(ZP&{KXCR=4oXwvI!*~gakwuB!u?n9_VgO?%w#wpEC;1ehBqWK_%oAr7V9dugV@euu!*s5 zFERWk$Z*?Zd%qXEO6)eVKZ`*Lvo4%QSobqAcq)=1S2oL4glp8qWYEMchg{iYm12l* zBs*MeoY(}hJH%#-Ay+o*){8wYwm|GHF?N^R#E>hS<&Z0zb=z7!vhr}v^u8DrGt0qu zndRWSOa|X&GWaf&oi7I8Win`ImVAT1_G3AQ2 zwgG&X$@UQ2M+^zJSPoNiv)qwl7|kUcFNVpv$>6)py6{~lL#iz@q}n2bo+g8yCVNZl zJ+ZadjLNMOu9XEli0v#^C5A*?Y;UO82(cMrNV>&x_liNilO1Pz1#4}C--K&LL@6(@ zTTG7caze~s?rDktJJVX@eXGTx;W+cwBzt$ zw#3uvyw(saMfx>c$aeL#L_+I@jUO?gHQKE->Fmk3?t%XIgx02xSp=WOnXIF9Tc~B7 z%9qI+y4tg4d#5`-j8dM$8hc2munr5io8x<7IBtQ2CM&b8%g>+lR;<-RUrab{Zb zRsL10yMie$9FVpe{fgp-Bl(*zZFO%Hvw4{yIQrTi_?KsnD+sPm_UV3JVqmLlgGpiB za9GDaK_hY6Y6^dRJ;nMiu2a?__aW1HrLB|1HZxBh&`Df_ie`7qazRbc1xbiczZw3OC6i+ zNgii|67NZRuo%A`^yC6>V_iE9JdxP@RA(8U$|wl4K~~Z%IwKU5G;5&+aI6!I3Mzt| zEQNdlO^jRJ7~A#1zWd>Bio|g|>f3wREpKyYFa^zVwwt{d{nRxvo=AEALQRbOe1$!6 z;^jJN5?c9eTS6n_s)Rc!p? zdru6CnB~3}Ti^Oi2ES#tx0TqoV!Mm&EjCnagcz=Swl`huIx(Cb?&Afqe~7&+hEZac zYj5X=?R5;-%4}A`h;-a!|D_cZk?Xu`y!Mw=4&J%eu&wMRuFmGhz$HJ{9{ytc%?V zSr@Z7vAvzdb`z@=>n*li44%oXn>ntGo7ullkIuIrE%oS|9%)-$cZbi?eE!nStEaba zWm7F-D3TeW->sfHRC@GutJrIZ&D`9SCmpnDQS(u@q3Omv^DE`iq0Q4kk7OF?ojeUR zo<}D8lm>c0lmT5Zz_KId)&J>^5hT?N5ay?6YzKcXw zLp5>I&{YzjEA50*e73XOUat6TvP(DZeEfSWg@-W`U#+(I%yK?1q(1QmRBIi_KaBff zJ02CjE8H&eP8P33yRE7c?QKf*qzomxkqc;iS7{}RPM>J}ejGz^ugkWBY%Tb1c|C7E%#X=U2cx@)uhq?GAjORAlvm1*~EWr|mbAovlK>4pjQnO3Gr z)tXkOGg8X5(Uj?o3}uS(gqGuSAs(LfQbj1!_5B>g%CwDdx1>xVj&AbiuWea{QX*!k z&`Z6gLMjw5gUJC#3T~=Uh!DIY;u?;HU^IdY*>Yg2_A$OqHv`l@#JAffjv~(pZi5%f zimm<1OBGV2v?jzZ%nx{JDpGXhF5hlRk>We$SpU%(iu7Y|E3HUzOTa0Cg(rDJr}#GG zXwj2%d^>qc@{HgP_d?m~^gA!tvO3+^%RCnBYif67&?UIOusV%WIv?RxF16d#Fnd^-@J_{Bqj;!XeoisWVxpm+{P0<@pMOhbTTjSkCXQ^eq~Oa_N# zmV?7G861|$;IK>vhh;K2ER(@unXJV6N47?|Mu%lG93{)aO_^*DG1#BTV0B`-QDXQZ zkxdkvEOv|7?PB=Bvb~4JmWzEZ*4oY)%i*kYA4ug)wu2aQ^pU~Mh2=1KPKM!gvY}!l z#4ZxMOziJsbHx^ly(G3w>{GF|P5yDe>x63-m@2lZSPwCT)v|vh#7-1LDrdHLh1eg( zZV`j)GRr+JwoD93ow<*7?TTZ$4Z<}Jx{_hgmE|f`u3D_GSbwomVke88D>hLKp`P5w zWnzC3yI;(HLt(ks#J&=P>oWU?q|R)wZMaqzAgMFi#$tPmRf)k(ndOFyO%S_44C$R& z4(Xk-Rugi$Q_q}az|{}Nl*-qTrbgK(`Zz_2PA<~L${kfLN?i)D(^nf6nU zL}_At^bYqVb~k5F>%=L4qb$xo#{I>miP0r;yGs8C4 z+?g5vBLC~m9C=6oH##%t#MMUr%uV*46zk7i2|X)+=4oAE*K~ho%gro*=5IrP=9%_k z?qF(a9Cc^Aq9|o?<7PxHJmqpl?Fzf}(<7=+St)p3-D1nt6t$UInQL9tHQC;BM^tk) z?1ipj;n_UbN}5}y%S?=}qK}bhGp_%90kwUsUKNT3Q&O6?s<~$~hCyHe=HFAT=GvSl z=nRJ9xbq6{GQOO&hQ&~0KFx}sU|O>dFh)=2DFrnv-j*^IXJ79r-^*cp`m1lh>M9L> z$TNL=6Z1m`i~W#~c0M{=arW|4T(lqZXm7WTe`D}NzT3B(yg%?mp6S~gnjbP)tbj2% z5(I}eQ>7h^R{Lk85}(C>$T#|DrSe0b>DybFA987iA2Ov>mLKv*b`0=V#zg=>HQWjPv;+(|(aGmCp~ktM{akAMy@~J)NS3*EivkfI}t`* z=i2frTS7G6Vav~K2~Xq|wuHEai3gs@_%f5DthkaPa`ELsi zgxJNiFA}>?*lROAk+H_nR?$#gyOKRA_N>@qvDd|5%R`N}o@bqP5!*nv zX1K=8%47((U^(2p$>0x8cD&dqv9rW5_RVsTxU7rJ%4Bzn-6Qs-*t24b#a zA+s{u!&JXy$gE6;%*tfQtW1W?%4EaEkXe}wnUz@%nU%?=h}|akXR*JDA+s{;zAm;@ ztkljs+gmGKD+@Y{brst|Y-h15u^KVti(-32#72vqB6gnG`C>PT%@VWUU)bKWV(*E4 zBxb*_upA}==JB>MsmZ>y3)jkm?ZkEz>o0b=7&1q(E;2{4J!Fm|L*^*5%fzk_yI<@< zu{Xrt6#GOBd8OFidUj9cJ~j;3%7XpGkVA^)jujgw*3o>3S+`TTR)jb4Y`aZFLfMia zSI$VNWrk|5P7>sQ+E2ShZ{awR)FkUj>@7UmDi-4{3{RQ-*`)G}((YSZce$&z{|~%{ zb5w3l_9#3eGRJHuhhw(P9)*W-l;~L5qYe&7Y;gtUD%^bT!vC@+D09EHHOgGvfQt3g}=rrK9 zJ21!Cq?<48UKwt_{k>xLKq|*{i^&1CN^ZV8`0isj-y3~9*Ufi^v)IiSPm(-;q2$DK zD_3&jZksDPA$8J{6SqmJ%gD_a_d~&!C-JFIbP3NQ^doDW^?WM>$%!L^8l@-MWa|YPq$e2!H5mjaS-sd>VvzkT2f@j5_+H4?w|tXTI;F}#(KA)JZj5M4rc zpx7Z|kohb(Obp_ZY@*oJVt)|3T?~Sg`+$2d>morh*|%aph;3owjpcq3u5o%?GNdYI z-6K@)7%_~gvK+=#Sq@{WWEfK=yI&0My<`YvWw}?x-WGdb>&0#qdtMAv>ae|~V(*B3DfX?{OnX=6es2ob{_S!PR@<~Q zF7C9=oN-O|ElE1#!aeAJ${81Tr~kX1ahq$q$R8KalT2-QRjBRE7rOK&9}thVoiSu% zI&PJ;-5d7pQQN(1Yt(k1*jlc(!!%B5zueR8t_E#~P>I#nc3FP8g~T27R;_h+OZ#VU zscwvq#ooo?c8Pbgc%irM>I%KbOz2%zQ0Sr4Ct8QEM4v4IOQA8%mWAIhe_Y{*R{;A$ zQ^N25l4@u17~-*Bu4A_F`^>+RqP=Jf3%~D6pnmbj7X)eH*R}=WcU4OG@k?o0_~F$* z+l%&2FI9x_``Qb}!Vh6eEek(%V3L=*zGad3q!-E%dC)_-GA$|c{^Fm;B9D4D_M*MN zS?#13?RKf!d;50mMZ2SKCtXJKc+uYEg|fV8|KX(yB?6{ckG*KG_R_Hz?Zsh7;%`1Z z{s=ExSX3+vPKFmPUQQt)V%>Yzz|+DZz#ur>KaEc)9b#iI+N1sR_NYcW<`27cJyap1*&ez)Snl;(3H;=#QoEppM`-i@?rFw#2!__L_Er zpzm*XvyG5 zO9nq$GWgMw!H<^g?_wAdC4(O=%fXM9>kru);TrvD$zZ}^UHp#8;89EVYq4X+ zP8Nep3d>y~h94!_?P9o%lHpuot!;q!5i*=NGQ5wFeJ8e33|#iTJl?B_2{Zi~#Vm-u= z448GH2)U0_#m*4BR&1tNy%^FMu`UJ$*&ZgFA%oH*n-#7Vq4_H8GS3;}^T|%_f0vxG za^*@}&iIWj6XSjlb-B>0gkG}cj~vv#S6$CeHipzGX#BoIjYS)d>$H}sjG95_D(g9c z%O9SLyunIaTVH~YEsgCb^;Tql!_t$2`3-N5?es)R=~MQ}^zn8BvAn`xpI$z_hh5{5 zL+v1?kBzbeH&_TJ9BOOoKCjucseF21r5n#3-KV1Qoc04MZS9yry@z~^48gy#9uF(u zyc$llhgI~cJ7`Pmbempvhjp(jxBS3gz0z>3^}k_a*UJ7^4VvD1qq$9`y@%A>O8H%r zx{SBV4HGQY@5D}6^q$x}sqeh8m5XXU$#7M-2AiX`bUrlXMT4R|x(zp+_`}gx58@6g zEvE9RhPkI6F^|o4w9w1DXQDB%asCQwI;%cVA`kx;3Q@o;tWYq-w&|_6)1n%+>ad(A5^# zd7gnt8;v}m-u~Bbw*Bv@nJM?%%k7Gyc10L|u44UFo93gDaM|c9)*kzAmR*^f3r__g3)xDCBf| zl<%%7C0LKja=P88w0%jFoNggBgP@J?3|~{RBa`E;Y%r~X+`pyC!)TnZz+ z>ewC722rR$$J-XeSEnQtXn_<)r=f1Jwx46XovrlkPhA$CZLdFX`St;=Hnvo$(GCU~ zJKOoAi5=ugVFYP4%Pu>-a3XEWvsYH%4W|9!;Z~VfFj%bUnG))z zgp#Jp?r61t?QQ%e@mcH@{H}ktnY@D8{vqZSyjcrg!FWwvX8$lY9V@bQc4E9j9_(lL zde1BauV%KtW2+NOuFoEFMT(BV@;c+36e69uT@vt;uSs+vouT-2tF^Y*de^%E zM{jmv*H&Fi-Yu-oafEoWM53NJ9n)C@PZ=_Jxux^v@w+5hoS|TO_1uq}Fi_PPBhfZJdvD6Wi$UWZ#abb;W2+5bT;bfRuViqt(94 z^%5sBoy8ctn&zeQsdu!&G+?nCGlp=Yj-q3~@a?!`+xd3VV>zwnVM3W@|AJs|zr(?$ zn_u^gWKUCBn{Y}Y0`jYS^px5^?kO!WvD$y$_Zj!}72ocXIE5*-k4CTaLhH7e(Fv#F z67L$ljZfAAZDChtlwo_w=#c|Q4;yCJ7Xo5Y=a=60?j`-~8o55$G5FjvJ;e?o`wl#7 z=*go8ju|~-)T!Y`iF^MxUa7(F8gKyNCBIjz+MuDNKzgVs0GH$4Yn7~FUeN>XNqet( zL*wd2Uq*WW*R$i~WK+j;H+H@58jKDqJx~`f0eSvH69b;5c-qEp%Xlv4jv?cWrpJ(R zP%s|mxGm$!B^Zt&xY*%$RJx7@U;JaQv+MxmaNR zC4-YP%XJp(Dz>v2j7%&ySPWMW*#%;7QYO1f48It%=fqwRLy}~C+BU#=2Ftax^F-D@ zT(dO8VijWBh;41T#;{fv+$)AOQDjexEfD)oY^4|` z6JmSi;aXV$n*!P9V!sr_h%)zakXS#l6U0V~oh^2r*v(>aS7v)pi9ILws@NN1ZCXeD zYZtDS1)aoTXJC6wGH5qOonk} zve(5v5c^mRNs?La2eD43?a7chne8E+6dBSZ?O}_P7?c_*p*^;i`^$S zSFBNNq1a1eOT^w5!!$+gGo~qGpW(txwz1gWVo0XUaz}{`6gxp|wAk5V=ZV#cT`V?R z3`v){k40kt6njtXBe8boI?X<}57)|qDzO@|gT)4jog+40Y=+piVzb2V5bJMRllwh9 zTr0w|gNH$3%TB_gb9om$!TQuKvPqt*${*R^EIRw9EIO@zEQ`*!wx=qanzp*`!%PXMXjbkC#us!)ll<#|%d2wC@9}&VWJm{kC4Vsj6Gp ztM242tGX?)JY5ylb&u9`D}^m;w?1`ebq{k!i>BjuZQZ0Uhg%gGvZ^m?=kqGfge7OOHC*|)<<6SlFn8<^{U6~ZPBUSNv5(N0 z&4YE^5pz-0O*-B{n^Y|p_i`)DNaAH$H-2%#OWcteYffCYoXx>HlisP%^VDu;v`VX|?ZIn?8T6&`xX)3qg_p1kLWAo7goLJbLG=cF*{6E-FjxRizXDtd(+BCt1OQ+@_3zS=DhN0 z+k{>NxGY;V#H4I6$O=Lm%n)14wZXh#zmT*?_#D2Ek%~M~$5Z9t*ciypKV!ledCvE}snst@96W*;O6-*>*n%BU+V*!`3B1n4}a1 z=m3fmO)e|pnB3$Q+L&@)=uQLpf{L}ljIpw58_Xfb@KPECGYZ;Za8oJH26MBwlxu^z z%h~P;6G;IZ%qTBNqXcXp7)68XfsZU~A5FgfP07a2Xp|Ua>=fr?6FbObgTb*6w+qQ4 zJ^kb(+Cna{Dw#UFE zgqS2t)a*YTn1q`AaNjQZzNO69xJq{P-EErC;(6>bbG`Mp2@49mB_eO`75?cq*2H># z_&BPq!e;Tqh5-TkBCyY6_kd*f>rvi9NAm~g1%Vc5mti(+v;YR1Wo$UgV|Rg|!5t$0 zM$>i|GYV5drcaIzhdvb*mR2k7%vZki*mD9B3Z04flh4+2gO@rbQ7Rp|jq7Wd zX1#&uMARDyJ)4Nj+KBx)<~Zx*CmY8@Q%T3fsmWtcflh?MxTOr)xF8Pl?sT#a6t;I9a zQf{_GM*_3<&b^)F|3bS+U;8xIE`n!Ou3ZFAy0l#c4;h(2BHwy+Lol?9;J($&E&|&? z^oM6U>;fCuKUfd&2fGNK^RSEHc?dhj1L1a)EnyeI%?frAJ4)8ZLh;%I(mwL z;2k8gn_y&|b`z}8ZbCNMdPz1bTw`!I8SEx3ha)G$xgmqygbe4040aQ;!1_f7y9rrm zG1yJWb`XOFg>0}G>?UNen_#W2CzB_`?|}?wgA8^PvKPc)HzC81p5<_xA;W!*40aPT z*iFc`5rf@?40aQigWZG-b`vt#O~}THO%Q|Kgymp2VL5zPWU!l%HHgg@`=?lw80;pj z3%d#H!frygxmY)`?ZuGYjOBWY4HQG@H`{~Vgyqf?s}s9e>^`x-iNS8dx^IYmDE5h1 zN3%Sz94si@Zx=CGP{?)?+fD39F<4Mo&b*hx_C|^QPV7ptIbsiq!4sc-fdz%_!EQnZ zy9pWWCSaoK0c zyS**LHJ+uzF0mfBsi}LiPu(e7)^wX+_f)ST|LRr_yG2D+-Q1cBo;b60)%2>(d)4); zs5*aMMZ>&pD?9#m?$>Md8S=#R))$uFHLVMtqNVlbqhc*}uO0G4!ym_WYCRFc|DU}} zpJOM>#ueXw7t=v#xc^@V8T||mRGYttweS&27<;L4*$+q0 z4hzZOWG-7oqVcAjf^mt)XMX5y`Nz#6rE%E_|1|T%!p6oQ+RcU=%6koSM;`HtqZLQJ z$&D39J=(b9gs07@vTT8cD(uv_Y`9IKwb*_bb~0DI?*2eYYs5mcl$_UbH~(6ChK?ca zT1DT;o(s#TT^BiqRM(BOlVMT>$TbziL5w=U;EqaVRn7|#Qp`rDA)VA?3$Y>Vw1#tuQdQymDideHxYC?%S6$~ z>KuV$!O1B%kt3SBiR79n`uPhwVijo97~5*L#n-~KWpzyy{j5!zC<^Bj$BSoiCJMZn zrQJmCakfdqf>6LjK{pGkpx8t))!*P_6U7^5q5z9c6m%~+Jh6iUCJK(+Z|`3sOcc1C zS{k-26UA^p0J@FDCW?!^U$Kc|v6(2qViN`3O^z+rL~)7lhKG+cP7}rNd^Z>XZt~Ax zG7|+@Y@(pM$?gd=#0Wpaj#i!~ioS`@XrjQV@0;G<{oX$2E@%56my`zgXQVj)+Rs~? zgwRZzDBuUdtZ9S2FGwcGv;EVAa?4|)fI?;RIGQLREf6@0m?8Wm;vx5)61z{EC}1He z^ID_4R_uB3hlFZPMeIY`;;kBv2eInUW1jff3YsM&q35Gu4m@vgIQs{%P#raHVg@f%Jd%Z91Hw(+~OfR*6ohOcf|p?njo)8=p2+ z;8fF&(lyaBm@07MacGzLj^aa`;MKh($pj_!PeCk%I~@n-_eW$ry1g@L&(jW{lqB47}~z8zxRu9lrp4UED|8x8B^0fhz$Ws`j?82~N&(|G*jA+6({L?@b%DN2g96eLDO~P8kX>5wwYu zFx-pv4Auzt)`5<6#%=r(?)2@W{}W9Z@a*7y4@YROJ$71`+p^sLO#-mLd=?BuS-_&v zWXq%lqrDv}JUVu?rHz)4Khl=4L_BIsSR!zZz!HHw7%UOE-eHNDWJ_2g@EDIQ5zu?I zL|~1vAY@Z)uVlXq*V=jvAsJ#pSPs1;Lo5i{Gh(lZy(VV2$*|lS;aXYHL2P|793R`; zMQmTO{lswDu^fJz?8_BmSBu>)hKq&en#A4``&VyB8t6T3w0TCtg8kBL1g_L10fvG2v~`4aY}Yq-`n*h&lu z=h(m9#cITQiJc&ZgmXN$v&E2bj%=pbO=5@-;XV)_!g6nly({*a7_zXl9CEL-E^@Dv zbrsu6Y-h3E#cITQi471tLhLNDbH!$g-6Zy~*gUak#gJ`){X>oc_8B<_$l91ifUI4( zX6f?8b`-;OVJz2I>}at;Vk5=Ih{0mPy0BPqzc+~8EcS%hGh(lZy(R`L2HS%bgY9h{ zuC)!e6N9$~%fVZN<%Wul5F0CYrdTJ_d#t-jxK@N`2F$XBtsQw|ZlH#enduR9v*#-? z83g9C(X)N?*c%J38EpT?)O(0V6m)~AnuG1mkR4Ss%mQX6+5J0Ed`)YsXV#58@in#9 zySRLIj7?=a#%7Q;5Qe~D(R*S?k5^ImpYRBid@(kap%LN*#@JLMgvEk*%*2q4vDw5O zGOT3_;)!Bcp70YBwrmE5=J7hIMw-XiymZ21?(aV}#>RfX_MPR4v8luf|1ZSY9B%cR z#n`+W#@OJxZqc4=qCfIi!( z>Ba$AYy>&q`JY-gf|w7Apf58$HhSa`zQ$ z^ti%{Q?$Xhvc$Jn`fgxb;cjXyRt79KdR*h>ewQ$8#6}Os%G{Xvj7E=9*8U4#?`MjY zVf%-g$Is7Nh?RkgnqmKvkqo?gl93E)BM4r(Y0HRXXSlyzTAfr(?93(pRmHJ0fA($q zY9P}&yO0$-^Nn{rj-6o__H4CPR_qMIvom977@txob_Q9!lCd+p`!-D*kkNyDJC2=! zMkpLR^RORDY#4bUu??j&&$r{ik0*RP8TgShj6gx}L4^7RSo)dTlvYW(#k76=P+P>dzk3e%FAhl$UsAtKQkMG6TG7{#co#z5j8n46F&Z zn;;o0!(#%^`3rqQDy>F^y73fBY-9gl*-0Tf(-2pUfg#;wnJY4DM!#n!y7IHkT$_!sdc!eq?hQ+B&ki zV2$1%WRuY=XBcB3L(~izyg$g`{XsTg4Bj7PI1elb?+-E@9~rzq$U2L4728>Ccd@}@ zL&Yu-gZ~TLyGjh+A7oF8Ju9|Y>~*mZ#6A{lW#^22UNc-{YJalr#gJ){>`*b}TqMKY z8Ea)WAQdr0JCglY4AvJiSXx;3?_#jDkUb*?OAFa+Vz9K3;Ty+VS+I^>n`9lrwX$F< zv2De67u#D5J|C<*Tny>{$xauuYd2)qi6MT5<^C-8ve;s=cf>vrv)rO#-EYMZKg0Ga z!ZouUi6Ku1J}nCdh#etzme{#sGsLbHn|!y*)UX`#IGpV&93a6oYRF%WWF2*>xqht=R5jdy8S#Jl5?aHcISdu`9%`7Q034cCjbL zo)uduR%(ik$6Fq*Sz0QwJ;e4AJ5=mQv0-Azi*+y3R**O`^dD0oaT+t8UicCtG*pataYB3ilbI~`!+2hh+1JoSy3x= zl!&8N>V5Y`M6ImryUU1Lxxo*vP}B;=d~Vdr+F`Sihr)7ip-|Myf&OW3)XK5W;;0q; z@G_%TKJr3YQ7hfNbA_T-?)NrwqgLqD5l5{&9(E?aCep4TxF#lesjR4#4Vv{P8MT6I zHyyPy!&w})Ld!-*)CzvfS?(XWZ9riKLCaAqzwowJF>2)pv}-;@@b5s>O1*Cv6}8gS z_t!jX1<%$zf1!n_BchpwqHhw077E|aZ9E7#iRuu#<75*7;Fu3({9VoO*iaPnZ8*wl7;y=_<6a*{3aO<;RFTOw`+$Ah>P z{8|vV0^@z9EswG#>f>^Pw+Jp7STydiB`g}a2w}N6!A&-k>TD? z2FnE*EEi<3T#&(XK{iorvKTyjSPnBtupBHGWUyS2!E!+c%LUo{V(=RwGdIbw+#2Cp zS+J>CXE9hV*d8nwtlLYhj~FZ$EC23aTjzR6(2AVUNT8EhD22aCamL3WxLY#3yiOM-P_!ytp@f((`mGFUFiV7VZ( z-&n{Ni@{#OavzJqUO|Rf7S`R|v_IJv;aXV$&k!mrtg z?9XDz{6Y4D*gwQR75hSLjWwh8)(+Ro0>rX#AFx-jy*gB}TkL66vAptzL61_L;Mu0q5K zeBSazoZwLxMx5X|xZ3e1c>eu=5^usR7>Vo`|AlxHyhAjPH>nNdP4L896`R2Nc5kLl zU{hP8O#stW=h_73+0~t%LE}y<1)IQGwp?wSKvv|*F|K~_%E1@2ksii@%dfeSAdWn_ z-D?f7Qg2&FEw7ZkrrwxKB{tgXSg?YaHn#GNP#V!+cwdkI0B-qMNaU3_o zFD^4~b!>vc*P0@6l;?upN6Bsku_k4CSD*;1-HAZ~>1e`GI2_hrP5EV9~gxQPW} z+DHJ;_piKE95=$VT}0dn+>bG-2jfJ7U_0O5I@~S^e{WfFBY0`GcrdR88wGct5jXO^ zSIUSR!OL>txDgI6$H5hcdg5}mJCTeVdC=R4Pt#v~ zo4+5}njZ1(IBulTw=?2KAQ7_+2k_SD;9ZO3MtF6#95>R%+b%M0q|6Iv#*NTekQ+DB z+pETLBNzEL<3`v|9AnPkzt4z}jvT==9G%RH9J$`gz=kl-markcWPwt{wEv zw%o&(um#*`OV|Q%mB1E&dl+m1xZ+_8IL?-^1>i9p*#d@}Dxoa^Ym6Krn__z*gNc?5 zvrmyBa)b<#BV>piAw%Q{86ro>@TeoRJ5|Wm2-g@n!gA}2?Wl6Qh#_)>=H3VldwHRld!$V#GVxUNNl-SnJIDB{Yki17W`Cf zGqJ735Ld!|z-GX@uo;jc3nEtmK{dg8ZqP@%)kP@lZ+7QQ&IOqw+;K$4X~M5 zYTDQwEC^-Utzn`iq`2B1lI;Bmzu1&bgQWP%xHEOn=?_Md=wlllP3G@r0gktMeE zyoZN8S;9V5TChar8{UW_PP}$l!_s0{*vbS85jo*S_A8u*<(n~zGf@j+l21R_ie#5s zPw+PFgN|2wU(nRi6IxrB-nV_XyTS@DjpA1{_lmXq56rz{bsue*dyXH$LQB2z8ZvIw zH?A1|Y~vS;>E^KP_3OslnYFg;_i0`qOULc+mWG=XJ1CGjgFz!}WM#@Y&iC8aziSMo ze(Bqby&rL=jII_m0v205n9Or_u@(=yMeuNOhM6hj4c|?iDWk%IM!;f=2i+njC#)K2 zi^nyI&uH-&W$nM|htWzwBTc@&g9VMWY9VL@s`pL%m+cl&?LUDDGCA-n-KiwDeC2VA zK=apm#mEgJnw)|bkT}6fpM=3Bohbt^_b{9W!GrD%6esiCpw-SL48+t zQUM_70%KCzCr(n@!huuWasWt0;xN!=d*7xj1e8>la67hqrClL#uy@-(%xDppN(X$P zEvTHrlcPNEZN#U9Mkj8g_RIcxr^G2q1$dzQfA&HxC+|eL@!pjhzi<(d#0$5-T}jQe zU__IlZ0~LVvgFuc$KcjrKbsw?h-98ydZk(Z3y+g|ZfujG%xWo_C;EsJYsuMyHG&>N zf6J0_g*{0k3n2!S5uOxHhC=s``+_aZVPUWtqyJNrex8ADhktn1!+6oo{*4S-l^G=P z{D+|f&rKL89t^jeYze~$ZfY=m;5)N{=J6eEX+yx{_qQbs8#mjz!RHHXiTZd>M+Odv zLK--*MrQ}I$<`{_tZ?AR~{jnS#$ZYQ>u~}mGip>#wS`6PP>tb>fwui}0$dIIytUO%f;$pVVo1`-_RbSCqg%+};=uNv5_?VzNjh2g4YAgC6_T|H*UADU=_KnU z)1)Ba_@-2#euA~ z$qklk6Rz2Dh#_4k%XJaMG$&+mabVp(VvsLnr-)4#yHxBtu^YtT?7+HkcHlnX>_7%* z2eNO(R)}HJ5|-OITq_HHA=XU{={i|%PcfvyAR8n$N$f(g%f)7h%@KP@Y?;`nVn}(x zeIVrp*2;n{!?m`-He%IcwPO9n4i~dAt*|{LHe`F5#thuhnj156LM(n`MgC_d9C^+5 z!~l#pJ%uOIK@KuzN;h}nktNkrhX&=F#}kiCu~H_K$6?};no1L+`q#mL?v- zq$3u7VWyGB-b23j2}i!xgd;l_oNxp~&dCsrIj~gt+#iqmgd_7X7fBlv_;5xT-Fr?Y zQfDq|Sc(>riu28!;$AK;fK;4JvFXj&Y$MA)VKXp3QMbS>$&CxzOK3sSgma%g$ z7-TAOw!;0`4s4>=j4YmJc4%LCPLUaAb%^E^85&fV-sBdfILczE&na?n=uv?yu8`Rw z6<~p}Q#w=ZV{5q&;Y4fvF)#as>(zs7+xip!(Gy~jv3y|`wO;D9f))teLW;9M zT<0z2S|Dz9wqe3RP{0B)%nMSI<1LyNh%@|!Jy!nDnGXh7Y=Jnz>-9_QpnwHpU%RlH zPcQQ8EQ`d^-hP>XhcFe;A~DYU6`Km4G#?DG*dj65>m5_9MdAW4&cnyBJ1r6ueK)aQ zX*3@Uu-GE;Td%iM!s-xPBo6e?`XoN1MFO_1H~q6ay#0835ngGrdQ1muK*#>o{^8|4 zE|t#*<7+QnH1w1$2)6b>JT*hu@`D72m@;U=YC9uf-s);Ha5-K`2uGxO%(z`{Ch7o zB=I$*Ee*IhZ&QT~C}drT@^23G zs*ePR*-r}A0=uv0`wRUfD(%w>d-wdGO9zoo27%`^)b)Dy@20S)uohI>I;;hF#zF1h z6K*%zGHETqFVTjb$3tZ!q+&N)+R*fPoOf))fdHWu_zfVmVzezQZF!b0;YM+(E#XFi z+YsC+@S}yb0Y)wC7w->|m0@dpFQ=6OYqT$fz$nbL^yG;zXH!@gGSq}CF zvd_feCPxM<1Ixk6Kn5!V8LSLsaHAlr6zeZ`xY#(c31ZX5E)|27f$hP{zi8(8-+G1wc(P7$jUyIAZRF}s4pJ~xUz zA@-rzCt}};trY8GS1b1mD+AWb0!#%$wwqYBSgqLcVxz>)5<6Gyda)bD8pP&{Efs^k zfqnT>41pA6urjcJuriRrwm^nSLC9cRAUjwLwgoZ-Qm`Cs3uM0+yIJfuv8TkI6Z=#Q z;S+4H-0qreZ@qA>EZ9N}wgr}}6+?)!4n3@@f7O7hL#qy}o$yhU-G*AU2wd9* z`lhz-)2h1NYA~ngqTY2MSYpf6RrZ;ki{7biv{KH^+5iqRYz0 zZ$7aYi&66%mYx*MZ+LTTrzc8EpEJqXXV9MC*hH1(({Hf-`8=d6b7uDVM>LN_`CZ>e zvwGlB2pa&CUm~oc^quq+9?xM4kG61PsK`Hs$Ij^~Jn+c$kcy(F?=YdCXZjA%K?SGp zfOEq<6ftYTaYxKIb-tuw0;U6rJQ!Xwt-m<>Y%W|E8ka2_iitd)ZJ2xN5%pGMk!}0~ zh5%GYkyjhvUN)f8)cw=uRA5&zRR8P83`%+wywEV$Cg|A7&o1W+#iS|GiNypRkHDt@ z6NbVyqypc>0j*a)US0Z77d&=LpWO4C@@eOW+8)=AY5Ur$@-0tUZvQ*G$+AfHSYZD< zpgyJTzqHEK_N8|3&DHjpj3yl(G095pWz7$;b!KU-0d_P53iZ0$*3L`4n)$r-gn$W7Xc~Mg@gF?hM5V{Xcq3 z?X0)44!_OWtBE1!0z#jHop1aQ`tN&1zB@zc^U(ijf*LGVx?{Xt$84d$mR)7A&fr{Q z44Yqhzbv6Y*xQeV{#(9%q`%Wc=(AtfnF0r6zffeDm)opZp-;iSw$5FpRlnGq_@V^g z8a*mTc<(zI-`_vmwV8;=Rr9iccAj@JebvPBJfDbXqQjQ5EkHjlvVR!ejP*|*(GHc_ z&QJN)-f2`^=@lapAAJQa1x`RyKIOG&K7-AGU))da*Sb%K%iE(XDbbF1<0W1x!-ar( zA3G#;XmY}hL%j*+KtPj+_%=tpVT*wU3@1AZsppk2qvX8cqeP2Iw}OijEucl5lOTu# zDK7JE%hRMIOSu(5Jiq9Lk~tpJZUs2?Gy`mw=ty$Hjgj6)d}vS@_KM(N6ol)!zRf%? z5UxD5-P8+7nA%z|My2Jh-ND0p(vDmbVE>;nrD-NCaSr*=d8*AQ~p2k`udeE<(- z==6KTZCr7%58z&wFN^}G>k<3>ICR>UumhNm8jma2w9|MPkv1^>kDnau1o&yePJqh- zb^_e#U?;%uWRWd#86cS?Zc#`kiCY)!2NRYAu^q+s5!+YnK(Rx_aPGL@ ziDEckWcX#Xf3UHTJs|dx*m5zu5W;e;!!>jJ7u!}0j#6xIZ!!39k-<}o`yDJcR1C=^ zS?(gStHu5xHdm}sY@ygoVoSu}m%{yiBep_pOH<|CFQ)QfIZWU|h6y~#Fl`6f(PEgk zgX|13vvq|GVHm87;0uyEeo&73(b4L#(G*Ke5Ba&Ja6C z>^d<7UvR$*MeKI5zluE|hA<5F4`CQ=55X5?pNnl^ik=L?7c94v*luFgVz4-{Tz|2{ z#m*EPCw7AvChK6`yT$Gkdr1sd2$pNL7T4MaYlds~TNT?@Y;UnDu|8s$%7g74BQ{tJ z-VFG(Z2)fumdo^Jz(b?C?ZPJMxa5qLD_17G8Xoc?>K|*84&0}r*o#5UVlNuP*b5|% z{Jdt*rUIcCA46Z`5t5rlz0MnM>nu_A3L z>0|Z!p;&NZ%2JXUgz-(QTfv+G6!oYV`tYrkhw6ta9Fk^kaz3U5FMh>?!u?8++ z*Er1^pIYAcUgcx0{DhobCMFY`ex8eu^_gD297`ancAH3E~h6_M>?S z#YtZ8!eZ?h`7=w#Ar#zAp%98;33~=@9UX1=hk6&zPJBj(g;91oAN9}T5DKx<80><#4a`l@`D1)L4xwN}Ss@f(cqii!ih48}6cIvkknb+3 zkyB0!(7um&>xDun@Cb~89&iNvg-zm!AZ+mhpF4|Y($;aJw;hXknm1z+kNxBH7qoSt zz(&4{tPqN$y;Pw&G@wGF!$6zRCl*NIU(Ct24sC+%*fN&Su5qc?$_kTwW3 zQ6Uuly=wjt3i@TmArwb=+1wBc9+U4c`M=QSffuD*n+Kjixi$|N#nUzqJUV3Fh~~(^ zGeWTC43x<_U-~nkTSE_Xe`b)(Y9IaEd_Ik-2F%@>1v0~wBo<`^L_c?RkjZXKLd>{Y+-L`4=RMDJ=6z=7%$Yf7M%fP1@I4S* z!|#itWu(Dd5nRJt5nO9XT2s3w(s0NF*LIP%m$ZLK!*5t{zXzl} zAq@vV;IFwgIK_hd;Y|sy;Ts{i=0#dv(pr$#nzT^ThLN_Bw5_D!+aGuy`1S{$$3xPd zl7_b;xF6n%;C}f@Gb0TLKXAVzq-Ao^W(Q~)A#xDMz$wp)F1x-ai2evX%HJpt@y9%k#MO=3u3%UPMkFdx>&hB2)#n9(GLE6gix z62l$F&%z`I3sy~H{DVeQnk}bIV&I_6CNZpG*>7N#o%XSdOeQhlWiO87N$XptyB~l( z{3*=o-zG7fV88x4iQxv*{%sNi4%*geex1b7Kv^TLAnC?9U_tO=9*^a;ddO&u175ak zH2*e@S|K=nH{^f|YNel@rqbyBgsLAq6o5Ya9R+awu|JN7=C`kHg z4A{#(+%`4o_$ge{pyM-nlsifAK^lh9_WuA8#ySKOfsx(a&GSuHl^_N3?i9pKEx+ zqem>uhH>Ba0 zA-JXodn@-dP%M}vPa1w1g8R9XR-Lp?q;(~23~3WcizF?Iv^dh@NyC0VFAMhbdA<)x zdrBJi?zx|dVi{?$ch9woq+#!#Yb{B`FGFyxCux&Nn?~AJ(zcU!hP3mf-60Ko_eeI< z7(nlt=Yj8gmphP`|4hhKrokRozV+bA<{)eZXeM(IOp1?#!og(0DM@vz9W8;1bYFP*{pkqU zh5dhJI)e0|J@Xj}`_wZMu#uq|3D|=283{v_83_qcA6cA{@DNt!9d2w6c!&D~Sig3- zH^6S1er^K3=oiN(;(K%cxWoOsa}%8O7HOK(+=LV;hM1dxq4}@53GG;~pcQzvV{4iJ z>)Zri_PZwC6L;U=otrR-4S&W_<_QeA3+5(_XD=ndic8-c#{u#g^}ccMPv84)37R)! z8vkqW8&3VdS*ci zF2az8LNts!XmH<{>A#c9LuaagW+M2$B{+Op-V?9jruJS5=Hn&sLtnpdg;{pzRfVJZUV-<>U| z%`CtHy?A3It>tu0e!50S1bY_98QjQ$TOai=n;LfzPnFYhx@>X_k z!h1?749#a2Brse0pVDi`7U|cg?1rHH`jkB%DCwWFWAhLK&|hX2U^_sf@{}FV@xOlO zG5y>EY^$-en%UffMc|WBr#%I9?6m7aPHW&V1`OvG;P3>TTYzUioLhkVF3v5$uBsVy z%JHDX&O07l*m=ht13T}r@E1Gpc%)o`zqljd+=4go_YVBUlNWaCD?quiQ;*jeqEo*b zF6a5fd$jo60{#o}`d z__|J{xsaa+X^! zH2kmw*DMsvNP{1C;94cpyhy7{T3^zFNt;dDeA4!k279`)ZKp^(N7{ALZj+{kJ(HIu zhho7eK}ah^S|!q|lGcH=E~E`5Z6s-vNSj95BGPc?0hYi>v!1lgq`@($ln>_{a6g=H zz{`yD4Y>AILP*1z2b?#Kv()0{OnejFN&m%u+IP-vO_*O);a%=Fdh+M=fJi8qzxi%4rzF6 zfcqtoc7(J?q&*|;D``JT%bio?iywR7yrQHPCk-F+^YY^qB&`9e!()FBeEA_6JSu&zsm^tu^+GzRA?taGt9*&Fh=b z*n@@#AM2Ue-}#wfV*g}H63mYu55K|d2gC6XtX=DqN%@q}947XO&vYjCr%lQq{R{pc zg*2FY7+u*mgOiVS0p=#L+!fL^p z#9dR*6t_hrY;IwP(cpRgsF_g11G@F>5foIlYTaJ_{2TY`*SPBt)(@MU4__q5Kb3~J zs(H%GE)TF@mX`@eTD-jVggVt~f}pj=A%lL}@p&Fk{P`ia#fD5c&(_9XNG|4M!b*;7^T!~3Mm^tDJs{`Qm||(! z?;$;cg1ZDM1y}`=Ba^GJO;`olpJmGpVD_k;ri!L9h4LNnbb}3yjdgkkd2|H~4A97_ z%P$O4YBC=^O%qe~>Thv%zudjiJqG}CqhXYLIR44S{}=(zHFq!OfvdBZKe5tC|MN73 z5Cob+ZEFyWWy=fGGci0^flaktVOXoFDVQ1zc~dP`5-R3t3d8`F5}+s-UJ!VkzzG6g zXC=Ttk#HYMg$4YUMVumb(`dqRZ|Dwx@u7)7{4D~1d%#~y_zTBXxHbHRN55Uw5g;mB`uP)J*4d??Fwl(NPA4$bJ9MM_Kh^` z2=KCC)One!k%kY}xQ53T_rv=tT*GS_uHm%|*A8Jfho2s>_gOXF=>cLw=?LscJkjZ) z`QWboLV6ABQ7d$a3+sXM6U3l_-Pu8yc8btxaO?=~fp}^t!A=eSgF={B+G&7$;eLr; zzaH>`P&%Ht7w)V1z-yKRo)!iKcOBR@Xb|HyPv=!96|g=)ls8;{!|DREs4Q z$4A=HP*6EOumnxDq6GXH!9&Cz{)vQf#6tw)4rSp94u^2G{FGf0CvNgfe6);UtPgY( znAT0Pj5Om&iy&74Z#+Z))P;EDfmA4p^4Ww}V_ ziOnr!pJi;Vv#F-^5JQbkU@lGZ5KB#Q|Du}WZWfy2@GVjERI`+VN%<4EmCB|X zrwEvS3U}%40*y5#-E1@^wH6wSRPz+Gq=E_ggH8QJ%{5aIFB6elNW{zK>=U8$u9T5m zNc{9iM7Ry05V?iKA3UP1%rw>FZ8WaYwwkgF?VuJ*K`oYsTC|5+gqd)e9DmM0N~=ZO zcSXWi!}CaU3-i>6^Bo-Hm5eo(fyLk$tdVI=;@K>Xe^C{8>{@X?w~&udA%8FKk5t4P zMC29{^Qnl2uy5poeZvU$jpDFxm}R(cgsF%(iO4M^V#f%|ShbL*!a_rhV?-`EU%<=X z0&~Opq6C~Tio^NB3eFei()ohV0Czggm^lGJ&QVJ2xkdoz_-k zm1>z{kz}4=7F@`$pmMhLP|@Lbn6eezLOMRThI;WS1Y~0%=K*qFAe#U=ACMh^Y@H$b zv8u-J5}8{_#tHY?BcD}~?-7|>NX9QE{%M=p7`P(cr#!fYd3f?XvejndZ7Y!;5Sv@b zuFE4+KCh-!pn;}jNMTJ$e=|6`3TaASD3)3@rASiYghH(TCaFj`QiAV;+(Ht*9uZ>f zln0JaLm(7|<1>GT$0xp0MWjbW;uex{t^trL;a&wtJhbC>a6H;U`<>BVrEUox6OCI) z!&fIj4gGQZh&PQydP4bd3-jsB^T}5GcwH3{9DBlJ%PnO4^N3V4g(CvS`9S+v2HHnE zH(Q8`MTSv%pdw<)IgwjP?5&D1w778Yvvacsnt6t_Wh&ZpqHzmp_(DvmvEL{kj)xNI z1rfQ0M4Vd!we~-48$UokBE6(MxrKR-;CW`dZTzU(CcLr8W6mw)k5R=u^ZnXLRoYad zaSLe?oaT}TYR?dA&k|}ce}=VpL{%<${GgPJTS%JDNv!q^VeiTXL?a-U0HRrj#Azzx z8zOQGiF0`*|L5)CB2_#1mh$Bm=DV2Z`^W7d-e4E$9p%F<%qNmZ%(DQrQgDktA~!_M z7@`K}KDSa3HOma6<^&{>-V>EuNL`yPs*S3wxQFw-j$26Glr8EzRSn~JkU5oGNR8!@ zX2*&RTu0W=@H$f6PyF>dvbHJ~@CZ!da|`+S;T<^sv(@I|m`ssA6PsJe-p?cAkXKU@ z+C0k;GmWKxA&sS5L5<~wUweq(p+_SUzS@{afm=wz+uv~B`%ThUB5?~zC$mNRMkH<_ zDTzn!k9zUgrxxiu<-sk?1IPSjw{{X#wey46+(PyZ9tqa^XWlaLR{-#`{v;Z=kajoQ zG`wrh>zi9hdz@{WmT25U+DjhAKdLEpe@~C{;1=fbj^~lB14+&F=AJq&oi_5}RAd#@C1c z-nCgV6)_hPxrIb4PW;W*59hawWJFYMA=Nfp)W20!9IMSE%Ppij@W}qrG4@Us4`a%M zTbPG4&%+U3;;VJha5+3#E+VYWKKkGA#sQ*_J**Zz}-mbec6N*hyBC~_LE|; zpTHGry8R?jMJ+;9ZXq=+TU1{a)q<$pLh3jk-T!&N&IkEzEZ^&zGIWi$NP~ z1#NK64BKG!z0DG^W?CDZLv^=k3R~@4zC=YI79??9HY_*p`s`8g4Hn)(yg-68^&a}|C zvvMn}vAQ5XKk`;ltck)cq{MNG13qUcqAB@CCKOZ=Y>2=uB<$gYEZhx!uWGMtiNq}= zCGcqdaa%ZFRV#Ls54SL%V?3X1wS|XN(I`c1ZXx?LkH&A@A5`}eOB0b>NWAddMD<+( zdm?fRiC2D`I6}46Wr)ZvB;Mi?{hzjfF{)a0pgg&SdEV!FX8Rr{z7Siavc%^W@}Kf3 zW%1tpVbz%tKl#MZVcbGmDyL1Q@-57eD&aeaP9j?Z5_S^OeAN@gIkz~A|v_+U00%V3#k^2%I_)G%ka8TeZTV8u_rhjS|m4Oa|_ueRP5g! z*V0E7jf%wP7P3n*w$=`w*@wVAO8)|IuQI>JGWOSdmAh2!2o7B4G36Fg9DkcqnJC;s zipy_P+=;?1q_{K6pN%w{sX7x@p**;Sd3f+VvUTTcjjBd)=r@lgw~+14*jg8OG7V2C z?cfPzNw`ig2~Q~HD+Ra*p`ukI8n=+vAltO+MB^6Hnq`|-gJ|4BT5Cr8vznTtswof3 zgIkzKN1jKvYHFt{wl#^(Eo66N?5x~bRM)vDk+_8w5sh0&3t=>VkHiM~O~BPx>>z*RLMsQHN;ULfQmQ z`;Dv2XjS>@5|LX-oci0uc`9N(B616fvl;Qvjwc({@#I5!a0~N*K~1oyX7AbAAXWM6 z6PsJeUe4HB7Z~j`#6V*km=nfKnZuZ=!Z2paOjG>LuWiz2Rh!g+Xxu_tRJLij*YdM6 zw~)3W+q6bR;}+6l812uGuXU>0YE1cX3-j5@^T}4f>wv1Znh={?$lk};d|YML?pD=S z5t|Z`TSz?gN5p1CK$RC$})qWS(cX`&~y>ac@a{ zZXy2~UQiiO4M^rl^Ppa5U%4blg@K6|of&xrM}soXEyi8o;>9oLL@+-BLwv zO=NB%`8gvOVSO4%;yu!N6W@l8o}tt+%T>r>vsy(qc+|e)K;#Xiq>P#eVA<2@H zGP$GaqM~#m3b&AA%_&v#YMi0nE(`7Uzg@+;sz`oB;uezZdDPf3@h8`vr&MRuu9OeA zFdrwLPqx}c^~i;8#O4;V-FP&9<8G1qNa#*PZXvNMCuZ^5K3o-Jf1+^&VSAMUoL_Uo`Srg?W~%pt-bCaU z68(7u|L5)AP{>Q9K9n!FFyG!h-#>2m+*EPzOZjjM^9kk=WB2(Cp;gKSQOmp^XsX)p zf{4m3q=sgTYM?4>KcaFAsUxyQy{y`2`V*B~NFBo?&5n`c@VvTKhW91$siq%zd&0k8 zpI2L`Vi8PyZXq8+n(e+IzU5M+0mSAOvZwQiu+fkDaBmXsN&1_@y~%tU%h|v715;G} zz=1^K7Lw*>i!_Kx+(OdgY>`5U#4RL6^2q(s$RYKdy1|qOw=j>jJdbQ$f#4TlL<%J~ zw~)PwM{lX*rN5>VuT1cdEln1vk510mz?NheKf;3l^e>}0dg=_=H&g#9& zW2!c40@1jIG~;a3CK8QXNXwsXS_IL!g*0;>#Xs(o1*pz~lPDiXA_lMNcH8>{h#;W zaJGy{b0}YKVZQBozJD}^L_IGb+hBgx#VyRkk4G#U_x|vG_9D$AGPjT%z{$Tk`a|8C z%_l0iklL41*?37q7%vI49HFmi2Qyzw!dUE*F!ME|XZL@p%DRB4+(PO=Riv|f?WrDH zvXJuN7UluP%yt{uRaO2)#O4;V!+2EK-9>#EFIhliHCv7Y|6UEaLt0D}ZXsnHr<67V zLLrS+tW0>PYF(EQfm=wJtSZO9kK4o-Yl^g#DBMEIOdhE}ZU@!#29{Ai+`@e3^L(<^ z4z5?#$Z}$H3)xF~B!1(X6JI_l(h4GS3yG_Ko2VXp5J^OCAu;;5iRx!ZD~ZT0ByQmm z{hzjd%T;H|Rg@>UFwZ!iXSVN8o>0YoHSxKH{5?EMS-3yBPIZ+WMI>$^DS?wR=^?7e zcCR4{w~%s-Q-0(Aq`ztl*AkIiNIcCW_&;yotRXLv)=|FP!hA3AeE+C@E2@flH08l9 z%;SnGT3MSFpR6LTCnC3yc#9LWct-|b8Y|KUqHzmp_c`r1dy?@g>PDh+3#m_eWZ9Xm zW~TQde;p$ktJ?le#O4;VQ&sHW9V4kee{3c;w~+mTM}v)#g!_;FCU74zuf}rVulFIB zs@jh&MBx@vzWp|3D^a+G6g_wkCiFReqbxB*;TBR18RgHOfQG8hfZHe!ZebpIc^<5` zf44>!tM;T=Vsi`G1$h*p7RtgC$5Qab@!#fBQi*Jlb`XhM zNGio6_eb|edaG(@C*{E{%)^oAk*(UVsi`GE<6&Mv?%I16Y)gh7E;_f#S!MK z!RRlWSoxWmo2neUiNGx+cx0&@fhx)#qHqf--aJx&eBI{;Y?1a-KHS258t{CwRU;2n z+pv$=+(LFU9*Hcr4(d1R{Y^A(A+0s1{l-cxzWMi#p~+G zac!z-98XNy^3VIPp%aHUSgYCDAm1{{-Dhg>R@1FNyT=wAaCzgZC!cp5Yq9f4*_qZ(?UNczsPVE_>bQfYQ?7TZ zVi@RJ&3iZyM?wXJJn}UwY~*%)(?7kVvI$p-p9NL)*5qY{Or%2^G+UqWnWG& z%Xh{53>w>Sdcpe5BG36;yj!b%jXS#!wLF*qk+ClRaPNBMW-dJ9P;7O9B2z|eYEb=I z{iCaQHoP5nz`WS!-j6$9F4ey5_nBVHxaq>DL6LO}Vwy5pe!?tB%?-fiEk$+xaEx1IlM zM)w`HXSm%6S$VUa?Zf=<67x(wa=pmnBC(Czk1rOmVbxO8Ij(+@ zjfSSgm-+Ithr5mEp0JurKL$-Y^}2JEUQ*q@x#O#}3vJaT*j7=+HKp-^S$4EJZ%x@v-6eL!I>95p6{O2YlnNewr{t0x_xj;?`@k#xa52rS)kJW zxKVqS``^6iKfIFaPfBQpor`d_025metzweXOVsi$qT=l{%wnIw8-JL53 z>=@QBY{%YS@3zBBkFs9tvHZeB zzXUt`7Ei_=)ipW1X+)`7ZTf8Lwya#fDhmRS+{;<^sblYUgYz`#($3iFt9I<3hbcXu z&n^-(VrKd4cgDKgzP=JruBl7@oL`nVui4w{;IN`!M{YJ9Rl$7AoFawof9CfK=vm-N z>xa`kuG$|cXqWTgo}<<;j~5!{8hUoRS%C+C`?)p_F#H}D6kMeK+>p4^^8yDC?5Ve> zx?^hhoMq~D-EqXdLx((93xvge2yD7x`=(>g({e0J{W>}Lvd6iRA=kexIvR4>!e-=; zkj87Z1C}jVp3=)Z?D!?qOaC}mpZ9RX!dC8vRo3+#x}$c3webOKwmWp+=a~@eI&ApJ z+HI!#pC5eOtNn};Lj$`8A2{Z@^!(A<-3koysUds62CQN=!PTx0QcWcwAK_63S>vN_@IThuzFj^*klui9_jKgi{*edW$; zO4#>IYX0+Va@X+-3KUD8;5xYRiJ^M0@)SL0QtOI|#R~6dHlr7j3&#t(yz+}_mg{O-5Tnhps^7bJVkjwzS; zX7~6Lb(2ffKKIa}wtdw?dq>Q9zjVonkr5|OEiZhyPm7Pk+UOnpG2DF7(?a!oUYgbT z^_@+Xcb9*s=Y8mOht%CyK-W&2;) zQL$g+SHolf)?0P%RG%aDKL>q3a%Os+pQrK-xV*hhtLk&=1Y3C5`>}KKiT8~wM&vgs z8r&vz?9UYwgQF@<`dC2UXxiso73#Mfcw)ufhb^|d*i9Td@=CY(dflQU|og$pKISfQL=c&4(C62Dr&s1{FC$f=gjhK z*k$3T!gj-VzwZ0!lz~&XLVc3^Yr6HT_~}88x|P?gb6Guak$)ZAQ~eUHqK)I4#+_-l zb-jPBu-aAs**|e}d5^}kY9`yw?zMN~mp;uLJMX(&tkU(-yKk4A_hnZ3C6C+oU;5@w z*xZ@TPIj4WSMv2L)41U0Wex{Fw`jGss?GQvrqRRdRUTBk_nLsWFS^Y=Z)LQ%kmH(` zSBLfcF?a2;DY}Po4WCS1da+$&OV?iUWqL-f@QAG(ZTnY|N9&4w+Fq({!(}nOo%TOl zyLd)~-=I>hS{=^weAeA>Yh22-2=(dG+vVDc;V<{p{vwuj@7h)$a&2a^SUz%fY{;=j zH3GU^h&r05M_|DkNuzI{{%So;EZ060-22B?wuR%+??HjSON7)CV$zr+U&}*fv*OZA7%d?)W>@@ODc*IeQ zj#tlLZyvU2>8*N`Y92Kd%O9L-em!=$@iMWT81*u8V$WSOlTACf^B;5A`puM9MkCkz z4i(EYn;aan%x_4rSRT~nLi3ZQ-|ee+_x8e7pRG%UwkW; zSN1yFi5G7{ZoTHwB={)ik*IO#23#gFZ(yWAJ#eS*s;KMD~B(# zj&@O%@^ z#1BIbxF6l$-m9eF57*bOUmM!&d^dMq+08x88d^!_YGFNS_YNLGw;la`}a;XSUB?R zmq*J!+twc1!Zs$tw@=P7<4T>_9y7D8!^Bcwnm4;!s_U%% zv-;SMVNY(Hc>AMipPzf|Hci}8J9$Ftb_2VJ=g0|F1^I1Ui*&hoZfWkw+@ra zt-IFc?C~qJI;Xn19`g7;t(Td8VxF?k#?;koJN>yPZq1_tM=!-TDnISp`iPQ^PU!BP zi>+vWqEYtMYb6g}ljA zdsm-t+dWjj``dNnFNC_SX_(T{$JcO7&kem+ZhGJs)2T|qImtWj&9ZN|@rR4Uz3_MT z-TMwIl)K{c7ms@%xI3m_^>ddN)~`^_x8m|#mEMopyhwX+_r&{?J4NZ0(G_31e$>tN zZR>n0zj$ob%RhHp`y1Q*ybx9*Z2HJAb=P$}-}bMu3-0ut`JiT{onIH4t{>6Qe6Zbv zzw~?-9-q;o?%Z=pl@>Kyx?s?Ou6urjN0xLLz9KQ{?9D~VR;{K5rVbiA@bFN}l!99S z_?BJku3C23C{Rs*Z0raX-LVpL#L&T;Q8XWa_DfAsR_a(bpY z-R-^CEf0G)d)>z)iQU@lPo1#iUX#W-pO!tpD#z#ki_Hv%EdRLu-B0BO&-g)=82YxeZ=ys8VeqbHP|pmEN{5KA+hAT zYKx{6Y;e5esCJc;+PBgr-z|PeENiAWn()H@=LoSpU|{g{J%xjes$SaCqD}7kkNqBO zv}xkIY;&g&J$a^4Lg2lShbzuCvgbgb93uO3G_?3(iN zK*VXW{9x#~Qcqod>WJmaR>$t1YTPDud((X8HQpL_|K2dr;M(TyA!7O1*XmA>`-GPj z%Zb-Ms_A~t8aW_`n*eJN|fAU5dOTtrw>)1F8k(J!YKES8h5N7 z&K~KjDK_n7{n!Wblgjlu_h_5_MeF+pljF)wC{wt$euLIg#`jMrzbd`Ivsk#3s{T^eFNuvDc3_R$&1rHgzaGZ|b8_ADgbwH&`>&OndRzBisAE8aFaNd_T77 zluk=4wHetdesG;d?*^BRO5JthrOg++$<^x2oIiQOl&=*&wX8MbWXo}*x(x0xtNyUc zA5$G3wf@|E{LJZ5TMKt}OM0?xvdOp4FVAimw|RiSVfk(&Hw@H_C|&&e+eL*fS2#|% z{VHHa*YmBN*Ojw>YTlyGs8e|wd`pc^_E>&!@}-aH+on1 z^3_7$pKQ9mx61oZL!*qtnhogd6WJ#Gp{LdGDVD`t<9fJu*im->aodlsheMjpU0$bP zheBuFF2ocm&^Yvd)z&=?!Xw5cn0Bh;Fgn*z=LN^ZI&8FU?%`^7yVu_j``0RQIez?% znWN6W=y)c2PH@D9PX^9z=VAv>m^P!phSz5%Chk~eG$tVG{eh!vLON}MP^H-^ox%OgmK#M4C|awrnZY-+nf@>G53hA#Wr^xz9d>URecn>}{Br(n z=M0!2)*sWp_N5d4I^!2hBUbJ%KmC>a$M<1REvGg8nYz*bd(?aV*j@E*53{^{XwqHh zFP$31#s$o8{r*_qzZTX>2{l|D+j7OaQlk>9wyzczer9jC#mgdV%x~xa<3+Vn(aVc3 z>ZctL(!t`%0mE_;;myTz)APrg54)njQN*vVW4($K*EG#9;`e!C$b!KOKM!m=;93i# zMQb+o*nNEIcFTnoHxKFF@Y&H^(UCDbDC}n-zv{qaq!kUvD~Wf!%D@Mw2D7t)O115rdA)WpRfMm!IS92B7W~GH+Gw~ zr^uASHxg679`C<@&TLuKN3Ec!xW}QR`~=VCLgfZN+hUwVyh3YrSpGcu zO^wW!1yAgMg6H8L`r+B@Lw!WNV)`BI;dANqF0uScv-bV4R&#Y?x%uwKtxh)hP-oi% z!}#sB97O8!8>aCC9wOJ51WMth_^+o*+DV(c>vEKaGB3{iN+i%;pbx?P)yl8D* z{YM&uo1%Q17nN$+vEf1c?*(n+YF=wy^Mu3XYQ;BvJtU4-*Um@Uerw;~B(r6){~h^I z=wPEIea?ve?}z2?qxlA!R)3kiDCfL=Z3c`kuwd<)AJbeiTNe9Ye*dw}R(Je*V|ko; z_fhSR)jJei$oKfbo_EFZ+UDKlw|iG^8UMI>lHTF>IS*cKzj$VokzZPi<-%P%U0$(t z;##r)wSLojMv)5L=Y}~4beJ-0$+*4^YQ70AeW-;vK5Kt`JH)EOjiANV4!CxG9+4>O zXT#LJN8?J@ae5s4wZZ)gUpI;P9n1M-OF;KUBSid0?k;Zs*YtzqtF5$Y+T#A3ULPEr z?i^Qt^g73Vch_84axBlzp9d?nZ|i3!j^{Gk5v9YHtPTWVktpBpyA_hPQd z_#N#E&C6_A@QQwm+WR`!JTJj>Uu0DJ{?i?eM7#ul!>98z$MrI*ab$0Fo*9lijYNDe zK3kT{x$UKdk_BB-QlwJiQiNj}pAEcBgMn*fHTB5iikRhIc!^hXG-E>7}%VPg4Zu+YGKcSur#s0T;^eqj9Vny<95l~fU%2v&#;M0sBWy^ z)Ke^%-~DF(Sp(g55x?~9$M9R08v;7HH|cx!?BzmHhfetxJMTMA)2N*|o{tqf)aT8l zyT(q_o^5?&`^w3wRNW}zC(ifccoO9i`$wI37ArlE=~WiL zi}q5)ue`C(>(k?g zS+U^`0ZT>;-o7&H7axqS@leF8@7+!vX6+305X+WJY&yRyVY*fzL)L*mujS|1VNM0l2CH6P5Ty0MDu#f}(Lq++n z+j|@A9vAK^+KcV-@z*7f7R$LGG>~G@wLL9$+ z*62q(I%zpm#P6c3c|`HGIWLI#1zo>=XTYW5-Xi}-0V5vWGM{%_93Q({Uf4V1@W~Zo z|4aB<-0xIwhn4TvHn;j{bZgtrtvB`^f7-)a?9Zb72e#aF|CW1-W5lGlJ=ixf7p=v}F!)ToRdvqJz)3zlp;j?m`_DT@{CGG!;k1s0t z99^fKoA#q(Ife8X5@H*KUsToL^LL%rMEPaY!!Fx7Tb&m^o9E@hu>qL$+4Fiehds9^ z&LYD`@CAD;qnZP+ZA`~8kT{rwPK)2hf+FkPC~@!uw>mA3B}C55 zXYU}Wu<%QK3Ns&b;NWMAF_}pm9QUZxj#4;9dilXon4W_lL)K}-fm0PBkjy2H1*keL zzHw%qIyr>~sW6))L2-`QZCOk<&b4ik-{>3G%|xs>oYKrE5>{;j7N$}X#~xIj7T;2WoO@LR6J$;q=7V2E@&XH!y~J?v5Iprjdqr`DyJ~;Lex2vPo(E&22Jo8bX zSDhq|GxNEqaK>z|0aXhNztnpL=7aSm>fKr5;1R6T;=8r5JpCQD^JR`J^O>rucNd9+ zd%8}$O5vRNzAZ)ORAfFOs(N>mIC!SiX>raRmZzNI_DGonr+)AW1WqHcFjbN`cm(UT z_|-tN6jVv#56; ziGwwu(;iSbuj<}xC36}wA8+7bu9zA~9Lzzdja4}93!nWca~d-rY7roGSNZPE+P%3ml9lrX~^xj~Ja6zXFJyqg5VVlsV0r4}M+GA1qAGBu)!Zby|E2 z4stf!jW&}xEt$_g6{m&7!84pri*NNtPF)Yr47LkG4?fW24C5|8S!T0hZCw1q}1#)@1G9Uc1D|%z{lQ`W#)oJnVfylY#Vv)gltUL3; zFR=Q7g{hmw!97%`#V?>Br&z%)`{nZVU_O2-j=#hSU_LlA7&+~?wuQzjeVltTA5Y+5 ztzZg}IDw$*wD{fQd|>^v<;_8v(~J2uGJ>}QfrTkh;`9bpr^WaD6##42HVrhw>B|F` z!r)^JoS9%@>Me2lf~wQ%E9LRE4UU&NLCnV+HhdgHAoZ0v{Xm6w8RCrP8GNq66PW{D zAn?HehJ%HvpTr3Um2Vfu=g`@`1!WFgT7eI~mu50pn1UtFKu~pBd`A?P$MSD1efsze zVm{;GmkD5D8Ypo>K;_$oerl0;*11L#Z!Y*V@K1XDIVARMqo5*Nf%(?xhcnL;CVeWIj`Y za}q2}6C_Rqs5&ivI}|d@jdxio+qtuLr~Li*EHti4jgeloGNjqGasB+hUKY!eqb+|GlTiy z8-r$og=xCP!C!cMaQ`{C>|Kh?nZ`ubYLd{T46JaDiuEtEKmL51TTI9Ojzz4nxoIZK#N3~+XW zg=w+G!95H5l}dTsO5b`QbCxk57&R*t&`yu9vX^+Ig9CX@-N_j@jS^iAs{LOsu{3AH~ zB+h>3bCmhOba|@xBN7K& zB%K!D{)F{zQ~BmYnRA@^=mQ6SWa*g1Nn}1a;}@gSH}I^5%sIh)^x+8Z%qZ}s28nZ$ z`Lt3v4QJRT$edHm#}YUl70yYCbDH@yQ#iIW?);EBXP6IuRo749oR&CenGe2ET?bak z`bHUCC%`EiVtB*=`mYM-ti(xTKKM>`NPs>KR+sT|e%x5NW#Bq@#aqeb;b5G*jl{ojazjm>U4d?JiU}1VLabAL|)BZQ-74uPZUP>Hj_A=!_69PVJPO8Lt!+h|qH@H97 zZ`BbR>h$&XmibJD?eYN&(;JC{EgST|{*wczHtSw>I3Fa=C+1@Sn}S<2 zdZ%SqdAmL{pA^^+yMcx2lf?M~s!sdgoUhDB&G{m6zGZ>)UE+L`IMD1tdGM=QSYIz& zAMPP<*H7lN6t+vW%ReLz4(`c}qh&s7jz$YR!7tCd^|HW$dzp||q^dl65=Wo;+=KsP zeZ8=%2}MUseV)(3eB6N}+GTx-V~_<-PKje6ad6a=PW#{G$t7_NC5{pE!8ZqEeJvSZ z2@dY`+m)O7yo1n+cG*bc7-xZ#N8%VuoV;1!m@uDDs`BKOIQf{55nN~D*5nNcC@OE4 zDf4lLmVXHNV9FqTz5sls-NMm`^_7^kWowH?qVj$b9e{C%E?b7So{< zke*YB`QX-IF8I`g~IvPDgc(#m!}-_F;~UMQQ|l;AN*1Q#yMYP=V+Nzp7~&$ z#c|;zah#bCeyadE zd~o~*{HM9vZ&_EFQ<3?2!*=Q;GSsP&h5hPlshBs_VTn^TGC1Tt8QmIPS~` z-_nX}e_opeovrkoD$FNA#c`K7RhbWt^Tas6aoYo((e#{Z%txczf2vBH>dXh<^@f~w zFDu@VIW?FM_D_Am2UB&4z zxjeO)PYP_;Ff0rtPl@Bje8Ln?Tpv9*nNyqjTmg<~2fZYYH}eTrIHwbuS<9R{%;&kP zJl+zgF7v@JJ7RoX%_pprIrW&&Qs9Vou&%`MVLmtx7CAxN=2Vn9^_kB&;D~m|N8&VK zJ}niFZ$eeLc!z~w_UDGoXOpTt4J1w@<`W02;@Z82^*$_f8Z)1G;D~m)k;G}jd~p9k z&cx?orZT4~^O>sJt|k(v8T0W~I71gTo-cEnGoNr3r~=5$%Pq#A(fZ@U8V&o*LU;tdKcvn2)E5(^}%R zWj;7g2L98;#~jVzcyGsiN7r#7=V>w}IvdrnkeEfkUj?<13r!(`xZ~b9>hEKk{UgmURK1mQCu^)DpIDX70T;VuJ z`mJw<|#61TvoyN_obbuDBqVrx)`vSH&k#;`C-d ztCjdX40f#~bNVo!VA!s{P-aZMB~D-FgY7BCIs8c~G`+Zm>isZ?`Cz?^BoG; zdG+taAGc&qf9CU8#px$;f|*Z8B|ewhwvClJ1DFqAfipOz2 zJ`t+&gh-rF=2KIN^XQ~<8Jt)DVm@xF_=HNFAz9!Il{iBr&afW3Fo27}6 zIFqx$nIdr}OPr}$;7pS^QzZ`eCt%#5QeSoEZwZpOYXs8S%PnS3|v%r}pab`-K z*;(MskvOv@&RphGNjbi{6pTA3Z`VBLa|5nX``h;J3vmE1b0wCz|=-7iW>Pcc-Jd%vsNT za2%|-UX7MG8<@{!*e>LhXp)@4b=^kh6A7Gfr92xX&L-yb8Xy?m)=g&Rmdmr5`RK!T zMJSw25@!qZd8@>E_xPTWD-zZ9wUzncb-1|h-6C;fm=Dg^!?l;(sR6zH^qg(X=Lm4b z`8h`7#4;cJE(gZvLF|(@GG{w+#wptsD{frs-VYBlpC~AgsILTxbBOsYS2&j!#>dE< z!^{Wo0|Y=0m<~ytBh2Tq5})FYCQX+)N12Z$Y?nCRk4T(j%qLpmwCmXsvP{2S$C*zI zltFKA?K?0=0cfshWX(2vp)tI(rJlvmidG#oJPJAHp-lH%m=S~MSY!>I7!UM0}2Bu z&GL86AIqHc%m>HwLuIm*BylbNW_sih^ z^Ht^(3>fWgZ|)k<^!L2r1>e=B+d=yQ$(pR&D9PE)p!9y3bDR0}Rc+TTiIc*7@OcwPw`^3W4>IQt^TF$l2=Kv_B603A zAN*b-*4MqnT4plm9`kVrPB=m!-IX}^nU6S+g>7A2Tjo4qK90Z<=b!r$=OOd?ODWHn zrWZ4)_eacUCU8WYA4;6Z%*P!5k16Ns<*=*ag8I04!hK-FEtnU)7enGaWj+`iOio^9 zXUm*t%m;g*#@tQwlyPuDpOg};rqag1Jk|5~$rC__jo4*6j+Ck3zd zF?}MX815X9zK{~BDBno2Q#LM|l$DC&LrO7aGa|I2gew(gGAVXAv;)#qt?TBYHn7ij)LJ2_{Ze7*T*JA1QH)l0r%)I6GmwOG><=+$W`a zVeS)2$}UCmB*n#?`vhx6nRh74KvLX`aGwxT{#KL-;dGJ-gs#kr3qDf<*fpVsm$ z!F>!!Iie_rq%ir(C( z0V(?w1Nja)0owXv)^&4^@KT^&pN;gtkHsU^Gh;vR+tVwCunERN~ zTIUqSoRpSLxQ_)XR~03eIIWv zx8ObxNx7*gk4f=s$$g%Z@<35mk-BFLW-^n_gPG9O;VJlq?q_| zpXH=1P?Y(^Db$twR3c@LqO_p3ige>XzND;Bl-8t_?#_LV5ofKU1dw9q&wcLFTALK* zJ}Is}xX(jU_9;p~;H-0lv|{D1#+L|q#RHb4^nUxF{a*H zu|FSFl)j|Y?TsFA?^7#$&M3+R;xy>PT;Tn6!soc6SkqdK`f?vzQqC(%DN@=7ai5*U zxuhtqN%8H+ePU^?Yl;#_ieG<)L&{@C2_=qyF!wPdB}Gx*km5gp`@AFNv7&q+rQbmA zvzC-+ic*)9z(L$+j8??wg`$ijWnc*RnLx^CMVU^V(81itiIlgBQkB*k63Tt5lk#0r zJV+V$7x&pu9K9i2=}bzvqMW3)42E)_)1*vQl(VE19L9Y##IgHD3F;BlJy=9H2SxC%-IXBRGEA(&FxRq-(=MXdEnQocyaH=z(YS|~*JAw`fQM0qL;WwNA9 z6-otXrHC_$;|3NcXZXjrLU26ATJHE8CND|x77CFQz>jkT*^d-KP7-So`GTY*3x%SA zr-onSY)FbACklngCne>yP>8%xC=_RRQUtlJC=Zd_OG-zfP&C4cBcf4)6hY1>)*^C# zNhv54BF7L%keiVr$dO_#BCnE^D4|d^f{7#c=X|6Ha*Cwfm6ZEJA#$WpD9)aw2y(C} z5ADwbB_%{CM2;YiD6>5&f^09=qG&isilb1798VlUZcB@35Ccbh$G0B zqzJOUSc{@zASs4IA@U~T2y!D*1bMDli^%gOWuZ`LubWF8vDe|>U(Xwj#99=MCX&)j zC`3L&96|OcMU?r5Sc}NFBqc>C6pg;b5z#OvMUaz)LgY)5az!XaUMdvYpR1E1$o`@{ zL=KRYK%r1H#t}zE!940B@LLu^0;)pUkk|M}%Vl9eBB}s7?3X%5_ zN02*`BFIO?T0}l3DTzWMGG46w+74QfBFH}^MI$&AA3dQEc>}E_%G`hyL7pzwBF;=n znJpCB>t+*2>~(nY^y_(}u2_qr;Ug&xghJ#)#1Z6fq=+(K5^E9pilkf<3PqzgaYQtX zND<_-LLqXJq+AdRkrxYv_UEdk2y$mp9wPfmN;jcUG{z7|M8ldCK{gX>5!qZ)EQCVj zSmFqB3sMAmtyqi5(UP)3C=`u>#1V1MPl_Nvl$6Jk@>D29UL_QYvll6X94g8~`|}V< z8734WPbQ8ivjZuD>?GErXgEuXi%^KXhd6@Vo)kgeFV-S*f}|W03X#KzBgj^y2(pn_ zi=tsHDS3rLyeh9*Lq}}`^p~8@;#bAGN0!fdgf_8((*!SNAtOj?9nWD(^xVZJq$g)v>s{s zJ!wa?JX-c>mUn6_Y58MA&!^PWW?FV;NhTP8?Kt$n-?HIPSj~_N-|+XYU`Njxyqhqr zO`*>El~}ma4uzi)@g&5 z&gwNAxr`A;(fZ4v#lHsz(K`f-FpAb0gO<)3eiw2fMu<_g&Jqi&2s~irRlnNEze5ty z-w76B6s>cLR?qRR=IX4RQslyC&Jm+%{Y@+_Hy%mrw<1-}*RfI}F~TTX|0r6iHx6B^ zv-0oeEW#*S=cN`Ro*Qs<`2F%dUB3$!VHB+kiq`K(-rAzG?!;vvY=lv?{xxXv&+j0* z>KyzH@8m{|qIJ=rrLz_b7GV@Ed=pxz3uA~Ij*EZB2cKPq7r3|~M$vK+ORF0=E}iwV zU=c>q3L#b^7z9_@moJ?Bcafa0F84wgY=lv?LKUr|;ZxV@tc`+27)2|LSa_X?TB)^% z{uM8^uE!H8Y=lv?swi6fGHN!~Sz85*Fp3s_o0A&DM420X@8ep8QM9TlT7!E&&`4+P z5iG(eTH%V;;iF}9dP}X_@8?>CQM9TnT4jq<+UTsG1dA|=Rs^xI&g1st_~|z!ORbXm zyq7SFRt-h#tRxL$q&uy=+)LCuHk;@oi6s_9C(&p=z_ug#G-`9=kw*`wZ ziq>U{)`hmKw(G1La47^EVHB-81}*;HZ$y7CScFluE?2Z#CBJ@FXN`Ffx$xcSh*7la zN-fL{TyNyc`8pw3gi*AvP_){f`m(RiDq2KRj4+B;Jz{C+6kKnFORZCaMHofvN<}Mu z-rTEn*5rq{7GV^v`oxOmyV(;DU3hwj)cQ-X2%~5W4MHodZmRQ=cjO&eMGKTG!axKCrT30JtaaWygp|jQt7GV^v zCdATm1J@f}q*j+lxE5g)t!osmtUHf=th3$`EW#*SaZ(HI{?$8b_DETKdp^pw2%~6S zt7x_A_Ub5|wL`E7qi9`6EN#9H_Q;F4RcZ}c#y22cb+2F%M$u}mXw8UB7^AbopWs@AQMB3+OPen|zG7qy z9}q0UC|Ye5tpi;i%+XnwKgqQSqiDqwE0*tOPd482M{TL~AHgDwqLrX%eLu1|PiM7w ziW?)0qD9|+EZ3q5wfe``ky`5ni!h2-dqu0s-7C{|R?>gC7GV^v4#aX_Z>*K`^@(5+ zM$zi1Xw51;Ge&0(Uct2pqiEr$$l-5t!EVe$TyG4LT6+bHFp5@ZMXTnwgKpJXBUf@Q z!YEo@h=qBG+yAO%Z&@K@ctEfSqiA(iw5ASwy`Ih*_cXUf7)7g_LF;L$byBbhqiA(k zw3^*jtFO+Q{S4P4jH1=Up!KZOs{Slz5k}GKsc1b`rufXjH1;`YT{t8x7Y<9U7BBE_!YEpO6s@)qS8mZ+>jaB1idM2gi+@Kn zs`Px3YY|4#N-=2ZtnGqD7)7hEL5qKnG^z}GiE9x?(YnE)rLztS7GV^vRAS*88*>BK zqAO$!$F1U8gi*BmDOwK=IonHT{VZ67QM7I(mV0g_O0E2txfWp*t(z3BMmKl;RcHM! zScFlu(uk#)&PT6k<^;EhHDW<(Hdyb z(pj}$;Vi-^T7wK)B~t5g!6J;JHQ1n~vzooiwFskV4I!3$Zrm^DYn@;bM$sCoXyN@0 zoz;0Q*CLFfHH=v9xzS8&y(L(LQM86DTCXKv*+FOZ_%GKYjG}cjv10jdS_3qbF?>(3 z2%~6?P_#CFadeo@>h~HqMi@nFq(Q5N)cRbo2%~7-qG;tV-MK<%jeebL5k}Fv)u7c% zY8?|S!YEp|DOx3OUZ1J6Ca>dKgi*9c5vvRA#=7y{Z&hb)kn`nwgR=;uXx*-8{e5OG zd?Kt02eW$xi!h4TXkxkN#z@HuU(e};QMA$(t>IOht))->BuWk7ELZHTV8}K)LldeH4OVQfZr^ghX z^_pN2M$yVPXz}lkK#iVn;cxg>0E1SJqLscU|7)G~kzf%<(HcuE_qs7t&ezDdxfWrR z7>-l4TC)#t)L91wi!h2-F0r)SnEBAPjT%a=^mn)xVHB-AMXT4DjkoBmqk=^kMQgm& zLc92SN|-#p#%|C|dajt*TOM*1KGbFpAbhgO<*!{vKx$ zM$wvN&qnyhGjepAoAI%~idu0&uzZNXQC|XkuS~_doR<1=DMQfTtYndF^ zDZwI)qBY&1rL*qY#&L#qIIXDwdUT81v+b~ zU=c>qnn^62FVs4+AugnY%#ABQKqdIr4TIJ!MQi^bXYSHjPYD)b6s_6B!WxAezQ&Oz zW7uRn?|AcE1M$vkJSXypuO8z*9fA0$7 zpB5~_C|YHT7Os1BR{W=2i!h4T0!8cfSKj*lupHOBf<+ibt6b5lzWldDoz;IA*CLFf zwUAiaeBm{0zMQXvf<+ib>p?~9rxsNTbyoUru0pQ_BjH30BqSfcw zI}Yls?9aFsVHB;!#A?WQv$?JV{q{<&9|emriq^x5R;-E`I+d$=*eC|XO1RZnQ0 z%D#L&|C?pl@sD5;M$uZTXq{~{sX%AV` z)>8(pu~KWmey&9rMe9F`7T(j-SziklVHB+u2CZDFmGc$XB8;N7QqlTrX-2-zIwe?y zQM8^WRw5YH)*Ex>e3gF9wFskVJ)>x?JNZ@L21-gi*9! zBvvfnP4`k7Nv-Dui!h4TON!R!)nC7+vl71H#t5Tmts)j4U%27xDX~&(qhJw6(Rx|Y zYFe6eMrYmlEw@D&MQb&&+}9iH!6J;J^{S$EIP0@7bym)IT#GP@)>>j|F~qvDTE_4v!6J;J^#TPKi!h4T z21N^BPpKoddj80@2%~7drD$zi)2P1A+9X(nQMBGBmbTs)I{%)e<5H{l|F{-m6s>m@ zt%$wJGj!G`f<+ibYokGn{|!e}8F7qj5k}E^*Px}djtCZE6s`9RS|{YV@_*u5gi*BK zH)!dsbAm+}MQanWv|}09qR&dLdw=Fygi*9MD_Tbu?K+^ds{g`Sgi*A%5KGHL%#CEJ zwLq{4qiAhav^Jk@H(6)J9OqhuQM9%ZOFNbaKRK<|OnH1gELen5v?>&>Hw&6y)LD&x zxI&g%3V*CLFfwZouQDz!EX7GV^v4-Hy6YvAu(i!h4T zPGY&|#sWEC2L+2Tiq=Ppmg~{P;W{hh1lJ;rqV+Mc+;gLq)H*6ygi*9UQM9UmTJ=7i zHSQ0tMHofvQ)0#P-LwYiAY=HmU=c>q+NEe=9_p+qC%G}gC|bLTrCq0FO?kAUlhksZ z;w-`_TAwLei+^~cRA-e77GV^vJ;c&-9@kM_q*lE@xfWp*t#XMmi!h4T zUSet2DSPsp+_zEASNqdki!h4T7m8NHKXzTGvo;78VHB->#B$G#QBteNUtEhYiq@Bk zR-=&5TIj6Jf<+ibYd^8H7~&dWn~Y(~8LmYbMe8d?3*W=3v-SuUVHB;e<+#u;u5CV$ zTBFZ$Ey5^T2NbQCUpxP%vwjgQ!YEn?4O%;-*3@%ci!h4TAw_HB%NI}Rtg3%=7GV^v zZ;0hyH)hEBS|V74QMA5Qw4P5|KSyWP`-f{0M$tMf$A#A^cuuJ=wH_BN!YEqbDO$JQ zm-vRxYJ8q+5k}EELM$yeZajB9j{hBA#6K-qgi*APDq3HJ4ehJ5+FamTgi*AFLEuyC|bved$@%(4 zun41Q{iJBk+qr$1&MKrURK^ISX#Gqqt#07`jT@!bpMpggMe7$utH~Grmg%hNF3d8x z?}->i>$udy{dmuOp^V`L!6J;J^{b-QbZ>dQ&Uzq(YY|4#`puxVNNUv$uJVHB;C#7cztpRbUuu0LScFluLKUr-`ycG3v+|?37GV^vFoV|DQtO;x5k}FfqG&z({m5oIYkoA> zB8;L%e^tljzTOxw=c`@}XYs$hQVk zql~M%q80OS%Nupp3c(_bq7^}`SeRklT;aF2{Zwi-sm-+rqiEGov<^LcAVz1c5iG(e zT9L%k?xo;sH@l@)=gYVjVHB;JidOTKse^RZ7QrHnq7|i#>-)U>|JWn7(&}(6!YEqN ziq@Mchn~<`UkMgr6s;IyX~!~Nr!1FRIhS)S!YEp`6s@AW=6tTReiAIgC|b3NrR4^$ zH@Zo!{JLC=FpAb?iWa^nP-mSGEW#*Sb)**N2Hsm9Esw9_E4UV66s^k@Ej*TW)I<@54Z91#&m7GNwMXR1+Tw|oxO2HzGqJ@74 zME{%=eg;8jwXM&!2%~7#CsrbOj|g0E+#}~}qhJw6(Q2S*oen?oU!9fMfNK#((P~I6 z_w_~-skK$G2%~7>-+R)>mDQHj&{;P$FjB7RH1d9$$5340j3^VHB;#idNMI z^M>oJo3G+pgi*9&iKW$zkFL6AcRi`~onR40(YjjE>i*ZXKXlf_MqG<9idGY1Y3rz0 z4&8NEeW`U;un41Q;op+eV~BaEv*w9E+CdmaD~?$1>y6jsd|ejHJtT~xb*-Y+@VTbn z>8xdfMHofvI$~)x3fCJ$q*lYLxfWp*t)_}r{TmA6b=FG3B8;Ndj98e5xXmwHy6Slu z!)x)&OJF06qSai{dj9g!$vW#b!6J;J)xx0lqSWej4c8)!qSaE-s(;yiXLQy_f<+ib ztCc}(mDC!6U%>(!VHB;_idL&hH$1MheiSUiC|dN_lw8`ed?2FrtwnOaO0MNv{4XxK zuQ%E%S|_S^xKd}G5iG(eTJglv&MCOws4cao;}^-mMi^yW35r&)`718ytn-3J7)7fc zv0}+4W3yJhTlk~Yn%$IZ5k}E!uV{VI=7no@R`q6_MHoe^gF)+<)OuL32%~6qG-&Cp zM$NevVHB-S2Cbi^){BBg7)7hIK}%x9274th82Ki!h2-4`R8m0j9{~YnNaVM$zi2 zXq_EY^pVcGtu@yojG~2qg)P+Ot~E_+{U}(3QM9f%Xz8q?He8D^idLdQYlhUiAXtP^ zw2};3I;*TL*CLFf)tgvaZoD`6z_|zHeASEREW#*SeH5*A`}-&9tS1DEFp5?(vD|Z` zh16=Cz_kdYXr(Ax7Y47-(pk?47GV^vzQl501GJGbysjPBB8;MSgQB(Yt;s*=tk(pK zFp5^HK`UNr^=!|z2%~8AQ?#1?JAJ6m`cSY4qiEe|&}t{OhIQatgi*9^QnWVDYx9}T zIwDwvQMA&C)dgxV+I^;LSn=C(z6v^WEy5^T{S~d%OHZEDS$_%^VHB+a#B$G#Tcy^t zPF#yHiq=3yYw#7bXX&g9f<+ibYmn5!(+XavyeDILcW16e7)5KaqV;6`V+lGdstack zM$sBVEcdm|CaJYtun41Q4OO)8o`cS6+Ldb&M$sB((ApxkUKcFFC|bi6t)Jgqx<+U9 z?#8tUqiEesEcd!GTh7;R!6J;JHA2xkpA=VDXASSpwFskVjU<+q8~FK@MpEl5!6J;J zb&H~P$C1mTbk^t|T#GP@)~&?S>O8)l5+P%FM6d{>Xx*l0g{8b#UuPBcvn^d&bq%B*CLFfHQJyRCAI2Y&sl^~w9*Y)I_nw1B8;MyK`ias0q<|T zBIhe1k!uk~(aKb`+LnKGL}$GzScFlu#t=)(4ZOcGKx*|!;#!1Jw6YYfQ|}FWL1%3d zEW#*S*~HS0W&B*gav8%LdUGwpC|Ws+*1fZy`B`UuAy|Y_w8k2=o{(B&`fx46C|ctT zS~}~5U=c>q$~9;`CADTHb1lLsT6qR7omDf1vk0STjVD$j)Llg2{f$TDd_5spgi*96 zC|dK@9XO%0n)Kybgi*Be<+u=mpHJx|wO$Y`!YEo36|MBug|l^5>l?ThVHB-N#KJtp z4PWCJB4hZPU=c>qDp0hVTzvN*ot2o%wFskV70Pj;)|vcQw+)k8p9mIV6sl48ujG{GD(K^&5^}Zja)@=j17GV^vS&G(*`4I^^>$qSMM$wv0EN#BAzX_viq<{E z(ymkP%HIF{*HY^N!6J;JHAm4JJEK*$&Z;|p!W-_TiohjT5$C|dUu%YD7^ zl+^lMun41Q%~!PW^RPN=y+{G`1(Px2%~6~D_VX3xczmVRWy=o5k}EkXwb@+TK@_bVHB+g6)k+9 zp3W+}g=-N;(OP8Cnk2QZxRtXAqi8*(XyN+-bk_5NMHoeEF|iUMerOlh8}sCRb-0ad z5k}E^Skc;jPs*n{>n*_|jH0!KSlT^U{CvuFQmf}Eu0v=eJ)spQM8s3ORG`%dP8u8sT#GP@)>FjN&R@9RxLIng5G=wdTK`eB4tMSvqqD9X z!?g&bXssZY7DKEXugMs`ELen5v{ouwcaC}gb)D5Ui)#@^(R!L#+VwKN4!cfjeIQtb zQM8^>w08B3-J`RHWpgdUC|b`FOIzFE>qP6N)^~zM7)9$jMXUS41&ejo9XVW!FpAdm z#M0_~z44tsnj+_mjpZ!DC|WNlS~q;u{W+aASFi}9XuYUt;p-`NrB=0ZT#GP@)=P>O z)%W3U7)9$9gVtYCtH*e*MHofvRfCq!+9_CsQMA?)OFNeF^C{0utq~Kr z7GV^v|0-Hx8w;P(SqBA+FpAb|#M1H*bEB`+%E;$hgi*9!SF~c@`nI{w`a!S=qiC%o zRxF)S;JR|#n3wL6$5-A&u0kUQgY}oQho%NSs5k}EkZ_t`6weFe3wFskVy=l+2%~7dqiAI; z%YH*=^}K^?5k}G4NG$i=@bk+$YeX^EB8;N7iCEfoN^IgY9ePNuqk=^kMQgL7^-5a3M|4(U3D+Ww zqP4{^u3l2>ykHSV(b}qL;dg!Ltg^{mi!h4THe$J7r@SZU>&hvdMHoe^LeYvXdVi?S z`j21{M$!6!SnjzoT582jx zE&Q7cI_q7*B8;N7Q;rME6s~PPmRdJX=URkOv_4X_@cXfK)_%bvjH30iLF-egl{15D z5k}GaMA5?cIq0m@f<+jrvvcp{cJN6EYg45E9hOhYP7I+KTeU9vGWw$n^!yRQLX67V zrLek{PkK~my(?IVQCYhc*6KB{*VI|x3l?Hj)@KT9V0k!vTLrw~n6?@-`M40Hvi1-w zR^WJ}vDyh1VpP`W3Tx@V@4`1NXsqdig&38!S7AN>?^yWUuEyFVScnN>>0|RtCKu;q zYi@y^tlSC3o$zR7>fP4V%HCzoH?l^Z$?hilxgh}$Br47GjU>K;w?!d z2c`7xKPV+BB{eCf_wZpklZp$91T?cGuVB)k%%aSR#fb@tiMO)r`{qr`?w>cQf95n3 z3;h#P5p#Xq#Jov~{S%8zve7NaUXmD}m`a_}w%ed+H+M_!&golF)HlB% zv&0ukh?~!nnq3}GQ)$mp-_WOE(iBra;^Nx}X7rchLg(PS&^sy787IUnh}3g51Fbb3 zOhFGvf|7#ON2eg0?7>egq-?q&9!x zbnapH>n=Ewe3p_9W}oUu&0kvfvC)`Ay7 zNJmXwYrzYr^ARNZfAqnbWVDoYFcVcjf|7=wE(9S5Jza1*0V(g~1&cEoXszjB3VJva zlpHkp5QHQ&_~3LF!cic6pvK7sq9*O3>0lOW9>^F3;e>4PAqZJ$@WJUU#N2Z{4J#3K zFb_RF30gUHpM#(-4E2n3Iu$_~{=+LN6bp~q6qt(?OGyXQQ1v4yN$BZ9aEhT@0yuwR zB91{vGtgSo!4&jxBq%v(@F56EXz(EjWe|OEGz;Odv(R)f3q2eON){S?2tpPbd~iAo zF&!*YA~2`pN<1g7G)APx9LhjzO$SrZ!;zrmpuvYA zB%#3vr?XHbKTkx5vd~)7!7TJ}Bq&*E@F56UXz;=5EX2b=KNK8F!%9RQ%tKF4f|7_v zCxVcPMkkz3MK~7VKmU}TLy2f9>0lbFegq{6JzWSw3VOQWbOPcb06sXHf!3N1rl5x- zLCHab4?##mgAYz;A!H!>;Aj?FYdV;P9*zVh3k^O5Aqx#YIGu%H>q1g7G)A^|6BIc!2wGqdz|9Gf6n2&}41SKC$ya+-*nt0)K zKI$oFFx02C{lsB zda>4YFc&=>2}&j!dhUz`Z)cMFMIy^Vf~Mfih~mx`@Shp^EQYcm<7S~lRcOXCWH6?*fa}*!ycFg!7~$_ z`CHg#!3dHk{szUqisf%mZ1lw6px9YJ{SAw)zW5TD0RIhhU*ZvH7tF%oQOrak4Q4$J zxC>@suqtC74%xZr@+B+WYkMGF@jEOwRx+_rvRZmrhQ&r#{0@t)mCWD3*y@Zgq0!f< zn7bl>gSAKvj0K z@F=ZT$~G()O;W;H2h2iXRaBLD1hb+B&JjODVq^U>$&O2y8yg+*GbFawE&qaIt1G^Q z#ch44r))54D4uBp;elBYJZhMwv?EnC&>olt!Rk;|{Rw6r4WK9f2F1n#c8Nv9GblEC z;%`uFEnt3z#a3T@35>oTb(^7J6jLaq0r9{r2p+}MQreMX8fXv9f?!olReyq6OathN zzd^CFgk56Q@C=HLp7xUGY0CHdZnp4i?YX`rWeD&M&Z*@8!Id_Z=v{*jmZ_4UDbM_>wXD z8Wm1_{QK*I^-6yr954%kM@a=)ugZ=T)BrnR76Pkss<;!(Vj4I{{0vF(3aLj(Y;?rW zkl0$cF0qJu1jSZYdrtE`+e?`b;Rvt?W87BD}b;;i z`i4{63%XetJjy8!-TUwzsiy&V!7L0`1=Yi$U{=&Xy5e_OY^-FLSUe12vC$R3!(wYC z^EWWII^#=d^cA8w@!?zr@vf=LVoAuKk!;&)hVtz`ZN##U#1360xo6=o>_Ma_qZ1l$L+KzI~ZI-7V8fFqSP z@IIIY!m6}-dKAps8c<*S4~&f^&4)wA{YHQ>FgE()e_(7aX?};sR&RU>j=pZidD4Dt zTs8U-l7RbQ76^|5>vndbg!On!t|J9D@IIIY!m7Y}dKAn88&F^T4~&gP&4)wAy}%j+ zW1}zr2gcTt=67gp^~RUrxUC+w>S|VC9KGYJIbvg(Hw%SFiPcoK)>GY)8mozGZ2+7w z3x!pY)m+h9IG)>koPGkE9qz?Mb3$wFb;gF!*jU>xv6yHf(OL+Njn3E*8e40d|G}}< z9bdwuuOne<;6F~3=kXobBPqZhm<7S3$nueU8N4GEHqaiJ1;MJks{RDCwg%7>e}iIU zA-lxF;TaSgJ@GdvwiYly!(yv1z68c?-O8sZXw{U215wb;!r)O(t+gGgrvZ1tEDTl! z)x)7+R@6Yc;&)hVtYnv1JPcv6(G|bLVrwPyH!!w3<4b7t6{46VY8<`X3$zPnVelxc za7M8hz9VHd;4YYj!K$o!I26pX8c0|C4vUSo>=LVoAuKk!;&)hVtz`ZN##U#1360xo z)uOOCa`BJ)f$_mC5FUlK5^+Z=Yv6q_3xri^_4Fv1wKbr=_#YS>OWGwC5o2I%^u_4+o3q8zkHg+MvG&K=H-aO6G51Y<0$$jM3Mq zn7hF&t1u~naltGM9%a>9+mW&wa2L$NU{zK<913Px4Wui6hb4Gr)esgNUGY0CwpOxB zEUSjV*y@Zgp>bO(2BxU_5RriUU=|3E!is~p2!JD%HSj)|1;VPddU_Ph+8R(_{11$c zCC!IJ#&gl?p1MI_wW`k8TGIRujji7Jk~R9eRTO(_g7lt%z_?%*29NRz!WP<&RM&vJ zU={|e;_Bg0Fl%cdUGY0C!Ku-NE|-(j(}l3ik5H3Y_1XM72b+nN!a!7~mu!KkZJ z+;$(#0^w0uD-n03vIgDAe{gJd$CvQvYgsrY^8ZPRL0ZWM!~?S+7}v4hr5&lT zQXU={!f%5IWTo@C=HLp7Q6AsY5+a)Hz+oi zu}iEQoLa0e8VH z3|0lz!=YeS)Ihr8cUWw!WS3Yx3}La+6~Dt`YbEnHFt$45OK9{pDo&F9HO+^B1lk3& zFnE+zIHOn$-;uH!a2L$NU{zK<913Px4Wui6hsDNP=EK3_e)Vq%i;b@M9Tr4Yb`r?0JY%FS*SVfG1vC$X*17mAR^E))QdgDuQ+*YtKQGuv#KEx#OPMC$l zqr~FatwP{Pjdj2Yvrt$SS%XW#tg?Z1#)i~+<>x>Pdv9Y>c zVl^=djg8LO5E@%+oBzSF)g52L4Yb`r?0JY%FR%95NouS0#0W`UhZ)vt(;Y^E))QdgDvhxUFE7SqjFj z0bbh!a(}`s6donEGId94tOHJ%g~F=H8e9ryl?|*jHiRa4b=D*_HacTNXl$)*msn{{ zf@7;YzJy0#N5WYMKIX-J3Pzy?{lK_j76y+pYpv}_kqx*DW?`@@u^tWuv%m(@6~Dt` zV?DdX`e6u*jjs3|7F#Qszk#vU8DB!~Pw>Wlw@v9Y9GVi7S0#ztTK4~(rP&F|3I>WweK(bugwPeCoPP+SA#gIORv z3T!3fjuhCy`(PFbs{-rkQ7{W^Kz;E)Fg6yoOROTsz}V=E|ADc!r1>2hTfOlmIBqN0 z097|1ViI^K%tGN&VsZFZA#kL|I^cv^D6ERC!KGkU*}ytuLuhQQZ9W`0?yq&3gvLf^ zYzU35wax$F*y&DsdT~zHFHSo1rzdeSbF-ji4zm!y=-H}<^`th)>;pk*6rB} zm}j@=bAU~^@Xd?MhHlkNXR*J%Xgi6vooG9XHbJxn*6cshBh&sjvd>WKXSYsx2p)9m)Ct-So#OefjEL-cm$L(tB5psMz*sNFw>H^DIawuBa>P^i z1w0rPsy!2JL=0<_oik?gxVWL(p46(0C4_0uc$>&#dJEhzZO;;TFg-$h=7uZDnN|`v zagw$t!o^t48rn17#>NcF@YL-P!q^p&jNKH)F^!-}i~(H_doCEm)M&=0)#f-Hl9|2K zYK%Psm~XD+ICR|1UJQ))x~n)w`)y|LEZFOG4P$Lua#WFKZ(4R{Nv6hREF2y_-By2& zj}l|z6Wh=sulK`H*($)T7g!tz;XLd$g#Nug+B*Tb^Oe|(1AF!3Ay423H>qpl-zdQ3 zdg46%ivb(+0P~0^&cxn2z-;owdDv?P_P!Aq^ba?0@v92?UK@UJljA5-=OY&O#S0AL za7&EINNkgpJE6E!2i^g{Uxc4Q9ykX14ToIYzDfPk`t|MKZ)pFdVSQ4&HA(M3v`_lY zDMNZDbd8U1VzR&Au#_Q5!v+pXPfqDQ{D$7O|vr}yN% z{OsaUT{1HhGIPdsXq`E>OIGXlUE6239y4}Kr`8=ijcGr&V^)VwIb-8TjX~_FX`MQ> z7S3}Mee?2jibstdGcIT1#PrN5)7rJk&MO*KT$D9xU?B`WZ$?h`O*uuAa`KCD8Qmr; zt4+a}@vIx`QOlLV*luqB#4tD^DAL`!4B84s&CB1}wZ*CmhDgy--S53?@fC@bMl+qV6B$7}WkUfrnlWo5CfZlel|3dR*>PK+Ikr7^bIv@V^RkD4gU+LXNF z$(i}F#U+!o^9n|_i%;l03hGBrQDISDan7hIS))n{3i69bO)Q3o2_4!bbZXPCb9}q@ zqw*$Y#LS9MnsDyY)Pl#_5-zK4R=T7b0#E+Vs zRa7!+VrG7RK^9ozwJ;posw}NmlPeg*^T&mfsYXmbE*rnLEZt|NGI{=z^~ha~5}q&YjTtS-!Xyi9cfu#_v`-Atls^~wx)subIyj$ysBUM(2I+r*N4?e z4&4;JKDmY~Idn(roXsJr-FMDBhUjY1>+4*Z9J;4Zd6+9HbW3U(41Fv8PAc27RXfW+ z52@dU-9Tq~7(Q*dEUyBOT-f6AMq4RA3wG;Pl#@Lf&Q@?nhBZ*HTH%0ju_QRPCe`W( z48ETKhnE`^x9s~@_F;^;FHz!A1ZyJuu9JQ3 zWM3!Q*H`vo4)bx0m3^2yyl0e%FNb1EMshf|5l-Idu&X%y~#-#|^cF(MO9qJ`T zzu)_O|MDT>n-5p%H)m5=YSs?y`U{?JTRFR+VOP;{R&<1Qt@t71aK#U~$Hb1m#g00N z+_rLJMp#$)om;1C#i7jo6^HV6RD6+nq~eRbE!&Pv4%@fwNKV*q+m75G_RY2EE{<9bR>RY{d`%4mgZ`hY|CA zLdBtfqu^;2?md=J@x{LbVsUS5|L&Wk7hDS^8^Tm^c52J7?>!K`usKAf{id|6t@Be@ zYS!QP9++M4#_~FiQ_K7Fic^-t(#oEQ3&-G{o3sfAA2t()RB^6w6WexlR=rIwu$__! zgPZe@3#`tr1CZpIA%IEE+Kn*^>-I}n#kt?n>y+3z|AgqEFdb9}K|lSzhX)nnHxY)F zkpqqwOirx$CG6PLliD_X27Ox zM<-T291G!c3*WC8FcH3qRd**~zzzq+FDec-~yi)rOg#BBoU zer31C_bV&z(y#2!#H8qE#V6p`orn9C9ZgD48l7}|(x}_F&Ied`_z7dLLoi`I0FN>t zt$Ckz`ndyYRZox858lP;0Qh}?t`-9iXuDfG{S+5Y%$~?E3*|Ye1RyRpfa-2-L0~L{ z>U3?`lRxNu9~Lq42yjkzNQXb=82yQ*td# zK2&T?WH{yJWltmWFxYW3flEVbvy@r`;dwK9j=x+)#wvj$EP-2BF4jjIp$3olfMbEF zjd1EO%9)r6OFt12ENMS2%kcesJGSZUJ=PS6A0b&zY1B7oU0KCgo(4CQkNr(ilWG z)XA>r8sds!ce{>KJn>S1vCQ~{^!x%MmwA8GSPy|GrBE~NF00^H7VJAPY*L4Sg z)55}J0GuG{Zsc@(7*#mSgjwuUtC%cnOP-$8hX!K-k0*7;nou8lkqFPgYuKVagxsRj z2@XNVt};XijebSsC>gEpgzW(WDYXU&fefUxm2@od6f4P@SeQN)KuL7NGun2J<0@QK0poTbs0A0L>xE1YhOe?!hy2tL1 zVEu(FJDe;W`#{$KnDkn;&cb7{($kL4%b*($42Cskpi9&k?8H)lHDsVm)DY~%{K48V z&?RaETEznykNJTvdCZelAwIw*0?${Hw3OTg6m7$ri(5zdStt5GgC!qUz&+6RhV~X0 zE6sp*FFeP(0L!w{Lh&4rL41BMw4Xz}7TW#L;voc!w$cxwJqRs4X5T=I*?1IMJp8_g zHVia>fEH8#M`&@3fGMp5?N89+@Lo>QzO7?A+eILobFJ<2W*>_&{VJ>rDa23k!;f(UWuCfmzA^Kpf z0*7Zvj+-g_N@d@RvTwEQ+aUWk%D(-w@1X2ECi{-dzVosVuCat}J@KdpYbg8DWZxj! zhjYuvhU*&cGtM9HE0=xG$v&J-j@v5xw#&ZHW#5;w@4W287cn^ZD)FcWyIS`3l6}2p z-)Px4M)noSzA3VAp6q)-_PrqcUY31ZW#4w$_qpu*Quh5%_WdIJ!ePP3;}R(zMX*@e z7bp9A$-dsQFJ1O!$-YwAH&^zplzq?1zRj|)LiXJcYfkRV0`Vw<;f*19p)-QAqK$159uL|zXO({EC9`-me7R0SU$g-5s@41O3af`46K!qt~pOlB) zmO6h)-2KoMx(Tgp!9CL^Y^^4q%#ACBeQ;Cgw$M$mCoBneK^qP|k5f-1^wfZ!7{ru^ z-3w?2PwPN;O=#;u8wG7cXrrNxg|-&7aR5I+2AV<7Wzf?KP3i{Xq5E>^?f`9FXuCjr z1++c5PKD4(gzhV$I~m&g(56D$Ag!zIK63*t(_EL^>wt>~HgIb6XWgB^8TWnpnip_}pI zw`_A#%dM%)Dh_4rtT>c=q~epz%@uEC?ycyVx3%Jpyn_`LV-8oG+ZUIx?dZ9>kVh4t z^b zyyrHM1yGL!c-mX!4Rj%NwtaEjxQax^^q41<3kHhl`EkJC z!NoS%M8z--gptPX6QUcT;7c=)ZVV7~w-ViFRdl>Zw?KCg-8)ot7msd%?jgF5spv$H zZh=k~-MdwEDvo70+?w4iM!79()|ESu!=E$$9EOrMXM1dF89%52&c`=MC-RPA(2)|l z3$5;VQ*wXqO6q<#dck41j}2G%P|k1_l?=C;@Z6jn-HcZfc))86JmBRix*0EA(am^G zOX~h_boslGEXn0P8kBvMR33JHQuM3;ocSubJnpvS@>W#DlFK8jkCn$`65NkU%i0c= zEiLqeq|kqQN3V~popgVA*^ZVUrDlByPwUi5x_=Oko*tJ+)`O=ZfEUQrP%u%=?RgkH zhx=1SM)(2Jz*pGGg32)BjsUdCY<1l-Gj`xwQkZg(%z z76mJ3Ggy5wQzqbxAh|^V;ev2ZWo(}<3JTVVf!K+QKlAoQf5$eFs zzu|<0YhUg5e0w5%TxE)DZYbW7$OR*OYfm>2Fegwt7Dnk}*~R!`c2cc0=l2|#E`gxyuNn-9%|5E{l&;@MQci7=jo>H($`;i5>W9$+!hupB2WzZm2f z|9H833Vv`pqU42Q<{0f#85jAutYhqUWq($7d(y)Fx6}SjAvmaMa6P=6>@{-3=Zw|v z7vMdC`h-t++n+fmE4v-pn*saV(*A>FZ-%s&4hwYcLUwuw0^@c3{REys;K>r)%59+I z2#Yr=8WWbk9>N5Uz?Dyzt`Y zU`J_q0TIWMgGJK8TsK2w35yry)d`#I;RKH~&QvDt@8d4iiOADo=OWrUmxQKEq2BIb zZV8p{CU6mf?~vfGZeEQc!#MP#@B^1_Zu|5*IEQE--Zj+12Du~de&F=r#R1-))Awg( zYq^CZs!M*{+}Sc2V;Dj2!j9jY+G4|ekm zMkYp%;)7{_9k=~>lyK+p-dRi9Kg44{A2~+8J?$UvHfx#WD;}}G!4KZFbW2!H#B82U z#C0@~DRLn7+`&A4=*27ozer%bgU65knsiU+Fp3yA@oHTRu(9OfJso_tt{yz+_jIuI z;*A$9qZ}dXF~XWbC)U-0E-aG>i+6`%r`!D*OkZuoa}I7;QnN*W7q}0A&!H}tqU-&j z#d3HPv}d6m0WGHRNNBMP-U4kDw44jD&@P3x6||2++X-5@ z+fdpQ+Lh4Gg!UO|%b|S^+Lh400PO~7UxIcQv@b*ZBeZLvy#Vd2(AI&o+JB*K3GM69 z;??3C(B1;=o6r_Q`xdlIpnV70SD}R!OX((P--os}oKm(yn*r?y(Bdfu?orYGl*=Ke z{C)~P;`dW{Ulu&)eRw~G_u>5%-uIa7!vzrUgE0skcrW^P%Dzu!AKn+^_TcWAz~Kv< zyziv!s{`}N`!MCWMx5-!{NjBnvJYR9=Y4ShNN{hLeY0fW-LkJ-_B|x~UXp!lWZ#Ff z?-SYglkEFd_MMY`7i8a+kSBZ`SBXavtgGznDf{}#J}j1e9AjnQc-c2a_RWxei)G&< zvJdl@>%J!YK9zl#&0OPW+4r05s|LBpxi!S22zG_+YasjD%f8OCZ>a1WA^QqtUy1B1 zm3?z%-;=Trj}GqN8rk<>+4rIB`$YErAp3rjeO2HX;Nz$+9!0QgWnXjImm>T6$-W%f zmnZw~m3{MN-*d9>CE2${_I)7x_Q<|{vhR1T`vagTq zOP76FvhOb0H%In8F8lr?`!>kFjk0ft?E6^u{UG~(l6|LT-#OWLKh$v^`vu}r1anmh zVO2W7p#dA-|JF8K3WqWnIOC7P;n7vp=eh>QoP903ro%OGH@YEfc%uH{db{gxs#6HN zVH?aF`h6w*=2rM4v~msgU0oQ#3SX?w(w9ZB*7w(74W>o1@Z6fj(k!4d=QWg%V&M_D zfGDh_+GeKJ9Cn7Xs|7@}i=)#0;g2GrkK09?+8GatW_H^`e<*WxsKr`Ni(pN2AtEKQ z5S7Ldl|~SidU{j_p$)j`fFnTL>cayb75)gL(vkXHaKT~q@9#HiPaQq+pWIy?+YSBg z*WtU%UisqmKU40zbLEp6Ro4#~_}2&X3(rsd^6JIc4SL~&J&T(5dAIS8hw43>^kVrb zSKD(h{r%9{Mf-c-yYs$vt50=o@yA=ysilnv_pJZ;@YkMgwD5y#m-L=EYqx91;yaQy z|MW~uTIoNp{m}24>*wC}&4GlMeqWZ*aA@|4es{j@Iy3syjFU&>`i#kJ{px|doqw(< z`TdzzUuONzhVFxtBi|CU;62Z6^0FVvtKzx{FM))z#IZSAeeZe zt}isYY~4SL0i_Ty_$d%|bN~DY207HVLY~f@FAK`d#3uCeTdXw@Pd%|T;e|!pCRqC0TGcl`hI`4rsA+X}o2DED@ zI?x`x**tAPd#wjOtZcy|jG`4oEcy+P)?IlZR%bmb zSbVShxN0d{DMe2o(^)$Oi!h2-ZDKWp-?&}(;{Dh1%*1E7Qo@Ze%D66rUVU7l^?P5? zS-672jWDc@+oxUXKyRok7JlQl`RJYd#GMg%Rw7u431RM^eZezTs7uQNcp|oUfN{P4 zHc+?kT1y?2(mN$J@m6-d$89h-%t#Izl#;}e9&hL|Ai*Ru`oK?LAffd*?K7L!PQ=mY zjN&SBRT}fzWzDYdgsfb6V@+I#IN!q}G%kt4RXSd}$|X^>*#Cd#O-#uDgDOASi6684 zfB(Q|DcdEqi*?)NXDPe7b;3jWS;}^uI(BH^1$ywal*dC4!DlJ^_Jy}=U{Qq4{TzzV z!{NyN^9g+9wwZjEvScjWV1f(9!Z3cQ<3X6{<+GG?1xB+-IFHXhqC55AEe`IR*=H5; z*3C75Yg&zC@r+>h*+)EYmI;hz)65>;!B`BqEjV!4@cd(D4=Z%->XrM4cRgYOw@54x zy!nUc$Un6u1U$yg%s;#)J`1>5d@BNM-u%Py777dj;g^}cXTi)=z%{`46T{}sKb()d z1crd{%gkO6n2!el_dyJ|hi6T*_~9$oy=rlcmAzzu3o2~7kfByic zJKp}QWPS@icCj;`TH8baF5l&1ckip*ze8WS*yw}S_RznXfZ51j`SF^6SHa@{#E&jk z^Dk}>>#JG)dl7Cv`~;YXVxS4OlK`vu2&^AG1Ed0z-S{yWE6&BwaG zLs)qPRx0c9=D`i!;n#AU)qJGHhq5u(TiZkbdL@RkA8+9}EB{`}4rTY@&EHD=%bFa@ z20m!*AI``1%R^ahyj={N)qD&9%#>$1&f9#nhrE3Lxls1i^BjlKHk;oLa8u@2z+AnW zjl9ch$S0YzV%g5H_p$)jtx-PW@qR5A%7~$x!wVzVQ$?tNDHT ziZHh0I*zmQ@7bndEG>!Sto*B!7RG+P&DtLN_u;58_Qn)z|8RbvyE}|c!#6y_W;MU- z0CVsGj`KFZSYIN`!&vhtto_4$uJU9UTeF_ytj0ST-flVIGi!S|-WG7%zwU94vzm{p zXTsPuRq?K$_v07+y9k&UVmZ#rzk-%kSZa4`d+1+Wk18zYW{&gbAI=BMtinFW7ZqT$ znvdx0Dy;K(j`KDjc)X3vufmp3bV zwgAkk*&K&=NzIO5w0H9EDr~{r%I&QJObdK-Eo@fyiXH_27FTYs1z=V^R=K^CPk?{; z;tgz8<6Q-q7B5t8uV@wcx2AG?EdaA>UFG&pz6t)lUAetgfN8O%a(hJ;;2*vmRmpf; z0A|&$%I%%p1O9zcxxH0@X>rKf-U+yMS#-Dx8^hnd<#oN0To2Bde^z1lMB)*|@1L0E zAKKefw<_Dxoa4OJXSDZJi>mBMZ)}%J*Dr+{7fx}{BXS8`d4Eu zGAj44YGyTd<8*8P(BASn)z|}%RBmtVqt)1|SFP>gc>jK@8awe0$61ZH-o|R|{AP~x zHeSq^dfTe8Ar%~Fl`mrfvwkPXdCQmofq}jsS7WVrah$h$KP?>I5e1m7UvgXq2uZC+4I)+ z(7(DDsK?ZfwevK?|#6XxQ64b z{ClElIGfYL+8)MlC19>>&v91%ebp(P9qejt5B>WaFk^aioRxo3eZ$%5RBL=W^kO9f2G;sY}`0&d+6W2fVoh>aaR7lTO7_dO|iCz z{(S?OzISn)m47Gi31?s3Yi$qxI|G=4g&b$)U)#mutld&;d+6UFz-)Mu&kK7{A&gUuTJmktVfEqy$vvqX@H4OMUfhwLP4VUV!=QYmW0aA7f#? zG3HQpw&NR)^S0i=%eI4nS@b=}wX&Gs)E}#}`NysO!}wJIrq%_Hvzp($un1P82EKym z{rJH6-2j*ojX2I~e#bS9U@x||wuk@@J?%cINmbAd|AeER^x5^a0L78F>8A`-kQrJ z*i~yd&T2kdua97pHd@<5|5gF!ua7JDZ@}INcH(Pmd+1-Q0}-spZ`S_de59R;U?qQB z+r#--3z%H-h6iu+fydk4&>E~;75>HutK)3|U_xqeoVVkxA^7)pWDT}2n&aT9>G^G0 zIMjQkT2h8Cc zInLX7(cX%ok*wdXmD{@)Fz52E?P2`hnjFc#oyu`m@v8=(1h{iH$9an%`uBNhB%66( z<^F8|OsnPA{-M3&Pe-z)FI8^uGr**5w6=%itx^%mT5jh!tMLv6%=S+>&f9n~UmEX@ zWD7pyIIDbF0hs(RIj)tLd_jNWzTy~oNSoR9+g{ky3~=WK7W2yN_&|GIzV^*t55UD7 z;8<_=VsU){nEY=z&RhJN!R@FefZ6y1$9aoiEwESTSR}jl7i)X&RrsOYlU)u!xLNtv7%&H}vi5Hz_&2UmP4<2) z$9aq2An@-HU|KijxOmX^=3mX`HCb)=sm#BIfZ5-|+CRKp%x+nerFP^vyen&V{BDBX z3js5r562aG@eeP1x7<*ZJu=wZ-bUDc5HQQqt?m5@>*C04z)a*g2$QLQ8z9|#0Orr> z9GC0GKeQJ%yCyq3&)VKvu-D!`K8A3*2O_chs;A2`n2@r$3l{unTaf8w}iUi`!QQus?v*6KHo!}?%0-bFC+qXF~V zX^u*^~cR+gk?qb^&G_e7gy5-t5f)d+Al9*jM2k=PiCQVDAiIs>N_z zUoZY;fxWG@qu6zqTie6)Winv$8(Q1@3?`si<0y76*4iF^j`WJFqu4V|t?j{EFT&fp)cvooNT^s`sX)`;1!$4>e;FgqfEDYc4`S*K)5#~(iBNFWW1h~_8 zTicrpMpEvHVjsc}ZV*mWd&gnNA;jIwaSZ`ywm!HG`oEjUG4PN!v+-hG>v3NcJ3F6a z@$KLxJF*=p?_>q<^I(Jd$$3u_+gIqHeUQ_O4*Vq z_SjNu|L}aY1~88;=QwZYw^iWY$|s`OO;2*1xBOcH_J#xI^h%ENwqC>c@qO`36#MB} zYyXnrK)e8$q*WZ}?frFY!M{6JN3pBdSo_xy?6m~U4X<&WxADTVgbiHBG4PN!vwXq# zr`!g(tT#B;+wp<+UKbc)&NQFV-ur;tzP@sMSG?()f7bx6<%Y`b-6=4Y`F9`S9(>E% z-T^p{z6H$N8#&Hf{Y!;Hv>!0B+c|Clh?^a6cYuLqA4jq3yRGfPSG%yLfN6Sw;}&_b zhu`G7{csc;bky1&z8o|OFt`57aWlNw`xfjqIvK?-{Aq1(CXBbi=_vN-MUGqO#U6eK z$u;n`Z*9Wijl8gViyw|R4KQh!ah$jD;iZEJ0BBgC&pd^FqMmE*j{55Ik-tXDK!oMdfp1H^AFU~gF@{2JUJ%|2huar3;`s{sF^mqoL^%Q+6px@o=Y z0`|@Wro(d_=k53%2>z{J70v!xV{LCg*t_Nxz-{0-Z~2Vt`^VplW|wcaws#QbqbFdl z{D|Yc&Bt!AcW75MTeHX79(<_^`vNd2hd9pLcpJfZ-#rq|4*Xzk?>yKG{V|$dJi&3^ zj<*!B_xfLuFXyc7g~Rb1`*$>JCjPRMxA@`bFL%J_mIu|0ftlssabb3TTMO}<2AF17 zaGbaJT?YOgZxF*SHnO%i7slJPaSVGIzYHHXZ}}Vr{{7P?hE-3nws#ng-vq$a>c(;7 z!KT^y1M{V0uNd}fqP4wkFy1c#^Z1P%=WVg8$HC=;Y5X9qSb@O6 zL)y&pWg7_109-HpbCa;a^qAVik3}s6ObGrVOxV0$zX2{4FmvzXIB)A`eEZYd(ik>x zzO}t+P{=+8O!7jGYvRQ}c$u9odl2#zesF{7Hm!fOUIWbbMI7fXerPZBp%_*desJ?< z53i3}0H)JoYkRnEEdtE*0_V-%LBPEYm`5J2+}<9*R9#~2AKI%An7;(h+j#N%x!qE3 z3m(#DmMGCme&%ar5@Kr2j@9v#fT=HV z-r^St{4N3m4{7seFBx!yDzS%oGzl=v1cM_G42 zn}m{HXqs{nD@v#V3Y3~&C?zd5+ipsVNn3kGEiAcgnx(m!Y}$f|p{3Zc1S@j?5Gx8_ zRiuchh=}1{E}E2X9kB3y-}CJB+3xI_^ZwrZ@XiTf@9Kc|ZUyh20QPnUwD&G}!=6yt z43^(jvjSFVB-crtJIR+~(!^*xL@ypS?VfJ?1y>>0tRC z1Ktrfcd-0Mz&XXs^GV+-aMpTxp7b$go4~or%k#1K2slrBc^-Sr@0&IUr{>7yZ;kpr z{11G143^(=aOQb=KIv-)XN{NVNgrdo44mt|JRf`afV0iZ^Vnm4U$!~S2aino9r6tB z)8NBnu>6*R6Y=tV(pLvgtC#0VA7i@^oU6S&AA7ffbDx*zvB&)W+2$}GJTm3?9q|GDe&0e0z9`n1+<}iIcGUay{c&}x# z$NY}|BgQp+cnp@`2sjJ8JWu*)r4^iWygZ-$UJcG>FVAC-`MuBPFnv5S<@Z_eUdUpP z`7PXmaSb0HgXOmzoH<^eCw;Wi08YZo^U3eU;9TS7dF(O2x7!@1k4L8b{sz1qS?n>t z?}9V>PucTZ2F?sG&yzk{`8+shczHhg-2l!dUY^Gu^Lw++!Kpbi<@Z7Gp3Gv8`F#zX z!kyXkdjvR>ygX0(Xk`I7r+axm`8@}m3%op!J?3|_%`ut9f0^>z2j0V3>@mMDfb*`G z=aare{vY&Oe0aF+-jhCFodeENFVDwb0-SCy&ts4Iy~gG+ULKk9`&01l%3_cC-2u*P zUY<|-M*n$~^$C1<43^&+;8c5gKK9N4XO)-dliy2hj)@rmWyrU>z^+kNDLy>h zcJE_v7C7^~JdZt&SIsuZ7{q^>+VgqfZOmei`MnXGySzM~^z8uWB`?pDKIU@R^U&4s z;W1c#r-2jk@;vsK-#VLP4C22``8^A~^Rw7ve!mOOtzMo_exC$qrWnP}g9`ifT=HS#Eney8RUPl&t%0>Tm0_Pns&&S@F zm*A)1!^3R{9(&Ah#O4?x{>zl#<={1CvB&(L56)Fyo=^I21?L_w&yzl;ekVAudU-zf zM*R(b3O+mr%Wt{OF+}{ADZdNBJ1vVn=J#xHHhOtJ>AMk}+q^uV{B8$lmzU>bZ^+B= zQ}E$2SbocFjv?Z|O!=(>Z*dlT%x?!c=X-fR>H97?H+gwJ`F#YOXT3Zhd+&f#`1kDj zJ;LT>%kNC^PR?SF`E3H{Y%k9zeOG~Vy_e^c-+RFMjhE+R?^ST#_3}LS*nbb%tuo1$ z-znf#*xXF*h50=VoDMI~Cw&{ix!TL~q>m}P4V(wPJRf_zzGSe@>^%$43tpbb9`jrHYOws4fH&Ue4wm1O z!8y&#^GV;?;B4^nJn3V~t_SC*UY?J=-+;5j%k$V{e&4k@I5kJ6@#>&|K%d5k$6)!b z0B5n6=aaq;aL)1aJn3U>SA%ndm*->eL2#b*@;vsK-`8vo^T8uien-4E$~qVy9)sm~ zDmW*5c|PfD0w>|+dD6$&E(YgXFVDwbA2<(tc^-Sr?+Z4E`QVW$zj^;0WsSs#N2dJJ z-guj1i1;s)y$E;<0@!N}XzzURE)HPtrhxXgg7=#M_FfHWZ`kYDe}xZ^!OCxn%`rs$ zm#O@!!8;{@y;TA2T?F1$0qorx(B3xio(W*@?SS@1zk&N<`0yC4{ASo3L&SfX%5O1v zrw6ciPC$E?fp={Hdwl`zZ3pkq0qo_ysWQoy-!b4FVRHv7zeqrP%fV|3VDJ2Z_O1c% z`vL524QOu%crOL8H|(uoVocOiIJ1hDthfc72%@96;c-VA7O)E?ab z!zWw$mD`+b<+l*L(*oE#JD|Nwz}pnS-W>t$Jqg~<0QT^gOP#Em?@a5{gTX7cxr6oJ zSpn^R9=yf?_Rb4v?`rTi2e5ZvKzq-E_d)=Bg`r^OR|4L6n>$$fogC0!9eAw)>|Gep z-gm*fDS*940@`~HyjKI*8>ZpRto+Krn_+VYE5FYNwATUNIRWfl9njt_;PnNt_iR9WuY;FYpt2b(za=)u z83Z$pf78H=1h7{Z(B4_#ogcv7cLUnH9lWgp>^&FI-n-xp+b4VZjk7t~%C7>v>MZug z!N?ilTx|0Od*1YFa5iVL$LCk?1LxHM_TB-faLC~KErY!yY)-cPE(NbHi#?{V8=M<# z-eCFtDL8j!vB&i70B6*`%Ew^##(;B#&CAq3n7)Y3$(G-C@XpR+kNVM70qxxg-fdaz zvAt|3r_fVxKKGwL0nX80p6C8ESBF*LoZ{vA*jok8S})IIkIzePvN&ts46E*<@agup37p7 z?YUr>@|w-w6!0o+Zl?B3d#44ocM*73WwFQhd}}~^4}-Toi#^uw>)?E1c=r080Zx^d z=cy0YZzDLXygZ-!y#$<1UY^Gu%kK`GgHv;4s^6EvdpnCgwwHrP1gqZ&cnfT9ruwD5 z)`0e|0`K}P_NX7-6VTo>;O)v{kM%obzhY|wK0Mrf_|!)=IH!1dp88<@t^((LFVDx` zcfq;I%k$V{{XSxIm_8nv>i1pnhK=;(!^hq@n*p9es2M6%-`l{uFN?hrSokwI1qWo$?^tjqczK@malKIu&govBPkzq<=K?R! z$KGaeZujzh>^%+6b6%du9?P%bK;;#u=Ezijhk!TE=4NWoY%g=bIm65IN#6!=F7fg_ z>0^7j8JxSkJRf^Iza8!D;sLeA0ITIG1^Op7b%l zw}5kxm*-<|Cpa&Ac^-Sr?}(2C%Wo-oWi~fcep$cs!0GVveA2fOoU6S&Px_eO+rW9y z%k#0f3!K-yJdZu*cl5!*@_RIRQ*Cah{FcDpVsH{(o=^HN2Im?t&yzlm2e*T>&CBz# z_cAzddwD+g4*n>{U3_@B`S7uKGB``UJdZt=U&7`XgZMAgcyJ+jS7for_VQD3p7!#5 z()T7f1!Fw<@T8CJWh^+8ygVO!3&2_K<$3It!QNTmeBb5`w%)h}oW20|o&{&*$CQu3 z>>UhFsm*ii&67Ts-z=MhQ*&ghk4Er10@&La(B3WJ^<}Zg_VR2%d#{6+cgW!NI}P?q zz&XX{4OV`2;Isy?cOf|U1hDrAI8SG>M}P25a7sR&y?&1dXR6K1RKHVTZ?Vn6sW~#$ z?*{NL31IK$fcBmP@6{~!*#1U+LV2~5mvKCp0$zp9&6M8=jGP9}wKi|C`n>_1TLaj8 z5}YxIX0MM@aLR06ru4D>%>(EB0QNS5b9EMbEWg`qPPX!U7Q7d-*kk$%OHj`E@EEN8 zP6lVG&C8Tuw!ehUF$VEpruMfPyj!x^WBoo3&OwJ|PhSZ*<85B1^l`pC+2&+R-)Z2T z5y0MtfcCBiZ*vxV^e^uVXm2NYF9)zUVr;PfHV(WgHrFk`hT24}Q&9Xy16k;`XD#TP z(9xgBd3E6@;dkT1gYT`a7*$c>$6E!?`M$hT{+Ty(u-#t;&K)*S>4QB+o1$4Kg7*wK zA1zI9FBNwBQbe|cPl9uNf1aJb6mJB0VQ^})@R+`3;C#vE_0Mn09(i8_=f^fLQ$9Wo z-ecg5Iy`$m#(*=SKhHNG)4+*j;W2&l!0E7g{quo~_ECcG^KFiURg5})K>d<;eL&uQ z0eQ~_%vFdnTM{UQMC)O;EE=trU;`Iw-&9U0f7;Hx> z=Y*G3FRq?Dt9)#8yso~zCE*w2?1c!rI5KyBbnc@0(PeWZmC;2DDre7~Gq-Z#A`^AG zAy}JTv)E&OesyGCxTbQE2VOdRe$9fhB?TyySMZ5p9J#~_nF^X+zI;h%tfO=JnQdJw zWVivbS|G(TRX8fRyXC?+Qw=1$4)+Ba>KE8jdfFxJ#K2<+2`9)tzCLTQiD7r{k?a*8UdW7Z(q?-!<|2j<(LW#ze`ug-4alZ%2{G&x*IM zD4E^X(vHYFtkG6!aWuNBK3bbdbj0hr60uI}0ANkA=Jr@eIywYsZKBy4if<*+BGj2^ zus(wCXjN@zG#sgpcCM~%k2b^-wee;&qtfDzSaYDZF8rI zz7r9*u86vrUEcFQiN6P7?OS{Y2j)?Tro22lE50IH+uYn%kEYVr8cj5H#5$YWnj2hV ziRDYIZH}*KZHcuetV3*zN7}@BFw=#qjCRJ)idhrvt3$irwhv!(ov;*djWt^(;Q#!i zoMObA+tyh(Bi@>rSzg|`GTy$r(>ly%5U5(Z!ZAyU;r5%+}kN#kdwMm5o zNhpEUb>ujPM)`Bv+mJFhK~71fCmDv?+d8dJ*l|@1w9_1GOjw87*SiN^?})ExO1O!& zLIa8gNr-i{);3!ow~bhN12_$_`qbru0hinA6RFFE11>kk)?m+{>cwd{>6xsLv-TR} z^=>|_k^x2<<7;9KseF_TARv~;=C;~|Tj*BB08U+7TXSu5dsD4zD6iY<9zes$*AQ>B z4!6_XH?YuZ8_wwJOf<%0Xz5NRTO$V;Vj~dA&HuoJn^cR%ws|{RuQjk9SG2V?)Ws8~ zfs^9Xz+-KVWH?5A8h9gyfOI214ZN<_M8jVBEB|QXt){Yh)hCL!F0}<-_Ay_Zh`FJC zdQNS7yJ-Xr?bB%z9bK*Uroc(?=?&3&ZS@^7M#2Io$)`(z)DdfMuC0%m(j=>7V1d`g zTQLus#NGvlPnFcgR=Agas-wQE!@cZNP_fnqYdR9i(bUc=yW$Fr!qNJ+*3Lw2Yhp%M zYbVCZSVKuPii=aHjGs6aBiX9j=B}7k**~xuQDY5J3}S7qGbWB#V>~I=8taJHM`Ilw zZ5>gJ(y7!rnyO|?jQC&Cc&KpqRP;mAQ{5Ikw6W% z43eXoAgRi1$D6E3kYp`7vojGJEK#2clBi5pyh*CFjoFP7!wjlev-XVorrHiwKF^|w zj@o#l)1Wg2+tYPhhrklk7-B8;?Po??+E&G)ZHSpX%hx zueAQ7xJe`Nm=vlFjHnUbG!4B>aqOGx7n9%YpE$-+%R4Q6Wth~Gl{)%F= z#vzuQN4p91B}JRtIy=oGEi*IT+8D>8{!HtLK@EPfD;`Tk=he=$jvkc9$rxiXd?RwX z@biy4-Wjc9!C=M8)mTPSY)vhe&h#WQQsS}ttRXA=Zf1mT49KCNV+57v*v-ty!g2xEfGr;>_ll zX?z?VoN*)7b{gU$WsL)RAu9nt7>E*CyG7{BDnbARMP9r=f8!^2BvBfeM z&ic52DwUI?z)2gAOw%wv)4*ZFFK#c@9TLX-B-Bez4Gd)g*!_nGrrTR$>Gsp=!0AWn zb>N}e<3IqW-HDIQ9({UEaNtWDwg;4QdN*We_10Q?a#e4h1hjUw7*C`bcf6bhX?Zyc z!INIQZ)c?2;~3M+H0-M-fIr`&58GMQnWWokVNPk^cH=UJu#@vC zn{KAyyCmuF&EzNa%Zo_XO=roP1=p$QyjcygH^v>bW>1tX?@DW zRZA-25(z<4O>3fE;Qno;e<(Ym$#xKXU>Y_x99GH*CL zg~tt=LjOQWW7fsxe^R+-amib@gVa$|TLHh_;r4L;%9cV7EdqB9D8 z+P8J3LfA=iCj8NX=t$8C!5V!kqSV0U3_%sXak&+f3Tmfun%!ace{bQCSf@K5>>JXo z3sHD>G)~D42|$KzXQk;ZjH}z~d9T1$^y=H1n|ZI_ncw}Z+(|({_s;8hcnl}bY)_f( z9yF~q>$-1rrqV{)Xn(UU@EH>n6`TqGh`^Rn7S51nb+0pb%{&9Lo6R&o(Jepcc3S`R z*cM!`*->2jv$!k3a>hoh`}aX3N0Oc5u=}pJIB<;^E?r zHR@mH{R8zYcqgFcTkrz|y4{!KE@SjB{mcQ8SSM(vD)0nOuLr7tj5Sj|WDbgW54Hn4 zT?~6}u>Ynee2UpR!IhKUXUzSXe&QlstMMsG#j|eDz@LadB)N|J1&;fOyV#A+hG+>N zKiU+lZNSY)-k{=T=hluu(PCuI)kB-WmX0wgw7PGhVQ#f!K{+Z@`cQ(oRu z+qtsSGW(dOA8nbD((J2Izpv*6Bz>;DJW-3GOWR=%#5)Rpc%)@TXV)gMlmWh&Gu2qY zCll_BMi(w#Ja-m~VRmzEXJ;hVS>F+FN6}cH24^PrN^x&dMG1j6RaRw z+Y-@^SYxas)>e*^PI?asj7FJ{W^6q|>6ooF*#E_Z;tlS_uN}xj zVkxe|-W4W&A(AwB(x+J?!NVpetAjTtq6w_+3yasGXL|Nym6yXR?xl9Nb#>Io!p=S| zi~)tkX9uR2m$zyAS1cOCtW{Wi)_VxboFUju1Y6@5Y`|_Y^xBEosg+>BREp;yeCPWQ zkG83FFz4+7?R0U{PbFa%g06gDLCedHDHOxeu+ijagZt5NJG^y8D!${vtqsOS_MGsD zgpJ`ezkszXR&>NxU=K06V=5lj=(50cwV2k~NsmXQ3>SJ+?XQ_CYpU=F90B%f2 zB4CwYK;svt<%CH(F#yrE%mcyzI8rP|D@qM0rn_#+7Jl*+44Npbv|*2X zD4S!6*nA@wV-?FeVUo2UXqTAuw@f;2sujl1FJu{?GK?+GP{nbgR2C;rnij_-Y!AyA zfBTQjEQ(vRWo6ptgQxtYPvR{KHCNRZT56nXZBJSXkULfMqt-F2knU6m!N2GqJ??GwDa3 zbo@-~V_=U>GhK(NVrA3}Skzqo;_aU{)^ht0ycB?WH#zrKbGW0U_RN0s;KZpW z{>@l_nWZ&XN7#Ui%~)oQg2h>3@g6*==3Z=wa|11Woin4cR*obMQRv}K^mNBhoM~q7 z3dAjbB-U8l)r?78J8!Y@vA6k{%P#87;WJBS;=WvAaSa-d3@nPXr!J1Q@;qC~H;s7&_RJaBieaYpQSdkSiw>~La||xcbjnk!Cr7C^2|leXPn}53pjhRAAoQLPN`yVCAQ#U4@3#jGBiyS zTXbP9wor6+U{^t7M_Ws@k$W83K7Z|G)!z#=l{3w;9mVpkHQJF##O>WOru{wOnDFR% zCcncEcT!NfCRUG)=Fv7R{GyHRs99D`$^Q2B=C&2i7DAJ-xlTbu`?Vvtz*!kf53&Q> zE3;^4ZDR}@F%HAElbrBe20CTv-kHM|bc~r!aINo5)o<;Om6tbS3unUQ^RM1ZyhDIC zZl;)S@5MMb?G_eSI?3pt4JL#eh-R8wLF;%x}e5L0@WBZATpIDLkCev7!?+Hssk(F{an1lL1q4i=C$4UK{T) zBbw>XocM!B2$tmYPCoP^P;Fhv;Gu4#krEZU86Krj4Gc)0>8Em+*nG#=) zCx1+OQ9hoXz{Xj}M8N8*b)#5}dvkPfIE`KoXWIm>Z^M55RSBoDjAtCN z>n4z|Juu1-uirh1Ht{>5Q#u&XILWs6$?PRV+w62mT;$yw_V#Pm0J3hd{l zPn=|if;XKpd*C$D5SxiD-i5{E-ToDH+GB?8RG9H`GyQ##T`=E zsxmFCctZ?BLBFKiHzUom!ZIhQ?yroF=B;`}6MUXCt|HELC<#T?7C#2Gf+%yR3mS+^j+ zc8{|cq;AnA%vgI@hN#_P!K^PYc4|e7o@P`$C!_})*r}|Bn4y2)sQ@=UrWd${I-nPr zcB1jvrv>@;0%y!(I_`Jdll!QYnQi$jiiR5($FQ&{7F&t-Y)YQbvkYX+Z;!RYvF9T% zh2|NfffPOxK{<^}G(JN&kTItv(TIZTH2w5vzLZE;OAB3?t`^*FEi4}AEDqdZ)miTu z>n>{UbvpTH+h*yo}y>8+i4c@HuiyfGY{`^N+8~bjjl+BX`Tm;v5U?@Q9pC=ED>BpGt-Rp zxsIY2fh)BidCiuW&&4KYGbmp>@ERZL>cSRVmfC{@FPr;D)={{4bB2E54rVN-nWnY7 z3lThVWv|NEZtlonfs?~bGu4iDil5uAwN1Fg-2c_hY|}q~?^ua?CUnGPo>qZAT3Gy1 z$CBGdoZ9qhIb5o-+1}!q_KZ6AI+&DBrzYhkx|z2roYD_*M^t;+ILMUl%v_w(r@i-Z z&XhhM!S`|ICOpFRMHD;6J9~{ahmMYEVVYMPZ2!Lj&t@lL14t~51u*y#rwE;izn;&P zxs#o)N;6|jaH>GP%bpTA0ln-Dp`61^ zt(rD_oioL7!+g4D!ucoaT*PyQc$OsEQQNwL&(#(d-{ZvNscLsIP8Sv_(lC}LsF=|2 zy7PNK7dACT@b*=U)0?6k8hxHR!=IigEZz}xCFU$(0GSVzpo`0b{i@+&PWV)O`Q8$Fb9P%j|rsjNGp8_%dpV>@3yJ&52-UnPIO@yAV(e zY%2G-PSxjXJ6i4GiVLJZCyQzK@Y*}t)|`pQ9}0`#aT}89q;8*0YjB8wx0fPLEv0pg z@oC+|T~K#ToSL>it9GKt;}i+>xYYWCtFU*StI-HVwp!l5>1tis+P1pYr0#YnPERLs zM?J(ZCwGn4dvv!XXoXOaoiZ=ed3=uBU=B$RgSsL zxWJtTW;c)$x+&@xoeW`oZ@@j!+WLAvJk)O_;uu)w zM9dVgj@7O-P5~D;eHqHmZRuL0aDDZ(lkKSK>24*lldkJGVMOcBY^`mvg?O_LbnoM> zw!0|n?WPQIr|&E`bK^&ynm`^Q2G~`>c%K+pa>K$jHH{h0pfJe##IQ9{a=PNK6ZtN)m1S<~j#(ABG}Y;Et{PLpoJAw2t#+qJL{p1Ivl4&PP3W@O zX?5vX!FED)8mFY3TZV3LF{|MLErjc?Q=Mf<)c@TA#(&uAkhE;lJ4_0T|KfD>cxyY> zbN#yaDX@LHbKSpf1fm2-4k zYc(Rr-G1(XxCg~m%6&%tbnbxy=#(;R1n7G();7cKSY>e(GkcVxnRX~KHi+sLBAbTT z_!C2l#Yf2Tb2xskFhdO9@j_`fAu8uRv8(8Vs{ly73$;bw_ z4^wJflVQoz5I2>IKbBP7#FFKmr>RrolP8&a-a-QlY9b~vB?!@Fw+Yt7HcX;siZkL0 z=if8ZGH15ERmHI%u%o`|%(P76>2+c;gX7PmVIyB;fbF;arrq|uq`n7-}gVT=iZG!{==r| zth$MFzV$u41J>YYE}6OgnQJG1b>nxx_m!m|z4k)9(-MY8g*N~DOP~MT*pdJIPVDzb zo^fdRH&G=9pWM`vnDCFz*m)Dbe$kkPlbS8-R|bFTuEU3%@UL}wGhP_+z>j}E?KhUS ze;)08;~U36KI^}3`}b>)e)_q2(>_;XS?3u1+jVz*|IV*m`K1vZTb};ip%qVB)-Hp$ zJbKd;S5!_PA9`cytDm^z@*i5(XY*<2zQWoyL&x0n*&inhqN^@HVWDOH+~9M=x6S$G z_U3Or_Fp?^zV*aYz;+w_-|bJ||IceKzco7T(WAas8@U~OY*9v|LY>F`;^^d*<90rE z-aD6!I%3b)EUVYxpa1(CznuH$-%Q-M?e81Me=2g9W&P9O6)&$nbxZTl-ahQ*FZ_7K zpWAO=??zXHG zhLAtyM~#n+eQUx|J%2v$kyG!gUxB~rW$@l#-23>%$inaa>#Hw}{L*RTH{y}zeaXM* zsRjGbyZo$O7qsm^?bz?%_B#xl27mUVb?d|Tj@f_Q_YS?Tci!1Z<821-eR#%K#{Iqd zhmTxWJ^$cyw?2Wt5mZP!kH7Nsnx^CKpLfKM|MH8k-MHU6vIgFD!YZ zx)1MOH26t77T$Yf_Y0QwtsmCye^l=$(WvmQ%2AyY~GcmMqP>woaG8!YR;3_kp*%NGCZzC-h$U-apVFZ;#gKgP!L;k0x9 zAzh~(wE4)Z#(#U$`2T+P#VagpvB5tzU$G_e2+MoV%#SfyE^>u?^`Gb#~x%8;V>RJxG zr+M0gx4msy|1x;NiI2YVYR$Bc@4d0?u&GUx&%xhB*^hR<@bhp^b0$^)<{n)|NL!5AGt7YS=Sl7`Jn0FzxcOX55D+^ zi@P5i`tZw^#n+CH3LSUq-wOV6d(Er^PapoPU;h05(=6*$gC9I;`|49ijVSos7q4D& zKu^sg{LL|Bc?1U`9H(H=T!6NT;m7)1+WA^csA%F=Ek5gT-iL2A0c!yUc>bY)DfnK9 z&lRw9Bz`Z#`EFcV2V0A6P7%JB*!b^&FSXCdq5mzj&%1H{1)S5)FY)1)37+$vp>;Or z-xzP|?el|h-hgu_4UIPb6Yy8yoM9XBX|m5>0yS=P9>)0@_IV*VEA8`@IR6K(-GuV8 z>dTkorXk+$)>6{Qmx`8*TQhCyQOmKiZfINGS%S@Woy#YdEswX>H+MC}a5|xE!o=f` zpE`L$+43%IbYEVN*QjvA`|5bxgr-d9uszRyz38fVXIE`=NoS%9uQFUdscho$%c1GU zI`EpM&e-x*^~)1@`lEAsOD9ezPMI)q>V!$hmra`NCSVQJD{~JaL&EoBOpp$+g+Ei4 zf|ro)$Yd^@;Fg@SZi%(v?#5oW$QP}3X12K(3pTN$r8d#@p+wb{h~ss!XJ)Fzy_cyE zEh4o4#?=j0L-}&NEV9jg2{0N_v>tCnZZ&TK*5>B;vp*cdy}EeHe`$E#rf4=W8%3;ttRUACXa`KKN+r zITU>e(ZDamyU8;}6nqMhoy^VYJy<%kb2)Es>h8M z)u^GAeef-%wiz;7Jkj(ajQ4u~+6SK{PGIH^ZHB?)Jybg<_r2Qmc{eY!!M!&CU=qY;ZdiewUzBMFoM8OdJZ7mn)9XddH z);#<<>=ewKc%Y|4hMA8Sl*}u3K8IMNkyO8&bUiml>e zHW-$~{E#V$g89l}RYAGHG9R#j0ZAItTO0s$!L3s4&OhAkgsyca2R| z;pV!jn&Z(&RW&>tW_<}9VXP!!SGm|VE=H+OyaM}dn3c@y35AnZPq-kxHpb%mVoz;A zW-te23v&-%zf>Fjp3p2W@AuR0^62w)Z zo~}M}(8~Hc*iGoOs;ttq3S)xo3LD+O!b~EFvF9;j^hgtNsum&TxChIq4fYH?RGSt4YSU4u`klV^?3*t3a~J!ii%mnl$=(e63>_`4 zmpu>WXFc`GH1S}|lr*}}wD%~b?_qN)butRaw^9qxFJ=~V102TkD0YkE zZiJ~fMyR6z6K2HplfgRM*WC`rxa|5m(5^4qN8LE~(+0$GPAvFlUfIjo2(W$F95XTC z@#NF?z!w2Yy^CkVtW~%s>@53C^&yP)As%~|F!nBCnpfRH?sQzJ^%Q^lAjg?OM|$y_ zDbuY3{qn1bepBlhQ{FyP>%Q5mV_&yAGR?xN*_Gx}(`DM+I9N>>B%Qm{r{~6uq&!C!AVl?ZU-QPm@SrW^uuwV?dJ_Y?)=2Wc~J3>mzMM_E**vM!bMA#E(-aBhgB>COa9XO37pRP*YUOEZ}uB=MlQp&lkGWk}dXv>AAyKp_6+!cP~ z?aHDpJ1Zymh4(y=d@9m&p)pdK+!g6<4<+xtGSYKP={9iJ?L0KR=i&Uj!pX+UY8I`{qx7z^LZ zi*wigJ=AqY1gT=4s*^}Z1hKN*5Uh`1%9T@YPO(K$j@xlLlKeyEF@H#Wxhi>{$yn9> zNF#96&GOO(bL)Ry-Fv`>YSdBHtlk1nC0029-u%Am`ab+dx!hZwe}7f|HYbb9O8#T8 z#7YAf*01MsCT8(e+=F!pZ zycWz=&ZBtwR&*u)6e<2XE&hC=`8N~2XQtWc9kQy-Qj`5nM;(?P;bLqdvg!O3w<8ai3a5@&H zahnVmf9BUdPBL+{;rB@fnVNT@b(4vM4JsAKYVazH=Yu!S35pKOwCuod&fE!n^QEI5 z@dj+@cIjnl^pV!AVpC}byL4c%)h2akAeO@o?vB?vp1VKizkkM%v3noeIKV~CiNB;&s79QkrGT813%|U>n zOYG2QhrG*P<6*OxAK=D?ZRqyj<>TLbRs-zzgy7i05J#B;#GBR!gxKH-QGpP=7pM?* z@!FK}jUFm9NGe-5Mh0XZ=?E7$11>YrqP(NgC3p`(lx5NROBNdr+u9cm=Qp7#)(^m$ zRlE(qIkn;rUtY>=lVg_s@^o`~bEw_wHAT$5%nW8(>&@?YC}b64gkqu6v7r*Jkp3wm}U6C#J^7)~#p|UH=Lcg9{1S{-l z?4k2R=%F;rZd3u&^F#f59WSxF$V|W9bpS50FR&LelzI_Swt4mrqUs$717*7wH@z3| z3fnK+?)*@CYv(GG%~e-St4`f9YDZF?gY(>O{GiX*V1wrrSlyia&c*jQe6PbdXPxu$ z&2=B~9r*q#zS$Zs#di7ZzPZAD2(zwMm^>SXKM>_$9roEU z3mcYg-dY#C-o?J}Vn21U+g$8f7u)G#FS{7m%8GXw(x|kIw9kfFV_fVLE;h%-s0=7< zmy4b0V(VRur6AsB7rVj5Zg;Uh7kk*nwz=3pTo_E*vDLKx{H0< z#TL2PG8a4F#V&BMAGp{S7rWcV?su`*UF>ZagEwKP?^ye6n6=2ombusp7hCCKm%7-M zE_T0*G1n^7w_J>CI$@;+PQ2snvtia}Tx`CJwYgZQi=kcHX+}%7?Op3)XuUS}Qy06< z#qM*l2VLw*7kk>p-gdEfU2GWKGp2sHHPSvCW*y;TN4wZm7b|zMcQ7<4EdGX^oxb3I{|09Z9#uHmcctx7!Qp~amA}pLxxz{O-T~+8Avj^X(5==# zF3+k8t_erAaQ?&2LcSUv49Cl=WA4To$6qe%PlxXZ*UNFW;9gbLzg3lgyK(#+$E(SB zJw=B)-qy*z1;;S(DR4L7%q)N{*8`(#Rb@OccwE(Rwx-G9I=jEaB?qc%-HY8R=j;BG zx#!-;QPj9x-$?Vgn&3+HArg2i%%$U5F-y#4@(nZFO(KM&l3G z^d52$d?vtQxKiO9$>&RsB|vuk~P6{)MJMV4MYKdsp9GEa4``!&)vcpjtaBtI@p}?_lbF5g@{2Hr4PSS;j+0|D?}>64hl`bPG2O0(_*mAl9=B^@TAW)gHLcD1KD5uSxr7%MBw`YRiVb(qOGwk255q;Oo1jB6QRVSe+!Om3HW zRURJbX;JrEs5z$1FkynpbRmLxTrAl>#bcZO@_*8uT8j8r{v(Lwm=a zlKXaniSIeA<1cgTrrT34|FbSubzY(AkdAMYabk~Vm-b>lX!HUgwb^p_MnSB;$n22#G~sZ-n%ZAhtvwA>m(jsCt*js z*aR2jfF|DOU5u`iFuG0(d#Q_E>0-CK7@a8b=sGDZT_<7BxY!OCqw6FdT_^GQkhd^8 zQNoUNv143pzKhX`5^t@G(TNg9*Gcy1IthE$#df+F@0f^ppnW#Xn(ktscCq;`Mkh-4 zzU*S>y4Vj~Y>SKC?P7GI6!xDkMkh*G307xH3!NxoC%G7%C}AsIj82rWuecbUC}G#Q z7@a6#H@g^}C}DTG7@a6#&$-wOE|v$UhgYHW+GoS8(Jpq7i*3icQoLvEvz!2x6QHEA z{cjbZ^3lrSEnzgJXS7!Zs4~7h%2|cv1gM+#7oZ+7t4VrMJd+nCj9!#5dQrmGx){AEVf3QJqZcKNUX-wB zU2La|(Tfs~UX*wgpoCF?5=Jjd7`-TA6rhArfD%RlN*M1r2%{Gzj9!$m%`Qd(N*H}7 z@%mhhUX(C;QQ}d65=H?^7`-TA^rD2(cM?VcN*DzwVHBW*(RUI?0ZJHsCt(zzgwcx< zMlVVj1t?(@poCF?5=H?^7zHR{$J%ESpoCF?5|09uu$3-G0ZJGJDDmh!38UO1jJ}gF z3Q)r6I|-uzC5*n4FbYt@=sO9c040pRlQ0TU!st5*yWYjnP-bf%HI>{Rz6^W# zK5PN%Q7m}1caLXsorKYK5=Pfa*jg8(>m-b>lX!HUgwb^pMgd9~1t?*3orKYK5=H?^ z7zHR{be)9JbrMDaN*DzwVHBW*QGgOg*GU*%Ct(zzgi(MJMgd9~ohV^+orKYK5=H?^ z7zHR{be)9JbrMDaN*J9eVHBW*QGgOg0ZJI1C}9+!gwcr-M%PIgT_<4_poCF?5=H?^ z7zHR{6rhArfD$&}#V9}tqW~oy1t?(@poDF4F$z$^=tPM}0ZJI1C}9+!gwcr-Mgd9~ zohV@xpoGzh5=H?^7@a6#6rhCBi4sNuN*J9eVHBW*(TNg90ZJGJC}9+!gi(MJMgd9~ z1t?(@poCF?681h^BIik-+!Ii@qLb&IfJ)n#xA&ib+6S#XW=tT*m03~dvi_wb`MlVV{3Q)o*KnbH4C5&E_FbYt@ zC_o9L040n9lrVZx!stZ_+w5W#poGzP5|09uFnUqK=tT*m040n9lrVZx!stZ_qwge) z0+cWcP{Jrc38U{Mi~^J}`cA?qKnbH4C5&E_FbYt@C_o9L040n9lrRcV!YDuqqwge) z0+cWcP{Jrc38Mfdi~^J}`cA?qKnbJoB#Z)-F#1lyC_o9L?<9-@lrZ{E!YDuqqwge) z0+cZNPQoZa38U{Mi~^J}`cA?qKnbG&C5!@;FbYt@C_o9L040n9lrRcV!YDuqqW~q0 z0+cWcP{Jrc340&jR_+NX_w^IGC!pL{P3-L_poXByr`)W)C_vH8$_Y@I(6s8^fwTAZ z;)$F9l@p+13BL0oCqU(%fC}YaB9ZpWltEu2@nH*4k72m-b>lX!HUgwb^pMgd9~1t?*3orKYK5=H?^7zHR{be)9JbrMDaN*DzwVHBW*QGgOg z*GU*%Ct(zzgi(MJMgd9~ohV^+orKYK5=H?^7zHR{be)9JbrMDaN*J9eVHBW*QGgOg z0ZJI1C}9+!gwcr-M%PIgT_<4_poCF?5>|}b6Gj0_7zHR{6rhC7cQHCq!YDwAM*&J0 z1t?)#T#N#gFgj7gi(MJMkh)b1t?*3qJ&X^5=JLV7zHR{bfSb& zfD%S0N*DzwVRWK|J?CN+poCF?5|09uFbYt@C_o9L040n9l(6^V66FM_+$;D#v{&%$ zi&mcUjP|Mkm3sope)U&QfWj1*6QHnSF~{D1XGBhbGVgJ5w(I3y!I$%%f_hKwPM^g0 zhxrP=$FcU67sWGqQNrj&38NP!Y^{sYixNgJN<4Z|!stZ_qW~prr;E{x5=JjdJPJ_4 zC_o9L7bT2dlrRcV!YDuqqW~q00+cX%QNrj&38Mfdi~^J}`cA?qKnbH4C5&E_FbYt@ zC_o9L7bT2dlrZ{E!YDuqqW~q00+cZNPQoZa38U{Mi~^J}dQrmYMG2z-C5!@;FbYt@ zC_o9L040n9lrZ{E!YDuqqW~q00+cWcP{Jrc38U{Mi~^J}`cA?qKnbJoB#Z)-F#1ly zC_o9L?<9-@lrZ{E!YDuqqwgf_dKaVbB#Z)-c=VlwQGgOg0ZJGJC}9+!gi(MJMgd9~ z1t?(@poCF?5=H?^7zHR{6rhArfD-mTysex7l@p-MyUst*SMU|0$*0_`y(&QEo`Ax1 zmJ^^bKmO+~R_+O?oB)-31s`7GopYgrx=gi(MJ zMkh)b1t?*3qJ&X^5=H?^7zHR{6rhArfD%RlN*DzwVHBW*y$_ctCqU%{s1Hqm8j4n) z@{IPH0A-$k$~^&*8+&*VA@qw6G$u9L8}E=Jc$7+ojv z=sF3b>m-ZTCFinB-KpHP+AAjJ2&k9`}uOO;LE*&FZTpg{r}b{pniu1uUsdd$#oJ& z*GU*%Ct+({jINU~x=!NJbrMF`Nf-qvVHBW*(RC6=*GU)!C}9+!gwb^pM%PIg1t?(@ zpoCF?5=H?^7+oh}be)7zfD%RlN*DzwVRWK|(RC6=*GU)!C}9+!gwb^pM%PIg1t?*3 zqJ&X^5=H?^7zHR{bfSb&fD%S0N*G-yVRW5@QGgOg0ZJGJC}9WMXA+=uL<98~K)$S5^oNe_#_Jl4S_zv^cCUs{ZHHVv6e&NPCEiKlO z_FujcbViyx5{{k~>u8I%w_#CkOcjIkCzIJD&F@y6UuVMoz%^YVJNREB(wjJA#tTOg z37Z0{M!1OKh4WTfce-J#JYf+aEjhWr0hRk3P;4N-2~Dwnfc7`5n2$1@ZQ9I;y!~v| z*0P$S@mAB!*e_2vmpA7GC~VK;XjNRi10|9Zpc)%H`Re}vV^2VB$J$q36wl;E38NP! zj9!$mwJt_4N*KK;@#sYfqZcKN0+g_wE=Dg(7`-U*C_o9L040oGlrVZx!YDuqqW~q0 z0+cWcP{Qa%38NP!i~^J}3Q)r6I|=J^F?vzL=tYS~0ZJGJC}H%Xgwcxo)LwA0a!)|zo`A|d0hN0K z>VNSQP)}jOE7yrgi(MJMgd9~1t?(@poCF? z5=H?^7zHR{6rhB?50@w>K;;Ce4^4n7LMu;sMtfC&$~^&#%jhaNU^NWU)6QFDXsmcgSl+Pt7 zQF1Rfs+@jPN-r*^^sV1H0Se2j`JwcUtN+IYsHd^^l^4Y`c~QdXMG2!9C2XyW(TfsB zFG@UmQNrj&38MfdY^RISixNgJN<0cs!YDuqqZcKNUX(BjP{Jrc38Mfdi~^J}dQrmY zMG2z-C5!@;F#1ly`do}&lrVZx;!%JSMgd9~y(nSyqJ+_R5=H?^7zHR{6rhCBcM?Vc zN*H}7VHBW*(TfsBFG?5%C}9+!gi(MJMgd9~1t?(@poGzP5=H?^7zHR{6rhArfD(44 zi_v!yMgdB^w_J<@lrZ{E;!%JSM&C&o1t?+korF<<5=P%i7zHR{^qquJfD%UENf-qv zVf3AZQGgOg0ZJGJC}9+!gi(MJMgd9~1t?(@poCF?5=H?^7zHR{6rhArfD-mTysex7 zl@p*ocmZk@ntaO5+N%Oo?g=PNXE^~1^JDG_sJ-D%<(`1bJpq+_0xJE6K{32fdi=aG zFh68&4CGn!v5T3GM3WhP_20Qz1no6MT;c@Ap{n%5~zITqj|4orKYK61LXG=sF3b z>m(jsCt-A*gi(MJMgd9~T_<78u`miy!YDuqqw6G$u9GkdP{Jrc38Mfdi~^J}x=zCA zItilyC5!@;FbYt@=tK#l>m-b>lQ0TU!YDuqqw6G$u9GkdP{Qa$38Mfdi~^J}3Q)r6 zLvtbqmC}DJ>#G?Quj82p=3Q)r6Lz;E?BxxpE8qC#CVW*Tzc?zKe5*3KtE#7K^Paw* z%B|tviaofx8CUUpeK)?tz2QTzT(|Sk{JUt(DXNbaquFd1*JUAl#mC{ymYNOHGvFH{OaQYqmy`yCXd{FGY&B`~){7t;*@o zRTfphR@pmaNaggu;`E)$$-kfc`19FfW$-d;>IC(QNeFtnXP&oNW zW$!s5I~@MF>(2Uttk?GrD^e#JtJ(Vw! z8wT0VToRYJY>P~P9IVIl@3QGV>sbIjOCFB&oVu%W`Wr>P^Y>%Ht=|MVoV*ud`ihRa zrL+vW{YTNLpKK$$64|@AXv_V)*i-pt80jk7awmYQD^Sn}C-25jUN=8ci}&HTL#eLc z4xl>!o6K_2mb6J&(Bd-Y^XvoHe87&dj*1{&4-g)^Y20$ z-gp|#zGulsl*dKa;ZpC(d-5MC+VW^s{VQ|xU#m{WHt*Sb=dOMB^r7@_x^u@q_}X4{ zXob~Sbm)1YGlvo6y0>x@Gq)i8^F26NGUzP#R{^R(BrJt#$%j}DWh|g^5te|HeWqyc zx|h4F*4npE$2^sY_ZIY2)jyWs zS9L$qKyr0`Uv+-v=49o@n%)tYMNmr%_H3>0sV=R`zmHAMq>h~&={1R)oBuE~Fs&;8 zL6bU@=<3pONHo*yq!Odcn=oJ1yJSOEy{Unk5-~P9l` zzI_K9Y4Xxd26?0s15|j=1C_};advOGw`va>P$YTkOTDxAM5dp*yQgLohK;);J?pPv z0JYYhpD@SMcNg_;!1$l6e3(6QEv7Uj42`y@vb!?5O-=U-jninWMHeyLAkA}!(;~ao zBH8@Xnj!2*vbn0ha?{-U2j}MBQen( zH}l@RY)f|2vP0ujwnYz(nWU3eA zN+kKs%`g$hpkU`ZlEkdG<^T26Yz*&txaiObTftUPl6OvjjN=cg^D(pyrw)JLQ@J6W zd}v@DeuF7Xc5R;BJEFvlHcwUMpSo?}P}8_E*}H+<-pUOebWcGsnPN4g3aXXatKgX8 z)aY`KE)DMJl2)B}msFj5Uutyu10^yaKXUh=JI!uV;)PyO^nVU!_i=YGr*YA z;uvEF7cK_WukETpYEd2Al(_r{%@BhW_Zwo^!JKraDz<2gGrZVUYzD6Xpn6RW?zzuS zUuFLPOV(_(Ya2EE{D~NG*1Z`jy6C8P_UuXSe*Q>&ncuJco4?TqpMM=^>4VY_-3pE* z9|IwIKbpi7?4X=vuD~*fGs{EaJs47#Y%pE+ViYy!5>)pnO)gdWT?4CoN712}Pf*LK zZuDM^-l%TVd+qK}F`)OR#%Mc-?7ER8#_7~-5;j$B=9I2Yso7+QJ&m-ehTX24-)Ew> ztD2Rp&S6(I7D<|J)$5SbKvnHbC8?@Tnvnxt*sf{x)zkLvvU-~!b29&ni&AC^y87hzysfMroNN*}Wn z1~iVe$hsNK%rIl#*)S`RnOV{U`0)Sv#2mz}hKab?nStHQmqe z?^!@y#DdYf$qXTCtjUKyu(m4S2StTh2sIqOUZNZp}!lBhEEHh8eSef?F*+q&pPkIry0Gpz$(S%v&`8u zxJ5ywwZJ&g)K?35^U8YHoIQ-QJ?3m3mR;hV0cTshLU=^teF10Uolxjp{S(e~wS1^^ z^>ExPQ`m=brmN$IJ6DTGIC1O)$56Z<;7q*2kq+-(oGC3IEwZeYCVfb#^$I?VKVn&* zvE~%x)IB~9)*<+{556l7bl~pBx)rgOmh$q3SbL&rb-XhcoxEmpw7s?=Ix#w9A|7?Z zzw~!X%)`c%aPep-yrtI}a2L>#1}iY{d}t1ZtPmoyto?DnPCJ4^7B{_wi}!cK9%dbc z-v{9LVFr7B;8SuG4Fl20ya}&^-hW=U^+f+Z1JVYpEke~BU4LuZ-^W4+TE~a3%lm8o-TAleyJjC}Pt`b@GKV&7-Gng3Zt-*`I^qqn=xS3c zOv)W-mX`I#@Uhl`LzfM`Fpp2lFcbTud9?alF6ha3D9 zo9xPr{cz%|aV*|o!gH8;+VCsf@EbhgM6WliD6xWwu8;qfVtUu&)_q z*u}qRexEh`80%*1ze8h(zCP?LCKqSm+IW00MVrPF$3VNl#P{nFhgoZ^8>}ny7UlQl ze{$cO@}3)dRUZ55cCb_NYFnlg?-CQwFy$I^n$_=bJs)8=h4E(0n=^E%bwJ+ayaz&~ z3(%Q{eBHWgNLk*F!c$F3*bLSgYr6{*R;~5xP=Pf%Z?V;s_h{(T1>Y`U37(A9UTSQ$ z4=u5(hw@D%TM_1~CfsEhrk%~BP0?7Z>Wv+X!#wlg40g908!LvavcmZf=I;u9ZpctG z2CszuNhTN97S6DyS*xvktj~rv6<9;AF(vVivEszBx?dbun>dD8%rmE`k6}i&tck-X zS>Fx)J#=;6j|y%JT{PsX(1gOgQ2WsT#^B0MAF{4B;l8xr7^`wWPG|cAyU3((>Cn;E z_x8DSpL6q_;pf{lX`MZCvh|2Mzxmx7t&6R|i&q$$TV-!B=p~^M*yJ=0b2jU$6*<#{ zT9j6ottJOd3CC>8SXc>!hFKF#m^L@eevr>*79TT;*Jrhxi|pYwCYi;iSgY~-1YWoP zF!J%dQ-&Njv@q|>BYI84uXd~LM$CrS?aOMr70lU&|MW;Rj+yf&J1fqJ*a)lW`B)>p zU=@bGWL*&I9KvjJ#`?7haV0XDu)c=>odVr$n)9O=Ilp0MJXWex$gS2ZBiN+hMA%fU ztJ4aQ&4n5o7DD^(YtR__3qh;$;Zb3LE*|JhSBN1=F5`oEULN-5R zvbxNKX-X@ok6;z@5w|zB+QXQMwbc#$XA@xrd2_l%+K?A*G86qFh`|5)YIFVI{@0y| z+8U#Zhh|e+>OI z^ymD#(DzWVY)mYN^G(&x%rCV%tT(J1Pyxh4);6=6`xtn$iZ_AhjFtCddMg~ZCT|Sz z&1rZ+=$oN)^DoBuIVR6pP-3^cNe36bFPg@A((rw(eXKU??9d^3H3e&iY$;$mn9egz zbJ;KNNQ~OQ4;_u3$fo!I*?SZ4s;Ybaf9-Qlrjr99TnDTZXjH1UqO_J4l|++z(4f?* zv<^W54JwEw7M#k#5G5W$Q7;w8My~_b0qWJNtwTUXM8%-CqV?LSw-t(3aHwci{_nfq zeZD!FtakX_=lMVTXjprH_S*B>YpuQZde@uFqwmG2*w^AdUlp4a-w-quEDt6ZT^f{@ zyx@i>mjs2W^TP8DT&v&CZSjNfXxti%a^H&0ah352SblIh%bVPJyx`7`-5H-29948f z@NUWcm_N_F<=Y0A28SgV1>`K3a_h~FXK{|t#kO^0nDqSb2e?SV)jqTx9+k1*^^W8(mqzHBi}A>m7{P-@!5M9#!>}E zseYwpNX2Pi2dq14@@e%Nca7QHQ2a5w=wrKE^}$Pe=b6v?;K;XnIi+~x_Ni1M7v&%N zF}#|wi`JK(Q#JG6xyfZu6U+l(_g@rt|U6VM)Q;sRzA>L5Bd+9EPJEabvJkb4+ zU(?+Qh{?Yb@t4b9Pzlbjoievxim3eNj+$Dmk9sekghhyuOCe_$+!#zNY7B-Y?+!Xg z2s7OWD`J2-Hs$)UC>BCe(VXBN*y>73(tA6xJU-GK+LH*QcT(j^(*z4N{pXbGh zmb>b|azBloU$7`Rx#;rXnB;uaGWx!3xA~e*&&$SJgS;8P{rqL)ZBec0YXNITy{6OH zCAXyFi9Rk{@g)toY!}6Fb#PU1Y4^;#v`nnLY?u7TF5CET7VZ?EQo0Y4>JPq5Qth4w zKXEr(8vGI$@V@w|fA<3NBF!k~Me2D0neHC*8SqY)?xy&-*I;zYefKkM{9C?lrDD zyu>{p?Rkf%2K1Vj78a#q^M)`l^||)fyC=_heBN1w#VA{tM0eqnJ^s5$qGy$U4N3IG z318y&^mQSaRrYa9xC%4i+%x_zs*XR7U+2r;&U&5R%MJ?f#}&Nn?$Z8MuGNi+6}gLI zZv^irirkcfhM-^3qrrQqQm0C83@$1yaSKX|+})+`2mALga{Gt(P1$Zm>7MTH;Emv% zSY2XoH?&}wJEfp3m|6I<*w!W6yZ1}>cjJ-=yAPAQ`|NrF(!ue5zsvh~CKEph2Yst{)_whA*LPz=SUFp7oRzK!ZyP$~VL%g2(6RI}Mr}F<6 zZo>xAfdg1Y==mF5iH12>U(Wg?D@MCw?F#&{9`rr@vGT#Kv0(WD{*K4r$@n`BfA#pA zfgQpO#wv#oS#5!(uj zHgXY)p(BG_>lW!|P_|H!%CdqwKHRXdt7-R6sBF^7>x8-uA`bx4Np`mIa3ieWis!w? zAm=JC=L<-WC3wN%vO&Er;~UmV#`ArB9L>Qp*Gz#G*2#-bhCxpjwIl^%W>+pO}L3l61 zsR$23iNg5_2pNBRIDaw1M$G>VVH3h95zaxlF`WMn-X6JpG(xU772!1qXCb^E;X;Hz zN4PbzoO=*XMtCnmv=aFF{SdCg{J{vHM##5Lo;O=;UPs7pE^i`h}EjQ!2n7GqR~>O6K1r;`55hS(9t zPO^BX7`xaQHKIC?dyPF{Y`w9!jq%k?`(Qb&b%z++&DdCD#~7PstkKxdjs2IgCyhO0 zY?HAMjO~nPNyoBlIF&@RSdM+q*ty0oG}dD5GGq4`yWd!cvHvypzOl{52IINau~dXp zN%sw7BaEG5>{Mgt8T*N`OO3I`RoD5~#vV4d%GhdS%u94Ee>V14V;>vio0H=0A5JBi z_k`>OV;N(}S;Awv$=G6J_ZxfA7&~!mAMC`f^H^u>En}zvhVl5-S@8}qR%xu(*jQr? z#%3CuYiyn|wmj)rZZfvQ*eYYM8hhQ?XU1Z9UDELl45v!nHpccbHr&`KV}}?!-dLTn zvy4qQcA2q-#_lxsOJk24d(zlT#{S1x5y}N!15~fW^Vr$guEvHN`? zG-Kx*n`P`_W6O=LHujvcca8nk7@8}?eH4aMNw<%&;l{2sc8#%zjV(9!vawffE*or;=_LW4jw;8LM~)89T+;sm9nyuXt?J%F_uJG zt?l&>r*M46b~m<7zg)bpgi}ekBg%2DJ0zS+x@u!-V`mwgZmiMRCC08WcBQfVjQz&g z^Tu8@_J*-Hja`BIgpTjZa4PA3ZR}xVe=+v1u_WFF!s`-FCEZty?PP4Fv8Rpw&Da)W z`=A|J+Z!HECEel1jx=_>u{vW<8*4X)55nR0%EGCn`;M`Lj2&g{IAf<6JJs0b#(rk( zE@O8ad)U}=V`tz6Q0I15IF)qQ8@tiiDr2jSCAZ40+drI2x`T~<*VxI%CK)@|*oDTL zjkOrN)7USKEi?AGvA2zFGWMx4v|sqMOuDi7l%exwPa;kwUA?h0jiH}g524Zh5E`-O z2OUOpODn3B=;O~X&{Q3vOa4b!L1FZv7njs#Pad$P4L!J)HrK2RwMP7`NUhNVgwPCq zzSfAZFSXf?>g^Ae1*9EMUT`RSvSMphknNx^?I%>@WH&-!GysuDt}d(HR1JAi)yQY7 z%W6MDzgx&~HV=Oey|%eW^mF8)u74c=WUu2I6dX~j6zYveuEe&uiJt`FCbFy1r?GFX z5jVFEe^3D&yG~l8L7~=Y5va6AJ9N_;b<-K)2y@3L$x(C7Csoi1gj%DV&gf+b2A-$R zXf2fk&0b{Dd~ACaHAZa4K##el*kzu^s1hegjZp`1b4?;&V^mz_MMcZm^^h8)pFm85{XtM+)P^Z+ zdw>aJvNfA2X*dDZ2#Sm@=JKrX0-5VGi&=-G?o=4@;(Q4G3Zr%` zQde2Z7ea**6Bv6XS3CmD3_2#BlEXgk9 zU6<0Rx_J_$Mog%o)adt^)S=b1oKQin(eNibrAEGUCVGYU&>205EqY18U#sFzqrYtJ z1jzTsEFBYn4Oe?3Cfm|`twzVyja$&Yc+A?I{OEqkkI+dxgAKUDS`zz)f*xEV3Im{( z7|SK;mYUXXy~aBg0=#_ao0;pc%8vhGuiC{}b#&unc`AYXgLAG5Z*pz+oHoQopYciP z@Q(hpUf$I(4f-SWs*Sh##4%>6R~{vx3cwN7qC@Q#cK?JtAnaJ{S3;*+^cY{v9?k4x z>kpQFY%y2KyE#H<6)IG^^{+(+RLnFZB~mL6xz;O@u%9L-pPUj&35y%cKFeznDnF`C zg~Z#4j?=vRTb40>)Wly89^wWmNm`%q9cg_=QZ@2<_4>}X=$WGu$sa_g5-C3&qvJO; zS3VVc6_O4sOvzZdG}q-DXRp+QGlvQ(FCDX2|1f}_DkNoXdir_PRS)7kkk{SJBd^K6 zfRx(~l%DYZlnNd33i1vTbuoBXR__+Otu}pCfkjpTN3?iva5KTyCv4l=O@-K5+K|% z9yF7zlJ4iE*V-(lvHr?6J>krCyZOveo=FE;tfwYQC2~K%fM1 z+hS^yBzmA2iMJ;!DN88jL(=Bibk`^?z~u=wNTiQROh8#iRa8=dwtmYi$}8PcNcb=(1OewZ~?cPb*! z#^VLO*_zsxgoFmjnqvfK$dkvjQzW>7vmg5JeHCsJ>Q z4`uFfv%Yr2+l~JItvE1bM`2b}l_y>5^QQX)Mk4A}nda{-wM#w;)Wlztc8M7^@BRAi za^j_T|3>Lbhl;HAA7IXFm{xn`k{2<(;3qrtv!^UARA*mmuE`945((#}E=dBq5mcxo zV7eQ-M3NYUXTGz*LKptUD4;r&GFW2u@#uS@+&}#McNSK84?pCUbA4gObYQYJrjIaH z;Gj0L(5lY9S-sb54O6-oRQ}#WhS*tFQI0kiLJ*%ykVHbIMhgT^xIvX)Bvs@})y`BA z=0ACu|ISBDE2vge zV7maBJ&N@9s+Oa+G%sYrs%iS)vZkM6BFv`zlbk0T`R7ZzY!lP9FYl_eUUrN^nstF; zD%O~#=1BwnQEe!!YWyUa)i0O-snk>PUE}jU3`Z~~gEYLnYRgK!q@cQwcL)@At=Jt( zNIt}6waYNfKv~N>dC7kj1y8(W`MF}NDsEezM2T0Gg=os_oqpSVtQR}++f5a;eE-%(wO*V-FP zLtcfPAZmJ;I&-?H+6nDm4t4BOWr}%^#|}SZLcqI0&4SJc0e6b^H?K_1)!!iD?OJoo z<&?R7=Q2zRq?q9H9*tZ*@gFHA{Y9ukq7M^Ae0kQV99*W%`VF1AA4FkGt42QcuSw%7 zO}o99AWKKKh#W)7*}s^mLYGrTA=%^3`@5L`W|HL0h6Ilgxfcb^Ls3i(hDvGxR8sBX zJD-2^!dqEc)79@vB~^)AIvTa$*^iCGR1cBVOk8A%r1rpKiKN!z^9DaPB%nyZVl?U} z+>Q7b{k$=?9{-}RJEmGuFI2o6(QvJJhv1D)@vgv>;x!iMmhOi}Of8*1 z8clgxdLyQ^^a?cFX&-m=%f&kZ@3@Nh0Hzf0M`gKqzr|Enkg&GWD;jRlzm&h5%kFV?uA`_GA8)IDf7Z@*y9=jA_)Z)Af;+;m z`?IpnxShhOZ}?@`m8pY=pTR%h=&4q!_lJ&FJJ*bRg3r&;e=YMGFtnbb%L&<8zFTWO zc6pCKy=4C#g6-S^!SdJ+i8te4DSo+6fm9pf_aczWbs@PANz%FJN_KNsl(5S=yQn;b z1cooGiAMMR;NxH~_g}FS1NI_jFDD77T2lA8!-B=Jt&19xCk5xF`r#BLEgHsE{ZA^V zl~{+JdK$psz}vM~r8?ao;t%2cez{HPqH-JBL_0RY3(T$;GV$$QD7CiBuDW!|_&ScB z7x+fMtJ_msyNlheZogP8KGn^PKNozjpddD}s4n(Q$y|3@a$j_{>|H3;5gy9l6H2W^ zJpFD6UPSMQKDssKb~7#)J3Ql!UCyLSw~(AH;1Yi(|DD?2{U-KC?0*WMDS6S3dVIId zZN}z$7fN}_g5Oe8zaIu!SDu*T-YB@vrHU_g+n3I@yIS9Eiw^5>i|mF%@t3-x|;%Rx|79{XBJlY6-2j97K*DF4hIhBJ8%StR=5 z*s(FA)lbQVeOvkvN}cX^b91W9JrH{lyBS!pJ}4^c9~)9K&n?4lMx=gVySdZvrf;DX zyUOJHW(BUZ8x0?KW-G7zV;X6PIX67WT&f<7^?MgeQ|K zmF*qBIJUXqk@)J8jeQEFs{Q(51b9bVOw zthwJjT+Tb(uVO!rZ;Jn<=*8INEGn?|jE&=Yysc-IpkenNJ!=z)y(IJ%P@W zH+s9vNK!NAYm?XiyfrJ+?OSmtaHTuT1o$VroSH zZCsJp*0N`I3(8Cu*}OhHuG{@FR}~I-e@6a(u?c{gD{X4}KOqF!56SYsDK2c#SCgzS}3MZ^yFkE;l+j z(`_Bw&euvhu+>~y+J^w@R_@t-(@r+GE2XmBJ?|=jI>Mjg(S98Yesb$Tci#W+3ZORl z9ai=saFPIuec?V&05!Dh@?Hf{JNtwDy8@^keYCCus9X*JH6VU-<3GWTC<&ekrs4T# zUdLM|Ipms>bCA;&$F>B!C(aLo!ctdNI4{m;hfkxP&aJ!-dA#Bb&i6_cE8O{ z@$aHKv?@ruak1N7QT$K1Lit_joBRFF&5Ny!w*+?;JrXn|AMYlTYVNm9a7VBv`3Ux@ zaw&Ih7RMjK`G4xqzo$@Yw^U-gRKj;h??ou}AiE&<;PK4ez4GMjD|_XM+0c~tvYn#Y z)5&}EkWb7$DV6ZzmwdI6gi`yU*Zb!SrS`&oVs_H&E)%mcSZFVYl9-+9ooBZ6%Hx?2 z$a@@mym5zAzg|1{|2?5pLF0i?UL=ZBgG+FVr=0@j#d)QBdZE;z)4oL4{+v*18(#vf z=}TeJRRPf4?}maE!OEhWgZq*%byrs~*FFOm^xfa(oO-@p?!KC<4s@aGPe1;aL>!gQsOT@m#(k`_>hl#ovTJk z&zzvUmJCX&&A31%g{iHPzEbtaK&UglbZ-cCe)x~;s5%RQQwxGv3v^UdP{U(k@FlMX z6BZLyr$EL_B3aDwN&g!fupkhDj%t8gocFZX`dZZweY?;i#K;vfyf4f^n0fT}~wggirURlaWNvFWm!^XU1@kj+_2ZrqWz?`Z@PyDNZ+c0YAV%6IaIDt(@+B^XU{+ z@y4|C8o83Pl>1L8Z%f^GWAWGv=uf+#pf+$tZw3QPirii$ZNU?GkNhnJRevkpA$CiD z44dF@Y~aE|p>%K89>imZ1`Fcfa;xHxW9;$@cXq-4?iU5S1kV>f82eet&hF~uNS7=< z)ZJRTtIytBvHz3&{?9M#OgCMGRUiB6-Kh{(ZS8)AmnhzM3ad`@)hLceBw^JMHg~!; zxYvD`5^wuBN8sW5j;{)}VvXG8VpV5G!N%b7qGW7Ma!s!#M#8T@%+k{aKi`Tnr^-LD zUsPB%4yWDo_pcIG@uIRqniEznws{GwSjp9L39DGfYF@&sHkj~jLKIf9twy}Nu!=87J(h&Rs-e9uldx(A_R?Kgr6lSX zpiM}^szWhbjlJ`iFRa=P^At|)9db5QBR!x2(o#ix-1s|yhR z5#d6FkO5@wMo59xy$C6=YC}kY)e{IQu;Pm|g;;FIpwQ|Sgzq3^YZ3)kahx*+Rs#@H zV6{C$3at17NI{jaVM7SCPdM-2UNQfD%u`^+2J8gF=?L+DpE)m_r&B-y=J`TTVbu)? zDKO$Sqp<3DgcMdyKuBQ~2ehM*>RYJ%QCRghDt(+Eh(-zut9C(1Vbx6tDR{!Zy|4;{ zt9W76l_;|*r1}#=3ab{NSmXSC2pON9(mDS+LJFtYA-xIV5EMOg5ROO4`8!eUarsJw zT<6$HGi?QpB-C>Dy_TB*fYk~7^9R* z@jf+Hf_o-L`5mW{j#4VIuNwQNv2Pn=R;qX>7~@;E*fe7o7|R%IHAZ=r_WP7E%B#fw zVC)TJ6jf;-6jdo6rBq_OhEqvLDV5k!#weu{JJlGaRAP9q^;1c-X6M*L#wevyyiLX^ zr4rjNoWk`pMhiYJO*%@c6mOg{N~y#yF-9qs*saF+O;K!(F-ob#=pMnTq@$EdY>#j% z=_sWVJJuMbRARG@QA#DY&={puVh97^PHV zZ1z+fw2$4xsiaFAt2IU`mE!S5 zOZz}x60*6*ZZfvm81jlR9_3Y9x82wa#y&BIR?jeADx4~H1B_J|8)~fDSlZY)W9+l6 z{ni^h)7S;ZGRAH+c8jrJ8++K;%f=|V(mwc3#i>&Ft}#?xLsk|}CEd=(b~QHA7$sNQ z-uI1BawT?}u``S{8*4Fkv$5NZ@f}acx6IfEWA7M?qufwDO0KkD3a!K_v=ZCL7$sL? zql_J5tj-uESBiJFvFnZf!q{ELSf**e6k2KBSBr9XJp&T?e&e$|#4aPFY<`{d&*n7slj>a7A zW4~}J=}t6uvaxfFoolSgShKNvjXhv&jj`Vvd(GG%ja`ASSUQ#~!>OcOVT>{?E={_> z8vEGT{#)hN{Z2TQbU!wBp0S0-t};fUmG(=amG<$FF-oq)UNiPbV;>m%$k-6{G1B&S z3#XE9Ut`}iHV2_4YBUw}?feu+Lwmx}`APBma(+_0zMP-5 zv@ho;Ev565_R*K~lj1#&h9|}Q_c}ifi=3a3IwR*NTm=7(#}P&oE_wmxdi`}3 zKCR;7$oVO9esbr-^(k_GikzRGNkz_2k@FKejW0#Pz@BrF^Hb#fb`RQcz)QX&+(2|8wVIt=z z2z~ZIUKu$*Mb1yXIX_*6>bL|+oRT0(>};eOu?xegQa9TerAdl+nK8B&irs09&QD^L zD`|UwG`8N@2gWE@Qan08XujlNjYnVuu-{TuF@1Pl`vmlGw$@C|44r^ONFH zt|Uh1Co#&E#OVAa#+NQJIzNd~t|Uh1Co#&E#OVAaM!Aw0ou9-gR}!Q1lNjYnVsw5I zqg+Xh&QD^yhEsBW5~Ey6@#y>{M!Aw$i!sWT#OVB_c$6!Nbr_>uNsP`kIqkGlp%@H`AO_7V|0EJqw|yE(fLX2PGfX_5~K5z;&IqE zF*-kq(fLV?G9)oNKZ#M0Bt}7!7-dLebbb<}^OG2zpTsCb5~K5z7@eQQW*MXNlNg>^_q8>91+)}`~4;!%7gM)8pt#YbWk zABj8l&@**qg@a{3J%_C#_58Co#&Q#pwJbM&~CnIzNf^52xh(Bu3{a z#an6d==>x`=O@LZ^OG2zpTy|=Bu3{aF*-kq(fLV?&QD@=eiEbelNgzsVvK?$F$$6tk1`}NIzNfM zZH&%OVsw5|JdCUqGCDtr(fLX7==>x`=O?k=ghr9`(<(SGt=a3Pz1E~#yyw+A)T_}= zJaT@DoS({Cevg-v&?F~veu6Dn=O@Lx z5e?S>_`j?3lj8N|{G_FQIX`JBou9OizMP*FkIqku_wRLn8n$=j{1iDq9f^1Q5lHg- zMb1w$RyuFekIt%_b{4yZP~CzCUKcq(<$R`Q=28npV6^SwNWGEslfNkGx+9N0#t(tb zBvb8KS@ zH*f&^|M&bw&QEk_IvA#?l1NqJ649+`$~k8?OrCb;M&~CnIzNfg`ALkx`DU}$VpTsDo68p#)rBq^ceo{P2 zsl@2~Bt|Kf7@eQQD5Vmk^OG2*RAO{~68m3cbbb<}luGgF{3J#xl^C6$#3-c_qw|v( zrBq_48>6U7jG`*Vqo_)ZqAD>uKZ#LZCAPvCx`=O-~bKZ()# zNsP`LMt&kKZ#LtB{s^~A;#*AQF5htbbivhbbb<}^OG2zpTsD% z5~I*cjLuJDbbb<}^OG2zpTy|=Bu3{av8#>I`ALkx`=O-~bKZ()#NsP`x`=O-~bKZ()#NsP`3aeGj@ z`2yHh9G9N`)&f*RvwyB?soMFf>g=Y**LO}ov1|)|@#R(7j~Y8Vs!hb1DWQXr5Ow7qBY3x_M4T22*J)pigcZpG0yXognkG3p|=|_6X5d% zmkq-7{(j*VJ}!q4)^C4TZpUC2=egcTLvyjPA@)9M!+ekEGcX2PO*#9v4eJ)2&sXQ* z-d!xw+0%;$e7s*}RO}HnY8@YJ=laLCvCWO>wmIf9Ut~;iKI)yj?P9Oa-J+0iv;3xu zV%xi`+^)f)?svhSw%fE{#(#9u9hl;7X~~Wu$}QelM)5zzTZd`xwh|#n?9OYj%Uz&x zK05!=DjPA?d8Az|+;gjpXKeq|Dw{FQqZy1h73{^1rap|7+cXwpl%<2pS!)PfzODN< zxCVz~>8XB=dxCm5$rS_>ZA;U-ZHYO0w=QS7gPYkcDl_VV*r}3Ur#*Zwex_S|W~Dqx z#m~2m&#f`H+ZuzhhQDGseU}>+y{=)>lxaSlkH%uUwmVlB+sXY9X6MeeqLi%c_IE8F zEH*(e@Ns+mQ(2C!m6x|-0T1oS;0Qmz*yj1PmitjNwVY36dHMae{BRs(xqn==+@37r zl@=|;Gkl9~yWuWR3OBqKi;8h0j|{4O)DDZ2=hz>Oi4!u`6;O!OL)t*;2N0JF@&;%yr&*=a?671VfG5U4TFSwkmNMW{q1TJR0FP2$>#7BfJ9PVF_TG;ja_BzW@EP*TW0KWV{454*4W#|SaRuDJ~bBL48YHT}PGj6%m?HEoaU6rvKV-3b;8oSll?Z#MIYkN!u zI_6J}1=y1q8~wul8ml&zHr8NlrmzDJjP*9|L1eIM zgB~ygWIlz8>#-}Vv&GfTV~VrsME23_b$sfts`~Jw<}s=2?Aq$=@~W1Dx5SrS_2EYl zBHdqIwq$M1$meUyYG1C3KUzJq9V00$4-NL3Yu;kqDn!Qmkm#(;V)2sp>XFYO)^qXY zVf5xE?!Nihr>mP!Tvs#l!?Knm(WC&gzE?BiIct?aA!R18Ck#>>jR%$A#L-p7)% zB~Nf+bIs-|99P+rWdP|#>E@b2*+((U<$iWRRdxmDb5Uw1tOclzU&m7}Tk>da^NDR? z^x6p>weg!-Y}D-aS#9&A_S)>d9hj@iK2e>0Xep+!b+;kGbI;bS!)sMj0_}Ntt7^K4 z3$sm|5!4KSG9A|UL6}c+HiIzR#F8V8YWGZfG@ei~&8#c4O>J0&m!t-~GJT%=5?_#p zR%f3;njQ?Rs_?!o%wX*eMcGx29|yAv9>BH2C2U!YpawS~yR5poW)ZGOb`#EYQ*+Y- zEXXe8g|EuCRgG+q3|8CHcpDvu3*u^JikGcfKwrFS+@-NwAoj1VNXJ(&$vwdLaO%8U zYFhbrSUEcWv^NY($5&P31s-p&Pw*BG#oft1Rn7b4Z&Etj!qky&Id*P(g1-Y}@Ol}q zX+v7iC5uW#Qi7?i7{OG=4GpWxK8x)R<<7LJY-nidZ}667)!CXwRq;*1L%gl)D?Zz@ zC42^}Mj~al0A^cS@h=W2Orm|HPYi2-J^B~>nl?^M)1g@^MTE#;H8NObT8s=RtMOju@AXCs5v$Y3=xSVd($JLc)gU^OyWZK;{->r0Wr>ffp2>)i*{ zuc6f}X8(^2R@?F9_HD3QfVO%WtSVk#2CItKm%*x*_GPfDr8HR8KKe3PRXiH3D&D`- zU={YNU*w|#JA3J)Lc%xjpNWPb`N92%Z!TuLThEUSL%b4jD=ZtriiP`Y{|DSe_e9Z8 zd~5$AY_szNg+4m4&0fDzXpO5Xr}N$@51#Me8MpJJgFaBa)Sq!XKRO)H^D9EV-yFZd z+{;!zGz)VTw$&G*StT-99l?v>fBFv47Tz2-3kR;sTJI~vSh?>2D-o~r6GNBg;;-WZ z^Mc=o)->lfmu~GYbl1CWW3R@hxgW$I2zD&^UGTTU17fu$=ebu(iekAo`E<8kw1{8M*G$*|b0(i8l% zco;6&Jbp2CyIE?>_zTK^bWw|O1A6-a#Alh=%*#H0rL8~L&+{Lx&*2UF`XHnXB;(gu z?7r&uvaPTgLcaixOb3?_^2vZ?$Fi|8d>D?O5nEf3jZZI`U(l%y?>c@!jo(0Iu&R%7 z&ehjVnmW<)Jq zCTY%b>~`mFaPyI{c0!iG?heoTE$r97z}@2>La*Bu?l-O}Sde0t!S9e$(BSRdNWblY znBNuEGv}7Y#=9lK@Ys277r&jr@tYOC-u1f+)x>WH{oT32d9lypkH*&&%`afzg;%gN zR}ez*Uq_|CzE4^zznNQb`ygM@u5+9GimU0@nQt^qo>(`-ufxg^zoa|&eZLM3MRV&! z2CMAg5x90=o7*0@@8I%QT&uA@Z+<5AxXZ+flQUyq0biWQ6Tx-C?D*yJU5mbAJzkuf z;gbt%gB$$%hn5!LbMaa3f?$W(n8eJ&B?z&Or7WVvn}G&<^olt_(ATc*wQHXt=Jq_ z8K2K~0T|8HT~ z2{Y}h(1_UO_pcfUU;(zy*3VogPPS@lUY$5uDb>6>ak4V1d3EAkYV%z>ao*n@s}pCN z;U1khS7K3qCr&n-X-8dJ<$QlnEXwc1*@5{=e-2%`6?2Gyb+(l*En<?tJk19t`5DNlI~yZY&>dW?8YlL+Zg*diCt!l-FU_BH1>e8hm8Hv7_}?f z#|OqZDu5UbR<%7EtcndW#!&&ps*N!d6{Eqb)*WYTqOp2o8Dn#d-DGUBF?O2P_GqxG zeb8W4Y=be51t7+;02Ge~t704#K#T^fV(f-3MuSx`jtU?~gHw-;|Ks^G*}hm2moRDP5d&Sr~V>DRR zy6kf$rswDn^4l#TZ8b5Mv{~;$3Wv2CHH;SXDe~ z{Kcs87o)~sj2eG2YW&5h@fV}UUyK@mF&eCjQSC2AgH1bMPoEr)%Iwx8g8#F$7rxBMuSzwqrs{e4OYcyuqsA_ zRWTZ@ihacx4OYcg8l%Cg7!6jnE)7=2Xs{|qgHG*}g*!KxSyR>f$rDn^4CmW-|su&Gc6^{n1Vl-G4YcWRMzZiA@ zibu7-7!6j%-Zr+$7!6hxj|QuXM}t){8mx-ZU{#C;t75$gjUt29$Y6EMN>doYL$hTi zUM#AqFdDNsoxOJ>=A%)lP^ERzsy+WU#t2-5eRLE`m1BXK2~g$Uq;XqBAmBg(5NSJJK_o zAK@j3%;eEoe4C;C^vkKKE6D=&}Wp9!77T2Gw}%_;r2i&a-N@Bi_aSz zcDQeY)j~A%%V1UU`Z8EmyvAbe#ILa*8ZiSuH4jr-+LytqmeOEV`{>JHRq<%Bs(AlS zgH?*8B7@b)V3mW=jXo$k^H!HY+uh)pc2i#<~^*PWK!*Ca=29S8o^p|Bgr-y2DeLaB9#r{4|1 zi|FUf!>R=Mfg`S2}l(2H+ld8|`jS-2ODK>FNIV-<&FQ z55!)K?GqbVus$d%>K_|YGS4k5IU_bA^#k8+k3y+Ck(+*=P>K?zT;Hs~b&iw9ZhvSG zcCY)x8i_a>T5+zQ=RZnHgRy?^LMd*6O3OyS#$wyTg1jxTGvQEwHiR#g?H#{3wz=Su z`0A34eF~(i{rZu?YGkk)8LYZZ$m5sBC@W&yc;7;($Y3=xSVeyPAIP;^P!ndJO@R>m zl&T|lbH5et+SvTaU^OyWMViojVJcM+w7L@zlYb}TuaA0me?A(B8X2rc2CMaW6e5GwSbMxF zcrY?pbLMB3-OP*o^QRgnX-|z>Hm; zUq`qM;Ts5F2e+jKPC83qr+2PWpYqoeT#;!AVgE2~}wC*xv&lp={j8ZDa z`_xzoo(C}+ta1tlp~k*yj0UTUM}t+x8)u9Lt705;Pw}Q1qrs|J##pN{%B!^QQ^qK- z5~IPY)}_I!7)4cL6jg~)N+m{vRWVAb#AvW8Mk$pT4OYb{r4r*St74Q=iP2zHj8ZBw8mx*@N+m{vRWVAb#AvW8Mk$pT4OYb{r4pmTsu-nI zVl-G4qm)YQbYm1%iBVLgcobEMQB);HgH7Lh)@7e%ty^!52CHH;SXI1?F&eCj(O^~aXs{|qgH5Vyw;>C0B|^gH^3dgHf$rs(8OPMuSx`8muZF4OYb{ zxe_acq)99pPRU?ZjFKzGJIvx8VT|%B#XH>?4OYcyu&Q7Lh*j>hGuqw97*lJ^c zHue`|6kll{6klmwim${dz7nJ4N{j}pVw7Bo(O^}Kk}EM9tcp=`B{sttC0AlJSXDeq zuEc1tDn`kb7!6j%Xs{|q8I~9gR>j^kMuSx`8muZF4OYcyuqwv7tQZYe#b~f9MuSx` zR>s9>uqsA_RWTZ@iqT+Ij0US>gV9ziMuSx`8mx-ZU{#C;t70@*70Vc-!KxSyRuzv1 zt70@*727YIN;(>>iqT+I@o2CrMuSx`8mx-ZU{#C;t70@*6Dn^4<#iPNh7!6j%Xs{|qgH_$vWTd%ytP->7#woB0VuxyX&JCeEP;K_uj@s;`_Uye2{Ca<=+3V$o zlgxwcke0;d-gzwEmY&d_jvu=?J)x#`^n_KT6S4W>O3p`@f`!F zY8Z~ef{adISU$AN(Pf)5*cbfAsH$3vN%)VIEup8GL{)8O4u%uO^hUMXeB}HTIX|IsFmis%uFOWxPm%Le z8#2rWiee~u0 zq3-W|5S?!YH^<(D z^~0}=W)|cG6hn2*@UH|E?1Eb!b|DH}IJD~wbFg?AU%j0Buyhyr%YVOQ9=dx=Rq=`E zGh;Wo0|Sa_*yX4ZUjUq25g+L;2|9vZ-M+D(2D#l1<0AjlZtq9@&fbsR#)z#dKMRWw z^Sih&mD@yjK~o#zP(}3j1WhZu?5aywh@7A)a(;@OpCacc|9D2uPr2@tk@M4k1}S`< z9_n`R1#IN}ga_pOepB716a`P4G1s@?DRO@57dbyg&QH!glZu?5BIl=2(8Qi|k@HjJ z{N#mwJ%o3$OKwTU6I9mil&7qt@D8fF;!7F|`^7JcVKV8(d3)x?q52LB;+Om;RZy5J zLxj}$Zx-$ppHjLHw0A$4{3Wz^z4;5#QfFB5ZoE~(tKaxDsb9ID#_o%s z8k}5od2mc}KHhUEg}O59p^fl?**lKbt5uAJ;JP=mj3g#0BqKJ9?{$oa`-BIhRw zl6rG~x(?NG36eM^L6X?nNIhZ~hEt_(wlPYR6z?))w;8+B7@eOKk8&k#?~lgT8~eZ* z!x-gCVsw5|Jj#{C==>x`xsn*2pTsCv z5~K5z80AW06e)>Oq$Ea>k{CrwVw5I{(fLV?(j+lDKZ#MKBu0^v7@eQQJ~l?@Cowud zDIT4l#3)x1qw|v(-VT>{)vHOfs zh9pMGk=A|D7@eQQ==`L3SD^kWM&~CnIzNd~4lPFKCowudiP8Bx`=O-~bKZ()#NsP`x`=O?k=ghr9`Q{?;X)UMqn}yTgV-~CU~XAwoLX8q{T5aU~G!~ZuvaOoA>B#v>$~5H{Ql_<0qSmW2jf%UfEh|D3tjPH(ybx3; zR%avUCo2y&l(KlP&feUL1XCSf$ijgpIPsc9c1&x0OtE*qR2i}UosYI`sm>m|u4*Lw zI2RzZN0HuM)pFF9X77|!)AYY(O+Up3P^uN9Z1K9xITbADV8!{bDCzPXJFBy+ykjTS zo_q#WtYIq82~Pe!8w#r$KM7{_3k$%o)C*01BIhUEoXGhpO!SfSQ`JdTCsv(Mchd1E zuE#ZTGx67(^V9Qq_WO2zDn>)UoSzi0FXt!4>&y8`OZ#$u(o#A-;nu>s+e%Qn!8STr(Z2XB7XF{EeMWC>~%p6mC)DyP2aRDbvD*bA^^+Nt2>ASf!2eXXR)Jq$~x>eNwQ!$C`? z=a5A@S8T_|j8;GGjeT1p=O=$f(sf53dyKzu_$J`0J&T;5BIl>b`6+UKV)r}>e4IN1 zf6g6^3{`@k8~pr;{{P{U{kOHg-&567aC+UeNk6KaYAbK>nMx%D*SSvOUHb<=ns1w8sGPM%mdV{$|NJ!b@gyT#W855@e!<@Y1b zaX!wg$jEYeQKw*q+hHYWZ@i4lq_iA3oafaUyIxx6H{dV{C8RS6J zVE6RfXmsCphhRIsS#UMI8aY3)Xo;MkBIhS7BO>P~^bRVF$LD=mSe%NLrs4&uA;pmT zO*$Dyd*_$#fypyt(EA-1IX^|tPm%Kz3f{>1sTVm`UF7_Px5msqs9t;mp_qSr#r*d% zA0VV>dII5egm8AwoEOeFAuPZ=t(*%H@-3H+M|^`Bfbe*PBjbDzWQ~-C>Dy_TB z*fYk~7^9R*@jf+1=O-~bKWQJ7Qi;*|NsP`F^a0hD5_FCimJpYsuH8~lNjYyVk?YMUL{6RmEuuU zB}V5bF*-kqrH#?~NsLk|#T##oQYx{z#we;1TWpNZPhyl;aVqKP{3J%_C$UeAVQj6C z(fLX7==`L36~=}dt2UN4M&~E3OXnx8TW^fcPhxa_QoM{YIzNfg`APBU{3J%_Cowud ziBWPTM&~CnIzNfg`AMuSoRagC7@eOKkIqkGlw66?`ALkDD=|7hiP8Bs>WooxrFe9H z(zKPet1SBiI-#XG_n&`bu=O?kd zjM4c?Y?ZOq#{O*VFUBao(mp7@(z+C1iBWtdM#+^Jou9-gxe}xElNcpeVsw5IqvT3# zhA~R6#OVB_c$8d;(fLV?k}EMfKZ()#NsKZqF*-kqy=RQhPhxa_Qan08iP8Bx`=O-~bKZ()#NsKZqF*-kq(fLV?&QD@=eiEbelNg%Joz)OG`oq`LWtGd)*_G=sht8aHD%ugIvuo4Y|EysD&KmUjpEDj4>89t(=J4Gm zJFlX{$9?v_vH^1{He$Nz$+D(agjh6~G3HbZLeMtEGw70>SvB5`o{=2(XoyrWRsYb#?`U zp+Idwm>ld+ZT4QgS&mTHE`$pAC>#@KciN8sx z&?VfSxD?bKCfo{4F~KJs``%pRr+$Gc+MgxdvzR*7Pcb=f!rwepMh|u0MON1NdmgcN z#OyB-&cOqzU*^|-Mjhm>nbv( z@TalRr{duD@Z?r#vh^`1aB2TR2ANR!+u6@=gSnkOxiK&9_x&~Kb7U! zT6uXZ7Vyw=V{TrH#vWjr3)D1Ouv@ur7wC+X5E;hz- zxpcm_7+Ym*wXu(keQInwTr;h^V>p#`RmN(JH5i*|>{es98)Ip$?J*VTm_IcZ;9iKa z(J$Puv1(&!V-3b;8oR~VFO2=x*h|LdqMo3Au*RVMwi;t&xma!X71XI0;jW-2$7ElN znlYagtNm6rAF?i2mkcYme8$E!J=gH#G0jJ>9o>9LJFBgH5HYgaf^-TTgc?HWrv zuo@!T-gA}MFR}Z@*aX{N6K)TjGB0BS|EYTRA|`6ORnPcHqv^x^0L_ zIom5h=Qv-qGLK?~jLSPKWZjp4-!HGnVd&R=`Qd(fXGKfPd(@u(UsSeo#qWsRvEuYt zo#XBKCo5FBO{cqUx+C25|6eLix%2c-h(A|dLO!%TDo0i2M*(l=qYVtM_`L4?i1+WW zF!e`G2sx&Ijp4Jxnh&4(UVc%{2Y2NEQOzgjFzV5}xCxyPk8{k|HiEI{s<%7mE<;s> zdEMm*4?=ha!aEQ?fbbrKw;*KSOwO~aav^e0RaH2Zbf+0RBb-XQON?cW-D>Q1V^0`+ z${3GX`}nJ|A=tkdzgcR(9O_YQoH2fZ6w4S}V(h<-y=d(B#*#?HTDN~Vm2?LhVOd}Z3v{uYd!K> zkG$3+ul1Hhb9%xWxLKp>Jq|f3M!;#Sjn^#B)+`v)lKPqVy553ue`+hzaXM00J-{Y; z`ck*lw6cy}IXeC{Ew4wU<9^TsG-N+P=j)*u{D8y!91}KVr?V}rz+(iNx#vX)PE%dzhZrO;$f``*1F!Ij*oPcNcv+}pjwTa8j8=l^*uy0vbovm3^72gy*#Eq`6 zfb%g1)W~4k4+YVJiEK+NXVF@pZSn^ZHcewzyPH#yL4=&0Hrn`W+IVgl? z?0E3ST}C--!%{53C`S`3D*ccTv@9j|%78iq#%Qk>auoIa!@ujZuwO zR&9B*Cc7+ed?O47@)ia-svddbMvQP&RaW~bhB#Wp6`H-Bd9(K@AGxutg)fEKnx{F= z(M2crE!jGvh6auU369FP)EE{=?R-g$KQw7`Qj}rm#r(Ynzyw?9vz1K?(Q&fytArf!%1xHwxupr6$ zjLVlJRq^FvqC$D1k|dLkk7@U%2}%)Pnjo`gerlyjeo>Nb$}sZ#Q$S9QtXP;V!#VBJT*nPn zH;>6wZFwrYa3yA{vdgMQuBmSJNw*0r=RuXQb@*Shx6 zm)E-D(Q94t{(WBS@LA`7U7N6b>tDcr-+SP0<0AQQ{b%BvP*BEsXx{B(Yxn*JL%gdv z8VB7FMsZz*1cZFa7!{s)~+-=pwhuBv3m$o`vm1u*RNzNXI1ZPr677J_ElZ6LCcAgz zr@FR+2Dh~6G`Apm8rtsFXQbX=X8xl~y%Fnu@gamZBlhPHA(ZjQu)egvyF7MBY^&Il ziQff_3;!7WtoRytV9Bu9k4sPU4+)15`X#bQsDJF(W^&g5bZidW>h{SZH&W45zngb3 zwEktWyRn<+63+*}EPOrqdGSx(A=u5V(i3bq^Zaghv)yFEo#ZY7jk$XJw83YY+04s6 zex8P*1A<959evQTMt8Opb3SX}E3y>2WTu#p~{dO!H8$%zl_!+Ua1=;xY zlKBPw!^{x%!7f7y)%X=6ul3wG1CiHy_uf>(UF&XkSH&j9Hv|m@%Y(^9mj#gWhgUaYOKMIfQ8R6r4!jv!J zH@`a`*WPmYy$jw9_AMIX-YYJ2%<22+_TQKH{P_ioy!Slc!WwugF7jHBpH=!bqu+Qq&{&s2a(tM5-4QVXr@sNuSO%Lb4%@I+_|0u$JO|Cs_``K|eAlI4rr;+5z*Q&_=(Huh|ni%5U#V_YL%6V-~jydLZ&4UXQ}G5PfP{ zsOJAI4D}eSBJ}(XMz6$!Tzxs~k3YcLyJGDM{IPQIJ^USoKW>dB`48}SJpLlD^|R`x zorRVxG@_K3vpuRodXBJlix{nxg=GAwmub#MgEX33Lo@FNXxG zL&zb4PD98cfjSU!2%vAGYC05QJKFa+zX>6S1PajB&LM#=LCB$cIFtv61loj=Vu6;5I%{JLjr9K=jj8><)aaDy{QN}B+x8` z91>_DLJkSEHL@HI0W=vQhXlg#>VAGdgd7s+V1!R25NCxF2(_Z#OSpy_JJ{at&4GhAjPBCx)=ut5~J6;7zYRvqu06^ z2M7|Q*SZ*$`C=W$I6#mXz19_v0|bc;#&avi0fNNnwJyd1g2d>xF2(_Z#5gRF7>5NC z5NCqu09F$HwTjE=I3)#Y5f`G7b#CA8v zGFFUU>xxIObukVGBu1}wv5Sq-Yh8?9>x#$We#AK3j~Ivh5#w+_VjS*AjKlqiakw8b z4)-HQuXQmF{3AxMbukY7BSx=vF%JAAMz3`-4*VlVuXQmF{3CXSF%JAAcAqg0{3G_f zu@{ZeYhBx;*Lt`;)Gx&7wJt`lbuoIai_vRcj9%+v^ja6A*SZ+J*2Utgg;7dzP)z1GF(wXS&dS{I|&x>$=b4gn;_A%GN*1OJH8YhCPZW1Ec8 zYhCf^wXS&dS{I|&x){CI#ptyz)|=2M@>-9))+4WV_MC028rfdmJSo%M!uRa3+borZ zp)GaG2^CwKYg>DZR$rtI*AMG2X$vYy+KlKYY2dqoeuQ1yFg+e-Ipfbm3K96@MC5 z*zm3HG_Bs)X=zt6r&{!@(Z`D@HUeXlJU|A(h zQZYNi_sM40V)V^C!IV8?*2=rqAMWVJ$MRGH_Xp=(mCoL~5!;Ns)^V#_mf}kAqRNIg z*Y_K<_{t06)0~P7B0y*qd99CGo0BEozpSGG>ZFaMTN3?iva6UNyCv4l=O%r8m zt;4lN&v$PS?-easCf(0Tum2x=?*d>)QKx;^IcF}J$t06uK|xRgjv5II!R%_>pk!bM zPGAD80 z0TuH-zk2%ATr#7^f!UoVADa z)pVCuTivj#Jg1|7X5Y@s%ewm=-`Ur+JW3zRr8N?_tt|J=(ymE77weIRSJUoCeFJ)i zs=PrxormmTCCcM+uXX7Nb)t3T23?hMuXW30s@IF$Ydzkq{z&~FxLFOV{JGaUt&7Zh zJoj4fsN93pN&4Hn+P1B&^r8HXa;J4tZuN7ddT(*VZB6_1P0f8zrIs6Z?-D$5XP5A| zV`=67>m+ko)bEDdwsmT@wN)gVnzf>>`L=KL9(ye)dnvfMjs z(?A1m(})eYt@E1qY?mlg_rt7#c6{Z*!B@$6JM@)ec(YBnLN)ZVJK?MqZn^dJYj@(E zPf7FuJg0IV^v445n3h3?kpX5NQQAW;<^3<%QS>1N^{@CAq zo3(CH0LV}BMiCX3P~r8AiH$0&|J=aR1(Sm_#gcOA;Uq854n{l(ZQJhAHR2AftD|&R z0Qnd;w`+)V)dSx2kRWgUvnvqFmWV_-SgqEfrByxq5Zpf!3TG|CUA3u#OTw zgy!*JRC*{?nX>#C?R8ZZr81|Z7k9j~GHYeE7B}+c$r|=hWr(9Nsk1a|qU5?03YBBm zOZ3P3I(uhP%+@ydT>p!0+nOic-a13aX{Pk}m)*l#nzq?H=(YY0y;d0AYrRnq`fCCL z8Rxr653U9D{~>xXT!>RwYRdT*kEzamogQL(?lMhz?r2`?o{QJIw=tU6y7S?+?tFha zuXT&0rVJHH(M@EPj_j-S8}*K``RU>f#gEkfs&sS1Gqvtg6ldt4+QPo!q|}MwMlBP< z#PJKl?c)XnO~Zv#`z7JjesMLoG~XU(hfVeUQ8>lBvw>xY3#X>VzQer1ofen9MkB^o zN5dQtPW@e+_u*!g69dAjzZ*$7#m~`GyWDF%4qxko%F(m(Wqo!s7)rAc+H|6=-kCv` z_;J3C%J)ZvQa9=SVhEpT+9g~bJ{DeAESAm=7nZ(S_{ZAX;;Hqsiw`v{4W~68pic9n z3@V%L${!jZ4F|`~%ID9Gg-6xTd6b@J)y*7V&7J*X`h5)ik|D!8XsYR((unsTpoS-MYa`llq6`lPP4TzsnM zsC(E!MWXbFgi=3<`7!i#QQ|h|k4x)BEkCkQs(Qg#9J8-XYgkcTfleY9$4T+T0UvAL zw{&^&x!V6IJ>0N$RDo1`Tt4?&&%M@jul3w({emQbif(ne*SfBbx!3yf#ZQF#(vOCT zqt}b zG&1krHX<)|KLg)oklZN>u1bsdXd3q?38iWmze&oAnlVii8}t)TJ4MQii^lE~g;Iwt zd=Xvyt3oNem*-yVEqe_*NAz>oQLgX#gi27|xYCjueH$Oo*KjF}W|S`9si~y-xG|qL zLnq|jnH1(|b+ z|FA)a?(OHkh(kA2rqVm^_u-zx+WK1xpKE;V#T>@Om9m1(bF| zk${y{`0&XzC$xvZkB4xdLB}ZyA4|Q@=>RpuCWAG@!FI&c5ebyNd4 zfqAusVxM$WXElrsU5)qdn2ySYilafnqEL;XqoO?hRu8Zw5Gge1GP+VF#pSFJI@KR7 zgty1g=UPo!=;HHoW9@zMyo^>(8C-|nc%vO)6<&P^_I!Z97Qf$^4)Atmw6ttPQ_F~# z+PeCt;|E7Sv)*jVUh8Hb zPVB~%4=I(GU6a^DiET&>DV6iRkXVCm<7UY3ybYvOX0J^QuXQuL)}4=+bY^(1o6#&V zTbLM52WH)g^(ThB%F8~O81gDJyw<%eUh8Ivs>~2onIWYz!)x6PDU}&s>t;x)%;qJA zl*(**Vo0gXzLXeJDl@#+nQDyVSD0bDWQLT=46k)Fq*P{jt(zgGGQ;o33@McvUh8H^ zsmwMc_Do`Utver5D(Ay%-3%#}8D8sVNU6;5S~o*VWro+f8B!`Uyw=T-QkmhkZibY~ z46k)Fq*P{jt(zgGGQ(@#3@Me_>4_n#GDB45e2A*d5LKDswQh#I%Iv<78I2@ zA71Ouhu6AUYhsfUYfr447+&jM7O!t;x<%o)}*1W_Yc8+2x7hwQlyQ#PC`- zdmyog6Z=VGKTiztmA8TT%F80YGDCc2hUCf&uXQsdS7vyvn<2R}!)x6P$(7j!i6OZ% z!)x97kX)JJwQh#w$_%e{GrZQ#kYSnOwQlyy#PC`-!)x97@LD&+Yuya5bu+xy&G1?` zYssb>t=YZn<2C^Luh4&*SZ;!E3?NF z!)x6PuXX3cYu#)wJ^VDoYuya5bu+xy&G1?`!)x6PuXQuL*3Cu|WaZ9Jx${%*{G_hE z=&|JAF~r5-PK7mgF=+2Q4mFk>3g+0SkJODcYe~j z$eo{DA}M}S0alA+<-u4QqjV{j#UZ+-Ayz>n7RMWtS<11SVsVWAEh>>6lfTtzNO^N z`q!hGi-$E{Rdia12?WaOgRiU0KyUlp`H3?qzaM*Gb(DTsnYAJ_&dHsha_6Vq`6+jP z>MY&YR=Rsu-_FDA)(h0jtCpSUO+#pTUS9Cq@XIAgr^_{&Z$N}?hzqF=l3TRt7Q z!Y#!gm+mfoq5eW?y)2-ZR6T0yyjB<5qc%(f%*LUZ3*r2(^A?;sSMSEL5zqs-fJ=A? z&yL+Y?9%YIhGi1RScdhx$S*A39NttwOhcEW#d-q}!hNOb;flhh!Yji8#eY$kH*as72)9u#ALOwhD;$fkz$ z>)S8;mAUiNP(f4f{FFOCg^$PjcG1|$;WKjn*t_A^jhDs??dhsd`|3~kxM&l$Ks{2c z!)5OkE)2KIRphOu)p9ObRs2!I^%|zPy@jZEz;WtrLma1$joWFoj7zUY)jRf_3ud1) zZ{GQHyUq`lPzo= z(-lq*4@t|rQ}LKEzxYVupqd+NxOg3|y}dfFYhlC0(AhxO-nCldin!c0GU%%gOP@PG z<<3vJ^ApwL;&Cqv9peUSM(+Id{lYVa|16!NPS|x>ulx`gB{YOLms-Pl;n{F&fp!eN z^>gQ^_sKfp+5*m3$bGB#ilwEB?h#)tzI0IEZVV2Z>R;1TGrp;&aYWTlP4|7H@Gp+Y z8!x+dL|${*zreTal3sJ!f~L~gQ5Mm#@kh1{wok%*p>)~4e^k&kDc&zf6*T3}PiHQe zd)_cj!b43}JCiAmpBXN5;8spUG(&@V`)pm(_?o{kj zy|oW`fUpd)%S+=jBkM-q9+%%W$r^T!c`wu$3R6aw-jsUoY}&n^vND47rl}`)ej53( z<<3u(+uZp{k-+;h#S+zdy*i#rp|{=Dp=x-<)!`3S?GxkjBUkOut23QseW_}{Bj){( z6iYVO+V1vHT6RR!gwYjCd*PxN6-)aKx~m~3L$P#D-5A|lnriEs_Qpx?q(dbr%blN& zZ#+0W+xY5uxn7}5@jEyr;jZxm!nXOc?TO7S1^_xw4<}{HQWz6<3m?=2QLf;GptU+I2I`TqCW_ctniOy3X6z8{j!FDw0?IJNL`rT?t-38l-F z=FU(5vz(u9RKwPSB&IA#GCNmi+w9%h)Yx!YVtt9dKQW#bdf87V_W8t+D|vlSB(^27 zXA(oMrQM*VmBwYGBKn{ULVd+ zULVd+W=|%DNXZP5lJnvGWQJVHj5ix*$d$}+elkO@WOjLC?@J8lC+9=1N@h4eIUjN*vx$24Yld9O4Cf~^&QCo`O%oDV^g8G5FeQ#J~BgmWQO?24Dpc}&QE5@ zkj!v?GDC)BhVzpdG9)vcpUjXUnc@6oh78H<%EXW%nSCxXWJqR6j=b!niQ)WYhVzs2 zT`4J%8O~2;I6s-;{A7mnlNruWW;j2Y;rwKV^OG6QPiAWq!}-Y!=O^dG`N<6DCo`O% z%y51(!}-Y!=O;6qpUiN6GQ;`F4Cf~^oS)2aelo-P$qeTwGn}8yG`vv8aDFnw`N{ck zelo-P$qeTwGn}8y?n(^jCo`O%oDb(GGn}8ykQ|xe{A7mnlNruWW;j2Y;rwKV^OG6Q zPi8njnc@6ohVzpd&QE3tlFSe!nIS_m!}-Y!=O;6qpUiN6GE=9OjN$xbhVzs2;rwKV z^OM;~LZjUIDR+L#ouAb4Fn4~cELyFen48)wbGxH)KpeHdYhiiL2C0y6(pjrf>Q_bO zQL+oz`I>EX|4bwLSK>n3IS2IP7f^K$=xm*45mly8>1>^ZBxJy;Dk>VJ0a;p})6=gg zw^L)@mpZhap3V9ewIt1px?7-SaEp4kvoqJdZD60mIzL+*E!eo*)lD{Pnh z!IqleC$E6E>0fK^>D7$f`RT63n>IBszfH$NZe_B)P>EPItMvE|`Ca7BPk*APMYOt7 z@m?9rX0)$piwdRoC{gOWL%gl;h;1E}A4!)adD7$3CP_GSHGRc0VNyHFq~EDd*sEf& z$H}$P<0PKg&uW2k<-=Q*AWV|m+bT;T%cVstb*@*JCqEd?Og0U(^^zn~5~Z54vc{_P zU8}{Tnk#pHO3JiRjZ9<{vKkH7NMLxke{7KGl0~i17K-ll*gry^z-CUcQ-Ke;3#jdcu^5gc&k9VCl z{6-b4(;#b`ov%SQceI2c@ouC7oSnSBj&6(WlHm<*aML6x+f~PKx zHz@fB-BapV@yAQXpu5%(i<=QH2f6c8?))@jWyqbMbSKK4pQ<9M*XV*}k<>=L-gs26 z6-IY{YSM##J3l$!XwFa0H=6U4=Z@z55eN4EdA+a=lDGeyM+S_cNO=j`EluGW4<-2Knj`B2m&cO zAnHM3_?(Jl2w!b@Rk*SNArxIyzNB{$==Bzdw-tUm9Z@q_}Kr^BL!Q+-Xhhr+fiyC@wx_6+K=dhH$?+OWqWPQt0+Bk4qlf{PW_c!l!CK z685dHEgsYGLU@BTyPt0Qco9bZPQZ>~X=r(@=-R5a3oS_POa9rc5h1&;t!F*J~atMdTCcz!bb6m{x zJuOph5)25Xs`1Xgahao};K7vh;1-ti-hR6{BYZCg8eOb*kdWoDxZZu@zU-t|T>Hdz zEDmoBhm5QQD)oG?Y3Hc2$eo{bBy;Dd9Wc;Z9oxbq%Y1U@CkbS7=cjCtn;}m&hF?nY zC#zmg|A5UK7wM{X)-xJt;)A92itSz~zHuCir0V6cixQ)rI>NM}A{$efaIzJ&GS4KQ?&zLiMJQJ3k%SvbVhZ zc*wQ|Wu2l^6n2CEu6{}G{G`K^J3r;lPrIvu_KXBvdn)~%(#cA5=cmy9*K~fmNey5N zt(daV%IsX73^TfFne`?1;lyrC3@MfKtx4>m#5N>`l*;*DNDSvEGn}88Y79uJ%y51( z!}-Y!=O?q{6T|t*jL6^laDFnw`N<4%;r!%$NU6+lelkN!Wrp*U8B!`UoS)2)QkmiWWQLT= z4Cf~^q*P`&Kbaw=GQ;`F3@Mcv&QE4YsmySGGDAvbhVzpdQYtf?pUjX_nK7KC8B!`U zoS)2)QkmiWWQLT=4Cf~^q*P`&Kbaw=GUIKV8B!`UoS)2)Qkk8e7@{gOL{-j*sLBjc zl^M=YX2`3|?n?}Ll^LQc=R;IwhVzpd&QE6L#BhExLrUd*I6s*or7~NZ7@{gOL{-j* z^OG6!Dl?p)%y51(`(0vsxSlbbpPUcpC+BNTY*J$FiIo$>`N_-T{N!clCx-Kr8O~46 z*PR&7Pi8njIUmkXW;j2Y;rwKV7pPUcpC+GWkVmLpU;r!%$PbP-*lUYgfCFetOWrono455`7&QE4Y zuFMWf?2yD}Cx+z8`EY*nvN%7P;rwKV^OG4uD>H;vW;j2Y;rwKV^OG6QPi8njnc@6o zc70+WNet&F=leopI6s-;{N!bEelkOHWmYFilUZXnW#=cey%Re;Iy*%#d7}U62@(D>Iy*oDa#B8O~2;NUqFqelo-P$qX5m8O~2;zf270Co`O% zoDb(GGn}8yaDFnw`N<6DCo`O%%wC%q&QE4IKRF-HPi8njnc@6ohVzpd&QE4IKbhhD zWQOyT8O~2;I6s-;{A7mnlNruWW;j2Y;rwKV^OG6QPi8njnc@6o){_{{Pi8njIUmkX zW;j2Y;rwKV^OM;ViQ)WYhVzs2;rwKV49g7XCo`O%%y51(!}-Y!=O;6qpUiN6GQ;`F z455`7LMtFrd(OOS>MzdZb|EUrRB=Ta^Jzs5J z@-|IWmbGq*c^~;@^Moa>TQ%LYuDPdQsb)WQLNsu4!UxHI1Rk zl=jMfN+t=d7SiOz_|vvr>>XqOv^(sWvc%o|^Y5HG0L$0sVRm?j_U-%L8}_ZeGH_V4 zBmEcO-JW%e4tSBf{N(;Ga)GjpW%_PtrEJcuyko-5${D-Pteih_X652ZGb=xAJ8_F< zgroE~S2w4cu#XM}y2F}qpQb4AHQ}EX9xysWP54($VSiQ=9?{giI7M-OT7S#b7(FZ; zs!Q3x=WELKa((@@(j}^p{&def<+QnHo%Ij@aO&KJUGpzkaQ@ucQ!ku4d*R8a&OTuF zfjcx7`q|pK*Tl7bOqy&zjVx%mUi!r8B9wjhjNkR7EbJM{O@s2jEzVP=4hP10y|F(X z{pb8SFumw0==B8a-BmMQA7@?}^U@K{pVx2Se{R0s1&ur&`D!gQVt2byy2mN|DBa`e zxWW#5JgI|<|Dge+-l0x^Oeje?>XY;~#H!lcw10J5?GA;oy|-XtKnQ^Kr=-<)L(S)yHQGR9DS)aNqzP zGI)ff>LMo%TjV)?XKyAdQc-uev^VpI<9B!`8$Vo%fbr(a9CC@)YbQsh0rzsyt%3u3kFu*JuE+$(w$Qk zID64qv%So~7P#j6Y^0pjX7a+x;`6MQdwm>DP=d{T)>}GJao~zEfLUT2`g+)X0{WJJRGW;uN{srP~mTBS)cH*5%*i-2;GdNe^j9@X5Eys4k_L2!)DE*r0VH%%(_+I zXXx*!!XdNIT6AXDf`xH3Pc}$V^3NMm@>mO{hZ3fxGyq>q3uS=we^g;`Novs?PPymA zlJOED&Z^ftPy$#9=gxfanI0cbY&lYU5<+d9TMS3F{6gOb&U^@5i}@jHQHM+P$7kmT zJvLRki_*6$eTh=a<4mPjDm`53e%j*6N~bD4MJW%nUaFMGQM)RAw^D_U-IpqzqV!Wr zHSR_?*ZfPAKAe5uSL^H59m9_anQ9CR5<4rKY7C6W<$Ui>?CQj>OYD}!KAzZ`#O_UO zLt@`Z>}QE_=kmV1keFU2W~?=v8XG1h_SVE`lzG|rCU$vZHzoGb#2!fO;l!R#?1jX3 z(=qcgczHI}7}^r+NUSTd3lsZTVxLHiyS3LxDad|rVg;QGGamY7+fA%Jv2tQviCvi3 zM-%(k#J-W(V~H(QJHgwa#o+DsCw6mUBaM5I8?5FAtGU5yHtd0vrn$kYf>m9&C3}~7 zRYz{HO5~dxtnxhK=&BQECV8Kc8>})0$3%_0LGxD6JK}&HQa@iCzg^v}BsW-nw0-i1 zt_7WX5O#dk#TirRqpKH~c(i8qHC=<)ysQyavg4RU=}XI{S?j5WGoneM9z>}iMm2gY zh}p0yRU_C+4~HG++~(VQR%lYrxjH2qHC6gA`sYTA+}vPQM|bgAs|t(y2kg2xWj@_! z`3xs-XUcB=0heyY27=u}r7#RpDxC0a8NoM~Al4<317b-B{>9Hk-_ zqDl`zTYRFaF)8j~V#F~ih%Sd#<|pgCp~}_$T7ybLspD7er3Wgr`b)E(OK#C~*Kaxe zH;hs-OJiXyTG=uEvF4tCRAap|>nk07ZQI&=V~OWMb1(bhU6XxeIeR)vsjlwhlGg5+ z$GfQ~%^pKz_zh`?O)FLLCaLyt*{VF4UA5xwJXyQ=`NE}5nRVso_0#p+AM21GbbIq{ zvEaA&&g#}c8FqngtE_38zM;K0j?prt&9qL7dUtTdY`(3pU#pTaee{Q3Yk%ne>*7c( zZQItxnkFL+GQ0@GGYOuOd zHMb^=X0WQRyPCULA-404X0YmfqZzDv?q~+9o{Pb%w=tT*s`Fv6>U@7mgH^p!$PHFi zI?qp9S01>?O0aHzIHUBV(z?Rwbvvi=1LEa0H(2EW=@BmGj%#(77nj5Uno_qGGdg0IxFK$%E{>JJd+^=W9th!p zv9Aj^7oIBIQ2cbw7YfVkt}pC2rdU|m@G;p+v2TUI%ZjQEChs=?MvR`Rl1w)26h2|!72qLibU(`no2#-V#C;s4dd*|b$gV$#=d&&-gU2Ndh3ZhsZS5h0!C`s zfzcXxD*Zyda9t|%f;YF!(0AJUgF3>z0v}r8YJL4|_?P0vwbvF-uKz&cn8xLQ(7f{F zv8A)Us-2h=lx8|A6R&@llX zYN|TTF05ZXS{-MRRVrN7I{u4rTj5=$4@ZOZk;Vg{{_-J3Ii@}|HQZEquNJx6tPg=lNHv zNVMwjX#M@8{&06^RDomk_fPuE4ORztAI=R{bA#2o(9N@v2BpWS!{h{|vz4k-boXgW zcT&1Z>1&k!y{=Z1l&;rX7QR2N^i4_&YCIjJ^a`cNDn-A09P^dR}v}m_Jh~%bl(CLrO1I`Vpm9EB!x8 zchS2UX?eRBDE)V(O}aYp{dGzoP`>8{!LeRWYm-|Y^) zOj-ADc5XJeG0=_I?6SoA5_^AQA5ZL)iG4n?FD3RwVp|e>CNV|@@V;QM>g{5%YKFn8 z8KVN2wI{|!)C_}F=Q}>JQxlt?Sa)Je61zFEm5I@5+Uvt$)!V>e)$GZ{7z@CRu>hP8 zgHh1PNx`ht2j%j8Os1 zFj#dyMg=g#VAX7KVn~S0_?N%Nur4u11u%O$v1by)VAabqDuDCNN{mqf%rIDWK1KyF z>r0GL0n9L1bw1>HW}6aYQ~)y!R-KPg0n8?<-kLEgfEfm>W{e78hQX>CqXL*Q7JwOJ z0hlosfEiC*%@_f|41-lOMgTCwVAYIki5X)7m|?JL_M60roXjv-bv|8tGRCL?X5EP~ zDu5Yd0eD%)0x)9)0JA$2+nCsQ65E^@2CH6H4Zv(!HUBbpU}7^8!(i3hz+lzOVz6q4 z!K&HP#4uPj>rd?F#4uR(vKXv7A8LQIA0&pss#$SNH6I46W*Dq`+5Hp4VATw@zw=?R zYKGe141-m(a}vW~)eM7G=fhyt41-lO3|7rBST$pGWHSs_%`jLsL+x*d!KxX$f3sI6 zhT7i@gH`9lVATwRRWt5KW*DrRVX$g;X<`_xnqjc&eCYno(EXcXuxf_Esu>2WW*DrR zVX$h3!KxVst7aIinqjbNhQX@Yw-Uo()eN=2^I@=RhQX>C2CHVM{movT824B+3|5`* zl*AYT!0f$=U7i>Qt6mm^Rp&$FZ-&O-42{1T8hC2CHTmteRo4YKFn883wCn7_6FM zuxf_Esu>2WX6XLS(EXdC_BX>|)eM7GGYnSEFjzIiVATwRRWl4$%`jLs!(i2HB%x7m zu$mjJ<_4>|!72`Mxxs2~uzJ_x++dY4RpdN*2}8Zyj}Bpy#)6s105_ZU%thVkSLk1) zESaOCycIE3{IfbL<_4?s&)k?sgXtJ5?pUZt6OD#6p0V~Y`~ul*`#VQ(5-N&$mv6xN8ezCN_x8yHt0SY!X3q_;X8#l6fceXv+>4 zyDa|nuQZGew-)azK2`csU8%%8`n4R)@?m-N1I16&UDhzE_ONCgotRe$pNZ>g)c1p1 zRx4MTtl|C513Ityna+!~=`~lFt2KtOV9ZY8I~oD=(oiYfU&o0#L;P-L$+>r2xHw!Z zGXQ=fYO}5CW1>N0i#vr?QPDO=c?Y&|!5}3WTevE2;Yf8BSWxqxnmgm~;PkFzCE=Oz ze-l0t?$X*H4CCTy#IxfCy*X9yrue(q;w@^h1)T;=$~mpoQ~!DuhKmc=6yH?ySm|%a zY^~*2oyl>CMe2>kO88WmSvWK7Qrt}iuDxZG)>6H}p?O<4_q=Sl&1_R4e2guHzfV85 zx+#~ks%)=o!Kt%zgVo$%wGh_FqWN^`P?ekq3gwU+tmX!*O{bkA$<9S%_lc67!xp|s z$<8WWfO3P?++cOS?u5JW_(CI?b5EYHJKIU|`*SkMJv+QV95YGia8YEV)2(!2c z#}6t2Ny{3qxV3Iv>AkPgHP0)IQ5*TQ3ZjTA7U-IDn)>=V z)5P>0?b?eoAbH6WBc&zxT`3yDeVxa)Z_L&Y8XN9F6_2)1=Yyb^3;kj<4?=(%fLR zdK?Cgw;v<76I(8bMQ-Sb`<6$>839wOt(rVcf9)~k)pL9qlg=~GG>baJcGlFt1vlwwQpOQpLj%?(yV_gXzvKv?xtr3kD3ng**k z>w%7iR!muFWp=L4nptl)HCDrmRN03UyD>4OR9<#XVh<&@Au*&>&i6uM4Jr#}7_54` zNU6+Tn-~VGW*DqGA5tna3|7r}SYx&@F$`ACx)bY940)B8#bDL>kXM;uuxf_Esu`jx zGelKpNU6*)ST#dRWro438B!`U3|7sMQkgAJ3@Mcv2CL48l*$Z)RWqbiX1irmGN?`L zwTU66a=zmeTbvjMt6m>nn4J%URWqbiW*ZViN@ezZVo0gXFj)1nNU6*)ST#dRWro43 z8B!`U3|7sMQkh|}YKD}`41-lOq*P`YtePREGQ(ii3@Mcv2CHUBsmyrnX@;oE3{jQY z`w~M`Wro43^C7P?yDzZ^5<^twe7c%u`Sw=WWQM`2mn|oT!KxWjD(556GDAvbwlpzB zRc45)oDYLlGvrlf7_6FMuxj?Z#PrlNV;HPD9|o(=*P7U*#M%=pCx*eQm&IV!%g#>> zgHC2CHUz0g|!iY^pI}uHNVX*3a7_6EhxiYKMz0<5Qo3g>G+1`mAp7I@; z*u2C}PYi=qFN?vdx3N4i3|7rPl^6!AW)CFxaAH46?B|IgzVbE@UwK)?S7wN>%#d7} zVX$h3CGAuI;R?RS2HN#-l41-lO3|7rB zST)07)eM7GGlW)V2(8R8ST#d(Wro4383wCn7_6FMuxhrKp81(!uxf_Esu>2WW*DrR zVX$h3!KxVst7an!vU2CA-1#YYe#)JnMshO9ou5WA+K?z@r7WDgWauop&e1ly&{Vmz z_te%INO`8U%Uokh$JBL7CZX<;qDLZ;&5HNcf%xj~nDq23iPk{25^1kI+|hj7S{Y(= zRMu$f-nPE-HrZdaS5Ek8-yz%Dr=PI3cUFH}bLZWedq}%IH+rM*$@HzweRu#?I=&+N zjGp%@9L*d!u-)kHsI0f`#g)wiK_lwW-+<*Q||oq^b6azRko@QZuzyQvg+S9{j1GApVV_RSJL*% z24$$+uWJ7_DyS|mi>~dM`fWV!A8gz9Kzr}8OJkM&fXXVB^^m@%3AV;5mlmzxzUnsb z8KT>kL};qL?`%C9jaqM)ytP~%w)btrkIUZjgQ{+O<=CZzs&CsDwfPa8adPLU-1({V zuIX!=`*P=}(%rKp4)vX+(y?M@=_@o6qHCShqDd60ddO{Sn(Ac~|8nb!O@()6wXo9D zOs}5jB>wiYuZb8_o})UO5*f4zaM(7TF%bcrYv4U zmb5MvsXQpQm09vF>n6?2)c&V9KYdH>sL`FD)WuhGaei{X(VU;0Z#3s8&mGPA$#Zdj z@-{|uesVsXpPcV6>-;2H)0cY6sdHz~kBjIjSNL}wSNFB>&#c!u z&z+y9aS-Cif~M8k$)U?bK|Mlt3LPy+iOtAzL38J)h7YTYz7Nbv!w$@59jw@8U+AzgzNMQzAOBT{v9fAdPG;wEA@vE zYQSFW?C^9Gx0UC#j8w=yZKNl`4{=Rf~X+!<;TK1SeVt+*VbPH<^`=$g?U-{3xa^WzPC59J$q z{snRVu0!WnH+0Ut+1?JVJ=PII)1coUf5d{W`3ug@ve5{9SE*xKmg1>NRGdt2)b^p*$~*%WQ99q!<0+?Q!{C;{}fE@6Ivrg$g;Qj4Zt=_1xLC zd;KWVo2H)SlHM%dD1}Y^XwsY9`Dr`F(wHl{P%IS@FU7;Q&-RL?(iLA!hFK^5ecdZc zXN}!oc3J z`l25Y%iVCzR#U0>hIn|Ic5T)b>7;|C**YWBY`ycW7jX<;t%@_!6@K5^9~Ta&-z)rb zOkKd^??oOD?8LAV8&IBHP*t9ADE^N+w#nyIilv>y`!dB6)p@--p2_7*O~iv+uGLia zzEd?k;_C2+s`iO-`H>It=i@d9s`fi#-VcTL>=19~PF7lWMAL*={&pKw{-Ua`zNxru zuezqD+6HR=euHL5L0ze|?3}tWO{K9E(Y^6>JLymr(R0TBt%~Rgb6&*h8SOyi&QF5{ zO4r9$@+#Gy?XUW7=La=vDO&BSP5TcMC_U7KVP(1Y!NTk7Hx~|XTs5dkIN;5;IBw%_ zQw=>RuJ424@ATA8Izlsah>j`@aV=!lX3gROEwe6~ zn-o#G^Hc8pB!LgnXzu(pvh&j|YS>zk#FPa|X6LFtnY}xk8XGQ4tS_TnY}78oS)2)D>)y|PiDxK%y51(L#||o^OG5JB{Q6# z%#bUY;rwKVT*(aQCo|+qW;j2YAy+cP`N<5qk{QlVX2_MyHYJ8!$qeTw=R>Y!hVzpd zawRjIpUjXenc@6ohFr-Ek&+oAB{M`yW{8x`kS3Yo{A7kS$qeTwGek;eh?LB5elq(_ zVmLpU;r!%$I6s*oS2F8P47rjSA|>ZTq-2IP$?VR=aDFoTPGXxA!}-a})+i`(K0Q&* z7|u^-GZMr3$qeTwFN^b&8O~2;OB2KS$*e!In-jzN$;;yWy&0elkObWQOyT88Rd@oS)2)A(`R)WQGjM4Cf~^WJqRLCWZ{j>~o1BLo!2h z%Uu{N!bEellawJ2RZ0%y51(!}-Y!=O;6qpUiN6GQ;`F4Cf~^ zoS)2aelo-P$qeTwGn}8yaDFnw`N<6DCo`O%%y51(!}-Y!=O;6qpUiN6GQ;`F4Cg1a zQxe1Z$qeTw=fnBQ4Cf~^oS)2aeloi&F`S>waDH+=oS)2aelkOHWQOyT8O~2;I6s-; z{A7mnliA6M;rwKV^ON)8{A7mnlUZM42$IYYBsm{4Br}|!%zl;_&QE4IKRF-HPi8nj znc@6ohVzpd&QE3|35{~+r`-7|cYey9pK|A?Ud7Y7^V3~X&2XEJMdzHyJ1f7I)x@e< zrN_&?vo>jRb$Rl`<;tQ>+tvzftjdz^uqNSq;8o2%~>s_+^Q%_>f3pFS$DtVJNufJN7aP{O65|= zO0-Fx>fmklg)~YXl?P()@7(!m*vZ`};Yzu$W~>fSrH@xjQO(ufdb~QF>+~HhqWKsr zy;Gg()q#Ensut<(04#{A9NjH*`m_jAf9S$+eXavjI+i?);?Q z{)?Y0G%x?fZ|U~Gskf(Jhi1z&`WErSQMnXFOka^x{cFuV59-)fde$p@d*!j- zj_#@JBw~83a{qNv!z3+}G)&thfjq$irq3vn!X&P%_f%CFIp^-nW73Z3606bw#p+ct+N%YMeh9MV{wO)j=m$d<<3vJ z^OK4~+qQdDL1LjO-Br1IWh^AQ^V7)APv6#Sh0&d#Cg?%Gou8ae%C7%k zr-zvKg<7U5&mGPA$#Zdj@-{|uesVsXpPcV6>-;n&cYX@F^AiH7X)4tIb@U z!=Y)+`@-&Hm#Sls*El~rA?yxa@zpyhu5qH?t0BEwq%nM_rQ8dLw6N@P*>ni~AQ}QTwezp}wW~w+%hv%ME7~+ne4J zyT{@y^Hp8&G*EEb+J(cmHbLv6i=MZ3%7T*zHgkI1%q>mL;q%2uwV9o2w-oB@bLXeG z=FU$lM8B&)AbhT#;@zZTHXs+tou8;-x${%*{4|09>dAQZoiU2Q$pWZ-Rh|Bj0BX|E z`6CLT_Eer75J0^=W*aJi;+Cm*iJ_ntZwO0E?@?dbhQdRI-1#YYernmShkWk*lsi9h zofzcfahO8(pth<-u!XQsY=JJ8*!PgwO@6yFRO)XvY5*LWH;8{7d^`UfWB>&E@&2_^^SWc zJXH8#X?^KE^)D3PI1bCCs^aKvvyVFVnDFLIW6EnQi4mHc!c{^09}X>+ieFcM+7-2W zj8OmM!cGnKVRFOj!u=XR{~KfL!*9p#QT*t5`NhWN3spP7vHOPgg;Mdb!iv%x!ULsy z!!=~|aoE{p-9u__}_LWfw+pmiWUNgD#lMcb3OIXE0 z?b7;=DjcbU+!Dpc{?0H{EiJ3ke1?u{X}KePKSBc$w6v^F-_sCK;I+f?1yKTpfxFk< zHdyDBJ3mzq%g~V{UZx{SVB|wKcpjd;x%1QhIyS$PGt`kv^-Qw+ZAuGD8KbO5>G?|K z?A(1(_Ps}`h7sw;%DGPIjY<(#aTT1P^c_lDl+IDQlhW@feT~vLsPQ*R>Ce>Y^Z#EuKYdikz(OmgEVME^R|ns$H=DBali7z8yD{ZM zO66tOB=%5Z8xliG<$NzB)}V7{hVzrRft1SZwTa>UWQOyT^C6`&!}-bV%*1ehGQ;`F z`MMM1LTQG)%K7NZVTQcQ4Cf~^oS)1PRhc2GGDAvbhVzpdQYtf?pUjX_naxWKDU}({ zPtJ#w$_(cxGo(~zI6s*or82|$$?R2$;rwKVl*;*VelkN!Wrp*U8B!`UoS)2)QkmiW zWQLT=4Cf~^q*P`&Kbaw=GQ;`F3@Mcv&QE4YsmySGGDAvbhVzpdQYtf?pUjX_nc@6o zhLp+-=O;6yRAx9onIWYzJ3TQ(Rc45)oDWfz8KNpPoS)2)SDD?H81gDJL{-j*sLBlI zCo`O%%*u)3{A7ld%K1)83@Me_(!>x|nXOC==O;7dRZKMooS)2aelq)AVj8j}V>mxK zAI?wC*P7U*#M%=pCx-Krm&N(X%g#>>=O;6qpPa8dF`S>waDH+=oS)2aelo-P$qdPr z8O~2;I6s-G;hM4LY|740&WH1p^WpqthUCf&=O;5HS7tarnVpds&QE4IKRF-HPi8nj znc@6ohVzrzlZic*SV{K{=R59cQ@i}RBi z&QE4IKbdh)Gec-)hVzpdcRDkipUiN6GQ;`F4Cg1a>l6D(VmLoJ-xm_Y`N<6DCohZh zlNpjLvpU^N%^I^QJ3pBrxpKb4Q@$e;Ltf>4rzeKnb{%9XX7 z^-aA;m$a@|TCS+`^i{26)DG;B@5qvqG*RyPYV#6em&&r%O)>8yzighcq;;#Nd)77g z^efe@iR4()O5g0>WpOm|w#vh481W^o6Ihjccxx!}C9NTTIi_`??F;4$J*U<1%T7|B z(ue1?*^u4!UxHI1Rkl=jMfN+t=d7Sg2qjc>WwJI4NLcb00N zvc%n7(%P+WtoYBha~PKBXS2gQv~S<{-mq`&m4U;W9qGUL?)I!(bij+;^m$lV(}12HDMne3Ur4x;XX}K z;A;YXZ)nKZgn!i(_GdNW5lzjDQxxZ?^|wro(Zj-_x|9ukzNTC+*Vj)gU7`x;q;WxLr3tMzGUGDf@UV^s}{duZiSi z(q#K-WFZV6`+QG52We}0iSq3k^WHQlug5x9rOw7L?~P-s^Gd>>1J$~%u6JOp^Vi3j zSH`>pW1YW#q_NJcJ)YbHP^v`JsmPa zQgxA&hAr}(zOy%3=d`pp^M~cru8z_zjge}P(A;@(iQ5bF!`x6?I5n+l;jlGvMIYAX zth#d-4$I25usWnyA-Jo<>;(g>?jDw(ODTQlTtEgr224A%bl4ITwM0B(3e+SY(GYcA zQ(rf`YwlSKV||&a*?y8fyZDOmc0KY6pc=P{rCK>@;CTTK$lT87=XVaK9kC zmL4Ie@7W{d^gVlooW5s|kkj{}kC2Ds>=AOpLuyKVgsfSV`lAYM9&jsV9a6f)h%tk09L}eGar1WdE9%X_CyabZ3FzcmCc^tK?(swIW=-7R!(kV(mrBvf?baTzW zMCrrX_kFd#_vwz|$AnBZh6Rb8l}$BHz(v-gnov@WTLnqI3(s;e4@!y34nxk@uPdFBk3*{EsZeod}7o}$UM zo}8q~)t=16k(b@Wj^2&U-a<{p*75x6wrP%@q(w&Fcx+v3?j|j!jA`rDM(m5&{7P1# z16mVaqo2%StkHjNo;|9Gj$zF+sX+0x(5w5 zqnTV7I=?!mx|e20hR#1CEuPxI5QZc=qkI_H1h##Z%qJ#E+)I6eQx232*{ z|=?2BC-1udoVHf+1vPaVhsOi z#yd-Im!Tfbj!%p?kY?S9-ImzDCH81y-%hMih1koE&!!r~n-Zgu;qA^%Y(ZilOzcC6 zeL1na68mOi-%6}bN737D%%&Q{w8Rcf?5DaOSk$N*W|V3$?Fl0pTwg+ z{iOMAeKWSjQ5)JT&&3fr)Pm5E4ZG=i6fJ<`%Y98Rs}9wmM{IGVhPKMrqBpPhfZIL92F&o&B25l%;w59UmdndM6ZgA7aV{^=0{8VAl`z!v{?OGkjqFm{0UD&SQ z5!bDm{DR8QCvRz9vN-fHxE{{ozIxebTT+ zs4fFXV#rgG>LT-pEkdo`UVh4HprWpcx&17ckbANI^e9J`2!7VMaM;?oT(P!^s&(5- zPcIyndxmny$7eD3dFNb^q^E=~Y~M~-wQ~9TbUIGm!`4E+kp-;Py<9qM39d||JP}+m zEGyTyVU`l?8-0@wido-M{F~6KV(a1;f_vqXXIIUi8}s~I;ZV75;TW??-+j#XDGsc5 zK28-Jau%WK5|o`)P%m99?kK!9+#T~QNOgZ~T&0e4I3+Ig+fkh=^t4c>Uu0^Qds=l@ z2doC(9+y8PZW9w4-W#zei^4>)w=>*xc&9Fu{GFgb8sGy5H7=a}GqLPPkL&f_&&vF$ zabx=KM~$1(cdk}`?C1Pkky=`|rui4BM7P9ChvyG@6v>Ytg8e9ROv1yi27cZ*_)#R) z(a-;e)aK^wc^|b@k0KYwXOWYXm)bbOvq-9ucVtMd!zX|`(WMg3Qt@%GN^46CWuICC zwLqIkjFe^P9b6ljLzx}<2_{8!&=btqDi5(;exVLx^}{`ngQ_kG;miz9tPR~0HT_DZ zla;EcRX4YqS1FyLRQ;{GuU5)@mVK>KipF%MeM;5$s+-%;kxIFjo}l#V>^trFpQ_xr z9nVx_=t- z#LiCa+Qe>1?7qYvNbGxw{U9;MIPt#VisfxgPwb$?PDyNDVwWfOzQnFi>?4VNKCv$) zwl1-U68mmqk0(a2WAES7iM8k(Y?NU;o46ub-(U?f8o+$6sc|UuMK#X2f4+#9wA>5+nXHBmQ!}#}j)ZG2$=hBmQze z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9Ae zFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipV zGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9Ae zFS9)U%Hyy9UhltlQ(gG$#$QwN_g}p3$m1^_U+3??^7miyMbfTn6VBD22ExtXf93DL z{ujRg`ng)tj=z|4{AEV`Wk&pEM*L+){AIQ#G2$;X;xFfWJh3MdBmQze;xFeT{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&n zBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY z{xUl|G2$;X;xFeT{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}GRxzy zJpTIc6@R@Xiaf93DL^7u>r#q#(|ZHm1A;W2sqwO$Qkzt771AF59L@9cm0lv>h` zznF6TWk&pEM*L+){AEV`Wws_U;x9AeFXwwau_qEE{&GIzFXtovG9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&n zBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}GCMml;x9Ae zFXtovG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&n%j2&+{`&6~f9WbX=-Qx?~K2;swM6Ciz&xnX2f4+ z#9wB_UuMK#W@{27{xT!}a=ynCdm=I7FXtovaz5fOGvY5Z;x9AeFEipVGvY5Z;x9Ae zFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipV zGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9Ae zFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY6^JpRh#um4{0*B+`1f9?1yfB%)g z|H|Kgwa2jr^7mi)`>&Qf{>tO8@U&Xej=z|4{AEV`Wk&pEM*L+){AIQ#G2$;X;xFfW zJh3MdBmQze;xFeT{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&n zBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY z{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}GRxzyJpTIc6@R^4b>XiYf93iw-Trd@SKj|HkH7Nq5A*R4FPeMa zIkOk$@mKhTTGEccm~#AOM*L+){AEV`Wk&pEwk9#+FEipV=X*S{ClVw6az5fO=Og|y zBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY z{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!} zG9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&n zBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOcY{xT!}G9&&nBmOeW z2uN{Bo@4xc+D}Vo$zyHd|pU=miAM6+VzkB@oU#cbT_=_pWUuMK# zX2f4+#9wB_UuJ6(BmOcY{&K#@6MG^t;xFeT{&GIzFEipVGvY5Z;x9AeFEipVGvY5Z z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9Ae zFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipV zGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z z;x9AeFEipVGvY5Z;x9AeFEipVGvY5Z;x9AeFEipVGpxUc*?vt@!I9mU?7O-(g%{Ty zN-Mv(uC4Oevv;_h8@2#!$L_EWUoKZ`UakMR?n4*+4DSL|b3m zp4Z9vt8~{FW@x?)zhp#JUh{a7QEBBcCRZ`#>a1+iRNSPNTctT|)7M?{j~I|;E7?|B zqfM|g*R}Uv)w-H2&pzK?*(ztU+grD?rRfj1H+MeW(RW~7+w?~?`FzLJ)$LjxTbjCV z%MX;ubt`7AthtVbS6A-8uI<_9rpp#f>tWgV6tZPyj3tM%we8c_wKaFHJGgJ>Li_Xw zG`V5&^OHA}`+Quo-!KNdvoYF*O1Y+TzqS#NgmzE3Nw~eTzP)l+-(mUI&H8>Drfa=7#9!Q|<+QfS+PFvZ*3u}}2u*T`nHupU}LA!KSJZWu}dz)_~&sNQhU+(tzU&ZftHQ#o>{MgFZ;@-6@ zzvMn`ON(S^0>?M9ENADOwff#EE3u85C6BRAZTJq>l`S^sxv~txR%&L)Q-td z?q~gRRrh!5Fs;{QvJ2aDM)ZK|`T7|7*{o~sS)u%RifvY=_R8JlTbiLw*>c_bt{iLi zwX^rs)(bmj0JpGX>blO}OIqh+4$JbZW9H18c2)c$C$*|O?6nbG(mJEPx2tuUmYJeT zAy>3-XjKy{4|AZkb(g{?YZpIXxU}~3T10GFEax3MO_ep2GWJ);tMa^#!1MZkt-fo) z)kYn%I~6_xAmc9jhk>m{)UsIp95SGr64@a(tn$GfWa zDy;*1yx%MBJA50qc>IvkBQqyI+I-uCGb`K5%Ed5bT()1{!yNu; z9KxzqUVH0AowRuP*daDlUi?%yyB_y*F`jw;y)!SnXNG=F^vvt477KQD6$@$MO-fZ- zICzyO2ixUw5TadPG{h^dKB}*231vWjc{^WGp0hsNcX`kCvx+70K;omfZ7sHX<dmlWUgaA)d;SS?)>e0|v=+BK4r45WGR&(ytU{8x6Kmfno!I5gxiy|}eiOy9T)DSg zS`sS+KiOrl6BA>0fQ`iN$YOR`~36n)) zrrm0Pej)!(dlP!IaKZG>C=6*a`)7A zz8k4#H820UW~KU))hg8~s>UYGQk9uGXA4C!ew+MAx%3qCD8W>qtEv@9HE`SNs?nrM zfaW*rdZzl*+);V3T6tbttvsFsS}0<3y@mtUq{5QX*V`ymdS$DOl92 zKU#m-ooX4W{C4v5UFUGReEa=Fv!7u4AsK!uxs`aT&7f{aVdd+LE zyr2@tU79Ow+qOqIY4hgIt6u-Kn$?<7Su=eDm+Q^VeJ91s^^h_YO>(ocb4Xv)wCa(b zIr;1H+WfnLYxC3P((S$`tCDkdn>qO_z7TKHm+V5kDY})~9(~yU>~MKn)_S3S9*dAw#S%r?E?;)<3@tx%@lOkjDynaBW!hy*o31q+&gvngJT))5 zoP{V;%)!63sk>Uls@LNs)utXr>quFc#w)N&Uc3aCR_k(&Noc>;ke$WM$CP%7DoXXr z96e0$u3nWXckwDy{b{MFePy(wR*J2f9Vu+pUqt=QF498^om&9g?yVlqj?(iwNED@N zLCnhDmVeXLx^3}uh34gBeygd<+AXz8BRE>KDx0(R!8$*}8(^#Ll?ONjo8x3^x^ue|^~GHG0WHpVuK0g)yF<%g ztE~()KVn^t7X@xXT6KuG2px`*?hM>lDT1t{LwAPu%DuEXr~tY%+}*Zqt$xvJeZ#PN zx@@g|`)cl7!`wcl~a=7-a^pz*GnDIe^%$_CaJZ!XQZeM86UTCIwf{CMS6 zWt~v1xUtfz-7!KdKDIb0;c8@LZTR?Y@*^FU=ZD=rhSk#bT8djp^?v61n*6U*d*nql z<|%F6+WBgI-K0%a8zURpNjK())K>?3)LSYjg^(jc#bpO|L*Xn`?w-6xN0&LxOL&A= zt-_km26bCGyzfAauW9G+tag6pi#@dBa)7H>c;iB$ma}-PP87D!DF}z22!Ur|M1-Zy>L& z-at5tTlODx1F2Tyw&@Qz9vGocRv!LKR7tABK$UD)fHOtMT`dsZKUHB=-m-R^+iBUg zs#7K64xKQbD(uP{^Z;Q3sV@FOX+iV3=1>M%zWP+5`sks#=ZjBmqmIZ`e3J5S=PLdl zEp(=?QZ?aV&Aup3y;A?;##j?h(S^;P zIp41us&gM{tj>Mo*y`L%o2ql)GOjxJ^P2LqKOSG*#{Ltk`F3xq=3AmE=etDycL5?`+qfjH~*|gahPOeneqrsj2o|HJ@mh zRs6@Mx7JiG`zEni{Map=S~7x7I_A)9wqEBog#R2nG5p`LugLcHkjQEpz8jVlURL^G zv8!%*VV^Ot4IE|6T62q{wvC1J z3m28%U)ZJgpNjtGLq1wfHENgk`9C6Drjm$Ybz=>D3k`J?Qjnw5(H}Mby}?rA`+;VP%rtv&wBS+ zGjlQn*!EXG|8`G4Cu{F#uYEcD?7j9{`@7zEeSGAHioYE7c;V@W?-Y7>yD5!MUas23 z9oeCr^%f50sB|c^2jpXL`Ay2_zuO9Thr{Dp7-AImH)n@M7$bzaO&ZXZqtV(&+!X#gEGq67#+PmlT#Tn_+3L8L4~$-0xF$Sa^C#ib+C{}Nqb@8S zx!V`gdFiOy+m>u^Lyzv>S(SUMat}Q&`~KO9yEh)gIZaLBy`@u2pDH%io>)A;?vi4m z;qSsX8h%*(`tDc6`}O_W(#7$Hs!PYVqH1q_%2PF_du2DDi@TZEG&USkI-~UIVomLw z;zzWbm+S8jPd5Cx_~7o{X*b8l-5i{D(^a*bZQ9M4R4gs)xNyEoyoK}P`s0-M<(U7- zwEmi^^|xmy#lQdJix#ze8%yIBC~W>Du5oYL!ny%lpb*`mJpS80=F(EB^mOelHJ7On z?H_`^+&=4+ljHWdoF?gkLD60rlHNDP9>p((d4VFMS=ZsZc!uVT-ZQKT_vkFO)ZA0^ zLfvR>lQHQF;h9*vjg18?af|+adAR#j3bHGOTq}c0s+d`_ynV@n1xx2ITN;mm+xQec z^r&(-1&?UY&pS#FJ=Wy%^$%a3ZlLsRRS4JWG@uZ(!#(LR5%e=7_@?;<9jGY5%akrY_uB^JA6Or;e-?4|lU_m^$Y0aQA?-u{>>`Gyizp-pKg9 zIe`bng7@3ScZSP~+@d_rI$~knR`-r@Q1SBO5v8X_oLO|?-hVuA4*xcq?>I)k#G3d^ z{9)YYSk13cl|zsBkt4&YBWpujbFb!NfdCGR+dn-2+=Yu{(NB5(P^5PiMum~#o#7AS znOw}7i(B1&z&BYOzX?aOMCF`JTv{;)(6h-Vcd;KeN;vrj$6b+q6di9+wZ; zBR}=SaqZ^OBg0!p95|x8X3MC*E`GZ4tPj31OB>i^}rFS@9J2usCHb#*A(Hty0GR zqO83m78FD^hhDxsF0UKByt1Pu3);OOZjVFbes&x1UE={ST-LF0ah8oz_q{ROJ5t3@ z_f@=%IQ6UJ9kOZUg<*Bgm=VVmsRps>!(+MlzM4ZqS9nw5jCj{^>iu*4VAO>0iBVh} z+-1J&My5(^rM#>y>{t9;<5k6Dn>NK|YjhlyW&axLONZCos&e$PVx^K9!tzuatdCEc zCr7+3d~yW0#vVGRe~p(yanxa9-{P|3LE%fqg{8Y|Uks0oyf}QP&ZQ&Acv!qRZ>^sb zT8g(9uT}GPeR1cAvx*njO(=2`?4djp;=@0TI3j#sTbmUgE&edvUD~NK|EOl3DV5{A zFC18!JnD_bp2ok3-(wFgzhS`g@rAL)?=+sG63fq?%6e@zx9GcFRhry1M%T;nTypX6 z44vG@y1g4~-q=_-vKUU#_1wR3wptdAHC^8whCewZZ&>;0ki2GPFZ_+WY=7m=jdgWH zuF|+ty;M$U-gCezC+d(&E03uyHjZd$Y;J7Q6Z^47-E=*GPo zI2Ugna4zDR(pkE4MeS~-)i2l9HrB4oj9dEOW8OavhpN3XgWD&*V>`ZXnz&i75XD}N~tM* zjx!pL9Wx{RMfg~;E^I0;C_XZxF8sE3YGHZZ@x|xs>%zAiPAnXymkO8czP@;1b6xnB zJ}1WYD|NV!?LML~GhABuaCmod|FEDqsrcENIpN$9qr+t*riD+`9vvD-E(qTpc}6&* z?!dw)N1YkYs((-6mWKVqHyh6gXYO`h*mw6?`uUD(?$j~Oi^p{JnErw)j^EBzK>O*~ zSj*m8XwV9mgs;Y5ojUCOk$)l1FH{BZ9r1}(A2-YM?)cPi_I<5> z!BgXNwE8*uvD)@x?Wm9Ic{xdKh{|^Q3$)YYKK4&n#no|p4~CP%vGJ<4NWTFspDvoL`~6dER4bbftM-uQfMQ>+$mAR8ebgUYF+6h^XM3 ztMT%%r|^b^2fboq!?$S>uZ?CET4UBbG|%gyS%sqpzBnQ4CN1K%juyqNy_)Bh&8))F z?TaqDVA;aX3l{3B!wxrVkq>v!>leC8Z()%NX4aB@>%s3*OSZXrt0=ELpce^G3FAcj zM13%Gmt2tAxXoI!EiL)*!gH3YBHG-%oejrR+W*RcjqFH#;H!W)YYDYyuQ6bWYSt{P z8*&-d__Lco8n25Haou8=)%=p?%dvSM7X;q(Zr9{7`W&9|g(E{3?@|{j-COArrTZy` z^L`b6NXBWs!H>x1X^pYGtaPzb?#_=WyV{O=agQe zlorxQm2$k7DQ(tw>{2>j>E%lISGr2+dz45d~PANU6nWqWE{2wX3Rw?be z8a6q?G2;ElR2GZcuuQ(p#0@s`Ss5?yI`VSCn3* z^s7p5RNaU9e^C0A(mtinDDBjR^E0JCRQiI_mz2J!bhE1Zqv~{C^}tji+I55;7|btH zO8(2T`J0sPBmP;XElM9%`ZlH0^gv;rmmcJQyYBiUG=HelBbCy;nWpqarL9VzQ+kHd zHWfi1RJumRAM@W*dZE(oN|(WPeek_`C2)=Ai3faK>2ulqB>g^rtMBZ7^-R@=)rqai zrs~6n#6FYQgNbcU?CHd|Cia`ef_??>gZshz9i7<3#3m&+C9%U3J2SEN#Ax$-dpsPx zkINEUmDpz!`$A&hOza;M`*C7FNlcIPY#;U6RDGadz4!Z$#6FPNDTysiY*AwLruVux zCf1wSgNgk-u^oxguiop@uioExQeu-6J1wy@6I-6xC5io2VxLcpe)Zl5{p!6BTx`vL zo>*f|h0(*_`QDw_l*EX(d0qO|JKws*ZbCGyx+NrotxP5#4btfFBAJ*V)rEWcZofk*yD-)BC(ef8=>;T$2TgQstBo!F-m`?JKpkl4n=o=EJe#D0(%C7jRI3yJ-EVoj>o_!U`NY1M*z<|KnAmRvEB6C^m%E{rs~7n6WcE_;_%ML6WQl#Mq#=u&Zg?aI}+CidIJ3TpoO_;$;t z8Uk;<&EAsO-ifs)Ha)RZ5~Dfkbyp^KMPk<`c4J~cQe)EJjov0+_cw{DQIs)t(aQB@ zxxP%DCv$xnJq&YwnR@@K{WqXLYOXI+XUE%gPNF8r{ZG|zd7DV=AgpkD1t=AX2U`H$FzR^)+^}aSGRO2 zf37ckee`9|>uv1t`m%9~SJedcWzMGv^pN^8=NnF6=B2~w%e)kQnfEcAzRda1mpR|- zq%Rv}d7A6Xbjv@d+c$(&rL~xxOsdmkm&Rb?El^VZ1+beVI-}t}n~=Wx2kL@*oP3a(!8@FU$31 zxxP&0L9Q=TZ?B4|G}o8q`m$VKmg~!MeOazAdv$%;CdERmFJsF3GP5;$u$pbirW(R$ z5_>Q)dOtZI`ZBLeXwVFOnHl;rGxTL<=*!F|CWgMu41JmNp)WH-UuK5B%nW^*8Tv9a z^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmL zX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%uY)TeVN(v#L$XSFUuK5B%=yrlnV~N;LtkcwzRV1LnHl;rGxTL<=*!H| zmzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qya zp)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2 zGeciyhQ7=UeVN(VY|8pFGxTN7hrY}VeVG~hGP9+Lp)WH-U*>$*Cx*Vv41JmN-I*Br zGBfmL&i7Pe=*!H|mpLE$GBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qya zp)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2 zGeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WJr zO{y(3^krt~%goT1nV~N;LtkcwzRc|S#L$0WoGEh%+Qyap)WH- zUuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2Geciy zhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WH-UuK5B z%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=U zeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41JkdS;o?JjB+u$EOpH|Gy)S^Pc5(f={i*c zyMGdg9&77*vy{w z(SCnM_nD(-be}(NM)%SQGrBLCIHUWAt!F>0W0i$r2=i4RtO;+`QF0~JL_1D&h&ACQ zu?2C8U-w0QsAV|na>-F98Yf=)22I4j|GXpik;&t%5HQxUGo-h=Z=A=wWZwwB zriYj9g9@SA`+BdKR~NV}wfK8zKwkS@T6ndbW4_-?g@R`aYoASb) zGVQ7CbxLIX=e=?la-v?%TbkQpn7%)iO_$kf{PU*!G>x@s<`vzUj$yWQ>3?V3rYW{P zgh6y=+oxBp!cE-&?FYFKRaTi(waO07^G)#~l5cc6nvSY`S~!g9GUxR;EnN`T*coPr zkEE@1R&9kMt;$Tfa$qZ}vQi{eEVBxs-`%K^zpE-gB}%jh7B7SW-bU$Q)>N%AL2Jab zrLV5n>NaL-Up9YHXS{%CXz|%`mCFlz$Ulo)dV?MWnAr1KuR8_LtaUNZF8|3h3;(DM zTChx?is#jBY2Kb!ccgiHUTsSA_Ppv%^MgFEHdkYNUTsNu(526TY}KNHo>zUEAHwsh zNm;jPQRaEo8J(@R3k+j=#r5OwH|TO>k6bdDy;U@};>RD_*oxoUOOmk_Kc~0b|I=e^ z#gAq_@F&Kg>I5B0khwogF{lbJY2tKP_D8wPH>uNSX2xWv*F|Y}x6+f8{;kp#N}o`= zukvqF`lQkyE5*6#-;}_+srs--VsB3Dh{ReGyCAX7#6FeSpCxu@Vt+6lTT^3iQq z>#9)uCLi5);J{efP>QMPNb7Z*9w@o{K<++}k8YcfZmZ^5KDuo_x~(3Zl@UiZ6yZ9} z=A+x@quXv)MIjkrDpT?gM7H518RTV21mE z8SVpSxDS}&K43=Ac{AJx%y1tt!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{R zGu#Kva33(keZUO&0W;hO%y1tt!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{R zGu#Kva33%mn@!n$zzp{R=fi!#4EF&u+y~5-CWiZf8SVqlcYR{G518RT;Cy!`hWmgS z?gP&ERARUfnBhL)e7Fyo;XYu7`+ynl17^4nnBhKPhWmgS?gM7H518RTV21mE8SVpS zxDS}&K46CXfEn%sX1EWS;XYu7`+ynl17^4nnBhKPhWmgS?gM7H518RTV21mE8SVpS zxDS}&K46CXfEn%sX1EWS;XYu7`+ynl17^4nnBhKPhWmgS?gM7H518RTV21mE8SVpS zxDS}^CQZK??gM7H518RTV21mE8SVpSxDS{epBU}~X1EVHUr%DV518RT;C#3bnBhKP z_IP5?Cx-ig^Wi?=e7FyoF}kf8?gM7H518RTV21mE8SVpSxDS|3%ceZKtr_kE&NnCJ z!+pRE_W|d_eZUO&0W;hO%y1tt!+pT4H!<7?%y1uYKHLY)a33(keZUO&0W;hO%y1tt z!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO&0W;hO%y1tt z!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO&0W;hO%y1tt z!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZVZ&m*x7hTwf-e zh)Z*QS!HC-KDGby(QVc3xl2t+4P#f%N4M>eqHYd7!FzQ|@NJ1+FL%V53*_}8l8TTAn~zDx!E##~=^z2UfoFGXMGeGI2Bb3XKC z&i6X$%kt4}L-NzD(yJ*O!Iw<@&N*U)H=V774k&Otq-P)ELS2Wi=~v zeOag-Sz7gODW^sh(b~N;9IxIh{>$}c{paFj^=!%YWsRqg)s=G5x%ud}xCk79J`AV8n`|mum60Tf6MNnxNpz)ok0inCZZG#p66@0Zpphg9 zggRS>J(9$Rgm)Q9;x;WBIFbaf`iB@vf@9vKMcGIaaUh9a%?~q>#AYq>aSR$r;tkr$ z78a>X{H#JYNQVac(0~@A+3*tW7cE@8Y)TwtqEB?_K_<4PC25d}?QA%n(!qmF>_~jz zAQNxaPSMkNjR8yAI|9qKi=8RAC1>V|KJoaX}%mU(YPRN*L&gZnmk6I!!y2c zWawhVokdFbR;vE|UHd78ACU1wGCnoqM`ZKUv*l%_i0WoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;r zGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0 zWoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL< z=*!H|mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh z%+Qyap)WH-UuK5B%nW^*8TvA_Ce>@q#%5F2mzkk2b3XKCX0sC0^C)9W6GLC-eCW%( z?)8bGFEiVi*qw=?FY~(S%bf42#L$0 zWoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL< z=*!H|mzkk2GeciywwqL2X6Vbz(3hE^FEc}5W`@4Z41Jl|@rj`?GeckId_9SwFEc}5 z=6vYO%+QyaJ)YR}iJ>oZKJ;bIhrY}VeVG~hGBfmLX6Vbz(3hE^FEc}5W;QLGvcAj= zeVOykN%_#1nV~OpKJ;Z~=*!H|mzkk2Geciy)|(jmGBfmL&WFCt41Jjy`Z6>0WoGEh z%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H| zmzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGn2FhgHv zhQ7=UeVG~hGBfmLW;7?w(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%<|D~H>c<)hnPxTL*viC%uhcUK*1;^d>-`qJ$0haa8qJmOzg{vF}khSy+1KVw>4vQ zTW{|tiTy0GS`}>0$LO}sw?|@(ZfnNqw$8`swq_S3)|nWi+dAK$CC2EsW{hs@bs62( z><5W4x~&x~&&unxAvAs`Q%qv`5nfX^lq2f<94nZ9gh8Z#UQOQvZ0_c)Hf(rO>q`K}2EYX6+g z8vP65MSWh<#M$8(wL|*qNk&Dh*ylmmc=PX;eWN~}vIrG{NYCpVcY!Sj|YMNw!?<#y= zBV+ThB)(Ayv1PVKxlv^Do? zewXaawbkj(v$WO8RoS@$;%M$l)uFll;`syWMqXt~)hhgMd|`rFuha*c-~ zhT3$wYSVq1Z_p|PzqCqcI_%k18M(sPHc_#H9@C31x?tJD&I=aC@TL#@^r}_1XRG*k z8Wb(7oSZpTtL)JHf3+UugXyf=GPn0`>DfVC!vZ-oe&8(!7JMo6@|4t-WbJUFUDqW;a)32V1u!Jm|XRK(=bpz+h{i=7$Kj z{{ORLQw+SPc}ZWXtWV{SGLLIqFR8=f8T?H{=%O6_t{hiX!E1lRFx~OA}RZ1Dj z{3OkE3wttRDIZx*k=;!P3*zM`V#w5Vh!4t_tBJ1)d!B-?8w9xCH9fT zZcOZ!#5O1P^~7FG>=%i(=rZ;`CT3Ih;k3lgOzg76RweeO#Qr9+t%-d%vF(WwvGVyN zI_3SgWK;Fw#KcZbtWI?V=WEQS>O<~TlzSEZTz5`u_x(*9A3?%@b6I*(it_G~7P-=?DC-yjGK1*w%el z%Lbv=?nh>J-`lc9lk8B>*J#Ty{fgeEE1s@XhlNL*u2`r1v8Jbm(TqXz{|xnI>vd^f zwLEUoIGE;4c?on*Klo9U@g3A4S-&@C9@1b%OK*-#bFZT0v(l$?I=~y_(AImf9HBye zdby`@YQ-KS+G5BaBY6?bYpK{`T+lL+*RvD!qixg1^dnv!E8^uLD#( z-O{DgKV#*C(@0jmP`Ie4a#_sm{)~Mhv~r(T(ABa=nYy1l;Mu0ttKwE=Bq1UV^9eoU zD;nlTX_#lCVSWP|=Iq$}mCQG2MM;ro@7#>kxme!F$&W7E4In65HsdQW+$ z$YX6if3&krh04ujI`poVP0HPK%+5*?(lk>p2SRNsKe#g5)~=7ra;?Edb=c;no}(0) z)(=U2fr6!{Mzaqb{OB#K{$;Py{j2)=ny%qR2AAiCnBEXCs5mCAFS;4sU!O7eSuVvs zWtefm6XntiILmMa<1uqXysqUjGoyQF?o~8}Upx0ID$C+vL%dv`#(|>yH(V#N%$xgS zxpY_VRdi{~Y)b4{0?T?L7Qv<7!Il%;(Jq!t)Ju|^#N5qto}f06Imq$&iKWb?6@Q6} zvBcMhv&E`7S+I#s`kg9u3?6aQnCXlA)(a$NXilQ%w04onyKLE35?9ThzN~>FFw6~SM9qqN^hJn5iGRq?7fOWH-FYi~UeLU>}#MWy|U zU#Y#W=In-3NA58wmbF)5KgqkE4tx$yU6jv4jU z7%xa{6BtlJ+0`@>xWS{-o} zVtyZrvADOz!(#H^^a3^aDk2iNA^SmIElziv@(mNGJ5awvS&M(EH`w(!OgXB0m^@)Ci%bDI=NE1h0Czivyzm8HrdoLTu{kcK$KvLeHN7=VEiEjaTldcmm#59msoLCTZEiH1Q@Diyq&kxw{4VL)TANSi zJE&i{0#GAv3V$6I7559{OScA$vrf~p)p0K$7`?V|O?bTKPr{|Oi;81LU06JFw=bmg z(owayE!o~CWP7u4ukuwpt8#BuZlZk-yv|*6LHm*g3zp7bwlw!Ds{H-~L!gzy=1<~j zyf^&_>js=w3eg?>u<>vEm`h8g($lrK)Lf=Qw0}6ua{H`PPEIA$Bt0-fc$2o*+B{BE zzA5%7eksh8-j++Q5Gs1xJ;R!CPdKF1Qgctu3w5KlP1Y%dXJY9#HeSJ0^!xX9+Ms=X zh+Qdk7lXV+0K%T4jd1Kd9@?iillmJ%)N?q7ILp5N(dWrG}c{tYh!72<6aFI zf4p^oZ&U77wDyqNhTN;jwONTbbFZT4Qg-?gve0bfyimC~;&y-_cGng0i+^1bNlQ~He3PQ3>CnNplQ zUQqgy(ifF(uG348y3loq-qF=7#TkM5MM}wkSvG%@(tX4~tF%QaJ_m18I!!N3n14nO zGxEP(uX*H@(RHZOBbDMyFiq)+N?Vmar}PY^ZBqPyQ0W@IYGnRfN-tEpUFkBoUiI+3 zdF6AB=C>;SxYFmc`AK?oepTPu{j-^}SCQG8Y^nQan|&rRTCHZA6MH(bt%>1P-%h;)uOc(Nip=mTGQ+FL46hh>a;#K5) zPbG#|kr`e^&WBf#8D2$Zcomu9Rb+-&kr`e^W_T5u;Z*7`9e0UX^@eDV6JhA5! z!>h>o@G5dXyo${5Dl)^X$PBL{GrWq-@G3IHtH_Kxf!VZd%3eigcojL{oRkl*A~U>- zoDZ)eGrWq-@G3IHtH=zmBD3Da@G3IHtH}BADl)^X$PBL{GrWq-@G3IHtH=zmA~U>- z%h;)uOc(Nip=mTGQ+FL46h- z%h;)uOc(Nip=mTGQ+FL46h&t$w`hKo2Yng^L zVoK~tesar1^@LT&^$BUzTgzge8?ox;!et{eS&Vy^PwzQuXZL-GT-NDCsw;f=%Tn0A ztofTYuim4*@%;9-9=1gqyEST_NS{JCf~J}Gqk);CMnd-kZ9T{CyoHT)X^G@72R~Z6 zU&rvPZ%YNE9=tbb8}v&3ozkB7@2qr6Ej=MEOw=$)|5ed2Je%|~rQ5A#&<;0iMTuol z%EZY7p70RNTg0;=n0I7DF!u@N`ZD#N=BF4ujAkZgKs&chXDKR^wkem!_%6@A-9jj> zxLK&&{ZLu`x;MpKx(GnJqNHDtaw(h97@X>NpT@?lY*jh9KGIOj_tAlzi>tNta_KpZ zuFQq??4SR3r=(WCE-T%^%M#(;M#jUw*>pv`{M_3+>(|YpqUKx^52~W0&=<|-IoK%E zL{HI~eUaBH)y(x}y=_UO$vws}=H=f*e=$g(IcOyLwr&dvtk1N%4SgoX05|Zf>N7c< z$~Dsc>jPe1wj?TNxE$s_IkV>t>*4^j$M@9SFjK$$>BsjpenN_d*-{1|)+{snx?~V@ zU8I#ukE0BlRxaJo6&|lQ55b=6%fit;jZ!7$`m&zJuF54=(fh8ZK%-#e(#rK^BYwSe z=c*lrrfX)aAn)Fm)gPWcbwIHmi}3@gAH3ZCz%A4&vYNrpCsdW-i4ZliCGO))rXtD8 zlqgksI@2{$K9uJUStsa=rNY%wL5I+Y`56a%y-jrlw!L$6tQb_}v^+Q{N3vSN4oxmt z_xqG@mk!pS&&!^nuU@!4GL0{|>h`$%!bROwVu%&#kU37Cs2mMn;SQxJ(3f4BuD-$Q z^Q(5f8l^UmO2=LGjTSyWGKBhkNR6Ye>ud<^SC=^E`ZDUe`nC9-{!8@^t$#DuhtB?h zDtr{3@(|T7)Y(;kxWs@84Y520!2!jjhgf!3Xyb$Rt_ zuC@EV*25mHRMDGe-gnrxrk?9{b4p65`g_wxSGCZtQ+d~&*}C)5*6t_S8r65*zR#$R zezPWXeOWvs+_+nGIw~X1=lZgc>&sprec6kOjt#Fb8?Shk^<~aCoW9KYhSQgM>2Ufo zFGXMGeGI2Bb3XKC&i6X$%c{JkbA4HOjt5Hk$>@U$Zw$v3W|SI7{87ypMjc=LK;ua@ z6_L(Fvcx~@k=ALQA$=OHMH;EAGK3#BObCC`fLH`I+|iNMjQUas>Qhj9d*R%|mx~Bbtj3xQj(^@Y&&*1{?p-_|Tbt>N{>OT_X$-Q_ z6Knn;ZDxwMq5r*^4a$e8E7zCh`m$VKwq19_&=#AyzAV?59Z`B}#F<5Oc)KcsuoEr> zx_=>nCeK*SuTTdLjkY;*WH>bnk9sv11*c#Mk3KyA+=Yu{(NB5(P^5PiMum~#o#7AE zx0Z6eq95E{^GPn$K7aANxEH!#^~LqpO1Qg!TxUsIr%@fvj%+?)G#QTSjLKHzg5QhJ zpl>yd4R0wOQ~E~Dq`FU*-l3hHpk<7v7{VQ6_AMqesU88cuZ!}2!P zbjc2PNdDxIykVux`-jTEQk^SLXnx}^+h2KeV_n^lt2D0sD16TWt1v=vY2`7s#l{g0 zjm?csy3)$gSG@hih39rGi{)Ke#nv(UP__=a-I|pL4a3!objU*=Hw@nmzROoCt?FoO zR(I7CB#`Mp)loGyvb5^m8sU6Ik)`h(u#_Q>ORKJ`9ocyJI~ymo)z&pGomjh1sq4Ci zmWFrKwltpFwQ7v4ySbE*nN*}E_>C{)tA^#WD~MlT9b&@XBS3GBkAx1Lik^50PAxng z-X8OGsKkC>%)?{FULr2RFP1w(Wt08GOY`VaCzEiOTFyZywc_(JaOmhy#eK+6TzPr; zeL3d6F?493)#oB!S-HNfwsH4(ne1PEA>{h9c$r4Ggj`=%ebe_>{A?I;r0%%$beDgQ zn=>3cW=8mn@UdcD*i>9ld_g>UI| zVqCvchx^#>BMLLarG*cNcNg~$3yPD9pRJh_&K)s2TsC4__(bi|p>gDb@ZFJTNFi}x z;gh4z3}@B9r*KQd{^6UAXM{6%J1^|J`>a^1bn2Mq#bdg9On*VuK|#gq4PJj78((j1 zEi`C_OTt&Z-&=NGE5dIzPNik)0vcCJeQ4t*Bs!>9c@NLf zmF7J>13?q#_wWq7v-13);Td?JGzfcmh7AetGCad=S~PHYhC4Jr#PAFp^Cm6AHfUDi zC=HF#+}x{qM*Uz^`vIeVuxPUu`8WpI`!MSL78a>R#ag!Nfo>JWQU_{FopGXlqCS{` z);Y~uvMnu%BX;Z|w4DvdQ)(N9_J#8T8`+WgK&u(XyGKvsH3lqc?@$Y?xw&r0W!OZV z-TYCmYI$?9E+as^q_iAg)4}CArUvA8rN`)Vc*Ys?9kWQHHa|UEURJtToV)WQN-tM>kpONS9k29qrTZ(DlA!B7O0Q6g^TgFkPgS~3>4&m;nz_vXklSm{klbA8$H`m%fV_^`f=DeKG3*67zW+mKB)gwG`QU}BpS zdpfbLiJ>p^_JV#5Z;$)IY;0WoGEh%+Qyap)WH-UuK5B z%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=U zeVJL)$O;>qO<7;&eCW%Z4}F>0ti<#@%GlDx(3d$M`ZBM3ePZa#%r+)=XJY8fye|4O z=X)wK^krt~%bX8=nHl;rGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^ zFEc}5W`@4Z41Jjy`Z6>0WoGEh%y=nehQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z z41Jjy`Z6>0WoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkdLn^aq7 z=*!H|mzkk2GeciyhQ7=UeVN(uiJ>nuLto~6J&B<&GeckIeCW%}(3hD#p4jt=p)Yej z^kvS6zRV1LnHl;rGxTL<=*!H|mzkk2GeciyHZ7a7zRV1Lne)v_`Oue{p)Yej^krt~ z%goT1nV~N;Ltkdrn;7~sGxTN7hrY}VeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy z`Z6>0WoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;r zGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41JjyFLlk( zmzkk2GeciyhQ7=UeVG~hGBfmLW@@cxtgO*{>dG3>XLMQWnsIc}o6vfy20&e>N?`X- zT6N3nZ9$~+h@+*vPlo}FymMMQf``dU%7kpHqEKKoktBV zcWU-q_^4z0I?Ecym>=oHU}wny-L4RE?yt1!gQv38@+dG82W8Qwh>qBpbY43n<3FEQinLwEh4nAFqADvzLM8t4|85M}c&(_euij^Van`d9H*rgE(1YNp?6Y2X3PZ+H-VgCNEgv#gCop^c7~dK4;}8Af zk>A=&(u+qP5JSCqoRIi{*NghS&Tc*-{s{dq5aS54rw{mLD0g|x)#)=cvGWpJoYUr*Ud9wTXu1;)SV!8W3?mqBy-JQAn zz(DCz?mm#a52&@*rAI~XK9IW)?gNKCiCK*FrgKz}s~REsB|ShZ>hIirK#djz z-?{sMorZ3YZ|a`CQ>Fr~(QKejt^=*zo1e|y2mJaqcOQ85QI_-3ZB-Q`CJ71` z8>s$*mWf19Cdi6t8(E3|P&2MrA@zw>tdMxmoLM0aatpaPx`p5xa!1^P{TOohfeDH- z4LYg0`@rktKJb#>oDJ_j@D{yOxBGze4d*`Ke8ag9czfcG(+`+)P|KHz+> zi~GPX(Y)M!Aa@@KGBHRIwP`Bn{pX0#XFX z+j1uQbSA0-vioP3&F9o0muRJFtb!E$i@Ezi?miH8wjPSMMs5vF;gLd1?mm$8%eni2o+O9GQX_XCko0B< zseVoFKEOjIcOR(oX^=<5l`Cp@E3JOHwzjb@iu5N{i}bI2NrS-dJ+|?{{giv@yKrl` zUTzKh$gN?{lA+8RDwRR`xy@g6&cem>=3f{qlT-$OAsx$b&I-Sbef98uv$Mj#$Gm?U z4pqO@A+PG<+I<^KH7pux*xogL`eK*=xxQHLJ}@$j3hxO&jz0rk!p2LIp`i)0pG96v zx%&XmkAapWx%)skU*eV!CdpxISF03mQfWrQHKWOMt!~g$ql?cCnya|+;L`K4+V*1Y zsE^C7XHx2NKG06(^tg}x(@lSM+}?xXq;PD!>Gx6ZT*`rG)ibwN%>#T`}hx%+^e2GlCg-3O$) z-&+@t)SkKffNrsTblagvxBWXkKHOKFDZ3At@w&@wLpIeA-3QDbOl)(?hx>rnB{b-K zxDS}At&=g_2h8YW?|c-4X1EWSO-T&*0W;hOoDcT_Gu#Kva33(keZUO&0W;hO%y1tt z!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO&0W&p$GKTwr z8SVqlhx>rpc>PjlxDS}&K46CXfEn%sX1EWS;XYu7`+ynl17^4nnBhKPhWmgS?gM7H z518RTV21mE8SVpSxDS}&K46CXfEn%sX1EWS;XYv2n;7l`X1EVHAMOKYxDS}&K46CX zfEn%sX1EWS;XYu7`+yle=gn{*FvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO& z0W;hO%y1tt!+pRE_W?892h4CEFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO& z0W;hO%y1ttYf`<&Y-~1V_W?892b>T00kc_&>3Ni~rHSD_;C#3bc-`w0!+pSPV`6tE zhWmim#eKl}o=Ob&0W;hOoDcT_Gu#Kva33(keZUO&0W;hO%y1tt!+pRE_W?892h4CE zFvESo4EF&u+y~5XA27pxzzp{RGu#Kva33(keZUO&0W;hO%y1tt!+pRE_W?892h4CE zFvESo4EF&u+y~5XA27pxzzp{RGu#KvR71|#-q}<`zLS zzzp{RGu#Kv_D-xdG29275BCA*qd94|GBMl-%y1uYKHLY)a33(keZUO&0W;hO%yNBM zt}n~=W$L;*sB>zrFB{Mw_0ZOPupOy%NuBzi=#pCLNa`-B*(kgkowL$a^;Jjb>{F9e zW7n-w+m5m8npUq8pl2~%jia1ld{-Qxb5>8|>dN?aS82Gq?AS{;v}|A#9#c+cE3$WB{!|UvQi)`&Gltg zVeq)p_?hc;N^*S}kC8!=m3jH-wn4+EGHiJ1>Zq!qLbFZR3{|aZ%+RZrnJKxxEISjq zzO4J}6udiBrp!3tiRs-h@7&Dr(|wxH#-biH1~uh%t}h#0iPsIK;jGU>Os7^~rRtxl z>Efu!Y9LtqS{Zk@GFtT$O&eEdVI*w} z&glMndF6u>mA|#U_3YLUwVu^}_J_`VR?EZ3^tnKhu$pj`7GE5v-lqTmJx&qvdZRuy zfymPyaq3!4y){lIP!nwXIo|AI>tW!>oy=fK63X}A}2pWHoK9b`!Rc1EqQT&pIiDD!~ z#!;>Gk=!$^3HL~a*HUwjMlKvZ(yF=2=%-_&r17U(ySh6n`PkQo*tEJHMFpK6hU#>^ zWO@6NY*1s4psb(i6a{^AeVLyLxxOqP-Bx`=)akeO1L4sTR23McXwT0E0N@fyVee+lB}-K=cC)INa+`h=K8W_ zIv=wN`RKO!=(f5L^3iQIW{2VdUy%;)Dy3gldZViI%>RQ@X%)Kqls==hQ_YW`DMfwp zg3_0izNmDws`{hqLf0YcZ(pyJ-t5dTQcC{IviX~o?j!zLr7cS7oBlSX)6|okdBoD> zf4iD9Qa5xRs`N;uG;gLUJyB_^(&v<(p|nlCv_GhHjo#NV|1G5#D&4Mh8C-29bal@t zy+-p}m4005bJ_eP{r-Qi?`(Y;Q`VQ6t;v?whYcy;XA*lbvCWC0FY~%v6GLBSrnXm{ zst@SP%+QxHRUZi8n4vE-LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=UeVG~h zGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a z^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmL zX6Vbzh_;!bFEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WH-UuK5B%nW^*8Bsnn^krt~ z%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz z(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WH-UuH%LXNJDa41JkdlT=$~W3wsi z%goT1IUo8mvssBPNo;9i=*yfBeVNz2J~8xVW*ZZ`GcojKUKf3t^F5Uq`Z6>0WzL7b z%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!H|mzkk2GeciyhQ7=U zeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0WoGEh%+Qyap)WH-UuK5B%nW^* z8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL<=*!G@({s}deVG~hGPC^>Ltkdb z6WRG@BzAma=*ygMVPZXrp)d2g=*yfBeVG~0aI?n~dp0WzL7b%nW^*8Tv9a^krt~%glNc zLtkcwzRda1mzkk2GeciyhQ7=UeVG~hGBfmLX6Vbz(3hE^FEc}5W`@4Z41Jjy`Z6>0 zWoGEh%+Qyap)WH-UuK5B%nW^*8Tv9a^krt~%goT1nV~N;LtkcwzRV1LnHl;rGxTL< z=*!H|mzkk2GeciyhQ7>~vJFAW>|PY0%O;!sGbPU2?&o-Vw)X z>4LaMKDupo;_}gLxn#JkW)+V9Upu<(bt?D$Mu{npZfka>t^>2{vZ;n}Q(|9C?8}KU zx~7w{<>7w>4vQTQf$tHDh#JvkMZVzp@#l+dAK$ zCC2EsW{hs@e2i{u_JhP2-PVlJZJm$NZOs_n){N0@%^2O*jL~h)7~R&4(QVBb-PVlJ zZOs_n){N0@%^2O*jL~h)7~R%vL1O16#^|=r$LO}s$LO|ZjBabj=(c9OLos7?TQf$t zHJjP}-SWzx-mKq8{kbRT)XY>ra9#kmu04925M994TYm*wm^w`#f5kVKef0RXCvBav z_UJ7@ZXQi<)+9x=@U-@$aBr*C7L_mRr8L6ciFsZ}RylG>UkCu*|S zlNJJ~KRJi_Kw0YNPEABvP6ylJ5YXt%(RB4Oz-N<*gJ7o2OkXw=B|aQRl=x`D?{OS| zrPUZUxF#H^{c|>J^iS`e^?6AXXNP0d4(b1!s$6O;hj%Hx@(mT>ePAj zJC}7VU$}IB`@t&?Ztt9bVf);Ua~3b2e?j|4E;?uNvV|9)vuxp##qFKv%xgcW{qU-~ z6jg@)y41T0pV!FPl`b&_bx35n8V=cC_43nFmBSgTmme3GFVt}lTrR7{W`b6Quyoml z3+K&m??}(&iItpM$TPWD^SflHqmCcpldH0G1yuSYsY7%7#q$T$jl9Z~s#W;i_`-Fg zIiTVDK=WG9A*4W#kHD+kX;|;pp~77hSMyVdn)4 z<%e6@_UTotY|mElc^uSPxw6Wfs#SJq{=Zre@;U6R+A_CyRR>?Ln-hiij9K?AT&wG< za$qZ}R;SRYScewEvW^Sq>r!1fFXiv5%1_BM^gs}mJ*=r(V}jP$T?U~lNaG#ET{J$7 z8GUogH)tZS*ywuwpQ_jt1Mg{G(htyo|Aau9$2G3Xm0X?u_nU^$MLGCerSDPtJEfmj zIpb)}VcP-KK1+ zJ|4H(k%=uz`96}^jfvfo*yhB(p4f|t{UWgzUB=$W#B8cQoR-*`iCvc1s>HsO*xw|! zHL>p|wmmT-R_v`l5S{XVTe7M8aAIPoCRV39g7Y6BAXYdyL3 zl-5&QPg5O&yOn_|DF=Fv+S!&#ByVYxf{zQawR=Zf)5hCljbeKDi)CR#iPH~$wDniN zp5C3a)_b(aFk{sVg^RlMC3xz{m!(fhxqD{IitfAOA+?F|b<=6?{>Qch zo@`ouP0Y7Cep6~v9h%qAdUDI0^4uq9_SCF9|Gx1vb%dw4bw4t5?&hO=YIJ%YnW=M9 zdPt|`VYJfOw~b${T>2W?X-<>Ac=k0p4`qFSzNt|BGD1_jztR7dwaoRprj6_44@$~n`@}h&DL!*CQdz}shv9g{gu3D6~;QD zs*?9_$~8fodrIjkdKv4#p(}Z+=AA$PLTY%2A2cccJ2F1mT^X>xq`y+vt_Nvr^P9B6 z_PEh2iXRQ%h_x%Oz=4%L_o?=tLwI*wk6`?{5j|h{ogTZ?%DB_|*E>y**4Ab{QNuT5 zzK7y^<23!gIRBaA_2KI=4=3G!?~B{%(@g(%j*EHvD~q0AtaXlyD_x)ZYO9ivDhCt2 zLH+L4Gw z*NNv;<(sH{{nhQGhN^CNRIM^ut57FCV05Yzw`!=KARl-Bt}9*}{xKa_XVuD6vi(gH z*PFbd>d7mrR^jXRAL5Yp9`h@TaZI<~$I9bJN2$#hZ z;ms~%vn-~qJvA*2HzH|C>;5)*j`tuLT_ZtO8|xb1a8F z@7|w%X*p$bb2I1P%PC8no42R=WpRE-nx}+{a>FeDa@{n|&2>6rub(y4mqpAMAA~z1 zrVtv0m#g*H+&m%SLG5w&H&KfyV`ddvW7f$j>!7lvNm;q|DKlmj)Q)De(*&mJ`1R^_ zU^&2|a@x$G`<+n=X0u3lCTl5Ta`3x!_cb?j7jnNs`+JejNqGk>0rX?R+Ccfj;~_UK6?|C{^dEt^<@lrS#oO)kUg{ z=ij%KK9()#2iU54uBdHFdzC(`^kJnxQOd=xZd6^=CF)1GG4x$sZf*=QRUhwFGk!C( zu579ytWJy~!uf7bY*S+QCf1i24J2=m@91@Zo>+~J(TuCh`M3_u=qzZ)4{kJ1O0PRE zo2m~jiA_xG=){gq?EJ)*Bt{oOZ|}0iKAG636Z?E(Urg-!xUn~D8nVn0jlrNn-h zSW%@9N1)dmb(LeO$);Yp@l5@Iw%!C?)mh`28!I-RDJLq8XYMgIo>PliE#-p;G?aN1 z&~6H>tA;Y!V?%iy+vsm7-OlGfbJ&i@Z`sZQOwDBjctNyu9zG-oHS3KHu z4Z#{69z{WGPvfMrT9b~;k%YsJv<@%uH}^v(Ws_5-E_spGFjEP7Xw65b9dd1dt(2B=*KS3M0OOu=h%3ysppF(X%t?N1=x30B( zg=SL2x-*-ZtX!itUEQ$mh|9Bv^=!>>5z(;bsvgv^zDNtlssN}7A5l^GhB)<{riyV& z|HbIZKM&EcF6l>_pye+p_wU`X9uzI9Zdjijt9CT3Pu3@d+lw2*6GJp?%MvFKjG|mr ze0R7g{BdD?cve+q8q|F4{zhwmgZh!U-5J%TK{&xeN>)`ijP*eWwlD)b?y+dcnEt>Lx zz0oqBpw<2N>0LL|e~_~)xpa`MHZiUCw*#7wgDS8zdT2VF9J7urygfXYj^gwIo8xzx ztUR2XS8GsnzxKCQ2UUVAU$CQU)6=v{RfF1RqO&SvIotL$@z>aR=8o(?oI#c8%HecX z?S&4@(DrrzOmWkHrS{h{7IO=~daxAHf98V>hTNOq_dAc|?Pn@?&6wdRodls4SsDUDl#O4Qg)o zY5IOcG^n|$y~jai08QsPEZU_(%`eV9Jh(y4o#wpN4QlSFp&Qg)X-Rd1y1#tWD}FuW z`K=oCnWqnbuhgJEP0MLe_bR2qJwbb+LCqCKL;CwlX-IRC(~zd_P-#eiPy2F1nkhG= z%{WD7UD=cy(q^|MMnl?cYhvF`jBC)_`)OjoNi68dye`M=d@YG_X_&n`F@9Gwo>X3! z^KEu%VjB|sOkz(Y_Ecj3k=XANqd%0lM}H{qrIT`%Ev+vD(`okekrr@*;IWvF0omOeJnBJmtOa_#BNXQ-o$zn`$=LyOKhAf zrrCb8sroQEv3Dnin$G!9(|Nx=iCvr6U5RZ5$8POtOD$RYs9&v8~^`V#Ch_hgYt-Tr9n%@qrc%hd;^)v zV}HK>w&xUa|G7cJBYu|_Zys@uFF2?Y_qRv~Rcb43%8ts})OoD5y{j6?dBj=myx7DZ zSP`$Bz?-X1;2fRv;dC)>#&4)vgX;flY#`Gn=nwVg5$8POJh&yCN1UUuh&yWN1~NC0 zmsB^9SERf0^@%utU(Fyll$mlv*^Gv=SywjYhO!wAW#^-zY_=&eBF<)giTx-sp2c4G z=ZV$m44KtsQ?{ouBjW7s@$$rsh_e}QKFo+Xn{_3&IHMe|KgF25m6O%0K#N-iYX|Q#vC^Vc$oa1@$e|f|?k2wEl zBhEiilgJHerreM=qakh9l}))JZAL@d`DjR+ZB2}bv)Q)9h&Y=OarU|@^)oghn`#I| zoSkoSVnm$HW+z6(*^G#@w@1X;Y(rv1oXv-X(i=13A|? z=Ne~Ut;A4<>!$4&2fR`CpVyd^t?}BsWOrzdJmRd{XRdM9DWF+2tB`A)|F3DB`_v3_ zLzyWzl+Cyz%(}8EHd7_E zgL`)88t3`Tmg+rfb2B&4&|SDYUF^jF|0gxh|EwmF8`4a*^azomThP3n1 zkT%pio%zkqW;gf(_rq<^ zz~-Fuod0>5Gw1#LKhH5EaW*4yHY0I1BXKq(aW>=f*^I>5jKtZD#Mz9**^I>5jKtZD z#Mx|fFcN1o5@(m2rn6Hs5@$1>aW*4yHY0I1BXKq(aW*4yHY0I1yFJ*BU?k2iN8;>q zB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e z&SoUeW+cvLB+h0e&SoUeW+cvLB+h30x@caf?I};3^Tb&@u>Y=!^M2a!>len)a^jpn z?HoUU#<{9S+Uu<{xm8<|>rvjb)1TI!`WjT$erWWIX8Pn-?cA!JTeU~KK%S

ZXFU2icq9p6 z+2mGjmPu~auD_-#Pn=mJ^2E9E$~jM*pTCLoL%JmLg*0uxkT&B&+N{6Y=ET{I3u%|* zLfVYP*^I>5Y~bW|W+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUe zW+cvLB+h0e&SqRbn~^x1kvN-?IGg=(FcN1o5@(kqaW*4yHY0I1BXKsHrn6Hs5@$0K zXEPFKGZJSr5@$0KXEPFKGZJUB+k@>0M&j&pB+f2J;%r9ZY)0a2M&fKn;%r9ZY)0a2 zM&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2M&fKn z;%v6Bi{^FOp7O*wPn@*_`|p}KPt=BAzc79l6X*1VbN-C8_8`@ZyF78u6KB?j>gv#; zol3oudY(9E$@W`Qc5@c&kME51X=&}VN_PGpXBJnUIFELLoF~rD=l3{2tji!@DAVQ( zWiu|6&HAftzECz}7vXYTD4UTun~^x14F!8R7{_9lBXM>)5@$1RQJc*OM&fKn;_PxH z&SoUeW+cvL{lQ3_%}AVGj>OrF#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD#Mz9**^I>5 zjKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtZDOJ*|?XEUtYW+cvLB+h0e&SoUeW+cvL zcLgJHHY0I%ITB|x5@$0KXR|rMNSw_`oL!E@*^I>5jKtZD#Mz9**^I>5jKtZD#M$iW zU?k3FB+f2J;%r9ZY)0a2M&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2 zM&fKn;%r9ZY)0a2M&fKn;%r9ZY__k9=R9%F6X#9$l>etD&g_ip7uL^W;_M6S{2Avw zajsuJ=Fd2vx4frs`I2Szw~F=a#Ll1m8E2hWj=Z4e>~x~fiYe6#7*HYd(ztQsyy;%r9ZY)0a2M&fKn;%v4d7>TnPiL=X*IGd3; zn~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1 zkvN-i`D{kwY)0a2M&fKn;%r9ZY)0a2M&fKn;%r9ZY)0a2Hce-zW+cvLB+h0e&SoUe zW+cvLB+h0e&SoUeX15315sbvyOrF#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD z#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtY&Ul+~m zv_0jCbDlVB2ln4Jah{|NzkXr-EGN$SGtSz9)FQ?_an2LxVXH%jb}IE-26^I~C(e3l zc3MAH^ekDH9iSI}wW;$+ot$RbOR^WLyH~5s>QrQ5{vPMXOWyoF&i~)O$N4{W8RQFP z+I*pG#^zwwUv2Y+vKbf3F2{wk8HuwQiL=>Iu!n z*^I>5jKtZD#M!Jr7>TnPiL=X*IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1 zkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGb_FY)0a2M&fKn;%r9ZY)0a2M&fKn z;%s(TFcN1o5@(kqaW*4yHY4OQn-h%0*^I>5~bW|W+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvL zB+h0e&SoUeW+cvLB+h0e&SoUeW+cvL`?`3}6X!f}{vVt;Pu5PMeqsGAC(ijZ&iOOW z|1j+Gu_58ft=hR&o10GitisLNX`VQ1AuQJgK$d+dT~LoJyf9sO_ZAA-1v;_K6K5?T zuEHW(&Yy99{yyXUU0o9SLYg*TNSpB!n)O%Pd?9Vdg|y3YA#FzDY)0a2wmTS!vl)rA z%V~#GWz(x|PMlqi#M$LYoXtp_%}AWhNSw_`oXs`_BXKq(adtTpXEPFKGZJSr5@$0K zXEPFKGZJSr5@$0KXEPFKGZJSr5@$0KXEPFKGZJSr5@$0KXEPFKGZJSr5@$0mpUp^| z%}AWhNSw_`oXtp_%}AWhNSw_`oXtp_%}AWhrs?d|jKtZD#Mz9**^I>5jKtZD#Mz9* z*^I>5?Dk+gf{{489Er2bkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1 zkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN;}>!Nv`wx>LC&J$pg#C3B#x4|i%BZmQV*sodRZxM^yCTk5{P z>BHFSgPdY-iMp>sXw>$2r^jOc%%< zY5D#vJ1o0Xzmwk;ykNvKOPOY6$< zg6~QN|Gepn>|bNP`bG>)3c>U`#0yenFTLR06-)ZgU9$MR7-Z#$LHN4$SEAs=Zs6LB3F?%@@jM{AOnT)iz%!o81_U z3uUw0gONC!4F!8R7>P4|CuL6tBXKq(arUqrN6bi^&DhtQkvN+z2u9*;)*p<-*^I>5 z!;(0gkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1 zkvN-?IGd3;n~^x1amj2(;%r9ZY)0a2c6KllXEPFKmm_gDBXKqx3`XK?M&j&pB+h0e z&SrChkvN-?IJ+E)vl)rA8HuwQiL)7rvl)rA8HuwQiL=?$!AP9VNSs}c#Mz9**^I>5 zjKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD#Mz9**^I>5jKtZD z#Mz9**=%1I&(~`*r)DD#WmcTcJ}I}Jzw0jd3|SlU#YU? z+SR)bF6|jCRd#o-Uz4WVIWW1Ya%KB&jiGhZ_e!gG4y~_FZtcCTm$hhk%^tgg(!k`y z%auFY7c$C$(w?$rv8?viP5-iG_qaWqJ1b9@ieDGtTwP$2@V) zpK;c0tCcz&ah1&bW%I;Y3xX>MUHc8M!u=kdunw=nohQyO&|8T)nQFL1b`@^U-5PM7 zI6r?A=SOr&~bdtBXKrk59V?t&So2ekvN-?IJ+E)vl)rA8HuwQiL)7rvl)rA8HuwQ ziL)7rvl)rA8HuwQiL)7rvl)rA8HuwQiL)7rvl)rA8HuwQiL)7(&t@dfW+cvLB+h0e z&Sp0TBXKq(adtTpXEPFKGZJUBX*xSKBXKq(aW*4yHY0I1BXKq(aW*4yHY0I1yFJ*B zU?k2iN8;>qB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h0e&SoUe zW+cvLB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLB+h30x@caf?I};3^Tb&@u>Y=!bE`J| z`i1ecoH*ytIBN&O$$y?W=ZSM#73!-)hjuFUTLyXJoF~qsULfbsI3wlKF0^{(d!aU( zwl<=RhAZLx8R!3RpK<1b$QR1A`9j%@U)Zd_+U5&oGcJ@}jtgZo5@$0KXS1PT4+kS= za=9mikvN;Nc6nIRFf$ToGg2lq5@)jo!AP9V`h$@;o81_U#Mz9*nYKw8iL)7rvl)rA z8HuwQiL)7rvl)rA8HuwQiL)7rvl)rA8HuwQiL)7rvl)rA8HuwQiL)7rvl)rA8JEmv zB+h0e&SoUeW+cvLB+h0e&SoUeW+cvLgTY9g%}AVGj>OrF#Mx|4FcN1o5@(kqaW*4y zHY0I1BXKq(aW*4yHY0I1BXKr+Iv9zw8HuyYkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1 zkvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGd3;n~^x1kvN-?IGgS3;yF*8^Thdo zaN;~gJB9j%^|PEf=g&Cj&p7ALIM<$J_?0f6IYpa72&n@amBMA)Zqb-cueSLaXEPFKmm_gDBXKq(aW-Rl zn~^x1Z3sr5c2*u& zW$)YChg7C(?tgT(c0buQaMbv&x!+ame|62izjOUb?JH;B+g15mXXT+swrCWp(@kk= zgqNQBug=Qm%2u^J-8pwiBVz=m%6(k}A1YMGp<3O8opbLkwRYb-Z(wSnbM6<^`jwgA znE93Rz@_Osr5UYESbu5z_B6-k=E_!0BrO4EQ9T|=J+^mN?ko*-?I~`#R=9Q3_O7|# z(*(bk=CQL#=fKf>ASD&ipp`+UKKC1~-T%@#&|WCb{Z?md_d}iQuTRTF<7JUD^op*z zPqhx5q7_)ni}T9LJ=79dsJ7NkTU%dqTdKzXMC+8>+Hd8%w{E&y|Cg^}u`>IWEkavc zH*E_#BwDK6)4ECjZ>^T^UZmRd_jDI8Z68b{-qNjExRru2aQC9k-NiNS*Dwp6?JK)x z{<2j0c3JDgW)CB_Rd{%zt}p!9X{&q(_PuAL9S0LpDO!z&Azww;0`U4OWQBig!I_J+Ml#u_EE~; zU8+2IZRzuS^kHq=zIxY?#($$qY3{$N)pJ7{-6NV%zu*(CZMU^=P>bfB`$lW;Z@ugx zHOrOT+Bd7@yn*IJwY+#LfR(sZOp9*O4s9ZOBw&7V2Tuj;E^YbFxbF2gx9?C9o@?mU zkXp-#?t9Avo)R11!mi5Q<@GDt7j#y>RMyftu~hl0X6bSjFR#C-9n?`?|K9dFo$8v^ zx&F-d8T{{2Uj3i_wO_rwdh7J+s!(3FtzQed^pBc{Z2s#1EL_mvtyy7~ zH1o`n))c;Cr@k6Xt@7j2%pbR|xk9aMwtU<2w$Aomb!VE#h4UAEt$U#P^3pkXzpPs` zrg`3`HG>7hs=?CGf~hT#!P?g>s~cY>~5b^o$=AWtA<*~ zNCvKK-;gGxHHil0^K$o+tu1+tC2fFSx4QY78bPOad@S3}xj$*W z^v`~=XOB2@Rb7oci(ZBGu!^}hjZ5E5v6I^Scru1{JvNh~H<3MeY8B|KZ6$NH7ihio z{{PyuhXue)cWU4jY=ve0z?%iwGKW%AJ5@Eys~;`Yw#oW7>VaF=o~lu`&hVb2wy4Uh z?wX)EDfN_2E4{Du-k#Im_nt>IqU*~5%;PvV8?Q=D=uTNuLS@_3n zoX${d7w{)Z;#_K@ysyJgX#RfT;OtMEj?8v6eIkvkt{XT!Ri@xauMck}0UMV;A4y7R zlmy>OtgCrfVQO|?)5p`W3mb+#HO<>J4fwkrc>KiT=SC<7em8Ys1m)eTw0_^!*PgyfXV-jHWzdG;G$RX#q|g zQSO?C$u(#JYKHHLX>zZNNi7&LiVf9C^}U>hJ1@#D99HfCE%F;x&Y#Z}_=X+o7{d(E zTP;Wdmf@`pb2;j*roIt_+^#`JztvP$pF`LLb#Yl+JywnwdUJK?>D3!m7wAsaoG9Ht zqV%9jPt`Z_Lk7byQdx%YP~RV>>Fpn5ST|w}+f`=7V+yrF?@5Esj6p9SG3X8rdSI-* zD|*jctX;yAGt<^GRb>fRv#cYAzb19_kC*LGb>_J~Eo93sSlZ)poFCiY$I}oWi!p3S z-TYHfnR#uyH3ok^Q#hbHCs}q<&j}}6Z^AbFd#S+eCiEstWv}bg2}+jTQ@l6Y52YFf9X>0H+QciElA`Y@-)FlbZGNW&b^gpU428phN4LUC*MV78&~ zfnoEvqG$24Gt;==ukl`y#(hfj%d;bzncn*~kDlI#o4=L4Pt%*19-X)|mAkl^pPRdw zS$5v?3*(+d?Kk^n`psI?(iv2lPo}xLdR%?3dt>VCRX>}S#=}jo&#uwZVCs7NPa39} z_M+^ycc<n>B7Wpzid|a z^7NgT#cyy|@to|d!+wKhF=xv)?2a_-8f~zzPwT^S4R=WT-fgb3B|5#S6V0s>gV*N zC#138$XC-}maQ4~=lM6nw>sw|Lp46FK5}Zp1#HBRX3FM)tS_1Q_&}kRcytQGc_Em6XWMV zoMlI+nd8=KZRY+udqB%2rdgZ0#Ft3d>*4j9xX2->;*#dou z$rH|)Jh`l=^*$%+ByDnY{{v(2dq?N1J~=8cuF@A8ZTzbSM&%P%eGvZQVe|7& z%};UF(c_yYkDoaC!2MhG+m_Rjx#yir&hA~2RtPrKqtma|nvFW%=2fp7gC9BupEd?> zfzKQ^e9R))O~F!S0tN}sAf)wCG4>ux;Yd@1ef zZ%aGd(GNKD+PMGofi#}&<7$U$9XJ**J$=ctGbKb~8|C0dT&*8(*amb?>c;_eaykn; zF3oG>0ko;$Men8dh?drPt*lq+Ecj*Ffm0@=@0cb&?4Ri!ZP}4oZ`=N2tlEcasCNCa zGXK8*Sc&+d-mSm)=ovpuK{VmlW^#R%3R%rMW3S4TmwJnHl4*P9w3!|G$gSNC6 za(OIdZRr%;-N)(&Y)g-7x;sw6!Tm9sD)s|+>qcT5Y%UMnaO%wTIMv0HbwZ((y;$8F zqG-c9MbR78i__T?3iDFYTh-l>zG{QWim`B>>A7w=EuE@fo9U%vY0+eJsuI5@v^pX5t^p$ZQ%;xs$;(>+GAJNU3;hc4^d+8$784>7Edjt=zWsIhlH zb`++`5Am>MFW7*d}T(3T^j@RjrxPZ^k#%KLA)ZC-y?*aD9`Z<{^sCkB( zO=`Zs+Rb)BH|Gh>YJNn`32JUqbE29YR9n>ilA4p$TD9vtYjrc9?!WPvbS!ww7!`u=fTdesZ}#3HBGksK-g)mWw?4N>tKG@D+-wF1^ zU_TDlrtR3%eL=OYB|9t_Tf67wO~HOQ*m1#rKiKkM=LfqaSS8p^!8Qf^e6X#-z7=dJ z*!P1y9!&e<>i5vzw>rVAln^IoPeiz8>ts zVBZb)Xt1dhYTsj8wXG#v5o~3!HNn;eyEWKt!M+~s!C+4Wdn(v|6Km6*RBda?UK;G+ zU~dUFH`uYkP6+m%U_HSu4EDibm0*7s?5~4u47RqVHh=4@Z7tb4y{Fvk?xoeXmh75f z*9E&d*sa077;Jm6Zv`6)_GGZ9gH4-Uo9@f1Z7tdF1e+7AGgvv;Cw0%=^UU3MZ(DZ- z8w|G3fS8jC1xo#DDE+GZNC|N*`5m4Qb23`bhcDN0l2V*046ID~@a<|{e@FZE>Ql>y zH>>>_^5HAnpVn~Ql|eF^zOK2uE;_cP*}x1Y?wI&zwV3#Ep;2Sv+4m|QRtU7Rq#Wwf z6r4W7jS(d$3{$Gy`D=v5Ur|E*1&77=tM&d`Sj-pPt&wJXhs9}e{7PYQsd#7S+}#@O z-HM79YK)4Et8sC6rCYgbJu~L}kQ*xxqTsOjm_ebt$V;u1w;@J+a|KFWO#K zxWxsB^09vPRDj%7d8%9K*`T`0n&oTMd~{s;s`|k%Qhq;oOY6Yj>hP_&Yh`7ybOucqO-C~0bE)bgvhL+3XvZj@A6%7nCaZr)*ZUy+P9_ zj;6bV0D4&Ntd)vnxmQKfx?EdV6ejkIv&74*w@t6DOyyPgkUo<;x30ZF;X5-oq-pAF zZlG~nH5B)PuKc1j^B1jaTt`IP^Kpyre#gMnesX9QN?D0#Z#8z+Y%M%~U~<26=g>WuVVPxO=s`7=y|VpB3_VosPzerB;`GZK*R+R>|7>+o|dym!9p|nYpN~^@^P@S7(Zy;l$3F zighYB)~0oR+BVp(w9M9}DQ>2k&_&fvt8=b*L9LhmrNZK}#+nwaep+T&^R$p@c}wor zFFnZD&<4&jWKY2EgEgbJUDvmBkD+z#)M^NwX+&v3DS@8GjEqd6xi)tKeXM#rfqs#8 zY6t1BIio_?)YSH;YHLbuAJy4fV*vT+Lp1I`RrA+Ppc@=~BNOOZ>E;z# z#w?9K%PuQ?COar@cGYC~x*Jvf9FfX%W9*1Tj%q%$X?x)n>1J6WyFxoBUF0+$p&JvY zX<=QfTj2kiH5It&$EI4lgIvg*Of%jqwHS1pLbo=2Q=Bdxl+(>Sf`K3CkH@O6&puJO zsOgwAuD*tGS)gAR-+NeY7Ce=P9V&h#`}4wiO>@$)D;tI#nNBa!op4LtXE53HaXQ_f z#s&EJ_&b{Z>{m^vU!u%{b=IGckN;;q!uVKIA$zTEvHyyRHKSnr_?1R6dWy{`t{Gz# z_4~LDx94hi_@B^zQ~Qm=jBHT9y_Ze7Ixm@iV@ADUj8Q*R+Rd;xjxlU~wnp3vbntLg znk_0&?Cs!Zw6~5iT4rfqsrU;uEc+mT_`UYazSwkeag5v9waNF^hNv!)%?+cd|K#;= z)o^RFHk!T>qZu4yG-Ide>(b^ldWz0+*}ji4atoC7f9zzPG3*#)4Es*oPt_ie9YF1y zt{d@9hsGG|7zzC0X>_9}@XX}yF-BLr)nCZ!w`tk)Ol5Y`Fr%mPjPvm^#+h>GH2AO^ zxoJPi{_A7ui{6rTWc43qLt^~neaq7^<_A;*s6+e0TKLzP*dLXKJGqJdErEmgx8sNH zZzpB9q`bN|d}FwBY#RRmq8(NSr1dDsr>0_m|0D))HJ7Q7*BIz z!!#eu9?8C0ICrlkpCwCNcU77WKfY@--JQDsq`5WgP02nnvghS9&0o(p6knmGy-;PI zOw+za`TEA}3yM^JuJ8P2J$XBJ(qGn4f0#A_l73h8I6bW^tr_3o_tpRUG}f0DZ_3`Q zqszGLCECP~)mZ%rlaKn=}5;B_&-wrmyX*n`*!x9 z*=t5!GAu!I+Q-wd`xg(+22;I?bG2l=1n+MCo{qlb3$IQ~ku@t-xy&9J*P)PooaXb* zn$HhtKI`%S$l3{8)12IqYA3u|wG;RT)J|Bd9RyW)>ff-jX2QRx;XYb;>9Cp!9_E12 zYbKCkaG#vbYqQ!eqBDEd_<5Sm^EI2dG|cA6S_%)P(z~=**{nD6jEe>PiL`KDkXAPq z*5=f`w0KB1JuQ@#8t!{(*xSdos_MoATD8V$Y8?$z8(C}N57SsCH+N-gqt?RCG?vMQ zY1s$0iq>l_+?~q)TXACc4~o%Ub;S$f<>|ZjYdBZIyQg`Vey)SGaF5sNN$v13Ms0@8 zX>z}tz8FcnfARa8Uz+{JGuLKl(W;>9z%gno{PwjONCuP)r`4I-47Hj2oPL>mWnzD( z8V&Q(@XuJIp(7P*tkFO%$zE~qo1WBt=>2ql`j%p!>MXF;Gdq7#{rUFG4$a=JWw$!} z=cb4coVkk8@T9t?8?=Kf5oD>jJG%JDVnI8dPH8_wb5#h<@@{ zX79+p)WmAd$;7_LTdxiAbQOfK$|tST431L1d6nuCjaEzIglDWJ zaY9-H#;hfA}q613eF?WNk`?;D^k2Ioy;%F#y+nZ zuk)JM7W_+%Ua4YQS|)tw;h$%tQpG}znFCd=QU!Y&ce_$WUv#@t#meY*rHcOOcBP7S z(cMs~;_{K$l`5_oiCw8;L*Rz1wpvYz;gu?GjiMv2q+C;Cc%_QXD%z30>Ry#924gG@ zS1wea*iJ8<8Z=g_*de-irHUcZ(JNK#jy_ST;&DctW~Y9(PNj;cgAYIB{%tB%uqSP- zRFP4uf}gVX`MAE}du9Qmf+c)uh(Mm(`?p#aGp&b_FL@lhxd*CbcU#>!I9}YEDt}r)nzS>wk&niP{zI zYErvmMzx!apV}2~Rg>Bk9A~LbahjUcl3-^-H^*;kSM;k%?FzE|!_>S^P0HP*CbcO( zt0uK6IKER0f&(kHA~@%vmc)EEz^w()lDu1k+49|JdJJsag z7PTn;s@lzR=21@9qM*&SD9p~U_H``^v(>?>O*Pxs%0?F!bW z-@)_pf@)h!c1WL9jD}^#;2-*k1b!hD*w$c=274^n&x7p=_Bt&^ z&+{9rZ7tcnU~doh&S0km<56G_OI-{vm&=0vd9W?P?hf{qU|$P1K?f|4Z*sM*CFAj6 zGae82G~OQU9l`z}*lEG=Y@zS`E3 z9TDse!Ailpg8hE5lY^}awl>&R!LAMVH^Dv;jP2d?@?fyXf;|yzPcYpQtuBK@t8J6A zR|lIN>@C3-1v@(!bu2vH3xi!9>@S1e5R7^kUiJ?L`*yHLgFO~ZwdktTD5{o%ht)o} z%3c&~ez4<$y*t?Zf-MQQG}u6}D}w!>VE-5lg_6JTcY^&e*pGu1Ce+G}tG2adFAnzd zU`GUdL$Fe?u3+yD_P$`1V1E{DW3U^7Z4I_97_~C|J*bu8W%^{Wr-L=C{)5X+sJ69a zZw&V4VDAWae6T+Vc3QAy!Ojb|A=vf7ZVPsMurCJN9&9Ms!@(X8_GGX_RcFKBEKzNMsu}IHL`-u40d#|V}kVrTNG?{uuFp76l_zl zt--bhdob9ygX#LaI)D3B+jLwF_R?T)2{t#_Yxb`VdqlOZC7Y?bB3`DiueP;hbA!Dt z*a^Yj73_n-E)G@+_GiI11^aZcZwGrQ*w2IQ2{x&yCowndFZ))z}Gk zU9f%Dewm=dz9Q{DHB{{+*CXV5gj|o1>k)E2Las;n&GZOGT~ah>&$S+5#CtGCR!+(F z2)Y5GPp(I(EeCFlb3?S2lgVw+^GA#0t%^i#I$z3iJ;J^!aj+1%o4fA{8@V1Kt#r8_ zp;nb7*CV8ymzo^A(yEc`5m*e*U4;T_aOQdhErMK+P--h{WjH#m5Mu`auNx!CKXeS9 z>k-EC@#K1hT#rx{!RLB}{xh@-Cb)mUngshFsCMUigx_3`@Cn`KcCb&IgMBlCeY1^aw6@NieEZn4w2-IrIo-JA-{E7404u&4V3_XI&p+_)V9qg81 z=n-7*&S2;f%+Mov*#8WM9>EMfg3F;tFndk4&3Xhg^aw6TByWZu!3;fu8F~aW^ay5y z!O$a^p+|5z^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbcE_Fhh@Ewk;TX z1T*vqE{7h$3_XGwdIU4{2xjOJ%+Mp4y(JiW1T*vqE{7h$3_XGwdIU4{2xeCYLyusF z9>L|%BbcE_FxwgIJHgN+xEy)}mqU+Wh91G}uwdvB%+Mpa9C`#Z^ay6?5zNpdm=WEZ zT@tJkjF{f#&?C4UdIYnr!O$a^p+|5z^ay6?5zNpdn5p_ym7zy4LyzEc=n>4&BbcE_ zFhh@Ec5X2A2xjOJTn;^g8F~aW^ay4*2Sblwh91G?&?A_kM=(Q=U^Z3JxEXo`vlYS6 zBbcE_a5?k{X6O;jZVN^bZ$=RBa_AAv&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4 zp+_)7k6?x#!3;fu8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbcE_ zFhh@Eh91FeC>VMKGxP{9haSNUJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XI`J_F)u zI;7-!gyiGb^Vo9f=XwO41m$`J9#Pk(k?Rq3YL)8|ay`QHMvu^}^XBI4xz;0$cwywo z1UlCv5UJMX{ka}tc~9SRiRY`2V9H3O{bsI5$n^+X3GBkyp|c5p8f72*PTNn_)iw9Q zay^0;-v%!~{hR9%tW2mqO`Yozay>$>N6?O8)R#SMPWz2qk1+Cu54j$pQR$HD5%zvr z#;RP8Fif|Q>k)qScTOzS8Ji!{_d6$gqdV6lsE%!}NAOE0ay`PcuSfW#Za%mc1#PZH zVMZ+qGip(otqw*l3bT#D&?A_kNAR#)gKZ0j9>L|%Be>jt+T_j9BbcE_Fr$uz8FeSj zjtPby!3;fu%b`awTOABNf*E=Qm)jWZhG6IsTn;^g%b`awLyusF9>EMff*E=QGxP{% z=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;jwgy9wV1^#S<EMff*E=QGxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;j z&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW^ay6?5zNpd zn4w27LyusF9>EMff*E=QGwN8Fp+_)7k6?x#!3;fu8TBsA&?A_kM=(Q=V1^#S3_XGw zdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=Q zGxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU3S zWtdSb!wfxw8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbcE_Fhh@E zh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIYnD!O$a^Z4QPW!3;fuheeNI zh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x# z!3;fu8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbcE_Fhh@Eh91ET zJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4@%DfTWhv{Y@Xml~ zrl#i`B$=P0MrY+dy`f@F z`*gLhy|;DkXEeNeok)o_?L5LDW&JgRrOKD}_5?J2yi=e8KQq_aG)=V+G>Pj>JoXxUcI&h+v318zknd9dA91s5qs=vzr zt1q$k-U~W@yD#$STYtOZ%awoW9`GA2=2zY`eSYPf8S^XW&zfI(h&L7VJffd6J4t_w zbr#p0b!n+lbF4Xe^4W@-lWmk2@MuHOzsN|Mvv&P^R%&DO_@Vw@rsMnk?Csk2>Yo`Z z@lti}r{>zJHQs;L8D}qEy7b5+&s^NMqW8ij=P&M=y;A@6^y!V*Z|FJd4Trzst-5gW zGlp{DCWkMN*!eRfb@S5W83{byR4>cj1;5DFWe-c;%T)HT1b#8BtY0@kj{T<8Z&!L* zZLiAu^QK|*;MWQ8ii4S{=i{kt{q+Jfhn4l~1JGN(GLqhtz*B-~ZSz zv}a4)tN7S2v}b$XtN7@z0$^s?Bh(+EZV*V;?iDXokxV-4d$i{^~p9oxy@wx+1I2|&u@BZwkhi>K`#C4n-sM__di^H=<~Viq_<;!`1h-@|TY&&)z;s*2NoNKwlez!-79v zVjL~fZ>wE{p49WxLp>`NFZFB4=d1UkRQ^_(jsJiurDJzvVa>-M;0xJ-YoI*N*76k##7#ZDidY-3>-owkl7i;Wf8xJq_4K zRxPM1ZrGrhUsn9#Mpo8bkD*~BpeQTra3d>gVuvP(TXpFL=dM`NckU9so|xC%vc`EV z4J#;PVI`}&SM}O~wl-Fk(XFc;(I>2{{P-Skqji;^*nGHkl~>ZfOEWm!p86wo)mK67 zymd@qVd_M$s(9B~TT9l@@!}>m7pQr&nmkeUkeaMq539*q^9wbPSM!%@X1WvdDK-B@ z&0EyGOU-R+^4!Xxn$4Qdizj9M9D`n>=AqT@!!^E&+I_E#wwAP&nJusOZIQ&iR*t)7 zE_ZdXzYKOmu$zKy4Yn=VSA%^$*ki$-2(~9!QQy?l<*u3Mk0+DNcrwY1CzH(3xSE|2 zj2GdW@hV)GyCc}u%Z#UsT<%lB{w~xGZ?0)eD_yQ>%QbD;$oIqTN~=b$X=5?unl{~O$g-~#YXJi9bF01Mc4O*yRB>K* zRl3>FyWBP;R@6PKd2QVJ>f3Ah`g_|RO#O0ATdrx-QF*FXvv+HS$+DBv3dW08Z%N&k zHUC|9r(O=IH$toLsT9e+H?IDYw|cGP7pc#ebZ(zzw&7xT+^0o+Pbxspk+Hl&0N!#^>5d~i1)2E=@vKdUqdWKH%D; z4$-Y^-oJ*piTAG^RqZA;xk=-2XhNGS7@M76?d$iinXL{+1!J>~!Tu@O=YxGa*h9ex zNIWb732iMIVTc)Fh#6ss8DWSSnl`hegDnbnb}%$;E{CSg!(I{$O`92-HkYGPj2W6X zGc;{xXxhxsw3(r4GdnsMnl>{uZ7zqV&1`irq7pMSZ7zqV&5XFq3{9IEnl>{uZDwfN z%+R!%y{6h`O`92-HkTtvF(XJZL(^u4rp*jZn;DumGc;{xXxhxsw3)S3+w?xNU})N0 z4o#cOp=mQi(`JUI%?wSO8JadTG;L;R+RV_jnQaS(rp*jZo6DhTGegs6hNjI7O`92- zHZ!&+Gc;{xbA!Dn7@9VhL(}GRXxhxsw3(r4Gegs6hNjKzpMs%jGus*LJHgPjc~~@U zE=R_0hNjI7O`928yBV4`Gc;{xXxhxsw3(r4GouQz8JadTG;L;77dAuFW`?HC3{9IE znl>{uZDwfN%+R!%p=mQi(`JUI%?wSO8JadTG;L;R+RV_jnW1SjL(^u4rp*jZn;Dum zGc;{xXxhxsw3(r4Gegs6hNjI7O`92-HZwGBW@y^X(6pJMX){C9W`?HC3{9IE6@tyE z5Nw8~%?wSO8JadTG;L;R+RV_jnW1SjL(^u4rp*jZn;DumGc;{xXxhxsw3(r4Gegs6 zhNjI7O`92-HZwGBW@y^X(6pJMX){C9W`?HC3{9IEnl>{uZDwfN%!Y!YX){C9=5lD- z%+R!%p=mQi(`JUI%?wSO8JadTG;L;R+RV_jne8(mrus*&Y0EWjTU$5XUA~4&!Mk-5 zlxx~l1Nl~M8o8!Tr&hV9E!VU?Z!~S3;Xl`!HWKJr`>H0MYua*6TRcP1Q2Vxahde1YkpA{F7_ia!ngg zubitj^w4$&>X^%JF(qu-@lHJv}zLx&NDE z@GpJs>iht1DBH9y5wCyZ~JJbvQj1NU#$d1?9O z93p$(x#aBL6=@CN39A#*8ZhR|?2a5`daoaYw~xVJ2!HLc`8%2ARa|}4?W4mkvi_;QV9+e~>NZt1~pO?L@ne~E)5>L@zmUWNsZEDJ{ZITAHY5cJ9)hkR) zQ>odCKCi?}Z7eoD<@gGfU0SG3e|*FAZ^>R; z@Gs@1{D_wBc&*!4RiAt0{_ozO>3Z(*$gHn_L#u|gJ^i_+?b+9~-K-shYf;eV zS`=pIYpI)ETy68q?#!q~;c^?J+@@fk4z@MewqQ>M`+2bav2`nl_h1)8=w$+RV_jnW1SjL(^tQ zW@y@64o#aGnl>{uZDwfN%+R!%Z3>2_%?wSO%WVyYrp@fJU})OR(6o72G;L;R+RWw! zdwVc6Z7zqV&E?RvnNb(R3{9IEnl>{uZDwfN%+R!%p=mQi(`JUI%?wSO8Jae;w+DMi zFf?s0ho;Tt-X9E2n;Dumm)j5wO`92-HkU)wW`?HC3{9IEnl>{uZDwfN%+R!%p=mQK z1?viirp@Khw7DELpUl<{uZDwfN z%+R!%p=mQi(`JUI%?wSO8JadTG;L;R+RV_jnW1SjL(^uqF&LUQv#r6{uZDwfN%+R!%p=mQi z(`JUI%?wSO8JadTG;L;R+RV_jnW1SjL(^u4rp;_&Ff?sun}eZgGegtnVbQdi>EKyq zXxhxsw7DFbHnXFHp=mQi)8=xEf~^jQrp@Khw7DFbHnXk4(6pJMX>&O=ZDwfN%+R!% zp=mQi(`JUI%?wSO8JadTG;L;R+RV_jnW1SjL(^u4rp*jZn;DumGc;{xXxhxsw3(r4 zGegs6hNjI7O`92-HZwGBW@y^X(6pJMX){C9W`?HC3{9KaK5M_^dIY_g@ZMaHu<7pZ zMT4sJ(p}8;2)Q01*CXV5gx^e$u%9j|o_jrls;BI$a!Rg8&$AIplhTs6(*Q z%OtHHESAl>@iRq{Nv=oGg7|n^I3J6JwPD!8;_1)bD&x;*^fYKq0B}--EQAujmkQ*1 zgxb>S?OD9+%oI5(M^W6v;(55~_1QJ)!EdU%d?zijVtVeD3Jtqc_le`iWm~fQ3*T2c zFL|B_Q)T^TR#$=*((W@BpS@&R8WIof=Mw#NVPdvlHYRu3>kiVb>I1ki9+yOv^PKPwL@$oupxAPN99C?V99?7tpO<%--#4!=;#Ica5vh zUam(tZ}I78p5N2A_`GTo->c>J%e0(wJwnuXTdITK9 z4)$qtuy01NZ$_|hwmKNWzS+iL=n>35AMD$~9twsY!R4L^Ms#n69)Y%&3_XGwdIU4{ z2xc6F%oYVZI~aNdmqU->a+d@{k6?x#!R0s^HbakKh91ETJ%Sl}1T*vqX6O;j&?A_k zM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~b>1;NlGm=Vdl9C`#Z z^ay6?5zNpdn4w27LyusF9>I)EMff*E=QGxP{%=n>4&BbcE_ zFhh@Eh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7 zk6?x#!3;fu8F~aW^ay6?5zNpdn4w27`%WEMff*H}h*(JfyBbX7>yBvB1GxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vq zX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW^ay6? z5zMA48aG3aV74L{dIU4{2rh>n!3;fu*=@lH;>`%+T@F2h8F~aW^ay6?5zNpdn4w27 zLyusF9>EMff*E=QGxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q= zV1^#S3_XGwdIU4{2xjOJ%+Mp44FyAwV1^#S<k&Rw=$f1B5pq4kHy)jIR_no?svBSFZtu(W2s+m+uezsSr&hTh zA=e{3Z}bRMzkTlY2(xlM!cg%e*`F8AYnoFaMqH`gLG2NpLgu&h=?R3$98&&S8#(e!72cCY_ zQH=g(KSpuQ7^A2^!IkR~3JU0yku@x-{c=4*b;Zy12(bqkz8R18HpEG!vb$5+e=APR{y|cA z$`-P>EwF9%0N{61g5B*CVj^NS|Df(AI_>)s;Z@dZjd1I>TQny;@T#vwwf?SV~^>fZcEs7tj$x9{Pq#LfhWa2XAs&wPwJ6M}MPar&7_}(OHU>kFVD{-?TZ3&2h91G? z&?C^+lI^FJ-wZv18F~aW>R6ajcf#zLVCWId&?C4UdIYo8!O$a^p+|7JjlpgRh91G? z&?C4UdIYmw!O$a^p+|5z^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbaRs zh91ETJ%Y=jM=(Q=VD>sKZ8P);X6O;j&?A_kM=(Q=V1^#SjJgJ_D7HvFF!Ts!=n-7*Z-Svm zFk^dnIrIo-=n>4&BbcE_Fw-s3Dm%2=HYr1o;Bx2@Tn;^g*`i?R5zMG#;d1B^%+Mp4 zp+_)7k6=c<3p4ZxX6O;j9u0;b!3;fu%b`awLyusF9>EMff*E=QGxP{%=n>4&BbcE_ zFhh@Ec11As2xjOJTn;^g8F~b>9|l8@V1^#S<EMff*E=QGxP{%)XFfUR)!gR1T*vqX6O;j&?A_kM=(Q=V1^#S z3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW^ay6?5zNpdn4w27LyusF9>EMf zf*E=QGxP{%3xlCYFxwmqJ%Sl}1P_ZI!Au9wDnpN8h91G?&?A_kM=(Q=V1^#S3_XGw zdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=Q zGxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{ z2xjOJ%+Mp4p+_)7k6>0_|K9cq<@M*bPbsfo(LU|e(kZ1=%d5V>K_@AdJKBfT=&byx zv+~nY<%i{MTlVNvs_Y&(=HP2r?>e}6=jx%M?nT>g*O4Nd|J1H>YsZ#yWpKB;G}M}Q z-CM}Yl^x~EmF;uX+O>9b>zYepYwvGe!zoJT()KB8bXM+rqIJre_UUS0dvEL7&uDn{ zqT0E}!m~!}FKxd@uvGc7UQmIiudE&&>cI;u)>XS_rS2=+`_)~l{A+oAXZuPOo00H) z+xyhr%W%^Ko@j0BY+p!`1)vE++dA7%)Q}owXFKd^wRft$qqB0i8gqmm7gFmiwels^ zI%EAs?bFj2euNsS>cL!VrkGuIrI~-uW_f*g`-xg6zv&z|*XMZnPf-0;{$G8GwfA1o z@!NfoN8kF}4PUPOOZUKa?Hlxy&9A&?`uxf{Gv-&$pEbYoP^qVMTF)cm*-83ath2c0 ztV>Icnq$q`Q)*)?YEHM)c-f9N1pSM6q&aKXzh|X3Hjf|b?`1l^&(GejZLj{Bp%O1u z=YDFgom%7lXPt5O;-yQEJo3!NeJgq|Typ;6p4lt)Ur*oZXZF0I=cs3RJ;99X6MVJ& z`RS&5Hf z`PsNbQ^@8;xrHOjv72PF9cjp;dzM~s?usRS=Pr?)wDyg9M+`E(VNM(F=+p-38!^a? z>fBVHzm8F_98r#)`{*yGCp9@%ea}fle7CS%x20ZLI6ubJKVnQAiuTv{dQDpFXDm6r zUUc1vq8%z)ld~4G6}>#p*|UVFdHsuCKB9cNI+NTN8>zMT5Z8fu{< zevWG}y8Rs2_UQI=Tsxwhjk_&vj2`}2&2(GZ+T4Bqiu0D7xwvOV!jH#bRot8K(xe#S1$v~ zJha++I_E#wwAP&nJusOZBa^DE5}_km%BRHUk1A&*iFH<2HO_wtHHh=?6F`^ z1ltp=sBh}&a@WlB$CF8BJeg$1lSyW$1v?|yK(H%<-4Seaut$SE7HpE1rKizaZEMM9 z2YXAfGlHEJ?2m)-3wi!N8H{I-%>FId&R|pYt9ZIot8FdWoM1-<<4nZm-XDxRT4p?5 z(fV68f-ak+!4Z91j|n-^@FHYbc=*z)*(*O*_4mtV&YN_cij4S?U=v*=hZsqKMhg)Z;b-P=A5l~)r z7hhp(dE1)yezovP*CDl;qJzrQqt6w67pU*Z*{Yr3|JL($r{<@k->^CR1ATLTjb_eS z)7j~%x?^_Ch_lnPR7#iq8C%P*adz5hCvsgJjU|0fE8LS!NGBld_6j<&c7=*+Z%wMP ze>wH<)eQJEvj3^6fBigjWdCF1Xg#Ag3Qd*sFMheb&i=+ex8yh9tN7UGmP4ZzAN$;r z^{jr9*KjDToo$p=#2?Q;YXc{8&vgFCid`Rd!=bY_>IKyi*Uk}H0e_WqL^h~j^Bj>) zxjr`yhv3@W^o^JsHWudQna&ZXtK8`4h_x~Ij~FvMkiE_YOKtBIesL<=Rrr+F*4p6f zMhwpW>DM_wWZyMby}r@U4?8r(0Xp&1{yuGhY=oTX-57Jy&{93oyH(Jq96r&zJ-U6O zw>i3fqBj`bKG9>#@bEs-W7T(e!-?L|NbD26-6OG2^jKXzl7_XL^DfrV;U{{melFUu zw${!Gho9)NI(2C9QBL$&Z9JBSwT!W_rVKyPV?7xCM30}?ed0uqU(w@jJkjH~G9P}T z$BA9WnO#|bwGaDMmWK|1SrXu@`39YNv9Eojn)B40t>!&yzE92j)GVvXewc2~%r4g6 z+h=C9wPZ_zEv>e-Wb1hbke+gfx#vAcabHx;L?KdF7nyG}mk)Y3_%ldJb! z*_rB9%zbJM%-eHqXJzM8ceJlmSE=&T&Vk-SX`o}zwL<+ASfI|*p7qzIyPwc?(7pmE z8aJwS{cY{n>1QifI&@=qO*?ylfv)OZ+s}5WOSf*ziw4VEb~drS>K^Q+?YlJ`h1nf+ z&E0v?TbV?uc-OV1r+(U5dAzfA({1fTD$_OhKe}4GpX?eqYJBzCz^>W%cj_ChoPBRs zlzsGd|@u5)r**O87*t> zZS9-2vdtT4KD2eywzAR)R^n1|8*i#!v_qT7mfhp#7kBL0+*#RO+VY)o-Ro~oC+3~n zh!KWaA9LxC6ccWYL5uqoV; z=19X_qpnVUHLh4HKQ7JuaqF5Z)XHYdw=FA$=v8;7d0aSu(bu{Mav5zdqy5cfv=en@ z_uR^8c~7$bYrNbe2H>8wKe>$7%ORK1)|La8-q&bXSHB1xagtxsP2nt?s9kB6y)Lf3 z7I=95>#N@l8pShRqN1kcWXPY-$!+;-#TmCY%w@wxMD2>OuQu}PT-~lg(xW9hvoF-{ z=j#ftlx`0n!o3#VS*{Jea>US^t3y{WbbBsXvTVhiDBVAz^q@+Qe!0u^-l4ufOw-#x z#;|V07`Cg-h|AyFp!cLfXU3qHj~H}^1|99vm*JTz`| zVlJaiu3QdJyVAmWT$j+F%I;7I!(B*LJH201pA(8*nbYnAwR#?s7WssWcy5Z#jZL)nT3N2V-JQ#5bvme89xMK*UuX(e?NH~}2}zBlK0Bkp?-G2*o38}EDX4nF+8_e(VT2UOZa4C~Xw zQ>L2qsQp%Gt2wRhD)k#)F(OMh(AQ}?wNFW3@Nsp#PJb6w`On5@{WH|fV^hCJK|S`< ztwt`ReQsp5n{_Ps0QL;sRrU{ zQ02pnDj#N4`7oo(hZ!+wvBXJW>o4iLq=}v%~c3 znIWS!Lq=8Lb&IS~Fy{X2@vG#_Mv$tfku4k|CotLq_Xz z$Y{;Ff+3?dLq_Xz$Y{;h217<`hK$zbkkOiL3xmGmG} zGi0=8$Y{-w(V8KnHA6;ghK$w>8Lb&IS~Fy{X2@vGkkOhUqcuZDYle*0jK`kLcmE%GFmfav}VX?&5+TWA)_@zMr($Q)(jb~ z88TWkWVB|;Xw8t(njxb#Lq=}Gi0=8$Y{-w(V7hfLq=8Lb&IS~Fy{X2@vGkkOhUqcz)SKs-%{6a`BCYAF5Uv7DW?)de~qyCD|&P37v>F2Ep*Pk zU#<7g{Km|$lm{+t=L_!ENVCcWO@lMd&PuttvQ-mFi-SB{Jvcq>tZeVB+*umv+Ed(e zt#Iq6?Ok)fr*!&jrQ)5Pb9ZaBcPkQSJ_n9g7*3Y`Q!7vhORbv*=YFHL`(OAzodfL! z1%VVS&HYwqYxhH)>#t9XrgQz3?fsfm*W9OC2R7(XQp*u5+ei^tYKYcNTU%dqTdMb` z`i>4hHTJuSI2_IV?t<-Gp6 zmC-tZp0%$8dRDsY%8T`8X(yUxmlZyf9h9O&ZcR`Dnf&CS)Q^9TNacBu?TEyVYCf}R zd*Kx-^CFeLLYGat*lj){YsyYj_qD3S_pe!#^dp_=0kGN~O*(x_TvN4||Tv@C;&>L`8 z#}9XD)}!Wzs;9L({7>kTRQnA*t~aRPe(#6%aO-UtW7Pac`>KG=us4n|Y<;#GZZET% zkbv{>My9th)@EsV0?ue}9b>f2(!L(<%Vo4j?;}}f$?h0q$?iLC=K%^f+S>26j?VP4 z;&y0^v5t|zAD%`xdIHZ(?jB=w`%2}t!_sH;RGx8i3hqzi1~gWs^;||<+gNiMEulJf z#QwcK21g5$-pOU#sQ>HE&UKqMCoNW~-WC zR9RHE1akeZZxQq3u9{!~rnd;Krb5A*^x+trj9v42Ljn~eX3 z>VB)5)70cRd$5{3F8OLT7gxJU_77Dzt82TOWci1wd7YY+yGhL%YJOJD*Qoh`n#g2% ztZz3>mGb`|rVa1>>=3 zPXigP%N-I78Lin#!I06KA*1!M$Y{-w(V8KnHA6;ghK$w>8LipYV903AkkPsvGFmfa zv}UivG6w&8UlEc3CiFv}VX?T@D$o88TWkWVB|;Xw8t( znjxb#Lq==%_F(S_hK$zbkkPu_`-35)HA6=0aztNd$Y{-w(V8KnHA6;ghK$w>8Lb&I zS~Fy{X2@vGkkOizf^`K$M(c9OXkBhqu(iRE(YhQmT9^BqV903Az8(x2tr;>}4~vY} z3>mGNY8_YEq186aXk88&t;->!HCq(y>|oTfa5-eOE{BZP3>mH24Z)~)VTO#><-Q&4 z(O}4ET~7P^YB^-IX2@tgEHYX%WVB|-1w%$_hK$zbkkOhg4K@%A8Li79qjfoCv}VX? z&5+TWA)_@zMr&43y#_O6v}VX?&5+TWA)_@zMr&3IhK$w>8Li79qcuZDYle*03>mH2 z4Z)Dnnr#b4tqe12Wq4R*v}VX?&5+TWA)_@zMr($Q)(jb~88TWkWVB|;Xw8t(njxb# zLq=}Gi0=8$Y{-w(V8KnHA6;ghK$w>8Lb&IS~Fy{X2@vG76wB`YqmKU zGFmfav>p~2t(gvmGA)__h8Vnh& z88TXzLq=}Gi0=8$Y{-w(V8KnHA6;ghK$w>8Lb&IS~Fy{X2@vGkkOhU zqcuZDYle*03>mE%GFmfav}VX?&5+TWA)_@zMr($Q)(jb~88TWkWVB|;Xw8t(n(edp z%LE{=JSXZBkujt`zKPX?V;zC81RKl| zssT`m&eQoqaclNqwxRHWLVdUuJ&Tv&*LKAcG`v}MO7qM0T##99dLM57R`$O1_%^rw zb3H=MQP8H#;kUNUoWgROXNP!M3-JQF)r;rdZ&w?}=q$Ty+}Lly-IYq@dIT0jm+G(7 zR_w+HS=rA#pc13^U!l68*31rD!Izw=2ViS8L8-wh?R#3`d$j@9+zo%7Js@pFOta>0 zSbkQ|vNSBURo|S3ea6ZJzmtj`revWuIuy6A@cO@WS6V9ZVrTYn+6VAj;(d?obmC(y*6D(0hY$eVb@Jz`Q%H zuqTgkfSK3E{ftxfr?+hnGut#CaK5B-`z*UH)jJ#gfHSWRJ>~}}mTezbt1~ID&f=w~ zFIje`UUI=U%E4<&8dq`H26Rs9w^Z{vnMINvm*%zc0NPZrQNv5^5iPCp+Gww;zNcYA z`i=}c{PS!i`6p`391?Rqf|tTxN&aRQkg|35AMD$~9t!qYuqT2M-FsN{2(-y-9E_?HX6O-Ij)>oEQLwXv zQHjFk&?C6qCBdjtVTK;T<EMff*E=QGxP{%=n>4& zBbcE_Fhh@Eh91ETJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#Sj7Z)LJ%Sl}1T*vqX6O;j z&?A_kM=(Q=V1^#SjK`?V&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6^Yf z7EMff*E=QGxP{%=n>4&BbcE_Fq<3fJ;Bf;xEy)}mqU+Wh91ETJ%Sl} z1T*vqX8#lnJ%Sl}1eZgPV1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu8F~aW zqI)y+2xjOJ%!ujD&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu z8F~aW^ay6?5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&BbcE_Fhh@Eh91ETJ%Sl} z1T*vqX6O;j&?A_kM=(Q=U`7ycMi6g?9>EMff*E=QGxP{%=n>4&BbcE_Fhh@Eh91ET zJ%Sl}1T*vqX6O;j&?A_kM=(Q=V1^#S3_XGwdIU4{2xjOJ%+Mp4p+_)7k6?x#!3;fu z8F~aW^ay6?5zK~yp+_)7kKl6X5zNpdn4w27LyusF9>EMff*E=QGxP{%=n>4&Bbe2ZXkdj!Py8`7kch>kZ~^X&v}m9lk4i z5kh6KbKRX*OYNj%`m#S_fXMAv!C&q?UhLh2)>OKH2C? zd%pPSc$e>*{gX~j_)1N1?Vi@lzN|Bo%H4W#!k%kOm2c`byqyEhO{LXCL!I(O+^RyY zn{quuTH?7LL2E;|YGQ0>&C2x%?fq$aJa6;}RJwic^$4>L&-Dn!k7R#dIIn3=QY);~ z)=;|}{(sp!7dX4B^6alOGYMf5oD4<{7bW6=(MXL1G#Wmf6K3j(440}v)aoRJz(h!h z5I!PJ?ZhND;V^tyMM0^JLKVCK^+GAdnh-()Dn>+wTI~R8Ls0=+C8(J1`LE}kcV^z5jddweNeccdutXtCZXqvA)Zt>H14)iN1u}0>INS*G}^g_wS?QZ*Ka3 z&3kkPH+tW3w5wh}Y|-Jy1bbwG?XcHAyS_55oh&Z-b+Z_~k2$iqY>X_1zT=K)XhSW5 zqV+VqqMd&H;7dNwqn@z1B7T5SJDw#Xm5lY#Yms*Kgi z09OGo1ON8Sahf7WKG%=k@c=nca2hR#C$DWU>=B0b{IIcp1a-Lae5dT^-`%CZOP#|i zN8S)M=~Ou67Fg*dbq4Jkd)v{BA7)20FAk+KxI^UG;R3{D`XjQYN zYZN0<0qKhFQreWKmo`5?O>2Ih*0fS{|CJ=^#&KpTqAqWm0LXdcnR@j?;c=mJ&?`Q~FNx-sx{8Q$fyBO6&A=P3gILU;8!oPxu1sNj+2Zk5}=e{w^M)vxiMuETvb*YJC6R*>XU7|5JDN(3;oxyHhhzGoYH`e0ESX z)Hm)6Iy1MYolmJ~n46b>N<~9wPBv1}u)hv*pSYh*|E}-Qd&$1*70t_=TNBSMHiuT6 zsN&P1{XYe3TVa>dU)Endr`(*Lk|2?rYjx z>p&0T2cq_Wjav9MJrGovO*3SD!GxY0~(;C+)jWn*;=v=Tagm>=DvPkpwN8Q#yOp z_c+(ZC7OG3KBhh6`}8QiTo1Rq^>7>O9Y0G|A1Pgx@0HQt@#9nd9Z&vODYc8LhmSvA z`Tv>nze)LzC8VGnab_-=DecM{qjq5zMehFvA|f40{AK>=Dec zM=--4!3=u@vn|1}M=--4!RfF^FvA|f40{B#!)s%$33~)H>=B&qb-}PlFvA|f>Cj@B zVUJ+;7s0ScFvA|f>An^WdjvD=5uA=MwPx5Om|>4#hCPBA_6TO!BbZ^2V1_+{8TJTf z*dv%>k6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO!BbZ^2V1_+{8TJTf z*dv%>k6?y9f*JM*X4oT`VUJ*jJ%ZV{gJF+ghCPDQVUJ*jJ%Snb2xcz~hCPBA_6Sag zJ%Snb2xe$3%&4#hCPBA_6TO!BbZ^2V1_+{8TJTf*dv%> zk6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO!BbZ^2V1_+{8TJTf*dv%> zk6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO!BbZ^2V1_+{8LA93R2gR2 zBbZ^2V1_+{8TJTf*dv%>k6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO! zBbZ^2V1_+{8TJTf*dv%>k6?y9f*JM*X4oT`VUJ+8AQ<)tW}AXxk6?y9f|tb}!AvjD z8p9sJ40{Bp!yds5djvD=5zMehFvA|f40{AK>=DecM=--4!3=u@Gwcz}utzY%9>ENI z1T*Xr%&=DecM=--4!3=u@Gwcz}utzY%9>ENI z1T*Xr%&4#hCPBA9n(*!Y&ff9 zGF{kDJ*9kd`IO4)A8u43MIFo!YS2ZeAdi=;k5;yA-mPD`x^v()2VE|aK=Un92=px4 zaiiW8Y2IV^SK2x^SE}20YARO^E67%x@_Q=MAJsO5gUKnn6Dn>ub|9^Yr^V z`ZZmyev>XmRw~)_jQ_Y}xu*MAj+pM>+S`4XZk)fam&ya z4?hL9v-11eiFLQ1*ZG_6$U{f}X3JNqU+NjSqGO{@*}UqTr_HOLIelLB+?n&L50rb$ zZ|!|hT#o)JEmkS6C3Wklp&V<;KKa~5E%|dAo$csG(Ek{Zw4@IGe|jF{_IOl(2kQMk zFa4qJy`kT9r8q#7duh0CN}czge(D*Emo9zPt4>?Id_~{+OU_-~J8Px>^)6p{TJOtx zk9vw0;M4O*>$A0?({qlynuhA88@nbrB&S`bw1?!hPYg@z?gpsGJ~Gd{KX+E!r?mbZ zsUX2Ez)EU9!zXDPx6H^hAIWKlx)+!+EUoLu`F?Y7PFqqR#e3F|)2L@q^Y^IG9=%SS z>$x z|0g>hj~u@zv-&3;52vBkjxeKSwbWUx&8wZ)bU?Z;b(T&}dpGep->aRU=kcF=N#}{n zYM;!`pVji1d>ZxBEW!hPXiJ~6E}BKA*S4lsOB`FhvSF)vxQ|{J%xCgrHTTTC#J`lz z)@QYYOXo&T{S7(sDr&IDuOF)Q4M};!)W1HK(ohej_4K@n{PUsZX+M^&y-&tv^#y7t zgSSOKVtO>F(WX0B?Om~WsoUAk)9gigk?Zy8{!K{8(?)&JE;syLp?mbewD07NIqe?* zb++r~Y|nLJ!#_E0S3*zQk#$E*+mZF2n6@M9U`*SQmAlQ$k8ot=t{Q>u$hxZ$+mTfV zqLzBZ4aN3y;SYCY<;r!&BW?ncaGC>XL=$s;Z{Fos;i;hb|N{;S`$>Yv-%;@z(a&YXgUON&6E`4PAnIc(r#tDhAmppeiKta@1XrPd_iscCCYC=sb!5$9wZ^3p4Yu286y?izE_R%uQjFw4ev`jL?#?|c9U<1K0 z32?fbgKY}-P_T!CP1Lb;er>g})-)^FD}tRG?DSyo3dR}odey0VEkD{GnSCeNeZeN{ zta!asYGbWwb}+ho@v@XeobGMG_@ZU@?qHt|Mj6M;eka&{!P-=)@v_gXjp>~dY;Lfr zx}BVk(vS0-8SIE)PpZP_K~UJV6*g_xZ7pothDyr{A?r zTkb`$uxaC?03Q&h>rV#zyB<@j37Eo|EIHCNcQ)eWrr!X)mQIx;sYzEUjR^wKPQw{hI}^AT&#vP^n@)pZoe^d8S!}^MKND+3!63_HkCUHn>H2k4%2bisNu^r+@ay& z8aC+@_bd%DmSP&QBJ;1+kp9~KMML^;qtNv_4KZ$dy@p5Crl~aftnzTB31b!*o1I&m z+nT5epAYrUEQ5j-JWr!J-A!byDm|@dqc1*BE z!OjRqV=Sk`rp?R#d9Z3QY}%ZT!m!ib7Yv&=Gi=(N4x2VJY}(9@35HFZ88&TBhfSN= znqX9wm|@fAbhiYfy2}ikHmAd;%?z71Gi=(-uxT^Hrp*kSHZyG6%&15)qawu&n>I6S z+RU(NGsC9M>?gsnX*0v7&FNZeW36(lt}|@foDQ2dr^BYr44XDHY}(AQX*0v7%?z71 zGi=(-uxT^f77UvMrif_*<2Hf>IaO`FqE$8LsAn;AB3X58&&*tD5p z(`JTEn;AB3X4tfuAt5%yrp*kSHZ$bHX4tfuVbf-YO`91uZD!cCnPJmrhE1CpHf?6u zw3%VkW`<3h88&TZ*tD5p(`JTEn;AB3X4tfu^#{YI%?z71r^BYr44XDHY}(AQX*0v7 z%?z71Gi=(-uxT^Hrp*kSHZyG6%&=)Q!=}v)n>I6S+RP9Hn;{4`!=}v)n>I6S+RU(N zGsC9M44XDHY}(AQX*0v7%?z71Gi=(-uxT^Hrp*kSHZyG6%&=)Q!=}v)n>I6S+RU(N zGsC9M44XDHY}(AQX*0v7%?z71Gi=(-uxT^Hrp;_H7&dKY*t9tvHf?6uw3%VkW`<3h z88&TZ*tD5p(`JTEn;AB3X4tfuJ!u8;RJ~FPo3_HHZA;sATPwMbubnCc6*g^BAQLKe z&-#%B&!6kKS_ehWbQ>91H;pFyV%qTMN@el611&G8sC%^?V)UCJsUpNF(}@nyOH@_)koS6>eonm?LfB&Laz)2V(yw`_U%Wq$ZkyI6-T89HSKrpJqSd@LKQ6VcJ5LIBHf~VMX|atA zK9{rV((rvuK2e_WMB7@d+$+_tj^)QK+WOjoDg8*oIVcr%N!M4)SzEN=_<>3NVs!qulc@3+4S#uCuFm4%_#x zn%-OqkRMhCL9j-I@>y|8JMoV>1l z*;qTEb>+*hIHZM5+qcrj?4`v^```0%x>oh;SEf5;(((({biZ77Fu8$+K69vFowLvH zJ^S?2&t1IY+*~bF(Ngcwi&dqeJdMy}p+4*|jbYOtX_|78+DbHkGW~J-QE6P-U$^+N z%Jtn%6VstNP2G&Hy=7c_aXLcHnNCYzQqNc)O*=~;NS|oBBAX;FS1#A&ydEF-ESc5( zq0DMlk1)frqJZk6!lrFhWdgNDd?)Aqv`J%yO?-+wW8~);9HgYGa#oeUlHAhX_dw*HU z^;h3Eep++?H4_h+cCj@BVbf;z7s0S;GsC9M>9A=t!=}v)n>I6S+RU(NGsC9M44XEy z*9OC;%?z71r+Zs4Y}(AQX>+=b!LVsF!=}yYuxT^Hrp*kSHZyG6%&=)Q!=}v)n>I6S z+RVzqx`ScU=5*M!Io;}D>w;m^=5*M!Io)3e!=}ybTfwktGsC9M%VN`JhE1CpHf?6u zw3%VkW`<3h88&TZ*tD5p(`JUo!VH@>Gi=(-uxT^Hrp*l9g&8(&X4tfuVbf-YO`91u zZD!cCnPJmrhE1CpHf?6d1;eJz44XEm!=}w_X|RD{*t9tvHf>IaO`91uZD!cCnPJmr zhE1DUX+oW0(`JTEo6}*_W`<3h88&TZI6S+RU(NGsC9M44XDHY}(AQ zX*0v7%?z71Gi=(-uxT^Hrp*kSHZyG6%&=)Q!=}v)n>I6S+RU(NGg}Z0n>Mpe!LVsF z!=}y4V$)`(muHP((`JTEo6}*_W_C<4Y}(AQX>+&Sk+RU~D!=}v) zn>MGzrp*kSHZyG6%&=)Q!=}v)n>I6S+RU(NGsC9M44XDHY}(AQX*0v7%?z71Gi=(- zuxT^Hrp*kSHZyG6%&=)Q!=}v)n>I6S+RU(NGsC9M44XDHY}(AQX*0v7%?z71Gi=(- zo>cuZL9cyPX)o7cVD9eAyQ=p+c5|)}FIOM$8t5xgNq@OeKMCe*vbPhh5y9bUM-#sU*Rk~;0)uk%$m9uW|u70Dd z`oM#mm4)i%NnNgPRtCRRFL>208rjt~XHc1uLAiRT`l>6{@^Ht5w-@#Zg+0Opx_SzG zgaz{odjvf`3VVeApgjWKKEHc=gs1Q@d}(eH$hVf4>Zayv%-^ZYmm$BwH|VpDtJa8w z_0_87hoytkg-x$Y_cUFTD@~pW&cBx=T7}P7x}oLmZP=HWCD8|S!(Litx(l7|mitRn z)F;bF^Rf#@EPG1cwy9d`l{x=3e(&Zl)2`uap zp2`dHG9AAm%}T>3&B7jmPp(hi*Ir?d(7sdGkbgG}>sxMQpLqW#Cn)R@>NV1ZJwkpx zpyaqeUmA}ny8d*!S+69tay<6c^OSxkGY>XK-WFc4d%29$L*6o z5D&^R`hB}8C!H19U7I)Vy)A`3LQ4NPAKFhgcgX~V!~IO1%(@N1`1DEj#WuGc2~eHY z-|6({Ru@lRne#ZRd2YHiUl17n`jj>23*tl_?>(0v>bLFNdHKQ~p`5K%sa-+HHAaQ~ zAB@3YFb1DG25*JW*t6n)?@1GK#ee^CRQ!LqR`K7zy0AwWz4Nt6xzKxDz8RK=x6N(M zSEMFs`Ec48y~DK$`84Mz)3E=38y&83ySaAQ9j*}$c-rpA3u4;t#}w5)-|okJUh}lw zkEtH^^a%Ik{zhEbBMinN8CjE0LCi<+w^fsWX})I0+8lI}5B%=Lqt4+VXHvOj>s@Ko{4{A6;Ln@f}){vKKtA=0E zaH57Zf|{h^poVIa+kd}?Z5s0Jbh3s&(ohX_`=6n0d6tG#HB@C%|ADpX7i#!yO&_k| zR1IIQ;XxX{PQw>!_{Q2af$Ix2y+Xqd4cBUTh=vzxNIJZYna0_eX#%O~8dAypA`N-@ zV{5<*37Z3&*B_=KFRb6TJ;LYpY;eUsW3JdYqhjBTihZ*+!Km0byD}K|2xeal_Rql{ z2=;KWe+x#{y_dxvfw9(vJ%Snb2xizLnDG)aTNLb!VAvx#9rg%L_vgW|M=<+LFzgY` zut)H+*dv%t*0DFk9>MI@!Hx-rJ%ZC=kKlBiKeIK#utzY%9>M9bM=--4!3=u@Gwcz} zutzY%9>ENI1T*Xr%&=DecM=--4!3=u@Gwcz} zutzY%9>ENI1T*Xr%&99vI!yds5 zdjvD=5zMehFvA|f>=nVVM=--4!RfF^FvA|f40{AK>=Dd95Da?+GwczZ4toSM>=DfF z3-4#hCPBA_6TO!BbZ^2V1_+{8CCaY*dv%>k6=bM zy&3ihX4oT`VUJ*jJ%ZU_FzgY`ut#t@>=DecM=--4!3=u@Gwcz}utzY%9>ENI1T*Xr z%&ENI1T*Xr%&=DecM=+xz-i(TPGwcz}utzY%9>ENI1T*Xr%&=DecM=--4!3=u@Gwcz}utzY%9>ENI1T*Xr%&=DccgJF+ghCPDQVUJ*jJ%Snb2xizLm|>4#hCPBA_6TO!BbZ^2V1_+{*^^cf zQ>ah{F-22V5YwO6ef65(;q}9pFhA;tFVTCFRFo73R_6NQ8#TJYZHp^a*AH(}s;8(Q zeqYBfE!R`sPQ&Kq-E;0=b!=Iip(<#&tBHSJuO@y(Xw+)rS+}ctSgoN~maAV;cVLv( zALPS`DvMdBT)pLKR2F|tmBn|=9hg$;nsb*%@0xM%jIUJ&F6v+hcPi6VouGB>&`4Lc z(o)@`mE^-geYR%YooDRms@_r_=-%DD`Eud5>vnX{`B&9Wf1})dOV^y8%66-&#M#b) zV^kSVUH0QW)t-*!)$P<+&$+j)=S%EQ*FZ<9)=s%R=byUTdLHQ7a8*7uT^rul(XUl? z&v~qE0F79^MSFEyy~(>;=(oM-a~(-*Q5S0Vg;(xd!v}$~ZuK^yTl81hBi#GY#M9dj z>f&>gY+OFHioSB-jQ8UzI;`DO! zZNAS~Qui6pw`uwD(S=inQsp{mn^S1Hev$eu+{Lx0WxAxdc^^aF;}@Or#O~eVY=s;q zc9FdKstz8@yEmc5(a4-z+Ac~;)+K9-Rb5p-D5_G>$Wmgx=9gu~q0LyOr&M0u96y5E zF8a_fckiYr#>?1HJc6l zn;hEKO;NsW)BOm_2dc9A)~RfWy3Q-VWmpX~vH4jl!?&bkHT!}*_8dLbo~gfxd%N&QpYv)pocX7`I>*BhV+Z&-&6OXH}fU>_hbhBGxNtQ zYM^K4JTETYke)fL_O`Jny~A?)=P2!AIsH*Br#0;;Jx^bcpOfm1N?ThF(-+LQYWi~3 z2Y)j)m8g*8Q&as5aw)lHn%e94vEOPJNh4Cy_;-1(ig=r*f0Wt3HeZ#lDXnUHP0nli zh`ih@-%zczm*r!@V|m%Z!XBZpM|fCmmtQRXHJ9L%^$PfGReJCY@~^N*NU6VmFmyw6 z9o;kH=rpDJkuB^I((q5b)R6J~`ba)}*QUJN-oE2YFU%$ryL43l zE?6_cna&&5sraFSi>B+EiIE*Q zd^a!m;nD%aI&SbX&(O#4quXapCc#|y`&rSZQ8QB5BQ&LF>fj!)qD@`)HHMmDQ$AdD zdy5RLe#Y-?IUv3NsXKdU%{SQZuCoVf22y%UF1Yo@<#=by7o;D#J?)gzBQ6?Ok)11J zlip?d;-VlpH!r`>uw$@d$!WdkFImyIWLYiQ6ea7-$qIXf!X9A^bwgo~(0+;TlauwQ zSay7$p7@vRiGTM9bYXejlD<2aSx{l(?6EZDskOi zrPRw=Bpp8fc;)|R%Ks+iU++g+wndAVE?lzgw8iJ-+naTqnd7|Cg%qz=08A&)#E}Ro zy#Dbc)?e5o6!r+cOHNZMy1kuJ4Mm3&O2bRsg*`%6S9Iv$pstv%A!-Zi_)%9Jts&|P z-m|DH=p7oB#NygC?{Cx<{TiaKpe`RZ#T6Qo?vom#rnpH%)D#pqQB%BMMN!lgTQx*Q z@ly>C*YHT))U!0aSPvJbX)eut6?1a>V>M)X3W%@MkZ)VB*YGcE(=X8pzCn3dMZuU= z6lSO>%urF7tqF#T!tBams8`HB8*EFkZNacda60S}7;8;?XCjG? zy(Sp;2xg}Q!ydtGO)%^c%svq8%3vQ4hCPDQ(I|wm)`UHR+5N$=M=--4!RfF^FvA|f z40{AK>=DecM=--4!3=u@Gwcz}utzY%9>MI3!L|g$9>M9bM{qjq5zKZ6!yds5djzMO z8w`5{GwczZ?&M(DBbcGZa60S}%&=B#}djvD=5zMeh zFvA|f40{AK>=DecM=--4!3=u@GpfGKutzY%9>ENI1T*Xr%&81@KGhdqMR{dF+x5zM&Xoep~hGwcz}utzY%9>GkX z=W6T)wXul_djzM$9>M9bM=)Cy40{AKG!{;WJ%Snb2xizLm|>4#hVH@)djvD=5zHP6 zhCPBA_6SagJ%X8@xiyA8g4w)a#|6V4!OLQg;B-rZEe$ph>^;G-NAR-PBY0Wt5zM|H z?9pJ@BRC!Q2u_DRf*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO!BbZ^2VD|A~ z*dv&23x+Dg3{{4g#U8;7djvD=5zMehFvA|f40{AK>=DecM=--4!3=u@Gwcz}utzY% z9>ENI1T*Xr%&=DecM=--4!3=u@vjxGhM=;wI z40{AK>=C>y_6TNrdDa;A2xizLI34x~X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO! zBbZ^2V1_+{8TJTf*dv%>k6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBA_6TO! zBbZ^2V1_+{8TJTf*dv%>k6?y9f*JM*X4oT`VUJ*jJ%Snb2xizLm|>4#hCPBAHGgK< zBbZ^2U{+D@0TU{!la*(|ijJwLlus_7Qd#}OjVh$b3x809E>tg%m#dFfwry4y0K3c8 zodd6_*&3`F9F%MSje1k0d5_&+Y3tlvsczq?DJ``&Ybz?%d#F2|tpWr+Cak?kBb9Y` zwXLOyqFU6+8gy0flmugK$25(vyS;7Q4O(8aP9(+J4!-EAPL?{>d3$q*AJpd)eN>;8^z{oDsTMnY?D>1exi;EyYU~{ z;+CN;9)1dHXXW>`6YFk2uk$zCk%x}{&6clJztl6}9xdio-#l$z^~~w>s^`v}SABr0 z_}&L~D$^VEw^*gPmej4IhH|VWR|InxwWLidYtq?{ZUp_0@kmSR(Eq3BF>a4X^>?7& z@ADe!;HN9a0h+g$hU@qenSZ}V9sHBkzfaHKN9go;dd_ZF(@-6GW8M29Iqfo~JtU`n zVpv+&r_%}jk$K+zxiwaw()xF#f&_nlEhX2NQ-?nz&wM1O9jZT{F)XcHZ}a`;;GDLk zMtI%rC&cywOS#qepvKx`9noB*o_r=L^Uuk-EB8oFyE5lKyOy|~JNNS*$!SLB#H(l@?H8t*rIbxG>l?SeA!(L+?dbOY949CZpXE& z5ucyvbyf$ewoxN)D7Kdif4CznSFX2z#7#g_F4p0WtkX2zsTGdm$jTMxTt-}=3 zvv^PG6RoMA_r)hQoUh@hG^9n<0~&I1eNRKKnkO_oUc+B$sG8gKX${}4;k6pxs^K;b zX}7Xn!xpV)zlo`zchLSCzMwY!66MFY4u30StTo?dW@p#twx&OcbbK{)x(@{VP_T~& z`(&^!!L|kada!Q=dpOv?1=}60S$pdB^3}}SN6RELS|*wC)zXablxC*}qotP_ZM~fC z=3twGJrwNWU=wvLy=+@;tToLF_KIMq20K02yMl3syk2#xUdxZRM`qs%c3-f`IxAl9 zl-gKpnjP$@V3b6h?rp*NqGd+wBCq4q!Tu)LcY@s)45u3}iwe=p9vy6Mu&KJ8oQ~3u z)6EQaM6jOfKPl|ET4z+HX6~*YMGf4KIWpkRFK;+zr{-}_YIKkSFSqh>1QQo&x#Kpx z{+@Xoj@iMCX_}$&?Hc9Dpz?7p`SU<=$zLye~(9}Ym|qk5;zEuf|Fg?QrWh)qhBNZavW0MC?*h3kG@qo9?{&!&8iphf8*`C zMcXr=bJ&vJq&??sv`}WvrPEXN#q2c=rPI@uN@0KEZh0D|(~)IUYjKoI{=KzyTbhsy zAUqUHDp*^6Qk$(+jhtV}^ZT>`{x#0GqtH+}vvK~hQ4pA3&q8aZfWw*Fqx82TlFw{N z&N<&B`PgO4!O@bBUAE*p8!F_Dcoo)58x?ENsnFl@X^Y-RNb+dd~d~Ir9M7qbyi% zr-|@>Icaz4)4E#gi?453oafWiDIfCe%AN2ipI@(h*r_F+p@N^D_xT3oCZwQub!^Fq zky=6TdO;UCT+q8Qrd`n66w@x~ZI5Xe^mfFw3wm7o&Tm9PZ?F-&ptrLTyP(JA>P$vl z-ITkyLWdXhxcr=S#MN3a6AmxvaXEEr@lgtTTsF>S#8pNvTq(l~dRzyi7xXyA4fgt+ zMdv%RpvSo~A70SQjp-@bRrFW?@m%G2==GP5s==43!5AD*UJ$!Bla>Q zfH%7!*gJ!LI@sR?`$n*PgH6!2?))az##+Z;vOZW%c>PHypHhB9 z`J~z>*UNO2X`IH9RUO7a)5G+())(%s&4c6WA9YW^q%zR5Uw8FR%0GRT zfl2!-H*QGvb{oCm()%s>ov6(j)RyrroAjN0&sNvBXVtpCxThTFBLqr{pIK10y$D`7GM@0+pV5(ge9Dw@nXxZ7-uM8at4n#f*ZP(qL zi<-yj6S5NbYjKXje6qPHALx7_)go(xP}_B@I+l}er6!o6)?Kyz)5J`r;#sWo{;MwS z*!0SxZ_ym8FDHIO$25-giJI{7Uwtc=`|y@dTkzdDNQKcBSy84mEsqhI($ZTds*+9i_d`_ z^)hw;`m*6=SbsA`WA5Kcdh3_l`7U}=23wo{UwSMToCY;bQHA#a?U6>7Z7*&*A|07s zvC!~4{QvvXktMrqa%6^kAsvaT`-Y_6N#_5|Ayu#_0~p1;N@k z-oAz`I+aCT8ec*hUq3R-8xmG(>$-3yzS`hS+!w{NK}eJytSvW8ri*K+YWS#ooY%*~}~KF;Z*pXy-h z^7tWbF4rr?K~*~C%baWCJ$bs?#1?xq`s$lOGtOCg_V=1zq$U{kfTvN8OV!U0lvb%r zL)ok#8=hLMbw$2x_pgchwX38~?=H2gb?Bs$uK+~;7w2>bVw*gPP_ToKSMK zv@@n1Ej<#`j+S=CG|$%IU$j@`=?U5*=f@MPy`5*Dr_a}ob!eX`_bl{0;o12cJ9n1T4!<|~aLyg%?Hqo;wQEj)%*kKh39*Qvy(>=2`)|&X@ZT5m-#{}aQ;1IvtN&vvY%;AM8WH{wmm(VB3QIBp8oU=l5J4 zC$9r_sM*Vc9T}_~tUDO$P%nE{u$9491^cUDR|fmrV1FO%pMyOR>=`;!UdO(*vDS2O zutS3#8|;K&XhOX#U(mg8d@VOa8({YBVE-KKp|Mb= z7VHzjwg$UB*f)aR8|+8H@YwRUJP`~PxY>TSv5D!q!SLAf_8k^%`h+@rNo}l%(o+$o z7g4(2G^)*_r1Wb=>1f!0|55q~7m^}MPkQ|hMc+k~URP%oQTlMLT0LUz*S+nZEBz?Y z7g4%`#v)4BWl==w_3*Zc(%akr2Sn*VRg~o@oiRu0W<=>`MCoQk>1IUfW?O<0rJE6@ zI~`HF8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr z8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|! zQMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QM%dZgAt{h5v4mFQMwsXx*1Wr8Bw|!QMwsX zx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr z8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|! zQMwsXx)~L^MO}JPmtNGR|N2q-etLrZzN7Sp-YSbIy@=9lm5dE-sER1Ph|(Q;Z&U%= zKYy8^Ca|bWPeomNZW887=b|qCe`HuN0e?xlx{|pZbp=DMwD(w zlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|p zZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=D zMwD(wlx|i;>4oCGP`p3Ay7c|^1o{0(=|x?-3IzNda}-f}5vA8I1w^og;=NG37mD{% zp?KE|0s;Fl@2CHTiucD9WjRV`%u%`-QMwsXx*1Wr8Bw~~mS9BbW<=>uN0e?xlx{|p zZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=D zMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(w zlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|p zZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=D zMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DMwD(wlx{|pZbp=DRz&GV zlwL&XPcKS8Ku?h0ZB1$;PE&sX7_L7@tnhf31G(;#n)!KAm;S$^ zF8$|=|uZYrh5Q->$=u#-^(u=zEqAp$c!wE%QdQq27Pi((Xl;tR$F-PfUMCoQk z>1IUfW<=>`TY?d#n-Qfu9Z|X&QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr z8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|! zQMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr*|&ocrJE6@ zI~`HF8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr z8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|!QMwsXx*1Wr8Bw|! zQMwsXx*1Wr8Bw|!QMwsXx*1WrS(o~*o))2cLx<_~yQ$8%7wAe?f9dKc{c&}3PA8LH z0}FC5=yZF&JWuy&y1ct2l~&_Zx5We1j*=)(Fkjc)i{$41SmR68fr`}p-5ALu@a zuF;!sxv5iAYX%1^i?&xb-`BKLQ|c%CqKR+5x zbj@CVO#Q%bs&3K9uC6(|s=HXO`qi$1^GmfQ)!+8^t~s}h-#&L>N~vqk9UA@GjC*H% ztuk;?-nG2F)d?Fe>e!Swywallx-@yKCbel@)g4{cTgn4n9lM)1U;fzR^o_n;J;m?n zo^!vp?e22(EnRa4mFulKFzj=ApnG>&N%OKk4AmXljvZZd{#D7o(KT>XQ}>*Qwd@mJ z>WH30pf2xMcGWw3pYyA>f$Mac)7d>8j#h7@t91&q_1?#o86C*0yYsD;%QzqGL%F&| z>=ylP3O*=a-KMVQhcB_{p6a$r^A#O8vaRYdy|Uqoj;qOF*wTx3&^h^f*1oo*Uo*RA zJl0+PK~ME_9oGx?Y`Ci9Mo~JbzeK}_CZ67QP#33L$42LSmCnSR&20lW=y_3P->Tbn z#4gHLUwQZK99=HUhsJq<_vI6zgFNS3Z3DBlOjq@OrN1g)o#owMerQtLL2QKcpY?5R zaHkeu_p`Q(cj`G=-P$$qC%Z2%SAU+bM_r0KBx{vid-V82lTSZg8^>gA`R-Xi(qX6P zwso7@E}pK8=zE>rnmf?)!V0^Xlm4R?-=yo5qq%5TrMkJIKHS;c%8Xm+-2M`l)S9+u zbcYTUE7`3J{-TZ@8r!^c-0n?X)t%+d-yhebYyBRv9^H_n@2Wo1c2H+}?LflD;rjI%8bvS!;fZbuGyW%X_S`S8}SCtqRn*8I5Cw(jiRyLYpFY-iO6sD8e6 zC33*4KP}JrY1`UQ=QZiBK5o%hUOO;lJr4;Vj5l=joiH%v;@ah^EnaZ^z=7*~ba~J3 z(axM$K6C4VJ&QJRoZGHj+c95x_B8u((xYQgsc&=rsBf<{>o~JLJ~H!~Z&$L;uIg6= z@($_f%+}6vp9~!nrJ0tGM}523=I7dR)jQj+)5XbA+RpCyp~gzD)+ATKMh&$=mocfV zJ4w2!HZeP2th$+)GgCAAHDkkB9n+h)_=eMhS954$^E$PzPSg(w`ntey?-lVIJ8Ra?K+AiApOC?;TTaNoohpUej zZqxO#s(zczxj8?V+Ag|SN!iPM5$o8nwe=@8M_7|R)!Ln{+ka5gJSaGn^Va;ZRKMGY z?tW*{wyq^-tb1QwBJNV^;aws=sOfa6{?5{s*OI!mc^BldNi8WIq`#JQ;JB2g<*^TG z>^XVtul3?PGpEyJdQ^XpO{S~jKcu7V_+6t^&)3v@G(1MVSNqq0`l)9uUb^&EuR3k< z@)dpOFFAK{@2r*j*Smc2IlYVe7A{-1_^jTypSN(?iX|5;T(RWrWxdN6p4R)a-lOCn zG15We_*|_^KidA$y?43&Z0*BKGq|v{IgL|6+P#98(opXW`)E&+`t7D#_?cc<|Gy;n z?R}<}|GxfG+Pmei(-%q?HoZyZfv()W^iW>(rtM`v%ppkc%F~=)|1QsYy-m{(XSTcf zL+SIS{-)REyiODw%IlQqQ~a@<_Oa&orVo}@Hq9-ujwP9q>*a~=OpQx@fS9YPhV{MP;+}6o9Ml}zW7Y1&5Q4^#fNsyi$B=3 zS30D5V%j5%p`IIOH)OFq_T(wDI6wNa>ukt^8`i&(SqwQGj_5(W-Ydg5b*-v%@Y1W& zU}RmXWyJAe2YF!lH!^F^6Frct!-rV*#Cp=&vg^lKmd&CU@KPGlTREq!qD3ywm(HP) z?fiypd2ai+=O!(GZq7EAT~JS2+aw-A{(0HWP2`P2{G> zOljZrUH!SMg_WTE^C`UkN!EhlJp1g<+v_>>*Oq4<7r~8Vp-C_7NVZr(K)*rk=-Q?*=-*qyM48hd`c;$6@BL{)H>Ek+ zBIlUW8+uPT$7Z+-a{ zy^ELmX!44?IPcTMI1awNKhrWTt!a6-@?4_r@madBHA8#e7kkbb|8h>-k@uW8?IU^mgQaolOZhE!cASaUd^A_3`*oD(wsT8t&1t_k zZf5#=+NyPRXk81muKW=F$Y6(-)=#^IE6sd|jH)^`w^Z z>Cxt)EnSwkln2H#ZQ0Je>^j+|9G35bWpPbS)@Qb(+hU)|ylLPRl>&GI?%kuk=Qp%%KCaqPCX$i|lPVmq(hXrR_Oq z3K$RNHMf*n(>ducv}V?b#>*G=el0IkA?VeqssmMSe_I@c=jJWw*Rpr#E%-*u4cdZd zX$$^HTTp+zr=;SS%4NRs^P%GUzdf(=CHZ`?s`|BlQ2OhZ=fpj;toFFbIV{Ty_i0_@ z^ZJg_cGhpMVUL#HK5a!;`>k5R=<`?P`I|I<=)|vBa+;Q7|Hz+XlG0oA#`O)m5nq}v z)y)~}tZ&@er}ui>IEzQ+W%n7j|KxrCk`;YRmerE+G<{W0c5zWp5}3BZ8(o7-&^!Ibf*lxLODNw zkP8|U@{@pXizn+ZrPq%?LQlN+>WTN&5if#~uh8zi>@{ihSLj5&4*Uw`X}VFl@}f!U zmE-qU-tSP}pI6@X0?TOaY2Th-c+JBu{WJ49;MLDpsFarGTVc;v|3tm|Us3XL^|5$R z$6&lJ!%Gz>=_N9G7mM%lo7UN`LP%fxKAOWt`vNW1p+7FuKhhtU2WR1p`g^nf7V7T| z{q^Z@ss5fJX)QMc?~xNq1gGt}l<#Q@I_>Qz#(V4OEgWuazd#gU}?|K4}uQO%1JjW zDS-rr(!&A?R(N%+a71u`*Yx$wQYnX}mMmK_>u{BO+Hy6Y|QXNcf00n zk2$94d0>R*EC$K;awVDs^Q%l za=O2v;pa8{riPrIdzr5{$#*p5ZS-9Yf2!d>Yj~t?-$4!kN{`?qzQac1N&p2K&cg zP1-N7mm-Xp-8a~?gK<2a?$BVz2YX$x(}Qt@o!^DQh%wAQ9PFdPt_}8?VBZOLU$7qq zdoSX9_&}aT6IRf zeN?ad82q1L{40ctpTZ3&6_DHaQ50)m>x38r()|#dSn;ML2Y#+aw!QL9|)L{QN*ag8p80^Er z^q#Bb_nBZ{3ij1t-wviXL+v`)d+++P`_#r-(~E)~7OWhsJJ{L5&JDIE*q;acP_VxW z_Ss;68*EFkZNYvP?B~J8x7N40wKmq8W(9jiu=fW0K(H%=eLUD#gMB^N&w~9t*!YRD z-r87eIyBgegS{%)(ZNm*_NHJL1bb(&_Xqo6uxo>TCfHrUz8UNX!TvSa_(`$NwXxRp zXt19K`+`d5z9u)<##+-?f_*L6(dHhi0XCQPK8>eT2t$x%)~mb=VMrki$@;W&3t>nh3@L;mqpD(7 z7s3$UoM}%Bw?Y_F2t!n=vPz{8h7`h(k;0Hos&=t3gfR<4%r4a9#0+7G8Nv`Vgdt`K zL(Faqc1JLTAx?)d#OV-*m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()MvP&G zFvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop z3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXD zh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F z!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5V zGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f_78?I#0+7G(;*BoLl|QAreFv| z%q|FqFvJXDh?hkeVumop?51D{L(C9{I32~=?Ll|O)FvJXD zh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F z!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5V zGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!Voir zA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`Vgdt|FwJ{4r%n*h+9l{VZgdt`KL(C9{m>~=? zLl|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_mewKis9h#A5Vr$ZQGhA_koVTc*R z5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`V zJOw6H)M{Nm9xFPgo>D%!d`e~Y4>ziyp;qRD8gx~EBCC*c_0h_<&2j|TU9RpNcumdf zW6j`T&!QbS>S>?mJ$8Skt#h*sK6YwKOReolddpX;_t3%|eQ&qy71rLVd6ji{wXNN% z5w)2=Q8T)#cVZbZQ=_s<;A^mER6KPF={4D@NsJWMYr0(hDoxM(^XNBeUcXX{PtPg- zIQvcXk>9io1s=F zJAnr`iD*V`CuoH)pv@D@+bR0l9m!#|&*k^K8jn&uxJEj3E1vHo?sbn|YQKRvJKXKJhFc3AaB`LwF8{|pqH#sf~ zlq97;Q^VWov5}R`_-~{s=aV&ZXLa10tZtMB?e+bi-;g!;HmCZ@8mp5v4+OIQH5;b0 z2oIE@ogdN8q`o!F8@6V8ZEL3KhER=P9FUa_>3AHB-r$>R9U7;fnU^?F&VBunOMgQy zJY^dic-J>qhq6*G~wF8-xa#&XgIYor`h8_HJqX`(I(Ac zlj@D1w#T#^KkbNVH-5S&rrr2yFs9x3iMz-7jcEMDT`~f@@zbtG?8Z+z0JYR31ORL= z7xeJPPh72IG=Aa&9p3n9nx@BS{KQq}%^Pu1vO+E_EGe7yBN{5D+*E0P#{Q4@0LOEJ{(5SB>OQHTcf)=fl6HR$|5U>x zG@PrI9iU;8mOD;E-XF(n`25=ROO)=tdi(eb3}dZnZ7{BTv-btNEZFtIJ{N3Luv>%u zQ?S8cKMcm+cpaQxZ_A|GSZm@unjIMIh+v!#FU!Gpx}IP-8k+q{u#W}%M6hoJyEj;i zj*pj}P#bGa2L|K2i`Ux~tP+gxDYf+m!?V!r%3ypuG5cn)aXJHLd)LNV(_z707VOQz zdV`%C?EGLK4))PtcLuvFmgJI#Nmybu`rME!u+W%%=di&`$_@5@3ZsF=Q{i^z_lB@Y1_r z@i}MrEmuL6OK*JPrB`_A6<&IUmtHEo z^a?M%!b`94(sLX|MbKsXSTVdhX``mcs7|_Cvs}G1qB^PY(ks05@Xq^fc_UY#X78_!O-x9p&29*WWZLYuV0Q%jW-uhvPKRWgv55)Ev>B3VGbGbyNT$t@ zOq(H@HbXLPhGg0d$+Q`gX)`3#W=N*ZkW8B)nKnZ*ZH8pp49T<^l4&y}(`HoNnIV}r zLo#iKWZDeLv>B3VGbGbyRPmW1nKnZ*ZH8pp49T<^l4&!l{mhU|n<1GtLo#iKWZDeL zv{^YAl4&y}(@wWA7;+;sB-2iZWZDeLwAn|3A(=KqGVOForp=H{n<1GtLo#iKWZH}> zLo+1PW=N*ZkW8B`4Tfae49T?9A(=KqGHr%r+6>9G8Iox;B-3U{rp=H{n<1GtLo#iK zWZDeLv>B3VGbGbyNT$t@Oq(H@HbXLPhGg0d$+Q`gX)`3#W=N*ZkW8B)nKnZ*ZH8pp zZ2w?Lrp=H{I~|f~GbGbyZwiKF+U$a0NT$t@OnX@*(`HDf&29>YWZDeLw9_G(Hv4%n zB-3U{rkxJSv>B3VGbGbyNT$t@Oq(H@HbXLPhGg0d$+Q`gX)`3#W=N*ZkW8B)nKnZ; zVTNSd49T<^l4&y}(`HDf&5%r+A(=KqGHr%r+6>9G8Iox;B-3U{rp=H{n<1GtLo#iK zWZDeLv>B3VGbGbyNT$t@Oq(H@HbXLPhGg0d$+Q`gX)`3#W=N*ZkW8B)nKnZ*ZH8pp z49T<^l4&y}(`HDf&5%r+A(=KqGHr%r+6>9G8Iox;B-3U{rp=H{n<1GtLo#iKWZLY| zU`VFTkW4!rl4&y}(`HDf&5%r+A(=KqGHr%r+6>9G8Iox;B-3U{rp=H{n<1GtLo#iK zWZDcbJu@WJW=N*ZkW8B)nKnZ*ZH8pp49T<^l4&y}(`HDf&5%r+A(=KqGHr%r+6>9G z8Iox;B-3U{rp=H{n<1GtLo#hv2tx{CNFfXRTvP8Ro9`0<}83@N)vmS027Ek;P`3t>ncslrRI5Qe0plSQ|?TbLgqMJJ1M+n4iz$Va2-WHBFx zLKspALkeL?Aq-KI-=dSnqLW3nFfKY-%uT2ZVMuS$$>M*RP8M%bHNAx)j9D0BcA<)K zW(Y&f5Qdl`3^79(Vs=}wJAxq$aXN${PKPkW3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn z#0+7G8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_ko zVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7q2SXTQhA_nG z76wBYVumop=@5pPAq+A5NHByUW(Y%^4q=EH!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G z8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R z5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7?7-F`6FoYpy2t%9> zVTc*R5VJQ0Ll|OqK`?|NW(Y&PEW!{ogdt`(1w$BOhA_nG5QdojJQ%_dGlU^dhcLtp zVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G z8Nv`Vgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R z5Ho}!W(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`V zgdt`KL(C9{m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}! zW(Y&f5Qdl`3^79(Vumop3}J{F!VoirA!Z0e%n*i{Aq+7?7-EJn#0+7G8Nv`Vgdt`K zL(C9{m>~=?Ll|O)FvJXDh#A5VGlU^#2t&*ehL|A?F+&()hA_koVTc*R5Ho}!W(Y&f z5Qdl`3^79(Vpe$R;eUf4U*Vw4*N<=9IPanXO8m)?`szZPD43KdGJ<_K1J>7@la z9{$3v?+57M3rT0F)hx`SGF?;Emt=IK!hL@fhMg(Tp z2g6IxY*VmXgW;v;We0=(FxVr(C`&jUUV6@t2aOqCdS-a(navJ{m!273dQOLzo*76!8U!|CwS zGs8>I3@<%1y!6cQ(lhI+{*z3luhzM(St0kx6B=vE@`ht}3UTH%I!L*3y2g*gA^N>a ze%yxF-!pH+F*}$sO*1sUU87tFDrRPzXDl>@ZzKuVjK;5LT#sVLuV#FgveNiQjdFo$ z(N2x-^e8URgC51@`5uo>*XRz9+U&Wq`c`&eOJ&>Ij(&~s%ihzhfNf)9xyBne4xT-z z**Ye&2Q{0=ME0O&`rg`OL=5%k?>CL4z^#Ql(OynHFpK z+c0JxY0HdMYhUH=-{R7(`cg66ikbOCR?Lm_E7=36Pg^$n@~7nabbC1Z{A0s0WqLh{ z4m|d>oZCH&kH@xE=Bn5u`PfCl!O@bBZL7?cIHuIiKF<{;@z2}OWx{cMDhp*UEwX+Z z7Rp?jzmA16*YndX_HnZg?dXW+#dROy<+a_UELAsZW@~twJeh^^G-W-Og|ZJ}e?!jG zwZtBF$>nxFl+s5`So-xlzGUF)rIMv9ig)8om%1<@T&9a}PDq~?iyy&y%=x+n6DF>O!eO)+gx4FYj_3m%gV*cp?urVtXR* zY{d3N=5lo=BZQo^Oy>$6?uo3kJeDUiSKRPM>|9P`G-Br|>=ll<%2**+3RhFz6PfE^ zbWdbXaYH$dv*>(BdLnbK%!hj-H>DHWf1>@X=&%0cdCTF@7NnzUaIp-Hc?O@Z;VU&f zL&IA&yi~(4YxqtLc|J4!5>2nsQ{9C$##+<6g1x&o)|x&V>|?=h33gks`-A-;7!_e& zM{8}YH60m@P5`{#(}MK{dvCB01iLNR9l?Ga>?gr6b@qA>sg1R!dBKhgwklYEu=VJR4YpSAS+8S#ZLC#IC3-rh(YwUd z4R7d}{Q8qlKBfGI@=3K0H@FF^pXykkBQY>{_vPwwAi z({y=v*M?7ZT%m>FD_LrJiPFPm8r|@@j*XgAsdlPUkF^~<4+gqxJ$BsGsi`%CgOx?w zE1U0Y;%-&{8y9ulqvh0h2ZfO3-E$sa^%}3^syw6Iyp7eZSK7#kXMbyuHw@asx1mrzobtDCiC(Qd40H&(P8 zE830m*ia`rGrKmNrrvbAYQ5KqnY?mYBwf3;g`H@WUxv9J43-{VJKV} z3a1g_M$v9eMKZUrE830a&k4Qf$!L0Z#Q6O>KK|KP_eMRD{TtaXar6%jJbqrQC6=hn zPgFMAmxKM&-1bdl`xx1c_Hlwdr)c?K<~)jaV@12M z3rY`{u1?oBU7xR|<+?7e@%1(SK7HQrY}zZW8qr*!uXpjX)AU)Y+|FlLK2D_6p=;B< z?eSqFr4N?Ir7xvtXI8goZf$vEQeNr)rswIJC6?Tp(|&K<%=GoNRqN`|x)x|%ll8Ta zPbn!a>pgYx8B3PsoTh5}&P;DEO;AoR)y#KTELr7v>PkhjTP<2igsgqLKN-B zM!X14={?~jY83nm?aqaiYtr-bbp4@U_vD|b*TIz1eWgDwQDE7q2Q4p}lwLW0f9b;C zp$Grxm3O_slF~Vgmo8kg?6k$_ME-B53Ntn5iYY&t&&>1qnm$|KT~k`B7g)J{sUGW> z0KOuP8O8;}sgdshy+`s`b}+RXSe!Y}cpo@paGNG}WiPD)qHXYntXO z^$WCAN6~JKj#&yr;l&G2JGW>zmj27M8{4A$+7)7qx!suAg}QysXg6j?yD>A`jhWGI z%#5lrvpa&(Zp@5!V@^lAF*B;m%=QgNyD>A`jXB++!Hy3`b+gl*9_($wE(~^YFxrhd zKiZ9X*=vJ+CK&C;oQ`&5PDi^jGun-rp{9pW=6X)Gun-r(QeEPos${u z#>{9pW=6X)Gun-r(QeEP?UNbp#>{9pW=6X)Gun-r(QeF)c4KC=8#ANbm>KQH%xE`e zM!PXH+KrjfZp@5!V`g^;qurPp?Z%vrc4KCI12v=Fm>KQH%+O7l(QeF)c4KC=8#7xP zjCNyYv>S6e+KrjfZp@5!V`j7)Go#&@8STc*#_82;M!PXH+KrjfZp@5!V`j7)Go#&@ z8STc*Xg6j?yD>A`jhWGI%#3zpX0#hKqurPp?Z(V#H)ck=F*DkYnbB^{jCNyY`v;@l zm>KQHoQ`&5X0#hKds8sljhS5#jCNyYv>WrXv>P*{-I&=;!Du&TM!PYmqurR<&x6r! z%#3zpPDi^jGun-r(QeF)c4KC=8#ANbm>KQH%xE`eM!PXH+KrjfZp@5!V`j7)Go#&@ z8STc*Xg6j?yD>A`jhWGI%#3zpX0#hKqurPp?Z(V#H)ck=F*DkYnbB^{jCNyYv>P*{ z-Iy8e#>{9pW=6X)Gun-r(QeF)c4KC=8#ANbm>KQH%xE`eM!PXH+KrjfZp@5!V`j7) zGo#&@8STc*Xg6j?yD>A`jhWGI%#3zpX0#hKqurPp?Z(V#H)ck=F*DkYnbB^{jCNyY zv>P*{-I&>UwZ<}|-I&>-!Du&TM!PYmqurPp?Z(V#H)ck=F*DkYnbB^{jCNyYv>P*{ z-Iy8e#>{9pX7+<%v>P*{-I&wSZp@5!V`j7)Go#&@8STc*Xg6j?yD>A`jhWGI%#3zp zX0#hKqurPp?Z(V#H)ck=F*DkYnbB^{jCNyYv>P*{-Iy8e#>{9pW=6X)vqBhB2tx{C zNFfXL2DrlDTE<~Fr=m)Kr!rJAq=7Bxz^645Qgm8 zCWX2i)ZD`_mj_jMb4c^VF{HBHo*Y9P%s`|g9;cXgzSHqub_}l7f`c_rb zTNuKag&}4as%&qDFvJXDh#A5VGlU^#w*|W+7{U;zLm1+82t&*ehL|A?F+&()hA_ko zVTc*R5Ho}!W(Y&f5Qdl`3^79(Vumop3}MLsXYWkltEkTRKQ|j8L_%CqP@+UZp@QOq z;*tQ-un8_GRgsGvI5CmeX{b?)RW3_c{YpbocwKYJ3;EGyXwc18di`E5O zZPhCNpXWR?@11*dCj_zU|0bV%-7#%~z=olhK#}F|(hKSKIM2wChVss17#%~z=olhK#}F|(hKSKIM2wChV*41QV~7|XLzIq=A!2k45j)!$9Ye$} zH%7-0F*=55TRMh_(J@5q7sluqB1Xp$rK4kr*aybw7$Qc;5T&DIh!`D1#ON3zM#m5_ zI);eRF+_}xA!2k45u;;>7#%~z=olhK#}F|(hKSKIM2wChVss17#%~z=olhK#}F|(hKSKIM2wChVss17#%~z=olhK#}F|(hKSKIM2wChVss1#mI(M2u|C!6$p!* zwrz_R`(s`K&DZv7GYEc|fGkEfJcfx-RsmEtJPyY2E`VzRIB80S+sjfn!3oy~yY=hK z6K;dw@&FbB_jmZ*@;O*{Q_qz-;Lq;X(_eA3yY=u_-0W^UaT3)H507p9W_c41WK_dB zWuqD{=s&7q{@_s!n=7YOo;T%HaPJI!*oVpUI4=)tjwTeQbJ4$PLScmFcNCbPolqE; zEIXm_a5DwPQ6BnB&iS^3Gt2Q_+`*aUct3Y$W;s3%G6#G!wsZ`By}%uO@7P&h!pB z*WEj>^mJFv^4+hTm=3F)n{JZB`Z@@@H5*lQslBm2@41VY=!NCIA(?g`$VHGxKn};w zdO_x3yK#{Dkf%U$9yhKJM7pn_9nuI4mo%ZU80)>*^_Gr}wAfw7?lIPEti>276jpw1 z#$GqZvr&F*hLw&J3X8GP7UP7%Vw_M|jJK>93tNm63X5?k1BaLxFVKI)UC>8JH(aLoMKkzsbD7?b%AR$Z&bmxz zU8bSR(+74H9=iI=x=d$Xrn7VDnY;B~xGSc*J0t5dopqVcx=d$Xrl(x&X6eH8S>AE3 z)y%p~XI-YVF4I|;>Fiv3Q?f48S(j-#^lpU9^A{lhQ(UH7VYZRMj!SZx7F&sCT#PQ$ zVsx1nqsz1yU8cp>8GFbWU8a?eF4Ia!muWG&OpDQFT8u8!Vsx1nqsz1yU8cq8GA%}z zX)%_t7+t2t=rS!vmuWG&OpDQFT8u8!Vsx1nqsz1yU8cq8GA%~aofuuF#pp6EMwe+Z zx=f4FWm=3b(_%F7iP2?Rj4sn+beR^T%d{9>rp0LX6Qj$t7+t2t=rS!vmuWG&OpDQF zT8u8!ViS$gWm;^8F}h5P(Pf%TMIK$I#pp6EMwe+Zx=f3`WQ;D;Vsx2SI=W1Y(PdhU zCPOj0OpDQFT8u8!Viy^s%d{9>rj?E^(_(a)7Ng6w7+t2t=rS!vmuWG&OpDQFT8u8! zVsx1nqsz1yU8cq8GA%}zX)(G?i_v9Tj4sn+beR^T%d{9>rp4$oEk>7VF}h5P(PdhU zF4JOknHHnVwAenz=rS!vmuaP=%d{9>rp3-SMwe-^%Z<@xT8u8!+LkWUVsx1n`-L&O zOpDQFTIuLAE%t#ix=f4FWm@UzGA%}zX)(G?i_v9Tj4sn+beR^T%d{9>rp4$oEk>7V zF}h5P(PdhUF4JOknHHnhgcx0>#pp6EMwe+Zx=f4FWm=3b(_(a)7Ng6w7+t2t=rS!v zmuWG&OpDQFT8u8!Vsx1nqsz1yU8cq8GA%}zX)(G?i_v9Tj4sn+beR^T%d{9>rp4$o zEk>7VF}h5P(PdhUF4JOknHHnVv>08c#pp6EMwe+Zx=f4FWm=3b(_(a)7Ng6w7+t2t z=rS!vmuWG&OpDQFT8u8!Vsx1nqsz1yU8cq8GA%}zX)(G?i_v9Tj4sn+beR^T%d{9> zrp4$oEk>7VF}h5P(PdhUF4JOknHHnVv>08c#pp6EMwe+Zx=f4FWm=3b(_(a)7Ng6w z7+t2t=rS!vmuWG&OpDQFT8u8!Vsx1nqsz1yU8cq8GA%}zX)(G?i_v9Tj4sn+beR^T z%d{9>rp2<3Az8Xmwm}%_9csWLhkF_2NPJz z&ZU=i4Dl{Ql&4<5Mh^lsJ9faMll)C3<*UpXJ4}TpW_&^4rY2e zhHy!aAz~}h>WI-XM2wChVss17#%~z=olhK#}F|(hKSKI zM2wChVss17#%~z=olhK#}F|(hKSKI zM2wChVss1mDM2wChN=L^KF*=5boo$ScA!3&sqhp8| z9YeG&9Ye(E7$WuyV{{A=qhpBD(J@5q17may5u;;>($O(QjE*57#%~z z=olhK#}F|(hKSKIM2wChVss1 z7#%~z=olhK#}F|(hKSKIM2wChVss17#%~z=olhK#}F|(hKSKIM2wChVss1g)`6VyyRK*ZWJ|JkF&jc9*exj5Ql;F~+&{ zlwX^%*Nw4uC_i3arQ=+BV!V!GoJ&uPbLolkdWcmU<6L@ToJ&vXIG3K-_l@0djC1KJ z9p}`h0 z6XRTZVw_7)jC1LU@m>)d+wgpK?d!MV+WJ!=k8QZ#O@UmwV#F38UO6nbab{%RfaB_) zELnOTl8;+)`ZJ?ejM&H@c++BeJr-FHf*F!|+XLUnB&;)7zKhGAmT%=UA1>fcSY!pm zrWIJ^0}6|r<+)9ZoaOl$E%wLaMlEWx=jz%Pp1|7b2bY#7u)sezZ8LYkW8=ykET~`b+=4 z+WkHYN0#~bv)u2qBYz)*-}NfU{y`Qfc+Qo}Yg=mWcGf}#$p@z;XRTR>9;lCvfzXBJLr>4HtF(>7r>VL7q|jCqxo^@Ox^tXj;P%~8xtX&Hm8C#7w| zs>!U`Vqz1^I-Zt}^?3)UQ0C2=JkgXEHaMC&X?x-A#G~QUB<78 zx_f7rqCIeTChyMP-mX1vwR9;96{SdBT6z~C_c1&NjbD&N#Ok$J7e!&jDbAaerbYJ7q}K5tnZsL$SUKAeGx zjQI?n33&|U=O9}kzXJIPx^wO z_KGnY!nBVff2qhj(imR?p#4raHp|#`#%?gS&e%i7-Zb_XW1Q4k`#sQKD)L4d8)xha zWAOIykL5?k?l$&2W4|}Xu9Wt%i@#Ll4K{X!v1(&ujV&~Gg|XX>{m|G_w6ogBa(}4^ zuYp)iOOI=ZuYo+qJ2|Bt=5PE%CSG5-tmrQ>f5G%)kDXaF@3N`$rXM%^;ss>~o#38Y zU3>5`GNp3f=Wy3nmX1al>MzedBZ9S5SyugJr=S!9GlTIlmw}tAoAq>XC^?VkRvZkE zr`+cU7I+CR!d#Oe?+3~HSdSIeM=lk4=NUWSUn=r$Hg>DAaG5lM9^FBilt-7zKdD4C zdHE+TlVw&WTq^R;#rcWF{Uw!&7|TTI{9)t;#c9ud`Zs@v9Y=9>;KleZSgpdUuFEN;8z@9jR_ndWs@(Pec9$qRA8_5 zy~`GUZrIz4rqw^WaIeZ?PcGWEW%ZRcv6eN9uFl1_RZP%0JXYVluvg`yr0bjgtzF3+!s9y79oO*2yor1iwA)>qHmF;Zv#>w&?(3h% zXr9LU+zr*!HdtZR`xmCVvAztMRW&?5uA!9*3SJNMo9f1K-WFtrdd8~KC3ns0_j2o- zuROZCv5~2(2mGU?PlY$Lq|Z{OLItSF>r>r$PFc&QMHOAE8%LGla*S!KuGX7c2ai(Ns>ZJa_Wi0m$<;G?sbZ4b0sweGeN9 z%Om>-mQHd@_;Sx$Dx_o5be=NYk!)wTBK5iUT&WFop4rfD=1rM1uVy-x9%wexE|@WW z%5jGc#DDm$GweWx)tj^3`o%b#vyr*yHRmK%9d@oNOOvY3c(?v}q#f_lo*hc7=pQzz z`dgL5ek!{cF>ltC+45`UOo$`h?+?2d)hD%}k*tc-1C=Q@~2`!m$tuK=a-Zs z-tb|if=7Hhj^c8cZct9JUGkBr?fCFWcI#7mc)=z^({{xNRX3c}0JK6-GUm9_I0O4S z%O!l!_EnL#A)jRt3Jni;9I`myZ8Fues7}j*&p>x!P&24fa68^KpLgkgV%tqf+m6r7 zWRHK|jEknuzIZyu4Y_>{wjb-#KVsWYO52`KWTyeR{m!eo%mxja!zko1&uwv@JDQrGWns~Xe8E8C>`PwPMtFUl8c<0aSDDf!$*HxbLbDN z%r2Tb{{m!kD2`+f60iq|3m?meF4Pa7aR@TF3i8M3UHrlOjMqEX+Zz|#$16q#KPzY2 zso0uVo4ZK^EkC(f88YhK!e7NJE6N`Bp-FiMUOZpU_kDIi(s5VC5-aXQJ zer=!Pg2ITODvJw>BYs*~T;QCSGh}#UaY2uWtsaZ~X-RSPPsbD&?9L3sCpC>HwcnD4 zBItN5w3%4XywfGKiVJq-PgMW-bZq}uR!jA7mQxm4TFku2JGQh5IgdxFQu7&%Ev2ML z^&4vIQvE7yeT?fSR@=IJ>Tqa#f+7FOQd-KpKdivTElIe7dL`JJH!tsa48+etWhyP5 z10W46Tshpq@@`vzJB7pTdm?GVvQ6H39ERsTp>*LpYFGTiiq0G0l$dUS9$+ju)>pDi zQIgWnsDlQ7Y#X*>r8hpW`>-@udfBd!yho3LV4sqOnx}ol8aD zMaJg(OGVxyW3|S2Z!dqT$m?fpfHA%$ zOXG*wmrgMg1)=r-15Te6)3xJ$u?UyHeX%Tz+NSe^SNeG|M5Lb_DFCB^7~iztC3% zc1&@3@t?1_ytsYEtiVEAh zKv9XAqQWI9Dq>Vr#Hgr<)f%IsB1T0;>8PlPQBe`2q9R5`MU0Ay7!?&UDk@@BRK%#L zh*41yqoN{4MMaE?iWn6YF)AuzR8+*MsEAQf5u>6aMny%8ii#K&6)`F*VpLSbsHlih zQ4yn}B1T0;jEagF6%{cmDq>Vr#HgrVr#Hgr< zQBe`2q9R5`MU0Ay7!?&UDk@^M@rsrEOGVzRxLRlwgVAEtAvZ?67-&@57_qZ>FggA> zn4&-Pi-{g1KEVx&?Tvwlm>U%1;9k-^m+yu>^{YSZ$qJ#rd{k)+^zv~1amfjx-=doE zl(jS=zbLRLJrLEAb60>iU(=-$K=s@9?O3#KBymVEC#~cBgacn z?g`0pi-U0iHEzKrJr2b9I1pQD={Wo?_I+cw8+*vuW5)hw>;q%htNf)5x^8Ff zbsSEsYIt^R!#&Od*Whe)al-oFkA=C8okp7J*n=C}aCLb#B+YcR#D%PIUsCuc7xqIzoeD%Ufw~g86Owz ztY&9oU)3(}^G*M%aO?a^wa{^_JOAb?QoT4GK;u>n&H;zENnN~Z8yet zJ;ALfx8ox$Q_B^!Na@NZZC(l)iR&1fLnf8zVSk{xjzPYC3oJ ztizY=AMRJAr_s4A=|R48#;h`wpgI>aX7#sq8M6l4x{O&vZQYDnymOU>8MAm(hK*Ug zT}?U1x)-2+K$7F8EovwjarYZet4TC?(T{Iq7# zGmzFSmhp7R*^pEY6Oe&&xCh!Y^#-}5-k=zJgJM(;#cGXFITWLEsB}Ly_H$#c#;6=B z9m`bPvc$w#W@1zh#i$&LQ8^T&aws;y7ITWLEDE6>1 zmarJhR_Ul5icvWfqjD%l<9*bd`7cNp6+1w-`XXoibMx9>O-{K@IGSKoNZ9h zHOCl}-!>%dU2`#Z&6SQ_b1`xfpMAG1_CqXpa%2Jw}Z77%|#o#AuHZqdi89_82kRW5j5W5!)&4Ze+uA zFvoC&rs4g-FcY@LaEORDn1*#0DOCaFBBeH_L`rQ+jg;D0RlfmNhMfwPnl*CRh9Fkz zDaA_VKXPLQ zpX4!J1*ur6r-EJSn4ArUTQfkczFAxAxDYF~RIJ)VFB^#!B_89QM?r z0~jl1r;(buuP*x5n~@EVWwh-?#yEAP{INk)eACsaU&r8%%JaT~rBpRAlck0aads!J zh3vc<2s_Qd4%NU>&ScUP8IQsT0W%$_f$8Rz38A#<@lA_Erk4zsnkzy-n`MUaOnjCh ze#Eql{%)OLQmJ@brdwthW;4~L8*Tc~;I!>{hksg@nNwZfvu$4$Y5QWYDZ(u_lT!`jqZ; zV@r%JGj@xyJB(53)3y&7d(+roj8VZ;x;_1+BJUt$hZsA}*qO#=8vDGlRmQ$)Y=bd6 zSaNSg9u++8_iba{@rxMjgnqgS#?CNC>x$CRx}yA8H`y`rJ1ywBe9CH_b1BKHb&8BE zL-GDo)w%<50d@OwNw=>UZ(lL0bz)TO#HiMZQLPiBS|>)ePK;`u7}Yv4s&!&i>%`L4 zx=u}r!A>At>^S)a`*mEBvz>FXySjWRV6;o!VE($3I@>?prEcxN&!ukdztyEKj;rHq zUSk{9IhVTXhTkP!>LS&y9&n563AZ??Uz;bd?TcC`m$-q*a470{o;M6jsfyOwc)X?C z+t_u%Ug#=5C&lRo{_hgbu5%5%-RuC=);sxR(2lS0JjEnU}m z*HIz*6(kj+2O+5t{RWZ>Q5)n5kZ(d#A!6Tx3Q-B}SSm#9?o%N;5RwYfagbDqsv)Tm z(V|0zs0NY>(Mm`vM6~KqA!>r8Ld4EJUDqCjq(bxxBo(5!A*m4Ug7Y{JlGiLyh(1F1 zSv@K)sYfNo9+emsBC%RyREWf=5Gft)VPZcwMukX>3X#&iWQ+=t*gM8(4-=z3Oxsd5 z665e=>|SFmJ+X(4J#LIvFm0QI(o?zv{H1PAH4>v0 zOzDobbX17Ms1PaLDq~cL#HbJ{9Tg%mDnw#bh{Q^8rNnmgmx|nkx?&5^J5V}Pe$8vf(eBcKaDDRW>{`7C}XP_1VaV)tn*lJuK4J zwVvtjFK>3g+*y7fWc7-w37lGRBbKQrt$_V1J^JW<#`2(p?#qrgcz}ZpqQObGZVmQ>hkWGCvDB_CnQ?s9>&6ghT zNAfaXR`@bb`QBwE%NgNOIjp&)F^+|*`ZpF{Fml+Ni)Pk;jNqhyEZPIRsVd{_f(6K> zKEYgJvO1Y2t9@V@+Ec$Tf$ivMCTI#0&`h;I671^|z+`q}FiGHHOxLV*m5o)%sbqOA zE+xW^7GbY_vDZFsBWIg>FlDp0Abak=9NFWXaeOkEr6zSJ&O_yqgp<--mYUR+4WqTH znD!hT$XPg$avX@V*MT@Y)^~BDgLwk39D|bRk0UA9k(h}rbr*hdL-|Ue;Kwzr9o6up z^sh#DRoTSgI@MIgQ8U~nrRh`U4e5$CC^kq}4v?O%W__72D|{KJeDAV`iQdBFhCQ@s zsDEmMacToqA_wA-_YW6d>ggr?f|gNpKGueC3Jv zl{Bu4;@7wu*80&l zx1C>`TwP)fdv}|gq7nz)*yh)jCh~1H_03$zW)&=c)-^wU++S}~x~e!V-8)-1@xW5; ztU)B;1%BDpHwpP@xXQQT9*Upnbts1e@?6lPA__`^ZqZ zac$FaeMOqo`kKHovqE|-7F5V4u5YXPN;9P2uw|^bE*YMQly7vRzD8A6oCs=A$?`jq z(XcA7r12&!RMxfTlq{#-;Hty5SU`2?kFxe3@b(l*y;h0f+Oj?zijqD=CGC<&Rfr~S zRQF)c!qC+WJ$h32(X8Yh7c;s8(Z$?Qm1iaW?lw37Ea#*}2?bA&#x|TWu8QjpxRT?w z<>P_ee>}K3Y`IaJ`w`CyNX%(_91d*)A5&NzHQ+ZXF}W=_URjfOz_`Zg<#hVj%(n0o zSvk3Z~ z#~#2^>VV~o*nV8vtOhS3)iKEX9?0|dz=HdWnEAG&xo`REj^t#Yijn;+UdI@o5BIZ3 zqd9N8;H)use_nI!4{nT+gNUDXyAFpEC^9%XN3N!3Wn}OOm-gDOZh8bP$2<*#?v7=S z19)Do+c)#o-zjbYk|Px{jN0bass9(cq-(nbV~m5-Hf5JP zxv7o1_C&HdF6jw{!KOpgHf66pebn_#>~gkCcS+Y^yNa~!*y-QSsB3lFCL6Iy%BX8l z8JIWivY5-g>z*h@4ifOS#;mXVde`MVnlsvkX|ur^L+mB}O(wW?eyJ3xs&wEQ!-;0O z$rWz1S?-#0?55JKtL||3UqDr7!33v2$^FhR?WWBywqR5@I2E4rJudBFmp9G6XS(&t zHqsk+>-pydU?DoiC?&F4zW7;J-6Q7@|@+?`K3I!BG1gD!K~G1AlIpGi$QjQ zRZ)^TP`leLHO_g;f*BXhaYe$z!k+xUd%8vrBlB`{zTW%X4KOEvtaoP4YdN#r9B!Px z>+3Fyt1Z(w);Bt~JI+;6)`gf6g)@Ha>5VAmHvIcO9%h_sxQBOEEZ}7zKXZIute-am z9$q8dzAM1baMzPBW*JdB4|$ON+NC|(@{NPN;MnJ4{k@}|huA5YsWkz<2S#c<@9{ic z=U7e?*7&Au&$~BZdCgd>a_l<+D`*3o=-4$r^W0Pj!1%kTYrDCWFZT=VWtZT!E|uOv z-i3(7ZN%rNcxJo~xmT>0H#}DET?+ZV*mp4b!SUWrIla6m@bPj7VgR$3cM(3{$}RVv z$}NfgEw7h%Q2z1WrTM+Q+wplWf1ua9pqDoupSps#yk%W_dB5zkw_SuX?E9~7|8E!4 zUcU4ou7u}mPWYFBr;xn%;c$u2T+Lr~W#0?gvo6HEYu$P|rkNp2cbuv{Iennrg@1Re z#f#@N&f~m1+H2}p#RYJL&$!i}A|0xh>>B;kNzp&eEiTwC^SriykmO=s8&vsy=FSidGR!}_-)Gh8+z^1POz z40~oS!)4!#&hVAu0ymvO`-8t~ui}EeGG}+?43Eo<@fl9u_tCo z;v<_gbB8~)|F9$1{*Tje7hxK1-c+pYoPyg!%3uoaYq88u$@rMw#885F5O(d5Ddb zvQ0G)u?cE#9@-0s&$cHtOLx#PX~OT&4{j1FKCJi%JH@8&Mp<&sgY~Xa!UF!-$K49a z#+qJr{9BUiC8k26Jl)LPw?Wdk>vqVKA@6|XoCQCC{2b&DA?YE9@=h#={4wPBAb$$^ zQ^=n|{tEJ5$mbz{4*4g@UqE)jKJSO5H`lKq4~JY0c>?4bNP5hzg`|O^6>=`*I>@g; zJ_5=27d;Nih8EW%u?x!XS;)O1pMxxi{5@n9BrZ*266A}JpNHH8IS+C(H{sVF?Bqm%-ya|~PS%AiJH^@66cZa+e64CdGHISGsI`Il*Psq0+_lDfF!1MNl zJQ%VM$l#>N{v#~5D|t8HtH%{TTPV>cQ5k+Hjttu^+bF+4Z?>OiLjMW&MZ|quQ*Bk4B=a`OVcYmqKJKWe&#?CP| z#n=VLYK*;yaUtdRp}&-22E*iA$Ktqa2CqnL5kxSVCE3t&8q`Y0e&)K%0Uh z@k1uF*2Tc6>{k_8Xv0P=8Y45^|ETUQKBAzu_>!R6;|9h>A z14GtDXGaWPH^Yd2yl#fp#F76dYvPZfi>6u=U&vrh{0&mkns_thj#?8_BF!^d6Gu4< z$*#y=f1kEBG0}lFajx5U=d6i?%#x$y{Qtt5IKzxh75?Q3%vBs{{PeAf{r)yFWBGq}(cyyMozGPkc!*Sffm)yeO<%d_*=#d5pgJ7HbCIb>l9 z4?Dxc^vSJ@c{tbqH0$C6fz>dl3FR;A;w=HoYX*~bF|Z`Y3fjPAU0j^Ox_I+{x^?lz z?OGSZ*FM6!_$7BQWqhGh){+QgQgMD=){=<(M5eXk{H!GrMUk~6&R8%X!H@q8OCtNX zSxaKplK7u)NnC+?dOj?NY(QyAWT!N+B*OCQTN1e>OQIMpiDGsBQa8NgJz%sXDjh9} zN=Hkg7%honYmL#8C`L=7($SJAMoXd?Es0{ZB#P0JC`L=77%honv?Pkrk|;(?q8KfS zVzeZR(UK@eOQIMpiDI-QiqVoNMoXd?Es0{ZB#P0JC`L=77%honv?Pkrk|;(?q8KfS zVzeZR(UK^ZFh)zF7%hoPM@ym@Es0{ZB#P0JDAsI@mP9dH5|xgYL@`?vcvGY0Phe_PrV zm5%fID?j)U_>3bLVzA5mtj5@UV^Jz$JJ4odesW4|}{mND8Cl`a={ zJRJ*dieh^i>t$?^G1?TBZiF$~6vfUmMw_D81;%Jo6kBYp&RCPNn~m``dEBC#$JgX( z+u_DeGDb_H($SKrbnjqHNQ^c`u@2%C%WxqxTM}7)8K>BU6fQb(X*oP~5VBZ}ki}J4 zVvyo}PTo_#oWI;(o`4wHaA)~KNCYYNWRT*uSax%BE-jyfMGTy$hAP6xe_RiwoF;1m7e(MRT+-Aw_j8)Ud2wU zf_a&lk86y%8Vbp4eg&`jWhj<16iaVsrf?BctK5t5T)D*k3|zSB0oc!eA!`N#3~f(| zAR&v}6`)I)G?`h-CZ+W2#FaRW%W(+%;}G^$%wPI3GzVF;TIkw@hS0+Y(H5lSLAZ!h z<(T0p{0r_E=D|3M1CvJ)iU_pB_OCle!0|e}`esy(WEF7N9#MfkGDZc)-JMvYfK|lr za9h7-pg!PTgNVRt19LVKmv(VgoB=-xuJ~butT_oLviLzi{>nw)Dju+cr~e6Sm|5&* zARch4Uo4+Ru}oCxnqTfIp@3F4ZpXZPLgk(s1!zg*{v}crAfG<0kpm%J6yOS+q6_=t zx#EHxfNG&C$6AqExf6a?u5P+`MOWq0LcO?W_bh4(lG{Oss&Z=1%}S~&TC$uW@(lZ1 zrekuUw#jI`^y=Cc?1$CWh1w?5TCks$hhGm>bnYaQsWVjDPS-+Ios@^sI9JV@aR#Va zjAgI+N&+ir!T5w~Uk_bE)K}is@Ds&@#E>`v*VwqYNyhhGk8Bm+w-QV4 zLf2ve7up4#C-48%>*4Yap3crHuar@j+8x7tmochO7c2Cvv>Q{^` zFWZ8{bw!v~o`~0L!!O9Fr{5%?P#XPPzp;rM`Qd)9u-t8qJDfauhx;LbxYO}F!uNiT zUXFKdTHKy{o2JF>;blF3++H&;c_?m=eV0ze?VXCv;A86Ld6V#cm|LPhs+JCgCn!8m z@$2JQO0};&0i7*9TJIgCqJ6Cea>wjzY46-jwXbbKmKaSeOpVq%6;U#Z*1HWS;d$Nj zmdBp=_5<71i+NP2;5O{->y;uJuM|OLrQRBRk`a07(R-&j8_8bSN)dWrDNLqKM(>Se zw%#|~u_WznBSZNnqxW8RX}h?z$>_b8L-BGc(R)|Bp?s+88A!ET*jiq&FYG`VR?^qyV^T4RBK>Ct=a&Sr|JQf!2g9|?fr>Ky(C%b$cdp`DI?u?uz1%qtD z?Fnhe!k+qeBKCS>lP|c}*$|3lNRhJLj&Pp%#s&NY^v1Y|4=7cU{J(p=_elw^b`L!+>t6czG^h|bWCQ@gh(|b{C7f0~#Mev6s_y?K!5)4XZ+V7GlGV?lwuw+VK zhuJYRa=hj?BmZgyM=P5#{hu@QrM)xr`k@hgS_J=L1b-njUsf2wt0MUP2>#v7{Hj$E ze0>D}3wcM;Y3#1F|H~Ae#*R)9oi+hmDSAuMX_IVS(P?p8S9IDeTUT@%TTX3n(P?ZE zg)BOaEuD}>r?C+WN2jrcQj+jv&%Jh+ZA?mZ8e4;i=rlgu!_jGccq`k~=rlf(Ge@U2 zqYy(uXvyd_K5Ifhxac%KT{1?eZL|jyBB}5cVIy@GJ}o|9lItbdY~Xkkc+hx^PU{BA z=(OD+8J*S_lF@00LNYpy(Q7!L1joS{oyHCtqth;iWON$Gy&0W$EhMAUz5&VTw4XsT zI&C#1qtiID&FHi(kZT}0s*O?DL^m8;D!qtiA)wn4rN$>=nWL2rWOR1ljX`$E19c_`#7keF)GMW>ww$>_98AsL;9vT@OA zD|I5R%bpg=icYoz?@A(P@YTbJ1z%LNYpS0pxCw zyJ;je!RQV~ryT;x=(JIgj7~cPlF@0L;E&O1S3&lH{2F8#UO? zI*ko7qtn>fGCHjW4J%r!blPo@j83};at!3Nkc>{-1j*>MT-3+YArFUS zblQoK&|MN!AkT(mt2YIbQEk&9sfz@*C6t42TjG*zOJbLyHjCByOR_DA(YB;?H(0t` zjM28FbhINW-S3QTHufiDv?D1U?MT|T#9!*>@eUFjV2pMov0=t|M~R(nj20y^-d9}e z=5f*iG0yrgw$j)SjM1W`biCJ;uGJXtH!)h2lpWjNNRk*%+TlN=NIFw*9-YT-;k?yo;5N)+4dKjP)}%z}R?WryHa7 zNZU>`Hs9D~#%?mkyI=d|eXnhwHiqYk&+r8C)8StB**^YKk$0l85ynvW{dDITyTBN& zN80Z-#;!AVi?KV5Jz#9Dv8RmDdZhgRWbDtzJ~l?{k<#%w#iec@tw&=0j14e0%-D&> z#v41`*j!_m82h@hZyUSa*bj|u!B~Zk<#m6l!w9l6T#T&UCm3(JZ?_A;)1%Y zormFA1&(cUAr*%<9TVzl=t z9qm10wD*Y7-XlhPj~MMeVzl>&(cU9Qdyg3HJz})?h|%67MthGK?LA_&_lVKnBSw3V z80|e`wD*Y7-XlhPkC=xigcwIg#c1ylqrFFr_8u|Xd&Fq(5u?3FjP@R}CS$bsh&3Cd zy+@4p9&Jl|j~MMeVzl>&(cU9Qdyg3HJz})?h|%67MthGK?LA_&_lVKnBSw3V80|e` zwD*Y7-XlhPj~MMeVzl>&(cU9Qdym+O#%S*mqrFGzXzvlDy+@4p9x>W`#I7?&dyg3H zJxce0G1_~?o-#&zj~MMeTW`#AxpkqrFG$8H`7WJ?}4dXxoXSvShaHc=*M=I1JK%_ZXIfcaLEa?;ab8 zu%5fH=-xfXba?le`vvbFgY5WyV@R;0?;FE&gO0>aPU3xISn*#o=Ks>>OAHCPlx4nj zV`p?h#(Gu#OBhDeo5re!wRQ4MW6U@(0cCvCSdf52m>s=oOz8q|`dX|au3-Cb8Vj=L z;9vOlVKL`%KW)96&Ca>@yK@hW5aFa27sR)QX)Zt(&yLOosBPN47%4SE$acXJ$1dOo zp1@Mto5pZbTkxR^IfFdl>g`s7V2;K0&Hj3uFWtG({={4%D|{J;)OBqb=Pf*bORQx10_2EncH$*t%1!08{g;ddsci(DWm1+FHe|1RYx57)e_!2>zGRH0 z7F6c&yRA6zY*87uGgsI_h3d>SfXdzvo9Xm4fZ7T%0~LN{30^XWbB%h**r!%&s=S5s zM-FRSbg_TbJNELg@Wl%kP-@tZ6=;*vH&o#nss||Q9A5LpNrZ3^JM|v0;86ISNWLyB zQ!J3(qHV!8tb57BPY(#P!}sj^VkafbT|CPUz6VS>?9h9_f=zgk$rH;IA7ls0rvjfn z9abqcLn*N=(N!vFIQW8`k;7)}>>Ij*Qshu0^lpJSR$d& z8Fmpm5jt}(_k}RxJnviBfbM#E-c$JIbo6=NL0y8Seo#yXId&A5QiEukG8j|NK`I8( z41nB`AR2_w4D7&|vKU!<-t&m((7U1zL=Os{nCRDi9wUe0*I=#5`>gj>=a@eW_35)r z<{gp439tC_gTJ`#dd70SsR%aso(8y)BIczB)TF--t4qd!8U@?D*M-47+@Jn3zYdF! zeRzGkw8?;)FNdrk7Ep5`vYP6WSLHvEGsog7`a8xi8By4qi!h=rbV*;044z?+FS%(- z@Qh+8rbhP#!e2~6hxbX*)r>L8G2JhDzVj%^sLLEcb8#!-P26G7wIZ|euUROy#iT^ z*E}5;EBEF=ekb;IZ!})_v8i*#8`b?Z zdSCBGzF=wT0g+oRJvH)uNK02nW>}ARFYT7Opz84^s@)@gdOY&H>OYEni_)^gA~Rfu zeeIEXU(0Tad}GtH-$w51s)do~b=57AZ*02ixz4o7&8%)fiV|AGBAb#Fh7R84w8W} zH$gHm<_^evAs>QdV9e8y42*dfl7TV2_za9GgJfXLK*%+a9E4p9c?M)FfbUxvI2@)bywkqfB7xx0WG`^nI=2S?eNn8rafSM(c45(QN$$*+WA^Ex|w!%e_FF|&L zd>yhF@*T)sAU}lc4w;9JZV6;DB!Y4hyF=pbYKh*E7;sDsgv1m=iJ_1T#Np&e48-{y zBm;5gK{62M21o|t{1g&S=!pj*%OD?zWFQVZ)O{g8f-HwT2pyt>AP`Aagmh|$cVblUr7iuO@wj86)&8;pI!*al;d8vC0uJPG`5aS!_Yz+LCFqm2zS zhC1)38*l6aV>QOEHg>HsJ~MP4J^ZC2?@(ih8{=!OmF_-&3C4?tmg*Ig3aeMl=~2C6 zVb6(`$yXaSp^!@sp0mE>;Dz^%EIGL1R(w^j>e!so11BzXMo%okoWUu|`eTtpacoR6 z%0qKTA=BMo?zvy?EN62{b4JvJWC`3?Nz=sQ#%6!L&6ghPD}OKZWrZ)}keZNeEG8tw zq-2_qj6b{4XWo4s%t)4C_Q6PQT4YL7@Eb?WGS5iXtRylPcu)X!2#VFWY?SPOD?n0|t?YaS-{a z%Eoay^&2p%_t>IC8pm_7@34m#^~pFZS%UdvMzbofvT&)GN0!s3SfAjjf5N(;xx-+EnlWtk4$mFtb{yP0e!-*~ z5iHiIiO1-yRj?eeGwq%ipJ?u|W@QyJ>1DDP*qz%JB)6-VZYr>Y^mH*)hjw)Curh5N zvY;hL`^`#ayQ&CM^Qg8TLlRf=?N<{#_>;rq!P5fs)WP%|4=e(WC=po%>afH)y%sLw zbb<@AxQe{xZ0=g$efLkTFV7aBrU&p= z>#wG+b15nW{@D7f*#dC@?5+n={jT-H9*?$co?Nh@4D);802UXvY@Si@m&5q|WEo~p zt1bhqE@PTE4r_gSTQ$oOUyp99j&n)J-p&NUhCXb9s4VF}e=EMe*j822f5b+7jcFd$ zu(@(d<$0BpE6<(0mM8w2Ge&Tx-b2t4$)i^k6$dP>fGVJ++@K1R z(?FpSO1L>ed9gWSQHz1TU;Xq-T`^Jn%N3Lmpf^qyp7$Ppke0(EH=U{98O?N5Y&MWjjj_sbaPtFN>jGfW%75IIkE4|&os!C6Z z#k?{w{k`of9av6!fMNBwV7cucuT}?zjXn->2|4;S0N9G<_Fei6(!Q9-u zf}Z#k;d4rEK|x{e?)da5C@d~28XDT{^VqE5sNAcLEGR1OomX&t?*82lEa;t^U$8qi zEY9s(kds?lkW*aBue?16{$Xxkd_r(4AB>Tv?Q7O- zKLZYMPj^Y;;UqX>OR^A2Cb%R|g%4+N#No{HcNjYI!y}S%dLbSgpI$ohnu}-n2X-0u zTIqg&A-vbaBC`5|pH;XJ23sM_&G73G`}Jn`YjH{e9&W#0=WokO6j8zYiJx7H=eozr z6@1Wf>cMVf`#)NY_VZ?yqWyC5P~2iRKCELG;Bz59Mc%QcsJmuG+K6RYS!6|e#@1y; zYO{4&k+#^ntVry+XnV6Fu}30gRwVW`gv^S>PDI#>#2$i@w6h|yEl;r`v89f%BC(+j zTann1D%(^m5}V6^iWLcEPeT&RE%;;KnjHuljDF*D8j{%G#-L|{jsLBXIjEjABo#x_ zkhD7_4N2vYKY$zrNkdWvLc+ilLymki@}a8j{#^q9KWc!ZZl|8j=Q~hahPXdK!`jAsUuw5Te&S4MLo`jRqkO z1JfWh60!_(EF=v=pNH%VNx%1UNY31L5ai8}GzhgoLYqrG14)BW8zc=vG2GxZ2o*!p zAT$W_aL8jJX%IROk_MqhNE(7}fE)q&O-LGoegb(Cn6jD{ekqajG_4r4R~iLEvE zpfQ?ywC&%G<)ZG2b@7*qJeqpM`WoXsBsRnt?;|mG#ko}Eonvf@F}|lq>1h5@x-S{~ zKVv^NM)Qx-Z7}wzG0teH{Am7Bx;)fR`r<-qVl`-Ch z+V)mscN)9T*!{*HF~(Lx+w!@l<9o^2Zn%%c_VSmCykW*pG&aWAcw^@nV+@Y=QDcl| zBC+onyU7@yJ^p_2l=0JTGnRwvsBLjC`RO<|B8GbJGt_jSp-%hkB4aCzH5&Vtu^Wy3 z*w{~vwHn)C>{(+k7<&Qt6OXE!_oBaqfg7|LV$h=e=^0#>N@D(AXSf*BHCb z*e{Iz+Stp+UNx2vla=<{)n6*|1{*uV*eS+NGj^e|ImT8R<9orFbCEX(V^Z430)NRF z$R<|TzFvl#1IDl|For!tW7v8c!)O+pK(pASRrrc8r){jPy6)w+>IOU|@jM#><63WrE5^b?HUYU0X~uhc_Q59ypCWvA#peKg4#DT}6;;R1 zTv2rbLV8*@Rpqs`Y&tUUk=A$Lm|WIUdGK3LwyeG={+E`@i)LY1@p$}j$s3zzK3K*5 zee}jeuVvG?yq4BQZ#;+nq4x#as^P{Wj4Hm?+Nz<((HK>HcT>yiBd0cVC~|ZKR-eP_ zY^}zzy5?&bf_wy@&2^i)Md@eB(e4ms(n; zEjs4e7>=hehElIwJiLsc&4Kj9RElh=NXnJSASAVrL&;Q(v zi!M6$*x46fJY(LJIrD0!Pnk1y+6B;cjvE3uYW$zzy6AgDsfxLA4C+(#+|l1@F|Efu zcCK(rZWK$YhtF~SdA1(<>mCaaA8|ugF8!V3GMa$F4rTO(jKe|PX0~m!1>2lZdZ>SB z^7X7N<>=RJm+4{~Zt5TG1}DMsMDCY^L0>tDKR7?pz#EDqE=pbF#3Q)*0*BIR{)miz zr&k0zo?cbDx>{UZcv3;P;v%?M=AKxPS6q}|P?B2$y><8EuK5MKAF=zHrU&~&A&E@QW zqemr{3M(H6?fAK&p*pPPz?Na*eHU!c-+5y~H%ssi-vr4gML~|2V1?@f z$zCUVg^6n*iy--|+XZqU@;MS^D2FUA6?uHChuB2O8-bQmu2*#Ki0 z3-#G3W3!B1Xsp55HOA;9seSyy*o($qHU@5Z;TUKD>7-JV(Rk6kftA9Z%H zl!H$ZKD+uIpNXj*AJ=!n&j;Xh2tJ38Truw0nfN-v^`+29daCu&sjck)uy3@f_0idG z#r2I|gO&EdWY-8LTCK5X^rS4YNegYu>Pxm2vTx#Sv`?1dP+^`Wz_;t;I3sP#>WQ$@ zI*jjeu+zHbc6}IOC@iG+XugeFemH(-1KIf=jeEZmANo-}ns3Kwhl}XJe7l1l%(v0oIH8o! z)0meWOH1vsOme-CEl888=fdalMEr(xjqJIw%5Asj;_^VjoSD`$$vp}^jr@EJr56@q z;CwgqO|Vvyixn;u=Jrz0q*-%*iRi8hyC)m5L2CDe{l1j$NgF`8d&1i!Q}=|oK&QGV z>W;8C(u7a&$L9vKD8=V`AF?;XX9;^FJYTN!PGm1+lFwO{xPFzt{#}3lUVoiqZ5!}A z$JgF~+zaw;NRF+smqMk9y%e_W?4@wrjJ*_A)1a4vQA)p;!X@=m#JG2{mHtvUkNqaG z?;E?_*h9u1Gsdz|et$Dofc=ZHZ=(E;Hbx~!>?g*4X6$KW?1v~F`y9%TV|8M%D*EgI zW0l6(4^g`5#@G)LW1mC$U1yAa4lxF4DBW|$7?>f(K8MnAtX!DJ zW^Adk<;KS1N-5n0f5}E>+EBDZOo;c?%H$d_SjGkzXy?}MW@JM)MgtM@?;gZe`ksQP! zl9E0n5o_7eI7#LK50*t*FTgeX<=`6JI0{Q@_Pe1nr4DHzzNK9UU;wKewn2N_=?(y6 z!?!b}@q0WjIm8k3K4Rv$3p?l#+<~9b=oj!H6y}{A(s;@g{vf32RDyn&(?vn16(nKdmBI54X)CN4+ykFyfElk-q*}T{K z+XgLnn_Ex++L1v9wwtM~_qs;{mh2RBkb(6#+=^!&OJIXF3!mVR z_dE}x2p^4#sj&_k@9V%n}f9~>&sKG>^`xjnSN@{Af;7 zel(ql(R3yTw6k} zg7U#72VYT6RsMJsv&j70 z;v9FDNf$>pdBypK1!(nlFF3I{zk5oH2X(eHE#5B3)blP0H!)#_mLqW|xFnrw@py=y zcTUO!JE_o0`R}mBJI`g%JA63d7LN+;AeZFMaN85r;th96JEhQ`?~-P2@z~CmyWf8t z-fL8gH{K=PEu|2WE#Argj>0NyvcvvDb`s%?Z-sO#O667#nJgURl}) z!+&)wi;dM8yWQ9ijjcCEUk`2jiZS|QiRIy$t9=yuOGVy2#_ltQkXDXBQ%6YA3(@wp zv)F^(y71$VKZd+(mgIetAUn_@udMkDE`hH@Qhod=KATP5sTHkn#HYqv-E z-lMSaKP4q~Tg}(nxWy&!e$e{#Y&ZtMbYF(GzXV2p=&6@rBs;M3$FYyvgtnkVz=|d= zZ>#xoGv!C|x6w|o#+hu!iTtAl`)WlD(;H8;v^&H{r?#}LIX>@Ew7hRT=guxTsFew3 zMc0BybwDed1U3aD=qsBAo(l%iSGEbjlVS3<$68j;*!B`f%;5>Eyzm24QWx(+&r#Iq2>j4%~SAhIEGmf54roI(9f1q^i5g{!y;~Yv^#oz6n&k>)gKXkLn+>r&8mRJRd%ssQ%G|F6mL> zq){UR9J69qHROO1)&J3OQ0hr0`#*2EeeRjkJ4*I{=(EYtr*Q8msqvTk2SJVh@U&_G z$CKJ&-OBlo>zs#AQNcjxgK`Y|!7hFR)_k+tgs(FT_C%L;RY%wPWS8WOa1lgZ=eu0e z-YKV%yv{ecUnAoucoE0ABqxLqC%qp$!X+8g8N<%!UDERKkw;zU@3^F;PAN&+OfJRa zp|X@iQfuAi`@mc14G3o!RTsYGXBWPn$-1yC!>`G@@N@TTlpo5i{%3>V z-Vc)75AfGHG- z!o)r=ElkHl(!z8KBrQyAV*_pA2-60*ByB*9+JG1bx5R3Vecjl%js48nFN|?;OWRUc zP=2o&dOv|&0A zZ@9Y$9^Mu5@ZLhVrvE;L>tZ})wy$vAi&SjrI0&*M3KwS5X5WNOdL{?(q{4MDGTsS= zYY$Vnlu>6Cu6wtma1H!dC|nz|3KzmAvkKQgN8zdnYdGnO^}k->IwpKLnH8>+vIh&^0mH=E-WEu8N;#QSlO^;w47KON@$_ z7!@xuDqdn#yu_$@iBa(qqv9n-#Y>EemlzcEemlzcEemlzcVBz`SJvA|08xCepk$dE1|8)OyuL%Wf$1omXaTd)_yC z`C%h4dWFMA;LNi1u&9U;r&hrtmJA#5%~)VN6E*_#)U<1z!3zE`uFAkeChm+?bTN>L zii31AOr3_Bk?Jbb8#OiEFm*Ye9AmHz2hlpOCvDeN=JK2lyhtzazz}sgbeQdhm3+~; z%Jj$iWimijgD=Dxg{Ol{H$d^O*{%V~<$;2kkY-sZE*z6;V8T33F0v%w0!o~HY5z}R zUz+On`rEL6k=_lyz$GaO_dBEP4qb7c>G2RryF(By5i*0M+n46JeLWC9oNzaorV05a z{xLlIA7x+qsmpC>IBAsK;U6w(r|e4{e@;Kiq)p)k_iL1Wsl*jflzj>2P?vL*eQCfa zw=WHHSvQ9ZHhi6FYCPH{>6Cq`$t68J!oGBs`?W*+l7hQ?bfoSKak=apu69w^neQp4 zi&V%s7`U~uVXRAL`w|U%>AxoJOSidSqwGto{B1k5 zFP+Z(y**9cVOOC2-}Zuss8giclzJNRZAxVTVVe>khM8YL{PiC|;-Vz(g;NTe^uU9m`VeZ^jC6oMO~Ul#VyK z7;izbJB{6C>~UjH8T-K4He)?;j>@mMzl6zmjGbugJY(k@TVbrx7$@gae)k#sqp{7# zFp;I7j(*KLk6!*#H;;bJV#gSxU$fZt##R}_gk^qyzchx)zkJ52y0k6k0r6R$zf{CY zw0y>?ew2qwcH)`2oH?Nd}H zl9M^_gfekZR+;d!%0y;owVhTbym0S*r7?lh$Dj8x_GQ_B4h*8N9qmm&;B}0r# zhS*$VR5HYD01FlsbfL3|*HCOkCJL;+Dzv;=v}(+3I)dJ(N+z*4ooAMsI3sjc{| zRa+BCG$O62h)6{vY84fWtyrW2-~WHk?Ag7Wdv9W2U%%hit6^sUduDcac6R2R|DKtl zk)ark48>?_?NgL zj+X^Q|JJYk%gAQ~A|LuoO2~HuBJcRiO5D)kQmx@fgO$|f-{(3}pHa214yMGc&I?a` z(djYN`Tv2HIFqk1!Kc%V87}gDpbyF(EUBqq1@1xmL_ewxXcW=!19Z+k5vgxN*j$yq z8LLiQWD9A6@WC(bsuKg!EJJmLEu^U#4QR&1mDifR>ckKfF8!(#H)ZsVd>-KZ+Y@co z3CuO*&{uXLn71P$B=(;71QJkjL2f8w}%SN{+()hd;BeQ zIMs6Y`RGyOe5uLJPjbgUbH~4S$G>&QU6Bvf>irQeMR*uOs@F##d=Vk<3o3T}c!UiI zry=B|4)!e@5wiF8)#mR(*->tuF;$xtquQ((eTNjAXBgj7#ppYva`YWijJ`vPy=WL; zKgB*Z>L>!{|k% z`F(8IXNJY_l=ZspTr9?oEgVJ$T+@y)>|Dd>F{Ej)Gweabernk74SU_N&kfsRSPVLl z=GV@}u;GGX-1}Y2@|9uR4EwLz{BKm7)6Q)8*J#XSZJO4YS5XRm$RdQ_MPqJ)1P?Uk z8<6oopfO)+8nb599F2L^KT%^|^k0pcTFrkoW=zcff21)tgw^1Go5no*zZ!Ec9Rid2 z#?ZJA`^Kcvn5)?1x*z*@YRtL#Q{z;20=9c>sjXT_*Fs~k0F z#i%(eM$K6&O&WcfUR*agnV$_@!qvos_ zHD|@BIV(oZSutwPicxb`jGD7z)SMNg=ByYsXT_*FD@M&(F>20=QFB&|nzLfmoE4+y ztXMl2Q_Wd1YR)Q0%~>&O&WcfUR*agnV$_@!qvos_HD|@BIV(oZSutwPicxb`jGD7z z)SMNg=ByYsXT_*FD+bLO%NNc9bDL4;7IC_x0j&u@u$a@(3X;)abU`BfY%v+!G#f$X z=fjcy0A%C|tt>txdH-HY`Sa#c%H=yhprgo2QpFKxbh@B}ZGPJk&Er}}oYBMUC_2aV z(DnsfO%a=A6>zhx0&bR7z@@hexLH;KH_Ix>=gNw+R(v0ol;|%013aE69Xube-#L&x z4`HC(rdwID57t#|(p`Lhx?Kq7r=fr6H)9uq`Pd+FP%cT2sJnQol@+TXdrXAp%Wjp` zI+LUxwhqWTyLBFy63%YR9eoeXD#$7X-ZZC6k_B{1m|lyreLY0vrm(V*itW2rL{o9P zQ{`-0PWK=Yxg=biFx$rgrl*M987@vL-Q`FTZPs$am%6frIK)(JUVjliHU*n^teibE z;cUR>DXpD~)AjjaeqL|P=`Q*EIo(I3PzkFvN1yVOA~G~wt*M?ezwu)K&V*YGI9)F6 z+XSckjEJP=beD<9Q{m#Iddkm>XtOxo1}B<|(>-6#cWI7>s-$x&PPfioC^@IAGYN7M zDlp)5H@URFlf^7Kj{cKZlzs`#I1(iCxLmSQ5y#^$1`zhxW`C4SE3E<^)=Gn29higv zV?VqF2RZn=#(|`5cSD@CZF_{IZTEA>d=+@?yJT0nV-NYzE20%b(zh{$^$7PuNLOnT z)O58z4k2BwPelmNhdBFxU+Ls7nyPd%rb?$`R5}&A!Nt^-TCs->TWWGtD>W@2xypTQ z*mlFt!Sz+{`!0t4tPGoH81Bu<(ceMy10C(Km4{Y|CL$l-(<6g)vzpoA34^LFD(cQ(czKmgIhD|hVieWb!cAH_38TOQ6uNwA-VVe#6 z(y+Usk7!w{Tuj!w`A=0G>2{F9A7Ns1d=U~AM{z@-KLT~wurGpB9G6o&tXDnpyLlLc zEB$*lM0$KgHN+`+f2bjzi152;h)vL+Q_^XOQ_^UNQ<|Y6PWiWLi2H{=z(&9Nv)|_a zJt2{(?$AR-=7tCBRJy~pBKltq5sSem*9}xd?4EEo&<(;GVt8*6t}==lpPBKyu%u`% z6vaVFh|IlcDWuvuj1C^lvM6ZHRvnn+urN~%l$VP8xwF-lc7r6sc6QHy*CA+^S2S|UI0cSQcw z7CR$+8R3Bl;UyaHg^<3p^gE^RY=4BH>*5>}e68^ijO(g3GNxLiV$>QHqt>X{jfPQc zRE%1q%28`nj9Q~&)EX6|)~Fb@M#ZQ#Dn_kQF=~y9QEOC;TBBmr8Wp3~s2H_I#i%tZ zMy*jXYK@9fYgCL{qhiz=6{FUu7_~;ls5L4^tx+**jfzogRE%1qV$>QHqt>VxclT1P z)Wu@nM8mktsLI`J*lmVAX4q4Ptubt!VShF3Z-({26V>Y;=wdOi+_0gBO*QO7!)`Z> zUb0%grwx17ur-FQGwjcXeQg+ZMJ>xiF7|I&A(3Vm$xE=A&03d!1()u4g^SatS(koA z!m1cetV{ojzXW^?#ZWFa{krtQ?(^YEbsLd-L6zQ2bX|IzREM?UWnKFInV8}xOVFQ@ z$-=&l#26}36S9N{?hu9ggD9|ZqM>bFzth4UQ+L*qE0x9RH@GD{#;CkXzpE`rbDMJeMJ z=yG)Zkt~ZfAeQJYaUjN>sJU=3It^0KZ$dgS7t$?^f2=Hw--Nj^A4?_bu6B1J<0jT= zJS)uE{v}kHD{g1gIh+4W)@}`YD@Lad;hU@8p+|%sV*BwWPMf__%AFlnW7)iaA}E}B$=ORB$*>CdF%Odt^Y35 zn0%^XXkGlQle12C>*62Y{B`k{MCv}lbNwWFJk46SWSNL2^16Z5a+{n@yKc!TBC;@C zobV@%SvWmVM3#n)s8r-Oo$5WWb67N$!*cW2#Xl;XU8>IbK~YEg`a zq^AAUurCbb#}SpIe~Zd>HH>RXDt4M-ryDliFg9jU2)P=*SNvYfsh8=C# z5W|KWcClfX8Fs5-cN$h>Se;>?8@9!;z0u~heEYgs%sa|3PO{YUoo?7!hFxgbOv6?g zw%V{i8TJ>$I^!wpb-TJ)%iSEjA0`TJJ+z| zpl55Di(KrVy>3i@yb~T=YfxN->FnId8kN|EZE$9N6KhmfJ406u&L-EWtZ!uM^0Uid z{mGr!gKcNv1N0|+mia8$kd2ZbC)tsWJFel>do&)ac#m#7w%U#5nJ_+S?ruCIvz7&| z-Pk{KmD;@<2WPHgVZ`3$cbYhoau(^wn6g?2doj~yn*J$q1Ge8xdWmxHiA;k_^%lSB z$i{2f+D(?EJZ~p9S;g9uV(MZ>sWWvqVN162T7T0_Q}^WrQx}GB$kbiutX!D3t7+eg z)vNYy4NTotb#Lzc1MSXE{HG)j_U$LIemC-{LRPGq_`kJcjT)k7q3YBS_0XkiY`eu; zqH9wO@3;3Q{#qn4NjE_16r8I<*nE=|UQX$^4%m=x6`KudOwbL@SjA?3lS^X8aYyjm zN@;rbL@hOQNzAGiqe6?qrm`-` zh4VJTAHjktksry~#PwbR>w=sW(rBY-k=-W#EtX2ne?UY#CEQ)` z_!r9Alrv`eY>yC;CsI9rS_f(m-xw}Vs>e^`iSAq>$GB9FpRUum)uf-zuEIH8it^eP zp{!ChkNNJZ$(zUJNzMk%W0{;yhWCs_hr-v0{JesM0J@kple zekN9S7q3Hmjy{cl|EUXqdNM7zAXH|KfiyRJ;IE6rXL@G&kovoMtDF zA9Tl$A*6ZfPZwtY#`B!i&V1PS@I1{>9<#&cG3%GdZ4vUAuQZR@8}XR09gmN3$4?=o zc?y$M%+ps9A|B^*{WMPp;(lnB(oE$SN}5mnLdg|s@X+F85rRjEUx1KiDNQMwrLz#y zEWI8f%~I+FzFFEAN`#uFjHy|w7|l|}Zg4U6dsFOT!nH$JIFBdIGW$dhH;lY#ikl|p<%Zh zcDG?G413A2FAV#$Vc9SMwJhyjEan|z*!K)8F|5q6XAS$UVH*s4->^1#QhJSS7mIoP zMyVLT6Y9N{8piLIiqW%8vNHn_$>v!+vGhbB4Wb7#DY8E-`PrVLJ`m7sjJrqnnGxyduK}88*bQ;f8gB4y|ds zxR`xmrFQlIYXzZ1uRYpyNmmf!Ob@Ob1pD$w94nmTUhox@OACEKtIj@Se)*@DVY}AQ zLcEhnux)}I*k?-LR>K!c{X+N|EjSpebN64zPn7&r8U9RJ4#ODVD*2g`pD6jMl3yqZ z7U45xKDUm`>2?kJ9xvq>dHDRaK z%lzn*@x_m<9AB6;TN&J)#`AEEO|;gGnlsV*5PWV-+*3Q%9F-v`T&gdCEH>-qze^MS zccQiKfctV=idr{)=p(;eeSYKZfJdw>e3%~GO@(D?UVKHBv} z4v8P_`WWIq+7$r^f3z#%F|K-A#zo_%j-E7a?6`}#Fjnpm;Ng#UNvDKUFX!wX|HpBN zwo-fiEp#AlBaTh9QO-q3n}x&`kALQleVgdF?wFsFeune=Bcv@udyKZrVuZ9^Zb#UF zuomG8gsa{08iW{O;v8js+k{(hscpiT+9rz8Hc^b9%oLkv7(baQ#!qG{$4_R8@spWi zFB(S6MX^r}`@%4ODATkZTuhfxQmn6GWHS`I!?0%zd)}}Q4ExwH*ljK?CxUBPI1ya) zD=_RN!)OJn94CUS94CS+#);sH-EUaEVVnrAa(^`JQ^PW0X=qwb1Xnpu1Xqj`!4=~~ zaK$(gTro}rSBw+E72`y3#W)dMF-`7iQt-+ z6TubZL~zA85nM4&1Xqj`!4>=5Fir$lj1$3CjuXKZ<3w=9I1ya2X@*^7*kZ$OG3+tJ zo-*up!`2#hH*`e3MwN^GN6!!9_odp<_oeT;R%xQ8k#c^JSQ`H9aWgmIDT$2-@b7AD zW9BT4Z^U8pR#XEMVim28cVTTLd?k8n z=f=8U?EEDBmFORsb;Qr0QCna+N_Gx<9rUvyTOi%M^kZdSdK0$5@^rI~mZ#DEmp5b9 z(ei)mtfO>mjdJ8d-Gxc)FsDw!vxkWMcV0RP-x;V6`2z0!C(TPw`{iUNiu{N?!n6v# z&hTlLLQlq)%_6MJ247CzlCvqloLq$BJR>4$75Zf&a%FQA`hWU_qmj)~=+)v#^j{1< ztkt{(CfVPR0@4~!cq4|3Vk0{=v&9K_ifvHH!LKaT&L&vvcdS{_lX1v)5zH_ zhKWSK1eM*)0=D(zL?De$Un&BDPOslYpO7&{J>c9QvgQ$;?xK&p}B2eJVog@BBQ)V?GtS%3Y3--y-<6 zj^82{A>_A+n-M~%jdLvX^?ABKsXot`>hp?GpI7V#7t?PIial)DQj?>`u4&nPRgPa6 z6x(jtIk>)J-*+*cW3L!zOKI9ACdb)QioIyqO2al8w%IVwmeTw%ly&av10ILBTw&aqeQ z5yLpgUNO$GS2@nHSB!J)72_Ox#W=@avFW3O_YW3Sj|!@e}^XlTQWInB_trkO;gwI!(7I>6D2#j}wgu9T)?p%{~0PF}ICnyT1V;*P|s`qx!3?;}+O??A=! z{`?B=VHM1nR7&>m6%koAx(SA^`x2Q zrhEZw1IuYeXOu=A*$%Q8@Apgm@sss{$jfvMaQ^1wkG>cBvzKY==&@r*jhiv4*U_V9 zj2<(2+*rO>zSffAjdm|i@Mc{jjf13HhIfACrMd=ADgGynF+E+{l8g?c;Do*_}j^_PJ%`ehnD>f)JGrOYL`Dqlm-E0W9fLCn7lH@6c?qx_~k8A5ph#HT@s)$OGC1PHMz9mJX@SIQa+bEWY;S_{Wlj3Z@afIwN2OzO_ znvBI{Hi%*uxU(_uYLokcVfP#M6T_+vt2OL1!~SI0UU(jQjm|Enbf#i`4Wmk}*qMg$ zSt>?{A;x0ft%luc*slzu6Nk#JGwdD17Vl*myV{HT@J1!-OEH?EYiKG_meU(&FWqp?ZU>Ita}-mrh6HgzI#bH z-MD*6IQ=i|Uf64Wx9)_^V=vS(yNu+v8heX8k?5PyTUd8OTTQ!@wh02(olt;72Vb1@ z1M`1QcTyzf)qmK|*qv~sJyIbr+MTew7>IN1F3v&dcNfRYtHOSaG3_oCV|Sq#y9>qG zT_|?HVeBpxt2V6GFm@N3mfeNQvAa-=-GyQU3_H=Vv4-JpoZJr#yTP!Z8V38)$<-US z%CI?jNANBj)Gj|Sb4WyGk)C5Yg-u1N z7p$Jz%IXX%Dm>GNJf-K7eD6|nhSGorsm@JLTsA8_QAORGQ!=RujHiaSyCxE z+9@0y_ATWsAUN2?EwDvFD?e<#|^$ z(JN6Cp`=rC_zVcO4o z|Hrnw1%Fxi8|%O#^ht~sBg{mIz97!3K~E6pGb%xN6vERGvewb|;zbBgM>qoEaD)>P zo`I0>1e6O=g$5!sRfQOfd6RHXv8gT=^R72+pJQ9Ts!3n77)n zHHN`-YYVNY1s2)J%!5`Wkr6@HCaRGtMG0M7L(sLYOVPFQaej%vrMOtM8w;yP&FsdY zAcguVzq1LtG1X&SFW0ymYpg&8-B^MO zw%9A}EfND0dL`?&s4KM*fi%jLcC79Hk9S+_TKu1{V)53?92oAlh_id*owIvl9kP2m z93i`>6A-d{qUz;$PbXOS#F%zZim`iA?0Of=5*Hf9*gdJ-5|d;1q!_y=m1Fm$7`rFM z*gYx6?nyCrPl~a7QjCvTF}4}SZZeGanPN{GMz<8j-ZTt!zQevYY`bCA=o9oBwJrvN zcmC%iq&sqRQ#ntz3ZPWEWbVe?U-LTv!VlR zxan=sgKftFW}qI1si-?Px8YP&?4mk26RU(CmAMAg)9ZjSO_Tc9R?JJ?Ih`Lfz`$Nb zh3=aCI1XxS4x3cN9udpx|7P?{j7=)4FkXAA5-CV`6T}G(GZQP?KoThTX55YmF+D?1{!h~4RqiRxW9~qv| zE`C2&fYR$PLsYXk4p;~$%0l>}J)I!)V0Xy+4~eosz9=;o{i!G)OGs3;n8CStiM=8x z=;z8OmP&{0B(ijq)3ha#v{g7xF>CX3z6m0}GAt5yF#ze~x|?UhUwcz|lRhZ9Wf?~a zR_%QK_nRs1dzU8xw*3dg|+xNjw4z; zUjzsF87&SVUyEnMc2d3;Uu958i|6>ka`->?Gh1*FR^suQSCscQ2;1NsWa8}+=DJaD zPlPjZT!8Ruga;#h4&fmPIc60iqUw-6}>UUa_AV z_H)BHN~#>MsB(Wa>{G)~W+#VIJGtWw!?^D-I_+wHU~L`tL&I)1>_NjwRp@npZ`kXG zeQDTM!?=~aUL()NVqRavjyLQ)!$unxH|!e2_$gTP#?Jd4`QPjINz}UAlJab*l_}$gnwh$|}d^q;k*-`lyjIu?OiB?09nt#(&WX z=E(4$$>G0+Km4zUHM89xt1*6W7>$)AW&GYSNn)cbY|M9KduSLRls_?X3?Gy~1ArUA z2jycw9vi*$IC?|>D9Gf9-DkyFXhVtP^bxp==s3L;Qu+A16@Sefr<<~Tmdaq3((*|- z-FTc{iWlqn+~ni5CUt`^4I_W3D?sV>rz2_w4niYpx^a49qG-Btn*B~F*L34F`^D%u z?H9HL=kymS6)DDLj&gc$bHf8+cwFXKNk#&nYr;cPs&TnoL=*j?f^k`OfO8|)vT>Qi zUoXkvkg!O2T;|Y4T@jP&!^KH8%VU&??iChIH7*Yn(bVIz+VK6tS*046`4Nj9c=B<1 zteo95;cPH2>#Y5O31@?Gxm41Y%aGeg;?aNnA-4!8B10}4hJ*Z(a!3IAL+%Jb;UTvI z$B`kIZ8|dKvh6y^A9C479pn$Waf3obt~gHcs{W6C#uogw!C#pJqhoH{3?f8r#<5i~j;$)k zu~jjSt%`AMRg7b+VjNo)0 zFuFx)esqgsEU=_GCc@s-rJiViF2SP+Old6IH1SxmjiW|&ZSmZSwqh^`=CyB8iwO;z zS1fC_rPJHd8|P-uP&*p2iE2k9R^jYuM8!5{V-tHiu%X4SUQArRsyO@EhGRz$1S4`f zdz$Qw+S8XImN-t+DJ(ipcSP%9XYdvNxSo&xlK6+rI)6QLCZoejKGR73u4BAR;0 zze3K>4ri5W$Y12LNkgvw%xGqxOdI( z02jk5kA{(&)UK7_U{X#M77mD>VjBAxDcCKL+hFxyhm4@AJ z*xiObZ5S7s(fs~k*n5VpLTk{ntadTcO?-R4v*;&*QBQuqJ(QA)W1)5UR{h+GZN-{b zE#e~(@2W+71cqL;hy-h1l?ht$Wabkz30m@^M$n26=@4-=pEOqe891G&EyQszQEg!v zqz=K~-S}&!w$PMSzf4BklvaJh>BibZ!s*7^!ZN(2iAFz1Q97e|-cTDw3*?yp=qOr& zSV#p(r!7oO6iufs%t#bXr!BCf3l$)pwh)JCR9i@;9k6dop&f9n{T@z}h7O5@wFCAh z2Z+eL@W7Z#J2*u|QxO9F5ZCH0B8P`X!rB2jgKQBg4;Lquc5t36PH4f3 zRN6tYh^E#K&X)57!&gnE9USAb3XkvsQGdRiO&uK*Mt_}Tm;13#O2WZ-9WjtuUC`K(nF=_#d zQ43Iv@2z6g0u-Ybpcu6P#W?mWMz&tDZw%wI{ECsSSGhbFi+NmNK`}0{pmO63n`9Ul z(^EP6FsmHddd0}rD@L|nF|zfFWnjEjtd)z!JhJtQk*!xbE)A#H5W_AoY?@)$8@AA} zhYee5*e!Uha9X;!~Nz*PlUb^ps_8fdhm5m#J368mcjW3)nPAcyFut^gps=z5BfpiYNKf)-d zu5--dv<$t3Kb^^8nr({dOmYvX4gBxryHaKC>BSk%Mwo?g4}^yzTm>P0HyG2|Hj1(T zRqS$;<7^wn))~fmH;N^zbt=~tS8mBUcvd|uvZ81F?G12H zvIGaaL(D7Ad7--Qn`v8$bADT029CK37>>8qy*&Es;>x-^=5OrAMfqDUulr`k*O&-c z>}{+4puE%9*f(x)r(GHI8ya$M+<`Qum1{~Dotjr#xVF4T5n} zwX1qiWqonY`>k@i6?x;w`-?D^EV?QW24Ha!%*O3ySw(eQcMgvg5y>lAv>?u+d^>V| z-G^tFWmVVh82P@6Rf$*?#O77(+O^9>svzrU@ez9)H1ngHZbN%YN&F^AEgQJe-x{@a~l+6DmBj?&aD1uhh zZY`J3Z`d}dUwzIZzB$e;J-GvqP+0WrHi^eTVA-ayl{I? zH&(V+T$$OWxMgP7!j|K6rd8*Rto}=J&e@sWpkjL7v%Bz5$)b^&i?VwboF~f`ots}#S~)VWwDRJD(#k7(mRFAHQ(E~! z@u=eO7oWFc7^J_6Vcltu-!!*KjB3?ZRN}VR@&Okb~rJu5G83KJE;PN{no!-Yq(=Qt}ef;=~$6a!<A)JWZbCGRDOP| z7$bu3>S9b1zd-6{63!hj$BU(%PQr8b+v#BMRB5OCS?1BAVk)$LbfQDy-IC2y8LUPv z>*b>^nm!ZD299-77eT5w{&@F+)a)cuTA zZjlEzYt+SKMo+O~kbtR>gjpe>%Nq+FE<)ig==?{?JE3Ljd7b1=m}P%C?jXmkIJ%-@ z#(jw6xXhKDsbQ((x0|KjBFC*QHLYT3vY`UbZd^bby`oJ%1Tvgd3lm#)ru}cF@Xi?0(QsnlN zB3>9OVtcIYv|q;Y2{+idbssMMXSNiTt(zVSoY=|s$=1C^a=!pxAOl0iP0~JdC1Lwe zjf9)tHp!-y)d&fU@H1;EsXHVTw(*;-+fL+4LG=Bf)wuEOCs}3MrMk;;XSpNRkAFw# zS!G<&anxN*QW4?T5H=#NIbFmkoO8_)XtkIb;k^7G4BJ0AP!NEP+06F)OE|L>j85pKP@9+C}*n0mMYwf zqE2&jZ^tPPBTS`3FwrWY99D_?!)k8s63b>9I*Z)gWtKKP?C}f-D9trIoRO`pwlgwp zg~*INdGa`oA9bQF2)pBtpAh*&1C*7XPC!U%t{1`vggpKb;fV;@72y+BoM~@B*aB(c z6cgtw^AJMz5K9oUA6bf!uLy{(IJ<7pT5)#ik0azT^fbb+5I%#Dt@Jkt**fbH?uPQc zijdC-^Vs5Sf3G8C6MGXOn-<8iIA0o2V{yJr?;z}l@O^}IKhyVBadOd3_HrOV#5X-Hqca~GaU6A>kQjr7%5Y| z#y&2V7C-><5P3VAvyuan&om#+!z{ZP-VKZ8B`DVc!_GKU$n# zm&BCj*W0jT3_I7b3d80ZcCBIe8TO!Ij~e!bVJ{d)9#^ls!LauYFkd&#gj413G4Zw-Sk=}Q6(V%V#OePY<>h84g@*SbH{ z#o#6%-Z8J!lCv^lGIxTCG+eI7^8SXTLbK1ZCg%0H92`UKIzy75a#z#7A(dqu)-JGHB(lyw z)~i+u%h6c_`**5VI_pq$dgYaCOBW3VN%C5G<$GldhUbIA*f40(kX@yf-_qrwpuF(y z;+juel_J=+ytMEe%7E$s$+fZ7_{Amjr{w2llq@PGy)^*b)%M|gRKr)ZXwcr(b(^=Z z=2j`^?orK?V6AYnN>5htB+&+NKCh@$Roz!-m$gS8Bi|<|M@R&EibN0-H)!Gpm$(Jk zp)WLT1I~BcgQqk1ma~2a4Lsjvk5H!B?Ilm2@_KVZJ;^2TX;a@_hxwSoc@;O5EV?O< zs?E8!Gia~Mg+et3)$KE2t8*4n_bD!1UOHwa_%CvREm4uEL)0JWuLY>S(#qeX_DTz1 zM(ypYE?H2PUrzr9h4ljpvkO~hfkCVB8MLl!;!QJXY`@B&`6coT*l*jMyUP~cogaq;I5qHS z7Z(|qR($TL6=&dt(rM=-rgR#8i4y6wTkwKLeP1qwR9B=Kj<8vDn(PZ25LgL%b!Qkm z0iAYX%kAD@yzCYaV12;hLZL0gg_=d7sZmNM>tH0ooL*zHSWa{}O;KnkP|7=?B`wE* zLOY>J3T+FRAUfv4yn(*qK{Gr6Q$4wUQF7X>uN0@9STsXWTz)>mgw2$+8J}^OHuEW~ zo|2CLC}qVCjCOZ&nv7cDagyTa#$5H0WB#K#G$0n_5Vk8=C&Q6LFA77C_sdWu$NWcg zV5?0+o1KNE2a0H|B`uL-{-a5^Al5W(Mq2C{N!cFH*SF3C+KgM+NKMYHg0FVbY>Zj|HBmRfn!JwpXd!m*t$ ziQ0rJB+9Xoh`d^)=)A5(q+gv^!=XL6wJ`5SOE7$Ed&;rWhkQT>$?<`q>qdDu-qNo{ zDi_!Fo1}Zk@lLQG=ELeDGCx5@?2cFVNGxr+S18vgPsd!xNs91XuP`4+J^+)DtXikb zaZs%bX7uLonS#-aGex6cDn^egL<&ZaLrDroui8?D`)xdx+}wJc zYRu?yoNmnMaa7ieE2Gz7X`_rD$Ij%89*0IfV~pOP0gb?u@qg6GC{}>KGKYg2i&IY{ zeOH5!bRORb9>)+4K*)||Btmv5*a$Yx*Z+Kkc?dzc2%UEuLUwEqAmq#bFhX`Er1{vl zljh@_N1Bf#1!+ExAf);Jh>+Lgs7ShRI9d`&n>b$r(tT|F)d(jbtVK8(A?ZK91f>7? zDy>Gy=1%&LEt>QnTQccCzN4i7`0kSa`z1ose^kXjM)+5RqygE;NCVPG9_Qkn5mK(J z!`VWKQ&;q9z)qkw)GvcEr2!Qq4X7AZ2*udS6eIbkShZm!{}f|0QMtbu_E*D5{;Aw< zE*6t-M~Y#tr%QX3$sKFhNW(@M#t$@_mL0ENm!IDhBl)M;PYwIIVQ(7twqctL;}g}i zd!vpuzkOW{o46WAVpipb8Fr>&;|(MEr*d~2R%O`phM~1MxsMFnWLP^qKP^iK7t;-N z72^iFdJQV!iXCql-yy|j8g`pucNz9m!+vfUDL+k1%1^Jc)36q3KZIY-e>UuE!&-ne)3UU7v6$Dvu)PdB z#;|^dooCo+!=@T`p}JF6Gweab9yRO^0SBS=@`#K1bYXHow;3g4p3mpVURL>OS>@gN>~Y1ci7vW27vD_@E_x+y;>j&1=nNN@Ji$~WNXRw0gVwTaGc=GO+g zio{)X*1(^>pomUuTX!yuN#ygh-vh1p8d$3j>KMoV$>OWFW8@0>pVec& z_7`=VAuJ#TkuSZ~%3^Vm52p>%tgU-jyw=VkeaD;j zDZBc)(|w1v^2#?z*A=eaRekEB%&-gr09HleXeq zTtsQKNr)+pmXF~mkw$wMjX6rAEr8TGq~T!L3>uBwHe*YI6f_z&b=AF*qxXbZK=ewZ z>khoqGxoxQsE>mgqgUEuAkCuG)GO@_WKqCeyowgV)f1FrUOn(a>nhA!K%sqiFSNI0 z+h}^B4MpW}`IY2!+He$2eZP7ducSp~D&>`^mxbvwmX%Dv0Tl5z%9YCPN_)W$Qt!2% za-RQaKI~8eaxAR8`ziWFQdB3T2-9hPilRu0dK@RA&!~KBakd#tSQ1IP8pq9}&mNSN zzf0dqn)#X`ItYe+|SR{TzLVB32o!qKsp_>E2uLh6@btD0xO3cyCF4WP0+9M)qhC zRDT=V_v7W5vvQhGksg+SmL!OcTq4NMI0R~P@5%P>e{iGb( zA%@8(&sb_q1;=ln&{>}xDW^ zgq-4SpCm%*$tlVy``S&~!Ns8)KOBy;+r7esf+vzwEZT>1m7rD85aS~!mBUuGOpb>l z_rpU^DMC)M7%N3E89BwvJR@g2Cln^p8H{z!8FF;sWP3~M493b^D?&;>@xtrncwZ?o z@82(RXWTmVu&rDZ@H6H6SA!o*Uwr45fwbQR;MlAKuLyB8sr^mXeg&kASp zqn@$_;r{r`cQ~mfdeoEJAq8?6LT1L}H3*9lvPS`J6aOQ^n-Q{;B9%mn<9>vDAS9*4 zj+K-WUtm&7rz0e##P@}i(rARFl-Q4wQsPkZG{Pqll1gHeC1tc6At@s^a8gEWq@;}4 z4oMlY{jEjFH;I%H-yl*(TM?29YLEN;6k%tCq>R{nNEsFI{t@CHMGiEY;B0%usiOFl z(Jq+$N*OVxl#ybjj1-&aVp-m`hVfyl9LXb<<6~2dG3<83o-_=t#K~_W&4J}I>7XvL|7!6}hJHW6L4Vz%t zWWyF3w%D+xhW*ko5=DAlF6*J!p!b(z^!`$84-hPh?d@VQZ-2uMGHkG6rG`y5Y`S4T zGVI5Ok@nGRkoM8*zF^o2!#*_Z6T|4=rD@x{Sj?k;mtyQ#HNTS$JJqmr4Ew%e7Z^6p zu$v9L&9KJ|d&)5Sa_M#dU>Lo+6r)#{mL=Q8vb^?&bulc@F#2+7+5v`D7)Gxy&2O$@ z^9*~wji0vK#lo~or;-Ij+6Qh}^FIfxa(6!aBzRlV9jgJ+vdZQ3!Ky}72o~j9bRwa+ zV;Q@W#wRQ1g%>%UuqfxZFB-YBqFLn1=eTZZ4VC(YzFaSGY124$tCH%V)RSSqItOg zfP#5TDHzNa6c*;-L>5MA65lJT_Wrs4d_kX#@oydc(d0F*JlG5*p*-}28sVl3a_Z+2 zK7cXblvRFE((i91WhQf49~FVJ!YyTm+e#L7c_yL$LDO_=G+j`c8b3cy7?hcV$z~0L zu{m6(3w}H2$pprxC;gk!GBxyh8iSH4Su+DMC2KB2r=LjHlz?T7k~Qx^Y6{XkhOn7r z4c7Y!=#sDpihVdme)#=bECP}>_hKx>6V2$1U4S;g*M?+G6_93;HAhG`Ly*N0%*Fdf zi#szWc%?1cNH%;@L1_{cPoKOwNRr(aA#V;6>V--oCjcg!LfBvK=S*CBbBjOW3Oub3&v#FVf2ZRb3A%>U<|I*JqWEIa5sXamk@kt=2=f~~v6v*@CIGER` zbaQxiW0WM~vu`V@%0rJU8A-&xjQeBwL}3FD@vsl}ehkL$yif^!@`xpDC#iy}+Eb2$ z67DOz9eTxt}2w@yjpr}RTfm4Z&XR8A(Pldj8d*)G}4 zLuwg~f2ruCDu^nhlb|hddLf7J6m-%uOO-IHlTIS5(3noD2HKcT;s~i1S3f0=Z<;nr zCvhCpGvQH>GY=c=OoE?Mp42_g;YU4c3&QUB>*#QjNnLPE^5}7dB$FE4@rMYLO*`BH z{65I>ec`l2_HrbT*olxl;`>4Ji2XRpBlh1Uk2qA3JR%81@`z28en@PZB$2iuB#Fd^ zy&54KA4w#(Cz42PXC#sM29ZSKn?n+*H$sv~6$nWpO+!c$iH(LN(sc+)7SV5s$82dl zri$c~M=fAdD|y72l1GY>JW_0)iz#`e7$39Bkvvk2k5e&{M~aasQjA2AVmU6RQw$aB zW!O^j3L4I@#ca*rDJoMEJlH0`T~y7=fuwxDDZ`ee`rWi&aBuz^nBrV@>468Bh1H(QxEE_FJ)3$dpkROKS8Fr{)Jq?G1jx?-~ zVP_jgZzQe1s|=fO*gb~bZ`dykd(5!k7*=E02ZnuYSX&S~TIStdEavTF*nWncVAw#z zPBZLu!$ui4#;|J)yWX%z3|nT{^M+L$_NifC7`EN8orcAB^Y5dbi^aU&h8<(rP{YnJ zY_wtH47<>~^{tU7yH#)r0q{p;HmgVLDlHWv%adbO(@gvHQ`Lx@n;;tyNn*dq%ZCFR_y)Tkr#(WW<9^=UxoLsnvx+Dpe^aa!2B!EkBO9$tiq}j$3vIR%x zXX_JXn(cv~!&(fzE821ng#iFO!OCJSqYN))2X9;Fi+wI)(r^$0OXuNna+4C2|QpF)Vmx4tE@ zHU^$fm_qnmdEws{gGB3UQwI~K4F*gaFVh?*%@qY#nX{y1UTqx7bh$0iW#1pU;yN5F zVRkEGN|@0VH<2(~fmbI=m}OxI%0rrMkZu-X2LGm%gc%L7z`H4{<%Wz(Xm=&TY*phT z%_7K-7Gr4$-qxd$gzDUL88fhkg%V_}uqFtyLP_>+gdi(yk{~M2t)Rj43tEtQ8k7$p4IP*M|SrIN$*#uOFEG5-meq9|cD99LGk_bgq{NV-pO9MD-| zmEZSO>Jv#pEj~yQHU(&d(S+8Or_Ql!7DaMkZ<&G+>n);FENMw3Df`@}2{C#z)k(^y zl6pIFYG0+C-xOM7D{Oo@m!w$`{Iv+a6cS7_S?~%7-YJ4>k}k$oCg<*xmPqD|->kts zX7q=9-cB@tCYUpw*?4GknX?zhrNjW#H`;VJ65L(EOkg6$nL` zGYa`!Sl|uhyF~GpN4YH6o++(NnNHN=olbJ+gVbT>EKAM?bP$^??K|ei)E!*vpnwtP zjF-Gcgu<*=)TN3BSCvSmV$QCWvq3@T$+7z7upkTNIDCy{%$a(A&~3{nrray<<_IPUuwve5zwN!{Ubl?16K=3E(^e$Cd+;_4 zmgD`UJ(EW!&j9xruLt%U3W}SgJs&87shBoS!3ndYIda^_8e3PFI`z|Fg9j0VUmB*3 z^>MTa={k~Zd`HN!GHo*FQjTxNV%=vaZ7hEKZ!2=T37~##2+t`5XCp=7NT?%!bWY(K zkx0SSJtrrVa&@axaCLEBu+c9SS62)pA&)r-*#+K!kX;?gI=*rw>)4T!tgAvuvW~A9$vO@lc<^x!86@jCI+Cm-30IA9 z4}>6L;%x0C>)5JaMaXx9WF6Zt$vQSkl67pVYZ3A#Bw5Fok7V6MgzqD)LdZ#izd-mY z!Y2@tv}5ZbNw*3iNxIkEF`FEZsnhzTT}x0DO4>1|q@7|U?G#($Vp+1{fMU-WM$%3( zdUt7BdUq*C(oQjwc8Ya(G4<|Jte0Wz{S+frr)l}kR58A?igAQgj0Bxxm4^M?FcNes zM_Nwh=*gwnmxgUMjHIZ_(Wgu0NRTRajA7>(MxQQ~n`YQWhM~>4v}iR>4(-5UXbBF( zvvb&&hV6wmtk>m~Prb$wh8=C#0K-l+j1vMiEhhwO+A71ijj>`c8n)7~&kfsR*q&%1 zn%_PyhRL0V9c^;LiFzio;{l&0dhP4E- zrPtlh#j?Eah8<>DFT?s9R&3b$hK)DudczhP_OM|~4SU+KXAOJNu$6{=ZrB#X==;Ue zXL%i6EanX`>_o$c8Fr>&vkaSK80j|6Z=qqg8+Ny0zcH-Fu#XJeWY|{2zA^h&$|v(VD9XBG&kL0&1m8ol@EBIcPv+3ZX@V&i7i&PqVD^mN!Or{zR z$m2?2d*{_Y(7$NVA$hXa*Jdm+mIs21ONPo;77Rao2Yw7>KZmasmz0QTj-X`7m3jE?qf`!En z0ORLB$i#K^v8veJcnG?xm|Iu>U0GNE9a&fZZ7wV(ONwDVF@IhCj>~;-LEl@BCGAW` zFte|%A$C8OW9uBgCs32J%0A00esiJnk_;?Szqy~246H|?!XzM95F4J)6~xN&hv45q zTtaM4{*0W0C-dWw#ZqGBV>Y{W_~&2^FZ$0yB5|L7iM=!AO~Br zp`)ei<*UD-k|ypYHi zzLJD3oB^pGNOL;EX0nB;-RlzB!WsB9e=C*~J6+hdYFwmQY~f*Ilb((j?l2_5hFmQ- zWn2esvDs|lKuMMtSx#(VlWbvhIk7mL-O?>5R*Wj+axt{4hM>6f<+_W_ajig(`A>3& zkQ0ceBHbY(_mdVYyw;YsXCy6KP{8?xSwdc&JmV>n;#MQ>`b2V|sSxB4Wew>IS0pL! zv>f!9pq8H|O%m3SgyT_^f(7AImL+n`e>5q(_vW#N{AdoFASG)!UCtMW+~`uUhSehY zZPTn_l?cWf`P*H9GHWH?FD;TwjQs@|9OyCz!zHylmKbY-H5?&XPiih}_@aonOyh5N z39dgt$}l2Ce?(bBGD5!#da?FrQX_GxDnQn#l1iTQ?Z66 zE?d8;mdkO_R5`yY;ig$Ra90&e+{vEaP&Q01_n2!0- ze}`*%Hl8p_mQYtB%at)UIcr!Tqh$)#Fh@=%V-4r$rC<%I3pM`zzhezY;L-R$>T(?B zsd$8!BI|);vV?CUJPaXU2_APwI0zx%PcU2$y2zfn@o@5nqO>qI_TR;~;w*G2_bPTIS zI1*t!LcRrL8Tl@eWn_CN%g827mXS@HEaNo@$ujcgCCiAi3Cjq^(3T=2%eWir2TWC* zt&J>W9_tGsH}2#yn<$T|3;Qf1=Kw0p$e6N>ijiehY>A60%c$5hhLL4dj4Y$3CCjK7 zSw_XkGAh>H#gt`Kte0VA85N__rfEkSHo>sThVfgE%3WpHeTH#-(fod8*mH(`WY{Ld zzA=mrftr>xNA((oE`~2rhT-WuIX+XB8*dn&r(%4fnjhUW6R4clQ@ z23S13Mxl#kd50U;&#)5=JH@cm47=VA#WkJ#N_3hOIMQsYkKR=S8<(E?p1JdUTNX$<&`g$7cMWa+>khF zwtUgZJi7RChV1rIcm(=eYhp>TDQB0psm4jpiKQpt=IvOu?AwuV*Vc^18Dd;<%q>5* zb^8MG^dmQhb;p=`VH-i1LEO(EUZ)#vx=e_t*-Mklk_~k8Vuz`_U&(^I^W(U)u-o6< z3j>!w?k+0xW7n4!uDAKIHw3GciPPWx^!LLZ$3hE~m4W3;vOhquvu9{~fM!`8PDm(1lu=0cV%$B%$M{4U&$f{)I)~2s zL>SL<;Y=F(K`A0c7ln!>i!cclCAwH;YJ?`+(vo@fl01~y;`34mn6{zk6(zPvL4790 zD-KB|b8k~>Yk(0|)spJ0j6$&VUtlG?z;nEW9TVdEa7Tw4j;JO0psA=aM?uvhrg&4wlG&T#m=V_7ATW zm!$ptUIdd7Q@oky&L(Q+~wsdRO}6swR`un3KQsYsL?_2 z?nBrMVYNH1MR+2P*^SIZxEA4k2>C97m=W@cq!IfDl16+jNE)#_C27QNmZTAfhNlre zjga$2*)mBM)gmM*#C8lCAkMZ(Qi#osq!61KNg=)7NxtPuuRg43-%8?XOtlltw z`ciD2Vec5m(M;2lC{no`7t=YUiuE##LyTg@hS4!iv7v@hJyMLr8e>`F#I4wMhS5b$ zF}kR!99`5Dd%>_3hHWulI@h7}n`*E3B^*E7A3iwwKeu-gs0+pr~uanngn`#Zy4GivBe@V!aJJ#xTzQRJoH3|VpF4XZVbu4kH-u4kH-u4jtT z^-Qr=AXpS@>tZpFPH2je9#Xkd!^#aCZ`cKfJ=DfeyTrwO_pFp#K8|_BO`6;8Z~0hW z`5L{n8qf#1HO6G;N9_7o`8GVO{+^8=!Jf!&kLouRuqVyf^l=-CF=$Z<_IG@}xMou; zu0mG4tERM)u0>_on-KKMTF$pD+$qjj@3-=|cog@mvREF7hM|~vi7g(p#QjR6^sC}p zTxB_AV1|Q%*}VPiqPmZ;1*C}5=Ll|AAYLd_B-{j z7;@W(nT2x6&PhhMgf>8gyHyd~t%L*0x%MbwTimFq)pR_}5t zGOdch%__SlfpX~}CDzH7+|kliLf@q1OQ>ui_T|w*U__SjsifVN5mPF6a7yI`<%JvJ zNA)ZynexKzZcE0>tm2$USBUpi*YJLf-9ACSa2M1IH0eCG8p7&4g^SI3IPFG^q-K<) znTVK@G{3~)kVw*SqH>g^;g=+C@Pn^P%_C{jZN!*J(v*vVz>hcOxPa&Vq{WXiuJFb= z_a|0zvq&1yh8oR*BDcY4m!5LW ze>5F+tklE}Nv7i@#i^DnFog~_+*TaImQB*6HTtKdRHbIbJMdRH)j* z>=C*J`~t_VR&v}URAounBN}Avz?x!@?i9K1#vb`aYVSPq_vaP5bUx{1>hl-n|Epoya9g_n7EZodua#o)kpGZ47?vhZr#AU`Pb%;nc z!6qFM%9?DFk~=z`Gno8%C2I0C+?CJ71v6#+2GCLR(M9)2p9G8(zmrtS*>sH4iz1PN zar%XvOv*U@Hia8goJDB#OT{=XgQSu^jTonAa4H4kRBfpmF;4Y3)tGTw4YV=i#1T_3 z{!iafkpsW~qi)64%>n#4hogza{S6gY04L!@84^&O`wX%s&ev}uLcVtNV&bbl9btQf zvk>l$kX}s3AS5Bx4p((NK9qoVUw6* zqbD)N#$JuE6T(`AT@aF};wwXC9OsPgAXo{eJ?P*j&bMo7|w zUQ29TB&vRkkc89T&Nf{HOEs1zeXrPw@^yVfu^B9$XSrE+W{ zijkmFj7>o?5>$$jpi+zkm0~2Q6x+iv5>$$jpi;SF!}v55BSEEdROb|%V!ElKV%$_wufds-j$FN0)afN)9d&00^ z8Ae)4^V?up3$!G~TDurFBQ$I;!+IEYpkdrfQPU1LtirIdhAl8`kzrgHU(-Hr7}v#D zY_(zU7}j9ecEffW)(-TLrsYLB2?mWXTGwce(Dh=aSikg;7mTB6j z413eCw+;K7VY>{=0u`ibb6hOu^)l=z!-@?XY}k0ixK_TFM^uqtv$_>2{}D)9+AHgU~-ng93gNc9F{S;AM^I$=%ZN}*YPb=az~wDQlT{nnFd zD$3~=M{RiHi*u%B-kUSB`Y*+B_mPRG>h7~ByH8Qh(=YW~nN#*!zwJ3sugod0hpSIN zY))9x@7*~sB`kHH=llAkTkihc6+>`ArIpS@OlhTR3@?ea(sDG!D6O;~I)1J+^aZ4w z;rsJlHYWV3SPu04>50stQQ!;zaPJoH0FO&wEe6soS}9*l2yz?w%*ET5k%mU<1cCk{ zwp=`QCd!ow^ly?ziu(F|hGk!l&!tkD=-YD^@}$P&d7r1Gi1-#pDWYLW*Im**C}kr- zMGt4vR=zV$PdU$jl&;~69S}ZYUmrh3pGb=8gcMP|{A=*4j=ytit zl$4ava5?`}(@f835&T6`rbpLyn`2Cm-_CksMIyE{Hk33Q4-Sy5U(0AFA#{_7pPPmd zqMaL)CmPBh^qL z$0uP$^WLGy5+O@i3}%*o78hF|>n`mqF1D39A`~zyaXHp4{dk9dncCy-Qm@>!N>ZgD zFK8c;6WY@&r}K^c#_kEqtkt~Q6y(JWUoIg`MP4k1R0{Hf9aIYPqRLV=A}^NURAcgD z8BmTBiS9D&yTjx~wWSUBS2g_!frv`-%X5F#XM6=6)Q*_at78kn}|@ z!kZEDMZXOp4eI+5ZbJ9~LUv9MBV<=bdZGiur3l%^Nl)Y3MAPo= zVo*;E8(`QahRrgJbdaXK-7wHk4y!TjZNoMgw#hI&Sy$#xFz_|ME-n`HPBQEi!>GNg z92aL|Eau&87*|SH?0Lhg4O?Z{YQx?!tiiAta@A|JbFr9rxM95wBZ;7LBMh5u*j&Tz zGweab9yROtb+TG_1(5v4%}F>=MIf8TNo-Ty#n6k5ixY z8eDWqF;0C_jEgQQ_L*USGAtL}p33d%VlnS%!}=P=MVC~L)1fp!F1n-`r$Z@5e;38# zhTUPS4XZU7v$(ZMznfxi2MXAO+ER%5_5{o%; zGS1&?E?2(PENi(I*Opm1r@(FGjq=Lc#Q8X8GQ%WIF);_;9cA3Bjn8E7!{On>WPH?hZSnq`g`h&fJvyZ}i&ucAd7 zbDUFe28&qNh*!tpCe88Qa-lPEAzdK)Eisxo^=2q4hYP?YH_yow^pk7$G%pA(DnT`? zsjx-ObFRK|fTY_fS1PwF?e%w<=luA`vA(C|xXp-yK61=|v?SGt1qS~Anz{ZUOXU47 zrMX@q=l2YGa3om}n~oV%MR4~s&rlEB9et_M@ z4i8;6YOd2V?kuU#G|SjeeX@6EYBP`q9`#Lb7@yLAgF>xDsAq}~>pJNtSlXtYNOufL zhiCAt=k;)x!3869054c9S)}vTpb4w1;6!~j_&%|M50yOj3sqb)VgQW`zr+bHiCzG- zO>{-@-turrANbYV3;p72peh58gRsxV7G~`o7`oMnj|Fe_8xhJz)~RU&$_Fr z=VGAgC!rj!E=Vy$X9iO=`u+c-9Kus{sHgBn$ipAK9{3~WKnj8s!y^buIj}wRm{h}w z2-#1}MEFO9^oZDj@F9fkhDkN>_r{mA;QiGNkOogkb)@SeImrY zh#b5?1ZNu~PF3V9g!~@BZ`4XfFs4+5Vx%Gzn`d&@8b&HYfWBV%=R#sR+e-8OEVOv0}qGyeT%uFp?6A(F209Ebj`#NJS_{DnjLcZWu`k z#Yp9-96cZu;}jjmzBDWY^{5yv!xkAvZwD<4y&W_^dOIjaDnc<*5sGmY zX~nh}_N`&?MRan9xLB6=J;VAMcD!L@4C5-&dX4K1TWHwBhAlPhX~Uj1Y>i>-4Ex5g z9foyl<=@BtE*A5+I<(#!SBKWJj4^D2VG9gfWY`^s-DB7bhOIE{L&H8XEE|M@UYBb^ z>vg#@i(=$#6(cF37}sb~jHHBOT%$!X`Zy>y&9IvdyUno2413D3w+#D(Vc#0&fyU5l zWV=|F*WR!$hUFP{ykP?jt1xV=VRH?eXV~+wJ@r1SUCj3|NHs-={K-1>HnL9glWrkp zP$>`PvyaC{Xk0Z}Jq|EMr^YChC77&J!i~^^wUbk9h*tTMte9*Ypaon?GgnN06$J_N zDr*vXmGVWyWa~3dv)MU4Sm*@Xeesyl{lDj# z|MRtms{7umy47{-F7@kEr*dKzOmPvsa@*0RojRbBYS-XZhTMl0(Jx_$C9NWcSRN^d zhu{@#+aPMG_E3xAs(#5fhb4Eyl$y0AD;q->uMlz<0WeL#m-`7{W`rlxgt!YfN7d4M zGJK+ZXlt3R9*@c@e_4u{@|RO_e^2Kx6Jc8u<}c47)neq~Go@MlCD%%39JA2GS3f$- zh6V{b%jS|6Gb3kW1ygl0%=T>-f6>V9U3y%ah0qsQEcjb@rmY(~uc8lx-i@ zG)SeYBCVafF&IBSF#gR1Ym~W+Hdy239tQU`J{HL^_&!Mv8}xn>vbW$sxvL~PIKU*l zSrGgtN-}g<%_iWhy#t#NpI~^fhN;z6OBcDmnVMaculIf}Gr*hiQe;!`hQ&E3$UpD} zZpY+->2QG;s|RyVknO@i!)AgxyGyD8XjqS>_;W1iSD0KuU9`-EBaGIt&FbKsiW`%U z^fFn0jFD1(_|w3u0Y3G0E^t0|Mg+cr2Y>htI!dm+>>J=7gip|*CIbvAkVKuEgh6v8 zoezT=ST?HE!&&OIiv$^E7zVLw1#+MCdvjaa?j-$@Fle2Wemd^C#|O@IhW_}aB+LYZ zc$j~=d%bh0K2{2Hv^M+1H5uU?g~ZN6#)QA}PpX zvy!r0jciudVpn6Em32@X+pO?mst5lcop2+ojkTB4f&<{g_blFay5cY2)ya>%88`LN z$&VTkk{^A9@O*^dBfJ#9KW15pV6}M|_HrAH9x{ zGiN?PScj0kn*4};pZtjZmHdcZlKhBWk^G1k8Tk<}F7l&p2tgBMMh$1q48ry%gzP2c zM`s}Ev9WXjhs%?b$4S?hqOpFI@n<>>W~(xHo-K~BGqm(jkHL$Cru+QQte~Y zqTn*BwR2-pXFt;pFpX=DX*zlXW-RKAH*KP21nJgG?K4+9=Z|nnrKHTIXe^U1!=uraflbYo@(v+J~lnVp;@*M(fMqF+G(a$n|7gTb4|O#v@18lDru_6e zm8K0c?HtqQm`1Y}JqFEM7>hc0oAwve$c;1|xsj&(#I(;%``)x4O(QMRyv1$|LoUu1`<7{6<}gc}ovRkzQyUq92WX z8GBBx4Y`JqRqmN)X-6emQ(kB0g}Su~RzzF2T_by}*o0qf!YdZ;ebYS7NU=a(aH6|u zL;c4SUj*%1zlv5xuMl~y$DEzfF7f)c`u7da+bOKM3SY!WUh3HQ4K#^4_Px?KE?Smm zQq(~f91FXd(fa&6<#o3{FNEsDnLQ)axaj`Mk`Oj2<(het1rj4La^T81npp)Xjuo*^Q9W7wBr8_iki~ zb9ajq9L@ns;ULX|Ug{TN5FYzF<>MbMo`_7tas7-aH^yF9&%bD-cV{{!ap&D>z~b0I|m;VSONB2J0ohuHGOCU(Gn#s8aUb zVnA#a(&MB?j>gQJ?dSka05MLo<8S26v|f5zl0TaRh;cB^S8@2x3=F=*K#ZxbmsCx7 zpW(pyuaZy^0-+P%$LZdL*D$R!*iSH3Aaam%C+748o!(>uVy8>)@p83rUXPc1Dl8m* z)D!{`9Kdn7z#D^~C*MnL+&%ib=*TPyn=bYX^QsXr&y(c)1kCPnPzS@bw6S#k- zGxWz|Ntg+Uv64~QtLKl0Z7177XWZcy=QT;glAP93EV_>*(q|I8Mo;%pJORE3nyc~D z$i0wOC&{O_d+9)oKTm;=NL8{=;(A?DWC3DN$j*!wRWJOH0I?Kmmj#INRVfP)TW7f% z0Wm(N8w0Ths4)zj(kF58;Ti;D?1Gv%_)8Nfaq$tTdx8(pBJAPqME9IIdCV+yyMaXl z#Ck&aK78k5+56y6uPZgEieZ}o>?MQ*Vobwr4i9Kh#akB*s(1_nvUUhBMz||N0y1J7 z0y5qZ3CMUqA|T_XML@<|J~}{xw|W9HKGg`w@^MxHLz;9o!)s$5(y>1in6Zlzn6c{; znDHVcFyqDdPlUuT1ZK4e3CtcqNMOcZL}11Rmk7vSakts^xXpLbeL}IG#YZ{T5YKNM3glROO(!3K*n`PR0rrl&(t!XPw zt2gaE(}=|MxL=v}t!Zt6q4c;!Vw!GG)4G|~&$I!iarT$yJ=?U)Od}H0@@_EgX46up z)tN@4DlLyjRhqXQ?vko?aAQ$tFVprmt=zP@X(LQK&9qBQn{V2krV)whacN4W$9>GS z4^1Ny({$gOMkJ<5s7KuQ%yU=w1jC3O}o=HA~DVTkZD9> zs@0qJtZAD}``WbD1zw#C-53^fGp(0tCzy7UX^Szg*YCU3jd=!MGC!_-Nog0KjV<`1 znqZ9g$I9e4^diHjXGQXB@{g3L_lgA&ZM=ZQ9)&^C(U9SM7mk<#ywEjdBYr(L+H#|; z@aYu%EpP5Eo?w!?T-CEsr`5 zy|3v~k}idGYPlt3jA>FxgGp|eY_nCX$`Sxq>yh3y{L+Q10Mf>dT-sV8nO7<-y>N6F zh_N&}vdOWo6|k4LtW}@J*z1zH0pwNTI#}M0Hi?&PIH@Lo|9HvvIQ%RJJuFWU_=RuP z0!!Gp*{$x@Iomra`^xEFIpf+^4++&G;mWeM)#!?nZ^YriT@iJm@j&@@ZnX3C#~$aIRD`Ij%UElE`G>e=W0|>_`f+k z&9;lPfHpr7h1V)$a3VZkfD)L$ye zbOB`F368qtnS^4z+8yjvTkUF*kAncdtEL07N>t(4yOjocjLAWPAaWpZF2yy ze$3#^GlUyN)D6!| zC8QAZO9;Nddp2aHNTDCiu@)O4;KhTHl^rbUy4%6xZbt85>#*IOQKu|i-%H3m1#*W?(HLG_=|m?A=`dM#>sXr8!vv@ zeI->cx4rC3yt+7{m>t)juy{+P*ztE=3#{Ux_J9f=4_iC)BMkXoDtKS0Oj z?w=eEdL~f2v}Zin^)^Z|O2+u*IH1IQ0BIqfjc;YUsU^8M(9DUj*ge`xBBf(IRFQ1^ zOLBSN2=`KXa`4#&d#&knri~jvPAWuYvE(YmwRB8TDHAwj&Xk?EA$qRt%m|zC-)H$_?-{~RzGP{S84-N=PP7?eSN!F>`T+ngd`}{9;!h)hJ83ie z%CXI>YZAf+gqI;?xi=!@&F^-Eyl?;$61-0UB_wz^A%NnALIA}ZH~|!I>;zDJ7!g2y zgpdG=eU&r(*hhgK65k_y86kW4I)v1W&wl2%dOJ5j>S3BzUSuNZ`a? zL*T@&MZm=ODi1L2j-j{$CdL#nsYbx0+I%;r7FnwCTxmK2Ce`?&pxO(jy=B@zO(S&D zbc9Ztx6qC0Vq2 z?C!=|J6%mHG3{W}&NOX;X^TzcQe0ZzUrqa)X>XeLj%lBm_PJ@_oA#q=gj3pngj0G9 z!YS1Vr&K$`v@=axVA|!TaoH`+d#7pln?@ThEswxT(-ByyMqs5Hft6|mR;m$LskXzk zJdgm@irrXir-x}Ln0AtB1XfxeftBW^yCT&tHf^zKOHKQ$X>?(vd1=k1d1=k18X=Zy zgjlN4u1huIEY)b&r5bUTY6qD1Ytsgr#wEKn-Fc>6WZLDXB~2s5(qj-}Y2JF%o;B?s zroC<2cGFq_5@}w}4Ak;y>!ljOmTClBsvTuoscC1L#$~)T-Q}hwP5ZrRH=1^rY4@1+ zqG_*~_L*s4n$`nja;@KiZp?#IoP8K82b@|GFYx)){4Wx?J9<`HaYa-cMY^N70inmG zvhyhH@h2b>PSMh49kP0mY8|$DU)EWNx=@(FDsOeL5Bqo75eaZBzOt6r4B7!!Rkrdo zED1Kax@^ZPZ%MGB#j(odmsKTiSMdL>RVAOw!eAe?EbO`vM*#|1aCUJl60Z>#PyK+N zwhiqFC&pMHtaD22wpLEvep*LTs(xTs*jcMPYq^tZ1K`&4@JftSwRN}79$9$+`PKWP z$|kY_Kyfj4W^d5!vS64%BCssY-GKd_I?9qS*W&tNVt-b+Xcx|5w1)M3MXf8Wxd}~BR~lNT${J!DYz;BA8(c;$3AUZj zxjLV#%cGH9%n7b-`4ot=| zo4(wmL}y^$*2wRicUlY)EXD?J78W~5UiwL+4q_?rT1n_a`oubNz|#a4`w?H<8|h3! zG5R@QYLUW5X8gtXEB&kk)gk`Ez)nh<2P+XBxbV$A-A^=UH^rn{ZKc|SuzkK1bD6n= z>fsfPyaa?i(?JC0;isOVbi7mfz$!QrV1o?Bk%Z4XNuyAQhe~KjDmoB@FmS{@)tZ?%KRWt;zK*3F1zLSQ6YionWqPje*V`jZq|5J;H8s*4jR zm6i2EU-2f8O*K|V=r%+O8s2CcU^6++3y~4dHBRry*#USEHuGZcxa1z*v`^`^l2Az# z!5e>BCS~AZxiph^*h1$W6goU`*f2xm2>&6;9h?G}!GuP6S>FJ($iT~fDEdzr zki9;n*ju)DmrIM^LJh)nnI&!FATMJHyu8`Di8vCW+&9X04hL2oJC#eKw7G5}lP`{| zcrM`o-=Lz`oC~4(C@?rLl4C#buEffzQ zY^|qkF9lm`g8iN@2{W04v4szJi}gC%(Xzd>9d?LpcTB5n`kg@9QBviYFUdnqmP9^V zn=0FTOUEO7^uE!4&Jh_q1*^z%XcIz@X6&3qP5yjIKCn5QCg#g-9u|>@ToyX_m1NIE z=N3e=(76N;*yxju&aFVoEOhQs?8-vtQkJU`om-1tjp^JvsC>q!n}_i^9Hes%mN)oJ zCY|HcQcDd!YiS~QJt*ozO}#h9=de+Rlc@!a0nCZ2l;A-v|@DfPasWx3=35@^b$ZLf%Y>>v(r3uH&^wT*qex zaUGu_#C3cg64%k*i@1*6pSX_QK8280Mz9-7oQRP4j@_U5ju#PLp%P;dzKW22nfQ)< znD~xYC-EJxTH?E_5WbHPdBkq)O@zdF>|Vrp?1P&T@;%VQczXa#D8^$R) zH2y@RBsx>NMSbroC<2yQUGNYPzpY+Z}DF8eyrHcd%)+9aHTT(@r(* zEYoO3rs=LR?RTaO4uI_}cEblk;Q z)LCR2U3jVXThr*mOSL;qyW6zqO?%lidKlMZd}~@a+4up$ z#u5Q*p*}&D!V21{`aj(2o{r zJEMSfi!|3$dq#t_XXJ3N!62<#4AP!SpL40c#ett_X$G%hTeia$yKiqs|LRLhfxWO} zX;dO%FYDW_1CDVPQ zve$syDoftbmC$HP=1-z5>{^bgr;692G$U)%!Ck!V6>%8<3w^VH@A_sRNQYq+o^SSK z{xNyET(kG9X%ow6BF1&nDzHx4Q~#-R_MtevYCtvyF*P8YhL{?Vb;Oe>-GJ=AaoOfSnw9sr0Gl#{unUlPyrhjZ4Y_7Y!mSw%!U(xW$o^K1AQ!!5&69nH zNzbD}nAh_Tbxx3;N2r=(9kmlB`@YSc+Q{k1j&pw#c4NI!FZMByg{8O&&*_K3Irp2+ zNrCZmn1yjjt5e_l<_TwaX4i?6jcn%wy zbk1HPCAA4)7ter)wSHEL*fp@%V_n?aP4ey)fG(lwi>!stdF&Zz$MZ9=tU@WuH;_xo zfqMae4i6mMV_7`-2a;m{K)GR7#q-lw$_=iP7iLeSX!Q0&H|{<)^Vk!Q)J4i@C3h?~ zJ_lUs(?&zAiC_CLNtBy4G2QW!&iBQYV%g|^2RF+x`|Fb1?^PU$@s#t&1NN#~*=~ko z_Qxe*Cf39jST1|h*4GPBWw!NGk}3yFV&komM9Px*{hyO<-!Z#a1M3;7MlUJC@ifaZNwb3UVCt8M-Svb-4*p-D7)mpAboM<_AHRePs zpz>*#&WVT=f}AL2dBdED&ymcWh>wX7C;A3bPdQVF;KTQl%?S6wU$Lu`677j?QleWB z9)vLEZr33!!#3|H350JTycQvEQKUlT8n+_krA0c#+bZc0FJICj-tS0(1x)U=yS<7`8%AL)=DgLFtW8gHpa z<1N+bY)iFPZY=81*_LX%n#LvRRqJaSm!wy1jA;u@yWF%}O(W}Jj;O=M={4^|roCa> zdegR>)&jRs%}etv&CA(_s?qJ1YNe*p?UrhEyQS$k$56Fu(-xVw#I)N@yUR41VrkwN zO(W}3?Hkj`dQ_uP7Gqd6$BngiI+#XNEKS$PG&Nd|dkegDE8JL+poqa# ze955pzQq*Qg^!QDNn8mDIeDK_wVBhmV2>8TwqwGR3XooUS*<@&8EBWhtg7-}i3N#MKtxCLMPT_rA< z!>7yQr9&5g_Fmnsmw40nJnrJv*00aU6YCn)$J|yuMBBFIZ2PfWE0aIQ`~Jwa@y@O& z*%GhW*%w_N{f#IRNt2CF(WcOS} z+Q1(`_48NbGFRa-h;xe};e{}e=N1=Cz9W)O;)%Tk;nE&4^qVj+3;WW`p_ zVZr2cnz=ISnTlfg*mazfr4~Gy*?Eonz-kqmz*izAk=OiDO3KD-hT>Sq@lXMqq8_r% ze-vb-5DW7fxVs;5U4eRnX-hEFL;o1GcmA8&X0H2AbwK=Sjljvl=O#F8doBDUsh? zF8hv_u0?+1_0VITVbVjJ;5UOL`>f{jo0}zhehynEj!nnmk7fhDQAU{4aD=~A%BsrE zX*fQ;P>S=N`OK4T_EoSR`ObXef9VVfn6ZV5Qk-FAm;mN;%!gQHdfj1sb0 zVxnp(DboO*dvPxC(M9ywD6M7NM;4uAd*=Y)@Hh^0b3DK?3OhY>_mpjaNxmaxI|t`s zAzN4m4&a}MOq{2W6g5f!1$}k#`hh84WE^H$`}D=*JS?|)v+8~wIBu8&@%#^v<5oz| z^=I^rj_mLJ4legc26Bh_5E~>Wxr2@G@gdgmMA_@l`x_$Lz5&??H@830GsKHnNIyx` z3Kh(5o5Xbd7y}s?XW<1 z%+O~yOTu7_c-W8?d`tFf3$Xn*1h!d?k7PTzzCxx}|F@*d!IPvd>7j6Qf!bnS~hf zL7jydt*~5;h!N32V`7wo8p9wWofxgPV$>5mJCx=PJ`ZVG#3xr~V$@)JLY756r1h~E zAHFkfMz}Zr^p=WJSEAW85zj0m+K^&>|~(9U)l}J1Z@T*h$HZHX;NQNNh%kIw#l-$&A>u z$c)H@$c%U`ks0wCA~PbZAPX9V@H2$8JR&n<_aQSn7vW}v7rEQ)bKK_ZkjIKRa#vQw zn6e_($cj{(@5YoBsm2qo>Bx#y!kZOFv(!Bg> zOtq1woo*Uwkf!6aQ`51{RlC--2TXg!G}0hV_l#+8n)Z%qWWbt^WKNH#ZJ22zO&f38WYb8Jw7jHg zzc=kh)9y0u9@Abl?G@ABHSK-VTH|)B$0&4TQD=YC4l?Zs(~dT+!n7*WrkOU=wBMO_ zgK2*ePr53)BbJR52o!2LZsiLn;VNdeN8*ov=dDmXxb>#Mw>R@v_+=< z$+SP4_ONM>o3_TZ=S=&;w9TfqD)8#u){TK>VJxrTqtcBD-zvvMKyoci25jugoqTH} z?weJ~PdPKt!A&TR@CQibTMY=ow=#Kf_soNI8lZ==#DBYIEL5Ic3+i=Ru`Nbef5Ehs z^?+Q8yYf~Ay|^EDbeB|cW+1D8#rfiJ-7diEUGZi0Q#$ri*X{7H4yWxa#euu$v>l7| zf$eM`+94(GtC*w~K#G3L;lYYYfwjCe9?Z(%yq&;{4&2peyPM;>4z88adhQyMNG})& z%quwIQnFv*zWsm<7>qym{}K!~!MuBkD|ghfL*1=&r?*j0R;=BvY?_9&+BM6W`+c5o zbd?4pctjITUaAK)>Y&*#tOgNrL4lJU{r1SE404qsOuE4xb^#*6EC^&&66 zx9Cw`Cmx~`X7KKQ1`egP>;lA;mR*6E(lWeXg=pE|GtjcV@GQCvc|JwzW?3=iTI`Sm zAD@=(mA5Xk%(Lr0VI$GE6HDORdb%~!P~mLKrA~Q8*Fq;kJKkC9pg>UXIsJ1fXCAgM+*Juk@=Lpg;q6OM=DNp;2)xFfs z@70~Rw48LT(z3=wS$Sj5M9294e~F}B9H3)aJh>DM0kQnV@q=$2p$g$28fT|;<3+FI{8YJ0oZ!SiANRl6s1EX=I)LZJ% zHNX$T7>%ixN~&gfc)v{&`k>)1+4eoWH%O(HM!Gxq0euAB6@@X{9OUE=(T;PDQ6n$+ zcxXOV31JF?2hObIh{{hCLTgNPuq4Vh9Z+eGkJdOK;$4P4nk?fq*$&QI3!=53nAq{z z(ATo9sExTf4CB@6S1@Ek#R@K#M7i8&k$hYyslp!MSxuff_Q_yH*sOGJ=zWcZYzL2# z38U>SsYc>Nc$CV$$la-b{7Rp(U|_Xf@wJY2x;rxw2B2WbaA-DiOp{XMay<~vHAo3d zBZoS-Iz?Cp!v}*w>=u(;e$y&b@V`-a*}q$$X+zkJO_d(o{*tG?T>17@V=Qc-@Y}cV11Irv%>Y zR)x<(eddQ9c45jFK3TxspJiu8Q?DnofV%{XYxK!x>XkyuEZ~l>Xj#A=-w!lbBXIW? zb~Off4Nx0{J3gMZ7zKBH_+|!oe5hsycYJ2*o&dO$Ie|fP$M-juzBm3dk-Mj$liW2R zB)R(t;rR%8+=~(NCJ3$~)?K$C5>x>O^%Q;m-d)%JH|I^RzYu)>Vz-8Aw#O}D_b+fC!y z(7YsfnvUd7wP#Fw-n6Zz{cIZTxHNBx8v{u&4ZX)L?@ZGsn0Bsdv_I72@)n`x@vOOZ zG3|NNaQ@tMIAd03hUz(1VUyNaK5!0$oyVNv#m)CT)rrlv07hKfx zxZt9uOPN+@+P_Tu%rv^_(!8_*)4a6&Qmxc9T7IcU%P&nw4yW3=rd?{7`4vyIImhq(+)GOuW7?f8)@2krd?#(b*6DXl^)}D zFbO@zTW%~j^;?E}E&bH<`9;*%9(OOIzXA!L6a+{UH$D@8G5Ra@(cd~je?1}4ZrbtC zAJ%rHsatOL*9LW=SFlR+3KkQuVDD8Vw^uG1T8zaUU6)`L$@;RDAGA#O0~UiXuAv=c zvS?%HnC}NH2Hy32{lCosu3xDnEY*aie!|i~Lf1_gJ=nV*z_1i#XY~LU@cxxvGQG#s zq1Mr*`Qi=CbphrlS|={?Ki>3G{g#RKG(vk7NjALAwH%*^^RGBe_FU&*-M|C2(LB32 zIR2sux;g$z!pA>XbVTE5yU@2i+@;a~SJ3BIIu(RvpVV?UspXO8_Ul&oLfegG&}%99 zwukpKt*Y3|1*%G)UvXXL@g9=Fw@@$F)fUj(X|k~6&|+e^%Hl!zx4gKTjBZ9@j|Ynr zNLw)$&c9@Z$GLR*CC67cZ?tszWhc| zp9{y&oIZNSbS$eVBS;?(_Rsq~@&~-}RXQ}CTZiP$f`ikfm_fL}I9=0m-fZ#P$c4_L z79FJ+o3$7TV7M2K-7&iUTR zSP$9eKR%WV13`jBO}A^Fhx2-d^6~!X|Y-Oax zdC(cwBoO52g>zOp&y8keUhWH_Wd$fRkV(z^3Id5jK#+-IQhKWZcM2Le4UxURb({jg zT?~b=SvYmrr&d8A$T3^c1((7{zg@~?e{fC85D;VqZ;?bnSQoanW0T(_sj>mVOJ%Pz zDOUM%*;dD8Bs#x$w|!U?27=7R0VQ67GBO^|7g$?Ka}C;j7tFET&6$drUuxL$i$HL$ zb|A=f z_enbS5k@;RuZo2*X*?|Fl94<$ljRBF`;;3e+vBdnjBJNwz-Ip#DgUgNy5JRkpQKGRr{QzelCL4LziWZSpH^S)6*6H7?; zmVBM%6!Q_A?j!62w<_6zAieppLuQ=&2{SBwG6BH_1zCKA@s+3XCmRstJBBj3;Nt`+ zT&?G_0Ko>!mG(M8U;P`gt1%GV3^j(KV7etAAJ=+t1%kXDG6O+AR5Js?_O>Ss1WU2U z`|zEOb=VhweOw*TEfK>u0bm_M0zqCB+~yqca)hkWB!s*&ry%5=VLHMhgfPKM@Cv;V zVGQBz2zfEyfp7>yfdpf}(J1@Z@JM|j~$01ygke4mN;X;H2hsa}2!al?Hzp%Xt zA^REr66}lwhe2myZHP+b-Gwm)g{l!0s9s*wQ0Mfy))_OxjqnYPh1ItJ5Y@a*X^Vy4kCm} znvRaaR2yyDSktaF?Hbb_H0@E-)|J;qLM48Ihb*4Z@TK~2})w85qgGmUsq z(~URn3e#xVr{Ci~(;hJGMblm}jd)PYd*3wTLDjxCtu;`aYK3kr>JSgA*3~rPLDgv3 z$5_-EWZF>E&Nb~q)2=h^deevpwY=q~5f7^NsA-%Ss@ewAI7L)7P7!4+>hyGDt(`um zm6}#=+62?iGL3jp%e&At;z8A}Fzp`G?lbK*)7~_Vcu@0lK}xM(5dfQNJG-%{Lp-S3 zzNQfms&=SpTzpbBEI#xEU?3A+M zaapzUNtx&m9|lxZU?tkzQBbj+RFS{W^ceO*PVKc^Kw=+7QmSwJuUoR>=X z{hPMIr`S)U-sT#Ul?Bx3QOtD?wq@HAG2bKBf}=3zwzVobb=55(0p|6kUkLRaD^YSL ztG5*85J7g&#FyMuoWR!%PHDa59M6sqY_lRsfBjV@>nlpu>vUEiyDNMjVq&~^EA961 zjiUkgVU+-M-m(M%XZ#sgp?qB6Tjo_LcW_E;rL98wO)|f?zwTr|c-Zf~5%qUusY#Nx zCa0`E6;JASc9))<)uSSLQO~O6SbzD+)dNtXa;8~T<)#jsDho`76 zMyg$rhtm(5G`e%$>SyI@9@#>9`20ZzW^^5Wfu5T!|gbaDPt|R!>o;@ted1e)AIEUgfMGc zkyb*GwRwE6R7&7KT35bOhS-{yG#*Nt!geN>MnpDT(*C9CwIs($!W%LoGQM1%Ap4)r zxR&HOvhNt_d9<1HdLFvH^t@)SCAmb3S(Jl)am34aXkI=3xHuF0Do4sD*%!ygXGlUH zt&Egy-w%F@<-h^@Qdn5*7TGIs*i6Q7he~dG+E(`E<(>oY+gaHchva)p#==0N5c^^x z4%OHze0t@vFYe{Q(E-Bo_j+uLd!OS@pg_MCA$G+?rBXyLcEu(m-=$AK|3KcHP?ph4 z_UhuCJYPLzyH%j!!mNr^Ntd!L3#7`#o(`0<`Wbslu_8D*CC&opOLS4cMjm@&jW`ktmWHg6YF$*507=oyObiIJ7^AI6mbs7C&y+I%;r zQ#@7UxzKdvKdP-YjZ-{Td&@MAA5`P>O!JZnsaEL5blRtCy-ednMzu22XoaoXVAJ^X zqiWwjls#aX{38v-WjHy zX&U&VoA(OS&}Urj0n^YwTyzglW&1_Pl9ZOxtQ&OPmWWkB*dNe@NWG~JH#|nA5AyHv~i|QGVKb}t~Bii({48HY13Al z_K9hqoA$kFKbl5&@LCs6;?z1HY8oeTs@C5$PU2LJQ#Um)O~zDPYT7c>{$|?4rjhz+ z-gitR^-=9})4n(DN7Hr%Rnc@b8Pnr(>ZWR(x~bX-(@ryuCRCd4CO76Xk`bBLPlPux z+D-AcB&6g?v399SzCtUib)w!|oI>b11cL!oX)}F+l_yu4FEH0a3fo%^on%`+u@mkd z70Fl4%ldmVNpeHH1Z?5w^r@2KkUU;9j}F%>qbU%D!Ku1cSShh}%0nq!^$*Iz)-eDK zkPmvyTH0D%f^DCET~yRf+x{To$6nfMFD%rmg2m4DOvOvK z)0voWFx6(Ix)alxllTH1WOYZKL&@bMwSg*+%*bLgFN}C!IgF%gQ5QfNn6U|(=_z(m zrQ*v=JbqzLrE3&5xvFFvjG`U~p{XkQrm(BS^$?S-%L?yHm8>mkvzNCPBIZx#@3j%d z()fyRJcKpgp|YAm1?->hBCnZIksJk1Gb?=!#8Xh5;+nCDDXy7;nBp3|c!Y4xfDC3( z^cc{UxskeAxF(kwR64G?NX9Y*CKurVj&psBL}VJ?v(%oc7D}^ljsCVvTyp?Rac0Rx zLgE_Mne?Iwv#7O_ENm8akzf;^k*3U|4wcc>5LDn$9uL1DatJ&G6{wYxXF1(qPQ+3g zqZ_`b^p{-E89?bF+x$myND8qqx*3Xm-I)Xv13Wg;Q?~h!=Hu-yJG$Y!svu$KbSvTT zPzmeON(iDGo=M`PLnOx-E1@)00^j!k2L zHtuhbTz`#>c9wuIj0p^~!z_rokCoiRa@#=>Y&<3j6>mY zCMkS-C;oQM>&k~GVP?cHezP1W^IC_zHd&AFq@)#rlEPexCGkGMJFVYbL4L&h6UNBK zkH*OM-hp$Lfgjx2OgBrbT;uf%Ir47OHc8E#e4Yfv zfX_bCd1!6reS@eo5Sqv0{F$xZi7u9Wy7Ff_b0TBnU@|vz0$RW)i*3@AvNI!ddLxT% z65lZzf3h*BS|rWFoR(u(7UskUv*v2VoJb@ZGp7{P#>|P&NG<07v6nHvM{UNz_Qqdu zt|H&9h@2io*b`xcyZsSD^p6B@52FzBwg7yP_!%MKL83Lf;_ngmMR*%RUZ{UU$QuJu z6z{-9QM|^8qIf??PY}~2q9{JIh@v{-^b$p}n-fK`d($MT2f{jpL`pypGFNdOLUwGT zDfV5WDfU^SsX++WBji;{G)0pnqA9)taSkIenV>0lC!#6#xy=aqj^-h#y)Xz@1jU#l zDAkCdRGaU{6hW!R6Rzoqpj6|DRgDNrHNK3fMg*lA5tM2~P^u9@sn*f7BTS=NlBS~> zn5L^TjmS^6(@mRa8lOIz_gd3_Z`vcKJz?7OroC+1R?~hqtu5M7k6Yx%bpE1h1b}+n zftK!M(=Ie^u4(8~Zh7ccZr>3W5#buq2j()BQHh-s&ocAjY$nMRNF zT3)SbkDK##%dFO)D|&VAD85QQK;$Y3G}Ev1we9P}31c=`jeN zRD0aCHKsjh+8d^=H*JS$dB8|o9>J5AM`I+_I9E|M8Y8K8vT4IjqY09xqY09xqY08~ zG(l30CP=E$1WC1&X?3P;Fzo}=@LKEU<%)!wcTdy0nRXh+?Rt!AH|F86Eb9waj=ip! zH_giA2b@31XKDZ5n`kQp>mjZy*gyG$`NV&;fAU341;jZQ^Pza|)dk7D%95*eqG0{j z8Q(c|A69kwF0!DZq43JfQN)h5bcerkV@o*3-;si!v9gu#x10cPU_&^^ufmAdzem1_ zZ8i7#dDI7(`^Rmp3-~kG(z{bu*Z(XgK5%jW6ZUv1xbx#Fb=dh*{XZs>*&qPTYJf=9%6!7j~MjyO7~AM>jC~WN=UD z`iik{S0pi6ZaqzQjxOw)U`vAk1e zdEP zITDkV$>?S}w#&GQY?+AP<+5K_0F3V2>sI$_??3RoxxtCBuTktm16k`sj3v3Z_2q9B&x&4FimkZd21 z7lz(}(-LAGtVXpYOtW$dIlf}=8z-r9Si`UmXG|NY`1EEJ5+G4z-lz}m&462bbs5ak?DP>TNr$*C}GN>jGLn)6fq8dq!rsL!% z)kqmsBV|ynn;UB_>s+YT%d|r+-N~l$2Nz8@(X=V1u}$IGLn>5`-)0UW4W7^+Ld)Ty9rmZpURny)u4L=;X#~@qKHrUs+{Y^W; zw3AGmZrUuslt;1#veX6f9%|w!(mynKFiFTl{lQtyg3mwZ(gfPK9g=IqU)Gpq^gD@ zjV9ZO`A99yAuz+{i;?ePIQVH}I}Zz9Ca;UX$VX5tvVt z^ykw|NcVsvh8-PHo8Z)%_wZ$a1*W{Dk@Z@O6_vA?fE&^76`;G(N6jZXekN zi_JaaPOn)8}-2J`g=-=pBO** zuNlFQmx@IKu4*!Ppy7+#Oi8#k>j-{?>~EFP)MB3OJ4|{Wr`LEr?@)~3*z=kg!JjDE z_ie5Rn%_zC`*V!o`9i9Lqf%0-URZAN zGK{JZ4vZATBltPU8I|0p*kIYqeS&jJ_L1Q%WZG9Uwh1H(jp~_bi`1oOmY>QyND;mV zn(dOhQy_KYSvk7|4j3W?JRo0)`P+P8cpM@G9M$qCIWA}C|AHU1u+X1d_Y9Pgi4~}$ zl=Mj}>A&lf2^=Ix>Mi||9$->DFlyNgcfwb|Ed3e;ttc8K)qa7m8nT+;SN&8HwL#I@ zNdnWANfmUZHLi`8l6<|eWZZdxOSwn_d)w=h+wW~}%eImP_O@Et9*kNX9H?%HB(NHf zOTuhc6KdMgCD7(!!oq9hDJdfdXW+?QBZ+)#o#$lRcT(-ucsG1^Ha~~YzIPVRx}j0N zce~feS=e2(T2N$}b<;tzXCe-lcFr>EW(E(~==1*@aZr;fZ3e!VY(_MOzcz9P(}}|^ z&`BIp2uU335MqqkXvz&&S%WQs5C)P4Uiu^ryedc#BZ*LrBtkWk z2-Qd;R3nK{t(zND5}_Jzyqb;=GS%3|s`WQ*tZ8Htn(jQ)E;5ZILi3VDXu5k$<1bLE zJ#E@*(>Ucu^L}kw1nsCAuWrrT+qA<>gUyVa?o87zGwlk~&}ZCq4YAGbV9Yy zP5a6;PPx%^J4`zOyg;>IyRoP<(zMe}n`hbr)9y6wZqr^ejZ8x8LWeARj1Nrv(zI_( z+XKBz)5Y8vM%r~fG~GzkE;MbfY1f-Z%LlE?a?|cLZH;NqnfAG9UztWup~v0X zjp+(%s&NH1JqA}$Q;jRAsdkZR^Gr*cc9m)OoA!`tZ#+?Yp7IOV1bRwx@@Hq>=YGXIMN?lMFrZz1fT zY~VDTjfhqzK~0{N88=nQw>jAcvu9v$@Tk#}<(PRBzaGe>Et1n)$)Ls9*qIw}P*Ug4 zv`Jf;%x5O<5@op^6u*O_f(+%&bcRCrwS`?5@_VWd;!O%Q89Iky-Hz?@t_B@`tpD6a0zI12%_CiTp>wPXl6pj-bH=&sZhu z(2I#N=I}}=m2Li`CGbYrOaqF|NSeujf>cF)Ko1Dum*CHp0Z+ky=Al6?d{?*`I1OLa zE|YY<(Rkhfbn4`3Zb27d{}|bSQbta*MD`so-ISc>I;3~taP&~jdw3S7x(QA*P_j>N zE~jCKcbt!Na2gJEb*vs-%{$C#n2L5HO>!CzR{7hb_ZvLNr3+=-=QIu64ac*sox3p( zKRz(@&BSRG4~;fXH8!L~)5f?||?1^eyYd z3y;$<^^e@?bdhU}4aQE%aZA?OGNc3JrO2kZ4U5anZFu03vbU2R_ypN53^Z&ejeS=POnB014fjPIFxe}urXh<6a-T6$s?U8ISaF=}zRm^C$1vgW zo6h4t{01E**IxDwa0kFAXkL?n0~JW3&P{S4(mu}3q@@ozPy;tnr5?^wr(Gn7Co|2S zDUds(&w<*?b|>kN)cpj0A80;@Ov&Ad9t1%5pWbR#}T( zjjdJILFF$c=^k77_|${z?3o738+??~T7?h2%+@Nro#-C3R)Io;mG*LC5W$D{crhC9N!Xnk4F>Q-!TTP><98K5VjYS>8BGvkuMp&fU1k(tMRJ+nN z!XnjZ)WTSl=h@X7O#8_+2UteaadM32-NUpVrg6b9O*h=MQKnsH+7+hVYuXCa2#fR> zG+@#4el%@|X?H1GSG3`FnUNh}Y)4n!si)mb- zOOM;xjiJt_^)YP*#_XDIwi^?KRlcOOOMJv#MQkMui+*)0ldFteVNr9BXn|ZIDb_cKuqR5gm9~o6bee@}HS=OXMAIuflmaXbDXm+DC9pce1Mf8n z9#34lP$A1&&xAue)WQ|a%h+>jZOE0&%OF6o>zO-!X$J*ZQ(k9r3p>izCQQfKvh5m; z)58plP58woykf&1Vx(9QF1Wzmw4wgvi7x_ftzSj^p;t(_)??DnXcut(nt+RKW9CDJ zH5_?*kPGdBhJY0jS_I9Zj(Ho;<1y=@4t1bi`C+#(TA`n(yzW-$g;0e!w`YXf5#7%P ztJb0haNd$a0RJzQ$sgl=e}q|4Va@qas!F!RYxej%VopWgyA>r{%CSHj-=@^q$Xgz( zb=WD48XbAX(k6#t_C;a_w* ztecfUJ&b{!F>3W0*j5U50b)wQu0TvF*o}CGrBkq}xQ@aGNHo>F40+y0>Sj`~>;_02 z+xP}Z?eboTEO3^zD0et5D20PGi+W9#A<`hc158F1_y}onVPrg33NDahc=lKf-(zV9 z6j-&3BLsh%&}13LO+_*MEz`j`oweYp%sgFaFjDHLL~;(pc8LfBg-;E`7=?|5g}$?p zu%XCtoaA`MXi^W^=084<3X?F-OV)Jda-<47QXhPTsaeU`nXeTRM@BpDIkom};@4YGMs7j-YRrgvt^KfB347=VFLVTogG7?s5?(HL{*54F_A*Sb^&1Hbsg^QD@E)k*DJpnuUQA* z!^b-w8Dnb2NIpa16+oxqS;-hPdDzfz`>-DwHsLZkaM+@W5S`WHn2D~tTfr8cmnU=Uv4Hkw!fqro)L(!be?yPnH}e9^vz%o33D1AFCnEJA}5;GPz{pvqDTdN6Sa+; z8E7_AF|YQWBv(mV?e`QdKhSg`GRCG;dy<`TIInXzo@>Q)*GXYvzcLqr%s%7n9vK>F zqA(d_;~p&cWWP7}k!|1J=xZtcEGL$04J=k;XUUj}jB)ReQjDHI9(IRp2c2w(Tb$=+ zS1B_G8Dr7=N+R`(#;!r=z^=g)z}aHnH;BSuEgKo*!PIU_?f3Y2^sTD+Oy(e%;5_@> zt_LMW7BY5&?96CYbLg<^Y?F*Vg)mIU zXjsLY*Bpe*OH#HSAxT*)ltEIqFG7+s-V#a5mLVi589Oa+Rdh=Osg~PU#5L# z8tH)^x1$@2I%H$2akVQw?!l(@HjQjd({Z&cO*hpvdNx+=cc$H7+7qTdZ5r8_miMk{ zpPIJGG_o;G*V>ImozAB1ZrV|%m6~>@X%kE%8`JU@n#MJ*RO4hZ#-h&Srafia2Gc$; zjg(CDl9FlOE~f2a8Y!8kBPG*xLrgoxwDU~6$TYGs%}X;Y&C6A+RC~ZQvN6@(H;rse zwXaS4*|Z2Kl$N)j8*A+xU>ez&=8c;+)HHfF*1Q**HrF(=G0l6sX=G!nJz(0?rmZ%O zt5|8e&rNGt;AySgSk&om8dtH>x|Ew1H*F!t`C8s$Hx{J>la=2V{wwtFXOU@3OygUU z=H-zzZ#-ESUoxW0ur$bAiU*xXkO6~B6&@qD?0mpZgx62fuyzE(c+J_xrSSzH!=e&5 zP_XFUC~{E(ho~I8El!_a(AW#NmM^)W;PFyyRwOqcSChXZK6dR!?yXEVR3=kpkMWA_ z-}^gw)2>La>Az^l++AQ4S@{H5Cg!jm&wYtmgQx z3Z~0y2A8hq;AzE zOQUs{Ub1wL9*o9LtjRy3?zT&oc8bN5{kt%BAc%ke7-M}HbL#d+V#W#>D`u=c_vSmF zSJkbYw>erjf8J)4hG?wr)_I!`jPTz+Eu>_`q-3YA8xmVK~Ow*Kcy@6^|iSy^8{`B^~ns%rexzqJ0L z$uHDDG^Va@<-XOgL#e4niy>T;VEFgR&(;5Z4Cwm4lfEcRu8G$Sqbshm$JpnYxIx23 zDTaTU{9^rI#?<31b5{(#jOFl_iSVIP4gXk;5WQf+GPEaJcqKh~#q*ADzhGqy$sFlA zAH|bZUCI`$iN%w{Vr2_Hk5w#4#rpUCzHsUB+u^VNhx00uFK8c1J`+#YBv53$w<@&f z^NiD1)3EF_e7SsQEMjXVHXpIGaW&^V|Fjrc zSH0|wVGrLM^PNNR?*QD${f8QVrJoeKo6kn7(@{=$gu}cii8Ft~nG?rPoqFVvgT{=V zGJah7_!+Y&4WB$~{BhH#om(|#_RPuWA9chrx%UbR9kVBI|Fb7eo+WpFHG#s>uvj^t z1RN?%<))LL=TqPlWnV|p`A@ZM)2^~wv}hhFYl%ErW=$QVk64;6lKR8#s3i9vAQylG z9aGevH)@B;N6(%)X4hP`CxH0;br)3D)K<7waz z>@gTzM)pAB2a6d$b2J`!<7b{de$2V!$BiC8V{-39@MP;>L}Ry7CXi3}yh7wqtT}be ztSK(aoqC_?~?NdBG`1jjg=&QFwd-k9VpZP_&ryp6w{x%R5E5dUnfs}0Z zff2XWi;5OQ()WoICCM^uv-Ji>r1fIRRf@v-RZfaz`YP?5Gl~*&_?EyWB-dvY(GH#F zgxvOCP}Yg2|FBRcUe*ZgzWp6ADf^(#0?19 z2)80^KzJKMS~&1{v}L#(VKKr#Bji(b8shpw#aPs-G3{zM7Ipq;8cLNMQHNGNn)mOf zJ!{&Frt#LH>AoHUVnZ{P+-l%i7X){cVFFCC^^nNe{Z!4Lf`Z4Yq`;GTNTN0^Eem!N)?()!Ia!x@- z@~!YQe|I!Q?`Ic%H+Q#q@-><0@Nf&w#AUZnm7HThel5O=$MP5yV@ zzAh*eZwv08*pxM`y?75CU}ZxKZ?or=HEq54;ZxSM^Wvpb))ad2cyi6khJ0^p(3CYr zUVH=&=^v(g%9@Veo=K=c;4m|$tl8Pyb^eq!ySVX134DPmYdU+IyiH2|@;qgkB-qv4 zyk0hU^EPX-8FYP-T!*)a?s&tgDtQxcIC#M+tij#G$@PxY3GX;lm$s<8bn4PJJ@^1m zp-_Z1c(6ElVQN=*+tj7`crq{INnwHpgv=uwF{EM&yb>lC@hmZGsVnC54Mvjj_TnY$ zU%)hpV4C(Uw>Frj3)7^6X<|&%5KP1B%J-8h@>o5YCWdcp$2y#8N?k?jQ_3{e!8CEE zNd(gjVw&1unh{Ks3Z|)MnucJSNlfG58nF7zV49e#u!;Jg&orfhG>a12Xp5=?$riCE zFl8c`lAVDC)>=v?Nns0Z&#vJecTugo(Z>1oWwzMLd7E4Cy|eKd=F4ZBh*Sy*<$Uus zGv=MR29|{Ea^M)tVg`Pb0><)NutL?)SwzC!FYWA~Q!j^qPMMQs#F$&osh7q-r(PQW zoO)^ebLyojm0Ivzcxn7|>ZS3|sh7q-r(T+BsZX#R|D1Yh{B!D+6q_T{ACj>EKKv z^QT|EDw(fq8RIG5QoX8v3q~BwnecK}Gv_wTSsKn+9mrX80{hF-mMJfDZTevo;moyZ zne!zx-ip{d3tKj%XHJDPH>73$m1HjUGRF`Jeqo&S^e+s-`@%IR6iDWTm$^DUb7?qp zbs)3kOnEtL5s9~CRTr#kw_sbh!leT+#Oa-?iSM*bP8*w3PFGALH%Uv!YZ3{bzAa;L z1~5pI6O2=@-wDrGL~s%r*)(Q#{iey!i%_k5aJCSM^QnC!2E6f_-_~MVMs|CkRDE!# zjnFFc282gw@=z{Yuo|}pc^tdLwDP_i3YT8GHEoy%W`JRuyn(pGG_D?=;v32QI=-^_ zRz;_;-Ul_%G1y^V?*?VV!HwJqIvGg(62S{~RXbkoSQuRVP zoTNIu6C(0y#u)67!*UM7uvm4%wzeR{D2%@X zam-3j!yGR8qp*R^7=@iGInOU>Gz#MnNqlz0sB0(MGW@-m@iXN|c^ze?D9GpD0*=d` za8qO-m$8R6BS){_*hr2Y-pi43B*)*uhvWP@&IjT_i4#%y`v7envW@K6+cJFIksZ6J zH?muSIL~BYWcMf}42B@-$c{ZYFtTfpkzESA*<2jio$GL{$1l$2$voC$3VzWnDbO74 zF!~!JRrW`N%LChrElDjCXba0$XA5nh4t zI9ET%)%p7i^@OY6-ixIMYZbf)E!n+av z86n579EcO?$$$r%ij_w3ki$%rv?a)?;iljlayR#^3k# z7(LurRNMin*59+# z8%*O}TFd*vG!g>U{4r}hS<(Jed_M;zxp=M+F$Hkwf%01tn{0CEmfsN?Kke{$N+*fp zpG5Pp+uU4KE;n~;htq#*5n92}l6UM-SbDXvBJ`plPLgYzmv)apa()+H+GroZD*gNg zUnYv^eQp=z57^oT8H+kJNmT6`Hx_joO#8^REKRLX;l@o(rZOc*Q;+y1O)XE?9Btj? zAx#r$W_mFL6|3BdfO2G!mSG;qe7u#FFdrD-3ry->r_! zZu@u_#Nx@9MTsX*EAANw>RwP68@;+c_V85ZJ9{HGPGVlye+f#MIQN0ZuCX8f*8Y<7 zLE>CJYwVb_$IOh48-Mor*^{SDL}b?NnRCX@o<6gC?w+q}1Clti+wi9|g4LMB!~ewG z-ro~D0x(ri2=A!pSo<=jL{K#nLDfhERlC)+|FnI1o-(!X%P2W~YYDdRv%jQ$rN1}X zz*%1=|I-Ez_jtB1mr>H&1!H=

r_>)o!+Qx0x1CmbJGZukV38EOawy35_3i(2?Fv zxViUTz3t-n59w(Wv%OWfY~nY5V)_Ccp9Q!tb9P^4kMlmP>mXDVEnq+NcEgGp~nZ?OyiZ(EE71X_#_ zqW`qP&b0<(EGldGsx}X$s+MwNt>qUC)nFlE9gICN9%?V16n;>sGjeB%;vd&KV$O`S z#Xcgo-+>sI#P-7tZS?WuXOEtVo0#;eUbrb{{JktcXP@$))hr;;LXE8lSii9Kz@d3g zf;IgOLe~2*gzQw$S{pGIm7nTVJICD{bsn^IkD3;`%fZxUHBD{g+3`(z;ejde-m>q1 z=$~A>=+)UBCM+x4u@(w8Auf7#&gSIm@+CXTLk7vVE;&Bmo;7?5r5oxTgd6%X&VVB9 zu+OQ8@d?u(f4^$nXUCq6-~MLJ9z9_U9~-kqBduq$qpnyi3oO7nisenT4gPo}FD)Jn z7}J%(uEbl(cz2{}CE6s^shgzl;{3`jV=}ZOM9agAl|P%FhD2>zb)?Zr(4u7l<-C5d zXvu5Cec7K?PH7PgO(qrb#Ii8vq(MqC{@USAb2#+b_!}6RhBq{`C>bGj(SO)poAJk+ zQl&~{b_uq_69~H_gj4hcueHA;EJgS6V(d%(Pn5?lA2^)7bAcZ#mjnj~jPmm?5ceH1oe;afv63yI>C%Gk-g|wQ%8E zs98C{&ss?3$?J+!2p4SIsc_M=6#A|zTyzammo3;9DO}XQ1^QFp^@WSJB7#|S9Kn<) zSAAR9eqr%uBww5Lg^M0WOuiAPlB2x;J>N*r$OB&eIY@Nt|G(gLXyaSu|BP?(zo+v5O*K6&XRj~E zq*7kj2vP{-vVI5=xvbAZk6Ex4S9KI!zr63Wg^MsXz&j299ar^|%HkgQi+y@KY9^Pk z)&5?7pRKrpZeuzZ;A*vs9wik;qgq9Oj~f5?^mnW1 zwf|MsD);JDo_yZuKqkGxgTf{j@-t0Pfxt!fV zFUrI!cQ?@{?eqyA(T{=PpW7nh^v;Vo@8ZvU`-|YcpXu$WsKnQJuUj1{f^X*{Ebdd} ze1bne{czCi;B@SnJB@a3$8^%$)6yTBzs&g-e}29bjGk1C)ZMDqSpJ)YXMmqO8SQtM?_!VzGMfZL#YAJkN8^ znK}1dDW7k7eZK$SXLjX2^US-P=bUrr&YgQ_t{-xKcG}BGGg* zLsx9?VUZ?3$ZP@1_e6R4*oNr8IUQxB1x&sEYqD29K#)Vo0 z2SU(cE^g4qO0wmF1Y&;)2M154bY<(M@9ehU?qqdL1_O^g$c(vQMB|T2_}nU*`qgmJk1`Pu*Wm(5nd-|y0ZN#v$)OH^RmY~>~TMPg#0=7_?E5T zV~_9H;|TT|1o`Rg(Um=}V2>%;V|n(<1NnOFF*|#_$JgJn$6{=qU0h)lXO9QjV>R~p zfIU`ckE7XZRCD$?n?1H*kA+yD_GFK6=~plI7|R}qv&TQ!<9F;4J_-aMc)JY0_eLi? zA#%m9dsD6(-|7kaY?QVzb9N^R*caf4)T zfy}WBe>o05-Xku2yhmIc>siVr<&mqAU*-zSTq~JtCv$MaGUCDw%SdmV%l{JlOo89NfvCaqRec~LO+o~rnF zYG|CND*l}sI+N`={+*tN{FGWerz%FW7j8OLK@ps)pa@P?Pz0wcD1uWJqmaQ~+woMz zd-j6fGLBOfDKao1E?+xU(WPIzP9YsaL%Z~9*LOgN(D3d52lkcO0?SSxZ613sgLjO65!8?cg@PGFt z#yEF&4@w9B{*Rx;pfd-`oK4Hl2v|;HgvqlQU%9>2|ADg@c)tdCk0cgHr!mT>qSF`^ z*rPWCR%DNG8Y7TB!f6a!YC4V4kgdaMj7IDcPGdkGPGfXo>u?&QD|>{~7?6k47!hn8 zPGbyak8m0T@^Bi1UH*orF<>1|V=Q6oa2jJNdxX;%Ti7F<#@Nap;WWk}_6Vmj4zou% zjqxXYgwq(8*dv_Ac+4K*G{zJ52&XYX7fxe@Fl{=G0g)?w1DoeY@W>Ue%H_FfG6&!4 zChi`Ydn|L%o1_QdASN7~N1|L!nS+ZHCgSP0*oxK5!T3*lrd z%jRQYEQQ|TsR0V#pMVy)^ zJ|V$e#eIXdPl3WE+|LdU-K@bwBs{R&r3f{EeiGoxTT zX_!l zGm5aoc{3c`dzB6>ttbA4@(4QyJWapynNg%26rSC2eP%Sx4i9^cxIQzAvcm`R#?!OH z`oV%|{1Q7xO~y#WJ~Lu_RbwnW)PjF-coi*iO29uOieX9c5T6lkm+Ou3God|lojx-< zBG>6Nqgc64pBY`0>-@u^a5lW@@Q9skl#i0`+oS2Dq~|iN%zXG{=N;Q*{U`~hCJ8Gu z8K5v5VaAG&l3=O@GP;A0l3+@aiZTO11!j@;qa>Ikbl7K|QSea`ObOZ|KT3j8_d`qd zqa+wz@>4%bf-$7Lb;RXINi~_2SYSU4{ls3R_>c23YD&69WdM8J$MOT&<6-u=ggwId z>L;MEBAn+yt3hDRFCL?x4)UR}D$;fy}`dCMXA8 zO?qBDVn1jlb9H16?utSOOCM=zACBXjU=f65J&xivDkUgj3E*I&ZH zHOvsjaV+v*KLmj%2s_Hy4o4~Qw9q z8J>ZH`dix_iCkja;7H^QJSN~sB;n_v)I*dr#=$rbQS7(dAEMZAw?9O&-~OLN6qsai zP(ZKT%8p&U`ZZ?|TIlEv=4fILQQ|lw6e5(FCL9>T)8apeD0r=|(&g|X2FF+Mg2jIF z)xp5XJztgG1qXAH$slf9@ru~&IE zTQG8AQA3R4W7d*Pih$XinCxrsuC|P50Y}DSsVx~40i&(PfSD?mI+8&VFgjx{OPef4 zl_S}VvfJ4Ym`3L?3PyPwc_Mubp9o(gTS!hLi+@8Si;cbdKX<#HCN|X`*RT}rU4lt> z3>*1C7J9RP0qkEjRAd2Oe-Dqa1W9j-z`F#D{ z7Dd1s=8)A=7_mpV)`{i@MZg-xSunzMmXn_jo76hBKAwC}Z(}&tA!cB48|ulL4R65`!XOtg~)vzRT7~Bujte-6oE>zM61k}DincAszj^Ikt!5{N_tZ9ay7hc`3>cD%P%Ru za_zCqmAO)ZB2e%^1s`_u$0daJw1NR!371MnZju2g8{JET(bvKr>rBy16V>7j#(lFP5sKWYrWw%L-VkW^b(6A{elha8NZxP&F|CwcFAwDIJV9sgL*msL{)Rl_eRi>mdpCLB~v5mZeK`>}SU-B_vGI@YGY=|H;n zW9>+{rKRfl$9jRRnj)yWy`^gXc}+N|nj)y081`fBNE?1nhxI7qz+xS1)8BL;UHh?i zqzk`m&Gyf@H;aF)7s{$Bf~xyjs@BJva8NZxP&F~^$J&v0gi^J2tWAH@fpqQ1+L7*X zOV#m@^&(j{MNst^OV#>V6Ar4T2&yKA{a8EFo~Tr99c$CybRb>(v38_8%~EyzV-3G{ zMX$pYLDh3CRqJC-IH;N;sG1n|W9>*AK20!ZxOJ>ef75|S#;V`dAYVs-_64CWif3JJQ~uRBat=)8BL;UHh?iqzh+068^kiBCDnds)lp{a8EFJ!Gjm{;^&vtELEAcEVD%KGuYT zHc2ErauKid$(!FS@I{xSNGFdf6P&NF{+SkXLa8NZxP&F~^ z$J&whZKZ1KSeyQ)1L@k2wIkh!ma5|)>*ca)ilFM}ma6r!CLB~v5mZeK`>}SU{YI(U zI@YGY=|H;nW9>-yqowNj$9jdVnj)w=3HwO?>tjtgsG1_Eni%$D?MNH${pxrhZu*-J zq-#Icj&#$YE*+)BKh`T{)f7S1@NGF!wLaE_gQ_Wls)=Dg){eA2m8z{{ZTg!Iq-#Ic zj&!{(RmVTpt7O#_LDhLIRqLJwRa0n>&KdCT(i8>TrI1n2&xXSR84R21Ov7b4yvXIswM`Y zd>B_Oje)O)bN-3Q!X} zgTooHm2glsMSvg%pvGES=SVx4v>68$p$k=;{-y)zg66V#bm3Px*-=UY$9j#dnj)yH zHMI<@*2kJ~P&GwRH8JeR+L3lg(qZN4g=Fs^cH)wX$l8pz2Ud z)%sWy4yvXI5X7(_Ye(AQO4Zh}HvLTp(zPFJN4i5SRmcCl{!vy<5mY_GQnfzTgoCOn zf~tvOKh}=4$0=1?$J+Ea9e7I^$2!)IbSGP?j(@Dz$*L)Ws%KcL*2kJ~P&Gw>Acp-| zJJOC)s2ErauKid$(p_w+I{vYSZ?u_hc;O%YU04EwQmq`g+D z+B(*zzv;kR!Z_BkcBC6)sXG3#-XN={2&&#@sahXv!a>y(0fHFzW9>+Lmr}KLtWAH@ zfpqQ1+L7))OV#m@^+s7WMNst-OV#>V6Ar4T2&yKA{a8EFKB-h~9c$Cybl@#v9P3y+ z(miLXI{vYakyTR!RbR4Ht&cU~plXT$K@9t`cBFk>soFZ$roZVxy7pu3NcXO#>iEZc zldPH|sQR&`YJIE;2USx9RTIO0tQ~2;RI0X)wdrp<@Rl%+b*vrfzOz&v|5$I9RZ|32 zf3{Suk2T?-YKj0s4EwQmr0vYUyJ8-ZTF2V-HyudVeyknox}t7EkIlBoswsl1-Kb?> zIW{94R80|7O$__7cBGw2soFZ$roZXHTVglXj&!{&RmVTpTV>T0LDjyNs`arZ98^sa zAc$c<){eCEDOFp?+VnRaNY{R>9qAUaR2~2G`X^a6MNl>T>WX-Otv{~`2USx9RTIO0 ztQ~2WQL46%wdrp<@Rrz(wIkg?OV#m@HQaoNj#4Oss;gV7*2kJ~P&Gw>Acp-|JJJqP zs2ErauKid$(rrjpXLdC*+j6U)M& zCQ4z4RGKOCQ9K~sX!4Z3`>;4ZmB>K zD2yTnd=H+(;hBdu?71YC-=qRXpb(iTg*{S%B2bu;D22UJfg(_tl_-UMQh_2+n4c(x z{ZfGMs$E#p%!x^bS5hz@VSHaqbvr>T~P`DYd zg0&6jqyj~ta6euJYa9NM3KW6D(|8rEZ8$F#C<2Ao@hVu`a6u|i1PbrtRj{_OMIrt+Jd_F)fkH2fLi}xbBo!zEg?<(V$8DHz**`p%3KW4tM4}X)NCk>OVR)hx zo=OFZKw(Ux6rM>1ia=pvq71&TmnTA~zQNCk>OVNRkHUP=XuKw)8`6kbUMia=qR zMZxim@?tEpyp{?SfkJem6#kY96oJBqL@B(H3KW6D)il7+7yBIMba+8u5k&k&aQK*#;$Rx%C2zz$UYcY;~(1hL-nVpn2$ zv8%AW&Pc~FP`hSnc@n8g5vaZ)Rc{x=`=xTU=?Cl}_`??`$)*U{ACc`-J(W>?dlJK+ zeMup6l(UgOGO6Jbk<4(0TdA={sK9I>uD6VGv4~UvHQUl!M}F)lLRjDXFeEHOmo<#* zLYs(}ibbRgnkg-P>qyrHb#1;W4d2dL4PSru>nZGNwye*yT+bYvaZd*9NyDhaG{pk< z5u;f^5jdxz*40gA)ZOl6cttz2-bl)NBN^+BN%zUT`(9SRDDe zE4?9F2fbmoi6ZC?(gn>hOK&*R^`zct;cm2GGcAbCv>I%tm0>fjIGbr%*mE!odk$t{ z&%rEbGCt36J>4E?C0x}kmZY+GD1yqpN%g<@>`o??DFWp@q#T*ah-9;{1Dl1d*(_|v zW?^kM3#+hMn7y-yk=;MDk^N=n=NYeOh)uUAEsp6iOPk?`WN1Vw0_8%aoPcMCi)2#- z>|)4H%(H{I-i%98E~x@@eWs!nQ{}RT!$&wU^C6Tb7sSv?bE!^ zcs=*L?h%`5k2{W9tVNYw8qG(IB2cYOs_d4d`A~<=hZ<}?INH}` zw>&FSN>z$LHJDV{H1=lu27O9-;_Z6GF=W#hI2yDpLf?p`<)KlQEpliLPu;v|w|haRT-c z$1QD6E0rk%<#19?0>vkN)f0ILDejEG8#pr@)(68^BQ?0@)RnO(Vz?k761#k4DU+A@c}| z)5DuFO^;RL{>-Bv z^XR2{II~d@eWiMwV;W*{mmU;>#|-lLmydel z7NdtPTPpUPWyjRv?3lW9I(BUBW@OuJ9$Qxs-M}6*OBIShWd*AEWOFgHMI|xZBAtws z5$rCaA?&O4Y!Phuel&65JqJawha)ziT3dR=kWcu6hubuoO|{w5>a-?&7W`y0WA zc}5)Q3yPpGhz+P9)>^SR@?k%zhI4^1LKk|#Yzsxu1EdR@2jkKGOg-T1%HC^cVed6F z8@?~qcberzv)Cj2#1_3yQv^NW%xgOQ6-##6!xTXe7opbSJWImnnG>65&g|Ki zl)bvsmsW%V^eyqAZz=MY3V?$9!i&X`mp|_b!U$dHEweooL0?nLKr@47o;lJjgSuJp znD)Qpcr%CWCyJn-0xcf@>hb1nwpS2~xAdS0JlGF1vYBFQnbv4|#4ws{w;93gy;mJJ zCe_)PRAOUNkc~<5o21dp@)}w|s!;@LL5WfuZBZ*I)hGhBhNR}7!tjq`pWH^;46leJ zM&=N<-x@{jxkC^5w~$nz2o%`QOtR{-R8MCVI+D`JyFHcR8=aaRPo^n3f}ymKYKJs;Acx86_>sgf$ymQAz$m~B>vKu zBJhnQ-~X0p%Y94VmXKZ)f!7oosRZtK;0~B#36N?Of!eJ9#+Dbbj819kMG<(-rBRTW6i_Ooo>M%aw3AxRA9j%*WKV(0xi-$rPx78-UuhdH-sI5 zFC)6UWex;N6^cNG?VRA1tZYi7EK@1TR0=VbJWR#MqLRj%1S-2xg&e>B_X}IT zzSS^bJ1k$r&{q2E2xHw2u|39qyFJFyb}Qy<+pQRT&y%cYYzFjEKQ@;v-Dm;+B{8hq z8OAZnLSP{rcb&#whhyAx7{Sps_*o_`%fM2kVx^`Pa|L(k)U!+Hs$IguLz;E(-=#{Q z(1CS3g!k(n5tz#_Ah2u4Ze4o!4h%%XK5P<(?2olUV0^)$$t5i-IwnvESstnJ@FGi% zw|ZD$N(kQ9(FKaWg%K1D9r&*lT{$Q)F=gL%X!AhJ##J^oAIhfYvmoAuJ>hT&lCnqG z&BD<>lnbBd!fc82KU>^OTyq+>fj9hS7@XlNXQWP7KQ9Y#q4`p#%vWbsxzZKNlxaV( zYj}r_y}AGmhUy6Gt(5pp2Z&q^p%3RY$|q94SSi&2^p8t@h+K^c_h(*dm9JRkA+J*3 z>oRHS!ATqb`1`}a@k4L@{4tf&?+a=#zn*i;`7Bj`C>u4WLGqZ{!BI6oeDC*d%Xi&SY2Ac(l_!^;aji8nqhX5MPl6j) zzcT8mQ{Gnh-vzt%PaeDc+^cJD!}5Dae|P0ZNzaAJHXUf+zV645-)%qIq4bX21taE7 z9rz(zzT&B>H67gJa#+=(-~A!B7no3Z#nBA|r-|*=!dzoxw{MRW+Z$x~ zz5mTk?oIP|oh!C)D|s(@NRs+aVtcLfx2}#&n!-tJ|M}|nvU!7?KZxyTd|yY zZ;zOCSZptH)yuEbj@MiFb-LsCZG+rV*_)N^mvPnbn*!fEFiE$^-CwU2_-yz6Uq?F+ zzbv+&-nOIM**haDi0w~jAL~AGiLI8Sn8M`p@_wUyBJtsy)3Y z_%H9B5BPcMvEhOrQFq_sowKtK65F$HUFGaKV_#*lz3j_!!3VB<&{s(0YWckkn^*c|o!=MPKT&K?;+Ny-n>*e~#d_`gTb!dW+#DqEkB>(8Tk`FL z+QP3&g(u&)n~HO2Pu zyxp1=IdR2RY)|^=T=9#)&3Y`hH^>;U=fmbA8O8PvA>+EcFZwNP@}>^cuTA{1`lYdt z9(Ee}a^r80$9{O6b;6tM%`5%9YUuo#VtawV`mHG5`L_;Y`xoDk*VhO28NI$+#28ZPjWCkl?s^LgJ5vE6g(@}^y$dyNv?7hHMq;^ot2 z9fV$~$^+7zSi0^);H^?8zhAlQRNh|Gf}(rY9wPYb2Y%!IX{g%)!Eewg=j^N{F3%I& z{f5-_-L>Fu7O~ws{k`w&+$q*m===A6-#ql`%ew-9zwMJg&foWN7Wi+T&&X7_(jUFV z_WDJaG&=Za#|2{h#2Ss89;{p_pU}(X_vFZ!eLv?E?M+pnT*vHnuVnmu`o^FhW#{G@ z-t)ro--av?5$kOtrmvn>=bIv8JOW-)9_Cv#QK^{9=^lNuRSN$ zz4Oiu`n$lXH)4Ili`@N|$KD7Q>n_O)ZCqbsPr$w$FF#%g{XMPo!1_xod>Qpjtamt( z;=AqvxeFXFHLCWzQYQxN>Y1W%mp+F*#P$+rx>uPss>}efK6-mOTe3roo(uekj3vc*Jgbe7CtZd3xE5eZob?Z zCyVW`*EYRWrSW)Au|75Sx7^3Or8WdUXVE`b9vVG+tnhRGt;O@vfu60z>-W}`MQ{7= z^!ZM#mq|73ubmAFU0JgJ+??yrm%ZCE)cxD~wVu1XZl2k5uG9LQRi>=n=hrt?vipxN z<~{wY_pp*TV)oVE*k=2lD^326dEaw?*9`R+H%hU>x!9VM;}-_Dc<{2<{W3dl_S!7) zH*z$*aqM}DSE212oJzLanCe`$KyVnuhpRC*&x%`(G(d+xXY8BJ?PVl=Kqu#l% z-``bi&mLA~(5$z+`W`F)_S@aho%$u+knLQ+`4KyxG$+^pR2lE3WQ;N5sj zMDJ$R9&{S~`1P_R#~$n-{JKL}#@L~^Q?+`ZCI)71O$cZ1|D-F|#F_Gxs*v_bEG&a^#x`pu)S7EN-!ICRsrY8y+}{H^`ycUihc zyJp$CcJr`I!4)ce|Gr+?q3d=G`KhX{|B`Up@clti*d+)d2RX4x9o@?Xmsz*Hk zdH*tMMWgE74^12RIe7)oBNfUwc~J7t6WtDsTJdm{(_baqY`EEb{P3}VUW&ecHfz1; zmVrIi%$PKJ)NiAkjJWprM8NyJGb?@B>+J3An|>)l=H>T_sG-z@C^L@J9T6{0>kbrKs z0wJe{T;KmCtkuc>?|#TU``D65zcAlVwHwYIDAxD43VOS~0E6cbw z&HJqy=ltW_1+5m&KD=OGUjH%q234OismRUmUyVNR-7}(0vFT$fdQAB?^k}hKM@kP` z{W#6VIsPd=Y^=Ax>8uW|8n@M>d_& z?*60#Dcc>tUTodL!?|1L9(kf=@9%Fno*2<&*1hLLZyvt2e(S38BeKmsSm=v?>X9KM zCQm(CZ+zX`|17=MZKY4gq{I7f-Mp*Xl>Qkn9~^(7^}?6^+r0X|se6-^OPfwCchuW= zdBbIMeCrix(*3sYu;Wu3pZB;s^5~|sL1pvp_PG<wfyteP8`LXxrU4ZU3H<>t4lF+gkm#Q@sDUbD_}Z zu62GYy4dgPw7+Zi+WC#whH;Zt9K6!r=YWwj@L8VL>BFjIOX@rOm(WHRHrAefb#}w9 z-;LeV{~xhFU`g7??cUT2$(}Fs*3QY2{*d9q`YAV7Js2z2eO`4*w<_vwCEJc~3oj2( z(X+s&4VhoN+?giUA1^kBO|L(`p4cB&{O0-Of{k9Ex>K@F?6?Zc5B=Rb!#_sfi11lH zdm=ZbcK&Hh(YAj#f8Xxy$RQU-p4fMw!pjx)+()lyyd-s&9TRsaAJfleV2N}l`rnju$;z8F5K z%-m6`1{F}tC3qRk9kDeY|x}#ZQ*M(n|y-j25W~kjUvFkbhnh}<= z*Y}?PG<&-}vdyW{d3QyW^2zh+-P#$u2l=n}Jlr%-$t`oX*L-vGr)yh}FX>zGZ2rc5 zKduhAv%urP$LUL_PHEb<>CZde-HsM0`)je@zgE9Ju|xg!>vzn{;qTGrVCHH!;lo8&g@zK$9KgWEx-Nx?TKrV9e@1mP;#%sJN#z#ayzkY)te7D ze!G=_Rfhvp|Eab0>Mx61ck0o7$-rYBX8X@8zi9iCB5qr%?5sGcdETdyQv=%vZt*%* zd+XMArw?Z6z4AZ$|LLE9=lQd!X+_T;4yD|Xxy*O9>sM@D;eUI*`tC_{HhKH6$urm8 zr*DJcYX95oD{9mkTJ+18u3m%A9PquA`;u4>?CH8R^i<4rvHt(p?N_R`N;7N2@x`NB zRylvPvs0=;?OPXmF{VXOlI6E@oT|2a#Q!VXv#m{;%&Bd$H=^GEt##4gPwuY0zqj4q zE&to=5&dS4`k_xb*PFFdtlB&xZ>@B1JB~cz-EQ0EUf15%TbHTgqV=6V{gP+N%`Q6* zFWll4RYGik@T$zWr^kintvzyn>WJC@8Z|HookvPJiyZz5U?JXG=f0+=y&9Z_DaugR_2^HM!P=f0`tH;&*D~ zy{>;%_Z(7k+FDOrXwSJh=Rb7L__E*6TQ{w}yltgNk6ZuLE7bDE(V^y_V(><*l}p`O3npyR9#-?_e0SU$G+B{i7rcL=dHZ- zrD)F+*BvcB?rBsoq(|QUkL#B1m}9qhvlAZ|emMQA{>pn3g9>?shU_>$J|?_fEebhf%>&RxD6I;L3jrxkB6dUeTb)9?4&Wy;*e-mhczf^EF-c3Ly?v{#acO+EdE zU!iw%CNwPKKHjZh*!W)Qzy0UY!?2%clq$95)wUG7kNw%G?d3wA$9K0+SNvdxn>_*p z#&4<~dgp1Y6#1G2RccxJ>F?Y6hn}f(<92ZR17bWIgnWovQmxz-F`i9Iq^mOvdv0n4mUM;h3K-nmtds)+kpBy=> z)Z~BKFPWX~VU@mTUl5<(5pHUK+og$oZ($m{Bt$AG)ty-|})~ z#`B(6Z;UuM@L}?eML!3$Y58f#?#*X5bR7}Xzy8(h%Lcc);=Cf@Op3RyqrVLt7q-c3 z-v#@6-uv&`Qg%?-U-8vbFHfdVS5lSQ@Z?UWo~cj0yp(-qi)3j!kIGiDYuS*r9k=+q ztsHmK=gsl?KmI;7sO)BTQtNb^m3!K}s`0h)?Ow0=!%GWKl>M$#iCTZIT(Wsu|Cv(? z-fCGrOTT8hFvq-G> zPI}&bse4N2*L%xPXmmOvTUfs>KLxorDiv;PT4i6O=QVBhQYN2uyW|Y9E}k#>!>0W?Dpk5j zpEIxih{>4ZykCkTrE0i*dE0l=hi1jWbK=;r+f|9HDt*8cO8EnbhBV|@urtnS1fd)(=(&d z`HlU44;|xsAo+)lb21D}+pkl*%O|6H_wmbKu;yHkv2WSd1BEPsoHcZmqk)jJeY@ZAC@TS~sQ&61C~NYUyA->Go2 z!Bu+f>jJnq8<4Z=;P)+T@NElNCKjC=rt82~Cz~Iu3%=_EOEOIdwm8|+;8w%v7I3t- zt^;2@cCv*r9nl^aO$WBHHglbAFM>MhI#9KfEuW=5u9^;foyJMlm336WZCxidV&E7!4<`RZ*MZ+fakBMd!`77hrPp-e`y@^_xJMYwwreXF-qm&BmwlaV zflQ|%*KyZ$prZKNlNmAA_GHp@;9EoSwI>T=tnKm8bUYK#o~&AXJT)D-x*)#xWYgN? zrRii(Kznj%?a8j`cq1n6@$x~8wST-d9bd%6J%4f{#yVcUnhx9D#M?i)wf5xFbn+yi zJ$beEO{UDWd84C7?Y; zwf6XFI>itJzcLHY{IP$vW*x?MB8K&Kr#NDw+0#{gL${cwA1k!KP<&C`4|9PVJV~Olw;ZW>Vj+JoNU&*%4s_A-9{%{+;zeC6P;|xcj*pVyyjKRnw`CnBr`H!7Ogzn`XB@UNsQo z&kmhnF9S<;O{XTy#y4KI5M%A1nwm~+#Khe{br56ipW2#EPy*UhS8Gp@rc*Bg?WwP| zr=F%0jF`B`s{vxH;}xvwz>!FN{R{`%>hq$ZrUM7s@jWk^AjaCyjWwO73209PYbO*%{85th>5$OTOr2U&n-2b)`*GwdC>+j*5^fQO{Xnl;7*ZnoY~(PbWh*U zw?oVbcAN>L0!v#>r#;I$+2YpefEcSzdrhZf0(3fQIvq8g&I!=zqUm(jbh;)$r<uHydkGAXmGZZnwme{WB6V*8Ulx>5M{5+;xpcjJ2*&n$DO6=#15L z#%Ma@5EFM@;}K)6Yn-MNiI}*@eF9>v;~uH$OhgRac^qCZMtROC8w#duD3wnW5>-NC8z$d**8GnWO1MC7?a?wDv@4I`b3I zo&{Qa=4(0&6VRSTT6-31I*Sv~o*%UKEY@_EASUklvlKDb`LjgRS%#Rn=ks#JSm*OH zO=m>{+Otw?&k9XvRRY?xT5HcLO(!}5?OCI>CtA~4ix~JNIrhKNu1DR>)NL2u)~yi`+0}fp6!~>&IGjQ7p*-zH655)>>N>?{roFpto^)8)7g!f zuKalHK+`X5val89_4GHyY-Pt|g;)g3ZcS$o%f>ek_aeqRUVAj1eTa#By!Io;I$rxU zodXGI&q1v{2Q-~S324t@tv!b{og;{ed%TV!#yVa{G@WCJiTinR95L2@KBnog-4+|K zxcm8c#8~_Jgr;*c0qr@ZwdbU!6PtkcoYvYCtLdCUOx*o^7BSX-KBMWJLrmPyi$4%! zeO{c?bj~9Ne))j?Z@m37W3;}XzknEa+Zm%Hi(omg>0D%4CtKV)e$No3`He_ROZj!Lb`gy@U#IPG>8rN9_%Uw+ejv$_um!t#3hQqy^jn7GI5Z)%Ta++S-tZxW#MR?~T->AXWs z-2L+p=~(*bou>0X0qtRjjBIXL$Njyg^DzNBpERA1n$Bm$#9h}H(y`R_S<`_*cCy7i z?l#0&$K9~8NuYe+CJAETxdn^k@3%N124109lW<9*={P4qC#k06tmz~}j5mAnf;K(Q zU&~+L|0G9DzU1cfJ(;EhXI`Cb@XI2gvt!1o(z;Fx#CR~BGE^2jUu)k#u84sw3ycjb z59HQ$QX*y-`!d~u947>Dom83*+#SlvwutM5-u<#t z*GY?*7^YK_>!i_i+z=CYUFi^Gt;pLYdV<_)13ED)wadz>Fvpkm|)f(Ki-~9nht!zA76X2AjaB1 z9-59PV&EQ^Y?>LVw=`o1-S+*H6)}5K!+CjHJT)CJ#K2veLFaJgckF$Wy-qf2k448z z)5(q)xQ{$5-|($)ae=Or12ODYltu_|Pj*emThaNr{xA08VBa1e#KbV2u3X1k)A2m;d ze__ATDJa)%T_-PMJlKTKto8Q&gIjH37M%i`P9fB}!!%f6r2ge@Z(XM_V&G2y`FMK@X*xv^1NUi! zea9br&eza&zCp}1HlRUVr--Hl-`;ex!F?t{r`jJa*wEV7RTME%79BrLrx;@3uA9(5 z6mchhw$BBmUB zE{o@T1x+UqF>tR+m_GyEzK_s#Dp7lwPB0Y3mOxFXGGgG4n$XWP8+?~r*QtUSf7W33 zVuDL$O$WXu>tutwfU^IMUhNkb)pe>N<{j&wnp~%>wT&?pC?M14mq} zG+e4{IyDgkeFMwhn92=womz;2BNL%hQ`4!9m@!NPX5Zn@BiU0LBJ2CXI*579(q3!} zEVVVAAjGty%|?%R<6G!DbrA#m7ItuiOOU2hPiar!ACt!EI`t6~$Obe!*QuxJ1S{>? ze*1VMU8ezJJeZECD_GNMh!{93gI;k>-DiTX(+DxJpBJwe4K_K=k#)S9A*L#u9O5{$sixB$F)_S77ss?`FPZi_ zEfC|ybVOavHJz4-VK3ddyk38AxUSO*F>u5#=3z@sr*#5!+GskhHJ!GKj^Eozc0$O$ zu6CMETTQ2Z0(3fPI_))`j);L@YJvUFiF$!q^!9W@Oii{^n9TdJqo>G4NX_pyP9L z#C~0;3u4$UA&qHVr?aL5XCR$y@T-!bvvJ0?3c5}=#K7-a4BWbv;8K~(*AV!S)*_>a3be+M7f&O7H0=PtIIztcx zzf8h*o5rWYRoN**`}Pb)OaRjn&#xhx&M?HlZ+n1Fs#H1INg8{dZxM5m>4@joFimGT zV&HfcbhhqWzDw8n4l!3O{XAUL8G#u1JwVV|+p$h}U1ua>;C}X^t`VBfD8#@o0D_L! z@^`y+9XNV)vVCMa;(0Mj(;0&p`29T4@w?HOy@cEM&sfB`SoS|-G@WsXf$Kp)XH0=Z zfx6Ck#K5mtiFr6q(}_e3{6+@oEUh(trmiyqF=JR=Vjf0nIuj8CzupHrd5uiopQkAmbv3w_SD-m6ES-jh8_CiGDFju zg%~z9aB+P7;B3SkV1HRx<1$OrnS+>cRu{a#=<>~qe0qE4BBmU}Wam0_G@U5KoM%R$ zv)=iZgSzG+hW!MS;mdWRG@bc~$-uV5DD^JWdWGJe1&D#;AJISaHJybC&{>2SIGz_e z3pE`$BjIF&<27hcn!A1j_4fRLm@90%&{?eMEJ=XQQcY)xrn3w&;(bc?s)Zc%^K!&I zx3p)Orn4deIx97u6`IZ}#K3a`7RTqqS0m;&+b;TNm8KJon3fE~0^>_>|FjfYkHgm> z21b^h)WIcM(^-p{(QFcemsjR&>_Y}nwCenbnA;2>j>Fe#I_ngj<1K&9rR%Im3`}^T zvrg05fEbt?Fn{J;_-&A`vys}v2DK2wz_LNpi9yUIwheUh9I$QCbv7a9qeUkMbwE*! zkYRYE0^RW(wwM`KB3}fiIA33D6PdX@vrc9*GgnwP*aQY%M`4MviA-{6G%g~OQOWe` z*F6-jsa7$$IcAPcu-fpT(o!_-EDBp&l@Px?6CsJMA?KF z{N4d93vD8^o@Z9b%oCnjA~ShqM}yZOwq5t?z%kK`MFKNUt1nt;`(dnRPN#ot*)I zWrNJ@=b0FpspLtRtunKlXST^qIra$%EZb#fC(rDZnbKZ_*&{RCcxIo>_-7-`0h!s% zGlyiRC_j)oCNmp(=7h`?Vei~wIVm&ic_vn73VS2O2$h*#JQF4}K|X{DmzgU(Gf-xF z`x0iD%zWaR;W9HNCt*g(OpaWX86z{Bc;>FmRL@PA`!aKdXCBK;zdVF_Dl;kaQs%YH z{J=ABWTs?3!n~K6-+AVv%yh|5n6@^tj~dA{vt%ZQXI9C~Bc5?%I<#jiT!3`aOXX2K z(^9e)^NhR1yylrqGE=)C>3GP@Ql9aYnYTRSB{TI35i7gQtmGMQnfb&szB1FOFtKvU z%vzqwBQv%lgvlo}EqJDY%*60aA(?UchFC>prajO2$;=L(nJ6Bvwb6Imk1eWu`Q%>2zW=Vd0OG_fwq%oU!wBs0Nf2y;bd4)V-ZnJHbCFxO>f1JB%) znVjVab6aNS@r*Oe&~xw@&t$TRBaqDHiIrYvI`K?bnOV*=X=LUZ&$!4;nF&&nFyIl7HHryTxM$U%m|s8z%!#{<~Yxc zk(o4=3|uD4Ofb((mYJD6Geu_3^UO4v@vLm%l1pZK@{F&{{K7NdGLy23fs2>SwB(so zGP9CrlF7_7p6Mtv-&8el=`1s0JQFH2n|LNnW?u2k5Sj6>X5cbRW(M-iIGNebGm$d$ zfoHPIOzr9hE}k+I#WQIPF*9HCOihXLsX;6cnd!+h)n#TL&$!FXO`d7bGD!?rKJ!d# znenWN8w@v=al)_@&$!CWk33UOX5R5k8JVeC3&}3W}V2C6s!$e zCPrk6`ia+EkrBeiVv#B8&tEU1L`GtkV5Ue!=MLc=$Za#ruz+A!e57+SkQmJC zLX1%)%w%mrRw{;p=fcmsGI*G*JDi0W!|JtF^P=-M;1%2e{<2RR0Hb*QqXtau#9~ubs$(-3kkg$s1nQrE|jI(xx%|H4MZkUcV?_Z^ypOVzL%-7Ge~y zU6KW=xqokwo6fJo+7-@1jNdtLRV)M zqI1-MZ^c=NQM`Urygn9P^vYx{;w;1{UV9`9)S)gq-wSJ3ISVn0*IvmIb(t*hrod(2 zUjdBbwNJ8OJ9yE#Q&{W3S%^`*_DdFcrFnMufXQ0GS%^`*4j@a+dCwA~3RK5$9f8+5 z&O(ghbx`pt(5}RKlU1!5a18@7iq|2{tF`p{iL(%+cpcWfOjh3Jxkk7&t{X1(O51|G5Tke<)4b@~CBTp6EW{{Y$2BjL^@+0(qj;TAykNYB z$UY2jNi9T-;`O`YWve=5v&pixA{Js4ualY=-9rHI-*XmX6t7d7m&wZ7n!FIBc*QDS zFkW;W2jHVQ3o(kg&4)_jOJy?z0L2Og&4)_tmb91I$C|-XkUgz7V&SA1b+L0Jy6tDA|S5@hCnX?e1cwNxEOjdAv z@kY1Czkr!eVuX~!8$?^&z7Ge~y`9vlt5TkfK(7a4m`R?R}7{%+M<~2@w{mxm4QM?{$UM8zu5As5c;`LbZg69`q&j|SE zoP`+0>xts^ZpgT3lQpg!s#J*U*8>fKc*6jN+qiAbIwtD}&O(gh^;Yp(J?O}f zCM$bi@we#_hSxlB!Kk`D1;`JU`q7OghJ2u^4 z_Td7~LX6_|LGh}#_fcV!<=LOS5Tkf~M3(5ohJ$~*X0oC<3o(kSxtxU<#Vf1g1@q8kd5k76#3)`~$P#@B^U!3?<1EA|UU-dw+(*GYG+CZw z$cwHcu%27aj$Wby1bDk`Vi)!$y&%+h*7+J z6fc;ECM)|m@T!22=mZnt>i4kC|)HLFPMiWE6+6YLX6^75?P`TVIJ0#eYl#l5TkgNQoLXu znyh@&$qO-xR{*j^AHx37WJPlpVid2^iWkg7lT}~_c_BveDuXQ1hcFLK);i8YjN(;R z@q&41vWm*XtJs- zBrn7$UiFYA`Vi(}bJ>RnISVn0SAE3`=Ap@|v5347qj&`)OY|YkLz8ucvk;?rHBh`@ z9-6E=i^&TyidREqi9UpRXtGXn7Ge~yMv52ALzC6u2l7IU;?)>gq7PT~UKYGo_Tf3s zLX6_oMDcR&Q+u$Y%$;mtlXbEWpK%sq6t7l_ zS4pQUM@&}ua`Hlq;?){iq7TdX=gMia-f~idPrK3+AE8 z`jN8`qj+^imgqy6hcU7bi>@Ir#3)|f6fc;EChKR;LX6@Sf-KR8Fb_>uxwYhl7{#l* z;sx{2WbNlH#3)`pkR|#M=Ap@|{UdoHM)B&Yc)>h0S*JM*F^X3&WQjh6dAL>fVe@t5 zg&4)Fx8eo!&}99^S%^`*LXjo<5ayxD>ad=?5TkhYQM_Otnyh=Ag&4&v3|XQNVIG>S z-W$jZF^X4T#S7-4$$G_Eh*7-yAxrcj%)@Q64+n1~FT^Nb{S_~mhb9Z|)C~(_6t8e( zi9UpRXtKt{5DPJi*8s%}=Ap?-xrtbaQM?8sOY|YkLz6Xwvk;?r4N|;d9-6Eyo5>3? zidO`(L?6OD+%EfY31=Zj@fxgn!8|lsdA5)jVid0-$P#@B^U!4d$XSR{yoM@XFb_>u zv908V7{zNCvP2)kJTzHsBIAM>#p_$e3+AE83jB#!h*7+TBTMul%)_0s4-az|Vid3M z6fc;ECac~y@o#X0 zM)4Z0c)>h0S-rNC7h)8zF~|~q2=j1{?8Dcbg&4(atl|ap&}0qXL0*Vayv89*^dZbc zll6(S5TkgFSG-^znyk?~$qO-xS0u7TAHqB|St))Y7Ge~y35plYLz6X?vk;?rO+=RH zLzsvAWFKbSMP7(eye27LFb_@E0?tB=;x!prq7Pvnnk?^M$qO-x*Y}DS%tMp4nzIn2 zcuhf;=tG!?CacJ9@3dm5TkfaSG-^z znyh`Cg&4(a2C_sS!aOutHTIAfVid2LiWkg7lXa4_5Tkg_LYC-5n1?2-$zJk8jN&z0 z@q&41vaWCzVid1A$P#@B^YD5(T6Y(O%`h`E{IXQeo(w%9-1uQ!^A?2; zq7PvnnyfXPg&4(asp19m&}8`?Auq%zUdxar`Vi*f3E77~a~5J0ujPst%tMn^;V5|_ zM)6vKEYXKB4^7qq&O(ghwNmkdd1$hNj*%B)6t7jt5`75s&}5zCEW{{Ys}(PphbF7_ zaq>cp;uVc7(T6Y(Ps%>L%~^<1yw)gQFb_>u&lBW@7{zNXvP2)kJTzG^I14d~*N=)9 z%tMnk_;>O`jN-KpS)vbN9-1uMNn#;J@mjBV!8|ls6F3Voiq{5Yi9UpR7%TfQ-6`@y zjN-LX@q&41vgUCXVid0!WQjh6d1$h{W629Kiq|H^3+AE8TEkh0QM@)IOY|YkLz7kP zGb>*p0nYqBnL7Ge~y zZJJk@^y+(-ybz;!{j7PJtQ6;ng&4(ayXF-xy_RqmVid0(nwQB6_=CIGqyar0I zQ=Eku#p@T%%VdR|CojY(Ub{4}VbaUEKrF;4UcYKyCM$}w5TkhQ*1U#GuWv4r7h)8z z-!w0ib%e7Jqj>Gnyhcf{_J5KWVid2vnwQD?hqDl)c!jxOMtVKsEW{{Yr!+5)I!84Ue^>asLN!nz3jLb(yT(kEn%+QM_(zUcS<63}+!m@w%gU zL0u;6CTAf=@w%&d<&s_%A5#kvqj=p@yr3?VwV1OIqj=rdyz)q|kDP@V#p{9M1$CLM zR!^vfh*7*AYF_!I*AC7?jN!s%9C%v|C7Ge~ySBe+ZWwP8~kQZVUuh)uKyW(FuO_W~YoP`+0>u<%YLHUWY5?FA~_2&iq|K_ z3+ggik2ni4iq~hw3+kFBz3To=Ekums^+oZ5x=hw4&O(gh1(yVx*MC7>bEKF18}dSo z;$=ftDz=#g2GnJ;A~*{%idPcF3+jrJUY9uwF^U)dUc9^p0qQbYfp4jW^qcb5YY?0j zFQ{w2^jg7Lh*7+fYIT{cRK$l{QjX9B1Z8_q19!wW^oo`l)79MFQ{vY^m@x#h*7-o_ZO79Ojhgn)I$1= z25ViZ6fdZ2ne^JjS%^`*QfqaYtXv<+3o%Mva0h#JyjDoB@tlPi#Vf7i1@q8kJ?1RL zC|+)g*Z;@dbpS?DZSC0v6agU!DgtVNNE0v#p%_4tKp-GZIv7nt5+D*tOo4!)t6)VD zeHQeo55a4f zUpFaRllKV^8Kc>1r`e+RWog|nwHTw>I!Loc<+WMaI_ytji!qul`n?0|Gb5BQOY01& z#Td<22hA4c>ttnXkJMs}X6sC_qMd={3SeOjArYQFk6Mn)~iyBF`BJI!)#ev zqyHAR7^B&uUv9GM2IXt6vh|qMVvJ_%urOPe)&L{G#u%g7>J(yKmR8#YVT&=EE&62;J%$UEt+S;T zW3+r78D`7U`crB#MzfV1X6qDXt0qx+$QaF5*Dza_*0)lNF`BJzVYbSZt>R|F7GpG9 z-NS5IS|3O)#%Q)uG+We`-&VG!H5axRquDx2v-R6?`#M=#uSzY(XtsKU+1jOSjonY! zVvJ_1XP7NZYrE89jApA>n63Aet*rfpEyifJ=(qo@K8o^XX+0*j7^B(h6K3lJWovK? zVT&=Et)s(iSz7l=EyifJ`i9y1NZCp~K-gl8W~*PAElX>Q)MAWgtACiS-O5(amckZe zG+W1n*|M~5kXnqrfYB5H$HHfwP zN?RBI^xrK9DO-)F(%K@m7^B(B49iy=m9HPA7GpG9Lp56m+|{YOr8TCl$QNTYTlC92 zi2=$NA!almHl&5Jb+y!DjAm=NW~m)tT~)g=&(bxwX}+)7GpG9BQ;wupK!%?OY1hN#Td=jDAr1nwk8fQea_PQ zuhe3UW-C{-wcof)w^~{w4ifodjAm;zYju$^Jaj@e60#$QaGmSk2b!D<62z(wZ-|7^B%5$696#XJ%}8+0wd4 zYB5H$b-ZTl&VT>sJ4@?Nsl^!0)_B%3>-?xGZyzXr?~j6-*FofqF`BIjnyohmcCECu zE|FS{(QHj*t0|Gwjkx}Z>hx?&DK=S7PVzdYuq6sUyRXgO=B&y z9#UJjv@VlcjL~dO*KCcO^;@E)wOeX2Mzb}8waj`*ZQ0W5d#Lb`F`BKJnyoLszw;PN zt4?Y$Mzb}GwamKl<DZ{HCv}%)Am+N>%fk}L&j*f=CGDo=U1G3 z@EJi>H>OK1#%Q)q)NBo$f7T_I*1x0{V>Da&nyrajJIxiphe-MQN@_7ivvrbYYxS}3 z&a$-94-@%fjArX(*6K*qIL^5I!YuKdiS+I?sl^!0R)J<~*1L7TTUyUbEyifJ3R%mn zhwC;Jj<>XuItdRMquDCbY%Omw{zpqIUurQ%vo)8s%zAj=fMZ5lTAQU7V>DaEnyrZk zwmZYp`blarMzdAITB;uI`~L0=RNctxEb_$|%~q*qt8?PPXIWYoNG--_w&v-!Q0FgH zw%(9hjL~e(*KARpx3rGvB0OY_W@~|F>%n=w_Ft=PEs$D_(QK7zw(`bJz01*(i(fX$QNTYTji{k1!h8=8vOI=6qT>b zq!wc|TZ=SXr?-FlFH7q?sl^!0)?(J8x%Yew*f&|(`blarMzeLQX6v14ujE-;`GDfpirP-SH)9+)tC|jGP z7GpG9)taq=!(Tnr()wL$F-Egh!&+1~$QxP{wN|z!Ckqc5quHv}Y<=_D%2O<@O;U?7 znyn>TzG&^RpR)B|sl^!0)>6&Zi*?uSu(ZZ>6&^B1vsK4h=3I2v+9glVRkqekEyifJ zmT9&oWsg~BY3-I;jL~c@XRTz!4=2X+x4hlX((2hwc*q#d)@hopM-FPUz|yLeT8zV$4f27 zXtvJKY`vEKUB0Dtnbcy8W-G{ADKduNe^B1h()vJZF-Eg>re^Ds_cO~atsW^NUyRXg zoyA&a42R$L$mf<;rPN}KW^0vZYw~`JUbM6xky?z=Y^`Q3GlpxgzVVR7Du&ID5*{)} zvvsy+>;6dxcCfT2NG--_w$5QKvmOq(`I^fut;?krV>DamYPLQe^^YNz*2hwdF`BLO zSc~S^I5CJ^=^5ArByAp7^B&`K(lq@gGW!av>umQjL~eZ z(QJ+W@v;3at^Ion4;iD`x{$TZ@}fFlsmkjFsl^!0)%674Txu~!vvrAP>&ETpBwAYA zr50l}TbHty8Nxn^t6_x~MeY3-F-jL~de!CGbvR~+;G>z3BQJ|bU?(QI9**?OV# z2RkgSGo%({G+XOg%Zy>?iJ$)@evclRG@h4QjL~derPpIplV@Q4PIjSw6C$$)(+4`4ei~3$m>vgHc7|qrO)=Ea{;KW!x zuuqny)w!SWkTIIA>or@{_gY%>q!wc|TN_y`wC}aF?v`4N(QIwfY*F89Y3-9*jL~e} zpxL6n*U}o(U*wB1nyni(Ti<+g%}18jrBaJAnys5yOO@BG)qT%X<@KS|VvJ_%X3f_5 zFRd)Hw0a#QJYILXp_LTWKavvn(LnRULj zN1t7mR?Af3A!9UKw`sP{JG9~#OKXPIVvJ^M3u~pK4iYb|Q;twE+$gmequIJ$vqkF^ zOY1kO#Td=j9hxm#r*u-bCZ&mdF-Eg>r)Dc}al)mR){Rn&F`BKrSc}SwjwA2Qoit3@ z`dw-pH2$7|qrLtThDjBi>1SJ`4^~wtkRW zjL~d8sM&gH_UUUZt=tUZA!9UK4~5wpq-?F1T8z%Q8B36|D6sl^!0)?=)d0XuY56uh-xhO+gk)MAWg z>v7H2sSo_|yrng0knoT(nyn|oY^5t(=SwZdXttizZDBvk()vJZF-Ehsm9@+gDy=X`Lyx7^B&GnzcqDe#HAghs+_V%GMiFi!qw5XEa+^ zzOvP2Mzi%SYlXJ29?DkdV}*x| z(QG}Z*(w~E^RuP3NNO=gv$cb@%<`iCaFnw3gw$e;X6t#))?df}be*NuK2vze7|qrT zVfo5Yw&qJM#%Q)))NDOi(fe;p>rtu27|qs8tQFe6vXrehLxqQo(QLh}+1gWeZc9sR zuGC_TX6qH!3ay97DO>kTEyifJUe#>vE`IGDORL2&;UQx*TRT~c=4^D_bHlMm%u=@I zNG--_wqDb0-Td~?Z(CZoNG--_wqDn49sTR+Gp8wAzep{{Xtv(aY>oQkqM?>n?r@PW z#%Q+QWG!=_En&*dGiNAUmr5zv-P%S>y~-UW&* zkTIIAcQjk)9QNnEmez8q#TYJ`XQhPRBpCgzpZ7F&F%NfHxb|Jup<4+fj(4RNV~qY) zmfE|lMRPwq7A}wdWcmZNTHu^8uoa^9UIeYKBZL-VOs$kEqI{pVs87I&(X;fZGcBuA zp+yH_WWI!w@d0ZM$2CHHX{79LX|0x8gwbt%$XZmNN$Wb})|V`;2c;HabghqAtF_cZ zg2Rl<7gCEby4J_6MfoD^m?iIKT3QEZiF^@8*V?VLa8fTC6QmYlphd+cqwxuAjerL6 zJ~@0rTgz5ZY7s`a^(kxFN8^5}MHpS{GuCp5hSVaAuC<4??4xmLwulB{L|jjh5@bg4 z(79oJ&OFI{ZvHXLqF8DX#4pVDzylsYkjF{J@wgX_XT9iewJaJ6U z&B@Kn&dr%IB4u(>@xs!QQKU8#Y9lKaFPTtOQ&m=%m6AFnD=lk69&F`8A#Xsz#JoHk z8>LkR#S4lSFD@-FIJLHDaZTCsqMEXb#Sx7YpF8_N3K!DSZ0$$LWgc92mCu~Iu0ysH-_6Nqa=W7`JL;_`~>((0Omc}2xF6;;&*Dju`5M`TYQky2SyR+WWDUtLp@mBnp; zL<;>P{v~(TLRS-0Zx(l+^4j)C^6D>l|H8Nx`Ya+jLR|=x$O&%zN&sQbT(x z(NB#?aWDio&4Q1iPmFHK?30`_<*8%xX3jQi4Qww;H|UgW^aDJOxas#(vqJhohtiDL z7n2jVv1nK?qK*kp!SFgTL=T87S8PMlAim72s9IW7Rgy(*UGx{Tv+11QQ}3T*BGhHk zC0GE5)|%RgJp*jnhKOp7s|ic=)+**jqJB1F&axvlhz81<*6-Qpt%19udKPWV&PS6J zRJ&2jXOoSQO*fsqedZgshjr(NZA2Q~5vS_VB#tyfNL&p$4&8Ty3_99~;no}5z%-FN zPBo?p-D#xoSi>My5b?hOX60(PBEwjMl;?NDa+j~`=-gf3V+j?Z^EJ%yb@c@vP=CJUNdb>tUL2NsQrt-(BzBHvj|2qMa4cc!0&?Etp{`_wR=zppTku&0D z(+4+vnYrDOb?AjeuL!qX>=Dr;4%R)=NIj7Ur)+Cd(|YApotoaO|FvLor0n+{n=n|? ztN+Dd{SQ_xa!I^w`l|*j#YJU{@o*DA<3!IaaCeiJ%8#C<)RdK~2cYPoC_FJ)Q(IYH z%Fj9QqYJ5g%J70Qr8Narc%TYTLDfSsJ3R~q9gQO^ogb|mkum~zEO>etSvNH;i=Uea zc{(dCOZckY4J%k!T1Ag!B7$OK<bWu^AV{oDxw6Eq- zEJkK^tfg_YHng+ssY3@8yR(wj4`+P{Em4Y>T?39W3Ni#SR%oZ{WwADBY$Yux4Icxd zZE$zZNHnm!DiS{M`Vn3c+lGiQrkrkjV?3YlrB&E7+t-NrsU4%-Fx0-dEo+_KB9%T1 z5l^rswQp|w6>j1cFT0i)Pf%n;oOFtiOSd`kDJ1^I+l;v%j;p}`8Ntn!Uy>KC&fV4bU1QZP(Rr54Hs;+d9h8Z3FaT(!LOVwCPxoKHF^EjB+j=*3N!q zRN8F(j5d0al0osZYoM;;D?akaBY9h;wmad3wE zUbE36#YYSWi19Ka8D^h0W$4INRVqKVTx#u=af#Hgeqf-Xo4S$g+`Ke%>~{2p{qW{X zB(8?bR=wyr3P#6S?E3!Sp`N&lh2B-rl!d>@Axe)Lvf&|nGKW4j zjL^ROiH*;>XN;LIPWM8hC!GtQOCn$%M*=QPJW+0ilvm&rT1EATgb~@(3JMmb4;)mM zMc)jNJI%c3sb?#s8HWcNdKO74Pbcw5F)72Lfr!m+Cl~g$8g~_>VPDz=;`3vot zMbE?E(qr2zi}kix?c=GIf@$>dOTm(&a_+N3wE0;W{+t?ELBL7HArcJ zi+LKNkw-yzZloXU!mZc4<%a8+D{o!v`$iS6~p(_WYmKeLzafmVsp)nT&5ow~a zM0X_`LtT1wj_8>QiHX(>Vh)KTOSJr0xpAO~Mi3eh2MULH*f(meHKTQhG{>;Uj7m~a zFlK6A(c+Tw(yEaQN{bglP+eMFTUA!G4By}fK|ygv#lo^uBc*on(z3-RDUI=Mf+>~! z-I|SJ${|}3Kba8Phl?pTFW*qqE}lxJFHi)s-WM1;WVzAZunh+<{O!LrQm!tde}5M} zjBWSLk+vE4)u3Y*y#`z4-Ce1R?7J*?%6^kLV}>Ti?~gRgGFEkEKDKX#ROVw;SLS2= zl1Sx1%mm7XXjyRj`bOnK%mk(jp1%juH0@o{H0k)|jM~5$(&M`7Zo|KR&GfX<>_yF> zKL2fm95%Q?hb)>sIp0UK88ugcYa3wybL)#)hwMffq@0QzgJ2xr`RUkp zBMz88`4W3zY|E*sBIAcM4xx!UwQ(vbuahn7gMYd5=+@)}^wc9Tup5c{0Z2DceCqjZ0uvFCh#*mRZA1`n^?J#n1f zhTjJBukva$G5bRGe^Nvn<~H3!yoMc0w_cWQi#J}=_C?$nmR)4psU`WJX}>JH%ybhK zwPSmZH{cH0b~Ep{Z*fgE!FA{W;&dmj@YvRsI32R@72Fk_bbp1*!Kj~VY=vApq^97vee^HY(DrJ4&U=%Y7r9Ss+T1spdH2=cwm)dJdhd&M4;9}R`=Tp! zRr7tZ@1g4ZVqfInRo(Z+zRSw*G)Om2<@d|F+sf~keYch0FZ*sQzhCyN8SM}dV z`>v|LkM>>Fejn|7sQkV(K9T>3{lCn;FXp{ee_tH%QvZE%z)J)0#Q`rZz!wMHHUX}O zHaDXiVF5s<{W9;m3HarJ>n7lr1FoBZUk&ew|GwDw)Chd>z*8&m#RE^xz!wiZwF6%~aMcjF z6*kR{0nq?_TZ_1B2EOM%(sL1w5`jn$Civ`tyEfos5qN0=zO+F%V}Z=SFZMk(0$)6E zHx~FG1DDOf#~3uF2d*0e7eo^+49K=$_T4uGzdUf?5d8AM-K5CRQgA;Y_!R?B4Z+9G zAoB9SECXL0@YWQ3alu<#@WlmhjlmZeytM{jTySU(^*+u^9~ZD+BZ!YIxcJO~_%xG; zPYpDx3rSMs6Fv`QWK>_~wPD*5R8Mo|=bm zUU+IBzIoxgfjG_O8nc1;>4WM!fmyQZ;<%!gPVrpzZY(riT_@>X(ax8;iifB?}euZ;#)Hnv4}L7 zSds9}32#lrH#fYs5#QYK)<}GF!&@uy%?gez*3txaSz@zV(p&BRYPJTwzO-SE&% z{B*-ZGx5_6Z_UKF)+(amVmP5#_~wPjhT@wa9$SiUet2vuzWL#?t@!4L%f`a(wJC3$ zh(_Y;3dwC#X_Q!mf8rzbW^1EGBm8mmMs~w(JMpgpYib6SbXk0 zqAgdo?ROUBDDzv2rRJLgZukE??D=vVv!^Q$Q1j+kGLIfRUM z21~07FkfA~xU{_B)Y_uOHD$|-YRW1Why8k&O&t@@O`A2{10Z6NRL>zjvC_0zdHL+8|i{7;0=XXhFqbbgj>eASL(=cjk^1dq$r*A69) z*!k~V1BA}cl8rxhetH*g@O@kyjaXp5}xQV;;JqC8y&a(--9JkemypMt2o%?3-`V;$@ z**!TQLvUQhUV0w`y?giBhJV42jozF4F~rAp_{skm_#HBUojtd+?mV$i=5bQrfCfBe z!9GL{nTOacxX*-RMQyN5#B9M^CSnVeeS@&6PRVzc2UBeK99o~A-+&{*Mh7f5U2aa!<} ziP!?=qA;vn#Ad=nHrSL;Z9`@wW)mK>5nHU>7YF4dHXCj;A_JD{V|m15!ecgKiSNFHg?&{+{R`$&g`=7#_f^GWXxt8AiJ@}Iljs)@*GQ?9Wsp7RD6SVjyKzmt;+vw$78x7vK@2G?c25MO77gSUBo=DjB~^tMrIeb ziNrW+#LO;X-6-!a>cJLBwt-6+xs1s;#>2FZBLVUM?}Q_f>HZe_g;pPZ;FsI00e zDHuTgOHuK{(h_>$PVn2>peeRFJ|oS}%Nw9AhZ~U7rK8vswuwbtetM#`+^i~%N_rz#98KF$- zRlb*c@`nBfc(RUJ%cEOrrD}f=T!^sKI<{2qt$0=cWp%S{^Ch zZirkKof|5bW#@*hgjY6g zF9t5hwig4JW7~^?D^ej6ourJf@p&V1>5}^rv~7MNKDl9X;}pj#t_Em2<&w#*Q*JUC zp$Ue1<%Y^~OJAqC-GK;s)(hQy^gW2Ud0jhfzqX9qUyrn|+?4}oz4F4Qo!qyh-HDOj z!y5ywTX#K1K~>lYl?;>~S)H5>**lXV(7QzMVU2^)os*P#tv8C-M=ykyi*~D6VhY=y z@-n3p1nz0X@H@b1$G|<2sC5lMBs|J!D32O|C~A?3aX_n5F_oEHp6DTR%OpxIQjFX( z37>4{gsMimWfDF$1XF~9BWJu;?y%_+2)yw`YHi+_A~iQ}Opz*+H>OCH$s1Fo%H)kH zQf2bS6umMLU%hFC$8JD+aXhCdUdDqfa&ii)W_j1xJ)|v^&cqw?r<&Z&WTS6%oeSDM+8(GPx;0 zw_I*Y&@GpnB6Q2;rVQP3S%nyRqk{Ot*XwcZEtwN{t<#}h07q80&eBxEJ*x;O))kQ> z;EmMQRqF|-!D7)8UP_p&PAe!_ls<4!SyommKj4Hv)6%k>o^pzp)4iiwNE^ma_molG zOt<3cp3|=2t#HOf8O2Rm>|FChD`@=D^hW90&4vo5m2vlsQJj?JQ@>fp5z1H?qd19t zbIH^n2IKHDCQ8?Cx-Xu{t;h1xJ!2FnQ9Lr1?kOWX84F_+Cq=%!wZ_O!#=_{<$%YD~ z^3gqE6c_{{^wS$7CM*o8EFBwa`K5jnQ&)QBg@rL1|@K+Mv1&dDkv%)fFMvN#JT(hjQw4iM9yo#)> zK{;8*h($%!3#-)(cEIH9*{Or`vJHZcNSVwZ7qr_@`!>d86HhcNga_r!>vx%5X!}r0dDaEp6(5vmYp>M8zZ?$ zTt-WniIQ`>U`5Y+sFu6jhibUWf4G9%24VdgLyok8gHn$j zlrbcIaEK76$nT&*gOZcehh!WpE)}*cIj4*9{^FJ=y-{KqQw(wRFR3amu31tl-x_b> zq^6nqPNprBjQ%C1b8F|POf|3cZD|-|o15=Mnq?&ANZd5@%4ImIZEL;@!PS)3)ub$1 ztgZwM9jR?HZl$zWog2$?k?x-A7#mMp`VHCABHao~Fue3wm(&UfDdhEa^lbZEZ7USaapl@!$!S^6Dt`eVp@ zx{{TYn$@3YV-~J8PWPW{813<*18xXhJ-%ef(S}g)a*v=ey*49=Nya^A#^xSUo7ibj8?> zvmfj68R#R>6&*7=d+hkJqbH1=Iw5=7$h^#M1rw%@ESQlyWq8`Lsj1y;u8*CTJ0*MC zq$vfXa&xASDHxwSBX|68JLHsF{La^+((z?;tBR_Y4WBTrU}_D1^{aSFX?bZ;bt$+< z<&MstK7QKxu{oLD%FE^!S1zilC&i_a6Qt|+4KPcywE_rIJi{^*8kOS3)F{x%|G8LlN2OJCRIBFJD zl@^tt;#gT83~Z4*EV5_n!w6M|lkF6X=FhJxosW@ZZh1()s6=PFhO0;(1GQ;(YNE|AqV+Fb`#jPquyiH3VQf9Qp zWol)$Ws7SDWuTdLfZz)NMC;7VEGU?lHpn;(((gm0W3Z%~TvS!GsMJOXRQi6;Q;rN?YZ%9~ ze87T5HT{3AE?u2=$=Xg+(#s74mw7({L_K#Hw!@g<#{Jjqkc~kW9NrAa*6Y`F) zZ2$3l|9b7om;P@FW}LlYASqB+xUcKgYu>zR`26cXD6dWY!Z7BreC@lF+fTf5`BxWI zv>(#%)<@nkjAvN>(b`pKXK(M+zSk{B-gwc(73dQWOdy_fy40S0$mX6`_q~2Y-~Zhsk6wJofvYeeBVP+~%_z>y$2U97M*%0#qX8wkSKW|7z4MorEiS28 zTAjSGtfo3YEj7PvadCNVNhwbIr}j@9Jb2K6{;B!3#Z@)=)ffmC&+oq=ex~9Iw6m(3 zf<>i^Dyo(>Fk^A$vVv-GmSpBvl~z`$ffE@mD8`gvF^`<0EzCm&t`*Ra6zA7eRFqfe zFRI3AT1Nl0LH*MQr=|~3MNy2g6!R7*LtaXdc${5!&!kGQsAhpPU&Do9NpY2!<`pb1 zUFvMmdj^&^T&-SIR9;?D4Ci>hgEbcBKjYhV#{!h*iffI753W1z_Osfwn3c2ffev3^ zGx4cxH08#fGcGAk%Sgp?Yp(G_-ud|{E%MH~i>3OTZEzG;9JS91A)#tm% zuQ_2z-kR)Q0C_ssXV?`t>Rz@L)XnXd;zyJOl>UnGOQ!4Y;)TGqqt$8dZ2v74?Z7V$K87dQ+zmG=W zc)T%&=kItc4)oQB^#{W~CR*%TAq_lR9r?s?o|g zuH9ZJ1&sNX7~QhljRTuFCSzJn?*qnU7TjfYGKLtfq3|7BsY**_jpLy)fDtbEoGjXY=^#TxHpYH=J-gbV|Fj;Tygh0AB~rnqn8Y-5uE6 zb`yc6!z>JqE{*A`0lAbxGq8ke$8^;pbS;dIMh>o+bjeP11K9aW&@E@WR=V>U>hjrq z`Ea&g2}_my{#dM|d+9179H)Y@mLdIE<pq?iflhA_x52A-c@i3H!_Z{%sBIVfJ2~yiS z5-C;V6r@xodm^2M6nkdDlaQh!2InH}hm=gAJp?Iq8AvHcsN}(oNHG-)-j8$`(ua{A zhxAdTSxC1c%|W^y=}4r{Bh5wnD$;RCcOk`mFi7nj`&_}kNR#nuWu(fpmd0Wwt5UMl zmFz4f+n{9BGbZEJ(s*3So>H>+mF#0B`%TIIRI)=6yJWm;WptKjEsYE%8=_>Rl#E&i zX|yz^DcLL~TcBj+N_K>yQwCmFzktdrZl;D%tx=_OX(EuVgO9sAR7y+51ZNv6B6*WQmEUZd-ZQ z%4n}-`ASx(WO-}0<()qZBWooF*1C@4&#z3Lo_FTAccEa~_OATOZivlWvlGK=i}7p5 zCXMGIc3Cop+EgB9nn^e+ARl&)kq!@x;Ha0?QBO@IE6$Kf4JWKuHu zNfyPMkD}Z_% zY7aw`aKeY(h@S3gL@(q{mqzqLX6zK`i>@?$M5mz@Jrj;$0h%Cm=kt&eT@3gA0-*z* z7|~}yHHYE-`KnZvBRUPkqxou&P+d8q(?ntrUme9}wGo{r1L!?*^x)BlM)(W)y_lzr zRk>=$xiaT2g^aOsDjiA_LmW*&(6{0^LUY#|@2Quiew_Gdyf5VUodVxGj`!3XgEgQH z&;yu&2IvI53}c3|ztNjBvIOs+CJ`nbG*zJiyO7^EuWEzw+fm%mxl)`dY}Sr>8NE@$ilcpvRY+wDn>NIiRnicZ~|M; zoHd)F!HEirrXM0VhH)dy1~WC~PUL2)aWIBOlbRrfjH^d&&1}Bo%ehe7pQ&q*lUrFP za$6HZ-3g&Gsgr>zWL!OJYZ`MoUmn9A(lo~Ou(>ftpT^Ms6~)XjHn56_7sb$2NkR;@ zk{9iQW-(W@N)B70S&XUD5-rAQ=FhX&1b{6(X8q@rDu!m>^Yz(~wfBtw4Gg(jfkSfOHko?~$I3^hc!UA*K1n`AGMNe-|J< z04dshkb<%nX?vs>Bc<8LB}flPilPkmL%JU6aY(O4N^_0tkWznoJyL4ZUGWk#Cpr@| zCn2MmlaSHONyyGpvJFZ`Gbe$g_9SpmDH+Y2gp6iR0!K3^A)}lL8Tl?`G; z&76dcW==vzxe+p&ISCo%Ovq^FBxE#m5;B@O2^r0tglw~tZBep^l#J$3UEyO(<2@z& zNXdRvvOkrKb_Tmbqm|K4p0zX%SF)~3)=$aOlx(b$O;EB0N>;99E0ip#WEU#gIwjkn zWH%_;LrV6TlD(s3G_NA>TN-Z0| zPR(2M!i2M*Z@YRm5geIi@UBhhjB@MDzqe%%nYk4cv9PHnWRC|?tlL+#J9`7tJWMs0 z=dF2eJZ-ev&Woi(&elcV|f!Ot2N1g+ctOQLzwts zC+H#5gfdA}TfE^(uAGsMy{K9Hyj6eIo0Sd)>L7EjIVahiWzt0T@*q6K&sZR?}7xv^$8VrA)vR+&Rhgo@jWME4Q9(S;r~(ZV1(t z^I4jrPUWi)htFrJ7pL*iFqZP1f~Lk<{63lA=LG^r7szNomgZ!%J5|WnFAL;Cho;`r z89A4o3iM<~thbrV(wu%EUp<=nPEdSX0!sri)8?r8E=>#!;|pxB2zPFJ`vMtv4KVli0w>0 zlgosr(2xm+6iugJV3jFMP19+Ux)YwhGpUn-DP&wd z>VeWD+}&jAGqLeDhNRQd599!ICp1lS8=zH)S6KW-?V)+~{d{>o zo2NOl>2-5szCINfd%;voXwMCg+p!AO2vemMWLmuy6MotW-O4KYtU_x9QzgmBN2Wry zc0=|pskjX6Q0lYyU|>2tdk)PX)hvlRQ=3E1-ji`vo4r#UX|`t`)Q?gJ)eA4`j_BBg zIWG0B4O4O~Iuq$0q~%CIN4gm4mq;nJUn8Zu^$pS$NU?Sao`v*3NH0N( zbwqFj(qE9?j`Vk=&msK-DfLo+Bi)JA2*59-iAZT}&j8fmF#UL`$EaSQL>+v>~|$QIKlMgFnQL}=&fY5 zO6Z2zv^0)Yvf)ZLUCCxES&fp_DcN~SMyrHw%tyPgbk@qC-B%&oqGWd|*$yRpNy)xc zGFl~cV?J6N&{->k)&@e>D$$hDKCF;+SF)Z;mZM~}N)R|&8_-!RgVqK@Hc!b;QL@vO z>?|dtwSmy3wSmyRTgmQMvZs`6yOO=DWFIQouS&L0$yzrvZ6wLFmPQvPOIEVJN|vf* zc}g~3$>uBBLM2Wz ze12PYFWk&_vHHOh<)W>qT1YS7jTEb)wyUq96Iv0yXxa0+CbHH-yhds**GNgUJbIq~ zyiQAq+Rdin5J%450Q+!WEYs}QOt|r0f6X+9mOEI=$c2tt<2bBtwgS%Usjb4ASWoT7 z8(vTG-+Uwa>$6E>-U*0zNwqa>NtJZ@Rv3UUmi@7b;V@iQE3ds+dHwmPT6s|wi?Z^< z^6U7m_%rn`Z4LH4YH$7f4Ak!pg-1(}(ldSHuv>#=cv^#<V~YrPU5SR*sOL>f!1J{o8Y*L=cTj;8^`a(JqB8X z(Yzyc4OYq5J0%oDCwvV?qwX9=OkrnCSH=B=skl0VuZn5-RPYPFHVJ=*t-)x`MIPS6 z9z19q4c}-YN$aqY%u<$65jfh|gE_M{XSLQ^SFq|@thHuv25H!D$+EdjMfVnVGu2Un zKE_;37_HSwA>_ijVF?<&wbt6mRM%h{JDcfgt#vTVW->jks|5Yvz(A}{oun6HNUO zZU7ar*URvpw$`ZniCYikJ+0w(^ZP4NE-UmVB%UIo z#pUM=IhR#vacQbFH_k;+LYIBwX($TZUe-BN1eK<)tEJ!RC-l#UNQu3Vp=o{{W`Bym?T6PisY58`5l_4%BddqNO8J>bY#~`J(AbQf^)kw!7y%{Nb(cl9}(dP%BK{^rXE~Jx?eu(r0q*Ny8 zvx8qEorZKT(iup9KspDhfq0&Tl%5qSLfQps3DQAG7a$#pbRklDIvTzO=ORTJ2InC? z6)BCoXs^8Hq>)0bIq6KSIfabYoI*xxP9a;TWVGfKGFo#A9K~J8Xw4~P6ni0~HK&jz zA|FDwzdUQnpAZrByH3e2Q?d<8c7u{V ztz@+36uNIK*?USxccFxh03s@IbQemlI! zm29<=(W+ALZBVjDmF!6+qi1>q?hPgTSjj$9GP(;Ta6c*;tuKX))|YgKr<9bezmgrN zWVF)k&bWz6Hbu#3eMwha8MMBnvzEr$N_M`IU8Q8S((KN-Ta@f}C3{NAXnon8al4gl zkCOeQWVF)k&bVe+q0w0@qlG*Z>q{Y{^`*ekT__@X<ZPaU9PjU0Yu#RWDTJ!Q+I7vLQmeqme@6w-7=|QpD^``j{cP`%zFYw-6q@TTk-g{mf zFM&#ftc|fY1r@KGIbi0xm7=Dvo43Hxre8NF;ghW#FtNNQH{C3+FR+%^RI6Ptud&8H zaRdGouhSOVccSUkUue^3-E$G6uTjB|!xT*%b_?x+o)+5kZ5G;?xEQn8z-#sk?a6%c z#?Zxb$j#3DkgL=~VI2{2v$K#TSB2_^EVSqI)u-(j+TZcTJ9#BR3+-wAz6-ynh4wX& zg)g)(N?3{8LG}yn5@y)$u+SdMS9|g#+*)X#nNXQf|7Itx256xzrr^J0A!RMJCoszg z36}@P7(Eio^i_hj(5CqbJ&kr4R$X+5i{?5sJ2i~6ct%ePZJO=UJh2aCZyUvMOf2#W zA+4lK#=}kv?bn&=AuIxFnVuHfn26(8%Jj6*7WCCvXfHKB4Y?H<1j?4)T4?{lRL_7u z$n?wcK9gl)n}8P9g8t^fIBXNN3RtVI-A?q@!um0$x*48|kC@QHnx^mumNpc7x<4xD zAH>3%Wc3%;^#WrpthX@D6I3?IJf<@HCutQrMK4Y^i(~4n!>E0v~I@=5Jwf$)4E;I2eEFiGQLL>GXu5(C|mktIT%!= zhjPGP$NS4UV6>kyh3Uu1VmAXOc7j$YbFT#E!c@<|LCPEr3#K`JF)nVgQvu-yA&%?W zKJ9V{*B`{B^m^C}UG%51eJtkWe$LITLkoCQr@3)+Jsq)8P)YvN%eaPhXg|W#NrFtb zV{H`BhY$>Kn)j4NUX~@C4cXv8{{Y=nrg6;XkQ)WNaW#CS;HwZF8n0-hAQP_-q#RvZ zkjAj{ky6i#M+t+};a`H3HU_YiElSBLW7mFEk)DP06Qma--GlTNq$t1O14yw+5!{LN zTcmFx{SVT2k^X>`<{v*H?Ta|3;3YO3=uB)l2pMfS2pMfS2-!L%qYVckqYVdvqYVck zqYVck`&`Lr!$HVs!$II^!$HWXY=kUD$*5d}EJMktOoVKdl1)~!X-Y=zSKw$6rn8pD zIweCbm9h;=c7u}9hJ(;WsYx8-FJ&kJDWeSsfujuvVS_dtgp4*Egp4*Egp4*EgpA5o z$Y{es$Y{es$Y{es$Y{es$Y{es$Y{es$W|*EZ8!)SZ8!)VZ8!)SZ8!)SZ8!)SZ8!)S zZ8!)SZ8!)S-Ov`Yzm=>L`U4?5LY}oWXq!RE=&@I!OB)VCMjH-7MmMyDjBaQPS*?;S zQ!?6S5IB15Rp4mDLC9#sLCENawvf>cZ6TwFU4@Jub`>((W)QNkl#DhUgp4*E1deWK z3mM(e7BYJ5RmhG}GTLSkvXM$g8xBH78xC~V(x7byA){N{LUx9dtx__&jZIfu85@<1 zwi$%%0VSgi2O*;k2RdtMe57QbD%qb(Mz^*Fjk%Eec^H zNy$1WS%#7gQL<4=mZxM3l&oCI>XdASl5JD69ZKe~A(D#55_LlayCR$G@Ch-bcP&7w zZIECm#CizY+91KR(WztC?7*kw+OAGOt@X4)vMoDl=6Bn&TiAc{PVFto$D>2`9?ZQ6 zZL0*=Z>Me;>HURBdFuoZbIGj}Jg+c{_W>Go1I5}n;SH4zazmvf$-kyQ_y{02RL(P< z3UQM6cktfKgqbPV8!Y24YL*c8@Il+vmm2j09!(Fd>tHfP%9CnbMPPuYnLn5Xf`H~Y2?XKZA8mhjN^ z)8H`b=FtUvm|YfZ*urvS$0i?y7*Ai4Ko3F$teqNaZZYiCV7q2v5P$BiMC;SGZ7TK; z-?pKz6r7I8o`rNo5>EKA+qN0xY1>9Ty{&(X!U-tR1LcZ8P(f;{}@{p_D z`8;&nMywIKh3bZE+bm`LEH)eVe8i#fiymjDT@6~JmN3}bw!t(LhxLqlkgp$-cqMeg zw{4CGx`q)avNNWu*7Fgw`Kq;TBlub-{^q!CGXzxu&#wfEjPblEqDi56G8JvxB=h@Cfj-7YW1q1SYc*@zMmRSq(c$@sUQGWc zR>hYv{Zdd(XSyqyp0;fS{r%Xsxf0(5b*iE#YFm11+vWtO{}}YNTFh>@9Mq>Uz1Vc3 zZ5u)VHMVWe!mZalMZepLJ{=J82}hdHqE|5vHP$eFEvR#uUMzBH+eXm8gKZmo$21ROZ$)f?&>ElCV21G^o2P9XllowmS=%;2g4L(CwryxaPER&>Mp02v zYCxOG)b}toZQGdCoeYu*>SSmI@d~51wryx`P7fP*L1u}XwrwaV<}Ms<(Fp3qgz4CY z%L>pAhV&e`AVJ}|zOuJmKcD%3!)DeK%zrv`E@l2X{GOsBIK(c;>B`>Vgep9CR2E25 z_6kGnX|>bZ5*o$ktm;EsLZxi(T&|t8B_y)*0k(wB#S;M275!FZM7UC~jIAx9flO~z ze%cb6!}OOjJ#7gI`lqoabSZWOOBDSECwgm3Xi>ewSOIEk0OpPpZ2<}Tq=Y%xahee5 ztLRA{BxqP+SXFcd^Y0fp+_;;AzY;nZF@FJg#KSLwV`<Syq6A-sKbG*ap<`y!mY;NYju~ z>q|#UojA6Hf>b9mkWwdu<`ks*jb}@OL8O^TY3#*zLU4b?Y6Q~ONOO?Vb}61c2__@W zMM_W8k48ESX&%zCNbw5JLV7&XQl!{w3erZyM5JdTorLr(v1ssq!IZbCX8 z>1L!ek=}t6TU5a(ke-b6Wu$YFzK0at!Jm+pAq}8>%aL|QT8Xq5(rTo%S+Er8B&5rc zmLWY2DSamP45ajl*dS7xhpj?N+gC^7CAP2VOl)5X8Esz)8Esz)8Esz)8Esz)8Esz) z8Esz)8Pyme`&`Lr`%1{DUI-j*UkMr24IxWWGO8CsM%z~cm!)K*l#J?wz|r=Vz)_nP zGMbDC**Ybo?JFVMpky~F8Esz)UE019INH7vviFpXwy%VYMoWRC(O$^VlB5iEU&@YB zGTOcpxNIfMS27wR1s}>+;;2>$8Ol(~Rx8;xO144C9#yg@m5jEpgk9Rc5_}&k8Esz) z*&j;wx02EJmC&W_D}kf!DwKV!FS*nr^ zQ!-kb3BJinHciQBBTMjAD%pifwob{eQL+t6wpGb!!%NuMsbp^|*-uLLtCBUtx>?w5 zAI8=Y%98J$ZGGdgE<#J7?iVssEZa+U}dUVx-$>iAG@P!AAP# zgN%Ob+k?3Sm=6Z?Az&T==5#QpfjJe-{T1_;5a!;BS)37bmSE03%ow)g5M%H|hr&Wf zSU3z8I>AC`Sm*)^BVge;SQriq!(bs(S)f~e=Fx}Q$air@7DlryF#T|2{8ycfQM)?B zQWsb{9F~rNr6XY}8J4=jQa4yS9+t+z(pXr^gQd~R63y<+<7j3jOX7?yjc3{K_aC(M z*dHb3>-umw!OU^y5Yx8eU{B-r4@}Fjv{!;g7-BCvu)ZDYH_ve?S z-C8~2va>#Y>7?GRa;_bA+D9i9t$%i2x3ypS({35N|BydrSKjm<}Gd-I(yAI{#{ zy7`lRKYXlU@yILh=sUFIgqtTnS9ELcUqy4bBV^y#wqn=d;3;5RQnB7aJ=yZ$wy;*7q@6ZhSCd*R{fExv!Q&qa^TJz>Eg>o?>M z`ylD-qgx();FW*xz465tFKTt}#=E->Nw}b5z^mhroYM23KfKf$nd2j?7++f7zpUhn zvgU!m@d4CCBWqr1QBD8URE+XLAqoA76Aab(Tq^DF=|||#j8G*>4A2jp=tt;Jn^5o* z1DELlybJvq5-JbOamXZ&AD%kwpQ71?+G>s~i2?di6ROp8G&4vR%7e}U7@Dopapxg9 zc#4MJgz})l1G56E;=?=i{j>FXs8%Ef%+}RaXU=jmn zTl)RsFQF^ju0S*rX(mOEtCYG{^p~llI)X` z)=H8Y5TZ6LNf|#eojxl-2ie3gC4@+*l!YXf063_;bO}XXmrx9K2}NT+qkqWHr6fUu zI)=Y=ob_6lL86Y)JK9E|gE2rJV}7|qrptYwzW*Au6W5cQdC9VUBX#%Q(<)ogw9*~(Ka ztpcgV7|m8k)*3?8IIgd2y-U=2dUu1=VvJ_%FkH1_xV!kZcPy=Mq!wc|Tb)=d3xDYt zIP~=2t#)yYY#5BuY;^|KvbEo&#g|%IwNi^QM*onXiRr>xyYW{)9)ZS)h~KNMespej z_ALBV;&kih3bNAZx41*K!@tAPi+1?8<9N{y|6HdR?eK?Hy=aHuGl{01V*bWqT(M7) zKc^WViH$4X3)04w7)Wef@m?!zT#13i#ue{X%*K@%NNim3USDloiGjq%l^802c(wIT z((;%A0|q3A9MOJ_Z&2Eh!Gi~q0{t4_BhBg8_}+YHKjT*#e{qCPRq6fZ7@ku;r?Zg5 z=GXXoRpI&AFYpxGEHR$b)FI2}*Z9^$o;F*K`I3@x^?p5%CK1oq!zJ7N8sBbEy#w5X zCx+Oy`87V8xa^;=+qL7P`ByvO+MO)0q-Xc*el*>!l^C4zVaK-5M@yHp z$^*u^c+o+B?f8;$;Syl(ULYSg}_<< z)#05urXndkD`4D<7agD%eB6#K9L4K7VBU~8LbF{P+=sw?ZHu#hC6nUvBQU?#$JP6F zPKryys({gLweZg=F0@@-1x%ZB1zwk&#Ao;Un(A@4n6GnvH%$XT=g{q z>l9~t4C5JKx?C%8#Ba-ofMo)+9FJ?$LBoO-XDh$MA@8&yV9dme4i)Em_24I5UIm-qI+_UP+aW% z>w?RhfO+yZfpe;Vw7l!SC14E2iw^p0$F~g^a)CJjkKfbb#7DOj7Xj1xZVP9}HwqQx z3}BvlQs5SV(<)!9U!(r-jb{Qz4{`v9Dqov^Efbj05=UsZYlE8%OujA7s`pe6=1C0M zrNb^Cm_it*0Qc;6;h$4nN}%88*?_SYFFI5{>b0W|xUdnJ2X+V?)gLP^mVeZ~PJdot zh>s3C|EL~Z2;Alu1lFnFAiiHEhRGuE?e`-3`V@GFAcl!I}TX8^5}!h z`*5eW#|M%5%E31mm{&fF%(o1eGrz#N|E0jucBx%{OK{;pOvX{hF$}~XJ3iW)y$QID zbT=MHB)*R%hJpBF$JY^Y{uH?1@q7#&PJA@n55n^!Z{WEZI%wjze-F(2c&>&H8lUX=Xc+pecY?7N&)3jF>jFFfeg)t7 zz6r)CJXb>pZG_tK(K;$+fWQ!r4m&;?SC0nnjSPWx>JO-Y89XpLUmo}}fjbM&^U&ec zU(qM)UI(UPu)sOZ!(PJWi-4Jp=Yr^);LJ>fq_F4jBLEF&&w*~AzJ}tr6eo}P1V~Y}u>3GrMWOpVm z7Xb4>ab&w+q2cT*Nieo9if(t*sR_npydv3Mipz6=d96CK-TPqo$C?D==9L0B4Wv$S z{yvys{DD^_yEI*IekRK8EPnq7U~jRvt~Re~|&0)gur!ET!i6O2xHh1=EpgA`y= zqu`Q3k_pTlTb$KztV2FZfcfbnVb`gDE=0$B5AJy#bVc-hyl`cL@it!3^06D3Z=>KS zA3ppiVMt^@9onr-(m2x} zxPxwu%tzzkM2U&Sw;LCy0GEDSWWGMYEtZ%_eEFbR58O|;3qGg$293X?u=q~CQ{c#7 zyZr9L{4WofI=gUl)8%^_1%6 zR}vG6?bjcHxrCLc+sKqQLmnE#@AGH zc+ug+*9;d@fa&#iWdDe-1eoO#=fu|n7ghsv;X9G}t^nT-U{2m;;cUwT0lONQpFa{f z>Tj(4>h<92-3i7`c+sK!tG91P`=xgQv(*Le1z_HDf!hPjPcCqO0n_G_=>ByCrk4v` zIxso5IIAAe_&Z)=C@yr^#hJ#-I^bG=Cj6uNXvNv$qke5YFylWLIO;d-`1YaUYy)QN zzXdMGfp0S8(|$ldgclu7>&qkLvwDf%(@@g3qa4 z(gR!X1M{uKLCuzr*3&-&bKK8@&#B)a`C5s=DIa$J?ZK5RfLs1cWdA4*4@it)v-gkU zx0P_e2|lOzk-S7=BJqs}-(uh%`8_h0p^i*0_PNGT8|%b zP@++V7adM~S*UjFfZ5qW;GFy$2addh6OBvpio|yp{JRU7Li$xm98P>Iacvhc(>n?r z)NJ!hju!w^K%bez5osRQzDuHUBwlnVzIye5>T_>kR!N*wea-^GZNTh3T<|%?`5W*} z!l$Z^JyPJP9od!}>ht-)Jf0$OeSov-m0q8VdL$Z)@S?*hA9Qu8#4r$l?Ces%u^hM; z=vQ}eIL+7S>ikLurlf$J@w0*}OZg88JlB(&rE53WrEZWMigGY%)d?zmPC z%*7I?;!Q`0WC20Wev^BlFSK6%vC}KJ4=QCa#aMCRK>IAFSF37k{Eb{{JGslXgH(!$x*=YHU; zl^C4zVdo$9FB^e-dsJjT+E?w7i+&d`I-J@Y<#z%wM~xOZr}z=y1Yim!&WUdaa5caT z8xxt2>PrbQ6%yxEUpnBzGGH#s6MRnfkFGv1F*xPJE`ED(~JdD?@EEohrdqweKVM7^gJ_qeun}x zM&csn_XJ>0a)FxVVmNiNHmwFVikhG;YR=j!5<8QDB~vxJdDP4VVvI;JyUrM_Zf~Kg#c)!0f#; zdVXiGPc+WIA$ooXZca4L!z)^TuK{MW#6`;QJ-|HS0=EO0H*IlNeyLr4Br)tD{@B$Q zdd}z@;1X^X`Ect0j)49YiHXEV`zn>d-FI7LJ{pgjZ4sDAd>?@CP~d*NU0|K+Gx5#5 z!yVr#z>U5uG9UH#w*vE{#L;+S7r$%ZU+23MjR)@$d=z)9f3eDg)LdoIeAxL%`?f293*H}{?=gvq#__Z#fumk0T~z;t~?;8g!oZ$14L@d0!A;{w;wfsghx4tyff7>pMkPX4t3 zUp6pfw?_7l_J_U(X5Lc*M}O_gV=^wR2IkFe0@npNr~H1qJ<-T`R^XiS`zC=zQIGB^sr8(c#2Lj;{h{)w_}Ts63to<{OD~Dvw^^`wN&J??>jN z{LTgD28na>kM>`l2Ih$mBJ+`dzXNmBhXUv1AFbbx1t#~S$b97AnZP_FaZdix`ur1M z{`+xszO>z#cj85d6CaHqmjHA5C(-#{1E&3_k@+aUX}}EsEHWSU&&z<>Bymu)t$(zC ze>*TIeJ=Q%?!(gf`!z8CmN+Lq+TZ;Zn6JKw>|ZzFQoa-zobqAUK4^R!4cxe|1=eZ) zLVR~fjHI*8FY)aJ?#;cC`RF;GA>T&#F9Uqlz}@_Bfpr>x=?g2ye4l9K<3)#4{AmAv z9xxC5C$fK^fuqw8iN>FJ(cv_|nv8-w>i-gr+Ft}N1z5ZBgPtS%@K=E$K055mgZ5v( z1#ZW00_)VD(m2}e_vn24kl!xAb=)VgPUZU?_*O^^PWiC&?>{Jyk$)x{12`h}?;lz7 z$=0|*_$4qu?oW|(e(wLVuK2|j{4!&4AA#!vf35gg>wOyMPaD|GI2SKE02A@ES;t=j z%(X6XHv@C83*6(tJa3D$@=IR5Au;419d_|chU4!5_Y1y)h7PClrK>xCf7o&iA9lMB=0I?HAyVq~F)X5h=eXN=!7q0^rUa7M<@IiHXEV z|Zb7o&e?xiF3;DG+g)(Fr!KYpVPdFW>uRd2B&=3<(K$w z1Mcrq!AEho%kK){Ce3SRyhuOxgoE0Z9Upz^>Bt4mj58_(ZZvQfpVfZ5Lq4>snQ<~+ zbRg`u{Th9tYZWjF%LI--r(kEd{7CE%0rQ{r0!Qm>JGT3=od+>JMgd^pv6`uc%SfVmi75=ciY zyzKbsOEv!lruW?f=TvSKm(TCT_=Xo9k>d9UF#F$UhqLla^|YPDFd_cf`Bx4@9e@ko zFZi75DXnLAN=zia4DfvnT>A$FpHuvZZ<54B<2xC+OCO5NM^|5wm`Hpn@b7itettMQ zU-}~g6OC^WaC07w%tz&Mg~UYS>kI#`0&e?b(fI<8NB3_Jj5P!9D~WX~k2`@&d7_!| z23~Z~^PYC~g_ewmJ=x4S_!)tt`!II>0eu-__O@pDehYy^wnF*?tDe$6@H~mZDIa!r zX`f~#aOZCqd`|hGeSz%~Bk64OLGAoK;68dbGT$8_>GYhyMB>XrTy_B0;zfb|fA-!4 zK8hlXAMcQZkOWCM z>gVK_{#ueT2s<|?empS(m=UkL^D6+43xRo7;fS`gZxQ%?3Cxf;-TC#$_2&b#Md6?( zTYh|w_yREhe9N65^YZySlZ;ESbJOFcMSs&#(apeo`!|VmvM=X_-@hv{f+7CcwYL=V zwS6ziIPHCjb@FGfN534HTNKVoKIRD@0_OI=$LIG6Fb5RQiQgOGm;OPL@d0*jPV(_N zXZH``FW9-ES+?!{808~?IqD;Ke)E7k6_~9G=VafDA^5An_&=8XqVMy<{excso3Ja% z=!Tt}lYDRBL=G_DeIjvA^FK_7jQO7?8RuYk$ zkNHr<+rSLjD{;~5xWnyT3m&_Hsrw>6ztzBeu-}~@pF3anRg&=@c5Y7k8w0^V0p^%* zB~Hu-ZN8U|`uhVg4=S7!Kk9D>Fee<4{G9YR4Y(D+{PjDDa~ih>fI-Lalkl5byu-kl z#ob1}Q^9Y0VzSXCS>l}dWrN=tz?|3ConJ51^GmyAqi3eXfxT`2sKSZ8{I)-o%F|fe~ZGHEbaKcjgwaaSKm?cbBYI?*WIZwZv5EZ zhk@&Tq&vUfVB!O2mBKmckL~>(Fq=BN^W)hM6~;|I-e2wl?(r`1`TeXgZu~yP$yQyH zjelV0=A=LN>!Z6R8@brIIkk5gPK*NPzU~s|)ZbQt$8kp`8+T&oMnAB#FQ4Q5L`+YK zbBZsVuiX#K3kv5X-$I;t6PTgB+~uRcY*84HVzZO40{k8UF1xqn=cK=};4mMUs}&B- zvTg4pII#to;IWcl^gK2kPuc%3?<+AvI0t_JLix48UD!`zo%$Q^6W>-CH-4NS{1dnn zj&tY7vnv$Fjo)sZJRi8~<0U^Q{hbWlW?;TnI4AqQf)hUj^U^?f`N*%wpk$*8J2xkO zFXBW4FynIF`7z30q%eXZ{@D2oa&w|EgJj+&T^b=(rS5bX$$m(ls0%`=mY`>}J=`m?+@#;OXSsjN z#)*FclhGpiImLs8;8zdKD+=etZ#z!>4VXWym;9W@OUgI(qGV$}c5Y7mC|@No7i@?x z-{-)bbg?@>$~POBj+eU2$N2m#FdZ&)=g0Wm1DMZ#FL6%e8~ek^%O!^Ge*C_$n}K;k;c%HPzs)%DA~2z=BtNJ5GxNX(f1fR4vy+eZXaijTKS->T|FJxv zFeXboes|*JV&FPoBl%HJ;rmD@=ddRNSAMPJ=VTAw|87+nH+~mVf56TD zqvYq*Klq;YpB2W9-@m}`0C4}jLGp7NFR6#s8zsh#p8q(91YvBF4h&hq^pxT?+W{JsN|O~71xo5azt z?BpB6if>Og#@r=w(ev@>ai;2S^mFXov^^~I4YE5=VFVC=?CQ-1gH^!Yc#nHK$nWn8 z|5l z(eJxFigEuhVVqq*VxF~FVL*z_j^D>Pa~g1CA9Lq78@T1byrghM+wt23`yK@5rpF~e zC;M`q-uelN0Vy^+e)r%^d*Gf>SnAoXy|mNk9{6QKzOR65^`v`yDc?AS0Vy^+`9^?W z32-Z)a_2V{93BNG{b`BAwANOCIXKZ1n2FCyoRj{Vas92p^nFg^ocxdPYfJ-XIDeZJ zn-f3grLIvJ0mL7>_GY8Ke*y0AFG+q*{ex!*{T1;ZJ2zZr%dZ$G#sahEWq0{l&u75w zS2!nr%v<~j%%`tNeopa#{CdBdY#fiBn;XBO!1Q{}ogdFGP#D1wf9&-4Hp#j(%V#-?cdLJTR3XN!)bc!u>5gZt?xq z3qDRZ-oegI`?Jk@*1f=duW(w=Hn_xH$woWu+@f*e`Q!+k>#8sUh(C7i;P(m60B*sj zl3ylp;dX@ijY0X(3KNgt$lb}tq|e>?aXr^Ag^9j zH886cPW!z@KHkr)2d2}{lAlu?=h@{7<0c>Px6T9Z55Ks}$NckK3ggD_J@D&rFxl8A zW-fBwm!19?*A^vE8{y5a|I)s-!2PY2#3H?8YY+09nHZlR$A!~?o0KH6PX0n|T&^%6 z#bzfT`^O)EyC_-m<6pb>(qEoe7{LsG?D&0*^1lLiUy3`wb`~yZ?&Qvo>jmBfX1Bt@Jht|I1La=< z^YoGK{FVXNs&k5QHg;}K{P;Dc=L0jpo5WG?;r5b?+}LkQyQdhZV&|s$S?({{kIEEA z0P)ApPdkD^5V&)XlKhW@t^mSN}SB;O{STLsLqeI<_Z%c;FnfjLFtocL|V^$USX?H8Y4 zE-)h$&W+y$U_ST2uYLa%qZ@W^Zv1k9c~Rlq+WQSKKY8Gna$JgW7j|w={1`Xh0_HP? zbCT~#;JyZC+wqbg|2o;@YhZpVaSKU~lbh3tU%)b!sp5GTbA-iNH;Z z!;kH)@yM?UxRoCGUGI_Kt-x*d!0%O${N4d>mj`}{`QGf24qRu2b+QNbH`F7)vA~sh z;J3&lzdGQ~@xbqDkNh?Qw~H6KXF}Lz`YQMAMNp(M}9v8*Jh~HwUhp6kG={6QfyA~`2^rb z$Kgl)o#v5W0JyW_@MC*70CSTSCw^62%FDm3_ipCpO=!2b` zlm1xmM1>JR{BhFXG~njN;m7ta2WGt$XCvPoz-+VPY})%KFuSZc8-9tyQ;c@lxjD6$ z@(olN0mL7t_7(y+#RI>^z^t(1Y~;HRnA@y4oAy2f%uXxLhTpfqB#emP-k!kZTX8o0 zW&pFminGW^{hgyQ@$|O=xNAJ{dk~nHtT-F__5gFhinGYa`BkTp@O$jsv>({;n+VJ_ zE6&1??X3Z3r4?tx?|NYFu;MKIDBp7mL;1Kl`QJOh?TW*X@+FSKxR0HiTYHBBQ)tE6 z$mav5&Wf{X?|NXaw&E=O7@xN(j1UlioZ9;sa4*E+$MNzrU=l{hZ*Navj<@10@=<>! zz$~!hY}$JcFl((i3qQ8^W`zMMHmCM(0q#K${C0Ze_W^KwJn(BXM(Qq}d|ANtP+09( zHtiklk>5n%rg`93Rd0dLo z7dtm8Uo1biV>B?+qHy;2$>qS*DV$q7)&ui^!Z9yUkQrMq``cr{JmUfPs>0wBu{r7a z9pH8;td`HB7uK6N-djIP2d=Ziy73$8k>6P0N<8pe{d<0D|&6iU6g^|ubdbdQ6he0_i!s&H=dl>jqG;j}$$pZ5UswZcieVO+52kN)yAFl{GFy*Q0CjJv%Q2Bg@W?A9N+ zp$h9}w^<(f%>!>tw)gJh5^T_Wp;9iKsk9z*hBfsx~OD=NvQ;z$+ff;Va z+038k0OPmfZ2VvaFdM8m8-81WdC-cp;kQ#^q_;T7@lS#KIu1X!H)|5^FR^peeq|%y zL|~>_aW?YR0CTPtXT$FrU~aYIEd1C%o>CYgApSV@kC%XZ#{<6u9{C*vE`72^FBbV2 zcl#?$Jo$zKR~Uz1HkkN;S)p)laqTi-uD9~DiEEDnvnz^UY+U1g==Z>6o#@`)T;NUs z#;0&ZJI!OSQ5ZnP=47{5ss* zKFPg*kY68#QS2P!RUU9-Sev3WwTMFFjIQ-b&jUM^k2HXQ4 z_`U9t-@Cwl>VaQ!skeC57Pzho>*V)r?{JU&#shb<2Yx}1{LTXI+&KK`SJwcuRpH$H z{uy9ixAL>``~ARlo#Gy^sF#7j3{^NczaJ0GT!oW*w)gucg#l}^Ir;rc;MPa+v-kTu zJo4KH+zt=?KJv(KFL2+-;m3I2c`EX2*txmcZ8R_^DxCIH8@mO7Ss%qOR?njU0&|!31rl00CSqcxwWGl zn8rB#*p9Wp+!2MdZ^vW6yr6Jy?f49stXc8f(Fd4;3g_02;lP|2hacN95118EIQw>7 z2F&#e=hlu#f!P(s&)%=T2PR>*Y_D58(t+u&a8CAR{5wHmK#I+&|BeRkM1|Guu*4Vo zbHF3NCg4`a;m7&L^}uYi;%xl>O<+E<;%wwQ2u!;!jJaYrZ6DI=46i@z`Y!YAMLT%Bfp=4YjcW4FBbXO-oC($x8iKty8xJ= z6=&f``PKq+r4?tx?*U*Qx8f}PDBlOb9I)bS_;s3_V)Vk!P5Xg`AMG(-VFVC=oa}Kj zaC77EV|$xC@>>JkC2{y=gWui2yllnUw0AEs-&=8(_L5)cQ;}!I&Q0sZhTj}u{8pTW zAKSYEm%}5p zHm*Mrm_=5cO?%e>bBPsa!|!fjUbfk3L_ZekCQ!?0=L=&zl|RG-3HtParp6m;dNj>RXDf%o*#i}JuCncMC9&Dx901Uj^o)D4hK~>|0&!5&zA#po5H#2c`Go_#KBR|uL9GnQr7FH=QY4wu5fO8z73cM6wXP{Vmt@t zQxE)p1g7;ODWB$Nqvt-rOo_t9>NywfT@1`M3Ma?u*ztq%?Eq${!nx`BBVfLbgBt;U z2Z0%06=~mC`7(h!7nt=5=ceb4z-&=Ctrv?wliv%#yz7DAK45;d^0U!%*C3v=W9KI2 zi){z*-{t^Qqi|00^#iU2n5_!O_dD(8O&n)-0h4fMWWDzDgHFKoQaD|&O*_T|vnUD| z+Yb868ekq!I5&TJADI0L=jJaz0@H4>Y=;xB0QGhS<}`)l{JbF3emuVom* zz)kVMZ?Q*yjlivl!;kvA4wx+p=VrIZfqBWw&&Fl>Cdrtqh5Ld()+3g>3umw3X3uJOR{L67_%2kxaf{BoeQJ;1bUitHcu{jE1JCn%iu zQyagU0!&jBzgWLwe_IF4Mk_y?{`MF!`xMTpzmebZOAyDgbJKomlP8=4%xMaz+hM7f z{FVZw(`cVD@|9*KR5D+1TT?HxHPx3g^_`0bnu@n6(P$7GJIb z=2j~|8$Wmon7vW_V*P;qH(?o`zhRGRhs7Qo-#P%(Tj8AIINLECnDQu`y+1DnX0^g; zz1Xy4BQVcJ@w0Eo`@rl{IPHHn?brv*!8rWre;tg@*} zK48`=oU46-xz)f{U^KoH@0K$ zIhc21=jPUq%YeCF;j~_C+VLnbyQ28nx8r+Y`u`?=I|9JGq;O8{;JkC%xssP?2>v*Y zgXO^0DXeaX#lGaX9+-_*oQ=Oc2FwdqoP{6z+h@RhZ^ha0>wF&iC3bFZ@)atK0OF5R zd*=Y>_rPxjFqc_zHuBvI%%fJEO?%%3W{(wT!>`SX6r&S%ZcgoGT+3G&0mL7t_KpVb zL=XG|9{JS(w>%C%<}I!SW~;)v^`mEidELs-rXTGGrfaj*i&K1NJQxVf0)->mE*^|P z`PIO@s&F#D8{03b=guo-ogl^Lr01T%9j~x%b}RA7Zw7D+Jn%cmBfr(aZScTvi${JB z0Qa~Dejj+`_bG5+$Kl7gk+lkW8tmNM{AwsLg$k$bVdGamVAe+Qv-g8*fVoNG^mu9M z=k%8?z&sFzi{;1u@fa}gDx6z8_5t&w!nw7h>uR(YJ2$8Pk&Sb6fN4@Vr@S8L2iF1f zfWpaoW81;@zOFDJ#pcvM-UaT{D1NbdF7iu0-&=oc3tU%))q1h9$8eAQ#shb99DcN0 z5STRz=VrI7fw|er&&L0D0JA3w7prIbeVa8lda<;Z<8)Wx1}dzRpAt75n3Ln+=m&Fv zsZls5KcIXqz+9tnZhE;Jn1`eI+55pe3Io<+bMmW?fZOYVU-|_$`m-5-I{?>HVcq;{ ztVe#6fScigU!6yOOMzP*hacB-ZS=@*3vds{;YUB%>5<<);C_t5kN(nitv7r02X3gs zI@u!|{AK~uXvNvs_Yz>Pv*IlMi2NP~=3Oh!hTr$VBwr}o;v^s2+Z&jCE6#@B3}6;m zaTfU)ht2`!N`-UlZ+8H*&C1Uv4!sG?&r!J8IFyTeyRSpsz|PHS{9t@(1ZJ(mxy7MN zfVnvit{D982Ihdm@%PN^?%$~A8NbE+9y>QTy)*%{QsJEJ#(s3Y!U%@=;}nPP0B)O= zUs+?Lc^!`&->h46X2JGB7JIP2y$N1>qxiud7979#Hoe8#k79A;*8{kr3M<=dkDCR| z;y5_!f1SdB6k9k?owLj=qn}<1++7MQ?P2;=7)O730+_d~IDS1+PJ_wL=m+X^VA}sq zwpscW^e6g9m>=_wM*&lyaJpU#KjLNsv&;j(RlwY0PViWz+qCfI`47iBPWTVMumkn-vK zK?^_j-$sQImEw!iO(i_JDjCoG9Fw6-0cdh`=tfPJnLV8`CQ@jIuuL)Js!BDE)?f!{;G{5=jo#;dP@>2_&+`Er0MRyZx6#s6r}`3eJ4Y<7On-x2i1 z;1CbIjz<=~^aj6k!S4peQ|s9)KMRg=(z-Qg#*Ty3!|gHch>er{J@q?daIx`^{rce; zTr5BSzQhwTxLAIa@0A!_EI;C2^N4%HBknB^IO_TB7~CQFfyMt8g5PfNE51znvr`lYViZF^PysJ*a3d&znAsO`CqI(SZ_dKWUXqxW@!g;RUUARr;B57hoHYh zYHxU=!+FVS)O)L{oAQR^eHh1i(^J5_qj0KUs&P7uqn-W<%)U4{*84p$Jua7ej*dfN ze#GSgGe_av6wa+3-vN_xRaARp+QE6mk-!XAIITa6Uf7Q5z=RY| zx5Faeall;&%zX;yCf_5#yb_Izk&pfDZ@_#K2S@q71g7KFvb|b9n|AaArc~kF+IuE2 z7g=$Z_Vz)&cLMXO!nw8gU10V_<6_#&c6<*^k3YzEB;m7NydrKAFhPZLYwx+hY*aYi zUQ0X3?=fKhsc>%X*b7X;HST`Jd1pE>!xc`?J8kR{1ZKU$>2_G;>kob#fq7ox+~j)$ znBCF17(LUkz5?c#I5_Gh^;-Nc6n1V}KAUzF08^oGTF(|7AcD^0D3qV6ITOBz$)A)4u@o4~27+@87_5 z`=doZiysU|JtKfQQ{mkFAOy^X(YP2r(;k-sbBn^c$@dU2fA@g<3Ya6WcW*~N>Ky{i zB878n$63IviN?jWgZ{Dsm^&2C$-cDPbHMCXIJfpD+<8N)kFqbKulyCZWnIDQC_pbxy-YA?oFAn3V zzlVYOK;hi{zV(fGj*Xq0TRX-BQ?77Y&z5#@++PXIW`%R}m-~QuIvN+_FKovvz78v}k%5_7YMdVx7L4vy`u24+(n+z{aY z49veg;EY>Rj3L;$Iq9z&=Zb;3KMt-QxMzVGajQE&%2x_Za~vG?_j_PoiG!nFJ_hEf z&F=L!1D6ZTJ#ld4_arbs#li7@I`cNz4Ldg{{Rz7Pv&jSQ&%k_X#g$hrZX8Cf$?cVa z$T{CG>om*wWAoYB*(XdcEGaFSJf&pTl)~8)i$@*fn=)&n@03Y1$L5X9%{|8Ee97!d zGYe-=o9Qc>G-1w(zS2pjOe!5~hn(3|8w%D0N`qw${)Xjar_A=v3i(69@|l6^fWI*S z-bIrp7tSf2U0O0>)G^h;vhwIO<<(6Ufl=dw6$RsSYMW|&^-ZCIqEp7_)HQ}00{)t^rb=Hh6lm~=>KY1)=8VrN zuktteLJj_4sIj1^pfOZYP!Rcd@F}^~b>;r*K*8XG!t4`b_}4epmZJiBi3J?lDl~24 zl-ZW1PAe^%S~z9WEKBhz6Q@m?eoQul>wfG3MD4XkLKY1+an$@dje&;7`OE8?76<1~ zoZ?>`n9vlgu4tS;!tc-X2g-&F^jD53ADBNfzhYonW!bQSLx+{+R}L*7GAvM;JHL$B z`OAh48K{&mSxgRA2O8&BmMsd@)c6(!Y6A_y@4%%*m<>RM!U@B1=JL>LZE%!Q8R;N{wXj^RzuMq=n9Ask zqp#TC=qoHL^))W_*ZV30A%CzMKAn@*5UBRm)i(-os3A{qs1~-bZWOY>BOsqomqEYO z_f*U>`WmgXiaF|JqMA{od=r9;eEw>BLMT{Q>kCyi1RAUAsw*@w+pq=ttAmSbYXY?) z<0!?kuL4d5nWH73)Yc!5lbrr<&sSZ?zjGpRzbL{zs;G(LK7S>3JQH%DS9*T+4Dh5*#C^z)! zq?prn<)QHD)R@y%fn`RH$z1tOWFyOiv|eSfT(`r>j$u+6To$MZw<9+OfLbc6>--_D zbfX{!Q&v}3?XRw{@@qz|nvLcd7{YKxu+HeCWZoXDG=IgJO^u<-U;v(O8re7^h7p}W zf^_?11s76F6n*mqc&`y_$3=B@6=lJYaBxCw96V50NrcJ7#=$EQ0wS5%ICz;~RKs35 z*uI)ztuQvv+Eh{3Bz=LWZP@!m0e!Vi&+*sS3n$>!Hk~HafKf##oB*5N;G0@kj?oO2 zP~imGbP0eO0`=AY@_^7LQQ5HyFALUUP!_@t1;WN8Wr0Qdw2e8+n;P_K8$|_bE5MX- zwLRkMXo$Od`qFe!twiw7A- zXc0=IuRTN&^l0Fd4VvtwK?osY=Hk+#DPE*<=3^DB$ctFBmN$k1H7-J(>_sRiE>@As z6f@9=8o28kWMQ^&hCoeu{c>MT-I9Q>uClT*5Hc33 zI#ee)ymg3C-t;`x3J;!_t37YFse$H$!qNY0@oG8ZE;hN!z!1fBsZ?jIYSoDt?V9CI z3J$s``0!OGG`+xVmz}o$tGa~R%;{m9=03fTIsfBpzj{FipUG2CQQqXEDjP-MWxPvk z-kN|Ah&>FfZ@^8FijBwO$WUGftT|rHs)6JBFzF7IhkW8zCe*;yY>cHJ0#|Fc(Z+T( zUEVNXDAsu9nHcNo6Hex(Awlat*|8nD4 zSJ+ui!9d72)j!oZ&K1W28u1x-PQ-FrXkT@((N{*rU_Q&anjMEM5>)bJN0wV7090h8u50nR?gtO5&XEWhmZFy~=5qbzzVlF1U z!k#4(5V%ui1c-2}2(OS1)EHV`9T3jPI1%d>6~RSFV+d`#0rGN%!`MS=FiVnr;=%&U z>g#IR9T*OsxtTC`e!*Iw33THZ0RunYFxpq#;I9ZS(>64xt?WVY6V=rs9MtSUjzH<8 zY$y%r2s54V_;|ob2cpNa73SX0zNMo=p_92jyv zprgYB<+f@pl7EB^%zhN9Tnkji0S`#ki3i57KGG5ld=X(gM$3`i&>XHKtrjV+w3|hM z+NK)j){t|mK~I(Q(NUwI1dF_?Z#%+O9HYGSoYr!Lu!pOtKubq>nHv^8%VKE@+MMYL z(UmN0Dfz2gJI5=OI43B$x_OQRheYwFJNbj1raL@iJKYf{{?qBsVXIF}cT7tv=eMYX z2_4)CDAf&t>LU7-1uH!%n^PoQ5_YXTrEq&+iM~ozl&>yPF78^9yaaX{<|>`OzYoi& z{Ku?JO@A$wS+7$P4t15#{vya2cXIjP7*|X#nb38yQB_?7bG2dXFle*)w6pEW$g~Q) zYPH$b?E6kM7_A-dDvMT7rIbrEEY@91J+iX2=`N4FP`j;k)QIJ&)LF?a~&KH#ftY?KkrC^X*I zyTVr}k<0h69)xiVAG&)CLM)s$?CZI!=c}yADE|-MWkI zm12GfC8}yn4dTUFK4r87muN6}n=?*xf?Qfx&YXdI?pIz{UCk!~<~VO}dXs6?ZJzZ8 z#KJhVygtmf*;Q7u)wglHsfviUzG${JxOWWk(StegAM2SjbYX@zqj|NP?rI|L7q$M( z1Y5K`O8ffm*?cY&jp9NE!;Fze^SfRvhsdTnR3Emg19<}fB0Le#R$5ct$^t`5P)le? zg;@}_&@1X{n-yY;Kx{(DYO#y~L50t=ODcTjxN$dQ>rh+OSJX9?RR_ZEY6c;OdRv2*_$nI!wgXoK+3}Ieq zZ;03nbpT-GW{vjNaJ~>io0mG7#By<@UHFWVwL00w`BKzt&f8Vr68W5{2b8=O8xVdh zW8zgzcEIPM6Uj$@UC=S>xvW)e<@HU-#MJqMjehYA^;W#_X+BM!h)1Hx7Wk?H{tBcu znd;(cGuU^vq*3oXjU@eqo(&Q^q3#3GDDHq{lPVG`8D1A$-0$8+C zy96l=Jjk!G-dIFmmp9n?LeU3Kf(m*;9sBwAs@Oy?D(7&cGb0ruY>K5gvP(yu4*3zF zQnTjZjR4<7e+V-mTsfNq+c7Bq0jYXlqt7>U_Uw`g z7*Z!z`x_gJ0*&Pj!FntdGsXk+JGc~AFExaGI2piUi64(3{U9*msI3e68UmGphCpq3 zV2sL-WQ)hz*~g7T<~=nl6?NzMSZA1X4n|AlekUx)S}5P(A(+0VX05{8FsdTdP#tU( z6@-~{_GWUH$8pAqxbE^p5YXpiJLHhy6YF!ZsEmSUf>8hl@lB)|mS8~)Meh%h+(jQ@ z906PhXdRw6Y=|!;KKI7w2Ie>Lc_7Z8rg&m$Id}Y#c6Yw7V>3eHuD*9+rcJcwt!L$+*7uKfiv|C?yA5 zzStw5Ce9hcB22)gi~gfaM~xD!poZhXqr#2{7rh~?Ub`q<-*8|Tc*CN3=JlwA&TzWj zh5kj08Ul;(trO^;;d*G%S)SoqlPBP^>2{Y%mum==2Z9J@;dx2usqd786{(5m_z(A!xZmvfm%j@3LhSaSMdgSK;Aep3V(@_S~4pNRQj8$F&N8bxmfHaS86~^safB^eZ*OT zllSm~?~*76(9<}hEAXd8Rfl4!a9aP~&We?sNSRM^#58EX$<1fGE(>$eRPhi}h$5s= zOBFB=;*)94Ub0Q~X$!=+OGjMS*ip%am6=$!hcyzMn4M!b#QsiBsQy;-N*`unwZ4W> zD5zG=2=`lRvameO6RqiuH>9N5CM^q;V@0~J4)Zr(CHn&vzt;5K>bgZRsxXca>?pID z(O#m}NmV~%I*OWL`5OI|0W8Dlg>&7^>p7bG)T}P17}3q3AY9Hf4%vv*tofz}Ek+Per39O2 zO@wdmL`!=lo7rGh}C;aW3elFOjXJ=+M6Q#vzV%wRa=D9w>$E&hKPLMV16pJ zfK`7D@ZY7%pdBGyxC75Yf8hj{!`(vdAJb|P33&gr#NQCaGzf1oG%RWaj2cQe)fj0_ zvjN6fvd=cFj(rbB-0Z|EfV>#BII{l}3&e!vmHz5R8IpwW^I^3t=bCU|9t$ zCz}lz0LCFxP$x5)Ias_Af?0;?uEK|Lhb+e5uDG2Ll0yRMUn??>{&K*u(xG}SsIc}g zrjg(QmUYSEwjW~+KX6Uot){FSH*c>WiN3M#%IHLdz-ZORX^Ey;(nc*~y{{3|CE984L z!s*7yoEX=Ii{U!gR7kYc6C zZHvC#g8|j51rHLUpV4rfiezM|1`E?fX1hcef7tBex)N(A5O<|)fGPlO*G z9KuAJQYOeJxq6j)XsK<1;aR%j_TN15JtI#!|Dk5`=^H?l#ZjC5lC%Dm8X;GPB&&kj3&56V-3bn8^nyGGub#+!s@f zhhrFy>Kd5yN{U9b~2- z$lyUmC7bq=)52=pBX9&5V-}lctO$tdcwUS=qRVM@$8&=CDQc>z;XPhc4RXlHFW@dy@X;~c3_j7NqPv>yr*m|-AER=` zT={$LIhY7@(jp#|3c>$sR%q3_h-V^-j@GQB?^^Q)i<`i5(*uHa>WShnAVFEs;sx6cF->o_;Y$5`F|rpIhGD|VSh znIj@6{jVSTb>>uI8ZPf)PJi%*+gVUMLYtbqP3B#n@Gu!6DpW3{0&moZ0x=+JV>%c< zZmQ6nP>AOfdJr`yhH~cb(ndKpQh9&XeQXwdg;UWG-e|7J<2c;3E*7rSWX41eSHgC} zGc(M&Fy9b}ycy$A^RzkC;JFQ6sPQ%UYZvh~W8{k*461J*c-P(#ZBpsRyn1t30k*SW z;f^1gU7JDG?$tY^60FYAwv987Gp9eE=HVt-&gQ6!hfHZ2BVM>O#^B7QrYfx6#xWHp z6aimg*FD0#8gC|u_g2hrMG%QF4zBN;M!?&sV$_z|4_@37Wm85nyxFppZ&zYsV3Yyz z+7NSS5WcEi_pnUM(1bat<7YFtneHsSE82@0h1Jcf(yet&4XYe%+3mLK$gg_4J6kH% z%2O#C4C#*fx|5vdZ>Uv)kMpg*x~A|LQ{PaxY&qU3NzJ-NJ8QHGBc6bw0uB%(A|}1! z3n^3`oWbbnepBACh^gM|rUmhSN(lW{PoTu863~Jb;qjxNDY2&7#kF-yYlUQIn!aO+ zs(RoMQ(=zmd=4wda5?i){Yi)?bw*8@gT3jV<8fio^q)hx5Z`@(xhhr^G2=e(KVpD6 z5;@4hn|X`R&8%9vx;dwiV%A0?^6va(pmn}QN-g#6)>;S05wphK=8c*+_tt_r&zayZ zbP4opX%rdV@cOH`dDvw7o9$37rd*5sQ-oBujgue7YI(^$bLq zR<@Y4tsiwqpNI!vjEHhT76aZVW+#olTACy=L77B1U8uUf*>Z?jQHPfk%ggyXSyZUz zAhFS`m=Z4y_!o=YAkOh>O%6_7RZi6yjdIN%vgNh@8nuK&OqO-#cd3d%wCWiSl109WAnrSx&2b1Cz|$jTDArpWYY;6LhpAH? zst|!j+&~puB8emr0yJ%Xx;ll{Q<9nnW6sF98HZBJUHk zd51L7Q2fPejE7C%k0Y09Jp?MwaNgIgpXE9y4PRzXpE>M)q&+q~T^4D@KkD|(ofu&- zlM5Crre+P+j-led9^1ryL5v%4cKM2FXP^C-P>A5G)9*6)%CEga($17ASR3X)cnI7iWT=UhTU1Pd%TU^@9!0!~Jay(|@ikW(-D~GEuqSiXZK8+ueYh(a^7CJyR zZt<3gaaOEx^Es_>V{LfI%+*^XP#ZrYG8z0f1AI)!$NO-asUNYhDK_?z<;&2)LdR)nb9F^<2^!4BcZ*?e-^3*$K3Af*WSQO)W%F2X!LmaR@c9%f*0F?k zIb+0xtRg79Aq(iO)HmH>*pOiUVBsM1$YA=EA|XsRuE5$>;Y_sHc+9E98WgT~6|=f- z5lRpWC=TLl`5MZrmWP)|YG1?1DSU~()=rpOHJsC~Q^4B?VdP{DUWXO6NFw`%GJi&w zc3-+UT;%Rfq4r>t*;Ng7OG_vuSqn==cN%FYMvDhTy%7s3St%YmlvU$9zPMaxps>K~ zs0CS3%Vq`-6^VbL3Q>!rBG~XVgn{vz2Q5q^&@4V5`<+;E$xh3nN(?IS33l3L(ADIU z(n%tC7f|DJ1*`4CQZ)dY9YHPzWml__&xpa3VPz`Cm_B9lRN)#JqJ^Qwd|i0Aa3tFx zCET);P>QYhN9hJ$#RieTT0v^IPb4GcqK6TplhgG223DG$*e0r{BEbH^84mknQ!N&! z7smN*INuviO9605xqyKW{7W~iv7-c4`fB31_@``5O_T$>mk2vRy0T&y_Z$ctuEU6dw?DKKKM3E7eExpX>NSN= zb?eaYww^bypZXifd9x^gZU2K)s)jx@_1LYu9^PIbN%0*`Sy=j{@FL;<^H$%iykqI zM@4z#(1(v}88Y-A@0|bB?=z1*xWO>`pzh3sf-hDrxVw7W552yaw>9JA`Zo>Z8d09! z|BBf^KHW3%vsvRWz2f25wql_?zE3PO;ew-@&Ny;&-#-l4c*B5O|Gwuc!d<8GMM3i6Oad(%qm$s_?_maLV?>Y9&D-7cYQGViw zGoQMv`E&d(-j=cs{nz(KcW*;JKP7!R@1|Dm2WIX)+n@W7j+60&eWE-w|JJd?YE#PJ z-tpjPx7@MqPWWLO@jvc7;*k%Yd~f>xlds$M(|~KBpOZxS+3l}<;)%1m&3R?xnSc7Q z|GZ&_ajhu7_`B`fD<&WH(Wrf=Kh&=C?}CQ$jVP}^a`f$&zWHpIOSjB!eznbuU*HE@ z(#e0bue$#|Z#*{qhlX!H+I7_(KKxLvC?C*tw(oV{yqc3nE!dX7q7>8Y4jIH>eMjfz zr}TfdtfteG)gzvN=m-4Zktk0Z`|^RWri^H~?ZDh#!>aPnGmNK2`Me)~eyeMnocx{H z7p}UoC}9MCD7r2A{PC_A&g?m@@~&?C{#jqtX4pV{HJ&J6H+)>b5!GMhp8fA%#(#C^ ztMIAMMfu$qmi3>vAU)}tPd|N&`7`gw$-_AL0=fZ>l{7{s4 z8T|Ir1(_L1r=4}}oQ~_J%rcBGM0wH~JAeAR%e1)k z-+$Qs^nfEj|F~$(_;c~YbE14-hfeFBx_ImRH~1!$ga`8?yftq+cP13w@y$}hWY_-hlcdFbbFULOC+)Dfp; zBQ}Zh)At>CwB+MA@>MSZa^$)tLT#%4-~x+^qHVv!Fia}Sgh#rpsN)<5p<2B=YVFrE(HE>=-=0XUWl?+ zP{*~3-U<3TMehUsM@4@N`g%qG0D7aMj{vV572N^!?Vzi0ehB#9q39)`9|Zj&$~S?2 z8*~@Q&LQU=MV}4&Z;D<4dKc(a;1ODkPeEtH2ScDgSM(jA_bB>a(0dg<2-ke6=&L|~ zt?0W!f1~L8K_3shkmn}Ei2;hf9rPeYXQ02Hpy+(ixr!bGI$zP#K@U;%DWHccdLHP- zpjkh-2eK+W{6#XaA4T}CN=w*sN8rQE<^fu63UrC-{V*jn8zXyGVqC?RC z)u2<+u56TFqv(@BU$5v3K;NY3n?Y|<^u3^ORy6CnMbVFfejoIQ;CUr{<`YHV1v&SC z?t&9^taiH?g9D~ML&nJ>r_RT zgPy18gAjARqBnqEpy=O%_JO8OO3?O&5p)@7wzn7fmn*t6=n6$=fL^EQ%b>&GD*8gu zEs9M>oU*br<2NZof{Ow23)ZrS)^OK^RLI14ib3kWxl-F&5ob46;F=RMW z(Z9eJV-!FV90PPXzGEw_Jh6(WfhR;ccA;>csf2`1e)^x z20X6^O>8H8=B)_ZSP#D26wN;PXGK2;`Yq6G*R7y;f+nBu-~*p3x(96W1?Xo{o(ehl zDVpB$6gWd$1_WdK; zz8UnhxV9R6Zd3GJ(6=jkIOr{)DZ`nd?^1Lb=zA6BWYG61`gqU}fF}P9pdST&6|QCM z`U~iVDCd~`Hz?e0D#)6x4q{5>gd)OB=KC>OuI%;i`gU%0QN5t>B;ER>9s^!;c zn>2`%Jq};15E5k$^nYKK&b9vUs}j!))GE;by<pRc|GK+mhy=U+q4WtfYJCxi}t-MjCJRg%sk9|9xuCWg;{oEaWI7YSmS z6P9273Q-8kW}|+cn((ruQ$^m5;cf%(d?Er0UvvF6%o^_bSQz+gR45l1@%f}fXF7Z? zy9_swSn~QSc@ln&h?JwQ(Io$cVAlj%!xqIZ}%@<(bKop~Nc6NVQ3TK6dK_g06m zNouvZldl}Uis1eK`${-8Vp*E}o-?FR@wZaEuyf8;XLjbcTm8C*GegDCq* zQEA4%a*&%9WnU^P%_!)DJfjfY2^Qg>zHv^XEH8p6%3eQV1Cw!)lh|5eDWo?Bd`W& zTozyXO~A?%yvBo6mVh9?t05^ykyzK)$(W*5#N>avkVzV4peWN4v?#5NX{homoZwRpbE9pL)Hf(3@M1eM z!DuFdCmGF1)*GF1gku|D;^{=XxSCX&@sln?lf~IIqo<|NO%^V zg>_F$2-huK%j#LQottODg|uf?}N%L zaQ7!|qu*u= zD8{-x%RqO8h*}(H4$GA4I2ghl4{fGY6ggr_)n1K<6c5EhJlnC*ZkC0aQZvNnT8(Co zmDy^R)mwWbDa;GtJF<)`OJvy)UB;CJvTVLCTd2zrU)8lqib^x=J@iIg^H&!Sl)Qa&C@jFldJri618?dAoh&)n|9fENz*P*^pS=(*84Cj~9;& zV-Z|7N!+sXX7E+Ot1M2%l`PySIG4-FO*&N_e&wEDT03Tp+iPqN4h4?mvf5{hKrszD z4G}29w`UzhNcxR%2Os=SoQ^cOoDC%i!3-lAZ=r;5(YR(OBN_kObcrbCDm#>>w@FFL zNNkmf>i6;#Sw>L;`3>C{CR zTz9&~TQT#%0E61Fb3vEf4#`{fYuk4IS%Nw<1k+bDk;!7F%6whc zeBad0mZ^q=c#p_CQQfHkqw{Ujh zoWfHI=c0?&7e7;!ll{xVg9nS+{ps`Kmi?3R4;Hq3utgVR%|v?S$}pjVf)Y+wDGyiB91GZ1;xiD+9q`mdqX-QZyqx1_M^ zw-n`NuG!x1_eD9mIIKAvZcyAZJFg$E=u*6X!oiZ3gThB9eP3?L%lhc}a+pX!(>((4R&#%}*?4rC=S#NQ1OJPCr>Jdh$PjSnPQ(IcC*?wO4 zgU_yORoqfiP~4J*N)nLxD!Q;$VY@$V-;;*FuZVVZDL(jeVawlFe|NC3-6XT=doEQK z91`o}!1O7>`c27NFfIDK(&w6Yo2n`NA{XHYYvKDj3Lho#o*NxnvS;0%&M;?j%l$d` zKzl8wO)GNxsrpOT{U%S+(-E5A!ki*~N6WLTpG(3Vqu6LS>DkpEbzXN;X3K<*y&rEH zC#-q?_NF!^EiVfC;u1IjZF^EnX)oaztwpOdFD`C*8WFy}&{udy;ex{XzBA_S*qjt< zQ@ozh3E9)ltrS_zb-A!;yP~%rVqhps6LGLwu4K|L?1?V#3*v(JNi zL~x-Caz6D9K3PN5%RsR`JTg$oEoOxJ4?|QAzLujfs}xGf5sdF>^>RYGF}GDKaoa_M zGKAIRXcxkIiI7%mKFn(V8J{@bg#-?B4SuKLF#vZ& z!EvJS!-UqxUZYjQRAVmoyAm>uXA(MMRGW~s9q01|=8vtONa!p?;ujXB8op_ZftZ*z zn^wh!LTt%s6w73xDM}s8KvDVu!Mj&ZYA~0Z?ynH7m;&?-qG*`v_6_B2Pwg^TpP3zW9@o)O@+M zoG(^njBNfO)%2tW927hT{)EHkvFD7rhzN3%}5(#Z-6U>`=5+= zJfs0Q7UIpbgdR}-a`@=;__hq22EA<+KsD$omd7^e9=8Tv%WJIsRD-UxmP_TYyoBZ5 zBaF=VL=snx$30kTo0PPSE^Us$w29&6OPsg|d!$f&xg&Aa3<#CdKC*GL(p9IK0`VBB zO}-$K(Q{RH#*77w4a1LNz!N!kMl{|=W{Zfn8gX)DRDyu7iez+K)hA=dsr@6Q=38`$ ztNI{5WTqoZWVA|7$xJLrFG?>-X`hkX(QE_ny_gj+V3 zLvHM^^lA2K#;$9jSDK!&maahy(n_>2gDQh0b)RV@v;EjIb4sV?jTnXon+xN%twrpe;Qz_g! z^_HLOEuTNIe6AFlm)DzjU$00O;xET~PT4!-)03*a2vc>A(I?`Vhhs60zs2z+952SP z6vxYPoQC7oI8McJv--Rp$LaX|5{@%*X9OsY3u>!~OI0kU6!LbrY3^Ag7Kk9o1`vPOKvJpOD%a8{H)18Q| zd7+rZL?nrvCN+IWjarz8KNZL&S1&WQ=Wln)_E&56?>KR+kn7nHaq;;U-H za~2Wk2c`GMzdk9c8Esm{aN`V{+c78xJggYu{vgImw2=p zs+}`MnegShaDc>Ds0bU4AD=UR*qre>5lN1Mq5|~4f&%&Xw8Fwkg{O?qiTZxff+B}Y zatnyg%LR|qb8>BBbYw`Linz*rZe&PC84?@8i!SH5K( z{PIb8Ex%eumT}30EaS*5%Xmj3%NFP|zb?B_m;FwcU8T#e)n%B>Dt?&RDSjX8vfa9D z4s<2!Jxx(*MnIPZb=fA|0ZZIwMWq?tac3*bdMYZ-n5fH6)MayZ**smgM3%Cu5X~uS4_M|R*MVGyy%Rba)AM3I&bs6u($vMqP zMV+#a3`HR_V!hIpIGb)HDJspFr^|f0>|9;8QkT5~YMHq94Mn9H$AhITJ3&!t#uQyP zLzmqEs$N{XNl{o2q04^KWf|!vE>lry#volbSeGr-Wfi*YDqVK1F59fj?$l+U=(5js z*)O`R6^3|9nr0lMsC464UDgOHC~Ub@QEA4ZELn;x}wsss%&;n<{X*# zM=EhW2YKX!ip!Z?dbQ=GOf#|6ln5#!wG>Lre=YyH-GCbxn+YL2A6T&C#R3#P0%4Pk zVo-E4Q0sr&47m~fV5-{~?YoMo9vIE#F>+VHQK;B^;uv+%HFN?Q>Er)F_V{L0A+yIwUa9Oc zo|~!c@eEKhdpsYM%pN0gq_W3LK*{X!1)ya1cs(eYJx2b-l0E(eJ;|CqMg|6HR@_bF z_~2oWH~n<>SQyX~n0Yu_; zE|P_RM;RW-8UfrQGG|Lmne%Gg!O4jr6I;B4IF4B3ex#2tGp;vw8y6+?5^{A!6qJls z;Y*a-8cQ)zWICCy#XD$x#Zs%Qkz%HdNIqbTN(RpnMNK*xY|3;d*iaUA$wdC}sA$M3 z5PF2TC|%#Z#HOf&0uh~}HdzO8Z(ydVWAe?F$TuHt=9|-9^Uaka-;9KF#C?j)fYq9J z2$6{%nQv~sl;gc0aqy4u zot;X2_ORBZ$~&_>GWRI5(JNcIBt)CP;5Al$(lX7TS<6*6`k&TveyL1G#>zr#x#Y8Q zjI~^4uY>lQsSsU_KWrh>#52X`enKg2v+^;PN8X>y^l|G$NgsE0N+0(kb90YPAFtebp<=*J@@uWjfAM*?N4C4Z%k7-5c^l>7_2juKZH>`d_4q*>NoC;^Fz$ba6iPl$CV@L}{Nb zbH2uK0+rdoxBdYvJ%(Xc#cg2z8T{j#{aO{p_C5}CCgd?NHU>rG!=i_**`g|vjBk}oJ9GNlZ z2saj=TSLxqIO1`ZF&@X0aAf&@@P7vT|3{hP8&I{}{_n^X-(WTe^CyOVKEC+xGvBb) z|6el2;RgP{OmXysz>&iAhbdFs@atuYr%#$-n=D3zjtt0VvN$pzqs$tR4@b6m6ynkV zXtTS{RgjXo3R%Wng)CzxL6%*p%eLyWZMtlSE@QNl*S@aH7;R-4hF`@mNl|#?PnTuu zGQKDv>m8xX7>i}uJYBX(mo3(1%o|JG#k%Y+U3Q-?dqS5zt;;YrsB8NvD$N+8%SPz3 zaTt?i9TOClW=z#(Gj-X`$R$hMZHhvBkq?$-*@{Xtiga1AE<071&DUj1b=f((EDzSJ zNM+t>sG^Xi230P~PE=HyaU&>M?=6Z-Gw#!6f6`@7=(4AE*=xG&EnW5xUB)|fa>lE* zx{TRHS@thbSD`%J_)<}6#))mrvXd2+W(0LvjV^lx)N*m{c15KboskcgW!)8(W{lKj zV|Cf@K%FJ7y;xDm#_F=&x{R4-iDRajRGQHZ8DCj;w4#u;)n(Il+4;KcLS6QIUB;ZV zwp4J^MC(H}D? zm8m9`W~}5%QMN`=X~vzpj5h=lCo&8I_pUAzjTdEnnM>j#%#6ereu*&pyA>kMd>XS| zVskLFo0XZeQdXWEY-p4zD?1x=HH89}`W4R3!0uKvUqGj8@=htjg0gxJ>hu$3KT(ym zQq1coc4k_n%!J&5Fw<&;dHFzrRGPtCE?KrpQEA34y6kpc)&)A3I1aD!T0C4rn*Z_Q zmEYrb(P+pmUj2CA;%A=VGlSyM&lb0vY^L*{Enc%dlvBKVd*9;KAAVT8uJ}8oll$#p zX&X|Xp&DkG`|a3G>c!sp^AX0kB;zDa%?7ngl$pJqt8c8m{WPF>p2ux5Zoscp#e&)~ zdBfD#9~7w{C{}OfnLq420Ivko4~o$&^vem$LxD!jt_rh?Q6>BmTd9$aUvDuNXow=5 z$%)8V@O!7Rs4XWYBk_pze(95tEJ;dDX`PXLl$nz(1EM3UVRLcUF#*yo_VGt$G+)_* z4p`tMg<@B`m+cQPG2@~$EHR^t=i*9g7GXzfHo~13 zMsTGKs~|WAYJ=A2cE5ppV;uj4Bb)eV97p4b=+In%Bkt*&C#laRIQ|8nr>f7ha3p>J zj=$(`Kq}2xq03e)Dh*k36L+O9+oQ|&=`y}=Bl)qDNPfe0*=SwHA4Vub|4uiyDk@F< zP=mboV~rEtYJFk2D?N^`gqOCS6`cb|_M5Dj-XyyYUs#e|X9TF|4zmJPEVjhWu|ToI z@JNfv&5WC>!}##cZ}g59lLa_Z2WQ~OgjEHOY-#|3u+()`>mo7sbR5;#fNn8L_4FpU#f{IErey?#?XEnZLZyPiukO3?Amv2r)AGhSrS|3@ zP3e}HDi(StrllliB;ze|`EDOOoa-nz9SxGcaSXyn8)50`mgPCG z6|4+tsk;aiBZOSTZC1;#VDZ+^DZZtj4~0N-IHR60oKa64;TU?U7t07ozbY1j$Un3L z{R$VuR&sj=eU~vK?H%7B zhRosoL~ZZ;yYla9Ba~!2&k764P!@gg6cjd1+wpV$)G)JhtHnKeU4NgW0 zmZ!8`zqxkDbDmtHmzGuXVY}Gh1Z8)2)qFhzSGRu{Qpi@CW;3K7B7iPNbSVNTKZ;(u zGA$oB>})?lPF$D3*3M?Bml_D1iM-)4&ij-WdTih^h2ui_?U+C`4iWkNGjpG^BNuZV zkXz}KZJfDt4UrJSdCU4cWseCUX?JI0 zMAddjD$KFhQ*4Q&!rnT=t~Bf(!#Lh*Sc7Z#e2Dj_TL z-7u#f>kF5%GyC2%;Ii+iG;>QSlRhb*M{WsbRJqEiu z2M2S{5X=g59O!m@ZgCV7aTmR@MloT(H<^wa!^c=x7eo5qDRB8+aMr2RB(_VfNunN&ivDbi^e%8MegKzFNFpzr<(E9Tp0F&eI zSHxfS4M4ZjOG`t~9cRPPbB7vwenCUe;@xTJISQSaGxVec9W?aJTYVpLxgg0h-K>o>*V$Zi5VXAD{@X(YB1m@4;Ce<-UcQsD6`5WO0M_R8>$ zL5o1?RsuBqRAW#W={aN2WuSB`0T|plV^COLIb+c4K&dh4&p@dKJ6k7tt&usDqmy;X zzl5*;ct-b`y+Fw2_ndM}5{~ue2JLNb&(u;_OcFt zZUDmY_o9bFpE;2Si6{4|gXR8sw41>a{#Rm>$>sg;0rR{jD1US^fuE6aIl+0FuX0Ru zz5^vLT51lIxNeuYP7K5~4?`w7x+Xj)YhEY*oOzGGV?nWi55na1T<>BSmS%?*d*_9U zyc@8noi9t%Q_b9JXgTt+-dashw<6p~3A;Sw$06k``)LBfeF!E9^nJ-2_aX4#lbeM3 z<{Syj(vzeVICa2FElX)c%gYZr2TEMN6*Jz#vh?_lTb3T*s%7b^zJfJqEa)stV|*4= zysBmC*+EDROJVce(m>MLm0lZ+S-fY(u*5lGkr#|k?Mlb-aWI&(9=$sich;lZ)7wbC zE@e5+!()gxd=-h*nTB2)_2Eun^XHj#U$rsi-Mf~T<{jWxGn zD;e9)Fsbo53)0t9ac4ogEfsebq?f1S&gSwG;z8y_O6TDEib!}#asg3!LP`HfscrQr ziV0?7OA|3vN>t+0y#|c^H$j#0hL^U09!$-MD!#Q`2$A zq-CkNGbX)*cuQl_p1v_@&kpQGZ~!vfDX9)>DPK_RXOwQRv~@3n`W=yDXSZTZx>6|+{EDp`lk!ANNE1*=l}dL@V^S^& zXNSr+O0yRMrt@&MYfQSsFEff$F$QUI6_*xxv%nws#1mGd|6xPY4ai2fLSe3Dkt?S? zIZSc51H&!D9kC{0PK6`G7S%c&EjawZU&6YSJ$UkN1oMIL`lK&MztIJKA0dNmMh9HNEQRz^)V{l&r_guL2w+b#;huC_!c>UN- za7V*^04|M7AB0Q)Prw}k_bIp|;XVVG@%<4l{c$in4EM0yN5}2OCc-@&E`~0_$v17o z@9dQ2@DX6VAi@MLw;aB*{L3>=yKNr>Rtmj3x1ome_c8dNea~Il-(ct8von+m%TvDR zG!#`BzY14o>g>kvscY(Y=e`Gt7yrL~4}xr-a*g{1hlRqVuj=1$--FerWrL-Yek1IA zklcV<+tQ%&=u+7Y;Y;p=a4f7_5k;EK5S)GM24s}ugm2#(94COa7$<1pO7SK*P9zz< zvc_JJ9hZ$CD*cpahV%TMd-C!7Z|JQUmAZi*ky;1BB zhP`CiXNHjzQh!*V=E9QeQS1ctIU3)|j>1+7hRrkVYzXV>?;J;ky-N(c%rMS?s=v1! z74~TAs@P0NLAz$y8pEzKjQVhf2zz%McE4c@@Z>t^406RZk`#|`k(O4i(7?2t^)%?xIWX08Zn(NJVRM>mbuvZNG z)UYoMt3YRej)a}+C~Om9*e1hXHSBf6hUG#hEn!DF3X+UrKQruJ!+vAf%ZB~Mur6RG zG`<2yg}q6Jl^b?~VJ91Qg<;<}>{i2mW!RI3(fpOEg}uTq*xnGOnd=of3ifcI^qb$} zsIXUr=@iZ1Ku5JSj%8o9Ta9B`$;3F8oz`3UiHVGDxsZhJkJrh<9An~0Do*8&O;L5B z?Z}0;=5fYNFQ@97Ty-W)(yHe!5j7X*YHW*38g7Zqg2wEwEK%t%MoURi=_d>VbYsbU z1mJuEt<6%rCKOQWwHLg$JzdEDAEn~&PbT^DOe*v&!h!n~&BG3eUifm|p0pJk{AAu; zo!4ZIjy9ejimuuoTK0Z4ekygu55tbX@nuTv$K(LTRdPDd3c9+toW=cHE;Tb1@z#Oc|!*&cFD5 zSdZC6DjW#Kc#LIzHQYkDKZMJ-f{rjY9PagSOX1!CcPiYQ;KBsWgBo2}o_SW^NrkuP9yV;VVcQM+hhaF1&BeF3qrx6LiyQD_xgK@iq_8H%{IS+* zUMT)*UYZQ!o+P8Oz4iFt1&vhxfb@<`=^@d(%f%AAth>AMjZ`LAt*C^h#j=_-mTBxQ zkI9!VuA&B}NXgZ{t5in!{J42@=BkH8U#Z*J`{^aSg;5?9q#YllKyM~VkjcfZ=P->n zVDwNNh=%XQ`(O`O=y@?fqJ-f4_Qm~yaQ}(^K$%=pVeD4yV`n)^&oNMc-!beR^Y^}C z91urm>}<(T_xWxs`FRWfHGdz^GGOg2RF>;X}OSK>tPI^1Z2>&w#s+!;gkFCrK>9 zlh06NT^=d5_EBsJd@086O)++FibW8wV)>2=)8@nf48s7n^%?kfdO42ltNjcd1-~e4 z!K6(`uJ!8J(xi1LFg<0FU#!Dl%V*b^gA{hHBcHI;eZukX6VA0yNJ>AUV*G@P@e?Y> zPpB9^m%}z27DWlDzY0ewm)7edC{?tE$z4t_W~_RZy8qm5aEa9)e7q#6HWv~dE? zOE|A68a+&Ggb#`~MiVOp0*h^MoWttxF_FdJd;I@ny2S&q1kKGd1)T==ZWb?)ymb zKEajV4*b@XE{xnW$2GbaqYKqb+m3F?*@0<$*-W>^FXO+btKp0qUNvjhsA1J3YpX}js;wD5a@fdOBge@yLN&v@w;Nki8Z*qk!UeN7 z^_5JAL!ZMVCy(45{V_Gm6H9{{yd%v;9LC zV;1&SaE-UaXpvp@^iT*iFP;0I3W_N&hRc&x%a+Vp>YuDi(^S3CMqZ5 z``(yQNlYAf$48EYW4C8^!|Y*pc5qJlI>a2wjy;+>Yj`bj1c_whoF1J44Jt~11r{%I z)Z7-v#*`G;?5ibbQy>)fMAw}y1sJsg?3rXVzy)ZTb72XPlX(!=uai$A;%zAlCMJusTV-jrCysp3LUO;<2X`U?O|m&i*vWS>p;)R>`o;uaY30(w2F! zU}eo%cI*P*gr0JVUy?2S<;H%T&L8~;=ZoK9Ne#;#gtc-R*t@e41x`lU^~1&QOpW_1hVvzF!t!t zst`QZYy%%<#-57|z@ojQgVrEiJdqiD8J{dClHIG*9+B)p{#$ZE0Snj}OH!_5C1%cF zkSq*pd*#H-;%R`?tE2H+0}zlRUY|goxfCHY_FBqPiXna$+FVM(3PQn2I%3GZ%p8|d z{72F~7%BA!YK0$IO5t;L6w3IHdY{OlUxal1th|nbYFjYA0IY#@U3jO{@MH62)4dVD zpW?^b#`oo6ZFBLPhu<0aHR8u_$M(t(#Y5bh@uMgYgCQ>iKMD;z*zK42{Te^EM}9ii zwi0|NHbC|uDSI6qGF+tdQbfl7+8Lhn%;zW|H~DP+F?VBTkIDWPYzq2Y`lJlb{*$Le z`~7D^LSKs~{>8Cs6n?Ph^;SFZ;5;wJaqJMdgW-;ZI}Gk9xRr3nz-5m;4(_;Ie}8Ii-z{Fs7=UM=C7#JBl$D#kQHh z*9^-+9O^IPsIb?|FfJI@u$-_~f8z|}*rnJhhFxITMTULHu@CAS zGVBw>&IiA*VOKdS>~XJ1#h?=rD(ro3SO}5=pN6@^lk+#oun~q~KNRPW(;6BdP781t zr!^E?U>MjMhh;em=DBe*M>3N0$t%9Xe|Eh(n5pjUF{(Xzhq$vuj5UA6i>mtekRU!wrt3 zt#ZtLWauKt+z0Q0RpO!lbckc#Pr|X+*-yenz>6x!d=7eWs4!Y^%>6KRIS9b5l9y#0scyO3JiC{icM?1Fzmuj=3A&u1${l7ZPQHW7h1g z#6}2vV+2lc%zKh!&Wzm_$?TllnM`v|kY{sD^A?WbL00G*jKD5bB&V}K(g*n{<(ucU z$~VuF{I%qpHJ^L7-L2G<7+61nDV|`LOzWI2+Dg-Nw z z{UW66fAgK^o7qCYO1_!>$Zp}A+3P0r%{nEP%r}3C!czfNN>Rowa?R*)y-OAHf?V?; zVC0%veaSU*{)${PYdg8-JK>URzQ^6)50~8YZ{7XlaLGMC1DD)!GhA}dJaL)a^L}v2 zJ@X^_-1EON&MNmz3T77wqu48s!s*P0QA4l(s2Wy(JscJG$ek<36x82%!;UcQbi-yD zc8Os$=h3h~Fzi~xHXHUk!`?NFXIN?2&kYNK7gmh>tCPz0))|I*LWgmgjrz+4kE{Oj z9Hsl;DaQTp)Zaw&hx57|Hpj3t4O?#5xrTKGKdbR|a}+pq!v+~P$*^+6wi@=hVecCD zfnlE;76P}cc|O5Wx!%c!)fhI|m)D9=NMd!Fa0u=gA&eYY1K74}M} zoyZe|Wi@d8to($k8T#YN@wdtcLkSCI^TT-;Ov0BV2R)U_W*jeDR5-cu&^^oJPwuSN zd3r^|?4p?+Q9ZNaw9ilT#Dc}bRDV1Fn8rg#I<8u@&Xc3@{czT-aM+XMo66&VK!)Xx zy66d;v-ZalG`CSkG=8*flo1{L5)|aRQ3k@lJFJ#<9gH zDK4NZghumQJ3@CXEfe%`TQ)6c~uK2_kip!ZM5(60zxg@hh} z&y6_y&S&U1N-Q~IEHV>x`wYD*-%r3Rhz83$C3G}&1Ps>?;Y|?0-W5!PMhaUcjX3;F zDwu-%fpX73JqltmD27icFjeVK-jX5@cU3G|*{WEw zNa8sj%PMdfF)9S(qrP0Tu>>Dzd`r<}Ws+zT+~Y=6`my?*l1(m@EObCNQT|64PA2mB zE6L*$oT$-?UjJtje?mS9mT(&R6MB8k zZqL@_laUha(9nPo6tMw0SM~Z;K64#hHzxAQ0ril5&upR8ug?(SBuA9`NBOVVic+7g zDOmAX%laWd=yP;qJGKjUTAb13(j2|_+fo{(etm9=aDvm#Ipm)iP{S69)t~t~qBpo8`v znEsZ+Jpt~42>%(FXcf^&g+0pTihbZHDEtjWJCLVg5jWM}L57WURIW$COyiqt*nGoQ z8@9%<4TjxmSd(F!413eCcMSWFVfY{}-GPqE^$s))x*7Lw(+!(%*kZ%3FpSe5`W`>d)cs04g12daHb!&tE0l+A%+b%tjw@UhOIR0TZXMO>`KG_V;HAX z^ga4JD%U%}u;GS{HtbZxW*Ro%u*HUDLRiy!(AiO7I1KA;*cijc8Frsx4;uCt!~SO2 zKMeb)VLTdC(>>Zzn5Z#qhGF%FU1-=k!>%-JKo0DD@XPfEISRWHg3>hZag>UhdRXAV zik710Cjn8Dh9DF*Aw^NtoLw|`a$^xh&Bu3E)SOv?SPDBLYKjqKyHht4^B$aEv|{q$ ze^bVs9N$vWPz#Z>=x|KplsBFps)*MWEvAJ%OqF2rbV`!&xw(wOr%Hbn@rgxKk812O z_UOjQus6|1K+Fx8Mx~QPQ5FeFbU;E94Mtnu*kfcxd}AU~v!a?=0eQ5LLg{qMsB=Xq zr72bWLMgjU6+1iQv#3?oUZ51C_DIE38UiRrJ(BvPc&gY{pj1$$&L~+>?Ttl4Pp1f~ zKZY-M`g9vHWa;%^u)azaRA*rxX?E49*~5m8su@;QTRgIQ1e_7YH8s`6qi2mcWLyi` zv?ag;o!ha|N|ji@9a`LRiS-6ZtjDBCtgm^##M+1rjw!J=w<59L5Rh2^C~PFBpf*Yg z=O;<5{eYR|iCsL&8YZToRAQy5rV{Jw&{9Z}tx2quIH}*j4rcJvw!}()!O|VdL$uU? zv4rQJO1LXP1w=^7bxXNElPqgO!u@XOH>h6BJGY$qKJvjCmewR*6#{X96TNwtgdX+o zm#}1H{jdQM)R%PqV6RHBI2f3dBpc+UX}th81rp&@iHgFzn&xztdkX!JO1N&p*GZCy z8KzYH{LsabHai`Yi2sNK=Jp29*Q!KJ^%K+6w+abU0<8*|oB>IK(XL6v>%1}El&`r& z{F3Bx9ky86DT!E1I4Ci=8N08iM6A1AClp2U;g3O;Q;E1XGhsKV1`noc4flp+yN_p{KhnH$?N+4Z1AteHCn67g~E zNW{Y>WI`hLI}%@fq#G1ek4Nv58T&{C)oxvb`UVj^v+Hvsd+d=atBNsd*I0TMJCn{3MB35{lGgd&*?FTT`i*$=YE3b`r7e1mu_Omy?K@%2#Ez$HLxy5-~^J zWQmxgs7byUMO%}IiE~8b=x6>h`iZ2>Im`9<`4{;j3d$%ak5dV}zKo#!6v9zZX8aVC z+4EB{=4JpCcvkG!!oqfTWRM;bLtr#b`73&25 ziiI5&_DT#JX&5)TQh(HaY1p$2TV`0JVe1UL*RbCh_NZY`81{i-9~qVnaZ}SpJ9Fs{ zH*B_>)OZ`f^y{o1hS413WqoaX7`d&jW<7{)b6S}qYs<$C#s z6&uEr!qi`>VVo&dtlBWnlq$B=FwT@J_5;JNHS89{eredNhP`gsr-prDSPjHs#*A(M z9fj3@hJDAdYYn@>utyAg)UZDo_L5=qA^K_>iyeh+8w^`x7-vt_-%kze2QgQ%{T&61 zL&GS?@M+lNd1&g7n?xwK!LU0GYcgz;VQ(Auo?)LF#uGg>zRMv>UxQz+cZH)c{|(tm z(;elgote|6^!63c={|ujp#3>r2-+Q;)P1BZ{#?qW?!jMfQa2bACUp_Z?wHi2c&{=r z`@xNJOEb*}o;r%0IxDZ%UDs&w($>%5)pPN?Ez)kXbdg zvx-NJsG40eboj`clG@^7L#sxV6qgi_96G!ftCdpcaXYenDOEH+A#`NNMdOtajr*sF z#+yA~G(OKw=6yv$%JUU$4-mJC*XUiXgniS;U;sFL|drAebIIxzMoz@ zGkJSv=#&mm-d2Sk!{lvoC^31(fDGC((e*ZLj85ahRz=s>BzK9)T$()X{>j{>nB@JsPv&lrJTBeY$y_brAZ!1% z7F~ICjMcl~A}tkN4-4(T)1oVml1Vd}n;8&YJ2IIYtaz=8u3t!PO-$x$yGSRxMp8uA zotw<1Szu6f)y#EdGPkFMOo*<2%k`y6xA2{65f+>kvScBKalsj5Vk%#y@hr<>_ld6@ z89OGvCW)!7PUw;+*&P$Q9E2#R{uM6eRCe=}Qz=nW{$xYqeJ0Z5{ef^Pw;lwSax2F= z%B|IKDYvp3(_bB2UvB*boVHHrk_vl&N1Tei<*2Yn`B6fyhLsw|30?Ka30)0~y{H_< z^=yj84a3IO4&!<@4a?2&)!%Oo+i2L6hCOT8hlYJ@SO%m)jjxlVumz!ELkt^j*kOj9 zW!N&q8VqYR>|ciMFl=8)h?;I6M}@s1h8<$q@rF$|Y_?%@4g1WnX2ZHelGHSMIx6f9 zHEe`ow;OhsVJ{f=vSDu+w%xG9QCl^Q362VTQw*DC*h<5`Wmtn@jfVAwjBz-AxgMu> zNrk-)_}4TzwMzItcl7$^F!kX=BLQvCx^Tk$y4AGp8=P~9q-@`wJ#tQ#&+sH zHp@{srp&OL4GWgZcYwBaOc|FF>!oEih4$>*anfYP_c4F0h;hw3G z)2r=bS%)!6i-i%OOJ@WLqnq~11^s1=$sjo?7iiz4Tt2XJA*K7%DOLmDim@~l`?+Dk za-qTVtTyV(U9FajY^Q;Jxp`a$cWiPe4Nc>zLpX1D<>DH5PFdkNY$Zg-@E6^7vGf>0 zp!BFG{jz2Exs@HMu<4gbBDkre zdb0;F_>^-iU^h~|y69$RruiWjQX#exT0Xt`k!bVgva$cjyR-;f6Gh|SEm{LF@zq6Z z;Z(ql!JU}-^P=@|%bGV$9J?*=(m&#isFJ2APXD0+a^655`Lt?Nsht0FS{Y6NI&C94 z9dG+9-^Kr3F!#&yehBJ%q1J=?rYO*E1@#S~ehun6d3v8c-DFR>alsw(^jUk#srTLz z_Ek_1$kTU0X&PkkG>v9Zdy2n3kfnRYUsq82v==CSS_n$xV?+D^KQaduDUFm9B!lsc z?wJWCUG1znH47Gu8B@Jz!Q!eVHI=iL)z;Q5S(?}aP;E(Q&+!=c#tC^#YHCx)fx~3B zmgYByA%33tSmOQDVIi4KLM_FfB|i`aRt!GaF=}i$i@Z*}a&|?p@8MXT*CYwfr7jk* ziv;EO4I^kBTv+u=ICew>#Lu+QQQme5H`n6QC6qjhLgNWbJggvzcy!;Oc@}1j#7iCM z7>Tz!bX+KmvDi;{KKEfsZ$Gjg0s5H71=vqU9Hdy2EQf0TZc6-Z?}cAn#KLUDb4lg|M82*QC%f z-gPLa@dbsr^C#H-^1!Af;(tEE)5sOlN@#W{%gPJVj6BCaIo>pRepIL^q^3A~;ypBt ziBOmp;SiW{k_I*xu0f@$Sy-*x>~e@$CSlK)xOi}qhAR#gLET4fMWH;es;0wU*b2#Cu}LT*EvYM@wSd?{ri2zCl7a$ubb7xKC0kIi-tJM8`vYLLa{*eb zu}o?Myp`fNB_uVItQ-eT=D@$$x4eiOZpLt)1JUQmwl72A@*|^C#ZHBLFkF7mVz}$z z9s>6-a3TN8LR2(9IU{5&Tv{hH9EE4bvm9<2Tv}_7hx_xrr2e>&LH+U6Va2$`g<@Pwr`QC; zrWiKOFe(bv-wMO-Fzjx_{%qK*hUI4XVYzd?rrX=FeukA8Hqx*uhD|eUkzq>>TV>eA zhTUq|uME52FpBJ&mu-eY)8?@E4f~g2CEzf%3`ROC?451cIfkvk_^1BPbChn=vP;a? z5EN$`W-9}5dG#YaZ*uU5(Z*pHxKka=tM?fZtxnHtvBmvYcrC|gy@~kAx%n$kTXI>^ zcn8_6lfh=K+MFeP)&U*ov;NXyKI;l(Rr#zRfKndlI#9~x+yYAZtUEy|pLMT2-Dppr z0Hu7^Gxqe4pp-BAD=6i&STdRhIWA3O2PoyF_$`&sA`7igyMfZDdxJ{mv#_Ho`k!QB z=qmUkpGEf!^xJyHTH~|kEvy@-TO49U?Fe^uJh-d(WZ2|RXNx8J_t0xznZ#YqlGxI4 zSL?uC{Y3`U1a}qb0C#mvXf!s?pb=}3yW;7B9B+K?0TXNDr*&@X^w8x054ov+_?YBb zzJV=hXp_qIXv(pah9-ltP3ESm@P_^wlBwL(8KKHHxv5E^-YvMP!@*7U7H*0|Sdg3Y zt$r0R3cZbr;M>e<{sxEod2b6xrG6YoMNgCQQp+X6KA|^UKps4)0S^lefW34YuIl*? za8=Jsw5@Vgdv+j-J*CKZH&^9v-b=2^=cw3Lk~u0x{wFvpRYU}ND)u8g!&C8#lBZ&` zBv18cxa6t!LpU6l|sc2YN2A6!sc(^oU^LeV*ji(|N#wmZEr`Q>e z!Z>W$Ifh+g*ky)MUe>Ve$2C4qt|-QSTru{~ihW=hDyPG;9TkSLg^y9>(Xhh}8*LaA ztS&6ySp8jX7&%wPHW{|nFdFD-UTC1F{%D}57!C9k+sm+h4WrSX`lHdF`a8j}lMS0^ z*aE}29f^j$(y*Hh`CIPb&F%V%`CzS5?H{h8tZ26;@gNt!Vt|NmNe%QOE}KKP@xV7-Rz zhrh9T12_LO@6;`Mr)(|DJO0tyyi*=3%R9I!@1kqrw>73px6NqD1|_(l6fKl;Q2t@g zl0ToR${YB5pG*mqH_&bE;cp@hwbfR>LGz}HvD=ncHE)To+L$%=^SnzB1}6m#)dpr6 zimUZ-CuTlS)C9MzdGo}v@8@0mu5nN|1Er>FYTm%}ib^Nu4V+k1jDPCER&AQuQg!v% zX$bK9e5$S(RGF(^fl|hYjFhrA6mOKdVgpp>>i71PosK?z6_hepZ`jj+fKq1YV^GRm zaj~hUK}Jf`=mM&jD6u$^ugnz%W_`N9Jv|syGIPacj1yCst6Sk~G*Y2EuYKmKcK)I& z$6oC~d8A{n#>;p~?bCR~i+O?2IB%Q7Fy$cFHt#_1`@$ow$2}DZ8k-v+bXmOkgqDjV zD-ExP39vkLMyOMOJxVC8(6{0m6B>%)Ihn0GRFY!d@Yx8mRg4z{70<53H6KcuPztMc zc*m4fO$fdVN-9?94l1dhg4b4+RJ82s4-cMqQRv(xmPlD9PLNZc_q_A5G%d%{m)nxZhdBy7J0<@*PVpS}~uMOoD`;-^qr7M2t;m3>0tG;ki5n~6y z9RimXjEvKnaH+&%eJF-|6I?P}|AadNE>{zf(V764{%68vcrsXJaL&|-#Pk5I@#HrV42MR$$eV0`-sj+uIOMw=g3@H*PNEt++U zbvY^I`(dx)-zB>|Q_NR)BU7@SIc#jjrox?hwU;j$%kGal2oWlKn3 zZ4FvX?6Qoh!oaukityvLDASA4T@06HzXUGFp8r^LB!#mR4WnL4u_p|B+OQ7{BRi}9 zRv=A{k0w`&Jxm@=t3w3f{FLgzEc(~4did0rtqwf+ZOI}vc%f}ayI((X)=`^ZzhfsI z70_zBgJ0C2S{UfRGF+AgKO34N#vi?)`i+rV;Xc=g_PI!fp%Q~QE!fqL3S*-*A8RtK z5b>(NB1d5ji%h@EoI4otX5Cbwp^jTZU>k%`w0de*G(It08jY{W;ZZg?gRnDpVQ8M3 zvo_jTQ8e&P#Ew~L@AfdHj8pU_7=*oQtu+-$thLpNA!nl%v?C}>vu6S3G5EX>Q*S-fE;X3#=zl+%V$k0QKzv}8QQI0?}N%bKDQriclXI4dlcLQ>r%AwNZuq?#mPOH zgQRJ^NfJW@)gw97Pr6B>!4<6UGL?ayvao`cJYEKvYV>M}GrXdBAFX*BozJKFXCyw3 z_xKfltxw+-gA3XlTu?SL#nkv-%m`nBrNC38rs5e(5}lYU$|V7r)tRHU~nbub-%?+vIB%*rqVxIRO3;RnMWGqDTceh!zK z8Z3&7v0gO8{QzzZzf)6AQI7!Q<&RW)=T1>?zXI()X3We(@He||(UPKKDMF2*#XnB& z<}0xmrdX(^aHPWCQp1)zD(tN?>|(?IW!MhG$c$-xM>;C(-D}uy413hDCk%VZus<30 zmSJpdn!hacQJO}Mqrx8dDmqu*gL@T`!i-eoC9G#};U`C0ep^~}(AY{;8YwN9t(zA+ z#1rx1_0krML40}DlG)BG?TD)Cnz~9@T-9LE$8gW4^ThzE?Ks6FrLA_-;`_tpm*x4>`ejM!fq;sg<(^_EAM`K`bS=o>Zdvov-ic#=T z?B|AUFf3R$*8weWqy7wPrEJol8YUIU9eMcWE0AO8tOD&y8TGX?BBjURDaNi*G1ec& zSbr1?mJxf%)MHYE^@qp&>815&!J@Nk(zD#IiS70h(q6s2Ha5()vEi(j!pkVO*iqWX6k{2wzhD`W$6ek=J-WQrGTK#pPgkbS;PcDZI%n`{wQkqS^dDBH zq{7}cC?CbvJ1PtlHXkD^sQxPAuO!=d&Me6Y5WY!K zdGqGU%)~>{vG`PZ7j(ww-mKAtSo z9CVRc9w+A>L9|)k1)x+5e<`Sk1zTqrt8`BZ@e@!wH*l*xz0009f%>h4eZ-!!yC4IU zB__l=BS7uZU(r~yvcG*{JjJh-*P!@)j)yYo*4OWfHn5gUm!XKQ36wa<6Z8nTH8BFymQoDvD*M z*DWJo*fJVK>WaxMu?@-Dj}gyX$e4q(4P)j0M7%Z>jktadWr*%F_P*RN3%vE28ek*7MNbeGCQIE#e`Nj5iggg`($*>$U-FP&1#$AhwRw`5!Dt}EvS(h z5~f81l&Ns}-kfzg7BtqGX8h302<2pOpJ7j>sFt`Ov571ODNFL!)8bl&lGE~| zO?AG6$Eq6%MNztkI>d&7_&YqQR%coP&WS!+CpT%lOs7RB4In(UuIxBy9*D z(^TCZA1X8RWJvZ|k*v;<@cBFulNMsgi#3zUQ+LK9YSZ1@WLl zfu|uxvZ@iCFQWTyFi#IMPd)1UMS9GnxZi&mt3c)Cs)h3wOPR9kp+cL}2t8mm&p3kN zJ2UZ$`u8H)XY~*0DjEQ1*1y*&oEh%aDRT%`0e8ajQ=Rf~dS-A!N3xz%{~luv;=14S z1uzasPEEG+9#sEXBpZuKby+!bJ7LkXg^42iOma(=sh${6h^~qfe#_m?ngfRu9qxbp zR(3n1u+uCt4?sC|k@+uet#1EsHgSHTh$J^1o(^x3gvWfp$KquUm(S`~58n-b5NRTj z`rrClV-umj5isbEAbrk#@%P;n_YK_pM%_WW^*OTqvI304a|`xGto}M}mK7^nKl}#b zHx$1r{8)Y1dgkEA14DUO=$ZJj_Fax&@UYMjnuC8pD7D2r9F%W$AAW4TYzS^1IM5#AsvI%7@W0ZN{Hi0K>_6k zy`LaaG}%4+dMfSN;j;gp5|aL#>a5{X+c31FN%H>6z`f`eC>macpMP;oei1k9ILNc` zn&x@2li{8O_a(Tez{7$l8phdKjgK6sru(vCe=+RehJ9w( zH&I_TY;Q+}J&H_ z2u(Sfq7LAiC#nPJE(G(WS6X#|Z8g`fLZDka3V}T{Hhafl#5)ZvXhI>7iPzz@N@6MQ z&MO3}L(fATJ)up7z&CeNAy5s0_HFE2(UwA>U&cN=rw|Bdc)L>}kO76j``*Ve#oSqi z0H*{|jZzc>hlL7Z!}Tp#0B(Xp;BzlkA#hF`3W2v%6atCzBgpp|1U|c?9xcVLd#C)LLcqy6DhEL!@Qp~1 z(^^#sd_&6kEXcU!A_b;V2)vS3A@E8E6#}nNwhYRjP9cD`m}wOPZ?&Zm;3PD+0P__B zUC~*kR|vejlL~=%CFXV%0-uVfWQBmAg)|C*`j2-}A@GHym#h%*(`#EHfIbS>PALR_ z@xMtMa38R5391A*F7Jv;fSoHv{qb6+3C{Xb5PBW~^u(J#!m#OJqX4rQO`-x$<81^T_{%Y7KhJ9`r*M@0c z`Zx;Pz8OXiRnwhp*s+Fjm6(R*DlzrPP2LszmSNW!_G80-W!N2t{noIJhCOQ-Peju+ z_JiQBY4mke*y9H5irrvXcL@24^>kF&d&n?un6B|XZ5TI9SBz(mX;_{;rdX+A6Ae4w zu<3?vFzimlHW{|nu(u6+&#-*({lg%b=XyOH74{lHX1$~>@C7o4bF+T~2kL>>o&A8b9r!gs zss?62HIT3ysP2f4pl!2(W;d&U$98BPUWrzBL@|I}6a4xA6jO=j%`g=G&p(~YZvvGvRYI$PV!-LG13G2= z*n7wO^h+oMn&h1eO&Or?dsv8jv7hTr!Wni4q)`TZJFPO{W8@EGhbaS6%Z|!`I`QKx z185!4U2>4H4ycj)RO^5np$zzuSqJzrcgI#>XF-wh-&F>5Yu7rUTiezF-FB~aK(~%q z2lS2f!0EoNSO@f#GOmR(fQyQf#dBvJuq~}JU|R>30oz(p2JD}jnzIggw=HFWitGtx zKsR(x>6HN=?4&Z_11X1glmRN7Co2Q|ETmHge7cLu0IU zt5F7=4eNk?@K#@ibwCFd0f&rg-7-LTOWB=@fMHV8(CH6T%K#bx^5T73%K(nSJEI0* zmq>A+t6M4FvsyZlXUa-C0>NypcQ&WBBm>Eirb}*rSH!L%7#8dN?ZVtu^eshGj#LSATmrD(u~1*xiQFHbDJtF{}%OamBc3 zj+E|FuNZf!SL_J$ca&i_81@sxxQD%lrTMppeZ{bChVg8(5fHv}y$o>pq{7~EP#X4J zN2!&7Vw40lzV+tsM#E+zhHv98*PHFAuy>1LzclPY!)PzS5ZukaLlyyxv1DMazhr=W zL(gonZs3FR_zqb&5RJd>uN%0t=qa=zt{-4OIk};F4SqM5o5%om{&^S6Ve?kgChM5SVHER`yU!~c`;)3QQNN@)%C!Ys0m>4B zX#7t}CjQtFsH80u|HpL+0XFUN&puC;+_#ix{yB;rhsJK^5`vPK%j3wc;aC34DS5Bi-VGE@CY(n~MjI$F|^!)dP{DX#JPpR5>2sh|M5h^` z=73$f3i=~JDMphx-C$@sC>8WMd}|t)gVGSR5YQ061f?OU2G9_{0i_{c0Hx_3h%Qi7 z15-h%)xZ)^$*O^4(RfQ!R0GT5YbbI}cYfQd0k=g_TIIkIhyr_xq$mf*+wy_&@_cy4 z(=huk4pn)3XM97f19W$ymev8KBDk}_N+o2I2%pD5830il*YeP-Ko;XZVcMTqE^wHH z86&xw4F99#ert*{V3fp3qjW#c44l8Z5cAK8ICZ%I^*6&Mj`0~|z{XSF=d_Dn!_kNO z%L`ya%7SJGl-h1CB>jCrVU(GS*|MK+k}&L9u24|EgbLe;HgVKpsYPNdtpt(*T>N&5R@GCPrdv7zr& zMPaz5`Tsmb$0xyGi`n`q^*bWjD}&;vn(yy`fSxPn`v=f`e-E1P9~>+>NqUd^9gIDQ zyV?5j#Dl4+fPMkuI5%z8jbJH6<-7D~zVD~kwt&8$gq)7{d<3Mve|-h~4ajej z>Hj?dbdN4yz;6Q54?o}ZpRAeQtLZEn*#T(%=B|C zWlZ__1&#k9j*Ia>yJL#ruep2ni!*Rf6LO03>^Nt_t#bH6hc9;bF?UaqkNzHVIJ@a; z+|%fr@$Z0J1D7H`cLAER3nIP_{$G`luQU8_LgdjIeo|q+o2S?uN9hh5igAYx^|#9W zU2GV+I`wy(VdUx*d&n?O^C32;LgsVUEJ~WQI*P>G4 zX_PrC>`gX|oVlhk*Rc794Mbe$<1W`b&{1JA<5!F`_oTw!8uNFlVd+J6?2wDYn@)_@ zzr$WMuyxdQ?3g>Zup<6J$-50d7dgD-nP_7)|4n%F)<)yP1xMq;U_&TgHi3^X6_4U2 zI$9nr>!}+AG&eP%RFJ||QAZ?s)_BU}$K*uEuI#n6do*5#74-Y;h+g<|-i5TV*`87I za%^RhSJz!*AG@O0(t>EbG~w$G#(v?)b=Z>N&u{$%H}@9fPkqN?5%*0+8vC^v^9_2=r2H|Zej8>`0tHp>)>*qExmLLlVwY5Dr>npknUqr&2#JY97K1%T87Lf zS5_=iX@+Yl_3CSaSYH8FhbX1&1MfN57_e4slG*(}FHdj9qSP)T-*DmbBZ9pM);=5j z!PW3P0sOp1t#V(5NUu?j@LpxP3VoC`CrQXzp{_`4-*3oMWbI7kWqbiyf)Vn3X^1R0 z^McS*msXvvhYeB8948*m2z`SQ?4R&ZIcFKROOkk=0yE=4e3#wap_{urN!8jkkT5<;W|lp06LkBR?C5J^@Y9_wlgf2 z66T`hyCrN`;)2j{j?1mcJs2WGR-{Kyf3vb0WUm&#)S~N# zqT?%{ERjwPecxLrk#-NfdT}=__KjT>!`>;U$d0bxMnumdr9a>oJP8>hlb2cFFL!^G z%wFN3oHUG4>{SmZAi{dWqVgP9ZC!As9Ij!O709y2U3Z|g5!r~A@D_6HUEtt3yPnB--`v# z7yccZ{A2jNgWtROy@%gF@PmfP%fv5>FM}Uno-fOn<8ESHs>^!89st{Xu?OI8b!=s9 zc?zA0Ed{Y;m#n?JB;XVr&j984~06w{}`2jhzK| zZ@6UsAezSN;r4~gezFMe2M9++=ZPI;>*X7_)uEy$UkTr?--VY`k`3FQ8**oFplCHmNUrek2*%hI9se3 z&4v}b%dqlI2V?%ZrwA@DL=JQ6_?vOo;p*aYZ z<&~a*V@?Y(QFUK29!BGh^rH;is!fHbZA6dZl<*1mdM$lodZSQxW=&?VH{fB1*lT*! zggU@pCp@>wUN<4m_StLJO=GVcadU4z{xmPjUNbqe*JT}GuXU^0^z3yq7p*?YMN9ng z$6S&vx#)Wm`4KL9E$-XqqF)ytwhaD$FVqR3l*c9)t=Mc(>9}Zpn#@HjwliGx(~|B) zv=Zf=+5VJwo()R#$KGE#byRfWo!5YT z+uTLiWx1wm0nSmaoL5(~1ZO2KX?s)MLjS1f4s+dSw#jvWCk@yATgP?BlDO_sct6a# zw#jvur{%g?KSoHYoYEfGeQaA?H|rsLOwapjxb8$b>>StqQ*hm@giGCJuA8r{T=z(D z-76&0AnW-%RJh23^i21j0XF(c_yf}omOI=z7(SIn4nErj_Ba}h$Favrm}8Hl|0VXA zJy6Sw?C~tH$KW&5vc|i@3iCs-ke|ZOKmRm^EoL)FWsCWJ+hL2}MCU@S_vupe6KwH^ z2pW7(_Cx;-o|w`ld1AIa^2F2Ok|(Z$OP+WxT=K;1|9qbKPRZL55H}Ln&hUb}siWL! zZkqgjy1W+x2gA}KUi!b0_l|Jkt>yiHy4*X$y&s6O9V~odv7_|(1jRUULMrUtW&Z9n z>^Z~8MyfxuDjJqNxnfzUpNf$?SFEpL0}LB)*b#vfnOP4@ZT)$3gv2u%{d)ygPd}vQi^ihta&I47B79|A9kFI6Gl1XQE%0LPy2+$P7)8rD9? zPI(bq*-|0?r?+mxO{)s=lRM0@uYC}e$7{H=s0ofK#=GDK=fvbr%)4uIH2##Y>Rzjy z{AQ;QPjK=K?0bsFTXOPS@m$L+zA;+AiG@!c_-QjX4g{$jdNHWCgnOozUOA5=0YUxWI8R0=2mUHHQAjU&c9?_#LG^lFWhU%YIgJ1nPNmV6&k z)6yboABl5mD=hir^7NJ@mi&Ifd`0+GNh~=#D4LB^t=>`*K2}1Wkd`Iqhi0eCV^Y#8 z!jD1J!^FesZL#D{@S=>#h%}1uRbNsOex)RwtO!pnht^o~Ur5-5B7BqF{}wDcc6o(e zNDG#H*iI)NwzL+7p=&dpdI9#7K`J?P9p8ER-J{Ns<~#7c0_yKg!og~bT*AtHl5yDVbh<- zBWkbrFH|;tzO=~%n~sO6oI3lNbewu0_@(xm0~)p0!{B}acNN@bxNHK{Ub9M5d;KU} zYOkM$OYJo~7;@@tkmS^B;gVBd1ecsTjTy+Pvw!nB^=~2U$-AJv2Dh8k5$-h0jr7U6 zrpremaIhL3<+@wXM@QIaq|;hH*ikYc1Mo{+>9yExmdCITDbricN5mzL0WWZUiF^$2 zkV5^xpO21+26$7J|53tulJ_7L_7)ko)G%@q>W?i({r$+W>kYfjuwNVYgkfw=8g{#3 z|1gXOVwy&tqr%>Sh7}veHACu;Ylbwwn+)Upm16fAMxCr;e>LpyhJ9ezM}{2%nLyJx z%28qOLdXP)H8?8l9ZOzE%I^e6VZD}N!U;*4o@ZFh{0)OlaDjv!yMnj97O zCZd7dEm+i1DlyzE7$pW$VefeSUnAIbM~M`{PS@lGlOmdeXsQ&!y3p1PfXx-eFJwHc zwuPV~&tG*DZ@!2&PIg?e$R2%xDfb9tou_bPd0B~^XL%(}(FWMXG?euU9&|DVU*e7w z9cfHH~Qulh2tX zSKLz^V2-#HquxsWrT@B5BFxG*zAoQHujH?LcGZ0U+s<3Ccz*D+LQqJ2z_$I#H)<+u z^&7Qa;(qBeg4EI-<#J8OFI_ZuCk}uE!BIebwgWDT(fb50OZhXnEaje7%A~Ndk6|}B zN{_Qqe{UKV-GTyVlYMA&S&qkbeKNpY4R=CHG3U`EE@tmvs33Z|Vm`TQ$wF-qD9gl+ z6j&ed%z-ruiAtT<#)!u%>K7Oz2o_k&lYf}EUibwotYf{Cvhwex zxJI^CTwOX?T#O)CTpuDI!Pd+CwN+Sq;JIH|XIf#A(&M=lTLNE-v4|AA$FN`-T@Uo= zl*+`4#)}_}J&#^mnNoV#F=J8yDWyLZwo<7&dd?do2$qsos(~&ibeKo>M;OLY@_Id+ zsg>#va9NqUS^1C(dy5ULb5z)4si?mP3`_KGdHET61);qBqhK>xj?rvybi!GCuLmQ#bdk1jM>EqYujh4}z7@xPtrTap2(2qop_E zD-B8--F||X%MH_NuWN8g(r39(W!dv%oXg?!u3Q7SHix9{fI!CFiPV zcZ|yUh?1oROD#R-)W+peEG>7jK5$%S&1LQ6g)PcrbYF(c5#vvA2f+O^T>3f0YA~s= z_mpAJIVvneg8KW&Fchu}OC7m}`O)}C(N(|Ci;n#uns?*}(Z&-) z{sP(|(Z-|pk8WwAx9Hg4Me~l>$}tV&*jCzHOpdN<+CRE#+qP&!bO+3tFMNh--`|Os zipEKyD6E#_FTk?~YB&?Q+8AwE3PzH7!rot_qB#Dm6bow$D>sU#$0-Y|_H(^aw}W~J zV@sAt-RR%JcV|i2e}KCx+@-eHSAg?9>2^V7&`W#0U2JZb3FWAq>a>Q(G7?FI)u^fbyImqv@g6h9(kiAH5>{dPt;=e`` zk<8f5#Dj|}f5jY5sETASYq5TjnPV4QC7GMx72=is9f!4G!oU)CzIP*u%qlKn%p8LE z&B^Z6C6Ya+DBxolx~$AI&;v&@yXFqeokRg7r&H%h_9S0p$`ywEcKsyH%JDF!JyX9O zQ)X>iC{MaW8Egu3SW#D2FIrevHE&^!nqt%AYVmjyKCm8Jbd-J1E;8DbvzJVqZXZRE zbrPggi_ahYB)71DbI$WKOcUvJ-$A{_u1V)F^&3p*w+T;*be2V}!LK78h*^! zPx0exvXZi**W%}&TQmfnPYdiMtAlQEU%K zL3lT8FTaj?Ly%nI;AMIDwU+FZl zFx^rn4Lk~NVoRB@$mPlcYz}$l%}wKDhh})oKGrcE<2eTJ85rH8D={$t1I3d% zFpq^Vc=Cqh7iMLa>+3l%v)e4kJFp^?_4m6`xdqB?0ceeiL+8+qy8c}sa!ZR~XLJ=_ zHPU&nNMe5LY9K)$KSV0@5yrxF<)wpDb!2QKIb2PXgEjb(%)DT?Dhup03;l3y_eMj} z8|5OCOpgJ7jvv`)YU`K^bIM*ORa=$PQILCPN5S{!q6WObG0M~ZGJwnP%}yc*ZiT}y zMk8k(z-qx5Ie6Y97jn-D=my*uy8C|aekj~afU|Qr7=cyMMM^sd#n?G0w$1##W>~Xf znaGj)%R)2MuoxyBhC1)6|3iNDN4+;q1m~Aht94IA%?l%%7eVRi_4TdR3Er}264Ryi z%*!3jO4d@)%fwDx?W;W3VXA1Zs5gDt%&%PODi&)-Uh>qPF%^UxIwDE{g^f0dSUvd!E$U;tgm~qL|BUA{3ke0W4 z9iwer0YU8z2x>>}*q%?>E!V`d_zvbw^BAR!1_4bvcji;xV@NcfTOJS#d+|hO?yVF& zm3fH2uTr#OETy_AWj%;^r4;t2a+KiSNxqmT2|RuhvbIqKyN5uj*!Op%*kbkR2z(aZ zU+5H275I=@-b_%*qTm~dDYa?Vz!zB$x_f-ho2D-cjzoS@Aw%6^osvB2j1*Cj)-ElM zI+ZZcKl+kdu>iwE-9kOZMCR8n&|HafI*LQDK@wsS?p4b*HZ(Yt1v(*}d>6$Q$tuq6 zoI5G^z}zA%oTMTs_B^D;*hi7I{3pq69NeZ$?B@bRn_zAR!YVykTDfp{_Q;p^&rcGCoI$Wi6$Zi z2pbSW7rPau+*RovdBK(5e2=F;PLLCU2<yo7-HA0gXW-*)U1aE;ATRM8_jyKS?V%Qk?pUWM=}FR`irN zrbMl;(-UQo_%tFo#(t6;MM)oo63)#Iij^WX#(wFf#{Q5mDK)k&B;GD5w;?3*12Bt^ zh{vAtkrLTkfefFTPDrGzP8Og1pUUq38~Gn7@m3-MX2+WRI?|@3m>c_RI)5zoAPe;d z>RIOU+(8)FGCGALU7_hSDR^?#(lcvjOSU*Nep|9xi;-rKYkq!K!}4a%*lY^+Y~h>G z@MM?-IV98%7AnCHO#u$|;3~fo1kI#m1EkFQE|CEc8WJe=tVy{@{D93yI9HlIrNLr zcU!Q)Su^-32umx>s~{X~&$IDSaK`qbvdLCvCsu^T7*W7F1P<6{eLb8YGR&7^hoXH> zngh49unB2$(yC{|RwW6=_}+mcCZh$_TSg?3K~^h2m}}+dO-0oPKVMpaj$&@XK;{zP zjM@dhG9$PVKVQqh?@3Jq-#b;y;In4Fwt;;;d#B)O$@pL`G!B#xf*J>&piHd;n;GYs ze60ifPL2sFrza0T1Q{+UC`HBt%pVo-Q9x{9kSg>jZyu^V6(Gkif^!p+#$3Su#qpgi zB319FI`C?+fE@dO0Cxaf-gA)SJ@t;fFNXU=xWn8%$6nr#L0s3s<-Rz)Co5TuTB4E% zDP<)Ud(J&owS;2iaMT~w5{gkRp%~Q?icu}0*hs_18g`^%Y`*Gmn_;gR_AkSB7zQ&q zmj(>rT-bvRD>00VWHm2bB&%sWV;FbJQS5cY-ZrcPN(K!()lp$@nPDppqiK%%qiK%D zM-zU|n#j8g{s0R~Ytv z!+vboPYrv*u%`|ClVRj)^?j$K)@WW%brfdB4V!P+Pr=8&h+nREtE0l+e?VziFUOBh z)HS7UhnR_+F>eS_X@iYq3R(NSz(z4(#<^Dv*znRB8>zFK2}NC&w=XE&Nd+Y)^G`RU z`lOm9jK(wM4#!tONQQ1uFO_ctTcV5^6S$F~vI>%0U|y2y6d<_zxc*&ZBI+MNLDS~9 z(1T#;s#rAZ3^!8w{`!bNsu<|6A6)o5uBKK;IF3Ae^-0pM$le;?=o!iC73^qrca5I@ zxE9=8Bj`;w*U0J;gx*Gv8p-P7_wMYXOZ}9STq=O~Mi-4LAbV0UpuZ0VGsRAjog2%^ z%|dby3BDi80R5em0kN6D_`bm8eJLq_h+i3$7juK3@)>)ajV10FEl;W_y`UY!$UG*1rGGokppq?_8 z)OyMRH(fm)6-bKTYa2c&zbWe)ubZ>gdE~eS5Q`lZmbqQUwi(7% zZi?kNO6^l!Si?4%zpaKv8!C!&sQ)gu6<6p>r7HScaD|}lc}P0ewbX5_+EhGaBRf!C zjzi-RU5?{d)isDb)o5uXe1X664#AIh`g&Pa4VZCDoeSJc$PQuzI3H*y7WgLH#ixGQlvM+qR|tG&p2E!aNTo9gs7GYPh3k>^-g}ueF&4&HXur~~&vACvj4YI27 zt#?!yCvm|7S@J?HJE^dDvSFtg2J_=mMD!MZWO6(;KZf!h_F2(}V{=RgPv-gZlrez1 zUtSadZcG^ic6y_UF=KW*h&0b0g-Ojb(YNP{4sK6*Tsdu)v>`I|dgZq9MGK?rq(QJi z7`A5XCdo%)Xi{NW z4X)TdhCO5csLoM;Tm#_3ItpWXz)|2?xq^AfEUa^G!h$wH9AZuTp1|J>J=3?`ymhDeU|M zZ;GvP6xIhA_LO0^*(2C^<$Z9}mY0}Py&67Kd2^1DyvXCR^VCZRluGQeS5sF>`+Ag8 z;zr8qY4po!`H-$1tc{EyxvW@+7TLR#Qod2K`R=Lld6-Da;pZ2zMc;RR()<1f4`-y* z8y)UEm@RtVg+AlwuQ52X26im-sw-=jELjB0ma4j%Ig6IChNOQW<*)@DOWJbSvz>Cl zyA`kr&qFT!`oqYQ%j84&2)0M&|0)CvwnvEZF*G>ub+~+`H{h}?@3*of1;*Sk)+oiE zGwemf{Cb4)<@gpT-z7=qyAcmhYomNGNiE-U*Dbaz;8vbU3BT}VOW0=&Si=0)ekn7; zU@3oy_h_$-KXGOJnJZ(iF=QD(Y-LO;>|J2kMUK+76zY#wFX~VDMa~G8Ciq268BV7u zX8fg+XWoQQZ!PVQC*-V|fJwCpz1B=9gfpcS3ZMQPOa9-s>DF~O;IRQH>>87=VCweWV=7En22=7 zdCEE(pqfk@7{^zevOSR;W3Ejbc__ZUFfU+m0HE~EI}hoWj-J5Y>)$FJhd z2_s52(;8YLb)MzI;&gSM&0snp78AyP7L(87(x-pPH$ykWo@Jr~E5O3?E0n_>0GId2 z!<`KGWVln{*1$zPv88a2f%^g6EAXq>8J5?t3W-XQhRsc?BFg3_UZ6v4Zfki%$;jn% zg?i^!i?)-uLt0kf)V#IDuz5ZU{@=*k(64dcO1{Q<8}>EM+wiY(-bU=Eym6$RiIGP& ziloBWjm*bt9Tk?bO8uQ<*pJNL^@jbzu-go4z_9QkI=fu2(NQ=|9+bxSxTC^eDOmT_ z;%}m(aEQHOjfTAfipSUIdfOZo_WGkgRqOyqK?@4%XX5WXN9i7;ialajk1Xu3CH{Ij zD(szRSe0QLK;0w$?sQbx`^2!%4XXfecc1v1>Zq`HrD0ba#udf)i@$dq6_#}*iVerN zAVrDo6863(eluK_k49HLlp{|f@WM%p;FDghJ{f3FIK2@JGpf}mQG1e_&h(Tg&R(?v zTk^jYjb~I;mu8j6EApe!3m>iP6pe?Pr(!vZGf+jb$@~Uow?Eaw`8R+aZO9yNjo}rn6X6E)=_)M;^Y>Z$=bUJ z?#C5$D-lKt zP`Cs~7vNT|Px?~c)%k{~!|1TwSX)_khZ4Y54UU2Lv*7OG?)j3laZhDw4cvZkYvD5d z0I8YG5UKG0WA9x6qbjcd@tbTClFh;O0Y#-&4FQ6nyh@TH zR&gPKq$@sJTWx*Rs#U9P)jm+GwI~W&ZL784s?}Ee_E{gbR%?~t=X1`??Cjl52!i(e z{r|nNnYo`kGk5OHnKNh3oS6wOci5Elu4%WQBgGNP) zamGloniEdcO}98i&pan`DP?`EvZ`-CyI&a&jI2-1he_XCB)X0|*tg+NVp*gQ|O`XT8<1F4tSd*Kj6zyKAoSHgDBcZM>|gVFNMPGGelDMkNLz zt+39+C}LXtz&bNknBcc@y%>fUCtxBGcwz%}3c;4T$||hnDjo4iq9dB;ap2(crV#C@y`_YdPw0nSLU=7XD@@7M6@s zF$!(5uKad9r(xU#Nv?6yvSo9Y%+)g<(qpONk{qpJ z<*1fX$!)MniXP^hDxHZ3MTL=7C8c$IwvR2SH=v8?jY=4c=g{ufFJ91E*VM|Th)TD; z<;B;@i+z(g&VIopZkxYBFV{dt@NO*i>yf?G&z66Qq&Ng`P1mH-OLChIzSFqcU#)U` zbl6WG?8W8E(8xf%w2)?zrKivrjU_MaIo1or?HyF^KVRH6)#_#Ju4ya*b8y7aVE2eZ(8SPQk*mZe`0X7S|D!}TDGCkvAghm#Tu z*kjJiYQ@?#bVn@m)(Y?%+mf|ZVo?30b1DX~=gG>+VffhM!fgB3b#B#UckExkC`Gaz z`&YL9YbBuCzjA@3=I(VUaGn)bo@0^Z9TFwYFf(chMIC*cb za4c+pagnZ)Z0dBC;{=i6IHWKX1mdR~w zF3&@UNJ*6)dG=L-^B#W&rTQ>FLOIZ&k?e=WmCLA+Jd5mvoedlFVEn4^8;)NSek5Zy zg8RP~8##P8KzAVm`)$bKw%;ba39wm~Y5QfJpn3}r=Hq8C${n0v;5_K52E^LNuT${Xn$6>S{()_*SFzh;L zFmx9ty{Pd;1K2Vc47&-MaAO=c-eEMI)Vxe~*p&{$jH`)@_2nkqZ4Rr9SggwUpb2r< zISw0BV8e|!zGyJZVRIeE9g#IJ+!0y7cfG?dblAfV{lQ!A_E<;V2n zGNs1NWq7Eg1X8<>5=OwMBLWnaXcC5jZo+5DO~W&`yjuKzxD(Pp`@Hn*h#L2>$F4xS zuyGQjoD{eO#NWo&`Aup*Pkn+6*|7hEcA63zf4HSlx{p~1z6my6))Q9nSf>-~8L|AC zLLGT`SK5S2k+$0FFrDAwXEh2QC!-zCSiG2d1Siuxni+ML>Fuav91p zX1#P+H@gXEl{K}ra&Hjzi3BhUT|~}_<$)Q()>%jmi$-m++oUtyikeGNN;==n2rP1t zf53P6GofPs!GgCJ^Ieh0;cME|u005ne7+hky1;<finY0-*7u~Da`{ZhQOdvsE~EyBz zP7i=lhF+tfxg&-j+o^_5tkw83=Eu{g@oNU-2kLkBy1I*kUp@Tx3e=5XJ>SA|11KDV z8{m?c@Se^3M!2jVo+G7ghrTE^X$Cvf_@co@4*QbB%0LDhx3BS$bJjye+Vwtb{5W3B z6KCS}()tM-T4df!Jw?F+ayA1!%+IEm>H1brOu(n{w-B`23(ow7Eh>%$DKkg|T$b$= zxGcMGxU!>9P2v<=W}d3mlVW!{j4I~)8^A@J4H|VyVM=Cg@uJ>yaqg0W9_FyaOes+( z!|Cp!ot2QTH)};|;PWRnEflWK7OR&&?kTYwk20M=T2m7hQQHR2bp7(>>rMHRt&ni} zN)RqzNeBS~z6KCqBYZqu3;gzQEvz@SunsPS9iI6rweT%h3-l=kD@F=dY_0K?%1N<` z-QcjBT{u#;#wAs2xa``Z;&gI|@A$Q~tdrVWJa<9sf+dU7Rb~LGw>l|#*}gNgzy4(E zjj!MFdQ%?cx1U@!_qeIKUznPE!qnWiUCq&_`|v47l2Pnp<0}=b0mZ)OudGidd3S{OVftVryjGF*l~}t*FH}qJiogb2d?a_v2`X7*7P1k`*xwk?EL8rh<<) z?)#9CwUeO_TgM{2V)Km;2d_BnY=?ctVN}U$+;2JT+YXDNI5a&rU=1ggkGYFybjpDg zAmVI#Mxv8h2xE)i7pW^9P*~5TgG2F_Fdft&6zSkF{HSNp>!Z@a=qx&*3f}AxW5?{4 z67+5ROO!MvaPdX9MwJ@<3gj(JC4L(=d52>;dky>^l~9L}0AzM-vH95@a0Os$aX~yz zi|goTEFRjc1|HH6+ZH=hUSmKnYzGJ@YbwXV1;#QJR{x%(Bl?t%6eAreMmkcAbfnnj z4&yYSVmCRA(}0TAp`0#&lsVqCq*zl zz7|z)UuyoE0<0}nP4!gx+ z>?Sl^73xLf9&CKhY&dg#aZ<+k;x9-bT%EQCSq7By_0sCZE_rg~w4CvN*kR%byFUcO zLXX|2t1~^iU|Xexk2+(!S!P!urt7u5%gRbm%iBdStT?{daCSCMX4(uR+)KPFs@9K@ zzx=cc(6!Irzk{lw+>Qp5BrpB0sj9QV_msWUr&XmGt4gs;jIT7f%whj=7(UyC+X?N9 z4#}Mc#wcXEsfRfndHF&{?Q;S3q)=yn3PO=cC*k+~kd(JzNsAd<&TCq*ux%McbK?;j z$O8k&NdsYh%sA=tizIkXeazt@M!yU7F;EQb4#fcZnbpVq0G~D9U)vgZisRHRw%GWha^km!Tj?;eXvMH(N_FG>j@5Dl1?YP(3fc1bhT1xW~_jg$0Cyu|d6VL>J#T$p`7p=xCEL_C;cs z%!{AjT3JDphp}qyU=HtvG4%}YLnJl|G*3W`3>yXPXC!tCo{hpmlBDZTXqUivGp$WR zyoz7Y{@_w;k+27HpcV-fK^{fx#YRQ&>DG8F;G^!cAou}%YH9Fe_>_u&4&Np;)`H-7 z@MT&WP~RJ->^~wDSp!|3B%qh_wH={sov$4t(Q6=qsA1Cna({2|BQ$|b0e64tijsKF zj?LI=TY-W>(+qH~1lbea449;y&R~%AD78LgW*)Y)L2S=MV|@q4`t24Q*FRQxx)pn9 zOySQ%iHX6ul?dU_LvcmoB^b54;Ps*nbmH+~<<%kD>R zH%;N(p`A12W+|K}oQZdHo-ot?f7~cx;z>O8m^dZ3^?;21zioPi*=Mu4fw$Y{=0y1# zJhK;hMlpB_`Rhmn{vEObxxj01?}W?y`{BM0_c?>V376b{#*VB7=$sj4tO-!Nd#}^A zOJ9?Cp}A4o>}95&0$wN z?4J-82N$!~f6w@$!8Fi?#+_k&I9ma}YtT292KO3YG$;Y_DAv>XD58IHEDGc${A39) zXZm3ED+)QMI=vqc$A!c+3aPvhKDeDVC|9PCzF_WQhb@>ZqS;@dDak3C^^nKx7Cn_| zxNls+xVh#ESzGi9QL#eM-tSyMsH#diWOsMm}z!k##3ydWf@sJcPl{VL@V|e zxZxT&2pFVk$@y#GdU|EnfDV_Wh`H~8R|C|CflcDl9qZBqrrshP^PEA{h#sn7`&E5K zg^ZG_^(OSUI&rDgNe%Jz$cQ?x1Q2t2p-pw1uy6()im+ZSktXfM?_@2pEG(fxJCtPZ zhr;DZw;Jw1xR~OV+B(nG7JXVEj znq2)}Ksurz-FOw4fn1d2#AQUO#IQ+!e3hvHOf^wCKAxcn< zN)1xxcp_wcoXINo%&?QC&K9PryLZzv)#_0x5c*=MSfm!h61mhLNYHQ$`;?ZJc49+m zCHP(}66}oSOQ32$DaNNhtoD;`6J_RuLTPT!$KGfVgi>}YqMSy4tU=ou@bf$XCP)gd zK^TvMDF~7~c4{xihxTF}zks@DJx?&;w@}HvzX5J3T;8+pcwcMoC&RrF_iPnY5LopF z^l7V5jIBa3whG0P4!g-=w>qo_dC_oG?P}b29fnog1~V;XhbIfN+Xo6UYuey2V43ZM zXIh3i<*^9Ga$;-ZQeVAxmQ&V*+D8fApz;1}tu5;?pC`G)#}TfX>*9d>by?8}IqSyys*C?`z?L zii94IbM#1`(xYOeN5x2wiX|OJdQ^<`sNpygQtTdw%}0SNw$S)!NrC;NXoKeM9|g0t z1TE5Nfd&JD-ibLA3!p#hCkxa{>;knui8;ptexUu_{$)Yi9thBX8dtFOOpJfD)lPLr z5_4(_+RyD*7Q`>xwEh@Kc3Ww*mf$W_jhjD%tnCR-kzT~lBAgeJvCl&&j195;4T5`r z{EKK0h#h^Di6Ydw_&Vq|(ki-~;V~ymSaPP&hZAS+8J3%-ZqyOqK8QIZ) z4RWMcSA#dZ>Ym+E6*d&*(Yfk-tywla;& zVY_1NtP}&41ZyQP^NlYWa5q)Oe(SLCzE=kz4hPL2*{D_fxSg0*SrGSm$q0N=SowSk zN+XKjB>XtvpqKKJmW)+y{+-nEHE03e5FyPfa!;#VFTuSbLfXvrW1LpOxYKTg$Z-PA zT>U8(enV)yzj3h#R;;Q~1ZRHs&c3QQ^aSMcf7 z3dh{rpIUhtq4vj*F15dU?KpMo)CxE|t_f~xUFt% z^I%Di{VG{r5Zs4dQMnaP&KVg>hzClrBjk6ZgD&i~Xom1KLYu*u&sT_#nM?PgZx zB!9h<&Z;Ea*&+J6c{8nYDJEe$omRP2a+ID{p{PZbv_EelOEr^U^QKkYM)X}Q)%=>I zq>DgVx?h$1^t6h7Q>W7^H%f$~P@dc%Q!U+}Qx82G8t8o#5rG&-%EpYFK`)7kw`Xblgp{W(WbA>$ab-X!*E3f0V!WBPXukHv} z!fJj9F00ZNecGcb#vV;E_GpTcFDrJD!@lIOyB+ouhdtsjPKayVHyrl1!@%v#I|_^s zo3}ZPlQA0iOAg~?jAE4F=)=}74tv#MoQ%!YLT~bP9$~O9M{9XgE&6tP^RHQ!w=D6pUg$L2ioWo`QLR zTt=1u)T{g^G&IDU-Nhx8dTED;1@?=UF2iO3&2t-i44;HX{8+RT& z3@5U${5X;o`EvE(PUOpo^sXr2F3qyi>znL4QV!7=iwPT{kE#pyyF*-Kpf4J{@33vg z7Y)3k-=Fl(q`R$M$11`DHXCQ%}N_( zCsL7urIATa!M(c$Yb9!bk*+OVChirkiZutj;-xPdJcGhf?03eOQM`W!cup6^%L>&i zvv{|+&#WuKt^-OTZK~MGLbB5nT)}J$hlzTHL(1FN6%Bn+ZdPru>x~aPUFmtAw5Bhi zMyc>cJPHz9rL}XZ<7YRc)PtA7JJOmXmTX<95JC88JJ`e)zc(30E>nn~XL&8?_?fLtRzDaorRCb+0QYoOt(Tt2U4QYK|!NJq(#D4H#wkMbnn(d8p9)~#49f!E} zs4}F5W_OgFrMo49Eu7f&q%l4=9*=am`Wg5#ho%!FFhB~`^yvtNPZEOxO3r#|Q&e3X zwhKd_(bZOEopcuHly@=(p(UQ^WV#7!OxRO$IUQshHYV&D9N?*SInaz16qWWUibjfx ziXsP>R>@JMMc73sV~iui&VnLy61T6r73;6aRB{SmVJzd(NT1mLnCCmP(-CQZ$tQPZ z)gj+ua$g8}A+36r5ov!NkoM!=K?5v@$}u2i0dN3H2Erlu9w?zs2b~8=pb^+BLY3yU z<+C}c@N&RGXJk<5e71o$7hR4!nI8`R_m@|VfkCCdihDWPS9zP*c;~a(FF8bFQd>J9 zYn0kGYd#KRK(Z#gl?Oj#d**Z`2v(gnq2rlj)_gA0RBm*&VmRnE zu_y($qX`DTvP9=RuYwx|#(R=2?~jLjHQbrzp4|W~o3tCCFB&}Iu>Uc>Xh3C?hI`3j zEAU>$&NDvjKIt&-wxQ|$%3;58*uNZxU93$wo;a@QVaF_k&34#4hkf2*=Q!*thh6Ki zgTQlsFT;$X#up7{!l&;z)%cujVWv~5S-n*-l&-)em$^x#CfiM` zIWmTN8~eqP4VK_$mgt-Jg->M>7Vw`$DjDT4R#fJ8_!CeFVM(PPp;)a);WvWRCYQ2D z?{q4#I`2-{R?{AiZL#@NQW3t5rA<~lqlw|@1+Ot4jF-fs<9z595OxUS5v729<%z$I z(I_Mm=~H?@>5$Svus0|vu^UNK=-CA3(LLeEE(GA)mJ9uX{ksGqotGw7O%NJ1TU;yam^!h5!T%zxCbN139!VgF)^hITT zf?|Jg*eukOVsnizCyxL>DAQ3$YO)?u*Ck`nqm?L{D`ObweC#Y08wOt{%?w5k!V>Zk z2t~#)7Qeq{(~Nh}l2*P4NG%>x(FuD{1#w!3pUEDyR)_lRL2GrWw)B@PPw4 zt)0OG*#5i61MC>vk^z&n?VsP^wg0SP5X*?P;4^^l1YRHN#W%9C<$m7$mk;I8)W>0KzI02mx}DF+FN;jD=#`;su{; zYi`6r&_-X9KK4Y#UL0Z7)V8p-ac=X%=GNw{K&mp?1*r(tyt#h{)ugE;bHlyY@e#37 z*@Ug@Mm_*C6sh<%MEE;?j2HoJ6yO)b&o+9#*VJ^cV4SQXWQjeGASx1NZ4`rqWVdWo zzo>Z`{GZrygzh_Vi{ZWtmu3EMxTJw%M+5Y!dPXrS5ftMLm13+(#b`gH80}{idmyc# zybqCQg;+jG(q7bz=w&nzMoZSUb_V;c>Ty9U#Z}f^(P{P2_=(@#b$!_KeiaG%ElY~m zXS2*dN#AF)I5H^-I5^rI8#Dklt0E{7c|xp>LNNhL5M~4QO*|XuEx}=v(7|zEUjgoF~p{LpDPmjqQNSM zk%|=aKL0U*r+4xB7ll8+lkM(p@rLOw+g*@zD)fM&#sI2`+^urt;9W;lz!a%c@6Z-IPQtJP*XqzX zX^)KMnp7LddacC9&b~=1)wEcy2|aHK6g>q+R*7L{9STYQ+_t82R<2ae>V7KJkj8q= z45dO{WMt-!rZOp4DwDdOihj}XP%2U`!*?{5ePTt~rMr(!r6_0V_QEPqP)1OXQRY1S z*xymo&%Ua?rRFR_=A325ys}x=Ac}WNksWOKiILv=p)16HH^PK6CP@Nngva%tSy2NH)x zlwE>WIMzv?e?gKhp$2mZu%7r;;rCtD7Q&}J=MGsyHhO16kcoP?(#k?dMWhACJ*0*> zipjpXTFUkq_Na#%1oMu1pfp%%Vp=;=+`fgMpC-Tac5n_d5As%C3-F=*_A}uQgnJfT zmcv503`4U;6<_F!23I)jD&va=e{dK}SHu0sVbBg6EG^QwgGWd+x{L+NqTt;^X3p}q zM}Q}Jt_m~~GI*=p`L-F<-C_oDYh8iT7Y$g9ilKpv zFB-h+us0m`KzhJ-Gr+LEh0WDo6yEgGBF)?-62+PcNYQq>ZDe;PM@vMZ?IX$Q$hGXn zd|m;*;}_91a11`54wp1{tt%q>RM(*xJ0QjGb>TKSEKP^qYM$32VoiuL{pQK0p_eH} zyFndG(E6qDxcgI1y6lDA>#h$EnZDj8nd$mNHg7Ab^mK|}gv?`jZ>SSdbR4psWQy=V z;MdwY&==(~K!IZDR>X%z<_=?FYPfU}=4i@yBUre={H9Fhn_gOAq1~QVc4NDz%!aJ0 zJJ8DZSiR+Dj9>}T+u6y}!T1H4tGV88Xz>nHkUSJy3z9x9NX1x?ial<8rJ^oSEM1V@ zqQze#iKDxq#XCYxI>#OlC|=oANhmT8!cr#`>D!*}P=9KtJ8XZ-UVaUJw!ryLpFOfE zKkAB)z9@wlgDo+>ENKxt0`#H?{U|NCm2>Og&7Iv3^@chUADk!))p?j{0yL zjKh!8(@6Z@a8EMna%Jc1>(c_DZYnkw$z)%hESIOyhq93CP_%pA4>R{xB}l%QFc=-? zw*p-UO3BTUf=?#-4PS$cWCHAIA)EeP376^dp5MoN7WPV4*z`py;v0;@ykb{5>{^F? z$6+i<4fmkK9(LFc()2@!*p-w>nM^OOmWSF(?fCuYb z1TBXm$O$%5IT`y3hfj1X5Nb8=3po_%xURGq_+b37QcYgPom9$j&t!pVvMd0SXf=NJ z;&(iakKlJy8t`tABE1+_CW+r`Nb|2?6-t`)!F~kK6g$=UR6(uST8BO7!oA=ye;K5# zU>pCl*8PQTaYcI|cGQKj+ zxt;>mw}q;!9-%0=aw>Fp3_tZ!lHEz;xWf*!m43KTxMwG46{+$C*i0gm;<4r%6@L#J zP$xU+^ppskX=Cpk1mnSorZ`%w!)L?>?&*tJH-e!@_5?5N)61WJmX$wM#^%mXLXGOi zS|({-JC>l878zxQygmGfq&E@smh2FiklikgZ(~NRZPyXVC-b@izl-tPh~Il0a>Dk`=!I4b=dO`btn+q8b|?`~_0Y>D9LlhARvee<(>?^(zhJ~TK5zki{E^wKt94#J(?Hm`9( ztFo}JEKMt5Z=6Z>eJe5s0%?QrlR{Ll8w4%I%QL8+d`2&&dd?*))$6`8^I^$2RRVys zd#&I)P+m;!LVfsX5Lay6?mUL!^+7IY*4RuPYq!B_A;9+IODMB&{|3sQ-2mGJubcCAIB*MGmdbbFo(%VU za2W>k+NRymrxJ={mzbw2p(u8(!+zr8-tVxVJM0OE(a1pK{=s2fVWZeV#)sV%9d?An z%0TxTjxB(`Xi(*_gB>Q@dYo_)js~thIkD>1Yw;qt*t$-ZRFBx!ac|`d#IXa&w1!k( zJ$CBA%tEouESOzB8d8yZOgLa2_V#x9NED>%`j@TU6u+#laweR$-)GXm(7=~sQ#T&n zo5GQ~zrf+vmvDi0jdX?$sW%d9-)Y?^k=mWljxS4KoErDWskPfMy^+E;B8h^A4YB_9 zsao!+TZ3FjStpjifcqhWJ(%|)qQww#!AJs)Q_qs*roqqc*cU2#+|stz zMkx0rdne)5w@U1l2vb{rl9!z(RE)ozo4evfh7=V<_ASbf>;n!Rcgv5s%Zo27$qqC_ zN+%&ut{rG_M4+;8zdz6&Xl90)zV1MCoF7tmpgAsAA`-f^B`b76iw#ZJ-^z^YrbzPE z(!t2<-uZm$ui(ECvQuo2N_qE@^ecQMk(l!vZwR8gk9#CisuzqPD zyq09Yq>T5CM=w|ik*s;HkU45FI8DNx2Dx^OG=*|B+xrBIOGD;zcEp>~a)v}>SB3T$r?Hl-!21(af zAr)3F+aXEldi+Q-Bm>qKYlO9FH&0?YhfY=K!2?#)+TPUc14X9OWa zWI|*>5QNCfC>yVWW~LWlPp5%A)+qrBg!w}Dl*K4R{v zY~}N~lrlyh;FG++z}!0pkvP+`|BSj5l6z2Gs6-$*#!4+xEH~Nn3Tj>uY$jcc)!K(b06EL zPu)=Hz5~aNhx!Bo7wW(!T*Q8 zQMdK}#J=(Fd~R@ev}FKNfN z6$KaKeO@qB(urgovM$}$(9*&9w;#5JDap@DZFiH}K#4sJNXL%Tjc-32t_Y*tp$kw=a`vNEHxYwAusvQn2tFKb%R z+EP+$ zMPkBH;e8TmmwX?zBc_^sG!Fck1MJ-}icrNAebL|ohfy|D?0SdY=zCrkJt2m z@35C0_IHQvfsAOly^Jp!a7sb3LmjrjVT&BbwT~K(Yai*02KS-8D)xZ!MT7gHXi)4S z2?;Q4$!(MY3k0{ghK5$qG#(#=&B&gp}>98t?JqF(`_{!4Y zapQw>1NVv@VSLfxREN!S*zNFLfFJDPjV~Guz_%(k$oMc}=&(A6@yzXSNL-$|O9kHc7emb z?6Aun_JqU!$6+ry>@|n=EVb_~H$H6n4&PLKeQ7Y;_@cr0;ZqvgXnZ(xHD=>h7#|D@ zI%ZUsohW<5re02@Hn0Ey=;pQd$&Mkg|Zbvky z>`HXZEQ=qw^yzrjn=qL7lwOJT)I96Xwi!Av&MJ3rXWJ~D*WG;cF4K86Q0EnSbh|3A z_{op_BYw&+-intuGncR)>_E{rvH4OAtAXq3!ETi3{4)hR z{??Y?h=**CSVx;T2qMu5!F9plyt?h{zlKPp66n9)huo<8ui=6{l~<7WQS@JTNbI#T z%BTKo5-I_cYuFYglCA$54*l1F?&-gV%hvn3^k44yC0~S7{nzOf;e1&2Uo(72^)_+~-Ym8L?b!pf7uS=y+)A}!8b@UBTYOGiCjo4nh zF*m-x&8Wcs5XMCt)gy^*fiuG!6_}{fvQ=P`VoA0bgo={&yG^*wonc`JD_r zrV1%tgn>j|E*02}*z`HsDzF)uDzM2rt^%W07Y87Kdh0;dovXmklyGN4n^h-c9IC*c z7OYbh*aPx-yDBi*VuocFR)LW%XJ1r-9S;>4wdG+IST6M!Dd?l9zea@BU&Fdmf7Kv2 zJFfmZ)_k5-e_dVSC$EjdW|bQkh%(ujnp6)p=1u`c&tI>7Fior*T5=|0{J~ zQ*!FOGUrl4I}L+6&Wlf@GOwZG zsPj@Rjy^!KKE@Y?)NV1(o@hAEmuR?VhkedrZ4TRrzDvXX*!ZHsz0hhYw#oRS!3gwk zij6To=%XCg=&&UYYjM~`4*QbBYM@;@1zLd8pw9TB!SV2YMzE8NFB;qqpT75p#up7X zIqX4)J?^kyJM0CA{n=q}IP7hQVaQ|N!4V$c77hA2Y;T9%1>dbwzxNtnG}s^aiXCKp z(5E=8!C_ayw;sRJ;9BF022p&gV!ezn8XWAfYKL)i6UHhH))-$j_`Ad2a#&f34cFKB zaB@0)Z4!6B@kN7QI_yb@QRk)MsPm#P8Wcj~q!_kpHyHMhH`sWGEpb?j!_IdYwO^Xv zPaJl?!>H-fcRb^;_Z;?t!;Xap>$CXM(%?Abiw4)gr!;h(@kN7T2w|FD%=kV5{T7Kf zTW3X!^&Ql2ZTlk{h{y&1h8xy3pfS4Z@{dQ?VCRMOJ)9>wqWqyjqDPoEZ?OzfXjot$ zcRqXRpH|9wCvRT646&Ie1ba;ryVIGAuue2k#;&q3Tz)%c?0mSiJK}}8<0M^hOyRwK zh0e))=~LdTSju5H8egfH#A>*A9L7>ttPEeNar+t{=?G_Nbz*GS@QE1PQBm^AD;^s< zv$Y$$OBIg)2W;)wtE!HXvqq|8ydFN)F@6s|)iGk!W^{}X!>2k%4(nCN_&j{7V|)p| zOj|q7wS?RK-w}#5PxmVyPrJ{wwYwy*2s*|U(8R6H(lKV*+EHEL=@`?zVPD}DCnLvd zYgZz+b|)iASb5;uAzQnA=o;UKr0Uz+ZO;2(Cv5G`LuzVkXS4IEt)1FA?Q&bYZ-QcB zYxiPM6ga3jZj(P+ODFpmKDNZSkZVbqi0<^s%RwneEf<= z^06IPG}f7~vx>&An|rHh+@76XFDYScSZEZDJQaqQRW!mI)-|#Z2{9!>#$$I}*~kv%qbM86 z?KZj1=OHvMb&ZO>?6AK( zY!CDc8jkjL8jkjLiXG~(1rFm#NyE{;PQ%f@PBGfoDMtG`#b{rr813s6qkWxXw69Za zmcwXYrx@+)G#u^g6r+8eVzjSQY!bA8$BRCRmUi?-gOlN#E*LHC=!*tFf=}bp(oW+( zmPs3F>tkPkd;QNlmebD%#!BE^Q zHr)7dJe$L2IP3=aI`AtEZZ^JX&>LT<*zU#`4MsX_ti#sB_f3g=q47nz=d!`D+p_qe zGlk|*!|iQ+INKb)&r95;#up8qa@eyDd&Oa|JFF+Pe;OCErTC)3kq$e`Vapx%1&7hn zPUEH=_K?FKaoF!1_6LXML7S*?3ylxDY4}dSmzD-68DBKG2|lHvTa7OoltJsLar+wI zC!l{MsqPZ}+mh+ zuKiTOg~gr(|6j-+gETGaiWDl!%Vf5OIe zpDMUMT|n9!i#6OH0=r-VY2)M_S8%aI`M4BZSA#E8!L^4d4rtIppDMT%OF8U%7w#s9 z{mfxhaA{mBxHK*mT#CKyFeDrQNg7c6B@D!3G*f=e+f zxD=y;>$_6tRB+L!3NFQ{;8KhVF2$(eQj7|&FXE>PF8Wl#r5F`lic!I(7!_QKQNeYc z#HE6ZK2>ljMg^B*RB$Oq1(#w}a4na(RB+L!3NFQ{;8KhVF2$(eQj7{N#i-y?j0!Hr zsNhnJ3NFQ{;8KhVF2$(eQj7{N#i-y?j0!HrsNhnJ3NFQ{;8KhVu9NYls^Fqe6JMANvAQlIku|aHS>|C&twl$1mIDc13*xO@C5ocPo6ET=?qXtls9v zY!IAz7=X8bpl+Ml2m8e2+(|qp8|L(s$7U-?THB>fW@55KjeS|B>u-3^H+cKy(VUK{ zLb~@hn9M#zp5Jr@*b+ESP-#*>JqVKYHp1n%&VqX|+#AsRbmcyM(cnynEjGTW?60KZ zE_T?9F5IgQOQ6gfutQ`+s-}1f%D$~6l|L@8EpmI_=HcmDAOjv1s)Yf-cyZgbvqQCD zce*(Ia2OgLQ9eM6wV`=ls~8>~ix1UnfE2rVt_74?>;p373D$P3Ovd?L88>pb+WF-y zN=9hGuNA-E*o_C28t`qX4#^F-?O_!e%&(%i0N3DWFMcm;2z=9R0=f^nLOve=cTcz# zaQ8CzEEnFh#=hffj6T?AI}CLzPou%P4of<$0PoYd#l{DHb^D5bWy^TzP$Kfc=ER1A za@SbaBGBhU)x@bTUMPSd&)#J0ro9tKzT0~K>}fJr&dpVGwyr+%IDwLewr=XYc_CBP zJ{gPu<|3Ck7N;N<=}`vaS&9-v8#unX1qW|RL|WB=TjI(*2vg2=FBIzV7kFNXZB&q! zRStDUIOSM-JHMPY%(R#h?l;RlW|;`L=O6*`GeZ`${4~%MKh4{UqXvQPg*3AEkX;p1 z@>vJBC!Svbw-;RAv%Yvw@~Ls;Ltiww(qZ2)zG(1chu!C}e>m)44&%sL)4SIAc8a8A zS1u$D*5qx7v+GD|f0WzKlhhof;*pfuy6nF|Qq0I1_AE{02z zx&$ssiuWWb-jk$i9ZAurB&8ThN->g@Vk9ZW{^2l^lwu?)4M&nv%x*!9O@p7oZlZ0& zNBoLAWmdBf0t*`-wDn1`odw%Ab%>j|f`_`NgJ28LcAoIg35UNSL_x40d?&~zn1?vu z1o$#fDm(;Pm=@xJ^$3Nplylwx=BzK2VC}Ik3+ASUg^IStEep0q#&h!q zB77^8%9lpu=c%zCXJ;)|#>cE5$Y9u^gKVrq{NX>=xW5_?m}f&*?EVNbU+n&vFOOHv z^patDRTStxAPoj@K?`RYBAZF(F5I^rMk3O1uR4sU5Gcm(LEnLmX^UOwxEVngg$P3^4>-P6 zwhx4Z%jaa>+$Yr@HDNo%irGt+EEKAlGcto}(xOjcmjWgF8q?b_0aJxiU2h?WtVetC z8)$<&kG*kdLVl!*rOd46%MOiBApg zG-Y^~DMPNo(-91P(O?6@D0ZpwX&EZUGSqPXc@Dp=PrZu+aBJiDJt3XSmV)tEedA7S z=!Zy~k$C#VhC9I*J{v0ZAqYiLvI4(8>}upnPFUEqX!hKu#$hAI&KWUx%p5plM>Wl! zJ$B@rG4tk(nLV$0)R>WDW{>eUsINLu%6ZC&PKdP+dZ0RCz~x}}JlYGQ@Nvz{dDy)4 zAM8)qV`Gh@+^2`0J!Oz&r01iSw3*&u3L=zAggqs~G~DY|CQIOdgok*?QS;^LXWY}h zGlo=hIzz$Ul#YF&`5I&}7Q;s8dRSS7O*+G6Ph-!eWCW%~wnYZRf-sG4H$Yq%f}FK*E8wn! z%P?=d9*91G4qIb?cCQ^@swx^`-uTQQQ4x&lxvn-014j^G1#y zGqO6H{=d>0{qNo#`bVYX8W5s?&;**y-FV2+_96|kErV=>Is@*3=7~24{os-;dCywo z{Xn?b+%Gu}?w)W-yL-W98~?X!pe%TrswBjmtK!*zX3;uGsFz7YznEj7vH+95;^9aHJE(7CUU2!&WX;@Ue48533$I zZtUpc^G1)HJ8$f$;q&I@;04W{@q!)b3-<`|g6;N&{X?9fEZnLy8~t868slDUSbiIo z##p)oong4Wu_U@~Z#g?dwkoy`uT^;?Hore)-G%$Zg&=>np+#`_g1ZlPaVNLzWw zwH5jR;+bM>D~hqLD8{y;7~6_sY%7YfKUC}mhp|6Y>{i3X{^*xi}XE+J@TGUt%)D@zdrq89o&v zS2)Zzp(k^6r{^IQ^}wg&HzT_V%|?@%+cb9W$l+s~M>fr?9y4b&oYB?I&2y^9%^p20 zyE}ckvnDjOJKZTYCT!1{D)s$3>rT@GksM6ICAD(;#$e8j%k_R%Q%oP9AxZ5BcfQF}gM zdy%HeTG%z5YQVR^ElB78Ff4`lBsSht0OCEVg7;Jky$yFN+<(Gl+js}=+2)?z_H6Y1 zIv%7C4;@BUq!?L|h9fIdjI2m8vLeOUZ7cRihq2pMjNP_|W4EmsyKTkTZ7arZTd`V) zu>)7^M2B%$r5M(Cnz-!16@y8f!M^ISdmXmPVK0K0Fh*(cs`2^VdnZEza=V#kNHDFt zLxRj9@jARMEC#T{VO>+K>YvTSnupD9o;SOC?C7SsBZiL}(>!8c^~m8(qeoPas2($X z)Vv%+;`2J=VgDC~1l^YNb8)fUj3^h&@)=PlBLZbaJ3At<6Qg8a;UPOKYL{{%vYlxl zql_T{*$%mMi0#N)Y6i9!=`cJj7!KA@27Cluk|ystEa3e>xFg|?gF6Z?>31|-@|Sm= zs6!t>hmpT1M*gDV$X^sAe^HG5MKSUh#mHY2BY#nh{6#VH7sbe56eE98jQmBhqa1dO z!%lYCXC1c6VVvgGxEu^+Ku2lA~o-9&1>!XW9A+rv_8x@^Ol;GS#dk__A=%f3Pd#illaEtsbUl1 zJ6^)ES^rU<7Qpu=e%#bzENvE3Ul7);mN`ueo0e4w z=`C1%20WNpZkvO}`3DI7G@-IWGjiHtO&0n|lV1OfBS}W!rLi8-Sfr$Mf9y39-QVZF z2lHX{NFRCyyfOq)l?jLO=3#)Ka!r~}lv2_!__00F#`el|g>X+3!(J>$TqQMUwM)soEW`}KcSQZ0Gq~1dnQ|DY?W~z5; zh4qBEMvr$krj`LAN+zlxFnDn|aRm^GbwrZcWf+RSV* zzX38U#A^ovBd^^HzxALCy-blF)4aHO*@8KZVRPB-a?h>c+M(iQT|+}3;+`cUvQ@~l z@Sh6wYj)o7Ok}MCKU||~ zT}S)<#lirzd#FMTC?pI^2zL($xbERykrmk~WRmmyKu!wP0i{fK0I-G$&JWkY2`gG! zn-^&fbc08%$t?KJ`Trq7e@_Hh1jZzCW0Y*W(C(=A5ahY;_wP*PUi^|?9Wg)QI=aKv z5q(kGRRxNjZ+vRxt=Pp5>%4oAF4y2#VrXnpXRgYG%sykuw-#Js&H?_{iNHEm5JDX zY`ria;d=QHEMJ8p`l2)+4HP4TQ|tnVr5v`4nhwe(t7q|>&Xu92V=ZxU9(;3mSl1!w z3%sJ|M1OZ$Pj}l%*K(=lVRkfN>w%f_>Vcnlj%yzDY44&Kdl$u6_=>T2QLMYoV`n7* zYQs+HqIujNk^mMgX)z3LUekhwZOfXw(M9>noY@86Z7O(leD8dT|Lnk4JfZEw$k{b0p&HtE^-(wPpT5_`3G>- zU2599i6IXp+BdzMXn%L#<;97Pa_GLQ)E@J({Sb#MuhVwN6T!=l4VC>t2t~yW-CIL# zqp^1GnQbktjf2$XxT5j%%~W|eHZ7DjDWdt#N3Q~7NIP?q=3NJhWnx4hPh8R3 z+#=)hv@vlUBXu=hga#h*8|N%p+}gAN>Hrroj!aSB>zfWZv1!g(jV<#Rm<60PyeySa zr6|NxL>n?28W8qRB-QMTrowTzEkapk!*RogxvJvVm^m6w8IDDCegPo;0#$0i1$d^H zGRtm0GExk;!Qjgj(7WsW}Ri{gw+ ziy=jM8VxRS*kum8+hIR(*z*qilf!mHejgR}TYuw=1~cK)_nvBeQ9W=McflVrNa=?_ zjth|z0=VmzFe%lU**nnD5}ah-+}J9Il^=FkW354iUW}DanqJZbRp=^sCBz6;uzOpE zhZW4qW2u|@-gM=$>U_Fjo>_I6v_z76jX+hWEgHg62g}Jz)gj2LJrOkmF6IX;WZGyI*k4F(s7iY|GpDz z8yjm|xUQ^m+2^IzH7{;$QR_sIX3pMikkrmS@;K+J^g(DtN0fh9A>5#G-jZdDnifhs zCQ<37M?!!ZW-PM35uF%TMCQl(^z!x6UnJbCw5kXb9mT#bTZ|GS1FSCF>O1U9GF`5R z9|R>S2wlvMxrml2^QBx&doc=jBYtGnq!({NElG$K#&1UZ*9F-$fZ_-6I|177w8q5M zcw{fuAFT!=aM`eVm6s@G?FN_ix;tD_*hoiV^hJXk9CowuMT3VN_K3q?bl9s7qv@ii z_q6dvgYHw}j}YhbE+~-&r`N|viH*w_w9ZE-14epS{B^~gdWS?*(O{@x9rl64P`#3Jlv`_AP2k^gYrE-+Z^B zANI#&Wrlw2ZWdt<$8RUuupBIh+pmWT-Z*hF83m5@#UglD!DX1e968XZGkA(|22ZgY zTsZc>iv1S8Cq!WWz42iubG%#Q?rnTJHKl6>bVW@2IsM9H9o;fG?=6cYQhO%aKP+n* z6fcWFJWOMc7Yu_19lV(SA8m>BdZ@L^IK`9Jy`9iiD0O{kT4%T8%0s&SO-I>Tuu|=VM24KZ>UyVDV&*JHL+l zSsmio9Eng2ru!{6Yq@rnP;u0f#cH6{xUjnt?f@7a)|S)aVt{1jAy8Caa36lOyge6E zAP8Tjz)aTtJ&5kQOkpyZ!uKR{6p1+|&1I%@nwB=rS>a5zK4i(xXthiY|buH!WM%v_cqv!eya1R_{)cAPmOsK4OLO z(l~ds!6^5L^31JA_#*0vwG(*^n>&=^Q~IH8rqj?}ZQ|@ad{}q2sjvdMzvVG)dt;@^ zAMu%w{|G!H7D-}3d*)ih+nCFV3uA?!%d&Q6nRBqekY5u_#M+JBvp%@O%j}(aMPy-V zqNq4>2+#Z~Eb19698>9q7zyqXX)Ue900ZNU(n%PC6ciWrh!sw>tc;2TsvZW*I~ylL zAcgM1uHWRy^lW+3r*ln)H9VyPhh1AAw?~I1m7c;_>$N|(Y7~Cyl@g^W>nr_&mde(eoaet79Ge4hxXo}WADR12a2cKgH^Z^} z8tu9-`mpz&!)`UcC=N}u;qGzR3l96U!`^n-yAESVrthF!qwgT4D@M+x*m8$`!C^me z7uSv!;W*<@eW&re&{*r z!arwxI5-+U{k9((Uo?25y|#GUT_}JDZ9qH z<~hrn=Sg!9&CaEjT=A|HTQv}iAQ%al}>5lKlt%#D$*u32QG8o43{sR2N$ATa0cAH;m(IU z2<`&72f#fO?hv?V!L5e75H58fi{Y|>mcS+9qJWYo!#x`=M;t5Q&VmaYie!_)&o%h@ z22a9m#q;%W8SfIfjQ3@@jCYxNj&4Mr-(a5KWS-w_@XZD%4Kp5D!-O7|HP9Cgu5#G5 z#up73Ps43>*pm)>%3=R-*uNac0lCJd9HZ|T+TQ?a7G^3oGADta^15-j>)p>8BrZwB2!4C5Q_nZ8FLrG2+2Q$D|%pQtvf%O!-Db z$E3>>9Vgxt-`KV!kveU8L#plkDQn+pEst-E96aRa588@rsmlyriA);tS}nAo`;iqL z&Ofasi4Blz@oYcisoyXqzkd8LmhUz(Ible3Td#@njderv4xV}Q^9}L3m)yFkj=IX~ zyM`j-ypBm1zsM94sp9y?y!DeVPHZ^wVizv;Zhh*QnugTb#SI`cy&Xlnn&~pj)UFvHXfejfqocK3<%C>1$m++3h zin{p5ck9Q0*jizce({YDLrJ1JO2m(vv~_)KEJ8r0JGZjBVME*Bkk@U@Yun#y;~R@6 zoF89v3kqz_ruaqIvEbtOJ)B5wtsn9bbSm-oiwO;RuOT(*sfO`eNnItn=%G+V{%vR(K-MHw@WYzim@u!(`0aKPhE}&BC_EH*Q)V zd1YYT`g1F*5?kvkD@Y6tsb8X4Yd1{GbLCnO9ZhmkYfA0HSsm?z;ieWHNNJb@{+d8 zYvW}_@dH6Fh}n7?=<|X&HgH<|Y-?>IbyQ8FeQeOW9~SG)OyyxQ-(K4u=F+~&HHlO? zJ6WtD0W98 zZCj_LKA5uhwfLGpB4^COC6iDO%mdzoJd7LLbr$N@=cD2_z0wnZAJ376`fXe4nTv0~ zbgsN$(&h28pwn!$HZcn}2i+`%?q=a;DTs#Dc_{bt9Nmy9 z(Lzq#mT3QX*|K4YZ4V$HU}5(sR{dL4xcw9D55T6U9tJ)4TZ5kX1?|YBu#0)|CDOtC zBYys*e{GrcH?)=$pO>@_(X_UDY3-IHt>uSGIti|;ozf#Q;xTh;y029aZo;c-8*5Lm zJ+1cC#?wzz8=^}jl>U8;^2Wnu)bO4M-|sNOERgjYFNlu=#J7>hg5am{eFl&pddIABmM`(w+g;F;#-3-x)$R?_#Tz_KIX7z;oBtP-iJ@M z&0MLh( z=s^`CMAu=;5cd;h52jiP_h^yTh&~=6P?0wX&ZT*W%c%JXSc5`X$0{Q(x0mcp zLn4PWrQnxEd^OdUH_EH}iOf&EhVH}jje_1mO98{^J>!g%>@hpzcpKazLOhG&4I%cw zL0gLbh=XG26A!|+;`<}K8&@2?VBtc%Blo^|6-bjq7V5Z><%w;&&wD-KDe|iQ^K8^= zx#u6Z1wfq4_TcgCj2o{R@WYdts+BEK7~^63ZC#3m-NP*=zM|SdL1il7aq~aPE z$%lSo95&kdUfxU;KcAsuL75Ox10T-MtpTGsi%@XRSqI|Udi;Ifni~Ve= znHPc|B}>AYi^1~Mr(ui7J{dW_I!BHl7mO1}b>(LxE^|Dtg70jc# z_wihdSg%z>sb%~wf}xP7tZ1Cu)Y>HNle}-Zq>Q;^FGcF^iXxF!L-7$&Ar+(9@N|ic zJ3-=>qioX}O4TK<8g6q+*H2ptDEUEG1+-i8*{lLeKHqf#JM>fWV?lpa)iG^QO7VZ}sxV*!3qY+!y-|5)LgtL!Y@ z2K2ag_gGMp+vC*s{FsTubt?2I!=$;f=ignh*9-atPvNKgJ0ouW;+E!Rt&L5pyJJ&7 zMgs2hk(#6hS0^z6owIusSm8|{@albuXjRqNutUe}I9OWZF zaZ^L9$DoEvI%ydEnDO#4vFsWuSdB;Ntx^2!2m^|&z9*c`8`!*yS7&jci&>)utFK{r ze=+V1;w9~~ONSOkVkO6Ua)+JGCBZj}RJBOhz)9#y6C&d5o?F(SOf*{}oGB4bb`eU@ z52V}vzwqivdn=265?R?&kV6J5n~#KAC7~J-uI3}5rE-5|q#&pP?VOuelK+|DV)WmB z*&NME9pApf$G*vGDrmncTwVtMCUHM=$CJ|AFGS7tDhS89gJl+PUqRge`R~(4-9EjO zM$JM!Q9dOnFKwUR2^VR{Xe0enzamdyrV3a8x%6OEWYOfDQXqS@Bjv3uFkdJ-HK0)B z7l72x>Ni+BNBbWpg*##OKPf4i8aryfq*jO2>)ED*%|*9CAn;?EVy+AqrNPA@+JCzi ze@W-XU$*+6eo4@uT^egn1!wBAG&pU_Et73&G|TUvR#n(nV@H`COe`1{T$_ZP$%Um zw3J2{`c(7GFAg)>+!9IIv2C&wi>iYVPz2!OU;kIH=^ z4xUSs?p@@1>%LoBUsN0`+^wi2R(PoG6S)B#Nh^YANZa-flrX;QrOZL|Epo4$&v8s{ z-{kQmUDLu=$E^KWB22R+wT03>QB8wTlU*1z7 z?_fsnGa2ynC=$Tv}68ilUKdQBh#BELo7Fq|@B0>o#Ms(jv4DKeBRxs}+9lY} z55i!$Ct-YPPK4FZHvTxS-{C@2U{hi=8EauF}quIRcaSs{H? zEh5&IQ}}C>`>*D)C;1|jH~u%NT(Wkhk7>EC{Vu;V2QI>(pDB?#)_$pr9%SuGpQz~f z*|7E?Yfp(4z{WJ^7@FmGx_m{>PGs#V(uw3guaR-*OWe#a*YV}r)BT+JyC|xBu;zJ; z)XHbrkpH|ySRjqOOCf*F+g;c5+mcYNsM}c2Z^->#`K;#-C~o5U znP04Rz2?0fbz0R)T@NCjUDrLksOvTF`6U`oSL=GsFS@AcU&&|Wtmrkr=tNprBaPC& zcmUWnnc7R}-R{PZZX^2W67=^Mz@=TtCN{FHtGc#4_zjG#M&b#FsCJWLHi_%-yB@zA z@FNp`4L>Scs2bw9l4D4+2aQw(WNmNbvk@%?k;8>SSZDoV3x+R)V@&5z=1idnN89f&fW$;C6s7zSovM} zuBjlEvW(>zqehPclSbnMIeXSHG<_)pG^j$%Tx3E*7DEE#!~jfR?8E?rRsiPEh88hY z)HB5@VAS>kqe7mBGt`$4)?nIYa7*SI092qO92Dq0=j93@ED<11QQIFl)tNqSG^M?Q zapkY!5wD?8?(x1DZ{l^Rx#!MBytw-=pR=3hbqaL3yq{z4ZG7&j&dbKXG!*|DbAND! z#=qIz4>9**GJ*K^eyE7SSiOVK=lkvm-H#N)Pd~ptbU$3USo-;$q5GlIk?V7B&DGf^ zTgZvbd%b{UX_vFm;OE0mXFBeaaLm;WNZiD+2 z+&kbNiz5F%+y=NmfcqHSJK>&;GPw)xJ#g=a`xChLzO|yf!Le;6mz?kwZLA{u?guA=OC!is178O_ak#uPe z8HPV|FDptzJ5f!=`G(nWk!k zvL)-)qYF#DV0CYZkW*25|GUak4;b;LcT9!@A4i-9|Grwt>+48|IKa>TCM~CTh z_@{J<+MF}?Xn#4f4VH$FC7l1sHWufslBLT*P5rgG(oZ3 z9rkM%_i2ay*49L70S#g;p) z-C-9v>~e=)>9Cs|#%XEH>~)9rMn9+T<(!bl-QQu*2^wsw!%lG69EY9Z zFevyS~Ik1AVLHrH%Eu&OpV6DmGEENs3*r*!L8B zP_ai9dsDG@6#HDUe=0T#{kY_%)cRcK6vZYh_8rBRDt5bKcPX|_vDXy)K(UV%%fL9n z(!ly$r%17Zij^ofPO%2XRwy3WDduK-;o4iD>m0AxiHiM6v9}c4rPyx84#xq1d2bi%bDh459jn+5#okx!a@ULd zJ?nFwA1U@@#eS{WV~Rbc*jB{?&>JOr$+kY%IY_Yr#okctZN(;I&?a%uus+wRR;)&` zUn%yGVl&%$anH3r*IA_462->j^q7rRqP?fey!Lh#hy~^``EiD z@3_YLTxSC2Xo8(;eXetkVlx!GOR;+tdswmGD8@a5lHL}@-c#&viv3fuG>q^V1MSlK za-7MERVh}j*x{IG%R9PQAL^B2$0~M#Vizm+UBxa}>>9;>q}W4>Q6-S%?p zSMpM5eQ4*34ODEoVw~kmdN(O{i(=PezAowAV12H$R1Tbe^ErGH?w{7@I&HDfQ?Pc{=Q@Wfc7$R_DR#7C!xS5(Sh-^36+1(* zvlXjRY@T8lDR!x1jfyoXcD-UZDfUyvey-Rfiv3oxClq^Hu~!xQqhjwV_BX{oSL~mP z9dx)yzXI!Xo%?YBDT>yZ<2-DAt~2Ba4;x{9u5*TBXDe2(*mA}0QEZK3Saor3Gw%pk zpX)RzwnDL=DE3puo>uHR#r7!nrD7FbaCqFjW1{t;Z&d6B#XeH(Q^gJqd2x@hKGzwi zSXi;^;CsNl<3{Uqox{6&SQqPaol%OFDpsvnjbf`5yIQeu)8*402r}`*)yxxO*_8qu^^N8|E^Cjez#N_AE;MqkRDJ^C`S{iW|-OJ7uCLtm_gnKObtmA>=| zeFYww3O%ou9VE0>}>+DgC@&^gGKXk?>1AF}&=^XjGbY=qkPv_pYJ|AW4b01ru zdn=vk6FLhpVPm2ezB+;`j2OQLkwdVzOOZYakQl`)@KaA zIM&ja>lf1B(dRlpMOuRW-1?-y6YLYk_J_W~e=B_-==C)j1rvy?!23d9Z2XUaI8L#p zFXV@YAM>=D(3d`;uVAFFVC2Vwk-mcY>H8MYD^k>7qf7~5u(Bt1QhXbR z$ooR)mg@0Q$|vY^o%=uy!5*+a;m3lJPe{1^p|eqqiJ1RhIv;q3yf1WasXq5nI@2d~ z7L0TjjQm(I(pfM+onHZZL5lh;ONj{N)#n2>|FBwyzxL$AY{bA81y0ZsCLhUYn>hFQ z(2k8ivU87JcJ9$#X-l7Lbi@gEGhQUv`_`9ZW+8&@4{d9{skw)x?bn`oaM0rojT0sw z`$F4KEp2yO+J0tfTd1_9PiQL`X)73MD;PPlVBzS7)1elG!#ktJLG*%5g$~d;1>wfp zP&oQmDE=(o-FScDT83HvMDDUx^bG5tyL<&aO%sZCz`>dBSHg|UY@oGu&<#+Q4P~8+ zRToG*SbTt@hvydu!c7y4!c7&0VH5Fpn7KNe#pbJe{5cE zUKpC_PYOXh#B1TkE+xA+!j^6qb zSqUN`qw{xTbiv?I$Y!2%!?ax8 z%{{k0U`UGV(YRxIlpAM6R^;@;PP0Q@|5mwycoIA`f|ecIER0?6(3=>!&f9?dI^OE- zLi2~CTsqh$vcix3m2rW2;pzGhOMcAfUC!`BA2OMC=Xo(a)2-A1`2s?}gdgYeEX?Lt z5tKCc#NIYm8Rnbs=Q}>bkNM-5^BegRHnp_~N1|f~7KzR+UYh}tu!MxL=j4wAhUubn z5(KxtT}ML=EPqUS2Usg&Dtm{!qAPI(+*05p;WGK-;C=*mA$}7QcFlx

kovqyMtI zW|CE$gxxQsO~@ZL=GR>&oGnxPfQ-)QbDdd=&9Oe$=zfxL7bF%iKbz4tZpX!^*AID<3U^TB^f*i<+kzj3Slq0@!J1LILkyoB%EbN zYn+9J)chi6@l+G~xe60y?w-Yu9A!14CU6usibRew!n`5V*@3cB$*rHCupGCe-mvPh zz4C%GuE;TPuYk+2=n^A$z+DBG{Nx_|R_+f!Ij|LCzc~@VtDkB6Mb;-$QNdW>>2rxykxY-Bp_mz!B1S;b~!nbjCijp;d+tVFb1Ysp&`wOYL|`V-8e@ zZ45`DLFy@!=~b!NQ#g8K;lqqpzk%OCRXMDq!kDJPCyc2MK4DD!5V1pl3w)=6bOGlP z_~KdAjUe!tgrU((2*vJg9eyuk%qy3&s5ndW`!qTVDX$5cw0_wNuMtZYDnjE-_$Z`6~6AwiOn4L0e@p<6$r5#EFSoG}I_yr;?w-O{Lrq;Gza_jq;< z6L}AFOTNPjkid6XoQZsAxJfUn}6WFExn(tB@4fv_{FZ+=Hpi1-`|WfL$SDipN^zje1FgeN3-x< z`|WaAJI}lb3-F>k`iT{H&JW0am*fPEuB@gTv@@ z*7=6*(BoH^*YFeG{0q7TWymE1#csfaABCfruf>fnBSZ106L5^p+Y(L{G1~ltc^&7J zm`_5X7&MzDWkr(}3ue~LpS?hq z(acER%=vbq0ViD#hVd|{wwZ`DJRZcE7svM{HsN@O3hK!T7=wxW1i5VGMZVC?X8~xB z|0b_|REO5gyrVTonEwmlvK7q$C=&(xTxX_Y=USgwQ<89N6*IJ*n2fd{?5^Crl2Ejy zHwnB8bNEo>*xuCP*05os(0Ur)6`Nq+fmBB((3+eKfs7XCS5wpa0-0U6Lh0#9%%k)} zmeQnU^UI_3bUaF+bn_)l<};Dj7g<_g40j-&qfHuG&%_4_t?6@}Dm)X6!U@5Cqu5%- zd`3>Ae<p)9|L2XWCxRrq)Z^6#zt<9KW@Q{Fo?SH; zTK_10Ymq->JLJSnl4Z_hn_pg;GgJakUoc!HsFyP z4RsZA&5NvTO2P`zff9d|w`y^~)n-9PLnV0V)mpbWa5*|4)%I!B^0PytIyAXf84U{< zxj~zwRngC0uSNr0?3@P1PeqSsSnMcTY%cKI-TX_POpH$BeWPlS9d4Vq`uOs~nFT=0y2Z2rDe*&N1_&+^^qDg8DI>`CiZ zY!t{aY`e*q{9!%Vj*4<9+!rnWd!A;+wFxf2=?S=e?UQghlz?`c2szCbxSUEn4fj~M z&%o^u7bj36#c-d8I~Xp`p%`08FT&-wZ-Yxxz5;g&+}Gfq3HNolXT$v?+;i;n`S$rj z`<$IHpI=~~ue8svu+Lea`JA1^Xf$f+-szLmWP+`>PsRFxV4D|@0~Q|us= zxx_87KG*4|7!SM4d#TD%!connV6}?XDHc&|nPQw|N?cAdC2nW5XTd_&haEY4OS5&m zDQB>WmW#aZj7(ui0(17pq-H#diCFpk)-4xhrmDUrGUWolYSnTP^A$p-IDRR_kJoiz z#B{HR%l!TbE(`Pm)O#sV`owaPVBfM&<#?cAYZSBd+32fg+O-2XCbMyCq%F;6+k@zd z-CS=($Lozio{MI3$-p3fC8~UWjZCqs4v8DybHLu#L_5+l1uvDhhO@nQkYR)s3ucHu zz}SGkqhXjF7yG4XvJ~0Y8(MfPk*Vjz-o9x@_U6aTryZ4Z&5!S4UchTiUSy3gaok3F zgc3?{GM4iAg;U^?MVtS_te((!16&r}G0IKp z6K*QlYWvi6$U_BtN-=Uo8`t_^G&KstDdFtXjh&T-_os(9mE09uq^vE#7-vs7`q2&~ z0t*=*ve}CKs(5X<@px>tI6j15@3Lik!%d?YXmOxC`W+0dyl~-7>uDV|x63jz7nQKvXQAxtg;Sz)j;W=HdD9EMlGF~dJ4eaPp3k|H75pq-@ zv0f&n$!R^(+8WLo!9B;0Epym@CfrJdqn55BxD>Q3!r1`Bs;44RxeCoo7T~_n08`VB zP79dGt8mZp?B`&eB0(`hsWFPfBq*g%D@>zqwAeZbC3QhpRR>be#x)Ut5Y9n^*}Xg_ z&-o*R^z@}N3Rp(uA30??^PpZhE7Pc`g`&O8VCKo~p%{tH`^TNor0`<~IcVfY8AydZ zgRuWF8H_-0emiFYST)&EImWEtHs3AAal0c7l&FgeAJGGf;xhqGgxoRB46F-GLil6w z5MD4As(#Il4SN<6Kf$oT*jRcNEgyJF4%?Z~^I$ty6m$1#_Nl-2KDBL5A1LaQPmNsO z%C|CocEP_kS$q_=adGS4NkWWAr0phBDDE5ATI?|2H%ho0k?>Mj#LG(coll$ScVUh& z&lG9jgzpTtsmjUCser=7JQQnY+nm0!Z{4sD-^y&CV&2guZ2^|2nH0R!CL}K^VLfKtmqQ316f#@c&CQD_eUih3*-R+eq5y*JVKjbBE#7AAU@=1;K_qe!-@8Sg``*ksR`U?4 zYz5q&ww#Ej#%+1=JlPAw(vYh=G9mS4T1OcTg}g^`ZXOmP%T&dgQwdYt%*An-)YR0K z99Ib8hUrd9#+6`ed`U@pS!wCi(y}pAJn?7AfRf4yC1)bvDXYwcw7NMpv|*8-?^T%z zc9`t1GQgzD=(vj&$-79??WhA8aHSV9v+zhe+`oh};Slv@xc%UAD}hXY=!5ei6Y~hb6tX*5^8RD0a7E+-@P^)+zReVs9(< znPUG?4E9fLTx$4|_rkh?#Re<(eZ{U(><-25R&0}EPbv1AVs9w+fnwBPBJVv61tH&Y zr1iPZcNANy*!LB?MzLQh_K;#vDYjLyzbi&H&+^^|)PWylvwT-rpX)pepXB95>vK(A z5$w;3QT3x>99c;vkL6lIRg;_ zJQwc##`uz%VqAl#Fva+gXNvJd-uq24?j0X}6Q&sNgek_GVT$oynqq_*#wFqCrI*p} zT?YtYvs$pl2*K+pt}ZP;lC~GaBepI%cx+_kF|fTj7Pc3UQQM2KS`!ZHSgj$$D3ltVV#c9HYWS!R9K28jR<3Cu4jpmkLl0j9{}>t6n2tu`IXgNT_eJKOf5aXkm!X>JmiGiSrkZu_?7C6#5)43;YiW&>#o<>}O2`Ke5ola@LOUrULriIc5 zW@N$Ub%yf^Ty9>nCIBbKj0JK^>Nugy_GuqGdFgLrK6DsP;HNYMINiK#Z(0$|B<7~Q zkAY*UQDQBbcrM*D1o#?s9bX1RfUOz-d&!hoyz##v8WVR&ahFR3zNE(gSS?OR{zaJd zq!FZ1wfhK@!I9}3oViX#+KXl*R(GZcd0V_ez}W`vlSPAo9kTYxAmCa zM*Xu6q-2~0KDL_a=1uWRL0QX2`RaF*B>t0RVwA7`&n+9}tA7!dB9&3TSIoyGGs;*0ioa6$ zOUN4Ko66^Y+#2OOE)lmz`M_K!FJ}1;0`UVao8>#mj6`B)`MBF38}u@;+a7#%Zrbj& z)15`hY+_-b%HH7;uR*soFt9*YC{IrFgrAlnFZ!% z>#I$~m|;GdPV6;icAmm?;{r6=kJFQx+G)7HIUO4|C#Pe>x}q9 zCyln@f)tJRV)JQ98*Re{Ei~GOi~TS3cOS;y+mP=bk<>8X5h)Dw9l;d+DOtmOeNq|b zqXHSv;2I6{oow!t8RqN1pN9GR`%>nHony>>?0|&#CLd3VH_SKMM2I)c=SA|{4fIU8 z4gZRU`O3_DVwT*-*o^uM$n*QTktwK(<|>|7BvB zZ~33_vX%|=(VmLGFx!|fNn)69`8)e-nC~55(kxBNV7S=E4usQm?^P|9<}MQ<-Y}mR z$!{Oemge%me^tYLpPLuOigUNksaJ%4*Z*Q`m~Wo#9Kg-mxW2E6E$6ep#FznBgEh=I zR}J%twBoQd_M4e$AP*+){$do%a~HJGV=Vu+pYurXxE#xI@m0j)w-fiLZH6byuS~(| zmVev-@rmhL!+d8GZ`PBfFZ)sq^Ck8qo?*T(O&@u6+Qm>tx)NWVlynsGU?7U?l)#8U zz$ptPBTrW9o+Qf@Xv{EQy18${j-gVgKDlAOoFsG|p7{NiuBL}>PS-A}=!%8c_@3C$ zhNTBt_HJs!d^-*Ol6E4?-Zh;_@;M|&jh~vh@qMREzL)Lv=d4+&W-eQjdVEl4Ql6>f zgH{LmOH736CPK_|C}x%~V(xRzc;UOw=}tz{vBI*(<|V!C;p8Q~?3cuw zmGrWkQk3*En97PTY5)6dNzYC(R`{0rjO4{U+Z3}uL0gF5^1swO*q>;MZw;GOvx}zq zp2wb^Vm$e8HpTZ<%d^aCeD=w$d$RHXo@OzHDGg%(b?ZSY=Rp3%J{?7bI%sJ-{%d)uJB_*3okejzR>49owHz3*%9 zpD|22W+d+vGsKPEKc8?v+EnY4w9dk@^t;NA;&58Q4b!RK&$!sT;Hq5lQf^j}bX2SJMyfJ^Z?6u3s};AX?! z0kAnF6;1+-Yz-!X;}x81C61O((cDa1VidI_OXU zmkQ{4zYH$(ce%a)2;a+U_#ZKjSDt8utVQE^Y>slVpK96kUBnCKv}*cpx-V$Ud-^?9 z7XMvzYSHlB|LElHc{IirTQq9-KRU(HsYQcz|8Y8T!{u>Ula~#b^vQu2!B*R+a@tI= zHHy8cabHpF&x&!!m!!8xu`d;4$1dpwtq-Fv#X2j-O}-M2ou;HWQ?YXuTcp?$#a1X5 zRqR^DZcyxJim`W*cf6(;w|5Gbjy^;30!gXGXo60#!xUp5Bp8>~CGP2pO;c={Vk;F} ztr%vIHtru3qaaw)Ylr?y-jQ#8uG2%YqZB(^u`0#RS8TCjKUVBk#eSjKFBN-5vDX#L zLEk8O$+bS$>8@BW#fB(0La|wj%~6cg8cFX$#eSz42QNZ{XB4A3ZNc7B>@SK9MPDuH zjkG@3`BX7VBqT1yR}$_t>vJ9MN)hZz#nAs->@e$doof{Pkz#i%_6x;cP;8rGZ!7k$ zVym;gcie7$FwBF4hLV@C^|{U@#ilB@Sh0%~Tdr86V!u;tonp@@_Pk=(V^_Sq_a^Iu z0j?YmJJ|YMr>kN;6kDL!BE`A{y|~@2&vlMftXQ$O*c~qKZD)NjdZpMAiVat6v|=i6#oAk+>l~)ok&4YyY>r}eie0GKGR0OZ_7lZ^ zs@Nln{Z_Gdd0t+4AYQ)X2*tW8woS3u6nj^(or(>>s+xT7VC!?8F^bWEv%KR}#il5> zNUbP2NkT$AUeN1Iyn9C&+PLwm#Rns=y2PL+f*$TNL|=V)rTbfMSm; z_It&iQ|u+h-c;-z#XeH(Q^nE__41Nwec%y_9jsVa#d;{#U$H@ol`2-I*eQxlR;)^~ zYQ@f1Y_VeXiY-^{M#WYu_Q_#T6#{ONH zJ!sOawLaJRK(UV%I|S#gCEVfG2fm@$@ruD*TM-6%@8M@OSgB#Q>3(zw`ch-sEqT~T zlCTXI73AgqV6Cct_#MK`NjP1L9?M*4Ck};ka>?dU&yrhe^GnbPQBk$<-0Dh^{!~`a zKey`KYO!9kA;j&6<}ei3Rk+R~SKEC&v(5zU0X&ha>_U7}z`K}#4s$WG(o<3)2o?Ce zD_nk8H@N-a9tIN1pq)O~X;SPe>vN5s6AAZ7#-@hYpJES$U9(i7oPhMn2h7G z>pd0GT{xT`ZmcLATG2SJ5C=4%7!f+OXOu@L7KEeg!hIhPH|$AYSPWG^>E(?jdF9cy zX18)ic@zr!J7!EyU@3Uc$>BWAPldU1AhrMPOloq5)iX9XX!8N~)WaD9Fo}`ugG-sZ< zPqHDPzCzZJ3$oe^N$^SsJMb82SGk$Qg4bQ6<@`$srl(fVEg!cw*=&rVqJ z>KwOk^$vrFxZu>@zX|bjgCAqcJw3Zw(Wcj%#Y^vP{^#Gr*fQwWck*L>K?q|09Bo6m zjQUP^a{!`-_$4iWI8#AKMP9|e16a(BZPYV0H1aC8qYTKwv)}pOXI8D+MqXvQ{ zJ2vtz!vDI(`u8sBjh*6N{mVCGPO^s~@_~O3!&P3XC#ir;iYFpIFi9U|l4c3MZ|-Az z7`#{cmagL~=a(kJ&yc&TQs&QA&d8qF9)_E}{H3gwUQRR2N5}Rspef9cDCmXPOVnQVyQ7xzk|W zQy;72vKJ?^l=<@K$!zdFcphQ@in`tLeKvSeAefPre7T}iD>%EU+z4vH5xB>HE=9S-ZJASN|_u#hx^d`+&7VNG_W7ZtD zb#kii`1Qtb0De{YF>kCB{J?YZn}Z+MG0($~e1P1FeH;sh9F6VVzn;sFXu^-Y=}P>5 zf#1FOJ&51K_>mX&0gob|a-HG%(~$(F53`WDt&%X{&+2vyu*6Z>KuLjI69dN8m zAC~_hU|t6!hHs&9s-WgC#~at-hario!^Q6L2uA|nhg$~sYPhT-5bj1gN%#mg%UzHA zTj3JF%iQ7pDy)ap$y}HV`7W#dq4Kn(1DrS+e^Ih=i>KaUwWir7#bm+181{|i zvHyX~BS&68$1p{FMjt!=o=6uVWiI~4n+VyqOB-bTf!$3ZZbr@Vvu z83g;KVh<|zrDCXAHe6Th%W--rHb}8yie04`HZ0q?KT+(bic#-^`gVfIfm3zw8VybMgW#hO1NY&;Xi}SVM@p!F|o%gm>p+kC6 z#DVsr3iwu75l5pHZSZiPuoe1TXEnkKcDwbtI2-C=WDOE-9MY9=?8gNQQ*N{{08zqv zNCQ-Q^fDrykx-PFT$>B)0H!mQyZEAPReCi3qFPL7v(k|T4aJAqWei+ zn)sK>#oJ@tK&r97XP@nr0^ie2sucF6!0#f-K}#HEm5Wj2@(cF9h9mg}^H=#l0lg%l%9FG4Vyk=~bxJNv z^U3pT&R<+ziTYeQ0~(@7ih*SqSE#NtfS9$1zQ8I>J1S>Y&#kU&9%yuEGzCK1D!%C1 zLa_u=jEh3ytMJit%nT6crM2oav1YU{@;(DDzi%_DKEJQf;`6hG_6y(=uZ2q*<4Bt2 zI`m;BMX{e)pX+?0*dE39hsFV9UaoJF#-Dv%8q;dI$6lD&e$#l7rE#65F}im{W9-PY zG^S5zEEs7l7-=jRX)M?}DN2sr|At?xPMFgkx#C%e=|OEBwnmj%t1Ig+s;yq+(}kO0 zD?B6>_~eDPbE@XE8{XGWxaBQxWab@En_2ld@#BPzcFaABCzw@6s9pHA)`|WFtOCKD z9hPNu%koG~pLC*v-HuR#1<*7E%eFq(@saut1Pmu|3})K9n0dJ@smFQ(Ji{M~(C`!J z*4B(llJtqO!AmN-U8OW)Ld|MrB8du+xpe$G+g$c>ghYMevcvyjU)WIq;rT)&&gLkh>w|>v~ zaqhTRM60jr|E<7^OY{Zm10K4AvAnew1-5LN0r)!@1O%M<@YNgNV)(c%Apm=k5JTYe z15`#%&G+!#4Bra#^r!HNnz8_VY=1a_0zIWCXwd_L!c3~;5O)p4J90^-KqIS}S!p#) z``X#eIaLcQ>lRkk)Jf~J?QGHP3r!2Vy_@zEp2k)bIj-r>2^dyQLcvl<7F&c96)d#~ z;THz2LLCTZn)6zI_yJ@xnAI;)p7T?}nY{=fv8Rwo&TY;U3oBbzdvL(oz+yjEY%b_7;cBf98uAsRlupvST`@}P1Y57zCdDYZIWe1g zIm-H6hq^P%4ThB$5i%qDg|P_<&iP^S)S+@({)GoAI&_kNwu zP9H|V83{ZVyMD~o34iZnV>g$d$702`x@z8x#j{)GvJ>Z5T~JjcLz(pM%4p*m>LDoy zUGXmDt3)zq{xaVmf;uF`K#LE7JeFR}N+VV@$S@#+ua(H#Jn46_(6KM6aj4N<2n zQR#D?<#;AoqxA_<1tU=S3mTILOW;~l{mr-F~&Ej?LETLlcq*tuk$;H@7>&(hC38cez<823J8{-(xHWO z?ap?hP!#@W_f9O}$SF{|F)J;+cVoEW1E}>gMhdCCG90br7LVUE58>SWGAA6JTR0R0 zsZi-TTH0@-QD2Zy+P}azk#e`W`Dh9KWPF{muI? z#cFt0n6V3@mm6o)@i$i`=ilEDGQB>xPJ~SSCR63u>)VBWw&n^vHiJo+iRb+(xR>h) z{~~{EkHL5Zz`i$J%P?#Z-D$lg2L6ijwTbZg_br%H)4h3K^fdn%>sGX|V8#WBgDInn zBrrHQG&^1!39~l`Zy#;mHq;DS=J=@74rYq|$;)}*1x&s%b}RwZEyVU+ zX-+#bSQr>f5E!>+P`x&Z-5ozbUMUUBUj%>`&+qxnmtmE6k7W_jdN|jjeDOF;j z9toF=GQ4ks+Xe1ba6@p(O-htg(C0cgD8>nbU_VoggEYZjQ|t}Jb}IIPVlIkX-qGIr zu+LhtBNZE=7*)K>JE#><;-041BZ~c2u_qLJTCqQZCrDg!2T5;_V$_@@Sj!v_%;~-I z=!e`N2bOns9Lp;;dTU!{cO@(Pl;l46dyn10|B_{Q8RT|kb@v%oXE+@gTRHcQ8IA{b zM}@ixoDK{0%{bj1pfyXJoSnO<iaz01f{|MZMs6h- zxs_n#R)UdR2}W)u7`c^TZBeBJYiE5pjiMNjcuTn9ij7umf?}sCb{pEOq{o&kaet%O zTE&dC2?H$%nc8ymZRQSS3EJT??30_*rfa6m9Uff-bu zQMGWsnKw{eKdENcMW#)6MFIFmFBBrM+oKi5_{lB zZkb=m8R7=F*feC!t(C(?pX;0qmu>$vxU2-;lC$Xxz<; zeI>&G644WlKKTh>nUwHLWO^S*`yt;X?U}fr_D00~b%{SuiBF#x?hvd7R1s{k_2pnh z>R}|ggxjK+5I9_t2eFNrOM1_E`-j?KXjjlS7;da(g9(~g$D$L=OlUMgU}xrY z?#4#L-&)k=eQ!W~nZFR(fIL>qmzLlcLh?CqXW_@|Oh{k&P1E4!!95Eugm=z#xMU1f zaJ$+2UU1LG{ZVkwfjhwBgW)pV2z$?l$J3G0@aU6&PlfQMjo(BeL9EVJP%#-@ES`s&X?$d?0dc+KdB^8wku z^8sDGT%_EI&HQje(9^tX2xuy=;`spYV^g0GINHRdK9nP!eIG^-$Go3q>)*4>GW74c zWf5^Xm%FCCy|g5D{QAhi!73PF{fTSvrp3=O8I(V)%iO`kwTiPWzA9H3`48X{=RLE= zdsh4F;2sC}2Dm4{y%FxoaBqftD%{m@r^018a)gu6XoMrsCrk5!{lq?Xod-1BBZ`HP zF9}y@eXdia7}YmOdKW2nsbW7?jJrA|9QSfcdfdw?7%byitey3_&Jl`nSEq!-!7>}} z1jQbK?>)oqertWMLrK59<9!Vm-c)MYP^qw?G)IMqpW$m0g@`CobCK5EVlCGJ$mvvo2yBrSkmmq7E?# zj<-j$;Wol0F|UA2(lo(k9XnC$7=1ED6YOUD6qdD+wqSYIhy9uj@AnSNF+np5*n;wO zp33FaAP67L7sRGM_W?OSLE?+=L8n`1G*q0&zfq16D5J?nh!XrDqrod!*+{e^{Je|#yAZz&{0c3IAw-1U!{9RC7)+Wb z`~lj%5RX3BSZWdM6oeA&e(TF|9#+gWVLKiy3OAkDE4*nf4OfRpJQ>bC-X4?{19C@& z8`ky;H*DVy%h?z;ZB}+h5#^1UIc;#pG(9t`V=%K#p)c$}Od50AVY5jv zqdm@Bl|fn<$j)jL%#0gHaVW%TFpo;kUBI)FJaFZ8C2TWT7f*c}lN)V%nSjNZwH6rZ z5QH$e%u)r@3M0rs6Qpfi&App7H@l0+AgR;B8*%x^!X{Y87<{1jGzA;Q8cZjQ#aMWE zB6-Fo*+`D~aTbmxdGaQfnH^_wWo0Ph@(h};ti<+4Of_F2e^@uTF2!b4Vevm_2uFDb zE|d8S+);3OUuN&i?fpc1KiS^1A3R7o5Ph!0IhSDbtWWk*2=-&ex*;Ee(F~WwWla){ zOCo~JS8SnTpTf7*aJkQ|&vk~PO-NiEQnqo!4Hp-{u;RLfu#T8s5gh~nv!=~~wDw>* zss0guKSV+c6PlDvJ&+il^VFJL2WVJSRa;vSr zaC8ax8^u1=aGxu74@f5A)>xnGlncwBo!ey@!eKAW-nC;QsCkQbn+kR~pka7%tTW|2 zBC&!|n!yVllQ|k`ZYb;*3SmHKX!e=Cq1J0>#+Vv5BjoC0zOpqth>{Twj5K0A2#g@XJsd;_3hJ z1vmj26a^V|@QGRsk7sQIqf4w(4G6`N+=ckDk;?UNgQT8aSwUW>wLkA%kZQ8!~Kg(csww2M-xoG}^R$BQ~lvne;GP6GA1=j49pIi0w=>)sa1Vyd_QHFz z5HfC=7tkjxL@=@t!S2*>KU3^U#hy{@1I0d8j5-%2Ju26h^m-}QSFvjp`;lT4@^G`px#Hc8i7rXO=MG|*~qe~(%j!Y3qA#IwF7x{HYR6P<5rF<3{^y* zF?{OfB&>=gUXR)tFwEz3kS^Byy_AGitw4CPs!Q;@36)r`{~)Wn2%>?s=6r}eC)OvD z=g4L>Ry~SmCW0?<{};_D$}SFun2I8k>wL`R(5zeKR5KxG?h?m7gi{&8xt~-0m>qHM z*X304OmY7>RpUN$D(01+v5%aJY>AvIjxBNKKM^1a%o)xcZ0O6*|Nb}3sH*$VsGdEe zXz1XoS;Yee4XG}kT{Lh&)!^cy;-Vn~2F)H~8I=gTYHeB5VZBadRfS+J$i34OE?HGC zxMWpF!7YN@2QFDvU$|sZ{otMk_h`7&;0}R18*UL?win)$RgF+qMW3)L!N{rvBdZdO ztV*ygiao2?>xykx>|csu)y>A`d`;5pqZp@8f?cl|S(RX9Rq|f)p=^$*Fyi@wOKc;a ztm|pGj?qMO%$Hl-YRJk4#ZzJ}Ou^3JY6GuawxRS+; z#_!Gp&SDlVsyTPQ3~kI5P0xaHG1X)rE!vYzAP2be414)?%QL!mCIYvv%J69fMnvHy+1k1`ZIdx^fo%b{7D$b zlYuF9Y!j*H=Kv*Sz9sPw1$M_{fWj|l0^ULr-@*)d3rYM4suy5Bc@&?DTBx991AZIv z^N!^gAsyc!#*=f9c*$mn4DMYlcQ(NcxP=zX!OW0Zg8~M#+TY&J6pOV9(I-tvFg78< z*n|XQ6B2B*Vr)!;y`mT!lVCwqeu;~Dlg0L7_BaxmiB;~_W{=l_zH%L?{`^ez9tGpY^_bhjjo-%SJY!NQ-53~*wJm%K&Wh%Qnwnh<>KczlOT^?`4CesxKAe$^$C|oUqFde>l z5&FFdo7h`W=m_Ql?gDj7|DFcLks^BwkpvZ`9iG-Up~q;Yk#S%tw#+*t=io1Cg=rbh z$FSa*m7+7O9;F?H%?0su=|iz!G1k$nL_lahYoF1PXCeCLPMgp3C>BHMx)ZLrRe z9?Wd6(=apgH+E)zEr>Vbr8ZVDTV?is4Q*Uf37!tP&95*SnuzY^5NrpQ z?Qt|$C>ia9tr@XQQoNrLxgsYFsvd*h94j8}{JTjg`p<}5ZI?oNU}FoHLQ=1MTxZ^& zV&#LBWJcM8QC%e|8Oz`j6vtU+ER(PjLgN3%L=h{siNyC4$t-(hM(*(HgMWFXS-70Y zrext*G=9s%?~+zibNYZr>Cge$9>WV2`HZhxbRLcjd#hHJCYu*J!>|N(9+)@3n1_<2 z2=dL>abkh$1x_BAA>g)0#)Ms6qO30vF>I<$`F=>_j2fw&~yD% zmN_mY=Sie-*~UU*5@}pvN@*O~OfPl&L!G1vuJ26-PDcV674jQ+*N$XtGY2{P#$P7?|fLW8g(_ZDS>7Z)EenV=a~ zHj@bso;_&D>|wfE=T^=aEsv1hC2}Mxo}6qoddT7ZVg;6bIyg!Zm1USSh&3B1e?Qfkpj+1T)$4R$f z4=eT?#WpLpMKK;Jm$*DqE^*ISY_VdmD#nv$5-uInDv6tAeXi3%u}+E&Qf!!FWr~$6 zM(aD0-gd=4QEZQ59kK8s;SRAr*s4+NM~eMgvBwm9O0lhqb;tUoq(|j~^tn#4VpJ|D z813XqxGKfwDR#bMI~03gu|0}?sn`y@=`D;ub7(!#_*`cL?&Teutq(h!l1?DXav$7d zguD=*ytJfu;tay9IcS`lP zE@4BglXfg-AA@}Zk`6~M^4mFXD@8|6BizUv867x>$n+0h&448{vYX=rzl_aD`0eIq7Ob-Eef@Z_O?@z9yYvxdxsGjveZ zj2S}*&Kxp(=8ze)s|O7kIAn&d)3IOo&mP!|4aLttd$* z-0#302lq0#VYm^vb=#$xlV4OV&#@T~l9E}LJMX_fU zdtI^Zig9^P;&ORT;+F&B{ZCRra; znkqI|v1b)~QL)bz`=?@UF~*nlXbV*G+)uG0#YQStqSy-EEr^wAaZ(k+rMjYi(l8gNi$JZ=eB$hllJPn%oG0P~D zTrm02l;X;^CPc24dP#7isfMK;&6-lk3moTim6F_xLILIwvJc52L9Q`0+j>v}Ft+u; z+n31r-bETs#}9Q)mm5Dxb0Yj$EK8fytGnTnC1iq_!V>5cI|G6(uusL#fMAy?cB95! zt=K(^tx=4E2Z>9;gS>-mPOy>I=Q_C$rmsw6y*4HPJlw>RXQ}f-%`@w{=Em+$ZMo#B z=_w056eV9#4I@ULY*1onnS#c6n$MH%XwEW4CxH|HvgmM-#B3hrD^PUi6)d?t1ZJ`c zE5o}0`%8-7G#x*GNnsCp1d#IuTz>06Y)NHlNzvyz-@-G&=35_3>MF+JyM$valW^}V zMy0CD*DemgqFS~R{ReoQcurqd{iU6Ji0_e0np>P=imev70lZ(xUA;Gt=X#dF5$dzR)J%@m)*0-=13oH~ojy`5>(R$X5Gmy8~Muz)=3{5y^(H$7C zZuoti(8b`0Pu1M&;lpF)Qh@cEq~&rj`q&xAYZQEq zkLGL>6PZbSG0!1zPjt6noO1obGLn1Yk{_*s%g^CGB`Ul>(%$o1dCvm4ObdWM7b=N8 ztjhZ2&;;D$8Xa203x|#*^6DV}z*QF2>h@;TH5NNk0&}WuF zdn|!|rUar-4qyp351%9$$s!orpJ35-4NnHZcEe6?nVgCuQ+O6>#9Ax0)g_u1@kqs9 zf%=l=^2;`B7WkFWHybL@io(pf)m74a4M9stTHQLBp+X&!$526ZN^>m128DS8F0mY` z9RN`7A1J~TjxDO8j(d?D44jIukgEgeBP*fT<1=Mhhx4hc?CG9q&cJl)|Tf7g-k@tOVdA2iSdwvFeVjn}WrHb8beL2o; zilO0{7&%lQZR0+#*vpD>iC5B_WPRog^_*zwK@D#m(s)W<_jPq`%9gFI%io!aiNc)q z#m^T%!7+`qg$okX@A9d2Kj^MhTHQs>4Yj&MgL@HFyPH&eTHT8Tz^=~eAhK1?UJRe8 zc;~K7QSn{|pQw1B3g1npj?RJaN#k1ppQw1h2)=j~?!>)F*%pL!RBla`QXjx6(-D9_g<(ULyIL^%odjoXUOgho&~VMmSato@{6^NS}< z;t%%6^ly_EzSeLdVmE6I5rC|}$y$T|jii4z3W9@^v+(291$`pR2FF7q7&tiH;qv2q zz-23gMmp1%;=F|E9&lR1tnXs&HZly)kA*uPE(Sy9Ir*7Px9D?m97uRvjx*ExTmIH{%2b#Byft2LZAYkzeI zlEVG=7_9517i@GG~$CCy&3m4+sTrP9zRGe5!RGX;ZP zY<)S#ew1LGqf5Be8t%7>J+4?0s4L;9Q%2&pFagR#fZBv&X1n0U{`x7jN4fkx@Yhfa zHLI$w%CM9F-ucfid~k{VYC5C`+Pe`{47B)*nB36&5?nT(m*H~q!+X+=_Z(i1)uN`) zbvTR`?0V~S9g4*S<1|RZ866POb>;PY&>mEBXmHpb;8tZLyP;6Ch& zhl$=N6*)zskQrBq^C!`2I*&EXMNq2_#*-Qi%Rk zB`*AuMQ!_LLZ)sZ4!U8Iv_(!S^D~C_F4nU-$T#a88wsyJWr%sV?QoC5{hM$Hz~w!| zbkv4KpU5=@TVS7xTvIU0H6`4w8ut#xHYxU$V(fAy?rVy5LLLP>%=%yxR54CoN+NF}}f{p>t{+ZC+q0>K3q2sLTnKkpO=2q4&sF^QX`ul`hXz;15^|GF^&Sc}2 zOu$wn+O%OVc=Zi&-a_?G=@a`@f}O9} z-PV`mkSY>xgJPQ%8`qf5qpgi6#-tzFD2G!MxF*>>FS7rOFyNI-xTc-pTe-*Z;hyEc zgo(fD2s?tuB%Ptcge#hT9nuBYDqFY^1~m;2V$+c;2fI^ZBB_PQL=e`&u$eon6`@o~ z%t#<(c}2sl*rMSe33uM;+rB>8$cJ>e4q5V#e880=%p%5l5z+_z%@oru+KTCOoh5iC z7z)z(U?o?vn-t?{RKkUS^*9yiuBG|(k?l+2gWBxysTRl?I zQ$o-nig5*aLrjxs5?v#nb_ty)aLUYWgfS6LBg*511j3($PzdC3TK!h*gr8kCcTx5H zs(GR!lA{P<-& z+c+0Lo^51qdA9Lpe9AWbcH+kdOJ+x6*#_G}1!GxkPpDuYqe5|B@(Em)(Wh`(!A{o- zMxX1w?5ZNQt5E&_5fA7m3U3!Al6^$-fq3pox5r_nMej;h;r?zb+y}P8eP}D(6s>Ud$)HX!4(bHE)B1AE$s)lh z*^qFodV=LXxK4WmF6)&hFeP$QUgWeR6Y58thVoepD(6=%T)1F~sT$)ku}{j50%ktL z0Znt`Jl>dwjFLiQHXroim9rNt#9+#F;!}}Dxg;U>wJ8L~zD|WXvTfcoCC>CQ42R|d zPg-md?~N@m2ht>zz#M*ny+weXtKsrHerWd^T?39TLJRs_XEvS*_ATp!L3+hjDR!@7 zzf!C>K3?K-xlhvb7)Aw+WW7?+cij#Ik4dOlV(}8HpCu{eE(pb*hf<4C2cVF1bkfQh zqO}+m+i?<`w(_u{2B@g=?MP{ltaytFu^DXpmg>bBS}!`8cZU&~N!Q}DP=$>Bqg2+W zScgJ5=Mqm zCryS;^&_KT!rsNQV&!8cWbec4dQ>)+-wklFKb9RxetyjJS80c1B|gbWzkZ|b3S^cFYt}y3Mcj`jXs-u)VRWuhPOiH(a&?s z9$)@El;NDhWsOtAP4odP-Mu!uE^DLHJ-l~qX>{{OXKwwQs2NWE=cM4`%+fUrc6P=M zI8q&MX6?)^UAsM_boaBRk?o<<$mXfzqE{5|VdjwR^l>Xo(jG0rT?NvgxUnSNk>)mY zGl7zDBUe=5o1XjN6@?!%Ty$%=aa zww`r=a~_L$Eqot?C;>BCKMvd;Lr9Qrt@!ZtBm8z^ig#fG-{xthBk+t_e?6f|A0P{K z;P|zz*`O;jT$`IOi;xk>CrqpC7K*h>5f7P%_;a*r;GlW6 z#v~cDGZ!{V*5V1Gz+mlMh{lqemHQB3INl{W5Oxas6fdac4mciA;6}1B=zKOeBs;Ip zz!NR}ErG`@o^9aifQx)102?!LEz~P37Q;o@e{Pt>^&J6}&w{$|j zx3Y9K2FH_;A+oqNCR*a3P$^fIgvBudX5V#&=dVVU&~J?j^rGiTVCX9o9B?QB^meXhgFwP3ec zpXkIE>^{Yw)^N`$Mk+|SPZUFavl!d8yrWPt_V|KPL?&^0)=4l5!35i`7`Dk-Y_DP* zyh*r#^>L!{-^6wo|9_F~zEFTtHkU}Z{{PN)yMY$}W4pFW!1>RyU6#Z*!*<((ZF1NV zgv+<^o@|%*WV^g4+vPpkF7L^9c~7>R3zwtX!OC{&6SgbZE%vFfUBSq9B^=qVU}U?3 zk?jgbwksIfu3%)lf|2bCMz$*$*{)z@yMmGJ3P!do7}>61WV?cq?FvS=D_A(XVKVd% zO)8ySItAK?n&#YxvF~bRE4-=fv42tjPvg|=+@oqA_K$_j_Kv6h;~mgYR1tl%`1x?- zDMRfcJ;ppUd!wruQfVQ|LhlCr)zZ-5U+g9H2d0seoY}vMrfox~z z$vg+$_lxrA<3?ilfhF#Heva`w(~@B@awbm!1VPe5|xn#TNVr$0zn4N|r9;78Bp z8y~L5U%+{M`?zOUPE6le0<%)cXt?3h4@iRX+jiyVLy7RX=rhRFXQY;)Z+PZrOlQiR z+@s1ikz!tW_{{Wy(*=Cm5-Ro_Z{lu(R;Ti)OcPcioezwj(4%SpDvI}kxYq2!!8Cyd zJ)?X4>)k`lTg$2Cc>G?SZA5P@+>S5E?R|gYD!7<^tcFusS$aQp9bg+s#H$ z=BB%O4F}rBe1o^dE_0EyA#UhlQk;wNxNlMv!3||C-0TB*!nv?j!|&(nJXX|j$ux#a zRkAVH!7#pqGuA9Kuo#awL1@oztg#+WYzt$dwh*g9GQ>{yVN|q33~GRI4ZwDN#ue;Y z6Rv04shFBg$Fe-9bJKxwx`iCzs+_nGi&l%Uh1d|7v(uU8!G!eV)6>nj)Y*V_OBR>} zN^tK#OmG`!F2O)$^6A%cx&D;RLH3N(n1uK9pJ#8wq=qbo z5sMnX1|Djg%vd@Gs+f*;gPE27^UmhnL-x`!OwFH?(U#O_>B+&&vnZuw>7km$c7{<( z2PL<-WKRldUT5wzAz6qm3vu&9#?mQrGWuX{8-W^BSszNGTZt7TjFdqmO_>!1AB-!* zR2*)`@ZOYkiwUy^1MIlQqf~~NNG)ZL`!llIWGt=BnFXzAE;K`xdf97L?-DcmE%P3> zaH-Mf+KhUI<9GdJ=NXt_`=z=5P3s^%Bd{Av!i)O z0cvNA%yN8XDIUcthX1*|ul|iB@2h_!#rsJ7xAAkFP?Go6zwLjYU&b+&HLiYa&bX{V zMkmNOI-)MSFeB4Dr$<&+M%SEf(1x1doeapfrQ>Id2%mHQ2J^z{ElD@dGmoz`(hdJN zigcs?IOfT(UT!ba%!s@Qoo#`PgQ33SbM$Zv%lz8v#TXpo2KZ4M0O~G23O+vxGY2Gaxl~~K=z47`9OhE@C zZ~6(})0ZzQ0ZsICg`M%nqst%O(rS~@8c1A$;866 z8Yl_z^kIWfLm8V`@nJW>TKT1kH`eJ1j!43>6$-QMr!7#@62^^unybwGJrTWIic&jkqCa=s1ImYe{c&0P@kEi0L<=L^ZH=iyM*BNFaa2A zS@K>nu=-s{{3%2`9Unv?+G_I!iGPY9M7tG2v|f1NR~A72!+OgegY}f{)G%?l-SO*< z-vIpBo~rO;ZpbHCr972n>^aZFk89iH^jzO&^}G^4RtLX&vNW(`YC7P$x8FFVw+2qZ z-b!SHg1rQ#0z(IZo95?l$I}E+--m!WyuqZ=3~wmv+XF0#sE-9)i$|2V;|KSh&66 zPJ!DOZYA7)a2e(pxE$3S3-?;M1K{2Uw-_#~^kBHG%0u9C2m`|-kxSr0U3TPW_WmP; zqe_hx&haSX*cG&140nB3WV$ima6n~1y@n5;-XDKv)h$?9NJFEe^L;}zinggpF`~I> zV}h_UVGJZBmLSu8K{%2VG78DPWJ-KtT3&H=mePQnu{Ow=!!Qs*7M|#nX}n;!*rzg$ z7wkU8p3}H5DfXsf?7M##hTbYOD{2sTI3Ov6~dTMX~!7 zdq6QRw@Z4zSL`LlUR8`HVrBn@u(PZW+kF+gOtG64yG^lh)8*6A(B8w3tE1$Fn34TYfMo<-{|NyW zmYZRi>KheeWo#@#5&ut!<-~+o8DPF5#43ePgji(FBE;ejauH&!fiGT&H5JV;QHb>b zLQ#lC_rFMpwGaZS{S{(em_GNb2(iAMKJWjU5X(hjwIaki(FFNwLM(Rg$%R;bzKRg* z)btbo8--Z@SzJ2|n4!n9fwL^XNXiPgj!!DwIzEMP3o^v`l&o;;v{b?^5jn+#TNj#g zNG9Amb3cV!XPT6}qg+K}(tA9Ji`zYLc z7rz#STkrbc*NSi}ry6D!_DQ(4!o08*;nt1+6K;9L|2GJ?E=ZgAKjD_HCnUD!JhJ0p zUW5?zi4aV%TNL}b_2oGCDfXOV6oN_IHx+wFF$%#XJqp1j z9ED(lQ3xg&g8)27=>VhQ3xg&g%#(%Vx1K0 zrdUtKDCUxPoS@k0icM4OXNujc7!4grTpBu%Ja>UeOE8)@5bR3DexMkZR&2O{^|?+L z#kwm-F_)wlQEZiBS1Yzhu`dADn>DvU>_-#2Fa3Onbzkz7b$kBV*ii5Hvz1xsQ!R&wl;Z5Nm9C{Y!4bB&;s3n zwopi*#TN(=N`cB^5|We_+9os!6l8g&Y~@iXJ1A&b1l$o7P*DO@D2h-K70|yHMP-$O zD8;Sc?|073x%b_?BrT$Uzwi4``!eUAJLk;InKS3kK3~Vs*K$0S1`j$2`|QxaR$5#S zB2Xe2f#Iub!M!vX=bn}2-TAt$l zwT$O>Cn!~PRlO%c|-e?fO3(CDkvZ3QfFl5$16n}gxr%ZD~_u@m0y$LS`EkVlZNB>NyGhT zm(kCEGN|8A_b*{Ecq*13LeyKZhlxgr)3tsNQ-PQuw}+|l-@J$E2}G;|sP6+(d*bBjSq(o)W!EO{b3N}vjuyYo{1Tst>44MN!&qJ^|8cbbHd)( z&Esl;gA|JoTK3H0z990h0o``+%wH5pJ%)0lzV2KsREvcc7HRpgw)U~?soT9 zXWHVwEvC9x9?7-{|^}7AOD~6MaOKGY-6P990%3^KjRA_IL8+0|IP+dmc+lu z`0fDX$;L+#K8-J*X?*!ilwjPNs*DEbL^uM|LtrcJt4EzdrL0HY;6yWCph+zJv`-H(?{t zXJ?f^(0+E^`*mZ2TC`(p*mK6NmvsX`^l5 z;PG*IyRL3?#cHJT4-)vv*>7_v)f2OSj#opu5em1xr_a9-yQelcPJ5$Zb0c>2yX{Bi z6-Pdatve5vS04F?@=9#j$y`v=km>nD>XOYDB%0U@b%?~zJKH;f9Y;!8dgL0L$)dn`G~z&(txuwRp^m4Y<{AC#UJx} zKF?&NGPAEvJy}~r`OYuD5QvrMRVQ&n=F?h|r`&|n#BQX|N^z}b+LrEBkIU(~Nh!LO z>B!85o9f3uX-9XCug^T|iWt`#9QnTOW?H_MniuX~I>~Qess>uOFWm^}aMZxUfVBu? z+<<07nd$_WYemQHc$>YVqcO*dj=K?RA8@4q{D1a4d59b&0$qYTHQ(>nk;h}Pgw>u#(`FB|#E z=rWAANN^$cQIr&Og+l%)k$8A75R3alNp`7p-pDeVpp%i*vVyd19f{)aVpKb%-|+K{ zT*@A&Jw<6>jpJqON5S$Ph>xaKmW)if=ORr-+X!UPJp`LeeHlt3wqYFHHJNu3tjZY)eS?^~v3${ZYDPM3;~- z;R__*0#gi%+5e)&@Hr|&*+tv&U9{xpQUUUpf2(wN98y?v6stTpNukR7Q;%s~?iTR` zc|S+hqe!9_B_a~t%|^?}U6h<23xZGOor&sBqT|gBjmSiFwa~f0y{^5v6m$ zqhL8V7j07*r}-M?Rk8dj;-kMPUs*mgV)-3WtKgRFEWZO>D+;5brGgYM|3Z{ow1kdj zj77^Qm+p-c8gh6aajPYT#T1EL2@TB00c&T9q`2QmE$oQ_Eo?NGD)w3tdo(>xg0r&^ zjHBF7Mhu+W2!d1c=j0b<$=%5q`v`N&7%QK0$@t>~$r#OrY#IA8`^!*1lnVDnup!dd zad*&Pu|Q%bUeR7Nb$8%*Cw{zAl(mp%m$sUwj;4||gw~m*mutrmV`Ocixuw~R?j9Ow z=MZ-d-HqQp_|fQ5gZ_-)U+`lY{1ZQ(oXMR;qwqTjzghUP9H@!hU9<|ntMQBOGI{{d zG=6LGdjdbd<7hgTlSGdb%HDM}b>FGb5L~5!vW(4OQ~2J{{a6H_}Ds;{t5io;4?k1!~Z>eY|T$IJ=jo@egpp7@OMCZ-i5y>d|Y3dz8U_X z;d8G4efTiQ!6x`P!Zr8+J{vQCgTKZ-zXLwc;P+p(?aA$FVww8=N@=7~-9RU7hx?WO z%knkJ8lGr1{ww+70Ps}wadaCNAsvHI-0kkI4gf3eYli!=(Q*Zf(!OlC-x-dwQ`$cp zZUpp6ajddR%W(0>QW+&+ezVz`qH*KD|L3q0*;2PJ|M)MMp0%t48u%5XavuGMg-8LrE4XB%#X z;Vv^Aw_#}<+=iuT{)XWmG8`|s(r`~3Zu=sSOF9T!iwrl>aMv2{M#J50xGxy)NyGin zaKA9zuMIZ>y>Cs+7zbfy)^OOg>GHudb2J>+d1&5_GTa=)-DiQrAcZ8Y594Ts9&;w^I!j+Zyw_J&Iuj$5-d4sOlTG#_ZVTEjIMj-v`1 zuFY@@3^x^PQLaO-N`q+*!fGeOJ#4tLGJMe`T!n)Y!D_=@Z@Bo1mJ1+l?)kYQjjH<^ zev!>yI4i#DacIjHofZEIzgXgIc}Vu12pvt`3p#qbT9&kRHFdRjihWq0+M6L39Ha?0lV z%*NDstj2n&{($wV@{hfb1GBnuF4ZC)-MX-1b@>COsmxE=Yzi(fNNs*HwfXVX0sriI zHI-qAOgD~s#r~4CT+qtq!`*eIz;-%e;P#kAI4!`|Biqf7a{VCpdLv>|K zw*cc?(p`lZWi*X9e6yRxbyQ;|P%RE`P(|QqhPtt$C>)Ba6$IZ~pWDu44$olLhCiUH zABzUlO!pG_Dfnl?KN9{@__P4mm^KnpZB!h!QE^-}tvEKq700H!;?@}MKEr*(aJ`0G zYq$-Dqn%KG8x8lK;nFB0jpJem@j8#VFoS>S)LpJvzDg^WlYO+ zmMiOPt*i-YSu2iZtvFg9#nJL8j+RGpv^H=5%;d z3EM-RV2Vc7A?I}KdGy+dW_*QP-+1Pc=oKI(Q#56SY1ovB%9?pf#$Ad z%bfw?t`BWU6H4H8c$`}$xP=Z%U|QJYt}z^&x*G0H!)fKljUx4#r}|tf!ld(*ldzS+ zf%tfFaPbUc4@Tp6z`tG`Wip*g*|3f4ri60cDZ)843t;V6Is6O698G6WGi=ZD4(9BO zLij$MvK}Gjn-tpkpRN3=xTb6WmxE9-_-$|vUm5lG{ivry`E=%$Y-A+wJ8gNQ^8Sf8u<7*8{P(cEH3U>c|Zv-xSpzaxC+7N>!`+!9Iz z=NT^TphS?#pPm=ho}c4QeN1~OK|0tdnProQ{%_oFKeGwfbT)OL(@EQYQ0190aIp0a z?en|ERveDPQdca-$Gs2dgTu~r@gPEC1!wBWf}L1K!K;Jt=yA$cq#4&7Qa0lkmE~|J zOB@WEW`0LFSr(ft2_*u4Zxz?&AdC+h?s~)V8=&Df8ZMPNq_`oIDsGrwU))nXYkGc9 z!F_p;PY$c_WJqXstPrX|PB#X>PsXZn{G!kHGkX@E)v~DFU6$JK-JVraK4ZZW<#T99 zn|_N!@mJwU!-4PqtU@&nC+E*Vx-f9cdEJ3Vz{2`Yf)4yT@$>aPa{Z_Ourq}yUP)p9 zSAy!Dt+f=AbClW5kR>a{REL{`OF#)?*=8F3)w@g!)8`h{T5d7>L~sLH{>Bh9ke?_5Y$o!X)gt16@WQ3V)h z=d_`)qgSRSZ8lg|CLtzcHCvt-fUvPpmZd(+kk+y>qiHi78>!G7bE5?L^~DS}#@=|? zcTDRl?64DR-b4u+|C7Xpv!n2(q3o!6UHEn5$9SoIwZKQTkD8VOsvAFOpSygPI@t&e zQz$x#jB+A+1aNetfsfekT0!a`mP=e6F;JLhS>b5O^d$ILSDfD8Jx_PfRL46^9SNyA zDvs)?xGOE(YQs?-H5}DZ!}S_&t>OM;xc3cr4TP?=)H0?0lHu+%T=@fe_HFz&sM}(u zi}I$UoxRiBG57uGK*OWGW$~#!^P3iSw=ZqM*_#73LfGDz-?DH~dz&;kc1qH8<(Tgn zuor;_hc`B45N%xa`Q9?^8A!o&8N>fF>f$GX?6?kyDr;4`lYTNRGTITNR?1_kE z&!owWim$a>S7@l>zl%T&=pEC#0fLOm5V8@yhY3wH4;lE>bIwPq3<)KIg@#+~paf1h z_TeZQ4cBWp>a^m@A1K-m505k6H5fArJ6KDBiBBm+TRBZ?YY-|u|J2jk7cYj^iJxle zsyl1hVd1aM7_d^}JlMUb|KXu=f74v|9C ze|~4u_{DW_PvGeG!0%QTvbdZQ+jr6y%FNs6>%i}N{BFSSM*OH*lwPeyB7YhG3;0Dp z?~gm_K?~Bm;oXz)cZbhsrjyT<=8GmxLW$sd!`*IPG+NkavHM%CvPKn+LOV6#*FFZ!iA{1odJOW&1;s^%m=EO2&fXe zrxlSqNe zgi_pH4l0#Psud>&zBBdYtm>q0<#`e9zEo>tVQR&BiB!;iZYuL*ZYHmJ6~w{n^m#ab zx_c_Og|wy$Q<;XU6vA(AT);gl*oPzN%&e-46&to)v953>pv>H=5#!eP%n$|__x51U z)!3yn*!h#nsruQ+n`5s(jW9 zy5R@uoF>sUWO}*r1kWjXPOV~lT6V=N;%QdU8fZ@;7UavaI6&tB+E#F10(60x<%a+* z7l_O6E*9uHK(|54g~4wC{Xn4C09__;*<5*8xUl?aph{3sr^Iw^6YvNww|0ItXZ>`| zk3s9`qMoh=qK!*sIpgH440dk7f$8iOViQZn4ovHAX&i* z{z1D|ar{)^(pmXif{**G${N7YmhLx^hq57uuzwXrf1c=~nM99`w2N;8y5MxI|4o!& z;r{leetGRi;P1mPy6*qW40{1q>kb(eT`R1d(^o`S>PBy<{{qO$M$IidwlRvm->}iM z=x%|W3g(vXiMti@%kqZ|sVF!Q7cSHf8D2JVq)*y;2y-)j9Fy!u=Fu%Zwz~ym#FTX< z5c5PjXP~wgKyLerthv_pNx_#xk_T0GRPccdSJ9wELy`Rlc2nj{nE6;Cvab{+hYu0% z$ga~Q2#fMQ36eq~Ch!YaC;PP>pFz&pzvz^3@jnys&EFH>{1UuiZa|{Ococ0Pz|U+=x#uP*yfrI3|-MUT?KoSJ_a9v zFw*&Gc4&I63-_V2FmZEO6-i&WVV7z-<+-#J2w z;3C6a>Yzlx!9fkj+-tap4cBY9^@e-iaDRkaDeb!sO2{=%ic92setSBoG}zm4C=W-A z5^(W80*L3Dmj;hJC=vYG!hK-jieL{l4q8S{bJB3thU4*H*NY5z{1>4_Pyszr+z1C@ zoO#t16faG%3?uJ>kXY!J3A=tT;!Rh~0)t!}SOj>F?q<(PsfZ^R^CU%Dh!O{LqphO|Td~@g?x7WhIc7Y8fFMqGh;S9i#)# z8ty&A;Wj#JP5lGZQms-h)9~uqLSVc zIky$7oN=zd$ay||$~oWUOh~(miu;0ln-EK=xW60DjmCv7w3kw{D>{W;(T5jHTM1jD zrLBZPIF>x^9xKBEu@c!EIJP-hS!BBj{J?2^*uI6@F6w9kG~8EVvn}uxv>G zy9hasNj>AItN&rJiRd!;av)b~d@blA4P|~~_(7oK&W*@7C0YZR*Ld6cVYHR`P5CZ- z%HVsh?fmm0z6KCV1kHG-xHboAtxz0mg@*f`g`>0;hY!A^Ep$*Kh);v@do?{)HZYsw zpc=vIG47#n*>rU1=u*F3kAbHpLg}&r;O$97x+2ZEmNRW}S>o8NG*j6OpJ{G!vV6{D zNhlF~0`C-es)G{Jlu_I_43|C7gg!xRpy~4nd1y>#6cZvr6A2qv*o+dFd!XsGr5!z; zm_`)6-ZR;cjrhpMnfh2{rgZ}u(OSG$I5VkcrnEH?MQ){v>Iz=f?12Rv4O(Q1UnO`;gyf3$vlZ|Z=JXCKey zY_;{P3MZykehlHTE&qdxr8o!xhbC-afkguKn|o8M%BG-;^(u&i+Vb!Hw1Gzz`cn%* zO)4y-Mene?x{>n&O^@SBi`vT`wIQb)5{B&bhTrsjUy!ALILI+3)n$A2wZe4 z{a0M1x)=d(#*bv~=DMTj&zE4~{ML8SgP(v&;G}dovaW1ohl7(fR2j6~(XbhU&Yj-W z7S3O2=kA*oOjgsO3xqw5uo~jKgVSM*Z@`aT!jFTSx>4Y2_j}sCaEy_h$4a2v@nP97 zxERxY-0uE6m}xY1>qQhzEjo|N4$B+~yc>wZ`H-0?fwIC;WhF!LwJ1v8;yW)#0?#i) zwCgArz?HWV9qqy0hWN?~=9CqqeVV$c2)4f$mK6-){2*6XAaSv~`Lv6&6YO7@ zGTkdK`#z&+?tm;SKqo{yTt2<4&_(M1E0)ApY&W#sXg8T$=7jv2zyF#HC$mRjbLg)O?0u)kl<>o19`*yd@nDG&* z`-~Y)O^YVam~r4D%#F0QG@Uwq+J0><)7qv^p4L9KW&XtdTla&v-^BLz)``Lc=XK9FJ{Qe%#8b;drF0;&{xm z;(7u79=5VHSnHrfz@C(byU#&fr}JiwJH*xo%7!8JdR!!q)G6~LZ7bH+xZ9$^HnCqe^`2ktL@+b>!nX}?g#Lqh>sTn_LUr>!7-P5iyxgI{#{hVt| z=9|C>>6%i-bs&`DzTu$K;331Y{es?6cXfqq1j_t1wTgBF&72pVg0s1@Doe+!QY*XA zo5Fsd?&@hP58$?+iJa?fataTdBGYrC0C8|wJIs`#{Y0hgI_$y z6fh<-&Hmy|nk>r0s>M7)c{GoaZr!{HXUdJ$j>^{H@Hn?O&8)YZ;ibPPj6N`svolnB@^Q+{kODZi8~ z(L)8k&?+wH96lDae5JtACGa~%ixQCaX4;oxDdeJsix+lx9oWQGZ2L{c7}MgGWrI}B zoIN_mg@bppD&hGBK^o}b0{BeBh43k!JQEKgd@>Dpv4hM)S^j>AH|??fvB>E7<;Z7k z5~umYxz#$JW@|W`x|g;vtI~2YtW1W;;2JWncNX*bZ+2!;JDTQqEM459cTb15s~An= zSz?{|1@Q4^`?O)a>6U3EGpM0wmQ#KVw*UnkHDaH1@;eVc^GJD8ej`nOgc8A3hP&25 ziC}}_UNKw_wRsmWyJKRFs?9313M$rrxy5HvW?!Y>CA6hmDB&H^CWZ(^yB8v>Y?F{kXO@e*!E%cQf< z#v^e2Q8(083XyVl-834ITA(OX{f>!Da~cqqFTZ0^4SpNnBbNC__$>E#-1q6bKdE>dSy=gUD+ybaa9bP%Hj2uf$OJ>0D6LJI3!&p zJ<+V8XciaegOO^9Dq)}}%$CPKPGU2#j?+eC1sgqs-^z}(T%=xzU1J6Ej_HiI zQC@eFeFHwz|E81h!zN)u2`<$P6nCbBbZAR)y@rcQt|v=!_aMwbyS*}T$@Tk{H_=5t z@!1w=Yeg6NQkY2@OmaA5J&i0;QAKh%l2~-}ag!XOL_m#H+)@XrZ|_3$nq$kyKuf;Z3Gvb+gUF|g$)n>^I@7bL?PET;10-tu zuEM8@L|3~{)F}w0y$3>x;8MI(+!YQ=1dki;NyGipa2pJlTG>!tjLQHpVGu4EE`Nac z-A&oN^5N<=2+1%t4cI%@TFdpp#j>QgM3<>FWD4rBY`8wtyW+g6ilApS7Z5jc&F`?( z0YB_6eiif>q_suFm-LojKdX8oW*~-si{<%7t6Oo5TLD(+S^CG!%jCPW^z$)`jd^E( zmVWK!lWWT#c;bx&{{BqjA0te#Qn`HA2bpJAUMcB)qdb+lwVE~`S7)v9@nijS!A~-W zRFpoNS-0XTEQUTe6_n3jx8mi|D-W;895Q0uqdogdYO$ahnbV2y)mh7;`>tf>rG58C zmNJjH_oGzeid80Vz+$hE{?j=t(-tnK?CjC0^vF799^=lrFb?O!!sUGj z1HZfDJgOlCqY;X23#5#L2S#IhuP`8JsBDCU~< z(IvhE>er-4BbPPl%ae$FNLI#_`SRU;#Pp+U*=14sPU*MHj+x8;_Wt1qgM$ccYQHV& zaW2$kV%8cvS-3tn{a)EINAD4lnyg$;zX#>0kW>Hji?Dcof81X$i`Pf^JoLMGozYUG z|02;YLx+xMKILAhj#hh+KoW!Ccr4Y{yYjfe9rGw~+2**AIC-PN1>z71V@k6_6yp-hG7DKbTyUIFK}liiY{_zO`+yfrrAeS zPntLl8SzWs11x=?G)cbBG%T^D@3at1CwekiX~7xLHF)Y)jM|IbE97S)2pw0%vxoX! zeYA6dy@y}XEucsnM zJ_Y`?XO@Jzp)vdELxe4)f!bYm9`t`cZxf}e{HOBvb+iu8h?U#1kRZo38k{KAOE*W> z32pgD+m{SHB=>1!o{#HJbR>?B1J`iDA8n+KDGF(Y-@k0O1=D(mDB(m86yU5tQ= zyVXIZ!5YIoX1FH|_nhH=X1L!N?)QdcYg^;ssxFOVcf*Z09BrwFJJ@ji(kt$0!<~+1 zs^XS7C=tADxZfF$M~!Q^VGc?JJYZaLI~(pm!_^w@SAafI%nspi9E80cfKCzIdk#tj zN1~mt{El%@A~?lxErwfcIQ9%kfz98ByV7tq=wsX~Wi-J-iQsC(U1zwj8Sa~gD+JvQ zLd)yy31K0S;W`Yr%5c{hZUdk${7QpY9Fz!VA+L%{IVcg#Gu+9BJJ)a*8174kyU%d1 z817ZWy=S;hhMSD}0F8G)2PJ~T4R@5`mKbiS;T{3h1Wwpl<)B0`5(Zq$vdTd?MhTF{ zk#-OUCvm%8#c@0kIhA=mm086~$}tk|@h$jq=I{#NzwIdi6zSo1C#F0%<*8Q@JiwKE zC`-^`LDVUaKV1t@JBBGhR32=%%Do9IhTR>-P^dGGwY&r{Gpl+TZ>OG!n*cErv2s>* zP5A?}syoZ~0Pb>ZO6FC7IwesXZfSA(I5V;dHZL-k z3>0GH!l7AA*3 z8Q4d%`8{B~OdfDd9vi#k@kqx@Sj^0a>BYJZyxago1bgQ`$(U}ztt2elhu}|!|1J0v z;7^pQO=?2A8B%dv8L7B$Te$BT?tcvTvf(hQ;P@3e2sb1cZld8HHypK4Lh9bbskj&smOfJ%85e`Fi*)-yb_YpJIAWNcCKC)N{pA&lN{K zR~+?RaW5N=dagL?xrU>jD~@`uIO@6LsOO5Ko-2-et~k~66b(5Pn>HQFEh?WfYfeKh znht9|ZHu&g2GYwKKNY{*|Ch9!Go%Aq^Ra_#xw6Y^06-O;s9JO3=TdU<0w_3~B{QpVI$DyG9i`y(rUJfnav^84(l+*GbI4#FLNz!JS zY+6o8wOn!3a>Y^06-O;s9JO3=FB^_pt~hGBhNG4%j#{obYPsU5<%*-0D~?*OILs-h zY)<*Bi00$5Mz(YUfJggGMpWU!X(L;Q6EL$JX=X2xz0xCzsojY+0=U92&AJQB=Qp8%ivmtuq|9(wfY! z1vb8;iE+@eLCcXFbtLWRZV=0d4z`?Hpnb-<>xr^!xoQq`~cs}dc)YWtG9^6znp zd_(5RR}pvcMBxD#kvN%~#=%24&SI0XA@i*45&yY_3STyXMt6f>0_X#1Ww-~t2?ZA4 z1AZMsQG8e$5p0gtz<+s*#M1Vzh3Bevam%hVXB-CR)VsQs7(^}!8nG!@mlo%_3GG^7 zCrB&Ot*Se!{cPdpX9Jb|V%Vr1WsE%B&dm_>EP~}YXMuOPhhyYCs=TbQkYmJAbcX2R z!=fm?R9tRnj!so_vvNTtcTx|{y_1?3Qt7;WNmk6Vle(xf9<%PG9^(hC=|?9rZvri79KW?k=^mWQ-WKER+ivS8y;q`@#4-1*Rfv?uk&wj{5!Xpr3B?P7`)*@r zIF7UwOSrSJk|BHPsYs%vHJ)QFV)&-`IN%vdK{f)pZYpMZ7>u>sHBpjKk1rMsLvpn3okOIo>F!yT6 z0et2l+)!>`*=C@OPduOL zp3id6tfzcWQ+60?w|;#H>E3L`ecHWE1Z=4&?(2s8nc-eC+$O{Q-Ee0i-Aa3|gAxI1 zjl&f?2tz@J<4$Xr7Q=noaNV#!%5Ry2U_v7&CH)F9UljWUQciUI*33s?snZt1*6)n# z!uPJVyQyPIQ)|bP?v{nvD+H+X96sVin3kElnVg`~_MH%KslEORHR^Ej4lqAJ{!M#lc z?^!sOsD?u;tn{(W`V~LGTCH=qR%`uhc!vre1na>l{6KIy(aEs_;DR7JxvlZUNs z)^@dbhfTQ1$__vOsZ2+U`Rc|4Bp*&dTB!#^Q9lootwjhh2ggn?MB9ZtLTf)d%bdu( z=I|nKFEpC?U)l@5V4lTABH95~O{Ue^0j^?TxiftzQB5C>!x$hYD$=J(%UTM%25-Eh zEZO8njj;9D7|TlY{r>P-$lF-^gis<_Za99>6?cc>?l#=_4fmAcno*w0kLysB7X6s8 zO-@VD87pu$%IJ_zF_H3+Rp9@^r6blJ=y${9z}=f`tfXkk2fA;hPfcUKXo;gW?RT)S z%-`?fGrxa;&qA1GHH}aLhrxKiho`8C!ul z>+*aYHq?tP^?C5uV^!`LCV)MGG!oeqE0meQ(@n>Z%Xrj9JC}AK`(RznE=<+*S(D4p zf7m9Q86;8P2U;vxEY*&Ni?swBOR}n+u1njO)h_xKSM**)M;i;WT5XIiR*UYN;;N;x zLR?OPkBxrvHRJKWtipajJ!2sq+4^c6Sv>3wb0>J98Xw9VVy9P9u5`Sgo|X4qz$wA# zAIhJriJ=YX#HvG$OA~qmKFf~Jlm?$C!ACum85urPsXk#fn@~cIc2V5r?yZ`1#nGf| zxCe}uV;_n;50ck#)E32EZ@8Nc$ByRa=9E{hz3?KgT1bmH){7s(^r`#Ms-=tN2UzaS zUObn=!G!yLvEk=+sC)uw@$p(Ww$Q#eBHii+TDIgT>sSbp5~mU%zJQUU#2G1VIn6dB zq7roU;La850fj(U?DPO2RiTDB=P0lE!tx)Xi23>%SJX{b)P%IC7004h+?@_8m8;(r zx7Kj$EZiO}GvOztC;Vm^j{R_@)uK-NqW%gP#CO+#RvdHrcjjH37IpmYz^%BYdP~b> z&m^VohO|Oij|Vzz9K}Vw0U3&ZFE4bWPCHRIh-fAvCxmo&jpEpJRvhb+;wTlxQPhf~ zs1*nE-Dm?cUk3Vwm>G_K8@WnSU9_%j!5ClnLhTZBfbJt5;p;n=#q`F<_zrghJOVzI zrNIfZ+yqHTubom{8le28ZGqn89e$$*9`CzG8sNo@-R5rzZ!$ErJj^ZwPuKcPrYHBL8cGk*NvSDTIn3atc#~xpIJcf{PlHh{k zSs5H%I=8xW$}dtYmu;Apc_-C+9+rQsY<^+(lvf&7E?D1?dA+{%oz$uc-v$4mVdb)Q zsg)C|XHVIj$~?1r_LS%9R{XQD{K`+^-KxTGGRTpcU#40&)o0#ZRoIn!^ba^ke*KY| z?{hM;xncZE4FxZz#=pL*tgGOehE-*&Q?2jRtvc-QxDou(SMxXbX4cJGHR2L-nJ))8 z)MW}kU$-(ia#d}`p_#&4>Q>giSBHkn%GynM^Rx!DK&?ED#ZnaAo|e^CC_C+Z8HZD@TawQ9r_1&<%Ks%#abOWj<*sw_P_ z^Th1&k71uo!Bc2At(sFNl@#DA=58eWsz{t-1q#QGVqIIHJD+ zV`LQ#I9hQ`L#C}d$x$&Z99X1>EY^JjTTEEk@8I`Cn3VVtuq#nOJK`PvPxI>_&hEv) zp({(eIy<`Bo0hiEN7?%WB94@%5VXuW&feEmE&-D%f;l#i&s^VlIdD|wkoG7EYnB)% zg0UfX2Ey$sw5@q_aDxNsw~f&A`3wx8cz#^*G7sTdT{ht(mlt#`Z51kh9j+EmRXFu1 z`W;=FcSW!)n4HJE#E8!e4wKXsN&mJPG~x2iMQu%F$fU6c!W2O z_+%xKOnl(5erw=mAoEh^N^K^Qt!OsO(B;hw-z}7Prz0V)@^05WX7O~$r&*r&$SVkT z&12s`j6+`dKeg;k>d;U(a7X7+KlJQDeLhM|Q#0PNtd7p3XxSd2NHm0#H(RQmKvG6W z=P`Q>#;jN{BE_t+kwOOEQHNi^Lju2(9rAO|@ZSx;7&tyt-}&4O{~q{@;ol2?DCqfI z2A|JV4u(s^{~~+}?l(x68bd-@1ZlVy2dRP7a5Rt_?tBZk+;F!Tj-L<>_j$wdQ>-`| zKjrtR;eKSe9~iO4FQo0Tl+v0{4nQ)MDPYR450w|5l*S0>|o8G=LpV$fHGMvm!XaGMcS?W7vu1@^24n3gr$0B2cNT4fsGG=gOtU?#bb0jB*W zNAK1Fp#9|xWt&5OJy3g#)|uRGg?@c{BHld+{{;At!e0dcA^11Jhfbz%h5tSHpMj5A z$@HD@ABGRx9XtY`cK8Kou-aildaStOPIYf}=e**U8je#5io4cuj7M>I7_QfFYYq3j z;h0NO;5rM#ePFl}=%Ml(<{;QF!|h`@+E)!n`>OoD0%*@-=Iwq5C4#A_iyCg4gTnJ# z^d3I9hAtVl_4pkb=y>##t$Ov-6JFZWZs#Ph51X^o_Mbf-9c7n>(Al_44~M_n>B7;d zNU95j^muf|QL_|B-BBDRqqttfv5XZLxg&%M5;uVbz|o-`ZI5<% zdHPIF*%b^pnR1N{shJwTr^UZ#nIU-YXrK>;$ECWt|RG}G#vFLztDA*5?J6xWVW zio4fAx^_cx5wWnA#&494iiJ0os#DGKVp-C*kP}TaW`wdi{9ly~8AN6CE+VFN@h+EW z&=X4YJ;I7;{sf;Ef_>r%^22x&2VD8m9p)G4|?^JP}y`V+6!B0S#oSL-lK$?Uvg&z4qy&Bg!hKalH9R5J&Wd?fPlLZWzp0_9O)_e<>R^Iyyn|7Gv2Cs~Jj4*s#e* za}9|NFFq$bUG1TlXp^O1l+gqsvbR_((Z&DpP8WZ5CSzfKm;ff7s@ezwiq5GjFEO46 z6vc%xEhnz&mD*?4J6|}8UFYIC0omsBvF>>vq@2&J&vd^Y=|^7sSkh&99UnP74;s%> zKnG6l7rOxdJ9xelzXh>fv}M52vHpiHYZvVn43&xk`8~94xMUdHEBl{!_uH0f!iL+2 zkdUreRNO`GZ2}iV_;7a^jswIR?ta7l!*Cb|khgl6v4-Q^gvRldgGz%R8SX{H{g2^Z zGu&H-`@nFljmi&a9=fz(X^_L=+=$>t$y=v`c*Iur9bXqn?bi8#T5ooLj!evq4QKnp zVRfU*snbf{+I>!EyDm(fF(a(Vv6>W)#}0*N4OEf0bdrD(MAW?kX^N}6w+>Wq8~v#9 z$%oH;C6RGR2M^i+)oTgiE6kv4Y$#7 zD*kC+{Bpg316q?!+ zZwi$FM~9A2@VS^$giE4PA#pPSKW&HN;~e?4g%z>{Lj?!P&;be*tJai+g&Hb_hQS&= zxF%d_6Nce8jk?RTAN#ks(HU1J#8uo;C{OZaGYaKtk+ZVC3*0pPykmZg5fdY$;img$ zUXZ5$0DR&fgim>V3qHgA%nFo{Hn|kn>fYjFF$>26)o?6E4TnyE!xbWU$dA%4vX5uIoe-cL<$szk@4QNVM(y=qOg5L$&E!Z)A*xN3(#J_ynT=fP*%J`SHM zwb5Gfgc8A7z%iGla%?goz2{ED-RdZyDtetpbU+m=y*Z$o%HZrb9a3$;j8!vMY&T%L zIR;U28`w`thY2cwtQ!`1U(zd=f_+P_GvUo(<*7%}V#noRJGCguyMnpNh@yp850J^@Q@!d?e-jzypesV1vD0IC9C> z7Au8cHW9TbI|v!4gmw_JgCrtE;LEMi-Zzb^9_E2YjTiYE2Fp$ z!(D6PZZzB-hP&HvYYn%~aL*g=1;hQtaDO%25R|sYn{bfc3#GV8hC9K+on*KQ{+;Een-=Smae8{)Anm>IcwSE zfvgr(LyL(Yy{P&Kf0(7HWqPeZEz=wDX_?-JPs{X6v)6e(Mz4yy_^hugtt4tIjj9RAvJNGO4MX^%sXQ{GArlX&#+`<#1bzp@+>(i|#|IaC}plzo*PIcx!X<^W}Q-C#MKzeO4BGX{`L zOW$!?`X{HQ?>jC1AIl*j&7tC$L&YJ7yX4N{U|M?PU^zT%(H3QKOX}gyt_;VyGThyj z;ReegANZjuQ;w=9i!N=;NG$e+C9U{A6SJBp4t2WV5AAHyppb z@RL2i<0n~-(vwFr(+$~PSC=ta`B8qzr5b}V- zdqEZ|$V0$Ph$)DOj=|w7BH~!uzN~XVc@&ATx}mZV*2hJC!v0?o5_@7LQ*fzk4(p!#>0`{bnb=KO&H}LkJ~;qw!90$2uq> zCpjvv({N8)xb=o>zzTyg4LD~xdF*j=81pn->zi7&NKW9uwZ5++BpU6wYPTWtOatz7 z;N9VZ%lzCo$1mxXM!2@Wr!{@zl#?P&@LB}UZi4g5$$etQbpk@M&aiG=9~PH2z_+mZ z%~o;}EZF89p!vBPJv1lywQ2i&fQzMa2elF_gg+ziUTjp+rz_S2ZAQe-!#!-m&1yz`Fb!sc-y zd^SDkaEe8mvX}}#34dSsEVpU!8Rk!>o`e$I9Pe;!Fe~mf!<}Kcs|a`aTph8PvIFj6xw&{HzZJb$}oCXA}yN;x~OL%y8lD^G2Z%DR`(K%6)QiFu83=P(HUfC`TnKpIj23!ZWpy*)-LI=JaLDsX~lt&jE9) z;h0l}m{X--y65BPT`t}e`^{+>e;9-ECC|$|Q~0^~@;FRr-tgjJ_}oM=d@2U)CvUT{ zth2P4gi-3>T8lT%#%1H6nxTf%&|huu~LxlUXhxVi}xW}Irb@GIfe0uF(_a1 zWLw3%TzpfGZ$1`tZcUoWmq$T*my0(K6764_S9?qYG|yLQmeaOj zfCh;wpQDyP`Yr#1Nm3!kyC{^TvpJrEZ#5hxsSqWp6bzNY&%0c_c`z(W$p!IzDakPY zFb3sIo)3Gb7BZV=ndX*Xd1v-&A{4J$E)`-vF3^-WNwv))K_S3`}z93Zn;&66stq|9D+x< zXzSrkqfm$xH->SzR^wcu(?4$+g+ipbwGYMN5sE(=g+ipbBcyO`jDsT-ZySX|q_|fp zV&8g~C86DDR0@&mD}z$KV^j)}>Kj5;iJjkLref9iQvG5x5|-O{g@W4AE8&RefV+2p zGL8z70{ZK%9PZLB#V$z|$AFm(&+Jbcw|A z`C>^C(|(&k)aTC&#IFB01gZw~kU$dveOsWZfW9M85)f0s2?(t(LC_4KK+>vE)rdmr z*PyS_iD2Tg<@Kz|>rNj4{(BMw)2R@1|CGrq5>P;&@1%Rde2&5IIiX-L!b_?}(kH#2 zrP`N?w*UWZu`0w^e=0nL;##Gc0jzg_F$#r9@r#hc=`mX+-o0-W3X$UF0Vy^ag+iqG z!+;bY7==Qlcxyn4zZ!)?r1(=Git)gD_cx1reSqCL}aK0;BMED4fr+XdUt9Sdt(6_n6e5V0i$bJ0Jfv_^uJIATFER>+ zNYNNlxRR`mPz*5&g-CI{Qb_wSyYIVOgmS1+Dn!bYg;HAD=sDAQ3cl5Ftdk1y3#k+g zbtHb?<>Gy+@^(QyU&=F#Ka4^7l4rA{pqGp9f-TBzvE^1F=4Z)(xz%vYtwPMLQp9tc zi+6XwxeenFV^F^F+~(qYuI4tcNW$g^D!OcR$x1Wzx@-pMD5V>C~ zA!!ZtTR<9hL!6&NYM zPbmTg+n6L3BIP|w85uQYE6}?Nqfm$xUsek0+#GE+MceY6=2R=E5GlSMQaGCxZ8c#~ z#Zf3kif;);)gD8Fu~SC|CC?RYDCo>@#tU($;xDE6h?UOwB|Kkpx0-h&jI%=I{D_34 z=J@Z14^4^EO;k_Hl@#lQf@SqXfqc2TU^05Y+BD{Ff$nfGzYU)WdeJ>!8PSU`n_eiy z94W-O6sl?{4jS4jf?=499|P*9q03gk##1 zip-UF+gZvLVmjWI*yB`DDQ<663X$s1N)@r~oDK7C2cuAk6o2bOv9k{je@CNGh!lCH ztb5TGOFBZalTj!{ilKcdqH!dRLLpKNQ;L2p%q~Wu5Gl45imGwwUoIFD40%pr`d$UQr4Xt1R;mdF!Gx)0LE&RS!Cj=D{H-x6g-C?~5NJf@ zfPJu92}2fp3m^TnhjCMg+-69~Tr&6OD(N`*I(w*4Q09jT6qWfa==_IciX{BG$XI7u zx4}&7FpNl)1tYH2S*Uf6H@)$eI)#|JdWj)#$HHL8Ib%1BX&x;GWXA~A1fx=jR7VTd z&if~V{ol(Ec747e*l}%PFuc1cD8$?Vo#((@C>{Gc3MBz42Zp-bi^1O0HidB=r*Q#C z=lN!b(a>zIC|~kiISAjAH0RTY2h(rP4<=n+5bUw6FxX{5QLrPvw>#cHcH@{0qt=h; z9o|{lT+zt5zZl`Tmq|q-N~J|PjxP(wFDMAcP2Dyaw{)kCJ8al)eO2%1&QZ-Hg$p+Y zd$+f7QHWel+afNvMY!x^TofXg#lod(a#=7A_G8DXqk`ejVYX&NVtsM%kj|p!LP-nL z!J5ai(eF_b&^ba2AUHdpf7MO0SQKI`T@o^vZhOLMj+)6O#d$(Ot+-I2sBT{<_T(H# zu?Zyp?W<--6sp=Qfjo@}hTq?Ro*0MbiMhMn^3?8=2J95elR`}4MO&SxjOFR7K6yG% z^0X9u>HZkW(`%L|g|&Ad z^?AD6@^nHZPy1V*6k?v%Y;~SKZ+ZGspFCZVm8T7nJiTFgQYiin8qb1tZknZ2A*S=I z5?9_fxLkB9t}UIM*jT(_NHYndlrxM{AyR%*D0iM&7|iT035w`N_rqMJF!=h4Xw_ z4q+M;VqD6XJgW!c`)kKHyBs|KfzwUidk(_?cPi1?s7Q*LaPAEI6#YG}s1rp%%Foho`3Om4;sch4=+fYKB@8 z$ycs;-gk0CoV;rlaRrDg*s6FKNB1D{{KLhAB*FRgv)scxE5x$W^phu#!1pd!`U?=e z?;4?$CxKsQ(p8AkEm5jL%H>d_REU)2Lb>x6l#9}^TohutC^d7xIFhei@r=}X0>n8; zx%3|o_s#f3gy$~Y=Sj#DHg+jx-=vV^9l z(k11uj`EP=K%rnMA1sh>#d)7}+K<6&IYOYotlku0D8>|ibf@Fz>39@@jv^|=d?>`Y z6q1><`@KccHmByAait_Z0OT?X`g9GJYlWDHLnLH0*S>JQ6p|!`Bc2pT2nEx0lt4_= zT!DQ0y^j~-Na?wf?idMOEAPTT(*WW{&>^|s$G!Lpe9v!NlZ!%CiF~d~ZFE5;b-84~ zU1;7Zcb1Go%+U#wo~rGK2HWP12+B4lHWYUbZ5}ew)9Cn{a%OH@B|P8q)IjgB=n$nw z&I*z9DH4*>Mf*UUCrG$w9|`fKm@gER(n5h4Uxz?m3f}je2J?M*2u~bS_yv(9*anvg z#BYgeDlOG#1y^mj0}YyG5a;IkL!&XPY!^vdYk`fr6do8`w{RvGrs^~o|EL@bMq(=))BYBZV{HDntn?L}S-PuD_?;Pwgcl#)47h^bpD^4W4-yFkK|Bl}d|v8k!LrVu$VlaO1ZYs-a# zlDb$RO6m%Myv}&vM?IMD!?U#LnD;M9$$)LJQlM;IyIpYAc&qxiHM(}0q_q~bQO7c4 zY-b7QS%8?T(_E1LbZr}Cjk?BJ9S{bW!lzV!4j=!sbZwMy=39j*L4_zmg^~u<7(`vu zx70O-Y7Jt#&l1i9>sqK0QC(9?%3C3(Zk5Pq%XRG*2@k?7U7H71s%r|7^K}w(Yjo{a zp`fHbD-b31If1;+c%Ray8_f6NSz2_=`jT^P`0k!A-E5xYj;RmYr#3{SZ0jv zY~lRx(6!ORnKD<15>$v1R48dsjX~5keM?h2Z! zRE;aaI9wvwY3g>taLfUhZYb$2ZpLfpm}dQs@;mblq2a3%-`D_rGT5oUDMapHk&s)X zZx0FurS)xr7;~>cUT?fVn4bCfK0L%Gj`{zh$PjFU%LU5Tw>5(MaQgOLNo%dd6#ikk zF}5Dzd^jMcYOxEj20jJ(dHnbteHcHca(iTfwvGKM5C*)pgi?JMKK^IfwlTt)Zxx~h z6`}+cN*YvS5cN&pQr{G+HHhifG!LwAp+-dYO(~fZg_yd>MLty{aZ}B6*{wCoM|F&8 ze^$Z)2yS3joofaQ)iH(0@hJ&O9b=n@&J$2(bV>QEcv3tk6pZDk0#QQ05XkF__dO@? zdpz?*NB>@tJjjC^1gZw4-^2+9v2^)k>3Uv_{{@M)7Esi&B*?Ez_{{{wc$?wVeHTCS z8Vg2@e=2-x$p!Gqo2@IxL&v49+9OejaVo?-E0i>-#-NFSGy1L>yp4p+ zWh?oXE-8OQjTkKyqPqf7DoKHSiF==trSrbWGjDYCZ!(Qow$&Q3gTz`(RuShlqEq-C zi8qY*Oc!D#5<`)^1E1+S0uM~beE8G|9(hH4R3k8);B2Ns%)3I&yFy8WY7C-A=v$^q z(=)I}gyk012&H5y6=KPZle|~r3s==viuv>Gku#=H?F02<9|;Q}xQ&0DI|FP~8x$hf zJtQQxfij9ZUuH>%q>M;0StuxneFgGm?0wQ{4QDO=q)2jqKy-Nju-+}QoGHX~;v`$-Hae2Yuc&uRj6xw&9HbPHl{^!CaQGcYp%5u% z3B|zc&Ko5R8STYC&UG3$g~;u23CXhXp}bF-(Iw@t>Mf;mtWZ<~@>1Mgq^Nsj!ddCZ86lzcsbfTzZd%PmNdrpRxfMj5nDm#5%1IwL_t#K{W>Tx29Ip#B?e| zd908;iL~2B2iwgZh1HBBv7cf10Q(uP78(!-Yx&2yZj-h`h3jr8=uH&9jByW4Tg@ z6rbrsF(X28mQg4~in|7+INK-`BE=W_P(*9XGNVw46!!^56*j0;<&6%c4ZMD6FIS=$ zOw4ce3A_UUrTd6TY|*|xVDBAtKie!fFEPHmk)5i#g9op9Dh9LsiqMMvF<3u)F_lRsK%hG z9WZMxyLk$1)_M)zl0ICAye*@5?vo}dg(#`-OK!KG*LlWEA@cg+R`7~ev-6FYLge-1 z56SBS=*)D}8HK7obl%>zynQp0w~H)q z3NdeQZgt-NYw6a=%Y-EcXWl`XV5@uj9w`Ji!OJ z-{Z$}<2`^Z#~t9aoO!Mm%bhJiKF@RSsRu0IAL9?hf5Mgf#qfQ(A1pCaMhdaq6=Jz7 zlr*TuAeOtnWw|RCgcWKwC~stOFcRzjMz+D9J2V(MxpD(u28_qM%aJ%O zPlfOp>?CEPr~e=`aGrv1H5}W{3NcSg!BDI4^DYB;`vg(VfCSwZg)V8kMmh{zsHa9`BAq6@j%EJeDc{0+ZQmk0$Vak)UgG%33kQld4Ef?lIwDlyeDb}IUGs4Lk! zJL2oJuCbK=G{WT?OSwW!dB#%Cn4^yRMwgVoVal(!l;4z|%Ni;`SN%|?%y98pr z_^Lpx7m8!wnKWJS>DJ)q>)J6Amg!Z9I<8Pu6O+Jtr!g{53X%3xlHMpqRChHMq)>?gK<`f zoWCR?sbgL$o}j;y`le7&Ru2h8ZTp@;zHGdY7vjj#b0yt(By_F33;#?5$g*4_rE z_#}Mt+W~Jpk7KRe6e3Th8hAWEl*%|zc&C&sA%!TFMOr?a5@WJ8VXmV7D6i*IF zag$LfM2a5`NO7}KC`5{%^r7(IWxNb-F$#r9@uE`1R@i@n!JYe*Q7J^KUn|wvL@*ZZ zu(4=|jYT_bEOx|?ojhs-38UlEpEeE(k;CtlgRTYNh@Iy)+Oapuox9B_6(Z#uLMdy( zpD)B3{jy-2wZpJRzf@>EHML!7sC^2t&MP%T-Q<D$90pbV{SFly_ zFplUxI^~`n&-*SOL=NZEpB+Z&sQa3J^1O49^#4QnR_$7hv60cih<68@mBw{{@~uak z)K_%Gce|BOXGvTv!bQ65OkIqG&W|l}Hra>XfK%t~Tl$+vcQWRp^6siTJDIC~}g(VrCl~hWm zS0R?m7?DPNpHmy&cy}j)>9IK1446O@zpk*6n5pCQn9ncoAq`aNLqJmUNuG zB-!pSI?Y9HxJ|}q&X8dw^<4j|8*}ujpH~THdHds1t={FroaJxp{)&SyQRh}H1 za-BfafV)9?E+CecM3ES&>aGu*1n{D>B3e#e|UG5rQ%89iO1k^ zftZT50wn?Up9{G`tuQwe!@4-N~-@Zbs0A ztBTua%$VP@a8X}ud&hi_>_1NZoc$2-sqMQ$tO4rl7CTz1)j~y^c%wl4P~R#L1*jI2 z&9nY3CbQ79w7u46YKzPnwIJ%4lvNTfOY3RLRP;Sz8qKbAy1LsJZ-sTv714DT(F6TP z)O74e5z{Re(|r=tgk*3S#MR!G?CwY|T!OpSTNW)mw>{~u@7g(@Wyv!Xr-|ReK7!pN zC|SI)3-{KwE=aa?Cu@RacY9a&jG!c$T-uI%^V*lRw$BJo!R=`?f@AzgVliTpnEShW5gd?|&9Q69*h8nK(KczZA+3EO;_#4aN`Wd@b<< z0VO!n+D~2R2FB|2l36$^{-bjLHu( zV3}NDk@T1;a=V3s1>#l*EY$FJaS+K(gOCe0CR1Dyur{*Qkx=}$ck+k*zMpg z0(iqHeE_9Ef*x{1f+1Ri1bA=2Z7JqJSV!pR;s4%mVWzN|aDo|3Z5M_eZU0Y4ZpdI7 z9X2e}=#bUY=s3K8RVq_7*yv2rI$9+&Q4CVct zz&yFrMpz1~c3GJDxBv6dQLtv1I)XLB)DdJdbp$M&I>IgSuR3DN3hRg|D_CijU zgm-o=P1v%=JWBU!wYNPk)O?T&RxERYgpJU0f$7Ll+A?cpBI{_0n0YnGo)>^(fF**r zoGsb6RR>cnSRG8UI$9mf{VK?wcNoP0s{=j=XemeAQT1r6y};g~bOId(OURr6u!PJB z0Beak0f2>b0zfZJ|0;FM-8K__M@xMMMFaCoLTj+p@HVfd)G%k5m@Cf`e=mC}-LzxK^90FYZ7CL&mD55WZb^_3@=Dr$bxT38bPzTPm;JO#BwJrxYokb z1T1yHl5G=L`@?)IEaG|+E?jHj!nclSw8Q<&5ZVe1JB}~m!nGFeD6mhJE<1iy$2@-o z5AV1@$O|GoxYokc0{b*!oHAeqC8k}_y`qnwuMXNolT96XQcskd2wKg2*}?!9RQ{8M z3)fn>e`)VlFUGWuQH8MJS_?}N#5IgAGk%K3#h{o*6AoN!;S2{H1)4)%w&8+cohcFy zTx;RjfqgcB)7T5FpoA)k9T!nGDI=ww(UG&4Ge+X7g_Y~>43RMZ`y_w3){;LoID-7c zdP65n|C{-@J_id)3~;R_1_h9-e7kjk#x$ltY++zo9up2+YvHJXeIo&`?;q$6Z`Isjl@azZAL%1b`bv1{u;93g{ew1o_N6N=(3T~i~T|?M#t%VJ%Uy*Ky zmRoslkiZLBwuA-OT3EPE+w^zNpwt=$Sq_8+*Ti@O9!o4!KQnMu|C8!|>o?aYlZt&Yr=SQVDkaD0p7<=0yLUiY=5_F8l~x5r%8e9_}f z(bOS$`)7H+@{qS*n(^_+wvK&OZBSEyxBslue)i<1@Rc_=Pc4}G`N{YVS5($o{#2Z) z5YXgyBhx4RTcLSs?AHyKs^6C?7Jl>!-n2OKO^B)a_vdC7=ac%Le7I`R>-xCt>R1cs z6UT~YR(yPJ8Xp2} z3j6W@OZm06jptVTWo_-VcW19aIlMh{=Ga1YUopkw@_7?Y=GE*PaaC$Tl;y~uUwcP* zFTL)kJvUik{@E}qKL6IX1-h<@`uqc4G3OS}G>@10)wRhg;BxH2pkI>XO}+4biWue^B3W zr$xnT5SKq=%^r`h$H!LV|5wNPTPq(v_RzNT?uOiLYNHCx zU+o!J`h1d1eMemq#oJ$mCm*riH)x;mw(kdSK2yJ0Ta&%%((FIuf)3@TOP+A`dY`uEN#|Q5 z=H7{ke^k09AhTMrAth+(`=4Qr*JLY7KOSnDaIaR7R`p@1hwI5VO`aJkW47+Ua3gF6 zu9rifo)=uZ^2{A?$2g2`wy{sG!rObJWz3%Tt+dA5&Qe2dNQj@#xk4-Us zJ2!es=1PZ*m6IL|&sP&R>A6VUYV*9T4Nr%}h4ov$FVj?E__et&XYX4+B>(NIfS=hK zlCmb-rsDGHdwiSf?j|~$r@Q8!Y3Wa)!54%ERb1nhx?)7m&u$YoW8S+T>fidFpVa5G)?Nj(agDn8ytlgeVrFhZ z$w9mw>RQ>9H|w5tX>ZN!QKrw|4*vN0GB5vz_S*HSFQx9(m)LmqJ=f(?;aOa7M=YDR zIe$)6uK&HY{NTF1Dk4S}jjOeP8hJ{-iJPu=y4&quKgvT#R!zv4y**O6pOxvk-{ljg zIEOEkJ`}du$!tueS;4*PsGt2u=~y{56pOt#`1tGYq%EmWKbXWG>n8s}^5If3Yon^I zpERT#t9C@MjaAi85c%qFTkv}Hyf5;r4)hwUer?40_k+s<&nVQ`d#QSlS9zHgD0Fah z^R#{*-NU_;uHra-<|h1z)1Q%$`Lt*3vWY(DK3uc>Dz@UCee=P@(lF6}2iK(gJdlZ6 z;(RA9-hA@F<0BQUVtX|O?GS`RChk`|AIkoETc_|I$s&uiF=K z%?%A#bFM6rG8h%)d1_PP!5${oomEAQRthWnul|_7Z{W@Z>zbdtlT*UJ9g2<-I(J`W z+th@;mfMFaeykNay24tmP-(;rm$zHe)_=OS{q|P1vPCYMLgG2jGe63X${4U*Vq^Zh zlu^F%V=Gc0%~bC-xVPjqJH_3D zcE<5cw7NP&ykGWx=`l7_FDm69$*Pkz^%0wIfaAgA2wzQR09u&eun6?xL;NAnX>L1eFr9dk63n7EibD& zvtib)z|lDtt9}%#j+D5+a@LKqscSrr+XkIc9H3t@e#EX%nZw=XPe`lpSe&GNW6zB9 zRt|wBBhLAo#`^>J*{Grgii1w&A3w z-?EfU~NcYOT8Em%%g7 z5Blj}2I2h7$|cqRP|1i!(eyI~`8{H*-q=01khHTEY!uJk@LM5!uVCyK?H|K4 zc*n**#M`#QHqSI>xr>dx=r1vPcJ`D1@~24%e-0^(>f_gtv*K{K%$(Hwvx+73?yTFc zAm(o9k-h1^j^AQ)Z~c%Xd$tw|m&|Csx2E{N{7$_3$(tkfg+IQL4vI?M9NOt;zX7#1 z#d_BM2Az+^rJdOIL~B;*@{{7DcXc+se1GXpyuIL7<1x2t(}K0}a*`9PbmXIY&ng}+ z*m>Z&m{hl&q0f!VlSDss>hf|c9uGHLe+Y8Br<9Jj!(wlQE=r4#iJBB5S>%52Ovcc} z<|Vg>c^x{PH@^DtZfRcO_xnAB%f7|*KWhB*(UZFCphpfeR+f)wh`iG8q-jw-obB6a)WieUjg z`cE1)NHFrGVt=J~->tqKJ*cPr?*`k4EB7Xu@(Vsa znBv-LyWI?#=@F(kjkiq*==@v=Z%?WkoBiv~PUl@ts@E=se~$Q`=)O2xZB2B>i|3&| zG$!`mIC9-budMvLIKK(KJ6DbvsoxFnkBp17%Nd>FhxhA^@!ekfd(?8wbC&xdeR0B@ zD(n|d2{Qb3b*3lYzvb$mrQyAEnoHf2mDJ`oNAD?oCCl5bD23;}%CBDyPAG`r@9|Oc zy?V~8%a@I7gjYlcX5sBq)!C6c)4ruFvdajy-(?g3Jfe5Q>Co&Vf5o?@A+F}n_XYgv zHO^@6)55$zJG&f}wl5OfDYk5y!O|sKW@{7O52bk>Hy+&k^JN3&SJ(UZ*m<$Dtvkr# z(#Lu6MiIvX4=-GGVu`t>!OJeSNx$O$OrKcwR=eY>IAfcCeV{q~xaJ~D6dw&Al0>$eTe z{f)OHRmQ$`v5aoO+ip+mKFx}aU5&Q`%uIY5mi8Qpx4TpieKMi!tOedKm$Du^CS++L z-d=rHSGxcG6S8=_Y2c-CW^r=!HDn%j55KhH(Y_1I^zz ze9$b>5$-2{sqXB&Zw_0;xmyFoVL*q<_@V(`_0{BbHLR@-!6 zn)uwJF~^|GF!?&AMG>L1&du${@8n`-cKY(~SfR}U;nxkvOB6UI96TGm?u1MAu;q(9 zh8d)(Rqr<#`|yH@&pJofk=eK(?A#F^9_j5`hWB53-*sEm@o2RkeWun*Uc3^y?%~#& z3YDs$OA4Xudxy$d$i&=RFWt9~(+lxE)jJyTcnp~SU4gJ%YdR|-vk_p|Iygl<{S^}07~F8b+Kn9Dth&=oXx z*5B&B?x253FNdd3jCL$tS!o!Ol|L?~ck3}LvgVz`exoJZ)dy6loEv(q zD$8^KfCK%~-Bj^)el|Zrt>o$gTfFV-U7s^;jN4hOab9lwMs{sD;}!eM^pm5P;U?Pw zTWvhFBYs`G@2^{T1J4U?UkxnjmQlLGeAl#YLCr7WC1SnEt1z(=_#Fp76QoV@g*W$MUwU(%Rm6lJ>BV zzs3sQm2G+IJmWAuaU(LK)on)+_@y++rf zusq{4<44pwcQ!)+XI!?6qOV|(`_z( zZ+`ZZ%S(JdYR>P>uXA@^Z1SN~m|#(O1=g!}m|m3?l3Bb;!mQF-O=^{1#QEle$`9T4 zOyU{dT{d9+?X3#qzwbZgy?pQMcUrrzO&EB(pZ_zfJt3=vKd$^3XyAxB_&g8!EN>GL zeARu=>X1;=(*EUTOLm-lrDWc@_kGhZ$0l6dn7|Kxys>8BC%c`A24~{ZYhl>*KUHi2?>|@!UZDn0g`VV>P_CnIt z?$fREKFYTDgm*SKtDc>{{mY-O(GBkg^$^NfT_@zTCcJ3-OV8Ya{FHRi#?0u{j`L`J6BiFFCJdf99{Tq{Ji1${Tw$B=P#3fpeZ|O zfzTKH{2?-LsexNiv4zR8fZ{`rbCd9RX6XsOPyLzvUg_P&O%ncPtKGjJJa|24;VSKy z^A-d?d|dJB_S&PSvwm!LyBU9S$dv4fJ>JYMd29R2%pk?tSkLgqlTDxWYa%CI48hNz zp6G~;x8GJE^m+Q5%uCPHB)#n(zIhUp^ux+Lb;|LnqntkWvc0Z9dF=K2{J8~M(--y3 z3Ql|D+^hbI#}(_aCYkTy{XW`D2!+q9!}~9Mtv_%!z$9*&+uf&KJbnd-U0h~WElAqb zyLZmnlfqhVN>gr#-s&{A({+np!rGSF?|vyAdFELvnS-xmceYHN5&Y*)D87zamz@nM zD}Q|$Z=cDwC>gx}@xE<^Cr31DWDi)Q@p9y~MgCJZOvp3HTeu{qNzBhsUi`;Ev)92J zUrdcp9khohKcL~>`Lnv78T;MEMREP&|Iz+Gqh5FNPZeEo;!cfp?!LuE)*?5S7}tH1 z9~#to_WT}-`*Kw#-RLE7dEUuGc)GFJfs0>OcDT5aQwxm}|6_upGN$8c1|Gar%e z<4%l#m#ET&R?OrHP4BQFa} z%DuH#7e>wbBz;V*o4<-;=?<3^7ySczLOo+XZ?L*`VC(EI-8Rfgo{(o0wr#^(1@#@S zr=6Eq?u;5!ulFrlp%nLLwN>&Pra$4&yLG>I)-^A+r!V$D^w+6hUArh{_fToi9F5zK zDm(4IyV!_NO5fmaHaE@En3rcQtghPkA+N9L^yDc4jzerFT^scpZ@YaA&UvYkK6r`Q zcdPz0I=xi=Bh%lq$W8aw&A!LB1Z~n(4>_81Kd?Lm*ZW3o86(}Fe(P=))*TpNhqD(1j2JR0D zJ~7XWBT5GP4jmVI{D|Rrx$x$Fxm~VA;PY}&yzbnfIrZVEn}+6yWap`FS-QXU)E7s| zPNzhN?fRfIC1{TNdFB3Ts}hqtXN~ygmXK{d-A8}8r_94UW!NOGd{ zz4setsLY+Cf^lvRY>t?*|l`dsEY@(MV-j>{X>9n5j=g)Xs zB*v((V)4tp%D*lT-#+Q_yQiZYS8ti{xfjlFeWcX(_*cHX&?U3XD=(My>1Ai=omjPc z5{|dYruNON!r+3o+b5SSbv$(7X1Aa>29|N>^97if+an~)Msv&@`Sk5Z3Hqx<{M;Wc z3GkZcl-&DaWEL-Y{Ir)YD~=t_@V{{?s7g&QPI&RC2NB*)zPC-94%<-YT zrk=4+Lu5>U>!)>7g@zB>k|FVO?O1%gX?c0>nN-b$u*B7wxuMs;2=BSRIK2O(H&$|v z{he}NEci6^$BFw^qM6kp>+{ld=Gf-++h(I8F;gOE-u_QRdY0PY_MNov+nae$6-^%X z>vj9mhmj(uerAuHDjX%h{)F%EH+C~E<8xfLTBw#xTD|n7zproazN-SJC&_(@$s{2 zJN=RO$}qz1hx=dKZO5!`X5A#$D7+~Bnrgbk%43Y)#nY?Kd)3&?-xRo@dgOy{;mH-! ze>N!2x-;?pt@@Jror1k~sHv_ospy&cSZ-&+E7{Q+n^eQa255F3;G#3twjTG}Z}%!B zZW`s>Qaf5F+PmrLEF2f^7yq~0m+B-%-7LyS;Nx*W#oKuQKJ$rvPe&~7iQ~cX`aiR% z?3BAy$S+00Y2j^s$$t04HeGd@^8@c!A31N`gjqKCusT}`=K%U^@Mdr{Yz^3i;gdjdRVi3rA5*Ojai4Q z;`*%5!(3br%*Fi$@5kdybcwB|xTwV5RksG$-4LkjDy`{pC~(2|Il6)A`?7yXgwJ*{ zTi`GMQe}8VW!I6XbiDtkew?4&6cKTq^hR5}{a$jwFcls71Gruu8@mS-t_zm?d`4{A zD4U++9w^ry>Qd4B&1S{gPg(PAmuU4rqBcu-ugI6U?D~>U;RD*1+o|9|k=UlAi*P^2 z{d{^Sb$_AtQo6Wb<9>|e!2M@;>LCx;aPPuW#hd!QkK10-iQFwRdUv_U&_TAf*)sd{ z)@VEn3Y^pwIZ~_a@t?3@W7iwQm1QI*mg4w%Bh8WyyRV&qxAFPm`#jN*w^-65Qt|B1Kp{KRUz5ar`Z=#zCq& zPD}Q^OMi2A&DQ*NTCcY4-}-yMUCkjenV2Ira8`S2mR&OiH#*vaNsmbk|B zS#;$2_T;v{AC_{xnl&45W1o*jPSu#)sK&Owlv8syESw|U2m7{I4CsC_F@1Gg-`id% z4XcmuO2zez$D0`gQaj~_>21Z^ho>)l)4Vru%F1CU#vQc&D78EAW7y@`m$MozD=Lf9 zcNbgi_KEEOXJ4Y4a-`GkqKgYL506)vhqu2yxH&ql;?DDZer6}Xhp!EkdlWs?M@nek zW+8|5DGBAzCDb;Ji@3BYpyYGNDY@CwVwe|EUbuC4`i0v{nbq1hk7Mg`JNuL=t?vF{ zkcEWrPLVrO*OD|<{Z>Tvo%43Z7mw{;evL90=dW`B)0K#Coefs3H92{*qr8mQi=95W z{iw#got@=tFZFj_jCuF1Q@m!yUk%0m`}F!#p91GJ&d1yM{2#LVbavf0$CywJ$%5-~ z{4AT)2LpH4S=nj#coNpoz3Zc?%@Upo*Mx64C|j=SwLde!{Ycee(`d!t^ZvZoINqbk z5&IVK7MlB-6wGnmaK&rmZ~HCtm9KgvEIkIoW%9xdOi0Vx?E&(cG4d+&?;FYHix@wfcOM^~nPyLRSY z+x%`RsHKL+p5Hf7E8xrf#te`D;sl3REr?DqyxNxY@z$1CTU^RsFG$-r^=9qQZEgGG z@=kjzB~PZ`#p5`>KI49k=kIubaNUue19S5Z;(5s4sTcBtR0n;>^N?p+%M1ir=Dk-Y zMY@3O^9rBCmF};%$J^rhj`SUI6-U|O=|*QVP4=I1Gr{BK_V0(^ zdG3h(DczH|H6CF+q-(%PqC=sRV{hvVZN!d%cGa-3oNg^PrR?=Sbopvit*m>!HIv{E={&O z@6fla2i}jj4>ZqzX)Rdbz<7vdOTNu^i#Wh zXyD4qG?~f6i#EhtlnuHVoTKzHu14*hR^WzaH_g2RjVuFR_P+Q2gJoCMUD$Vbzu1U5 zu1D@%k9TUYK2l$Y=ZRK7`ppaWbopay^Xt#knQD3QpM9>t4o zebCf=By#V=vwn*&_10Z3;-%Ho_)d1TPv4_*%lmG+;T^xfti(+X_fvd5!`sr@$Ifc4 z4LBw~|3tjjfzZ_V+2fA8zH2J!*K}lfsGs75KBuoAbnlzH$yiatIK$#iaohfYw{ic+ z+#}X|N^*V)zxl82B@P~mtA)qklQ|zI?74R4^S1L9E2SB(B9FqxoCuHjbI^6@pYbva zg;xx7pZ?5XcA3!5pIWZ>3QFciEXs`kIC8&{RQ4Ji51x#*R5yIR@21x+s&>d6n@<(yA9C_OM3{DZ@&284 z=%_!2o8wO$sSv`~=Q|R4-!^!U4l3+brBDNP?x-jS2K+l-7vyT|n;cfi?8<=t)H#e9vq2i~s z?Bm?5I}7TY`eePjbFy#ojd`Qi`&X>e9~mruD)EcgY4PKWw|B?eL3VH4x9)e3iR#sN zc2n=;lFzQIbtzv_tm5hxc=ns)jpg|N67{aJf06^E@V08g>77x9P9ZpdUevfxwpLrV z;_afffybu@46o|tYcNs#Q|A)Ay-lY4R?ckodiT+@zid;yyb}9YZnK|$$51x?E<}DbN!r?_i=d&{4=7P&42bAop&?S=)uW|#3_5F`2(gT*{=7m6ubDQ1aChm zPc^VID4vYBb^X_T-lMl^c5&3Z2A#Dt-sAd=4i?|=_0_NscpLu@?&C0agmn+C(aR6W zmb_|89W?!=Nsd5waTm)(sk-TjBQA`p_1m;Se{`wE;a)b26=ed9-mlH7`TceMp#eYl zM^90Uu)OOYY7tqAdAOb;w(h%s6RV&W{#Zi=4R^2*Te(6!9;5u2^9t-5|< z&4ppNB1L{-AO7FtdJo%93S(l2)J}N5LH|`suiBu7i(ST~OuSWKsBuoHAKpLA+PX~s z{;8s8({eAYzVe7SWWl}qtl4Vfo%F$f%cpSf+e2@KPRqSr{!CN;X#B}F z`?{+K&dxZ1+sX2x(vFd4i)Ih+(wL)VAie5u92^h+zjx`wv&%j1xfETJyr^^ifU4Ie z`{F~7=6ne;5M2{_Sy&tUK1DyA;}jX3bbgVw|F7cH#^L@kT0Q=(%DS@Wve(35iQPkm z)C;y(4Yl~Md^ZeTYPwxRM<2(H%Z2m#HQ&=DtIvXOe~aT-NVkA|!^1oOtG@dj>-WfK z*Lb!6%I}NinDM=}Rf5}=2gjq;JbIDNkZI}vRX?#eJ`Jicdou8EzL9oA%FgEQKl(t! z?)Q89lTW--!WJA|XR)EK{!j6sdse=&FV|}fTs6zOuXo;DMVCVhtfoiWMQA>`J;qo~ z=;pgG#=V~Q+kI!}OkIz?B6k)f2#@{LW#j4Gvcz-sBQ9Cj-FmRP?-Z-pX1AjgjtJS> zZ*Y0&ACrCI+TN<9(|vzUuCDlFyyTL~&9NDWb>0oFmoS~ER8~E1)tlz3bf22n-S3Wn zm}5Dv!aV2NwE<(d3k~`9rr9Ta1@8ZPPTpQVs_xEv-G%w^=TZbbJy(~7eyXEJfuDd0 zVH8zoUl%7o@Ie)j;D@*b{5{lXxm2B99Q_x`qi<@1A1xH{hl98M!w=-OB-S$PIgVz&t-+5yw?)A2|x2BfZtk=l2@{@F7t&4+`Qli2?hL05D(5v2z_Lm%K=|7 z13Y33hSwlnx|imF?;Zgj_DFJ+r~T;>=E!oKqcezeF!IJh9i$W|K@-$1vq+~{v%IuU{4nC=ffrV zOBEW?M8^X^n+G@`On4T-q(E`t=k5f2FbJa3lzg@Xp|>p;{JfBW{|Ur{&x_#{M+tf0 z=nB)NMiVER13w!o;J*e-IC{dQL~)dXD&VVwC5*!Vt_JuJXxn&HkOz)qxLnEu@qrn~lKOV#b`x#85D9#w5GUbAKO?a-5NOQ&_ zk1XKe{xgQ+z?u<&A9G>O<>iNs#1>J$f5E8xJdvxRwG{Maji=8Qw0$)M5Q zz#2@N91e31Y5}$N_(OZfBM&|YhfK-wp*`@c##_dywjSo(fuD&J@Nt_qR_RQqW7a_) zTTa=vIWaRF*xSK450TI)fo+QC z5|b#7KJwsmt8|L`7n);0ar7yUA@cltY?+Ka@Mo2AJcbl!N(VSok;jKq&r>MQG~`hR z9@yvet~7!{9o9H^oHs%q_%&^?#$lR9aX>SJwoqa@gIvqj(VQ8`jq7KLduTA)xVoz}vBG13a4M*fL0vtSUETlM2$a4TL!QaI9 zRp2COThGqOa~H@mU=1cG4rc{YT)?vBS^(#(Tn{sbyCToO`}`v0c@37hkGXR4VmNLc z;J70Xcb?!zaTa%guMjmOtXT=K%Tj5UKP66Zqj)zN1kfX;c>YFsJvk5TKXz17uUn&wGw%jvw6MJ zHvto2u+i!|7>GP6ATKz=!nBggD+s8q^Wu7#yg-!!PXcIDX|Q(g%cycXuT{uX2^w7) zticpa<+U29EqN8XDm2obHLX0kS-)q{o)F}b14|GR_+f@&AGsbT=C#NJo(-Y#aE@+O zE**0y@;HF&JC1oR6*HI?aANkansS!*gthW`O*tS#d%}@N4lH5J>q^4Lb3IJV>yZck z_!z9F~7;M|(E4@?2Rz@h$Dyj68B+31f!! zOnbH<53E1vF-fjELVLC%4~}QEgmE0#!<2m+^5FSu`_Is}BM+<}Jhp7(l%3)1=l~}I zdEopRb9PXioygN2EMW@Xdn1-^i%75sJktY9%-KnCb|FtUu!PB{>2`0Lvm5MbJ@)RR zID3#s2tq(=e;&0L>}eg(9*VOMd6XaoB#V@uhv;~sz#akL7IZx@bfC14;_OErc>RJg zMP8`^9ieSsjYghQVfc)IN&6{I4Dx7$CG0<+&#v@Nx2mq-eqCdC^Xd80^ z*aP|)h#B{>curmn=M?h%yU(9So=;$j&%sj^=M3_|u?1wuyR^y|Ogq}fa~64EDa}jwY!4j5z$k|8RY0srro~ud1+i1__Rvzj2 zHJfP9l~$e|rxG^Po~x}qW+%6-r9IbLd5VwQg>pSiA4^3ZWw3vXF;+AD2mSZX*vq z*H7&`(9ZQV8+l%XB|djiRD-I>6pQ-6liyKz`=dv3B`f4ECGKK zT!O!bMjI=dQ$aYmuRf(X&yZ&}%dvg+6kLef*5PyH(E~D$=NZMRM4lNeXUu|tcH((~ zJm)y&s-!ruH3j@tU<2lJo4u@$|IAXM(lW;vuTf9M@fA_<; z$ip2s-f;3_I915=4lH52i$5Rro{qU1d0;=pZC}OVFfrF454Rt3JxpFO7Xcr(IZW-p zJ_xc09(XRp^9!bToV*xLEl~f9^8tBaKg66`ic{ACPCfE8fFrVemEBF}IRr-|ag*`t8J04!k?5tlr@=y*OO&tcFQaBPL? z6UF%gQ~`fFSc0bKRp`9CMRUF)PY5TTFBIn+@^l4DVB=-nD+V_%ZR_Vd@@xbKe69RO zaeg4rM6iVML?ji0iC$aIPvmjq#PfsV{6ZdSu!KoulP74vwwz|LN5DVK;ryaFzmezP z{rL~_z#MV8ep4J+Edu_(>xYj#a3seZ9(sob_x(1Vkmuj=famuhuOh&~{kapxfitN8 zI=_TTJeVUyaXKT`UrFkmur&uV8*B8oil ze8(ISiX( zE>Ce3kf$Cj;W@Ud=lyFmM-h2+Krrxn36lbcvjQpbYTNSujjjO~9@3r>$b;uU>$6TP z(;oQhe9OG<>w1q6+B341r!9w>|0pAmC@4EDR}cQzWpuezkY_ju$QG3Wye2@EOBH!6 z!G3_^!E(_aH7Xw3GpdybmW%d`ZsmdH;(C~JjX@qfjxC5w0GAFJ8TZ-XSmehg(0iXB_fy+nx5nXj`{C?HS+7 z1M7$DVahcDd2qRAZ?^TO>qi@TxaFEal?&dvx9-ohN0*9+_DpQ$f$dIvCbjawa&bLO zx%7w!G#4+eW^jMHT>8iZM{+#==yA%$a14-VAcTOV{UKp7%`rqCcyEq51{4SGW!_H< zIk{iPkDxhIkmo5_;(j=p;!N!TXBx$sN^y*k2ac<7jE>k7yOEA(I`YJVCG3kZ8Bv@W z$OGrjFgdO`4z}B#U&hD-XI%KYJcHtxAP>G?wExk=6>#}av}fuRqCfHaHZ%tjt1 zun(;8mUj7ih~`)!j|eB8*%W6E^1$Z_a7@%IcLzfejEp-T&P5(`V2}lSV46d5<{=L* zSFX9?8=5m8d1^W3nn!W0k*7D44bRTW>^#k}Vd4Qx+z+iOjxF+llN0r^^pW-7XpSB7 zgn%XPhqe@F0rJ3hhqW@}`s;S;#~yi{Ih+L)#{qfZ^&jSC>=E{gj%OjoaiBPk$b;iq zU?$Vf_~V2;wQOGCAJ38Ez-R^h!C)U);Yq7(yG+O9f;_Lmeq2A!6vwp#oJGh}&EdFG z95>`K03O&6rH&MhrQ>l&9`3l|MsXIm;yjsQDMNESkOvkVmuoS_@kAb6&+WIn7xE0} za6Bo_66C?{>UT5;rLOUfL5IL<5`Y8+;Rm_oE6A}$3xkuqkHIfhwnHQ@XvuIeoblx#ewev z{jbkf1R)Q6B8TfIkm3X*&k*2&>3{^^osMS}@(6WmxxNQeoYfuRtf4roDNYFTXfxsP z3bbRuLnDxJt8soU@~8tDU%x^qPAKy90!M-sp74;z;Gtt%&N}3QS4=TBzfg)3hCE;p zMDvX`%M&ipoN$H%CKK{3Cye5(N1pEN{sqO_w`k4=oQ=qXueXN{H-cfW zZMilf&o8jV<=RMbHX{#w#SNzV+9-3Hvjuq?03JV|-%N3~B9AzDK?Qn#e;fxc(QV_| zhCJ5?gX=O_gJ~;=vjQpG84g&7mf1qlooLUFRvz8gp}lBNL@Q7GoSm&aB`zsyTn;mT zjzpe+&+B#}j~rOy_KM_`i{b3<0A~-y*-dfwc7U^w;_Rh3Q61pyM;`9=B8uWfcYqT^ zaiS?sYzH_8kcV5(u@vWE2RMf)&OwTExC5Lc$iuCl!xSg31DvB2CywGA>i`Gdy9oI3 z+`|3v7{xi!0nSN^bAsZ;BhOoKj>5dIR*7z=&-Vo6xdMg(JU+*BIL!He3VGxJ1FZ0B z4^F$x^)TnoY2?WQ`{8|Bl2d3q&o0g&PYx%q(^Ot(Tjh0k`rB1p50h6S@)U!7V7Y|9 z7dzACNq)4-BSCvEAP>B6fSjKuTUvRT zn3ItQ&n3F&xdhTNryvh^j-AYjnc-YSo(XJTXS3jIfo;d|OBClK#kq_;vsljM43Tp* z=L+(qgC(xR%M|A-@`$jUN9KL?Y0fp|sRSO}=dV(nROEqu2dw<%v$xlg2V6u6Czayd zK%Rfca}#+gIh-35Ck=TbfCr{mM~HD9p)pC=OFNW46MPFN9A=FdGL7(KZi(r?zQs3ISK8#-^$ZIr=XQ5E&PlY zm&26n0rH44;qrQX%#)+bRfs&?`<(}zaxt8T$OA{ku7IVnTox3uZT&n#9%axma9cd& zaG16zLLOW{D^-pUr9H*1Jn-2m*Tck9f;|83V~>%?6D)BbE8)b$a7vK}&%-xtSkuny zAZ5s-4wiUqDWy11I>0HXI8P|fQ{?&gJhK9Mj39D#d$_?Dv{^k_51>P zl))0mQ%P}Nc7XE=dAMWmON#Rvd6GaM1gAVtyHT&5=PhrL=NAYXKkt6c;V}L1E%Jbe z>1eFGRW<(w*TeLUD&)b>^r9NWLAP$(=c|!N94v9!t2lWvoEqeT_nt5X^>GEa@ohQp zkcWE?)=-@H$dkxmcy$BLyV0Cl3C2*H?=39eMEl zNa@N=1v;J|$YTPQxF3F}I6sl62OCfLn&X)?=NIxc01xK;q&UqT;QU6OA`YjS;`~7# zDc}JsyzgUeOz3!E)wTY-%pZ!wM;;r%0M*F5G}!MM&FO?Z5nze$LHM2EjEVamAp!FA z1uBfk{#jNG&4Ew7z-LjMBh*2C-iqjQ&;A?HLe%gNz1N?-G zfUg6V@EQ+OSBldOdH!8L63D}?pKcVVJMzo~!N6<5&m*6I(B*>P>LcJQgG$ze!GP4A z;`BhCc`T>1f#(dG(-V2*IGi37M+$icgFImUG>$SqNORyfLJ0V7z<{6SN>Q9%$b-+n z{oWf#(Ht4%;f_DOC{AzWf%93IHmlqM9Ue%no}KnVo>HLV{CZQIzR2_MIZ{95f#t%S zz7(fF@(cxe!TQPk+WQh6k1X=Qa$!z?iZcLt@cquTr{ij9&Oqeh-tP>cID3D`9&kA6`=fz-(GZcA30RxT+2}6H?sWy;WwU<2dz)=XF z??WlhFyw*r78p-}mi2I&qkudPoOp&&97W{$ce@Wq9%sP8?WIU@Mj($ahy|?ha`ck5 z=y;Tn=Q>#8Iok+|GZJ~=xeim1zI8m!f$yFI-%$qk!#ad%B*jrd9(W%E(@R6E(KJUD zdEoUFb5tmf8uH+NICk+OW12Gxd8$D^xPH_q&S>NT2cz~n-82bImw?o&-Nzsgyqm-E zjHWnaTXDXn*|#$u!dFQI{JR{^Sc;>8Jop}DOiB+&Iv!2rfnz4V4r)-Gama(m`Kg|{ zA83vi@_Yjvd|e(#amFK$6!5^9x*yu1LUSe{j{~?4;`!=$ilg0%bMD5jaWn^BxxoAD zI2>(?ql-M}!4mc%n@y^FY0gCCxeE*qzyp&mhr`^1OhO(w9)O~u*EqNy=H5XMc`mYf zjUE2mfzC^x;Q$cWiIN_bmjRU**Tdvxh&-`?0Q+>t;<_PpUXzi>4=nL~)R2=G!ZSc}|o>`eAj~p=I^K>S~u|OUNmZRbE7TgvCsZ~6)kS85*@O8(6;=pg- zV7{~Q@A1q=9&S9A6vqmA{vFR8Djq9}1HYX^z{lgNl;=J$!~m&Px#l4ccRZX+apoh> zzsEys*dk9n;Nbb84aKoT9#L>4OhFBr2{dN` z^1yn=*Bv{GV~;#|e}mYFdo;%ZdCq`*aQW>i&O+qDA#(mG5C-PhY z`*GYJ6vqpBlvvNeLou#2X9@Da`*+OoqB!2jgU2z=VFOd?`td=YPk@8_nK#8*iaa=< zgB1b!G{=|3gE>nn&NAfr2J(V^V^6WqLz)AxT;To`WQOON%QzfnocBi_JkG=KkK%fm zu{QvDaLnHnF2&P%!Efvl@VVz;04Fbov!WHJ{r8KmM4sWCKDL751R{?rC^t;vo(`Z* zfz)bj2|}JxfP>GSK#CKLJh%@!TLj;wIjfMT63}s51XG;V$bmV;!*M~3mo=S7pA`fg+{0uFG;)EiPIvaEQ-=Db-dANNwl;VUT4}69OQ&i}u zcILF<$nzTP$Mq9Nan>WxY~X?CPfBXs4!T?$kmneuz1CBljmQJ0nJDck_pYWnn~>)m zaNz#Bk>YGd9^CHy=ID0j5?hc*lG6`2Q=F|Gh-Vw}XmL1ODb9A}F#|D!72bLMeM{+b z?LZ!Ru*Cg&JH?4Wo)EU|uaYMnr8zs1r<%iwpg57pGZb)ON`5l}Oe=uY>ipV;Jn($S z^%zNUb|VjLb71?+YtVa;#|Rj3xpq^Wy~qReft>bV6W@nC;9^Yr;a-Xp)d9|aiW5a~ zqLBw*3qJTRPNC~326;+3@kCRcSmeR?_|rz&gW()Vt?Kyz^1#%-$@1Q zP{Q#fK(^%^LY`P)Fb5u(4stlmwcs%FfHzO0qsx;rkI|kZ$iux}(4M$|dYEg$QRLA= zk$}%&X5UGoV?M@kz*3U+9p%K#aE>F74!b{WYR`6_1Hii>@VOff=QzbVi98cnPHe$2 za4Qd_R&5baaZXa4gjO849ZjIB+Hy`&oCJz<8hLOZGuUMYriyJjXOM?GhM%T5XORbw zg?r~F3206t@^Jg=S&EZ{Jb3K=pzzk7=A1(wPflM=qB!tLf`GpZ)GS!xO&$4YCe68k zJVhWKSI^;j2-8)HlZrfe zY}p|>I+NyHM;_Q-n3GCzZnWaO`93d%=G;V{S`Y^wqi#@~G~~hc)4!`#JFhpTBM-Mf zr%{|+$b;Lx{ri**9wHAucWy+zYv)r$jOW0 z6t&_cFH{G^9*|m{%f-kO0yy{?ND;*;K_2`Jr2j0Tc$)JVd2~4Olu#U4GtB2EKoGn) z-;#rAP8srW+oF`>JV73KuERQPe?C@@Jn+20ed7tmd5S#FzynrzCVcP7bUYQvqX(9_ zO`lSnXUK!&VV)I0BKKLrbL2S&R3Y#O(=&=w*@~m>8KzFh^8$H50}xIn#d+C^vt`O_ z&~!j*70)Y*^OEAcZp9Io)bOG?Zz#@diu1M==e>;lGMZCGao$oKc-PSK`!emnhoJ^} zvOsBZyH``3cdg<%U(wH&j^{nac}H<-kq2H=U_WgC_hmjH54@(p{sB`h#i>Ic@WvO^ zO6N{bZf9;&k36qIM);X<9pXSD?om9R93r%w%Pk6g63Qg(QNaQ^N{3QRD8-vqav@NtCctn!3-Dp;kZe!jBGQ{oYZjS5m7QkMDoIxX{A!bqgXy0e}ZjvA$mrxY_v zab=ZzJREZvs}v9geqsTpe4=ErN>4sM3ceE&rhB9m@V!AWE#YI&0ML_|B*$~-{|4Qc zwO|?i*I_*y31=azgb@Wk>40e%QQ#aDrrkt=qXSG*MCmDDlwL&9WtCe*ac7lQ#ou*wsnyl0gML>b3UC_;%6%qq=9 zxyUM~NM22>@|!4X>_ApW6jxT+MwBb8QbNeX+37i0Em17lNz#6z#Iec& zqP%037@}x)M_GYyGsbzXWtDwIdB`e5NIWChhYnpyPY7p~VB!&GAC9gj$^ljhBMSPG zV;=loHJtl&_7Wl>6lB4QbMUg1FGEDB5iL!}Rwi2a`Rn8M-1bp)jNOnXCV3iO) zj;D}SHWFnZJ01K^6mM4fN|bC?`AHN>_64_Rq%AyIrIIMyS*0I2>I|zqC7wc7@gts3 ztO7?rcfJKU`}w@Ksdqm{xkWgmSVf$0rnAaoqAX$+hCHWTayZA+31t$|O0l2j;^!3L z$7GLkx1a6p>Al>=myuvS1I6FRhpWJS(XBX8$)S`cR=GgR)xAG-f;k``x8QJANhZo1 zR=G%&5LQVg%4t@)N|XXtxj~dBR@q5%7nkLs)IbykR_RUp*fdtDCma`6*+q^DVU;GL z#IQ;uIqDj#@Q6~vDrd-1O|0TbJW^yj2z!f9yKSn@@i1-b$SS3TyqZ;v2zfuN+#Rpg1Xl2tN^vX@mP2>Cp#coL8$W)(>iPco~hkfWMe#f2!k@=RV%L|MTqdW3U`Rql|y9((&IeX`Lp(i(Gw}ov&sbJr^19q^URE3r z(>Io}N+Kc0u*zOiW4Wxdoha~qM=*8b<6dXWD$YdN#41cYY3;^yiS?W#xj$!>D3X_i z63hv_n};ZYtRlq6xgTW}SE3ZM3X^+wySe{iJ!44j^G3p)c)VhwY-bf2KF%wNRUQ#f zDXaJr&u>;?a=+JZ?()hoC-AN7q+F9(Ea+@gTtl~wK<*edClvq}QC1rY4601A|9sw$07OT7_%2QT(NtDm5Qb`mE zRdg`#Em0I%r6n(8R>=e&CU-x|bAnQyu!7v6Zu|(O(DhG)os>^tOk$4!8bg!?tfD}a1FWJ>lnPdv zK$HOnJd~7)V#_KAN$v+&Wf@VPv&sM-j%T7F!;v9MG^*BTkfTR&gUrBdZK2%G7DhQK>|UWEBxUF4tpLc}YA2jF_Wj ziN}^zW)LNoRo0TDOs6wPohF_%R{2Ie-DfbK+r;C`Do2Uu5mrcBcp5WDog_+U6GllO z%3)SHPL%PcjAuGgN?2t)QGCr9k2X;xW-`hYqMT-xX+$wHXFR$@dCe+%LcI-^Ks3?ch|$Tm5&u!OC<2ph!V^yc|?(&&3N7uOA)jWIk;GHPDmRGI%qpvi zVmFT=yA$OmtK1>V!1;{lILXV5RT4>r+gRlX@tk6nJH+#pRn!Ps#F`rlm#{n=QGJ`4XZd1sBeZWxyw6l~|1`%bN6LXXn zQF2)2a3_3JUuVWMoRHmEMU^O*Sw)j5U0j%>bciySRr->=GFU~9D5G4Nqs9~E9ILnx zW#A&l<4cq{R`Dl_f*a#mL6lRh5=0bLcgC}tD50#fjwl9;8P7(d#InjsqLi~r3Q@Xv zFh{*6iY=?u5G9&bM(}YxSF?&5Q8Yan&N!lkvx+WJidbbkQPz1eN1YG4Wl>Yx0saMIORE$5;;_1jxhe*$tlO0 zltqUr%tvm4L%cQ)+DW{l}J&#nZyG_bG zPPxaVoXsgXqOC;mt6eE~yHcKVrMxLpI-(tsZZ1=08B#jB7IGKqLoG<@iE@2bn4+dB zrOIQU5BD{#=7W@Soq{_|D5XrNJPW_bl&5vd7>(&1l8hPUva^9s2?T()GyE*ptUw7H zvMH62G7f%r(yU@XV{8gKjnjZ327@d-ObX?6BS<+)YgG`ynL^_0LCP;0W8z~{#-Qal z;OBh!p=bpBm<>2gO0ibI5KU3dDaGuvZ*^_iq^zM+0wq9e2Ay?n3TMob0+`u=S+KIY_5azWxBXTeKaMFOyQD8_#N*Wop!|u9Vw#iq5F--`cWi zm0!3pU%4>&z9!0rvD4y8Y0xPqU)?%IyX<>STQ(8fRAUV5PLrZ(?IuM#eYv({qH(Rk zXcxcL7)|m$qA^DOKRP9#L3``kVp3-66vNujl~SQoigjV^Z!GJ!jKhzu2{Yg?d?HMx zXmyi9^#^U?Q)~*O+m}tzIgPq7`0Ns~ig8Wv2Jq!2?N4zDcXlQvPzk1QgY)+r4=Xtv zHib#P2MtD-&U|fo40`wp{CuP_V>HI3nBMpUjS1+mF4T5Pbh|iB=fc#DR*fNLN`MHovc65=_h^11v0DW^bCV+un1U8nKlwx+# zkpeDaZR)ldHsnYF%yhu?xG**awe|tbLW^N7lR|li)9~dHU4j89!qM>81j7n2%A^?9 z48htN{%Ulw94ROS2aYQgpFlBKIvDGAUFy=D_*w zHO7>nNx_WzFtDRFCZMaPNinSFA&#U)d7lA{_q9JIm~?|lfnR*z9;x1E`r#0~F2MO1 zXUnpv4ekUsw%2w7D39ZSN_Airyc8+B`F_HfxzDTsca?Tj6gY{$U@TshdlI;2Vmvx>71m4bFg9If)To6C-a1-QS# z4?a<-?D(NRp95A^3Y_w_upPB7f$`CulwnmsL-BnNKRwz`g>IE5g-ZD;u)HPUlvV}! zd=W5I3MBjf#9C^d1hp!_A3O|;3I|8CjUiRpsVs-6fgP2C=#7t2dEeBhV@O>#p3y8* z>moWuyG%R2m1RsC2R}P%jOih53g#V;d*O#;NPoi)bnuc=%$)XJgTZ!z#}*nhMz@P2 zA(dDqx>f#BV~mGa>NuMee4de5CgnO;$}*Eu2B@pyuST0fZO)Nmaymw1Ov;|FlzLan z1d~z;S;d*@IFmAlLuOMdK^>pW3t9~4!=#ve)oM&xIatQ8F`S~~Q-(DP51WD$Uwrd1 z);x+&SsBz8JWNUnS25qG8e>vE*D2JGA?pC0V&Nq#%n>G( z9IKKE#ZN(3t3h-vj`)!z9!aC);c<}&VA3)8yPU+NM;d6XniBJKL7K@@amjZ*~T@@B8<@b zR%l&$?WT);G)|jL`btu+Xhh&(0a8zRiSxZHbK!4#Ehn z9}El2>x{?lT3}gq%p#1?`q7oIQRX?IgP27aq4kr{Dy}?ip=DjhEW!w_pIurT7_FC? zMHr#=i_m(a<&84S8g@Jd!$%mQ^{dc&am-`us_!l#elKPbMri#ev@SX9j`@~#7_$f? zw6KA!JVe(+)Y{49>wabtMraKomTq6m$3L-^Wqrjg!U(OETw3Z~xR|f)Pr(2Ce1s8N zD+{eNzGxb1S+ki%7@@U_OG|xI2{le+7GZ?eszR%|YQIv;`Wv$dBeYg?X{mP(qt>=3 z;{TzaLyXW`U1;4NSnnpwiZF{XLTf0obbq+VX`@d#-?Zf`nMD|(wT94|9=T<2%leU7 zgb`Z9h&5DYz=!?Dg+^=YNh+BzLTgQ-wM+9gf48iwm_-<&wHC29Wv%94&zgLx(fW>A zgb`Y63#|pq4%y4HCZ4Q}5k_dOV_1m5K5AQfw=@2A5wi#*wAM8&_{H~suf9}nSwAt0 zFhXm%VWD-@TGOuzQdMrdtDESztk)|}ICc+RpmKLeHEEd_`XTAK^48@m57!mWC3qTM^3{L(95=S%eWyp)Sr2YW!=dv!U(NxO}^0XQ!{q`U|kc#jn7tE zgb`ZX39Uy?`E`zEEn*g7gjSK!f?s^g*Z=h;%X)=bgb`Y!gx3B)?)QRajajNZB#hA7 zo>)4DySH8Uwq;$&EW!w_9fa0ZuiXBMW&Oe|!U(M$iKXj?uV&XjSXSs9txIN ziCKgZT7F{b^16M_6ZMw0?|I5Y!U(N^&|3Qa!2!#)5E_u zJsvZc@|tn#UPYF52eSwxw8jao)y`dfkY%lVvGR~GLTfK#=@{a?SA81=@$;EQ7@<`q zv~X@?S&uV|FhVORwD9^%efI>lMqPschm|+P2(45hJuF8Wx^Mt%J;U${Cko8hl}0u^ek(lMD-fAuY?g<1*yHog>9^tg|%{ z3)>egBlwn`{=?n3m>9l(86?3$7?H2ZLTlmuU;NXue3vU0VSKyA?dseVV(kd0QL8F( zUcF@%F^e!F8qLJ21jg#{v-@sK4!5jH%p#0nO(hogy{NU{&x`l4ti{YCj9~3eEFF#J zf4qFoY9{iRGK(-G8qReOy33Mc-xJ+lZS@-<833s;sbYky`DMriFL zv>y59@?F)JI#KIPW)Vhc%@$g>H?O^+Wj(?y!U(N7LJNCdry=Oc{JY7tt#!?w7}vc@rs zFhVQh(rPeTQDzZFXtfG0TrQ47M&zr*l`qSBnOTGp9(Ia+ zy*I4+e#`onS%eX+F5@9a8Ao&V#bNwwldJLnJ|AI3zPb$ye(~M=^~#%ARui)bBeZ^J zJVdM5AGVtq9>FZa2(7r#!g^>~*D{MRLaRq;;h1u{sfW)qi!eefA+)f(ENi7}&}-PE zh!~;OORSM_ZYB8H_M79bP+#kYKYXRkB8<>lD6~Et{b0RiEoK&BMBP|KEUfdW^~lk) zPqD0fnMD}k;bLOxI{$}}|G2=ihFpt&!meP%2-Xr}>9)Llb$_E}O=cEh1nXd3n}bvCmIBckzpV(D4Rtd@m8Sk{NkB8<>FL}+1+`qq?k<#nhJ_p}isvu{lk*RYm#6SD{-w2ly3m@oBRPSkpzS%eWBh^7^?5@;9m=wMHr!VqR_$^TGn!A5k_d8L@Z|vE$ba-5k_d8EVM9& zmbK|}6%E1&ty4_C;1`OaWi>O4Fhc88p@lKDtmBwP7@>6fI;+7?Wy5&~TGo~~LJ}N=5v=oxrE3(9M3&XgEW(IrTtKX~z#{zO!;#3co?sSX zgw}<`(!J<4M>c=1zD|OkZFdv?-{&KY(7H%y;VPD8wJ?h?LhE8;>3repocb~gYFx%F z!U(NPgx2wMj~`-LuQ7`-LhDkYg{yN>GdgT=Gb+MUHi!{gmkBLg#j>pZm_-<&b-B>O zQBQp_1U0T;7GZ?e6+#Q|s9M(Qw;&hT%0tJ>(v`&80?uLW{p{KJ`7fDPxqw-O5n5LX ztzp03G{UmpXBJ^ZOIb!N-BLC@;phV`YyVr(KUmvxd3ZIkhQm4Z@cP^8Hn6N$m_-;7 z!)u5&5*Vw%&#NbN9&TB)Z-XQ_2qRe65(}e&>oy-OyQAfP({KERS%eYMxQZ4K4`S=U>0G7)*lTEe)0X^f3}}#S?@86FhXm&VQH=T zrj*yc1JmFxCSruvp9~ANaNn0@?ZGU<2(254g*_XV*Q&Ryz3N9MU#-j{jL^DKXk9mE z`1+Q0A+rc0YSc}{(lu(y%pE?ktQVO@7~$c~#KL|9OBt^MENj@EkOT)|1nU-)FZjiG z_L_T^SXLvm2qRdx5)11F9$3mBm}nfuEW(Ir+-6v)waynGK5AJvFpDrk>vqG^S_@4n zzsfAa2(3E|3$?J6EoKYgv~v zi!eg#ZkJZ8$=7;!qau9QLwV>}X}?Ek;qE@m8qX}k2(5cvTI#D$sBti}2qW@!pU`^k z*gdCP*1woV7@>8)ORLS~D|`nMD|(^=F~A#kSu(Y*|k;i!h=Gc!XHa z9>B7O-lseyjPUSLVmW&N%W7a2VFc?jlP~zi_v>xV8(G$A%p#0n{U5QMJ-}Z~G@fG? zVMH_@H!Re`9>B7GV-{hA))R(>Img~6Zfex-_hTBol@>8V>q)~xE$jg-Yc8`0Beb3} zEVR1)8V9v*XJ$kvF^e!l>uJM6t!s|jYCX%kg;|6VTF($m&jDUua@3~k8-=L+DzgY9 zw4N1Ot=&I=W?91?KrXz09x3^epA5{cEqYtTi80wg@A%{$f~& z*krG@9#>!X!N0~bi!eg#uZ9J`_(p6Oh*?$#vj`)!UM5x%Fp%~7-_~AeSr;>lFhc8Z zLTk()k9fqgo@W+egw`v>!WxBI-#pp&f@KYP2$J9+jL`bK(7JH?_{o-4@(_`Hgb`Y= z5({e-9v{8@`c-pH4-kEb5`BabTCWMM&VNtc+p;cT7GZ?e>%`hZ8TFOj{`}qh8Lj7- zMHr#=hS2)=RqO9-St~!Rj1fj?{lld-$7oeBi!eg#O`)~)`mffvtafG*Mrgf7EZr)9 z8~)e(ml&)}nCjbL@_v-<#iUqOrxJN{cWe8Xpi#_W<8M zxO831I-gmD5z+XNSUU0;4a?f-F{MQq;o(Qbaz?|lmNJVlg7q=6bTke-Y3d!8HRAu2 z7GVVIpTu%TqsK(!JZ2F_MB@`;>1eF=L3xv9ZS=U(B8-T}r^Iqb!?Mm~7GZ>kpApL$ z4a*w-gwi66V0}(36OA|T`kiH+!z{uG*1w45jK)F}jSZhvT7(hN_<~qE8sC0;$P<=z zCbI}5qVaEHIiq1&Yd@v52qQfFl331YSk@`bB8*^tMJyeSrt@CA(z4cjT4@nRu>NE6 zg$SGxO*J#3Gnhpf5sj}63x4t8y*A5wnOTGpTHhECk@Wqvtzr{Q#vGM7@@V2&??&9$TH`aeuu?Qoyh8h-@*X&mp|7cnJ zGm9`nYYk%Qx)FQ#n9G|@-B`vf!U(NlLJO~MEUWu9 zU9kuwwAK<@uiw1!S(dfh8;V63p|!Tqy6xLTUbn25m_-<&wT{pl`t=c~S=J^0P+Ei$ zTI&)^*TWMo_~T5=I_OQsB8<=)F0`(BZ`7fdRsNP@5k_dOXIO~9x-revjg{V3EW!w_ z^$knc4a;g`7GZ?e1}0x<7wg6(Q#a0G7GZ?ehC&PLhGn(AgO=cn!H5xB8xd;uIjjL_OlXg$4l@$r^5^+UxXjL_PgSU9)A1J~KMH|4eMM=F&t zLTd}5_4&uKiI&yPEW!w_EnQlpjn@6lB8<@5N@(Glo@IT(EW!w_t%-$e4tT|JeDLS4 z=gpeK$d6UN2qUyc3awEm-F~=bg_%VdF{|8$SUa-T$I%PhE$eb-5k`2pEwQka(ZjJz z&p*+!-e(qJ1Zz8DjRXe#;=6REeJ{1FUH=J5a1chYiiFm<`m37WH$6Z%vj`)iF^X7v zG#~j*M~`LQ#w@~!Xlzd`j0U#Kk*lu#Ps{p_S%eWD?qEE`9AGpoYuqQ8=JOFouy!OC zmhuqz`Lb!vvnQ|n-m*6Nl*m59 z2-YrwweX7IY|EOsDqFMwD_jv7DuBSwAw1Fv7zcVmV9Mvg*E4 z`67&9)e_5D%9gd1S%eX+I>EwHwyZapMHs=VCzi96Eo#Z(pBQaQdAqNbhlCMYjY4asVK1CuSqCwTFrt(v z63bc2mUR=e2qQe4L@Z}1Thy}>NP2v#$(oTY48qrO!h5=KORDzTiUY*~jei!efKZ<8-9xf%p#0n%_NpCcHrlfO!7RdvQr@3fSjt$V_B`p7i!5vVAC-rM5gyJZ)~0Y8=Nq3M`NGkb z6=xP<1nU5!g>&S?Ho3URvL0m?VFW8gEIod0_|967Y-z6bhX15IB#em0fyC00Kli4a zzObxC%p#14M%Z|W`8x5JhT|;jAIu_*@UVqg&N^>d<9}8j5=O8h#B$bo%etLegb}P( z!NNLkSzG*~vHkR^z zC(UiPtnou|{Pg(lEVOWSZbzea7PAN=w3Y}he4n0WeZefk2(5#K7Ut_-qcv$YsKbbbH3|{Ak+14w zW>#3%B4!argQN1D2^oj8dAy4>>&o67dGSsp*T1T-6BUq;s3nPyQKI3CqFEEQRA{u89OGg8r@v*G))>T@B z5n5*wYb5Ak#z-#7GZ?e1um^?jMi2gDHdUb z)`da~-w5}{LhB-z)^$cJys^?EjL^E+rDa(wZ=zU)5n7kHw3ZpItC&R?p>?Uy zDjwhRl4aFzs-^y$jEKh7#L_M0ka4v$EbCcj5k_cTvGXmxL?vDv3rlP9iWS))fO7GXp*ZX=d68kY4uvj`(Ryq#FiXjoS7_DYK|f^`S6 zbTp2C^SI3{YnvSui!g$9C$X^KK*X1b3&P7fr&-n=%p#2N@L^&(qhVPG?xwT| zBUpbXmX5}xHUD$CWewk5u?Qnrj}Qy55b>zk`P!#HHPN_-S%eYMc$8Q=8u&I6%i7zo zvt$X&PXIs`L z#fn83q4k8&y6M6jt1atBW)VhcJ!$d>c zXsucF_xCJoSNwDo9E1^C&l3y#UOaHM=KvGKKQoIkLhA*gh5O7cYwHT7MHr#=qD!mD zX#Jj9gb`XV39WlC`}76NdV*Pm5n6vCmR^y*zVZI~$C<0_4J(z0gb`YQ63xZ^Y8Q&GX0pVU%U9Ge&tx7{PjlSd~y#c>n9* z^{ckAtQfNhBUpbYmM&$y%C@Wrm_-;7jaP}KdmFsUwycqRC=UrEv|dZ2btJP0BeY%@ zTDX2_wccYEVT9HjLJMo}VP-V1+f#W+7@_qKp>@g52i#~`%b7(Oq4lOqD`vDd7^}1h zBedRfX<61G%p#1?dfTP-xY2r_S%eW<@3^!qt7)9_kT62)U1DKxgR|vhulRbSWu~Rv z$}GYNt@ngh>%K!zx2#W@MHtai-Y1rBDfo=4Wo^5c@{lmX!w-mswHFb%I%iq?Gm9{S z^`T(lSZP@oF^e#Q^%1djzfpYAC4alY^ih9f7GXp*J|>oKDT^2XGR3mCtWxkFZU{f5=5uT~xsMri$8 zXkowcqp9;3FpDrk>r0{4Ts`3o%X*Mmgb`X_5lh#ox}~=r^0=u{?=g!oLhC<5Yr*L! z%(bj7YgE1nBWl#w#KIbdvy`^I4m!oMW-*H}!ozQfg*^Zsc-PFb&SDl}1nXO3>3gHN z=hm|R$}GYN)^~!1_eL#iom!PI!U)#)#L~6*pF3=Gqh(ED7GVVI2a_-C*>Kl~Wu3$< z!idQKNGx4@@qI*=^;c#QMri#+EZtJDMwOTzd&@eNFTx0|pM}=FZ+@4stP_|;7@_rx zODkZszG4<(gx0SvEz3HfUU^6uq4k?fYagTaB(n%3w6HPR=f5ytmQ^}lX%R+fVY9Gb z4Z|3IV_M23%p#1?LU*hd&gU)bab^)lXsv8mSm*Ism;;U07tA7z&|1Z?uzlg%_$_P1 z1eGts2(49#g|!!tnRmYU=UGOphFOFWTB`{yyc=j)r!tE$LThzm>9#y^)g6{iHa*)b z%p#1?8Y;B#xf{z`)}Zo57@@TWu|`6kFdAh;dTz9=c%xzwMraKaTK7Cq9Jj3MiHb!S zp|z&adS>^F*0QYOlN5_ELTfFd_4rnMK5bcVF^e!lYi(j-3^8B$OqONc(4@2oBed2L zTK~Lr(#@82=w!trjL=$_SbB8m{O!_}%T0MTO;Ieu2(95l>)iKe?`T<%F^e!_bXbpA zdUOcfwA}}mHLh7{5k`2pKC%1|3Fr!FPoQPp#Vo=I)&|7VqXX^==FhJ{*R&Ae}hWnIlI#7M1ChUM0Jh*^jMmg5T@+Y?KV zO=rycY_`?*6Q37S~(WsF^B;h>@%vi3R_oqk8kd9&cG=nS~h1+KE^r zfrEXWI<)e1U|}mjjH;RKYzJ}D&dzDkn*LX6ZJ zM=aexH$MBy7|Z&VS%{HZdl72~*4lEthIcKiX|{?yVx(488m$;PiC)RcVM~^>#@l$weSV!z* zW+6sujZdRBWM7qc#7NczV(HOSdf3h^!~n|?=LTYFpFX{Ge4X|1b7mn%vKooCt`hN` z+g`h$Wv#NG@(?kSHIZ1lm;Lh6)d8rLM>7jCk~N7~>#)|gs|;zitO?9QjAS(tOIshY z-A3T7{;Qo?h>@(xf_3RJ&2ud4Xl5ZsvZe^uk~gm#Wmy+63o(+_ELc|*9dnLl-N-D& zNY+$h=~BKdbkT8^^(eCt11v}F-J4k6wf7TdAx3IVBi4ZsdH0A9udAo=6fu%Domjf% z{_@M+>sb#^0~Q{Lksi(uLEW}9GY+~Vfibv(B$Cg;uI&)Pt5CbendCeh~cQnQ_ z3o(+lZyMIE%tDOzP+d;!M=Tt7Fd{odM;&QB{Ek_Ok*xiRrLW&F@m>FnWvzRF$`@iJ zYc8>{HYo)B0tj`CVisZ~>i}Y*hlri9@YOvmYfok&MzTVN<<8f>%tDN09Y`#luR~WK zRb#alG7B-16*erL8_PO_S%{IW7Gmjq9k{{I_gdEV%tDN0MGVWGuV@&z!D_hY-S-fo zj$4_97+^V~(LpRd8(bXuXVh3;>d+pp zd7vs~#7I^*v2--PxeUL$rCCMHLX6DU?})WCh@joMbC)^h8x71tjMRz~OXo|L@_c3? zMzVT{NPBPoqvp3h>@&?X;^PF3o*cQ)Qv^N8pSbu zZEpEAM5yBjW+6suEhd(Z;W;&5Y;9Q^hE*veMzWR=3!||LD_GVp%tDN09c)@r2bnm|2LCT89!#M`O*JddJG!Qf47W zvJNBGXwZRQ_|TERo>_>Itiy?=Yt*J6zct-@_z<%YBlC3xv33F#%-087EZ)bm-eMMF zq}Gwd()p4t3qo^`&wopMzT&YEO)-1W)@;3>qKJd zeEqo2#A6Vljt`lI7|A-xXsyCn%leI3h>@(54GR@ytK2lAN*OVdb&6rR^R)-F5Cben zt2~uh#h`~Y&|@XS%{HZrx8m>L(bVwU>0H|>vY4?k+WLMnS~h1I)hlcl;xc5 z1!f^evd%OtcQk%s7Gi+qh{jpOD&c6%z3JWytcOLdD)NYtT4xhWM?=>6$;?8GWGy9@ zcb$(j3o(*)4zYAJWSu{kS%{IWKNyxf8jmpxF_LvIv2?y#*LfQ*`*g{@&n(19)_KJ8 z?t6zsRSXd$S?3c==S%jzTQLhUl68S$x${-WEW`lIQOXw*YhNzq|4jSmm55NsShxMLJY7R z(YTCQdItN47kWon4?kfRVx-pP#PW{D>hn~-5F=Sv5X(CnBbkL5$-2_8+|d}rEW~J* z3iMUP(pU61>|U}BBGj=Tvk;@wdN>8oOFPTRj$Z#+zW&HpH3#-)G7B-1b+urPT<3$? zmbIK&h*`-OYHLsQ_C#CRHMG+jZRwp?1hWrV=_EU_>VcH#b z_I6=zCQP>jGtGs$#icdFg}Ie5&iKr9VQwRgGw(aOFt-!t2!}s&T$nptc6N4Q?j#I! z+@}8Q>%!dSvNPI+x!Z-=Rbe9Y7xa{48Towo5axwl=?};M{sQ$q)ZH78M#AmwzN)I? z;GF8I_0_XOlY;xgpC$0;Y{%cBSvmnuheADx)=(lE?}&AV6H#9%*10g;9%~JChvVUn zD4Ym&cEvi|y5OI9G|?OH6ar|LMEs$quE>H|A_(S!!M*FNtAh=6BH%tT8@vj)^aR0; z$&<;4_AYQ8J&ARNy5q6VL|dq`@Lva-^%HqpLHtu!qmX8T)&v-DlIQdOIWG z-g)yAq3GgBl*}lv=2VBKfVLyElzD>!jqW8J5u4Z=?McMDmO!@sCee}0z3Xf0R0L9; zRdMhnSJ&6f@&|${P)Yom7()!zqWR%O&~>r~inyi@&sT*exXxBn1!yG0q55>EJHm^D zRYje>9np9!5^9fi#1cKh&_tEP>D8f$DA6A6oR^pnxXAo)Jd}us5maX|lXMU;fT>VW zRqCK#z)ww`s+k>K3!~Yi!#zE*d7YW?p-EI4>A|zB!4HT%K@(k2=M$mClJ1nwG-zz} zcNwvvDFANpW8|YRCz^I0rlyf>Lfxvv4f)s>I@e7 zSs0qiBvnYEY7~X8AsDKQwuO7!6Uq%p(G`K3{B%c`6RK`KsJAB(>IirCg3n7r3!{-l zR~(2?g3(sv2NVN#An*@&RYe$vh0(b3DF~@#+j>*wuPUM_gu$M!fz;bzG+Lv44olJ# zHXCc863Ya|`l^u7L?}jJG&u&4DQ*q3JQS7p(1<2sLzjSJYV|$Qb_h)fI<|-@e{4kf zCv-c#5$pu}ty@*?5=BQ!ls3jPG(7>`?Yt25`N5g^XRM>U9SRm2gK6L@7beJ|$xFHu zarCKATns1G*#6LMXBTP;$LB?%-zK-ATMEyE3e`7fYScJT6{?*djVyqg-_{dN^w|av zuc6t{w)#Y#u9|L@KUA|gf!$@_0!d>+RUjvF|1=%-g&L2lF{&i6)6&%@snSssR+Y$A zm0$$mUd_6cvu*59l6+7#-r3>iQ;+I3^rX;;@@Up87llx&srKrfv{DE!W%Hk{tm=F* z)pZ)9^6Q1pa$mh@i^Y4Wr>O1f=#EEw;Ia}X%CUGbsk%{>hqNs1n_(9_*>uGoD!c2yyEY*FQEnv5zuWs^S4htIREV;U+FIMtli!V3wVXCMIf@b1XnsYv} zAA#r#v*xlW#~7mRKr!2fU+7w+nl8{n@r5*(jdsE%V$PYEt=tC*sD5yn7M&M|g#wx! zhdUF&&=l-CQ|gBn#S-(?`~v>JD7>U66#X4skqoR~wR$QaYR_wEuz7Lpv6IQMNU7C_ zySt;*yXR<(>xaX`JX^Ml2n8^VwK>mblb0p(6JJLZ^x{bV3{w>HYnG!B>+FF`{M;KM z*Rs$gzgmH-&bbkQMZV^*%GDT671$UJqPgbRUIX)7IH#hv)^PPFci%#jaCmmCl;r50 zvIkS_Ja|`)OW{;Sq8;6I#ht5jxQq^S4eLd>`mTHoo@)6j zG$Yo!1lBEKaX#k;q81&=pjd3nDc?{&6wlePmYJK2W@#nf+nI=UM4ii`W=h)MAn-c9 zKb390=vo!yFpX0f9XWkI)S3>B0~0Ksv1Lcr zv>8|&bDyLdrX5T21jfdFpT6A8l2kr=} z9H|?nvz$!b>SL|JcDzf;UXmxOf32$VIH$(1P-AB#9&HP@VIBn^omIVFT{H~@8;H}J z623Th8*BuFc*6u6N5Fl0GM?thd7pKLE)&A<3Wc-t3b@Xqu&RFYl%wuJ({!t1!-Q5b zOPzt9f=4s}&ui~$QEOCCBu=ZfSP5oP>+0wZ0*7*46q+1^yT{>nY*yyB5_nDbW?%|Q zfh&SLSM6}ODjsWF!uQOuQR0uOAi%+Sw<7qdpL5h+XT1m{pR`pwG{Nhfvk20jUlePl zp^`wh3#h5CqiIgc(>gPLP*QVdSWihZ0`&w7({xV(N6_IuAoLG{fOogRnk^+R3+mNp zm8xrV?34t{R7YqbG%pcnu`a_RzQYN=#K(z8vI7LhGH&e(CA!+X7C~cnsp-h1$EfgB ztMb+sd*PZmlf{%64~w3KGZ`wnDm(BnCBjvF0PYOKo!>>?Ii?~`i!l}<&s-xy6I{`g z{c76NMX=Q>q?#)%4T@6LEeH$Np7K*|<4WUKD@pZ>qmf>iR(Ewq%uQ>!2VXrC6JS4A zcwR6NoLXMpfDI-Ui^p=_WOn^J#q~zo|me*<11GMtJbOynidW>%MdXsR7|IBHtm8iiORmSc+@KB=Z zixVH9exbkGdQX*yl4*i)4dKo))l7_*zHtE)O*aZ{VBe#!#4PXd~-3&F$CysoZRk8`x4 zwhCutJsxLiw1#~WkysCIV@nPqOcQ8#Da?0a@w9JetAPu1-2fBGn#nkuoJ@U#I@J*Fsb09aEKochruR_0 z91PAjSx~T4ph$Nwt$*S2YmDlrPbbX}_kh^EbeO+sLcj;D#h{LppGM61d(8rd{Wg!?4+H zv2bDbKBXO$3LDl#37An~O$tpaF7EfViN;Tr9j9om!#gk0+CqprL*gY`UkFiWp?Qfm z3n!Y5*Uau`+N~ zwm9Q*<+QQw;f|Kpa41kx87XP4h`>LUW#N{V%F;+hTco0;Em~GlTG3Lm7e10xQ_Z{I z)ip*@XqwqBA8LcUGqd_&tquyUe>H$A6V7z7jle}#Xj0#Fu@FqY%gfu!D%!@Bl(rRD zww4F{rDH0~i`&XeTiYtjird=!GT*ptt2)+{Suy-DS&58aV7zNk#se>fp@e ze9C8&iu~2QFI7AkiP7P0&QW;u3G6t5&POKQ_l02n8Oome0-g!r4GXMY{**fv!3mxd zOsgu=3&5@L1Q1+yiMxD`0T(Q)+vLu>fNIo&3!ji;)6+OJJ7d4=jfUXa9CfAYwg`@8 zR)s51XBV9%2d}T}gF%IH1#62l3cM@bJWrH&RocP@4arLt?m(vAjGtA|D9Y|UB|i(0 zl`RehI+c1Cd{&mS^vdlw)qnwUAWgC0ea2qX4m=uFRxxlk0a^!wmIi%kJ`Lk-ZLfM2 zhu#2pLgZJC7OOpKF(Na?yul6s(X;7l z8on;@m5$DG4qc;i>P~^ zXd=bq9dM@)8+Ka^9#PhwWu-#P`T9n-x>CrFN7K_#b5Gl1BbX@^X1t;eHC5%>(X4rV zDFSzt`8WP2xGzl;`)yF&Xy`Xxt;qF1KfD{L-}U zO3+smSo@%xDu-^$909Y4TP$^L(9E5Z7zfW0H%D& zM22fZ9Y~xda7PLI0bmzRe2Kb#($yyG9!stI@}0Xtj<%@Z^Y@THeeXP_EB&@YPSo^| zu3xeM6GrEbFnG}nys9V>tb=CBk%j3;CsxIr4&r7d zoK?W2Rdok2o1q(|gV8ZHQ5=+>(T4QF=oxK#fW5JK^9{G~(uSJ8bZg!fP3_1Sq|Q+* z7^J>YZ5yP%sYW_jog2)N8_Z3$882rBX`#XeE-iqC#H0<^pnY8;&T*AKFXrqTjVp31(T#p5WBY0>WdRmFa~ z!&L)boSBzKffH}MjMTfq3t$wc+kKf<(6t9muE8(o0th_el5Vw^FT~;HGyPKIWmC2C zRhOF)y7sKk-3D%rr8fljBS)#PVY}fSvv3ic;(}D4^|-ohh0U-X@Cuu5c&RS;$r{at z1`mQ^-V^h?dV9j1tvzr@k6Vm45cVzfcEdXyRjYDgsEPD%s}8UI@b9iGK1cZpN!{mM4a^;6jCF)ds}*zbVkflU*JZ)6xKjCg1wjqP+%7@FD{>w6Zk>f>}0D==6HFN6-euVH9} znWK7^!0ruffxUpO;mX$1;>u`gxXoV?DTjZ`{n2Q|Kc=NT;M^O?TjR3Y8<&M$e0Op| zcE@FYBJ&-TI@&+)1%_pH*cpg#m3Ge~X!t`Z z{i>8?sVYD8w#nsS3O#xEIAk~TMmJ(7n}l~7>Q|uC^f~?BaSxw#-LSW^Qn+pFM6wUm zRxpvDAb^_@k@-dbAk1T`^y?Cm$D&m8hhS+#HN1MmX`$a+E1b4U$(sV*n1>N*z_f7m z@|GLW*6q%(T2;vI-a6GKtgm!H8AJX7i6gV>zVpV!r7vYCg4S37^i%}lu&bS+6`;lH?QbvdKYW?@-v8qfqYYPoj+ z;L?ZNvDEMU^T76R*9xr)(HQFKfcuphG)we)i-+5#+{a&Mzu9kVX86sEH9Pv|$DG&T zJGgH#xeD5@ABz4Yi^an%%G}W(*f%4&^atr}Wv{+@kh%(Y z=U0@ddJcKH?wxmKJ}+05j-I}sPk1=|3Jl0f0=ZmJDcALtR{L*VUupe(&Tu_#=3!`Z zO)7KPBb;s?n#zAMvbmhmXS1-J)fAPzNcR>ftWB_p>YP~O#&UPN>Gw3yEb2I$M!&7e zNhN2n-`0{Ql{uS}lS*Z7g_%^cyUtzdxvHfoGvNy`rn(U2owU*Qyoe4($PKWqvppIwfusr3`49ROwJRZ1-vDXw zN|)YvECC+{r|(rqy1Ev`qL8j`>%(W{`}?wlBEPlL|JMw_*qt9kIMC+F5{$9=u>?~I zr=c8YS58v}x}~hvAqGGQwoN!y`nog4$%TDe1G$w7q~T>gi(;LvMWKZ~bVs-?)Dvp= z6?NmM^^y}AYJeK%NMHzjmMW^-yBhN7_7HC2f*n(AJp(AcDO?j!Z>CRg&4Dup;l6}) z&iNK;V=g^P8;f*x(}!5*hvOl5uWt;#^~$@(rqgCl$akIDxu$LRjijZ_H)}oK|6#?{ z2mZ8DpEz0wzWsAhZ%-oS3kB$E8{QVJc75SL^n{Vt>@Qu*i|^?s7V2tigEuI_e!uKr4G5Kn-^ZZU^J2?5HXB|7 z-Y19DCk2f&{pD*1@L_};_To~1;%MBe!^+ENu^m?P>8z{s%RT?vWIo-L{YO4Zi7gr@ z3-U#8T`#K&V_iFM`qLb%F7HoN?|^}gijJ^2oyXIg;Xd|MUq{9o@)^s zFH{v0zUROG2=DV>w8Hy>nK#uYzaOzrZoO4`9=WW2TX_nb1FS!VQn39`p%l~<7;90X z6jUCH3#g#-P)LOWH3rd=^IZa>7s`Jn6}}hDe4!eE@IL>wU&KED6;pU$F!Q?7$LKjt z=R?JPdU(~WjPT_Pd@b6!o>w62ruQ$9ebwhcy)Te`Q}GLA-}L(hvac%-d>076Dwy-K z1-xa{S!?ob)`UCXhE1IFZCH6+QdE_1!^-0V7}iyS-UtET-f%38=D09|Z!Xu@-vzR6 zoSzNfiq5S%C$EX-%b<3#${+9TOvF0)L*9;CDY!`4-mRy1D)ayQ6{9rXiK;i?o;<9R2inQmj+}^rM@}u2%~w8 ztQoTDGd;M>80qd_On3JnE@@!&xrdo_b8vFFyC^+4)?G~a3^0?2fQPW7;a8rbw%*&q`HCO>-XTz3wUl}G!n#5Kj`+-H#QaJ4ctCmiq8Hz4586)pz_l2W{ps?tXN5`kKQ&hFIFk=qxm5N6wO`)2O3%?)fxQT@K z?XqNO5$f2&c#~QXvv|Bw_Kp2CVM{p|N0LXaTx{8Dm5UV}UdOXy{WnLCz33hxj~2C) z*lUZ@*YE4onyCTWgx>xn%y_+($5C7R#4$oV4!MOKU5Xw-yg}L&Jl2rtYVTSUjVITB zd~Y63J#j0Do`UkMvTr1_h|xEi(t@>#zR}DzaNlT34c5^5MiXI#PsH~qPj1o`ohv; zFeG~PEsza*Td?V0(xZ$q`7IR@CN+3TC=QUj4iCQxHyWSYQb`ZX! z6igDJnm!X>96F`Jq2JZn8pHki@a02C>(AAUcBcbAG9R87)xMY#z{z;7Z)!G$R7ee- z5qw=_6kiq@n!cnb5$(Xr0v~Lnpw{M=6mT2(HRIqw!co zb>K-OGY*eA4NLHBn5hE;sHiiEi=XN9XTRK`htEE7w2;YLqn_fVnr-AioC6@Kvx%|E zAm<&>>`6uno(b#d=!z%8?RfirR(f4Kk%aexrxNo!r zWKGfeher%Md%C;eL$W=wj@DpX%e-huM=%Hv<25wE`s8HzU-e|?Sd%@$EfJ5l^+_QE zUah8;7FAiFGBDIaT3;ASqpZ&wePQT8!QxU^M|U_L4Z)YB+M@BEp!40mR2=>WIN-@? z+nx1=lUm66qGA+k$U-O$C$*6Ct;{G?pM_8wjumpge%cwfg1z$CQ3U@5ZQH$675x|F zle?<_iW9cm`>)7**S%V254TTgM>r9WEe>^chZ69q7O3#Gp-IKX{oZM%7LlFJ-&yzr zq@Akn4Ul~Lx;H>hq^f%ZB=24KoE@cCzgJ(tx;H@Dsp{SU$)~S-1LQ=ix;H@b-gVD8 z8}O=o4F#-w1Eig*?hTN9`noqjPNb@P10-+i-n6mp@WIvAa41kx87XP4h`>LUW#N{V z%F;+hTco0;Em~GlTG3MB{1}~lPQZT9t{*mxBSoQUX196>zD*UJ)d$ZC@+rG4a^t+y z56d+xY7WeHiWN8yW^cy|oCj47@UYnmoCj47D|Q}KIl$vND||Vu*m*F6gBhjD%iGE- z+QyWWwiQ>lmIwT$@Cnu8w(`=}w#u^Nwl=>UrSL&@=NeCrW{f+jd-^Q=gKO7<3|HAo zJ1=@uj#$|m>+cP1v(%3dtippnp-E=A&sG}pkf!mYv8exUZiz=7@jl8IGDgPfTm;a5tn8C%2cjaZ#@<3Tjv^ZQ^RuT#O z%PPtvV=Bg!`^(!(%PUI7wVWF7;1N(q-=4D}<5K$ZE(<^LnhqZGEWnVLy|^PX|10^) zcI?YONQ0wikj53dmD^89agHd%m?wCwO zgGK(D>fp>!U9>IS+nxyNmpSS08PFnHk-s{q_ZH(roAPY~MWH+xbGTM#?$Igdoa(c^ zDdD=4i~;(i{!x8lY-brTR`;&R^uz z3!r(ns-5kBp7Cp8d8n_-Og<{CvXs9XtL)`oWtE+L)mddNAC*>F(iI^;^$Tr!bg{WM z25VNkn?t=7FRbc!`&2X0ywqEAD3A4)?B$`}lAXNOTe6mydP|mcMaWaV1)GJax5lh? zxBvB4JkQhbYD-q~QESOi{wgh*%e~H$nS51QGM0}TONMj}$WMg@lLe@+#;SHU$M0>L z`9aQhY?aN~RC1-w*`ThrIUAA{H)nIU>gH@p*9TR3)mMXaH0n`(^JrH)+}7UPGvD00 z;FkkrT6U?1w|G_0g8MO!SNi2{$X4OpZ6(*?+%4M5lDjP({DOXB*Yqs~RtP zEwG&KwF)dF`?La!*-BhsAzj0r{b4~RQ}&hvpsF)lASvexcOfMT-jFFH1JsI1tH4c| zGzO&ol2XANFRAFtJOC{h^ai@w8cChas-kFTYkT?$55Bat+B|HYX^w=iG=<5L-}!pR z0;@P%Zb7r-ZoCDRbJqBR3fbCFRi5IOBP_Ae9oK$V26(>@+{z41?+ACsx`SXWseTNw zuFbHyMV#`x@)>SH>#%s;|JApj^Jm*Q4)0xrm!`&-sCW76ojsWvr+M;3HPWQ_Gv&{u zaWjykO~(}$e10XjETb()SCJ2Mjux5^b0%#59xB|vUzmE9Yd(xQ0+$!(TwQ5itUAgg zFZN7<1o|CAQG;``;wXaL+;PP|H*=1l=Vr)6TfRrVv$rFJi%E4b2jLD5UTqbMC8F_g z0-lvmF=DBX+4+C{_VwHhX8q3g++?V$)`JVa9H-jF2J8IO57IIe|UePk(Xf-RE z>$;t$e#d@pHMQ_nUtD*s=-JT4VL#xLhw%6B14{!XY&))j&vbInGw>eRu4CY}b!GGX z%+tUWzq|4BuD=P<1T5RYq*9xdm&` z`Zt&Zh#QN{jikXCLFM3}4k2yFQI9NC4ayLT{X&D$!zqKj5|V$q4#GFT@)kqeYZirkQ^v#F7 zTtS)-`8w#E4|zGrn-9E(XZGN4LE1|l%+UC}troyd(tNEJ@GFh%+TUPz)AmV&-P7cL zX|Vfi`=-I}scBM;gR@Ch<6FWUO=^(4>1a}e+*4PR8sz>un$#fo)HJE7u>UA8_Z4#( zq}+qj;POhnXbWEoGr0Ux_nroyUn*XKK^m{Y=a*@W)1uw~^~RVqmJK48Y%?50PT4j% zh@7$wa1c4AB89DQ5ILor9(-UtFQee(de85W=ijz*ytE&ChxZ0AaZ3$+qJy3dQ$VDhRcP+5Y$fWwVEGES5)fHXZ&AowfZSOqh=9%lyezo4JJvZ*O#pOY`MD-u|55MH z?169Q0xreV0kEB(-zbVt5Cq{#DDgUnLd)pvw1wN@nUHWtw7~f(xQ@y*MGIzJ_v5C5WMun>Z^+HzX*6jXS<=GA8 zKW|30t5pGaGsiiq^9?^~G1^e5u`|XiMIxJxU~IPNGm3+v$LKrJQRVwA>Z;{^wxHiF z(RaV)8zcJ8cQmO!d##H}z1@Lpd>^fdmXo_7?GoJaPBZB+J7M!Mw;2E>N2b!vJo22Y zf=fBFm1Y8x=Uf$9NoOiFBi6daV;Txqz}2%IPcdi2X@h#PKy-xZuKZI2jY;#_n-=)? zX=}K$wY0c0S{iQiS47I;pK^aR8u5>5DGxZmh05QHOYy16)m+6q-V|1;T9rIY#VGAmT+{Lf^^G3YdQ z_C(`}Q2KQ)*m6OOWBq(@lNvf5yEfn@P6MlJFLMT5I{7MTz#Y&RSpzOCuh9lv-d^Gj zxU_Cx^8J4UTIW43F!hr9f58tO5GaYs;RWz*F6eLc-Kja%@Yb_B+?Rs?!t%zCR z!e}HY-~QKECVW=aKYf1 zU}$cRo!HdSIB`M{AXR0A@r7nI z7Q^$b9umG#?TqSF=fL}Ees5l|5}9cMRaJfuLAOGc~fwuP0iI4YwF-4 z39p7Jxgs<^9Em2>`|oTqg&LtiIupTIYfuTOlJPgFy3vLlamGWRUs zlBzQ0K~E||1uQ3bM&q%FG6r=p5uMi+U*d37Rr!j#+NPQ*lbfal=P3dQuaVy_k%?c~ z)HI>Cxv8ls2rrgM#G~O3GC(CdAvm?VuBNHBVX`?{Ri;4Dtf>7^lWGgc+I!{zJGSfArWaG}4%IiGH zxRV_cQo*CLN(oN@Xn3T1iBOfFR6k)#6Vy(;NdQ$o(%p;QYbe$e_Ekl?V9zBr?2@cs zJ{wRCkZNahaLUw%>hU$z9vpDTWabs*9s1DGt`xn6i;$0tBrpG`>9ycdTI$!}&}{M^`*S9t3BF z(zZh-r>5>HkpZ|#5iG^mkJdx)6LJoj1yh>=maE%i^EyM+5l*3Rb;4~>>i#hLfvJ-w zKu6jPC0kTG(KOzv!G^jCQzuq8P7PL7bi{f(!imWIs0efDXSlcwIcJjnYwcvtDgfEg#96zP8xv3g2t8jk-Rw}4f z?Om|dz*j|sLmu7%j8TPnUX3?u?AbIhIO!#>%CH zl*%5?sL5m&@=CNCUS~_aXLdY4b)Ccx7S5HpPOt+h(yb2}0ivs?5lNZGIl23s_Q!pO z9ZGc~P{HLKs)c2p&gi00YqUKIUqY6WaE;}0TB~}V)RFKas&kHLqz09`J=bMl>N!_d zMK&|}oc1+2o!#_KwjO$)hfSGuLeI*zXIMx{r?h@jGRu3WT6NFUC_)q7q=~pz@;b+I zsZUwW)RO?k#@ri7HbCy_?ceToHBov?bT5T)5%>ClyHTA(h8v>+Xw zbk|l2WBty;U}?&^o4sz;y<7VEpb{@hZ%^kUttnSFvOq)oPjewRjV{65*Lm!YH#gah zbSP-yc^LO~T~+-;9t`T5l^5?gSb4N%Yib^is-h5cAy-_*SrH_0#+hS}6G`qTAwnTo zk!TN=7W<3C{<5}WzdC%OP){_{8;>QHgd$yC3t~}7S=`YPJ6>BYDEd~NW9M=E=tfxUwf zcc;Ro@(=U3+Fc5x|L>LGyTINWfP46Eg<1>#@$wJtjk!l#QvZ6{>jryc0k`~Kg-KPu z+ra7J_hq*CGe81>+x32hO;x@a*Fzbjt$O*l1^9O|;I4Tfy*+GR-!Uegy|>}yPk?*< zL4{fk{_%?6ZjipoLq1GPBMM>!KdJ0(@QBY>4L^9KvNsg$ zO#;mH#}qDA{o5K&zXh0I8JEi5j&Ne=|M`6W$CbTQZ< zecI=H=Ou+Z4F2)T?;T+8*;jnNqhC*N&$oupcP?PgeP7|0rm%i{ z&>(ID%nPeviTQks;m6A!w(BF-P#DDF;pHEWAEy9r=`e*&)!xwF`-~xDY3%(QaNn<) z-X7M=hP5*LhmcwDzm~Pr+e80uVN53f9su0`t&`c_#_MMG597Bj;C3FK-X7-nV8*2L z5A%Bh;FhkJ-rmJ<`hCWvv)2yw>c|a-_&(oQ+1m}kUiAg--My*8WU@DFvmw4eZ<*d6 z&JRx8ILk1=e{GcIWFW59i1 zoY`JOiNa*EHyLnqOEcU16Js*jy9IE6F3W6h{qpSm+XQesRAjcdh%uS`I|Oj2RHnBF z!=>*H#-y{?0A}6;+%9{hw}<8ad&Z=*Hx|qs4Y&vQ%xrI61Un9|`iEKA1aQsc(%ZxO zcPV4ENiTaJ!^x`vcgkMMUaEGE^Qk8olg=K_6aNmludCAA!+yvgRG4)3a2`<#xRKTA z?O}fRXG}VKnBOqq8f!A!JC-q-?41O-cx`%nJA;uM0P}aoA=;}vZh{l<0_LoGWe>-@ zRQ=UfZAu}@D8^@S*_HZ6i4Y=sc%=Rv4 zOeTBF0Qcmq%=T8@M`6<0gQ3W`2H?ic&TMZ9W765%5zHJ4xMg!P+k1yG>FiFi9Uq`-2Ty6egWLKR`AC+=NJGdq*=Sojt5C#{=$ne0sh+=6-O?P2*|!Z0QNTf-4Nd-_`w7J?=}8@4JT#-rm;uiQq4E;I`n10e8{*|{{0njKLX~t zgtC`v{7nFEST9_^!4Do-K3@Jk3PKkF=D@`Yw?5#!<_EZLvEdShfnVwHvNseUBLTPL z!3vwozn7pxXk|<~d-LJKqWth7zDtf%_F4h#<=>W&e%wi!{loF?O2EBxa(e%M0(%>r zlG)x!$j3UT4)N`Froxt|@DE_ViDzfFw+cX}0B+h+g-sQ|8nAa7U|wKcs`|1NPP`A8 z#pfz}spgYkLI-omc`y!MqHwTW=~-XCUmMyRU>;nqa2JUD+I0r3FOS?Z#P>Y>;DP_A z`jWKHa1{9WDq{$!aY^m+QSfiS+lTnJzW4vxdl&eqin9;+Y&KvBktBc?6(#B_AQuCP z2nob2Y;dFDA~#WyB_taX$<16KcwH4LxIocb#nxA;we?!DYOPgk!$rYM5%F4Y#nvj- z)(f@XE#LqD%$(WXle0-^(eHb|-+N#)bN=U8w3{Ptz^=lJpBU$=Z513i5fG#|g&dAu)V!tVVi5wc39I0A^y14%K*~%hc7%QXId;e0+Ee=?Xf9&@Jx< zzgxP=?@rJ?*ag1{SbgCV4PhV_)-6scaz^b(B0StKc?^DZt~j2fFHR@cVu@`K<-rkGtSU{d=gJ{I-H_XBYg~ZhQcm zLmRu6Up{DxopcWQu{`F3rrt^CQogG}v))PP;K%ZKP}9IGSF-YW40KO*!EZP?>;X;x z^SjRPXwVcn=^W`}e&>Ov)=B4*->X1#os-VNkNJH-(@19VCs}?U1>N>8_%XllfTnL# z*ZIu_&3GrBBYn*8*`PVkN#~N^m7uxSN$23l{NAT&;FT*`ejf(i6J79Qe%}U7R&&?+ z9SNEOC!HgG%Lu%1P&v-{qiL=cIGtcaNs&D!-3`?x`;L`M_ikXfj&6&hH7J8RMjL zN#88cR66Ng@_RXG);j4N{HV`&Ynra|`vB-y!^2_$>I!z-H<4>~u-U7ONy5Pt5>KV{{;-qs)U*BaZ)=~KJ>MXzGK~v_W zbK!R$XqGtX9Qn;bT-So;0Zqr)ljV0iXkKvgbD5Xz1);o{xWnrsst!zn$eH3p6>J zE}2}I-+WC2uUyH>BLKSTn%3mQrF`o_v(`!HqAz!Y<{>AYgCEoP5@_CW(z)T#d7xRIz|TD&>p`5Tkb z_Yf8tbCc@F`zlyN#~NjHqb10(mB${{N4hZ`VVM z*ZIu>&2UYZEWhM;8fdDVbS~*z4w`jNI!F4L-+Mswn3K+h-|L{+0(gi=}_u6jq+W@+|yWq$CKGjWr zyFmA5H~4+sO@65>Fdo8>mnjcNzuynfMrs-%#Gm3U^!pXni<`&f+mQ7TUO4(4bQTuB z=H%p@IN4uXRyuh~>C7qqS(AbV$AqTLoD`ZVNddZ~H$)zPTW=hoYjKFQulvxh$)5-!< z{ZmS2I^a2zrcIfC4417~ALGYmPS;u?ostU6rV7;cJ3djc(Qsm14czHF65e2U}C#y$qt_uz(t<;@{~ zpe)q9w7e19m$YER0H~2KyD40Yy&9W^xTR^7)f-rCO-r~5=O0P*YU`hn!59I!;_GCQ z`-3yEp2|8Xy8#Vgj)eyW1)(_;OJ~ir1_KSUS&6iba0yh zd&iW6MT>Q8V%(vrt#z_rUO_=M%1I@~8UmujxQ@-vt_u;i-fV8Rj_Z^XJGXIfwPn`v zosym3T7z91rC2 z)zB0ct#D`A(o~K*=tGooY89syN`abNmeqzu`B>XJszqhZqMDXwk#=VUqjE)I+)3)n z+d_&@mt^6##)f)U2Z|~=H=C?;el_(Wo3Jx}ab)1fYUUnsQ+Z`gn<+zG4Mm{P6J6Ef z6x6OjNePvb+B%Vx%vQo-T6vNZ#w!ptmbplYtLL#)s+p8LcD0ecVQbOTw=|E(j{Z$W1qBd-BfYwACneYI7}Lu%9HcbI zom{sHEa|xPc1DYu&!q^kz&?)}8Rz9A@>7#>-DutX5#WG6sbC+{sD4m2n9Yj~^P9su5 zkjF8drmnRNJJ!fsjTHvpSJrHMUSqP-!+t)Qe2lv|$7& z^g2zp61i(dA~Di(a;IrAS+K*!1V%MTuB;sb>eR7#7|K0$NkSo+RG;J`qqdD2kPV?fL#-+_x=S@8KpG}Y? z$HHW2_x5oNTQN#s*jiA7b;fg!TY z@d#P!)+C-KYSL0}yDmd<(&*OSNz>Q>5y?Q5M5Y;zrHzcRnXHW9NK&knC7w@hxJu2m z_&n0_c~i}zYBg^lf#p%Ny}0}%3E?J8Rjea*hOCqhG?n3s*yGd=j~g&EF`?5sJgyG6 z$)H^lpr^TU323gU;bgO_ra}hoLY32jNmWf7rW@UC`eW+kX!^UT2?=FvO4!QnkQ!u3 zMln(|lk1GBE(wgGJhnbs#T{6)k1nrmtS&e4r$nqs2QrwNaE^pbQ7EzYjs;s@d0uOC zOI1x6#sd>ypAL+usEUYVHIaVD{7PmO6>H{+RHcr(wy2?j8{SK6KoWa^7+2V8=X%k3 zQn*xaZlM^@#HGq&*IiO{-C^X7c61oDjf{3rSxvok+AK_WEn)Yd89KTX_XeW_J<6T6 zgz>xN8{5U3$1Zus0SV@@OOsRH*eJD!%w1X=96M28AzDC!;*LcZn%Yo-y$BHrwSYts zSE8nHV=Z=$mm(&W7FB$Uw6LZg)=)|8NuaPg@>v*OWFEWd2zEX+k6i>6uCMH3;2pv# zCoN<06FL-o%yf!Mj-xM(h0sN754`(!iyS384r1#@7afZzHq5dIOuAScNi%IoET~T7 zSVT#6?dO`tq~VZQd+;&9HKbH_2v0|2ORi@(h{4;Q1-MrVn|Fc%#SWrX&LM#KAjbG% zkL8c=G*OJj4uYiF;~^_@g*1c7B8oGg$W(|zwj!@}Hwpu8*P{-vXyL0XdGB$Zv-M1s_CXRTCnY`z#&ZRqHkUDz=>4|v1b`x1Fu6Q}ymB17E z!@s*73bEO`<1T6TyE!I8tFEf;4a%i~-uSl?oZ9NX7t8W?aY?OxQ zq88xvgx0WFAsMz;ambsRS#~ZW`y{>pbrkiSO4tTDdkitN4jB&N4IK9G4WWjIaBfVqwb+i^ zC3o6L)lrDH^~xzn8&4+6nodM{;35;5o@?jJO{V`Z4V%qyBSnAE)y+y&eQ5 zp&w=H2rDO5UytxQ(s!s|qsExPC5eIMg<-fdgz0oc{rFL;(30#1FLeNYX>GXP)N!pq z-AKb&lY}yz1RZuilVDzPhE$6&xjPxc+(VB2avT&EpQ9<=NmX(_&SV9;ldPF>C!?7| z7vd9jdUq0)jEgf#^_rR1%ML9G4Qn0iv$2OcE3|o#tVY-B_>wh_2P{6R)OM3*PbJD4 zDv-mlwm(%g)Yfud<9vVQpyEML=7pa<|4O8(!+X3IPHpT6?KU)#Ob?Sx4`I2rHMR)< zQ>jaH!>KsCd3(Z_m<}ETX>z1XB9ur{hu5w+pHlUy zY~5%XMw^zwVb}?ka%}}p&y<&tx#)st*DuA}-_X+PFjh0vFTwjEm?J8kw(Z1lqedqW z$eH+U*zU(J>PZ-);I&hzI$U0f_v3tL%E$KnQ;yVt8rjdgs!=o)xR=NvgX4O?0BVe3dLCMK!gMqi^gNeJ$mf9jVnShgfOZvxj1j>`6Rnjrc6q z(_;XCi8qj;=1^$HtXZWKQQarimNz#C!p#*;HH}alsB&8NYN#|92I0UVJmhK9OUf}2 zEQbfv@A`%ooWGz?UmLG`{~Q^s=A2N3bs%ZkX^7hwVw^F~K1?YwY@E0Z%Zx%ed2|fu zF2}|TIwHhSThlBN#F(?X+nkkKWSxp|KiZE3hC<9oEf0|mhs1P@#am2h78r#Op^(^! z;rLof`cNdP^Q4c(F#u`VTw;f+Iwn8VBKODQzKL3Z`w@74iOyQwGOB_C#DWC^%?+(h z6=A=<=nB)0wCttbCNC)9>9<(Rg~MuLM@`Fa-H)KmDS{OtShHKO4okl<%@~D+RV@q{ zOYvlcum8`(7Zm7pFz1WA$)}Y~ZY&8C5p>ai6tti~SV0UYfJdbp4=x|W8jt!#vG`5~ z?YZ5fMfU9Qh=j^;wp+mRMT?rki?E*us%I=7mgvlGlN(8%h+xy*f+>~5F(cs`wEnSp zn4eSkBY=r%FaieL0?IpsxSTLah20=DoY``m+-1v=XB`TQ2)n!t_?RFcNUZmR}>g|x$nDX}SLUMssw z>!@*(r_;Nl8$C+ekP%%Z*3neki7Y`<7Vk%cnuHB=l+k5MbFrQ?Eqe=6s@9QWq!Fqb zHAc$hC43fPqbw6FW^4{u&TK4iLO0b=+lu|SjzHks91mNE$9UkF z(WaU3n@?{B~Ac9Xo@Pi->Wr_)etIAtz(dpK*yW%iT+gaqXLC32l2RTI| zj!Z0oQo%76!^&Hn4=tlV7<|&Qk2Aby)=qLg9M0 zOO+w4{i^1aVECva85?|r@x%V5R_jX<(mMzd9^;-QcJ)QD$){x>i9T7yfU39{FRYGa zD0Ht1^tUwB!7hb!g82p(Ma#j?ER(9KdL66OoJ;M@NiHP3K2)!vUgfn23z*ZgKY{wo zQ=wCqbK$!rfdRx&WE}?jISJ7LRuZQI{_1<0l^UDbnU}HG$2PZwQ})a}R+NzI@FXdc z3cXqvgD)=5WOK3|N-EFGsZUZeV~S)jm17ImC`uCbvD1VCVP$N_?lQvBK9$p&1oR6i zv7n)DYo#o0RqLC;n(8;)W*wSQjz!F^O<31k)znZIs^WV!4o5Gvv+Hgpm$JXjaKYHK zKGf9GQll+zBC(_Fe1{TrTndZr#Zo&8$GS!deC%L5n|@s@;RD@cwPh=f(KcI?^{_M4TwWE%Qt6|?_;5Qs72hr>`eicQ_se43?-Io9H$U94#u zFQ`$a@$sv})HEzK2ewbNQLrj0;zaq!2goU6^KmTV8(?QUk(e2mWlEwp^Xw9HD21%U z6X_jlvx%MS-6Zn|v0g|_+ifPHGbcGjtYVsljPWz^LrBdNk;q*)J(? zs=;^_Uve}pY6cnWUT<4O@d{=q)G7qUE<2gsuHvjy$Rh_}e>q;Ky9(cOsD4E%D_6Km zepLGvzo?bX*DpFYU+7(_?dREoP~U`zaPC8o?a)K1)@Op=dxU<5B^^AHTlJc@CMt;b zEZybV8s;oI8%Ms7v*BReV&@=P>Dh0TrHoSTiY2uT$JoQnm6lzVgolYuO7h=1It$2@ z_4IiWAjgKzo^vrUC1WmbBP2@!^bPcxcK(OxRn(=&-6mJOR^F z5oD-~YR})-kX&0Ju@=s-!K+w^*RDQ>z;2`+3TXyotQE8h$a7^q z7po8r9k<%i*~;3rKxJ(B*mMyLO*|wHADyeDUsEf&l#(e#asix`rI`7DIqsP|ElaO2e zg51i(ledkeJ7;FJh0~=hI};0Q@TDJmQ@m2bYhU?5PDcObi!9EfAF$~{lN>#n7!*yn zB5fl_3G3}@mN3wiQNnclkz*h%4U%cplQc-VYWN$n#lR7xIi6ld+-XLrXX8^f_uh#E z_6?ZCiQcs4jn)l5OKiQ}3Dnf1hSXHDLxXu4%RMRn4^4Mq#=e-kaTw{dqcycPVZZ_# zds_Ba_K4QdB&*Sh16jtr(M}g$NZ=JMvi=yEMwDH_pj$Xrc_#_lW9y6b*v$yqcaI+(m$#pGN5LB}?H7B@|1X}CrI7@D=!^@Pk>=(MLT&8d^+F%>7 z%MO@0AdVkBOT9h9ysU9?k{tB4mD5&oVNQq5|m=Wy!3I0bbO`{ESHj*(DAVtU9F+uyC6v& zHElMFqDX3<84Y69sUA>Mhij8?U|*{mzh~?s?l2ir%%_6ObvXJ#>A5a9X$19X8IXuF z-a6zQC)*tt)RqI>Rd%YJqco&k)&@zhxW!IcS52-Y6s)PS3vJep7FDG{NZv7HU%p6) zEsxS99C7`|bjV!~VWlb{c3R_}-~Vv-h-(^xA8C(0u|DK{6eHSlE#%2q|EugoO0<>Q z*jI^N4_d+F-od7#{DLc{h40|HN53l1VtL!45u)9dC$i`}~P*ZvRB7VvN6JwGBHS;JM zTAC7zRB0|l-yRb{ya|&8%2dK}!{>IX8AY>goK)7`c03Mu|2R_A|{>q zu~Ub3ATe&XP<`yU@g2JCQeZ1*s0D)!4szbGbpz|^xxPY=pBOlrplWQev5&!0E}6nO z%bE>}Z|wHft^x5)m#*CG2z2YE1@26DtQ=6W=7a4)wgIE7;^wGY7j@^wLPAx{Yzgi+ zotsW$=gDSqIMb|)2nl8l$usr#tn#LM_0}I-Nu*^DHZjGz(Z;5Rwq@8o02UWhf20L6 z4N~0N0Yt!&qM59^9JX=E4kArE?6M$OCwNUrys`YyIN-hNblq z`+2r*I2v=)|50dSc6rY>Ush;)hF!xu7&J~pS`l}*`!hu0l&mT$Eu#HM*Q+puESP@peyJO(vytPGl*~o0z3oI;cV_$EpRx+Flty-f9=GYv-?8 zNRBgq+$o(J4AQ!|E-9T|gJfjg-&Ts4A*a5zww7H9%9HPiFHTbO2{r=S+itYG%&vMw zsRhT{eUS7ox*_3&W{v%TeHMoWuyFHz({#I8PZX}oF{fu=nR+z|uX)+{s4iK$ymxF9 zU1DgZvemW{>Si;kqDl4gdUOFAK}iAjwdI5iRpQl1c|`?x^+;$#*=_vPPA2oWG+e$| z=7}6**4rGC)iE_xt=>Scw;MHsv8HE^86%m zC(K`%{D|EJAdh%SsSA$r)^{u*lP>lUbCX0NwNB=cK}X6tE1+im#;r+V!FOmL`()$T~cd9GO#HX1|0 zCb8I_YfGi8?un@tW!8O^?rl053JSIAsKf$cO9?TFC7B=ohs75AjxCb?8dYfiVzS8| zq?JDQx-!2=mMuME*DSlCOp;<-7tmO9sPcx9OXnGl1+$0e>_gu&`X3vPivj5-lhZRN z#WnXf7b)e@M$L2&WjZmLcXUuHj$U`#T9agnxU>v~_6SEc?dsEI*jDW{)zmj)Or6jo z%mweq?1s}_MY?KEwZ6mzk#hoP>#VHT*eap>Vl~mu#S_f7RWD?Z6uHHMvv;-UieedQ z>TcaIij0yz8&RETM`gcgl**N4${HJ$c1TR@k(vW>OzAbEn{DQ4lF1J9FvYGbSt7~o zYpU%MbF{>)_M$@1c2nso3(f$?f$84;3uCMM!3cV^Wm!uRIbI2J8CoNX!pBul32Tx3 z9CRlvOv%r+GC+PlDoZV##^*+7C-yJZExJ(Cu=Oq z)oRHWMatoc&@!R;`V}v~;GL~+O_Eeq34eeDpf{+)cfnjhjnM%&?PE_BAX~k7 zZ6Z25S;H&ED8fufUTQ0AmLMZsp0pUn7@D-C1vOT*Aluf0gtF|Y1z4lYAwwP`>!>gy zHI7L!67K38bs0JA~ z9v+uD$yl%kYucMCs+Yyq5(_Kp5c*=HG?`L_I^!JJ%>nNa5~GHLP_L?nP$kb#<9PTR zRMpPc`JOEH#yON8WIe08sbOg;6G_%m0*RGD$)ByjuO;5NRmY4p)QFE)u~!4*9w8-= zxOy?60**dOUdM7E36^iMA|HX6t4^+Xth+Db*4_eR7TmceVEF4r%#zYRZ0^5vXrb$0Yptjb6i^+(ZfST3a)CL zAPlGxn-lWa>#KRoJc=oyMG9oB)K-?YnVzb$CT%O_Q5j~1;DCn}tF;~Cnt(ARMWtHR5=isyp9*Kgi@{^7HC{`9;DcMm;h49+zZe9a?|?-~`DamQCzzSrl% z`NP*)*3p=LeDdrm?@S$gR*q%y_}wf|^GT1K5Y0d7&wpJ0^|e{Y z@B0Db7yR~6?a+H)`2ERWHhuEeU)SFi!nsQ+q(A4&Z(cgAmoM+-oRyc~67Y=0dF+D! zxh4CUekWXd@1JK!Lcct-;WV6+BlwKe!yf!%@wmJj8+P^{a8MOSHB-Ihv+LvEPN_a= z+tlOl|LY?^xaZ(2ENi{sc_V*X^Y3YGE1y~V`m8_qy#>o`dO{Xip07Xq=tl!j{?q$E zTzmZ^r-l7XQHX-Cyy3`SHC*`8FDBK#)b>gC8y{hRXTfjozh}S&OH=AUT{8HxdyhZw zI($itvk|jA#UEUL?#9~Re0lT-=iHz1cH@gUzg+Nd8eiS^ubZxaC^Yujp?8!A9>w{2 zg10UB`sg1=U--p@ML&JLw$=NNW&K0&h6`rQF8-g~JBA$B@|)+czXbUjkJX%6p0$6R z-f!yl7woyJq2JgMcRu{GW&K?6H`iXV!vFZ;{f68z=;v#uUW7D$D0o)hZ4<`S_pEqj z%Wwboi@SbvH}=&%hg{J&p9Q4=z7)M)ADf=RJPbGrh7u!Z<7}_)ix;cIT!`Z@4g{Y2&Le4=R2M=jaMv z_w2pTuP-Sa?)hxa-lHD7{+E_Dwio%_cXjU9=SRv4GK=@N-1GA75zG3i;7_ELxAi)F z>lyb)Q$tIxA2$OZ4yTj;!+wXXdVJmeZ)^xnTsUg-&3B-#3BGK0(JQatl6U30Tkp7Z z&SAH##yK;BfAw;oNB^|-_30m<{y)F@diYPKSk`gaOE}AO^ZR#CXgp~|_V0fE?0d66 zD{Hr`HG-eG`@GNId-lH3Pwewf`2AM@NXzy6dCE1{J+kY5oR6MC zKBw-U@%TNF_i&u&FBkS3y5?A%%PaWp-&}b1S;zGG*U!SQ4L)yBB7(mY8ocfNbEl_WaO)XAjK2GMCC-l&eAKuRznW6f@09^3 zynOzhJ@=v>`*G-amgoBeTjw8o``{lB|LKO|x4rrPdds?6@Eh*>-m*DEpIcaW$ky7i zPd)slWql=h?V*KtUi0Gi0oVL;R^+)}&wPN7Kyc=7mM3+>v!CsqGPdcC&(1u0Om*HR zmi36>14h5H^xUkB)U(dNad!VzQ)c2^Y{ZRvk^0d&KTGL5BI_R)l;{4r|74t_EBGHb zAJ=o-S68Hre=p<7`+qz31in0#{^x*c zXFlBevv-HAt$rWv3|hM^&)FY-_Pf%zUl?^z!-wmJpBUJT9fJiwbIq*n2OT|X&$jE2 zJEn2bc+?lZl*{rgpLfZ-w+-L)`oR;1jLIB%^c>5&RqzvgKC%1diJyH|QMdJyp68L`$KDWa~7Oi&pKS!N-1pa5>Z-;w3>eCMG zzXWZ`1hn+Tm!aMiY5y7UpQ8Q!;oqeF#b~Qu(f<3v=T-Qbp1*?6YudjX{^QU?ljktd zm%`6*Bax14;J+92bK(CD{0#db=pNGkC*dEAUXS!gfo3@Tq@NCdKK!IV7c|B2x5Hfl zJg8~L!9N=_$n~Jf05(Va8E$Tz|7!ScnOp}y`D{mCu8;G#YkKneC!Y5~-mHp(d9eCo z*jfN&G3%OA318w$k8X~j?SA`Q(sZrS{a@vuWpk?gK0)(!_wv)EZA9GpTG*!{iU=#BaiL~n>*Gq$<0uCngC zkgT$6{P#KL{tyin-$jZOUL$_%jX)cgqGRI=yyxk(7`babzT=cPwh6n|CzI@ftL<%Q zS75LC_9MavvifzVyl%kUuv3xP&8XJw)|Q&u=JF+NqxUBeBWg~rRR2MR{Ek{+lnD-U zNLD7@Pq*T8KqUYR*LNd?a#@a)q=t511a|)^-6!`5s;XI(Tp$OoUSYvivhq7{k+iW| z@Hl}aVmfd!eP`#uQO6tzKcpy z7p~(=!6XSjaN3IuDMQz{9~JsjSqt*5=cx^!yh1{`$;)Yf3Y=*^5S;AhR+VgWjt8!6 zjRo`DFI$c$>;;=i%D4M6HtomAfs1IdFJsYQ}XnE8)c8BLElA$-&J#4UW@fBH7(WO!N{cTyXqEo zM7c1BeT*MW9(X| zRDSNr#>$1(-jDOzgR4BjRsJ;mvVze~TbNKL{zDiHKd?@P=`?oPH?Z}JAyPewU^IZ& z{#|_Ihqtc#xsLVQ;Hef^m~~R)nJWFF@%SRV=qt*`D^l!E+ER@@OZ!;?%EH;LXMMz-LH6%T)fGkAl1Jd32$PA=&%4 zLKU-ZZ>#rlJySALQ*oMf_CTvICGB$M1hc`iI-A;aJgl&^_#%DT)Hr5fnnyBl2q1Fc zH`~)ZCeeKXGjb*nup}_WnwD+t1xV$!FD8w~X6DeR!9gA?LN5%^5xmQ=a^OP0Ef45= zOKAGit$xU@!oH_{>DCy76&P&3!4?_}xz}OSv@hLCs;ECju%|mw)XQN#SC>}QIcnp8 zPS*24E1c_iL{YFN21@x~h3`xOhc9WUsZ?6{hUku~7G}g$%dZshbVxv5O0ihv*>Q@+ zcLo^;cdlz)>sw#w8x?{2CT}6)bdJf@I+vw&PAeCZD5_<-Dpx!B#Vgl6TDk7l%C%7| z*O$nlQZD+^t&b2+VV`JUy443y6xLV!(yfOK_Nc*vSNwel8oJ336}xu_R|P*0Ua@t^7NWiA`M`Xuh=I@I+nQ1LlkzW7m>R#(5;jwd{R)EK3d&ef3i5E@(5 zV);^$?9hjHN4C1d-Uj~cAfm);DqLk&4%Q$fwrdAjzBpas`yg*tZ(q+0Z=c>ndY7;< zO6z%0MvsB%*4f!$WGw_me?-IVV*SBH41*V|egAB}DhK_ba4IpNnvRv(uobBdf+SIF zSWgfZBe9CMpR~QD)LbE!G&0>M3rVj8edfEpjI?9y1YL3{cYK_R#&BwOyxI#{k5XH+ zhX^OVlm`bv7YP}$iE6b2C;_$xPV(IgG zz_KDN0KCkLFpCetErh!nZZX_P;g;z8Qn-)deyYBo377P2W50$#R5hS4-MZ9ZS7=|l z^+SVQZ!o-v)nPw0SbuO({03-Wx^=R_3Jr!TXiY#X+}q;D-SnkfyA9pjhEA&08h@-R zZ9-Mrj9L_5o3cC9CRKI###U9GvG65Um`f4GVn>CU04i1(x-2nuIVw!3uDl*^9I=~} zdaHs@$z(C6USt_a-0sS*MJ{#7)@v2ZVV!6{H8xGi4o$B8HoK#R@Q7rZKT;mMXnyZQ zotkgovubkCkX+T%(TF3B@_IZ`%}3>TqUIbJERgi5Ka{&u$>%{yStfV`9ia^7!$r-s zD&aD#VYnEYSc~8?Z`E)|!>xzQTF&QGa<3>Gs~dgk7&Y72bp}I)&~&Ien(i%w{ncO= z@~ZfyYG1k)hHs+8f#ZtBmu_8a=++xLQRvIQlulwmS=5i#fAiqdVcUai=>2@q(u`7c z5q3YY2XR`@utB>S98$4yf>V#z0w3?t<9tw2W_0BpzA8}yQ8gw+=j4tSr2 z%Pj7MI~4A-aH-G4slVSKbE=W1FWo9P7&Kg-rd!`P*i{C5!(eY2>|=xd%U~Q%t2mC( zKEwgvDWcQVZu-)#TMgaqhHkS`H?qA5;B<#NdV$;5$t9=%8kVUqYe%_`V~ZgE zqU;u>qVZFt*euE`Qy;buNl0VLsz}*ilJqYFMXTev#wJvXlqyA1e#(vt5Knp*a;q*y z3JPf{HRy0SDB#?Q$z*PzH8wMQO63AGdZyArodcLDii0_Hpz005&%P*osvgP?DO1<7 zR4YQ?g>b1_oEIt8qA%TAX0VI355;b<+YJU4)nT7E7}k(!?B51k2PvsI*dVGn;#6i7 zz-gVR%p)CJusD@TqA?t?YK@s((}eTG?d}n&!WGjOPN_9vS-GZjQx?vTw6f^=k(-`$ zKDV~JNmc?jA90GH63HnvHB+mBB#1MgI2}lw`E*s(m+PWlrHlICkhLmm`qHf>h)iJ@ zXkWT@lfiB=*j9sWHyB4HDr}kdrCY&=IpM)|N^ni;8jxDS!lz2NwG%4MJhk1}fgZvm zXSa1U>2c-Y$3=JQ(%sgn*k?diO5-d1`tkVODsE0PiQ8s6O898>ux2(fBixKFNX_WK zuW-)nqN>^@q=)A!61$GEGq5kFn{|wJ&8I=EWsA^@A!sBAE@-5#?eo|LKwr95foBTC z07iW2)(V5IGT1W)LkFShf>BR!TuLx=;#2V!wiogIR7Z8{1B@3tnz_#EGzH6)@R?d` z+?Eq&69d+csnAHYdu2RaGO-r*kwVbVXUO4_i35W^or?Hs^xCQtkOdktiGpIst)(%= zHf~mw%Z%uY`I?J1%T4mAGt#jhrX_!ekvx6rRvp4B?0oG@x7Her!$U>)cY}Rouy}dj z0CbW=-cSa)n6As5@}6B^g)ho+f>8Ogg<%}CQ(G(cLKKc^i8#JU6m@E+lB5wap*bOg zBr;>#+VP8*Sxt)4*7M*JUksN$!xu(o^r87OSgrP@TlX650fPls&GV(5iJ@*zaLvFU z0fyb z$rvC3m@ZV4SX&h9MGo(Js4+kWFvZ}}G=?SYba^V;a+)v7u(_ z*z(wBiyXOk{^1D4?xe_&b&>T=_eZLeA~+TxS)$Ua408Ze48Uci!jvW%sl?Z1mM@nb zd*S-<<24P2I+i)Q+6bdN11|M41=^yzG5XT24FqPgtoLgPp)BXRqRll==Q%vjo$4wwv zyED?#Gy3$7x4Z3!f_JUt_FE1{wSS7v;aD-`uN6IUIK~I|W$SYf0Qnj`Q|t3wxEzG= z9x33`e9`Z7;8MT8f(|JCrZ3&P*xX; zp`z@1q4ZRZ>vRj|HgZ1{jc>;$t`tem5Wan-agMdU8Paj)$TR03Hg|Rt$0*U7#m3=m z;xs5NsdT3lAs^Tm^TtLC|1B0AFT9S5Fu(XVLM&AUaGCc)UAR3=<3(ROEm@Yr*u5%j zxxrQ%>{^4ZH`qr8`@~?ybNsXXX9h3(`vyS4DD}?@=4}nGTJDR$TNXX5I2ipn7+vn$ zEw^W61fu1sf#|aSWvj}3(@XN646F+Ha$%bq5L`9O7mS{Px431#xj~@CfmM~hfk1tM zRTug4%iwD(LubE$e;Ug`Q5^v7@W8lUnQQMy_JXT!^le8pc~1sc-R+}(=DilJ_0tSK!|9-m*)3w%G?<6)q1xfNn4TAwh1ML?_E3=p8V)6-$n2bN9y|f zy?@Bu_>@2TLgvOYD=;fS>195cpZ$Jnv;~7L+)j^fCrOTf-xK@V z{obd86;Eb9);}Ap24rr0$@`RKtMHSH*07BBZ7cjHbM?8Hx&;w_ZXk*h4@6P^{(YND z3b$vj?kkTV8GrQo07!B`F=XH7RcWxtr$WdgHGkntnX9+|dmluAGQuw{fao9w6u5ui z7QgouDLfWZN#UOmA4ri>A0nYDWt0^DA#?R%0Lu_|!M<%&27lpKnX5nf2856S;4HEn zju3wDYku!jqJm&3N)#yUlezjK(3VA&c9lhy$_1lJ1%tS!Vg{p1RfEyhz6kI0p4kJ( z3SPE#0~~+IKX3k)Echm*vci{Ppwb8W7#p?z3AKKlqwzZzRBR-Q;jY4@P+bxGL=cQo zm+;&vp4Pz$?M@lG3EGH;Y-09Y1mF)1gEw(TaJ<+&IrTnXo{#Zf<+&a2y!_c^9*=b| zaDI}u5F@^7e3_gM=1&WAzwiy>>2wKqu$Q*eScBO=Wv2CrN6q6r)&S%Xabu*5$q9TO z$l(HRf-AfK3#(kfr;^G#RFYUNQS<(Ei8k#h2Ae3fjPevdvW^m3Mth`0!FJuBjRBs% zCQ4W}3ZEoli}3_+0eDGR4r|%TTGlNRme#G{pM3 zV88PWvzB=j?h$;a;MFPbT6akrs&UVg!d-vuoM1s3H05GzG9(zEh8392 zTr4~|IO)h^k?>fIe0(B2{^qH*CO}#I*^7}6+S4uTZjaZhwN_d^MINzISQD3@I^je4 z9RpraJjEY$T^^l&T{R(?SY>cB{DD;$6p46Cw;+~aDST@pq zJ;d#7QTxWtSlS_|j0ur7_QYcYc(`1C)>*aOg&{WKcP*kzw+gZ&AfJX* zb2!=#KRbeH9@7zIXKx3{27H>wkCAkCHburg=`6_}a|9ZhZQKsL_CS17Zfid+^A<*uUjK^00GSuWRr=2%ml4!*JQy z{|+u~!B4Yp>a9OI)!7YJH`h{@G=Tf-je;M2t;ob$8@ot05bi4tV>G}dL)64$)b+~Ll z{)ITyY>d8iYk|QkwJ+VGxD*`=MbSkJ#ui9n*BWfS!5%W$qXuJfD}K)yjPh03Zi9J| zB845KeQJ%I!uV2;zI5wELzibT&H@$PB!jV&R2cgT6_#y?!q_z`>_UTGYOtumeqgYl z80=<)Z8R9`tm5~)!Cp4ly9RsTV0`Va(sGdYrCZ-K*bxRh(O}dF6$dv7QDGMttkGaC z23u~hl?J=VVD}m9F@tS3*h>a`)nM-#>;r=x0sT^G8KiwEBZK7`jH66NH{DQCL9nH})8@yS5W#G_mgUgld6E$uH)s>R|!b3;B&4RhfcvZ~fM7BmAf ze;XfIl`n>d+`za~GS~hSwFfi9i%Rl#1u#jj#x%TiRdW_5o^4ZBRScN2YH?0U^qgBc z`|BS7y8$vy&*Dt~0tAKOU>~g(!&mIeTw4Z!d_a~#%feu}{|9I7Y*=pi>>F?w^8pf2 z=1a>QR1EsJ$?I7tpqKN`CpeY92ov<&wFZva(`FXOXs0>hS$kM4=4`t0~7tr>q5vvilDPV;UNtAgLVIdfyG&(B+A z0?FSRtoSJCZShTC@nqYa!qvW&2ypQf7*yJVfx<_9t9e_5yMn@td;>4eN6Z7fo1!o` zESG1>ZnBNc2YSWH@orXPm>XQ5`Cw`qOb{sK%*XodL)6bg*D$BobD{T?VbXE9H=IS;HgaQqKAOlu|EU+A3;@$4w_qY^vpzt3sY=8<( zh9D}@kO1R*gZ2ifpMT#rQKLZNm&&j}1>Cm@T9Q?`BXjk2Afq{=KVulcwB-i8*NQ|L z(z}C!b7M*xC_E)|^$f93-0kDgsw{e|Z#Uc^>=mzwyIM?S9qkeGL+75er9ViNMdD{@ zl@l!ze>te5MdCqF9g8s6!@a5#iv&(EajDS3r8Hi5IUVyOY;fI7eEE2W>mwQg@IjZBE zBK6!Od7+)96DvhN0&qDC1R~MItQ7YOn`*G7NXUjsbXqOi_{b`furVve{Sx*JN7#6) zMG&E;OV}z2ORGhm+~?@*a3D^AlLIbd^JuHZb)Lc2X_!#DTQ1o8#->-wazV?(II^-X z^se>14E_`4O&wQ*!9MLJJQnYzCMw=}R$Ki91?G6a#x=!(`YM;Sr&K zJ>?{;EQL5{icIqDf`6DY*BYF{7+LF>>{A6-;^Zum7mEh9Bjs4E(>PpssBEjp52j>T zJyQm!@CZxtiO>GqDL=HX(?x!nSt>|@G5b`(n9&4Q7OFqXWG3T7rdi%O@Ga!6HOm_# z))p2LhvnfJ-rJB01{i|-?+cbI3eqBeWnpPRhLj2C0`HyHGO@5Upulzu4eQ@Tc|ZIU zxj!N3 z5uW+QKE_!W-cmyocyH(D!IaU!VwJK^=xGK!R_H&JwAUK?$5LkCd$>fa3prBfl+|Uu z=P11MK1wjK5H5qEKmAd}M_IqM{)V~kP)RX$w4(&imlRM}D<$l8Vs)X;Es*=vxF$qd zXeg;GisA%3*VtVdk7Rc0sr>G}sFU+ikG74Mv%(^nGG5+HVwgg!ZLdgAF#!U?&@_&|otScBa8947SK%%?4X) zupb!ghX$klMx}-IUCCgh!R|NM?+v!aU@sf&HG`qNH9u&C=9h_bR&n&xK3GKzMoW&0 z_f&(OZm^KSDh#&NU>6!JYOo&|>}G@g%wYE$>^BB`+F;Kb>~n*CZLkdJp~^>=_MtT~ zSdqbI7;KKg&NEoO!MJ6l;&-{ht~1z=4R)Ww*a9e7J!P;R2K&%p|1{V>gLzZ!umiQP zx0PeCT!ZBstju834Yt5wl?H1x7%e#}eYE7Lysb0X27}R(qv&YAQFIR(Y^T9qFxcA$ z+hefL4feIc`ok8Y_zlp$bZdygh8rwku%N*f7_8D@jRtEm*gAtDDm@JKkVL2AgQGsRo;2uzG_v8SE;9tu@$>4R({k z9y8cxgS~37Hx2fI!S)*L^MmYse64+0P9C>&z|YYtIzJOYVaf&7l@Ey3omyI?&+T&k zJ^<6%=9V1&Y5s1^60mU7-=xNOe2uKG&ez;tjR~7tb4vzTIz$GYthuE%mg)3q?#K1C zBJ`q!BE$#78D7~X=>=V*!x(iB%r6W#Ox>~KT4N+-ggt6F^3{L{$Qjcr`8 zF+MQvu*|hpNFr<l7^a>ccG_?(j9K|(AbuEV)?5vbEVelk!Tiz3OQWA)h0fPVe|}j7 z7R5Y=Kri{D?Qk|viSF?}$F(&>usW!$Vs9C2WUy$_p0of4BA*u-pw+8IY-wCSwAC+b zb~bw-$4z@##ph+-i?9l3kTr!dy;K@~mIFc9PGHuW;{8|fiVr-kZ@+(@Bxp607=2t) zvmF*WaDYYe>9UHyLB=rG$=V%6^)!u*nH!&h*>OtrYf^FL&-Sv4kD0!pvi>1I#bwba zN~1d@Lc~>i#fP5OJzRJ*g!WU|4t3Gcen!)pS|TJRNekB4eUJMK-^*Nm3#?%=qZd}* zoKN!@N}YB(s6uX_@Q<0R%K&nnP_=)bD%e2b6Pc@zSC%wb?oj-UV>|=2c~3KNQ~8>bzL^YSu)q(BMzPuZQsKjNY!Nj%P$>T9!#WlI@3Xjgql=+}r*tkcDN_PC9hn#yWx(4 z`zhSfa8Y7bKHR==^WYw(@4?HOg!>I}18^VJIM#y5LT>gSm|aJPz$KsgaLK@$){W4YE@oMURq3bcR+FJyVzA2%_I-o>!eDnA>_LNZt)7bG z4THU9uzwhgl|j*cV=&YOjZr2lea9FKdD9qjujz^mHql`GI!1+^Z?Gi>yTD*q8Emb= zZZX&|4EDUiXgg48e%D~{8;rIC6|WbPS9GJaueWuQ!Tbi}${jt}yh(;!C#<*S_9Z7;9sr4K~JLlMQy7!4?{< z%3v25jJ5;DmTp~Tu(byJg~9GL*kcCUY_MGhd&yuQ8|+^O>xF($n*kt?^-D!gAj_{d}(6MfBZ3&XKSVzI1{5Mq(D{P75%6#b*)od}aJLB41`D zvbdpa{onC>6fCv)v7kP%i*EC(-4efV9(E8^XSW#(j~UF$ zO~eYH$y~#X$(o0Az!dY6V$6TO!t7uMdOq#AKbRl8}K#q zo}X$!G}T}+1oN`tT%15@2P^h+X~P0wJ`#_$CvA;jutnx++hor6cuDkqFXwRfv~BK` z=+Z!7F{{|p|JZp^}VgT){4!qU^B)si0gkq%6+ zWJ0$Cgim{gyd?S+*O8R@ayi4p1n^^~d#?F#1u~f>DN-}K(%6g+dPjB9vpQ&wLVPXE5uBO&%>{g>1YR$J zV^hC#WO~YxAYXN>h3WQJ7n{7t;Kk{$QjSLi6SK=e(I|A6Cag2yT*wyWW_i9ZCFnT| zhO851p2XokT5;=TdG=~bmFIXYyq$u$)Tan(z{kk5Kc-BxmZXp$U%3)jvgz%e={eE* zg=ZyXrq&JQ3O)ZkCoj&Kt`6%47y;+4WHg$z)*C64tTu^&?`UHY9PItfidySE*O>_N zg`R&@1nb~)UsupyQ{E%)vECV0jjSuk#yzurgWzMmZPp}NSHLL?V?IalK_Ujeua!%j znv}`byLcx&Nxl2TgKWXyOt}pol!QDtgVV(lj?faqOpo2{;C{A;EOF6 zh+A85f)H|~uU@v=O2jfR#X$%`09rDoM82X?=QSXy*L*G1N$ZUTY|B^oNK*e-D$1!+=QhV zw~CGq!87}LY>uav8Ib>83H!s8dMpLEKXXv&9f_q1_4v-{IqUnL8-$n7m6w_Zv+{W* z?Nur5)&}bZ&mc)NE1}}I8a1a1GEwU_I;KgzxIYwe^bwd(Ym6dzB(lzUv)9+plcS?I)DLqcD(R8RHEkB7b@yk zQcA2t`DelQ33M*{dngB6(8vTs}jmklPGstB7Y7=0qwz~%F|;G$8DP}S<-GMn{q z*+ikGmxTn4aL3^Oe7MDMo8YpSY=*l4E~bt0nMVuU8}B(|2K$@AIDuFEzA)G~20Ij~RQwLt zzI3U&3WL!^hvj5lVXR$>AE)gK^+0= z^}dROA8)95hZ*cdgYiYbqC4GSoR}+YnZfYomBy|%*v}1ihrxbpFfK7s@jhp;7Y+8Y z!Tx2iUeFO0@4?!KrT+%wWL@zqGT20eO*PmIgPmuvdV_JYuHv}dV4SQg>>7h{vaYau z4EC_WIDuDbdCFir4ECnM{%o*+8!RQ&4tu!v!6ad@kp>%Wu+t1S)nHWyJI`R380>O` zU1PB840eyf?lah92HR|~w+!}IgMDGJZwz)Y3<644JY`qOWvIbM7;K!uCKzmd zulE(dHiNA+*lL6E^}eFJ-Cz$G>>-1_Y_Qi1_P)VBG}u7QR}{Y-?MugqXq+dQS(dN? z*e0|Ld5%|G8Nhh;#;@HTj_W-*>dM>H22|Ww2i!8E?kZ{7z(iri+Yg8|oJoVeD9x-J=U3>*xQCkjcU+WR1lO zc`x#ji=Tb%kBr|pGNun3B!P*-=4)TNb)ms7H5g^A=qO_qHW*C_cGTp@@%&_ml=}eV zbufNBc~)I&$pD*ubccQFnUBu2nfbG|`EN$j z^rg!#2ns6)mBOgQ3cK83l&`{e8!YPKmVx`%Vq9X+V(k{Arv86zi;<^W4D9bMEk>bk zG5&63OrL5o6vh@qVQeuJMrkXI+N>~2TVZZ8?yAM$Y02u+x(pfrZ*4J>$(Z9m)hg4c zS`3A;#ZVYq427}9P#9&bFv?hA_U4y~n%q^3!Skur6_fJ+<`yGf(rhtoZQf%fO`mEp z6jlx@g;9qU#uh_il&`{e8;rX;$xcvNv$3-jHUyi_Wn*R0U3hUaCHe|phYYwO5cLnl z8Y;dZNQ-`i&pdAktipGEpM4#y*dwnWPDe7hb}X2;D~Rvkiv7_|Wq3U@whW8Pa?7}< z&noqe98LIH{Myl&z2&t=V+r4Da9sdiYkbOY8t;eS`&9G^+UpT*D%?$F6?>wa@J%C@ zT}|iuSV7zt zbK}e)_FtQh?TF6Zg7J_nark`-SMQuktC_j}2Qu%AyKCWYiYdOm_fQMAU2Dd1)!|9l zgy_sLw{LAuvJX+bS({5<)MsiDiRnlWw%76SD;D+191ckqgB?p;U581C8Mr@8faRXs zu+u8J%z#y@U9h!S=`jVC8@1MlCY~&@P*-AmaA7yyuv50QnYgDl!m=*$`~kLU294k@ zX4i`?O`;tQ3y$JGP?;&wSAz#_f|k|lc@ZmpXlJC2uCK(Eh5Lyj&;p^UhzmbMXlp=s ztUS9w){xNHszy_FSb1b`rZR0(s4?6WQpq8|2@-^>a{Sp-!J=61=fwpnjSs?-8!aR? zaVcn)9I$vvh#C|!H^-CDjMdZ5%)G0lh9&G0_-Akya)H3XQ9aEDH6U142aLl z{&~~ozQdp5*mj|tkQ8oS!k7}4mvFw()B^*0B< zEC8!Dn1io7m}!{pWK+H4;8Hlp!!3q86mBWpW8g;Mo&a|pTo@N3?Qn;~{Rr*|xHN2y zgv;V4o(Gq7EE9%X2$%d=e@=qSiN$B|t1gVbbnMJyW6Ze1E;3leU^f};7K70cr1;%u zu+0X0!eBK2D7udg_Km@yuR30=htYIH4dypkiNR(Y>@0(Y4MwF_`MAtrD-3qE!G2`0 z2MqR*!L}M~yTLfIQu*MZU&R4Cy~g<2y~1b`QgmFwsW2Lc6h`Ba!e|^)7>z>;JJVq2 z80-auy<#xeP7UXA#5%Qs?9_7puT%T~aHqDpo1I$M?`r?AQ$u_5f1MhHfbKZfsqtG- zyHiW}_VfQ=ofWsc8kH-sVR(|nhLwwVC>Ws#!gMq zu~SnRJ2i!|Q&Si_HHEQLQy4ong|Smp7&|qEu~SnRJ2i!|Q&Si_HHEQLQy4ong|Smp z7&|qEJ!CL;Y6@egrs&wIDU6+(!q}-PjGdao*r_RuotnbfsVR(|n!?zrDU6+(!q}-P zjGdao*r_Ruotnau9%lIm2zIGsb|52L;92+;AA&z}scEr&7$r8+*T~E~dKpzDe_l;GtPBk6oryW1o%guhbi zMI9mjeYm~gUI~}e#Cwt2?MEe%T6e!pBZGLU?M3`*!lP2c^zq~pXb^_^>U!R5 zMd*7GE@iXP$cDajEQ7GIChbe-Nram2A%pQNGC2bxH@FQS8)2E*J~06BGqUN}LMl;N zv6OK63WN>*7IeH$2^Kz=d8I&W@_&RU*oT*&gjh`@f@`MYJU}^9CRn%w=K&@h26zrO z{4qyszJG9V#iuQ4LH!X_aMeMh0^IkTDjIXra^9eEY_ zk@(Sl)}bTngh48Z`$-iA1zk~y9`yi!*V>mO6TT@(V-8~Hp0B?IA*a&CxZ{#39)5%O zLF$kPuK$Wn>z}vI$3ly1tpZ`fwQARr2kwu;;cH|P#2pO-_7|45vM?hylTYL@J&wab zR7Y}{4i2N@K9MNuhsu=G`?%h|-hrO!-d_0i>zU%6igV_Ar}pff(Ic;C&x{^D?Ni@g z0{J0v!C%FF#_6RQJtEQg6YF?5s`rfEr9IQUBYLK0r1j`|a7K?{hx1F%b~(RvrX5w+ z=aa4?+c^z^&aP!=sh7zU}@2lzB2m_AaRF6WtECP8}2d_HK%)6po#J}*({o7UP1 zjz8ZteqD?ri>!8|LyGKfLr!SI$$n`_jq{u(S`o^SkuxM!tE}^2_@S|b-%mXv*uiLW z)X}QWWj3d+j&Sz zDbo8(&6{a0HEB&tNUOd4Cs$~le{wN+Y{C|=N8rJ^5Y=t?QJ`ays{;I}zs%v&`27LD z9r*nbKPu)h19ehEq<-J;kRoRvesj ziqCZT#ld1^y0}>VJ)^|$FAkmVFPZ3|8C6_-HZszAQBPGzw8t0qRQrsc_@V}=k1pYu zAYPf7r*{0}U$g%h<;n5VPvFv+dLvwp4YojnYTQJhI#g9**XgI}STt$VQ9~3R`$$Fi zyuscz*!u>{23JLQi1wvhoCYXtgu!}X0Hd&8+J~h%1{-ZKE+12LlMTjie-sur7{3Bi zSgXN4GuT%K>kT0(x=ig$w@x?MG=tq{u)7TQ8-qP!urS64oGA3RYP2uidK5ktN4xf= zTRv36oK)blI1Fi0Q+xKx=#d_OHqvXL$v9y^Mvn;t<3jR~U++K;U^03PV3+}ROT}|< z`8EaX;5m+q9W3Y3uOWKKFz;BT2anYe;^o*W>lkOR;?C5|oP zM?kMocn4+MLyzPwIF~D(jc%M0xD`2NZrO#8!;jrKONw3iGx*uv__Of8j2|OmDV=~a zvAgsW;b$4~C{rH(>1e}q^Q60GD;(Edt9dS)=dqGs^@lp#jvw`$O)#&EF;(NV7OMv% z+1bgye4nxi$TS8uT^{L@Ib^BhJr zaB`7`JMF7W&%c|R%9qL1YIIRSK?7dVmA5oBopY{Kmlu=Gys2JlxXaqqpjNEM=0rVY zSYSt#=u)X{a|ZeAQ5m5uP_hbQArX0+w_)-9GV?-Vp|h}j`0={ivLbYU1($ie4=xM* z1JJ1gr%#>RsIXf7G~K$=&_xZl+hA`SY`=uew}9&EhLB(VmW53I2SCUVY9V7_D?)x$ z3;9DMWcrkl6-FT|j6zlzg{-jM2BVM_wqHW#?u6=c3OTQ%Ca;D*xX%3#kS3G=0TA-t zTF6dK-fM(RpAxddC}f3E$O@y76}H=86tcqhOUN|+tE(GAzWQ4hGWj0>At%%1e;Ogv zr-ZC93Rz(kvcf22h3z&Ng{-js5^{Ue<|4nNCV%}cHy7mpEeiP~BV_uNkQGKDD~v)` z7=^5`-3FtO6}DeO=4VLia%%FJjzYfwTh?Uq{}zS(i4ih=O2`VMkQGKDD~v)`*lvSS z$O_{ML+@rYzVL$KBFBW{AQ1E7q~b>oG3(hvbIqdqaAi(Kb$OH1h8y>&h_6!|V~{b) z^!*?igJ{ca$1%u3osL0bM-XkpAay7Pq^IF$k4qS*qj1mypNm@6{WE0BdIFv_eT$wP={VWsayB)t| zWkPw_W%829l_O31(xu=Ow%lNRx23S387u?T3d_PXTz8waR7IN9R_j}tk&j>JvT?6G@Sb!30a<4IE<^rc&T@vg9l_N7}tF&Ilm(WQY} z(XnI{7F^>`Glg*rNXs0B5kEn?AXXUm646Nwb&XBo=H_ta%*OI2t`x0pt*b96fJ;jP zr$YDPy&dtq|U`P{SJ=bU@)InVQ)=k!go%M4ui&a8N|7e05P?!nGH#7!45sH7secQ{-Q zFkFf107G9C5-J~??R-&>B1(;inY{SeapT}g;dGj|Z8^#I)kd?8h>>k{v_CLjG5r3N znZWc$v(sl!8$B6>nErT>lSK|1rfvgusz=zI!XKJTfH&Rpj!o#|5}rHVBmmSXQXU%vOAVHmWHws!!{ zrY+l&7u$O2cGC8Y8JG4#+n&YdAQP72d{wgu1tgnRtYk&*D&F zN{98fGNs8O9*H{W73-e8>__Bq#d9gUW+gLL64~WTtb7nrC41ytbEGX6Jo^t1jE|XQ zuYXbH=|>$UqXxRL$EP6!4qKAFn_(Dobl4)pZZPamhCOW9Uk!W1 zFbdriD4?TZM*?6S7o780* zj~{_xzhIrSwWs)s0;UFa6<C&3nNM z)L$m*Ntr?BcC1}?_bzuEgQ(zixzuhj#*~cm2~{@=Mjly^dvQ=cCMre8XQ>n&(n+PL ztP7Q*qy4VhyGl_)>eC06qVptT0t3@wju!sIbgNQyktE5e6zQ{frBZZ;U*VuiQE(Qk zDn&E9iq@4z(UDzi6pccg9xs)u_0EQ}hA+nRv!${Z-|Rw(ll_N7_vx4K66*@fy=l*(l$JyEDRP zz~ytC2^YheHx};RaLvbKq9Py&CR#xYxt2f%{{)6XD(nm+`oOyZ|o08&Akv z!|w)tQGRzE#>Q9d2Zmi|*zXOy->@}?tu>6A*;>Yh&KH#OVt}ElG7LV3uwA}j* z`;%cS4dZfG5o1|ka zerO9+O0Qw)g7lr>y5i%YzkWCAi^|BW*!K}jvD=(4Uxe3+J#N@j7LOAWOJhZ=EkHr;TYmPW&`pkk&6EWDw(gwli&C!Ry>5GqA6; z4f0u;+1Ezf2aLA#MaAx>Vx+QS?>L__tQ6ZG+LGU<*Z&LJZuzRT4YC56*>=+wo0L*~ zzM*bhILeI+e>B?Cr?gd!v{j6>RqQ>(g0$r_v#E!E%TJPCjDKon|=1E81 zKJ8^X>?p|JsyGf$mTxUN+ptLRjNMpr7BdSCkW;g04J^< zx*llG54K)?q`63c`$laiDDINZpB?xW>C6niCOSWBbfzyVr~N4QN03G__GraEGc4G* zHv#=Y4|L|oTd%&+854`X^=U@zrZ0cpgY;MF%WXzu`l8-FAckW1I-icwijmqHZ+mFW zkGNj{b2R?C2kE}h_-Hp?l(_NY7&l%}nOMJr^eK%MBaIdN!1?k$(pa$|jh_bk-5&Ze zm%w^C8uz~5>vfJieyR+;UwMMed5}N9$eti~Mb}n52IFs zKBFsrQSYaQ-Q;{xk4#^U$Jw05+a9`dQ=492J6*r>t74?9Vx+5Lq^n|(Y4$e;#q)r(J8Uj6UJ?AryVUDzBJ8NkCb_vnw;i@O zP4}I_jUa<3TYILrPRr+N#V^=t&ve8+3of5`tRwD2MqK)oxQdauijky>acZX6b`h6* zH}v}2iQ8?j?JIG!H9UIjxm^)=oFneJj=1MJ;y!G|rB8{g7>TPGiK`fit5|SEpa5`g z4OTs#RCRA^U03hSoz5a z9C!3yXY{5oD(WSQ-HjX+^KeHo&WRNZ()$&}%(nBv7ls$>K=nes`cCg&xobfhe`)5L zPvdOm$4(5kuSDZ#jK=h-oe#xYKo-S5a6X+qC>Esg3%Eb4hdxb}LB0A+^LHc$( z9QVdx^Xc2$aNIWzad&-qsnM6dsJ9q|Q0!*sQx2wLq_oD{9{N(VO|Sn4^zC*;?kjzJ z8Wi|mjOELxivk{lRsP{7vK{2Y^D#j0< zVkEO-+eKXJX6p6z6Sv#W+gIZD(s}zrTnJrLe7@0+xMLh~A2j09r^Ho^#8r&MRgA<{ zEFhBQH2Hg3{gxj!|KbK*y}l;Wj-FXpJAOLFd_DKwjLf!Y5$i6P60^eGQQU`q+?&XB zcSmY$;g|kGJeI#*inpA3B>{=z7w~TJ&>3+*UOO@ri1IpPniz$wiJF2 z1rI%oXgluG4G*kuM7!Q5Tg)iH3)68ln?&TMbJ4Mnt0k1{QuIVeM!WNT{ zeb@P5Z^5uu!~U0S7+j8LIU}n%&PSzoz>hAsq3fm1(Fc90v!+b!q=DcoOzkLnb#130 zsPXiFHVDvp3WQi+$p%5s%1UQrw7_DcOZx*~r|khX6*Lqj1ynq`)>UPnYsD`Qze@ag zeGdv7>^PUf<>+uF+~IJ)4;Q-yysO}jgqwmp8tyf4$HHYizLIP0m836<<%^G9;e0sd z#;_k6_7B7UX;?mT(EMV~7xgYSY`S6JH|!e2Za0ittXdaNPBlOB4HTofn8Envdo&kA zU({QKzZ&lb=YxO(#_q-ZU12i)>1oaTG@_PXWqLW(EPYFJ&5=@?nHx&Q%Btni)NmH$ zxMJIi-#sYw?5x-SDBw~24#tm$r1a|Z>pym)s?y2_Eub^mQ3+{pj4+7->=FUpBD2CZ z64lacC($Jwg)p_%Xu_bA$v9>_#u5y|aGuee6ti7>?i7T?YF9@{jCSp`TUR!_lE|2Z zU~Bl=fWCdMLcz2DPz%uCD&uQ9u^KFw1#xMg^(s)YX&CZxicLZWYCx3%b4q<`9naMH zn1O$>y;|`LasYNk*C)XS!e#Y$bFac2Rv zY))8%iHNo0hnLi*2#p{<>pI^x0yUPk5$KD0*W(_T<-qu)^hIfQLo}Q7y(P})h?X&4 z*&JbXufjJ|x&C{(73tP3dC6~ye)gfs=HwyC($y7N#w%gCGGn|__G2U)Tk8vGr4pMn z$WBA+*e;Dy1XcfflKE!9*V3Vd&PJWH4WrZ}7N43f6`1wXVZ>?KXB#G+Bx6RmCP{a; zF{9HM!1+aH(3tL>40C`u?SZW!m*`3#1jCC(QCMI+6rONyL2f@9VT^Xtm_AZ8ru*d= z=auJ%S0@pa0QQA`mzaHKY40^tL zC$nbZU8^u3ol#*9n2z;#vveNqD%_jp)+Bk9vU{fbR>cl9k*jIW)>+YPZ5+tc}Q@`Yi;4I6LRB*Xq{*c*o7 zC{dOMTP?-s%Bq7-=eEN*5TTy)FRg5aVit%Hm*vp2=$v^B=$@GduQ?fA8R5s}BbK>VDuYP9}B30JRE2>6@=}#+h ze0&*vB^%z4^}o4@(~dX~#q~5yqRnomsj#qvZ;1cWmEOW%BI?N zmFc^ShT>*rQ{C!{^t(x9JdDBgeMRHoRXe(Nb^v?ikfv5=~ro6Z)kb* ztZ*f67BYjf%I1Oc6-j>lVzTXZ6n;a|UC5xK`J4!2Lx}a#dt739-GW{Y>AmXDW9IWnTMrnNcp?PSfmWEw{? zag$&e4?VhhpKJMa)mACoEgThowqi&tBVdzIqNo;RU^&f^I0#LGgcU$3$OkbREK>yP zkVT|9k_@*caZFNSa&g6w)y%GGVbNk#rV_WSlb~=40p!evo7?(X+vc41*7UkWbCEPb zqG@6g4(^y(RO`-rYW!$+9$LI|$Y)#fSz6uu@mT+YYZ}`D!xu?!$;IsvpS>SDx9$Dd zxOIr%4?w|?EjzqFf-y%nM{a9uuMa2F%Ymc!SiD`BUtUG?sLv`;=jMW|w}dOwk0DYB zx%Xr4`CIald}n$quMHM4K$C{sIO^H`k+K2bGJnOY$DfIhV}fYin(o( z%J9w7dda!VkwWR@S1>vEAE8*|=diS$069Pt?HviMF(O=Ol(7oMNJK*VJvLGLTCDkM z5ICM*QPF(*=R6>*BD^NyqE$2({4gF~o?JYtIdUzc#O~1s4M&R5rp*JT2`a+RmJfLX z@5G@nwhgi^14Z&0+cDuu#>(50sP?EZF}%U30IQn}uOsOz=Dr=8y~-68JK|;BdiL_1 z`kiCNp!~UEL|GOqMn6PfkGC&Rwxh>Ygf~=%aUfPQe7C#|{&Tz^8?Y;gf^)bQ77gO_ z@fDM|L|&!x(idV2pZSbOYT>0!;|;}uT4P9+{ybl6TxIyIqGbEB%J6;iJ}1MQlkLm# zBFj50udjQNk)ZU3qW&l$w(w}l@4li|kdK4MYH@EbS_e0YFh{c_js)`%J9+hrO$f%T zs=i}ih4>CU4C+*Tjx*N=40MALqLB^xe*CT%><7_RE!34>`zDAf`Fu@>1JybIIm1MAM zAOjvJHKRYXnkI2;@vwRw;=-B2xU3}UGHIrZ0mmtpW<#Vnv@?uVCI+w(Gx7AODpG1E z3~&74&`KpA^ehIwIgvN87~d;=xs=KLU=$J85aeGzU^K#+g@9T(F9gO#Wf@|Hyzx`P z>$SK;nN%)~%WntcZjzNL~SBI z3E8iF5>l8H*E#YL66V;M;%%;c+!K^Zn13{9jBMvICyi~-Uz9<|B%%#rl0LN#0^%o#wt*o6|)*K*vnQEL81!z5 z{N9@xR?=rWsDI+yeG+HCNG89FB|kPB=Z0)H=64DFPfOe{!heLwLa~syPOzLv*!v>P zCp&I-NY-0E&L=99IJ0K!YxCa-4UwEz%lTxMvhaxzxvT!66lTxOJ{8IiN9t)f` z=WFVhz&y&B#M7o_&99VKXE$&_;sb6)_}dbG2Fr(B%*<}|?gahV?04ec@8sUGxR;Mt z2S@w?AAr`nMeyCRip>#fmmw~87)qW@cY@q63NhV9-m%^-a7ksRy;okz9VIW0LBEu^ z!x1aTJ3z|ypD8DDF><&xbe{Jr+%2Koyti_W5+=YV7T5U#6M)MUWa@D=!<4SQw*u}&hkw`M>}SLmI-Kpi4B-bIzRKZmI6Q=Y!T8^S%knOQ%lLP}WqsIh7^cZ= z#@h_{QMm8JT?021(fZSTHlOEFxNG5}BX}>P?z-xrFX~M+jN^x5Yzd9mVAwpv78rJi zVZS!)e#1afm)}amP&{8^-mkV*CJV z+RF@^V;F^B8t>PJk$~DsFVd3J@ytkG?ljWLrZ^I5T4BH4@yi&uUK;p0( z!!9!HQp2t=?B|BDH)*-F4y@(=-mo=>tu^ce!?qb#h`ys~i<~d&oo3kShD|qYreUzN z?9yIi*pCeRsbRMocBf&F7`Du?Ck@+R*z1OYU*ewkQ^US6Y!LdoKF2=J7xj)YY?xt5 z!zvBCz_4n=8VtMAumy%KH0*Z6?lSBl!&(h{*|1H9Z8PjM!}4#qdtaI5J_IRx2f$?pyTBzcA0N2X&Tw~y zyBpkra5-UMcrUoS!zCRUj|1l>z&rv1rtgf2KbfmHq<9JFh7(X)SrT zykx`N_M9tE1WV$$=FwkN)@)q=_Q>YQk?n2i^@-+zJNoRD*duv~@S1q+k%Ao(;kF8_ zRj~}7{We#&xw`C?s*Q(klmAuhsPc{tT6p(Mf<87+aLY-N-!5{R&z-@CLLK2 zyJuAbtOta~m519B;gyLdu-aR(gai^VeLuFa2B%V0rcW&5QOrp$AIp5{U0(b<@e2a$ zx`lWSt~O(8)r`rLXHBY~Rdwl<`r4|?C(WEuRaaj%YuNajnv$bs zvFFbVLpVyHV<$XO!fehH@#9xi?}xlkB^(NL;M3*au7dN=;W8iNO!jcW*22~04VQU- zdggZt=A3?*OPgVcpYLYeT-r#+XCtGblF_3Qu0>d{p)Q=sgo}VY2tGsJaOgk|ll%Of z>7@rUpM~&20zB>RXVTM*%LGsr=^eplARmdw8P}X@LSNsnMkB4JAw{E zuh=RF>ZNW?!2XiNkBe8vCMn4)<>Wr5=!?DMfO6i>Gp*|c z=Mmex2Wp|q>Wh)jNV$KS#J?EvHJ=C3zGG$H&7q`=3nLguD$+?uD)vk#Fan#^$Gh99^!+XPJxD4*wa7iQR9i&)MhDp;ca2GoKdKYei`!4WT z;4=MtaQ_9D!|JOjSBF*l)Eb~-4eoA~_Pt!Z1%};V*kZ$$7>0y08uq4P zsG*b=^>##!v@Sb2UsNo7DYma+hgv*}l=V3(4CBmHv2zW(&@lE{&2O4viwwKLFzRDy zyt@p0&af8^+XXW^jR$==SMG6!aW$%S!QK%UkE<5NxNgz3wT9Ij)?nB?!xk8JvtccU zJ!Dv`VNV#g&af7IAGF-tosUwZjAcwtRkH15MS$@_0&j8V`_V_4x>*O_G@aGpOGa~27w1|&2 z=SmIA*fmViY1me2=HH^V4BSB&qA#^by}v0oZ?hhfVMd(<$ryi1$wd{K{_ zA8(@!PR ztJ2T9(`hT4=M@cQF6pO}%?pcKkh_#c5fTH9AH}ZY=MF2R1Vp665EZiI^pn24Gy&08 z!{o{S!l;B#ltQ|4F3J9c*nj51x{w*O5X#N?xPkaW@du5<>?HeR|cws5vY(&qIgyovBD<>ANtMo&U&>bH5UaW}+Awb)zP&XoRF;QV6O z_NO%$Jd8W&ndtMBQL@igHr=g4uMzCwuB> z7m-@*2wU(Jw@XQoaF>9fB^^sgHO3*Dl!qErcB%n~fj-xOrK1v31D|?AO20`eL2tH$ zPiOWKHa=A#>Z~!s(6-Qa85RDAwRd=H#oTv79GL22jZ@Kan7DA{Su%))6B?cn>< zC`^>xIgPu?EGk_A)6tok)C3_N2E>IrAR?<{S-I8B*PZj&OBQ=7^{-w*H(mvH%7tX8 zTzI~kqjp2?Ww^-?Mug0L?=|uL&9Kq4ojPy$m_6RVEO;?*d>Wfx?jms((D zPWq+)KxAiocp+Jn!qiRV4DaaBH*>axmO*~#a|}Y@1aJ~$lT29MW|%yJZ%D@T!<5~T z2e5>bX)m6$mv=^}BuC|R^*}-}Xv;K%;4fwrZ-Gca`L)ukSojEo0(lg8)uE$uB7u4+ z?}ajTVE6nu{4echpg>x{iNPv_)ZTYA8#*>f45n_OG zRbVA^%EU=kwNh0~!MsN#-(8Ws8#Yc?$uE`Crs0-yHvW|}By=z&r<#RMX_>F%>mVMP zNlK-7ko`b{Q2;xWoy5CUYQ=2dl83um!rWT$Ti*7l>o&rQs?CbWj|!*x z%HRv~ef;OH%SyqI7uPVFV#1X1v!pAMqc~A2wJEd-n#U7DM_Hi_Sy{9oF1eEI2pGPq zMvpy9T8gjl34$GlwDAFCnSo=D>$;pu@7xeY>SUWuPz*2bU&Dt%_PPMST_BogIVVZ_ zdvia?*(E1a#yM67`?@w!RLWq>pCb9)0YUh?7=@ITS18!J$id*r&-Id_Q*&MkO-DJ8 z;YT%qGTc8$%DO3+`opzYnnSg(3anA+t?-3giZDs5cV^DG)aO0R-4@*ShaJzWmAbHQ zocx1<9Xj9JL!Ot?cCE|G2n6_%Z$^&uu8SNf&znL_y|+G+n=?3|(x5LU>3N3acVnm& zc6#`hQ6*rEU{p64K7h^Z=TVijGL)%PhHW`X;-4L2`V9TUT`*Hg&D$jQKcqA=MO!4C z8Dy!*z(JIQ>??Bb76@L#IHqI|p#mA=O9Vd}@%F@fnORWtK;eSnjXx~NJ$ugp%sqPH z#vekN`&j6+R|He&KDx$V737|@Zy<#n=G*u?J6Kz=i$o65$k3fF}SC1gkppyj&gc z&@bHh$NY&q>{Aeh8vcQPHG5p$vvREPGRZsQDg!~kgu|fs=`b9lzg+_5QAN5SzY%=5 z%>CU1_l5qaH6{vjQ@`3PfNKLJ&;Z{;1Eg*WrjZsnqOlryI{0kb1eXMwV1oZ_T{MCJ zM3*>nvPsAe-6~D;mz;Z`gyAn46-i^_ zL-+e|`6DZav%bz(f5;Z5V-w_FxW$df>~L)4b@$@-NQ08QvcNmrO1&@KfvILWv{=A&mz%e(DprjL*C-gA40&-qmo~FxX_B z;#V80CaD&UHiT$tNG7#;+d>60!D6B6DWgwKI{E!+ukncwYjng4@u zS?(6NtWQ2>Bh7F*QNIrEpWyxw?wOclGF%Op;b&d=b-0YriSsYua%4FPbvnyC1usSB zDsW!qfBJZ;x|hYuAt#(reH8vogjMsRq4w>kn3+?p(@_#fY`yyRG#h19cFJJXG6bOI znLWE?)Knm9=b4IMHv5rY{koa8_%U1jh+a>JV0zc=g0BoU=ePi48!H=?z9^Iyee4qF z(@n`5?+U}NwRma6?lJ6M!zj$rw95^9%CM&md*85+4GW`2nzoux7&^H0&>iQK3uAphA~E2i3R~%fTD2*ul=1?|s9t;|x2&u+t1X-LT1q zU2ND747<*-n+&_fu-_T>2g9gHs`Y!$u!AuMDfSKLi+U#*c9LOd8+M*y-!trT!^or2 z{H`+^a=xf{m|;T=!&$a2-YUbM zH|!(FtG1&Q{-Vno%G;F6R6pQet<$D914+ay!XVbJd zIA7G;h>ur4iTASeMZJTep`q9}oG!kb8u zaLhUGs^A7pcVmW^tbtG>g)C$jA(?F`tmLD{cVpL`i~u&|HIz3`fWF$zvY)BEd1eUW zlD3Mu%X7HfPFc0Fdsd3{=4zHXh&+1BA2OyS7~B0ZKzZq9lNUh_^NrHwa}F+Ve6wNy z^3qLn_8Hzha2T>&N9oOm*gemNpL4RCL$~ClpNBr)$S-2|JO3e}PA4FNq{UN)QE+nZRBY6W zHI|7~2T$1kC}a|KiKZP@ssp6-!`Q+B6y+!}6A`#jO9b0iFBL;l7xPhlM?11O}7L-X#{*ZFnz|tQkgn`lX%=jqmMd9 z!u+G@mcp0eE-D9;OahJ(RA|E~_Bl6swX%tb3Z3^0Rtt{ak>EBA7p3DO!M@XpOUZo5 z=pQQg&&XK^byjUefcOtOAUs?Sk~&u6jIs_88~||QdEJ7A=>Xx9u=BPEmLqjY!OtJeeFW;DoI|Yiy+6vm$x=RrFDVKC zCv?0wIpl9R%cx@VsW}_RhM!JPoT=Re;p4D&wo5`=vY1jFo@s>37*$OQX_k zrL*8tUPGL!EX2R>aB@b7-{)|Oiy40%T&(p|U%+KKJY0_Dm%(MdIEZ4kk~$3Kb7iFK zOZs$usTkLnid|~)xV}{E$AJ6J`7^$T3?lSCd!yYoM)v!&5y=mA7hHW#9d=@QtALqjy-LRpCp-o&o zt`@Z}wT971hhkheYJMrhZZ@pNuqB4wXIQIYD-3(tuuX=&XV?dZK^)}D&2zq}S7=y~ zVaFIY%&>%ENyDlQn`l^_VfBV_Q<6R}Hzg?zZZ(XXk`#Nyuw{mEy{c)sUe$QqfTS2# zu8O@A@v--uuNT#DlLY@3f}HQyp?e{EwkB3vY~d8dAYJ(>ga%eT{DN!Y zjtW?VB_mbBktUvk_r4;%mW!$eC@uXK$)zKTc20`~)C5cN;}?=`ufeX1E;6Jn)Hs>` zBspZ8sE{F4nSP^k2sTXJF7=>d4M$@t)D@m+C>a2gUb6v}9(_5*R)rvS+q9!F*Rntl~zLZRFjLjo4D#LG8HXnAvC@kYD zQPNUCvUs%BBc7V7TBTIgI=wkEq%!buv| zdX!pHREuH-W~aAYTG?ELN*y+A5frrs!*a=fD1RT8-?8v66t=L6>!JcMR?>R~X8(wq zY9y7Al-i%9QKtDWpyHjRY&95q?}=VnX3ZMUzX=PiZiPBxfeE z#&Xv|TM;r1u5}E2X$`Ixmz8L6aR&FbYH(eN4431fB4Vb1-;an4qKaLB@5e^LUNPVA zbA1i256rg{RzxilZ&&jnfvlF#1V8CJSsGmA?;U{qbelQ#m#e$gGHe;w!0hw%NmHw2 zSJ#xw$JbApG5t%acD-q;UHwsCOowyFcqO65Im<(zU`pVx8#*tsdsFSI&Dq4@(ODhUP3j=7 z^;hyq$%23M2{^d3eSOY>A8$-nyat!GTBqqGXyf8$Tvn2lOTvXCWpldQ)HY92CbS4M zJZx$)`c^Y2m)33p;;?oU50gLP*RIx&=AMG%TGwh^K1bsuDd{}Rp)M_w!J>_l3MQWW4C}D&wUL_x(IF^~tDgk@O=d1c2wE&=`sS?qMXN# zCY_lue20E4so#~7slv5L!kNJ|$Y5D3_d2U^u}64bw<=r@bg9DiKo2ThPj;!o^=v?Y zs4ps9O{|qy4Hd4FO@)iHE^d`#N23bY`x4Hma8a_wPC+wX2f}B2vwRh<$0b6~DqOFc zZqW9raJ?;gXLN(8T1^&UQ1_<`6)s3+GrB=Rbu6dCbqUn5x=`V|q$d@wYr9tA`ca?> zCir?!J*aT~$dQxd3pMC2m!2{x67fEi&ncT=iNwh?!2=TRK{ZQtx4Ku&`Y$8@_Nivw zDFtW9f18ARA%Co~AxQrJQ_b?ZryaPToIEH~)xlp%HB0rOI(+N9Q_WH{QXRhC|1VXu zTJWBBR?TWPw3BKUHM54w%K#59&ZP?x=9p9DaB$6p<9RP!el)>56V0p#;9iaJAK^X& z_s?+OfeWpu)CX`GpC5sT;0}fh{#uG3l!xJtf%^zt4)cG3%b}b1H^5y5_gNR_ra^}D z;PU=37cPYh&Rq&zS?@8pXTW8CKZeWve+!r8Zh*`BY=yfHF6XuD;ZidT_e3L$;n|on zGJGjqhEp#5nhXCEF4J?4{W{#Ocs`67u#NbYG_`tNlx(L8SNFarI3kY^PCUb0fyaRSc_q|8@Ado$_g~UjfTB!*k;4n>oi^gO4oAP z^Ay|1F!pT44l``1VU>olcWOLNO*Fr$hH+l5*p-IeW7xfhEjNstU^Tyg8uo9)c0k|M z{G!em^^P^{n}$Im;o^-l>_WrF8+Nr}jfVZiup13qZ5WLiX#L3L*1C|(t=NHhR}?$M z`J&z#hMjHLWWz2t>^j59v)23`GOX3Gb%s4-*vE$b$FLaQL#@lM&KLELH;fBEE#p+f zMjCd$VO54*W!PN9erVW_4ZF>-Um3Q_u*VJC8{>o4Z?N-4y(0}PF>It^qYdNOqG@Ls zMm254<{HL%hhjfA>^FwpW7va+Q3szXq8_Jqn)Y48J~8Zb!*;`WK;!M{d{OUk!;Ug+ zg<-1=+i2L!hHWkoto#T9qD|~j5O?g z!!9vwhGAD2c9mf_8`fgj2Zn7kECQV4ReD5F52Lt!uVQIXn&KLDs z3&1^+cq^PQ>K(F^j}37?Xe}B>Jt?Mu_Nrl57zVAU77T`Oa(`fZ^J!* zJAFL8HJ;v_z<&Bq+ z1qfWSfofY2v5*Q(pu4Z{Li_~yz6;>`e0B(VWQ#raX)i{~zd&Bq4&ASMRAn3!-XJv! z%57w)e+pSNw#Kd-fiw?F8r@Y5t}!1b6CYJk)g;-rPuVJkQTp_<1)CfCA+P!s?!=o% zwxfmk34^$#jv8Y$?#+6{b!UHg85-eH%9d^rC9=xq6R+{<3RZtYL{w1qYQD_h;-6kl z%~$M0Z#X$#`s|z&p!#(jcJCjJ&HG1`J~`(b*t_4K8egl?xb17=;m6s^DmNlaoUM0C z?14yVOSn9}qCC7B86}v}$;hY_866LexQ3$>rLWF89M4W#s|FeK>r-w?<~9Pkos8T{ zL4pvfR)VS>*(tyFw%&78VHJ{+ioPj;SmS5l%hPngze4CLN!*Gxz7D9oDF%xI6?0cb z%1fVv##eo8;e&udUCwwYt641W!lD#h*KF;}T$3ik8{?(V#}*D_64zg}{p0OzG!4K+ zBBEl63ECTY_j5!=hx5Ck>tAY=Ap9wNUcB_N*utMMo@lPk6Sc&0TEEl23^|}ECC=Rp zlLLA;)_5%9`Aw18#`-`Dqe1Sp z6g79&2U_7BrPc?CQnICQ6s!-ZFoguW3%B^*FA;ZP5i_btKSy~B-#h8-Oz}`PD#q=Z1HB*TTp6Am`a^C|Or+bPcV!g0 z)W|JR zC$?87x)61d4*N5TS0j1wJ=bf0QKh3&pI?Vtz?JlQ(cIQV?&#TwpOyqa+u@u@b?BNM zb`+daAiwNgyyDQeax%)6#eNaUn?feis*{*Y4Ny`nK<Q|Szam+%UZrFQP=0> zd+&Suh89~kWrFjMX48P8dXT7b;k!YS)aE`NRt`P;#SVBCaF&IMcajo^e>B?^;;?Ok zinaWNt(16sN;StIJQtrA9M|gQdKKQ8IM2eD_l%Kv{G<6TK-^3FBLS_qwbs?sB!umb5iw#p9scwWQ5| z?o?KarAUz_S>u~Qg9%*R;;gRFv)EC5PUsk;_JXV|S`jxRT|3K+8L%5`Km3mLRT@kf zhaW_WIkh;dgHka{f0!OJ8%V{T4h@Eks(<*7Pt_|m=8CEc;8-*6EJkCpJ+9WwP*;|qM3DDs>n_s2vY4oyH~5X+1p zv{l0TEJmDZ67T*UHie6GI&;!EF04k3?j?2n+WDgq`OLl)@jvnm5uq2~?FkZI531~+ z!*-#hty-{s!S7t3^I=XQ=%G?d#h2w~@>RnP92He&R-f~8?}8BNJ_XOE@!N7spa$@? z_e|)>@ZUpKp!tK2?kSY-=Q%38I$(T3C4n5ymPuWTaxU<`BQ&S-a;so!z+q7!kDbDK z0cm1J8pwq%&d?BgXZpk79QKFK5H!z*)C;7zhauz&V<5`) z{tLgLRR^^LF~7AdA>#bj?uCf+H+CgNod4H=1gkGX#NY5~VOeu0?-2BJm12?Q5)>kS zF5!#>3pyLPdJrN$D+ziOBF^7zQmpM0BEBzqXQWu~Nw^m&R{s38K`B;eA>#bm1ziad z=g;mzh&caxr0Hx2V*bs6Cg_V0@n(5~fzkshM4TgGwfVrkxcf;D+C_9|&P4cd{GdXB zSMlkbQzCo75Tu@BL}BBfC2gjuS|yw(d5^Ttul6I(wAVV8I~&@$Pv^vPI?07OuCWp~ zN=DrZCFkE^&AEL-$t99+ra6Br;a-|EHXoY*8JTcUD0v;GdD-?Nwvk?)uKyp{izwl@ zCTI^rw`3$iZ;ni(L!3Pa?Lp`k4SpB(zHZUr@S|bfqQPND!%iAmj}ZquKeFLkfg7@6 zm0hL`?zD4b0dDy_Hz;-FbV$qVFneOv7vq+{se@5l5H6EP>m<9p%W}z@^o`sXD_cxD0>j!Z~Ow#^-WsDqKo6 z3-K&xe-)`_aG{zdzU*Q8PO}jtt6pl1q6~p9rsIs-5{TM)s>XRSW3fx=)tk_h-OP{K z!@(ZcsD+8c5cO}!UnQROMR72hkFi%NM$3j8Z;oM2hBX`Zd&9_C)3j}dtu$=CVb2=I zHq$abGz=dMhvhn7l;?&xY&XN$zcuYQ4J$XS!mzQ1oom=^!??SOxkSBY!)P8@v0oeZ zTf?3)Y@=c1xoBNB8}^xD9^Mqq?-1w9_l6jDykXP}(0HQ_JJT@OY;k3L->{z=c9UT% z4O?T_3x>U7*t>>Nzd*|!fOk@#V>joEdixu8kYSv^YCOErE^VD*-0P$HEjH|C!yY&6 zDZ{vjNAvr_u$}S#E4GXCMLpV_P>ePww2WHArW(eDqQ;{XRO9`|uzL)9+OX#hd&97| z4Exxy{}@JTsg^-$sg`lFVIvGX)39?4t1;{%!zd%w{H`^O#?ln~FT*G;RgBV7Etdw; z73=4Gu*qWB9)=xa*bu``Hf)4p=NndK*bKvF8TMnterDLchW*hnTB*?bJ#N_ZhS6Sy zKF7y~(Kdx*{qSAXwE4~#^$s=caKpZ7*tZNDXV?XX)fzU{u!V*#GVCtH?lz24TCEGE zv|2yPW)-V4j8a;~D5ceSl+r3jDXn6Z(kezNtzwkTD)yOSl+7wu;CxZ_x*~ zHHL2QJ|7s2wHVM(2aZLd?fXoDVvdhJDAdlwtD> z`)|X3XBec?tMPGq8^3^5x=ancJ9`|EKf@r$C$PP77h=WJ?*(mg45CdAexno3)iBf% zhe?fzMFR;g6;KUz^yfyrFh)O~N;L+OTOLc5EZM*#e#_Jx2Mo8X(GFkO1j!Bzfy1)H zWbWou6U<_tV?JmSe}q!mMAI=vPRdH74+XH50c#w%{jkVv?{=bK$YdcZ#yQfmS&Z9W zOJ0JQRz$W>A_PebnM@GRLMDrYn{SYFgdr(i0YMsBDsd6awox>*PJ*qtxeBL4=Ohr_6VB#zNY{j$(=M zdgMhCb7xts@kNSSwb|nBtrc_s3G6Sx66s@#a2|De)4-yNxho*e{cCLD&jd%Sa!+A@ zn&L0uDcl2bUN^fQ(rcTSVq@XXh>PQ^*-s%5mc9a+=mG@GOP`4?e3{}{?F2lsYV2;q`DKM(O<<0cUtcGK<(yw;u zeens~gE9mv^Riw2a>r$*Tj$d0`D<$8guJO-Nq4lRtIkTd5#bCEUu6#DYth8XffBbo z{9Eva74y#&9ts}%zrZ!tbCS6J7t#EjF8x|`}GZ3-klID;=nvDE_ zeQ1>2-#fg)yFt?QNB9iEcDMdY&I;+ve*7c3_jt}pU_@NvePO$E774x)GS8D>US?J3 zk{q5{tg;9Hp5Ol|able*U%6M(&Bku1h7ijjKi!Xipx2E1f0KGagUs+uKFZgxMLthU zK8NQ_^QKEa+-I{yuvy^CPlMg|KTAG&UFM^_eYVBlCC$Htw@*_u>m;o0(3!W-Cm4Yr z_VOZkt$mqI|H5F%r(*2B-s_E?n{9(8(|5>=K54wufvluSx&+p={ zh{s`t917+5(5D<9#iqHtQI8xS#rSd2ct0`hM#Jtf?AM018Me|ea(pzuXAOJTF!Fx1 zjL!`t??-mP@XU#v|88u~!Tu*G93ehM_MwjQkr- zyT4%v85TFJ+_15Room>44V!HkEsAQn&4%4!*sl$1Gi;?{&l>iEVecBY)vz5g+0!z1 za=xgyuVMQecC2CFH0;}kjWuk%VLXIg=`z=_YYiihNXz}1VYD}@*aL z!#*|a3&ZvUuSWAb!1OtX|PajlxmIKY!Y?rn(%v|RDpSE+hwlkW{=dC2%${mZ{2DA}W zoq5ygIiJMEci;4!-{*q44|>l3#dyB{s_L_5sXDuxowS`zF|qEof-bO4J*QSkkEUtJJ7Y!Rzln5N+rmAda*0!!91shn6~bZ{>$G(~eetoOU!U z?QEBopA>2Up2T)r`C$b*S^1%E^!8f$$*Mq~t^BBFv`844bISyglXtUAhYtr9+$E5!{#y_Y0ksHypFUswbPMqm8`x- z9qC^pL)C{az;p7e8<9Q2irtKFGNpOYea4qqeOuvE9qD7h|5x8A_}I}S-kIjZCtf&I zNAKi$)mb{yls|CQL3dFfI8?oEskK#LXVz73%G2ARJiRDqzwqHXo3~4OdQF6@O)5{* zmL679&aTc1v8&@R)qQ(G871l`x>cg~^Vu6Lfv)TYWt6B_b*n`E8?+7;xqDWkUc19( zk*h*W14`6>*#lvm<;*XyMEx;K@1#WiJt$EZ8b@xZ#Ni+1nURapiDyenRhYA8{^>+Kc#iDgsfGVyE14_&wHIb{o7fj1;* zmWb>quIjLZ-pmpuCCoosyA$0-A%hm47D^ZiAROh=w zO3IikN@bO_6mhbYs>z_EmPC^zJs@WhbpCwuEXYdUig+2OWX}rK+`_;E_b&GK%<;|X zFg=g{I?!7i+B0WE&avR8UXNc#W$How`b8%8qD%-kHW0v30IiqzYBP^A7Y6sgY@#%M;7`so~1q+X9_9X=q1 z@~WjBw&i{dvr(D)Wh+nbms16`c-A>vdAcExM9s=NjPmrJVA8JbQF&U= z%j>i9^qE=8)9boWo<37J)Er{`2hTk8b}LVR)TQ$DN8Kw=U%PWx%G1~G7SP=7i}LjE zKy$p{P@X=|l&86CE2unOg82n_htkhZ!SbSktEMY20tGalmIX;s3c`m-cJ0Q2Sfh>+m%XQ-t=#8vlEW(5jc*VFuKlB6K%KoH+GR;mp^o2+i+e zXGLg!54%%@{x|N8Lv8$vYo$Vjivd(R{3=K>4n=q_+%mZH;Esjc2$$b#9PgXD6Yc`I z{gDQi#Z$#_e+G9bT&CkE0Q!ic2YowSP7QB_+Y0w4xSQbq7hLK=Lt9bwpl^XoJ!r-! z|Nd8SsRzyQOD;@3Xoe35ADj272mK4U)Pw#x-0#5s4czf?ng1fVEN=;1mcI!u>jism zQr{tP??sp^mHXg+0{4EnEtr8bd_P=<|LDT4F1$b9W!^smF4J>f10KBSMZW>s%=oH$ z(Y-ECvlOPgS8Zk|(W^JrW_BDuX78%aV$JFAXh)TF(HHfuL3+jJJ6}|U+=~6wuv;zO zorXPT*pr66Xc+gXY8n49?4O3w#HE(Oa||@z0frrH*wKa^XBg+un)W=yCL2bfwC49c z!)O#xv7Z`tlVQIz><@-LW*8N`H0_6mePS3UqOLB~w9|O|8pcnbVoAd)4I5|J1%}lb zR&UsU8HST2U4F34=E{B0u)iDj55pk&bnze~b!m6TJH=XIHSK&+?`Xr0GYtC@Ts-Vt zaB2CzYT5~gU2oV=4EvK|)Lc^AIuu!Lbr!+5~2KHF5oco?r@DZ>^T zw#cx%4Wsc&eKw9|ns$X@<ARFrJC0@pvYlrlmAZF-p@E8)Mj6hD|iA*07Xe^9;Me zu*HVmZ5X$dX&Fx#w$89khP`Rndxm{rSbu!$w9W&aFX|N`#UbfCjSC zU^nN(o&&=UGVBe*-ZJcc!#*|)T1_s$e$EG7c*FKE42v2U?-0XE3_ISivkV(&*xiOL zG3+73S`E7eDkjHZRLb{mb-t*V1fN)+x6=8d-s+w4rIL70I3Fy-K?7N_Qs;|$;|-f+ z81$3*J$)NL4@7lVv#3pc7II9cS2T_3e`fp~XeuX@O|^OH7n6{$P;OD4-bOjVB6yQc zb%T;k4TUA`l}*)~E1POJRo2{Hl)|0Lrn-&E8nvWVQL{eTwh2c=K%=x0vaLKAg82#J zAo*EEg%#;_={3ApOkq=tzyphT)?j}*$`6vy#+PE(QD3sWO}KCjfh0cHR4WQ^nN_nzu-pvJvSMl zKk{q(z4v3|mc@$K%iZ70-2y=C7%t0WI8-sWwSQ&!DM4f7Afb3WH2V)wTCK(7@W?zU zDJyw>+Cmd|PI)rC3RSwFb-)=Asd$dR4MZ$GOD>+X3Yv93|H@BK69~14V91*~alFLNsOe5};jUv-SyE3jNJ`$H7_YV1mF}~mQwRsksln5At+TrTG0Evv zrgeOBT?x&HD^L9h$@PewUw9A0V05YFku5>s+n>o}xa7f_CP9VwB#%9Wy4;a?poEXk zSqbwwex1q$;~&RJhmZO38w{?vVgpXZJ^XI9zf7 zIjVY}J&H&wfJ_jC?d->4Z@^|~AiA*ERwu?$P%&*qVlBYf63~p4me3Q%<%l~*_bDkg zX{qq6@vb+Fx>br%#ZlwkVc7kK{mHOb413KmDm!XgDm!X^tMScI>oSAj2?0*r%~z66(7GMQ=w7e zwej?_3Mkbfk(@47NSp^NmzR80@-DYnVMEq$kO8JOUUG3o^NBk$#u%!#_)0Q~*aHRo zHJ67Hu}AvhMCCQ{(iL-djy-VJexYP~RmG5%u}AW7!f-h)mxfnwMqW@YE9fU4Dhag$ zNTgfSD{qW{{7HH9x4(#&ZoqtMXgpT_Br1YXBE7XdR=zEXslu4KD|3=KkvU%a&Xw27 z79Ku+F%3!bF*&6emuzoCZBJ@K#AOklXgm~?3T*b_Jg_*{I2cNGKDn3!44?W6P9UVx zHoUAN{GQH6l&(V6SmW)O!|*iXYR*JzITOLuLna}*--xrT7Sw~9E{o`1y&?rCe!=pc z(HNWXi+FpO4+~$>%~%T0@T`dn`5K&`&`otaQJFn#%CPY@HAfxS%Y0?Je7z{Yiy*(A z=PL)vcZy%SgBaEOYpB`#NYYc@=BM{JOZ-9NoEfUd8B!nNP$14@IC%SC&d&3g{Uk3+ z=&5_c)rLQh@$(MOFS6$`v!r&E!_AV^Kf+h6=(E> zUbEd5X~y7S_30i-mP=c9(_y zx!IBcqH{C0kLEdbVhz-L1`KF5M0zKuVeCzA1B>f*9X7EDwkzVXGqBfSx~9Fx-_62u z{fpyOA&5j;U~(nbRT#qP!5^8dG0Y*2VYbuNaCy$%Jh*&6uYvmz+_`X9y6`5ryw8@o z7A~hY3*deV7n7<~7-gp6?hO}{#8e$z<~s}S^>E2qPT~K5;NAe2leib**GU|G*u!mD zt@DAOWLSe?Y$;8PoWvLPS`540u=@;SJ7^gz3?tbUd(W^B3_}-oWkj40yimgi8g_(X zc&>cC%qEhCOQ7V}^0Z3UkT#xMM~03!y(K#-5?r z;f5V$7$?^n?_9&UPerk5hS8>oVzeou`7JZ-QNwWJh>Q1}Vc2!#Fzi!uX|a>ZVFk_? z^|(Vx%Q(!iVTPS(SfycO47RlZ;my`{o>`NYv=5jh~2X$5q_$oX3g-}BgbI!s@lky-D3W_)g{Hu za@X*q?W^KVJ5;p8g21DZ(obUxk8pEJ*4W5|S$cnxP-t4gDdie+%E;1UJRu`3eYYSG zYy6FyS{8C@Sxo-cP);o|vkz`QCzEST{KDmEcbPiAh)$6`b^I3^Wf1Puy^7nd*8|4^2_DavJUryQ?cyn;W+mR&!h^*&TL;@BIUdeUSKV( zc`A$UG?agh;E#JFu-%Z)L3O3^QctSIDWj`c;_#2os8a9+OX+@kxBxM`o8t|`J-swn zc9HBk9w#@P*#1Bw?i4!NtFu~_X=41<^o!x^cAl5airj**-#k9Ai(P1@#Jfml=rnXq z^5a}Yd)&;tj{R(@+#M!_!X!xh0232(vo=T4b#jgfGtKmX&ru68?5WoHaFwKz*?^cSQDRA@PGR($fm}I;d?&WZ&!6k_jq?uD{s-{kvHl=>n ziL+{F%&b4@gcGKXZ;&y-rOa{)lh?~$e8B(4Z)o5)%VHlz^{|Z7t1YB&*fO}R19+Lz z^KV1zY0sxmk7iSB9%3n$T*^(HxURBC#0%KTT1>u{giIiu8sGw0zI5@3Rj}1uf7~&B zf5_}dWLepdh&gJCrG6hS>p-2h!{FX-k4T?BqGEhR#q1GJwnuynHz#JbV(<}<7>EdR z=_6JpCQTkcdun~v_^DH8)Qqp6F|(?E=6GpPQozs?@#rTN>Z4c9nsWIhENV|0uxTHD z^{T2S&zL!F{8ahg@MEr5zDz*!5a?b-i~WK2VHt~=jSg96DwODogsUSRd&w0b8H>CU zE=fp|l7w7(YA>M=rW_5!!c^`?g$1SY-ZZQWvOa~IV|yU$3pN*1vVLz;4M<6%9VR`g z2Rf2%b@INXV}0$ESznTddwI13_=2zpvx$e|Hxj>b_>ucS`m&JGAt^*?+ZAbB5ij@x zLbD|W$b)HQA?-bIN!nFLTKb|MKXZzG-}#~*398s;!DB<};^i*fCQqE-=gJ{3(9#>0k@XZ1 zfb*SnR^BRVT9Zu?tPckkRl;K2(4xM#}HUEq-|8=NA;26;Yipm*`SJ%tid+XWbB@0e{M&HNmz*+lze(y zMR-M7?18n((q+k5<+^0)KVlCoi&d_M4n-uspy2pq`=b@5A6|Jqx1IJcdC4b@JTaeR zRCr@$_|qf~r%KAQ4zXg5Low3x*|L^#AR7sYH7L!84U1oxQB$DCY$EHOz7tKscjXqi zb21;i$rCP`G;P`mC(NAm-Pz-(*JJprnoa(HuZuY4xpU5}JohW*{pOsRC9o`ie(mqv zw1EWEK7T>y$MyNCB(-3*MjmZS_(^wMJ4pEF6aYze7Js*w`^Ow+(fx(xh<9bd~# z7Fk;MJl_Ed+>#_>S&Yfbg0Euc*ps03Z~sxpWu<88kRoVuoNH2kY)43n^FzPJqS4Q# zS}^|6r@srntbKm$TW2Hr4>YP(xmLrLbP0TvM6ex0-sIV{CV_7;b+#2%mlc00!dyPH zALn_~XHUFHHDJy_S%*tpeJ9QcyaT&0N9g&9`KRa5&^&x+ukG1i574Gok zz;6r&S}e=7MBJS*d-}vl`g$Z$<`De&4fN{-J6RK|jypy=Jn!q;IEt1XWnY`Ic=HP) zR}U%JF`rvp1_f@N&0Be?hYE753i=J?U4D6nv5d|a7QDOQ!^{sVvt-&<{HOrJo>7mV zFLYqLYxs05#s>_jwlH3G>VL3Zbbkk5XCkCLU(wMazM`~n-(ZP4yo!6_Rw{5S_{z!2 z-GO_7S4qt4+pP|-3~w#ATOD47LAcd|TUys(7v~*zM78u2wZ-(&}&l>8|hh;L%UdHgra9Pd>7rqE?5yBk7Pk_t0)AR7_ zw3NOm%;x#n#m*P?zGoO`RvK@Ju<5NjXBcG^TE?x0{m!sI7)GTRjknIQe;W30!)QHSn(UKy{}&b2cOz)s$$AN^g~87ST!aE$sCda0?#m;YqYHeiEW`sRVU|ZnGq}8u_cy;Mx@z?{a{y*Zr z12BrJYkQUi2oOm^Z$g03TM|NVDU?JKYADi{O|nTAl5E^;ASfydiV7$;?1CbS1w}=` z@*)TbC@3H*f`wNRP!SbTQBnTq+%_{iyU7Ofe*cB+%$#}7bI$E|?%bJ)NIx(Xw;ndW zhgg(FWbD59ZqjWTcnk0L(6oy?2b_&qT8xpXyn93IT(EGKFScRvW)a0riHwk6>4m)V zrUYN}0&v7FAz~z(NXuMO>4wA@x*>r<>;^V0r?oB%;2pgaw-|%knMpIY|Bd4(kYAzr z$FHq!LtgrhZr1~W3qp;yH=rZpLEWIJ^yrpf@zd&-m%vEl1LJ6x1}N0-^040a4q7@I zG?qb7&PbNuNuc`=lC-bj{;9S!dIqFGb-Nh2j1Dex5r z)TR4hcS|KnA0PTWKgkzV)L_XJp(y(Dw81YQr>!zkn7Ul@q6tp-!ymqUqQ+~9?3jM^ zp&og`hyJFnSw`d#QwdRCr20@$WhsA@I>tw1vq{6)wNoLW>yEZeV=0z2tx<@q zvCk+~-gx2ANw#O;m`wkfKI+uIv_#Bl7-iFn;6;hO7*8j%OE!BR28Q}gn|vSL{Wz}7 zDEBb%vaHMdHdLW*VZ4pmeH*&dvXC7c@7u`HYdxIVz|PXDlEG}fjd}D+h=!uT&<&+T zhW0p1FkQ!nuTMho7ZaL4>oyslgjc_*MtDd=8lcK2+1EAXGjTJSisc7`)w&q0qI=UO zfq+P8pjwP$N>MpbEmOyQpt?XEi-GEG5SRz5%ONojR3C(dFG@DmG#XZ}!YOXgRDDbc5Wd^qQVmldI;u?i5MQt>; zZ(}h;1@_7jrew#Jj6ROQ?VeY% zwx~>8)#o%`buL zF80taDA^1pqwy%`yIIL-Jj!MFDA@~2wqD6TP_lzc_LGwRs$}xLYIU?#7^U#|(qiqY zNI7~UEs|EPM5pfBeu0KkmJ?X=q|A!%7?)XNANgDWJh&660!XYl;Pg>+zm=*>TDx%G3<^;|T^J=Wd`&lv*D18J4b&XkTUrl>dm52mz~t2# z7tI|*gwO-?YxOYV;jf=hVwkqJ5Io&v*Jf(FaPEwjpb}fI>zxhV5e&PD@zI(OsapfD z)2HdINOs=x!$t^EQPDVI91&2O@Z*te>MB+pV?;pfT>C;`T#7-xaw35+Dj#h#CbMC7 zXE^{eP??OzAI;`062gyBn`!q`Y^mHH!It827+Z?pXV{`oW5*@p{1$Oc{R^F^bR@lI z*wS$;aZEi2mNzUIjxF(1uYtam^@m?zpZABfS6j;i7njWzd$sYDqLf{yWRED>N+o+< z$zD=2>LWQH^^x2L;v{77O2|$r*=Z%CGRJkp#a?ZERY1yGC>h0z<7fhz+n@y&Tt*8l zxNN?XEl{%MN_L-;tx~e5lx(w-;f|W%+pT1KmF#mRJEmmx#(VBdxY$EHl#FJ6xPR-F zY=e@eE=dYY?P5#acD|Zw@A&v=gqq()f}s}jPzRg6^~8a=?_crVTXqM$+&j1772J|+ zgt2xg%=STJ(GP!n@z=RQ@ONv;|E>5#rz+;m@tzsnoyDQH7F&wfv)IxY?p>rTFSE4A zXBWAQ#%o-5Na3ioaT$G&2M>T}#zeBuTG1>}$BZRcN2V?LC8NvLSiAaG%mRSZw*6f# z>Eio%T4(|8Eo!9$@%G*u`q1-1W$8;kP22W$HGI=4Bm4d3_z2YvwNsb;)pKjwl0CmZ zpB8eEJ;8hmb4SEOp9h*rcPwwi0UictH%Bw!G5huv9*%y9#sblJS}AGC-gN9XLHC`6 zo0Ra7-Q0}C(?-ScIbZbY>C3uy#^50R_O3B$1AdHH_7QlEYhhED3=d1sZn1XgfN5(( zBbF_~X_|8>OIva@qSa8EVHw^x;_dLsOKM=^Xh+Cy97p3}t8m-|{r!f%8QJ^Ovv;K} z`8O?lS1O)?$Ai?9)968XaYHsOWM{<2+K43G9kqp}g=|elKVA9*S;Z%rM5I$MZs+kP zALcJhU(z*3oJqR4i=9cmxRbnJItqg!OeYn~6hdm{QzC~6e8_GFPrA5+k|Ax$kHZH1 zH)82|vAh*4VkyD!6D>cc)G=vB(lq-FRZYG?dFGYI=)xSWr@2m{VB6 zLRS8a293AlK%{9evk$b3$8A_2HxlP~NM?P~Nc2h6qhP6OtbEGH&0uqEoAEIh>h5WL)Pc#i*y7O4XtptU2)UEC*w&rtWFYc(cD}kT zQahTj6UOu(KH#-LR&94=?D~O0k-i?9q4z z?jSrPFw$NJTSwr?2O8sSB?z*HWV#RfV`5Di#|(z2G1f7L--7wJ8F(I0zc=_g`ik(- z2H~MK!)t~cPf#Z!PK2V{;-3hwXV82A)h9f3!7(~#l&>Ip+nZV%W!umj_{!R$9{S4C zgiP9#TN=f6WHGS(U=%k4eNQ8t*bJ8`KIFRWhx)@ZS>J^}>XY{2k1Th>AN5aN@JCj! z!r!&<{5t&2$KU<hgFtroawhD1@i*GW->A06YSphoAZX(Kt+F-%Y_7=!J<*iOLqBDPc?Nlre6U_`$a+fZze zVOt$rD#A6ejl{Mlwp1K3oLkUR$djXItiO^CS27x+a$j|Fds)f0DcMdXJD_AAE7`Y7 z_MMW^T4C-ptrg}z(^_FJqqV|Zb`IkJE;}#wYHKzWw$Nl`BSL4sVZZgjPH&6F^f}uL z!`=wCXVegiOaqH1ga+RujSPzh)e{+LT{gGD_ef(rlQqSA`LE2u#X&+Xt@gMY)MBs@ ziC?;O2w`Ivx&@5LAsk`9{D<^&9+zeaEwH-Sl zm=fS0GzdYvDEFvs<2+6AN{!m_HO4@|0-v(?fJ<__?V#iqaS}F~`M;Z&5 ztabHjT6O9m5Tpj3h^UDoR>JM<&?o6?^3~w zO{IJ$-j$LVTYud5T!PQUxuyc%2N`dm5)jSKQp)f>O8dvThM^rXUXQm?8hnp5p5hvK zQZy_pbU!a=^5yQ_J(^(WWrpvO)^kkOdSbXX5e-?w@# z;FZW@#7$!o8Ddn-`5wjmE$&Ibx>~<;VOp<4wX{w`CtVvjQs>Lr7h{%pQuioN8e*z11rJ&HZ&CC>E~@b2cZ$E;72dm8MADr;G3 zOQD^LJ>R3)f51IyRaa|;d~c3?Z;X7e@2#mt;4Ou9GHW6BjeV|9vFCdfd(K3y z<{5*R{%jsH#hx@MQ0#TT_#Rp2yu|ra1-x`CCq%O~uBJ6U8@Uy#xCGHddlh@WN3lP~ zJ?UIm>wGR;Yk&L-tqs0{-qMS;200B8`v!oKi+>CudP*e+z4o)HTk(wal@G$KS`=#+mWHj8sDQ72*n@=DbNyi z{&*9u-XXlCV@ETsc4l+reKgLqo4ssHC6((@I`F+{>~RLNbAYHt@Mo(ww*wZT$kms#Y(h>UxX!FA}!w_>m_L17uE^{491HQ*{ zo@o;b*Rm@3vqLwH^X%%;fmn zq#CweRLuDv#k@cFs8g8M>0C9fDc;vmH!~D5=l#3XrkHabiaFn-m~(AH(Ur3d{_G^0 zVlH_p=DII@k7CYwiIZj)WbnsJua-uh*9_5WJ{dAbj&24?6>PhzSo1xKHNE-*`Oqs| z>vgV%*8O-*t@WWWt;LR7TBNtO7KVPFf@GnXa@}aP7lS>{La6ItLk55L#+hO-c`5d~ zPkfJJ&v}XS<_dTxGv25+webb2E4BJRM;(jUf5p~1#kFVF8WSd3eBM)svs9p%@wDIlTqq>;xosEo{Fd-Cdycxy5XW!NL> zacpJy8tVL-i|w6uts_icQSyeq6QK8J-%GCp%ueCAo=6UHoD2MvnP zNuf!bZ&ZBvo~=%GtP~<(KXY-=mVrd5JT>0^ZvgZxnjb zMz&U3#Lrh8tFyn>)|$mNW>y~)8lYs}$8^ZsHZ|#&hn^~R_#UMWEu|^HWF8aRGAHq| z3TRL&J;>t;H-nhCPx-lZpu6s+{C|m5a}66(9QSh77$_eE1&4=dnu1=V=um z`aHBgPeSpj%`B|K9>u4vAwDNmeE43|))?30!&YJYYitcI4w)(2&)c9&WeWoca8p>! ze)&&t6$8FUF`$J^C`avTXzk9`*CLNM(84nLXijQI0>$yf%=aiixF(^#0k7Z#^9zHg9;FB8B~D`->5unw#@n=QO)NEPpw<34>{yLry%l>9T;{s+DkL)WQ)$BYC{4cR zen!^NA`e}uh3{ygh34?GAvGzDIi8sL9;FZ0Bowu0GWfG`irWwn$xCUW`@#1pEjTZ6 zZmfX!4CA%oxtDtPV4OL#&KNS0Mn59-S25;$6k~c@21)^4Q=O}>HN}->16)~#dufba zi|I)QiY3>hQo#43vBwz*MGIzR@Mr5UZc9KUFU4H(EXNw&{mZ0eotnV*=|eB8$OPoMr}{S98% z{u)hQm{#(OJpO%ggZsaGYt*SPqkqU-c-6bZrY|p9e9xN)Q+9vh?ccC^{P1D7b$u)S z;bY6+-?nVVpxhUea_7I7^T~f3o_j3m<)zD;kF5UU`ZwO&zWL5iCt)cf7ix^MM-%-LP}tsF2+L1$W<4*6QP)ANFqabc?T(Th=+* z_zlV5ch|8KpY18V@wYY6`?e-_zECHjW%oaCKK{?I?~T1>=Jpq_s=4dp9nYTY_+H8R zm5uLzZ`Ez@w0Y~U(Qht`lE;U8Our@LkKU`h-kytMlbsP z?cXw6f7neP*UtWP$z7i>>MM`8hE7WT$nn>E^0@S-zn1p8Z}4V$eDp|3a^DXIoR`Np zY`Na?(9f@w{WQAm*JFP!S^d(*x^=H9X!m2Y{Xcsry4QaBeviLmw=cV?`oHh=JLFmO z83+IO{`8Vskj~hqL_~Yw4lRnHi^QimX7uy`} zc&{v%FMogLp(Rt|AC>7^e}B97XBPK}lgBMTnHw?m&)4GR@zhZRU+j1>>`Qs<7+7;> z&wCd2m&Y~t{u))|-mSyr@sya2q16`_%#+8*hcsI}_>N~5$>Z=(zo;>F^t&&~{SBBtM~d-hBk@N3Unt?E6WJN>#(3X1F2|7Y*BA2%AeXGe{1zX`pv z--G*0i@HA+@yUaWHhnkqrEd--{qe<&3G1eheS6sC!+#80^v3ed!^2Me`9^lb3wQ7B z)a#wopPZhX_NF)dx4K`%9Gd^_vtRtM{+iRr2RIHqGF+CYN&9{{xAyuEZk>Glns0LI z=B#+_w%`9;zGd>cveJl)g+G6Hwqr!I){&oNK5=yO>V9uM^3{VMEvSCQwzk>TLYx0~ zB)`YUYoD>5+IHQm#T$Q$8GX~@25BGKhdG#8m-}37{Z+*P4^XyxGYxh{MQ_i|c z%bHb7sUeT&{&LUJuo>+Wt{u@S>g-Ql7Oc-$`fpyTt?AIL0U!Q1WlP~ty}JJW-Tb%9 zR(#g(X>pO{uB zYs!y9!w<(ba{ShESJbPSPc-YC-|e=%pF95g_;0fYgn#(MpoqQlcx{aVQ{P(XeqSEj z{vPb*tmDC1B3xg6Asq<^QZZXCME63-WO5ycEQ;@+iXaj zwEy09C)e)klW@kf>SXk!tLC&g)qQcj%%|^Lc}>0B7L0n+bLj1oVVhD{esjx<2efw| zcy(}OonQa+Zsd{7dqUqH6nXc?=Mz1@{dWJfgLk^S9!=d=chQ$2pS_&b?4ie=Xftz) zeU)q17fGWpRB!8YPJZQdi|?Ng$z1c@4PUq4^z+y!$4(1*>Y?SX7iPWo>5P@Fjy?Ll zCw}ko)=mHXxM|HMi&qXfw6N%{_7RIR4t%t=`_Om3ifJ~s-3KFT=XZMXnH3FwZ1>gq zu(5p)KJfj}w%&IRWv}kIBy+%)@gJ{k96No=yZxSx>3{mOU)qhh)l*u%<e`w~2aDTCoUO;v)z#*VoO{cCtyk6W^vSalckYE7pKjE7>`%|NU3zb; z?ETa0ZQWgPF|pr!-~Rg2Z;LakeS2q%Gq-*}C^NBM%!m)Jf4SD;)`JI4-1p@TANO0i zzq&SePHsEP{~{L;sw{UpS{&U;#Te$d41Na13&b|Y0H!E zNjiTq|CO)SOVO8$35R|f8eVX zL!RkZ_tRl7PpNz#qxez`{c}G(&7u8Yyi2B|Y29(->z$Y{&2` z@76T3@X#bsh^;$*VNSd1^@}U4JT!S2Vv8}9pC%TbWhNjtxYw?@w_FMgF z$1wC&Sb38v#=_H{VQ6}qcs`pF_kxwD1H-gHAuPilh4vPnj*y1fWc@q0?j(|uQp%Xl zofwANXIY**T6j7K!PCXU)7iq)m0@K0dhES!H(UMb#%&q=>1yGj8llH4_K(}XwDR<@ z_|x6O6B`6ioP{UW!V}LhGG1$A2Jf}{)05jW#4FyyLoIWNO-{gmJN%jFtvtOLrT}Gw zQh`E(g$K>LfoJQ+6SrD<`Y?=i0u$aC|4*9H1P1cG8objTQq*DHeCNi(R?F@k0plS zOo(l&z-+p%@yk|Qi3}5oyr34J!VrTk&XWXbh>iTE@XbGWqA~WZgUL7(Vv97CjU)?C z3d5AoGt|P9V&NGU1kZ2_&oB$mh#+`UEj%MEJZVAjTxH=&v+#@zf+yX=Gt$D7!7wzs zqj0|EhKH>AH40}!Y^jF)%CPW^W*FI~HhpPIiIryz!wfX=jJEKMHStU;TeaEBGmc^U z8hFN9cxa%lm%~REPV>`7Wireu{K`5w-oi71VQAz=VdLV>X!d-|^F*BCV`Su&vT6rcj%xU~eo=FxSHo8{zEC01-80h)>Lp^PX%^0t%Ej(1y_4JABX1m+U zGtI&?)xtBKVW`JMc4#dgTz$WlX9mNp2SD~2(=9x9hN1dSJTI=<(ay>T6oZ%0~3W`(!S-!HQTK`D;VYwer3Gwx9~i`Fs#MHo;H0O1`fXdJjgIR zAg)Ikga<4<7@oqPaCR0Cgq6%i=itlpFvHL_zVzoI3(q4ABOj=k-gzb(bxJAYT*0FZ zlaIl14)!QKV&QoV(hw}d=3)NXh$dE^#~H@hUp!{vS;;Ve3!cwk*n$DIuRp68CK`=P zZ-HBB;dz2#0Aa!ZIP*!4F{IBE7M|4%lZ;v>IKw3ZvC-mY@;ULVA)T!}uQJS1_)KUDuNZiE z`fPwyPoKhJk2Nx4xUJV1Mn3knvux>6tF4U;GZTfrFZL+BX0XM1HbIK(9h6ln#beH= zce3(qW|;XX8>!f%u*t%MA_X4#2x7rXwg$C_X7GR%BL zYAE(7?6mOgf;7ZN*Rc4nRh#}E(6ndlFLpCbI?fEn9)(>N9+(GD4yhvCQPdV$>&x>V z&V<+!F(^UBVquSk=Y58e{owX{-58krmY=;06AF5`#KQX)o_!1>AGeC1cNhZ@Uw=Me z7<$wFKu)9WV?1;qOQxnpGyD9#yc{MaZG{%c6nOBdnOrH7nBhXOUGcbtXl$uc1suO#$P4~46hgced$n5raq3CScS zp+~$aOi+^FgrvWc)JINJSgjlI#|eAxiRxkUXIzT}4y+zLMC5WSf$#5R!+K zWS@|XRT4Qwy;n(+L}krZlKDd7P?DF0+Z*B!39WHA-?% zU9Od(B<~7Io|2@Yx2CXAN!AESiIOBna?EHY@e0XKC3#6mb}7k^Lh`zjw53Z5giT5^ zMMz##l3Ru35hd9rBzu(PPa#>UB!k4o!#*XsNk|@3l1GJPt&)5qBx{tU31(I)Jf|dc zgk-&vY!H%lO7fSGyrLx28gQPDN>U~y@k;W5kaSg&jY1NuBnO40vyz+LXx5+%Y`IONnR0>_DXV2NQNm%BXRLLLP`1x$w(!cA|x%9=_w$(q%S1N_**S8@v#D&ae7r`I_hOVujgVc;slHM{F>lLOa zGJ~#?*C|N?cqr%+!q8ReE0A(#iBS@kgQOKBF>yUbRNG5QKjAV~;K;bkSVT)nf{Z(D znJp!uV#Fje5^Q#aNho>P3k7LF!zD78l_U<4z#0ef0y9>3q(3wiU|OQ2@<;rcB#tx& z;g1G1Tp~eAB3-7Jo&wXoEXU!cVvF!0{;(Q~?c)K|emf)h9Xb-EP%hT*k=IdGug%-n@Y#m@)Mq7N=2x9FWp~WyJTL+mIr2yGlzH{?zUF#2_#V{sY zA2BVXEk2_NvDT#({c3n?J8>H0b%-Qv07tfFt!&m&*P1T07{+AlW2V&vztp;ZyEZA6 zud={_cBjx{7?Z6}a8|cSXf9?Llh$WUs{wv(`1>V2Vv?@)el$b~3}e!w7C8iu&2p{D z#d%^~3Za{{reEG_BjpX{@F=7qwrKp)Wy+5=%i8F+W(zHbF=>4cERPqZ{G2BfC+k{o z2rY&&rTiC6OO~jKS#$o>wHmjSvy{4>Q``Ni6>RPx&Xfce*)>lkR+FJfo zdYG>DoX}zzldWS)3p-l7eOI^R^Ye6cOlTd$zPa@Nmn5q6eltEVP1ow!7AFxH#$@Yj zNX_x$_rwY97FrBriq|(xOU8>@0A1?^p~WyJTgRCem5mVm-Ew4i8(r&^&|(;qt#3`X zmMo|>Ue{_*%{c(~_+ym3e-@31aQA&|(;qtshOcsLbnH^{KN(U>K9FGmz@#ka{A1 zFNx3@LW^Ncwtiw-GG4F6MXlGhZWme%W3u(L(jxZI#77?D^L})+LufIK$<{A8tJklu z_BS4=Yt`&Py8Ii0=9cX&q#-uhV(+^+>vml$O=vNU$=0t-OUCP=Jw1l$TFZnM!hYW{SK*~U!R6N{9iujN$9JE z7Q>iq{lT%G1>YHQoVi+`uOQ2UCSf17{+AlZ>A;V^+u<{H+8Mog%-n@Y+YblVfdvm-r1lL zzhA%(JF{PUJ+@Cf{|`x2JO6m+887KtgM}8um~8#av}787QyPA&u63QzVi;4rE}Cq8 z`%J(7y4D*)i(yQ*XuzPSp=WC?L%jYJS`1^dWn)@04M+D}_o!~GZx=X%z%V9T)tHt{ z!xx(Oyq({>A@p3K#V{sYAtqb#eQ%hfYi$%-3}doIGwLBWnT9(*K3}A3ofBFNW3pA< zWNTIP3A1&r1R_IV7?Z6UOiR|~FAsM5P}lMbErv1Ks%f(I>thWU=~^3v7Q>iqg()ra z+r#d>SO?6xzJ)5ldW*3CDU;A$KlI#t(`)PVNABJFxm24{X}0~>o1|j zFeY1dl`Zn=)_R{8^ECr>)T2B7VihD|OtvCSw!Zr78@sMmCbSsFWGj+s(ZGVz@TG6s zZP&H73N3~)*`n`e>Fq|n|2ChaYt`;S{_$^Xne(e2(~@b}GjhWTT`N;)F^tJpeUq&c z`=(vG*5g8pVNCI&JA-)y)MwN?l%hB3v9R>bM;;h>A-+UQybgcie?Y&A96 z+PCAr0$nSdCKM1D#$>CRvPI_;pWV=buN9%A5kiY$OtzYvY`xI;&>Oney+Vs&OtxsH zR){Sc%v26Tm)*Tv*ZNXuF^tL9l_pz{C*HYC*Xj}vA_Rsp*=nicMYa+TE)3DN%7hlf zm~6E&*>cr=GhEl&DYO{IWGhr#uTr%CR=}AQ}=FN>vy5WFeY0uDqdu3q4)NFx>ia8 zLD!$;4*XrA!bn!Al!kBFJ zWm?fVM`?KSwSL#?TGtCLhB4XdXR>v2)Z$LM)()Y?FeY35nU<_yUsr$mdR?o|0GLN$ z7?Z65CR@di?yse5jT2f7W3n|+#f!>eVxxVX*QxsTkkDcnldVA}TWhACyFu4FDYO{I zRH6nmEm@*IntI`LT`OTAiuWKCUAWEhjyFs3C_qsg}1mb%tap~WyJt>H|o1xT@@-S_?P$^7|cvNB=_ z{c0M+m{MZ|(~_yNed)hf>sogUErv1KN>#R~etmUnaBE%bh|ppfldUw9t>1tBXPU0n zE|L7hO92UEvUL^HlJ%=c?z5Pnlvh&4iA+gj7?Z7$CR<>dtczsZj+T7N%V`=xP&p;%3xYD4YxeeqfFNtBeWRCWNQ@DlH>WoXR961wH^{$ z3}do2+GOkI)py*ZYaJI_3}do2hH1$(95=FTo37O(na7J^Ot!|FY#ogMrkAdDtsr%< z7Q>iqO)%NIVdIr2bggHF7Q>iqO=Ma!4NqR{x<=RfU1%|k$<`#3t?Rm7HBr|}8Or0u zFeY1*nU<`}T_&_#&)vN&SFeY15l&ui3zdyD4hq_jW zVcbK8G1;1GvgPRdMPpsdE3_EKWNRAJl4-bhK==RZTHAyc!1e9ZVi=PxyV9c6c=xk|y>+b@g%-n@Y|Ug^vK+qf%d$Il zEo}r@)HH@M*>ad{eK~OMDqSm6Xfce*RuCjPh zI{j)I!RRiB7Q>iqc`Wg&tvu|P!EG^&$ySNU7R>|bS}TMW!qOm7}E%L4%4d7Pisdyp8SB1n@KQQXc30S(s;Lz zkkEcL&7ND}EiG~8&=G}dTKDYy*(GQjDa>Weo9nOx!_RVX3>gxicy-d4VM&wiBNM0K zuqO^Dt7E&XXpW=6m18eJ zT!k)gN#gLt5^qjo;`pTDlfhc(D7z$x{VGXgzlwP5qdmp>jv~9Gpditn;x6(!Tt#px z%TeNt<;bzal9DDf=;Q)tQJy#7?(}%v9)6O=EO{u!)1FjN;Ldh$7^&fm@qMOR*tGe_A-8paY78Raak5b$PW4xZk6uc$Q<19gDjz^l8I3X@AEK1DH%5xSLCMHt+Ks<3s z3^$&bie2G3qM6Fc`7b5IIQQJNoU%%@4mYui%+y&YQdCMR&6;9jP4V@0RB7SpTsRyA zaWayYqQ_K_M3qn(66+_yC|MJUFvls20Sc@oU0LbEPb$yYil8Vu0*MM>FL7P#1gUR% zb9-cgGo_9xC&weYC@DUG_(WEc4Rr$S6Y&B;P%_sNCvX9)Aa?J(V&qvl<%Q=5izBN+ zBb;7G0eM^{oV?T;lQf>CxVoo)T|WCIXdmmcmub&0h@D{%3AiA+@j9&Z3fon2xD&ZOxw~9wHteE-h0^?X0I3zA;5=suy3j}9Ywr-;L4n|21 zP7IxXWz@1A#g1&3cOKe}%BT%V3J%Z4k78J2pOC``zd_kT)uVxJsy?j2Sf<45aR=ur zsneB3DjV+FlS_kFvyb8fE_bwE%tOcB=(7?K3tQI|8Q)dy6()hf4M$lL~K!TF747*O8sw zGv4lT=Bn#kT%xC@qJ>GOR%RG>Q&VGOW9_5pN~cn)pvYjlv8i0U(BUmCEr3y1K(pNP zFsPc4#h+Skv41={7Yh>^WFk8Y<4=1DUElknCndnX$5m!`7u&sw zafz`omBC^bN86+f=A*|w*Y3{E#nY1|<)-4zo#VBV1(}pJs z%a&ATwWPQ($Ko(ch*WCr^zn~r9wRQ%K8n2H9N7h~VtZWgESJ|%0QjjKXR$YbuB*gp z?^)IpqsSb4oNww-!Xx;medRPsJtYIbNtA)9Jt;?g7%;<>jajE6XMtUmU5D4@E(*pe z#tvR?ATG|rUP0U{h+RTQE+vl2Ie)Pn5<`=IVix0Sx-3VrNalS%&SB0_PVTXj z_N#E#eia)UaMkioLtD%YGG)>=ss=mVg<-`A4uZvp+;n$zejonEZ^d(9igC{fw73 zjDtdxp9*2}!Wh?HlEZ}H>2(RIsW-hM%{fPf80 zpqzOgM*%OH7+zwOK|@=aZmM_%ymwPUiIX{m(rzqm?BbV~p}0#dM8SAJg+^z*q_TE_ z6&hZ#Fu3M@D(+=v=f}h*VnSs|6-=rmCZr}|AcR{9g41VARKa+RjR;Jms)`6NWdK4Q zFb&%9hql!)$SU91C61aZCcTt6lrxB82JI9@e?(yyrJ-4(Hyn-}Q1+`aMuM)28;$rq@g_x=+ zG+IoF7q{IfuyT{(D6S$>uV-G>k>O&GsudB&R*`?M;&>MCDx#BR<$uxY%aZ#);g*h9 zZpvP|R7ovq>@ughrS;3DQSMyA)VK2CQgktU;VPn{d?}Oe!t$kRbK7y3W&2XKx$XGN zvVE!A+;-2)vVE!As-RW+c8MxxmDf~8#lovf+;CA=FF{qBt<>Z&n>Up6%F_B}F!0Ro z6*J^AIQU5D@;JC3y{o{F%j3ADOkpE@s^UO<85jrKzjzxHNT@4VR{_lK;}w{U>{{_m=#pd(hee z5 znRJ(X!&0W_W%KZ;C&XO#EmYxARWPZHzwFql(s0>KGUFK9c~zObRd82bCU{j)@x;J2L{*US?D#%n;Rt1^L%Bmn!HNPszWUZ{+jSCF5<=P*aftOCe69@hIrBkS8>C!2<7wGFR zoq~IDSrnI+9oG9uYHGW3vGGTw&ud>km+z$Zc=jRM)hEqM|0`f z_EnjeKHv2~?^_x_m0`6)tBkpNe}`B~*I$u_fAn&@|+cCdgVDQ z&h*N2R-EaT=hQR(|KeE@y%qRBv1W)|l6+y*B|o8J@VyGi46X(D*o(n@kO~G96+8y? zzkiCu;Qjx9UknK&bF#{wEHOm1>bVV$2k}se!IdD;Lx$K>678{gzuM^ZJ$R7aaHGzYg=8rELC*B6;D{>Se5FfoJ zu*XB1mW@9O6kmUQUYwW<%ny$+m{t!PfBw;X2v@p3uW9@7M*X0yMX!Q z6OId%emOY(_NSWG_Ati3f{x0>zi= zZ!$2=PHm0{L4G7X-n}(p*;Uq0Q1g|92dwxDu4CPpq}H8LLk1=II#ejf}c6Ay!0#nMbqxb zAB8}C)c&mmX8T!=3*;Z=Z~b5KIc5A&2*gMAV;(Tmf8)47{`JRc$L|~?fBm&rFM#h6 z;6|M1sM^^0YybK{{=C4H$M+Waz6S2`zc?y90AFv&8~rVr_|~6)>%o@}+=UAq6NoPf zr@j9K$M+g=&WjuyD89tEM_?qYKmRtszuL&HEjH{R1mYWrb8V_|3$jgwJPQ#nhC&7 zug$T6>IYqKtrHlsOTl0I5#L+D?Wj{eUtQ=o2U(?d2<+S3s5FGpCI|aKp12d})$2AGS*AVh4ZEg5GJ;%|n zKR#+!<^fY0!*PMiAC>c+z zwgblo8sC%O4|TL@PvDON{rcsD{CQy9oj5L#fAt9m%oc$Q#CHm~-N3BxTs|LNKc5EX z7l8}Jw->m-f!Woid_IbANLQOy2Y(a-@jZoe4S{)G;AmXpFFz>0{{`k;1$;Yz`xlsZ zy7~CWUZ;SQBDBEvUxAUb#K_ph?rgqZN-%eY-DcfwT6_FapmNV}K72GT916^2Uz{Jl zIl$cPi}T?l|5gF>mM_i^-w|L=_~Lx{$iL7YHmwQ%D5!Y&;p+`dvM*T;{5pc zBrq@f;(YihzI%cB+!yDE?=N6##QDU-hmZ0%Mqntu6atm!zQ7F)f-fsbzB$0%90cDg zU^e*T{KWSlFkkxOeBw*#cLA92c%O9f!`B&@-o7{=K5DNr1%}d&LZI}U23&pxe3T!z z2Fdp@a8C!pw*#2NzBoVe{T-NUJ$=%}C%%+^(ZIy|;{5Q90%nRY&WDfEuS{UD%R->^ zTL9eB3iv4fo(qz13vllR!FLLn3%)o%@okoX_8xx}R6FStUrN8Bz-0R3{P1~zxxp9b z!$;}&sKAh23W3t^Dd5&uz(?u#L6Cf319zqZKJqWT7f%!NjY1&*8UPn9uz}K#{2Lr3 z-#FlA1i?2SnB~4WKl$+@FmL$ceDZ_x_b@Q0d~tsGYV@{g_3%d_P<$!>5KD|eiwiV@8gp$e)u{A)7uy4!$;|t zDKHhK-!$OzE8wH_yERC@hk<*#0zUF@N05Abfje9QANlutkbJfKVm<gX+g}U{?9!{P1l7<~?7W4z!do6{KWSTU{?6zeBw*#w*i7d} z6TW;NddsSw_%yE-I@nKs_`!t*fn)8x1(%rE-G`5G4T8W?d)2~%8bH;($j`#q>ax=%1lY^-^QLXKE*z9*tkJ){bFO=_&J}JIc!`~=IC+u zp?Kf_2z&aliNn$d`9qE?#pg>3o$0PDkHa%>P)4SGJl<&P${yz|aNW+%(RgRY$+L; zKJ25@hmK0h7&hKVJTYZ-#+WwIl+Sd_F&|mIRI^1Pb2zgSx;b+DW_Rn^uV+rTtlX^L-Fo%T>Y3XsJE6BTH+EVUVW*Y# zPUt4$%PEGt3Y;a=_*Y8pc}{$l)s@{n$K{z;;>n&imcIBh!Rsn0>7Je4-JLZ{>#q%r zvdzQ_39Y^ssjYC?)HcJF?Qxg5bG^|W$90MxjW-?_x~_E< zecX$0qhytOoh7&v9ut-CETE?(4bmErI=ls1Eo`}K8j0|-Uac{<_Eda#C2457y=1PV z*q(#0UAhWTd}5+J_|k{FxP*bdo;a-zWCbpK{MJ!W!p!5n-*5@vx)m}q09LH}W9hv( zQ`0(VHKS6gAV=av|Ni!5SDyUBEPXq~n~#sY=DQ1W6fj!z3uGUcEp!%nwHAW0g8+^K z4L8N z+8tS>#vGuNHq%M^S}mPz1=X>-s>fYglwAm;t^DAr?c1V12qfxDzcYgaB3-!P`{8qCw$mkYb5)P=7vjdG0A zI+u$p!54s$Qz#RJrI`44(p6&5qJ+Vxnp>9#a+W!=y;_^{ATDQiuU`Gix5eIzk1A6x zQK37hw7^|HhT;dGsZa-uSV^%nTYM*?oNPkrn^%PI8{4y-_$q>vl?wlqcq|2VFa{#*hLhc zfIwpbq7qB7sAj0=x|cjAoq#d4M=U0}1#X9oi=o-@XWJL5nuM-?9Q6h34MS`YXmj&G zep8V*CWMjSkO#84D+emtfgq5g5_hZEyfG)J;iC!|$OU6Hq{`}>YsQ%IeDiU@{N?z8 zc<#wFxPN~nf{(wVZU=;mc8vTbA8PWF;~y?s1x+tl)YB7;ib3GQYEcy4 zQTC9QfA0cl!A&YYfs>3%8(!&Zu1%E6fgLoc2-m2-26oMYhf_W+zfS>0|6^*Hs2wN% zR(^AlqlGIn!)sLtg%?x^11F72Qss^)@bSVOQNewj_fZpm$vpS(mtP$?SbhUek9V2U zDH@J-ZL0SmY3`D&f!zC&*{Uh!Si!LW*%knT?BjHaaGdXibbD)TFstgrQ+SVMJ^1iz#*^&dMRx3>y*yMEFu2 z8^IAzA}=gPo&sksztW=f3#{io^nE+!tY+iafQA^s1*Zp>D%zF8S*@A{ljF?RPlsAg zyK!Zrw8Jc?^POeEw%K|~5k;8B4-_+b(QI(MB-=%ouDPykO}6Y66m(NjU{?r?I@;oK zuB!~!HnQpWU#?VDFtMV5(Lj~)##(rh5^V6J3WkXo; z6TVVb!N}TfN2}F4!PKWv14Q=xB(1Y3hxJg@H4owL`1FN3p~!eS@U9mW9Qoc4=7Z>> zbwm0vOAeSsw)nWF+8JF{_}CXFDN6PI`WR3T&dR|@6Th0Qvg?-#ap``XL&>$J4+67)SPO%*AK2PX-TUElOAZ8`pJ?+m!<^GR7&r=*sAg z8i%J+L~>U~Z`|lf_OwyB#h8w7AP-aJt4U=zGE$P2Vq#?!M;R0~TP0FyBoq@xbi)E! z%*t~X7UB+7|rJs zx!A#9;p6sjVKmBtJ;Pa;U5xK=yXVjcoO4T@UTvnZ5zUKQ)!Hb*m!h>y8H)uMnZK89 zqKK{=XG>N;ev7$AYx2lRNZ%Ba@<55?DkqVu&KOIr=`vpaQtPs8v1F5z)^eOZz7XZa zj=R)&cMQJFUT81&dP=k+>4cxel~YO9)Rkuj`zEhC5h#5N%1K|TTqX8c63Zg5`Ag!< zHOEq!Mm;oQ7VWpSv!E+sLkA)8;))>vK2CKg73B|I>G@ADb`9t+f7ppj&7Uf>7hL{a zjyY2;kXLBw?rgM5PBBl1Z>tp0<)ZIxIUkgFUa?c%N0pbgW=tV&k9z5f(YM-}VPf>D z?%T?%Tr-N+;V65exwe}jg}B%3$ios5nl7MWg198~fuYJPTXKkDCMtsCYRDrlJFTVC zGUSO@CDi9ehCPiIkKhCA?9OW}x}ph1b8*YtJ~!Wqg&jq6u!IJ4FgY5H(P#uDk{}41 zalMsMpSuBOqjB+ptqwC=Wy?^UIq(asR|dxM>S7@m}p#saK5kdJzcmUbSvqCS~z3pBTkwzcm{8fki{~^T~tz9=(OiM9XVJcLATB6 zw46|2QA32}`QgE&2d2Kfm{%z*CqXiTiBw=7-&hR|#;x)5O1#eUoEjO7Q-N4~U239T z5mnrkT6zlE#ih8h?zX#19IU3Y3kH6LnN7`wyx;3};3AnXWooE}MNR00*B+%fyy$x% z&!l#x4a9%pMzp=eZXcJKnU;*^DW$+sQZm$8lI?L7qbi|thea(#U7_`19q#pod#Z_h39Mt2^J>674f1y=6WZ%RKDwdwYt!xrGTi5bc>B0(-X}&?wufx+v3E zEGq(W^%;j%cwSP}JsuAIqD$BB-(UDaF`peIPbqaux(mrr&ATMk{{0yhlA$v=a{O`7 zT0<=TDazCB^#OKza9Bi7>ca>XhAI9O9eH^kXC9Urqq^#LC`ZQ!4Oc1I18QUZsqrF* zw>mmq=pS`E6rT~5P*9dSKyj!)MRrHZ5EF7~KoCI1Of5;D3R{w%c*`_rC1#7Qi$=qa98strCSBg`ZVd%$zgD1b|o6#05lY% z@CrSR#r#%S)LwWhCtEU|MI~+zTL|sC7OTbGMPnQuMOL+v^ijF4X`Y96tnHDIr3se6Be}(ua1)(><#hI#_+^D zFn%yAI7Z8ChAghP))aak6ey|ZLf2@=ya>92wm5ad;Osc(HMF$b93$#qit(xt(I>Jd zBcCOiZ}~{_cpURgJuA!)f_hEZ;M06Vow<(E0`%l^>PAlMu+vI9kS}M69?X&?0@s;b z9fLfq>c(nxHps>`Kv+~e#Xa6dHG(Bqi6FpA9z-7RS5WFXF3KzznrC+wQR9|l$GXCN zx-RM(H<%4Le#W4ra^8xBVBGHsi@FjWJ-2{*I~%u4uOuz>(Oj71 zbr)h9EE_9}aZQ1Iq7hW4@@kY=Bh9MWjWTj97EE7^-rYq!u7j~SJ}l}S>Iyr{aQv@9Wq7W~U1-mxJF_&By-r5T|9uvg!_O!d3|our94l^5#F6X7V*A!O*G%eD@s=4y`)s-ek5`06b;5NR%K|>`W#y}%S=PUQE^bVF zS#_W6dy1~@QMXwO_ltCeb75g=4(9U0qI$&0`0H(}z1ZRM3}lv7Q5tAmUnnknCt_w0 z*Sol`U>l~#MGjdf!DQC?&Vfle$}}5WO4B~hsVNWM2D1Wi4i^;615B%XP;KXm0@MKB z>Fw486ji$7px$n#DvzU!&?eIy$Q*~qg>fq$^zr1C07ez7j!Z^l>5-0Nsw6Dxpma0n z`b!lCaioFRH|>Srxp>rvcMYs2(gJ7}AKnBR*E~~!oh75o??kX9cm-XrPf=nua}w|& z&BZya>tRn4vEn8R6Bp6!o9LTlA@R*1DTnv)zblBn($7G7B$pMi1~63C4OU@rm5xcd zu&CSs9LgHYuYaRO6c8z&+kR{&7sZ}3d%tSJGBH<2m#JMKt)+SdHx`@C@yhG)u4JRD zJtK>KFb+LSKEQ);G0GCsUO`PIW!8v@YAVt;Ks8lpUPoKNQl@P4e5c7+!!=uWRo0#} z$2l1H$-<&Omq{0N*)w>-TGx)M8YC5&B?-Aae}fz+@8woVLCFo4E@ARm|ddOgUEJq zea;hn7!>7nn;!{ag zGe%`G&={81K;Xhs^R!awqlh8)>4K=9akw`KA9>lbP*IOBa!98W1i>` z*(ll0o`|H*(d)?5=$d(h#Mx7P`YU$JLUkUjZKO+&2W2%gjR;vsKSrMN8L_ekNj2R8 z8l+f0PPNNqph2IrEZd3cCwCDW8i%PwWna7S+XvCe`v0-_9e`C-Th|j3iUGL^O+e59 z0qKM)O-LX>z)++MNC-)QU=koC6ya&?V8r_D4Y53X!-DOJfC`9;pn@GcqE8VlsQCET zI{VC-nOo+DfPU}${&!$9=dM|M@6+3ynOTV$mnt`NiE(HRf9;UE7wyHllCR5E14U?w9{NMJ*(Q;DkIPW zlpMB4eQnsz?c6*y0qGnP9#vgUu7p(A%HJDP;pmNRhAO+AZI5aw!tIAaINGl7408Az!4szGTRmXcmtSjb6h zX(4uv)JS}*qJ_(34#pI$>Dz2U@5t3X%vH7X`0%}FiWw&%Yb$|f)Hz0o@he=X`*!hU$NGVy#eRg>+D1xm!pO}#kQjSIe5wyQB_rCSul#SG8kKDu>ztx^>f2qM==lr=Tym|grRtXLG@(y zL$7gTbt#^&W%WKK^s-t=wc6m~v%lS0a6tqO>{0uv#&3(A z#eC*ko)IoqH%#R2k2L~Xu5D_UbDeLh5!7R1Kq@KSi#5UBtld4K*m4-V+zwUnSPUUo z8^cuJx84p}v?f=j0#vg~PPbJDwWtyTMfuWJK_C3ztv&N)Uw;fr<>+M_RZ zGtGE6vk#ojEHYC-9HoCI*MD3Ty+8XO%Q zuyf56G<(UBg>{$Rkg$&lcMI8?0oD>*N3>?p9!ctk(7$}yWEmJUp`NirGv9UNSS`K> zOhX){arF*R`a5%Wob<7)9HnoxXVVMR+KYSa6w(fOZL@`{ZQI5x z2hTaorV7-W69}8=N8Xdv=}dnsF8TLsXwX(G7XYze*g?{q-dsOrJZ}An&6fu zp1?tiXy*`KEK!2y!VFYZ=a^@Vr{flVHdi@Ii_R>Xv#>~6pKX_opvK(ve@ryBxU$Dy zXUx&77IqB}S+*F4K*6>ix9L1_r!F9Oe+Co5DJxlpc%KSgW_>ASY%5|6h+Tx#Y(G*WSH8inm4uh#N0s@k{C-1hTXJk{CKCGy7S7GNoww|_?2&APYglNfXS_PeQ9196L%jgNH6s>}P*UWoHbOOc&r7ed`^ zMl5Kmdd0@*3^WO)GO&)FPKZ!0?qOu-j*=MSGBFC6TwuVS` zOooaa<_@)$WT`g9WGU}v{_N?2v!*XD$}ZF!h}3Z0ta~rijRSLeY6RP!`3r+zW_JOI zBkmdMjHBJ6&d&i>5FP=|~dMC+ay zT2+{(RkE8MpP`^)OGl+vAna6vCAD76cRb{&71uBABsr{6h1Rb&96f8-pD5w7a~w@F z2P7Yw*z3+SLUbH4dl1f^@)J|aok6#nDcx$~Ghv`BlWi?3FT0q+r28S$RjJuUNYUXK zb*o)ABJ9~z^}mljj*$jjo<+wxsU4>LqPZA3d+Pl}=>DhOJccg_(Hc!%Dx8E!YRfq} ztK?O7#_Jwerqe07f;qMHHM(Jwm2Ty2YmyEhHP*8+70HmA0x*Mch3O;#KBXJ%oO93X83sk?Zwt!|u z+5k#x*tx*Xj6GqEKhcU=e6$BYYvSi)YGXsT?0YN`DhDBo%Yfo(<2m>l0GB_peO2~J z4hJyvFm@0{*)_!toNKAhzT2g?bdDt=x2h2aZVUVbOZ>(E=p6))986 zYJ{LlCA=H~K$~BP2feu+8e;&yg^~SHI@Rj!WdqQqsol6L8&zWB%gR13mz%G0IT7sDOh+!F_UM-1 zwTsI5$y6|Ec)AKqlFnHc66QpIjLp=(DQ;F)qr=9o$f<~BGDm4!Ou9KMZ*?kogG27M@A*bbe`7qW8H`v=iv`wl(ZvAT3uw^meAC@T>Yg0*F; z%3wSx!W?EMFk7;+IGt7-i&^^?A#nx|%^ai_-%^>xvQs;3vDCeuU7}^vGwV#D+#p8Y zv3(dpyQ78=9-(p){j#dFYM`%*sVX^PTN_ml>#I}Si_1{T7)Nnv8184l}XKFU11q2aMc}Fr7ZoMnwX4P z`E&UC8cu-E;LmV7#VzYc*gO!Y9-OrjV$P~_-MV%28jst0`|KxowVB+*vc}+gLd=S1 zU)t6=ee}J*U%aRGlBw;lu&ieke)ZKSZyRvaGk^T}M*lBI^gKNY-`tHMe}4C8+m?6h z{^`3H9Q=Dit3#Jt)(VB+lU2~>k+)tr`M2U9Kl$SN`?DdF9h2O3N7AzM zR;9=EL?Ng`J==bGdibpFuZ(E5`io~Tf9Tlr@g9GLw=X>_YkSt@!ePD7c&h6;nHcbF zRe0Bqx99(RklL&!t~1_3SUMzV__1woUo>j(xRaNAAQob!RL5;4gcx zX>#(%-&}p!bUZaz@o>RY zzcu@A^6FZj&wbmno>KUnvqq0gy(Q(|)-6k(+J4>H7)({eN6HdnE_-)W-4WNFwfmAe zb$fQW@0lI=Ca1zby6pT5(_U&^xAnaz-nC-HQs{VE;R#*;aY~P(YB@XCKmGOH_doT3 zW!1z^$%L3I4y;|9JGjXwy}zIIOyIan^Rb;*;eX71@0Fi!y>3lb&o|oKo1Oj~-g&I> zN#FnS{Ls(e>Rf%!_gA#$mfRRb4N>;?8@|8Q*Btr6^bN)N|W6 z51cZ$dxym5pLk=>xL-2YqFgDw)2`XS?0Ms{E-xR7J>`WBX&q6f6+UFw=$9TU+k?-h zJ~+K@n-wiAE4L=~)NFI@*xz41G4|^*{ja+A*=?)wt#XCm^T(1&r#G+t(;a#5pEUc# zvRm+->ssV@omE)UasT|h3p!tUS>w@N3NV0B_?KBHy)y5NQ8muG^R%nWzdDeM?=UO8 zbFU7M4$rB(vtipE^X{v*57jU2Sn9c`N$J!E_ndTN``cHx|Hnt)TyI%dD!iaUpZl(Q zdt<|^9voZt<}q8ov#c8U$YMgwqG<=4-B|wC-KWgF{eyzi*e~(DYK7l$|8a}Qw|R4V zVf_sSJvTq|n`Pah@VHan_+{Vlp2he6GND?{eFIxzI$qITK#m-xA47S zg|FDs_ng+>7d-g-U6~^rp1X0oWwl1!5@O<}?l`!=;m8Tkl-}`G>&s^0d%QO&Jay@y zhGPl_54+-viC-R5ShUNsUR8LvM`yg={MU|cmVbW1>u0RXnQ2*F(AXu!^nB>v(ucP0 zs9x`ue?Qb|+tY~eEQL>4F?M71W}SDxa$U>jb7%HNIp>;QLd>TniOuV_J?D{6$CYJ0 zIAP8(d{+X)xrCU}agCn*^~_#fZ=SQYX2a?;P#;Sa-l^KlyLJrt<(Hho4QF4_yC2Ho z&!`&6Hs!blVjAlIO0*?kp-hiL+l0Qu+6_NcSo3gvt^K>;Pe!|QJ?d^PkXmYgH~7c7 z@Ch#ee#m^p;buNv*Lfzs(d3fph09oS%dpyz{4YPMMNllv7Ze zn+I>ll#ZQGKDkHNjww@0(P2-?!K-oEabqC^?|texE0RWR`_*sDUXVY(G`k>aen}|= zrgTZ^eDV}*J;*D@I}+ySO<9mLr3BB{&Yw~^AKuR0I(F{SvCGLRUAmeu

IOH9JDU zkDi%DB{OO0uZYp28ES{;145_!(yMC1gjxI#Y1J|_p7a(hp~o1EsD z_p?JqRt{cPUo@p;Rz6l4 zYOmIxUZR*bc$hm<+8n(~QkYkWmofbLc92K1*74_?T7+j-kw7VbNr~`p%nqjvbE87! zAXi+I-4{sCM%H-r%8*&a`t$Qvt-k7=c@ZjMB~t1ynrk@YDU)Kzo(gx;jQjk*B0t&q zs1@!bPho~Ra*z){tMonlqoGo*8y^itt_sJO#Q9`V#p8>)&J^C5h#bD7SACsgiA^Ig zSm6RGgU7!jh-$ZW#RKM={{02371;%NpHY6vtip;1P*N_x=lr>axrKj40?G~WKfirm zB)v`wmbZSuf#qO)A1yQSjrNvxb}QU3z?xc3>pbm?$7_}>>nf{${dco+zWU(pCBvR* z*zUGnmuJ5W+F$++x&=9%yQOrTn>*dw_d~m3aWeu%F9s&B-5)r;%6bNq(b{%MbLk47IF&i6!9R4QhB2pthB!?mUZ7EN)HpOMI;@ zug+83Xyz-%saSy&7r>32I#@dm>rxXTpP0xrr-hL*v``NS1*mmojIFhbHIk9ohl5#V zJI199k;UpKcE^hw4TwIh7MC%OV{t}qTHFR!Q%TboZ)F=SPy6Dn^9^>f!FCz!bAx4^ z|839o@?kah2NLR(kB)meJvk|6H?CS4=Wpn_o{O(dU?UFyBp=?#d0j@>C% ztI&MVFV(;Xb=SsU+!*Yamzqji%EQT- z4Ezd{Gr{@7xRoLD?tH;2Jt1od&WtluseGx9@JPPU7jI!z-4g6f?TfdT8*GKa_8RO* zgQbo^&Lr&*OkI~&{w^xS=D_4l`vVhSRC$vgcrrbCRQdM(8G-g2kx@yLzlI?0Sy~pC zxZ_mWILVW5C!k&j=UX>$tKmTTRLO`FoT)#DpKtNlx@wX{{ zo2*weEmp<*T-O^JC3vL@UoL`ItK{U*=VO4R)&mtcZQz=#-}pGCcE=(K`X#bj;hW?l ziZ_Z$vBQfC1|^OJwYAL#shX9ZI10ZTLTqmWFJORMG#|Ur3vt62JI0g&7EIoLu&k=M zZ9d$zLA(`Ft7=2LByPaZD%OZZ`}c5NGLw{O$RCuVI6Ec>ZAY{&9ZhJubeqj+pt+$n%q zX8PgGH0h5s%g;cZJL%tSqu6RlU7%0uf?%u*f?aBOR~YO8gFRxf9R_3TBeGu^4EMS; z)&Q2oMq}-Zw=xWtX|QfpY+g_8i?;?{xc2O+8OsZj6Ec>UBsa`hzBD;0Z9MB>#_}o2 zDMQPD==%Q9oC6ut1QTxD(&I z#|tHSmEW@!*H8a;*x~zGW=?Y@BB49YXplO7nk%$+{bE#5O8CkSNc8d$G3uc3@+7Efu#ukHg z31f)~;|_Oh5jyHSoY6%9sU_u!zIZDiArh=m`{FH@7{M+#*pCJ~V6Y<}U9|J$T@u>S z;JjmxVUI4Dkm8Og@BTZ!aE~ue)1@=XPFwH!KW%(*G)EU~IixPoCv`zE)&;>X)4rOR z=Gxdp27An4?;4CPkjQd4Okb-e^As9y&f^iIifklC5 zU@^4(*P%H(WkjJG#tYV#w6i_U%T7^srlisN=hUcCS~s)&jr8*M8Rc&eEq`5czUjq* z3Pu|HyS|@s;m;*411F|hffFx4crgpmqmvzqYL8AnD#qwU$xPwsq`*5mQQa?M5r;Pb zVIZq~-QnZHK~-xceAwlLt7qXa+ZdkG>&Pe#u6OI zG=oyhI#0FpNfE{^Cnz?vg^jF*U_0}Fdz>;9SMjhN95)@i$1TUJ-$OC#U<8-|;m0k* zp`oE-T&N_RaZ5uKkA7-1Y7I2i)J|)26<5OM)?UXNsOB}ij7^zviFhuF7=++Nt_21e zRe?`Rj02W~0+NMeIu3S@{$n9~uEB>S?YTzixQMCg9T%0rQq*yg^lTFuQtg=kJ8{&& zk*P7%xw2;Pug1A6&KSXz4Z!&+oX6n22IomQV?0yF2tR`}lmB^~IijkKw2)d)pVWH6 zSnCB_YNurm!d$6)nPJH>88?ZccW>YC08kay}Y;~2cQ#!ACN)(A&y z6fSY0y|;c}!qHmI8K}!~PursR{MU}Sn3Gc8=#zCe!SWC+!Oqvdnij|Hf_-bSy#_n- z5f?|`PM$HhgY(Qi;$lLIt5lx-cckSWX^qsSGRaO;@A?1oNQ*6n)B*aW4hY6NAlOCP zSJS%0V0RmgD|^Cw(_n0I1pCBbY*htIfF=5BTJ^Lq-Wp)A!3JxOIv~6h?UQw!Y0=kl zGRya6lz+{2oIpbQEtb8GGb3$z+*G}i^F@@E9M*NGKHi02!S!(lV$>K1&-;AFK(014 z(%47Va7MsL2PoqrCPx%p!(ldX0fH@XVl}@t9FCl14X2NiJ~c=>`WlXHEBsiQdV{5S2aM-o@k6R*-RTx|ESfxU1I5JjAQpqNNm_Hn=Abrd#&b^k5IpP6I z`KM3HzhI1tVDq%Erd4XNa}9Q(!EQ6y9R_>YU>Fr?eJ>bny}>v)5xbuljB^vgelQqE z=7QDLzIf|IgS9YNCxdZBBJp_6U~3H)W-Jl3iqjNaryhKSkPP-&#W~5RUovw6SA|H0 zz9YRwG(Z=!qcy1DOg06F?G9%Z=NP1xlqdS)t=Ui_SdsR{TbCN_3WISKM|cMecI0D< z)}Fj;hE_B<@7!YwrlYvfF~#BJ-G2uc?!m=~s7;_fb^8ywO{DH!r`kmUXO@vNOcttY zKQEmE2<>y!`$TL#q(0Cm^+7P!2f?_ODcIEpW4kFB+fCuUXE3%xg6%QbcLwA7w#f3Y z@J=<@NP}_xMtFU-@2?zRu(mtx>O=SyTsPTo+2e~Je8v~y>gJIRG@2o6Bi}6gPaJ5l z->K|CgR!m5K!g265)Rujzkce(~q>s-yqm55rrQWh}v{o$Ab-_AHgNq{>TO5lNk}^bJyjn;UEC;D2 z*tyzQ)8e>Pu-yjx)?i0IwrK0guLfvAgY(NhwqP=f3mscT&9DCsDBJ^zQBj+^|F8iC z%Y3TNeV&eFJi%5%>VTT{s5&4R>wsVvXkSfhnZdYTEj+fK!h6GDY;6Sl*kGR-jO(u= zTSxoityF`h8;t7}!sB{{=;OYViM%~j!&$VLfW|N>qx=W%H)*$TX!+L}^!QvY`B*{c-nOe#BgU~DTfcnBUlFmHm#4)^J?LoGx}$~=A2?hD2; zFW4OIt7*+Q*f|C}-(a^G>~@1aXs|~O_PoJfG#GbhiCxwbvHOj|zBd?4r0}@=O?Vt< z3)akF9SqjlU~3G<)e4c-V~2XOhc$tFScBFmxbET9fe#UqAwFY=4xxRK`H-6HRng8~ zlM*O&7AflBm$*iFNmakZPulWogiXbE5#tL8S;V3bBWO%EGu71_m1tmOj5ZKqfAI;B zD#IKt7UyIMoppmi){Qu`{@;c($D-T`B;|>|czn0s##o93yVPJ@qY&&zgB>uKj3d&@ z*Jk9rSH5*nxoZRK#PYGpqcT?1YUu1>WzrqO5vLX^+j(+r3fj!zTX(YqsYff< z8dPK#Yc17Su<@z|`A|0!K&w`M+W3b$4@0TskpE(Pk z@tc)DLw4Zh8&(6gt5wQa$i_~;U9A<^4C^do$!6Gz2-tB5Q(YB6^)LrMGMk%)*J8-l zzF`QKl%IG+W>O+D#rg?-M)b`n_Cl^xej{#h<%Luz7OE>b8|o4-7u-o^Z*bLDg7*#c zGULYhM%e!vJPKjiDnpHNuWDc}R>u6Ei!(b)to)WSHPEJ(wb8#>gIXb#qz2I^HApbl zAi-E~1pB+eRvPS1gS}+1O$Our6Onz_V80uTHArl53@*Hm+J_ZvgXJ0QDuZ2Xu(`-u zkuA|a+*#&8MGa5*j7D1duXs$O{5?FVu}+Og^uF{RD{95c%@YnnzD8)&E(<1b9A4>Z zSqzNR94CaS*uB7t$AOm3V$&$jObsg69G~c_qpt1L9HU5ir|t8r$W zQJu=V>EEo3?EIxNs@or`I}?nRQ7~3U!62=%l?G#F6do(1@K_lI+iS2N4Tj9ryg2QP zw|L7%WO>U)WTzP{*I>!055jA$eNyr44P+(u&h)hN7uTzSo#|#5S!qjE4Y7}`iS5D0 zvDbq(j&yI6_Y%+BVHv5dp}O6~1p zL*P4zBM$E$mO+72%WnWTxcf%VsZC-_aXt__@$j$Z(;?sty4Wv?Yo#g}*C1Od;7oOE zlpCAEtyoHRdbOo}_GXBiBN8JwE(^FB1xyDuQ&P&YjVw zA-lP&ZBYu9Or2`e}Oae6opU?D0v%SY9M{m9SX)8DA*G1t7)BMup11_2RGI1>*} z2c?|Q7jG3niC}ZJFWy>VFtq>bYP@=frtl6J>_{datv$JR23q0ZTysx6h90$C`|r%7 z5;KqJ_fP-RXC5fCtTm}PY}6$+C@VXY@Z0v_Gm~J=Nzwu1iTT8d1VdX7 z2OYW)H8J2$U#S`NNzD+9HA65?Bm^rn*rf)$!eI9q>>-0aW3Yc3jP*wJy>2khmIeFP zU>r&c#?Fntc#AuS1mpb*!P*#%_bUYBld-}(#b6r@w$WhGClJlSb?U=*gd}*EF$dUj zI2z$-@7r6M`*%lS)-XgDEY{gD?l5N!M>1*PK36G2^vMog!SWC=!OqjZn$|@IW1b6- zc`m#onKXF!JB&ul8Jtt@NyD%s%c=j)6e=-Oh`ur6KYykWJT5tsi9$+BFjZ| z!6q533F?7heCdPmI5`-97@L^NYlYjwgu4yiSLdiLOeZ6R?3|sNcn5w3cdb*v;sA)} zUmD<#a=tY&(o7)oCZ@?sssARXMtbD0L!ckj;!MCT9cdJ##$iQJnfUmtOPsP zU>6$RZ3er;U=JH?wZT|N#KwAqachuZpBU_424m+Xc55S)!sB1T_$Z=atQCTFG*}md zaSN01xP?i0oCuT;sK+<)rI!zgD<92WOs%cu*viCls=SS_2|TL9`HS*$N6kUk)Ym*A5PeTmQnWh@b&!4XYCi7BIeL4Vg69Z~XBi?I zi*vG${?DPHu&tLO9t15;g%x7eTx#s*b*2MxyQt6=VTdHiSJ ziv1m-ms_z6xl?M`fCiVEtH7;~V=0c`{TfjJQWL3B{a=1lb{a}D>r!PK(8}GEWr-h! z!*e{vS?P+Em?G(Wz&G;CZ@V2dmPTJ5qv|Q)Cl^d zMhM0lAsCxf!Im40O{!r3Fc_Ou!JabMn+AK^V4oQ53xlyq6Hw zk1jU!#aj!ID1x1(eeu?H2D`~%Y)FO2hE!z#swHJwb8?!6COD^a5xT}Wc>eWp8$VZ0 zM{524k1j5=QdG7@tkmK%OKuVl+cCd_N0e1{1IMOA8aVo-qzcB8Dj4Tgf^l9Y7>5CZ z-Dt4K4YtN$8w|G5U~d_Wp%Q%zv*`QTU{w$z!K!IryxQ?47+>5gvRw`DWP=Sf7+)47 zye$UXYOo47gq-*})g#>Ac=w31Qw1Bss0$v4(fV<%^r*Fd$b=(l0C@*MiWPmbl}Rve zWfJUC?W<|=?u1}J8tj0<{_JTpEju~;AbtgJz?;Ib;NbbU3O0NuXCt+C|3@d#Ecumf z?J75c4t-Mz^C`H&LldG#blkKiC6hiWnS!xo3bsW1WRsm>9yF+!Pr8 z;`%sT6l539?OQOXXlDB`bLcu?%kI27*aOaA;*Li~ z)xv`->FSFCb<8s$-uIl>wdqz8US4j$*W7ubIro)MWXzQ7ZMk1f&aa-E9J+&Rw{uA|;(ZDd~c-qzlHO zr(j$T5{wHzf~_~$6lf4^n)cx>y6Ze;X`Pg%Pf?b(m2X@F8p_hHLmSikY(`nyoYv>X z8ENifGy#m@O$#SOh^_|??t&5LTZ~-e%1Gs?Dc*x0y&NUN7_*pTC-1WkRVq-fP7YJ9 z+WwD~t9^)tyI3(@%fb|^s=!%{nA9vrEKdB*;>6!gab~e%e%Qr|O^y^R`eZYLU~|C| z>}Kt&X{|Ka0|tAJg@}2Qu}IJ*BI<}gWYAYmkh=ZSY+9J ziN0Lzi?=e%H>u{z?iYtjzc>{AV%0(H7*#u#k+XGZ&f3gA8#AS692(j)@}cbDCM^x7 z=~8j7`t1iYdjH;FnPb+ z!4HJjitAMIW{d*kaoF7>18@Z0V{k!ZT81lVcJJ62*V$|}!I{n2iTZaloY{<};%pZ* zP9LS9(I?HAU~I+&V>2e$N`tL6*am~KPzjHPN@R~(p&AaA!-Ce_(qNh{6$c+F6&ImU zg&A@E|6QzP!Z6BCHk_HBWno&d$`mZ_=azy+U%XWW-GUWsU%Yjz!B!b;oxxr<7>kq0 z&eA^ig(zBfQbq+wS^&gRTMn=-CPYqe=;@YsucA_oMOslKpE*9QmRzMifJ0ITvY{Lk#QIE!fvp*-$2S0eevpbse;W0OR#ITuO?qs zps~9RcAvpE8;qMfMV2K@^c85|(J5<_pzTzjvc?Bf#6>7;rv3VFp<{0vZ3=lb%N!RP zq|DJ5Z*k%%*nI7a=j$*uc89?pHrQ%|Z8z8sgUy9j(O05<6&;NY&DqH2EoZB=Z&;*o ztx$0BH#Mlpu`1DVhRq$0;vy6|Gv==g+R+$)b^1Rs{$d)2YtAa#O)~AfLIqbRxDQy0 z7Jc#RrjlT@^woGiqON(@8*Giio--K7up-NK3ei_yrB_S_)*Uj+os}zgvE72oMJ-@- zHF0otM_d`c2Hv}OVfG@mXvNQS^-HXpz_O5)nWtYpG7PrF)l!>1^!3qvLy1~JIMqdn zyu6f@aU}|eEGJRL$QUuf9YgqOX-w2Qt41ldahw^4>oI&`J|t_rW__!S-dZ>_Zgp{H zx_3sPB;D!59u$L3)xLOZiNVe>7}rxp_5*|AO+r-yosi5!JjuKsgziDf8~{dF8%J`O zWS&tlC%Z(_`KO5K98l@3uZK%#zpszF)0rxQ(^+G#bmsKKoy?4bUotapb~4AAWTr1( zZEF)O3oOBwYF|y2Q-U#>g~wzTETgW;zkT|C>>v^W6bM=Vn zTwA5Hz8)@}YpW%GTfg`0PG_nJPG^m|(mC{26XW5R(u|*-(tK)C3N(H3>Mdx3WrHQy z+1iIM`x=btE4*C>E05__6~BmzqZlnsvC`Ny3y#P8x8KtlQgDJ8V$kN zXb8qeLohZPg0VCU#?%%JjYdqM6O#Gx8jY4=lKFpYqv1|w7IizBaZ_F=Gkwx%2*yT3 zFg6;3l^Kl5EEtnncp2p}8KH$aTBE_m8*xry{@>haxKo;qhMm$WCZ*|9~|URW7uvuI~v!l6K?-+`p!PFtIdHTW}a8U7R`C+$lQBy@nF6}adE{_V75!8v8m zB{P$)lG4D0O@Rq(%U@}?{)Tqz-aJsg@s~IE2ioE*;8V8(-yfK?89!f43yghjMqtt> zfME)sM4t4jhF%PG+O;q?(0W&3(pz{t)yDmSFE{ZmRXDHR7dU-0j&+cH6URH&Fd5syMzFGUde?>4Bmh`v(QS+!mO$ zmp?vB3$)%$;B$m^Us_1TFu^lP+6>>4DZTl77b0lVb7fBelccHDPnuDCgQC5X$X z#tki>oRCre@}NNLF^?cuUxxkph-PB7q2==%4lVyIYyCJ7H&x*-2rf3RSH4B?@hkUn z)}`?61tZS75WJ&t;QUoR2ZC{Wa{ue0e<(7dFJrQ^W0t@Z&uN9D@OMTH@Up zt$(I|cfzW06}&EmzX_8R@1mZt1lk z9cyk+&tm`?fP5s2xMyOnh-t2|7;C4(_uvC=aUL;kwIO6qRC24TgnMscZEHXx8y0cD zVoRQV(;Ii4=jRnK$jh3Kw_MK8FIfze@-^=in~oSTzHCgI8998&I|oAiLN>ldzA&#i zYtD?Ul7(}Oc+Vj5UHIJ`;%N$ZUJ{ozqj*kX7N$*EdBw$ZF= z5fzepxY9;lUsA;$kF#NzzLg@t(~#rZi|sthu<>p%g=W4yP` z-4PN6=8!~#946K?iqiz!oQCs8xN;nF)v9`K)ta-aRjUk)p1?hjY?lDP`1*kZdnthy0HSURF-=qtWG#$5F$#@3XkJl*Oa!{ zqFQk$)T&VE$W3gYAmg|+4mqN5mN{j_(3euU!={UHIuo74phg~X-u}h^Y zE-}Gqmx;q%Vn*_o99y|m9C!;M$`_Hf4uaMv5eUmPOVh|0RfY($v9TerP5{LAgKdi1 z=@QiDw8ZsY=6b}o#q|c>hAI^x0>}P=+LLh9(m1vylr_Zf#yA(@d;-o4(D68*tAGC! z=Op}Qi=T`$>t#!vhvM7{=hJZRjB_E*ycc*r&RmOn1m|uzKZY|#A7yNu2jcuC&VzAg zKJ#}R!j+D5b)5N|tqXs%X(O){&S^Naw(>XMT|NM3wj;xF9*gq`oY|U9!1-L9Pse!~ z&RByfy9DRSIA4nM6r9;a=HSdVn?G>K^ou_E>V;rT4Z+SbJdRrhyTo9Z8SGYrtuoja zgE6Z`mLpxUvCCjJkrsjlv`@a8C0GlC4KO_J6%&2q4aOS+g3U14Y=d2Eup15bw!z*t z7`J1I-TK-WZ#6SmOM{It7~eG@b}up*Q(drY4R)i!HW+N9!M-)vUV|mY+F?o7zIbbd z!A2Wws==lk>~e!$ZLrk_d&*#a;%poJwJ+Yf%wSg;>~n+dHrS5_J7BQ3XecE8I%;3M z)!ShG4A!lh&FiUs`pI#AS6SY2lQtf^s4=L<1Y}d7)0;!f_u~CFFK`I6AQe*qsNyVU z-GaK#+xaJMJM;>mg+Jjs=ViG5ru+KoTMxYfXyH#7N2jeAdWfS;Y$y9{e@39~dh;S( z{A#}u3%nUOWVBnC@#cYyKCh~WV*`U;Oj}-U#gIdRL9eLS=Pa+rFFN<+8?@7=uI~h0 z&m;5V)aRPlD&Gab_Tz8{uhW5N2Cc8X07iWrH{tjmjjOnz!K-;6ry9NvciwW{5FJS> zMEC|<40T*@f@LqfN^4`Y#L*+Wu5^W+gzbRx!0al>o~f`{GayY>zcUc>>S*|?;|-b^ z4j`*yEa!!{L8hFA>15UOux=(RH8;wuwv~8s(5tq&jjEYqU8kf*F;A3hX0@kL_!%{A zR>j^G$t-72B%|vLj+dcDGb0$KnQgs{k(PM|0{pU9t@yI@?I$NU;5YkQ|NX}9&v-OJ z-^6|^6$f8%?ahW}xY`p}WFKgdVfEauOJjO|#)*a33U zK39YJ=m^l+@uT8p?QmX#b1KeDab_9$8_uWVjFDd1G@Q%upR(uT%r4?0oLAudcbr*v zufv%g%fE3*$3kDcdaO~fGJRD(vnSXU23x6pHLW`h_K3kAHyAfLi|jWB`^8|t8>}gc zlJJ^oU%c9-B^Ylz(1$IohBwDx7aOeHV0;^%$S&1BG+O0vt^vU+A6jGkt`!-FzJ?zo z9^5qn>QwvZfdt>KW|gN)!11rpYCkrwX#Sky(@L|8O7hQg%~^Z*9**G>Cx)BavTBsf z%Ac>ww{zzxR{dsuL_-hfJQdvHI2r9JIP?!jdmNh8z!fSV8KacD##F6cG7LY96w(4C zNO8ECp>`D`PO2#U3RsE)eevp3>VmOA3s$CmHLVK__LjkDL3l{&uA5Z)q9F-N-v@yN zr*AwkI`$Ob9#;Bd80(w9RS^?;%KB)gC&L?@o~W*6fO#)YrRM?oB|Yhr^c0NgDcA<> zt7&o6Em&rbY(`8&5yiekZsQ3&yeX}GUGZn2ebh--4)-O8&P#v6k@ao8>io}D=2aIjgWTZeMM z)OrPH#tAvB_9Je>nVds7qq@LvkmV&gWYJbTMf3>U@TFhZ=gJU{@F{FjkG@)-z7BR>;`?fCHTmdg~!yGP^7NuLEMvG3 zER_>K!7n*MUp!}PmSCr8pX@{s?0SRAy86bzST%sf%Ib^j*)Vt$=XM}%gA&KCEG(;> zqKhk=VW&FX@p?q^rjsz@C({xK|%hlys7e<)xlU@BO?3CJ+A2DlS&u1>&t zI5X@hs44+}hF=npK1_Y^SFq9A7jLaG*mDL`Jp?i-XyAQ0kWN8Krpg|0I*tOxl@;Y> z7f)Y|-8!ztTYHhlNjIkx;D9l5%iu0ysj%Qmpndq+T8yf{3=fm13J$wR{a7c@E}bkK z=t#2A7jKQhU%@75U%VPV33iXc0#8;w4nI^)djPF zlQk=Qe%7L%J+fvLW_R_CdZMmMjA?L9V%XdTVN~c-H~*`?NQFY?c%&wU z_NO5g+zD}86-kJdIM>1T+i_+>{D?ymf%WBylVX2bQgrk4+wzR${GYSgVR##NUnte+jRa@g=|~ z64cuVhwZ}Yo+i2j)vUnASS$YeYF5={)m4aWN$L~Hdd9~}B9dec!(qE{Hn<>!@K>v1 zRohubg~*nO6n>E)iwk?U3#Yrl(Y+>4g~XPKq+2A&k~=n{J)G`h(G7w-B;LFcIi|ko zrOsAtKeP*{cahQSPQ6HaB`v5kJ&NA5wO(jKURuaYcM0&85D6D^RPvF!xp-;2aN)kd zgqtk8)J6)oltJp`;;-$(>Aghhjj3P5s$YU5x4Kmy>r?f+B<`Zr3$`fEJrt7`#m7A6 zU?$dzXy(eY5D+6k9Jt@7Q-Nh+gW6F^-T%eBfMai})Bzwr;3vjcQnpP|q5no;569*w^l}jZ*aK{r*~LL8nr~YY{z9VDYxQdEJuk%1NuNC_UcS-Y_E}K1%R2 zv&fSQlv`QcW*HD4rKc-t)!v3Fx58tdh>v+962av*obFko8!~k6a_iW4EQsFVavM%> zk#8Dhnlut0W67`cAdRYFHq)g_3PaP4f3zEJB*aIFvwS5+7zy!FqRdxfq>&IGB`)%n zkVP}*miQ>aEeVn1Aa0M*N3c%~a|LMA>+T_06yjFA!_ zrEZNXHP%Rpk5YGr#Hk}t+l@04;-kcUz7pe&g!m}&h_A#1BOyLYJgFq$>5rW4f8z~w zPs2~Qw?AL70(Xx3i}Ui`SAQ_Mb(aD4r?ww^&rWN|tNGLV22P%Pu@KmSqQDp1i;Dv6q!j$t@UAr=Z2B!`kOP zyW_6)XQ#e)?f1vl9)8*rDXFivxb))g87Jj#w)vA5ynXDAuU<0an#X=`mpOaiCv_go z8`F7v;6hvfh|A}dEbVdYA2z>e-}asDzk6@dzec?^@ZPkqHvGEc*_R48d_CFL|LB7Y zx=(+uYNpNa_sjd;7t}8;viZ4p)p>K^l)qQC^i2a6abshfy}u;T{J`e#d;faH=$TIs>izcaLyx~X^le-InaSmCw=N#A%$8rU z>&dDowA=rXEx%>O`1VUCt~}wnKIgoCQKNaU4PJ6$*|Q%unrO>^clW-Ud07W`+VU6O z5i{r2Ky1&gAI%ORC(vgj6Ro4 z&HOdD<2AMV-*@U6SC4t6>Rm7W`rF^mZL=fw>(zbj-}Ul0etu7{=U3+)y0G7;udQBl z&fqykk9YoTxBdIU#@E&?IOB^tvuk}f@bC4m%uYS);!}I?*;X?Dz-`A}vw7vRot@6O zaQH7h?ihBr_2lF`2LyUAZZ=`V-&SvGf6c&0zrLl@?zrWhAKhs!>{9iHXEUxnrth|I z?#OO+^WuL@etdTG`&(~2|HGKX8Xqnu-BCr}^U- z|I4a--M_Bdamt3qzkmGEC;xnPf^n4)|>se-QVmy>COFxOYeK}$urNX^LV!zpH_dW=!z3g`EuNI zoo4>DuDi{j-oE+V3wNyQYh5s_<;Vjc9J=F*cUN8TMe)+h|2<*m(#G$d_h8<~ixy0* zUVqOosi(g+v0&Hwx9;nB@-d6M*ZJYt1NC!ne{f#CE%$b4`}4P_pZ@kE&o+IpL+=kZ zj_I(i-?jJcK9Cigzwnw*8kL@UNy5}N!@3W+^TK~D{`tZeJHEFjdHlfPwZ6ac_)f1B z*1Yh|Wr^GFykk|*1=lyYbpMS%lsB2X^4^2rcK`guEoYu_q$z2&OAR*&m^;Ysa>J%2^o#;q;u)_!(TV8(ODE$wyB z#=J{U>-bK;zc;*kb^4hd+eearuTJ8UzWYA>(3ao0yW#l@Tc0`lXv^0dyyT?Cfk{tn zN-Wzjxks1r!TMKCOY)2aX34V6tFJA7ZatGoXDPn(|>tpCbYEmrn^AnU8l zx^Laz{MSyqa+dsY)1qnbuiyXTZx0{;ajO$vy8F&=a#jv|H0S6VKTf?74+>@ z^v)CO3kx5=VCo&`pLVhI)0UN*GpBHF$Nb!D^Q*=jI>fP4>WsYXl8z}USb;1Pq+>4L z5LdE5EqY-ffK3uLc=EW8xp~t|XD0K$k%fT)IJ1hx=g>4~ERk5n;oStU>sXSvs3f@% zvT}<$Hb!ouVfx=S z*q9`j9^5kk=GH?Cu`wJMGlpBPXptGJ=NQG|Xqxv}=xT)Mi3b`RlZ(F$&yX)48Z16{ zys$+BoLwN;Gqv~-J)Cto@tWDcc2;P3j#ZpwSG;P4=&7wZ{_$ci#KySeRXaov=RUD9 zcj7NoZbH+Seh7_MqT&=FeeBiUgb+PUIVWD>^NV-5Vq==P;#DU^Pd&x)k5_#Y9y`D4 zh3Mg(%Gj70_{;oy=HTe=q4DBf(Aby{T=~@?L=W#TIq?dgUyT&UonH+@^sq!kD~HD` z4)>PX@oF5Rr-|bD$E&I0eC~=@lMp@3Nhe<6^Xo*#Y3RzY6GHSPDUN@BHB%gxA3I)2 zA$poC&Q$z`IjhdN{b=pzbJwpHio=y(yM8qf(UYt=j2)#fZgY7-X#HxbIPUyP4$;#} zaX4o~u&r8S?*4bEo|6=ZvoQO8EQQqNgp; z*qE#Emowp>>#zN1X!^8M9ICg&(>6p;d&TijpAL%SPM`K6dYCe?G5+b(NpT)`g{NbP z9xhwN#`vcX@9&{+a-~m7h@LKrW4F&eMvOrug3C`=#c|h*E+KkYlAU@HzWksm0mof0 zx`pWJp*a5ObF$*x<4T_%A$oc$j(_^_YHSQw%c~p^$yX~S8-T_DE&_VJ-3IJ=Tj7?JqULH-Zw-~KgIDc&s>Nw;p7v)be#gc<07D$*&}D?^L2&_xC6=zs@b`WV)pIJY#Kt@a zJsjoG4GYne2{bl_wS%s#=+^8|JzP_bjd{eSCo@FP2*pXG61e-%J|2pL<24djM2{V> z5g~d;DUQFM(-epIhiyHhLiCJQ9M%rH-mN=t4Gj+m+OaWfT=5zmqGzn)`0L?4zSx+x zEyja-B2U*k5E0?A$q2T=$WoKcD(LM?zbgWPmbca<25}* z4_mOtR#Pn$Zd~~@XdN{(2jd_88ab#XOIL!EYrsC{|Esm1u zW`yXO1vECMKK>#*tkbVdSr8hYe8oA4zjpbV6{2Uh;;@`5J--!W&_thm9CfDRe2c%; zaL~;T(Nh34HYN#w>1Ka+JO%~9dXQw`YyezGv9O~mL{E|8Fuat0^uxv|m`Dguc;+b1 z2K=?dQxu|SuHw|iU+Vd_>*UN(J@XW2H~!js=7#7gR-CH%OFa+WwEI^;D+$qqY(^UT$7`YDxYK8W(o-;JW*2G|m!#}F zwk(TCK{0mavXClgbL{#f9Bz2FDGQVo$&Ew=?@hI(T4`jiK{yJeD>exGTDsW=VJgtg zGRPeQ;IFOus}Q7Xj1AeGQjO#qsTVae!yw#?L04dq*31;;Y(0H6QUVU^tlYCz9wSw% zIdKMAt&vFvc|#-94DyXe;tkSC44@p@*3Z;P6N5agk%k8OMk5Uj(n%MfItDpYBh3x+ zutrWW$Tu2EFi0mBe7IT$Ia4ED4Dz5x+8g96jr2B18(qP=86-y|Z4GjlMv@Kku|@_M zq#fE;x>SSA(a2zfJgbp(gB;RGKZBgA8^pc_xm+VP4DzT(8XIK0M(P>lCygW-;?(Hf;mlW#BkUfd-XK1 z5yPotBgR&98!?;{Y@{=1m~;s^yy2;3BSsIy?&Y+%5yRnp#LJ;k3A1g!t&P}#deBBV zcKQqm;iTG#u{GF6jGlBG>1;#&Y{UkvzBXb{dDPfMSvLr}MhUY)>V+UlAqe{@Z`@Nt z5YGC&oYo;o#}K4P2+~R+9T(+hmt?~Ww+IK;V3i+_R=rhTDq}wa7wGb=I9A8Gj?`i$ z#k^({{@IwBLmjD|T1s&&x2?036w`!K;~M|*qa)Rh%HR~oV`~XejNbVdM{4(W6C9~B zEu}afTT7J`<3*{?X)Qi*q~6w2isP~MHzj4;>XiBAhmKU+V{s8qaXhxp1{#Z}CdAeg zU7B9(NS&jl6vt!h93^Gj8u`guVgTZCXlkJhm=UQfw6&uOZF8{+lCp5~B;JI38OU19i$_ zi`%Q`I8sZsl;U`7EmKl<8U~(D+3QHXsihRhV{5s`R)<&X)N-U+CE_BS;&^Ob0#wQ& zuU{G%{FThV_-l!lQXG%1a$I%dRdd5T0Y_?$mQoy#txJ^@%RJrXTTgvP?rrhck6KD` zJhoQgs$=W34om*!NTsq1f>Ruit;>Kq_3OfY|1NZ-%C(f@cx+v+r0n|j(yCL}I8r;c zl;U`7U7@5*e!bEj!wx+xv|4id0;f10TUP>g;`MYwVNFM>SW796$JSL!N)N@X&gU#U zxtg){f|gPokFBdcwi>nQobO1*>I_jFkFCEeDR+L!Js`l=U=1sd$JR9-Tg)#<>MAXz zI38Qq8e6Pi%r8gkLoKB^9$VLWYz??S`D{n3ZGBvXQyh=2>w)6lG5_KSAio@`g<48+ zJhpC7Qg+!$eysFvM{0wXQXG%18p!m#+IF5j+FgjQN{7ty3J#Y`Q=D0 zJq{P)6vtz0B~WjEIZ~Unl;U`7-L9nE`Q=D8U_%b4I38Q8fMUH)UBwsx^-JzA6Ufn0 zisP|$hmx{wF~1zCe`qPi@z}akN!j@|_}(W{L9FMJkC@FWm9I2PIl;U`7-J_)Jc)i}CWR~3Pr>z=|`4`V{ z;fVFvx|fIN--lX(`3=*t*|i>&IvEKXs(O z)l!P%vGstGvfGUj$6a}+Bh~A8T!d2`kF5uRI`LXI?uUOkQWt3{#qrpBNJ+WNq3qwF zjqO@WaXhvj_Sj-Mbfk{sz!pw%JhmPI>Wl#{di~P-yO=&@l9o~&kF7_QlwCI1Za7kR zYAMC>*m}&^vfB+u3LS-VisP}h+GC6TvLn^AsYoe~$JXOY%FZvg^N!T{T1s&|wx00V zV!!N2y{@Gc$7AbBCFRbq_9hJ*ogg7p9FMK1Jhqr$j?`o=r8pj2Ym}5-zu0cbJyu$| zSxYI7$JWyxTg$&(Ud566QcEe0$JR4S${jDcPfc5$PUK&CFQ>OIKMT|;^Ng1xHBU<^ zj>pz>O3IGcxsNqy-`1qzV_HgaJn?$oW2^6u>mPHZ4r(dI@!0yOvBh$D()=$EI8uX> z0KqAa$JPr#op`<2rRz9H>WU=F$P>Ou4cB~mEl{Wb+Iq?xH65wxQzDUHh zhrgclMW!Rwsu?cADUQe1I-uTmKG~$~P$$3GZa7jmYAMC>*xIb5-1STL70|}_T1s&|wqEtv z>QZp+qmERcWXf8W;&^Or0qV5#Y&Qm*vT>1?QXG%1*OZiKI3AH(yLMgu1rfQj>pz}xazgl*ht;0r4+|w z>wP8Vw&h4|*HVh(vGqZSt$N1RK`o^?9$O!V*m9&=wZTO=#qrqs2&gw+NygSlEu}af zTOTVacf1^_#ac>nJhpa)*pfZKfUQ+pN^v~4K2cI`TaMHgEu}afTb~*!c&wJMo!&CV z#Oo(5r8pj2pBX9E%JCWJ-RMX)YfDX7amL~8mp><>dRShkE!p$NU+8R=Qyh=2FMvDs zi?$r8MOsR6Jn{O{NKq|qwKnm(LrW=+$JSRy%8r*K^@f&G9FMKtN~#78!LfdIG`4=z zQi|iT^)p!w zA+}l>TMuYkisP~MuMk^~)Vo?ralo^zs_oTD{i$MgEOzc+d=CiQk}IFIbNwBJW^#_-dp8J4b@WQ zc%}9!DfTjSNwvS2=tyO2DRQbCK3!0QqrQyRfGO|XbFlM658(im=(JqY> znOGSq85Ht=k*7;2Ygk4GCG5As>91_$pOt4I&kN%EOI~MW&u7|Yr>F?m9FiEW_JIh120qd0Y=;#S?xv!xY%yy6Dp4;&?1& zgy(o{Mc{a1joz6f0ZEw1#=?_=thq=z1=)pjQ?rJ7@&IE2+me0V6P7U7J?S6jx+hJ- zT=#@9QU`n<r4rI^ ze?~};vQi0Y_dg?~M_H+aw8x(j(xa?YLVEI_5z?cqR6^SG&j{&JRw^Ow^=E|iC@b|g z`=;|&4S%#5ytjQnOj+BoYbfRJ=sDoyT0uo7#CAB2hAPQ9bOlF4ndw)KhB66R=c5_2 zqogcKR#s$PB1#rkNJW@nxW{I8iK@sFN0{JLT2;6bRcU46N>rtlg)32&Rwg0ymZ(ZA zi;|TUS!s%rg%wf}CK&F?gk7R4vS<}1IF(ivu0&N@S-28aX=UL`RHc=LD^ZnJ79}ey zvUV6H3oE1|OfcLtM7u=El1aqd32GrOOmHf#DoU^_va%T^I2BS6CK&D+vYk4zfOFWX zQ)yM<>RY9ig{yCsRu-;↰TQsD%{3{SlaWh_$_S@w1U zbEJw~rRhjjxysU!s&bX2BUR-pM@OnEYW782>yDajQ5A#@LU2RN5)^HjJ8V!Yr6_8U zjyNb$6@(3fyLQ_3D%xUu*i@;MqVV;qQmVq&t4gT~U#}{qDtx`Fl&YxNcf{EiRYABQ zxNaZ?PaP`TMjxrlu%+rqm4+=}N2)YzNjp-dVawc+Dm{#>kJ>y-0I^!*bTtuz7gbf1 zU{$KA_77gAR31h!qc-Ce4_;JNQG!*eX39T!l~Nh4K37Vqgi|dVhAoDbQW~v3S4ydb zRISn+y^tP8)<^9YDxT$0RXM>jH|yOi%+Bq6XABaJ7)GVLqjK*5hfx`U7)GUgkz=sA z*&jw_1Y#JK?v}>cVGu@T1Y#JK?(#RtFO(68j!LH_diKN&F?$c3a7L$wRqKhBzuE}Y zuxdRq^H&>z8dj~Rq;PMAY=avm=s0umcfRaZv0GIh^7MT(h}z zezujBRh*YyklJ-digM2Hoz z7|VKml568Lz3~a2xA_s8cD7|bH<>=irCJFC^jiqW+UWK0G~uHuE~j{|H|6tEPaKbf z_pRn~PkK_qsA=xE8j{zI7x2(ZH$F@PR|`jK0_=v-_W(GbYo4FJ1K_mZMQxGwF&=mB zvaG&**aI$-z7lM$9|g_^K0E=}1V^~=4FdiF9+hau2Or=f=}W>R6WzfX$Hx!gxWhe? zzMsJ9%?A$P{Pc|gr-+;U;Uei{xNCDuzsrTok2_Ib+Jg5wH}b=Aw|}_sQQryNw(oM` z^lgB?-r)V4oAu%R^o`(VeV2=*Z#H-xe-fgfzOyyQRvAto<9j1`{eFqA?=sEt({~T_ z-3Q+2U;Xv*$(dD}AgZf_796x<;LEi@OuKy#tzJr<*P2VB# zwjYYF?_>+cT`pXHFuuLP+ZQ8Tzxa-?QZap}gO?Z=U0aNSSDlTy%Y_Rc^~HmC*)h@eeWW?I%5eG^-%r3BS~I%78#E`H zzMH{|JJw%cLns-mIez-epr!!4Z)-=_*Eb*>KYc4f84KQz3DNbXCss_~6z~q#iLNiR zZghQ&Z!UPB*Nd*NTm9(zSY9&0+uFciAIo{tanbc{0Hrl}-!>F(r20#JqZ&om_cq|-!j-S4bptJ^WZ!>>=eBy6x z^XU5cWbZ`qy0s84hl%0(59*t%Iq)hMu07zB-6h~Ep2F0q|~a6&GFO6^!ot3H77;a_oL?c>0|f~fVaDKbbZNfgcD6)Yw#Mj zjjnI3=J@H`jf)e(OKs<`kNLI~oN4Wa$Nnx{dHJ@gWt|Pq>@LDf2G1#fi||Wa5>DIj z3h8|~==fLZ3z`S40)7yjTf6z&U5Nty95_?>;d(fZSHszD3cLg#sJ{{iouA#Cz&WK) zbi0$m`JEpohx4;rH!a3W!9f?v?p3(f8=Nhg7b%?!aA6NP;|Gbp29S#s-!6k=tdThA z{Oq0%&NrIpXZM&PF;?lY=yp##HO4B!;b#}wW-ZbjMZ}+Q3PpT51C6HVWb;2jtdU0=76!tv9``1S;^{ix{rW@%0|eFfl6bbWDSh2y7>!=)PFH5nIO-$c!crf&*( zbH_*5_kia3=~LwoyvHX**SA-5qUrk)yaN;c^&JZ(-A)&dpT1mB(!jfGl5iu9>!|OT z$XZ#M0A8-fOE>l!dnAgxc-CdbA^}1Sc`Gc z@vqZQJN-HBoDa^`72w?h&Vv=;Jq6AyQF!dn-vsBo3h)kqQ~z@Rcu?Po;Peg4lP`mU zVlkZugLB&7MPHC-XRxj5+fH&Qy6gL6{_cz1#GL81btk{jzzfs@E znxm9O(8qOtp`*eN#Ao`q-Zj1@Gx?!u3nPpESo$AJeblcKmr;^!cUV zD9!QH$Mib`ykmFx>tp(zp*eo~n0^<6H}744eN4X>HOEgM)9-cg`n@mu{L=3%&GFO6 zB7X~bc^^g(-xHeSr;qh_9eCqE_SeVsyIXVo^s&gl4&IHQ`0Hc(?a>@ReWRe}7x0FB zCi?u+?|jYi)5q}L4PNmV(Zjb+bNuu%{oVjC=PQ4GOuyBd^rBF|SFgZDZ(yEQN3eNzPMAUI9F z^A8{OO$4V{^CIcP5Zk&4oIT(B>*Khv)!rDZ2M#*^4L5%2iwmjXEd9Y>AJ=)-fb+WM z`RUsUPQCs9`nb+BN^{^Lc!|G6*VjvP{PZ!teZk8-5M5u1=0wxC5WI_j_1AYQlsv3Ce)>L#n%}_t z=y%aq8{}~LL4C6i3dc{Mg@XPBcu)L0y1ukS71MVqc$dbo24OD(j!5BaSVcIBh(F=V zALE+?-jKNH`nGFMG<_#jtzuQF?yryK`7+J%)5rGeO7Ir*BhPS=+CSEtCp1S9@h4n- zYazZ*gZI)gqOS&c;nME{;PvB$Bfoc1k*hiHW@e<{#644ku)g%_zjQ{R)| z9MZf<`no}1)0RlTR{r|>L*EK;wrC#N;p#^c^z9-?e?Bx)_y$3~FF5CFo}a#Jz;1~_{)&rjcP;KX-~ zu5U0nGc_-gKE`(*IP*G1*Y_YeuWMc;eQ~(36PyoH{PnSaYuH&h@G2LsysU)29Pnbh z3fHecx?FSo^gRN7?}2wo_vrc>^oSll>dOW%v1fFBw`h)E_!!^a;4SMNU0*%>gS%Q` zr1Z-I?@(WVeQD59eoFNCrXze?z)R>a+(`RCkAuEtn&TJW1Gsn_c-N(hzDVsU=O15b zj-S4U2;Z;ZB@T$LFI#i`^tFb*`QY6&FuJ~PG$)$AKfr4_D7wB<&GFO6_+AX&ZG)rh z`(AVW^fiILm?0P+Wccel9!d(qS*Cephbw<$q3>pJ_6`+&k=lFCx7rV@V)eyA2eGjF zGH_uqIG3I3uaEPs&EV|PJU@MV!09sFUmxewMVbSza^d33{&xv@_m2>Lk;=20pKDGu zeKkf_v3id3*T?>QndbQE zzSF?V8yj8U?V1x!-`(InGcLNmpEbu%ALIKwc!$RSKlZ)^zN+H-{{|w4mk2x-K}CHk zRZzeLkWIvd&7ff^s|)%F$qPiXnY^&LhQ%t4R@|!gr*-X5ZELl)t+jT+x?pRo{?t~i zTDNM|+PbywmjCxVXJ+1g_uYHn3klTr@BQ$&lkYvVojG&n%-l0`v!{3DVxh^FUMc9N z9iKhDHcb9M{~gYNpe z?CI@QpS^rcZ!GBk)exTEp(xa<#_Z{BMtawQuJ|;e4bp#1uTIm1)tBjg9drk`WKVCI zrpcDx1)v+;nmxVaHBGkk)`PC^>g?&w)-+-1U55Iu0bSAB?CF(jnr!JcgRaLJ;pq)S zB2zR?w)9qlZu^wAkp~;qBHRwJ(J3Kx1-!bQ8Pwxfr@5`VoStqn%`ruYg6IMPf zwL1@j?gtyPr+4Va?B!#6$AE71SF)$~9ZeHfKGydk(A{!=_Vi+#vX_tP9R<35+db*I zzQD)wkgu0(8dJW~KEp^#$rlnn`=sf9hynHyv^qv6S^I6j4y5MJ^iC!S} zu=%QOmMocEtjFP?Inqn#DIe|e6F^hzrSnPeJkVU~rSqi6`hH*2NJ%tbeA2rcbPr@n zkM(^SG#_~Be9{|mA;w?)cv-%9%E$U14VpP#I-m6FL9^CN=Sh!zzCzP5UtU4-awF(& z&ypVN`#aFQ=%w>1-*(Uxel2@_hk<5-m(C}>#h|J3(s}C3_FboGSXW*__1y%z%d@1% z`rZketzJ5x^1T9@cf53-^09q;U(^jgVjjNuq<0i(W_sy7>9M}cH4TF53aW1-=+4NJ z9_xD*Xm0b;`IPTr&^+a(^OTSE{U>O4+nl|=2Z82rFP$en%JVEuqbiI4g6ew$=n`4d zV|_P(<_a&JPkny`ntQ!;p7OE2&x7VIFP%?%dtBVj*$Y2jLHb|_o{a;|Nt%w-LG^6} z%^BYGT;Kj;K11=l3N-horB_<&{?aSyO7Z(WXa-y&<>LpjRKA}2jt71cXf92o%S?}Q z`n0A&P+dXwcoB4e)3iZ+VS0s^O4)RBfqdx)y8Sh6Sb9h0nBH{IeL08p8gooZ<~=W+PkPbIVTa+z%kssO9`(?Xnnn@gzaV)& z26XeYq{sT62AXrdbUx*~5j3}Z={)6QeSZg<=e%@2>3ss4-M=1QU$*a;G!2643aal2 z&`rpa9_zapG%LMyKIPj4n#;X(KJ~p5H1~PweA0UnG=KBbdD2t*<%%5j3+VROv_bO1 z`W^+EFMH{H%GU^*GrV-3^09rd0?n;nI-m5mg60V?ohLoY^E;XbL3IVS?`NRvd1bh~ zu)c?Y=14D{Px(#&O~Om(DIe>*0W_C->3q`rK4^aKrSqi6arGI{Y}a(GO;CONU4{0= zkC$yw6w(uJF?Nr1vAx-0P+Dq(?pUyryA2cm>JxUqSalmh{-Z1Fq@j48xC? z<%>`GjseYlFP*1+tnX={S?8tmN$*y0>QmTNjJKS{ox@~QfQW?c^HU9M@6l)8fIaSiBh)wH%fJo%C@TS4=( zrVHcC2cYSGZFasK0GgSaE{rcHgC?Qr!uZk%nsahU?@~>p*%H{k{{y<4HEkGQ9stdY znl6kl{{&6mb=mo{KWL88bYXls2{cujE{rerpgAjt^e)yk+4yo5=)R?C!}xL^Xr9+} zVSITHG~2!D`ShcGukYqe$B$PSUrqo`rKStx%SzCkkwbbHX_{<&xdL=IYT7Wq+zXoL zG+h{9-T}>L-t>IO_tT0R*va?0J`osg|`Rmdw{0N)*d53Hz9}g7U!5=73dnXq(}Z; zm}7dEgYNn)>9M}Q%rU)(LHAS+>HRau^zyz1I~G4)Vg2JUO`{0$Uy!{q8gxhJklx8T zrk4QSY2NgF^yEdLxl7a0@2Ip7>@Xj?*Fh8gU#V{x-Bi%5%tA+d?HbVBrswq z=JhP;4MqHYZic?ckC!c9b!D=0NVCHDAN>kiL36F96MbL0Me3U(q&hi zqx@|J%@bMZIFEl0G3Pbx7 z-JG%b@d~QPB0O6Fnr~~muzLIuG!LcG`Pbud(7dYY!s_ucX!iTAlrM~K6lfM{x}bW@ zKs}P6`KhK0tH=GI*_KA@NxotAJs30-)9C!`I}J3)Yr3#{)PZKRm##XotabU2`Q!nwYW!{j zO~LJMzS#4XE!|Sk4F=7bS?HQTx0y6|gr`Tk{-D{Ig^u}N37QYH&@sKD@1uP9@#6n# zA6nIll`Uh(D2)Hb;8?#|(^wErp0QZ$@EK(@D`w7^HFN%~vISGiCmtA|HGgV+(e!yo zl}souKG5g+%mvful`WVvFFtMhl!eE{E2b}+UU8Hk<-FF0WNm$-qIOwxW%JsjW-W-% zPvWz(Rr3;ciOQA)(w;VbM%luO1r;-=Ogylzc3D+ZebO(-sq+wZ!L*rk;xp&Zi7%cx zZF+qE-04$i&X_rU-h7p{J0YYtb=Cq;>T@cl%`TfYeZB`?GV-q#SOO}y#$(pfaM(FxV5;JP+5-m&QQ|j^MiH1aTZPm!? z+U6xK%~eb0R#vTCoTy#CBH1#dv3Y7^!>W;0RU;dhor+U0>O``#w$6#* zL35%m-q_Tlh?C7FP9dAz6FyA=@XgN@)kDya(m&YsX>Kd!m_le`l70rp36^(V(me54^5LVXJE^nw$ zG$e5bprm++&S!RGL%bCY9dD^UJ>iVhPkXj~tS{euTaZ$1L!!>v2lTUP#z|_`iH64J z`gr9s5~!^BFfZ+46N=2I`y=X?Q&F26TN|q5$;Q={&DHVK6U~jzewsi>Hw5v9L?x0) zItQia9iQD=A76%!HF4q!w38GCE(yCw_u{@9Q(YgTcwJ*li}R&W%G!pS+J@TXTIb+U zveQ~?6Uq4O%Gt14!!m9`bD&Ki6QpHL{0mvz5?@BfB&wYQ!iW-UDyx#tfng-IiK-)x zm=K;<(mAaB7AuJiXe6xFHNn&qapx*v|t19bS6V5P1b}2K(hMKy@O2Y`3&haa_m(=Q$;XbwW5~s9E zaSqh!x`pLte{plsoxZC9O(zT?)k-7>O05A#WY6B+jMSR!y+K8A_hI&QZ17V^%-r7*x*f6PNxD4|XC(`stR8LDZC08u2ZAsAfm{o~Ud(EU#<0i)-1@T$v z)OCplXOT{@K%{7g6I2m7B*#U-Je+E1t*31So1q^1Xc9h#)r?Z0wLFZ~8Z3}pYilx= z%-UT_?N6?CA4Sr|*f3h?506NCs8_S#@uU=8mxVQD`h9%Ml8T{=bGRnY&xsr*GIiBL zP4nM<+f8iMzUohgPeD4Ca^UBKdPOJ(kXBw2q<`RA@#L1vSA%yZmfSY{rHb z)M~C&Q8lwozFJ&8`Fi8&!X6P?SpP^UF0IE9X0B*#)`FmAn2M5u(A&3VVc05qT_z(n z{$h|$jIzu|+MVk66Vki=LY>|YcY8_C+v~@LvdnVYR1f`Mw<`rVxD#bW)J&ulF_u3Q z17aybq#S-&;5Zch;5MfCm5lZDcZx-DQC(4VA*oS<3zE@PgAfQI9ZDQeT7q~b%Bi>_smL)vQ9#}6wD?e&cO;PDH(;5 z$tb~|8O(m$fVMJPT1M&0#C%KWiKc61B+S27zHv~!lhdmgHayByUxr!C_mAPAJISRy zhoct5N2PbeYMp!z-LM16dwXU0SX|fGuslsvVkp$`y;4@o%*K{wlcWh+R@OGHZm~m> zzcA?pHQ&wf>(BQzTJW3GLgsA@Usv9+agxhgYt)pFB)Zeb-6Ty|0;$5JSxx6iWV(=x z2+C(QC)PG0J#f1dB27~x>TC<7MJ51O)DyGLfzV&whS_?jaJt-syQ_?+mhL^mDQ)Vwj^t66Ikuyf{7|q zuZ)B!s6u3Uomp^IYN2BPIh-w$DQn9c8>^SqCb1+ZBxX|LYP3XS4Jk|_KAo@zi-aV} zK5mX4qre`Yy4NDEZgY8$Y8uk7%T@M4IH;RsMVndvP{EVqw+_*K=~oMp14mmSi^7L%V3 zixUmijX0L)yfZ%*WB{|TsW4)#VWqm+eeY)#!fo}yWOlZvX#I38Kj z@apGV#=v+etg*xN0*@z0)#<@oosHpK58rIkiIKLM^7(fiL!P$sXQF7)3n&hw*i}2D z`40{)lsgKFW{f+hZG%#WK^eK${TyT5Ic?A68V5NU)8q)>;#D`cE~`tVIBc|&|Mbp7 z?W#vKJoPjl+q%mz?y0)X!_6^U_c1cac_oQwr-Z2{*c%9 zyMN^z3mF?R@@G1S>zK_@%eE0Cf6n_wK|Y*{vfuf$KRr@f8<>cWZyRJrw9-!H!W{h@ zA~+%nYoZ1k4T!Y#nI4bfr%L;n-7B;<^q8AW7Gn(}IOb3<~!qW3jU_@5C z=iz)w)RxoD&C;o89xo4H^e!2bTJ=6TJ$6`ibDD0APOp7e&*mig3}&$3{%r~u{-x#Z zC_jG=Uaik6J6_rIl!@++dqEiQ=N2?qa`{#1_u=w?$?#Q;wbf8_v*y`&v5v%>JyNC#o$5JsST-4*JY;awZPP?ZM$lRPP49|sjSv`OY&1q&Z(j@ z40KnmXskjF6M7}Dspql=O7eLk6Xl;aHKLPy<-w znD_L$IGIxEodXUp8EZv!x)vE7*=wPs27@tX>v?Qubxh_U^7dEVh*Gzo%iUhpC z8dl+rGgif_of)|>5zAmntg+OGaB)^HwFyqgTOH5Kvh?bdj0&#Lv|Q>le{D-L5muYy za;Z&FD&D$qUFT*T;A>O_UVgOTm4AFiqOuwuSoF%_W3%j|Bo=aNo@dj4^Tsl;32@p%gt%$$PmG_|g>rDa;8rK-8M30({-upoK?R3TkYo0D-o zOyIVv5++_H0$3+&XiUbN6E%tEL_<|#k{_mDXAOr<;o%SA8< zr>un|Ykbrg%s~pG>#;>fXM`N;YFktWsl+)*UA{w6Cr(sJfenWujcUI%=#UJr!G`6j z{KkRy>fNBLCM?pCIS|^E+16#H)3hPnfrjXN7l$9M&v;oPmMnzdmT8!1{ zg6K2joV2Bu^j#H`Mi2KcO5)u~LG<~t>9PDBA}377W`%<2ePD)J22$%buvrSCUvgzU z1!KiZ=zcz%4aMgVID5myqWHj*za>#Uzp1jBPM3A9_*fguANQ2RQ~ptz<)4UH zF<0iPy(C#3Oz>KxAbO1_NB_FR85DaS3Zl2TsQRBXJ!Q;eX6TkL8ni0*|$ z4h=(cF7xQgcsp7UJ=j%dGqzRJop}|;(!D{@Iy{wly>e&8J{3G;OZ{%Y)H4+YZ2)6F zIcd~n^?ai%ZrXEXPDSEwf3+}LGS=A(c(W^BEgY-kToX$=bMUCe!x+6l?yi0iuQD5) zwnGz6BG>kg8ZK6-?%~x1WzEf%YtyVI6!j=X-yI?>6T)eUn#$HXSZT7i1~VkR2k7#Z za9TJH;6Kr50?tiJ{U*5^Eazo^yc#cv9%z=P;$_LkdaQR=VRs8Ar4SIB+6!zIXX|}B zNSO z6%Kk2p_i%Wl@x9zCSxb7sX)xMvX$BZSvF-tiTfkP7VOqmDZ;k|OCs1{H5KSb=urxd z=iEiq)}NS?DkROmFx(^^pI3=ZUaig8_f*r|SRb#U8wV#-XPD~xzl>F~bF)naGpB}l zb23@0T|5=X_A~X4r*$~h*i_l8O%|T58!GZahnQ+kUz4c97QuKU=4A02DoaY%d8Vt^ zH7rWXWp*3iN$^nOFB6 z*Gn8N1ewb)MmoH!Fa}4G12v87hz=QLn+;tJrZ%H~?k-mp`)d3g)S*mnyJuiYYM3W}xLg6QL>m^rHm_DR%{ z9e-O$8({a961$fV)^)4Fep(qwl|tY~A;opkk?y@gRFVIisUl&zuuq(7m?dx96a#+% zbI2pDN_8v-eRq5aOg)2_qT0x-RI8C!fw-co$9N-$-sUwp>Wx2VFncN3g zny^X3k05z5u2xFl=bO%~dU!)kBVI=oL~8=luw_@W^RJjLH(2CxC(v(<;qh6^u)dZ|V1!=*;-j#-BrTUk zGcmGKnp!RCh7M3mmRfYuVC1yGj1b;cX{D%Fnjt^af~ihS#{2Pt=(C3NId>yhDO6t? zYJ0SgK7)0@DMPVgh_w#Xs;7(>V?fZ;^mj{O#uZQRu%is}k0nQ(g@IaE6L7^MTvlI|F=<8N7&(&=x^kPy-QVChUl9GXZMX2mtoU+SrH`q^vRa%)0gFV<#p3D+ z2U;h|%HGx@-zOuCDnn#|gj8d?6$O}-g#WI13 z8#d5pPpqlJ_EnTz&6x@;Q%}TU3jX6er(D-zB}4V>*3qNT^RQe4pKlc043V`B=x?>v zGzsu}3i~_A{)c0s z&MJPgBg3fDwp{&5*@Su`ZUrr=wCCMfltc)dXht*zG=?3r)UNd#B~4n;21gP zClV{6Jk)T{tpu4AbD9zjSnKC*1I(FxN%$_DP9_v+Zg9y=WJZ0m28?P^%J)!Tl4-5= z^;|}3t%s{$LG;O7yPIVU>QAlsxQUoP&2rDllqTE9-}EU~g7laX(}2;&-`rDjN(K6n z?QfWevZE<)9L1kUO!2&pEAz5th~3?2qj;7hZG3` zSE7bw63oQLKxKjq$b82AO076&h05-z%*X1j7!JdL$rsSY?I6fznXEdloi1SCj-J1< z3)BUqKvPIYli!*q=Gi-PR!p;cyRD4Y$?(R8QpZ;WrJLw>Q=^KWoB>j-bg#a8yS=dl zAoWyTn@p*^b~s9|+%T1yai1w!wlPd`0ZFq*FEdGdhA1h4B6-!$ZumkWbs~u-p+S6? zwQ#bVo2n}ych0+>=y*J15OeWZ7k-fLKB412s~px=V_QA(d)hZy}xPxJQ?t1RY2_GHyI-qHl;e=Jz7RHE2w#t>Re zs_m4d7nzwV&suV?pU~!b(AXT<(+0=dcyndLa&Csj5`I8~+KD#T4x7`PRJsLY-;!d0 z`xOBRWIjei>_gLPR?y5l1Pbd5Mv-vGnx8k8Qv<3qOd8qLpfTFDA`F)=Lo72IGU}(M zG%YSyD)r{Hw^hVhlxw=hM?2>d!((rM417srx*TJF!&s`T)H3LH`*UmJ$ zC1ve-r?oj*ZECtio!86M28UUxqt1-Dt2qWoyyuLxVUHtSFSUuw$n9yu%OL=Vp{?er zG%$sog`oJ%Xl&D`+{#clCx;fT4|NlIIgx5vbgWa5?Cxd-E}H;x;-iHpyrD@bfBdy%{cC0&BsY<^l~iL>dXn0>M=3e;ML&}oqiNvAirpK^ z>b=RxfUnIVika5&H$v&e2IV^Y=0G~r&D4CLzY&Vm3SmQQT^&RayTWk@TS4@!fR;~N zIpR9-)n=s5HV~_>a*!GN)Uc+t4X0rjng<*zX^r4*50?<heRS5yN$SFy&%vM6s*YFxl zsm6=S%Bm`!L71kjIpSS!ib)==PE@W`6MmY<4JL)4ekxWjyUV9$isZi_p8EhpbAP5O zQCw+0D`T8Xyq{)?#FwpYsI1psuWDLiUt_J-+KOU#Nko|1ERL{Y!XPsdRbGjtqRT+cK4WGEo@3TT)yU-Y_z-WGuU z)re|(u@yO^z&v_J6rs}OCtFRvumMM`!ij($HZX~s)w_w44Og-asw}PQU3z5IV`PX) ztf*yprAa|YaxvAX%22>Vq!Q|_!>fiR9kQ-;`f4pOy1!05<%xjd?+9djkLjjHa|e{z z$O7~-O}6JOU&1Szie{!M9Qj|g@}HWPsFlfY+L|t&>KcDcDyo%7SuxW?rX5w){3}DK zXbC)Kq$cjat3;`pBaK}qV-2^0v11gOGSxOTVQ!J8UXI7}Ul>j9FCy9WyOx&}BVvbg z)qFLZ9V;^o6rn z-SZ+$xLkL8u=%|kGU!ZEhXxGW$PrLf>C_jY02p2BabgX33c+;Y`R5$@rZHLVwLJxc zr`oKd#6mGMg1XdI*V1j2Zj~z`7V)X8lF$IE!QRciF0(wWCKV0Xc~Y%$+q&6*`6(lmH4RTkL32m~_m?^FDbO-t)lYCDZ2J`pJ23^g82| z;hP=jrwSkQ{hCJ({AlE`O@H3_$jQH|T8?j&VY1XG@6t=hJwD}se)`!z9zFWi+2c=$ zInD(NZ#m+gL)*t3@zT>9KfSEa!P_rF{tCY(UN`KHCx1Eavi961cZcOK- zk3ajvUOk75ekykU`WvR@jdvV=R;5qgOUdYg{SH0nj+Yj;#cx~Oc&y{Rs_=RFd;R#M zl@mr^-S}AHpxtX=cFpci`p4h7d)A60w$48I&R6ca=zDu?z_&vbK6>Q!wf~y4=KSBT zes;l2eQ$t^UJvl1Pu^AE8Mt=Qu>V_DKj8kl@xS@$$51p1|MY{mFCRSa`M+NJwJYv9 zHc__PasH(6{G%TI;N4l{n{WMK@d0C3j6NIRp2ml&`sDrb=)3CvSa#^gYqxD|fBB)8 zKXaUu6n<_0*9M)wx?98hs}5Or$HAw5-Er<$_-9Rj-1^UNUh$Lo_(zA`S~=}!`1Y>C z8&99Nu=JYZTZeurdDjzHoQ-;w!>zqf-q)U<+i&(2r@!{q#(v{ReEX+QInMVL{_G)@ z`_3y}Qh4eEC;YZ&^lcb&d+tv9KYeY(xn&RR-EZiv`+w`g*=M5cOBMcui;F)!t*v5W z@6vaZ-+StoHhdFS;eAHmeAL*69#z}!yZiP3{mxzA#kaT$n9k-8w`{4NG5E!aZ=Lwl z-UBbFh0rT}&C*W~_(uDa|9jN(>z}P_?fxphA*1lO`VH9hz~(!jyDmOuS;>rRZ*`nr zJ(WT^T9Oo+v|KzD&KYRZEXXn0i z+%U5H+OWD(Q3<+C$RJmJ7z|GY8r%ps@l z-*%1ToU8EBE9#RY-)l*1EV=k=d(RtH=QvL({MGm&TTeTAZo%m{efiS%zkFEjI8*ju zI;Ce$A2h#i#<81UIsVn2^$jmTYbd;A!iewBs_M6G(4kMA_U#_;q9ebk@UIMRJ!Q{Z z4*ACL>#rMr^Pm2DCC=RK#dHq8^NI08&imzA`#n4J@w49eCHmHt3SWHTf`@iLpyai! zSA6Nfrsb0$SKWJ)zHZMWzkSK?9~yMYZ427|ujg<7?l`9_y!7wuPrkkGu8$A+`$=~e z{kiG)j`Oy{2aVdc`s6-E`6rxq^}_y}W=Wks2!>;geHO)5A&=-_xGH$*rT&TmR z;8~Flj{{9_7v9Gej=E_2y21loG<&-6fi8TI3m>e*=OW*IUE%#);r(6Vm@7P1hsPoA zI9GVQD?Gs!p6Cj1Mwoo-0o{C=E4&HI0p!m{mMcA;R~$Xzq8> zJm8{v&_%P=Me~r0=GQKo-?(TVcG3LSMe{os%_A@F!gO zlP>)CF8mKJ{3#c{&4oX!!!_W^bFT28T;b}A660i%3*WNC9RkSEUCgLaX1~of-R2E9l0VXMQpXwU&&fk+tOND7i&qj zBEphU#U`yeRch=;H)fEcD}~^Xh37tt|)Puf5c9gh&8B{S_IkoIbn+#zPo^T z*1=rtfCO^+CRFnH(ZD8rxo5R}G!Q-&T+GBr1A~O2>$=$*Xk4rr=iV(^leKj%m8;f_ z>Jru%y}yf;J}Qe3RPKDP!u2sIlicM-3LDJkE))XvEW8Nle39)q4*3mh(%hL~QdEWk z&jhHcT^`z(jFF+WIQw49F5w~CTzR|heC;Hw^+B!Gbz%XY%NpR$&kAZt`4#Pa=23s~ z1DHEKr(Kp21g~awbx?(J?`o=x?UJs7^K~Gl#ld-Xq|lU`u1Lvfn)*aN4p9tFvFrMm zU0~0B^cAhPD+;IHIcZPvfWDG@D_P#|qDe}n+E)-z?5=CT$~v64S({uDIC6GfRycgm z_|cmEal1kaQrS`-XnhCg+HrBv{#jj-qHyq-M4R!-uc{=JN%dK%c7;O^VBlN5HgWU&enb+~LgD2xf_vGl8 zXY!qKas`gD@8y(5XQE1Y1A=Q`_4l!4jk@;6aT=4Or-2`N&ZKx5PYuD&WX3tp(j2O2 zS|A3N20mz{nQLi=!XaXoWf@-~6z$%lX!rizoY~Qcb7GW)Q)3R^c=zpto6i+1kHS%O zMUoMRw8h4?O%gd)!K z7JJ!ZZ&{2h$BYqi`hh1Bcc2bMoN*RA(qeB~>}`v6M(+P$<$i3M-1FVo&qCy$>-2K% zpxm49Fn6@SsCcM_K&8wt3jZm zy$>H>iN_V277Qt!)?UEBH{~pMg+gF;>Y(^a%JsT!F%>Y$CKb_ zleot`W=;ddixhk>T^%sVFSo`H*8jUU3Zw=a^Nd5F-Fl%wELF8IvPsr+ifAec8LQ4q z@3E<|X|x(+W})#%EB;igap*bdZ#xd@v5MmxJdtso<{7*8Q*jQ#AYtQVj9p_hX>b+> z$G0ydUAcy-zvtucVG5|s923>iRMVq#Kr_@tL#{oXY0)j*su54nV% zB1rl_x$AmFik`!kTtZJN&KSpWUXV$@R8x}+EDyzx^gkQt;tLK&HT1|mKR zBSpLK?;hXhV8rJ5KG#G}yK{U$7~^{ZQP23^D+YHF{QBY7A3u)wOC86dXt=cA3P5ar zJFcVoIo>w`Cu=Q^CMLedi=$bIw*gPH57dFE(dhYJdNdK~$zC>vO&?7`oNP?9KfrS0 z*8wMMEj?J9vOkW;Q*$u|+8+hDS7?y-26Y4N4e&|X9~5rd9TYy|#TsYFAWl7E?2#2# zFEbQzXh{fmv<^ia8h3(GT!pUDVl5Us$6^~ScB92^w%AsSJ#4Y3EcT4Wwp*+_rVf(t z;W||4jJ8ha!%$mx_JHjHk=nrwv(}Hd*M|-WP)f z2Z`8gV5?KECJn_hX?RvC`SE}&(vLEX zDt(foNt?1{sPqwvCQZeLsPLs|yi65{Db8MQ6}Sa{R8)cNpP3g`;C@hn6bhpPd&b~3 zgI{m_r~;|_N*!#{NLPK(KEA4NE^xBKsv52OsASSqoK}5Hfv2f5RGF%Y1|rik7iE`< z2|hW>If!`B#E1$As$VIlgK_8IoJ<*oL`OcEATcDyo?tZg1fxg@#_?XTdW)TAv5PHs zxy62DvAZqyoW)+W7#*D?k0*7g3l+mFRx#9fNHNemNv^I@4Epyf6a!P~pkhE2 zWBB>oi)yjNqZn@Jdd2YT&~=J`mtweohZO^r!a)3tVj#<0Hc^;jAS*u4ilNmi28Kj2 z2u8&q7!`wHR1AVqF$hM*AQ%;cU{nl(Q85Tc#UL0JgJ4t)f>ALDM#UgldHa-t@^iN& z^WWZHikV_JwWv#To@azl&{M1H# z4^s83=GLmDI&(YiNY3EMY8~)mW#HR&z|=rH7<=v}M{ib-<8yF!aI+eTQ={x)2wZAG zkAB!i`8_$*adi(q&ISCr#6CAI)z7D7vGKR}GljOXO(jFoBZU7;W!y(@EL z^PM)p4!oV)dg49-cdpP)!o45vN8wJEPsW`qbAzmWF%)rrZn1lHDB}FdVlP>232Gp5 zm+BB!9&(U}1gL);A`kS?mMcsia#GMdjwT23$T=+~EpLvcO%*6D-FAtzun-<;VaDmw zlDU%G74kCB$_qmg=NUAXU|iD`>{E+@RT?`}hYFoGi*<#((7Ru*PRoln3$m$`oaBXR zDS3HC2}Gd0e6maAg@y3Q3p3so^0KFu7luS$1f#qNMtKp8@*)`JMKH>XV7bUkw(*9$ zJLC$J7ypA>^h>F10umfqNyIJT8sr!1@y;G^SO||qFynNI*cs!^-c}wMlJQ2cd(kX{ zal8?X2lKLK#{@!-0pgp;>TroTeV0fC3*nIn zW}Ge&J7-MU%gO{p5$Bg^5W()#p@{Q>#r|wDN`lZ;>rlk`mAr_52Sld>-0W$3q;Icr zF%#5HKV5y3D6Y?NRo`x*;x5U_nzT##{JvF)$r(-AlySIn!yVtVzSCVTKhr2A=~Xg7 zxJgq6sF2cFBq@*6&ci-wSE|Q@1#$`LAp8+*oI0gZoZ}(}skS~%VuYCwivy8x!%UM| zyWrG9?oi2H{ZHQ0T*dqWg^=rC-NjtpjyuiKb3r8QAPhyEyDdh;R4@vF(7k4{-l(Wx zd}S@8`=mCq?bBRv2LFtH?2#(`)M0A2o&NMX>M5OhK zHardI!-V6Gf0JI(CaNi-5e^n7H{lTbrK6fjdw8d(QXat1C z$xT3ZWZ+qE#e<=Ua}U}_FdDLgy=t*HEXGAZp&Ot>5r^U-7{x>4y2N7`z>7k}V^j>O zt1DbQy40fW1=-N6qzjBUrJqH@sg5>hvB>LLOHG-w;r40Ybo*3g$VF6_amvKS0#&|@ z*?N0ERzk_Lo_a;J%{C`P5$8vECK!!T!Jf6)3l=Ly+X&q>9ZH*xaAD%Mkftk%AtiNn zpy_h79Migaj&U$5uwk{enYCfHsoD94)wW|=!)n9PHf%1v)Z47**k)xY;yeLf3r0gx zu#YYFFN<;ELFhQ$6gpRr%+{=hQw8b%`q^=G$Ymfn!);)uWbbr)VTGocs zCR*nkPTO2*4W|t*+i$C-xweht1)cMBK_E1{mX=BJX-i|2Kvux8b6mjkXKLw-lBN$h4 z1$)V2Y&yZ%bV8T@;&}9y@=bV=JSFu8c?c-IZ;-icz-u6WJi|vW-JYnKoPlecv%**8 z@QQF2z9+jg-XITBZ;(GKSL<@~YL~Cz$)oOVBZER#n4Y0&pEB|H^U+Y?{{7b-c-|=ByDWt zB*0YDq#$#JzANNln3V&Dzwl>?sTC0D2AK)>^}b-#`+{X_=Yv7%eM9Tk`#hIVu8y~}TD;ofhL)wT&^NTw(w}y|rF~w%GS#$} z-Z{GcFx%D)$=g@K?gQHedqIZ^9jjGZ95;ID>dw$djjYQE#~l@J2P|Bgk_T3vrrgQ9kx|QKbt(x%XHX;bL(QGda>>s&A{XbN_v#k}(JO+2|bL|*#FkcGNJx?uV7TW=7Vk9F?ETc!7%u-vBG>Hz{QF&Ty<%uDY9lCw^qwENFpv5>^2=+sZ{lsD&m8ZSH47oZlPuiI5VtKM=<%i+g$t=`$MxJ=uYlpQH zBN=&O89nl3*7$agJgu_w#E{67V3a4pC{Kb>o&=*j2}XGmjPfKHf2$7VI(6lETcza%;I8@#L(W|QT#vHg}EVFEA-H zcXhbLm}SY%NX&@X4oD0m8Hr&TJrZM9G=n6jb7R(ID>Dp5oZq0m1bak>BF<|Td(&b) z(Dp(X(V>V#NfGR7itiV5 zEXOm!R_aj1`L4z8u$Y@S7Xh6R!kd0E5U4AiH|mrIvGC<=0ac@Pn=DtB$PRkT67_z0 zM|o0XdBRWxpCEP|!ItSz#JS00w^+>0lWT#_4B<)77zosr&J*9=hr%P-5ZnnK$(Gu# z;n5_^BZeYQ1L`7Jvkpa^A6o1u7IX9HJ3tqP@Th+b1nNrXk$HQNmjgXIgkJ?Q5U4AiU#?@fM3PSH(@6!y1Y8OT zd3wkGj_9BxEpHf#IMql`uoXHaI!G|;Afa>f=2oEdLwM6W1_E`Z^TxBFWIi& z2}zPo?kP#?9KRM=elZkr)}UU3ovA}2OM+3Bgsvm}8W=+~b>+Y>OojCBY2labV(tXL zWbbcR@hfTh#gOnzF!D>VEjm=_kY9qi`E>)(iV*qYDF||<^DF3xB4O0%F`ev)Odzcz zcBUWBw|rtKg5!wLrxezzLlK9XT(F;6ti1jH@=fIhi^~?3@pP-DxkmKO15k3;NU z-hRQ5EkxR*Ji@A!2Vb!i?S1gqc>YC=pbxkz=HDEV)l(dMC1+}xk z`Gv>zQKk56x6#j(UpN)z+`W8L9DAj=R$Pd)v1YcPG<0TrOYzM1m&@X1r^N3Yg*1QD zjayh2BP6F~u@rIeCw2Pb*-(3b^hn8{K^4P~HYvY?BbP`?edWpyt%nW|5~p2s;uViw z3P0aOj+$DGCLE?JdY3wQkLnPKEtz02TT2Z1%pLX6#$dOi*NYcwdtIu}~Ins)fMEYEhwAuoCu6*-| zDaat!lW8?`N8JPh&NuP*lxQ15W&9H>{e4|)-FwD(z-8P0nQQ0eNP+r~y?;+v3@&tc zZ@b?vk8_FKog%-QRFvPgh4?^Mya9|iqi`f5o)R%9Y4W5k_x5~s-e#{EjLV-PJ9|&r z5w#Sh$-Ii;gE1M~^?UIVrbGPR7!%?*xmX(7LOt=LUhacmKm2G9($S}KQ9{y=veBne6acK$ut4=mH4=Gl&@0hDn zuHgEaUf?xr(WY8tm4Sph*p^y#=_aY#zNX)9%3xCp==x~d8nD+V0)4p zY?tEMWD;Qd>K+>TVibcdzTG-Gn*M5$9sgVvBYm(v3^BU5of|mV^0R~(Y}a`Q+bH8r zRfFw;cCh7{ZkdDa)S$t(RF%j39|zm97;KC90D}QP^vS`NAIW5Cdd57*z~OJ!n>FF! zwZicqWSfBdo&UB5c-mm92iZ2@X@e$O%{>U#*g;UON0frxTZ# z-^?gTzo~7WQE*>i90dpBm%6fyf}f9$+2KKtBi?>WJ9ETiDB`?lu@5ZP9Re+MyX#QI zImBZ0{+75rCtT=wx~E_#SuAd`l@{aao-r8)sWqsbJUsaw(f4^6o_Mrg$A_oAVyWRN-BTfGc%oZD z*zm-!>G};%gBZ^{Jn>|~E+3xwO==EL{LV8=V}_?bC;`_sSrXQpHOV$S`JF_PVbmIG&~)sR5OPshGcjWjKh;)9G(Q@@FW6gpsC@i=1{qI5wNkUClQ8!?+cR6b1XKs%$8~*viCnJm z;mF_hV#4IKuK@(Smd#k>%lQ0^tcRwiI4d%zD|$VYvwxZ5ELGDL_Y{Yd2Yx=rPrsL| z=|?G^$@F7EnnK+pM*ZH*1k&A#sNS0qiIU=WgdK?p-J8=UA8rNR4UdcR8@zLmBE}Ns z-rLSSrUG=wm4&{*xd(f1=Eb?kNX$JVe1y42Oe^8joJD!C! zecb+B;AwLU#s1Vn+H9kw-vwbQe&(WVh0P!u@SHOU8cv)+PzKEm;`<0+*cCH~P6%s< z^(x9t%YR?=I`I%;DB|2=vG3}TycH5Uz7-O>XD!_e7JJiTZ(EE$I1;y)4n>^3Ew-P< zjuVmWA!K>M51~e ziwdUH;}TT#oDlW+VyDz2JkvgJ`jHbQO^j9LAd?nD=)A4ecUh<@CS>1FE22xEEm zV;R6M*N^*a{rE82mxVF>Z00LfJ`N9$F-E_p#Xi7NhwfbTk}QOMN$9Ah88OmKfx_l&WN1;#fs2nWps|#s5`^F^f=siXC*xa4lHlRbz zrvVR!^x-7M9e15%0&a&1im}`9V0pd1lVjj3$&w#|JL@qPcQ*a$ie-$=Pz2Vf!PZ;s zev3V1F}GjIFMzHH;o0CA2-KymSTE0VTvLo$79QAH7BT@h3v=@Q3gY2f%R`1Du#y}{ zu#0sl;yh`wZ5HbY5BH8Cnz}l}Lo6^^9`4;)9x{P0C1@JJ)GEFgUWM`Si1l&yPOy51z@{A$ryMnRr3dX)G82heZZl2u%^u&<9 zyI%|h>go{B==EmV2)03oBFcM2F!D_3+&sG$XqaF0 zJ~0re%f&PAmui<)wj@$4+OM-LVgfD}bxNPDu{>f({GtVGMy&+Q^Ckn!N^m=+&siA=!7#Ln>W zV9P^>BFa}tuuH)i!N^0w$V0(!{zpZ79=;7#v8kKg=*AOImw1@D2cqyQ!tWm*X11)Z zY|3#GH+)rNZM7U;spoMOi5fY0E42r14|Ot^e(rZN*B)vMEsrb3YR`iDCc9WpcMKk0 z5qwN(to&lIN51jI$*4K9R8I<3Y0UXe8cUtxg|yj7P$2dqU+C@W$ZnW6V7WU-H5HP_MaP*)r4iDqCM>*v93On0eHwJ|5prj7NiHLZ;`mTKd!XyC(b12YuCiHD9O z*yTDDaeiR2A6xAIEcSbg(O*>J(qB~4^X;|S8u*f!GR5|p9eQE9S%Cx zz?|8e2G&#Rv} zhYB64Wx=SHg^p@jFsfz2ighSJE$@lPCx@uzf9_bLss#jBKhaQg56@V`z+m7i?LC4dO8%)sJr2@ zcc-A+fO^-jQT6v=PB&`iK=5E6->6s&KBQg|G zP78vqLWKmQoh#TC7NZIlj4D{@2BU2Ri|G&>@&1ZU<*|xQrGvPOaUnJmEGS#3_A*|$ z;`;J+Z$5_T~C7?D+PQRU=HuJ+gZOYrHpF@|?(z5o2{kb=C`slw^VE;q$wDwsyPs1Jw zY=@k_niR2V?OR7bt`h%i4^Wl0KPP)2r{7h+>EvIQU%2qDneC6N=fA{0&h2$=n zf1MoJzV+?i_m&l)2CWB{w}Y}B+cO{D-bdk6i-<=QaeLnOxdpQ}HTId=9v_7Jpz?E{ zNe&Y-6@6cys692NV#W}2C}Ixna7=94jE_gm@rc=bV+5>E<=gw*kGt_@zZ4slIiqO9 zb~MMIE&BTJ5uCZ{Xe3fFbJIBm!hjBye&*I=`mp6$cZ3Hq+_74Wet+hs)rteScaQ_V zf>RPIIB@jBu9jJ=XSF|I!~zWZla`6Ckcmej6I*9qIHy=k83`K|PuE=<6*A^j^H>z24_c{|gfRi&xOz@)NYTOoARk15QD^dnh=pO9roL z=_`kYRk;E^f!#z!N^Rkp{uJu^Jc{)c>M9$AA-nBm)Y zzqHW7eESh_S6aIxt=-*i(3@$ENb$e*0Pw!INy`X#YayxtlqqeV*}k>B{pulu*c$8Z zr)J~!=CYH^mc&mv`MxFidmr?de6>9I8+3$xhc75)yYt5g#nn@~=X|7|{sI3B-3JKC z#-=D}WMdO2Ecq(N2K+BEevOdCcmpAE?)w));@r0n)Z%k0<>d&yrb2fjB&i^vK3(0` z7cW{pc2glXj+Nl~+aUUWYKS$y;yh_wCb2Cu8k?s!HmsU7%IzplZx(!$I43f-05KId zk?-e*$EgL0#bA+|-8jF=@8L{~_MzCr2~RdR$!QackZw(Kje%cAS#%-(9;1LC<=+Zt z#z&oK9yj+c%*%6LCj+p)!4vBX_GA7n)s@Lgl@9$?-cs>;tG`q3gG;edWm=R2-XO#~ zPL+FEU1QbC+J@!vrpo5Zdf8LO6n>{tI8ddK+FUIu48?Pv#+g(!v!~V8E3#U^uPSJe z%4jaaDYD}X8MWY771|PoegWHLVdX96);7;T2;}~N%Q!4juI5{5+d-{~_0>p?EnOw}>U6S8O=}vHCY$BBI1V8$G zDc%0=(hXFl+drUmLjy}!>RUQ@%5)I!zTv(eJ&KB^x;LHsJKH`69Yt3S_R*Q>i;GlW zyaB$c`#~4&i>@#wdORqeRCMF=+;!d~hB_B)jQD%uMpQ}zkaFrcT6(c|-xx`gOLvOh zm}36M9^|51qKX)A!_#ULU^LAU*Bx6NaW_0|+JJZW^M@(JlQx{^E>R!0!-n(N4!ztt zDjD8zc~CoSxXrg6+_`O4QnF!{8)GLc+HJ!sQ$qJM)x(b5Fxp)@)x$Q7X6g3W+r>vI z9b}<nMws$G815=7Tu=8pTkYOBk@lfoWuoRjYE>%RefcQuyucgYA> zS@598`pevP_15>d%x%j0dG7iueRbIST6a!LU#+hVsPD?a`ks@uzE80P1J}=U=k|iC z*E|%zcN!0+z9*6o8TI8-As!U0pBGl&2btUC_5I!TReIsD^@q4ip!CA}LjvkMEU>;u zWv%ZMEJ45Z{oT3!N!7PMir=TZ8<+YH1YVR;U%s)zgYN75ht*eUmEG3&cGs6$r3jdb z=YDn{OINn;5jUpN7`5&>cgd7yTK61kJG@7RW?J{M)=d4=T2J+Wb)O(kj||P^&hP(N zhTiKQap(6OIkfH(RJbtTjVlhVdmMOX58&$`Jjh@7h^eW$yK_@Le&RZx9>4BJcTTFu zue;G-BG!E?NFvtVomC=KkMF%Mv&XMfJ>I9jSNhgh_xSvE*?auNbtk%cAU%FvW{+PN z(Bszy_V{%<_W0iGPIPlu_4swtv)#C~?fP)1 zL(BM#`s;?Pw)C+L*G)r=Oxw_%GLORQ-gX5?)kg2Ant80zO&tF4u_-y`D9YS3dBD#o=Fl(UhxEJ$dj<3{5Y*miSlw9f}wkUfj$0^PfXlcxNeFZuk}8*AqXp$AMNb zZSG$9^~SFcezbaMWA($YKYna4O2FQ8~yQ4U7nM@yV19CHHCIln&P zm~H}GMUmqx^n9MnBWgy+^3eIJo~l%k%fep1<$%{Qa=!?~goxAF1rtRDN?jf1l;~ z`(n@EcYFT+UE1HLM7iwB`qjW0ll3`KS-h$IzT^3OOWNP3M0qR&W5pSYJB&1*j?;#r$~hEw`kNevdjsk_5_cYlkFL;m zAMP>SPe8d3#Qj^iAB6j>xKF|T9ptwHX>aoEvnc}3iyA0(wJqHl3um4!He4lgEu-~y zrbl^@0(+WVyF(JvSGo)hdzqmLUez+x?0<$jwKS!An;~H*%p_60&yZ{}6yQ~`kCCHK z3Jc;lWI|nqHZ=xz0l!PU%a*2IDSj8`!0*D4RYa3wir!0%Bx2>8Gp6w1I3DU^ZP6^ffAK%op& z3MH1tGW$pBs7LDX&lk_fWf$@}RY+}i`O@|`R1s2KKOHJ``dDnR#bOp4Zn0vEO|n?2#VRZ|*J4X7w$x&E z7HhKDnHFoa7?x>un_p?M>VA%M0p^K?&I%ohIB!_&?-uLT-_Z5dp@=idVx<;~Tdc}r z>n-*bi#>_Z*HpgSbOkvK?Wic$9tEUm??-qO4VuN8&N!*wYMVzB8Hq~OwELLN&A0l+6%J(Na6mcfP0Yb3J zI)o1kS!|ibF0j~Ui`{OqJ1zFO#r|NiA$vK_#VX&SIuvofW3k&U_Nc|4u-Io7%fpHx zQ;9e+9V&E&SZu1rj-cM|0wL{L$8%K8 zIG)IRN_OYS{{6nNgIvC_gXs$!>$ETIpf7x3|G)Kxy$)SQ`VEiM701y3ZeLiBu=EuZ zZ`T2sOGqAL5t1|Ajzvh$bW8Vzl~nLh`NGon3-g8j0bUNs)fTd(KRP#G*i14tH-2E; z4*9~$Ny=aN!bZ2Cf!v<0Jh<573(GT&|KogNe+6sM>kC`RitH3$*n7z}k1s6Q_V4qB zb^AN;y?6JGy?wn+GmkDD%@)dLY@9|xG_l;ljc$$_0b;lNehmBuy z`#ZRwi^uPVZEj8}54sKXefIH3-tc0O&&h_5e0@&bxzT3JJ)w_0*A1u%J5AuV(VWxx2{UNd)^@H6D5@h^fGY_Qq^MgGUF~a;{Q!>8rgY{Yfs>grf2McNX!VflOY3@8f*dJilK+jWZ z7PlWP|L&L{>?K%zrynf!k=qZJy2$MZ%UOcg50<%cM!~sQE`G4oL66}VTP0(dOD4 zzkTrA7r*`RqaQ3SD?dM2&i6fju=1UpFZ^JmUF8SMj|Fwy4|YQI1FR|WGJe5DYL;mJ zJ|)^;{k@yRbKIH!f0G|9USYL8i29?uxAjIn(XEvqEMDB~3_sY+)iB=;yN2JRa!}x3_+%QN%r4@&_`!w>`4>FzVxEg1?0+-Q=?B{b_M`a0G9-Smg59W} ziXW_CcUo+ljr**{_!eB~=m#t5(GONI`oRiDKUl%&2P+u;UG^nh2ZVDyL;x;-sMk66JfEJlx5 z!9Ii^ti+{9tY8K3?G&s?ha%1(iw(BeP>T(>*m#RgvKT&BsMD*k*zp!yVzCt#tFzb| zi=An)^DTCP#jdp2)fT(OV&ApcPb~H`i#=qq-&$;&#h$g;YZiOcVjo)UQ;QYA|3LFk zhrmCJ4Yt@&i_s%ict74^lPp$lF?z%b-SHM%VzCt#qeraJt+5!rA_UuPF?z%bwiKSO z7b>};M=V1T=T(c*BUZ2oyj6vc9Uw&R(IZx{ zJ1s_!SiuV5!FsvMmmaYU;UfSRqerY@cUp`dv4Xv5F?z%bmX8G!h6){e#0qwZ#pn?$ z*v|EX?S~TVfFJC27-~D~E<1V)zKT%3sjR^LnTeN?@P(dJ2Qe(Ei@v^GL6%EC%M+iQ zY>x9&MY(+*0jCdkG=^F^x_DfG^VJ?9yJq<%1RgX{fC4iCy_(C7?ds6pxuz<(vyO9N zGvFW2%8;uZD{A~0E}WQ=i(KdCJ8jqm?|cP!H+@fFq^EWDHX?|1#gH6?FW8Uu(}?q- zrTf%kz72xYwVdySGzgEkkSnY~a{aVHVEg2pfNHKc$9A&0n1H*v(%C-51l-JCJdeD=$ z2QefnPp}^&SHYa!Q+cUR32>%xx#voKYP{pje`12NEFp1)tzk;Cg5(8 zovYTKw~fLOc58xZg8f8?A`Y8EFb;h}H+}S$3Vtt%mO(e#2<=mF0KBpj@DZhoO;ZZ& zhm`6N$=g&1N!##%7Ij#X{3%V<;cxd%oY>(gs;O=YQfLRM6N+z8kK=w{5;zEBeaZuX zZ>I}+P*y>Y!sF;R)196p-lK90l}C*_E-U!Z6q+M_@nI%tEM-RVML3f;CJKwJg~NQG ziN(bCvr?ZMU>}B=VK4HUCJI&)c)*Uj%646Fm(%^!z!3BHD(mi9bY7|81c*c^@ zxf&cw0$*>)e|vjrrrxj$gF@q#g`voS*CF`z_2~8v_x>(d7(ih6JMw2}Y$M7?p-#JONCw zD|IM9X%zVgid$*$xIVc)KT2bVTQrL%FpZY2!+)SV{D->3f22G7KWt+$B#k8)8%wYl z_$?Th=LPG4A@Eg1y3J#=xebBGb8D`gW-X)Cv?o9_L$nVGxYMAlN^3sL=VyVl)7Rj#i`4b<~LXrjMMsjff~XA=iI~oQR3h zEmsp`992n&O^go7l-tB$L}uwDTgJp-(bG%}Q9zwFGV-j9F(fi37-dW_DjdP6a0L4y zH9C&?33dI)aRl?bZlkNE%ZxHl|>u zXb-`r=}^S!LL-E(>s{5Hvi8~;Bg8JgeyBQraz(9tqUcyuT`n;~-WR=)eXWAt^x;Mb z*Pc6UgfJ2v_nExW0W7k|2w^dIff3Tz>I8;Fj0B??2}YeD*asG)5h56k5TSF8JNMcd zYBnQ;`z7S+yiV8^LbSt1NJ>~bV1zJ|kC5ynBczWN28Ki!1fwtrMk7Qp8X$PI&G9)r47-dW_DjdP6a0H_fA{dPj!CW$yQaIU+5N^(r>pw$~(v6S~wYV`t0<=iy zjga29F&UD^6pW227>y9YXoLvX0V5>aD-nJ#B8TSme-||&Z0RF7ikhb0GZt2{6KH1K ztsdvF*1-=}gqsIuLNo(&yX~@U~F)~p398rlNc>b5lQ+GXMfR8iEM}T1uybB};On4zY+xwj{1}ZU*iUsR;&5yb>=}#E1Q6_e z9ST^5q#vnQP~5AK{34lLJJ7FplA!FcIgk>F4wwUs|FP^0(%4hWdY_YK_)fexTI zLofk7mI~7{+{su*uRGb`|3}_;z*kXpeeWgIODLfi(EtHyViG!15<&@rNJlJ$kc1+T zK!_9zqF_NlQLv#Rq9`H?ihy0f238asiVxWJ5f!WmD&HwHvpd`F-sHxI*KdExy|e#& z%FH=u%FfQt#_^J%hVYFq(rX0YxLgR{rz`qLOofeU3M&$+uz`arrh7(V^X4QPnM?3t z3Ii6I7FEhz)W71lc=iHH-w!{Oh1*&15>Aj>mKG-+hO_^||0J758vuq!efWP}o*3Fz zz`f4OE-IWfyW5OO6Y}$N6LThJ7q*#{Uj&P@=MK)!ne6p5*?V>;WpCW7)Vx2_03H=7 zr`>XU1+>F(2j1#k9hZ3pTmOel(lw|{a}9Q6iSREEEZ%IGyTUN5CBP3u?1`~zb_-Ks z<9hg=v71B+)`J2gV`W7uY!uHEb%Fm;A5UPh_|QBd9AVw3*~|Ja`!$odN+NhNYs=+@#h{uQKxqz!; zM3o9iP5!9o<&Wx6?sD*dZUPj~2<{IQX!VD)L&x%BJ+gYFTp9LAdHBW^)fM3z?NJ53 zv86jpc?45oW4*#Qid5LZTE%qOCotW-C$ELyjiUI>MGNOag8hQ+e`LQv2-1NqAwDX} zNA!MaC!4xJerd;#1tmZXiz#Eh<*Ng)eJze(P@unGu)KbL!NsWP7qo};3%(|nZ_!|i z{lXafg)#ICW9S#gcCh=gTPXwJZMG@_vea>EqD(qE>Vk0>27dT)uBAT@&PR0w-xuyW zeIEy_|9jIzj={Q{Pa*JRcY{JTe_)u8Q|81}*mzc9+eIpDSWK4)rrY9Ux@L*MMMp%_ z1&U2x{^@7Qz+%X-lS;#|DAZzDH0#;Qte6VZEHY#J#jjDBwPhmN6h2~OR$hxxVza7M zEx22D%;Ob>+RQplnH5uPR>sh*j9H>JQblhX+^p`iY@VMS-mVmfR>SNK%)#w9hKn5&B3cIM0`zGWchQHMe;#>2|oNtq5)VdHs) zy&zIyBW+mb{DX;^i$=qPF&`Io%v^YBQu6c1;aK#_p=#SRD$YL=x~O61qPWaOuN|)r z$R+XfkB2UT>mWh~!&5ZCkU9Uw#PihkBrZBxYIDH=P>7?IKJbql!1BS3g2()@*MJ)z zC!QNM4@Ir#19z?IahWsWIrcQlS!L~l33sTiTpli%QYu-WqQMPqD_1LDHm(jlMN_jP zzR%ot)!QE|fls#?AlCZa#b?mF#hr#%U}5>7jl2>|By#(Q^|3r;M%9=r>>`ta*z#B& zZYA1U@XM`7yIaoPXCTl&_G{^j;V~NY*(LCee!EmWR&%6`9LZoRY;1<#7<*Er!p2Jq z+oiBNU=5~gAW~uDI)z=YuoVi!NMaqhP=$GMp$cQA5W3HR@JDV0=Q)ocF$`+zQ#36yzO$7>7gIdYUcA;CJ{)s3xOW<~g_e0iM5v{+UGbP*bkG?;bWTOk>$u;C z=9jAqFkhHgyhH(es-4ani`#uo#BIK}l-V&AHl_nDV;6~3*x0797ZsMdXotFd-3e%_ z%}$_}4zibNr-EsDh1o>~g^Brjv)~G{M?zDeg55ZVO}aQwTu}-}n8KG6pw)2K4!EuY zx>2xE0~4Bic9iwKKZ?7vu)RN7yEcydB?P<3w zb744h;hC^)Ple1ySrrE^?46W6aA7~1i0GZUuqdhXb2Jh0vg<^|jQII~>k|>ckh$oU z%thxXrQw7`)(e~jh|aL=hvEMTD9r?$e-{9b7wR+NAKx*+E7SUG5lVz6V10;eggy6`1C@iiM+Iq}^(agAU~qh`S8U%ASbSob?&5=h2UMx zRM~KRdOJ{s8>vN{S@C)M>+eDnE z>4pFti(#4uV;NR0%)@-+ImQ>BPlWI5;d?TC-vHk@OZbPX=a_=-Mqx`uDs0@TuvH4% zt}q;fG4Fc{+pDm9z_qMnjYx%!M-{eNVfTS{rdumgu;Nu=SZi4awn0p{TVW?s`|=9p zYNk*7@&d#LAHMBNpetp*&7?Oso^wIo&5gE&NVLrj&WYKY8=S!@mFDJcu$`^B`2?U+ zZEk)B1-{MAJD4d)bA#KV#G<)D^P)ZRnRhHcYQr+iEmyF)!7O5Pa}xZ^=EgE}CYu{= z3zi_w4K2Bfb$e451{pjw##Gq211bY!cZ*aQHj|au28Hcb81A3Ryk98n8-;Cz7KP>E zfSBpFDeOgsZGw8rbX!F#Y`mZ_ti{ZW*MLlSKwxdZ`#qLeR_N2IZ5Z; z-e|i8MBCobJ5BZB4ZYH|G=?|yK2v>oL$5O}&EbuB_o?^r26u>Xwl}!*K`h!EG%wl{ zA8(bfQp7UL-4*2P4;v(MnUZ?xSO{%7q?E!Ey&irX8;?uKf?7`8WzVSB?E zwl|Dnd&3yEH;iF>!x*+VjA47j7`8WzVSB?Ewl|Dnd&3yEH;iF>!x*+VjA47j*ooBM zM7n;$x770C+upd(Lh-@`y@BzZO!aPHwC9AQZD4TP*fyZSDbP|G(0l;KvJGes092|2 zn%_Wy?|=rM{&qGn_{?)G8W=P$+7lm|!{VbhEVF#B8V59(MI6xJyH8~UW0`uE4UG1j zb&v)I=agbSp7{i7DUW9`#n(@a;q?<^c>ToKDur!V7+yay9bP{%9bP{%hSyJw;q?<^ zc>Tl}UOzF0*H4V$^%G-w{lpm7TE_7Di7~u>V(dg}U!H_qP4u~b+6A$}M={zLF*U9? zFrE|M-VKcQ^kK9O3{LLb8W^0WE|mu6J+PLof!PmGsWvdbf&$+L2A?W*HZb@kWh@#P zG%wl{AHl)mqc$wFd|DG57|bFzFek&mY+x+62V?`IJ-r#Efq}p*jW*^Rs1w}AU@8o6 zKY{rs!fq6)uyL2d?orqdg}tmWT%ymsA1Q1#v?7e%FH*2QzQVRBYz^pSx(7uH?#d|) zYcBo@?>bP}y9&!(QuanQ$&PnCGHjK+9AJ3Be3r4j1|{KxD|vQdVfLJE@Gwl9R9OBc ztM7Ri&;;v8Cwr`?D@Q*!1t~DuKDdClE;FZ6%SD)K$H1rv%qU;ntw*RjKqwPt+EFGg zs6wYF!@oSREZbnX34dy$eG3!@e}5Xjv5e2aH%`F)7fi&?#S}b1rm!g@6^17vB;6u~ z9Z(n!OPMz+Bja1y`LG)q7vkHDd?RGlbgvFjv2m>ak;TzgHKC~w;ylzl*?K*ZwPWF`S z!o=LXDS1Vc@+UxIMp5BKIdCWIRH`wvArlr~<1)rZq=9cdm^b+v!Q)-#dkFBcw)m>y z_{ufwSx|h$3edQkcr&4l^)Z=X^Y2w*{vGULpd8d7n`_LZWx*@27S2{~#uPV{jA28` z7&erQ-J`H5U5fX@O5)NMV9~|4=FjTV3hpi~$ungE+-`Z|43}H^xjndpeQvI#-G~*D zo|=-67h)S~drl471>=0>R!qVD1BGoCsj#tKVK|;7eMCP>P}<8@HQso(rBItA9V1*#Io=t_ycJHykUD>4I8{lq!wRUZC!6# ziAIWvoG067A~a*NkHR%HrvjO@v+dC}G;@wJGp54Ey$X9kq{7A~g>6+>G0coFWwE~} z(Q4+TM37ELL^B6$w(!}Eh;_jBEL&`>-Yn6yy6w3(1I<2InH^L7@-N0(NO-Hn|?a{c{*tuzKHy~Hw@bMT$?KffI{p>aa4n=l=u*?i2gmez4LMk z^TyLfq37#uoy&slYs_Nyl$_s4CqebXa;28+g)Q&Z#W}pz!YFc)Rd>s|n#{rFS@J-e z;wmfnqYyy+0(fl^+Ib9o6ooXAT+%T7Q-vX>!mxb=ylRgyyd}jLwp)y?SJ-C?`$}Ph!0${qRHVYj7=>X} zu)Ns{yHsKSgD}KJ41AP040(iQ4MWd&5ufm3Xn6_pKNp7as?)?2haqDahKyktGKOKu z7=|HZ7>0~t7&3-o$QXtpV;F{vVHh%oVaQl1gkhvk7;km+;TMK}BTj4FdA@M@gpWJR zo1O#39o|tXY3uhMwRAYAh$-%b8N*JPG3o&gGYo7wG;oLK$sd^yA|4KkkRO6L^yb7>9V1|LvSman<1+=rA%SSEJ1cqN&Uc3k> z2jA@xfMZ}BKWYIxsLg|W#JGFXyr%FCQ%CdAE<@oPF9zOL0f8yL`N-H5@oU(at>`XQ z*gl0FP}o6*p;uU5S+F6?t0+>inJT@$A^zgXUS_rn@`PY?7F_U*ugd87Q4+k91!MnFV3y;Ft7G&vJqfz%=1Y2#%r>3 z;2X9#5CMTKQvwF|e(3q1t}H^8#hU%;!w z!|;!_INxSjSspa%QCNDyhr_bbOm0J!g1fhQS+qeqK2c?x%r<@CyW=WMUP~WXCabZ6 zR1vvc-XEX2JUanfM(j%Efn`Duz<^FIai~{;Jy0LMv8)Z?8!G^KK(vUM3LDE5cB@Fi znh}Me1(^=#{FoQRo-uFBe-1R4`B?t1;Fgc>>Qr}xZGJm?M_A^st&W%_91&g;qYK@FU5JE@9)+oX?if($_EO_*$x6E=|O+O|E ztx>wNJf^rcVhmd&#;`SF3@ykQwnmI$Ys8qh<#8=c1VQ59BLz^=l4tyA!Hz(QLo;$rd zt-Y%EE&OcVpBsk&`X07bq0_$5<{EjughSb5!J8+qRaST>6uM|wS5+{Frq3i^TqfV!3qoH0SN; zRL~yw?!Bhy-fO`(+Nrka-bX8!Vk!)8iZTqwu%l-z32e?-3z5QIpl976r`cQ6$;@Pd;u?ED$SeMcC>eMcCZ_as(G-Vu5!B-lqd ztc&a;taEtOhW{fm_L1uBO&?i0dfi972ej2kxD1AT1d9;^`v{r-fMVeKAX$SfV*_^| zp$+_egtn1Bnyh?;DfSU#=p)9^M~tD57|UFgUOjXE_Ui54$vmDYuRwkV7JKt1NAq~e z9!i7{I*Qrz@Kah^^XA##SJaxeGKnx1TefUu%gZZxWRMoO4GFL)czn{4FHoonYrz^Kp4vdY@u}!1H;R4?ID63yyQ5LP1&MTzO z{86a8mIcygXKwC#lmM6HskM+SD{Fi~;neIYw5H5_`hZEXA;-BakpbHm_Te8whUxB9 z7+!}k2BoDG)P*JK=9I|uKwy6#3eKb90risxCUg>cn{eOp7Px6z5qroo+|rlw`9Jgl z#uhVDme~+$-tE&Ln18!rtBCPYRrtnoRTJYQn1~XF#T38ugt41|im`V^sw9S*rLSMy{9rFrntH@hSi-htnQ3qb!SX7;x>TogE(t-FP8`e zbQIHwa@=6CqUW$7s1+?^0=Ld6nn-M?jEE^7m@$R}Gse(}jG+-3J9IqVyvu<%ckMT> ztVTRD;*G2R`Dy{2=S_~F*0J2=pcr$`qqrHUm*}lC;2S#?Sh-2PH9lj>y)~xzrV?XQ zfQqq=B30RVTw$3D69z7{*ys%Cv+bR8MWDu`0{pj0#1tMP(4t>OI1GkQ&@pEf=1uco zN?6gT3B#t_JJGO--qMJCwt*89;RbRrwqha_jIAf$8Res*->53n?B*?N1M7-sJ5fE1 zs!a9xLp>0SD%B6DQneJFrCRt#+u^_feTAbcJSM}cK>VHp-w@5lRQN{Q=8M&WKPz8i zidziEu*G0(lSoyj%|sc)uxGk`iVkhS7~UObI=jztz-k{$Sbg3)5ya8q=ksWW5%v)9 zT*t@W{ROTfly^%x1S|{lWC);ozf}xiio0IM?g39QwnLaoXw=OQ|ef$#spX27y4PR0PL|EFW1 zs_OJG#WBDb#sFg&1B|__FpL4lFb0?oV}LP?0mkex5UK5Nn+W3Q@QVT8L7Ls^o-5ec z=^OvSif>txCY_Gzqi{N26qdB@|3KAyOmW-K7`FY4VcXBxR)t~P<A-OowehW7zgH zX7_nF_{Bc3w6^`HC4x9QqW8HRQ`(*GxqOVBKII>Dx@A3>bULaJ?(|sr{4?cqO!45B zF&x}7h9fq{o=_O>3dGna3PYbWhCXLr*j{-cEcjU%8o01bChWL8a8WvJvz!ilEcY@8 z#}@Ey44j6k{Vc#pd!wNeU;)Cg0i&WTHrLg-H7L#DZ{VVw>aj?mutPD2KF)Rn$ zcctI3;f9=+V8dZ+iG$5;lTQpb%P?668Iyn zhrz~l7;KDTurY?g#ux@0V}XJ#Qn!hF8S>#5Z2n_r8C{-(Wsa^#O1~+?ofs|Ah2!HA zM;CUR_8lHEx-8>y8C_UHTXe;u+k9WOWSHV^lQHZz8N+UqvGoeWZj-Uk6o%a4MGcmic4eZYc%ibpB6(MC_ z?kJ#2?cIA6&G}|fG|_5x{XSPKqk(1ZTRmEyQ@K41)N5CYFI(>v%WqxbxA?L~e3XfqLIIcod4Cwo zW8L=%BPZD+32iwWdM@}=%i}x1JK>R}BYdNcJHdAw`2Jo+45q>cHbRWuAW~rir*jy? z=^Unuz1M#VlGn1_y!K9*z1g;*BL{vGlpJg47}dLy{@-_ zx}I2!y8dS%uScxw?V_yfb+Aqv?sdHj{8rq$jvK!gyRKuVPylA2Bz3)usOw!tT|YzA z^&eDS#}s>=G4wiP=yk@>>x{+T>xTn*Jz`x~UJqE;|Ic{6ICb62>y=bp#}s>=G4wiP z=yk@>>x_ZdJG<9)c-L8avP)gZ=|KB>LuJs+^r$D@ZlJl_$1E3W79EhWYFJZ1_7UAjJHv0L*UQsjzEZ%!JidCb*j~p>p#aQ)^tyd8?_8OK_OuV@ zQ$&?dg>STTJ5l8isw$5u-h`O38-a>3bUb6|c*bJx_@4tgK4OixINr0m{~vX{t-6DI z>@_|b$5&7_9#iai#?bMMq2n1t$1@gaYUp+#_q$!U$2K+O@%CrG`*kzq%#i0ROU1k+ zUDfdlDc{?5cngu{cdEx*@5TL7ckv=TW&(He8o*<6xabZu>UB5I_To;woA(G^1_}ds ztmTt;SRfMA1i(6Y=O&yQ55YIy##;~HI00}}wHTP<`AEiaDuA)IiVo)^8T(aXe<&c@!k#=bWz;LQ!(iI9!rkFNAn`Zj`i z_c5jc7~LJe-~osk^}6$i#t;)~Ae(sb!Lpsv3TX z<4(oFDdgBmSYx=kR<4K_(n${dmFePD3eCzKNYw}a<>4m#O_nW4q+EQy!qjhSV7VxB zfE21Ib)I*9OL>9CZX!csDay_9aU4|tZ+)Ng8Fx%N`J$^D8;NfA&{ zML;U#O&%*%NjZfjJ!qCR7oSuy49S7NG98wbQ&>{wK&qMWFAq1_KWQbEa?v-Yep3U> zMVSkvP(`Wp9O06BMD5+Gl$^qH{B4%fC2E-tOUWrLC3D#Q=_b1@WG>EV3_N&Pqt?`K zYGAo`f4a#9M}l0hy17b~l2cg9>YNjgy2*4{N={)ZnZsU6H`#SOOKIviHLzTJDc$6r z#HI9!+Iv(#sCysh>mPRSuOVY}Rba+4qMh7VM{RSb%V#*m>|9VQ{ zLhV~9h2^@9Qf(l0Bc<9y>LyBcgVb_Lod+pM7;E7lb6y^@f#z3DNX&R z29|3trJLLtTuPtTZjCA_r?8}R%#ymac1(vQSo9xTAl3H6kQ@^Q!<=R_2 zH@R1FNj>`L`&B79g{53%meQq>IUGTK#G2H#M+ayFcCJ-s)S* z2UICJg{8d9x0FnWrQ{Trk~!?9bd!CrXDLnnrUsU4FQuE@2f36!QM*=^lv7yJ4Q5GQ zqL%5fq@2Q%GKW2C-DGdpN@|T-Q@^Q!<=UgxP43eqH~K#M72<;;Xnpj9$|{^ft86FA zX!__^i35ADHz*a0K6(k|46T0yrLd2_l~UM8-%hC{NG+ul_R-5Ig?;qRlnUBMCqka# z&lpO1hXfquNdj7fQ|Nn6VJSJ4s8TIdDp{qxdR)#g>f;pZ*iD+fIJjKM!71c;-Nhp8 zTzfm$*nY2}ZTRp~k9ReF|yrRNmpaR}+5(&xgq`Rx;o>W{@A3LmIAO)Ah!e*w|sZ(m3P z_}-{W&nc8ViYWa`KUheRr5{TiSo$1FSxbM7R(iZICXZ88={aRBeWE0SZii=ao1#4wgziXDq%Jr1u;uLZ{YUWPd!LXL+X~n@Q{SauO!dLe%TF z2YDf*I&gHLR6R&_p;RKI_CRMHg74r*H6=MsNlsUi`;O7Z|Dc}M`s-IRn2 zh&p(wphs7{qm}k^mG;~K+P$>(RFWqv$y1c%!U&Sv_DmmXd1*TCVw*OjC^s#Q)a`-D zF#TtyU_mbY{)21W9Co0xMP`{KSbcm$)EG72MaB2K;ydc#L*ZDio=Nz0;aS;5g_CBd zi8HaNF)brAjUQG_vnDB9{)PU_RoyRyS`9>^Tt3inap1Bzsx;*(g+m3Lmj++i4lahp zL9juqEQ;$wk+h`bUIWuq**tv-6;mCUrAn^qB}qK}ioKxeU0GN^?{Y!WyIfGoBjvh% zV{U4YK5SKLQwj})EkkS!)H!J+L!_NeY=IpKq*?y@(nm^q~|=z>X(6i+V;} zYV;A8>8f28V*vdcy@k1)XkjiVNr1Bv@ITC`k}wt*`O^WNB)qr9e0`yP=WN&3xtmx~ zHFoCsqU>=~@=irjXv_DZ|EO%BUQC5)!V`BT84mwLW1yGc!(ggKoqLr|Oqn`!;J56k zbh`Pqo9%J`HmS5`*31Hxd!Nb!rp!EygFK9dJRENU2|BD>3vid-%?f!vtl;wexg`sf zhy5xKm@@Maz3sg+fv@;6wW94mQ#vta>MSMO^Lk}73smkeRUR+}@4WqH*}n5yr1q+l z(B;8Dub=(xYm>%ytJZ1SpC8xz_28IoS9aK)zj?swRXe)N^A`8ldgz&IS2eoqoOSDR zYJdH4tL~MqKKf4Wg9Bg7+Vn=oy7Iqf{8H_O?G?Y2=T*OH)Tv#q50cs+OZlVlkn!Zi zcYliC{LvF-QVTd-u_!7lQA&FTYN!W-t*lIKUB8F8%s702ev z^YQh^6`lO}$N!P%q1t)Vci&WN(vS1@SIjw(wdUDH@AuzUls-qE_k8NYFFv|`#251X zw*&XB?^Sa;~z;Me)zYE`%m65Po8J3 z%73WtFSo9f=Rb73IeXdo9$n@6e>3|m{{8B-*X8->nYj;5-m~f}c|JHhb4m3&pIjo( zpB=Wnal)D-Qk7!mq>IYVO-pt?vB91!HzCPyhYC*IKRrb;Q>n?rOH8{$qE} zes|2mH|8!_-TnF(^It8fRq3xOnFpso*X`m;W5Yju@#4ujcR!ZW>fyfc?D%5uscU<+ z-CeitsQMQ?ck7^$@kv`6j#}{RI}NuVd#;+qD|R_@`NDBaTS;8zW6_9@KX|fr>Q(am z<0C__toYl459Rrc&#u`!_}QH&$@A`~RV}*euDl`geAw=uO|CfQl9%N9g;%|o^yi;* z56kn%c1=2S<-AAh%kza>I+UMWW$83|p7PqUp)>w7yoNmAGAe1$d7sw2NuFQ2|Mkc3 z`s}8SR}_7D>s{>$c2|q`0#h~#iTGXhz zc2An*U;9qQ_A6R{nIPqNdF+S#zR7-Zx#Vwk{)Rhm*|u=6{JqKa`CoswE-qX0H@|Sh zIo}-`@VWf`y{$7I{e9=VT_yhu-Nrl>w{}Uk{Jlqhn~OTV{8Tl`|MP& z{nZa{ntuGV$EsB6D9_uTKJt=}N4z^uo>zT;d-=IFGG3DB7p%D8o_U?p_sjEZbN_6Z zn|AG$QvSZwGWBYI*X5euM|S?(sPop9*+*9`YIN1%6$Ag6zh~hS7pyFQ*9*07{q+4b zdA_^L>&Iqv-qTs0U-j3Zlke(tayNPYMryA)S6}wtoAUhTMU7TY>HFrVHE((O&lj>! zJG=AXw7v_Cy1V50fTPPleeR6!{~OADs$loubGEgdc<#KT*)2Y8zq!rEmK%Hgr%c(^ zH#K~s)vlkWzMir&rTRyE&b)B?4R<|!@QeM;>fbxy$c~5ayRvhmO^@cDQ!n9-rpF## zJ^cHRTRq+UH?%6RqL#hW*;Yjov-LuE$am)AX9 zXY;FD-jnA!ZGPT=eBBMVOM5jx+~|zACse;&o_A}%zGC{ygg52+!dn;p@lE$jy2$e< zlWut=ZCL0ddH(mURl1D$c1e#@etzP@j8k4+vOoR6wjR5xcV518^qA!je>W$t;H#5{ zemv~6Uk`Qq>ETV+40z$KSs%SI=ahZ&{OeDD8J&I2ogsOC<~vOv`JzdFH+i1E=A}xz z+rCv^o_zG#e7tLSyf;?Z|=jj)pZ#?#IdH&q0)+g_LbztikM+~m^ z^?|~Kt=}A3_};X;|-}qO1!ae`)$Rmo*+K&!4+Jq3*bTDOJih9Y1kZ%dDdMV^?gyy6D>) z@_g=d2GxL7k10v++xhN-){IgXOuj@_Nl!)Djt5jTgBmVV;+05e8w5K zj6D5~@L&7ndBxAZUYYXSj7>5>W6n?8e?#1{>hgT{-Dj=(@3y+-V2??)+FaM`yKBxoZPdul1)uyf z>cymbOD?T^&Dt$jHaqjSRZm=*SLV*GMYH_mBayyliaF!uUCL7ae))r)$Hv zz4Ps?IYTx)me{9ae79|f&yRE-ZN8(WkG7oA%8!;l+WbVa`|``*-~ZCa1-&EL+5UU9 z`Hq%8+H!u<;`qa#E?>W5$c=Zsc-5<`pBeu{%NzF{=-RRHzM~y_f6;aRrM>@}y7-Gd zN%nFSmoM(qm37v>&~N;Iv*Necs#j^~69eMru0CzdJgIN~k@W)-hxdHju5Z=cmMP7r z>@V1{=Ew|a`g(P}t6wdyK56%y&NsjI z`%fKf%JV(vz4-i{kN>?%o_qZ++qt0!%N$Rdx+Bw0|JR}1%Fi$SW!%YYMwhRedu1a@ z|I>K~zx@2|OW(4e+xc%keA_qwU2%Q)7eD=@Z*KO2>XKf{v(wx4wQ2umqo2;0zu5lW zsb9t8zfZsGJv&~I{OQJU?G?+K9PGOPn)J^fuYK^6Lz^q4KHg*dTNxcTmTe%v^Q(Sv zXlVU*E9`U$&m^2*rg`d}_dovoh-cRI{k_?_=k859^~#xlimdnv8kl$4%K) zcS^bgi|&|w)%|0KN1}UZSbpt~<9_WGiLTiVH}t=HWZox{=%oI>9SbsQeDza9FZwqM zR{i?=Ny}C)YrFO73&y2hSat5i7TvG^d+K$Yk`D}>^~bzTU##7+Ys=!4(TV@5eCM># zk75GONXsdhI<4)b+*>A<3*kc^aYowsyzHX3$;nGY#ym#a<`(AV6wS;-9-Lc_3*p1o z`q4HwZ`?%_l7=CQx1r)fW9RA=aQlvFa0-s*nBzjX*-4DHMR~J}lBVY44>(U77wQb3 zDDldPTa!6u)diDWK+_%wa7G)Cvba)DlLOPn5#`_mCAysQ4mo&Ro%4+O`SZ#ARnGiW zBAOz|Ixd;TqoPAj7|^&-ANWK$@pl~D=af^KXcA#69_Q!r2s`9pi6hCWLNr!6@eVmv ziALu4NW;o+JM)7}FXKY@f#1;zcvN-B!JNj0{N*GN&5IT})g5wb1dvnHA*Y5z4$jF& z$`8EL%T}&h4mouK$f-*-Yc1uf7*haB`&BssWjN|$qzL(a(o_UdYCJ*@ojT4L&!@pJaiMt@`?PV$X-hQza!x0jRTepI9de+1 z0A2oaaJg_?$SNnX6edfSm3`V=Y%VhnyY(5ILYZTE~b&YiBI)`Yat~JXNJ6UO1*Jz@Fff1GKy4BNRa9Re= ztYf5ei3XQL;>Zk-(GFeoAet)IwM*`K#!ADw&Iiu8kiV`Ah{mexe21@~DPt_lqKoBZ6OF%|aZCdam2$Eja&iL5$#uxdamdLFAZI+$ zSj&~?kTW5GoQV!O6C83T1(0(g(R=})(mslr^q2^hC|LpM1xnUc+6_?$y{fBp9#Ojg>oT3 za%^*vL(VLs@t1?GYg}lWMb0dToH+sHTOamcwWfE+NH%|4eo zWPPu7WtU@48!jiB zOb8y?&M&jrkmcMQK+Y{hgV)qj&dm-vw+4`Nn?ug64mogX2lc|gTq}siTCUq2a_$Hq z=T4#-Xerkn4mo!bO;h;9Z3HgBCdtI^;YQAV2Gg2K!T4FCKEpd6;PY?ehrH zSncz$L(YZ(`PoP`R{Lyl$a$1#{OtoF4)J5P&!Y}Gn*!wLaiX!>XOlzDW}@-8&laND zY4O))hn%ee@`Jr`T*z9#wmRfINi_cUd5UPP_3KH8oTmfi=NY0IY>A(z9de!}8h`sd z=g7~q4mr;U$j>&SvD)W(hn($1<8PlGM6=uCuk8*wFA$A?`|~2v;CM{duNNG0ULu+r z@QKHP#ydW9wudjnFL9x7;8V(Z$sy+zqG!6zQ>_ilE>FL9yjpr|ZS!9%zXId2ng-IXzrG@xc<@(6Na69NL(bQL!u>D$6?Xcb+yI#D^&N8(7peuB zt|jQccF6fQfSmslO_D{_j<=_*K&;Pm}ntZ#Re}Z4Q9h3e#?2z*_(fG^x#UbZshn!!Draykd|8sx8-`1I* z--sp|;z#xuzdGdnuF2Vy(+ZT^%XNfk+FRuO?vV2b(e#8*6 z9Jm|>8}<=$&Z^#Yr&G?K4mrmha{eNk#_$Q*HkRDlr;}68-$a81(msDV2Vi2^HatlClta+v99fj)8r&i=mH&@-Ct#i##*j8hn#W&yU#xqs4{%^V7f~r@li@!vJ#dh4rvr0zPHA8am|Qd&eW?=VYQmX;Mxjhn!Q0rXlzO zy?uPgjdPrKP9z%Z_0A~{IgN=%UKcbrYPWRCX+kua@G0wiV~3okL?f>Y3h%l7UZxYM&a+(tj_FvLI%^Y%04Il?2CoW`_bE-p5OQONP6ptmRJOu*> zd;MxfG^V8BHr!~>Y{(VQoeuJEHPy(!TfKO@X)($zR1(4I0Xt1_QIj1?~;QLj$ z^~Z9#=LanV<3o#_(;aeB0?0`v8f&>y9CGk2FOl-o-XW)*Lrw=x&giwDoaVH1N1{0Z z<(Bs8;E>aaXrw>g*9dhcnrq=Rj7Ef`lS59I0CKt#&21JrT^w@G2p|Wq++jVfMa~%x zIo$)u!52Qog{vkb^C%Uao!)IsF63InyDhzeCOdqG=`U+;`YJ z5ZZP-4t$QkO8 zGsGchm?r1PhaVm5%n$aYJl`wrGt41pL;yJ>9dbrE~Nbkdx(*lN~_L zIHIwZE88I_CxDz>hnyUegD#dcK8BG<^}sM{vg_efB25MIr9wJ7XnG6eB}J1fkk1t& zXN5Y1q~vviCP5)M_lif&kfgyG0X$lUB!c}h9=9vW7Yn3HNYcD4kbVlOi`5*C?jcFD zP9RS!Ifn(ZRUv1HA=r}&!L3H|*sPGf0@<&SL}AwN6*5yG*&(SNHlEQPES$Pk6J5Y5d9g{%|EtxED?fmBvB zJ;cOuAB8Lx$QcT0R@#Zcs9)VO<$i(_gQ%51mcq+Zp3#D7PF2XQ0%@d>9|dAszGX}1`ansZDUfoC zYo|bvYf`9l9M3j0pZshf2GN;HPB(${R>%Z_lvl_b0_mZUHLY1MG>SWacvSyg?wZDaoG;~Or_pOFy?*>@%*=P2Z}0gRldkgEqWGDab7&thc0 zLRJc7u|ld2VwwdCnJbVb3i&}GOBIrJHgjF0klg~gRw3zwndU}?JSLEZ3TZloX>L-; za)Df>kg`LW=6Zz`3FK;pd?An<6f%4mb8$^c8aPPQK)jcP^F@H9h^Ee{q7d2q><4C| z=^&bC<9mrHu4)dhPbH1w`cWdCl7+G>B%Yq6iF8U9 zZTNK(QF1CfCyd!D4LNHQTZvS@*}bg z9D+;H#5rj2l9+ue{q=`JWPXk-L^ktbg-EX76(Y6&st_sXFNvheXsIZX6e;JJLL^s6 z&UQgYnoio2`-}1bWJ9%>OxoAky&F+u9_nNhJ$D{T~jn& z=bhYXpUE{*a1o8BYbtSBZOH4jP{*@^i)b`m`I;`Yp~)4G6#@>T(R39Mm(_+Q*SUg= zXf$2ZG+k&zlWU#eA{tHCbmFqw(BwKQxQIs6RjBDg8=72aj^c6=jizgc;zEiM>xZ4f zYoG9Jh2SC@O;?fPq8@YjH?NpnM+6tqXu2*UE@{Ii*B={ha*a3#H5vxdXu4)E~3$N%^@yp9P+vg z)bY09A{tE>&V6!ggEquCG`UVa7kLeXXf$1y5SO$e)@76HQo%(unyyPVT^NTZ*B-$| zG@7o$1t!>O9UN(P+BnYPv8EO|B~h7tv_CE+;Ol4SCHZYWYxb5sjv6o~8?J zXmYj0P7)5H(R5uwTvi*JTyq2$(P+A^)O4W@O|I7j7tv_C<`b9Ih9*~|3pj^Fqv=|p z=|UTtTm^!QXf$0{DK3sXu7V|bYWdKxjq$KM5F0iNL7moCJ)gsdWD7c75({-z+ z3tKjmt9>5pA{tHCZNw#OCH4*`*KEN>G@7p4HC@=UnOx5aE~3$NtspL0E3tPlxyp{` z91@MD>kds9wrnQXnSzUGG+lQRmyAQSAwO||T9yhfqS18SrRhQ&nq0dD7tv_CRuY%h zh9+0F30y9s(RAId=|UTtT!RG{(P+9>5tr45CfALEi)b`m_h`D%h9=iLf{SQ0U8@xr zmH~SQUcHZJH7DXHyhsvhG+p;9F09K2jPy*dae|9zG+p-*m$V`F4kp(|!9_Hht~Hu2 z>>W%lV-l)|`(KB2ZPx$Y2LM5F0?gt)9W$1spqu?SM zP1jaU7uIEy>#*P=8co*|#3gNrb=l+^QouPR8co-enl7x%CfB2ai)b`mPZ5`Oq{q7) zpq7Mbtcz$gT~BMeaHMB)T`ahWMjPopLtHIDs2u5;T%QRpqS12rEODg)R|WXTk)FwQ z)^vd2AR3M9IpUHdy}f6rOftEi6I?{2aXn95vR2|q&*W-Z$hwF|v&J^!vX1mjuGNBz zXf$2heRMUL!Mcb>)3w7#*D}FHG@7m#d~{VVVqHX|>3Y#e*8;&sG@7oLG+j7K;pah6 z%b$XaXf$0fYr3#yGr5Lb#5p7yP1h^LC2J-24kp(c!9_HhuAQ1LY}rh%V}grlG+n!h zOV1ZaqvGdYJuqv?88(}gXY$@PHXA{tHCYs4kv5N*iMa-x>M1Q*e0x?b0Gp$$#0 zvu1G)iAK})260(!XmZ^txQIs6^`@o^ZD?|RFSv+C)Abf{S#4-?^_a~$BpOZEZcP{3 z(B!&7a1o8B>utq_aDohF(uUYOm|S-W zE~3$Ny{qZM-ofPhOK=g5rt3Z8k~YNN!Q{H&V$LDaXu96lbYbscay=`!h(^=(0dbuK zLb1hOGi%~7-ct^>G`a*o4TESjT_0+?7H?WJ)#Q3ka1o8xJM1Mc**m;2|D44p*O*IL z7tv@r{D`=)ra;2DCGnFLCRgY(<{}!6>to_V8)CV-XAH?Oxz-6TqS3haDP1TRdk2&2 zthtE6K8Z-9S>qGpvi1%p*I$B*Xf$2_Aug-sN2?i_^_R0QqS18i*L0!fO|Jg)n2TsM z%O4;vtL06uF9jFTXgU0pxU7~pxfWc(x`;;O`i!`&mN&UtUddcUqj7z%bRh*=-sIXT zxQIrx#uvn8wY$<<{sa}kYZ`5%Z&TE0`68f#6ilqJkXG+GXSBrdDvO|CZAGZ)clT!)BD zTK@dhNtPU*b^~(}jmC9Y=|T#$ys4|jQsyEW%^E)um$dxIaWnrgxsq>WE~3$N{Y+fa z@)+~HCo0l+x{0}nM$`3+rVC@<(YTH%T}XkJH@Ujs%3MUFS>q4ll9tE$UXyFUZOlb9 zny#aoE*$CcQ{hN|&h5-aG@7nsny#JW_H{719ur(dqv`t7p^M)xfm$lBU|mF`>H168 zmDysX$u&oC5sjwnZ-*}PImNF87tv_Cj_bOhTqakKJ2;0#qv^s<9Nq@PKhr7#e)|Rj zHw!MJ(R77~OZsakl*{A_-O0j;M$=VBaUlh+j`&0M4pRgd(P+Bj6c>&v-%kJje3R>8 z!9_HhuCk7D9ap-(7hFW6=_;q`!q_mm`rgImA{tFsc}>^+oT|@+m9CY7i)b`m6*OH7 zU%xEdtqldT+@4)8txn4>1`;qv@)y zxR3&EcubY+iq*_TG+Mb56c_5k*f6B4fETvy-6IV2iNp0kD#{4AsOgLOzw3&6`M zlJX)o8Mx2}NTYM%b6~K0gDcl&!G$zBS3MuDzXTW3=v?)QOZJ|eHJQV{_j3-BM(1ka zqidevLK>Z`p%2%Sf(vPY3vV4K@wD1W#MKJ21|5gddE~P&Rso2PG7s=K(nwv*)`+<9 zY8|;&8tYy+xta(rq|v!fCNAtRkc$t1=;$H1kVfY^g}CJWYQLF#yPCSr6Q{r;BMyIuG6{OL* zl86hffo;O6dCjgcb2v?KA&p+HX2dlLIIvvJr|(&${N#N}>{1%eA{bgp)a%V~}G1Q*f(m&F?R2D-RVU&sntqeru)Es=tcY7g@_ z(&)N65SP0(1_&;s(YZP*E=Y(&rsfMSq|v!L5tq9)o)%n4qjPmuTuy8JDY%eE=j!6a z)#(w=GSUE-#fDvpOGeZ$PY;1nB0zLpA-Ir6*L4PQx!dp!!G$zBS2x843313&?F|ed zjn37bxZG`cf#5=}7!Gm3K4()#yF zPft4s1*W+EK0^MU)jvO{FmHUCE7cI7ItQg>_09`OH7kIyetAV%xp`Cait-BG^^WM% zi)@nKr`O1$In(m8CgqPWNJ}FFCHG9q8eCM!WgXcwX>fMVGD64~q+MjVVK7n77UT!ujjGlapFoT7rN zgwm3dpp;5pN^H#4zNBI>s-TQo=4NoXmaM$zSmG6fBXE`MH83rPmC{{W@v5bVEK5O) zTQNN(N3NPy##mKOtIQHrPY*%Gs-PaSid{uL1u}6izG$kWbRa|P& zfoDX|q#XD)Yg%?;_SCE*q)JOk8{DT?2K{RNqg zx(i*jZMcgFS!aDmRQ`OvEwn#t#0J@)JwW^GFMy)TwYl}>i>h_?m83PMZs8rcf&>Zo zsD$~4e{`avs1DHyHmgSfzG$mU0Fi;JPjqs8t5bBMB37^H#098s(Fydfeo@JZsyap` z*h0MEjbOU zmfJ*AGEdznP0gO1m&LO}vc?CoDM-zP)(I!8xTiqbNG1D_sgjG|>9R4|I$F;&b&zmYo zLfSlUlC37D7Pls*LILQOoSTt7-E4O0PNTb2_f!hHlE#*)G?LL>v)~n51vB$7>RcO- z00v<9$iU%|oZCEi5K-7#!i{9*kz!@dtoJ3(nnhGJS#zrnnjW=V_9cGF#Ruw*!ne2QnB)5iAR^gkY^3Eu#tV~B0P%xxCNz)emAmW z=?fHiE*S{K7ty1(0n{E!-fjQk(xtB8Ya%aI=v5l#Izxk^;KO);!IHg(%mYNVQN zGM2Pno07O-C8^bJvPx2?{pBTshE zSQM!G1y&zj{fl0#tbhSHDRA^TmlW(}+@@INTE6rQYT=<5HgBl=4o67|-9p07HVP)Q z{&{x!0m>M>ZHP*knK|zcznB75f3JH+PIhvEM#Fq2Sk|2cYTc;z;t7JDN}Pvd?Q$cK z;m;1ymN{O^7R^hVGp;?8z4GZ1YGTBKV!cLFfsIS?=W8t2On6N21sm7|$(&oW8idcS zT@Aoj{D#%#s*Yn2Eo(NUE@XikP8U8FA6-KkSqxq|VDrPRmAoo*;j^p>(#I6V;no|9 zl1UI3O)gocAlEBdN?ImeG{HxD2BsW$D8zj@VHOvu&eD4jnT7Ohs$DpPH_g7xrgHD5 zIG_MaTbXXT)S|mSu9sR4n>t%lyae+k^J(YdvWE_4tn#k{ifY+x<7kth>-n;3vCP#YOQPI21I=*31HT$=W9 zZIWCE7v}QHfP+h$l3;^Noylx8JX@^5*3p|u%hZ8o@-Hf3X6C$Gv|vnDP#qT(d)swbIs^_EIvKBz8Zjt1*)1);l;_=Dr7c7Tl8 z@7o10lF8{;B(|tauj43|CuL;u!6!Z)#S6k({0x{gI3C6IQg1Tam1p&_&Kc7(PQDKj z#au2uM=v7Sb6uua)~q)n;C7rDV}TfGV!pU#Fs)Le5}0BF<=;XcDEFT2ky&E`Wj}x% zE&oC20~Nrfwg$?u3!j!jx8DP0$g^Z-gB&P>0pw^I4npr+fE0N>9i(mqDnbxJS{Z^c zz*uHdeo@|p!t5!$d?h=-C@pJHQgV74$P)LiJldUzMKr5Vi3(_n2~<3jJSK(njD-Jv zosLQQT&iXuZ@KVk?s54&Ci(UB$N#=Xh+Mv0c-TKFISsBF-Pb_77)_tV`aiqwHm!YT zy3sRI`8}O$RyL-xtw(oGUtyWox zF>REODVVar!&Y(UmMo9fk+M~6@p5UE7CE0{Qk<(&%r_}<+Qmzs{aI7@8>!}8mvd4W z=i@E$_4dy{tgr%MRaY|lcJ0t0${8^LobCZH?Vwp6Jc3KTc zY!?l;-%7GKB|g`0&p<3h=1k0C{p zI~W$P$4HuTc#PyJU&lzEa&wI2asI%=F_Nd8>$=xeyl!|yWFiJ(d87HM8UWSd&4t8F#qh0?{(;zo`k>L_sp(;#bD^`U^Y{jauh^<%^ z4zU$0!yvlG$Qc1c;7%~^n=B(BX7ZHtV%VQ={c{^qjm7}9-P23$FvyLLT zcry%2HUBU~jIOJTwglMkmud-0QGp5}uNjfpT#T7*NNMZInKo@UUqb2Y#$rB8XC%!i z$}Y+qL05NYWlw`3DDARMnwCmB{{dApF6eGT>OVJ$CHtCxT(4f>e{K>@FYrI8S1)jc zDX!SG0yOcf7x*94D+}y1*mAiy$d^)442C1iQ2kyEa^gS#5i~FUa}$xJsG&`XN+ccR zK0flyMX+INWOn72lIh3c2CVOOK0q8a2cFXikp(C}6gLOT1+L>6-wZ@OfQ(w2>7ZEV z&U1KJ%=~!{)}oeYdNVfVmCp64qyZKOMBN}pFH5b?+g3ctDvHuwlWvZoPt^Qb#@;7t z79C^u6E&ljvHppgMOj}WJ%04RuR6|=w$IaICrWmuc~8XGddSPP6EbhbY=+E*vL$cF zOE&CWlWRcTh&Y8_xMGtl#sC+btA z{^P5Ef9WGMoBw_T1R5Lu`^^xc>Hqf|K(>OBI+uUDC8FxS{{JwB3=P)~(q*-WvqOw> zgUu7<^=`i7{eN@0BXo-Y=AsAc4*$&th|m%Kn~N^nxkz2$zt;v)b$<3PH6;P0td(Cd~cOS$*3#OBY(QcajBDRswIBv+cal3pd?|9hvE^ zizV=duUM7c5_-a}gcd4UZxdOb>;DH-sbUrLgk0EJ zBh~Kx8_T8~Cu@&qOKDy|V|(OTX`g4rmv>+?YIKql6$vzZ}B*!x} z?PCK|P_#Xk$?0C}JRANed$v*@Ev&D}0}G2}^T2{47(Gfk zk*yx3Ku@zr>Q#Kr?8zL-te%V!%ou?$ve_aC05cV{+aBs^VNXx_b-!K>Twr05JP=q= z1dl{1C$fj45O{ir4ylQjBc(&fPFk#ym1#zNE#-889UuD+Jxm9 zf2SGMdkMPX@|W8nvG@{sW}z#}n_ZMNHDCU+91=;F8padwk4G8!Ppm$8tX*#|{O~%^ zBwtQsMs@f~s}6c?@-_gz8)z<{$8>)3mI2N0S2CTKJj~y%^9`f*0)Kg9K~Ybjx%+B= zd6>VR*BHhR@Q;UA{!m_-YYk)Ebxh}#Kg7!{GK?DVj|aHX)qZL4!%0AM{$e6C%EOnN z{ptXI8_+x===|h80W?RJ_{+omwZGmldc!{+Uh=xZ4}*Z_x*M3zFMlrs%|1crC+{nu zX>y~#ymIjK7(v67?)Iw=igJK%#!XCB5x(5)hw@$!G?C=(1iBBG`OB*cKevD&?1+H~ zhgW&qzz=CaGv#Kc!z${Pzukg{Dc#G9irxdd_ikY-FMpsN5^j}}_}i`iqP+S*S05V& zIK1Sc{YD5HB7_e&d4B?aF3^p3p{4omTZMoMls@xx0UTdJqTIVkh^LOuh!#Mnizr19~U$qU; z7H#sEhxwbo*)TrZ8d=^EplR`pO&;yW${2p4Kc+lqo?|MldU9G@dZ5?D;~`lDbn^r) z+L`EV^6CM;GN8OIK(`};Jj~yLfbzZpx?dv5!~E5F9zHCGS9x0jT}MIdRbI^B@PP8p z2fB$7~CKCvFMF9l^EFPMqDZlG%Rb{ z#2SBf&9wQIwR1|R4I5K9$3L{Tw5+&hTED`=%tHUnUR?`I`W1EU*}rFT*O?_V`*iKy zXJ*fm-bKCo_)9XT%_MeOU7ucEb$*3nL}{77W}4j1^v&{D_^V5cx)qmJPphden#QS# zlY88)sHhv(jf^Z~U~1e9l&5hNe8Fo=vjUg=zlo?I%EsA2`N>9aI zVB=VP`|=8Fd^x$JeKqq6t9-@&Kw)W_(G(Y|{bjz&su~3jRA(B^AS){k`0*RCH7fZU zBwR7eXUUiVSm7JP-n4)JNrq8j9Fdww2g<;utSqcemKOOW=0Ln3qPk}fU!aS+#J_xj z>cY}Mjdf+PTq-Opt1K$Sk75Qdqz%r>@{RGA7gf#o@dw&{l_e!L{(w=Y(@XnLr&m+D zz;Dbjb65Yzq0e4LRb`EFs+mHHO29X?be2z>qqMRDKkQxYubEw0R%`)v#_hnuveH=< z<^GC*(N;5_Di-pM1x+oQhEL_Fo4(ql{#P9w)2#thX@$ScNCUpharEf~H?p#3=0N;b zFp8&UPHEM=8sj95h>t|nR+I*2)|MD2Ye=F4qzcu!2>pxEUW1Y%L9oC)u6kQUDbARx zN@!*&WLhdzWf0No?Q~k%kvwJol7MldzFr@By}ESP?0}`L5f@<={*ZEYMPZq7f+jKI zBQVAOqTuDkh|85lf#Bt&h|9D6bw;{LuIo)`CW}g0dL^YrRz8fh2$GW0I)8C69~lt< zq*78=Ss1X4Ze&MbW>!{~6_!=aF0_c^>y7#d7)Fxf(n{kLt@HXwqZJmPTU!$-DfOeK zo3+Jg5kbUCARsIMk%}v=1;x7A9kthpEXP@umBlkl1FC`(;!wf;l_f-&Bn}n41f6Lp zi9-dSS*Vg>tL)*NOlgHGY+iLpqOw-%0xvtzJJy+={CpNY`m1UWPbfU5mfWran4Wlf^eA`L#Xv;uyGQnoh`4n;E4Kg+u8P#i_I z)z)PPL-{L;F+v`ZT8YufOn%bPkp&ZsnUH#oQO@I&iW-ao{^B$rZtDj6^y-p5YJCJze$4&N)91DMZG=A}HxY1X0v_24yF~f#h5wi~@;@FUgM%)2d=?p~7 z)~Y7Cr8skzzq}kCo3A9ZkKN2ehm9L;jEZJ3F9w_8jtAG+j#d{^?60V-F83AAq|>6@ zA70cQh`v|_H;Pv|u%8)>T3uONQS1v;&MT}g_AT&NS1Qkd03F=~_$vH`ppd~ZByHbV z)Ig3(Sy{8;XG>~EN2GFGOG`mVkgu$=rbZ2k?nvy;D1{3(-{{~;cwTL(Kj0f%IM(Rw ziDUNwKM_7Xu^eWdlP<0C&7@)AA8_n*1^Md=;i%AWbA^=pi+cC&@12(F$uQ;dBr2~g zt}Uzd#z^<4JY`pfYO4H2r7*$*WX`>q;QP+6@Ylc&{t}FBs#Z8tVnTyC6bHwUY86u8 zQh{m$^UM6I^06y3yT4%FDK4FbX_2zF7a%i3RTyVTd10MTc%s7mbybxWYz~}#xU-os zZ@$tBp9%Eh3xR=;4!Co|)rG~SbygXgZU9Y%dZL?JNCl-&$&zseN+o4RJz-`gJS`e9 zv;wilG8d*d60W7>q!Tb9{gJ8~Y8;B}Hk{J7L#-~Mi-V$eaIj;@hyfkmJy37E#X|W* z%D`+#p=Dr$>V6;wq;AArqbDC)672Xwy6p%nhc-htR(q+1sw?GYQ=p=@oNGT=4JkJl zFyKMJ2sV9n+jfQPevJB3ha&ZwII5aZzs;h@B0;iBrN0avgc}T6ZVa3*I%tM&*Gqp<2)(nrO6Y8m>=%(wL zkVHXF=w-TVXZhbj{j`P6#Iz4t&X?vtEvIprCg*T~=8!E6h%2U*v2j4c7{p{jRe@VOS^< zW8V-Ku9m^3#To|oF~8~NOL&u5HmAuVRmUo{X(^PW$V+dY=+EpEc=;j zSK<0-25Z)5uFr;}cu&?a`$nVw!&noC%BDGV+qIhm^V`B%xUI-rJhYl_PBs`;+bj(Y zF_;Ze0lTWFjja&>_Z8^=1#NV^)F5{S{kzS!sk-Ak)sh z*nc|M!2z91E$L&CS`;?DEsB1a-4FE4;J0ouVKe zb9vI)=HZdT3~xp>91R~C(Q3ZzcUgy=t?wQY%9>H#iog~*v>XQLRNczKJu1>Qy=~nx z)G7jQp!@K|Gv2~d5g5qnO~^vLiPt@<3qBJqMHcSoKsmQt7kVxixpZ$%^5jbJ=DI0& zyN6M#6Vv@g*gcFxmzcVl!F!{3VIQaVhke6fFU+ugv;Bp|SPJCw5HFhxE2nek1;!fl z`gGGQPWLnZEmhTx4f9ruPCl}f>;rk19#)Cj{6ODG?x?`l2DRLifhF>Z74xuW=$kj& zkNff!bFr#`dzHoZxl(w%%pUF$*~_acB7Ig72T!$+0xAUx(e28Oe^ztC^e7WUo-+Gb znHnA*S#37(7ejsQQ5cvh^~|RV2}>mY>S|;c`#JoxDy!#%!#4(lbeX>b8)8HUJs!rO zLqAJ}(cy6qPz43&_6VP}s|wAg+ih4?H)ms!4a>aZcxzD25#5?rnNH8CX_>5tB)!YY!kQbB1Vx5v|54 zTExyi_yC#M3l?C+pvEH*zvAHIV@wb|nDaR?^3-yo5L96(qRu*~s99Nx3TEgGj$$Vc z?s&s3po&?+^!frjJvJ<|XU0b&tPE#6MJ${(tJ*)SPz{g4bXcMjVuo9m?2c&pPSK>w z;Ta}>DTeuAI?T`M2N1zZ=~P6_b&9AKYeGthDh-GMXfd<7^^Mw7^fZnEt{;gDP>X^c zifXQT(c<^)rQ%n0!DA{_wS4t}udZJo^kaos$Vy6mCj;B*%Q0(BN=;;kse49li7oR7 z{NvP?4@{2fP4zHx0L#=_lwL8i&vNN+%jhh?z3?`;@%yS#7qnmb9}0^3Mk^^`W*yu z&%s6;5Z9v0nK>#i#J=V_Y<o&JS~Vb;WK1?vcSvX@#%?&c*ie^)Wfh)6(gVEeA7`sdVQ&FB z)zyXb!`uYqY7n9yfx4kqEAUH~C55$Ra7)b1+vetOb=i^*l)F1=Qo_l&RWH#oFb_oW(;Y0Zv>K~-52=piVl#6p7m8F*r79~VuY<9pKPhz|dINP;;bh@a@y-n+ zfEfm>ia04OIf%vLY!?q6tEptxXl7n{r{9@uK1WwqwGD1;P^u^`^!6vXr`+wr-gOh& zfVy7D`lL#x&tTF?B+Luy3kqr5UlW7IRGO$QlOeFQ~*So!D!oY?V|6 z32IiqE+(~ad?9v))mCGpR7rJZxvzwqWa+S9oRUi41S(CM6?yD&*@ni4VNg!@x#;3i^g5tTl`GcoL;H+c0Miqg$_p0Xg50_Ky~#~?3cKxj8?C@)bC7HgbTN;i}9dIQfl{fGylN>2=5gvtsW?q%t4u^A@%ip-U)ou5K2cT(KIFIR|9_rCe+?X`6D^0+#g7ssijgN^5|}ZK~0r z=gGWXHM?MsHW5{|7wQ|`v>8@?99Mzfnk!m!3#&`vzT>&2>RB~_v0*hc%@|sH%nTR< zfOyMPGv@Zo27^3u0RCqOt#=8YT#`XUwM6a*QuRVQr_e1x?3?b4?3*w|vMQfz8bY39 zQpR}*UV^QXsWv+q_=My2H9Y5_9+^_bVOBE{p;XQB%&b24qBG^l7jo7G!~ru0Zl!15 zM^t5$Xto;Fl;BzFi3^97nkvbErRgjnQ~p_zE7)idH1%t6(Ulc=FKxbzBOE51 z^@xyXWmOdh_#}+dZL`%Y7T|ssC-|o!S`%ydsl*d8a|34W?Ly_c_{PjsZ!*U)HXGnU znC7dru~=++pN@^6kyXDW1~v=DrR7^_Y4&ZAQi0ix+81F}UpU7c7bA!1V*g-ll}<|C zX4*XF>Kvkl>aaUnEh~o)b}3J2pxoB!f^}Ktn`D><3ATQV2KS_3UbEd<0JH11)nv;> zV;Q6Ji=AchPAw?*f&30Lx!4V=JjRy~JRX*eht^@}nSo>*tmrWmyV=?6cIZ*$XkzT; z>8&-#1Ek6zsz5_;_ZV#5PnoSdtYfC74}KL@4GU+RwZzuCt)7u}(amAzeLzgvtpU+{ zc+tdo{c90oQE~K@vAXgRrRK}XAeD6u*6oGqep24>R)LwM?&1%!ftDt|66Mu zu{>f->*!pjo5@@z!lZbt~t*uISrW2*^n{8U!+E*|~a$E`tWyuU=|K@y#1e3w}WRsry79_8u=9S%K( zvWhdHhP5geor66fi>Q%VQ^Zd$LuF(aH_#ND6{y&+CQz9X`luejD(HwFK-FDU2biu| z7M2c9BfJ`!wC9qA=ipRqpY`~ra+vshoK21{af07J2W?CZLVU(95;3mIUxB4SK86Qp z*%88fDcrQfa_4iskt8F^10^t2jq0O1J3?}6%gZ_Gt}VxHODx>QS}QHXAph(<-ZCSG z``4k)xT-|QI}gYiYLfAbRl_iYpqr((C#ej+Zt~kpN8Ik2P$9u=bGiZr5nBhz z0fuV?&znXJ)M8a*po}^LtUQIUgk<(maf-|;T3X4LTBN+FYLlq65|5@JtE#Ciu}e`` zM&MaX%(#`GaEZfpR09EU6(BtlFcOa)sloB+$ZLG)wH6P>uz3%PysTEejZU~Y!=;T{ zojr?Xuxfdh|A{A3wF}QWT;{^SL0Wpv*UTd3gHrNfPqn&xCAYwhv0%CO1hby&a#Ovh z_Rk?wrVL$dvbS4v#6z{EOTE<8%I@AlbM?5P8cts@HH+3)D=UDSO>(A5ZR?6f0|nV> zIn7)g4LSG4J1oY#MFRc^5a&PnAbid=q3LpB{gwA6^_Gh88J4%Rr|tW?)mcnH-NT9@boA9! zVLuR-8s?~~SZfXf^bAzlshZy0oWbzHxX|e_$JUMJ*ovpM@#w3sy0Bsv-&TMXOP2z* zW^P=!sSYnvX%>uqchCT#E20FbaRg60CZ+ytmYP-3%q3G+*6n6T!ZF6_cDzvn)cGby zv~JLJPIX7-Hdn0>g*sgk{Xlry?4eqiY2!IT^%SXDIhE&2C+8n#-N3W<%6n2yhZ>?A z!m1iOY&>T=k6UHnpcxYp`FFE_HCuqnLwLpKK%jdsZE(BZ!E!(|I0v*xm`f(nikt4Z zE^6;$AyA5$J;B4);HE2xx7sWYyUx0ZpfFoVPpjRYRajl2`*v3TQ>~PO!)R4?W!-!{ z-HA0GtNo}R$m)}(W6Jk{nA9 zhIaue|M3p9cpV3S=^^Q7-r$K7XOQceD_wDB3sTLtm02l*D?k;sWn~;nP@cR=f1yjs zhb=B}h3_VF$c#23s#b8aIR>fmMbF?kp;>NT;Fv|X01M_^m&-T1_3*Awa!mj2H>+I@ z!%cK{J~AY$A#bO-C>S1Fs_bmD66$W#wW6u!l@O;3&@z-Nz?P<+kfCDSX(=o!;#1dQ zeJF>GTTL~Yzj^+`IclCrAG5+_aBE}YD$>mzX_ELaHUj%*&aWse*Sjc{ zJ8rGc&(oa)OZgm^w8Q2vRKMOF0+2`C^V9{$bSokY$kL0$%M_O+RI5{S**u%(oE5NU zJ?7M$X=f*Zykhk`3ut(JPHn^uW7JW*R!nWa}UyB!Z#P@!d@QY#Rqm5`EJg60#5ezl0Z z(=^Eec@>&pt@Wgys4IQ&?q{#TYTwvXX3L7Au~`?;4boNlhh-N%th!`07|nFgIfi~> z)&HP7u2!D@$I9vCVIjqDvZzw-VO29dLRo{Dns-E0DtfP*&06EKHCXiw156J`x^{Ux z7Zznrok}aJ;8Tb72vflQjM;HISCMGlsje?UL!@=)Y@LnuDzi%Hu~;VB8MuPkwv0l$ z%afejYPn3qP+(jcsgAX6SQQx&&ql=VW>V&TOOGKwq`R>X&LNlJoq?M^WrAh8iPk(# zWinzOCUntk30jUdv&|B-^~7xUstUakW2L7psB3?a?me(LW_3RjQBN`qV;&O6O-Swo ztU^(wb5B{=POls~-C_G&dSw{R0Y49or3}-}xoX5$J9T|-`*fKa_GB1HtRGV2pXlEV zz5@w=Hp8C};WF3V)DE10K~sq3WF&#;Kfi2>hdpdwRqyzK%@&+0n=#BR9&yB1>% zBFq9?tb2l^Y=Ps(+x4U~@3pC|xCNBRy>}$Z?Vg{&<%u;(mY;p6FmLjfp#(IC;|^mN zRi#z_2?}6%D9200_!tP>1->PdLsu?o^|uOt9+5F7O7^n2rJykYJpwS^dWjOvB(;vtVSmlK45XFD)v;MAEq5y?tOZd_Uxf* z!c-FEkI7Zar0J4ntzk(Ffg7iGym4=&awOY?WKLx)uN04yR2R*jAF}z9`C_HQ;A~Zz zWL1PZ6~{}B~TEypERsR_UNr{>?l*HR9p?g)^v_4U_6@2 z5@sQ=SbRPmBd>lX+BY3$nSUpK7fv zYiqtLWld}??NRJ!gP;e*hE-dEXYjE4p?1(o%VM2k9V!cHJ8Vim{WteX?99O5X3vH9 z7n7Z)^_5ogt)HBpPv@UbyZ)hsu@6T!UZx(0@07UZPrdk7X72dE{&xB97U!SSWhLGV zt>i1-&p&GHbqjV~QF&CquJ=5-#W3Df@{d+5T%7Y_o1;4Z^~Ap{AA3QfVH^*ll(@IP zdt%J&-Y<>q@bG6({ptQA7ouG#`SRBWUfl89vin~D%jj{fFWL0AVLYhhJ-gjn`mb?y zOJ19|t>Dv^w_zL3iSgvw|I?m7x9$7!mshX2?x|7!oOy<^T*+_wa?PNs-d$7w_UIeC zC;l{gy{s$;ovtnTZR3dvUrZQs)wNH(^)U2D-csT&Yg>EH zF?XMOLzi1uce(4MFIO4Hqe|Yr@y6|2hW_+ZQTc|87G(_{1N~vsl(@_`B`3Kh8b%k_S4Rd#z!dr{uqFY4P;O8@A+}X-<C?&pxYtiyvEBTAr;zXkcFv!-uUG2``#F`b8Nq} z(lF8_6MyT>XYP6K;_J_EUcL5%EhlEbXBaD#y!?#^-d;6)K$p0mChcwa%yswS_fM33 zV^U#VlQtX9e7GUeH}|^!2h?6pRy>#6v?W<-D#Jl}pQSzmI2Y2jO z_D#luAOAIE@0vH!R(heUONm>%WM-$S)0-vUxO4m8Pp!&MH;kK={M5TT+<)Ou{lD9O z#l*9=%!uH1zNzJ-vZg=Q^WxE%H$AW9*%u6NJ)vyGsFj~h+1aGLVmtajk$-dP z^tEM={d&?jQy*@=qv~D5;F?@Y+_qB-PZ*y)t=YLRp7mOj)IFG$yrJX|U74}}y!z2u z$=Q1Y_iwqo-Y~v545J^0S)(Xx8rD9seXBgJgfA{n>xkzoI!(h4Wc{}BxurF=g=J|q zfm*zecUq5(%)ZmGF~MJrmvGkjr_C*z7Qo}mHPgy#aGKexTV|hbJ^E(!=sB&lqNuF4 z*gvfZo9AaAhGZ~w;RW+<3Bju`uAEnshV9lh(=szG?J_gEWpvBz+qX~8ZW+^R;c`q9 zo^Gxx$IisJj#d^O3MSn7fL?_GcdkQMUSW4x{^PZy4$t-B)vQ_ld#g4;8Pm{Ae(eJ&-Xvkpg{!K41>J^^!A>fuP%yLUfyRi{vG zOLM8A2UXs@SARGxq&DxlRry0_pkVpqZY=WUK5Cgxxjfi9jXgk(yCDw3wq8y1&}Fy| zn;F$!Mwd!)2vd?JZMQOv-9pk>4y16Tj7s~}RFxN(AId@!=f<7!&{{jxJu9OPP@WQCl!0ht>d{T6T z=>;C1l<>5iTSBpSI@)QCc|2hCs41LV*W++>4s|8!$zFHm0<%l;6C2p(5tu(szx=~m zx6ywecz=30auytqIlyxAbU0E)lXI&vhpxS`5#VOM`Yk^!q2reTN2lJ~g)im9@6j4Z zAN{_s=<{vwosDlM-eH$6YE z;r*qlIk-WUoSKt}@4S@c)RC=llAQ)gIwcv9^r9qt^wN=2AkEENyuP+iZbME|PQ&`^ zH|ETa`)PjbhMeZPr#*i?&T^C2uE&v+#*em7WFxa9i#G4Vm+@sH&_-Uvt9eU@%}|oO zh8OY{Zb+N@0`I0x#*t`Tj?d}pjQ0Bu$G>oo;0oh(tRe@)H8{m>5@Q99=doQrAJt-{vL(DCA7fb zgVl>u0LS(`{w8FOy3r`t8?Y6j8}`NxPTh>>;NlFfmB=4Ooyx?X!-{*Vk(!!;b6R3toU%kqh;X0E;A>(aQn@`AvTVqhyK!-0 zxl2vW!v*GMT%4JkRHYTQ9~URu#~b5N@_T`#zGh(g(>A#EW#noh?i|;%PT*Eb{cwEa z7G(W=d}I4q{rUJ#!}rDb#x2GA27Kd|MLje%rr~IY_h-vlQ)8_qd)SgaYsp@)WFK3y ze_FC6=p=m2^;uJczt^qPuw(_6Y>Fjog@O=VYkk(#IM}wzcFNY9kJekj);j~Ow+_-` zw%-0$>%C+NR7V4DYc=4uW&=)Ft@w#-0IUXl#6}fC+VH}9TncZ-S3n}o_%@uC;7G(j z_hw8}v&eAQ&DeS=*^KYRx7mzwr%^X!o=Gzn8Jn@l*o;NSW-Kx`W0A2Li;T@!WNgMFV>1?6 zh9zS&78#qd;Mj~s#%3(CgKfr#Z^%6hc@MVLUy+;3Jciu$2iaD!{Z75(H>Q>ZbjE?ca~I$AQeSdp=h5gc2r$WqWcMaC8@vV(1>|0{;uLYT$735O218~%+Z9P?ny zCai~AdWQdh8){#Sxe133wX{SGP56KDP&)%PO`0&zqzQ|RHB4k|!Xjf678#qc$k>EM z#wIK>Her#m35$$PSY&L%B4ZO48Jn=k*n~w^KYz9nXp+}3a>f@1!VB0GhTrWCZ5fA^LiZnm+0InWg|Y) zvaeJQT@tQ@%*$j7rRGqmdR8FGc6jG>&^ZrcXcTYM^Sd#=>6jINT-^BQENXhf8 zsqug%dqkf#HCT>u$h>T?;GM0_VqPLi@ z5&v?J7IV!(mWt^P##ScI{$O~C9BgX|ajRTPZQ8$XGTa zi?y{ll}l)AS$RB?QQ^R*)G}8ktx zK&|On>xseczmd9pGKk{vTm+W0xh`=AG?LKBYc$9Ujt1n{J_l#~tjy#b7pFLtX}QFf zA*||?d0CBNH38r0TE5PZ;o&KKGmlT>o5R8+9AZ$OVVTg9mFcslYGGG!TznJUb_=(| zk{ytd=~qdx8<~B*&B#|m)esl-kQ%zm_QL#d5*wS!C^9y=T#1oWpIt z7##djim?1oqO1Y}zrq1_ny+#MkUi`A`A z&y&ER>$z6osPNh^Kk6f03mZU3rm6$;yFHP=p2VaiOm1YoaLX;(^_C0|5NI5G9Er;wN8;u+_wY zTc*#N8k;TIJC-c3;gWj%;b}u^%fOonfBLs<$WAB3h6@L{bT~;KbvM+8B+xxd$nY0x zL+b8jL*~h};Vhhs4S6Ou6d7$OGTKmNw4untD&b=g7kX5}KOdw4&4Dq*e704;1FwR1 zug+=#-AulzUcXRCg;C6}Q6 zVy%95@5^q#1F3v7m91EHc_#J~8SN)B+D~M(pUAQ&Oq?`1ryyry&ZL~lxN)7A+fdqi zbi;SJ|D4+}zwPz84JS2}rse$fd+ySw(^nx{!`9rTYtmQqET?(i!simN&s|!ao(@FA zo?Va{i`FMEdH_9a&-KF_URt^+eI;bW7ypsG;&lpNUOGBGZFtYG7k{0+qKbk)7XO&M zVgk+@7NxHSbj0%Jof_UwUi)rN^4iU~NA~K{g7hvq4KL>+V#d;C>35T^=ZA~eC$DJC zSR02oe3HEOh1`Z;lGnb7IO~_@rg!Z5^5PGZSHOKT7G98^9yhXK(v;f9Im;81zifif z=E-X}CcHGJVbYAeRk_LQnx`!MEwQ$HPQwd14R61>o9QN8qtvDr3GXJaeJ^3l!tHU( z6Z4lRUXr(J#FFMIU!K|U!u1P3O~_rI7~k+_&T~Jp&% z55bGW6JEsE#@vLL7H&@%x;(Ky7dfUy^NKcuvgIGo4R|$q+2}w1_yb!9IyNxZf4q>J z@N;g$r?~@eQnpE6b}}!Ep&G;%xec&O!YhBgq-`_cU&+h1{R=4AIp>cTa|e8#yzDyu zEt>qXKH=}WP*@T<1J)-mE63$8d(ucb3-`y>CXZ(#CPPqij`#GQ$AHLMPXpZ+V+giCklTd?NtE9TdH-)bhQ0J$sy#dtb zO7-{XxJ=_`B`XJ>YXb0O;%69tj^ivW6;5xxl2zfY#iNz1JTA_tRkF9@5{wt)S{Or9 zSL1JtHwZ_Cq%}e5aSWs#)U^&F*T$qLQ_X&xV|C+-#&t z>(P$QWulPO_-%XL$1bp|!_Y{2h$2V?n)?a~d;%~w?<8Q<_`Tl`#^6$Yn)5EG7 zZ`#&3#W%{ko-50<@qH4$(R0+FjPE)4X0@xpcNe6~{Yv5+@Jz-wk#TGj*|iq#221vk zC0l37Hd?Y*EZGN^>?2F|jV0r#ENR3e50WlN9+9=PWbG|kmL(f($;Mi;@s@0+B`dLH z)s}3oCA-v;U1rJtV#)q$$sV<2Pgt_8mTa3P+h@sswqz_9$y;-M*3?L|Wa*YH+mhv4 zvV2QcV9Ba1S-_IjTe3x#><&wIwqsGX*dQ|I90Tw4#o z|2{Woq_>-s7L%LvzRk_K#@o#~mu^mfxH)yo&1pHN;W8R0=;@p=vS9@!2jJ|Wt1=P| zc3NfXY$(p6J3EtsVh23}-`r0m;b5E{PDw0h$8|nzcXs-O4ApjLXIxyS@tYdHgU(K7 z{0w82&Dp8J@IB5p6n_Y3=XIO2(>5e6yR*|hy8lMeU!gO|My-;UI z{^QH+ zggQHHH;2>NITD>OogEJ7ban>fo6gR1eAC&v1>babUdA__o$dIhv-3T^>FhLvzI1kQ z3sE^c$KspL&Z+pOv(pLRbaq(nOlOBqusAzB6K6+cbaq5WXGdgoc0@*JM`Uz%M7GhA z(b*9hogKl^*%29?9g)%55gDBwk+WUjfqb(V|QHeXl zk`-IBDoYlyWc8M8ktMs`lHFy=xV#{CdCZc%XURUWWP2^ykCv=4JOqi$Wj4{Toh55; z$p%|8&RYe?Iw*0+Te1pER&B{HvSgQ9vfC`#otEq&OSaCEy=Tcjuw*+e*=|dghjB%8 zp{FDIIr%sLUCjBvde7?roy8m{M<;YKhdb6K9Jr$swv^*~Z`tnX{C6oQ%+dMpQVz@! zyl?e?YbnQRXG5r?^Z&v^P9Zu1S;*m;I65Muqa!jpIwGT^BQiQVBBP@tGCDdUqoX4- zIyxewqa!jpIwGT^BQiQVBBP@tGCDdUqoX4-Iyxewqa!jpIwGT^BQiQVBBP@tGCDdU zqoX4-Iyxewqa(80Eg2mhkzcfh>VVo$mr;ZjE;`T=;(-yj*iIa=!lGt zj>zcfh>VVo$mr;ZjE;`T=;(-yj*iIa=!lGtj>zcfh>VVo$mr;ZjE;`T=;(-yj*iIa z=!lGtj>zcfh>VVo$l{Q>#y#RQU>fBR^P2$+h1k-UXq_eDtWL?|Qkxj5mmayN+4d%z zk67QRDse_azQi{S=GF|A3QYz%Bf~{X=9Xra6+_M#W0Ay&G0pQV200@G7c>)$&@|UV zVuos*NIvC^F;*&>S3cRZn4w0YnNK+*1D8(S(+pTKGb?g<0dBHeIaAtODaj6duMj% zH}C53Y^N9B+;m>nh401rw>@0Z(iTLe)!iD*BaBGY@XZgqHj}f zc)H2&TaT~ZwCt<3SI@sMZQYo?1HVpvtH;$P=JiLm{bOFKhFE| z@=;wUPwH1WzV6l=(x-lwdjB)sUfe$D%cjeZIP1+@r@#Getk+9#{e1P98&3U~8NcDo z6OtdjfBi*fe7EZ;Ock<=D$A?7l@?!H+9>XiKUnJ7C4RgxIU{2wHUx>JTQPrZd@k+* zaHlFTaol?x9=a9#XV%V2$9)Fb3!M-*gH0qXXiNcx=pewu@gh8Q3;62->E-MmRPT@w zcMZ3% z;|LzMf@u0QQJ5Ja`Xoj0G*uXHeVRq^@Cm>0d^J}X=9|T_a5aBh+-{#E6($=1T7jW? z1WyY{6L9-Xc%E(et~in>Sz$tXT14=u2P2c923FQT488yme=pSw&u%O zXnj6TVY&doTEfsef(O|M(P!77qjDqj#VK4u91}9_+$Msjt-_f3esp0AWYk`NPKe-X z8^O~q27OLcnEm)P^=TKulcq4<`kWNOlNQ0#USa4fu$O5+Y*$WXzD`z{UKm!haK_L+ zf+ro)ggB0i&9?y7@cvzPf+KE zar@ox?~`$mf^9lq3S;1fbNp#66;8uvLYx}caMtX}SMCkQ8mT)Dg)u_Y>Y`$?cV&pX z{r+>L?eMQhbu5JmORFoc2GjcSxh3}oW9`$i6ecXKZYox2zQo^PT0Q7QASg^&THPTH zrp0^(W8I=-DNI;e87fw2zNC+08qKr{3KNzVYeXusMh}#^e{s{T#1;Wb9BBF^cLR(u8r`G1(ieTaJT*TIj4%k`|>BMTSh={RSD$B~h(dZl#rR#E)=jGvL^ovk%Cjw!Ab zp?%K~H+6zV;;K%M>?rDlayY9)iV$_uYb*E)>-59Y@1`ra>@cNeX6)eldQ;VdjBO?! zwH+RU9^6#*u%u;XEUI157Q^$#wr{F>kg?6gy>lAbXU60s*m^fp)x(mOnX#yLiOAal zS=BkVnYedOBm36>oK+7|T4u(gS~ViQ1G1`P?CyOs`=y11%-(%6`uFM8uSb|PEiE%6 zqfehcX=#1>^zPBW7bDDQT6|e{<82j9r=O62gi)=RQ$p9v>6-r;UV5uvL!5M;;s<_1 z*ZnBE;Bvg=_&38iWudtBoP=f@DMLfo&52uuqfle`H-o~3ta($$mBwFexU`h~8TNFy zBe^3M#TlP3mUKB=%1%MLHa<>9`f@*eNO0ogB1{f&IZFh~cusuO?=Ko-QabVV1K-2I zeaXi&5xn@uHAL5M4iYyrFSKJL$Ir`n?a#-&P7A*PWg6%ZdKt zD+Pm76`c4ut6K)#w5tW{rXTs%XpE+F)bA_MJPh0|tcVD1d@PS|HAVsP;iO*%^7}h* zT~`U8S9x5iF>f}>rW_{wlo=0#vK9}wJ(FupqcC1C=-zQe|X z+(!fldf`*;GKgdTrT|l{aYU&1(JS!&0H+~XrI4GCQh%)^h0 zes1;ZCMce;4)yPG!MWLG8tjsQ=U{vr1y_tTf_4t(mv%Y*l{n)D91LD|xgVG(HI8UE zyKDevvm-8OmlcTr0Wh;)wWVvb3mC~Vz4asC zvlq{k=9W1VQ(6PTf1C_ z3)#TD)xEfLq;C^z-a@HO7mN{rLC5 zovLpNd-2UPZyakvr~FodZvk+-j~4yh%A*T#V~>e9791-$w{|%N4*5D@TAV1j!ARSw zzI0}~cvZ|D+}erY##aWu?ZAxg?9DeHe2;gDH!`~kj<>a(@>`4x4ZwWcS8(HjbJFj2 z@SQv$-q<)$aNLXJ#P<&Pdf^o}Nv8{r{)`jf0`M&xg?*r-z4e;`z8S!*pXjY0UG{xb z;*BfM_SSDZ`2GOqV{Yq0nC7BiHsteW3kIhuIOUgq&SSuxcCKLE`f)XGX^a=&JGgik zaOckvKDTj(e2;027vEM~d3+^zFAaBq}H=WA9W7%#rouwOE8r&fCN zab8=bFxP2kDCVGBZ9>)OpjmEl_ z@6Et<4#XQT;b3qZk6y#&&wzPwp5VAIE!f_I<0|Xr^168AdK?VMmb70rUWSibw`+_7 znmF71V+Z7S0oO2J_}ucrdis*ac=6F+dlk657DVU!r^a~keT9pk1NZj%(fN`u5R4Ze z^LrF~aHbR1Kb#~Y7dBRH>oR9zcyT!4eYD<78ub5#^v8aQqO=4nS)e3=W2|~Fg~2>AKPyo zaLqSKe%<3u*>t;Vca5I2;@gu>}pLXIahQXSB3_s#?!3_n@DZgs>;qG{2APxq8 zI`K8fg?wP9eI+>d|4w|oP5KTnJ2lSDe*1A@4=`VSBYbZ5<4(-{Z_z*FVBn{de%vR1 zBQPiK6&&>l>KE?6yba998fWQe^Pf2{{z7ALs)AFxT=&@rT>d^u*R7tiUar;{P3M@8 zeYkiBaI1b0KDTk3 zz-<4?TR-x({W;#~frG(~F9{b00+aiTH{Z43s0ZelUxPTu@_@m{19QKcX~|yoV1C2v z!T9(D;~X4e`q|bUxqdwxn7SBnmjZKD47i(sSrY?p9WWbXz-Yj0 zy!lZH#xxuZ{B+{WM!E}td9sz@-1?W35Wn5g3C0i{46I#Fe605qfSG)Z;N03}D&k)a z%pDqsXpVebkGLP0ACDD2w|;~25v>J-Qx%-_+kq=%f$MRcw|=xkg~kY*vwpN+9dM7g z5k5EjQQr3W==zNYUkBi-+j{eHm;ZafG&@0XZsjo$d?x}kwVmMH#-nWTy$#G!CwlW; z0KQX!xjoIBuMPN)KPkcJje`Nv9LqN!7lr`yr}lz#vmfir_rN5c?9In@>K4Fcr3=oj zzHpuP3Sf3pn6w^(-cnAZKh`7T8KmB1XCB{)QL;!nA?X4&dtud!FNP0k3b}=y5Yn)rZy9j*u0h2ja_}s=p@|6Q~xyHHivERKBm}B$3`FM4L#^6*1r*dPz zI|;Zu&lEn|#mRoW`ntvln@c?a-3@tHyZo?ZL$i;L-}b`A!Bd512U`2Q?h) zX*w>{0(1RD;d3ir@_hkJ^GSkp5R0XH}uBLuP@F$DC`FM4o#%MZ6`|-ZtPr!Ze z7e2TA(th1b;BVkyaI@c5T+0Qf?JRFTUY(*bmrB1+M#S;qz+0lQl+>x$sfH zvw`bT>dkjMalrhpac<)RmST-D=i>c5<-+II-uQI!nHqyr6`b-*|KK9v9;)!><9(6% z$^_$R91L#ljrr{aOk9=V-0VlbPQVP*I5)mcz~uwe`aEww)|YHx&d@lw`qBy)CIVAh zEqreEk5~VqF*sGh$$mR<(e+yl%qop@<74|+1I(fe zz4fDA*Jun*RdBQ41He6fk?^_o2OQs;Uz}h}#KGWJU%EhXA21D<2+qyFp8<}9O9ewd z1}FVyfUgL+7ZwSY-e~!{P*E!R3Nu{dVFT2!(b5lYWih=m!M(>hPC@3HaK4 z9qPdDTVG@T`T{dd<1D*4;Kl%RRt&fzU@9GPLI0Ur&DR*w*g3!bpwI=tJ#xL| z!>xUAdu8hz;2+#1I5+!cAj1J*?zmfUUiQmhgZT^&1~2M_CKR0Su#B=C&`Zt~-TbsK-F&(#{E=^XV-1mAVQ-S|XwzAYLPjqgL?K7P`h z?=+Ac|5So80tW-saMlkECm)!CzX^`(`A+uBgM1$_1D_F`)!uCPQ+AOLn10U*uC)sv z*9$s6pJ0r~!QiG}68NS8Q@p`jKd!$dzL;QKf`fsdPUV}A3%3CC?Iyvs1M{_19SBI-hAv2&IINL zjU(DAzwG}W2WI~V(fRs)i28zq!Hq8)e76Ae@iuS1bHQ=^KN5_^I2dRbC;jTd_Z%=e z9|v(x^^fiMLSXLJIJf#&48EsOQ% zz5?!D;&yoRodn$Pz_j~Ja8Se1emfxV2u$;x(fRU#Db+YPzFpv}0cPsw-h4xWyAGK5 zHI8T}`*HmE379{2MdusvMS^i54hAr7y#f9uUh{jLY*LydFOkMrqY zfI0HJ=zJrAY0x+~K91*q1?IlJ(fPgr=Ctnx=f=nUjs#}nK5ssbmsbMwl*S>NWBud( ziWh;Y{ZaVb?mM%8kNXM!I1UCkKCZ{65c9LQe(ix9s4?VYaB3fT-Ihk%D#e59lW?(->iM)^9)Zdk1i3H4$PU3xL|&%)l`k4ehg0eO@rb-;4U5|`EV;= zUVTbq6qyI#v%r0PMs&X8M++tz--*Bt855oFe2wwq<9K;7aChlfoqFZ>pXTdOHKCLJ z$oDyL$@$UsJ40i<^kaF91FrJS=zRBTOf(Y1iP8B!)EF;5w%3n=`)yKmzP^*A=a+l~fSWwUn{OwSx*C}IX9aOi_2qH&SC0Zy zK231p<594GNdw=J(-V!ZI2fQtNdFRS=NlmJ3(RjC=Qhvd7VfU+BpN@=h@S4B3loh; zaWJ^0`!23M3(Pe$1;=*elwaJ6GByL#uS9U+>#^baHD)CmX*d|d(+%Rf;96&3CPl%q zf1UwMnIkS}Kl)*Ffmvcrti;ZtRF5Cdj*=52PZs(wWT#syD4u1m&gB#y=T>i0W+-9n~(Zk49w*k=f<}kxGRAVSD|p76Q(P4sdz>Z0pMzMjBc zK3}l3yHkEI0B#E~6V4YL+m)032ExC36__Iy3T{LgU$Fh24EYO-6OElX7+`kCer+W> zlwX0lVwvFhoQG4oWq28X^W}*~-5S9qLEm6Lg6UH4P3sbkcW^KO7Lsl_?jvA!$AH^M z%wy5{ngVl-BQDr3n@}H(p?(Zb`Q?1I6L4K0mvq_h2J;)_yC1kxV2*uKaLru!x}YL0 z2Ij#Hf^#c3+OPa2jBhv?yzI9Sm_Iq=@pNE@eI_`!c22&lfmx?WeZv7&^aj**rhNm_%&O1(U*)IA`0pEAW zH!?<^;LVo?zCQu8_7rcvlaNmLj*X0b91Ng$tbem{;cQ@@?<6?4`x@+b#&#AA`52tq zFZsG#;2lVe9osac754Z>VccF;X+4XRu2-KTmO71I9d;GWZZy*f&GS){WzZQ2IiY=!MXVdoG;v#)5v%N2ZNh_ zGjQPrU``(9tsm>(jT)l}@!_OjG5GEQuElWSbF0sT5PuXfmusBcc=VX+#dHH_>3mO?yaWJ^)_cJc|ftfx@aBlwTQNaBUOv)6&QC}zhX2AdJ z2h6Ief^(Z+k}og~^?7=9zO}$47kcyYbxdCZb5pV4+}6!r!-b~)M#g(pf}4SS1pU9@ zIs^NiYXgmp+i);g_Om_r`6$v|15DvO?{u4k@C{%-To^sw8!vBU+=as{-J5XnL14x# z5gb3A$|C^%E(YfND+K3uf8jV}@TrDI#yiUd=T*M@ftj{aaBk&GzJ6CWGS0jzI^TR? z-v6^V-<`S~;KsKe7d8R2WUb)b#!Koq@PS6gC>&mV)UN=Tz=P5C+X_s-hrIcy-%w!UAMw_Y z{qs6t;vV(pWB=R?m@VrB=Qh5vJ)HWOU{r?j;Z(ky&zuI_u*bdmPC)#tfq6jV5Y3VA zZd`a0n3+!spWA$k_aVN~7@Vr$q#w)UC*Y2KTKL@RALSD?M$2ggZ6Hi|;G&eFof{ z&kLX1cu6~)_kv)&_zdvX0$001ux{ni8YB;Dj29o*>s|)#i5G>>tv<6JHQ9)IfP(>Q zIF@f0TsR4sF)sots5Wm-3g7M>>gty0`Xz#| z5pYem3f8S2@ak}l@zRfcqktRxzBeDs;|h)O;^R8V3gFg#;LXNKZZ#ynN4ctxt@Yaw0##W8-;$y$D2e|DY37=cP@icH5A2%|-!ofg$ zI@M>kqgLCI-yMQ;yRX51yx_A&MmY`!n8}fk^Z0qdod3Dt-0E{22)+kq-!8%N(@8(x z;_UE6Bjd-f1sC4VgX0$0f!lxE$jHIL;HKYdTsQ-mFZT$}t$cfdW88O*jD$e^kKLPI6ebM<| z))+57w%^x)+x$cH{5JnlFwyvufjjCaZ$91!8>cZ|d^>P)25`AQd-D|lR}aiP8b`EK zdE9~gHvR?U;IG2xR$n-eZSk96aH@h6-vhXE9B?mdtXuiAU4It?Ukdd51-K@^d*_$> z4c8c@96p@%>j%Dk;FkQ$n{O=QZvdv{AA&=(cjQaQg$}@sien3M-_Ku!!5`cvkz_YsYaEF27o=BQsDF608U zEXi9xrn3i_pEb^n?04Q!NI^!r~ZI^otri`df;I2;u{D|$7bGqw12V2C_;R= z*>5&*OPWXL+n_OCe4JNo0&d5V(fQI^2*!&q9enA)aQ%?DQ~My_Q)!KjmM2H&>j2Cv9liM`A)Q}43&tzI z`&b^QH8yg)3f8TEA>TC`W0w(e>L3TzBA($P_F;o$QC7WHg2W z^GgrG(I0f`57uD8@B5yOjavo?F1$SI5K996!Z!vrHa^6`0D9rG&C8hXXTaPz#5>)) zk?x;#RGlb8reE$sGN5IsK5gh#p zC;Q!m3tNDxnIgD+;DY55v>)Gxa`n{4#!egzR{L;xKI&Is8c!2kxSfN1ERPgm+B)Kb zem2XqgT^pj2B&;9heGE7ckVfoZVGV0d<6NppTDV3FyvxzpSM1G4KWvc3;8X>t{Fb4h@_>89FMMwG>S{0~ zmNYi5!@*$H16w_P2NRHcfw}lx!L0jl|$MU#YW2hg4ll?f(+zQ-_=LsJ_o$|~3$#K<cJN-g$zU{y*1g7XB!Es*iR2~6bxCEFVE)iT?;GFbp4f&A^1%p!+ zocOrza|&=rFA}Vq{iu&$V}#9FKlWF%fy-VjeEf9Mk9-$sj29pGt6U7+)XTj2_}-)I zfVoHGP|Y0e_YUNb0JG$BZ@xOeb(OqFc)Z?WoO&CwF$V(fEjzWH(xPu zR{-;k#<`7y*|-pYO=Dx|wZa!(55o29aUJISI2hdY+kk7iz$C5m){pkf1!knix$&`I zn*hw|*9)IpJD&{Pb-;Y1ac=qDfrR33fS-7y@LBhX>xv5lg&@(P6~1q{Nie1?J0&B# zTU9uY?c;vnp43?O4+>}FqrbN`M!sFZ?Tf-kJG8txHvQTF*Fj_5^dnzxjC`Ykn-YbO zc9|O^-=)BzMa7Bje#%ae_|_-*1)A}tXFvqjgfB@aFb%- z3&hBG0dPxV;JY(MzWafDG6ue_G4gE(?yD$#Y`;x!iLE@^0oPe$y~-mmM!tODrbXdn zKf54CzN>)ypD286Z%@a_w+Xm+V&K~wBj3M(Yr0x&?PfogM@NmpsS0lW=|JG}qVTal zD2b7;0=NZH_-MaXG4kC3+yhbgnBUid`P7bc7(a|#8yn4WFu2)|<40#;vg|kqzA3;I z+i|w>i21!#W0)?3TY0Pi?uICQ%~2bgL*&OyJGz}#fVIq*HHF-k#vxY_S@;68|f?`L3|-w{2(oq_3N$Jz9w{R)67 zw&NV~dnqt0>^K`A?Y9P)^>&;C-$%ggw&QGk)Gz5Tl2@Fn;AX$pz@=-fRS#@@96yEv zbG9AlpkEy@i|jZXAN9Kpn1}2*2fod~Y_sDW_Jq`x9{IWcH0h4FP zIp|jcOtl^7px;VhR@-q7e9r*$iXCU;<9M`7W0(&HxANTs-2NzhEZ;VF;r=BK2ForE z`sD*N&5pC_NBb=RW`!N+z_$jN$L%;9AM^X3#!x>7xBPwt-0m3olJ1VJeWU=_R%0!@ zIOvxZBi~Tq#>T)`79(E(xcV6Q{wGGhJAr#B3LnSw%`x(A18!Fge2xDaTY0nuuARoZ zl?UtJKw$DUj`MT3`ce!`m7T9TeBRv=@mBz|E{rcbyPK^Z^aJh{VB+tQyl`DfrE9~n z|H}hripGh4k$lWY88G!>IOlbNYk*m;ah6>i^6?BXJ2j5$AlWIA>9W1;0p{lzaPjxX zR&Gs!J4RzI{cLt&y1ioL8wgxp416Uq@>Kx0AO^lwG4ibjZcPk)uf)jr7H}WMz_&j} zzQi@y_ke@Jt30}Ci~{1rtsV6NZfFdAGh*bM4O}1wzN=#7y8*a6qwuj^ZUE*Tjq~a^ zb^)``&ganHTHc3w6b=Tj_Ld9G=qNbqHwl;$jq}oP0Wg>uG1;`e>}1{W$Iy z05jW;bMT)p1Li6_&Q>1O?*U-e+i?zj|A)PIfsd-X+Qv^v2w@T-LC}b|fKdb=au;U!$Jp;_EcAQNf?fG+sNmjpK1LwIbxnCQu zFv;W<0XH#;Jo?>5DavaCE|f%G0px7}X0yU2@@rdx*>0ET;MYC`=CHdH`!&u(M*vf$ zaBlWe1l(d^HYgmWx{YI>Q5ZnP;pW#~1ny0RbxW7+=<5{a^}7epvGL*ICXaSisxShG zKW_P}0IoI#c}r827Xt3W6y#l-qP&g3ZB8PO>oA@L=6!`rWLIAUt0(qZ2pDqw?JV) zio?wgMgmuwg1q@D%Bu&iEr~p~%kKlTQQ;ER+x@_7waas;x3_`u+$Z(oR&Q({#{hGJ z!Wli=^vC=a0pnFT+1}#kwbXNy!hjTqo1R0!T^J)T-d-s0+7#t&0`8t9@>t#-Dav~d zxSc7;%eY_aSeDk^zvKXyuds>y)%X