Fix failing on some models (#20)

mozilla · Jun 21, 2022 · 59397a5 · 59397a5
1 parent 398675f
commit 59397a5
Show file tree

Hide file tree

Showing 7 changed files with 37 additions and 15 deletions.
diff --git a/3rd_party/crow b/3rd_party/crow
diff --git a/3rd_party/moz-bergamot-translator b/3rd_party/moz-bergamot-translator
diff --git a/Dockerfile b/Dockerfile
@@ -24,6 +24,11 @@ RUN sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.
     apt-get update && \
     apt-get install -y intel-mkl-64bit-2020.0-088
 
+# attempt to use openblas
+#RUN apt search openblas
+#RUN apt install -y libopenblas-dev
+#RUN update-alternatives --config libblas.so.3
+
 # Bergamot
 # pcre2 is requried to build berrgamot-translator with -DUSE_WASM_COMPATIBLE_SOURCES=off
 RUN apt-get install -y libpcre2-dev

diff --git a/Makefile b/Makefile
@@ -16,7 +16,7 @@ run:
 	docker run --name translation-service -it --rm -v $$(pwd)/models:/models -p 8080:8080 -e PORT=8080 translation-service
 
 debug:
-	docker run --name translation-service -it --rm -v $$(pwd):/app -v $$(pwd)/models:/models -p 8080:8080 translation-service bash
+	docker run --name translation-service -it --rm -v $$(pwd):/app -v $$(pwd)/models:/models -e PORT=8080 -p 8080:8080 translation-service bash
 
 call:
 	curl --header "Content-Type: application/json" \

diff --git a/scripts/download-test-models.sh b/scripts/download-test-models.sh
@@ -1,24 +1,32 @@
 #!/bin/bash
 
+version=0.3.0
+
 mkdir models
 cd models
 
 mkdir esen
 mkdir enet
 mkdir enru
+mkdir enbg
 
 cd esen
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/lex.50.50.esen.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/model.esen.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/esen/vocab.esen.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/lex.50.50.esen.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/model.esen.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/esen/vocab.esen.spm
 
 cd ../enet
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/lex.50.50.enet.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/model.enet.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enet/vocab.eten.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/lex.50.50.enet.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/model.enet.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enet/vocab.eten.spm
 
 
 cd ../enru
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/lex.50.50.enru.s2t.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/model.enru.intgemm.alphas.bin
-wget https://storage.googleapis.com/bergamot-models-sandbox/0.2.10/enru/vocab.enru.spm
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/lex.50.50.enru.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/model.enru.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enru/vocab.enru.spm
+
+cd ../enbg
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/lex.50.50.enbg.s2t.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/model.enbg.intgemm.alphas.bin
+wget https://storage.googleapis.com/bergamot-models-sandbox/${version}/enbg/vocab.bgen.spm
diff --git a/src/translation.h b/src/translation.h
@@ -107,13 +107,14 @@ namespace marian {
                         "\nmini-batch-words: 1024"
                         "\nworkspace: 128"
                         "\nmax-length-factor: 2.0"
-                        "\nskip-costls: True"
+                        "\nskip-cost: True"
                         "\nquiet: True"
                         "\nquiet-translation: True"
-                        "\ngemm-precision: int8shift";
+                        "\ngemm-precision: int8shift"
+                        "\nalignment: soft";
 
                 options = options + "\nmodels: [" + modelPath + "]\nvocabs: [" + vocabPath + ", " + vocabPath +
-                          "]\nshortlist: [" + shortlistPath + ", 50, 50]";
+                          "]\nshortlist: [" + shortlistPath + ", false]";
 
                 return options;
             }

diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py
@@ -44,6 +44,14 @@ def test_non_latin():
     assert translation == 'Как дела?' or translation == 'Как у тебя дела?'
 
 
+def test_different_shortlist():
+    query = {"from": "en", "to": "bg", "text": 'Hello world'}
+
+    resp = requests.post(url + '/v1/translate', json=query)
+
+    assert resp.status_code == 200
+    assert resp.json()['result'] == 'Здравей свят'
+
 def test_json_escaping():
     query = {"from": "es", "to": "en", "text": '"Hola Mundo"'}
+8 −8		.drone.yml
+1 −1		.travis.yml
+21 −8		CMakeLists.txt
+7 −11		Doxyfile
+2 −2		LICENSE
+3 −2		README.md
+10 −4		cmake/CrowConfig.cmake.in
+22 −0		cmake/cmake_uninstall.cmake.in
+90 −0		docs/assets/crowlogo_main_light_color.svg
+105 −0		docs/assets/fast_light_icon.svg
+67 −0		docs/assets/header_light_icon.svg
+67 −0		docs/assets/typesafe_light_icon.svg
+48 −0		docs/assets/websocket_light_icon.svg
+129 −0		docs/getting_started/a_simple_webpage.md
+0 −146		docs/getting_started/setup/legacy.md
+25 −8		docs/getting_started/setup/linux.md
+21 −3		docs/getting_started/setup/macos.md
+1 −1		docs/getting_started/setup/windows.md
+5 −2		docs/getting_started/your_first_application.md
+6 −1		docs/guides/app.md
+71 −0		docs/guides/auth.md
+9 −0		docs/guides/base64.md
+7 −8		docs/guides/blueprints.md
+6 −4		docs/guides/compression.md
+43 −0		docs/guides/included-middleware.md
+10 −4		docs/guides/json.md
+11 −0		docs/guides/logging.md
+90 −24		docs/guides/middleware.md
+14 −4		docs/guides/multipart.md
+1 −1		docs/guides/proxies.md
+13 −6		docs/guides/query-string.md
+119 −13		docs/guides/routes.md
+2 −2		docs/guides/ssl.md
+16 −5		docs/guides/static.md
+32 −4		docs/guides/templating.md
+2 −2		docs/guides/testing.md
+26 −10		docs/guides/websockets.md
+208 −35		docs/overrides/home.html
+11 −2		docs/overrides/partials/footer.html
+37 −0		docs/overrides/partials/social.html
+54 −9		docs/stylesheets/colors.css
+48 −0		docs/stylesheets/extra.css
+5 −0		docs/versions.json
+14 −8		examples/CMakeLists.txt
+12 −0		examples/example.cpp
+0 −19		examples/example.py
+88 −0		examples/example_middleware.cpp
+1 −1		examples/example_static_file.cpp
+0 −44		examples/example_test.py
+32 −0		examples/middlewares/example_cookies.cpp
+36 −0		examples/middlewares/example_cors.cpp
+1 −2		examples/websocket/example_ws.cpp
+2 −2		examples/websocket/templates/ws.html
+1 −0		include/crow.h
+122 −15		include/crow/app.h
+173 −28		include/crow/common.h
+109 −187		include/crow/http_connection.h
+841 −1,468		include/crow/http_parser_merged.h
+21 −6		include/crow/http_request.h
+30 −90		include/crow/http_response.h
+68 −39		include/crow/http_server.h
+81 −47		include/crow/json.h
+9 −1		include/crow/logging.h
+331 −0		include/crow/middleware.h
+8 −13		include/crow/middleware_context.h
+144 −8		include/crow/middlewares/cookie_parser.h
+200 −0		include/crow/middlewares/cors.h
+3 −1		include/crow/mime_types.h
+98 −23		include/crow/multipart.h
+102 −8		include/crow/mustache.h
+31 −16		include/crow/parser.h
+45 −41		include/crow/query_string.h
+188 −117		include/crow/routing.h
+9 −4		include/crow/settings.h
+1 −1		include/crow/socket_adaptors.h
+4 −10		include/crow/task_timer.h
+189 −22		include/crow/utility.h
+70 −28		include/crow/websocket.h
+91 −64		mkdocs.yml
+33 −12		scripts/generateDocumentationAndDeploy.sh
+7 −4		scripts/merge_all.py
+20 −14		scripts/nginx_mime2cpp.py
+2 −1		tests/CMakeLists.txt
+385 −244		tests/catch.hpp
+1 −0		tests/img/filewith.badext
+6 −0		tests/multi_file/CMakeLists.txt
+15 −0		tests/multi_file/main.cpp
+1 −0		tests/multi_file/secondary.cpp
+16 −2		tests/ssl/ssltest.cpp
+1 −1		tests/template/mustachetest.cpp
+714 −181		tests/unittest.cpp
+1 −1		vcpkg.json