diff --git a/build.sh b/build.sh index ada6b37f..d0c710bf 100755 --- a/build.sh +++ b/build.sh @@ -30,9 +30,9 @@ do_build() { # Check if ASan should be enabled if [ "$ENABLE_ASAN" = "ON" ]; then - cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_ASAN=ON ../source + cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 -DENABLE_ASAN=ON ../source else - cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../source + cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 ../source fi make -j${CORES} diff --git a/docker/workdir/TABLE/otmxml.c b/docker/workdir/TABLE/otmxml.c index b607be07..9302cb94 100755 --- a/docker/workdir/TABLE/otmxml.c +++ b/docker/workdir/TABLE/otmxml.c @@ -236,23 +236,6 @@ EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( -USHORT /*APIENTRY*/ EQFQUERYEXITINFO(PSZ pszTagTable, // name of the markup table, e.g. "OTMHTM32" - USHORT usMode, // type of information being queried - PSZ pszBuffer, // buffer area receiving the information returned by the exit - USHORT usBufLen // length of buffer area -) -{ - - if( usMode == QUERYEXIT_ADDFILES) { - QueryExportFiles(pszTagTable, pszBuffer, usBufLen, FALSE); - } - - return 0; -} - - - - /******************************************************************************* * * function: IsFileUTF16 diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 00000000..5758e384 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,23 @@ + + + + + T5Memory API + + + + + + + + + + + diff --git a/docs/t5memory.apidog.json b/docs/t5memory.apidog.json new file mode 100644 index 00000000..c4c0a0ae --- /dev/null +++ b/docs/t5memory.apidog.json @@ -0,0 +1,5194 @@ +{ + "apidogProject": "0.7.36", + "$schema": { + "app": "apidog", + "type": "project", + "version": "1.2.0" + }, + "info": { + "name": "t5memory", + "description": "", + "mockRule": { + "rules": [], + "enableSystemRule": true + } + }, + "apiCollection": [ + { + "name": "Root", + "id": 4550416, + "auth": {}, + "securityScheme": {}, + "parentId": 0, + "serverId": "", + "description": "", + "identityPattern": { + "httpApi": { + "type": "methodAndPath", + "bodyType": "", + "fields": [] + } + }, + "shareSettings": {}, + "visibility": "SHARED", + "moduleId": 845181, + "preProcessors": [ + { + "id": "inheritProcessors", + "type": "inheritProcessors", + "data": {} + } + ], + "postProcessors": [ + { + "id": "inheritProcessors", + "type": "inheritProcessors", + "data": {} + } + ], + "inheritPostProcessors": {}, + "inheritPreProcessors": {}, + "items": [ + { + "name": "Flush memory", + "api": { + "id": "20097720", + "method": "get", + "path": "/t5memory/{{memory}}/flush", + "parameters": {}, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28998661", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "msg": { + "type": "string" + } + }, + "required": [ + "msg" + ], + "x-apidog-orders": [ + "msg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"msg\": \"Mem ID9612-en-uk was flushed to the disk successfully\"\n}", + "responseId": 28998661, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "Endpoint is sync(blocking)\n\nIf tm is not found on the disk - returns 404\nIf tm is not open - returns 400 with message\nThen t5memory requests writes pointer to the tm (so it waits till other requests that's working with the tm would finish) and then it flushes it to the disk\nCould also return an error if flushing got some issue.\nWould not open the tm, if it's not opened yet, but instead would return an error.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 130, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28998661" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Tag replacement", + "api": { + "id": "20097882", + "method": "post", + "path": "/t5memory_service/tagreplacement", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "xMxiCFLsH9", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "OEUJrQ83Sh", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28998668", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "src": { + "type": "string" + }, + "trg": { + "type": "string" + }, + "req": { + "type": "string", + "description": "Used to test Fuzzy search tag replacement" + } + }, + "required": [ + "src", + "trg" + ], + "x-apidog-orders": [ + "src", + "trg", + "req" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"src\": \"Tap View o get strongdisplayedView two strongUS patents.\",\n \"trg\": \"View tap to got strongdosplaydVeiw two strongUS patents.\",\n \"req\": \"Tap View o get strongdisplayedView two strongUS patents.\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Fuzzy search tag replacement and Import tag replacement test", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 105, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28998668" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Flags", + "api": { + "id": "20120389", + "method": "get", + "path": "/t5memory_service/flags", + "parameters": {}, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "29040387", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {} + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n {\n \"name\": \"alsologtoemail\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to these email addresses in addition to logfiles\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"alsologtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr in addition to logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"colorlogtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"color messages logged to stderr (if supported by terminal)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"drop_log_memory\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Drop in-memory buffers of log contents. Logs can grow very quickly and they are rarely read before they need to be evicted from memory. Instead, drop them from memory as soon as they are flushed to disk.\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"log_backtrace_at\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Emit a backtrace when logging at file:linenum.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_dir\",\n \"value\": \"/root/.t5memory/LOG/\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"If specified, logfiles are written into this directory instead of the default logging directory.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_link\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Put additional links to the log files in this directory\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_prefix\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Prepend the log prefix to the start of each log line\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"logbuflevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; ...)\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"logbufsecs\",\n \"value\": \"30\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages for at most this many seconds\",\n \"type\": \"int32\",\n \"default\": \"30\"\n },\n {\n \"name\": \"logemaillevel\",\n \"value\": \"999\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Email log messages logged at this level or higher (0 means email all; 3 means email FATAL only; ...)\",\n \"type\": \"int32\",\n \"default\": \"999\"\n },\n {\n \"name\": \"logfile_mode\",\n \"value\": \"436\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Log file mode/permissions.\",\n \"type\": \"int32\",\n \"default\": \"436\"\n },\n {\n \"name\": \"logmailer\",\n \"value\": \"/bin/mail\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Mailer used to send logging email\",\n \"type\": \"string\",\n \"default\": \"/bin/mail\"\n },\n {\n \"name\": \"logtostderr\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr instead of logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"max_log_size\",\n \"value\": \"1800\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"approx. maximum log file size (in MB). A value of 0 will be silently overridden to 1.\",\n \"type\": \"int32\",\n \"default\": \"1800\"\n },\n {\n \"name\": \"minloglevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Messages logged at a lower level than this don't actually get logged anywhere\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"stderrthreshold\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages at or above this level are copied to stderr in addition to logfiles. This flag obsoletes --alsologtostderr.\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"stop_logging_if_full_disk\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Stop attempting to log to disk if the disk is full.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"symbolize_stacktrace\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/utilities.cc\",\n \"description\": \"Symbolize the stack trace in the tombstone\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"v\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"Show all VLOG(m) messages for m <= this. Overridable by --vmodule.\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"vmodule\",\n \"value\": \"\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"per-module verbose level. Argument is a comma-separated list of =. is a glob pattern, matched against the filename base (that is, name ignoring .cc/.h./-inl.h). overrides any value given by --v.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"flagfile\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"load flags from file\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"fromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment [use 'export FLAGS_flag1=value']\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tryfromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment if present\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"undefok\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"comma-separated list of flag names that it is okay to specify on the command line even if the program does not define a flag with that name. IMPORTANT: flags in this list that have arguments MUST use the flag=value format\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tab_completion_columns\",\n \"value\": \"80\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"Number of columns to use in output for tab completion\",\n \"type\": \"int32\",\n \"default\": \"80\"\n },\n {\n \"name\": \"tab_completion_word\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"If non-empty, HandleCommandLineCompletions() will hijack the process and attempt to do bash-style command line flag completion on this value.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"help\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags [tip: all flags can have two dashes]\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpfull\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags -- same as -help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpmatch\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on modules whose name contains the specified substr\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helpon\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on the modules named by this flag value\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helppackage\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all modules in the main package\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpshort\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on only the main module for this program\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpxml\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"produce an xml version of help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"version\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show version and build info and exit\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"folly_memory_idler_madvise_stacks\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler madvises dontneed stacks on thread idle\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_memory_idler_purge_arenas\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler purges jemalloc arenas on thread idle\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"dynamic_iothreadpoolexecutor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will dynamically create threads\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_iothreadpoolexecutor_max_read_at_once\",\n \"value\": \"-1\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will use this value as default for maxReadAtOnce in its event bases, valid values are [0, inf)\",\n \"type\": \"int32\",\n \"default\": \"-1\"\n },\n {\n \"name\": \"threadtimeout_ms\",\n \"value\": \"60000\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/ThreadPoolExecutor.cpp\",\n \"description\": \"Idle time before ThreadPoolExecutor threads are joined\",\n \"type\": \"int64\",\n \"default\": \"60000\"\n },\n {\n \"name\": \"observer_manager_pool_size\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/observer/detail/ObserverManager.cpp\",\n \"description\": \"How many internal threads ObserverManager should use\",\n \"type\": \"int32\",\n \"default\": \"4\"\n },\n {\n \"name\": \"folly_hazptr_use_executor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/synchronization/Hazptr.cpp\",\n \"description\": \"Use an executor for hazptr asynchronous reclamation\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"dcache_unit_test\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/wangle/wangle/ssl/SSLSessionCacheManager.cpp\",\n \"description\": \"All VIPs share one session cache\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"zlib_compressor_buffer_growth\",\n \"value\": \"2024\"\t\",\tfrom\": \"/home/libs/proxygen/lib/utils/ZlibStreamCompressor.cpp\",\n \"description\": \"The buffer growth size to use during IOBuf zlib deflation\",\n \"type\": \"int64\",\n \"default\": \"2024\"\n },\n {\n \"name\": \"request_number\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/RestAPI/ProxygenHandler.cpp\",\n \"description\": \"Include request sequence number in response\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"add_tokens_to_fuzzy\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, list of tokens would be returned in fuzzy responce. Could make execution a bit slower, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowLoadingMultipleTmsSimultaneously\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, multiple tms could be loaded from the disk at the same time. Loading multiple TMs at the same time was disabled because of I/0 disk operation limit at the server. Enable this if you don't care about num of IO operations and want some perfomance boost\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowedram\",\n \"value\": \"10000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought RAM(in MB) allowed for service to use\",\n \"type\": \"int64\",\n \"default\": \"5000\"\n },\n {\n \"name\": \"allowedtmdsize\",\n \"value\": \"190\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets max size of tmd file(in MB) after which t5m would not allow to add new data to the tm. This helps fight btree and lookup table issues when tm gets bigger.\",\n \"type\": \"int64\",\n \"default\": \"190\"\n },\n {\n \"name\": \"debug_sleep_in_request_run\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set, provide artificial delay in every request handling execution equal to provided num of microseconds\",\n \"type\": \"int64\",\n \"default\": \"0\"\n },\n {\n \"name\": \"disable_aslr\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, process personality would be set to ADDR_NO_RANDOMIZE, affect memory layout, set to true if you have some linker or asan issue at the launch\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"enable_newlines_in_logs\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines,\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_at_shutdown\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm when shutting down the app not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_to_disk_with_every_update\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm to disk with every successfull update request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"forbiddeletefiles\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Set to true to keep all files(including temporary and tm)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"keep_tm_backups\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving tmd and tmi files, old copies would be saved with .old suffix\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"limit_num_of_active_requests\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, it would be possible to handle only up to servicethreads-1 requests at the same time, the last thread would respond with 503 to eliminate creating queue of requests waiting to be handled.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"logMutexes\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true you would see mutex logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_end\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request end with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_start\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request call with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_file_locks\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, action about locking tmd and tmi file would be logged\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_hashes_in_hash_sentence\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, tokens and their hashed would be logged, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_memmove_in_compareputdata\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving segment and causing memmove in compareputdata functions, just before memmove, data would be logged - use this to debug btree crashes. In that spot in come corrupted data, memmove could crash t5memory(or cause asan crash). Just to set low log level(v=2,t5logleve=0), reproduce the crash and check if the last log was one that started with \"memmove size = ...\"\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_tm_lifetime\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, TMs ctor(when tm object was created-so before it started loading tm) and dctor(when tm object was destroyed - so after closing tm) would have transaction level logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"maxBadSegmentsIdsSaved\",\n \"value\": \"20\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Defines how many segments(it's id's) that failed during import or reorganize would be saved to the status response\",\n \"type\": \"int32\",\n \"default\": \"20\"\n },\n {\n \"name\": \"newEntryIfOnlyContextAndTimestampDifferent\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new).\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"port\",\n \"value\": \"4040\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"What port to listen on\",\n \"type\": \"int32\",\n \"default\": \"4080\"\n },\n {\n \"name\": \"saveDifferentTargetsForSameSource\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"servicename\",\n \"value\": \"t5memory\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets service name to use in url\",\n \"type\": \"string\",\n \"default\": \"t5memory\"\n },\n {\n \"name\": \"servicethreads\",\n \"value\": \"10\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought of worker threads for service\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"skipStartupLogs\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, values from --t5loglevel would be applied immediately, if set to false- you would see all startup logs and --v and --t5loglevel would be applied only after start of the service and it's init message. Set to false if you have issues at startup\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"skip_default_flags_in_init_msg\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, all default flags would be skipped in init msg\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"suppressTmVersionCheck\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"t5_ip\",\n \"value\": \"\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Which ip to use in t5memory(default is any). Should be in format '1.1.1.1', default is to listen to all available ip\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"t5globversion_max_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5globversion_min_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5loglevel\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets t5memory log level threshold from DEVELOP(0) to TRANSACTION(6)\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"t5majversion_max_supported\",\n \"value\": \"7\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"7\"\n },\n {\n \"name\": \"t5majversion_min_supported\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"t5minversion_max_supported\",\n \"value\": \"14\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(disabled)sets max minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"14\"\n },\n {\n \"name\": \"t5minversion_min_supported\",\n \"value\": \"60\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"60\"\n },\n {\n \"name\": \"timeout\",\n \"value\": \"180000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets timeout for service request handling\",\n \"type\": \"int32\",\n \"default\": \"180000\"\n },\n {\n \"name\": \"tmListLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM list(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmMaxIdleTimeSec\",\n \"value\": \"7200\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set some value, except, 0, during request, tm list would bee cleaned up from tms that are longer inactive than this number\",\n \"type\": \"int64\",\n \"default\": \"7200\"\n },\n {\n \"name\": \"tmRequestLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for part where request is requesting tm(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"triplesthreshold\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets threshold to pre fuzzy filtering based on hashes of neibour tokens, range[0...100]. It's pre fuzzy filtering of the segments, when t5memory tries to predict how high fuzzy rate would be for some segment based on data that doesn't require as mych work as fuzzy calculations. When value is low, it can select more suggestions for fuzzy calculation, but if it set high, some best matches could be skipped. In opentm2 default value was 33, but that value seems like not working well with segments, where there are a lot of tags.\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"useTimedMutexesForReorganizeAndImport\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, in reorganize or import thread would be used mutexes with timeouts, and reorganizee or import could be canceled, false(by default) - would be used non timed mutexes\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"wait_for_import_and_reorganize_requests\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, waiting for all import and reorganize processes to be done at shutdown when not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"true\"\n }\n}", + "responseId": 29040387, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "Return all available commandline flags. Do not spam too much because gflags documentation says that that's slow. Useful to collect configuration data about t5memory to do debugging.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 140, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.29040387" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Multifuzzy", + "api": { + "id": "20124748", + "method": "post", + "path": "/t5memory/{{memory}}/multifuzzysearch", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "29046185", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "responses": { + "type": "object", + "properties": { + "first": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy", + "results" + ], + "description": "key \"first\" is an exampe. it was set because request has \"customId\": \"first\"" + }, + "second": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy", + "results" + ], + "description": "key \"second\" is an exampe. it was set because request has \"customId\": \"second\"" + } + }, + "required": [ + "first", + "second" + ], + "x-apidog-orders": [ + "first", + "second" + ] + } + }, + "required": [ + "responses" + ], + "x-apidog-orders": [ + "responses" + ] + }, + "description": "Each response object in responses has a key and value of it contains response of simple fuzzy search.\nName of a key comes from \"customId\" in request.", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"responses\": {\n\t\t\"first\": {\n\t\t\t\"ReturnValue\": 0,\n\t\t\t\"ErrorMsg\": \"\",\n\t\t\t\"NumOfFoundProposals\": 2,\n\t\t\t\"searchedSrc\": \"First segment with int\",\n\t\t\t\"customId\": \"first\",\n\t\t\t\"forceFuzzy\": 0,\n\t\t\t\"results\": [\n\t\t\t\t{\n\t\t\t\t\t\"source\": \"First segment with int 10\",\n\t\t\t\t\t\"target\": \"Перший сегмент з числом 10\",\n\t\t\t\t\t\"segmentId\": 6,\n\t\t\t\t\t\"customId\": \"\",\n\t\t\t\t\t\"documentName\": \"none\",\n\t\t\t\t\t\"sourceLang\": \"en\",\n\t\t\t\t\t\"targetLang\": \"UK\",\n\t\t\t\t\t\"type\": \"Manual\",\n\t\t\t\t\t\"author\": \"MANAGER\",\n\t\t\t\t\t\"timestamp\": \"20250716T084347Z\",\n\t\t\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\t\t\"context\": \"\",\n\t\t\t\t\t\"additionalInfo\": \"\",\n\t\t\t\t\t\"internalKey\": \"10:1\",\n\t\t\t\t\t\"matchType\": \"Fuzzy\",\n\t\t\t\t\t\"matchRate\": 80,\n\t\t\t\t\t\"fuzzyWords\": 5,\n\t\t\t\t\t\"fuzzyDiffs\": 1\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"source\": \"First segment with int \",\n\t\t\t\t\t\"target\": \"Перший сегмент з числом \",\n\t\t\t\t\t\"segmentId\": 2,\n\t\t\t\t\t\"customId\": \"\",\n\t\t\t\t\t\"documentName\": \"none\",\n\t\t\t\t\t\"sourceLang\": \"en\",\n\t\t\t\t\t\"targetLang\": \"UK\",\n\t\t\t\t\t\"type\": \"Manual\",\n\t\t\t\t\t\"author\": \"MANAGER\",\n\t\t\t\t\t\"timestamp\": \"20250716T084347Z\",\n\t\t\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\t\t\"context\": \"\",\n\t\t\t\t\t\"additionalInfo\": \"\",\n\t\t\t\t\t\"internalKey\": \"7:1\",\n\t\t\t\t\t\"matchType\": \"Fuzzy\",\n\t\t\t\t\t\"matchRate\": 80,\n\t\t\t\t\t\"fuzzyWords\": 5,\n\t\t\t\t\t\"fuzzyDiffs\": 1\n\t\t\t\t}\n\t\t\t]\n\t\t},\n\t\t\"second\": {\n\t\t\t\"ReturnValue\": 0,\n\t\t\t\"ErrorMsg\": \"\",\n\t\t\t\"NumOfFoundProposals\": 2,\n\t\t\t\"searchedSrc\": \"Third segment with\",\n\t\t\t\"customId\": \"second\",\n\t\t\t\"forceFuzzy\": 0,\n\t\t\t\"results\": [\n\t\t\t\t{\n\t\t\t\t\t\"source\": \"Third segment with int 10 and float 1.100\",\n\t\t\t\t\t\"target\": \"третій сегмент з числом 10 і 1.100, друге захищене як ціле з точкою\",\n\t\t\t\t\t\"segmentId\": 8,\n\t\t\t\t\t\"customId\": \"\",\n\t\t\t\t\t\"documentName\": \"none\",\n\t\t\t\t\t\"sourceLang\": \"en\",\n\t\t\t\t\t\"targetLang\": \"UK\",\n\t\t\t\t\t\"type\": \"Manual\",\n\t\t\t\t\t\"author\": \"MANAGER\",\n\t\t\t\t\t\"timestamp\": \"20250716T084347Z\",\n\t\t\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\t\t\"context\": \"\",\n\t\t\t\t\t\"additionalInfo\": \"\",\n\t\t\t\t\t\"internalKey\": \"11:1\",\n\t\t\t\t\t\"matchType\": \"Fuzzy\",\n\t\t\t\t\t\"matchRate\": 37,\n\t\t\t\t\t\"fuzzyWords\": 8,\n\t\t\t\t\t\"fuzzyDiffs\": 5\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"source\": \"Third segment with int and float \",\n\t\t\t\t\t\"target\": \"третій сегмент з числом і , друге захищене як ціле з точкою\",\n\t\t\t\t\t\"segmentId\": 4,\n\t\t\t\t\t\"customId\": \"\",\n\t\t\t\t\t\"documentName\": \"none\",\n\t\t\t\t\t\"sourceLang\": \"en\",\n\t\t\t\t\t\"targetLang\": \"UK\",\n\t\t\t\t\t\"type\": \"Manual\",\n\t\t\t\t\t\"author\": \"MANAGER\",\n\t\t\t\t\t\"timestamp\": \"20250716T084347Z\",\n\t\t\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\t\t\"context\": \"\",\n\t\t\t\t\t\"additionalInfo\": \"\",\n\t\t\t\t\t\"internalKey\": \"8:1\",\n\t\t\t\t\t\"matchType\": \"Fuzzy\",\n\t\t\t\t\t\"matchRate\": 37,\n\t\t\t\t\t\"fuzzyWords\": 8,\n\t\t\t\t\t\"fuzzyDiffs\": 5\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n}", + "responseId": 29046185, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "requests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "source": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "context": { + "type": "string" + }, + "customId": { + "type": "string", + "description": "If provided will be ased as key in response block" + } + }, + "required": [ + "sourceLang", + "targetLang", + "source", + "documentName", + "context" + ], + "x-apidog-orders": [ + "sourceLang", + "targetLang", + "source", + "documentName", + "context", + "customId" + ] + } + } + }, + "required": [ + "requests" + ], + "x-apidog-orders": [ + "requests" + ] + }, + "mediaType": "", + "examples": [ + { + "name": "Example 1", + "value": "{\n \"requests\": [\n {\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"First segment with int\",\n \"documentName\": \"\",\n \"context\": \"\",\n \"customId\": \"first\"\n },\n {\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"Third segment with\",\n \"documentName\": \"\",\n \"context\": \"\",\n \"customId\": \"second\"\n }\n ]\n}", + "mediaType": "application/json" + } + ], + "oasExtensions": "" + }, + "description": "Sends multiple fuzzysearches in the same tm in a row", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 150, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.29046185" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Multi Update", + "api": { + "id": "20128944", + "method": "post", + "path": "/t5memory/{{memory}}/multiupdate", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "29051393", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "responses": { + "type": "object", + "properties": { + "fourth": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "description": "Key \"fourth\" comes from request \"customId\"" + }, + "fifth": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + }, + "required": [ + "fourth", + "fifth" + ], + "x-apidog-orders": [ + "fourth", + "fifth" + ] + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "Saved", + "description": "" + }, + { + "value": "0", + "name": "Not saved", + "description": "" + } + ], + "description": "Represents if new segmnets are saved to disk right away" + } + }, + "required": [ + "responses", + "save2disk" + ], + "x-apidog-orders": [ + "responses", + "save2disk" + ] + }, + "description": "Each response object in responses has a key and value of it contains response of simple update.\nName of a key comes from \"customId\" in request.", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"responses\": {\n\t\t\"fourth\": {\n\t\t\t\"source\": \"Fourth segmnet\",\n\t\t\t\"sourceNPRepl\": \"Fourth segmnet\",\n\t\t\t\"sourceNorm\": \"Fourth segmnet\",\n\t\t\t\"target\": \"Четвертий сегмент\",\n\t\t\t\"segmentId\": 9,\n\t\t\t\"customId\": \"fourth\",\n\t\t\t\"documentName\": \"test.xlsx.sdlxliff\",\n\t\t\t\"sourceLang\": \"UK\",\n\t\t\t\"targetLang\": \"EN\",\n\t\t\t\"type\": \"Manual\",\n\t\t\t\"author\": \"PROJECT MANAGER\",\n\t\t\t\"timestamp\": \"20250812T125637Z\",\n\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\"context\": \"390\",\n\t\t\t\"additionalInfo\": \"\",\n\t\t\t\"internalKey\": \"12:1\"\n\t\t},\n\t\t\"fifth\": {\n\t\t\t\"source\": \"Fifth segment\",\n\t\t\t\"sourceNPRepl\": \"Fifth segment\",\n\t\t\t\"sourceNorm\": \"Fifth segment\",\n\t\t\t\"target\": \"Пʼятий сегмент\",\n\t\t\t\"segmentId\": 10,\n\t\t\t\"customId\": \"fifth\",\n\t\t\t\"documentName\": \"translation.docx\",\n\t\t\t\"sourceLang\": \"UK\",\n\t\t\t\"targetLang\": \"EN\",\n\t\t\t\"type\": \"Manual\",\n\t\t\t\"author\": \"EDITOR\",\n\t\t\t\"timestamp\": \"20250812T125637Z\",\n\t\t\t\"markupTable\": \"OTMXUXLF\",\n\t\t\t\"context\": \"200\",\n\t\t\t\"additionalInfo\": \"\",\n\t\t\t\"internalKey\": \"13:1\"\n\t\t}\n\t},\n\t\"save2disk\": \"0\"\n}", + "responseId": 29051393, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "requests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "customId": { + "type": "string", + "description": "If provided will be used as field name for result in response" + }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "type": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "author": { + "type": "string" + }, + "context": { + "type": "string" + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "yes", + "description": "" + }, + { + "value": "0", + "name": "no", + "description": "" + } + ], + "description": "If 1 provided will save new segmnet to disk right away, else - only in RAM until flush happens" + } + }, + "required": [ + "source", + "target", + "targetLang", + "sourceLang", + "documentName", + "author", + "context" + ], + "x-apidog-orders": [ + "customId", + "source", + "target", + "type", + "markupTable", + "targetLang", + "sourceLang", + "documentName", + "author", + "context", + "save2disk" + ] + } + } + }, + "required": [ + "requests" + ], + "x-apidog-orders": [ + "requests" + ] + }, + "mediaType": "", + "examples": [ + { + "name": "Example 1", + "value": "{\n \"requests\": [\n {\n \"customId\": \"fourth\",\n \"source\": \"Fourth segmnet\",\n \"target\": \"Четвертий сегмент\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"targetLang\": \"en\",\n \"sourceLang\": \"uk\",\n \"documentName\": \"test.xlsx.sdlxliff\",\n \"author\": \"Project Manager\",\n \"context\": \"390\"\n },\n {\n \"customId\": \"fifth\",\n \"source\": \"Fifth segment\",\n \"target\": \"Пʼятий сегмент\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"targetLang\": \"en\",\n \"sourceLang\": \"uk\",\n \"documentName\": \"translation.docx\",\n \"author\": \"Editor\",\n \"context\": \"200\"\n }\n ]\n}", + "mediaType": "application/json" + } + ], + "oasExtensions": "" + }, + "description": "Send multiple updateEntry requests to the same tm with one call", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 156, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.29051393" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Tables", + "api": { + "id": "20146676", + "method": "get", + "path": "/t5memory/{{memory}}/tables", + "parameters": { + "path": [], + "header": [ + { + "required": false, + "description": "", + "type": "string", + "id": "k0UsRIj2NS", + "example": "application/json", + "enable": true, + "name": "Accept" + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "29077795", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {} + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)\": \"position)idInTable: value;\",\n \"tagTablesTable(4)\": \"0)0: \\\"\\\"; \",\n \"langTable(1)\": \"0)2: \\\"EN\\\"; 1)1: \\\"UK\\\"; 2)0: \\\"\\\"; \",\n \"langGroupTable(virtual)(8)\":\"0)1: \\\"E\\\"; 1)2: \\\"U\\\"; 2)0: \\\"\u0000\\\"; \",\n \"authors(3)\": \"0)3: \\\"EDITOR\\\"; 1)1: \\\"MANAGER\\\"; 2)2: \\\"PROJECT MANAGER\\\"; 3)0: \\\"\\\"; \",\n \"filenames(long)(5)\": \"0)1: \\\"none\\\"; 1)1: \\\"none\\\"; 2)1: \\\"none\\\"; 3)2: \\\"test.xlsx.sdlxliff\\\"; 4)5: \\\"test.xlsx.sdlxliff\\\"; 5)6: \\\"translation.docx\\\"; 6)0: \\\"\\\"; \",\n \"filenames(long, caseign, virtual)(5)\": \"0)1: \\\"NONE\\\"; 1)1: \\\"NONE\\\"; 2)1: \\\"NONE\\\"; 3)5: \\\"TEST.XLSX.SDLXLIFF\\\"; 4)6: \\\"TRANSLATION.DOCX\\\"; 5)2: \\\"TRANSLATION.TXT\\\"; 6)0: \\\"\\\"; \",\n \"filenames(short)(2)\": \"0)1: \\\"none\\\"; 1)5: \\\"test.xlsx.sdlxliff\\\"; 2)6: \\\"translation.docx\\\"; 3)0: \\\"\\\"; \"\n}", + "responseId": 29077795, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "Get author, lang, langGroup(virtual- created in runtime and not saved on the disk), tagtable(deprecated - only one record), document(long filename, long filename caseignore-virtual, short filename- part deprecated)\n\nTables are provided in format - name(internal key): \"position) idInTable:value;\"\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Tables\nWhen adding new entry, for new Id t5memory uses basicaly size of the table before adding+1, so Id should start from id=1. \nWhen doing fuzzy search, some field could also be saved to the table, to generate then id(or find matching) to compare with saved data.\nThere are 2 types of entries- older, fixed size, and long filename. For older, every entry has the same size, 39 bytes+endOfLine. You can have up to 389 entries saved in that way in each of the tables. Longname table manage size dynamically, so it's not clear what's the maximum size, but for filenames there are duplicates in old style docname table and in longname(which are used only for this type of data). For filename search, first used longname table, then generated on tm loading caseignore longname table, then short name table. Usually entries in tables are caseignore. \n\nIn the record t5memory saves only id to the table. Maybe it make sense to implement endpoint to edit that table, if something is missing or corrupted. Exception for saving id are context and additional info, some data, that could be saved as int(segId, time), and source and target.\nTo clear or fix tag tables, you need to do reorganize, but if some entry is missing from the table, you would not restore it on export or segment exctraction via id. So tables could be used to check health of the tm. And for some ways for concordance search or some other checks.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 166, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.29077795" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Add to table", + "api": { + "id": "20502616", + "method": "post", + "path": "/t5memory/{{memory}}/addtotable", + "parameters": {}, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "29731405", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {} + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Invalid table key", + "data": "{\n \"ReturnValue\": 400,\n \"ErrorMsg\": \"Invalid table key(0); available keys = LANG_KEY(1), FILE_KEY(2), AUTHOR_KEY(3)\"\n}", + "responseId": 29731405, + "ordering": 1, + "description": "Endpoint have a lot of checks and response. If you wouldn't set tableKey or send just empty body, you would have this message", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Table already have entry with matching id", + "data": "{\n \"ReturnValue\": 400,\n \"ErrorMsg\": \"Id match in Table 2 already have entry with matching id = 100 and value = \\\"Teaser Page.xlsx\\\";\"\n}", + "responseId": 29731405, + "ordering": 2, + "description": "It wouldn't allow to dublicate id in the table and would return in that case already saved entry, I added valueIsEmpty check after checking if id is in the table, so you can use request in read-only mode - just skip", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Value field is empty", + "data": "{\n \"ReturnValue\": 400,\n \"ErrorMsg\": \"Value field is empty\"\n}", + "responseId": 29731405, + "ordering": 3, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "entryId": { + "type": "integer" + }, + "tableKey": { + "type": "integer" + } + }, + "required": [ + "value", + "entryId", + "tableKey" + ], + "x-apidog-orders": [ + "value", + "entryId", + "tableKey" + ] + }, + "mediaType": "", + "examples": [ + { + "name": "Example 1", + "value": "{\n \"value\": \"MY new entry\",\n \"entryId\": 100,\n \"tableKey\": 2\n}", + "mediaType": "application/json" + } + ], + "oasExtensions": "" + }, + "description": "You can add new entry to the tables but then it's better to reorganize tm(in case if after adding new entry with missing id and there are other entry with the same value but other id - which could be the case- in binary search that could cause problems - so just reorganize it afterwards). You can check table by just doing something like full export(or concordance search for full tm), and check logs for that 2222 errors.\n\nValues shouldn't be longer than 39 bytes(utf8), and 256 for filename,it's tables limits, for that it also have checks.\nIn cases like with those tms, you can also check if x-1 and x+1 id is present. Usually they are in those bug.\nalso you can check tables via tables request\nAlso logicaly FILE_KEY=2, but that was only for shortname, which was also 39 bytes, but real key for long name table is 5. But for requests and internal handling t5memory uses 2.\nId should be between 1 and 65k, request also checks for that", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 186, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.29731405" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Create", + "api": { + "id": "19983316", + "method": "post", + "path": "/t5memory/", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804546", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Created TM name" + } + }, + "x-apidog-orders": [ + "name" + ], + "required": [ + "name" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28812960", + "code": 200, + "name": "Error", + "headers": [], + "jsonSchema": { + "title": "", + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer", + "description": "Error code" + }, + "ErrorMsg": { + "type": "string", + "description": "Message to explain error" + } + }, + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg" + ], + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"name\": \"examle_tm2\"\n}", + "responseId": 28804546, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Error", + "data": "{\n \"ReturnValue\": 7272,\n \"ErrorMsg\": \"::ERROR_MEM_NAME_EXISTS:: TM with this name already exists: test-memory; res = 0\"\n}", + "responseId": 28812960, + "ordering": 2, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "New TM name" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + } + }, + "required": [ + "name", + "sourceLang" + ], + "x-apidog-orders": [ + "name", + "sourceLang" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"name\": \"examle_tm2\",\n \"sourceLang\": \"de-DE\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 0, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804546", + "BLANK.28812960" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Create and import TM", + "api": { + "id": "19983317", + "method": "post", + "path": "/t5memory/", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804547", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Created memory name" + } + }, + "x-apidog-orders": [ + "name" + ], + "required": [ + "name" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28812906", + "code": 200, + "name": "Error", + "headers": [], + "jsonSchema": { + "title": "", + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer", + "description": "Error code" + }, + "ErrorMsg": { + "type": "string", + "description": "Message to explain error" + } + }, + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg" + ], + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"name\": \"test-memory\"\n}", + "responseId": 28804547, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Error", + "data": "{\n \"ReturnValue\": 7272,\n \"ErrorMsg\": \"::ERROR_MEM_NAME_EXISTS:: TM with this name already exists: test-memory; res = 0\"\n}", + "responseId": 28812906, + "ordering": 2, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "New memory name" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "data": { + "type": "string", + "description": "Binary TM file encode with base64" + } + }, + "required": [ + "name", + "sourceLang", + "data" + ], + "x-apidog-orders": [ + "name", + "sourceLang", + "data" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{ \n \"name\": \"test-memory\",\n \"sourceLang\": \"en-GB\",\n \"data\": \"UEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLk1FTQEAAADtzssKQEAUgOFjoTwGWR+5lPW4TCjDxgtomqI0I3lzLyBvIEUW59v829+GrinTJMaK99iqWY5Yq2VFMcpp1sotjN43swSCC/gSzw551bHu7MGLlMYqh6d8xrpBhBFjPnhACCGE/MMJUEsHCKEIpyRnAAAAAAAAAAAIAAAAAAAAUEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLlRNRAEAAADt3bFKI0EYB/BvNzFaXCXH1Vdcc3h7xR14xXEsd5pTwRgIEWwlBBRCFPEFbKx9BJ/DJ7CyshYrCwsbX0A3GtykFQnE329ZZnb+u92wMMXMt9FupBFxlERxRVTi+mPwCrNjLe9b+nw/qcRqq77UbC0HwCSdR/3vfafozSVry4s/f2Qr9Wa23t3tbGer3d5+1tju7Oz2u5+X9vqHB3u97+3Gm/24uv1s5V8AAAAAAAAAAAAAAAAAABHzg13/hVqcRSuPOM7P8sv8NG/lvTwAAAAAAAAAAAAAAAAAAABgasxFEjOD3rAO+EwpqQ56Jx+eRmqlpBLlj5JSUhtJqqUkGUnSUhIjQ5UYG4jiLIA7O/4BAAAAAAAAAAAAAAAAAABgAqrPJwHMx2zcfvryNdKbP1e/Iy5+LS58W3h8BgAAAAAAAAAAAJgO6Wg7rK7UbDe2NrfW/5dfTYJp9VI7a1i/CwAAYGLGVqq8U+YBBfOAgnnAowdQSwcITWiheHEBAAAAAAAAAAABAAAAAABQSwMEFAAICAgAUmZ4VgAAAAAAAAAAAAAAACgABABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAO3dwWkCYRAG0NFTzlaQBv4cEsg1JLqokI0gNiDLQgRZQ7ABu7MaLxawKijqWWQPvjcMM1PEx/xM8nZErCJa++J2dV1fTR5b+9inazDOuqNxLwCatI7sc1sctqfWsPf+9pr62Sh9l7Nimgbl/C/l0+J3VpXP3UW1/F/MXyb5MO6krFL/KwAAAAAAAAAAAAAAAACAiM5F6r8Tm48AAAAAAAAAAAAAAAAAAAAAGnD+TS7/DwAAAAAAAAAAAAAAAAAAAM3YAVBLBwgGVIGewwAAAAAAAAAAwAAAAAAAAFBLAQIAABQACAgIALRlVFahCKckZwAAAAAIAAAoAAQAAAAAAAAAAACkgQAAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuTUVNAQAAAFBLAQIAABQACAgIAGdralZNaKF4cQEAAAAAAQAoAAQAAAAAAAAAAACkgckAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1EAQAAAFBLAQIAABQACAgIAFJmeFYGVIGewwAAAADAAAAoAAQAAAAAAAAAAACkgZwCAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAFBLBgYsAAAAAAAAAB4DLQAAAAAAAAAAAAMAAAAAAAAAAwAAAAAAAAAOAQAAAAAAAMEDAAAAAAAAUEsGBwAAAADPBAAAAAAAAAEAAABQSwUGAAAAAAMAAwAOAQAAwQMAAAAA\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 6, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804547", + "BLANK.28812906" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Get", + "api": { + "id": "19983318", + "method": "get", + "path": "/t5memory/", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804548", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "Open": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "TM name" + } + }, + "x-apidog-orders": [ + "name" + ], + "required": [ + "name" + ] + }, + "description": "Memories loaded into RAM" + }, + "Available on disk": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "TM name" + } + }, + "x-apidog-orders": [ + "name" + ], + "required": [ + "name" + ] + }, + "description": "Memories available on disk" + } + }, + "x-apidog-orders": [ + "Open", + "Available on disk" + ], + "required": [ + "Open", + "Available on disk" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"Open\": [\n\t\t{\n\t\t\t\"name\": \"test-memory\"\n\t\t}\n\t],\n\t\"Available on disk\": [\n\t\t{\n\t\t\t\"name\": \"test-memory-1\"\n\t\t}\n\t]\n}", + "responseId": 28804548, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 12, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804548" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Resources", + "api": { + "id": "19983319", + "method": "get", + "path": "/t5memory_service/resources", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804549", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"filebuffers\": [\n {\n \"name\": \"/root/.t5memory/TABLE/OTMXUXLF.TBL\",\n \"size\": 3660\n }\n ],\n \"totalOccupiedByFilebuffersRAM\": 3660,\n \"tms\": [\n {\n \"name\": \"test-memory\",\n \"status\": \"open\",\n \"size\": 134976,\n \"activeRequest\": \"\",\n \"expectedSize\": 0,\n \"StatusInfo\": {\n \"status\": \"open\",\n \"sizeInRAM\": 134976,\n \"activeRequest\": \"\",\n \"tmxImportStatus\": \"available\",\n \"importProgress\": 100,\n \"importTime\": \"Overall import time is : 0:00:00\\n\",\n \"segmentsImported\": 2,\n \"invalidSegments\": 0,\n \"tmxSegmentCount\": 2,\n \"importRuntimeSec\": 0,\n \"importTimeoutSec\": 0,\n \"newEntryIfOnlyContextAndTimestampDifferent\": 1,\n \"saveDifferentTargetsForSameSource\": 0,\n \"invalidSegmentsRCs\": \"\",\n \"firstInvalidSegments(num:segId:errCode)\": \"\",\n \"invalidSymbolErrors\": 0,\n \"errorMsg\": \"\",\n \"rc\": 0,\n \"lastAccessTime\": \"20250801T153723Z\",\n \"creationTime\": \"20250801T123723Z\",\n \"tmCreatedInT5M_version\": \"0:7:14\",\n \"segmentIndex\": 2,\n \"sourceLang\": \"de\",\n \"internalDescription\": \"\",\n \"tmd fbuffer was modified\": 1,\n \"tmi fbuffer was modified\": 1\n }\n }\n ],\n \"totalOccupiedByTMsInRAM\": 712165696,\n \"Run date\": \"Wed Jul 30 16:08:09 2025 \",\n \"Build date\": \"2025/07/29 16:08:46\",\n \"Git commit info\": \"dff40cb9 * updated to v0.7.14 + added skipStartupLogs flags(default true) * fixed non-asan builds * fixed nextInternalKey is empty log on failed export\",\n \"Version\": \"0.7.14\",\n \"Worker threads\": 10,\n \"Timeout(ms)\": 180000,\n \"Resident set\": 1.4483e+12,\n \"Virtual memory usage\": 3.74334e+15,\n \"Requests\": {\n \"RequestCount\": 6829,\n \"RequestExecutionSumTime(sec)\": 14.484,\n \"CreateMem\": {\n \"ReqCount\": 44,\n \"SumTime(sec)\": 0.065,\n \"AvrgReqTime\": 0.00147727\n },\n \"DeleteMem\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"ImportMem\": {\n \"ReqCount\": 29,\n \"SumTime(sec)\": 0.016,\n \"AvrgReqTime\": 0.000551724\n },\n \"ExportMem\": {\n \"ReqCount\": 13,\n \"SumTime(sec)\": 0.09,\n \"AvrgReqTime\": 0.00692308\n },\n \"CloneTmLocaly\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"Reorganize\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"StatusMem\": {\n \"ReqCount\": 109,\n \"SumTime(sec)\": 0.011,\n \"AvrgReqTime\": 0.000100917\n },\n \"FlushMem\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"Fuzzy\": {\n \"ReqCount\": 98,\n \"SumTime(sec)\": 1.034,\n \"AvrgReqTime\": 0.010551\n },\n \"MultiFuzzy\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"Concordance\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"UpdateEntry\": {\n \"ReqCount\": 6,\n \"SumTime(sec)\": 0.013,\n \"AvrgReqTime\": 0.00216667\n },\n \"MultiUpdate\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"GetEntry\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"DeleteEntry\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"SaveAllTms\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"ListOfMemories\": {\n \"ReqCount\": 6474,\n \"SumTime(sec)\": 13.221,\n \"AvrgReqTime\": 0.00204217\n },\n \"Resources\": {\n \"ReqCount\": 22,\n \"SumTime(sec)\": 0.016,\n \"AvrgReqTime\": 0.000727273\n },\n \"Flags\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"Other\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n },\n \"Unrecognized\": {\n \"ReqCount\": 0,\n \"SumTime(sec)\": 0,\n \"AvrgReqTime\": 0\n }\n },\n \"RAM limit(MB)\": 10000\n}", + "responseId": 28804549, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 18, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804549" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Status", + "api": { + "id": "19983320", + "method": "get", + "path": "/t5memory/{{memory}}/status", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804550", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "description": "Statuses:\nopen - memory already in RAM ready to use\navailable - not loaded but present on disk\nloading - loading from disk to RAM\nnot found - error status of absent memory\nreorganize running - Reorganizing in process\nimport running - Import in process\nwaiting for loading - loading requsted but not done yet\nfailed to open - corrupted TM file" + }, + "sizeInRAM": { + "type": "integer" + }, + "activeRequest": { + "type": "string", + "description": " " + }, + "lastAccessTime": { + "type": "string" + }, + "creationTime": { + "type": "string" + }, + "tmCreatedInT5M_version": { + "type": "string", + "description": "version of t5memory on momemt of TM creation" + }, + "segmentIndex": { + "type": "integer", + "description": "index of last inserted segment (not queriable)" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "internalDescription": { + "type": "string" + }, + "tmd fbuffer was modified": { + "type": "integer" + }, + "tmi fbuffer was modified": { + "type": "integer" + } + }, + "x-apidog-orders": [ + "status", + "sizeInRAM", + "activeRequest", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ], + "required": [ + "status", + "sizeInRAM", + "activeRequest", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28828312", + "code": 200, + "name": "Import processed status", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "sizeInRAM": { + "type": "integer" + }, + "activeRequest": { + "type": "string" + }, + "tmxImportStatus": { + "type": "string", + "description": "Statuses of ongoing improt:\navailable - not tipical. If returned and \"importTime\" == \"not finished\" then import still ongoing\nimport - ongoing import" + }, + "importProgress": { + "type": "integer", + "description": "Progress in persents" + }, + "importTime": { + "type": "string" + }, + "segmentsImported": { + "type": "integer", + "description": "NUmber of successfully imported segments" + }, + "invalidSegments": { + "type": "integer", + "description": "Number of invalid segments found in TMX" + }, + "tmxSegmentCount": { + "type": "integer", + "description": "Number of found segmnets in import TMX file" + }, + "importRuntimeSec": { + "type": "integer" + }, + "importTimeoutSec": { + "type": "integer" + }, + "newEntryIfOnlyContextAndTimestampDifferent": { + "type": "integer" + }, + "saveDifferentTargetsForSameSource": { + "type": "integer", + "description": "0 or 1 - considered as bool value" + }, + "invalidSegmentsRCs": { + "type": "string" + }, + "firstInvalidSegments(num:segId:errCode)": { + "type": "string" + }, + "invalidSymbolErrors": { + "type": "integer" + }, + "errorMsg": { + "type": "string" + }, + "rc": { + "type": "integer" + }, + "lastAccessTime": { + "type": "string" + }, + "creationTime": { + "type": "string" + }, + "tmCreatedInT5M_version": { + "type": "string" + }, + "segmentIndex": { + "type": "integer" + }, + "sourceLang": { + "type": "string" + }, + "internalDescription": { + "type": "string" + }, + "tmd fbuffer was modified": { + "type": "integer" + }, + "tmi fbuffer was modified": { + "type": "integer" + } + }, + "required": [ + "status", + "sizeInRAM", + "activeRequest", + "tmxImportStatus", + "importProgress", + "importTime", + "segmentsImported", + "invalidSegments", + "tmxSegmentCount", + "importRuntimeSec", + "importTimeoutSec", + "newEntryIfOnlyContextAndTimestampDifferent", + "saveDifferentTargetsForSameSource", + "invalidSegmentsRCs", + "firstInvalidSegments(num:segId:errCode)", + "invalidSymbolErrors", + "errorMsg", + "rc", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ], + "x-apidog-orders": [ + "status", + "sizeInRAM", + "activeRequest", + "tmxImportStatus", + "importProgress", + "importTime", + "segmentsImported", + "invalidSegments", + "tmxSegmentCount", + "importRuntimeSec", + "importTimeoutSec", + "newEntryIfOnlyContextAndTimestampDifferent", + "saveDifferentTargetsForSameSource", + "invalidSegmentsRCs", + "firstInvalidSegments(num:segId:errCode)", + "invalidSymbolErrors", + "errorMsg", + "rc", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"status\": \"open\",\n \"sizeInRAM\": 134976,\n \"activeRequest\": \"\",\n \"lastAccessTime\": \"20250801T160913Z\",\n \"creationTime\": \"20250716T102923Z\",\n \"tmCreatedInT5M_version\": \"0:7:7\",\n \"segmentIndex\": 5,\n \"sourceLang\": \"en\",\n \"internalDescription\": \"\",\n \"tmd fbuffer was modified\": 1,\n \"tmi fbuffer was modified\": 1\n}", + "responseId": 28804550, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Success", + "data": "{\n \"status\": \"open\",\n \"sizeInRAM\": 134976,\n \"activeRequest\": \"\",\n \"tmxImportStatus\": \"available\",\n \"importProgress\": 100,\n \"importTime\": \"Overall import time is : 0:00:00\\n\",\n \"segmentsImported\": 4,\n \"invalidSegments\": 0,\n \"tmxSegmentCount\": 4,\n \"importRuntimeSec\": 0,\n \"importTimeoutSec\": 0,\n \"newEntryIfOnlyContextAndTimestampDifferent\": 1,\n \"saveDifferentTargetsForSameSource\": 0,\n \"invalidSegmentsRCs\": \"\",\n \"firstInvalidSegments(num:segId:errCode)\": \"\",\n \"invalidSymbolErrors\": 0,\n \"errorMsg\": \"\",\n \"rc\": 0,\n \"lastAccessTime\": \"20250807T130822Z\",\n \"creationTime\": \"20250716T102923Z\",\n \"tmCreatedInT5M_version\": \"0:7:7\",\n \"segmentIndex\": 9,\n \"sourceLang\": \"en\",\n \"internalDescription\": \"\",\n \"tmd fbuffer was modified\": 1,\n \"tmi fbuffer was modified\": 1\n}", + "responseId": 28828312, + "ordering": 2, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-GetthestatusofTM", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 24, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804550", + "BLANK.28828312" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "IMPORT TMX", + "api": { + "id": "19983322", + "method": "post", + "path": "/t5memory/{{memory}}/importtmx", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804552", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"msg\": \"success\"\n}", + "responseId": 28804552, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "multipart/form-data", + "parameters": [ + { + "required": true, + "description": "json pretty print string.\n\n\"framingTags\" key possible values:\n\"saveAll\" - default behaviour, do nothing\n\"skipAll\" - skip all enclosing tags, including standalone tags\n\"skipPaired\" - skip only paired enclosing tags\n\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, default values from flags would be used.", + "type": "string", + "id": "nBnfbWCL1O", + "example": "{\n \"framingTags\": \"saveAll\",\n \"newEntryIfOnlyContextAndTimestampDifferent\": 0,\n \"saveDifferentTargetsForSameSource\": 0,\n \"timeout\": 100000\n}\n", + "enable": true, + "name": "json_data", + "schema": { + "type": "string" + }, + "contentType": "application/json" + }, + { + "required": true, + "description": "tmx file to import into TM", + "type": "file", + "id": "xBgBcJUP8I", + "example": "", + "enable": true, + "name": "file", + "schema": { + "format": "binary", + "type": "string" + } + } + ], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"tmxData\": \"PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPHRteCB2ZXJzaW9uPSIxLjQiPgogIDxoZWFkZXIgY3JlYXRpb250b29sPSJTREwgTGFuZ3VhZ2UgUGxhdGZvcm0iIGNyZWF0aW9udG9vbHZlcnNpb249IjguMCIgby10bWY9IlNETCBUTTggRm9ybWF0IiBkYXRhdHlwZT0ieG1sIiBzZWd0eXBlPSJzZW50ZW5jZSIgYWRtaW5sYW5nPSJlbi1HQiIgc3JjbGFuZz0iYmctQkciIGNyZWF0aW9uZGF0ZT0iMjAxNTA4MjFUMDkyNjE0WiIgY3JlYXRpb25pZD0idGVzdCIvPgogIDxib2R5PgoJPHR1IGNyZWF0aW9uZGF0ZT0iMjAxODAyMTZUMTU1MTA1WiIgY3JlYXRpb25pZD0iREVTS1RPUC1SNTlCT0tCXFBDMiIgY2hhbmdlZGF0ZT0iMjAxODAyMTZUMTU1MTA4WiIgY2hhbmdlaWQ9IkRFU0tUT1AtUjU5Qk9LQlxQQzIiIGxhc3R1c2FnZWRhdGU9IjIwMTgwMjE2VDE2MTMwNVoiIHVzYWdlY291bnQ9IjEiPgogICAgICA8dHV2IHhtbDpsYW5nPSJiZy1CRyI+CiAgICAgICAgPHNlZz5UaGUgPHBoIC8+IGVuZDwvc2VnPgogICAgICA8L3R1dj4KICAgICAgPHR1diB4bWw6bGFuZz0iZW4tR0IiPgogICAgICAgIDxzZWc+RXRoIDxwaCAvPiBkbmU8L3NlZz4KICAgICAgPC90dXY+CiAgICA8L3R1PgogIDwvYm9keT4KPC90bXg+Cg==\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Rout is async. So when import is finished should be checked with status call", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 36, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804552" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "EXPORT TMX", + "api": { + "id": "19983323", + "method": "get", + "path": "/t5memory/{{memory}}/download.tmx", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "CWZEZZXTgD", + "name": "Accept", + "required": true, + "description": "", + "example": "application/xml", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804553", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "In response headers \"NextInternalKey\" will contain \"startFromInternalKey\" for next call.\nIf \"NextInternalKey\" == \"0:0\" - then you reached end of TM", + "contentType": "xml", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "\n\n\n
\n \n \n 2\n 7: 1\n OTMXUXLF\n none\n \n First segment\n \n \n Перший сегмент\n \n \n \n", + "responseId": 28804553, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "limit": { + "type": "string" + }, + "startFromInternalKey": { + "type": "string" + } + }, + "required": [ + "limit", + "startFromInternalKey" + ], + "x-apidog-orders": [ + "limit", + "startFromInternalKey" + ] + }, + "description": "", + "mediaType": "", + "examples": [ + { + "name": "Example 1", + "value": "{\n \"limit\": \"100\",\n \"startFromInternalKey\": \"1:1\"\n}", + "mediaType": "application/json" + } + ], + "oasExtensions": "" + }, + "description": "JSON pretty print string:\nlimit - number of segments to fetch at a time\nstartFromInternalKey - internal key to start export from\n\nWorks as cursor pagination.\n\nIf no body provided, export starts from the beginning (key 7:1) to the end.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 42, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804553" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Concordance search", + "api": { + "id": "19983324", + "method": "post", + "path": "/t5memory/{{memory}}/search", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "YAsH0YBsjZ", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "wdM5iC91yw", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804554", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "Filters": { + "type": "string" + }, + "GlobalSearchOptions": { + "type": "string" + }, + "ReturnValue": { + "type": "integer" + }, + "ReturnMessage": { + "type": "string" + }, + "NewSearchPosition": { + "type": "null" + }, + "NumOfFoundSegments": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + } + }, + "required": [ + "Filters", + "GlobalSearchOptions", + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "NumOfFoundSegments", + "results" + ], + "x-apidog-orders": [ + "Filters", + "GlobalSearchOptions", + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "NumOfFoundSegments", + "results" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"Filters\": \"Search filter, field: SOURCE FilterType::CONTAINS SearchStr: 'FIRST'; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: TARGET FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: ADDINFO FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: CONTEXT FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: AUTHOR FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: DOCUMENT FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\\nSearch filter, field: TIMESTAMP FilterType::RANGE Range: 19691231T230000Z - 20260628T220000Z Options: ;\\n\",\n \"GlobalSearchOptions\": \"SEARCH_FILTERS_LOGICAL_AND\",\n \"ReturnValue\": 10010,\n \"ReturnMessage\": \"ENDREACHED_RC\",\n \"NewSearchPosition\": null,\n \"NumOfFoundSegments\": 2,\n \"results\": [\n {\n \"source\": \"First segment with int \",\n \"target\": \"Перший сегмент з числом \",\n \"segmentId\": 2,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"7:1\"\n },\n {\n \"source\": \"First segment with int 10\",\n \"target\": \"Перший сегмент з числом 10\",\n \"segmentId\": 6,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"11:1\"\n }\n ]\n}", + "responseId": 28804554, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "sourceSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "target": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "targetSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "document": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "documentSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "author": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "authorSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "addInfo": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "addInfoSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "context": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "contextSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "timestampSpanStart": { + "type": "string", + "description": "String with date in format\n \"20240121T115234Z\"" + }, + "timestampSpanEnd": { + "type": "string", + "description": "String with date in format\n \"20240121T115234Z\"" + }, + "onlyCountSegments": { + "type": "string", + "description": "Instead of returning segment, would go in search till the end of tm and return total number of segments, that returns true with selected filters", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "yes", + "description": "" + }, + { + "value": "0", + "name": "no", + "description": "" + } + ] + }, + "searchPosition": { + "type": "string", + "description": "Point where to start search in tmd file", + "examples": [ + "8:1" + ] + }, + "logicalOr": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "x-apidog-enum": [ + { + "value": 1, + "name": "yes", + "description": "" + }, + { + "value": 0, + "name": "no", + "description": "" + } + ], + "description": "By default source, target, document, author, context, addinfo, timestamp is combined in logical AND, but by sending here \"OR\" you can switch that to logical OR, any other value would left it in default AND state.\nDoesn't apply to sourceLang and targetLang filters, they are always in AND state" + }, + "numResults": { + "type": "integer", + "description": "Points how many matches return in current request", + "minimum": 0, + "maximum": 200, + "default": 5 + }, + "sourceLang": { + "type": "string", + "description": "Filter segments on src/trg lang attribute, \n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + }, + "targetLang": { + "type": "string", + "description": "Filter segments on src/trg lang attribute, \n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + }, + "msSearchAfterNumResults": { + "type": "integer", + "default": 0, + "description": "sets how many ms should pass between first found segment and search stop, if it didn't reach the end yet. " + }, + "loggingThreshold": { + "type": "integer", + "maximum": 6, + "description": "HACK.\nAdditional field to set log level on the run.\nSets log level for t5memory process in general and not this request only" + } + }, + "x-apidog-orders": [ + "source", + "sourceSearchMode", + "target", + "targetSearchMode", + "document", + "documentSearchMode", + "author", + "authorSearchMode", + "addInfo", + "addInfoSearchMode", + "context", + "contextSearchMode", + "timestampSpanStart", + "timestampSpanEnd", + "onlyCountSegments", + "searchPosition", + "logicalOr", + "numResults", + "sourceLang", + "targetLang", + "msSearchAfterNumResults", + "loggingThreshold" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"source\": \"First\",\n \"sourceSearchMode\": \"contains, CASEINSENSETIVE\",\n \"target\": \"\",\n \"targetSearchMode\": \"contains, CASEINSENSETIVE\",\n \"document\": \"\",\n \"documentSearchMode\": \"contains, CASEINSENSETIVE\",\n \"author\": \"\",\n \"authorSearchMode\": \"contains, CASEINSENSETIVE\",\n \"addInfo\": \"\",\n \"addInfoSearchMode\": \"contains, CASEINSENSETIVE\",\n \"context\": \"\",\n \"contextSearchMode\": \"contains, CASEINSENSETIVE\",\n \"timestampSpanStart\": \"19691231T230000Z\",\n \"timestampSpanEnd\": \"20260628T220000Z\",\n \"onlyCountSegments\": \"0\",\n \"searchPosition\": \"\",\n \"logicalOr\": 0,\n \"numResults\": 200\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "All fields is optional, but some depends on other, so error should be returned in case of not providing required field.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-NewConcordancesearch", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 48, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804554" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Concordance search (DEPRICATED)", + "api": { + "id": "19983326", + "method": "post", + "path": "/t5memory/{{memory}}/concordancesearch", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "puP9JRtUNc", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "q4CWk6wlXh", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804556", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ReturnMessage": { + "type": "string" + }, + "NewSearchPosition": { + "type": [ + "string", + "null" + ] + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + } + }, + "required": [ + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "results" + ], + "x-apidog-orders": [ + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "results" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"ReturnValue\": 10010,\n \"ReturnMessage\": \"ENDREACHED_RC\",\n \"NewSearchPosition\": null,\n \"results\": [\n {\n \"source\": \"First segment with int \",\n \"target\": \"Перший сегмент з числом \",\n \"segmentId\": 2,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"7:1\"\n },\n {\n \"source\": \"First segment with int 10\",\n \"target\": \"Перший сегмент з числом 10\",\n \"segmentId\": 6,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"11:1\"\n }\n ]\n}", + "responseId": 28804556, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "searchString": { + "type": "string" + }, + "searchType": { + "type": "string", + "enum": [ + "Source", + "Target", + "SourceAndTarget" + ], + "x-apidog-enum": [ + { + "value": "Source", + "name": "", + "description": "" + }, + { + "value": "Target", + "name": "", + "description": "" + }, + { + "value": "SourceAndTarget", + "name": "", + "description": "" + } + ], + "examples": [ + "Source", + "Target", + "SourceAndTarget" + ] + }, + "searchPosition": { + "type": "null" + }, + "numResults": { + "type": "integer" + }, + "msSearchAfterNumResults": { + "type": "integer" + } + }, + "required": [ + "searchString", + "searchType" + ], + "x-apidog-orders": [ + "searchString", + "searchType", + "searchPosition", + "numResults", + "msSearchAfterNumResults" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"searchString\": \"With nice text\",\n \"searchType\": \"source\",\n \"searchPosition\": null,\n \"numResults\": 20,\n \"msSearchAfterNumResults\": 250\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": " SearchPosition / NewSearchPositionFormat: \"7:1\"\nFirst is segmeng\\record number, second is target number\nThe NextSearchposition is an internal key of the memory for the next position on sequential access. Since it is an internal key, maintained and understood by the underlying memory plug-in (for EqfMemoryPlugin is it the record number and the position in one record),\nno assumptions should be made regarding the content. It is just a string that, should be sent back to OpenTM2 on the next request, so that the search starts from there.\nSo is the implementation in Translate5: The first request to OpenTM2 contains SearchPosition with an empty string, OpenTM2 returns than a string in NewSearchPosition, which is just resent to OpenTM2 in the next request.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Concordancesearch\n ", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 60, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804556" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Fuzzy search", + "api": { + "id": "19983327", + "method": "post", + "path": "/t5memory/{{memory}}/fuzzysearch", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804557", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy", + "results" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"ReturnValue\": 0,\n \"ErrorMsg\": \"\",\n \"NumOfFoundProposals\": 2,\n \"searchedSrc\": \"First segment with int \",\n \"customId\": \"\",\n \"forceFuzzy\": 0,\n \"results\": [\n {\n \"source\": \"First segment with int 10\",\n \"target\": \"Перший сегмент з числом 10\",\n \"segmentId\": 6,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"11:1\",\n \"matchType\": \"Fuzzy\",\n \"matchRate\": 80,\n \"fuzzyWords\": 5,\n \"fuzzyDiffs\": 1\n },\n {\n \"source\": \"First segment with int 11\",\n \"target\": \"Перший сегмент з числом 11\",\n \"segmentId\": 2,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"7:1\",\n \"matchType\": \"Fuzzy\",\n \"matchRate\": 80,\n \"fuzzyWords\": 5,\n \"fuzzyDiffs\": 1\n }\n ]\n}", + "responseId": 28804557, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "targetLang": { + "type": "string", + "description": "rfc5646" + }, + "source": { + "type": "string", + "description": "Search string to look for" + }, + "documentName": { + "type": "string" + }, + "markupTable": { + "type": "string", + "description": "If there is no markup, default OTMXUXLF would be used.\nMarkup tables should be located inside ~/.t5memory/TABLE/%markup$.TBL" + }, + "context": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "x-apidog-enum": [ + { + "value": 1, + "name": "yes", + "description": "" + }, + { + "value": 0, + "name": "no", + "description": "" + } + ], + "description": "If yes - will return all potential fuzzies\nNo - only first exact match or most relevant matches" + }, + "numOfProposals": { + "type": "integer", + "description": "Num of expected segments in output. By default it's 5" + } + }, + "required": [ + "sourceLang", + "targetLang", + "source" + ], + "x-apidog-orders": [ + "sourceLang", + "targetLang", + "source", + "documentName", + "markupTable", + "context", + "forceFuzzy", + "numOfProposals" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"Second segment \",\n \"documentName\": \"\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Fuzzysearch", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 66, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804557" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Get entry", + "api": { + "id": "19983328", + "method": "post", + "path": "/t5memory/{{memory}}/getentry", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804558", + "code": 200, + "name": "OK", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28839642", + "code": 400, + "name": "Bad Request", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"source\": \"Third segment\",\n \"target\": \"третій сегмент\",\n \"segmentId\": 4,\n \"customId\": \"\",\n \"documentName\": \"none\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"MANAGER\",\n \"timestamp\": \"20250716T084347Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"9:1\"\n}", + "responseId": 28804558, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Success", + "data": "{\n \"ReturnValue\": 939,\n \"ErrorMsg\": \"Requested entry not found! Next internalKey after requested is : 10:1\"\n}", + "responseId": 28839642, + "ordering": 2, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "recordKey": { + "type": "string" + }, + "targetKey": { + "type": "string" + } + }, + "required": [ + "recordKey", + "targetKey" + ], + "x-apidog-orders": [ + "recordKey", + "targetKey" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"recordKey\": \"8\",\n \"targetKey\": \"2\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 72, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804558", + "BLANK.28839642" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Update", + "api": { + "id": "19983329", + "method": "post", + "path": "/t5memory/{{memory}}/entry", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "1Lk9zS6LLP", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "XvSWvz3Bwl", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804559", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"source\": \"Second segment\",\n \"sourceNPRepl\": \"Second segment\",\n \"sourceNorm\": \"Second segment\",\n \"target\": \"Другий сегмент\",\n \"segmentId\": 14,\n \"customId\": \"\",\n \"documentName\": \"translation.txt\",\n \"sourceLang\": \"EN\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"PROJECT MANAGER\",\n \"timestamp\": \"20240111T111443Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"14:1\"\n}", + "responseId": 28804559, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Segment source string" + }, + "target": { + "type": "string", + "description": "Segment target string" + }, + "markupTable": { + "type": "string", + "const": "OTMXUXLF", + "examples": [ + "OTMXUXLF" + ], + "description": "Should be \"OTMXUXLF\"" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "targetLang": { + "type": "string", + "description": "rfc5646" + }, + "timeStamp": { + "type": "string", + "pattern": "^\\d{8}T\\d{6}Z$", + "format": "yyyyMMdd'T'HHmmss'Z'", + "examples": [ + "20240111T111443Z" + ], + "description": "Time of segment creation" + }, + "documentName": { + "type": "string", + "description": "Name of document where segmnet comes from" + }, + "author": { + "type": "string" + }, + "context": { + "type": "string", + "description": "Field that will be considered in fuzzy searches. Plays a role of aditional search option" + }, + "save2disk": { + "type": "integer", + "description": "If true - segment will be written to disk right away, else - only set in RAM and saved when all memory is flushed", + "enum": [ + 1, + 0 + ], + "x-apidog-enum": [ + { + "value": 1, + "name": "yes", + "description": "" + }, + { + "value": 0, + "name": "no", + "description": "" + } + ], + "default": 0, + "examples": [ + 1 + ] + }, + "saveDifferentTargetsForSameSource": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "x-apidog-enum": [ + { + "value": 1, + "name": "yes", + "description": "" + }, + { + "value": 0, + "name": "no", + "description": "" + } + ], + "description": "If param set to 1 - new entry will be saved and old preserved even if all fileds of segmnet except target are same." + }, + "type": { + "type": "string", + "examples": [ + "Manual" + ], + "const": "Manual", + "description": "Should be set to \"Manual\"" + } + }, + "required": [ + "source", + "target", + "sourceLang", + "targetLang", + "timeStamp", + "documentName", + "author", + "context", + "save2disk" + ], + "x-apidog-orders": [ + "source", + "target", + "type", + "markupTable", + "sourceLang", + "targetLang", + "timeStamp", + "documentName", + "author", + "context", + "save2disk", + "saveDifferentTargetsForSameSource" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"source\": \" ... \",\n \"target\": \" ... \",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"sourceLang\": \"de-DE\",\n \"targetLang\": \"en-GB\",\n \"timeStamp\": \"20240111T111443Z\",\n \"documentName\": \"translation.txt\",\n \"author\": \"Project Manager\",\n \"context\": \"\",\n \"save2disk\": 1\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Updateentry", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 78, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804559" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Delete TM", + "api": { + "id": "19983330", + "method": "delete", + "path": "/t5memory/{{memory}}", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804560", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "Response will contain json with memory name as key and \"deleted\" as value.\nMay return 500 with \"not found(error 48)\" as value that is simply indicates absense of TM", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28898466", + "code": 500, + "name": "Not found", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {} + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"test-tm\": \"deleted\"\n}", + "responseId": 28804560, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Not found", + "data": "{\n \"test-tm\": \"not found(error 48)\"\n}", + "responseId": 28898466, + "ordering": 2, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "text/plain", + "parameters": [], + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 84, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804560", + "BLANK.28898466" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Delete entry", + "api": { + "id": "19983331", + "method": "post", + "path": "/t5memory/{{memory}}/entrydelete", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804561", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "fileFlushed": { + "type": "integer", + "description": "Indicates was TM flushed to disk or not. Depends on \"save2disk\" value." + }, + "results": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "x-apidog-orders": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + }, + "required": [ + "fileFlushed", + "results" + ], + "x-apidog-orders": [ + "fileFlushed", + "results" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28988335", + "code": 400, + "name": "Bad Request", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + }, + { + "id": "28988337", + "code": 500, + "name": "Server Error", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ], + "x-apidog-orders": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Example 1", + "data": "{\n \"fileFlushed\": 1,\n \"results\": {\n \"source\": \"Second segment\",\n \"target\": \"Другий сегмент\",\n \"segmentId\": 14,\n \"customId\": \"\",\n \"documentName\": \"translation.txt\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"UK\",\n \"type\": \"Manual\",\n \"author\": \"PROJECT MANAGER\",\n \"timestamp\": \"20250811T101311Z\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\",\n \"additionalInfo\": \"\",\n \"internalKey\": \"14:1\"\n }\n}", + "responseId": 28804561, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Example 1", + "data": "{\n \"ReturnValue\": 14002,\n \"ErrorMsg\": \"Error: to delete entry by key you should provide all three fields: recordKey, targetKey and segmentId\"\n}", + "responseId": 28988335, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + }, + { + "name": "Wrong segmnetId", + "data": "{\n \"ReturnValue\": 5005,\n \"ErrorMsg\": \"\"\n}", + "responseId": 28988337, + "ordering": 3, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "recordKey": { + "type": "integer" + }, + "targetKey": { + "type": "integer" + }, + "segmentId": { + "type": "integer" + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "yes", + "description": "" + }, + { + "value": "0", + "name": "no", + "description": "" + } + ], + "default": "0", + "examples": [ + "1" + ], + "description": "Save memory to disk right away.\nSaves if not passed." + } + }, + "required": [ + "recordKey", + "targetKey", + "segmentId" + ], + "x-apidog-orders": [ + "recordKey", + "targetKey", + "segmentId", + "save2disk" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"recordKey\": 8,\n \"targetKey\": 4,\n \"segmentId\": 12\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".\nThere is also \"segmentId\" value.\nFor delete entry request it should be provided to validate that current state of t5memory data is same as at the moment when you got internalKey as internal key value may be regenerated at any moment of data processing on import or update rrquest handling.\nSo if you try to delete entry and getting error - try to fetch entry internalKey nad segmentId again and then retry delete action.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 90, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804561", + "BLANK.28988335", + "BLANK.28988337" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Delete batch", + "api": { + "id": "19983332", + "method": "post", + "path": "/t5memory/{{memory}}/entriesdelete", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804562", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceSearchMode": { + "type": "string" + }, + "target": { + "type": "string" + }, + "targetSearchMode": { + "type": "string" + }, + "document": { + "type": "string" + }, + "documentSearchMode": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorSearchMode": { + "type": "string" + }, + "addInfo": { + "type": "string" + }, + "addInfoSearchMode": { + "type": "string" + }, + "context": { + "type": "string" + }, + "contextSearchMode": { + "type": "string" + }, + "timestampSpanStart": { + "type": "string" + }, + "timestampSpanEnd": { + "type": "string" + }, + "searchPosition": { + "type": "string" + } + }, + "x-apidog-orders": [ + "source", + "sourceSearchMode", + "target", + "targetSearchMode", + "document", + "documentSearchMode", + "author", + "authorSearchMode", + "addInfo", + "addInfoSearchMode", + "context", + "contextSearchMode", + "timestampSpanStart", + "timestampSpanEnd", + "searchPosition" + ], + "required": [ + "timestampSpanStart", + "timestampSpanEnd" + ] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"source\": \"123\",\n \"sourceSearchMode\": \"contains, CASEINSENSETIVE\",\n \"target\": \"234\",\n \"targetSearchMode\": \"contains, CASEINSENSETIVE\",\n \"document\": \"\",\n \"documentSearchMode\": \"contains, CASEINSENSETIVE\",\n \"author\": \"\",\n \"authorSearchMode\": \"contains, CASEINSENSETIVE\",\n \"addInfo\": \"\",\n \"addInfoSearchMode\": \"contains, CASEINSENSETIVE\",\n \"context\": \"\",\n \"contextSearchMode\": \"contains, CASEINSENSETIVE\",\n \"timestampSpanStart\": \"19691231T230000Z\",\n \"timestampSpanEnd\": \"20240610T220000Z\",\n \"searchPosition\": \"\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Asynchronious operation.\nThis would start reorganize process which would remove like reorganize bad segments and also would remove segments that gives true when checking with provided filters combined with logical AND. So if you provide timestamps and addInfo, only segments within provided timestamp and with that addInfo would not be imported to new TM(check reorganize process). \nEvery parameter is optional, so empty json would just start reorganize async process.\nIf you provide one of timestamps you would get error - please provide both. \nTo add parameter you should set it's SearchMode to be EXACT|CONCORDANCE(non case sensetive)\nIf only searched string provided, but not search mode - you would get error.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 96, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804562" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Shutdown", + "api": { + "id": "19983333", + "method": "post", + "path": "/t5memory_service/shutdown", + "parameters": { + "query": [ + { + "required": false, + "description": "Skips saving tms, for now value doesn't matter, only presence.\nIf try to save tms before closing, would check if there is still import process going on\nIf there is some, would wait 1 second and check again. \nRepeats last step up to 10 min, then closes service anyway. ", + "type": "integer", + "id": "6G5xEpWiRA", + "enable": true, + "name": "dontsave", + "schema": { + "type": "integer" + } + } + ], + "path": [], + "cookie": [], + "header": [ + { + "id": "xMxiCFLsH9", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "OEUJrQ83Sh", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804563", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [], + "requestBody": { + "type": "none", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"Method\": \"update\",\n \"source\": \"The end\",\n \"target\": \"The target\",\n \"documentName\": \"Translate5 Demo Text-en-de.xlf\",\n \"author\": \"Thomas Lauria\",\n \"timeStamp\": \"20210621T071042Z\",\n \"context\": \"2_2\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"de\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Safely shutting down the service with\\without saving all loaded tm files to the disk.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 102, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804563" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Save TMs", + "api": { + "id": "19983334", + "method": "get", + "path": "/t5memory_service/savetms", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "qDDSGk9gOz", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "7WSgqqWX2C", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804564", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n \"saved 3 tms\": \"ID9580-de-fr_next-2, ID9612-en-uk, test-memory\",\n \"errMsg\": \"\"\n}", + "responseId": 28804564, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "none", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "mediaType": "", + "examples": [ + { + "value": "{\n \"Method\": \"update\",\n \"source\": \"The end\",\n \"target\": \"The target\",\n \"documentName\": \"Translate5 Demo Text-en-de.xlf\",\n \"author\": \"Thomas Lauria\",\n \"timeStamp\": \"20210621T071042Z\",\n \"context\": \"2_2\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"sourceLang\": \"en\",\n \"targetLang\": \"de\"\n}", + "mediaType": "application/json", + "description": "" + } + ], + "oasExtensions": "" + }, + "description": "Flushes all filebuffers(TMD, TMI files) into the filesystem. Reset 'Modified' flags for file buffers. \n\nFilebuffer is a file instance of .TMD or .TMI loaded into RAM. It provides better speed and safety when working with files.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 108, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804564" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Clone", + "api": { + "id": "19983335", + "method": "post", + "path": "/t5memory/{{memory}}/clone", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "38R4HXnKSc", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "LXuAOp7SzH", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804565", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": { + "msg": { + "type": "string" + }, + "time": { + "type": "string" + } + }, + "required": [ + "msg", + "time" + ], + "x-apidog-orders": [ + "msg", + "time" + ] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [ + { + "name": "Success", + "data": "{\n\t\"msg\": \"clonned-tm-1 was cloned successfully\",\n\t\"time\": \"1 ms\"\n}", + "responseId": 28804565, + "ordering": 1, + "description": "", + "oasKey": "", + "oasExtensions": "" + } + ], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "newName": { + "type": "string", + "description": "Name of memory to be crreated" + } + }, + "x-apidog-orders": [ + "newName" + ], + "required": [ + "newName" + ] + }, + "mediaType": "", + "examples": [ + { + "name": "Example 2", + "value": "{\n \"newName\": \"clonned-tm-1\"\n}", + "mediaType": "application/json" + } + ], + "oasExtensions": "" + }, + "description": "Creates TM with the provided name and data cloned from provided memory", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 114, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804565" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + }, + { + "name": "Reorganize", + "api": { + "id": "19983336", + "method": "get", + "path": "/t5memory/{{memory}}/reorganize", + "parameters": { + "query": [], + "path": [], + "cookie": [], + "header": [ + { + "id": "AVqjeFavWC", + "name": "Accept", + "required": true, + "description": "", + "example": "application/json; charset=utf-8", + "type": "string", + "enable": true + }, + { + "id": "pEssFEtOV3", + "name": "Accept-Charset", + "required": true, + "description": "", + "example": "UTF-8", + "type": "string", + "enable": true + } + ] + }, + "auth": {}, + "securityScheme": {}, + "commonParameters": { + "query": [], + "body": [], + "cookie": [], + "header": [] + }, + "responses": [ + { + "id": "28804566", + "code": 200, + "name": "Success", + "headers": [], + "jsonSchema": { + "type": "object", + "properties": {}, + "x-apidog-orders": [] + }, + "description": "", + "contentType": "json", + "mediaType": "", + "oasExtensions": "" + } + ], + "responseExamples": [], + "requestBody": { + "type": "application/json", + "parameters": [], + "jsonSchema": { + "type": "object", + "properties": { + "newEntryIfOnlyContextAndTimestampDifferent": { + "type": "string", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "yes", + "description": "" + }, + { + "value": "0", + "name": "no", + "description": "" + } + ], + "examples": [ + "1", + "0" + ] + }, + "saveDifferentTargetsForSameSource": { + "type": "string", + "enum": [ + "1", + "0" + ], + "x-apidog-enum": [ + { + "value": "1", + "name": "yes", + "description": "" + }, + { + "value": "0", + "name": "", + "description": "" + } + ], + "examples": [ + "1", + "0" + ] + } + }, + "x-apidog-orders": [ + "newEntryIfOnlyContextAndTimestampDifferent", + "saveDifferentTargetsForSameSource" + ] + }, + "mediaType": "", + "examples": [], + "oasExtensions": "" + }, + "description": "Under the hood it creates new tm with $Org- prefix, then reimport all segments one-by-one, and then deletes original TM and rename reorganized TM to replace original. \nThis request should flush tm(from RAM to the disk) before reorganizing\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-ReorganizeTM\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, defaule values from flags would be used.", + "tags": [], + "status": "released", + "serverId": "", + "operationId": "", + "sourceUrl": "", + "ordering": 120, + "cases": [], + "mocks": [], + "customApiFields": "{}", + "advancedSettings": { + "disabledSystemHeaders": {} + }, + "mockScript": {}, + "codeSamples": [], + "commonResponseStatus": {}, + "responseChildren": [ + "BLANK.28804566" + ], + "visibility": "INHERITED", + "moduleId": 845181, + "oasExtensions": "", + "type": "http", + "preProcessors": [], + "postProcessors": [], + "inheritPostProcessors": {}, + "inheritPreProcessors": {} + } + } + ] + } + ], + "socketCollection": [], + "docCollection": [], + "webSocketCollection": [], + "socketIOCollection": [], + "responseCollection": [ + { + "_databaseId": 1073315, + "updatedAt": "2025-08-07T08:42:35.000Z", + "name": "Root", + "type": "root", + "children": [], + "moduleId": 845181, + "parentId": 0, + "id": 1073315, + "ordering": [], + "items": [] + } + ], + "schemaCollection": [ + { + "id": 2408559, + "name": "Root", + "visibility": "SHARED", + "moduleId": 845181, + "items": [], + "ordering": [] + } + ], + "securitySchemeCollection": [ + { + "id": 680526, + "moduleId": 845181, + "name": "Root", + "items": [], + "ordering": [] + } + ], + "requestCollection": [ + { + "name": "Root", + "children": [], + "ordering": [ + "requestFolder.7454461" + ], + "items": [] + } + ], + "apiTestCaseCollection": [ + { + "name": "Root", + "children": [], + "items": [] + } + ], + "testCaseReferences": [], + "environments": [], + "commonScripts": [], + "databaseConnections": [], + "globalVariables": [ + { + "id": "1029997", + "variables": [] + } + ], + "commonParameters": null, + "projectSetting": { + "id": "1030033", + "auth": {}, + "securityScheme": {}, + "gateway": [], + "language": "en-US", + "apiStatuses": [ + "developing", + "testing", + "released", + "deprecated" + ], + "mockSettings": {}, + "preProcessors": [], + "postProcessors": [], + "advancedSettings": { + "enableJsonc": false, + "enableBigint": false, + "responseValidate": true, + "enableTestScenarioSetting": false, + "enableYAPICompatScript": false, + "isDefaultUrlEncoding": 3, + "publishedDocUrlRules": { + "defaultRule": "SLUGIFY_WITH_RESOURCE_KEY", + "resourceKeyStandard": "NEW" + } + }, + "initialDisabledMockIds": [], + "servers": [ + { + "id": "default", + "name": "Default Server" + } + ], + "cloudMock": { + "security": "free", + "enable": false, + "tokenKey": "apidogToken" + } + }, + "customFunctions": [], + "projectTestCaseCategories": [], + "projectAssociations": [] +} \ No newline at end of file diff --git a/docs/t5memory.openapi.json b/docs/t5memory.openapi.json new file mode 100644 index 00000000..5df2b67d --- /dev/null +++ b/docs/t5memory.openapi.json @@ -0,0 +1,3526 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "t5memory", + "description": "", + "version": "0.7.36" + }, + "tags": [], + "paths": { + "/t5memory/{memory}/flush": { + "get": { + "summary": "Flush memory", + "deprecated": false, + "description": "Endpoint is sync(blocking)\n\nIf tm is not found on the disk - returns 404\nIf tm is not open - returns 400 with message\nThen t5memory requests writes pointer to the tm (so it waits till other requests that's working with the tm would finish) and then it flushes it to the disk\nCould also return an error if flushing got some issue.\nWould not open the tm, if it's not opened yet, but instead would return an error.", + "tags": [], + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "msg": { + "type": "string" + } + }, + "required": [ + "msg" + ] + }, + "example": { + "msg": "Mem ID9612-en-uk was flushed to the disk successfully" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory_service/tagreplacement": { + "post": { + "summary": "Tag replacement", + "deprecated": false, + "description": "Fuzzy search tag replacement and Import tag replacement test", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "src": { + "type": "string" + }, + "trg": { + "type": "string" + }, + "req": { + "type": "string", + "description": "Used to test Fuzzy search tag replacement" + } + }, + "required": [ + "src", + "trg" + ] + }, + "example": { + "src": "Tap View o get strongdisplayedView two strongUS patents.", + "trg": "View tap to got strongdosplaydVeiw two strongUS patents.", + "req": "Tap View o get strongdisplayedView two strongUS patents." + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory_service/flags": { + "get": { + "summary": "Flags", + "deprecated": false, + "description": "Return all available commandline flags. Do not spam too much because gflags documentation says that that's slow. Useful to collect configuration data about t5memory to do debugging.", + "tags": [], + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": "{\n {\n \"name\": \"alsologtoemail\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to these email addresses in addition to logfiles\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"alsologtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr in addition to logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"colorlogtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"color messages logged to stderr (if supported by terminal)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"drop_log_memory\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Drop in-memory buffers of log contents. Logs can grow very quickly and they are rarely read before they need to be evicted from memory. Instead, drop them from memory as soon as they are flushed to disk.\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"log_backtrace_at\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Emit a backtrace when logging at file:linenum.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_dir\",\n \"value\": \"/root/.t5memory/LOG/\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"If specified, logfiles are written into this directory instead of the default logging directory.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_link\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Put additional links to the log files in this directory\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_prefix\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Prepend the log prefix to the start of each log line\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"logbuflevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; ...)\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"logbufsecs\",\n \"value\": \"30\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages for at most this many seconds\",\n \"type\": \"int32\",\n \"default\": \"30\"\n },\n {\n \"name\": \"logemaillevel\",\n \"value\": \"999\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Email log messages logged at this level or higher (0 means email all; 3 means email FATAL only; ...)\",\n \"type\": \"int32\",\n \"default\": \"999\"\n },\n {\n \"name\": \"logfile_mode\",\n \"value\": \"436\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Log file mode/permissions.\",\n \"type\": \"int32\",\n \"default\": \"436\"\n },\n {\n \"name\": \"logmailer\",\n \"value\": \"/bin/mail\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Mailer used to send logging email\",\n \"type\": \"string\",\n \"default\": \"/bin/mail\"\n },\n {\n \"name\": \"logtostderr\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr instead of logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"max_log_size\",\n \"value\": \"1800\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"approx. maximum log file size (in MB). A value of 0 will be silently overridden to 1.\",\n \"type\": \"int32\",\n \"default\": \"1800\"\n },\n {\n \"name\": \"minloglevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Messages logged at a lower level than this don't actually get logged anywhere\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"stderrthreshold\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages at or above this level are copied to stderr in addition to logfiles. This flag obsoletes --alsologtostderr.\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"stop_logging_if_full_disk\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Stop attempting to log to disk if the disk is full.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"symbolize_stacktrace\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/utilities.cc\",\n \"description\": \"Symbolize the stack trace in the tombstone\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"v\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"Show all VLOG(m) messages for m <= this. Overridable by --vmodule.\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"vmodule\",\n \"value\": \"\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"per-module verbose level. Argument is a comma-separated list of =. is a glob pattern, matched against the filename base (that is, name ignoring .cc/.h./-inl.h). overrides any value given by --v.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"flagfile\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"load flags from file\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"fromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment [use 'export FLAGS_flag1=value']\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tryfromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment if present\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"undefok\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"comma-separated list of flag names that it is okay to specify on the command line even if the program does not define a flag with that name. IMPORTANT: flags in this list that have arguments MUST use the flag=value format\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tab_completion_columns\",\n \"value\": \"80\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"Number of columns to use in output for tab completion\",\n \"type\": \"int32\",\n \"default\": \"80\"\n },\n {\n \"name\": \"tab_completion_word\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"If non-empty, HandleCommandLineCompletions() will hijack the process and attempt to do bash-style command line flag completion on this value.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"help\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags [tip: all flags can have two dashes]\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpfull\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags -- same as -help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpmatch\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on modules whose name contains the specified substr\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helpon\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on the modules named by this flag value\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helppackage\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all modules in the main package\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpshort\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on only the main module for this program\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpxml\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"produce an xml version of help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"version\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show version and build info and exit\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"folly_memory_idler_madvise_stacks\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler madvises dontneed stacks on thread idle\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_memory_idler_purge_arenas\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler purges jemalloc arenas on thread idle\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"dynamic_iothreadpoolexecutor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will dynamically create threads\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_iothreadpoolexecutor_max_read_at_once\",\n \"value\": \"-1\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will use this value as default for maxReadAtOnce in its event bases, valid values are [0, inf)\",\n \"type\": \"int32\",\n \"default\": \"-1\"\n },\n {\n \"name\": \"threadtimeout_ms\",\n \"value\": \"60000\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/ThreadPoolExecutor.cpp\",\n \"description\": \"Idle time before ThreadPoolExecutor threads are joined\",\n \"type\": \"int64\",\n \"default\": \"60000\"\n },\n {\n \"name\": \"observer_manager_pool_size\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/observer/detail/ObserverManager.cpp\",\n \"description\": \"How many internal threads ObserverManager should use\",\n \"type\": \"int32\",\n \"default\": \"4\"\n },\n {\n \"name\": \"folly_hazptr_use_executor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/synchronization/Hazptr.cpp\",\n \"description\": \"Use an executor for hazptr asynchronous reclamation\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"dcache_unit_test\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/wangle/wangle/ssl/SSLSessionCacheManager.cpp\",\n \"description\": \"All VIPs share one session cache\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"zlib_compressor_buffer_growth\",\n \"value\": \"2024\"\t\",\tfrom\": \"/home/libs/proxygen/lib/utils/ZlibStreamCompressor.cpp\",\n \"description\": \"The buffer growth size to use during IOBuf zlib deflation\",\n \"type\": \"int64\",\n \"default\": \"2024\"\n },\n {\n \"name\": \"request_number\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/RestAPI/ProxygenHandler.cpp\",\n \"description\": \"Include request sequence number in response\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"add_tokens_to_fuzzy\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, list of tokens would be returned in fuzzy responce. Could make execution a bit slower, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowLoadingMultipleTmsSimultaneously\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, multiple tms could be loaded from the disk at the same time. Loading multiple TMs at the same time was disabled because of I/0 disk operation limit at the server. Enable this if you don't care about num of IO operations and want some perfomance boost\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowedram\",\n \"value\": \"10000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought RAM(in MB) allowed for service to use\",\n \"type\": \"int64\",\n \"default\": \"5000\"\n },\n {\n \"name\": \"allowedtmdsize\",\n \"value\": \"190\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets max size of tmd file(in MB) after which t5m would not allow to add new data to the tm. This helps fight btree and lookup table issues when tm gets bigger.\",\n \"type\": \"int64\",\n \"default\": \"190\"\n },\n {\n \"name\": \"debug_sleep_in_request_run\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set, provide artificial delay in every request handling execution equal to provided num of microseconds\",\n \"type\": \"int64\",\n \"default\": \"0\"\n },\n {\n \"name\": \"disable_aslr\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, process personality would be set to ADDR_NO_RANDOMIZE, affect memory layout, set to true if you have some linker or asan issue at the launch\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"enable_newlines_in_logs\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines,\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_at_shutdown\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm when shutting down the app not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_to_disk_with_every_update\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm to disk with every successfull update request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"forbiddeletefiles\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Set to true to keep all files(including temporary and tm)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"keep_tm_backups\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving tmd and tmi files, old copies would be saved with .old suffix\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"limit_num_of_active_requests\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, it would be possible to handle only up to servicethreads-1 requests at the same time, the last thread would respond with 503 to eliminate creating queue of requests waiting to be handled.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"logMutexes\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true you would see mutex logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_end\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request end with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_start\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request call with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_file_locks\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, action about locking tmd and tmi file would be logged\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_hashes_in_hash_sentence\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, tokens and their hashed would be logged, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_memmove_in_compareputdata\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving segment and causing memmove in compareputdata functions, just before memmove, data would be logged - use this to debug btree crashes. In that spot in come corrupted data, memmove could crash t5memory(or cause asan crash). Just to set low log level(v=2,t5logleve=0), reproduce the crash and check if the last log was one that started with \"memmove size = ...\"\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_tm_lifetime\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, TMs ctor(when tm object was created-so before it started loading tm) and dctor(when tm object was destroyed - so after closing tm) would have transaction level logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"maxBadSegmentsIdsSaved\",\n \"value\": \"20\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Defines how many segments(it's id's) that failed during import or reorganize would be saved to the status response\",\n \"type\": \"int32\",\n \"default\": \"20\"\n },\n {\n \"name\": \"newEntryIfOnlyContextAndTimestampDifferent\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new).\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"port\",\n \"value\": \"4040\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"What port to listen on\",\n \"type\": \"int32\",\n \"default\": \"4080\"\n },\n {\n \"name\": \"saveDifferentTargetsForSameSource\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"servicename\",\n \"value\": \"t5memory\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets service name to use in url\",\n \"type\": \"string\",\n \"default\": \"t5memory\"\n },\n {\n \"name\": \"servicethreads\",\n \"value\": \"10\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought of worker threads for service\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"skipStartupLogs\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, values from --t5loglevel would be applied immediately, if set to false- you would see all startup logs and --v and --t5loglevel would be applied only after start of the service and it's init message. Set to false if you have issues at startup\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"skip_default_flags_in_init_msg\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, all default flags would be skipped in init msg\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"suppressTmVersionCheck\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"t5_ip\",\n \"value\": \"\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Which ip to use in t5memory(default is any). Should be in format '1.1.1.1', default is to listen to all available ip\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"t5globversion_max_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5globversion_min_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5loglevel\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets t5memory log level threshold from DEVELOP(0) to TRANSACTION(6)\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"t5majversion_max_supported\",\n \"value\": \"7\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"7\"\n },\n {\n \"name\": \"t5majversion_min_supported\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"t5minversion_max_supported\",\n \"value\": \"14\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(disabled)sets max minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"14\"\n },\n {\n \"name\": \"t5minversion_min_supported\",\n \"value\": \"60\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"60\"\n },\n {\n \"name\": \"timeout\",\n \"value\": \"180000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets timeout for service request handling\",\n \"type\": \"int32\",\n \"default\": \"180000\"\n },\n {\n \"name\": \"tmListLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM list(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmMaxIdleTimeSec\",\n \"value\": \"7200\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set some value, except, 0, during request, tm list would bee cleaned up from tms that are longer inactive than this number\",\n \"type\": \"int64\",\n \"default\": \"7200\"\n },\n {\n \"name\": \"tmRequestLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for part where request is requesting tm(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"triplesthreshold\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets threshold to pre fuzzy filtering based on hashes of neibour tokens, range[0...100]. It's pre fuzzy filtering of the segments, when t5memory tries to predict how high fuzzy rate would be for some segment based on data that doesn't require as mych work as fuzzy calculations. When value is low, it can select more suggestions for fuzzy calculation, but if it set high, some best matches could be skipped. In opentm2 default value was 33, but that value seems like not working well with segments, where there are a lot of tags.\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"useTimedMutexesForReorganizeAndImport\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, in reorganize or import thread would be used mutexes with timeouts, and reorganizee or import could be canceled, false(by default) - would be used non timed mutexes\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"wait_for_import_and_reorganize_requests\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, waiting for all import and reorganize processes to be done at shutdown when not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"true\"\n }\n}" + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/multifuzzysearch": { + "post": { + "summary": "Multifuzzy", + "deprecated": false, + "description": "Sends multiple fuzzysearches in the same tm in a row", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "requests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "source": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "context": { + "type": "string" + }, + "customId": { + "type": "string", + "description": "If provided will be ased as key in response block" + } + }, + "required": [ + "sourceLang", + "targetLang", + "source", + "documentName", + "context" + ] + } + } + }, + "required": [ + "requests" + ] + }, + "example": { + "requests": [ + { + "sourceLang": "en", + "targetLang": "uk", + "source": "First segment with int", + "documentName": "", + "context": "", + "customId": "first" + }, + { + "sourceLang": "en", + "targetLang": "uk", + "source": "Third segment with", + "documentName": "", + "context": "", + "customId": "second" + } + ] + } + } + } + }, + "responses": { + "200": { + "description": "Each response object in responses has a key and value of it contains response of simple fuzzy search.\nName of a key comes from \"customId\" in request.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "responses": { + "type": "object", + "properties": { + "first": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ], + "description": "key \"first\" is an exampe. it was set because request has \"customId\": \"first\"" + }, + "second": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ], + "description": "key \"second\" is an exampe. it was set because request has \"customId\": \"second\"" + } + }, + "required": [ + "first", + "second" + ] + } + }, + "required": [ + "responses" + ] + }, + "example": { + "responses": { + "first": { + "ReturnValue": 0, + "ErrorMsg": "", + "NumOfFoundProposals": 2, + "searchedSrc": "First segment with int", + "customId": "first", + "forceFuzzy": 0, + "results": [ + { + "source": "First segment with int 10", + "target": "Перший сегмент з числом 10", + "segmentId": 6, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "10:1", + "matchType": "Fuzzy", + "matchRate": 80, + "fuzzyWords": 5, + "fuzzyDiffs": 1 + }, + { + "source": "First segment with int ", + "target": "Перший сегмент з числом ", + "segmentId": 2, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "7:1", + "matchType": "Fuzzy", + "matchRate": 80, + "fuzzyWords": 5, + "fuzzyDiffs": 1 + } + ] + }, + "second": { + "ReturnValue": 0, + "ErrorMsg": "", + "NumOfFoundProposals": 2, + "searchedSrc": "Third segment with", + "customId": "second", + "forceFuzzy": 0, + "results": [ + { + "source": "Third segment with int 10 and float 1.100", + "target": "третій сегмент з числом 10 і 1.100, друге захищене як ціле з точкою", + "segmentId": 8, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "11:1", + "matchType": "Fuzzy", + "matchRate": 37, + "fuzzyWords": 8, + "fuzzyDiffs": 5 + }, + { + "source": "Third segment with int and float ", + "target": "третій сегмент з числом і , друге захищене як ціле з точкою", + "segmentId": 4, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "8:1", + "matchType": "Fuzzy", + "matchRate": 37, + "fuzzyWords": 8, + "fuzzyDiffs": 5 + } + ] + } + } + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/multiupdate": { + "post": { + "summary": "Multi Update", + "deprecated": false, + "description": "Send multiple updateEntry requests to the same tm with one call", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "requests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "customId": { + "type": "string", + "description": "If provided will be used as field name for result in response" + }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "type": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "author": { + "type": "string" + }, + "context": { + "type": "string" + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "description": "If 1 provided will save new segmnet to disk right away, else - only in RAM until flush happens" + } + }, + "required": [ + "source", + "target", + "targetLang", + "sourceLang", + "documentName", + "author", + "context" + ] + } + } + }, + "required": [ + "requests" + ] + }, + "example": { + "requests": [ + { + "customId": "fourth", + "source": "Fourth segmnet", + "target": "Четвертий сегмент", + "type": "Manual", + "markupTable": "OTMXUXLF", + "targetLang": "en", + "sourceLang": "uk", + "documentName": "test.xlsx.sdlxliff", + "author": "Project Manager", + "context": "390" + }, + { + "customId": "fifth", + "source": "Fifth segment", + "target": "Пʼятий сегмент", + "type": "Manual", + "markupTable": "OTMXUXLF", + "targetLang": "en", + "sourceLang": "uk", + "documentName": "translation.docx", + "author": "Editor", + "context": "200" + } + ] + } + } + } + }, + "responses": { + "200": { + "description": "Each response object in responses has a key and value of it contains response of simple update.\nName of a key comes from \"customId\" in request.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "responses": { + "type": "object", + "properties": { + "fourth": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ], + "description": "Key \"fourth\" comes from request \"customId\"" + }, + "fifth": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + }, + "required": [ + "fourth", + "fifth" + ] + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "description": "Represents if new segmnets are saved to disk right away" + } + }, + "required": [ + "responses", + "save2disk" + ] + }, + "example": { + "responses": { + "fourth": { + "source": "Fourth segmnet", + "sourceNPRepl": "Fourth segmnet", + "sourceNorm": "Fourth segmnet", + "target": "Четвертий сегмент", + "segmentId": 9, + "customId": "fourth", + "documentName": "test.xlsx.sdlxliff", + "sourceLang": "UK", + "targetLang": "EN", + "type": "Manual", + "author": "PROJECT MANAGER", + "timestamp": "20250812T125637Z", + "markupTable": "OTMXUXLF", + "context": "390", + "additionalInfo": "", + "internalKey": "12:1" + }, + "fifth": { + "source": "Fifth segment", + "sourceNPRepl": "Fifth segment", + "sourceNorm": "Fifth segment", + "target": "Пʼятий сегмент", + "segmentId": 10, + "customId": "fifth", + "documentName": "translation.docx", + "sourceLang": "UK", + "targetLang": "EN", + "type": "Manual", + "author": "EDITOR", + "timestamp": "20250812T125637Z", + "markupTable": "OTMXUXLF", + "context": "200", + "additionalInfo": "", + "internalKey": "13:1" + } + }, + "save2disk": "0" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/tables": { + "get": { + "summary": "Tables", + "deprecated": false, + "description": "Get author, lang, langGroup(virtual- created in runtime and not saved on the disk), tagtable(deprecated - only one record), document(long filename, long filename caseignore-virtual, short filename- part deprecated)\n\nTables are provided in format - name(internal key): \"position) idInTable:value;\"\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Tables\nWhen adding new entry, for new Id t5memory uses basicaly size of the table before adding+1, so Id should start from id=1. \nWhen doing fuzzy search, some field could also be saved to the table, to generate then id(or find matching) to compare with saved data.\nThere are 2 types of entries- older, fixed size, and long filename. For older, every entry has the same size, 39 bytes+endOfLine. You can have up to 389 entries saved in that way in each of the tables. Longname table manage size dynamically, so it's not clear what's the maximum size, but for filenames there are duplicates in old style docname table and in longname(which are used only for this type of data). For filename search, first used longname table, then generated on tm loading caseignore longname table, then short name table. Usually entries in tables are caseignore. \n\nIn the record t5memory saves only id to the table. Maybe it make sense to implement endpoint to edit that table, if something is missing or corrupted. Exception for saving id are context and additional info, some data, that could be saved as int(segId, time), and source and target.\nTo clear or fix tag tables, you need to do reorganize, but if some entry is missing from the table, you would not restore it on export or segment exctraction via id. So tables could be used to check health of the tm. And for some ways for concordance search or some other checks.", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": false, + "example": "application/json", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": "{\n \"tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)\": \"position)idInTable: value;\",\n \"tagTablesTable(4)\": \"0)0: \\\"\\\"; \",\n \"langTable(1)\": \"0)2: \\\"EN\\\"; 1)1: \\\"UK\\\"; 2)0: \\\"\\\"; \",\n \"langGroupTable(virtual)(8)\":\"0)1: \\\"E\\\"; 1)2: \\\"U\\\"; 2)0: \\\"\u0000\\\"; \",\n \"authors(3)\": \"0)3: \\\"EDITOR\\\"; 1)1: \\\"MANAGER\\\"; 2)2: \\\"PROJECT MANAGER\\\"; 3)0: \\\"\\\"; \",\n \"filenames(long)(5)\": \"0)1: \\\"none\\\"; 1)1: \\\"none\\\"; 2)1: \\\"none\\\"; 3)2: \\\"test.xlsx.sdlxliff\\\"; 4)5: \\\"test.xlsx.sdlxliff\\\"; 5)6: \\\"translation.docx\\\"; 6)0: \\\"\\\"; \",\n \"filenames(long, caseign, virtual)(5)\": \"0)1: \\\"NONE\\\"; 1)1: \\\"NONE\\\"; 2)1: \\\"NONE\\\"; 3)5: \\\"TEST.XLSX.SDLXLIFF\\\"; 4)6: \\\"TRANSLATION.DOCX\\\"; 5)2: \\\"TRANSLATION.TXT\\\"; 6)0: \\\"\\\"; \",\n \"filenames(short)(2)\": \"0)1: \\\"none\\\"; 1)5: \\\"test.xlsx.sdlxliff\\\"; 2)6: \\\"translation.docx\\\"; 3)0: \\\"\\\"; \"\n}" + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/addtotable": { + "post": { + "summary": "Add to table", + "deprecated": false, + "description": "You can add new entry to the tables but then it's better to reorganize tm(in case if after adding new entry with missing id and there are other entry with the same value but other id - which could be the case- in binary search that could cause problems - so just reorganize it afterwards). You can check table by just doing something like full export(or concordance search for full tm), and check logs for that 2222 errors.\n\nValues shouldn't be longer than 39 bytes(utf8), and 256 for filename,it's tables limits, for that it also have checks.\nIn cases like with those tms, you can also check if x-1 and x+1 id is present. Usually they are in those bug.\nalso you can check tables via tables request\nAlso logicaly FILE_KEY=2, but that was only for shortname, which was also 39 bytes, but real key for long name table is 5. But for requests and internal handling t5memory uses 2.\nId should be between 1 and 65k, request also checks for that", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "value": { + "type": "string" + }, + "entryId": { + "type": "integer" + }, + "tableKey": { + "type": "integer" + } + }, + "required": [ + "value", + "entryId", + "tableKey" + ] + }, + "example": { + "value": "MY new entry", + "entryId": 100, + "tableKey": 2 + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "examples": { + "1": { + "summary": "Invalid table key", + "value": { + "ReturnValue": 400, + "ErrorMsg": "Invalid table key(0); available keys = LANG_KEY(1), FILE_KEY(2), AUTHOR_KEY(3)" + } + }, + "2": { + "summary": "Table already have entry with matching id", + "value": { + "ReturnValue": 400, + "ErrorMsg": "Id match in Table 2 already have entry with matching id = 100 and value = \"Teaser Page.xlsx\";" + } + }, + "3": { + "summary": "Value field is empty", + "value": { + "ReturnValue": 400, + "ErrorMsg": "Value field is empty" + } + } + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/": { + "post": { + "summary": "Create TM", + "deprecated": false, + "description": "", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "New TM name" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "data": { + "type": "string", + "deprecated": true, + "description": "Binary TM file encode with base64.\nCan be passed to create TM with data right away.\nUse with cautios!" + } + }, + "required": [ + "name", + "sourceLang" + ] + }, + "examples": { + "1": { + "value": { + "name": "examle_tm2", + "sourceLang": "de-DE" + }, + "summary": "Create TM" + }, + "2": { + "value": { + "name": "test-memory", + "sourceLang": "en-GB", + "data": "UEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLk1FTQEAAADtzssKQEAUgOFjoTwGWR+5lPW4TCjDxgtomqI0I3lzLyBvIEUW59v829+GrinTJMaK99iqWY5Yq2VFMcpp1sotjN43swSCC/gSzw551bHu7MGLlMYqh6d8xrpBhBFjPnhACCGE/MMJUEsHCKEIpyRnAAAAAAAAAAAIAAAAAAAAUEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLlRNRAEAAADt3bFKI0EYB/BvNzFaXCXH1Vdcc3h7xR14xXEsd5pTwRgIEWwlBBRCFPEFbKx9BJ/DJ7CyshYrCwsbX0A3GtykFQnE329ZZnb+u92wMMXMt9FupBFxlERxRVTi+mPwCrNjLe9b+nw/qcRqq77UbC0HwCSdR/3vfafozSVry4s/f2Qr9Wa23t3tbGer3d5+1tju7Oz2u5+X9vqHB3u97+3Gm/24uv1s5V8AAAAAAAAAAAAAAAAAABHzg13/hVqcRSuPOM7P8sv8NG/lvTwAAAAAAAAAAAAAAAAAAABgasxFEjOD3rAO+EwpqQ56Jx+eRmqlpBLlj5JSUhtJqqUkGUnSUhIjQ5UYG4jiLIA7O/4BAAAAAAAAAAAAAAAAAABgAqrPJwHMx2zcfvryNdKbP1e/Iy5+LS58W3h8BgAAAAAAAAAAAJgO6Wg7rK7UbDe2NrfW/5dfTYJp9VI7a1i/CwAAYGLGVqq8U+YBBfOAgnnAowdQSwcITWiheHEBAAAAAAAAAAABAAAAAABQSwMEFAAICAgAUmZ4VgAAAAAAAAAAAAAAACgABABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAO3dwWkCYRAG0NFTzlaQBv4cEsg1JLqokI0gNiDLQgRZQ7ABu7MaLxawKijqWWQPvjcMM1PEx/xM8nZErCJa++J2dV1fTR5b+9inazDOuqNxLwCatI7sc1sctqfWsPf+9pr62Sh9l7Nimgbl/C/l0+J3VpXP3UW1/F/MXyb5MO6krFL/KwAAAAAAAAAAAAAAAACAiM5F6r8Tm48AAAAAAAAAAAAAAAAAAAAAGnD+TS7/DwAAAAAAAAAAAAAAAAAAAM3YAVBLBwgGVIGewwAAAAAAAAAAwAAAAAAAAFBLAQIAABQACAgIALRlVFahCKckZwAAAAAIAAAoAAQAAAAAAAAAAACkgQAAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuTUVNAQAAAFBLAQIAABQACAgIAGdralZNaKF4cQEAAAAAAQAoAAQAAAAAAAAAAACkgckAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1EAQAAAFBLAQIAABQACAgIAFJmeFYGVIGewwAAAADAAAAoAAQAAAAAAAAAAACkgZwCAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAFBLBgYsAAAAAAAAAB4DLQAAAAAAAAAAAAMAAAAAAAAAAwAAAAAAAAAOAQAAAAAAAMEDAAAAAAAAUEsGBwAAAADPBAAAAAAAAAEAAABQSwUGAAAAAAMAAwAOAQAAwQMAAAAA" + }, + "summary": "Create with file for TM" + } + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Created TM name" + } + }, + "required": [ + "name" + ] + }, + "example": { + "name": "examle_tm2" + } + } + }, + "headers": {} + }, + "x-200:Error": { + "description": "", + "content": { + "application/json": { + "schema": { + "title": "", + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer", + "description": "Error code" + }, + "ErrorMsg": { + "type": "string", + "description": "Message to explain error" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "example": { + "ReturnValue": 7272, + "ErrorMsg": "::ERROR_MEM_NAME_EXISTS:: TM with this name already exists: test-memory; res = 0" + } + } + }, + "headers": {} + } + }, + "security": [] + }, + "get": { + "summary": "Get TMs list", + "deprecated": false, + "description": "", + "tags": [], + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "Open": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "TM name" + } + }, + "required": [ + "name" + ] + }, + "description": "Memories loaded into RAM" + }, + "Available on disk": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "TM name" + } + }, + "required": [ + "name" + ] + }, + "description": "Memories available on disk" + } + }, + "required": [ + "Open", + "Available on disk" + ] + }, + "example": { + "Open": [ + { + "name": "test-memory" + } + ], + "Available on disk": [ + { + "name": "test-memory-1" + } + ] + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory_service/resources": { + "get": { + "summary": "Resources", + "deprecated": false, + "description": "", + "tags": [], + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": { + "filebuffers": [ + { + "name": "/root/.t5memory/TABLE/OTMXUXLF.TBL", + "size": 3660 + } + ], + "totalOccupiedByFilebuffersRAM": 3660, + "tms": [ + { + "name": "test-memory", + "status": "open", + "size": 134976, + "activeRequest": "", + "expectedSize": 0, + "StatusInfo": { + "status": "open", + "sizeInRAM": 134976, + "activeRequest": "", + "tmxImportStatus": "available", + "importProgress": 100, + "importTime": "Overall import time is : 0:00:00\n", + "segmentsImported": 2, + "invalidSegments": 0, + "tmxSegmentCount": 2, + "importRuntimeSec": 0, + "importTimeoutSec": 0, + "newEntryIfOnlyContextAndTimestampDifferent": 1, + "saveDifferentTargetsForSameSource": 0, + "invalidSegmentsRCs": "", + "firstInvalidSegments(num:segId:errCode)": "", + "invalidSymbolErrors": 0, + "errorMsg": "", + "rc": 0, + "lastAccessTime": "20250801T153723Z", + "creationTime": "20250801T123723Z", + "tmCreatedInT5M_version": "0:7:14", + "segmentIndex": 2, + "sourceLang": "de", + "internalDescription": "", + "tmd fbuffer was modified": 1, + "tmi fbuffer was modified": 1 + } + } + ], + "totalOccupiedByTMsInRAM": 712165696, + "Run date": "Wed Jul 30 16:08:09 2025 ", + "Build date": "2025/07/29 16:08:46", + "Git commit info": "dff40cb9 * updated to v0.7.14 + added skipStartupLogs flags(default true) * fixed non-asan builds * fixed nextInternalKey is empty log on failed export", + "Version": "0.7.14", + "Worker threads": 10, + "Timeout(ms)": 180000, + "Resident set": 1448300000000, + "Virtual memory usage": 3743340000000000, + "Requests": { + "RequestCount": 6829, + "RequestExecutionSumTime(sec)": 14.484, + "CreateMem": { + "ReqCount": 44, + "SumTime(sec)": 0.065, + "AvrgReqTime": 0.00147727 + }, + "DeleteMem": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "ImportMem": { + "ReqCount": 29, + "SumTime(sec)": 0.016, + "AvrgReqTime": 0.000551724 + }, + "ExportMem": { + "ReqCount": 13, + "SumTime(sec)": 0.09, + "AvrgReqTime": 0.00692308 + }, + "CloneTmLocaly": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "Reorganize": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "StatusMem": { + "ReqCount": 109, + "SumTime(sec)": 0.011, + "AvrgReqTime": 0.000100917 + }, + "FlushMem": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "Fuzzy": { + "ReqCount": 98, + "SumTime(sec)": 1.034, + "AvrgReqTime": 0.010551 + }, + "MultiFuzzy": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "Concordance": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "UpdateEntry": { + "ReqCount": 6, + "SumTime(sec)": 0.013, + "AvrgReqTime": 0.00216667 + }, + "MultiUpdate": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "GetEntry": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "DeleteEntry": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "SaveAllTms": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "ListOfMemories": { + "ReqCount": 6474, + "SumTime(sec)": 13.221, + "AvrgReqTime": 0.00204217 + }, + "Resources": { + "ReqCount": 22, + "SumTime(sec)": 0.016, + "AvrgReqTime": 0.000727273 + }, + "Flags": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "Other": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + }, + "Unrecognized": { + "ReqCount": 0, + "SumTime(sec)": 0, + "AvrgReqTime": 0 + } + }, + "RAM limit(MB)": 10000 + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/status": { + "get": { + "summary": "TM Status", + "deprecated": false, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-GetthestatusofTM", + "tags": [], + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string", + "description": "Statuses:\nopen - memory already in RAM ready to use\navailable - not loaded but present on disk\nloading - loading from disk to RAM\nnot found - error status of absent memory\nreorganize running - Reorganizing in process\nimport running - Import in process\nwaiting for loading - loading requsted but not done yet\nfailed to open - corrupted TM file" + }, + "sizeInRAM": { + "type": "integer" + }, + "activeRequest": { + "type": "string", + "description": " " + }, + "lastAccessTime": { + "type": "string" + }, + "creationTime": { + "type": "string" + }, + "tmCreatedInT5M_version": { + "type": "string", + "description": "version of t5memory on momemt of TM creation" + }, + "segmentIndex": { + "type": "integer", + "description": "index of last inserted segment (not queriable)" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "internalDescription": { + "type": "string" + }, + "tmd fbuffer was modified": { + "type": "integer" + }, + "tmi fbuffer was modified": { + "type": "integer" + } + }, + "required": [ + "status", + "sizeInRAM", + "activeRequest", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ] + }, + "example": { + "status": "open", + "sizeInRAM": 134976, + "activeRequest": "", + "lastAccessTime": "20250801T160913Z", + "creationTime": "20250716T102923Z", + "tmCreatedInT5M_version": "0:7:7", + "segmentIndex": 5, + "sourceLang": "en", + "internalDescription": "", + "tmd fbuffer was modified": 1, + "tmi fbuffer was modified": 1 + } + } + }, + "headers": {} + }, + "x-200:Import processed status": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "sizeInRAM": { + "type": "integer" + }, + "activeRequest": { + "type": "string" + }, + "tmxImportStatus": { + "type": "string", + "description": "Statuses of ongoing improt:\navailable - not tipical. If returned and \"importTime\" == \"not finished\" then import still ongoing\nimport - ongoing import" + }, + "importProgress": { + "type": "integer", + "description": "Progress in persents" + }, + "importTime": { + "type": "string" + }, + "segmentsImported": { + "type": "integer", + "description": "NUmber of successfully imported segments" + }, + "invalidSegments": { + "type": "integer", + "description": "Number of invalid segments found in TMX" + }, + "tmxSegmentCount": { + "type": "integer", + "description": "Number of found segmnets in import TMX file" + }, + "importRuntimeSec": { + "type": "integer" + }, + "importTimeoutSec": { + "type": "integer" + }, + "newEntryIfOnlyContextAndTimestampDifferent": { + "type": "integer" + }, + "saveDifferentTargetsForSameSource": { + "type": "integer", + "description": "0 or 1 - considered as bool value" + }, + "invalidSegmentsRCs": { + "type": "string" + }, + "firstInvalidSegments(num:segId:errCode)": { + "type": "string" + }, + "invalidSymbolErrors": { + "type": "integer" + }, + "errorMsg": { + "type": "string" + }, + "rc": { + "type": "integer" + }, + "lastAccessTime": { + "type": "string" + }, + "creationTime": { + "type": "string" + }, + "tmCreatedInT5M_version": { + "type": "string" + }, + "segmentIndex": { + "type": "integer" + }, + "sourceLang": { + "type": "string" + }, + "internalDescription": { + "type": "string" + }, + "tmd fbuffer was modified": { + "type": "integer" + }, + "tmi fbuffer was modified": { + "type": "integer" + } + }, + "required": [ + "status", + "sizeInRAM", + "activeRequest", + "tmxImportStatus", + "importProgress", + "importTime", + "segmentsImported", + "invalidSegments", + "tmxSegmentCount", + "importRuntimeSec", + "importTimeoutSec", + "newEntryIfOnlyContextAndTimestampDifferent", + "saveDifferentTargetsForSameSource", + "invalidSegmentsRCs", + "firstInvalidSegments(num:segId:errCode)", + "invalidSymbolErrors", + "errorMsg", + "rc", + "lastAccessTime", + "creationTime", + "tmCreatedInT5M_version", + "segmentIndex", + "sourceLang", + "internalDescription", + "tmd fbuffer was modified", + "tmi fbuffer was modified" + ] + }, + "example": { + "status": "open", + "sizeInRAM": 134976, + "activeRequest": "", + "tmxImportStatus": "available", + "importProgress": 100, + "importTime": "Overall import time is : 0:00:00\n", + "segmentsImported": 4, + "invalidSegments": 0, + "tmxSegmentCount": 4, + "importRuntimeSec": 0, + "importTimeoutSec": 0, + "newEntryIfOnlyContextAndTimestampDifferent": 1, + "saveDifferentTargetsForSameSource": 0, + "invalidSegmentsRCs": "", + "firstInvalidSegments(num:segId:errCode)": "", + "invalidSymbolErrors": 0, + "errorMsg": "", + "rc": 0, + "lastAccessTime": "20250807T130822Z", + "creationTime": "20250716T102923Z", + "tmCreatedInT5M_version": "0:7:7", + "segmentIndex": 9, + "sourceLang": "en", + "internalDescription": "", + "tmd fbuffer was modified": 1, + "tmi fbuffer was modified": 1 + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/importtmx": { + "post": { + "summary": "IMPORT TMX", + "deprecated": false, + "description": "Rout is async. So when import is finished should be checked with status call", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "multipart/form-data": { + "encoding": { + "json_data": { + "contentType": "application/json" + } + }, + "schema": { + "type": "object", + "properties": { + "json_data": { + "type": "string", + "description": "json pretty print string.\n\n\"framingTags\" key possible values:\n\"saveAll\" - default behaviour, do nothing\n\"skipAll\" - skip all enclosing tags, including standalone tags\n\"skipPaired\" - skip only paired enclosing tags\n\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, default values from flags would be used.", + "example": "{\n \"framingTags\": \"saveAll\",\n \"newEntryIfOnlyContextAndTimestampDifferent\": 0,\n \"saveDifferentTargetsForSameSource\": 0,\n \"timeout\": 100000\n}\n" + }, + "file": { + "format": "binary", + "type": "string", + "description": "tmx file to import into TM", + "example": "" + } + }, + "required": [ + "json_data", + "file" + ] + }, + "examples": {} + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": { + "msg": "success" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/download.tmx": { + "get": { + "summary": "EXPORT TMX", + "deprecated": false, + "description": "JSON pretty print string:\nlimit - number of segments to fetch at a time\nstartFromInternalKey - internal key to start export from\n\nWorks as cursor pagination.\n\nIf no body provided, export starts from the beginning (key 7:1) to the end.", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/xml", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "limit": { + "type": "string" + }, + "startFromInternalKey": { + "type": "string" + } + }, + "required": [ + "limit", + "startFromInternalKey" + ] + }, + "example": { + "limit": "100", + "startFromInternalKey": "1:1" + } + } + } + }, + "responses": { + "200": { + "description": "In response headers \"NextInternalKey\" will contain \"startFromInternalKey\" for next call.\nIf \"NextInternalKey\" == \"0:0\" - then you reached end of TM", + "content": { + "application/xml": { + "schema": { + "type": "object", + "properties": {} + }, + "example": "\n\n\n
\n \n \n 2\n 7: 1\n OTMXUXLF\n none\n \n First segment\n \n \n Перший сегмент\n \n \n \n" + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/search": { + "post": { + "summary": "Concordance search", + "deprecated": false, + "description": "All fields is optional, but some depends on other, so error should be returned in case of not providing required field.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-NewConcordancesearch", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "sourceSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "target": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "targetSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "document": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "documentSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "author": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "authorSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "addInfo": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "addInfoSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "context": { + "type": "string", + "description": "Sets what to look for in source of the segments, based on type of search, specified in sourceSearchMode(exact, concordance). If sourceSearchMode is not specified, returns an error." + }, + "contextSearchMode": { + "type": "string", + "description": "String with required \nEXACT or CONCORDANCE \n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED  for applying filter in inverted state, so to return false on match and true \nif no match. Logical NOT\n\nAttributes is not case sensetive, \nSeparator doen't matters" + }, + "timestampSpanStart": { + "type": "string", + "description": "String with date in format\n \"20240121T115234Z\"" + }, + "timestampSpanEnd": { + "type": "string", + "description": "String with date in format\n \"20240121T115234Z\"" + }, + "onlyCountSegments": { + "type": "string", + "description": "Instead of returning segment, would go in search till the end of tm and return total number of segments, that returns true with selected filters", + "enum": [ + "1", + "0" + ] + }, + "searchPosition": { + "type": "string", + "description": "Point where to start search in tmd file", + "example": "8:1" + }, + "logicalOr": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "description": "By default source, target, document, author, context, addinfo, timestamp is combined in logical AND, but by sending here \"OR\" you can switch that to logical OR, any other value would left it in default AND state.\nDoesn't apply to sourceLang and targetLang filters, they are always in AND state" + }, + "numResults": { + "type": "integer", + "description": "Points how many matches return in current request", + "minimum": 0, + "maximum": 200, + "default": 5 + }, + "sourceLang": { + "type": "string", + "description": "Filter segments on src/trg lang attribute, \n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + }, + "targetLang": { + "type": "string", + "description": "Filter segments on src/trg lang attribute, \n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + }, + "msSearchAfterNumResults": { + "type": "integer", + "default": 0, + "description": "sets how many ms should pass between first found segment and search stop, if it didn't reach the end yet. " + }, + "loggingThreshold": { + "type": "integer", + "maximum": 6, + "description": "HACK.\nAdditional field to set log level on the run.\nSets log level for t5memory process in general and not this request only" + } + } + }, + "example": { + "source": "First", + "sourceSearchMode": "contains, CASEINSENSETIVE", + "target": "", + "targetSearchMode": "contains, CASEINSENSETIVE", + "document": "", + "documentSearchMode": "contains, CASEINSENSETIVE", + "author": "", + "authorSearchMode": "contains, CASEINSENSETIVE", + "addInfo": "", + "addInfoSearchMode": "contains, CASEINSENSETIVE", + "context": "", + "contextSearchMode": "contains, CASEINSENSETIVE", + "timestampSpanStart": "19691231T230000Z", + "timestampSpanEnd": "20260628T220000Z", + "onlyCountSegments": "0", + "searchPosition": "", + "logicalOr": 0, + "numResults": 200 + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "Filters": { + "type": "string" + }, + "GlobalSearchOptions": { + "type": "string" + }, + "ReturnValue": { + "type": "integer" + }, + "ReturnMessage": { + "type": "string" + }, + "NewSearchPosition": { + "type": "null" + }, + "NumOfFoundSegments": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + } + }, + "required": [ + "Filters", + "GlobalSearchOptions", + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "NumOfFoundSegments", + "results" + ] + }, + "example": { + "Filters": "Search filter, field: SOURCE FilterType::CONTAINS SearchStr: 'FIRST'; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: TARGET FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: ADDINFO FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: CONTEXT FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: AUTHOR FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: DOCUMENT FilterType::CONTAINS SearchStr: ''; Options: SEARCH_CASEINSENSITIVE_OPT|;\nSearch filter, field: TIMESTAMP FilterType::RANGE Range: 19691231T230000Z - 20260628T220000Z Options: ;\n", + "GlobalSearchOptions": "SEARCH_FILTERS_LOGICAL_AND", + "ReturnValue": 10010, + "ReturnMessage": "ENDREACHED_RC", + "NewSearchPosition": null, + "NumOfFoundSegments": 2, + "results": [ + { + "source": "First segment with int ", + "target": "Перший сегмент з числом ", + "segmentId": 2, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "7:1" + }, + { + "source": "First segment with int 10", + "target": "Перший сегмент з числом 10", + "segmentId": 6, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "11:1" + } + ] + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/concordancesearch": { + "post": { + "summary": "Concordance search (DEPRICATED)", + "deprecated": false, + "description": " SearchPosition / NewSearchPositionFormat: \"7:1\"\nFirst is segmeng\\record number, second is target number\nThe NextSearchposition is an internal key of the memory for the next position on sequential access. Since it is an internal key, maintained and understood by the underlying memory plug-in (for EqfMemoryPlugin is it the record number and the position in one record),\nno assumptions should be made regarding the content. It is just a string that, should be sent back to OpenTM2 on the next request, so that the search starts from there.\nSo is the implementation in Translate5: The first request to OpenTM2 contains SearchPosition with an empty string, OpenTM2 returns than a string in NewSearchPosition, which is just resent to OpenTM2 in the next request.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Concordancesearch\n ", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "searchString": { + "type": "string" + }, + "searchType": { + "type": "string", + "enum": [ + "Source", + "Target", + "SourceAndTarget" + ], + "example": "Source" + }, + "searchPosition": { + "type": "null" + }, + "numResults": { + "type": "integer" + }, + "msSearchAfterNumResults": { + "type": "integer" + } + }, + "required": [ + "searchString", + "searchType" + ] + }, + "example": { + "searchString": "With nice text", + "searchType": "source", + "searchPosition": null, + "numResults": 20, + "msSearchAfterNumResults": 250 + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ReturnMessage": { + "type": "string" + }, + "NewSearchPosition": { + "type": "string", + "nullable": true + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + } + }, + "required": [ + "ReturnValue", + "ReturnMessage", + "NewSearchPosition", + "results" + ] + }, + "example": { + "ReturnValue": 10010, + "ReturnMessage": "ENDREACHED_RC", + "NewSearchPosition": null, + "results": [ + { + "source": "First segment with int ", + "target": "Перший сегмент з числом ", + "segmentId": 2, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "7:1" + }, + { + "source": "First segment with int 10", + "target": "Перший сегмент з числом 10", + "segmentId": 6, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "11:1" + } + ] + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/fuzzysearch": { + "post": { + "summary": "Fuzzy search", + "deprecated": false, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Fuzzysearch", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "targetLang": { + "type": "string", + "description": "rfc5646" + }, + "source": { + "type": "string", + "description": "Search string to look for" + }, + "documentName": { + "type": "string" + }, + "markupTable": { + "type": "string", + "description": "If there is no markup, default OTMXUXLF would be used.\nMarkup tables should be located inside ~/.t5memory/TABLE/%markup$.TBL" + }, + "context": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "description": "If yes - will return all potential fuzzies\nNo - only first exact match or most relevant matches" + }, + "numOfProposals": { + "type": "integer", + "description": "Num of expected segments in output. By default it's 5" + } + }, + "required": [ + "sourceLang", + "targetLang", + "source" + ] + }, + "example": { + "sourceLang": "en", + "targetLang": "uk", + "source": "Second segment ", + "documentName": "", + "markupTable": "OTMXUXLF", + "context": "" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + }, + "NumOfFoundProposals": { + "type": "integer" + }, + "searchedSrc": { + "type": "string" + }, + "customId": { + "type": "string" + }, + "forceFuzzy": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + }, + "matchType": { + "type": "string" + }, + "matchRate": { + "type": "integer" + }, + "fuzzyWords": { + "type": "integer" + }, + "fuzzyDiffs": { + "type": "integer" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey", + "matchType", + "matchRate", + "fuzzyWords", + "fuzzyDiffs" + ] + } + } + }, + "required": [ + "ReturnValue", + "ErrorMsg", + "NumOfFoundProposals", + "searchedSrc", + "customId", + "forceFuzzy" + ] + }, + "example": { + "ReturnValue": 0, + "ErrorMsg": "", + "NumOfFoundProposals": 2, + "searchedSrc": "First segment with int ", + "customId": "", + "forceFuzzy": 0, + "results": [ + { + "source": "First segment with int 10", + "target": "Перший сегмент з числом 10", + "segmentId": 6, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "11:1", + "matchType": "Fuzzy", + "matchRate": 80, + "fuzzyWords": 5, + "fuzzyDiffs": 1 + }, + { + "source": "First segment with int 11", + "target": "Перший сегмент з числом 11", + "segmentId": 2, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "7:1", + "matchType": "Fuzzy", + "matchRate": 80, + "fuzzyWords": 5, + "fuzzyDiffs": 1 + } + ] + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/getentry": { + "post": { + "summary": "Get entry", + "deprecated": false, + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "recordKey": { + "type": "string" + }, + "targetKey": { + "type": "string" + } + }, + "required": [ + "recordKey", + "targetKey" + ] + }, + "example": { + "recordKey": "8", + "targetKey": "2" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + }, + "example": { + "source": "Third segment", + "target": "третій сегмент", + "segmentId": 4, + "customId": "", + "documentName": "none", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "MANAGER", + "timestamp": "20250716T084347Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "9:1" + } + } + }, + "headers": {} + }, + "400": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "example": { + "ReturnValue": 939, + "ErrorMsg": "Requested entry not found! Next internalKey after requested is : 10:1" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/entry": { + "post": { + "summary": "Update entry", + "deprecated": false, + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Updateentry", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "Segment source string" + }, + "target": { + "type": "string", + "description": "Segment target string" + }, + "type": { + "type": "string", + "const": "Manual", + "description": "Should be set to \"Manual\"", + "example": "Manual" + }, + "markupTable": { + "type": "string", + "const": "OTMXUXLF", + "description": "Should be \"OTMXUXLF\"", + "example": "OTMXUXLF" + }, + "sourceLang": { + "type": "string", + "description": "rfc5646" + }, + "targetLang": { + "type": "string", + "description": "rfc5646" + }, + "timeStamp": { + "type": "string", + "pattern": "^\\d{8}T\\d{6}Z$", + "format": "yyyyMMdd'T'HHmmss'Z'", + "description": "Time of segment creation", + "example": "20240111T111443Z" + }, + "documentName": { + "type": "string", + "description": "Name of document where segmnet comes from" + }, + "author": { + "type": "string" + }, + "context": { + "type": "string", + "description": "Field that will be considered in fuzzy searches. Plays a role of aditional search option" + }, + "save2disk": { + "type": "integer", + "description": "If true - segment will be written to disk right away, else - only set in RAM and saved when all memory is flushed", + "enum": [ + 1, + 0 + ], + "default": 0, + "example": 1 + }, + "saveDifferentTargetsForSameSource": { + "type": "integer", + "enum": [ + 1, + 0 + ], + "description": "If param set to 1 - new entry will be saved and old preserved even if all fileds of segmnet except target are same." + } + }, + "required": [ + "source", + "target", + "sourceLang", + "targetLang", + "timeStamp", + "documentName", + "author", + "context", + "save2disk" + ] + }, + "example": { + "source": "hello", + "target": "again good bye 1", + "type": "Manual", + "markupTable": "OTMXUXLF", + "sourceLang": "de", + "targetLang": "en", + "timeStamp": "20250112T111443Z", + "documentName": "translation.txt", + "author": "Project Manager", + "context": "", + "save2disk": 1, + "saveDifferentTargetsForSameSource": 1 + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceNPRepl": { + "type": "string" + }, + "sourceNorm": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "sourceNPRepl", + "sourceNorm", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + }, + "example": { + "source": "Second segment", + "sourceNPRepl": "Second segment", + "sourceNorm": "Second segment", + "target": "Другий сегмент", + "segmentId": 14, + "customId": "", + "documentName": "translation.txt", + "sourceLang": "EN", + "targetLang": "UK", + "type": "Manual", + "author": "PROJECT MANAGER", + "timestamp": "20240111T111443Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "14:1" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}": { + "delete": { + "summary": "Delete TM", + "deprecated": false, + "description": "", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "text/plain": { + "schema": { + "type": "string" + }, + "examples": {} + } + } + }, + "responses": { + "200": { + "description": "Response will contain json with memory name as key and \"deleted\" as value.\nMay return 500 with \"not found(error 48)\" as value that is simply indicates absense of TM", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": { + "test-tm": "deleted" + } + } + }, + "headers": {} + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": { + "test-tm": "not found(error 48)" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/entrydelete": { + "post": { + "summary": "Delete entry", + "deprecated": false, + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".\nThere is also \"segmentId\" value.\nFor delete entry request it should be provided to validate that current state of t5memory data is same as at the moment when you got internalKey as internal key value may be regenerated at any moment of data processing on import or update rrquest handling.\nSo if you try to delete entry and getting error - try to fetch entry internalKey nad segmentId again and then retry delete action.", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "recordKey": { + "type": "integer" + }, + "targetKey": { + "type": "integer" + }, + "segmentId": { + "type": "integer" + }, + "save2disk": { + "type": "string", + "enum": [ + "1", + "0" + ], + "default": "0", + "description": "Save memory to disk right away.\nSaves if not passed.", + "example": "1" + } + }, + "required": [ + "recordKey", + "targetKey", + "segmentId" + ] + }, + "example": { + "recordKey": 8, + "targetKey": 4, + "segmentId": 12 + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "fileFlushed": { + "type": "integer", + "description": "Indicates was TM flushed to disk or not. Depends on \"save2disk\" value." + }, + "results": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "segmentId": { + "type": "integer" + }, + "customId": { + "type": "string" + }, + "documentName": { + "type": "string" + }, + "sourceLang": { + "type": "string" + }, + "targetLang": { + "type": "string" + }, + "type": { + "type": "string" + }, + "author": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "markupTable": { + "type": "string" + }, + "context": { + "type": "string" + }, + "additionalInfo": { + "type": "string" + }, + "internalKey": { + "type": "string" + } + }, + "required": [ + "source", + "target", + "segmentId", + "customId", + "documentName", + "sourceLang", + "targetLang", + "type", + "author", + "timestamp", + "markupTable", + "context", + "additionalInfo", + "internalKey" + ] + } + }, + "required": [ + "fileFlushed", + "results" + ] + }, + "example": { + "fileFlushed": 1, + "results": { + "source": "Second segment", + "target": "Другий сегмент", + "segmentId": 14, + "customId": "", + "documentName": "translation.txt", + "sourceLang": "en", + "targetLang": "UK", + "type": "Manual", + "author": "PROJECT MANAGER", + "timestamp": "20250811T101311Z", + "markupTable": "OTMXUXLF", + "context": "", + "additionalInfo": "", + "internalKey": "14:1" + } + } + } + }, + "headers": {} + }, + "400": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "example": { + "ReturnValue": 14002, + "ErrorMsg": "Error: to delete entry by key you should provide all three fields: recordKey, targetKey and segmentId" + } + } + }, + "headers": {} + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ReturnValue": { + "type": "integer" + }, + "ErrorMsg": { + "type": "string" + } + }, + "required": [ + "ReturnValue", + "ErrorMsg" + ] + }, + "example": { + "ReturnValue": 5005, + "ErrorMsg": "" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/entriesdelete": { + "post": { + "summary": "Delete batch", + "deprecated": false, + "description": "Asynchronious operation.\nThis would start reorganize process which would remove like reorganize bad segments and also would remove segments that gives true when checking with provided filters combined with logical AND. So if you provide timestamps and addInfo, only segments within provided timestamp and with that addInfo would not be imported to new TM(check reorganize process). \nEvery parameter is optional, so empty json would just start reorganize async process.\nIf you provide one of timestamps you would get error - please provide both. \nTo add parameter you should set it's SearchMode to be EXACT|CONCORDANCE(non case sensetive)\nIf only searched string provided, but not search mode - you would get error.", + "tags": [], + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "sourceSearchMode": { + "type": "string" + }, + "target": { + "type": "string" + }, + "targetSearchMode": { + "type": "string" + }, + "document": { + "type": "string" + }, + "documentSearchMode": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorSearchMode": { + "type": "string" + }, + "addInfo": { + "type": "string" + }, + "addInfoSearchMode": { + "type": "string" + }, + "context": { + "type": "string" + }, + "contextSearchMode": { + "type": "string" + }, + "timestampSpanStart": { + "type": "string" + }, + "timestampSpanEnd": { + "type": "string" + }, + "searchPosition": { + "type": "string" + } + }, + "required": [ + "timestampSpanStart", + "timestampSpanEnd" + ] + }, + "example": { + "source": "123", + "sourceSearchMode": "contains, CASEINSENSETIVE", + "target": "234", + "targetSearchMode": "contains, CASEINSENSETIVE", + "document": "", + "documentSearchMode": "contains, CASEINSENSETIVE", + "author": "", + "authorSearchMode": "contains, CASEINSENSETIVE", + "addInfo": "", + "addInfoSearchMode": "contains, CASEINSENSETIVE", + "context": "", + "contextSearchMode": "contains, CASEINSENSETIVE", + "timestampSpanStart": "19691231T230000Z", + "timestampSpanEnd": "20240610T220000Z", + "searchPosition": "" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory_service/shutdown": { + "post": { + "summary": "Shutdown", + "deprecated": false, + "description": "Safely shutting down the service with\\without saving all loaded tm files to the disk.", + "tags": [], + "parameters": [ + { + "name": "dontsave", + "in": "query", + "description": "Skips saving tms, for now value doesn't matter, only presence.\nIf try to save tms before closing, would check if there is still import process going on\nIf there is some, would wait 1 second and check again. \nRepeats last step up to 10 min, then closes service anyway. ", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory_service/savetms": { + "get": { + "summary": "Save TMs", + "deprecated": false, + "description": "Flushes all filebuffers(TMD, TMI files) into the filesystem. Reset 'Modified' flags for file buffers. \n\nFilebuffer is a file instance of .TMD or .TMI loaded into RAM. It provides better speed and safety when working with files.", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + }, + "example": { + "saved 3 tms": "ID9580-de-fr_next-2, ID9612-en-uk, test-memory", + "errMsg": "" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/clone": { + "post": { + "summary": "Clone", + "deprecated": false, + "description": "Creates TM with the provided name and data cloned from provided memory", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "newName": { + "type": "string", + "description": "Name of memory to be crreated" + } + }, + "required": [ + "newName" + ] + }, + "example": { + "newName": "clonned-tm-1" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "msg": { + "type": "string" + }, + "time": { + "type": "string" + } + }, + "required": [ + "msg", + "time" + ] + }, + "example": { + "msg": "clonned-tm-1 was cloned successfully", + "time": "1 ms" + } + } + }, + "headers": {} + } + }, + "security": [] + } + }, + "/t5memory/{memory}/reorganize": { + "get": { + "summary": "Reorganize", + "deprecated": false, + "description": "Under the hood it creates new tm with $Org- prefix, then reimport all segments one-by-one, and then deletes original TM and rename reorganized TM to replace original. \nThis request should flush tm(from RAM to the disk) before reorganizing\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-ReorganizeTM\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, defaule values from flags would be used.", + "tags": [], + "parameters": [ + { + "name": "Accept", + "in": "header", + "description": "", + "required": true, + "example": "application/json; charset=utf-8", + "schema": { + "type": "string" + } + }, + { + "name": "Accept-Charset", + "in": "header", + "description": "", + "required": true, + "example": "UTF-8", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "newEntryIfOnlyContextAndTimestampDifferent": { + "type": "string", + "enum": [ + "1", + "0" + ], + "example": "1" + }, + "saveDifferentTargetsForSameSource": { + "type": "string", + "enum": [ + "1", + "0" + ], + "example": "1" + } + } + }, + "examples": {} + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {} + } + } + }, + "headers": {} + } + }, + "security": [] + } + } + }, + "components": { + "schemas": {}, + "securitySchemes": {} + }, + "servers": [], + "security": [] +} \ No newline at end of file diff --git a/docs/t5memory.openapi.yaml b/docs/t5memory.openapi.yaml new file mode 100644 index 00000000..be96ee10 --- /dev/null +++ b/docs/t5memory.openapi.yaml @@ -0,0 +1,2981 @@ +openapi: 3.0.1 +info: + title: t5memory + description: '' + version: 0.7.36 +tags: [] +paths: + /t5memory/{memory}/flush: + get: + summary: Flush memory + deprecated: false + description: >- + Endpoint is sync(blocking) + + + If tm is not found on the disk - returns 404 + + If tm is not open - returns 400 with message + + Then t5memory requests writes pointer to the tm (so it waits till other + requests that's working with the tm would finish) and then it flushes it + to the disk + + Could also return an error if flushing got some issue. + + Would not open the tm, if it's not opened yet, but instead would return + an error. + tags: [] + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + msg: + type: string + required: + - msg + example: + msg: Mem ID9612-en-uk was flushed to the disk successfully + headers: {} + security: [] + /t5memory_service/tagreplacement: + post: + summary: Tag replacement + deprecated: false + description: Fuzzy search tag replacement and Import tag replacement test + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + src: + type: string + trg: + type: string + req: + type: string + description: Used to test Fuzzy search tag replacement + required: + - src + - trg + example: + src: >- + Tap View o get strongdisplayedView two strongUS patents. + trg: >- + View tap to got strongdosplaydVeiw two strongUS patents. + req: >- + Tap View o get strongdisplayedView two strongUS + patents. + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + headers: {} + security: [] + /t5memory_service/flags: + get: + summary: Flags + deprecated: false + description: >- + Return all available commandline flags. Do not spam too much because + gflags documentation says that that's slow. Useful to collect + configuration data about t5memory to do debugging. + tags: [] + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: "{\n {\n \"name\": \"alsologtoemail\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to these email addresses in addition to logfiles\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"alsologtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr in addition to logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"colorlogtostderr\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"color messages logged to stderr (if supported by terminal)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"drop_log_memory\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Drop in-memory buffers of log contents. Logs can grow very quickly and they are rarely read before they need to be evicted from memory. Instead, drop them from memory as soon as they are flushed to disk.\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"log_backtrace_at\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Emit a backtrace when logging at file:linenum.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_dir\",\n \"value\": \"/root/.t5memory/LOG/\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"If specified, logfiles are written into this directory instead of the default logging directory.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_link\",\n \"value\": \"\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Put additional links to the log files in this directory\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"log_prefix\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Prepend the log prefix to the start of each log line\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"logbuflevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; ...)\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"logbufsecs\",\n \"value\": \"30\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Buffer log messages for at most this many seconds\",\n \"type\": \"int32\",\n \"default\": \"30\"\n },\n {\n \"name\": \"logemaillevel\",\n \"value\": \"999\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Email log messages logged at this level or higher (0 means email all; 3 means email FATAL only; ...)\",\n \"type\": \"int32\",\n \"default\": \"999\"\n },\n {\n \"name\": \"logfile_mode\",\n \"value\": \"436\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Log file mode/permissions.\",\n \"type\": \"int32\",\n \"default\": \"436\"\n },\n {\n \"name\": \"logmailer\",\n \"value\": \"/bin/mail\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Mailer used to send logging email\",\n \"type\": \"string\",\n \"default\": \"/bin/mail\"\n },\n {\n \"name\": \"logtostderr\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages go to stderr instead of logfiles\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"max_log_size\",\n \"value\": \"1800\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"approx. maximum log file size (in MB). A value of 0 will be silently overridden to 1.\",\n \"type\": \"int32\",\n \"default\": \"1800\"\n },\n {\n \"name\": \"minloglevel\",\n \"value\": \"0\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Messages logged at a lower level than this don't actually get logged anywhere\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"stderrthreshold\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"log messages at or above this level are copied to stderr in addition to logfiles. This flag obsoletes --alsologtostderr.\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"stop_logging_if_full_disk\",\n \"value\": \"false\"\t\",\tfrom\": \"./src/logging.cc\",\n \"description\": \"Stop attempting to log to disk if the disk is full.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"symbolize_stacktrace\",\n \"value\": \"true\"\t\",\tfrom\": \"./src/utilities.cc\",\n \"description\": \"Symbolize the stack trace in the tombstone\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"v\",\n \"value\": \"2\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"Show all VLOG(m) messages for m <= this. Overridable by --vmodule.\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"vmodule\",\n \"value\": \"\"\t\",\tfrom\": \"./src/vlog_is_on.cc\",\n \"description\": \"per-module verbose level. Argument is a comma-separated list of =. is a glob pattern, matched against the filename base (that is, name ignoring .cc/.h./-inl.h). overrides any value given by --v.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"flagfile\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"load flags from file\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"fromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment [use 'export FLAGS_flag1=value']\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tryfromenv\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"set flags from the environment if present\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"undefok\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags.cc\",\n \"description\": \"comma-separated list of flag names that it is okay to specify on the command line even if the program does not define a flag with that name. IMPORTANT: flags in this list that have arguments MUST use the flag=value format\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"tab_completion_columns\",\n \"value\": \"80\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"Number of columns to use in output for tab completion\",\n \"type\": \"int32\",\n \"default\": \"80\"\n },\n {\n \"name\": \"tab_completion_word\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_completions.cc\",\n \"description\": \"If non-empty, HandleCommandLineCompletions() will hijack the process and attempt to do bash-style command line flag completion on this value.\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"help\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags [tip: all flags can have two dashes]\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpfull\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all flags -- same as -help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpmatch\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on modules whose name contains the specified substr\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helpon\",\n \"value\": \"\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on the modules named by this flag value\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"helppackage\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on all modules in the main package\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpshort\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show help on only the main module for this program\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"helpxml\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"produce an xml version of help\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"version\",\n \"value\": \"false\"\t\",\tfrom\": \"/build/gflags-MeLouv/gflags-2.2.2/src/gflags_reporting.cc\",\n \"description\": \"show version and build info and exit\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"folly_memory_idler_madvise_stacks\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler madvises dontneed stacks on thread idle\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_memory_idler_purge_arenas\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/detail/MemoryIdler.cpp\",\n \"description\": \"if enabled, folly memory-idler purges jemalloc arenas on thread idle\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"dynamic_iothreadpoolexecutor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will dynamically create threads\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"folly_iothreadpoolexecutor_max_read_at_once\",\n \"value\": \"-1\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/IOThreadPoolExecutor.cpp\",\n \"description\": \"IOThreadPoolExecutor will use this value as default for maxReadAtOnce in its event bases, valid values are [0, inf)\",\n \"type\": \"int32\",\n \"default\": \"-1\"\n },\n {\n \"name\": \"threadtimeout_ms\",\n \"value\": \"60000\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/executors/ThreadPoolExecutor.cpp\",\n \"description\": \"Idle time before ThreadPoolExecutor threads are joined\",\n \"type\": \"int64\",\n \"default\": \"60000\"\n },\n {\n \"name\": \"observer_manager_pool_size\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/observer/detail/ObserverManager.cpp\",\n \"description\": \"How many internal threads ObserverManager should use\",\n \"type\": \"int32\",\n \"default\": \"4\"\n },\n {\n \"name\": \"folly_hazptr_use_executor\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/folly/folly/synchronization/Hazptr.cpp\",\n \"description\": \"Use an executor for hazptr asynchronous reclamation\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"dcache_unit_test\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/libs/proxygen/_build/deps/wangle/wangle/ssl/SSLSessionCacheManager.cpp\",\n \"description\": \"All VIPs share one session cache\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"zlib_compressor_buffer_growth\",\n \"value\": \"2024\"\t\",\tfrom\": \"/home/libs/proxygen/lib/utils/ZlibStreamCompressor.cpp\",\n \"description\": \"The buffer growth size to use during IOBuf zlib deflation\",\n \"type\": \"int64\",\n \"default\": \"2024\"\n },\n {\n \"name\": \"request_number\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/RestAPI/ProxygenHandler.cpp\",\n \"description\": \"Include request sequence number in response\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"add_tokens_to_fuzzy\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, list of tokens would be returned in fuzzy responce. Could make execution a bit slower, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowLoadingMultipleTmsSimultaneously\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, multiple tms could be loaded from the disk at the same time. Loading multiple TMs at the same time was disabled because of I/0 disk operation limit at the server. Enable this if you don't care about num of IO operations and want some perfomance boost\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"allowedram\",\n \"value\": \"10000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought RAM(in MB) allowed for service to use\",\n \"type\": \"int64\",\n \"default\": \"5000\"\n },\n {\n \"name\": \"allowedtmdsize\",\n \"value\": \"190\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets max size of tmd file(in MB) after which t5m would not allow to add new data to the tm. This helps fight btree and lookup table issues when tm gets bigger.\",\n \"type\": \"int64\",\n \"default\": \"190\"\n },\n {\n \"name\": \"debug_sleep_in_request_run\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set, provide artificial delay in every request handling execution equal to provided num of microseconds\",\n \"type\": \"int64\",\n \"default\": \"0\"\n },\n {\n \"name\": \"disable_aslr\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, process personality would be set to ADDR_NO_RANDOMIZE, affect memory layout, set to true if you have some linker or asan issue at the launch\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"enable_newlines_in_logs\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines,\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_at_shutdown\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm when shutting down the app not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"flush_tm_to_disk_with_every_update\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, flushes tm to disk with every successfull update request\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"forbiddeletefiles\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Set to true to keep all files(including temporary and tm)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"keep_tm_backups\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving tmd and tmi files, old copies would be saved with .old suffix\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"limit_num_of_active_requests\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, it would be possible to handle only up to servicethreads-1 requests at the same time, the last thread would respond with 503 to eliminate creating queue of requests waiting to be handled.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"logMutexes\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true you would see mutex logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_end\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request end with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_every_request_start\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets log for every request call with it's url, method etc...\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_file_locks\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, action about locking tmd and tmi file would be logged\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_hashes_in_hash_sentence\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, tokens and their hashed would be logged, could be usefull if you think that fuzzy match is wrong to check how tokens was parsed in sentence and what hashes they had(or track hash collisions)\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_memmove_in_compareputdata\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, when saving segment and causing memmove in compareputdata functions, just before memmove, data would be logged - use this to debug btree crashes. In that spot in come corrupted data, memmove could crash t5memory(or cause asan crash). Just to set low log level(v=2,t5logleve=0), reproduce the crash and check if the last log was one that started with \"memmove size = ...\"\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"log_tm_lifetime\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \" If set to true, and --v=2 and --t5loglevel=4, TMs ctor(when tm object was created-so before it started loading tm) and dctor(when tm object was destroyed - so after closing tm) would have transaction level logs\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"maxBadSegmentsIdsSaved\",\n \"value\": \"20\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Defines how many segments(it's id's) that failed during import or reorganize would be saved to the status response\",\n \"type\": \"int32\",\n \"default\": \"20\"\n },\n {\n \"name\": \"newEntryIfOnlyContextAndTimestampDifferent\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new).\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"port\",\n \"value\": \"4040\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"What port to listen on\",\n \"type\": \"int32\",\n \"default\": \"4080\"\n },\n {\n \"name\": \"saveDifferentTargetsForSameSource\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer.\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"servicename\",\n \"value\": \"t5memory\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets service name to use in url\",\n \"type\": \"string\",\n \"default\": \"t5memory\"\n },\n {\n \"name\": \"servicethreads\",\n \"value\": \"10\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets amought of worker threads for service\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"skipStartupLogs\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, values from --t5loglevel would be applied immediately, if set to false- you would see all startup logs and --v and --t5loglevel would be applied only after start of the service and it's init message. Set to false if you have issues at startup\",\n \"type\": \"bool\",\n \"default\": \"true\"\n },\n {\n \"name\": \"skip_default_flags_in_init_msg\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"if set to true, all default flags would be skipped in init msg\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"suppressTmVersionCheck\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"t5_ip\",\n \"value\": \"\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Which ip to use in t5memory(default is any). Should be in format '1.1.1.1', default is to listen to all available ip\",\n \"type\": \"string\",\n \"default\": \"\"\n },\n {\n \"name\": \"t5globversion_max_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5globversion_min_supported\",\n \"value\": \"0\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"0\"\n },\n {\n \"name\": \"t5loglevel\",\n \"value\": \"4\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets t5memory log level threshold from DEVELOP(0) to TRANSACTION(6)\",\n \"type\": \"int32\",\n \"default\": \"2\"\n },\n {\n \"name\": \"t5majversion_max_supported\",\n \"value\": \"7\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"set max major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"7\"\n },\n {\n \"name\": \"t5majversion_min_supported\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"t5minversion_max_supported\",\n \"value\": \"14\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"(disabled)sets max minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z; by default it's current version\",\n \"type\": \"int32\",\n \"default\": \"14\"\n },\n {\n \"name\": \"t5minversion_min_supported\",\n \"value\": \"60\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"sets minimal minor version in which tm was created to be supported for open in current. it's the Z(MIN) in vX.Y.Z\",\n \"type\": \"int32\",\n \"default\": \"60\"\n },\n {\n \"name\": \"timeout\",\n \"value\": \"180000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets timeout for service request handling\",\n \"type\": \"int32\",\n \"default\": \"180000\"\n },\n {\n \"name\": \"tmListLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM list(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for TM after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"tmMaxIdleTimeSec\",\n \"value\": \"7200\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set some value, except, 0, during request, tm list would bee cleaned up from tms that are longer inactive than this number\",\n \"type\": \"int64\",\n \"default\": \"7200\"\n },\n {\n \"name\": \"tmRequestLockDefaultTimeout\",\n \"value\": \"3000\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets tm mutex lock timeout(in ms) for part where request is requesting tm(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout\",\n \"type\": \"int64\",\n \"default\": \"3000\"\n },\n {\n \"name\": \"triplesthreshold\",\n \"value\": \"5\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"Sets threshold to pre fuzzy filtering based on hashes of neibour tokens, range[0...100]. It's pre fuzzy filtering of the segments, when t5memory tries to predict how high fuzzy rate would be for some segment based on data that doesn't require as mych work as fuzzy calculations. When value is low, it can select more suggestions for fuzzy calculation, but if it set high, some best matches could be skipped. In opentm2 default value was 33, but that value seems like not working well with segments, where there are a lot of tags.\",\n \"type\": \"int32\",\n \"default\": \"5\"\n },\n {\n \"name\": \"useTimedMutexesForReorganizeAndImport\",\n \"value\": \"false\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, in reorganize or import thread would be used mutexes with timeouts, and reorganizee or import could be canceled, false(by default) - would be used non timed mutexes\",\n \"type\": \"bool\",\n \"default\": \"false\"\n },\n {\n \"name\": \"wait_for_import_and_reorganize_requests\",\n \"value\": \"true\"\t\",\tfrom\": \"/home/translate5-tm-service-source/source/otmd.cpp\",\n \"description\": \"If set to true, waiting for all import and reorganize processes to be done at shutdown when not using shutdown request\",\n \"type\": \"bool\",\n \"default\": \"true\"\n }\n}" + headers: {} + security: [] + /t5memory/{memory}/multifuzzysearch: + post: + summary: Multifuzzy + deprecated: false + description: Sends multiple fuzzysearches in the same tm in a row + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + requests: + type: array + items: + type: object + properties: + sourceLang: + type: string + targetLang: + type: string + source: + type: string + documentName: + type: string + context: + type: string + customId: + type: string + description: If provided will be ased as key in response block + required: + - sourceLang + - targetLang + - source + - documentName + - context + required: + - requests + example: + requests: + - sourceLang: en + targetLang: uk + source: First segment with int + documentName: '' + context: '' + customId: first + - sourceLang: en + targetLang: uk + source: Third segment with + documentName: '' + context: '' + customId: second + responses: + '200': + description: >- + Each response object in responses has a key and value of it contains + response of simple fuzzy search. + + Name of a key comes from "customId" in request. + content: + application/json: + schema: + type: object + properties: + responses: + type: object + properties: + first: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + NumOfFoundProposals: + type: integer + searchedSrc: + type: string + customId: + type: string + forceFuzzy: + type: integer + results: + type: array + items: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + matchType: + type: string + matchRate: + type: integer + fuzzyWords: + type: integer + fuzzyDiffs: + type: integer + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + - matchType + - matchRate + - fuzzyWords + - fuzzyDiffs + required: + - ReturnValue + - ErrorMsg + - NumOfFoundProposals + - searchedSrc + - customId + - forceFuzzy + description: >- + key "first" is an exampe. it was set because request + has "customId": "first" + second: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + NumOfFoundProposals: + type: integer + searchedSrc: + type: string + customId: + type: string + forceFuzzy: + type: integer + results: + type: array + items: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + matchType: + type: string + matchRate: + type: integer + fuzzyWords: + type: integer + fuzzyDiffs: + type: integer + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + - matchType + - matchRate + - fuzzyWords + - fuzzyDiffs + required: + - ReturnValue + - ErrorMsg + - NumOfFoundProposals + - searchedSrc + - customId + - forceFuzzy + description: >- + key "second" is an exampe. it was set because request + has "customId": "second" + required: + - first + - second + required: + - responses + example: + responses: + first: + ReturnValue: 0 + ErrorMsg: '' + NumOfFoundProposals: 2 + searchedSrc: First segment with int + customId: first + forceFuzzy: 0 + results: + - source: First segment with int 10 + target: Перший сегмент з числом 10 + segmentId: 6 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '10:1' + matchType: Fuzzy + matchRate: 80 + fuzzyWords: 5 + fuzzyDiffs: 1 + - source: >- + First segment with int + target: >- + Перший сегмент з числом + segmentId: 2 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '7:1' + matchType: Fuzzy + matchRate: 80 + fuzzyWords: 5 + fuzzyDiffs: 1 + second: + ReturnValue: 0 + ErrorMsg: '' + NumOfFoundProposals: 2 + searchedSrc: Third segment with + customId: second + forceFuzzy: 0 + results: + - source: Third segment with int 10 and float 1.100 + target: >- + третій сегмент з числом 10 і 1.100, друге захищене як + ціле з точкою + segmentId: 8 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '11:1' + matchType: Fuzzy + matchRate: 37 + fuzzyWords: 8 + fuzzyDiffs: 5 + - source: >- + Third segment with int and float + target: >- + третій сегмент з числом і , друге захищене як ціле з точкою + segmentId: 4 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '8:1' + matchType: Fuzzy + matchRate: 37 + fuzzyWords: 8 + fuzzyDiffs: 5 + headers: {} + security: [] + /t5memory/{memory}/multiupdate: + post: + summary: Multi Update + deprecated: false + description: Send multiple updateEntry requests to the same tm with one call + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + requests: + type: array + items: + type: object + properties: + customId: + type: string + description: >- + If provided will be used as field name for result in + response + source: + type: string + target: + type: string + type: + type: string + markupTable: + type: string + targetLang: + type: string + sourceLang: + type: string + documentName: + type: string + author: + type: string + context: + type: string + save2disk: + type: string + enum: + - '1' + - '0' + description: >- + If 1 provided will save new segmnet to disk right + away, else - only in RAM until flush happens + required: + - source + - target + - targetLang + - sourceLang + - documentName + - author + - context + required: + - requests + example: + requests: + - customId: fourth + source: Fourth segmnet + target: Четвертий сегмент + type: Manual + markupTable: OTMXUXLF + targetLang: en + sourceLang: uk + documentName: test.xlsx.sdlxliff + author: Project Manager + context: '390' + - customId: fifth + source: Fifth segment + target: Пʼятий сегмент + type: Manual + markupTable: OTMXUXLF + targetLang: en + sourceLang: uk + documentName: translation.docx + author: Editor + context: '200' + responses: + '200': + description: >- + Each response object in responses has a key and value of it contains + response of simple update. + + Name of a key comes from "customId" in request. + content: + application/json: + schema: + type: object + properties: + responses: + type: object + properties: + fourth: + type: object + properties: + source: + type: string + sourceNPRepl: + type: string + sourceNorm: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - sourceNPRepl + - sourceNorm + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + description: Key "fourth" comes from request "customId" + fifth: + type: object + properties: + source: + type: string + sourceNPRepl: + type: string + sourceNorm: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - sourceNPRepl + - sourceNorm + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + required: + - fourth + - fifth + save2disk: + type: string + enum: + - '1' + - '0' + description: Represents if new segmnets are saved to disk right away + required: + - responses + - save2disk + example: + responses: + fourth: + source: Fourth segmnet + sourceNPRepl: Fourth segmnet + sourceNorm: Fourth segmnet + target: Четвертий сегмент + segmentId: 9 + customId: fourth + documentName: test.xlsx.sdlxliff + sourceLang: UK + targetLang: EN + type: Manual + author: PROJECT MANAGER + timestamp: 20250812T125637Z + markupTable: OTMXUXLF + context: '390' + additionalInfo: '' + internalKey: '12:1' + fifth: + source: Fifth segment + sourceNPRepl: Fifth segment + sourceNorm: Fifth segment + target: Пʼятий сегмент + segmentId: 10 + customId: fifth + documentName: translation.docx + sourceLang: UK + targetLang: EN + type: Manual + author: EDITOR + timestamp: 20250812T125637Z + markupTable: OTMXUXLF + context: '200' + additionalInfo: '' + internalKey: '13:1' + save2disk: '0' + headers: {} + security: [] + /t5memory/{memory}/tables: + get: + summary: Tables + deprecated: false + description: >- + Get author, lang, langGroup(virtual- created in runtime and not saved on + the disk), tagtable(deprecated - only one record), document(long + filename, long filename caseignore-virtual, short filename- part + deprecated) + + + Tables are provided in format - name(internal key): "position) + idInTable:value;" + + + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Tables + + When adding new entry, for new Id t5memory uses basicaly size of the + table before adding+1, so Id should start from id=1. + + When doing fuzzy search, some field could also be saved to the table, to + generate then id(or find matching) to compare with saved data. + + There are 2 types of entries- older, fixed size, and long filename. For + older, every entry has the same size, 39 bytes+endOfLine. You can have + up to 389 entries saved in that way in each of the tables. Longname + table manage size dynamically, so it's not clear what's the maximum + size, but for filenames there are duplicates in old style docname table + and in longname(which are used only for this type of data). For filename + search, first used longname table, then generated on tm loading + caseignore longname table, then short name table. Usually entries in + tables are caseignore. + + + In the record t5memory saves only id to the table. Maybe it make sense + to implement endpoint to edit that table, if something is missing or + corrupted. Exception for saving id are context and additional info, some + data, that could be saved as int(segId, time), and source and target. + + To clear or fix tag tables, you need to do reorganize, but if some entry + is missing from the table, you would not restore it on export or segment + exctraction via id. So tables could be used to check health of the tm. + And for some ways for concordance search or some other checks. + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: false + example: application/json + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: "{\n \"tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)\": \"position)idInTable: value;\",\n \"tagTablesTable(4)\": \"0)0: \\\"\\\"; \",\n \"langTable(1)\": \"0)2: \\\"EN\\\"; 1)1: \\\"UK\\\"; 2)0: \\\"\\\"; \",\n \"langGroupTable(virtual)(8)\":\"0)1: \\\"E\\\"; 1)2: \\\"U\\\"; 2)0: \\\"\0\\\"; \",\n \"authors(3)\": \"0)3: \\\"EDITOR\\\"; 1)1: \\\"MANAGER\\\"; 2)2: \\\"PROJECT MANAGER\\\"; 3)0: \\\"\\\"; \",\n \"filenames(long)(5)\": \"0)1: \\\"none\\\"; 1)1: \\\"none\\\"; 2)1: \\\"none\\\"; 3)2: \\\"test.xlsx.sdlxliff\\\"; 4)5: \\\"test.xlsx.sdlxliff\\\"; 5)6: \\\"translation.docx\\\"; 6)0: \\\"\\\"; \",\n \"filenames(long, caseign, virtual)(5)\": \"0)1: \\\"NONE\\\"; 1)1: \\\"NONE\\\"; 2)1: \\\"NONE\\\"; 3)5: \\\"TEST.XLSX.SDLXLIFF\\\"; 4)6: \\\"TRANSLATION.DOCX\\\"; 5)2: \\\"TRANSLATION.TXT\\\"; 6)0: \\\"\\\"; \",\n \"filenames(short)(2)\": \"0)1: \\\"none\\\"; 1)5: \\\"test.xlsx.sdlxliff\\\"; 2)6: \\\"translation.docx\\\"; 3)0: \\\"\\\"; \"\n}" + headers: {} + security: [] + /t5memory/{memory}/addtotable: + post: + summary: Add to table + deprecated: false + description: >- + You can add new entry to the tables but then it's better to reorganize + tm(in case if after adding new entry with missing id and there are other + entry with the same value but other id - which could be the case- in + binary search that could cause problems - so just reorganize it + afterwards). You can check table by just doing something like full + export(or concordance search for full tm), and check logs for that 2222 + errors. + + + Values shouldn't be longer than 39 bytes(utf8), and 256 for + filename,it's tables limits, for that it also have checks. + + In cases like with those tms, you can also check if x-1 and x+1 id is + present. Usually they are in those bug. + + also you can check tables via tables request + + Also logicaly FILE_KEY=2, but that was only for shortname, which was + also 39 bytes, but real key for long name table is 5. But for requests + and internal handling t5memory uses 2. + + Id should be between 1 and 65k, request also checks for that + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + value: + type: string + entryId: + type: integer + tableKey: + type: integer + required: + - value + - entryId + - tableKey + example: + value: MY new entry + entryId: 100 + tableKey: 2 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + examples: + '1': + summary: Invalid table key + value: + ReturnValue: 400 + ErrorMsg: >- + Invalid table key(0); available keys = LANG_KEY(1), + FILE_KEY(2), AUTHOR_KEY(3) + '2': + summary: Table already have entry with matching id + value: + ReturnValue: 400 + ErrorMsg: >- + Id match in Table 2 already have entry with matching id = + 100 and value = "Teaser Page.xlsx"; + '3': + summary: Value field is empty + value: + ReturnValue: 400 + ErrorMsg: Value field is empty + headers: {} + security: [] + /t5memory/: + post: + summary: Create TM + deprecated: false + description: '' + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + name: + type: string + description: New TM name + sourceLang: + type: string + description: rfc5646 + data: + type: string + deprecated: true + description: |- + Binary TM file encode with base64. + Can be passed to create TM with data right away. + Use with cautios! + required: + - name + - sourceLang + examples: + '1': + value: + name: examle_tm2 + sourceLang: de-DE + summary: Create TM + '2': + value: + name: test-memory + sourceLang: en-GB + data: >- + UEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLk1FTQEAAADtzssKQEAUgOFjoTwGWR+5lPW4TCjDxgtomqI0I3lzLyBvIEUW59v829+GrinTJMaK99iqWY5Yq2VFMcpp1sotjN43swSCC/gSzw551bHu7MGLlMYqh6d8xrpBhBFjPnhACCGE/MMJUEsHCKEIpyRnAAAAAAAAAAAIAAAAAAAAUEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLlRNRAEAAADt3bFKI0EYB/BvNzFaXCXH1Vdcc3h7xR14xXEsd5pTwRgIEWwlBBRCFPEFbKx9BJ/DJ7CyshYrCwsbX0A3GtykFQnE329ZZnb+u92wMMXMt9FupBFxlERxRVTi+mPwCrNjLe9b+nw/qcRqq77UbC0HwCSdR/3vfafozSVry4s/f2Qr9Wa23t3tbGer3d5+1tju7Oz2u5+X9vqHB3u97+3Gm/24uv1s5V8AAAAAAAAAAAAAAAAAABHzg13/hVqcRSuPOM7P8sv8NG/lvTwAAAAAAAAAAAAAAAAAAABgasxFEjOD3rAO+EwpqQ56Jx+eRmqlpBLlj5JSUhtJqqUkGUnSUhIjQ5UYG4jiLIA7O/4BAAAAAAAAAAAAAAAAAABgAqrPJwHMx2zcfvryNdKbP1e/Iy5+LS58W3h8BgAAAAAAAAAAAJgO6Wg7rK7UbDe2NrfW/5dfTYJp9VI7a1i/CwAAYGLGVqq8U+YBBfOAgnnAowdQSwcITWiheHEBAAAAAAAAAAABAAAAAABQSwMEFAAICAgAUmZ4VgAAAAAAAAAAAAAAACgABABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAO3dwWkCYRAG0NFTzlaQBv4cEsg1JLqokI0gNiDLQgRZQ7ABu7MaLxawKijqWWQPvjcMM1PEx/xM8nZErCJa++J2dV1fTR5b+9inazDOuqNxLwCatI7sc1sctqfWsPf+9pr62Sh9l7Nimgbl/C/l0+J3VpXP3UW1/F/MXyb5MO6krFL/KwAAAAAAAAAAAAAAAACAiM5F6r8Tm48AAAAAAAAAAAAAAAAAAAAAGnD+TS7/DwAAAAAAAAAAAAAAAAAAAM3YAVBLBwgGVIGewwAAAAAAAAAAwAAAAAAAAFBLAQIAABQACAgIALRlVFahCKckZwAAAAAIAAAoAAQAAAAAAAAAAACkgQAAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuTUVNAQAAAFBLAQIAABQACAgIAGdralZNaKF4cQEAAAAAAQAoAAQAAAAAAAAAAACkgckAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1EAQAAAFBLAQIAABQACAgIAFJmeFYGVIGewwAAAADAAAAoAAQAAAAAAAAAAACkgZwCAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAFBLBgYsAAAAAAAAAB4DLQAAAAAAAAAAAAMAAAAAAAAAAwAAAAAAAAAOAQAAAAAAAMEDAAAAAAAAUEsGBwAAAADPBAAAAAAAAAEAAABQSwUGAAAAAAMAAwAOAQAAwQMAAAAA + summary: Create with file for TM + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + name: + type: string + description: Created TM name + required: + - name + example: + name: examle_tm2 + headers: {} + x-200:Error: + description: '' + content: + application/json: + schema: + title: '' + type: object + properties: + ReturnValue: + type: integer + description: Error code + ErrorMsg: + type: string + description: Message to explain error + required: + - ReturnValue + - ErrorMsg + example: + ReturnValue: 7272 + ErrorMsg: >- + ::ERROR_MEM_NAME_EXISTS:: TM with this name already exists: + test-memory; res = 0 + headers: {} + security: [] + get: + summary: Get TMs list + deprecated: false + description: '' + tags: [] + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + Open: + type: array + items: + type: object + properties: + name: + type: string + description: TM name + required: + - name + description: Memories loaded into RAM + Available on disk: + type: array + items: + type: object + properties: + name: + type: string + description: TM name + required: + - name + description: Memories available on disk + required: + - Open + - Available on disk + example: + Open: + - name: test-memory + Available on disk: + - name: test-memory-1 + headers: {} + security: [] + /t5memory_service/resources: + get: + summary: Resources + deprecated: false + description: '' + tags: [] + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: + filebuffers: + - name: /root/.t5memory/TABLE/OTMXUXLF.TBL + size: 3660 + totalOccupiedByFilebuffersRAM: 3660 + tms: + - name: test-memory + status: open + size: 134976 + activeRequest: '' + expectedSize: 0 + StatusInfo: + status: open + sizeInRAM: 134976 + activeRequest: '' + tmxImportStatus: available + importProgress: 100 + importTime: | + Overall import time is : 0:00:00 + segmentsImported: 2 + invalidSegments: 0 + tmxSegmentCount: 2 + importRuntimeSec: 0 + importTimeoutSec: 0 + newEntryIfOnlyContextAndTimestampDifferent: 1 + saveDifferentTargetsForSameSource: 0 + invalidSegmentsRCs: '' + firstInvalidSegments(num:segId:errCode): '' + invalidSymbolErrors: 0 + errorMsg: '' + rc: 0 + lastAccessTime: 20250801T153723Z + creationTime: 20250801T123723Z + tmCreatedInT5M_version: '0:7:14' + segmentIndex: 2 + sourceLang: de + internalDescription: '' + tmd fbuffer was modified: 1 + tmi fbuffer was modified: 1 + totalOccupiedByTMsInRAM: 712165696 + Run date: 'Wed Jul 30 16:08:09 2025 ' + Build date: 2025/07/29 16:08:46 + Git commit info: >- + dff40cb9 * updated to v0.7.14 + added skipStartupLogs + flags(default true) * fixed non-asan builds * fixed + nextInternalKey is empty log on failed export + Version: 0.7.14 + Worker threads: 10 + Timeout(ms): 180000 + Resident set: 1448300000000 + Virtual memory usage: 3743340000000000 + Requests: + RequestCount: 6829 + RequestExecutionSumTime(sec): 14.484 + CreateMem: + ReqCount: 44 + SumTime(sec): 0.065 + AvrgReqTime: 0.00147727 + DeleteMem: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + ImportMem: + ReqCount: 29 + SumTime(sec): 0.016 + AvrgReqTime: 0.000551724 + ExportMem: + ReqCount: 13 + SumTime(sec): 0.09 + AvrgReqTime: 0.00692308 + CloneTmLocaly: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + Reorganize: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + StatusMem: + ReqCount: 109 + SumTime(sec): 0.011 + AvrgReqTime: 0.000100917 + FlushMem: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + Fuzzy: + ReqCount: 98 + SumTime(sec): 1.034 + AvrgReqTime: 0.010551 + MultiFuzzy: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + Concordance: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + UpdateEntry: + ReqCount: 6 + SumTime(sec): 0.013 + AvrgReqTime: 0.00216667 + MultiUpdate: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + GetEntry: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + DeleteEntry: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + SaveAllTms: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + ListOfMemories: + ReqCount: 6474 + SumTime(sec): 13.221 + AvrgReqTime: 0.00204217 + Resources: + ReqCount: 22 + SumTime(sec): 0.016 + AvrgReqTime: 0.000727273 + Flags: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + Other: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + Unrecognized: + ReqCount: 0 + SumTime(sec): 0 + AvrgReqTime: 0 + RAM limit(MB): 10000 + headers: {} + security: [] + /t5memory/{memory}/status: + get: + summary: TM Status + deprecated: false + description: >- + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-GetthestatusofTM + tags: [] + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + status: + type: string + description: |- + Statuses: + open - memory already in RAM ready to use + available - not loaded but present on disk + loading - loading from disk to RAM + not found - error status of absent memory + reorganize running - Reorganizing in process + import running - Import in process + waiting for loading - loading requsted but not done yet + failed to open - corrupted TM file + sizeInRAM: + type: integer + activeRequest: + type: string + description: ' ' + lastAccessTime: + type: string + creationTime: + type: string + tmCreatedInT5M_version: + type: string + description: version of t5memory on momemt of TM creation + segmentIndex: + type: integer + description: index of last inserted segment (not queriable) + sourceLang: + type: string + description: rfc5646 + internalDescription: + type: string + tmd fbuffer was modified: + type: integer + tmi fbuffer was modified: + type: integer + required: + - status + - sizeInRAM + - activeRequest + - lastAccessTime + - creationTime + - tmCreatedInT5M_version + - segmentIndex + - sourceLang + - internalDescription + - tmd fbuffer was modified + - tmi fbuffer was modified + example: + status: open + sizeInRAM: 134976 + activeRequest: '' + lastAccessTime: 20250801T160913Z + creationTime: 20250716T102923Z + tmCreatedInT5M_version: '0:7:7' + segmentIndex: 5 + sourceLang: en + internalDescription: '' + tmd fbuffer was modified: 1 + tmi fbuffer was modified: 1 + headers: {} + x-200:Import processed status: + description: '' + content: + application/json: + schema: + type: object + properties: + status: + type: string + sizeInRAM: + type: integer + activeRequest: + type: string + tmxImportStatus: + type: string + description: >- + Statuses of ongoing improt: + + available - not tipical. If returned and "importTime" == + "not finished" then import still ongoing + + import - ongoing import + importProgress: + type: integer + description: Progress in persents + importTime: + type: string + segmentsImported: + type: integer + description: NUmber of successfully imported segments + invalidSegments: + type: integer + description: Number of invalid segments found in TMX + tmxSegmentCount: + type: integer + description: Number of found segmnets in import TMX file + importRuntimeSec: + type: integer + importTimeoutSec: + type: integer + newEntryIfOnlyContextAndTimestampDifferent: + type: integer + saveDifferentTargetsForSameSource: + type: integer + description: 0 or 1 - considered as bool value + invalidSegmentsRCs: + type: string + firstInvalidSegments(num:segId:errCode): + type: string + invalidSymbolErrors: + type: integer + errorMsg: + type: string + rc: + type: integer + lastAccessTime: + type: string + creationTime: + type: string + tmCreatedInT5M_version: + type: string + segmentIndex: + type: integer + sourceLang: + type: string + internalDescription: + type: string + tmd fbuffer was modified: + type: integer + tmi fbuffer was modified: + type: integer + required: + - status + - sizeInRAM + - activeRequest + - tmxImportStatus + - importProgress + - importTime + - segmentsImported + - invalidSegments + - tmxSegmentCount + - importRuntimeSec + - importTimeoutSec + - newEntryIfOnlyContextAndTimestampDifferent + - saveDifferentTargetsForSameSource + - invalidSegmentsRCs + - firstInvalidSegments(num:segId:errCode) + - invalidSymbolErrors + - errorMsg + - rc + - lastAccessTime + - creationTime + - tmCreatedInT5M_version + - segmentIndex + - sourceLang + - internalDescription + - tmd fbuffer was modified + - tmi fbuffer was modified + example: + status: open + sizeInRAM: 134976 + activeRequest: '' + tmxImportStatus: available + importProgress: 100 + importTime: | + Overall import time is : 0:00:00 + segmentsImported: 4 + invalidSegments: 0 + tmxSegmentCount: 4 + importRuntimeSec: 0 + importTimeoutSec: 0 + newEntryIfOnlyContextAndTimestampDifferent: 1 + saveDifferentTargetsForSameSource: 0 + invalidSegmentsRCs: '' + firstInvalidSegments(num:segId:errCode): '' + invalidSymbolErrors: 0 + errorMsg: '' + rc: 0 + lastAccessTime: 20250807T130822Z + creationTime: 20250716T102923Z + tmCreatedInT5M_version: '0:7:7' + segmentIndex: 9 + sourceLang: en + internalDescription: '' + tmd fbuffer was modified: 1 + tmi fbuffer was modified: 1 + headers: {} + security: [] + /t5memory/{memory}/importtmx: + post: + summary: IMPORT TMX + deprecated: false + description: >- + Rout is async. So when import is finished should be checked with status + call + tags: [] + parameters: [] + requestBody: + content: + multipart/form-data: + encoding: + json_data: + contentType: application/json + schema: + type: object + properties: + json_data: + type: string + description: "json pretty print string.\n\n\"framingTags\" key possible values:\n\"saveAll\" - default behaviour, do nothing\n\"skipAll\" - skip all enclosing tags, including standalone tags\n\"skipPaired\" - skip only paired enclosing tags\n\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway.\_\nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, default values from flags would be used." + example: | + { + "framingTags": "saveAll", + "newEntryIfOnlyContextAndTimestampDifferent": 0, + "saveDifferentTargetsForSameSource": 0, + "timeout": 100000 + } + file: + format: binary + type: string + description: tmx file to import into TM + example: '' + required: + - json_data + - file + examples: {} + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: + msg: success + headers: {} + security: [] + /t5memory/{memory}/download.tmx: + get: + summary: EXPORT TMX + deprecated: false + description: >- + JSON pretty print string: + + limit - number of segments to fetch at a time + + startFromInternalKey - internal key to start export from + + + Works as cursor pagination. + + + If no body provided, export starts from the beginning (key 7:1) to the + end. + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/xml + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + limit: + type: string + startFromInternalKey: + type: string + required: + - limit + - startFromInternalKey + example: + limit: '100' + startFromInternalKey: '1:1' + responses: + '200': + description: >- + In response headers "NextInternalKey" will contain + "startFromInternalKey" for next call. + + If "NextInternalKey" == "0:0" - then you reached end of TM + content: + application/xml: + schema: + type: object + properties: {} + example: |- + + + +
+ + + 2 + 7: 1 + OTMXUXLF + none + + First segment + + + Перший сегмент + + + + + headers: {} + security: [] + /t5memory/{memory}/search: + post: + summary: Concordance search + deprecated: false + description: >- + All fields is optional, but some depends on other, so error should be + returned in case of not providing required field. + + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-NewConcordancesearch + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + source: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + sourceSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + target: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + targetSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + document: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + documentSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + author: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + authorSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + addInfo: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + addInfoSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + context: + type: string + description: >- + Sets what to look for in source of the segments, based on + type of search, specified in sourceSearchMode(exact, + concordance). If sourceSearchMode is not specified, returns + an error. + contextSearchMode: + type: string + description: "String with required\_\nEXACT or CONCORDANCE\_\n(or CONTAINS, what's equal to CONCORDANCE)\nwords and some optional, like\nCASEINSENSETIVE for non case sensetive comparison,\nWHITESPACETOLERANT for\nmodifying whitespaces(result of this actions you can see in filters in responce)\nINVERTED\_ for applying filter in inverted state, so to return false on match and true\_\nif no match. Logical NOT\n\nAttributes is not case sensetive,\_\nSeparator doen't matters" + timestampSpanStart: + type: string + description: "String with date in format\n\_\"20240121T115234Z\"" + timestampSpanEnd: + type: string + description: "String with date in format\n\_\"20240121T115234Z\"" + onlyCountSegments: + type: string + description: >- + Instead of returning segment, would go in search till the + end of tm and return total number of segments, that returns + true with selected filters + enum: + - '1' + - '0' + searchPosition: + type: string + description: Point where to start search in tmd file + example: '8:1' + logicalOr: + type: integer + enum: + - 1 + - 0 + description: >- + By default source, target, document, author, context, + addinfo, timestamp is combined in logical AND, but by + sending here "OR" you can switch that to logical OR, any + other value would left it in default AND state. + + Doesn't apply to sourceLang and targetLang filters, they are + always in AND state + numResults: + type: integer + description: Points how many matches return in current request + minimum: 0 + maximum: 200 + default: 5 + sourceLang: + type: string + description: "Filter segments on src/trg lang attribute,\_\n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + targetLang: + type: string + description: "Filter segments on src/trg lang attribute,\_\n\nIf specified lang is preffered, matching is done based on lang family,\notherwise on exact match" + msSearchAfterNumResults: + type: integer + default: 0 + description: "sets how many ms should pass between first found segment and search stop, if it didn't reach the end yet.\_" + loggingThreshold: + type: integer + maximum: 6 + description: >- + HACK. + + Additional field to set log level on the run. + + Sets log level for t5memory process in general and not this + request only + example: + source: First + sourceSearchMode: contains, CASEINSENSETIVE + target: '' + targetSearchMode: contains, CASEINSENSETIVE + document: '' + documentSearchMode: contains, CASEINSENSETIVE + author: '' + authorSearchMode: contains, CASEINSENSETIVE + addInfo: '' + addInfoSearchMode: contains, CASEINSENSETIVE + context: '' + contextSearchMode: contains, CASEINSENSETIVE + timestampSpanStart: 19691231T230000Z + timestampSpanEnd: 20260628T220000Z + onlyCountSegments: '0' + searchPosition: '' + logicalOr: 0 + numResults: 200 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + Filters: + type: string + GlobalSearchOptions: + type: string + ReturnValue: + type: integer + ReturnMessage: + type: string + NewSearchPosition: + type: 'null' + NumOfFoundSegments: + type: integer + results: + type: array + items: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + required: + - Filters + - GlobalSearchOptions + - ReturnValue + - ReturnMessage + - NewSearchPosition + - NumOfFoundSegments + - results + example: + Filters: > + Search filter, field: SOURCE FilterType::CONTAINS SearchStr: + 'FIRST'; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: TARGET FilterType::CONTAINS SearchStr: + ''; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: ADDINFO FilterType::CONTAINS SearchStr: + ''; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: CONTEXT FilterType::CONTAINS SearchStr: + ''; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: AUTHOR FilterType::CONTAINS SearchStr: + ''; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: DOCUMENT FilterType::CONTAINS SearchStr: + ''; Options: SEARCH_CASEINSENSITIVE_OPT|; + + Search filter, field: TIMESTAMP FilterType::RANGE Range: + 19691231T230000Z - 20260628T220000Z Options: ; + GlobalSearchOptions: SEARCH_FILTERS_LOGICAL_AND + ReturnValue: 10010 + ReturnMessage: ENDREACHED_RC + NewSearchPosition: null + NumOfFoundSegments: 2 + results: + - source: >- + First segment with int + target: >- + Перший сегмент з числом + segmentId: 2 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '7:1' + - source: First segment with int 10 + target: Перший сегмент з числом 10 + segmentId: 6 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '11:1' + headers: {} + security: [] + /t5memory/{memory}/concordancesearch: + post: + summary: Concordance search (DEPRICATED) + deprecated: false + description: >2- + SearchPosition / NewSearchPositionFormat: "7:1" + First is segmeng\record number, second is target number + + The NextSearchposition is an internal key of the memory for the next + position on sequential access. Since it is an internal key, maintained + and understood by the underlying memory plug-in (for EqfMemoryPlugin is + it the record number and the position in one record), + + no assumptions should be made regarding the content. It is just a string + that, should be sent back to OpenTM2 on the next request, so that the + search starts from there. + + So is the implementation in Translate5: The first request to OpenTM2 + contains SearchPosition with an empty string, OpenTM2 returns than a + string in NewSearchPosition, which is just resent to OpenTM2 in the next + request. + + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Concordancesearch + + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + searchString: + type: string + searchType: + type: string + enum: + - Source + - Target + - SourceAndTarget + example: Source + searchPosition: + type: 'null' + numResults: + type: integer + msSearchAfterNumResults: + type: integer + required: + - searchString + - searchType + example: + searchString: With nice text + searchType: source + searchPosition: null + numResults: 20 + msSearchAfterNumResults: 250 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + ReturnValue: + type: integer + ReturnMessage: + type: string + NewSearchPosition: + type: string + nullable: true + results: + type: array + items: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + required: + - ReturnValue + - ReturnMessage + - NewSearchPosition + - results + example: + ReturnValue: 10010 + ReturnMessage: ENDREACHED_RC + NewSearchPosition: null + results: + - source: >- + First segment with int + target: >- + Перший сегмент з числом + segmentId: 2 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '7:1' + - source: First segment with int 10 + target: Перший сегмент з числом 10 + segmentId: 6 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '11:1' + headers: {} + security: [] + /t5memory/{memory}/fuzzysearch: + post: + summary: Fuzzy search + deprecated: false + description: >- + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Fuzzysearch + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + sourceLang: + type: string + description: rfc5646 + targetLang: + type: string + description: rfc5646 + source: + type: string + description: Search string to look for + documentName: + type: string + markupTable: + type: string + description: >- + If there is no markup, default OTMXUXLF would be used. + + Markup tables should be located inside + ~/.t5memory/TABLE/%markup$.TBL + context: + type: string + forceFuzzy: + type: integer + enum: + - 1 + - 0 + description: |- + If yes - will return all potential fuzzies + No - only first exact match or most relevant matches + numOfProposals: + type: integer + description: Num of expected segments in output. By default it's 5 + required: + - sourceLang + - targetLang + - source + example: + sourceLang: en + targetLang: uk + source: Second segment + documentName: '' + markupTable: OTMXUXLF + context: '' + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + NumOfFoundProposals: + type: integer + searchedSrc: + type: string + customId: + type: string + forceFuzzy: + type: integer + results: + type: array + items: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + matchType: + type: string + matchRate: + type: integer + fuzzyWords: + type: integer + fuzzyDiffs: + type: integer + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + - matchType + - matchRate + - fuzzyWords + - fuzzyDiffs + required: + - ReturnValue + - ErrorMsg + - NumOfFoundProposals + - searchedSrc + - customId + - forceFuzzy + example: + ReturnValue: 0 + ErrorMsg: '' + NumOfFoundProposals: 2 + searchedSrc: 'First segment with int ' + customId: '' + forceFuzzy: 0 + results: + - source: First segment with int 10 + target: Перший сегмент з числом 10 + segmentId: 6 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '11:1' + matchType: Fuzzy + matchRate: 80 + fuzzyWords: 5 + fuzzyDiffs: 1 + - source: First segment with int 11 + target: Перший сегмент з числом 11 + segmentId: 2 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '7:1' + matchType: Fuzzy + matchRate: 80 + fuzzyWords: 5 + fuzzyDiffs: 1 + headers: {} + security: [] + /t5memory/{memory}/getentry: + post: + summary: Get entry + deprecated: false + description: >- + Each segmnet entry in seach payloads has "internalKey": "9:1" like + key-value. + + In this example 9 is "recordKey" and 1 is "targetKey". + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + recordKey: + type: string + targetKey: + type: string + required: + - recordKey + - targetKey + example: + recordKey: '8' + targetKey: '2' + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + example: + source: Third segment + target: третій сегмент + segmentId: 4 + customId: '' + documentName: none + sourceLang: en + targetLang: UK + type: Manual + author: MANAGER + timestamp: 20250716T084347Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '9:1' + headers: {} + '400': + description: '' + content: + application/json: + schema: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + required: + - ReturnValue + - ErrorMsg + example: + ReturnValue: 939 + ErrorMsg: >- + Requested entry not found! Next internalKey after requested is + : 10:1 + headers: {} + security: [] + /t5memory/{memory}/entry: + post: + summary: Update entry + deprecated: false + description: >- + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Updateentry + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + source: + type: string + description: Segment source string + target: + type: string + description: Segment target string + type: + type: string + const: Manual + description: Should be set to "Manual" + example: Manual + markupTable: + type: string + const: OTMXUXLF + description: Should be "OTMXUXLF" + example: OTMXUXLF + sourceLang: + type: string + description: rfc5646 + targetLang: + type: string + description: rfc5646 + timeStamp: + type: string + pattern: ^\d{8}T\d{6}Z$ + format: yyyyMMdd'T'HHmmss'Z' + description: Time of segment creation + example: 20240111T111443Z + documentName: + type: string + description: Name of document where segmnet comes from + author: + type: string + context: + type: string + description: >- + Field that will be considered in fuzzy searches. Plays a + role of aditional search option + save2disk: + type: integer + description: >- + If true - segment will be written to disk right away, else - + only set in RAM and saved when all memory is flushed + enum: + - 1 + - 0 + default: 0 + example: 1 + saveDifferentTargetsForSameSource: + type: integer + enum: + - 1 + - 0 + description: >- + If param set to 1 - new entry will be saved and old + preserved even if all fileds of segmnet except target are + same. + required: + - source + - target + - sourceLang + - targetLang + - timeStamp + - documentName + - author + - context + - save2disk + example: + source: hello + target: again good bye 1 + type: Manual + markupTable: OTMXUXLF + sourceLang: de + targetLang: en + timeStamp: 20250112T111443Z + documentName: translation.txt + author: Project Manager + context: '' + save2disk: 1 + saveDifferentTargetsForSameSource: 1 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + source: + type: string + sourceNPRepl: + type: string + sourceNorm: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - sourceNPRepl + - sourceNorm + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + example: + source: Second segment + sourceNPRepl: Second segment + sourceNorm: Second segment + target: Другий сегмент + segmentId: 14 + customId: '' + documentName: translation.txt + sourceLang: EN + targetLang: UK + type: Manual + author: PROJECT MANAGER + timestamp: 20240111T111443Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '14:1' + headers: {} + security: [] + /t5memory/{memory}: + delete: + summary: Delete TM + deprecated: false + description: '' + tags: [] + parameters: [] + requestBody: + content: + text/plain: + schema: + type: string + examples: {} + responses: + '200': + description: >- + Response will contain json with memory name as key and "deleted" as + value. + + May return 500 with "not found(error 48)" as value that is simply + indicates absense of TM + content: + application/json: + schema: + type: object + properties: {} + example: + test-tm: deleted + headers: {} + '500': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: + test-tm: not found(error 48) + headers: {} + security: [] + /t5memory/{memory}/entrydelete: + post: + summary: Delete entry + deprecated: false + description: >- + Each segmnet entry in seach payloads has "internalKey": "9:1" like + key-value. + + In this example 9 is "recordKey" and 1 is "targetKey". + + There is also "segmentId" value. + + For delete entry request it should be provided to validate that current + state of t5memory data is same as at the moment when you got internalKey + as internal key value may be regenerated at any moment of data + processing on import or update rrquest handling. + + So if you try to delete entry and getting error - try to fetch entry + internalKey nad segmentId again and then retry delete action. + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + recordKey: + type: integer + targetKey: + type: integer + segmentId: + type: integer + save2disk: + type: string + enum: + - '1' + - '0' + default: '0' + description: |- + Save memory to disk right away. + Saves if not passed. + example: '1' + required: + - recordKey + - targetKey + - segmentId + example: + recordKey: 8 + targetKey: 4 + segmentId: 12 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + fileFlushed: + type: integer + description: >- + Indicates was TM flushed to disk or not. Depends on + "save2disk" value. + results: + type: object + properties: + source: + type: string + target: + type: string + segmentId: + type: integer + customId: + type: string + documentName: + type: string + sourceLang: + type: string + targetLang: + type: string + type: + type: string + author: + type: string + timestamp: + type: string + markupTable: + type: string + context: + type: string + additionalInfo: + type: string + internalKey: + type: string + required: + - source + - target + - segmentId + - customId + - documentName + - sourceLang + - targetLang + - type + - author + - timestamp + - markupTable + - context + - additionalInfo + - internalKey + required: + - fileFlushed + - results + example: + fileFlushed: 1 + results: + source: Second segment + target: Другий сегмент + segmentId: 14 + customId: '' + documentName: translation.txt + sourceLang: en + targetLang: UK + type: Manual + author: PROJECT MANAGER + timestamp: 20250811T101311Z + markupTable: OTMXUXLF + context: '' + additionalInfo: '' + internalKey: '14:1' + headers: {} + '400': + description: '' + content: + application/json: + schema: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + required: + - ReturnValue + - ErrorMsg + example: + ReturnValue: 14002 + ErrorMsg: >- + Error: to delete entry by key you should provide all three + fields: recordKey, targetKey and segmentId + headers: {} + '500': + description: '' + content: + application/json: + schema: + type: object + properties: + ReturnValue: + type: integer + ErrorMsg: + type: string + required: + - ReturnValue + - ErrorMsg + example: + ReturnValue: 5005 + ErrorMsg: '' + headers: {} + security: [] + /t5memory/{memory}/entriesdelete: + post: + summary: Delete batch + deprecated: false + description: >- + Asynchronious operation. + + This would start reorganize process which would remove like reorganize + bad segments and also would remove segments that gives true when + checking with provided filters combined with logical AND. So if you + provide timestamps and addInfo, only segments within provided timestamp + and with that addInfo would not be imported to new TM(check reorganize + process). + + Every parameter is optional, so empty json would just start reorganize + async process. + + If you provide one of timestamps you would get error - please provide + both. + + To add parameter you should set it's SearchMode to be + EXACT|CONCORDANCE(non case sensetive) + + If only searched string provided, but not search mode - you would get + error. + tags: [] + parameters: [] + requestBody: + content: + application/json: + schema: + type: object + properties: + source: + type: string + sourceSearchMode: + type: string + target: + type: string + targetSearchMode: + type: string + document: + type: string + documentSearchMode: + type: string + author: + type: string + authorSearchMode: + type: string + addInfo: + type: string + addInfoSearchMode: + type: string + context: + type: string + contextSearchMode: + type: string + timestampSpanStart: + type: string + timestampSpanEnd: + type: string + searchPosition: + type: string + required: + - timestampSpanStart + - timestampSpanEnd + example: + source: '123' + sourceSearchMode: contains, CASEINSENSETIVE + target: '234' + targetSearchMode: contains, CASEINSENSETIVE + document: '' + documentSearchMode: contains, CASEINSENSETIVE + author: '' + authorSearchMode: contains, CASEINSENSETIVE + addInfo: '' + addInfoSearchMode: contains, CASEINSENSETIVE + context: '' + contextSearchMode: contains, CASEINSENSETIVE + timestampSpanStart: 19691231T230000Z + timestampSpanEnd: 20240610T220000Z + searchPosition: '' + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + headers: {} + security: [] + /t5memory_service/shutdown: + post: + summary: Shutdown + deprecated: false + description: >- + Safely shutting down the service with\without saving all loaded tm files + to the disk. + tags: [] + parameters: + - name: dontsave + in: query + description: >- + Skips saving tms, for now value doesn't matter, only presence. + + If try to save tms before closing, would check if there is still + import process going on + + If there is some, would wait 1 second and check again. + + Repeats last step up to 10 min, then closes service anyway. + required: false + schema: + type: integer + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + headers: {} + security: [] + /t5memory_service/savetms: + get: + summary: Save TMs + deprecated: false + description: >- + Flushes all filebuffers(TMD, TMI files) into the filesystem. Reset + 'Modified' flags for file buffers. + + + Filebuffer is a file instance of .TMD or .TMI loaded into RAM. It + provides better speed and safety when working with files. + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + example: + saved 3 tms: ID9580-de-fr_next-2, ID9612-en-uk, test-memory + errMsg: '' + headers: {} + security: [] + /t5memory/{memory}/clone: + post: + summary: Clone + deprecated: false + description: Creates TM with the provided name and data cloned from provided memory + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + newName: + type: string + description: Name of memory to be crreated + required: + - newName + example: + newName: clonned-tm-1 + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: + msg: + type: string + time: + type: string + required: + - msg + - time + example: + msg: clonned-tm-1 was cloned successfully + time: 1 ms + headers: {} + security: [] + /t5memory/{memory}/reorganize: + get: + summary: Reorganize + deprecated: false + description: >- + Under the hood it creates new tm with $Org- prefix, then reimport all + segments one-by-one, and then deletes original TM and rename reorganized + TM to replace original. + + This request should flush tm(from RAM to the disk) before reorganizing + + + https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-ReorganizeTM + + Both newEntryIfOnlyContextAndTimestampDifferent and + saveDifferentTargetsForSameSource impact how similar translation would + be saved. In case if source and other field(doc, author, lang) are the + same, but target different, and + newEntryIfOnlyContextAndTimestampDifferent is set to true, you would + have 2 translation, otherwise you would have 1 translation saved - with + higher updateTime. but if you have different attributes, like document, + you would have 2 translation anyway. + + newEntryIfOnlyContextAndTimestampDifferent would also increase number of + saved segments. If values was not provided, defaule values from flags + would be used. + tags: [] + parameters: + - name: Accept + in: header + description: '' + required: true + example: application/json; charset=utf-8 + schema: + type: string + - name: Accept-Charset + in: header + description: '' + required: true + example: UTF-8 + schema: + type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + newEntryIfOnlyContextAndTimestampDifferent: + type: string + enum: + - '1' + - '0' + example: '1' + saveDifferentTargetsForSameSource: + type: string + enum: + - '1' + - '0' + example: '1' + examples: {} + responses: + '200': + description: '' + content: + application/json: + schema: + type: object + properties: {} + headers: {} + security: [] +components: + schemas: {} + securitySchemes: {} +servers: [] +security: [] diff --git a/docs/t5memory.postman.json b/docs/t5memory.postman.json new file mode 100644 index 00000000..d3e2f6a8 --- /dev/null +++ b/docs/t5memory.postman.json @@ -0,0 +1,1056 @@ +{ + "info": { + "name": "t5memory", + "description": "", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "Flush memory", + "description": "Endpoint is sync(blocking)\n\nIf tm is not found on the disk - returns 404\nIf tm is not open - returns 400 with message\nThen t5memory requests writes pointer to the tm (so it waits till other requests that's working with the tm would finish) and then it flushes it to the disk\nCould also return an error if flushing got some issue.\nWould not open the tm, if it's not opened yet, but instead would return an error.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/flush", + "path": [ + "t5memory", + "{{memory}}", + "flush" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Tag replacement", + "description": "Fuzzy search tag replacement and Import tag replacement test", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"src\": \"Tap View o get strongdisplayedView two strongUS patents.\",\n \"trg\": \"View tap to got strongdosplaydVeiw two strongUS patents.\",\n \"req\": \"Tap View o get strongdisplayedView two strongUS patents.\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory_service/tagreplacement", + "path": [ + "t5memory_service", + "tagreplacement" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Flags", + "description": "Return all available commandline flags. Do not spam too much because gflags documentation says that that's slow. Useful to collect configuration data about t5memory to do debugging.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory_service/flags", + "path": [ + "t5memory_service", + "flags" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Multifuzzy", + "description": "Sends multiple fuzzysearches in the same tm in a row", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"requests\": [\n {\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"First segment with int\",\n \"documentName\": \"\",\n \"context\": \"\",\n \"customId\": \"first\"\n },\n {\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"Third segment with\",\n \"documentName\": \"\",\n \"context\": \"\",\n \"customId\": \"second\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/multifuzzysearch", + "path": [ + "t5memory", + "{{memory}}", + "multifuzzysearch" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Multi Update", + "description": "Send multiple updateEntry requests to the same tm with one call", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"requests\": [\n {\n \"customId\": \"fourth\",\n \"source\": \"Fourth segmnet\",\n \"target\": \"Четвертий сегмент\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"targetLang\": \"en\",\n \"sourceLang\": \"uk\",\n \"documentName\": \"test.xlsx.sdlxliff\",\n \"author\": \"Project Manager\",\n \"context\": \"390\"\n },\n {\n \"customId\": \"fifth\",\n \"source\": \"Fifth segment\",\n \"target\": \"Пʼятий сегмент\",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"targetLang\": \"en\",\n \"sourceLang\": \"uk\",\n \"documentName\": \"translation.docx\",\n \"author\": \"Editor\",\n \"context\": \"200\"\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/multiupdate", + "path": [ + "t5memory", + "{{memory}}", + "multiupdate" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Tables", + "description": "Get author, lang, langGroup(virtual- created in runtime and not saved on the disk), tagtable(deprecated - only one record), document(long filename, long filename caseignore-virtual, short filename- part deprecated)\n\nTables are provided in format - name(internal key): \"position) idInTable:value;\"\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Tables\nWhen adding new entry, for new Id t5memory uses basicaly size of the table before adding+1, so Id should start from id=1. \nWhen doing fuzzy search, some field could also be saved to the table, to generate then id(or find matching) to compare with saved data.\nThere are 2 types of entries- older, fixed size, and long filename. For older, every entry has the same size, 39 bytes+endOfLine. You can have up to 389 entries saved in that way in each of the tables. Longname table manage size dynamically, so it's not clear what's the maximum size, but for filenames there are duplicates in old style docname table and in longname(which are used only for this type of data). For filename search, first used longname table, then generated on tm loading caseignore longname table, then short name table. Usually entries in tables are caseignore. \n\nIn the record t5memory saves only id to the table. Maybe it make sense to implement endpoint to edit that table, if something is missing or corrupted. Exception for saving id are context and additional info, some data, that could be saved as int(segId, time), and source and target.\nTo clear or fix tag tables, you need to do reorganize, but if some entry is missing from the table, you would not restore it on export or segment exctraction via id. So tables could be used to check health of the tm. And for some ways for concordance search or some other checks.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [ + { + "key": "Accept", + "value": "application/json", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/tables", + "path": [ + "t5memory", + "{{memory}}", + "tables" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Add to table", + "description": "You can add new entry to the tables but then it's better to reorganize tm(in case if after adding new entry with missing id and there are other entry with the same value but other id - which could be the case- in binary search that could cause problems - so just reorganize it afterwards). You can check table by just doing something like full export(or concordance search for full tm), and check logs for that 2222 errors.\n\nValues shouldn't be longer than 39 bytes(utf8), and 256 for filename,it's tables limits, for that it also have checks.\nIn cases like with those tms, you can also check if x-1 and x+1 id is present. Usually they are in those bug.\nalso you can check tables via tables request\nAlso logicaly FILE_KEY=2, but that was only for shortname, which was also 39 bytes, but real key for long name table is 5. But for requests and internal handling t5memory uses 2.\nId should be between 1 and 65k, request also checks for that", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"value\": \"MY new entry\",\n \"entryId\": 100,\n \"tableKey\": 2\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/addtotable", + "path": [ + "t5memory", + "{{memory}}", + "addtotable" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Create", + "description": "", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"examle_tm2\",\n \"sourceLang\": \"de-DE\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/", + "path": [ + "t5memory" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Create and import TM", + "description": "", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{ \n \"name\": \"test-memory\",\n \"sourceLang\": \"en-GB\",\n \"data\": \"UEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLk1FTQEAAADtzssKQEAUgOFjoTwGWR+5lPW4TCjDxgtomqI0I3lzLyBvIEUW59v829+GrinTJMaK99iqWY5Yq2VFMcpp1sotjN43swSCC/gSzw551bHu7MGLlMYqh6d8xrpBhBFjPnhACCGE/MMJUEsHCKEIpyRnAAAAAAAAAAAIAAAAAAAAUEsDBBQACAgIAFJmeFYAAAAAAAAAAAAAAAAoAAQASUQ2MzItR0VPLUxlaWNhLUhlbHAtTWFjaGluZSBDb250cm9sLlRNRAEAAADt3bFKI0EYB/BvNzFaXCXH1Vdcc3h7xR14xXEsd5pTwRgIEWwlBBRCFPEFbKx9BJ/DJ7CyshYrCwsbX0A3GtykFQnE329ZZnb+u92wMMXMt9FupBFxlERxRVTi+mPwCrNjLe9b+nw/qcRqq77UbC0HwCSdR/3vfafozSVry4s/f2Qr9Wa23t3tbGer3d5+1tju7Oz2u5+X9vqHB3u97+3Gm/24uv1s5V8AAAAAAAAAAAAAAAAAABHzg13/hVqcRSuPOM7P8sv8NG/lvTwAAAAAAAAAAAAAAAAAAABgasxFEjOD3rAO+EwpqQ56Jx+eRmqlpBLlj5JSUhtJqqUkGUnSUhIjQ5UYG4jiLIA7O/4BAAAAAAAAAAAAAAAAAABgAqrPJwHMx2zcfvryNdKbP1e/Iy5+LS58W3h8BgAAAAAAAAAAAJgO6Wg7rK7UbDe2NrfW/5dfTYJp9VI7a1i/CwAAYGLGVqq8U+YBBfOAgnnAowdQSwcITWiheHEBAAAAAAAAAAABAAAAAABQSwMEFAAICAgAUmZ4VgAAAAAAAAAAAAAAACgABABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAO3dwWkCYRAG0NFTzlaQBv4cEsg1JLqokI0gNiDLQgRZQ7ABu7MaLxawKijqWWQPvjcMM1PEx/xM8nZErCJa++J2dV1fTR5b+9inazDOuqNxLwCatI7sc1sctqfWsPf+9pr62Sh9l7Nimgbl/C/l0+J3VpXP3UW1/F/MXyb5MO6krFL/KwAAAAAAAAAAAAAAAACAiM5F6r8Tm48AAAAAAAAAAAAAAAAAAAAAGnD+TS7/DwAAAAAAAAAAAAAAAAAAAM3YAVBLBwgGVIGewwAAAAAAAAAAwAAAAAAAAFBLAQIAABQACAgIALRlVFahCKckZwAAAAAIAAAoAAQAAAAAAAAAAACkgQAAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuTUVNAQAAAFBLAQIAABQACAgIAGdralZNaKF4cQEAAAAAAQAoAAQAAAAAAAAAAACkgckAAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1EAQAAAFBLAQIAABQACAgIAFJmeFYGVIGewwAAAADAAAAoAAQAAAAAAAAAAACkgZwCAABJRDYzMi1HRU8tTGVpY2EtSGVscC1NYWNoaW5lIENvbnRyb2wuVE1JAQAAAFBLBgYsAAAAAAAAAB4DLQAAAAAAAAAAAAMAAAAAAAAAAwAAAAAAAAAOAQAAAAAAAMEDAAAAAAAAUEsGBwAAAADPBAAAAAAAAAEAAABQSwUGAAAAAAMAAwAOAQAAwQMAAAAA\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/", + "path": [ + "t5memory" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Get", + "description": "", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/", + "path": [ + "t5memory" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Resources", + "description": "", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory_service/resources", + "path": [ + "t5memory_service", + "resources" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Status", + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-GetthestatusofTM", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/status", + "path": [ + "t5memory", + "{{memory}}", + "status" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "IMPORT TMX", + "description": "Rout is async. So when import is finished should be checked with status call", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "json_data", + "value": "{\n \"framingTags\": \"saveAll\",\n \"newEntryIfOnlyContextAndTimestampDifferent\": 0,\n \"saveDifferentTargetsForSameSource\": 0,\n \"timeout\": 100000\n}\n", + "description": "json pretty print string.\n\n\"framingTags\" key possible values:\n\"saveAll\" - default behaviour, do nothing\n\"skipAll\" - skip all enclosing tags, including standalone tags\n\"skipPaired\" - skip only paired enclosing tags\n\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, default values from flags would be used.", + "type": "text", + "contentType": "application/json", + "disabled": false + }, + { + "key": "file", + "description": "tmx file to import into TM", + "type": "file", + "disabled": false + } + ] + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/importtmx", + "path": [ + "t5memory", + "{{memory}}", + "importtmx" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "EXPORT TMX", + "description": "JSON pretty print string:\nlimit - number of segments to fetch at a time\nstartFromInternalKey - internal key to start export from\n\nWorks as cursor pagination.\n\nIf no body provided, export starts from the beginning (key 7:1) to the end.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": { + "mode": "raw", + "raw": "{\n \"limit\": \"100\",\n \"startFromInternalKey\": \"1:1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/xml", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/download.tmx", + "path": [ + "t5memory", + "{{memory}}", + "download.tmx" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Concordance search", + "description": "All fields is optional, but some depends on other, so error should be returned in case of not providing required field.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-NewConcordancesearch", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"source\": \"First\",\n \"sourceSearchMode\": \"contains, CASEINSENSETIVE\",\n \"target\": \"\",\n \"targetSearchMode\": \"contains, CASEINSENSETIVE\",\n \"document\": \"\",\n \"documentSearchMode\": \"contains, CASEINSENSETIVE\",\n \"author\": \"\",\n \"authorSearchMode\": \"contains, CASEINSENSETIVE\",\n \"addInfo\": \"\",\n \"addInfoSearchMode\": \"contains, CASEINSENSETIVE\",\n \"context\": \"\",\n \"contextSearchMode\": \"contains, CASEINSENSETIVE\",\n \"timestampSpanStart\": \"19691231T230000Z\",\n \"timestampSpanEnd\": \"20260628T220000Z\",\n \"onlyCountSegments\": \"0\",\n \"searchPosition\": \"\",\n \"logicalOr\": 0,\n \"numResults\": 200\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/search", + "path": [ + "t5memory", + "{{memory}}", + "search" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Concordance search (DEPRICATED)", + "description": " SearchPosition / NewSearchPositionFormat: \"7:1\"\nFirst is segmeng\\record number, second is target number\nThe NextSearchposition is an internal key of the memory for the next position on sequential access. Since it is an internal key, maintained and understood by the underlying memory plug-in (for EqfMemoryPlugin is it the record number and the position in one record),\nno assumptions should be made regarding the content. It is just a string that, should be sent back to OpenTM2 on the next request, so that the search starts from there.\nSo is the implementation in Translate5: The first request to OpenTM2 contains SearchPosition with an empty string, OpenTM2 returns than a string in NewSearchPosition, which is just resent to OpenTM2 in the next request.\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Concordancesearch\n ", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"searchString\": \"With nice text\",\n \"searchType\": \"source\",\n \"searchPosition\": null,\n \"numResults\": 20,\n \"msSearchAfterNumResults\": 250\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/concordancesearch", + "path": [ + "t5memory", + "{{memory}}", + "concordancesearch" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Fuzzy search", + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Fuzzysearch", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"sourceLang\": \"en\",\n \"targetLang\": \"uk\",\n \"source\": \"Second segment \",\n \"documentName\": \"\",\n \"markupTable\": \"OTMXUXLF\",\n \"context\": \"\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/fuzzysearch", + "path": [ + "t5memory", + "{{memory}}", + "fuzzysearch" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Get entry", + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"recordKey\": \"8\",\n \"targetKey\": \"2\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/getentry", + "path": [ + "t5memory", + "{{memory}}", + "getentry" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Update", + "description": "https://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-Updateentry", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"source\": \" ... \",\n \"target\": \" ... \",\n \"type\": \"Manual\",\n \"markupTable\": \"OTMXUXLF\",\n \"sourceLang\": \"de-DE\",\n \"targetLang\": \"en-GB\",\n \"timeStamp\": \"20240111T111443Z\",\n \"documentName\": \"translation.txt\",\n \"author\": \"Project Manager\",\n \"context\": \"\",\n \"save2disk\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/entry", + "path": [ + "t5memory", + "{{memory}}", + "entry" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Delete TM", + "description": "", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "DELETE", + "body": { + "mode": "raw", + "options": { + "raw": { + "language": "text" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}", + "path": [ + "t5memory", + "{{memory}}" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Delete entry", + "description": "Each segmnet entry in seach payloads has \"internalKey\": \"9:1\" like key-value.\nIn this example 9 is \"recordKey\" and 1 is \"targetKey\".\nThere is also \"segmentId\" value.\nFor delete entry request it should be provided to validate that current state of t5memory data is same as at the moment when you got internalKey as internal key value may be regenerated at any moment of data processing on import or update rrquest handling.\nSo if you try to delete entry and getting error - try to fetch entry internalKey nad segmentId again and then retry delete action.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"recordKey\": 8,\n \"targetKey\": 4,\n \"segmentId\": 12\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/entrydelete", + "path": [ + "t5memory", + "{{memory}}", + "entrydelete" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Delete batch", + "description": "Asynchronious operation.\nThis would start reorganize process which would remove like reorganize bad segments and also would remove segments that gives true when checking with provided filters combined with logical AND. So if you provide timestamps and addInfo, only segments within provided timestamp and with that addInfo would not be imported to new TM(check reorganize process). \nEvery parameter is optional, so empty json would just start reorganize async process.\nIf you provide one of timestamps you would get error - please provide both. \nTo add parameter you should set it's SearchMode to be EXACT|CONCORDANCE(non case sensetive)\nIf only searched string provided, but not search mode - you would get error.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"source\": \"123\",\n \"sourceSearchMode\": \"contains, CASEINSENSETIVE\",\n \"target\": \"234\",\n \"targetSearchMode\": \"contains, CASEINSENSETIVE\",\n \"document\": \"\",\n \"documentSearchMode\": \"contains, CASEINSENSETIVE\",\n \"author\": \"\",\n \"authorSearchMode\": \"contains, CASEINSENSETIVE\",\n \"addInfo\": \"\",\n \"addInfoSearchMode\": \"contains, CASEINSENSETIVE\",\n \"context\": \"\",\n \"contextSearchMode\": \"contains, CASEINSENSETIVE\",\n \"timestampSpanStart\": \"19691231T230000Z\",\n \"timestampSpanEnd\": \"20240610T220000Z\",\n \"searchPosition\": \"\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/entriesdelete", + "path": [ + "t5memory", + "{{memory}}", + "entriesdelete" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Shutdown", + "description": "Safely shutting down the service with\\without saving all loaded tm files to the disk.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": {}, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory_service/shutdown", + "path": [ + "t5memory_service", + "shutdown" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [ + { + "key": "dontsave", + "description": "Skips saving tms, for now value doesn't matter, only presence.\nIf try to save tms before closing, would check if there is still import process going on\nIf there is some, would wait 1 second and check again. \nRepeats last step up to 10 min, then closes service anyway. ", + "type": "string" + } + ], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Save TMs", + "description": "Flushes all filebuffers(TMD, TMI files) into the filesystem. Reset 'Modified' flags for file buffers. \n\nFilebuffer is a file instance of .TMD or .TMI loaded into RAM. It provides better speed and safety when working with files.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": {}, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory_service/savetms", + "path": [ + "t5memory_service", + "savetms" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Clone", + "description": "Creates TM with the provided name and data cloned from provided memory", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "POST", + "body": { + "mode": "raw", + "raw": "{\n \"newName\": \"clonned-tm-1\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/clone", + "path": [ + "t5memory", + "{{memory}}", + "clone" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + }, + { + "name": "Reorganize", + "description": "Under the hood it creates new tm with $Org- prefix, then reimport all segments one-by-one, and then deletes original TM and rename reorganized TM to replace original. \nThis request should flush tm(from RAM to the disk) before reorganizing\n\nhttps://confluence.translate5.net/display/TAD/t5memory+-+translate5+TM+service+-+REST+API#t5memorytranslate5TMserviceRESTAPI-ReorganizeTM\nBoth newEntryIfOnlyContextAndTimestampDifferent and saveDifferentTargetsForSameSource impact how similar translation would be saved. In case if source and other field(doc, author, lang) are the same, but target different, and newEntryIfOnlyContextAndTimestampDifferent is set to true, you would have 2 translation, otherwise you would have 1 translation saved - with higher updateTime. but if you have different attributes, like document, you would have 2 translation anyway. \nnewEntryIfOnlyContextAndTimestampDifferent would also increase number of saved segments. If values was not provided, defaule values from flags would be used.", + "event": [], + "auth": {}, + "request": { + "auth": {}, + "method": "GET", + "body": { + "mode": "raw", + "options": { + "raw": { + "language": "json" + } + } + }, + "header": [ + { + "key": "Accept", + "value": "application/json; charset=utf-8", + "description": "", + "type": "string" + }, + { + "key": "Accept-Charset", + "value": "UTF-8", + "description": "", + "type": "string" + } + ], + "url": { + "raw": "{{baseUrl}}/t5memory/{{memory}}/reorganize", + "path": [ + "t5memory", + "{{memory}}", + "reorganize" + ], + "host": [ + "{{baseUrl}}" + ], + "query": [], + "variable": [] + } + }, + "response": [], + "protocolProfileBehavior": { + "strictSSL": false, + "followRedirects": true + } + } + ], + "variable": [], + "event": [], + "auth": {} +} \ No newline at end of file diff --git a/include/EQF.H b/include/EQF.H index cd60ebeb..7d23aa69 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -555,22 +555,6 @@ } -#ifdef TEMPORARY_COMMENTED -#define CBQUERYSELECTEDITEMTEXT( sItem, hwnd, id, szBuffer ) \ - { \ - sItem = (SHORT)SendDlgItemMessage( hwnd, id, CB_GETCURSEL, 0, 0L ); \ - if( sItem != CB_ERR ) \ - { \ - SendDlgItemMessage( hwnd, id, CB_GETLBTEXT, MP1FROMSHORT( sItem ), \ - MP2FROMP( szBuffer ) ); \ - } \ - else \ - { \ - szBuffer [0] = EOS; \ - } /* endif */ \ - } -#endif - #define CBQUERYSELECTEDITEMTEXTHWND( sItem, hwnd, szBuffer ) \ { \ sItem = (SHORT)CPLUSPLUS SendMessage( hwnd, CB_GETCURSEL, 0, 0L ); \ @@ -3642,9 +3626,9 @@ typedef ULONG HPRINT, *PHPRINT; ULONG UtlLongToDateString( LONG, PSZ, ULONG ); ULONG UtlLongToTimeStringW( LONG, PSZ_W, ULONG ); ULONG UtlLongToDateStringW( LONG, PSZ_W, ULONG ); - BOOL UtlLoadFile( PSZ, PVOID *, USHORT *, BOOL, BOOL ); - BOOL UtlLoadFileL( PSZ, PVOID *, ULONG *, BOOL, BOOL ); - BOOL UtlLoadFileHwnd( PSZ, PVOID *, ULONG *, BOOL, BOOL, HWND ); + BOOL UtlLoadFile( PSZ, BytesRef , USHORT *, BOOL, BOOL ); + BOOL UtlLoadFileL( PSZ, BytesRef , ULONG *, BOOL, BOOL ); + BOOL UtlLoadFileHwnd( PSZ, BytesRef , ULONG *, BOOL, BOOL, HWND ); USHORT UtlCheckSpaceForFile( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL ); USHORT UtlCheckSpaceForFileEx( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL, PUSHORT ); ULONG64 UtlQueryFreeSpace( CHAR, BOOL ); @@ -3669,7 +3653,6 @@ typedef ULONG HPRINT, *PHPRINT; PSZ UtlMakeFullPath ( PSZ, PSZ, PSZ, PSZ, PSZ ); USHORT UtlWriteFile( PSZ pszFile, ULONG ulDataLength, PVOID pData); BOOL UtlIsEqfDrive( CHAR chDrive ); - BOOL UtlGetCheckedEqfDrives( PSZ pszDrives ); char* UtlGetFnameFromPath( const char* path); PSZ UtlMakeObjName( PSZ p, PSZ drive, PSZ path, PSZ fname, PSZ ext); VOID UtlMakeFNameAndPath( PSZ, PSZ, PSZ ); @@ -3707,14 +3690,11 @@ SHORT UtlDriveType( CHAR chDrive ); VOID UtlSetHorzScrollingForLB(HWND hLB); VOID UtlSaveWindowPos( HWND hFrame, EQF_PSWP pSwp); -BOOL UtlAllocHwnd ( PVOID *, LONG, LONG, USHORT, HWND ); BOOL UtlInitUtils( HAB ); BOOL UtlRegisterEqfHelp( HAB ); BOOL UtlTerminateUtils( VOID ); USHORT UtlError ( SHORT, USHORT, USHORT, PSZ *, ERRTYPE ); USHORT UtlErrorHwnd ( SHORT, USHORT, USHORT, PSZ *, ERRTYPE, HWND ); -USHORT UtlErrorW ( SHORT, USHORT, USHORT, PSZ_W *, ERRTYPE, BOOL ); -USHORT UtlErrorHwndW ( SHORT, USHORT, USHORT, PSZ_W *, ERRTYPE, HWND, BOOL ); VOID UtlGetMsgTxt( SHORT, PSZ, USHORT, PSZ *); VOID UtlInitError ( HAB, HWND, HWND, PSZ ); @@ -3907,13 +3887,7 @@ USHORT UtlFillTableLB( HWND, USHORT ); // get languages USHORT SetSystemLanguage( PSZ pLanguage ); USHORT UtlSetFHandState( HFILE hf, USHORT fsState, BOOL fMsg ); USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ); -BOOL UtlAllocTrace ( PVOID *, LONG, LONG, USHORT, PSZ, USHORT ); -BOOL UtlAllocTraceHwnd ( PVOID *, LONG, LONG, USHORT, HWND, PSZ, USHORT ); #ifdef UTLFFST - #define UtlAlloc( par1, par2, par3, par4 ) \ - UtlAllocTrace(par1, par2, par3, par4, __FILE__, __LINE__ ) - #define UtlAllocHwnd( par1, par2, par3, par4, par5 ) \ - UtlAllocTraceHwnd(par1, par2, par3, par4, par5, __FILE__, __LINE__ ) #endif USHORT EqfDriveType(USHORT iDrive); @@ -3945,11 +3919,6 @@ BOOL UtlKeepInTWB( PSWP pswpWin ); USHORT UtlCheckDlgPos( HWND hwndDlg, BOOL fShow ); LONG UtlTime( PLONG plTime ); -#ifdef UTLFFST -#define UtlAlloc( par1, par2, par3, par4 ) \ - UtlAllocTrace(par1, par2, par3, par4, __FILE__, __LINE__ ) - -#endif VOID UtlWait( SHORT sWaitTime ); // wait the given time USHORT UtlLoadWidth( HAB, HMODULE, SHORT, PUSHORT ); @@ -5383,8 +5352,6 @@ ULONG GetOrgLangCP( PSZ pLanguage, BOOL fASCII ); //------------------------------------------------------------------------------ struct TERMLENOFFS { - // int iLength; // length of term - // int iOffset; // offset of term data in buffer USHORT iLength; // length of term USHORT iOffset; // offset of term data in buffer TERMLENOFFS(){iLength = iOffset = 0;} @@ -5428,7 +5395,6 @@ void LogSysInfo( VOID ); PSZ_W UTF16strcpy( PSZ_W pszTarget, PSZ_W pszSource ); PSZ_W UTF16strcat( PSZ_W pszTarget, PSZ_W pszSource ); PSZ_W UTF16strncpy(PSZ_W pusTarget, PSZ_W pusSource, LONG lLen); -int UTF16strcmp( PSZ_W pszString1, PSZ_W pszString2 ); int UTF16strncmpL( PSZ_W pszString1, PSZ_W pszString2, LONG lLen ); int UTF16strnicmpL( PSZ_W pszString1, PSZ_W pszString2, LONG lLen ); int UTF16strncmp( PSZ_W pszString1, PSZ_W pszString2, USHORT usLen ); @@ -5657,19 +5623,6 @@ BOOL DicGetCheckPassword( PSZ pszPassword, PULONG pulPassword, BOOL fCheckPasswo #define XLATEOS2_MARKUP "XLATEOS2" #define XLATEWIN_MARKUP "XLATEWIN" -// support additional pages in document import/export -typedef struct _EXTRAIDA -{ - CHAR szFileName[ MAX_PATH144 ]; // filename to be used - HWND hwndExpListbox; // handle of listbox containing selected docs - OBJNAME szFolderObjName; // folder object name - PVOID pIda; // product specifics - USHORT usExtraPage; // number of extra page - HWND hwnd; // handle of parent dialog - OBJNAME szParentObjName; // parent (sub)folder object name -} EXTRAIDA, *PEXTRAIDA; - - // defines for SGML Formats #define SGMLFORMAT_ANSI 1 @@ -5745,7 +5698,6 @@ NTMCompareBetweenTokens PSZ_W pD2, PSZ pszMarkup, SHORT sLangID, - ULONG ulSrcCP, PBOOL pfStringEqual ); diff --git a/include/EQFDASD.H b/include/EQFDASD.H index 23f8e683..cc50b09e 100755 --- a/include/EQFDASD.H +++ b/include/EQFDASD.H @@ -212,15 +212,7 @@ typedef enum _TERMTYPE #define STYLEPREFIX_UNDEFINED 0x2715 // -#define ASD_GUARDED 0x01 // Open dictionary in guarded mode -#define ASD_LOCKED 0x10 // Open dictionary and lock it -#define ASD_FORCE_WRITE 0x40 // force imm.write of any changed record -#define ASD_NOINDEX 0x80 // Do not process index dictionary - // Caution: this define should be used - // only for dictionary organize!!! #define ASD_ORGANIZE 0x100 // work in organize mode -#define ASD_NOOPENCHECK 0x200 // do not check QDAM header open flag -#define ASD_READONLY 0x400 // open in read only mode // AsdTranslate mode: return terms only diff --git a/include/EQFDDE.H b/include/EQFDDE.H index c05efd28..a2bf3ae2 100755 --- a/include/EQFDDE.H +++ b/include/EQFDDE.H @@ -587,10 +587,10 @@ typedef struct _BATCHIMPORTPARMS HGLOBAL MakeDDEDataSeg(USHORT, PSZ, PVOID, USHORT, USHORT ); -static BOOL EQFFunctionCmdLine (PDDECLIENT ); -static BATCHCMD ValidateToken ( PSZ *ppToken, PBATCHLIST pBatchList ); -static BOOL CheckBatchOccurrence (PDDECLIENT, PUSHORT); -static DDETASK ValidateTask ( PDDECLIENT pDDEClient ); +BOOL EQFFunctionCmdLine (PDDECLIENT ); +BATCHCMD ValidateToken ( PSZ *ppToken, PBATCHLIST pBatchList ); +BOOL CheckBatchOccurrence (PDDECLIENT, PUSHORT); +DDETASK ValidateTask ( PDDECLIENT pDDEClient ); BOOL DDEDocUnLoad( PDOCIMPEXP pDocExpIda ); BOOL DDEDocLoad( PDOCIMPEXP pDocImpIda ); diff --git a/include/EQFFUZZ.H b/include/EQFFUZZ.H index 75d5f9b2..6aad5807 100755 --- a/include/EQFFUZZ.H +++ b/include/EQFFUZZ.H @@ -1,4 +1,4 @@ - +#include "lowlevelotmdatastructs.h" #ifndef _EQFFUZZ_H_ #define _EQFFUZZ_H_ @@ -31,9 +31,6 @@ typedef struct _LCSTOKEN SHORT sTotalLen; } LCSTOKEN, *PLSCTOKEN; - - - //static SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); @@ -43,10 +40,8 @@ SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); //static VOID MakeHashValue ( PULONG, USHORT, PSZ_W, PULONG ); - - //static -BOOL PrepareTokens ( PLOADEDTABLE, PBYTE, PBYTE, PSZ_W, SHORT, PFUZZYTOK *, ULONG ); +BOOL PrepareTokens ( PLOADEDTABLE, PSZ_W, SHORT, std::vector & ); //static BOOL FuzzyReplace ( PSZ_W, PSZ_W, PSZ_W, PREPLLIST, PREPLLIST ); //static @@ -54,7 +49,7 @@ VOID TransferSource( PSZ_W, PSZ_W, PSZ_W ); //static SHORT TokStrCompare ( PFUZZYTOK, PFUZZYTOK); //static -PFUZZYTOK SplitTokens (PFUZZYTOK, USHORT, SHORT, int, PSZ_W); +void SplitTokens (std::vector&, USHORT, SHORT, int, PSZ_W); //static //static SHORT Snake ( PFUZZYTOK, PFUZZYTOK, SHORT, SHORT, SHORT, SHORT, BOOL); @@ -64,13 +59,11 @@ SHORT FindMiddleSnake( PFUZZYTOK, PFUZZYTOK, PFUZZYTOK, PFUZZYTOK, //static VOID LCS( LCSTOKEN, LCSTOKEN , BOOL); //static -BOOL EQFBTokCountDiff(PFUZZYTOK, PFUZZYTOK, - USHORT, USHORT, PUSHORT); //static - BOOL EQFBMarkModDelIns( PFUZZYTOK, PFUZZYTOK, PFUZZYTOK*, PFUZZYTOK *, + BOOL EQFBMarkModDelIns( std::vector&, std::vector&, std::vector&, std::vector&, USHORT, USHORT); //static -void EQFBSimplifyAndCountMarks (PFUZZYTOK, PUSHORT, PUSHORT); +void EQFBSimplifyAndCountMarks (std::vector&, PUSHORT, PUSHORT); /**********************************************************************/ /* macro to calculate the number of tokens in the list ... */ diff --git a/include/EQFMORPH.H b/include/EQFMORPH.H index 97319b4c..c03a501e 100755 --- a/include/EQFMORPH.H +++ b/include/EQFMORPH.H @@ -98,15 +98,7 @@ #include "LogWrapper.h" -/**********************************************************************/ -/* Structure to access MORPH_FLAG_OFFSLIST type term lists */ -/**********************************************************************/ -typedef struct _FLAGOFFSLIST -{ - LONG lFlags; // flags of term - USHORT usLen; // length of term - USHORT usOffs; // offset of term -} FLAGOFFSLIST, *PFLAGOFFSLIST; + //+----------------------------------------------------------------------------+ // Flags for flagged term lists @@ -190,12 +182,7 @@ USHORT MorphTokenizeW ( SHORT sLanguageID, // language ID PSZ_W pszInData, // pointer to input segment - PUSHORT pusBufferSize, // address of variable containing size of - // term list buffer - TERMLENOFFS** ppTermList, // address of caller's term list pointer - USHORT usListType, // type of term list MORPH_ZTERMLIST or - // MORPH_OFFSLIST - ULONG ulOemCP + std::vector& ppTermList// address of caller's term list pointer ); @@ -330,26 +317,9 @@ USHORT MorphAddTermToList2 USHORT MorphAddTermToList2W ( - PSZ_W *ppList, // ptr to term list pointer - PULONG pulSize, // ptr to allocated size of term list - PULONG pulUsed, // ptr to used bytes in term list - PSZ_W pszTerm, // ptr to new term being added to list - USHORT usLength, // length of term - USHORT usOffs, // offset of term - ULONG ulFlags, // flags for the term - USHORT usListType // type of list MORPH_ZTERMLIST or -); - - -USHORT MorphCopyTermListASCII2Unicode -( - PSZ pList, // ptr to ASCII termlist - USHORT usBufferSize, // size of allocated buffer - PSZ_W *ppListW, // ptr to term list pointer - PULONG pulSize, // ptr to allocated size of term list - PULONG pulUsed, // ptr to used bytes in term list - USHORT usListType, // type of list - ULONG ulOemCP + std::vector& pList, // ptr to term list pointer + int iLength, // length of term in # of w's + int iOffs // offset of term in # of w's??? ); diff --git a/include/EQFOSWIN.H b/include/EQFOSWIN.H index e54dfe3a..3f2cf910 100755 --- a/include/EQFOSWIN.H +++ b/include/EQFOSWIN.H @@ -130,7 +130,6 @@ HWND WinQueryWindow ( HWND, SHORT, BOOL ); /* one might probably use NULL as default, but we use our UTL-function*/ /* which stores the anchor block for PM - we store the instead instead*/ /**********************************************************************/ -#define WinQueryAnchorBlock( a ) ((HAB) UtlQueryULong( QL_HAB )) /**********************************************************************/ /* resource strings */ /**********************************************************************/ @@ -1048,12 +1047,7 @@ USHORT DosGetDateTime( PDATETIME pDateTime ); /**********************************************************************/ SHORT GetNextSelection( HWND hwndLB, SHORT sPos ); -/**********************************************************************/ -/* force use of GetOEMCP- needed for 32-bit */ -/* GetKBCodePage returns 437 for TChin/SChin/Korean under Win95! */ -/* which causes that no DBCS chars are detected!! */ -/**********************************************************************/ -#define GetKBCodePage() GetOEMCP() + /**********************************************************************/ /* allow to fill the rectangle in the selected color */ diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index a2ab3533..8e6de758 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -347,7 +347,7 @@ typedef struct _DICTSIGN SHORT QDAMDictInsert ( PBTREE pBT, // pointer to binary tree struct - PWCHAR pKey, // pointer to key data + LONG lKey, // pointer to key data PBYTE pData, // pointer to user data ULONG ulLen // length of user data ); @@ -392,7 +392,7 @@ typedef struct _DICTINSERT SHORT QDAMDictExact ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for user data PULONG pulLength, // in/out length of returned user data USHORT usSeachSubType @@ -438,7 +438,7 @@ SHORT QDAMDictExact SHORT QDAMDictSubStr ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -484,7 +484,7 @@ SHORT QDAMDictSubStr SHORT QDAMDictEquiv ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -573,7 +573,7 @@ SHORT QDAMDictLockEntry SHORT QDAMDictFirst ( PBTREE pBT, - PWCHAR pKeyData, // pointer to space for key data + ULONG ulKeyData, // pointer to space for key data PULONG pulKeyLen, // length of space for key data PBYTE pUserData, // pointer to space for user data PULONG pulUserLen // length of space for user data @@ -781,7 +781,7 @@ SHORT QDAMDictNumber ( PBTREE pBT, // pointer to generic structure ULONG ulTermNum, // term number - PWCHAR pKey, // pointer to space for term + LONG lKey, // pointer to space for term PULONG pulKeyLen, // length of key PBYTE pData, // pointer to data PULONG pulLen // length of data @@ -894,46 +894,46 @@ typedef struct _DICTTIME SHORT QDAMDictNumEntriesLocal ( PBTREE, PULONG ); - SHORT QDAMDictFirstLocal ( PBTREE, PWCHAR, PULONG, PBYTE, PULONG ); + SHORT QDAMDictFirstLocal ( PBTREE, LONG&, BytesRef ); + + SHORT QDAMInsertKey_V3( PBTREE, std::shared_ptr&, PWCHAR, RECPARAM, RECPARAM ); + SHORT QDAMFindParent_V3(PBTREE, std::shared_ptr&, PUSHORT ); - SHORT QDAMInsertKey_V3( PBTREE, PBTREEBUFFER_V3, PWCHAR, RECPARAM, RECPARAM ); - SHORT QDAMFindParent_V3(PBTREE, PBTREEBUFFER_V3, PUSHORT ); + SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, std::shared_ptr& ); - SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, PBTREEBUFFER_V3 * ); + SHORT QDAMChangeKey_V3(PBTREE, USHORT, LONG&, LONG& ); + SHORT QDAMNewRecord_V3( PBTREE, std::shared_ptr&, RECTYPE, bool fSuppressTmdSizeCheck = false ); - SHORT QDAMChangeKey_V3(PBTREE, USHORT, PWCHAR, PWCHAR ); - SHORT QDAMNewRecord_V3( PBTREE, PBTREEBUFFER_V3 *, RECTYPE, bool fSuppressTmdSizeCheck = false ); SHORT QDAMDeleteDataFromBuffer ( PBTREE, RECPARAM ); - SHORT QDAMSplitNode_V3( PBTREE, PBTREEBUFFER_V3 *, PWCHAR ); + SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, LONG& ); - SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, PWCHAR, PULONG ); - SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, PBYTE, PULONG, CHAR ); + SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, LONG& ); - BOOL QDAMDeleteKey_V3( PBTREE, PBTREEBUFFER_V3, PCHAR); + BOOL QDAMDeleteKey_V3( PBTREE, std::shared_ptr&, PCHAR); SHORT QDAMDestroy ( PBTREE ); SHORT QDAMKeyCompare ( PVOID, PVOID, PVOID ); SHORT QDAMKeyCompareNonUnicode ( PVOID, PVOID, PVOID ); SHORT QDAMFirst_V3( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); - SHORT QDAMUnTerseData ( PBTREE, PUCHAR, ULONG, PULONG ); + SHORT QDAMUnTerseData ( PBTREE, BytesRef , ULONG ); + SHORT QDAMValidateIndex_V3( PBTREE, std::shared_ptr& ); + + LONG QDAMGetszKey_V3 ( std::shared_ptr&, USHORT ); + RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT ); + VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM ); + RECPARAM QDAMGetrecKey_V3 ( std::shared_ptr&, SHORT ); + VOID QDAMReArrangeKRec_V3 ( PBTREE, std::shared_ptr& ); + VOID QDAMCopyKeyTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT ); + VOID QDAMCopyDataTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT ); + SHORT QDAMLastEntry_V3( PBTREE, std::shared_ptr& ); + SHORT QDAMFirstEntry_V3( PBTREE, std::shared_ptr& ); - PSZ_W QDAMGetszKey_V3 ( PBTREEBUFFER_V3, USHORT, USHORT ); - RECPARAM QDAMGetrecData_V3 ( PBTREEBUFFER_V3, SHORT, USHORT ); - VOID QDAMSetrecData_V3 ( PBTREEBUFFER_V3, SHORT, RECPARAM, USHORT ); - RECPARAM QDAMGetrecKey_V3 ( PBTREEBUFFER_V3, SHORT ); - VOID QDAMReArrangeKRec_V3 ( PBTREE, PBTREEBUFFER_V3 ); - VOID QDAMCopyKeyTo_V3( PBTREEBUFFER_V3, SHORT, PBTREEBUFFER_V3, SHORT, USHORT ); - VOID QDAMCopyDataTo_V3( PBTREEBUFFER_V3, SHORT, PBTREEBUFFER_V3, SHORT, USHORT ); - SHORT QDAMLastEntry_V3( PBTREE, PBTREEBUFFER_V3 * ); - SHORT QDAMFirstEntry_V3( PBTREE, PBTREEBUFFER_V3 * ); - BOOL QDAMFetchFromIndexList_V3( PBTREE, PBTREEBUFFER_V3, USHORT ); - VOID QDAMAddToIndexList_V3( PBTREE, PBTREEBUFFER_V3 ); SHORT QDAMAllocKeyRecords ( PBTREE, USHORT ); - SHORT QDAMLocSubstr_V3( PBTREE, PBTREEBUFFER_V3, PWCHAR, PBYTE, PULONG, PBYTE, PULONG ); + SHORT QDAMLocSubstr_V3( PBTREE, std::shared_ptr&, PWCHAR, PBYTE, PULONG, PBYTE, PULONG ); @@ -1240,13 +1240,6 @@ void DAMLINK // <> keys are unequal //+----------------------------------------------------------------------------+ -SHORT NTMKeyCompare -( - PVOID pBTIda, // pointer to tree structure - PVOID pulKey1, // pointer to first key - PVOID pulKey2 // pointer to second key -); - /**********************************************************************/ /* validate the passed BTree pointer */ /**********************************************************************/ @@ -1305,29 +1298,9 @@ USHORT QDamWildCardList ULONG // in - size of buffer ); -SHORT QDAMDictNextWildLocal -( - PBTREE pBTIda, - PSZ_W pPattern, // search pattern or compound - BOOL fCompound, // compound search flag - PBYTE pKeyData, // pointer to space for key data - PULONG pulKeyLen // length of space for key data -); - -BOOL QDAMMatchCompound -( - PSZ_W pKey, - PSZ_W pCompound -); - -BOOL QDAMMatchWild -( - PUCHAR pKey, - PUCHAR pPattern -); -ULONG QDAMComputeCheckSum_V3 (PBTREEBUFFER_V3 pRecord ); -BOOL QDAMCheckCheckSum_V3( PBTREEBUFFER_V3 pRecord, SHORT sLocation ); +ULONG QDAMComputeCheckSum_V3 (std::shared_ptr& pRecord ); +BOOL QDAMCheckCheckSum_V3( std::shared_ptr& pRecord, SHORT sLocation ); // get nect dictionary of an association list diff --git a/include/EQFSETUP.H b/include/EQFSETUP.H index 3fb8b6df..979bb9e5 100755 --- a/include/EQFSETUP.H +++ b/include/EQFSETUP.H @@ -79,34 +79,20 @@ typedef enum _UPDPROPTYPE VOID BuildPath( PSZ pszBuffer, CHAR chEqfDrive, PSZ pszFolder, USHORT usPathID ); -static VOID DeleteFiles( PSZ, USHORT ); -static USHORT ErrorMessage( USHORT, PSZ, USHORT ); -static BOOL FileExists ( PSZ ); -static BOOL InstCheckAndAdd( PSZ *, PSZ, USHORT ); PSZ InstLocateString ( PSZ, PSZ ); PSZ InstMatch ( PSZ, PSZ ); -static USHORT ReadAndPrepareConfigSys ( VOID ); USHORT UpdateConfigSys ( VOID ); int SetupMAT(); USHORT SetupMAT( HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives, CHAR chLanDrive ); -static USHORT CreateDictProperties( CHAR chPrimaryDrive ); USHORT SetupTmCom (CHAR chPrimaryDrive, PSZ pszSecondaryDrives); USHORT SetupDemo (HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives, CHAR chLanDrive); USHORT SetupDocu (HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives); -static VOID DeleteDictCacheFiles( CHAR chPrimaryDrive, USHORT usPathId ); -static BOOL UtlPropFileExist( CHAR chDrive, PSZ pszFile ); - USHORT UpdatePropFile( USHORT usPropType ,CHAR chDrive, PSZ pszFileName, PSZ pszDocName ); -static USHORT UpdateFolderProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateDocumentProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateDictProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateTMProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateSystemProp( PSZ pszFullFileName, PSZ pszFolderName ); BOOL SetupUtils( HAB, PSZ ); BOOL TWBInit( HAB ); diff --git a/include/EQFTA.H b/include/EQFTA.H index 5ead7267..71859037 100755 --- a/include/EQFTA.H +++ b/include/EQFTA.H @@ -21,12 +21,14 @@ /**********************************************************************/ /* Structure for start/stop tables as used by TACreateStartStopTable */ /**********************************************************************/ -typedef struct _STARTSTOP +struct STARTSTOP { - USHORT usStart; // start offset - USHORT usStop; // stop offset - USHORT usType; // type of entry -} STARTSTOP, *PSTARTSTOP; + USHORT usStart = 0; // start offset + USHORT usStop = 0; // stop offset + USHORT usType = 0; // type of entry + void reset(){*this=STARTSTOP{};} +}; +using PSTARTSTOP = STARTSTOP*; // structure for commincation between analysis and MT functions @@ -94,30 +96,8 @@ BOOL EQFUnSegRevMark2 USHORT usTrackDocNum, // not 0, TVT tracking document num PSZ pszConversion // ptr to conversion name or NULL ); -USHORT TACreateProtectTable -( - PSZ pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pfnUserExit, // ptr to user exit function - ULONG ulCP -); -USHORT TACreateProtectTableW -( - PSZ_W pszSegmentW, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - PFN pvUserExitW, - ULONG ulCP -); + #define CREATEPROTTABLE_MARKATTR 0x02 #define CREATEPROTTABLE_NOTRANSLVAR 0x04 @@ -128,10 +108,10 @@ USHORT TACreateProtectTableWEx PSZ_W pszSegment, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function + BytesRef pStartStop, // ptr to caller's start/stop table ptr + //PFN pvUserExit, // ptr to user exit function PFN pvUserExitW, ULONG ulCP, // ASCII cp fitting to Segmenttext int iMode // mode for function @@ -139,16 +119,6 @@ USHORT TACreateProtectTableWEx -USHORT TAPrepProtectTable -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // ptr to segment check function - PFN *ppfnShowTrans, // ptr to Show transl. function - PFN *ppfnTocGoto // ptr to TOC goto function -); - USHORT TAEndProtectTable ( HMODULE *phModule, // address of user exit module handle @@ -218,19 +188,6 @@ void TAFillWSList PSZ_W *ppWSList ); - -BOOL TAAdjustWhiteSpace -( - PSZ_W pszSourceSeg, // ptr to source segment data - PSZ_W pszTargetSeg, // ptr to target segment - PSZ_W *ppszNewTargetSeg, // ptr to ptr of output buffer - BOOL fLeadingWS, // process leading whitespace - BOOL fTrailingWS, // process trailing whitespace - PBOOL pfChanged, // TRUE if target has been changed - PSZ_W pWSList // list of whitespace chars (TBX-CLS_WHITESPACEs) -); - - VOID TABuildArchTM( HWND hwnd, PSZ pSelFolderName ); VOID ExportSegs( HWND hwnd, PSZ pSelFolderName ); diff --git a/include/EQFTAG.H b/include/EQFTAG.H index 9980fd97..8b2aee8f 100755 --- a/include/EQFTAG.H +++ b/include/EQFTAG.H @@ -223,8 +223,8 @@ BOOL ContainsClassID( PLOADEDTABLE pLoadedTable, USHORT usClassID ); VOID EQFTagTokenize(PSZ, TAGTABLE *, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT); -BOOL TATagTokenize (PSZ, PLOADEDTABLE, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT, ULONG); -BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, BOOL, PSZ_W *, USHORT *, TOKENENTRY *, USHORT); +//BOOL TATagTokenize (PSZ, PLOADEDTABLE, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT, ULONG); +BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, PSZ_W *, USHORT *, std::vector&); /**********************************************************************/ diff --git a/include/EQFTAG00.H b/include/EQFTAG00.H index 5d3362dd..90f56d34 100755 --- a/include/EQFTAG00.H +++ b/include/EQFTAG00.H @@ -168,7 +168,6 @@ typedef enum { MUFILL_NAMES, MUFILL_CLBITEMS } MUFILLTYPES; BOOL MUGetMarkupTableFileName( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); BOOL MUGetMarkupTableFilePath( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); BOOL MUGetUserExitFileName( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); - BOOL MUUpdateMarkupTableFiles( char *pszMarkup, char *pszPlugin, char *pszShortDescription, char *pszLongDescription, char *pszVersion, char *pszTableFile, char *pszUserExitFile, char *pszMarkupFiles ); BOOL MUDeleteMarkupTable( char *pszMarkup, char *pszPlugin ); #endif diff --git a/include/EQFTP.H b/include/EQFTP.H index 705461ee..bb6623de 100755 --- a/include/EQFTP.H +++ b/include/EQFTP.H @@ -982,9 +982,10 @@ VOID EQFBVioSetNewDocSize ( PTBDOCUMENT ); // refresh doc size VOID ImeSetFont( PTBDOCUMENT, HWND, PLOGFONT ); -BOOL EQFBFindDiffEx( PVOID, PBYTE, PBYTE, PSZ_W, PSZ_W, SHORT , PVOID * , PVOID *, ULONG); +BOOL EQFBFindDiffEx( PLOADEDTABLE, PSZ_W, PSZ_W, SHORT , + std::vector&, std::vector&); -BOOL EQFBCountDiff( PVOID pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); +BOOL EQFBCountDiff( std::vector& pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); BOOL EQFBDoEmptySegCheck( PTBDOCUMENT ); // do empty segment handling @@ -1017,28 +1018,12 @@ int BiDitransform -typedef struct _FUZZYTOK -{ - PSZ_W pData; // pointer to data.. - ULONG ulHash; // hash value - USHORT usStart; // start offset - USHORT usStop; // end offset - SHORT sType; // type of entry /* KIT0857M */ - EQF_BOOL fConnected; // true if connected to pPropTokList - EQF_BOOL fNoTagSubst; // true = do not use for tag substitution -} FUZZYTOK, *PFUZZYTOK; - typedef struct _REPLLIST { PFUZZYTOK pSrcTok; // ptr to source token PFUZZYTOK pTgtTok; // ptr to target token } REPLLIST, *PREPLLIST; - - -USHORT FuzzyLCSReplList ( PFUZZYTOK, PFUZZYTOK, PREPLLIST *, - USHORT, USHORT, PUSHORT, BOOL); - SHORT StrNLFCmp ( PSZ_W, PSZ_W ); @@ -1169,7 +1154,6 @@ USHORT TAIteratedCreateProtectTableW( PTBDOCUMENT pDoc, PTBSEGMENT pSeg, int iM USHORT EQFBPrepareFuzzyPropEx ( PTBDOCUMENT, PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, ULONG, ULONG); USHORT EQFBPrepareFuzzyPropEx2( PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, PVOID, PVOID, PBYTE, PBYTE, ULONG, ULONG ); USHORT EQFBPrepareFuzzyPropEx3( PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, PVOID, PVOID, PBYTE, PBYTE, ULONG, ULONG, PUSHORT ); - BOOL EQFBFindDiff ( PTBDOCUMENT, PSZ_W, PSZ_W, SHORT , PFUZZYTOK * , PUSHORT, PUSHORT, ULONG); BOOL EQFBFindWord ( PSZ_W ,SHORT, USHORT, PUSHORT, PUSHORT, ULONG, BOOL ); #endif //_EQFTP_H_INCLUDED_ diff --git a/include/EQFTPI.H b/include/EQFTPI.H index 7e5ff5df..ac0ed078 100755 --- a/include/EQFTPI.H +++ b/include/EQFTPI.H @@ -1482,20 +1482,6 @@ typedef struct _USERSET } USERSET, *PUSERSET; -// Profile ONE Dialog IDA (initial values) -typedef struct _PROFONEIDA -{ - USERSET LBItemArray [MAXLBITEMS]; // text of listbox - CHAR szBuffer [60]; // get choice from listbox - PTBDOCUMENT pDoc; - HBITMAP hbm; // menucheck bitmap - CHAR szStyleArray [MAXSTYLES][40]; // text for prop.wnd.styles - CHAR szMarginBuffer[RMARGIN_LENGTH + 1]; // text for rmargin cbs - HWND hwndPages[10]; // support up to 10 prop pages -#ifdef R004422_BACKSAVE - CHAR szMinute[5]; // text for minutes CBS -#endif -} PROFONEIDA, *PPROFONEIDA; // TOC goto diealog IDA typedef struct _TOCGOTOIDA @@ -2301,12 +2287,10 @@ BOOL EQFBDoCopyData ( PTBDOCUMENT,PSZ_W ); ( (c == LF) || \ ( (c == SOFTLF_CHAR) && (c2 != SOFTLF_CHAR) && (c0 != SOFTLF_CHAR)) ) -BOOL EQFBCallLCS(PFUZZYTOK, PFUZZYTOK, USHORT, USHORT, BOOL); - +BOOL EQFBCallLCS(std::vector&, std::vector&, USHORT, USHORT, BOOL); USHORT EQFBMisspelledHLType ( PTBDOCUMENT, PTBSEGMENT, PSZ_W ); - /**********************************************************************/ /* worker thread defines */ /**********************************************************************/ diff --git a/include/EQFUTPRI.H b/include/EQFUTPRI.H index bb5bc61b..3925f036 100755 --- a/include/EQFUTPRI.H +++ b/include/EQFUTPRI.H @@ -187,8 +187,6 @@ INT_PTR CALLBACK UTLYESTOALLDLG LPARAM mp2 // second message parameter ); -VOID UtlReplString( SHORT sID, PSZ pszString ); - /**********************************************************************/ /* Private data for the rename object dialog */ /**********************************************************************/ diff --git a/include/Filebuffer.h b/include/Filebuffer.h index 83ad6391..56dd03fb 100644 --- a/include/Filebuffer.h +++ b/include/Filebuffer.h @@ -16,28 +16,115 @@ enum FilebufferType{ TMI_FILEBUFFER, }; +#include +#include +#include +#include +#include +#include + +class LockedFile { +public: + enum class Mode { Unknown, + Read, // read-only with shared lock + Write, // read-write with exclusive lock + truncate + Append, // read-write with exclusive lock + append + ReadWrite // read-write with exclusive lock, no truncate + }; + + enum class CalledFrom{ + Unknown, + Create, + Read, + Write + }; + + static constexpr std::string_view CalledFromToString(CalledFrom from) { + switch (from) { + case CalledFrom::Unknown: return "Unknown"; + case CalledFrom::Create: return "Create"; + case CalledFrom::Read: return "Read"; + case CalledFrom::Write: return "Write"; + } + return "Invalid"; // fallback for safety + } + + LockedFile() = default; + + LockedFile(const std::string& path + , CalledFrom location + //, Mode mode = Mode::ReadWrite + ) + : path_(path), + location_(location)//, mode_(mode) + { + openAndLock(); + } + + ~LockedFile() { + closeFile(); // Ensures cleanup + } + + // Delete copy + LockedFile(const LockedFile&) = delete; + LockedFile& operator=(const LockedFile&) = delete; + + // Move constructor + LockedFile(LockedFile&& other) noexcept { + *this = std::move(other); + } + + // Move assignment + LockedFile& operator=(LockedFile&& other) noexcept { + if (this != &other) { + closeFile(); // Close current file if open + location_ = other.location_; + path_ = std::move(other.path_); + std::swap(file_, other.file_); + std::swap(fd_, other.fd_); + std::swap(fl_, other.fl_ ); // Swap the flock + //mode_ = other.mode_; + } + return *this; + } + + FILE* handle() const { return file_; } + const std::string& path() const { return path_; } + size_t GetFileSize() const; + std::string fileName; + +private: + std::string path_; + FILE* file_ = nullptr; + int fd_ = -1; + struct flock fl_; // Add flock as a member + CalledFrom location_ = CalledFrom::Unknown; + //Mode mode_{};mode is disabled since we would always open file in r+b mode + + void openAndLock(); + void closeFile(); +}; + struct FileBuffer{ int status ;//FILEBUFFERSTATUS std::vector data; size_t offset = 0; - FILE* file = nullptr; - std::string fileName; + LockedFile file; FilebufferType filebufferType; long long originalFileSize = 0; int ReadFromFile(); - //int WriteToFile(); int Flush(bool forced = false); - int atomicWriteWithBackup(); - int SetOffset(size_t newOffset, int fileAnchor); + //Methods below applied only to filebuffer(file in RAM) + int SetOffset(size_t newOffset, int fileAnchor=FILE_BEGIN); int Write(const void* buff, size_t buffSize, size_t startingPosition, bool fSuppressTmdSizeCheck = false); - int Write(const void* buff, size_t buffSize); - int Read(void* buff, size_t buffSize, size_t startingPosition); - int Read(void* buff, size_t buffSize); - bool wasModified() const; + int Read(void* buff, size_t buffSize, ssize_t startingPosition=-1); + bool WasModified() const; + bool isTMDFilebuffer()const; - bool isTMDFilebuffer(); +private: + int atomicWriteWithBackup(); } ; diff --git a/include/OTMAPI.H b/include/OTMAPI.H index 5f2d4ae3..b60447fb 100755 --- a/include/OTMAPI.H +++ b/include/OTMAPI.H @@ -308,24 +308,6 @@ * EQF Typedefs *-----------------------------------------------------------------------------*/ -typedef struct _STEQFPCMD // Pipe Command/Response -{ - USHORT usCmd; // command code - ULONG ulParm1; // parameter 1 - USHORT usParm2; // parameter 2 - USHORT usParm3; // parameter 3 - USHORT usLen1; // length of buffer 1 - USHORT usLen2; // length of buffer 2 - USHORT usLen3; // length of buffer 3 - USHORT usLen4; // length of buffer 4 - UCHAR ucbBuffer [EQF_SEGLEN*8]; // buffer1/2/3/4 - PVOID pvMetaData; // ptr to segment meta data -} STEQFPCMD; - -typedef STEQFPCMD *PSTEQFPCMD; - - - typedef struct _STEQFGEN // generic EQF structure { HWND hwndTWBS; // handle of parent window @@ -361,7 +343,6 @@ typedef struct _STEQFGEN // generic EQF structure USHORT usRC; // return code CHAR_W szMsgBuffer[EQF_NAMELEN]; // message buffer PVOID pDoc; // reserved - PSTEQFPCMD pstEQFPCmd; // reserved pipe command struct ULONG flPropStyle; // window style of prop Window ULONG flDictStyle; // window style of dict.window ULONG flSrcStyle; // window style of prop.src.wnd. @@ -1483,17 +1464,6 @@ EQF_BOOL EQFCHECKSEGW ( ); -EQF_BOOL EQFCHECKSEGEXW ( - PSZ_W pszPrevSrc, // previous source segment - PSZ_W pszSrc, // current source segment - PSZ_W pszTgt, // current translation - PEQF_BOOL pfChanged, // segment changed - LONG lInfo, // info handle to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - ULONG ulSegNum, // segment number to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - EQF_BOOL fMsg // message display requested -); - - /******************************************************************************* * * function: EQFCHECKSEGTYPE diff --git a/include/Proposal.h b/include/Proposal.h index 81f60047..427c13bc 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -72,8 +72,8 @@ class OtmProposal int SetProposalKey ( - unsigned long ulKey, - unsigned short usTargetNum + const long& lKey, + const unsigned short& usTargetNum ); /* \brief set the proposal key @@ -227,7 +227,7 @@ class OtmProposal /* \brief get proposal fuzziness \returns proposal fuzziness */ - int getFuzziness(); + int getFuzziness()const; /* \brief set the proposal fuzziness \param lFuzzinessTime new fuzziness of proposal @@ -281,7 +281,7 @@ class OtmProposal /* \brief set the proposal context \param pszBuffer Pointer to buffer containing the proposal context */ - void setContext( wchar_t *pszBuffer ); + void setContext(const wchar_t *pszBuffer ); /* \brief get proposal context ranking \returns proposal context ranking @@ -308,7 +308,7 @@ class OtmProposal /* \brief set the proposal additional information \param pszBuffer Pointer to buffer containing the additional info */ - void setAddInfo( wchar_t *pszBuffer ); + void setAddInfo(const wchar_t *pszBuffer ); /* \brief set the proposal memory index \param iIndex new value for the memory index @@ -377,94 +377,99 @@ class OtmProposal //protected: /*! \brief ID of this proposal */ - char szId[OTMPROPOSAL_MAXNAMELEN]; + char szId[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Source string of memory proposal (UTF-16) */ - wchar_t szSource[OTMPROPOSAL_MAXSEGLEN]; + wchar_t szSource[OTMPROPOSAL_MAXSEGLEN]{}; //std::wstring strSource; /*! \brief Target string of memory proposal (UTF-16). */ //std::wstring strTarget; - wchar_t szTarget[OTMPROPOSAL_MAXSEGLEN]; + wchar_t szTarget[OTMPROPOSAL_MAXSEGLEN]{}; /*! \brief Name of document from which the proposal comes from. */ //std::string strDocName; - char szDocName[OTMPROPOSAL_MAXNAMELEN]; + char szDocName[OTMPROPOSAL_MAXNAMELEN]{}; /*! \briefCustom Id provided for the segment in request to track segments in response in multifuzzy*/ - char szCustomId[OTMPROPOSAL_MAXNAMELEN]; + char szCustomId[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Segment number within the document from which the proposal comes from. */ - long lSegmentId; + long lSegmentId{}; + bool fSegmentIdWasGenerated{};//if true, for import, reorganize, update, means that id wasn't provided. /*! \brief source language. */ //std::string strSourceLanguage; - char szSourceLanguage[OTMPROPOSAL_MAXNAMELEN]; - char szOriginalSourceLanguage[OTMPROPOSAL_MAXNAMELEN]; + char szSourceLanguage[OTMPROPOSAL_MAXNAMELEN]{}; + char szOriginalSourceLanguage[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief target language. */ //std::string strTargetLanguage; - char szTargetLanguage[OTMPROPOSAL_MAXNAMELEN]; + char szTargetLanguage[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief origin or type of the proposal. */ - eProposalType eType; + eProposalType eType{}; /*! \brief match type of the proposal. */ - eMatchType eMatch; + eMatchType eMatch{}; /*! \brief Author of the proposal. */ // std::string strTargetAuthor; - char szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; + char szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Update time stamp of the proposal. */ - long lTargetTime; + long lTargetTime{}; /*! \brief Fuzziness of the proposal. */ - int iFuzziness, iDiffs, iWords; + int iFuzziness{}, iDiffs{}, iWords{}; /*! \brief Markup table (format) of the proposal. */ - char szMarkup[OTMPROPOSAL_MAXNAMELEN]; + char szMarkup[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Context information of the proposal */ //std::wstring strContext; - wchar_t szContext[OTMPROPOSAL_MAXSEGLEN+1]; + wchar_t szContext[OTMPROPOSAL_MAXSEGLEN+1]{}; /*! \brief Additional information of the proposal */ //std::wstring strAddInfo; - wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]; + wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]{}; /*! \brief Proposal data has been filled flag */ - bool fFilled; + bool fFilled{}; + + /*! \brief when saving request, if set to true, save segment with differen context as new(as was), otherwise if only context differs, replace previous segment */ + //bool fCheckContextWhenReplacingSegment{}; + bool fNewEntryIfOnlyContextAndTimestampDifferent{}; + bool fSaveDifferentTargetsForSameSource{}; + bool fIgnoreSegIdOnUpdate{}; /*! \brief Index of memory when looking up in a list of memories */ - int iMemoryIndex; + int iMemoryIndex{}; /*! \brief ranking of the context information (0..100) */ - int iContextRanking; + int iContextRanking{}; /*! \brief list of replacement values */ - long pvReplacementList; + long pvReplacementList{}; //searchproposal std::unique_ptr pInputSentence = nullptr; - char szMemory[260]; - char szIsoSourceLang[MAX_LANG_LENGTH]; - char szIsoTargetLang[MAX_LANG_LENGTH]; - wchar_t szError[512]; - //char szType[256]; - //char szDateTime[40]; - char szSearchMode[40]; - char szSearchPos[80]; - int iNumOfProposals; - int iSearchTime; - wchar_t szSearchString[2050]; + char szMemory[260]{}; + char szIsoSourceLang[MAX_LANG_LENGTH]{}; + char szIsoTargetLang[MAX_LANG_LENGTH]{}; + wchar_t szError[512]{}; + char szSearchMode[40]{}; + char szSearchPos[80]{}; + int iNumOfProposals{}; + int iSearchTime{}; + wchar_t szSearchString[2050]{}; - // - TMCursor currentInternalKey; - TMCursor nextInternalKey; + TMCursor currentInternalKey{}; + TMCursor nextInternalKey{}; + int moveInternalKeyToTheNextPosition() { currentInternalKey = nextInternalKey; @@ -478,26 +483,73 @@ class OtmProposal int setSequentialAccessKey( char *pszKey); -//otm_put - //USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - //ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - //TIME_L lTime; //time stamp - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import -//otm_put end - - + BOOL fMarkupChanged{}; // Markup does not exist, changed to OTMUTF8 during import + /*! \brief is source lang is marked as prefered in languages.xml */ bool fIsoSourceLangIsPrefered = false; /*! \brief is target lang is marked as prefered in languages.xml */ bool fIsoTargetLangIsPrefered = false; - //searchproposal - - //char errorStr[1000]; }; #include "tm.h" +#include // for std::swap, std::swap_ranges + +/* +void swap(OtmProposal& a, OtmProposal& b) noexcept { + using std::swap; + // Swap raw arrays + swap_ranges(std::begin(a.szId), std::end(a.szId), std::begin(b.szId)); + swap_ranges(std::begin(a.szSource), std::end(a.szSource), std::begin(b.szSource)); + swap_ranges(std::begin(a.szTarget), std::end(a.szTarget), std::begin(b.szTarget)); + swap_ranges(std::begin(a.szDocName), std::end(a.szDocName), std::begin(b.szDocName)); + swap_ranges(std::begin(a.szCustomId), std::end(a.szCustomId), std::begin(b.szCustomId)); + swap_ranges(std::begin(a.szSourceLanguage), std::end(a.szSourceLanguage), std::begin(b.szSourceLanguage)); + swap_ranges(std::begin(a.szOriginalSourceLanguage), std::end(a.szOriginalSourceLanguage), std::begin(b.szOriginalSourceLanguage)); + swap_ranges(std::begin(a.szTargetLanguage), std::end(a.szTargetLanguage), std::begin(b.szTargetLanguage)); + swap_ranges(std::begin(a.szTargetAuthor), std::end(a.szTargetAuthor), std::begin(b.szTargetAuthor)); + swap_ranges(std::begin(a.szMarkup), std::end(a.szMarkup), std::begin(b.szMarkup)); + swap_ranges(std::begin(a.szContext), std::end(a.szContext), std::begin(b.szContext)); + swap_ranges(std::begin(a.szAddInfo), std::end(a.szAddInfo), std::begin(b.szAddInfo)); + swap_ranges(std::begin(a.szMemory), std::end(a.szMemory), std::begin(b.szMemory)); + swap_ranges(std::begin(a.szIsoSourceLang), std::end(a.szIsoSourceLang), std::begin(b.szIsoSourceLang)); + swap_ranges(std::begin(a.szIsoTargetLang), std::end(a.szIsoTargetLang), std::begin(b.szIsoTargetLang)); + swap_ranges(std::begin(a.szError), std::end(a.szError), std::begin(b.szError)); + swap_ranges(std::begin(a.szSearchMode), std::end(a.szSearchMode), std::begin(b.szSearchMode)); + swap_ranges(std::begin(a.szSearchPos), std::end(a.szSearchPos), std::begin(b.szSearchPos)); + swap_ranges(std::begin(a.szSearchString), std::end(a.szSearchString), std::begin(b.szSearchString)); + + // Swap primitive types + swap(a.lSegmentId, b.lSegmentId); + swap(a.fSegmentIdWasGenerated, b.fSegmentIdWasGenerated); + swap(a.eType, b.eType); + swap(a.eMatch, b.eMatch); + swap(a.lTargetTime, b.lTargetTime); + swap(a.iFuzziness, b.iFuzziness); + swap(a.iDiffs, b.iDiffs); + swap(a.iWords, b.iWords); + swap(a.fFilled, b.fFilled); + swap(a.fNewEntryIfOnlyContextAndTimestampDifferent, b.fNewEntryIfOnlyContextAndTimestampDifferent); + swap(a.fSaveDifferentTargetsForSameSource, b.fSaveDifferentTargetsForSameSource); + swap(a.fIgnoreSegIdOnUpdate, b.fIgnoreSegIdOnUpdate); + swap(a.iMemoryIndex, b.iMemoryIndex); + swap(a.iContextRanking, b.iContextRanking); + swap(a.pvReplacementList, b.pvReplacementList); + swap(a.iNumOfProposals, b.iNumOfProposals); + swap(a.iSearchTime, b.iSearchTime); + swap(a.fMarkupChanged, b.fMarkupChanged); + swap(a.fIsoSourceLangIsPrefered, b.fIsoSourceLangIsPrefered); + swap(a.fIsoTargetLangIsPrefered, b.fIsoTargetLangIsPrefered); + + // Swap unique_ptr + swap(a.pInputSentence, b.pInputSentence); + + // Swap TMCursor members + swap(a.currentInternalKey, b.currentInternalKey); + swap(a.nextInternalKey, b.nextInternalKey); +}//*/ + OtmProposal::eProposalType getMemProposalType( char *pszType ); diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 38fc6a18..3c2f6c91 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -16,6 +16,19 @@ #include "ThreadingWrapper.h" +using BytesRef = std::vector&; +template +inline void pushBytes(std::vector& buffer, const T& value) { + const BYTE* dataPtr = reinterpret_cast(&value); + buffer.insert(buffer.end(), dataPtr, dataPtr + sizeof(T)); +} + +inline void pushBytes(std::vector& buffer, const void* start, const void* end) { + const BYTE* beginPtr = static_cast(start); + const BYTE* endPtr = static_cast(end); + + buffer.insert(buffer.end(), beginPtr, endPtr); +} bool isAsanEnabled(); bool isUbsanEnabled(); @@ -34,7 +47,7 @@ class ChunkBuffer{ void setResponseHandler(proxygen::ResponseHandler* responseHandler){m_responseHandler = responseHandler;} bool isActive()const {return m_responseHandler != 0; } - ChunkBuffer(){//m_buff.reserve(chunkSize_+1); + ChunkBuffer(){//m_buff.resize(chunkSize_+1); } ~ChunkBuffer(){ //T5LOG(T5INFO)<< "called dctor of chunk buffer- sending last chunk"; @@ -79,6 +92,7 @@ typedef char* PSZ; SHUTDOWN, DELETE_MEM, EXPORT_MEM_TMX, + GET_TABLES, EXPORT_MEM_INTERNAL_FORMAT, EXPORT_MEM_INTERNAL_FORMAT_STREAM, REORGANIZE_MEM, @@ -94,6 +108,7 @@ typedef char* PSZ; MULTIFUZZY, MULTIUPDATE, CONCORDANCE, + ADD_TO_TABLE, DELETE_ENTRY, GET_ENTRY, DELETE_ENTRIES_REORGANIZE, @@ -122,6 +137,8 @@ constexpr std::initializer_list> CommandT { EXPORT_MEM_TMX_STREAM, "EXPORT_MEM_TMX_STREAM" }, { EXPORT_MEM_INTERNAL_FORMAT_STREAM, "EXPORT_MEM_INTERNAL_FORMAT_STREAM" }, { STATUS_MEM, "STATUS_MEM" }, + { GET_TABLES, "GET_TABLES"}, + { ADD_TO_TABLE, "ADD_TO_TABLE"}, { RESOURCE_INFO, "RESOURCE_INFO" }, { FLAGS_INFO, "FLAGS_INFO" }, @@ -282,15 +299,16 @@ typedef char * PSZ; typedef LONG TIME_L; // new typedef to avoid conflicts with Lotus -typedef struct _RECPARAM +struct RECPARAM { USHORT usNum = 0; // record number USHORT usOffset = 0; // record offset ULONG ulLen = 0; // record length void reset(){ - memset(this, 0 ,sizeof(*this)); + *this = RECPARAM{}; } -} RECPARAM, * PRECPARAM; +}; +using PRECPARAM = RECPARAM*; /**********************************************************************/ /* typedef used for all vital information on our new approach... */ @@ -298,10 +316,11 @@ typedef struct _RECPARAM /**********************************************************************/ typedef struct _TMVITALINFO { - ULONG ulStartKey = 0; // key to start with - ULONG ulNextKey = 0; // currently active key + LONG lStartKey = 0; // key to start with + LONG lNextKey = 0; // currently active key } NTMVITALINFO, * PNTMVITALINFO; +const long LASTALLOWEDKEY = 0xFFFFFF; #define MAX_MEM_DESCRIPTION 41 // length of a memory description field @@ -318,9 +337,9 @@ typedef struct _TMX_SIGN BYTE bGlobVersion=0; BYTE bMajorVersion=0; BYTE bMinorVersion=0; - CHAR szName[_TMX_SIGN_SZ_NAME]; - CHAR szSourceLanguage[MAX_LANG_LENGTH]; - CHAR szDescription[MAX_MEM_DESCRIPTION]; + CHAR szName[_TMX_SIGN_SZ_NAME]{}; + CHAR szSourceLanguage[MAX_LANG_LENGTH]{}; + CHAR szDescription[MAX_MEM_DESCRIPTION]{}; LONG segmentIndex = 0;//highest generated segment id, used for generating new id's } TMX_SIGN, * PTMX_SIGN; @@ -352,7 +371,7 @@ typedef struct _BTREEHEADER /*****************************************************************************/ typedef struct _BTREEHEADRECORD { - CHAR chEQF[7]; // The type of file + CHAR chEQF[7]{}; // The type of file // BYTE bVersion; // version flag struct { @@ -363,19 +382,19 @@ typedef struct _BTREEHEADRECORD unsigned char f16kRec : 1; // TRUE = BTREE uses 16k Recs (V3 record size) } Flags; - bool fOpen; // open flag/corruption flag - USHORT usFirstNode; // first node record - USHORT usFirstLeaf; // first leaf record - RECPARAM DataRecList[ MAX_LIST ]; // last used data records - bool fTerse; // tersing requested - BYTE chCollate[COLLATE_SIZE]; // collating sequence to use - BYTE chCaseMap[COLLATE_SIZE]; // case mapping to be used - BYTE chEntryEncode[ ENTRYENCODE_LEN ]; // significant characters - USHORT usFreeKeyBuffer; // index of buffer to use - USHORT usFreeDataBuffer; // first data buffer chain - USHORT usFirstDataBuffer; // first data buffer - ULONG ulUpdCount; // last update counter - USHORT usNextFreeRecord; // Next record to expand to + bool fOpen=false; // open flag/corruption flag + USHORT usFirstNode=0; // first node record + USHORT usFirstLeaf=0; // first leaf record + RECPARAM DataRecList[ MAX_LIST ]{}; // last used data records + bool fTerse = false; // tersing requested + BYTE chCollate[COLLATE_SIZE]{}; // collating sequence to use + BYTE chCaseMap[COLLATE_SIZE]{}; // case mapping to be used + BYTE chEntryEncode[ ENTRYENCODE_LEN ]{}; // significant characters + USHORT usFreeKeyBuffer=0; // index of buffer to use + USHORT usFreeDataBuffer=0; // first data buffer chain + USHORT usFirstDataBuffer=0; // first data buffer + ULONG ulUpdCount=0; // last update counter + USHORT usNextFreeRecord=0; // Next record to expand to } BTREEHEADRECORD, PBTREEHEADRECORD ; @@ -397,28 +416,29 @@ typedef enum _RECTYPE #define FREE_SIZE_V3 (BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) typedef struct _BTREERECORD_V3 { - BTREEHEADER header; // 16 bytes header - UCHAR uchData[ FREE_SIZE_V3 ] ; // free size to be used + BTREEHEADER header{}; // 16 bytes header + UCHAR uchData[ FREE_SIZE_V3 ]{} ; // free size to be used } BTREERECORD_V3, *PBTREERECORD_V3, **PPBTREERECORD_V3; /*****************************************************************************/ /* BTREEBUFFER is the format of the buffers when read in to the buffer */ /* cache. It maps a record number and its properties to its contents */ /*****************************************************************************/ -typedef struct _BTREEBUFFER_V3 +struct BTREEBUFFER { USHORT usRecordNumber=0; // index of rec in buffer BOOL fLocked=0; // Is the record locked ? - //BOOL fNeedToWrite; // Commit before reuse SHORT sUsed=0; // buffer used count ULONG ulCheckSum=0; // CheckSum of contents data - BTREERECORD_V3 contents; // data from disk -} BTREEBUFFER_V3, *PBTREEBUFFER_V3; + BTREERECORD_V3 contents{}; // data from disk + + //BTREEBUFFER(){memset(this, 0, sizeof(*this));} +}; typedef struct _BTREEINDEX_V3 { struct _BTREEINDEX_V3 * pNext; // point to next index buffer - BTREEBUFFER_V3 btreeBuffer; // data from disk + BTREEBUFFER btreeBuffer; // data from disk } BTREEINDEX_V3, * PBTREEINDEX_V3; @@ -434,12 +454,6 @@ typedef struct _BTREEINDEX_V3 typedef SHORT _PFN_QDAMCOMPARE( PVOID, PVOID, PVOID ); typedef _PFN_QDAMCOMPARE *PFN_QDAMCOMPARE; -typedef struct _LOOKUPENTRY_V3 -{ - PBTREEBUFFER_V3 pBuffer; // Pointer to BTREEBUFFER - ULONG ulAccessCounter; -} LOOKUPENTRY_V3, *PLOOKUPENTRY_V3; - typedef LHANDLE HTM; @@ -466,65 +480,61 @@ typedef LHANDLE HTM; //table entry structure -typedef struct _TMX_TABLE_ENTRY +struct TMX_TABLE_ENTRY { - CHAR szName[MAX_LANG_LENGTH]; + CHAR szName[MAX_LANG_LENGTH]{}; USHORT usId = 0; -} TMX_TABLE_ENTRY, * PTMX_TABLE_ENTRY; +}; +typedef TMX_TABLE_ENTRY * PTMX_TABLE_ENTRY; + // name table structure (TM version 5 and up) constexpr int NUM_OF_TMX_TABLE_ENTRIES = (BTREE_REC_SIZE_V3 - sizeof(ULONG)) / sizeof(TMX_TABLE_ENTRY); -typedef struct TMX_TABLE_OLD{ +struct TMX_TABLE_OLD{ ULONG ulMaxEntries = 0; - TMX_TABLE_ENTRY stTmTableEntry[NUM_OF_TMX_TABLE_ENTRIES]; + TMX_TABLE_ENTRY table[NUM_OF_TMX_TABLE_ENTRIES]; }; typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; -typedef struct TMX_TABLE + +struct TMX_TABLE { - //ULONG ulAllocSize = 0; - ULONG ulMaxEntries = 0; - //std::vector stTmTableEntry;//[NUM_OF_TMX_TABLE_ENTRIES]; - std::vector stTmTableEntry; + std::vector table; TMX_TABLE(){ - stTmTableEntry.resize(NUM_OF_TMX_TABLE_ENTRIES); - for(int i=0;i pszBuffer; // buffer for names and IDs + std::vector stTableEntry; // dyn. array of table entries + void restoreStrPointersAfterRealloc(){ + for(auto& entry: stTableEntry){//during insertion to the pszBuffer, it could be reallocated, so we need to update poiners + if(entry.longNameStartOffset) + entry.pszLongName = (PSZ) &(pszBuffer[entry.longNameStartOffset]); + }; + } +}; BOOL UtlAlloc ( void **, long, long, unsigned short ); -BOOL UtlAllocHwnd ( void**, long, long, unsigned short, void * ); #include "Filebuffer.h" #include @@ -558,41 +568,33 @@ typedef enum _SEARCHTYPE struct BTREE { - bool fGuard =false; // write every record - //BOOL fOpen=false; // open flag bool fTerse =false; // tersing requested ushort usFirstNode=0; // file pointer of record ushort usFirstLeaf=0; // file pointer of record ushort usNextFreeRecord = 0; // Next record to expand to - RECPARAM DataRecList[ MAX_LIST ]; // last used data records - unsigned char chEntryEncode[ ENTRYENCODE_LEN]; // significant characters - unsigned char bEncodeLen[COLLATE_SIZE]; // encoding table length - unsigned char chEncodeVal[COLLATE_SIZE]; // encoding table - unsigned char chDecode[ENTRYDECODE_LEN]; // decoding table + RECPARAM DataRecList[ MAX_LIST ]{}; // last used data records + unsigned char chEntryEncode[ ENTRYENCODE_LEN]{}; // significant characters + unsigned char bEncodeLen[COLLATE_SIZE]{}; // encoding table length + unsigned char chEncodeVal[COLLATE_SIZE]{}; // encoding table + unsigned char chDecode[ENTRYDECODE_LEN]{}; // decoding table NTMVITALINFO chCollate; - unsigned char chCaseMap[COLLATE_SIZE]; // case mapping to be used ushort usFreeKeyBuffer=0; // index of buffer to use ushort usFreeDataBuffer=0; // first data buffer chain ushort usFirstDataBuffer=0; // first data buffer - BTREEBUFFER_V3 BTreeTempBuffer_V3; // temporary V3 buffer + std::shared_ptr BTreeTempBuffer_V3; // temporary V3 buffer long lTime=0; // time of last update/open - ushort usVersion=0; // version identification... - char chEQF[7]; // The type of file + char chEQF[7]{}; // The type of file char bVersion=0; // version flag ushort usOpenFlags=0; // settings used for open - HFILE fpDummy=nullptr; // dummy/lock semaphore file handle - //USHORT usNumberOfLookupEntries=0; // Number of allocated lookup-table-entries ushort usNumberOfAllocatedBuffers=0; // Number of allocated buffers unsigned long ulReadRecCalls=0; // Number of calls to QDAMReadRecord char bRecSizeVersion=0; // record size version flag char sCurrentIndex=0; // current sequence array ushort usCurrentRecord=0; // current sequence record - ushort usDictNum=0; // index in global structure - //wchar_t chHeadTerm[HEADTERM_SIZE]; // last active head term bool fLock =false; // head term is locked unsigned long ulNum=0; bool fDictLock =false; @@ -600,332 +602,70 @@ struct BTREE //new fields FileBuffer fb; std::vector TempRecord; - std::vector LookupTable_V3; // Pointer to lookup-table + std::map> LookupTable; // Pointer to lookup-table + //methods BTREE(){ TempRecord = std::vector(MAXDATASIZE, 0); // Ensure proper initialization for arrays and complex types - std::fill(std::begin(chEntryEncode), std::end(chEntryEncode), 0); - std::fill(std::begin(bEncodeLen), std::end(bEncodeLen), 0); - std::fill(std::begin(chEncodeVal), std::end(chEncodeVal), 0); - std::fill(std::begin(chDecode), std::end(chDecode), 0); - std::fill(std::begin(chCaseMap), std::end(chCaseMap), 0); - //std::fill(std::begin(chCollate), std::end(chCollate), 0); - std::fill(std::begin(chEQF), std::end(chEQF), '\0'); - memset(&BTreeTempBuffer_V3, 0, sizeof(BTreeTempBuffer_V3)); - //std::fill(std::begin(chHeadTerm), std::end(chHeadTerm), L'\0'); + BTreeTempBuffer_V3 = std::make_shared(); } - void freeMem(){ freeLookupTable(); } - - bool wasModified() const{ return fb.wasModified(); } + bool wasModified() const{ return fb.WasModified(); } int initLookupTable(); int resetLookupTable(); - void freeLookupTable(); - int checkLookupTableAndRealocate(int number); - int allocateNewLookupTableBuffer(int number, PLOOKUPENTRY_V3& pLEntry, PBTREEBUFFER_V3& pBuffer); - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMUpdSign Write User Data - //+----------------------------------------------------------------------------+ - // Function call: QDAMDictUpdSignLocal( PBTREE, PCHAR, USHORT ); - // - //+----------------------------------------------------------------------------+ - // Description: Writes the second part of the first record (user data) - // This is done using the original QDAMDictUpdSignLocal - // function - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PCHAR pointer to user data - // USHORT length of user data - // - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_DISK_FULL disk full condition encountered - // BTREE_WRITE_ERROR write error to disk - // BTREE_INVALID pointer invalid - // BTREE_USERDATA user data too long - // BTREE_CORRUPTED dictionary is corrupted - //+----------------------------------------------------------------------------+ - // NOTE: This function could be implemented as MACRO too, but - // for consistency reasons, the little overhead was used... - //+----------------------------------------------------------------------------+ + void freeLookupTable(); + int allocateNewLookupTableBuffer(int number,std::shared_ptr& pBuffer); + int deallocateLookupTableBuffer(int number); ULONG GetNumOfSavedRecords()const; - SHORT QDAMDictFlushLocal(); - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMSign Read signature record - //+----------------------------------------------------------------------------+ - // Function call: QDAMDictSignLocal( PBTREE, PCHAR, PUSHORT ); - //+----------------------------------------------------------------------------+ - // Description: Gets the second part of the first record ( user data ) - // This is done using the original QDAMDictSignLocal func. - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PCHAR pointer to user data - // PUSHORT length of user data area (input) - // filled length (output) - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_INVALID pointer invalid - // BTREE_USERDATA user data too long - // BTREE_NO_BUFFER no buffer free - // BTREE_READ_ERROR read error from disk - // - //+----------------------------------------------------------------------------+ - // Side effects: return signature record even if dictionary is corrupted - //+----------------------------------------------------------------------------+ - // NOTE: This function could be implemented as MACRO too, but - // for consistency reasons, the little overhead was used... - //+----------------------------------------------------------------------------+ - - SHORT EQFNTMSign - ( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data - ); - - - VOID QDAMTerseInit (unsigned char* ); - - //SHORT - //EQFNTMInsert - //( - // PULONG pulKey, // pointer to key - // PBYTE pData, // pointer to user data - // ULONG ulLen // length of user data - //); - - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMInsert - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMInsert( pBTIda, &ulKey, pData, usLen ); - //+----------------------------------------------------------------------------+ - // Description: insert a new key (ULONG) with data - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to binary tree struct - // PULONG pulKey, pointer to key - // PBYTE pData, pointer to user data - // ULONG ulLen length of user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: BTREE_NUMBER_RANGE requested key not in allowed range - // BTREE_READONLY file is opened read only - no write - // BTREE_CORRUPTED file is corrupted - // errors returned by QDAMDictInsertLocal - // 0 success indicator - //+----------------------------------------------------------------------------+ + SHORT EQFNTMSign(TMX_SIGN& ); + VOID QDAMTerseInit (unsigned char* ); SHORT EQFNTMInsert ( - PULONG pulKey, // pointer to key - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + LONG& lKey, // pointer to key + BytesRef pData // pointer to user data ); - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMUpdate - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMUpdate( pBTIda, ulKey, pData, usLen ); - //+----------------------------------------------------------------------------+ - // Description: update the data of an already inserted key - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to binary tree struct - // ULONG ulKey, key value - // PBYTE pData, pointer to user data - // ULONG ulLen length of user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: BTREE_NUMBER_RANGE requested key not in allowed range - // BTREE_READONLY file is opened read only - no write - // BTREE_CORRUPTED file is corrupted - // errors returned by QDAMDictInsertLocal - // 0 success indicator - //+----------------------------------------------------------------------------+ SHORT EQFNTMUpdate ( - ULONG ulKey, // key value - PBYTE pData, // pointer to user data - ULONG ulLen, // length of user data + LONG lKey, // key value + BytesRef pData, // pointer to user data bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed ); - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGet - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMGet( pBTIda, ulKey, chData, &usLen ); - //+----------------------------------------------------------------------------+ - // Description: get the data string for the passed key - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to btree struct - // ULONG ulKey, key to be searched for - // PCHAR pchBuffer, space for user data - // PULONG pulLength in/out length of returned user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: same as for QDAMDictExactLocal... - //+----------------------------------------------------------------------------+ - SHORT - EQFNTMGet - ( - ULONG ulKey, // key to be searched for - PCHAR pchBuffer, // space for user data - PULONG pulLength // in/out length of returned user data - ); - - - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGetMaxNumber - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMGetNextNumber( pBTIda, &ulKey, &ulNextFree ); - //+----------------------------------------------------------------------------+ - // Description: get the start key and the next free key ... - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to btree struct - // PULONG pulStartKey first key - // PULONG pulNextKey next key to be assigned - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 always - //+----------------------------------------------------------------------------+ - // Function flow: access data from internal structure - //+----------------------------------------------------------------------------+ - SHORT - EQFNTMGetNextNumber - ( - PULONG pulStartKey, // return start key number - PULONG pulNextKey // return next key data - ); - - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMPhysLock - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMPhysLock( pBTIda ); - //+----------------------------------------------------------------------------+ - // Description: Physicall lock or unlock database. - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE The database to be locked - // BOOL TRUE = LOCK, FALSE = Unlock - // PBOOL ptr to locked flag (set to TRUE if - // locking was successful - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ SHORT EQFNTMPhysLock ( - BOOL fLock, - PBOOL pfLocked - ); - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: QDAMIncrUpdCounter Inrement database update counter - //+----------------------------------------------------------------------------+ - // Function call: QDAMIncrUpdCounter( PBTREE, SHORT sIndex ) - // - //+----------------------------------------------------------------------------+ - // Description: Update one of the update counter field in the dummy - // /locked terms file - // - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // SHORT index of counter field - // PLONG ptr to buffer for new counte value - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_DISK_FULL disk full condition encountered - // BTREE_WRITE_ERROR write error to disk - // - //+----------------------------------------------------------------------------+ - // Function flow: read update counter from dummy file - // increment update counter - // position ptr to begin of file - // write update counter to disk - //+----------------------------------------------------------------------------+ - - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGetUpdCounter Get database update counter - //+----------------------------------------------------------------------------+ - // Function call: EQFNTMGetUpdCounter( PBTREE, PLONG, SHORT, SHORT ); - //+----------------------------------------------------------------------------+ - // Description: Get one or more of the the database update counters - // from the dummy/locked terms file - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PLONG ptr to buffer for update counter - // SHORT index of requested update counter - // SHORT number of counters requested - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - SHORT EQFNTMGetUpdCounter - ( - PLONG plUpdCount, // ptr to buffer for update counter - SHORT sIndex, // index of requested update counter - SHORT sNumCounters // number of counters requested + BOOL fLock, + PBOOL pfLocked ); - - VOID QDAMUpdateList_V3( PBTREEBUFFER_V3 ); + VOID QDAMUpdateList_V3( std::shared_ptr& ); - VOID QDAMFreeFromList_V3(PRECPARAM ,PBTREEBUFFER_V3 ); - SHORT QDAMFreeRecord_V3( PBTREEBUFFER_V3 pRecord, RECTYPE recType/* data or key record*/); + VOID QDAMFreeFromList_V3(PRECPARAM ,std::shared_ptr& ); + SHORT QDAMFreeRecord_V3( std::shared_ptr& pRecord, RECTYPE recType/* data or key record*/); - ULONG QDAMGetrecDataLen_V3 ( PBTREEBUFFER_V3, SHORT ); + ULONG QDAMGetrecDataLen_V3 ( std::shared_ptr&, SHORT ); SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); - SHORT QDAMDictUpdateLocal ( PWCHAR, PBYTE, ULONG, bool ); + SHORT QDAMDictUpdateLocal ( PWCHAR, BytesRef, bool ); - SHORT QDAMDictExactLocal ( PWCHAR, PBYTE, PULONG, USHORT ); - + SHORT EQFNTMGet ( LONG&, BytesRef ); - SHORT QDAMDictCreateLocal ( TMX_SIGN*, ULONG, bool keepInRamOnly = false); + SHORT QDAMDictCreateLocal ( TMX_SIGN*, LONG, bool keepInRamOnly = false); - SHORT QDAMDictInsertLocal ( PWCHAR, PBYTE, ULONG ); - BOOL QDAMDictLockStatus ( PWCHAR ); + SHORT QDAMDictInsertLocal ( LONG&, BytesRef pData ); + BOOL QDAMDictLockStatus ( const LONG& ); VOID QDAMDictUpdStatus (); - SHORT QDAMFindRecord_V3( PWCHAR, PBTREEBUFFER_V3 * ); - + SHORT QDAMFindRecord_V3( LONG&, std::shared_ptr&); //------------------------------------------------------------------------------ // Internal function @@ -1032,146 +772,108 @@ struct BTREE SHORT QDAMReadRecord_V3 ( - USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, - BOOL fNewRec, - bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed + USHORT usNumber, + std::shared_ptr& pReadBuffer, + BOOL fNewRec, + bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed ); - SHORT QDAMAddToBuffer_V3( PBYTE, ULONG, PRECPARAM, bool); + SHORT QDAMAddToBuffer_V3( BytesRef, PRECPARAM, bool); SHORT QDAMWriteHeader (); SHORT QDAMDictUpdSignLocal ( - PTMX_SIGN pSign // pointer to user data + PTMX_SIGN pSign // pointer to user data ); - SHORT QDAMReadRecordFromDisk_V3( USHORT, PBTREEBUFFER_V3 *, BOOL , bool); + SHORT QDAMReadRecordFromDisk_V3( USHORT, std::shared_ptr&, BOOL, bool ); - SHORT QDAMWRecordToDisk_V3( PBTREEBUFFER_V3, - bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed - ); - SHORT QDAMWriteRecord_V3( PBTREEBUFFER_V3 pBuffer, - bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed - ); + SHORT QDAMWRecordToDisk_V3( std::shared_ptr&, + bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed + ); + SHORT QDAMWriteRecord_V3( std::shared_ptr& pBuffer, + bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed + ); SHORT QDAMAllocKeyRecords( USHORT usNum); - SHORT QDAMDictOpenLocal( - SHORT sNumberOfBuffers, // number of buffers - USHORT usOpenFlags // Read Only or Read/Write + SHORT EQFNTMOpen( + USHORT usOpenFlags, // Read Only or Read/Write + SHORT sNumberOfBuffers=20 // number of buffers ); - - SHORT EQFNTMOpen( USHORT usOpenFlags); // Read Only or Read/Write SHORT QDAMCheckDict( PSZ pName); // name of dictionary + SHORT QDAMDictClose(); - //SHORT QDAMDictCloseLocal (); - //+----------------------------------------------------------------------------+ -// External function -//+----------------------------------------------------------------------------+ -// Function name: QDAMDictClose close the dictionary -//+----------------------------------------------------------------------------+ -// Function call: QDAMDictClose( PPBTREE ); -// -//+----------------------------------------------------------------------------+ -// Description: Close the file -// -//+----------------------------------------------------------------------------+ -// Parameters: PPBTREE pointer to btree structure -//+----------------------------------------------------------------------------+ -// Returncode type: SHORT -//+----------------------------------------------------------------------------+ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -//+----------------------------------------------------------------------------+ -SHORT QDAMDictClose(); - - -SHORT QDAMDictSignLocal -( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data -); - -//+----------------------------------------------------------------------------+ -// Internal function -//+----------------------------------------------------------------------------+ -// Function name: EQFNTMClose close the TM file -//+----------------------------------------------------------------------------+ -// Function call: EQFNTMClose( PPBTREE ); -//+----------------------------------------------------------------------------+ -// Description: Close the file -//+----------------------------------------------------------------------------+ -// Parameters: PPBTREE pointer to btree structure -//+----------------------------------------------------------------------------+ -// Returncode type: SHORT -//+----------------------------------------------------------------------------+ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -//+----------------------------------------------------------------------------+ - -SHORT EQFNTMClose(); - -SHORT QDAMNewRecord_V3 -( - PBTREEBUFFER_V3 * ppRecord, - RECTYPE recType, // data or key record - bool fSuppressTmdSizeCheck = false -); - -SHORT QDAMInsertKey_V3 -( - PBTREEBUFFER_V3 pRecord, // record where key is to be inserted - PWCHAR pKey, - RECPARAM recKey, // position/offset for key - RECPARAM recData // position/offset for data -); - -size_t GetFileSize()const; + SHORT QDAMDictSignLocal + ( + PCHAR pUserData, // pointer to user data + PUSHORT pusLen // length of user data + ); + //+----------------------------------------------------------------------------+ + // Internal function + //+----------------------------------------------------------------------------+ + // Function name: EQFNTMClose close the TM file + //+----------------------------------------------------------------------------+ + // Function call: EQFNTMClose( PPBTREE ); + //+----------------------------------------------------------------------------+ + // Description: Close the file + //+----------------------------------------------------------------------------+ + // Parameters: PPBTREE pointer to btree structure + //+----------------------------------------------------------------------------+ + // Returncode type: SHORT + //+----------------------------------------------------------------------------+ + // Returncodes: 0 no error happened + // BTREE_INVALID incorrect pointer + // BTREE_DISK_FULL disk full condition encountered + // BTREE_WRITE_ERROR write error to disk + // BTREE_CORRUPTED dictionary is corrupted + // BTREE_CLOSE_ERROR error closing dictionary + //+----------------------------------------------------------------------------+ -BOOL QDAMTerseData -( - PUCHAR pData, // pointer to data - PULONG pulLen // length of the string -); + SHORT EQFNTMClose(); + SHORT QDAMNewRecord_V3 + ( + std::shared_ptr& pRecord, + RECTYPE recType, // data or key record + bool fSuppressTmdSizeCheck = false + ); - SHORT QDAMLocateKey_V3( - PBTREEBUFFER_V3, PWCHAR, PSHORT, SEARCHTYPE, PSHORT); + SHORT QDAMInsertKey_V3 + ( + std::shared_ptr& pRecord, // record where key is to be inserted + LONG& ulKey, + RECPARAM recKey, // position/offset for key + RECPARAM recData // position/offset for data + ); + + size_t GetFileSize()const; + + BOOL QDAMTerseData + ( + BytesRef , + PULONG pulLen // length of the string + ); + SHORT QDAMLocateKey_V3( + std::shared_ptr&, LONG&, PSHORT, SEARCHTYPE, PSHORT); + + SHORT QDAMGetszData_V3 ( RECPARAM, BytesRef ); };//BTREE typedef BTREE * PBTREE; typedef BTREE ** PPBTREE; - - -#define NTMNEXTKEY( pBT ) pBT->chCollate.ulNextKey -#define NTMSTARTKEY( pBT ) pBT->chCollate.ulStartKey - - - - - //typedef TMX_CLB* PTMX_CLB; //prefix for each output structures typedef struct _TMX_PREFIX_OUT { - USHORT usLengthOutput; //length of complete output structure - USHORT usTmtXRc; //function returncode + USHORT usLengthOutput = 0; //length of complete output structure + USHORT usTmtXRc = 0; //function returncode } TMX_PREFIX_OUT, * PTMX_PREFIX_OUT, XOUT, * PXOUT; - #define MEM_EXPORT_OUT_BUFFER 8200 // Output buffer for export #define MAX_HANDLER_NAME 40 @@ -1416,19 +1118,22 @@ class TMXParseHandler; //class SAXParser; struct LOADEDTABLE; -typedef struct _TOKENENTRY // entry in tokenlist : +struct TOKENENTRY // entry in tokenlist : { // !!!! Attention: below has to match TOKENENTRYSEG definition .... !!!! - SHORT sTokenid; // Tokenid - int iLength; // Length of data string - SHORT sAddInfo; // additional information from tag table - CHAR * pDataString; // pointer to data string - USHORT usOrgId; // original id - USHORT ClassId; // class id of token - CHAR_W * pDataStringW; // pointer to data string - Unicode + SHORT sTokenid{}; // Tokenid + int iLength{}; // Length of data string + SHORT sAddInfo{}; // additional information from tag table + CHAR * pDataString{}; // pointer to data string + USHORT usOrgId{}; // original id + USHORT ClassId{}; // class id of token + CHAR_W * pDataStringW{}; // pointer to data string - Unicode // !!!! Attention: above has to match TOKENENTRYSEG definition .... !!!! + + void reset(){ *this = TOKENENTRY{};} -} TOKENENTRY, *PTOKENENTRY; +}; +using PTOKENENTRY = TOKENENTRY*; class ImportStatusDetails; @@ -1517,7 +1222,6 @@ typedef struct _MEM_EXPORT_IDA USHORT usRC; // error code in non-DDE batch mode USHORT usExpMode; // mode for export CHAR szConvArea[MEM_EXPORT_WORK_AREA]; // Work area for extract - PSZ pszNameList; // pointer to list of TMs being organized PSZ pszActiveName; // points to current name in pszNameList USHORT usYesToAllMode; // yes-to-all mode for replace existing files ULONG ulOemCP; // ASCII cp of system preferences language @@ -1525,8 +1229,8 @@ typedef struct _MEM_EXPORT_IDA // fields for external memory export methods CTMXExportImport* lExternalExportHandle; // handle of external memory export functions //HMODULE hmodMemExtExport; // handle of external export module/DLL - MEMEXPIMPINFO* pstMemInfo; // buffer for memory information - MEMEXPIMPSEG* pstSegment; // buffer for segment data + MEMEXPIMPINFO pstMemInfo; // buffer for memory information + MEMEXPIMPSEG pstSegment; // buffer for segment data BOOL fDataCorrupted; // TRUE = data has been corrupted during export CHAR_W szBackConvArea[MEM_EXPORT_WORK_AREA]; // Work area for conversion check @@ -1535,7 +1239,6 @@ typedef struct _MEM_EXPORT_IDA CHAR_W szSegmentBuffer[MAX_SEGMENT_SIZE]; // Buffer for segment data CHAR_W szSegmentBuffer2[MAX_SEGMENT_SIZE]; // Buffer for preprocessed segment data int iComplete; // completion rate - CHAR szPlugin[MAX_LONGFILESPEC]; // name of memory plugin handling the current memory database int invalidXmlSegments=0; long numOfRequestedSegmentsForExport = 0; @@ -1591,6 +1294,7 @@ typedef enum _PROCWINSTYLE #define Err_Prop 2000 // property handler rc #define Err_ObjM 2100 // object manager rc #define Err_Nfol 2200 // new folder messages +#define ERR_SERVICE_BASE 3000 #define ERR_BTREE_BASE 5000 #define ERR_BTREE_END 5999 #define ERR_MEM_BASE 6000 @@ -1651,6 +1355,9 @@ typedef enum _PROCWINSTYLE #define ErrProp_InvalidHandle (Err_Prop+23) // invalid handle to properties +#define ERR_WRITE_REQUESTS_NOT_ALLOWED ERR_SERVICE_BASE + 1 //3000 +#define ERR_LOADING_TM_ARE_NOT_ALLOWED ERR_SERVICE_BASE + 2 //3000 + #define BTREE_BASE ERR_BTREE_BASE //5000 #define BTREE_NO_ROOM BTREE_BASE+1 // Insufficent memory #define BTREE_ILLEGAL_FILE BTREE_BASE+2 // Not an index file @@ -1701,8 +1408,9 @@ typedef enum _PROCWINSTYLE /**********************************************************************/ /* Return Codes */ /**********************************************************************/ -#define TMERR_BASE ERR_MEM_BASE /* Base for errors */ +#define TMERR_BASE ERR_MEM_BASE /* Base for errors 6000 */ +#define TMERR_LANG_TOO_SMALL_FOR_REORGANIZE TMERR_BASE + 11 #define TMERR_EOF TMERR_BASE + 12 #define DISK_FULL TMERR_BASE + 13 // for Add /Replace / Create @@ -2258,14 +1966,14 @@ struct LOADEDTABLE { CHAR szName[MAX_FNAME]; // tag table name (w/o path and ext.) SHORT sUseCount; // number of active table users - PTAGTABLE pTagTable; // pointer to tag table in memory - PNODE pTagTree; // Tag tree of table - PNODE pAttrTree; // attribute tree of table - PNODEAREA pNodeArea; // area for node trees - PNODEAREA pAttrNodeArea; // area for attribute node tree + PTAGTABLE pTagTable = nullptr; // pointer to tag table in memory + PNODE pTagTree = nullptr; // Tag tree of table + PNODE pAttrTree = nullptr; // attribute tree of table + PNODEAREA pNodeArea = nullptr; ; // area for node trees + PNODEAREA pAttrNodeArea = nullptr; ; // area for attribute node tree HMODULE hmodUserExit; // user exit (if any) BOOL fUserExitLoadFailed; // TRUE = load of user exit failed - PFN pfnProtTable; // creat eprotect table function + PFN pfnProtTable = nullptr; ; // creat eprotect table function BOOL fProtTableLoadFailed; // TRUE = ProtTable function load failed // new fields contained in tagtables TAGTABLE_VERSION3 and above // these fields are copied from the TAGTABLE structure (if available) or filled @@ -2276,13 +1984,13 @@ struct LOADEDTABLE CHAR szDescription[MAX_DESCRIPTION]; // format description USHORT usCharacterSet; // characterset to use for import/export BOOL fUseUnicodeForSegFile; // TRUE = store segmented files in Unicode format - PSZ_W pTagNamesW; - PSZ_W pAttributesW; - PVOID pTagTableW; - PFN pfnCompareContext; // compare context info function + std::wstring pTagNamesW; + std::wstring pAttributesW; + PVOID pTagTableW = nullptr; ; + PFN pfnCompareContext = nullptr; ; // compare context info function BOOL fCompareContextLoadFailed; // TRUE = CompareContext function load failed - PFN pfnProtTableW; // create protect table function (Unicode) - PFNGETSEGCONTTEXT pfnGetSegContext; // get segment context function (Unicode) + PFN pfnProtTableW = nullptr; ; // create protect table function (Unicode) + PFNGETSEGCONTTEXT pfnGetSegContext = nullptr; ; // get segment context function (Unicode) BOOL fGetSegContextLoadFailed; // TRUE = GetSegContext function load failed BOOL fReflow; // 0=notspec. 1= Reflow allowed, 2= no Reflow PFN pfnQueryExitInfo; // query exit info function @@ -2318,7 +2026,7 @@ class ChunkBufferOld{ void setResponseHandler(proxygen::ResponseHandler* responseHandler){m_responseHandler = responseHandler;} bool isActive()const {return m_responseHandler != 0; } - ChunkBufferOld(){m_buff.reserve(chunkSize_+1);} + ChunkBufferOld(){m_buff.resize(chunkSize_+1);} ~ChunkBufferOld(){ T5LOG(T5INFO)<< "called dctor of chunk buffer- sending last chunk"; triggerChunkSend(); @@ -2377,12 +2085,12 @@ class CTMXExportImportOld unsigned int m_iSourceSize; // size of source file PTOKENENTRY m_pTokBuf; // buffer for TaTagTokenize tokens CHAR m_szActiveTagTable[50]; // buffer for name of currently loaded markup table - PLOADEDTABLE m_pLoadedTable; // pointer to currently loaded markup table - PLOADEDTABLE m_pLoadedRTFTable; // pointer to loaded RTF tag table + PLOADEDTABLE m_pLoadedTable = nullptr; // pointer to currently loaded markup table + PLOADEDTABLE m_pLoadedRTFTable = nullptr; // pointer to loaded RTF tag table CHAR m_szInFile[512]; // buffer for input file CHAR m_TempFile[540]; // buffer for temporary file name BYTE m_bBuffer[TMX_BUFFER_SIZE+1]; - PMEMEXPIMPINFO m_pMemInfo; + PMEMEXPIMPINFO m_pMemInfo = nullptr; CHAR_W m_szSegBuffer[MAX_SEGMENT_SIZE+1]; // buffer for the processing of segment data int m_currentTu; // export: number of currently processed tu }; @@ -2439,25 +2147,23 @@ struct MEM_LOAD_DLG_IDA PTOKENENTRY pTokenEntry; // A pointer to token entries PTOKENENTRY pTokenEntryWork; // A work pointer to token entries - MEMEXPIMPINFO* pstMemInfo; // buffer for memory information + MEMEXPIMPINFO pstMemInfo; // buffer for memory information CTMXExportImport TMXImport; // handle of external memory import functions PLOADEDTABLE pFormatTable; // Pointer to Format Table - MEMEXPIMPSEG* pstSegment; // buffer for segment data + MEMEXPIMPSEG pstSegment; // buffer for segment data BOOL fEOF; // Indicates end of file }; class TMCursor { - ulong _recordKey; - ushort _targetKey; + long _recordKey{}; + ushort _targetKey{}; public: - - - int SplitProposalKeyIntoRecordAndTarget(const char *pszKey, ulong *pulKey, ushort *pusTargetNum); + int SplitProposalKeyIntoRecordAndTarget(const char *pszKey, long *pulKey, ushort *pusTargetNum); int parseAndSetInternalKey(const char* pszKey){ return SplitProposalKeyIntoRecordAndTarget(pszKey, &_recordKey, &_targetKey); } @@ -2474,7 +2180,7 @@ class TMCursor return setInternalKey(_recordKey+1, 1); } - int setInternalKey(ulong recordKey, ushort targetKey){ + int setInternalKey(const ulong recordKey, const ushort targetKey){ _recordKey = recordKey; _targetKey = targetKey; return 0; @@ -2489,7 +2195,7 @@ class TMCursor return std::to_string(_recordKey) + ":" + std::to_string(_targetKey) ; } - ulong getRecordKey()const { return _recordKey; } + long getRecordKey()const { return _recordKey; } ushort getTargetKey()const { return _targetKey; } }; @@ -2500,38 +2206,37 @@ class TMCursor typedef struct _FCTDATA { std::string fileData; - ULONG lMagicWord; // FUNCDATA area identifier - LONG lCheckSum; // checksum of FUNCDATA area - SHORT sLastFunction; // last function performed + ULONG lMagicWord{}; // FUNCDATA area identifier + LONG lCheckSum{}; // checksum of FUNCDATA area + SHORT sLastFunction{}; // last function performed // area for session data - std::atomic fComplete; // current process is complete flag - CHAR szEqfResFile[MAX_EQF_PATH];// name of resource file - CHAR szSystemPropPath[MAX_EQF_PATH]; // system properties path - CHAR szMsgFile[MAX_EQF_PATH]; // message file for error messages - CHAR szError[256]; // text buffer for error messages - DDEMSGBUFFER LastMessage; // buffer for last error message - HWND hwndErrMsg; // handle for error mesages (HWND_FUNCIF) + //std::atomic + BOOL fComplete{}; // current process is complete flag + CHAR szEqfResFile[MAX_EQF_PATH]{};// name of resource file + CHAR szSystemPropPath[MAX_EQF_PATH]{}; // system properties path + CHAR szMsgFile[MAX_EQF_PATH]{}; // message file for error messages + CHAR szError[256]{}; // text buffer for error messages + DDEMSGBUFFER LastMessage{}; // buffer for last error message + HWND hwndErrMsg{}; // handle for error mesages (HWND_FUNCIF) // area used by mem import - MEM_LOAD_DLG_IDA MemLoadIda; // pointer to analysis input data - USHORT usMemLoadPhase; // current/next phase of TM import - USHORT usMemLoadRC; // return code + MEM_LOAD_DLG_IDA MemLoadIda{}; // pointer to analysis input data + USHORT usMemLoadPhase{}; // current/next phase of TM import + USHORT usMemLoadRC{}; // return code // area used by mem organize - PVOID pvMemOrganizeCommArea; // pointer to organize CommArea + PVOID pvMemOrganizeCommArea{}; // pointer to organize CommArea // area used by mem export - PVOID pvMemExportCommArea; + PVOID pvMemExportCommArea{}; // general use buffer area - OBJNAME szObjName; // buffer for object names + OBJNAME szObjName{}; // buffer for object names // current progress of some nonDDE functions //USHORT usProgress; - USHORT usExportProgress; - - + USHORT usExportProgress{}; USHORT MemFuncExportProcess(); USHORT MemFuncPrepExport ( @@ -2543,13 +2248,13 @@ typedef struct _FCTDATA std::shared_ptr mem; proxygen::ResponseHandler* responseHandler = nullptr; - TMCursor startingInternalKey; - TMCursor nextInternalKey; + TMCursor startingInternalKey{}; + TMCursor nextInternalKey{}; ulong numOfProposalsRequested = 0; - folly::IOBufQueue bufQueue; - MutexTimeout tmLockTimeout; - MutexTimeout tmListTimeout; + folly::IOBufQueue bufQueue{}; + MutexTimeout tmLockTimeout{}; + MutexTimeout tmListTimeout{}; } FCTDATA, *PFCTDATA; @@ -2815,4 +2520,15 @@ enum FilterField{ */ int convertTimeToUTC( long lTime, char *pszDateTime ); +typedef struct _FUZZYTOK +{ + PSZ_W pData = nullptr; // pointer to data.. + ULONG ulHash = 0; // hash value + USHORT usStart = 0; // start offset + USHORT usStop = 0; // end offset + SHORT sType = 0; // type of entry /* KIT0857M */ + EQF_BOOL fConnected = 0; // true if connected to pPropTokList + EQF_BOOL fNoTagSubst = 0; // true = do not use for tag substitution +} FUZZYTOK, *PFUZZYTOK; + #endif //_LOW_LEVEL_OTM_DATA_STRUCTS_INCLUDED_ diff --git a/include/otm.h b/include/otm.h index d8aa03bc..9ef240b3 100644 --- a/include/otm.h +++ b/include/otm.h @@ -16,6 +16,11 @@ /**********************************************************************/ #define NTMREQUESTNEWKEY 0xFFFFFFFF +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(saveDifferentTargetsForSameSource); +//DECLARE_bool(ignoreSegIdOnUpdate); + +constexpr bool FLAGS_ignoreSegIdOnUpdate = true;//not used flag //+----------------------------------------------------------------------------+ // Internal function @@ -60,32 +65,46 @@ class JSONFactory; #include "lowlevelotmdatastructs.h" struct ImportStatusDetails{ - std::atomic_short usProgress{0}; - std::atomic_int segmentsCount{0}; - std::atomic_int segmentsImported{0}; - std::atomic_int invalidSegments{0}; - std::atomic_int invalidSymbolErrors{0}; - std::atomic_int resSegments {0}; - //std::atomic_long lActiveSegment{0}; + std::mutex importDetailsMutex; + short usProgress{0}; + int segmentsCount{0}; + int segmentsImported{0}; + int invalidSegments{0}; + int invalidSymbolErrors{0}; + int resSegments {0}; int importRc{0}; long filteredSegments{0}; + + //std::mutex invalidSegmentsRCsMutex; std::map invalidSegmentsRCs; + + //std::mutex importTimestampMutex; std::string importTimestamp; + + //std::mutex firstInvalidSegmentsSegNumsMutex; std::vector> firstInvalidSegmentsSegNums; long lImportStartTime; long lImportTimeoutSec = 0; - std::atomic_long lImportRunTimeSec; + long lImportRunTimeSec; + + //std::mutex importMsgMutex; std::stringstream importMsg; bool fReorganize{0}; // true for reorganize call, false for import - size_t activeSegment{0}; InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fSaveDifferentTargetsForSameSource{}; + BOOL fIgnoreSegIdOnUpdate{}; void reset(){ + std::scoped_lock l(importDetailsMutex); + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; + //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + //std::lock_guard l(firstInvalidSegmentsSegNumsMutex); firstInvalidSegmentsSegNums.clear(); firstInvalidSegmentsSegNums.reserve(100); importMsg.str(""); usProgress = 0; - //lActiveSegment = 0; importRc = 0; segmentsCount = 0; segmentsImported = 0; @@ -96,12 +115,14 @@ struct ImportStatusDetails{ lImportStartTime = 0; lImportTimeoutSec = 0; lImportRunTimeSec = 0; - importTimestamp = "not finished"; + //std::lock_guard lk2(importTimestampMutex); + importTimestamp = "not finished"; + //std::lock_guard lk(invalidSegmentsRCsMutex); invalidSegmentsRCs.clear(); - activeSegment = 0; } std::string toString(){ + std::scoped_lock l(importDetailsMutex); std::string res = "progress = " + std::to_string(usProgress); if(fReorganize){ res += "; segmentsReorganized = "; diff --git a/include/requestdata.h b/include/requestdata.h index 65e470a1..7cbd9600 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _REQUEST_DATA_H_INCLUDED_ #define _REQUEST_DATA_H_INCLUDED_ @@ -27,6 +28,7 @@ class RequestData{ int requestType = 0; std::string strTempFile; + bool fSuppressBuildErrorLog{}; //RequestData(); // json was parsed in sub class public: //std::recursive_timed_mutex request_mutex; @@ -144,6 +146,31 @@ class CreateMemRequestData: public RequestData{ +class GetTablesMemRequestData: public RequestData{ +public: + GetTablesMemRequestData(): RequestData(COMMAND::GET_TABLES, "", "") {}; +protected: + int parseJSON() override{return 0;}; + int checkData() override{return 0;}; + int execute() override ; +}; + + +class AddToTableMemRequestData: public RequestData{ +public: + AddToTableMemRequestData(): RequestData(COMMAND::ADD_TO_TABLE, "", "") {}; +protected: + int tableKey{}; + int entryId{}; + BOOL fOverwrite = false; + char value[256]{}; + int parseJSON() override; + int checkData() override; + int execute() override ; +}; + + + class ListTMRequestData:public RequestData{ public: ListTMRequestData(): RequestData(COMMAND::LIST_OF_MEMORIES) {}; @@ -159,25 +186,17 @@ class ListTMRequestData:public RequestData{ //class ProxygenService::ProxygenStats; /*! \brief Data area for the processing of the importMemory function */ -class IMPORTMEMORYDATA +struct IMPORTMEMORYDATA { -public: - //HSESSION hSession; - //OtmMemoryServiceWorker *pMemoryServiceWorker; - char szMemory[260]; - char szInFile[260]; - char szError[512]; + char szMemory[260]{}; + char szInFile[260]{}; + char szError[512]{}; std::shared_ptr mem; BOOL fDeleteTmx = false; - //BOOL fImpoortFromFile = 0; - //folly::IOBuf fileData; - std::string fileData; + std::string fileData{}; InclosingTagsBehaviour inclosingTagsBehaviour = InclosingTagsBehaviour::saveAll; - MutexTimeout tmLockTimeout; - MutexTimeout tmListTimeout; - //ushort * pusImportPersent = nullptr; - //ImportStatusDetails* importDetails = nullptr; - //OtmMemoryServiceWorker::std::shared_ptr pMem = nullptr; + MutexTimeout tmLockTimeout{}; + MutexTimeout tmListTimeout{}; }; class FlushMemRequestData:public RequestData{ @@ -195,22 +214,23 @@ class ImportRequestData:public RequestData{ std::string strTmxData; bool isBase64; - - protected: int parseJSON() override ; int checkData() override ; int execute() override ; BOOL fClose = false; - //MEMORY_STATUS lastImportStatus = AVAILABLE_STATUS; // to restore in case we would break import before calling closemem - //MEMORY_STATUS lastStatus = AVAILABLE_STATUS; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fSaveDifferentTargetsForSameSource{}; + BOOL fIgnoreSegIdOnUpdate{}; InclosingTagsBehaviour inclosingTagsBehaviour; IMPORTMEMORYDATA* pData = nullptr; - long timeout = 0; + int64_t timeout = 0; }; #include + +#ifdef TO_BE_REMOVED class ImportStreamRequestData:public RequestData{ public: ImportStreamRequestData(): RequestData(COMMAND::IMPORT_MEM_STREAM) { }; @@ -227,7 +247,7 @@ class ImportStreamRequestData:public RequestData{ IMPORTMEMORYDATA* pData = nullptr; long timeout = 0; }; - +#endif class ImportLocalRequestData:public RequestData{ public: ImportLocalRequestData(): RequestData(COMMAND::IMPORT_LOCAL_MEM) {}; @@ -365,7 +385,10 @@ class ReorganizeRequestData:public RequestData{ public: ReorganizeRequestData(): RequestData(COMMAND::REORGANIZE_MEM) {}; protected: - int parseJSON() override {return 0;}; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fSaveDifferentTargetsForSameSource{}; + BOOL fIgnoreSegIdOnUpdate{}; + int parseJSON() override; int checkData() override; int execute () override; }; @@ -375,12 +398,14 @@ class UnknownRequestData: public RequestData{ public: UnknownRequestData(): RequestData(UNKNOWN_COMMAND){}; + std::string methodStr,srcAddress; + protected: int parseJSON() override {return 0;}; int checkData() override {return 0;}; int execute() override { _rest_rc_ = 404; - std::string msg = "Url \"" + strUrl + "\" was not parsed correctly"; + std::string msg = "Called from " + srcAddress + "; Method str:\"" + methodStr +"\"; Url:\"" + strUrl + "\" was not parsed correctly"; return buildErrorReturn(404, msg.c_str(), 404); }; }; @@ -409,7 +434,7 @@ class StatusMemRequestData: public RequestData{ int checkData() override ; int execute() override ; - static std::string prepareStatusString(std::shared_ptr mem); + static std::string prepareStatusString(std::shared_ptr& mem); }; class FlagsRequestData: public RequestData{ @@ -478,6 +503,7 @@ class FuzzySearchRequestData: public RequestData{ protected: char szDateTime[100]; char szType[100]; + BOOL fForceFuzzy = 0; int parseJSON() override ; int checkData() override ; @@ -545,6 +571,9 @@ class DeleteEntriesReorganizeRequestData: public RequestData{ protected: SearchFilterFactory searchFilterFactory; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fSaveDifferentTargetsForSameSource{}; + BOOL fIgnoreSegIdOnUpdate{}; int parseJSON() override ; int checkData() override ; @@ -559,7 +588,7 @@ class UpdateEntryRequestData: public RequestData{ int reset(); BOOL fSave2Disk = 1; - OtmProposal Data ; + OtmProposal Data{}; protected: int parseJSON() override; int checkData() override; diff --git a/include/tm.h b/include/tm.h index 0595e0f9..60be0444 100644 --- a/include/tm.h +++ b/include/tm.h @@ -17,7 +17,7 @@ #include "ThreadingWrapper.h" -#define INCL_EQF_TAGTABLE // tag table and format functions +#define INCL_EQF_TAGTABLE // tag table and format functions #define INCL_EQF_TP #define INCL_EQF_TM #define INCL_EQF_DAM @@ -29,134 +29,126 @@ #include "../source/opentm2/core/pluginmanager/OtmPlugin.h" #include "../source/opentm2/core/utilities/Stopwatch.hpp" -#define MEM_START_ORGANIZE USER_TASK + 1 -#define MEM_ORGANIZE_TASK USER_TASK + 2 -#define MEM_END_ORGANIZE USER_TASK + 3 - - - -#define KEY_DIR_SIZE 4096 // key directory size -#define TM_PREFIX_SIZE 8 // length of prefix bytes in TMT db -#define MAX_SEC_LENGTH 30 // max length of secondary key -#define MAX_LINE_LENGTH 80 // max length of each line in lang file -#define FN_LENGTH 13 // length of input filename -#define PRIM_KEY_LENGTH 4 // number of characters in primary key -#define DATA_IN_SIZE 3900 // buffer size for input -#define DATA_OUT_SIZE 3900 // buffer size for output -#define CODEPAGE_SIZE 256 // size of codepage of the language -#define SEG_MARKER_LENGTH 3 // length of segment marker -#define MAX_TGT_LENGTH 2047 // max length of each target. -#define MAX_MATCH_TAB_ENTRIES 5 // number of entries in match table -#define CREATE_BUFFER_SIZE 40000 // buffer size for create_in -#define MAX_TM_LIST_NUMBER 500 // max. number of TMs that can be listed - // by the TMC_GET_SERVER_TM_LIST command -#define GETPART_BUFFER_SIZE 16384 // read a 16 KB block at a time -#define MEM_PROP_SIZE 2048 // Global size of all memory database properties -#define _MAX_DIR 256 - -typedef CHAR SHORT_FN [FN_LENGTH]; -typedef CHAR BUFFERIN [DATA_IN_SIZE]; -typedef UCHAR BUFFEROUT [DATA_OUT_SIZE], - ACHPRIMKEY [PRIM_KEY_LENGTH], - SZSECKEY [MAX_SEC_LENGTH + 1]; - -typedef CHAR LANG_LINE [MAX_LINE_LENGTH + 1]; -typedef CHAR LONG_FN [MAX_LONGFILESPEC]; - - +#define MEM_START_ORGANIZE USER_TASK + 1 +#define MEM_ORGANIZE_TASK USER_TASK + 2 +#define MEM_END_ORGANIZE USER_TASK + 3 + +#define KEY_DIR_SIZE 4096 // key directory size +#define TM_PREFIX_SIZE 8 // length of prefix bytes in TMT db +#define MAX_SEC_LENGTH 30 // max length of secondary key +#define MAX_LINE_LENGTH 80 // max length of each line in lang file +#define FN_LENGTH 13 // length of input filename +#define PRIM_KEY_LENGTH 4 // number of characters in primary key +#define DATA_IN_SIZE 3900 // buffer size for input +#define DATA_OUT_SIZE 3900 // buffer size for output +#define CODEPAGE_SIZE 256 // size of codepage of the language +#define SEG_MARKER_LENGTH 3 // length of segment marker +#define MAX_TGT_LENGTH 2047 // max length of each target. +#define MAX_MATCH_TAB_ENTRIES 5 // number of entries in match table +#define CREATE_BUFFER_SIZE 40000 // buffer size for create_in +#define MAX_TM_LIST_NUMBER 500 // max. number of TMs that can be listed + // by the TMC_GET_SERVER_TM_LIST command +#define GETPART_BUFFER_SIZE 16384 // read a 16 KB block at a time +#define MEM_PROP_SIZE 2048 // Global size of all memory database properties +#define _MAX_DIR 256 + +typedef CHAR SHORT_FN[FN_LENGTH]; +typedef CHAR BUFFERIN[DATA_IN_SIZE]; +typedef UCHAR BUFFEROUT[DATA_OUT_SIZE], + ACHPRIMKEY[PRIM_KEY_LENGTH], + SZSECKEY[MAX_SEC_LENGTH + 1]; + +typedef CHAR LANG_LINE[MAX_LINE_LENGTH + 1]; +typedef CHAR LONG_FN[MAX_LONGFILESPEC]; struct TMX_EXT_OUT_W { - TMX_PREFIX_OUT stPrefixOut; //prefix of output buffer - - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence - CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]; //language name of the source - CHAR szTagTable[MAX_FNAME]; //tag table name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_FILESPEC]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - TIME_L lTargetTime; //time stamp of target - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - ULONG ulRecKey; - USHORT usTargetKey; - - ULONG ulTmKey; //tm record key - USHORT usNextTarget; //which target record to address next - ULONG ulMaxEntries; //number of entries in tm data file + TMX_PREFIX_OUT stPrefixOut{}; // prefix of output buffer + + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence + CHAR_W szTarget[MAX_SEGMENT_SIZE]{}; // target sentence + CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]{}; // language name of the source + CHAR szTagTable[MAX_FNAME]{}; // tag table name + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language name of target + CHAR szAuthorName[MAX_FILESPEC]{}; // author name of target + USHORT usTranslationFlag = 0; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSourceSegmentId = 0; // seg. num. of source sentence from analysis + TIME_L lTargetTime = 0; // time stamp of target + std::wstring szContext, szAddInfo; + ULONG ulRecKey = 0; + USHORT usTargetKey = 0; + + LONG lTmKey = 0; // tm record key + USHORT usNextTarget = 0; // which target record to address next + LONG lMaxEntries = 0; // number of entries in tm data file + TMX_EXT_OUT_W(){}; }; using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; - - -//#include "../pluginmanager/PluginManager.h" +// #include "../pluginmanager/PluginManager.h" #include "EQF.H" // ************ Translation memory flags ******************************** #define far -#define MEM_OUTPUT_ASIS 0 // Do not convert translation memory output -#define MEM_OUTPUT_CRLF 1 // Convert translation memory output to CRLF -#define MEM_OUTPUT_LF 2 // Convert translation memory output to LF +#define MEM_OUTPUT_ASIS 0 // Do not convert translation memory output +#define MEM_OUTPUT_CRLF 1 // Convert translation memory output to CRLF +#define MEM_OUTPUT_LF 2 // Convert translation memory output to LF // defines for special names mode of TMExtract (only valid with TMExtract!) -#define MEM_OUTPUT_TAGTABLES 3 // return list of tag tables of TM -#define MEM_OUTPUT_AUTHORS 4 // return list of authors of TM -#define MEM_OUTPUT_DOCUMENTS 5 // return list of documents of TM -#define MEM_OUTPUT_LANGUAGES 6 // return list of languages of TM -#define MEM_OUTPUT_LONGNAMES 7 // return list of document long names of TM -#define MEM_OUTPUT_ALLDOCS 8 // return list of all documensts (long - // names and short names for docs w/o long name) - -#define EXCLUSIVE 0 -#define NONEXCLUSIVE 1 -#define EXCLUSIVE_FOR_GET_PART 2 -#define FOR_ORGANIZE 3 -#define READONLYACCESS 4 +#define MEM_OUTPUT_TAGTABLES 3 // return list of tag tables of TM +#define MEM_OUTPUT_AUTHORS 4 // return list of authors of TM +#define MEM_OUTPUT_DOCUMENTS 5 // return list of documents of TM +#define MEM_OUTPUT_LANGUAGES 6 // return list of languages of TM +#define MEM_OUTPUT_LONGNAMES 7 // return list of document long names of TM +#define MEM_OUTPUT_ALLDOCS 8 // return list of all documensts (long + // names and short names for docs w/o long name) + +#define EXCLUSIVE 0 +#define NONEXCLUSIVE 1 +#define EXCLUSIVE_FOR_GET_PART 2 +#define FOR_ORGANIZE 3 +#define READONLYACCESS 4 // defines used in TmOpen and MemCreateDlg -#define TM_LOCAL 0 // TM resides local -#define TM_REMOTE 1 // TM resides remote on a server -#define TM_LOCALREMOTE 2 // TM may be local or remote -#define TM_SHARED 3 // TM resides on a shared drive +#define TM_LOCAL 0 // TM resides local +#define TM_REMOTE 1 // TM resides remote on a server +#define TM_LOCALREMOTE 2 // TM may be local or remote +#define TM_SHARED 3 // TM resides on a shared drive // Match value constants: the similarity level classes (lSimLevel) -#define BASE_SIMILAR 0L -#define BASE_EXACT_DATE 200L -#define BASE_EXACT_IND 300L -#define BASE_EXACT_SEG_NUM 400L - -#define MAX_SIMILAR_VAL 100L -#define MAX_EXACT_DATE_VAL (BASE_EXACT_IND - 1L) -#define MAX_EXACT_IND_VAL (BASE_EXACT_SEG_NUM - 1L) +#define BASE_SIMILAR 0L +#define BASE_EXACT_DATE 200L +#define BASE_EXACT_IND 300L +#define BASE_EXACT_SEG_NUM 400L + +#define MAX_SIMILAR_VAL 100L +#define MAX_EXACT_DATE_VAL (BASE_EXACT_IND - 1L) +#define MAX_EXACT_IND_VAL (BASE_EXACT_SEG_NUM - 1L) #define MAX_EXACT_SEG_NUM_VAL (500L - 1L) -#define EXTENT_SIMILAR (MAX_SIMILAR_VAL - BASE_SIMILAR) -#define EXTENT_EXACT_DATE (MAX_EXACT_DATE_VAL - BASE_EXACT_DATE) -#define EXTENT_EXACT_IND (MAX_EXACT_IND_VAL - BASE_EXACT_IND) -#define EXTENT_EXACT_SEG_NUM (MAX_EXACT_SEG_NUM_VAL - BASE_EXACT_SEG_NUM) +#define EXTENT_SIMILAR (MAX_SIMILAR_VAL - BASE_SIMILAR) +#define EXTENT_EXACT_DATE (MAX_EXACT_DATE_VAL - BASE_EXACT_DATE) +#define EXTENT_EXACT_IND (MAX_EXACT_IND_VAL - BASE_EXACT_IND) +#define EXTENT_EXACT_SEG_NUM (MAX_EXACT_SEG_NUM_VAL - BASE_EXACT_SEG_NUM) // translation flag values -#define TRANSLFLAG_NORMAL 0 -#define TRANSLFLAG_MACHINE 1 -#define TRANSLFLAG_GLOBMEM 2 +#define TRANSLFLAG_NORMAL 0 +#define TRANSLFLAG_MACHINE 1 +#define TRANSLFLAG_GLOBMEM 2 #define TRANSLFLAG_GLOBMEMSTAR 3 - /*----------------------------------------------------------------------------*\ System wide threshold values \*----------------------------------------------------------------------------*/ -#define LENGTH_THR 50L // Length threshold value -#define MAX_LENGTH_THR 100L // maximal Length threshold value -#define INIT_MATCH_THR 59L // Initial match threshold value -#define SHORTER_MATCH_THR 50L // match threshold value for shorter - // segments, defined thru LENGTH_SHORTER_VALUE -#define WORDS_MATCH_THR 0L // Not used yet +#define LENGTH_THR 50L // Length threshold value +#define MAX_LENGTH_THR 100L // maximal Length threshold value +#define INIT_MATCH_THR 59L // Initial match threshold value +#define SHORTER_MATCH_THR 50L // match threshold value for shorter + // segments, defined thru LENGTH_SHORTER_VALUE +#define WORDS_MATCH_THR 0L // Not used yet //--- Used in GET_IN structure to specify kind of matches. //--- This defines should be bitwise 'ored' with the number of required matches. @@ -167,61 +159,56 @@ using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; //--- If you want to retrieve translation memory hits with exact context only, //--- use GET_EXACT_CONTEXT. //--- If Generic replace should be disabled, you have to or with GET_NO_GENERICREPLACE -#define GET_MOREPROP_INDIC 0x00000200 -#define GET_ALL_EXACT_MATCHES 0x00000400 -#define GET_RESPECTCRLF 0x00000800 -#define GET_IGNORE_PATH 0x00001000 -#define GET_NO_GENERICREPLACE 0x00002000 -#define GET_EXACT_AND_CONTEXT 0x00004000 -#define GET_EXACT_AND_FUZZY 0x00008000 -#define GET_ALWAYS_WITH_TAGS 0x00010000 -#define GET_IGNORE_COMMENT 0x00020000 -#define GET_EXACT 0x00040000 - -#define EQUAL_EQUAL 100 -#define TAGS_EQUAL 98 -#define TAGS_UNEQUAL 95 - -#define NON_EXCLUSIVE 1 -#define EXCLUSIVE 0 - -#define LANG_KEY 1L -#define FILE_KEY 2L -#define AUTHOR_KEY 3L -#define TAGTABLE_KEY 4L -//#define RESERVED_KEY 5L -#define LONGNAME_KEY 5L -#define COMPACT_KEY 6L -#define FIRST_KEY 7L +#define GET_MOREPROP_INDIC 0x00000200 +#define GET_ALL_EXACT_MATCHES 0x00000400 +#define GET_RESPECTCRLF 0x00000800 +#define GET_IGNORE_PATH 0x00001000 +#define GET_NO_GENERICREPLACE 0x00002000 +#define GET_EXACT_AND_CONTEXT 0x00004000 +#define GET_EXACT_AND_FUZZY 0x00008000 +#define GET_ALWAYS_WITH_TAGS 0x00010000 +#define GET_IGNORE_COMMENT 0x00020000 +#define GET_EXACT 0x00040000 + +#define EQUAL_EQUAL 100 +#define TAGS_EQUAL 98 +#define TAGS_UNEQUAL 95 + +#define NON_EXCLUSIVE 1 +#define EXCLUSIVE 0 + +#define LANG_KEY 1L +#define FILE_KEY 2L +#define AUTHOR_KEY 3L +#define TAGTABLE_KEY 4L +// #define RESERVED_KEY 5L +#define LONGNAME_KEY 5L +#define COMPACT_KEY 6L +#define FIRST_KEY 7L // Note: the following key is NOT used as the key of a QDAM record // it is only a symbolic value used in the Name-to-ID functions -#define LANGGROUP_KEY 8L - +#define LANGGROUP_KEY 8L // used in GET_OUT structure -#define GET_MORE_EXACTS_AVAIL 0x8000 +#define GET_MORE_EXACTS_AVAIL 0x8000 #define GET_ADDITIONAL_FUZZY_AVAIL 0x4000 - - /* The segment marker */ -#define SEGMARKER "###" - +#define SEGMARKER "###" /**********************************************************************/ /* indicatos for old or new TM */ /**********************************************************************/ -#define OLD_TM 0 -#define NEW_TM 1 +#define OLD_TM 0 +#define NEW_TM 1 /**********************************************************************/ /* indicator for organize */ /* TM_CONVERT - convert an ols TM to an new one */ /* TM_ORGANIZE - organize a new TM */ /**********************************************************************/ -#define TM_CONVERT 0 -#define TM_ORGANIZE 1 - +#define TM_CONVERT 0 +#define TM_ORGANIZE 1 /**********************************************************************/ /* default threshold value for get */ @@ -230,91 +217,92 @@ using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; // against the new computed fuzziness, the old value is used // for the triple threshold checking /**********************************************************************/ -#define TM_DEFAULT_THRESHOLD 33 // prior: 40 -#define TM_FUZZINESS_THRESHOLD 10 - +#define TM_DEFAULT_THRESHOLD 33 // prior: 40 +#define TM_FUZZINESS_THRESHOLD 10 #include "win_types.h" - - -typedef BYTE ABGROUP [CODEPAGE_SIZE]; -typedef ABGROUP * PABGROUP; - -typedef struct _TM_ADDRSS { /* addr */ - USHORT usEntryInDir, /* >= 0, entry# of the cluster in KeyDir */ - /* <= 4095 */ - usBlockNumber, /* > 0, indicates block number in TM */ - /* 0 used for chaining purposes as null */ - usDispBlockPtr; /* size(block header) <= BLOCK_SIZE */ - /* indicates location within a block */ -} TM_ADDRESS, * PTM_ADDRESS ; - -typedef struct _SEGMENT { /* seg */ - UCHAR achSegMarker[SEG_MARKER_LENGTH] ; - /* marker to beginning of segment */ - USHORT usLenSegment, /* total length of segment */ - usDispIndustry, /* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource, /* source length in bytes */ - usDispTarget, /* target location relative to bufData */ - usLenTarget, /* target length in bytes */ - usDispReserved, /* Reserved area location in bufData */ - usLenReserved, /* length of reserved area */ - usDispSecKey, /* secondary key location in bufData */ - usLenSecKey, /* length of secondary key */ - usDispContext, /* Context area location in bufData */ - usLenContext; /* length of context area */ - SHORT_FN szFileName; /* name of source file (short name) */ - LONG_FN szLongName; /* name of source file (long name or EOS)*/ - USHORT usSegNumber; /* segment number in file */ - EQF_BOOL fLogicalDel, /* set when logically deleted segment */ - fAscii; /* */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - TIME_L tStamp; /* time taken from c function */ - BUFFERIN bufData; /* fields with variable lengths */ -} SEGMENT, * PSEGMENT; - -typedef struct _OLDSEGMENT { /* seg */ - UCHAR achSegMarker[SEG_MARKER_LENGTH] ; - /* marker to beginning of segment */ - USHORT usLenSegment, /* total length of segment */ - usDispIndustry, /* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource, /* source length in bytes */ - usDispTarget, /* target location relative to bufData */ - usLenTarget, /* target length in bytes */ - usDispReserved, /* Reserved area location in bufData */ - usLenReserved, /* length of reserved area */ - usDispSecKey, /* secondary key location in bufData */ - usLenSecKey; /* length of secondary key */ - SHORT_FN szFileName; /* name of source file (short name) */ - USHORT usSegNumber; /* segment number in file */ - EQF_BOOL fLogicalDel, /* set when logically deleted segment */ - fAscii; /* */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - TIME_L tStamp; /* time taken from c function */ - BUFFERIN bufData; /* fields with variable lengths */ -} OLDSEGMENT, * POLDSEGMENT; - -typedef struct _MATCH { /* mtch */ - TM_ADDRESS addr; /* address of the matching segment */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - USHORT usNumExactBytes;/* no. of exact bytes in the */ - /* input and matched sentence */ - TIME_L tStamp; /* time stamp of segment */ - LONG lSimLevel, /* <= 100 indicates similar */ - /* >= 200 indicate exact */ - /* >= 300 indicate exact+same ind code */ - /* >= 400 indicate exact+same filename */ - lLengthTest, /* value of lengths test */ - lInitMatchTest, /* value of initials test */ - lWordsTest; /* value of words test */ - /* last 3 values are meaningless for exact matches*/ - UCHAR szTarget[MAX_TGT_LENGTH + 1]; /* The target */ -} MATCH , * PMATCH; +typedef BYTE ABGROUP[CODEPAGE_SIZE]; +typedef ABGROUP *PABGROUP; + +typedef struct _TM_ADDRSS +{ /* addr */ + USHORT usEntryInDir, /* >= 0, entry# of the cluster in KeyDir */ + /* <= 4095 */ + usBlockNumber, /* > 0, indicates block number in TM */ + /* 0 used for chaining purposes as null */ + usDispBlockPtr; /* size(block header) <= BLOCK_SIZE */ + /* indicates location within a block */ +} TM_ADDRESS, *PTM_ADDRESS; + +typedef struct _SEGMENT +{ /* seg */ + UCHAR achSegMarker[SEG_MARKER_LENGTH]; + /* marker to beginning of segment */ + USHORT usLenSegment, /* total length of segment */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource, /* source length in bytes */ + usDispTarget, /* target location relative to bufData */ + usLenTarget, /* target length in bytes */ + usDispReserved, /* Reserved area location in bufData */ + usLenReserved, /* length of reserved area */ + usDispSecKey, /* secondary key location in bufData */ + usLenSecKey, /* length of secondary key */ + usDispContext, /* Context area location in bufData */ + usLenContext; /* length of context area */ + SHORT_FN szFileName; /* name of source file (short name) */ + LONG_FN szLongName; /* name of source file (long name or EOS)*/ + USHORT usSegNumber; /* segment number in file */ + EQF_BOOL fLogicalDel, /* set when logically deleted segment */ + fAscii; /* */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + TIME_L tStamp; /* time taken from c function */ + BUFFERIN bufData; /* fields with variable lengths */ +} SEGMENT, *PSEGMENT; + +typedef struct _OLDSEGMENT +{ /* seg */ + UCHAR achSegMarker[SEG_MARKER_LENGTH]; + /* marker to beginning of segment */ + USHORT usLenSegment, /* total length of segment */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource, /* source length in bytes */ + usDispTarget, /* target location relative to bufData */ + usLenTarget, /* target length in bytes */ + usDispReserved, /* Reserved area location in bufData */ + usLenReserved, /* length of reserved area */ + usDispSecKey, /* secondary key location in bufData */ + usLenSecKey; /* length of secondary key */ + SHORT_FN szFileName; /* name of source file (short name) */ + USHORT usSegNumber; /* segment number in file */ + EQF_BOOL fLogicalDel, /* set when logically deleted segment */ + fAscii; /* */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + TIME_L tStamp; /* time taken from c function */ + BUFFERIN bufData; /* fields with variable lengths */ +} OLDSEGMENT, *POLDSEGMENT; + +typedef struct _MATCH +{ /* mtch */ + TM_ADDRESS addr; /* address of the matching segment */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + USHORT usNumExactBytes; /* no. of exact bytes in the */ + /* input and matched sentence */ + TIME_L tStamp; /* time stamp of segment */ + LONG lSimLevel, /* <= 100 indicates similar */ + /* >= 200 indicate exact */ + /* >= 300 indicate exact+same ind code */ + /* >= 400 indicate exact+same filename */ + lLengthTest, /* value of lengths test */ + lInitMatchTest, /* value of initials test */ + lWordsTest; /* value of words test */ + /* last 3 values are meaningless for exact matches*/ + UCHAR szTarget[MAX_TGT_LENGTH + 1]; /* The target */ +} MATCH, *PMATCH; /* More details about the match ranking method. @@ -351,106 +339,117 @@ typedef struct _MATCH { /* mtch */ described both in the LLD document and in code comments. */ -typedef struct _PREFIX_IN { - USHORT usLenIn, /* total length of input buffer */ - idCommand ; /* command id, previously defined. */ -// } PREFIX_IN, * PPREFIX_IN, IN, * PIN; -} PREFIX_IN, * PPREFIX_IN, * PIN; +typedef struct _PREFIX_IN +{ + USHORT usLenIn, /* total length of input buffer */ + idCommand; /* command id, previously defined. */ + // } PREFIX_IN, * PPREFIX_IN, IN, * PIN; +} PREFIX_IN, *PPREFIX_IN, *PIN; -typedef struct _PREFIX_OUT { - USHORT usLenOut; /* total length of output buffer */ - BOOL fDiskFull, /* for Add /Replace / Create when number*/ - fDBfull; /* of blocks is a cluster exceeds */ - /* 2**16 - 1. Filled by FormatMore */ - USHORT rcTmt; /* returned by TMT function */ +typedef struct _PREFIX_OUT +{ + USHORT usLenOut; /* total length of output buffer */ + BOOL fDiskFull, /* for Add /Replace / Create when number*/ + fDBfull; /* of blocks is a cluster exceeds */ + /* 2**16 - 1. Filled by FormatMore */ + USHORT rcTmt; /* returned by TMT function */ -//} PREFIX_OUT, * PPREFIX_OUT, OUT, * POUT; -} PREFIX_OUT, * PPREFIX_OUT, * POUT; + //} PREFIX_OUT, * PPREFIX_OUT, OUT, * POUT; +} PREFIX_OUT, *PPREFIX_OUT, *POUT; -typedef struct _UPDATE_IN { - PREFIX_IN prefin; /* prefix of each command */ - SEGMENT segIn; /* the rest of this type is a SEGMENT */ -} UPDATE_IN, * PUPDATE_IN; +typedef struct _UPDATE_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SEGMENT segIn; /* the rest of this type is a SEGMENT */ +} UPDATE_IN, *PUPDATE_IN; -typedef UPDATE_IN ADD_IN, DEL_IN, REP_IN; +typedef UPDATE_IN ADD_IN, DEL_IN, REP_IN; typedef PUPDATE_IN PADD_IN, PDEL_IN, PREP_IN; -typedef struct _ADD_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - TM_ADDRESS addr; /* address of added segment */ -} ADD_OUT, * PADD_OUT; - -typedef struct _DEL_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - TM_ADDRESS addr; /* address of deleted segment */ -} DEL_OUT , * PDEL_OUT ; - -typedef struct _REP_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - SERVERNAME szServer; /* server name */ - TM_ADDRESS addrDel, /* address to delete */ - addrAdd; /* address to add */ -} REP_OUT , * PREP_OUT; - -typedef struct _CLOSE_IN { - PREFIX_IN prefin ; /* prefix of each command */ -} CLOSE_IN, * PCLOSE_IN; - -typedef struct _CLOSE_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ -} CLOSE_OUT, * PCLOSE_OUT; - -typedef struct _EXT_IN { - PREFIX_IN prefin; /* prefix of each command */ - TM_ADDRESS addr; /* address of extracted segment */ - USHORT usConvert; // Indicates how the output should be converted -} EXT_IN , * PEXT_IN ; - -typedef struct _EXT_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - SERVERNAME szServer; /* server name */ - TM_ADDRESS addr, /* address of current segment */ - addrNext; /* address of next segment */ - SEGMENT segOut; /* fields with variable lengths */ -} EXT_OUT , * PEXT_OUT; - -typedef struct _GET_IN { - PREFIX_IN prefin; /* prefix of each command */ - SHORT_FN szFileName; /* name of source file (short name) */ - LONG_FN szLongName; /* name of source file (long name or EOS) */ - USHORT usConvert; // Indicates how the output should be converted - USHORT usSegNumber, /* */ - usNumMatchesReq,/* number of matches required */ - usDispIndustry,/* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource; /* source length in bytes */ - LONG lLengthThr, /* needed to filter out sentences */ - /* with different lengths */ - lInitMatchThr, /* needed to filter out sentences */ - /* with non matching initials */ - lWordsMatchThr;/* needed to filter out sentences */ - /* with non matching words */ - BOOL fAscii; /* */ - BUFFERIN bufData; /* fields with variable lengths */ -} GET_IN, * PGET_IN; +typedef struct _ADD_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + TM_ADDRESS addr; /* address of added segment */ +} ADD_OUT, *PADD_OUT; + +typedef struct _DEL_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + TM_ADDRESS addr; /* address of deleted segment */ +} DEL_OUT, *PDEL_OUT; + +typedef struct _REP_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + SERVERNAME szServer; /* server name */ + TM_ADDRESS addrDel, /* address to delete */ + addrAdd; /* address to add */ +} REP_OUT, *PREP_OUT; + +typedef struct _CLOSE_IN +{ + PREFIX_IN prefin; /* prefix of each command */ +} CLOSE_IN, *PCLOSE_IN; + +typedef struct _CLOSE_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ +} CLOSE_OUT, *PCLOSE_OUT; + +typedef struct _EXT_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + TM_ADDRESS addr; /* address of extracted segment */ + USHORT usConvert; // Indicates how the output should be converted +} EXT_IN, *PEXT_IN; + +typedef struct _EXT_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + SERVERNAME szServer; /* server name */ + TM_ADDRESS addr, /* address of current segment */ + addrNext; /* address of next segment */ + SEGMENT segOut; /* fields with variable lengths */ +} EXT_OUT, *PEXT_OUT; + +typedef struct _GET_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SHORT_FN szFileName; /* name of source file (short name) */ + LONG_FN szLongName; /* name of source file (long name or EOS) */ + USHORT usConvert; // Indicates how the output should be converted + USHORT usSegNumber, /* */ + usNumMatchesReq, /* number of matches required */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource; /* source length in bytes */ + LONG lLengthThr, /* needed to filter out sentences */ + /* with different lengths */ + lInitMatchThr, /* needed to filter out sentences */ + /* with non matching initials */ + lWordsMatchThr; /* needed to filter out sentences */ + /* with non matching words */ + BOOL fAscii; /* */ + BUFFERIN bufData; /* fields with variable lengths */ +} GET_IN, *PGET_IN; /**********************************************************************/ /* structure passed in case of rename a file ... */ /**********************************************************************/ typedef struct _RENFILE_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; // server name - CHAR szOldFile[ MAX_EQF_PATH ]; // old file name - CHAR szNewFile[ MAX_EQF_PATH ]; // new file name - CHAR szUserId [MAX_USERID]; // userId logged on to requester -} RENFILE_IN, * PRENFILE_IN; + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; // server name + CHAR szOldFile[MAX_EQF_PATH]; // old file name + CHAR szNewFile[MAX_EQF_PATH]; // new file name + CHAR szUserId[MAX_USERID]; // userId logged on to requester +} RENFILE_IN, *PRENFILE_IN; typedef struct _RENFILE_OUT { - PREFIX_OUT prefout; /* prefix of each command */ -} RENFILE_OUT, * PRENFILE_OUT; + PREFIX_OUT prefout; /* prefix of each command */ +} RENFILE_OUT, *PRENFILE_OUT; /* More details about lLengthThr and lInitMatchThr: @@ -505,78 +504,84 @@ typedef struct _RENFILE_OUT */ -typedef struct _GET_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - - USHORT usNumMatchesFound,/* number of matches found */ - usNumMatchesValid,/* number of matches valid */ - ausSortedMatches[MAX_MATCH_TAB_ENTRIES]; - MATCH amtchBest[MAX_MATCH_TAB_ENTRIES]; /* Matches array */ -} GET_OUT, * PGET_OUT; - -typedef struct _OPEN_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* servername */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ - FILENAME szTmFileName; /* name of Tm database to open */ - BOOL fExclusive; /* indicate the mode for opening the file */ - BOOL fOpenGetPart; /* indicate if Open was due to a get part */ -} OPEN_IN , * POPEN_IN; - - -typedef struct _OPEN_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - HTM htm; /* handle to the TM */ -} OPEN_OUT , * POPEN_OUT; - -typedef struct _BLOCK_HEADER { /* bh */ - USHORT usBlockNum, /* >= 1, points to previous block */ - usPrevBlock, /* >= 1, points to previous block */ - usNextBlock, /* > 1, points to next block. */ - /* 0 indicates no chaining */ - usFirstAvailByte; /* >= size of block header, */ - /* <= block size */ - /* points to first available byte in block */ - /* If block is full it sets to BlockSize */ -} BLOCK_HEADER , * PBLOCK_HEADER; - -typedef struct _CREATE_TMH { /* ctmh */ - FILENAME szTmFileName; /* Tm database name */ - USHORT usBlockSize; - LANGUAGE szSourceLang, /* source language */ - szTargetLang; /* target language */ - EQF_BOOL fDbcs; /* double byte char support 0/1 */ +typedef struct _GET_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + + USHORT usNumMatchesFound, /* number of matches found */ + usNumMatchesValid, /* number of matches valid */ + ausSortedMatches[MAX_MATCH_TAB_ENTRIES]; + MATCH amtchBest[MAX_MATCH_TAB_ENTRIES]; /* Matches array */ +} GET_OUT, *PGET_OUT; + +typedef struct _OPEN_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* servername */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ + FILENAME szTmFileName; /* name of Tm database to open */ + BOOL fExclusive; /* indicate the mode for opening the file */ + BOOL fOpenGetPart; /* indicate if Open was due to a get part */ +} OPEN_IN, *POPEN_IN; + +typedef struct _OPEN_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + HTM htm; /* handle to the TM */ +} OPEN_OUT, *POPEN_OUT; + +typedef struct _BLOCK_HEADER +{ /* bh */ + USHORT usBlockNum, /* >= 1, points to previous block */ + usPrevBlock, /* >= 1, points to previous block */ + usNextBlock, /* > 1, points to next block. */ + /* 0 indicates no chaining */ + usFirstAvailByte; /* >= size of block header, */ + /* <= block size */ + /* points to first available byte in block */ + /* If block is full it sets to BlockSize */ +} BLOCK_HEADER, *PBLOCK_HEADER; + +typedef struct _CREATE_TMH +{ /* ctmh */ + FILENAME szTmFileName; /* Tm database name */ + USHORT usBlockSize; + LANGUAGE szSourceLang, /* source language */ + szTargetLang; /* target language */ + EQF_BOOL fDbcs; /* double byte char support 0/1 */ } CREATE_TMH; -typedef struct _CREATE_IN { /* crei */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; - CREATE_TMH ctmh; /* parameters that copied into Tm header */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - USHORT usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - CHAR bufData[CREATE_BUFFER_SIZE]; - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} CREATE_IN , * PCREATE_IN; +typedef struct _CREATE_IN +{ /* crei */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; + CREATE_TMH ctmh; /* parameters that copied into Tm header */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + USHORT usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + CHAR bufData[CREATE_BUFFER_SIZE]; + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} CREATE_IN, *PCREATE_IN; typedef OPEN_OUT CREATE_OUT, *PCREATE_OUT; -typedef CLOSE_IN INFO_IN, * PINFO_IN; - -typedef struct _INFO_OUT { /* infi */ - PREFIX_OUT prefout; /* prefix of each command */ - CREATE_TMH ctmh; /* parameters that copied into Tm header */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - USHORT usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - CHAR bufData[CREATE_BUFFER_SIZE]; -} INFO_OUT , * PINFO_OUT; +typedef CLOSE_IN INFO_IN, *PINFO_IN; + +typedef struct _INFO_OUT +{ /* infi */ + PREFIX_OUT prefout; /* prefix of each command */ + CREATE_TMH ctmh; /* parameters that copied into Tm header */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + USHORT usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + CHAR bufData[CREATE_BUFFER_SIZE]; +} INFO_OUT, *PINFO_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_SERVER_DRIVES command which lists all @@ -586,49 +591,52 @@ typedef struct _INFO_OUT { /* infi */ // any of the secondary drives) // ---------------------------------------------------------------------------- -typedef struct _DRIVES_IN { /* indr */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} DRIVES_IN, * PDRIVES_IN; - -typedef struct _DRIVE_INFO { /* drin */ - CHAR cDriveLetter; /* drive letter of disk drive */ - ULONG ulFreeSpace; /* number of bytes left on this disk drive */ -} DRIVE_INFO, * PDRIVE_INFO; - -typedef struct _DRIVES_OUT { /* outdr */ - PREFIX_OUT prefout; /* prefix of each command */ - USHORT usValidDrives; /* how much drives have been */ - /* found on the server ? */ - DRIVE_INFO adrinDrives [MAX_DRIVELIST]; /* info for each valid */ - /* server drive */ -} DRIVES_OUT, * PDRIVES_OUT; - +typedef struct _DRIVES_IN +{ /* indr */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} DRIVES_IN, *PDRIVES_IN; + +typedef struct _DRIVE_INFO +{ /* drin */ + CHAR cDriveLetter; /* drive letter of disk drive */ + ULONG ulFreeSpace; /* number of bytes left on this disk drive */ +} DRIVE_INFO, *PDRIVE_INFO; + +typedef struct _DRIVES_OUT +{ /* outdr */ + PREFIX_OUT prefout; /* prefix of each command */ + USHORT usValidDrives; /* how much drives have been */ + /* found on the server ? */ + DRIVE_INFO adrinDrives[MAX_DRIVELIST]; /* info for each valid */ + /* server drive */ +} DRIVES_OUT, *PDRIVES_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_SERVER_TM_LIST command which lists all // translation memories that have been found on the selected server; for each // TM its full path name and its current file size is returned // ---------------------------------------------------------------------------- -typedef struct _FILE_LIST_IN { /* intl !!! CHM */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} FILE_LIST_IN, * PFILE_LIST_IN; // !!! CHM - -typedef struct _FILE_INFO { /* tmin !!! CHM */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename of file */ - ULONG ulFileSize; /* filesize in bytes */ -} FILE_INFO, * PFILE_INFO; // !!! CHM - - -typedef struct _FILE_LIST_OUT { /* outtl !!! CHM */ - PREFIX_OUT prefout; /* prefix of each command */ - USHORT usValidFiles; /* valid entries in array */ - FILE_INFO aflinFileList [MAX_TM_LIST_NUMBER]; /* info for avail. files */ -} FILE_LIST_OUT, * PFILE_LIST_OUT; // !!! CHM - +typedef struct _FILE_LIST_IN +{ /* intl !!! CHM */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} FILE_LIST_IN, *PFILE_LIST_IN; // !!! CHM + +typedef struct _FILE_INFO +{ /* tmin !!! CHM */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename of file */ + ULONG ulFileSize; /* filesize in bytes */ +} FILE_INFO, *PFILE_INFO; // !!! CHM + +typedef struct _FILE_LIST_OUT +{ /* outtl !!! CHM */ + PREFIX_OUT prefout; /* prefix of each command */ + USHORT usValidFiles; /* valid entries in array */ + FILE_INFO aflinFileList[MAX_TM_LIST_NUMBER]; /* info for avail. files */ +} FILE_LIST_OUT, *PFILE_LIST_OUT; // !!! CHM // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_PART_OF_TM_FILE command that transfers @@ -639,64 +647,64 @@ typedef struct _FILE_LIST_OUT { /* outtl !!! CHM */ // maximum size to transfer at a time is GETPART_BUFFER_SIZE // ---------------------------------------------------------------------------- - -typedef struct _GETPART_IN { /* ingp */ - PREFIX_IN prefin; /* prefix of each command */ - ULONG ulFilePos; /* file position of first byte to transfer */ - ULONG ulBytesToRead; /* number of bytes to transfer */ -} GETPART_IN, * PGETPART_IN; - -typedef struct _GETPART_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulBytesRead; /* number of bytes actually read */ - ULONG ulNextFilePos; /* file position of next byte to read */ - UCHAR aucOutBuffer [GETPART_BUFFER_SIZE]; /* output buffer */ -} GETPART_OUT, * PGETPART_OUT; - -typedef struct _GETDICTPART_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - PVOID pBTree ; // ptr to dict file handles - ULONG ulFilePos; /* file position of first byte to transfer */ - ULONG ulBytesToRead; /* number of bytes to transfer */ -} GETDICTPART_IN, * PGETDICTPART_IN; - - -#define MAX_DICT_DESCR 40 // max length of a dictionary description +typedef struct _GETPART_IN +{ /* ingp */ + PREFIX_IN prefin; /* prefix of each command */ + ULONG ulFilePos; /* file position of first byte to transfer */ + ULONG ulBytesToRead; /* number of bytes to transfer */ +} GETPART_IN, *PGETPART_IN; + +typedef struct _GETPART_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulBytesRead; /* number of bytes actually read */ + ULONG ulNextFilePos; /* file position of next byte to read */ + UCHAR aucOutBuffer[GETPART_BUFFER_SIZE]; /* output buffer */ +} GETPART_OUT, *PGETPART_OUT; + +typedef struct _GETDICTPART_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + PVOID pBTree; // ptr to dict file handles + ULONG ulFilePos; /* file position of first byte to transfer */ + ULONG ulBytesToRead; /* number of bytes to transfer */ +} GETDICTPART_IN, *PGETDICTPART_IN; + +#define MAX_DICT_DESCR 40 // max length of a dictionary description // Structure of EQF dictionary properties typedef struct _PROPDICTIONARY { - //--- common property part --- - PROPHEAD PropHead; // header of properties - //--- general dictionary information --- - CHAR szDescription[MAX_DICT_DESCR]; // dictionary description - CHAR szSourceLang[MAX_LANG_LENGTH]; // dictionary source language - // dictionary target language(s) - CHAR szTargetLang[MAX_TGT_LANG][MAX_LANG_LENGTH]; - EQF_BOOL fCopyRight; // dictionary-is-copyrighted flag - CHAR szDictPath[MAX_EQF_PATH]; // fully qualified dict file name - CHAR szIndexPath[MAX_EQF_PATH]; // fully qualified index file name - //--- dictionary profile --- - USHORT usLength; // number of user profile entries - USHORT usUserNameCount; // number of user created dict names - COLFONT ColFontDictEntry; // colour/font for dict entry - COLFONT ColFontEntryVal; // colour/font for dict entry value - PROFENTRY ProfEntry[MAX_PROF_ENTRIES]; // user profile entries - EQF_BOOL fProtected; // dictionary-is-protected flag - ULONG ulPassWord; - CHAR szServer[MAX_SERVER_NAME]; // Server Name of TM or \0 if TM is local - CHAR szUserid[MAX_USERID]; // LAN Userid of TM: if local '\0' - CHAR chRemPrimDrive; // LAN primary drive - USHORT usLocation; // location of dictionary - CHAR szLongName[MAX_LONGFILESPEC]; // dictionary long (descriptive) name - CHAR szLongDesc[MAX_LONG_DESCRIPTION]; // dictionary long (descriptive) name - USHORT usVersion; // Version of dictionary - used for - // comparison during organize if copyrighted - //--- reserved space --- - CHAR chReserve[3402]; // reserve space / filler + //--- common property part --- + PROPHEAD PropHead; // header of properties + //--- general dictionary information --- + CHAR szDescription[MAX_DICT_DESCR]; // dictionary description + CHAR szSourceLang[MAX_LANG_LENGTH]; // dictionary source language + // dictionary target language(s) + CHAR szTargetLang[MAX_TGT_LANG][MAX_LANG_LENGTH]; + EQF_BOOL fCopyRight; // dictionary-is-copyrighted flag + CHAR szDictPath[MAX_EQF_PATH]; // fully qualified dict file name + CHAR szIndexPath[MAX_EQF_PATH]; // fully qualified index file name + //--- dictionary profile --- + USHORT usLength; // number of user profile entries + USHORT usUserNameCount; // number of user created dict names + COLFONT ColFontDictEntry; // colour/font for dict entry + COLFONT ColFontEntryVal; // colour/font for dict entry value + PROFENTRY ProfEntry[MAX_PROF_ENTRIES]; // user profile entries + EQF_BOOL fProtected; // dictionary-is-protected flag + ULONG ulPassWord; + CHAR szServer[MAX_SERVER_NAME]; // Server Name of TM or \0 if TM is local + CHAR szUserid[MAX_USERID]; // LAN Userid of TM: if local '\0' + CHAR chRemPrimDrive; // LAN primary drive + USHORT usLocation; // location of dictionary + CHAR szLongName[MAX_LONGFILESPEC]; // dictionary long (descriptive) name + CHAR szLongDesc[MAX_LONG_DESCRIPTION]; // dictionary long (descriptive) name + USHORT usVersion; // Version of dictionary - used for + // comparison during organize if copyrighted + //--- reserved space --- + CHAR chReserve[3402]; // reserve space / filler } PROPDICTIONARY, *PPROPDICTIONARY; - // ---------------------------------------------------------------------------- // structures needed for the TMC_PUT_TM_PROPERTIES command which is used to // copy a property file for a remote TM to the server (during TM creation @@ -707,37 +715,39 @@ typedef struct _PROPDICTIONARY // the drive and path) which is then used to create a filename for the property // file (same drive and path but new extension (will be .PRP) // ---------------------------------------------------------------------------- -typedef struct _PUTPROP_IN { /* inpp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - ULONG ulPropLength; /* length of property file */ - UCHAR aucInBuffer [MEM_PROP_SIZE]; /* buffer for file data */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} PUTPROP_IN, * PPUTPROP_IN; +typedef struct _PUTPROP_IN +{ /* inpp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + ULONG ulPropLength; /* length of property file */ + UCHAR aucInBuffer[MEM_PROP_SIZE]; /* buffer for file data */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} PUTPROP_IN, *PPUTPROP_IN; // !!! CHM Start -typedef struct _PUTDICTPROP_IN { /* inpp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - ULONG ulPropLength; /* length of property file */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ - PROPDICTIONARY DictProp; /* dictionary property buffer */ -} PUTDICTPROP_IN, * PPUTDICTPROP_IN; +typedef struct _PUTDICTPROP_IN +{ /* inpp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + ULONG ulPropLength; /* length of property file */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ + PROPDICTIONARY DictProp; /* dictionary property buffer */ +} PUTDICTPROP_IN, *PPUTDICTPROP_IN; /* define union to allow combined processing in server code */ typedef union _PUTPROPCOMBINEDIN { - PUTPROP_IN TmPropIn; - PUTDICTPROP_IN DictPropIn; -} PUTPROPCOMBINED_IN, * PPUTPROPCOMBINED_IN; + PUTPROP_IN TmPropIn; + PUTDICTPROP_IN DictPropIn; +} PUTPROPCOMBINED_IN, *PPUTPROPCOMBINED_IN; // !!! CHM End -typedef struct _PUTPROP_OUT { /* outpp */ - PREFIX_OUT prefout; /* prefix of each command */ -} PUTPROP_OUT, * PPUTPROP_OUT; - +typedef struct _PUTPROP_OUT +{ /* outpp */ + PREFIX_OUT prefout; /* prefix of each command */ +} PUTPROP_OUT, *PPUTPROP_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_TM_PROPERTIES command which is used to @@ -750,29 +760,31 @@ typedef struct _PUTPROP_OUT { /* outpp // property file which is stored on the same disk in the same path with the // extension .PRP // ---------------------------------------------------------------------------- -typedef struct _GETPROP_IN { /* ingp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} GETPROP_IN, * PGETPROP_IN; - -typedef struct _GETPROP_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulPropLength; /* length of property file */ - UCHAR aucOutBuffer [MEM_PROP_SIZE]; /* buffer for file data */ -} GETPROP_OUT, * PGETPROP_OUT; +typedef struct _GETPROP_IN +{ /* ingp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} GETPROP_IN, *PGETPROP_IN; + +typedef struct _GETPROP_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulPropLength; /* length of property file */ + UCHAR aucOutBuffer[MEM_PROP_SIZE]; /* buffer for file data */ +} GETPROP_OUT, *PGETPROP_OUT; // !!! CHM Start -typedef struct _GETDICTPROP_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulPropLength; /* length of property file */ - PROPDICTIONARY DictProp; /* dictionary property buffer*/ -} GETDICTPROP_OUT, * PGETDICTPROP_OUT; +typedef struct _GETDICTPROP_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulPropLength; /* length of property file */ + PROPDICTIONARY DictProp; /* dictionary property buffer*/ +} GETDICTPROP_OUT, *PGETDICTPROP_OUT; /* define union to allow combined processing in server code */ - // ---------------------------------------------------------------------------- // structures needed for the TMC_END_ORGANIZE command which is used to end a // running 'Organize' process; this command is sent to the original TM instead @@ -782,16 +794,17 @@ typedef struct _GETDICTPROP_OUT { /* outgp * // original TM file to be deleted and the temporary file to be renamed to the // original TM filename; // ---------------------------------------------------------------------------- -typedef struct _ENDORG_IN { /* ineo */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szOrgTM [MAX_EQF_PATH];/* full filename of original TM file */ - CHAR szTmpTM [MAX_EQF_PATH];/* full filename of temporary TM file */ -} ENDORG_IN, * PENDORG_IN; - -typedef struct _ENDORG_OUT { /* outeo */ - PREFIX_OUT prefout; /* prefix of each command */ -} ENDORG_OUT, * PENDORG_OUT; - +typedef struct _ENDORG_IN +{ /* ineo */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szOrgTM[MAX_EQF_PATH]; /* full filename of original TM file */ + CHAR szTmpTM[MAX_EQF_PATH]; /* full filename of temporary TM file */ +} ENDORG_IN, *PENDORG_IN; + +typedef struct _ENDORG_OUT +{ /* outeo */ + PREFIX_OUT prefout; /* prefix of each command */ +} ENDORG_OUT, *PENDORG_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_DELETE_TM command which is used to physical @@ -799,16 +812,17 @@ typedef struct _ENDORG_OUT { /* outeo */ // before this command can be submitted, the file has to opened exclusively; // the supplied filename must contain the full drive and path information; // ---------------------------------------------------------------------------- -typedef struct _DELTM_IN { /* indt */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szTMPathFileName [MAX_EQF_PATH]; /* full filename of TM file */ - CHAR szPropPathFileName[MAX_EQF_PATH]; /* full filename of Prop file */ -} DELTM_IN, * PDELTM_IN; - -typedef struct _DELTM_OUT { /* outdt */ - PREFIX_OUT prefout; /* prefix of each command */ -} DELTM_OUT, * PDELTM_OUT; - +typedef struct _DELTM_IN +{ /* indt */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szTMPathFileName[MAX_EQF_PATH]; /* full filename of TM file */ + CHAR szPropPathFileName[MAX_EQF_PATH]; /* full filename of Prop file */ +} DELTM_IN, *PDELTM_IN; + +typedef struct _DELTM_OUT +{ /* outdt */ + PREFIX_OUT prefout; /* prefix of each command */ +} DELTM_OUT, *PDELTM_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_DELETE_FILE command which is used to physical @@ -817,32 +831,34 @@ typedef struct _DELTM_OUT { /* outdt */ // a previous organize process that died during execution); // the supplied filename must contain the full drive and path information; // ---------------------------------------------------------------------------- -typedef struct _DELFILE_IN { /* indf */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szFileName [MAX_EQF_PATH]; /* full filename of file to delete*/ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} DELFILE_IN, * PDELFILE_IN; - -typedef struct _DELFILE_OUT { /* outdf */ - PREFIX_OUT prefout; /* prefix of each command */ -} DELFILE_OUT, * PDELFILE_OUT; - -typedef struct _CLOSEHANDLER_IN +typedef struct _DELFILE_IN +{ /* indf */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szFileName[MAX_EQF_PATH]; /* full filename of file to delete*/ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} DELFILE_IN, *PDELFILE_IN; + +typedef struct _DELFILE_OUT +{ /* outdf */ + PREFIX_OUT prefout; /* prefix of each command */ +} DELFILE_OUT, *PDELFILE_OUT; + +typedef struct _CLOSEHANDLER_IN { - PREFIX_IN prefin; - USHORT hth; -} CLOSEHANDLER_IN, * PCLOSEHANDLER_IN; + PREFIX_IN prefin; + USHORT hth; +} CLOSEHANDLER_IN, *PCLOSEHANDLER_IN; -typedef struct _EXIT_IN +typedef struct _EXIT_IN { - PREFIX_IN prefin; -} EXIT_IN, * PEXIT_IN; + PREFIX_IN prefin; +} EXIT_IN, *PEXIT_IN; -typedef struct _EXIT_OUT +typedef struct _EXIT_OUT { - PREFIX_OUT prefout; -} EXIT_OUT, * PEXIT_OUT; + PREFIX_OUT prefout; +} EXIT_OUT, *PEXIT_OUT; // ---------------------------------------------------------------------------- // data structure used for the TMC_QUERY_FILE_INFO which returns the file info @@ -850,70 +866,67 @@ typedef struct _EXIT_OUT // server // ---------------------------------------------------------------------------- -typedef struct _FILEINFO_IN { /* infi */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szFileName [MAX_EQF_PATH]; /* full filename of file to query */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester !!!! CHM */ -} FILEINFO_IN, * PFILEINFO_IN; - -typedef struct _FILEINFO_OUT { /* outfi */ - PREFIX_OUT prefout; /* prefix of each command */ - FILEFINDBUF stFile; /* info about file */ -} FILEINFO_OUT, * PFILEINFO_OUT; - - -typedef struct _TM_HEADER { /* tmh */ -/* fixed part */ - UCHAR achTmPrefix[TM_PREFIX_SIZE]; - /* should be initiated to EQFTMT$$ 8 */ - USHORT usTmHeaderSize, /* actual length of tm header */ - usDbVersion; - LONG ldispFirstBlock; /* disp of first block from */ - /* beginning of TM */ - CREATE_TMH ctmh; /* parameters provided by the create */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - -/* updated part */ - EQF_BOOL fCorruption; - USHORT usNumTMBlocks, /* number of blocks in the TM */ - usNumFreeBlocks, /* # of free, pre-formatted blocks */ - usFirstAvailBlock,/* points to first available block */ - ausKeyDirectory[KEY_DIR_SIZE], /* key directory */ - - usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - -} TM_HEADER, * PTM_HEADER, ** PPTM_HEADER; - -#define TAG_LEN 35 -#define MAX_NAME 8 -#define MAX_RANDOM 20 -//#define MAX_VOTES 20 // 30 -#define MAX_VOTES 30 // change 27.2.2000 -#define ABS_VOTES 400 -#define MAX_MATCHES 15 -#define TOK_SIZE 4000 -#define TMX_REC_SIZE 32760 -#define TMX_TABLE_SIZE 512 -#define MAX_INDEX_LEN 8150 // 2048 - - - - - // name table structure (TM Version 1 - 4) +typedef struct _FILEINFO_IN +{ /* infi */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szFileName[MAX_EQF_PATH]; /* full filename of file to query */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester !!!! CHM */ +} FILEINFO_IN, *PFILEINFO_IN; + +typedef struct _FILEINFO_OUT +{ /* outfi */ + PREFIX_OUT prefout; /* prefix of each command */ + FILEFINDBUF stFile; /* info about file */ +} FILEINFO_OUT, *PFILEINFO_OUT; + +typedef struct _TM_HEADER +{ /* tmh */ + /* fixed part */ + UCHAR achTmPrefix[TM_PREFIX_SIZE]; + /* should be initiated to EQFTMT$$ 8 */ + USHORT usTmHeaderSize, /* actual length of tm header */ + usDbVersion; + LONG ldispFirstBlock; /* disp of first block from */ + /* beginning of TM */ + CREATE_TMH ctmh; /* parameters provided by the create */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + + /* updated part */ + EQF_BOOL fCorruption; + USHORT usNumTMBlocks, /* number of blocks in the TM */ + usNumFreeBlocks, /* # of free, pre-formatted blocks */ + usFirstAvailBlock, /* points to first available block */ + ausKeyDirectory[KEY_DIR_SIZE], /* key directory */ + + usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + +} TM_HEADER, *PTM_HEADER, **PPTM_HEADER; + +#define TAG_LEN 35 +#define MAX_NAME 8 +#define MAX_RANDOM 20 +// #define MAX_VOTES 20 // 30 +#define MAX_VOTES 30 // change 27.2.2000 +#define ABS_VOTES 400 +#define MAX_MATCHES 15 +#define TOK_SIZE 4000 +#define TMX_REC_SIZE 32760 +#define TMX_TABLE_SIZE 512 +#define MAX_INDEX_LEN 8150 // 2048 + +// name table structure (TM Version 1 - 4) // typedef struct _TMX_VER1_TABLE // { // USHORT usAllocSize; // USHORT usMaxEntries; -// TMX_TABLE_ENTRY stTmTableEntry; +// TMX_TABLE_ENTRY table; // } TMX_VER1_TABLE, * PTMX_VER1_TABLE; - - /**********************************************************************/ /* Defines and structures for the long document name support */ /**********************************************************************/ @@ -925,67 +938,67 @@ typedef struct _TM_HEADER { /* tmh */ // and increment for table enlargements #define LONGNAMETABLE_ENTRIES 32 - - - -//complete entry id tm data file +// complete entry id tm data file typedef struct _TMX_RECORD { - LONG lRecordLen; - USHORT usSourceRecord; - USHORT usFirstTargetRecord; -} TMX_RECORD, * PTMX_RECORD; + LONG lRecordLen; + USHORT usSourceRecord; + USHORT usFirstTargetRecord; +} TMX_RECORD, *PTMX_RECORD; + -//structure of the source segment -typedef struct _TMX_SOURCE_RECORD + +// structure of the source segment +struct TMX_SOURCE_RECORD { - LONG lRecordLen; - USHORT usSource; - USHORT usLangId; - void reset(){ - memset(this, 0, sizeof(*this)); + LONG lRecordLen{}; + USHORT usSource{}; + USHORT usLangId{}; + void reset() + { + *this = TMX_SOURCE_RECORD{}; } -} TMX_SOURCE_RECORD, * PTMX_SOURCE_RECORD; +}; +using PTMX_SOURCE_RECORD = TMX_SOURCE_RECORD*; -//structure of the target segment -typedef struct _TMX_TARGET_RECORD +// structure of the target segment +struct TMX_TARGET_RECORD { - LONG lRecordLen = 0; - //USHORT usSourceTagTable; - //USHORT usTargetTagTable; - USHORT usTarget = 0; - USHORT usClb = 0; - void reset(){ - memset(this, 0, sizeof(*this)); + LONG lRecordLen = 0; + USHORT usTarget = 0; + USHORT usClb = 0; + void reset() + { + *this = TMX_TARGET_RECORD{}; } -} TMX_TARGET_RECORD, * PTMX_TARGET_RECORD; - +}; +using PTMX_TARGET_RECORD = TMX_TARGET_RECORD*; -//control block structure in target record +// control block structure in target record typedef struct _TMX_TARGET_CLB -{ - TIME_L lTime; - TIME_L lUpdateTime; - ULONG ulSegmId; - USHORT usLangId; - USHORT usFileId; - USHORT usAuthorId; - USHORT usAddDataLen; // new for Major_version6: Length of following context and additional info data - BYTE bMultiple; - BYTE bTranslationFlag; -} TMX_TARGET_CLB, * PTMX_TARGET_CLB; +{ + TIME_L lTime{}; + TIME_L lUpdateTime{}; + ULONG ulSegmId{}; + USHORT usLangId{}; + USHORT usFileId{}; + USHORT usAuthorId{}; + USHORT usAddDataLen{}; // new for Major_version6: Length of following context and additional info data + BYTE bMultiple{}; + BYTE bTranslationFlag{}; +} TMX_TARGET_CLB, *PTMX_TARGET_CLB; // helper macros for working with TMX_TARGET_CLBs -#define TARGETCLBLEN( pClb ) (sizeof(TMX_TARGET_CLB) + pClb->usAddDataLen) +#define TARGETCLBLEN(pClb) (sizeof(TMX_TARGET_CLB) + pClb->usAddDataLen) -#define NEXTTARGETCLB( pClb ) ((PTMX_TARGET_CLB)(((PBYTE)pClb) + TARGETCLBLEN(pClb))) +#define NEXTTARGETCLB(pClb) ((PTMX_TARGET_CLB)(((PBYTE)pClb) + TARGETCLBLEN(pClb))) -#define PCONTEXTFROMCLB( pClb ) ((PSZ_W)(((PBYTE)pClb)+sizeof(TMX_TARGET_CLB))) +#define PCONTEXTFROMCLB(pClb) ((PSZ_W)(((PBYTE)pClb) + sizeof(TMX_TARGET_CLB))) // max size of additional data area // (currently 2 * MAX_SEGMENT_SIZE for context and additional info // plus size of three identifierrs (USHORT) and two size fields (USHORT) ) -#define MAX_ADD_DATA_LEN ((2 * MAX_SEGMENT_SIZE)*sizeof(WCHAR) + (5 * sizeof(USHORT))) +#define MAX_ADD_DATA_LEN ((2 * MAX_SEGMENT_SIZE) * sizeof(WCHAR) + (5 * sizeof(USHORT))) // identifier for additional info data in additional data area #define ADDDATA_ADDINFO_ID 1 @@ -997,73 +1010,124 @@ typedef struct _TMX_TARGET_CLB #define ADDDATA_ENDOFDATA_ID 9 // compute the size of the additional data for the given input -USHORT NTMComputeAddDataSize( PSZ_W pszContext, PSZ_W pszAddInfo ); +USHORT NTMComputeAddDataSize(PSZ_W pszContext, PSZ_W pszAddInfo); // get length of specific data in the combined data area, returns length of data area -USHORT NtmGetAddDataLen( PTMX_TARGET_CLB pCLB, USHORT usDataID ); +USHORT NtmGetAddDataLen(PTMX_TARGET_CLB pCLB, USHORT usDataID); // store/combine additional data in the combined area, returns new size of combined data area or 0 in case of errors -USHORT NtmStoreAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszAddData ); +USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszAddData); // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszBuffer, USHORT usBufSize ); +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, PBYTE endOfMemRegion = 0); // find a string in a specific data area -BOOL NtmFindInAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch ); +BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch); -//tag table structure for both source and target +// tag table structure for both source and target typedef struct _TMX_TAGTABLE_RECORD { - LONG lRecordLen; - USHORT usTagTableId; - USHORT usFirstTagEntry; -} TMX_TAGTABLE_RECORD, * PTMX_TAGTABLE_RECORD; + LONG lRecordLen; + USHORT usTagTableId; + USHORT usFirstTagEntry; +} TMX_TAGTABLE_RECORD, *PTMX_TAGTABLE_RECORD; - -//individual tag entry in tag table record +// individual tag entry in tag table record typedef struct _TMX_TAGENTRY { - USHORT usOffset; - USHORT usTagLen; - BYTE bData; -} TMX_TAGENTRY, * PTMX_TAGENTRY; + USHORT usOffset; + USHORT usTagLen; + BYTE bData; +} TMX_TAGENTRY, *PTMX_TAGENTRY; // macros to access certain TM records fields - #define RECLEN(pRec) pRec->lRecordLen - +#define RECLEN(pRec) pRec->lRecordLen +#define RECLENB(pRec) toTmxRecord(pRec)->lRecordLen -#define NTMVOTES(l) ((BYTE)(((ULONG)(l) >> 24) & 0xFF)) -#define NTMKEY(l) ((ULONG)(l) & 0xFFFFFF) -#define NTMINDEX(b,l) ((ULONG)(((l) & 0xFFFFFF) | (((ULONG)(b)) << 24 ))) +#define NTMVOTES(l) ((BYTE)(((ULONG)(l) >> 24) & 0xFF)) +#define NTMKEY(l) ((ULONG)(l) & 0xFFFFFF) +#define NTMINDEX(b, l) ((ULONG)(((l) & 0xFFFFFF) | (((ULONG)(b)) << 24))) -typedef ULONG TMX_INDEX_ENTRY, *PTMX_INDEX_ENTRY; +typedef ULONG TMX_INDEX_ENTRY, *PTMX_INDEX_ENTRY; typedef struct _TMX_INDEX_RECORD { - USHORT usRecordLen; - TMX_INDEX_ENTRY stIndexEntry; -} TMX_INDEX_RECORD, * PTMX_INDEX_RECORD; + USHORT usRecordLen; + TMX_INDEX_ENTRY stIndexEntry; +} TMX_INDEX_RECORD, *PTMX_INDEX_RECORD; -typedef struct _TMX_TERM_TOKEN +#define SAFE_CAST + +inline TMX_RECORD *toTmxRecord(BytesRef buffer) +{ + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif + return reinterpret_cast(buffer.data()); +} + +inline TMX_TAGTABLE_RECORD* toTmxTagtableRecord(BytesRef buffer){ + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif + return reinterpret_cast(buffer.data()); +} + +inline TMX_INDEX_RECORD *toIndexRecord(BytesRef buffer) { - USHORT usOffset; - USHORT usLength; - USHORT usHash; -} TMX_TERM_TOKEN, *PTMX_TERM_TOKEN; + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif + return reinterpret_cast(buffer.data()); +} + +inline TMX_TARGET_CLB *toTmxTargetClb(BytesRef buffer) +{ + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif + return reinterpret_cast(buffer.data()); +} + +struct TMX_TERM_TOKEN +{ + USHORT usOffset{}; + USHORT usLength{}; + USHORT usHash{}; + void reset() { *this = TMX_TERM_TOKEN{}; } +}; +using PTMX_TERM_TOKEN = TMX_TERM_TOKEN*; -typedef struct _TMX_MATCHENTRY +struct TMX_MATCHENTRY { - ULONG ulKey; - USHORT usMaxVotes; - USHORT usMatchVotes; - BYTE cCount; -} TMX_MATCHENTRY, * PTMX_MATCHENTRY; + LONG lKey = 0; + USHORT usMaxVotes = 0; + USHORT usMatchVotes = 0; + BYTE cCount = 0; +}; +using PTMX_MATCHENTRY = TMX_MATCHENTRY*; + -//#pragma pack(show) -//#pragma pack(push, 8) -//#pragma pack(8) +inline bool operator==(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { + return lhs.lKey == rhs.lKey + && lhs.usMaxVotes == rhs.usMaxVotes + && lhs.usMatchVotes == rhs.usMatchVotes + && lhs.cCount == rhs.cCount; +} -#include +inline bool operator!=(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { + return !(lhs == rhs); +} + +// #pragma pack(show) +// #pragma pack(push, 8) +// #pragma pack(8) + +#include #include #include @@ -1073,141 +1137,160 @@ typedef struct _TMX_MATCHENTRY #include #include - class ImportStatusDetails; - // IDs of TMX elelements typedef enum { - UNKNOWN_ELEMENT =-1, - TMX_ELEMENT = 1, - PROP_ELEMENT = 2, - HEADER_ELEMENT = 3, - TU_ELEMENT = 4, - TUV_ELEMENT = 5, - BODY_ELEMENT = 6, - SEG_ELEMENT = 7, - SEGMENT_TAGS = 8, + UNKNOWN_ELEMENT = -1, + TMX_ELEMENT = 1, + PROP_ELEMENT = 2, + HEADER_ELEMENT = 3, + TU_ELEMENT = 4, + TUV_ELEMENT = 5, + BODY_ELEMENT = 6, + SEG_ELEMENT = 7, + SEGMENT_TAGS = 8, BEGIN_INLINE_TAGS = 10, - //pair tags + // pair tags BEGIN_PAIR_TAGS = 10, - BPT_ELEMENT = 10, - EPT_ELEMENT = 11, - G_ELEMENT = 12, - HI_ELEMENT = 13, - SUB_ELEMENT = 14, - BX_ELEMENT = 15, - EX_ELEMENT = 16, - END_PAIR_TAGS = 16, - - //standalone tags + BPT_ELEMENT = 10, + EPT_ELEMENT = 11, + G_ELEMENT = 12, + HI_ELEMENT = 13, + SUB_ELEMENT = 14, + BX_ELEMENT = 15, + EX_ELEMENT = 16, + END_PAIR_TAGS = 16, + + // standalone tags BEGIN_STANDALONE_TAGS = 20, - PH_ELEMENT = 20, - X_ELEMENT = 21, - IT_ELEMENT = 22, - UT_ELEMENT = 23, - T5_N_ELEMENT = 24, - END_STANDALONE_TAGS = 25, + PH_ELEMENT = 20, + X_ELEMENT = 21, + IT_ELEMENT = 22, + UT_ELEMENT = 23, + T5_N_ELEMENT = 24, + END_STANDALONE_TAGS = 25, - END_INLINE_TAGS = 29, + END_INLINE_TAGS = 29, - TMX_SENTENCE_ELEMENT = 30, - INVCHAR_ELEMENT = 31 + TMX_SENTENCE_ELEMENT = 30, + INVCHAR_ELEMENT = 31 } ELEMENTID; - -enum ACTIVE_SEGMENT{ - SOURCE_SEGMENT = 0, - TARGET_SEGMENT = 1, - REQUEST_SEGMENT = 2 //from fuzzy search request +enum ACTIVE_SEGMENT +{ + SOURCE_SEGMENT = 0, + TARGET_SEGMENT = 1, + REQUEST_SEGMENT = 2 // from fuzzy search request }; - typedef ACTIVE_SEGMENT TagLocation; struct TagInfo { TagLocation tagLocation = SOURCE_SEGMENT; - //bool fPairedTag; // is tag is paired tag (bpt, ept, ) - bool fPairedTagClosed = false; // false for bpt/ept tag - waiting for matching ept/bpt tag + // bool fPairedTag; // is tag is paired tag (bpt, ept, ) + bool fPairedTagClosed = false; // false for bpt/ept tag - waiting for matching ept/bpt tag bool fTagAlreadyUsedInTarget = false; // we save tags only from source segment and then try to match\bind them in target - int generated_i = 0; // for pair tags - generated identifier to find matching tag. the same as in original_i if it's not binded to other tag in segment - int generated_x = 0; // id of tag. should match original_x, if it's not occupied by other tags - ELEMENTID generated_tagType = UNKNOWN_ELEMENT; // replaced tagType, could be PH_ELEMENT, BPT_ELEMENT, EPT_ELEMENT + int generated_i = 0; // for pair tags - generated identifier to find matching tag. the same as in original_i if it's not binded to other tag in segment + int generated_x = 0; // id of tag. should match original_x, if it's not occupied by other tags + ELEMENTID generated_tagType = UNKNOWN_ELEMENT; // replaced tagType, could be PH_ELEMENT, BPT_ELEMENT, EPT_ELEMENT - int original_i = 0; // original paired tags i - int original_x = 0; // original id of tag - ELEMENTID original_tagType; // original tagType - //if targetTag -> matching tag from source tags - //if sourceTag -> matching tag from target tags - //TagInfo* matchingTag; - //int matchingTagIndex = -1; + int original_i = 0; // original paired tags i + int original_x = 0; // original id of tag + ELEMENTID original_tagType; // original tagType + // if targetTag -> matching tag from source tags + // if sourceTag -> matching tag from target tags + // TagInfo* matchingTag; + // int matchingTagIndex = -1; bool fDeleted = false; - - //t5n + + // t5n std::string t5n_key; std::string t5n_value; }; - class TagReplacer{ public: bool fUseOriginalTagAndAttributes = false; - + std::vector sourceTagList; std::vector targetTagList; std::vector requestTagList; ACTIVE_SEGMENT activeSegment = SOURCE_SEGMENT; - int iHighestPTI = 0; // increments with each opening pair tags - int iHighestPTId = 500; // increments with pair tag - int iHighestPHId = 100; // increments with ph tag - bool fFuzzyRequest = false; // if re are dealing with import or fuzzy request - bool fReplaceNumberProtectionTagsWithHashes = false;// + int iHighestPTI = 0; // increments with each opening pair tags + int iHighestPTId = 500; // increments with pair tag + int iHighestPHId = 100; // increments with ph tag + bool fFuzzyRequest = false; // if re are dealing with import or fuzzy request + bool fReplaceNumberProtectionTagsWithHashes = false; // bool fSkipTags = false; - //to track id and i attributes in request and then generate new values for tags in srt and trg that is not matching - int iHighestRequestsOriginalI = 0; + // to track id and i attributes in request and then generate new values for tags in srt and trg that is not matching + int iHighestRequestsOriginalI = 0; int iHighestRequestsOriginalId = 0; - - //idAttr==0 -> look for id in attributes - //iAttr==0 ->i attribute not used - //tag should be only "ph", "bpt", or "ept" - TagInfo GenerateReplacingTag(ELEMENTID tagType, xercesc::AttributeList* attributes, bool saveTagToTagList = true); - std::wstring PrintTag(TagInfo& tag); + // idAttr==0 -> look for id in attributes + // iAttr==0 ->i attribute not used + // tag should be only "ph", "bpt", or "ept" + TagInfo GenerateReplacingTag(ELEMENTID tagType, xercesc::AttributeList *attributes, bool saveTagToTagList = true); + std::wstring PrintTag(TagInfo &tag); void reset(); - std::string static LogTag(TagInfo& tag); - std::wstring PrintReplaceTagWithKey(TagInfo& tag); + std::string static LogTag(TagInfo &tag); + std::wstring PrintReplaceTagWithKey(TagInfo &tag); - TagReplacer(){ + TagReplacer() + { sourceTagList.reserve(50); targetTagList.reserve(50); requestTagList.reserve(50); - reset(); - } + reset(); + } }; // PROP types -typedef enum { TMLANGUAGE_PROP, TMMARKUP_PROP, TMDOCNAME_PROP, TMCONTEXT_PROP, TMADDINFO_PROP, TMAUTHOR_PROP, MACHINEFLAG_PROP, SEG_PROP, TMDESCRIPTION_PROP, TMNOTE_PROP, TMNOTESTYLE_PROP, - TRANSLATIONFLAG_PROP, TMTMMATCHTYPE_PROP, TMMTSERVICE_PROP, TMMTMETRICNAME_PROP, TMMTMETRICVALUE_PROP, TMPEEDITDISTANCECHARS_PROP, TMPEEDITDISTANCEWORDS_PROP, - TMMTFIELDS_PROP, TMWORDS_PROP, TMMATCHSEGID_PROP, UNKNOWN_PROP } TMXPROPID, PROPID; +typedef enum +{ + TMLANGUAGE_PROP, + TMMARKUP_PROP, + TMDOCNAME_PROP, + TMCONTEXT_PROP, + TMADDINFO_PROP, + TMAUTHOR_PROP, + MACHINEFLAG_PROP, + SEG_PROP, + TMDESCRIPTION_PROP, + TMNOTE_PROP, + TMNOTESTYLE_PROP, + TRANSLATIONFLAG_PROP, + TMTMMATCHTYPE_PROP, + TMMTSERVICE_PROP, + TMMTMETRICNAME_PROP, + TMMTMETRICVALUE_PROP, + TMPEEDITDISTANCECHARS_PROP, + TMPEEDITDISTANCEWORDS_PROP, + TMMTFIELDS_PROP, + TMWORDS_PROP, + TMMATCHSEGID_PROP, + UNKNOWN_PROP +} TMXPROPID, + PROPID; // stack elements typedef struct _TMXELEMENT { - ELEMENTID ID; // ID of element - BOOL fInlineTagging; // TRUE = we are processing inline tagging - BOOL fInsideTagging; // TRUE = we are currently inside inline tagging - TMXPROPID PropID; // ID of prop element (only used for props) - CHAR szDataType[50]; // data type of current element - CHAR szTMXLanguage[50]; // TMX language of element - CHAR szTMLanguage[50]; // TM language of element - CHAR szTMMarkup[50]; // TM markup of element - LONG lSegNum; // TM segment number + ELEMENTID ID; // ID of element + BOOL fInlineTagging; // TRUE = we are processing inline tagging + BOOL fInsideTagging; // TRUE = we are currently inside inline tagging + TMXPROPID PropID; // ID of prop element (only used for props) + CHAR szDataType[50]; // data type of current element + CHAR szTMXLanguage[50]; // TMX language of element + CHAR szTMLanguage[50]; // TM language of element + CHAR szTMMarkup[50]; // TM markup of element + LONG lSegNum; // TM segment number } TMXELEMENT, *PTMXELEMENT; // @@ -1219,25 +1302,25 @@ class TMXParseHandler : public xercesc::HandlerBase // ----------------------------------------------------------------------- // Constructors and Destructor // ----------------------------------------------------------------------- - TMXParseHandler(InclosingTagsBehaviour InclosingTagsBehaviour = InclosingTagsBehaviour::saveAll); + TMXParseHandler(InclosingTagsBehaviour InclosingTagsBehaviour = InclosingTagsBehaviour::saveAll); virtual ~TMXParseHandler(); // setter functions for import info - void SetMemInfo( PMEMEXPIMPINFO m_pMemInfo ); - void SetImportData(ImportStatusDetails * _pImportDetails) { pImportDetails = _pImportDetails;} - void SetMemInterface( PFN_MEMINSERTSEGMENT pfnInsertSegment, LONG lMemHandle, //LOADEDTABLE* pTable, - PTOKENENTRY pTokBuf, int iTokBufSize ); - void SetSourceLanguage( char *pszSourceLang ); - - // getter functions - void GetDescription( char *pszDescription, int iBufSize ); - void GetSourceLanguage( char *pszSourceLang, int iBufSize ); - BOOL IsHeaderDone( void ); - BOOL ErrorOccured( void ); - void GetErrorText( char *pszTextBuffer, int iBufSize ); + void SetMemInfo(PMEMEXPIMPINFO m_pMemInfo); + void SetImportData(ImportStatusDetails *_pImportDetails) { pImportDetails = _pImportDetails; } + void SetMemInterface(PFN_MEMINSERTSEGMENT pfnInsertSegment, LONG lMemHandle, // LOADEDTABLE* pTable, + PTOKENENTRY pTokBuf, int iTokBufSize); + void SetSourceLanguage(char *pszSourceLang); + + // getter functions + void GetDescription(char *pszDescription, int iBufSize); + void GetSourceLanguage(char *pszSourceLang, int iBufSize); + BOOL IsHeaderDone(void); + BOOL ErrorOccured(void); + void GetErrorText(char *pszTextBuffer, int iBufSize); std::wstring GetParsedData() const; - std::wstring GetParsedDataWithReplacedNpTags()const; - std::wstring GetParsedNormalizedData()const; + std::wstring GetParsedDataWithReplacedNpTags() const; + std::wstring GetParsedNormalizedData() const; int iStopImportWRc = 0; bool fReplaceWithTagsWithoutAttributes; @@ -1246,125 +1329,123 @@ class TMXParseHandler : public xercesc::HandlerBase // ----------------------------------------------------------------------- // Handlers for the SAX DocumentHandler interface // ----------------------------------------------------------------------- - void startElement(const XMLCh* const name, xercesc::AttributeList& attributes); - void endElement(const XMLCh* const name ); - void characters(const XMLCh* const chars, const XMLSize_t length); - //void ignorableWhitespace(const XMLCh* const chars, const unsigned int length); - //void resetDocument(); - + void startElement(const XMLCh *const name, xercesc::AttributeList &attributes); + void endElement(const XMLCh *const name); + void characters(const XMLCh *const chars, const XMLSize_t length); + // void ignorableWhitespace(const XMLCh* const chars, const unsigned int length); + // void resetDocument(); // ----------------------------------------------------------------------- // Handlers for the SAX ErrorHandler interface // ----------------------------------------------------------------------- - void warning(const xercesc::SAXParseException& exc); - void error(const xercesc::SAXParseException& exc ); - void fatalError(const xercesc::SAXParseException& exc); - void fatalInternalError(const xercesc::SAXException& exc); + void warning(const xercesc::SAXParseException &exc); + void error(const xercesc::SAXParseException &exc); + void fatalError(const xercesc::SAXParseException &exc); + void fatalInternalError(const xercesc::SAXException &exc); - bool tmxEndReached()const {return fBodyDone;} - //void resetErrors(); + bool tmxEndReached() const { return fBodyDone; } + // void resetErrors(); - TagReplacer tagReplacer; BOOL fInitialized = false; BOOL fCreateNormalizedStr = false; USHORT insertSegUsRC{0}; - void setDocumentLocator(const xercesc::Locator* const locator) override; + void setDocumentLocator(const xercesc::Locator *const locator) override; private: - const xercesc::Locator* m_locator = nullptr; - ImportStatusDetails* pImportDetails = nullptr; - ELEMENTID GetElementID( PSZ pszName ); - void Push( PTMXELEMENT pElement ); - void Pop( PTMXELEMENT pElement ); - BOOL GetValue( PSZ pszString, int iLen, int *piResult ); - BOOL TMXLanguage2TMLanguage( PSZ pszTMLanguage, PSZ pszTMXLanguage, PSZ pszResultingLanguage ); - USHORT RemoveRTFTags( PSZ_W pszString, BOOL fTagsInCurlyBracesOnly ); + const xercesc::Locator *m_locator = nullptr; + ImportStatusDetails *pImportDetails = nullptr; + ELEMENTID GetElementID(PSZ pszName); + void Push(PTMXELEMENT pElement); + void Pop(PTMXELEMENT pElement); + BOOL GetValue(PSZ pszString, int iLen, int *piResult); + BOOL TMXLanguage2TMLanguage(PSZ pszTMLanguage, PSZ pszTMXLanguage, PSZ pszResultingLanguage); + USHORT RemoveRTFTags(PSZ_W pszString, BOOL fTagsInCurlyBracesOnly); // date conversion help functions - BOOL IsLeapYear( const int iYear ); - int GetDaysOfMonth( const int iMonth, const int iYear ); - int GetDaysOfYear( const int iYear ); - int GetYearDay( const int iDay, const int iMonth, const int iYear ); + BOOL IsLeapYear(const int iYear); + int GetDaysOfMonth(const int iMonth, const int iYear); + int GetDaysOfYear(const int iYear); + int GetYearDay(const int iDay, const int iMonth, const int iYear); // mem import interface data PMEMEXPIMPINFO m_pMemInfo; PFN_MEMINSERTSEGMENT pfnInsertSegment; - LONG lMemHandle; + LONG lMemHandle; // processing flags - BOOL fSource; // TRUE = source data collected - BOOL fTarget; // TRUE = target data collected - BOOL fCatchData; // TRUE = catch data - BOOL fWithTagging; // TRUE = add tagging to data - BOOL fWithTMXTags; // TRUE = segment contains TMX tags - BOOL fTMXTagStarted; // TRUE = TMX inline tag started - BOOL fHeaderDone; // TRUE = header has been processed - BOOL fError; // TRUE = parsing ended with error + BOOL fSource; // TRUE = source data collected + BOOL fTarget; // TRUE = target data collected + BOOL fCatchData; // TRUE = catch data + BOOL fWithTagging; // TRUE = add tagging to data + BOOL fWithTMXTags; // TRUE = segment contains TMX tags + BOOL fTMXTagStarted; // TRUE = TMX inline tag started + BOOL fHeaderDone; // TRUE = header has been processed + BOOL fError; // TRUE = parsing ended with error bool fBodyDone = false; - + // segment data - ULONG ulSegNo; // segmet number - LONG lTime; // segment date/time - USHORT usTranslationFlag; // type of translation flag - int iNumOfTu; + ULONG ulSegNo; // segmet number + LONG lTime; // segment date/time + USHORT usTranslationFlag; // type of translation flag + int iNumOfTu; size_t _invalidCharacterErrorCount{0}; - // buffers - #define DATABUFFERSIZE 4098 +// buffers +#define DATABUFFERSIZE 4098 typedef struct _BUFFERAREAS { - CHAR_W szData[DATABUFFERSIZE]; // buffer for collected data - CHAR_W szReplacedNpData[DATABUFFERSIZE]; - CHAR_W szNormalizedData[DATABUFFERSIZE]; - CHAR_W szPropW[DATABUFFERSIZE]; // buffer for collected prop values - CHAR_W szContext[DATABUFFERSIZE]; // buffer for collected prop values - CHAR_W szAddInfo[DATABUFFERSIZE]; // buffer for collected prop values - CHAR szLang[50]; // buffer for language - CHAR szDocument[EQF_DOCNAMELEN];// buffer for document name - - CHAR szAuthor[EQF_DOCNAMELEN];// buffer for author name - CHAR szChangeId[EQF_DOCNAMELEN];// buffer for author/changeId name - CHAR szCreationId[EQF_DOCNAMELEN];// buffer for author/creationId name - - MEMEXPIMPSEG SegmentData; // buffer for segment data - CHAR szDescription[1024]; // buffer for memory descripion - CHAR szMemSourceLang[50]; // buffer for memory source language - CHAR szMemSourceIsoLang[50]; - CHAR szErrorMessage[1024]; // buffer for error message text - CHAR_W szNote[MAX_SEGMENT_SIZE];// buffer for note text - CHAR_W szNoteStyle[100]; // buffer for note style - CHAR_W szMTMetrics[MAX_SEGMENT_SIZE]; // buffer for MT metrics data - ULONG ulWords; // number of words in the segment text - CHAR_W szMatchSegID[MAX_SEGMENT_SIZE];// buffer for match segment ID - - //for inclosing tags skipping. If pFirstBptTag is nullptr->segments don't start from bpt tag-> inclosing tags skipping should be ignored - int inclosingTagsAtTheStartOfSegment = 0; - int inclosingTagsAtTheEndOfSegment = 0; - + CHAR_W szData[DATABUFFERSIZE]; // buffer for collected data + CHAR_W szReplacedNpData[DATABUFFERSIZE]; + CHAR_W szNormalizedData[DATABUFFERSIZE]; + CHAR_W szPropW[DATABUFFERSIZE]; // buffer for collected prop values + CHAR_W szContext[DATABUFFERSIZE]; // buffer for collected prop values + CHAR_W szAddInfo[DATABUFFERSIZE]; // buffer for collected prop values + CHAR szLang[50]; // buffer for language + CHAR szDocument[EQF_DOCNAMELEN]; // buffer for document name + + CHAR szAuthor[EQF_DOCNAMELEN]; // buffer for author name + CHAR szChangeId[EQF_DOCNAMELEN]; // buffer for author/changeId name + CHAR szCreationId[EQF_DOCNAMELEN]; // buffer for author/creationId name + + MEMEXPIMPSEG SegmentData; // buffer for segment data + CHAR szDescription[1024]; // buffer for memory descripion + CHAR szMemSourceLang[50]; // buffer for memory source language + CHAR szMemSourceIsoLang[50]; + CHAR szErrorMessage[1024]; // buffer for error message text + CHAR_W szNote[MAX_SEGMENT_SIZE]; // buffer for note text + CHAR_W szNoteStyle[100]; // buffer for note style + CHAR_W szMTMetrics[MAX_SEGMENT_SIZE]; // buffer for MT metrics data + ULONG ulWords; // number of words in the segment text + CHAR_W szMatchSegID[MAX_SEGMENT_SIZE]; // buffer for match segment ID + + // for inclosing tags skipping. If pFirstBptTag is nullptr->segments don't start from bpt tag-> inclosing tags skipping should be ignored + int inclosingTagsAtTheStartOfSegment = 0; + int inclosingTagsAtTheEndOfSegment = 0; + InclosingTagsBehaviour inclosingTagsBehaviour = saveAll; - - bool fUseMajorLanguage; + + bool fUseMajorLanguage; } BUFFERAREAS, *PBUFFERAREAS; - PBUFFERAREAS pBuf; + PBUFFERAREAS pBuf; // TUV data area typedef struct _TMXTUV { - CHAR_W szText[DATABUFFERSIZE]; // buffer for TUV text - CHAR szLang[50]; // buffer for TUV language (converted to Tmgr language name) or original language in case of fInvalidLang - CHAR szIsoLang[10]; - BOOL fInvalidChars; // TRUE = contains invalid characters - BOOL fInlineTags; // TRUE = contains inline tagging - BOOL fInvalidLang; // TRUE = the language of the TUV is invalid + CHAR_W szText[DATABUFFERSIZE]; // buffer for TUV text + CHAR szLang[50]; // buffer for TUV language (converted to Tmgr language name) or original language in case of fInvalidLang + CHAR szIsoLang[10]; + BOOL fInvalidChars; // TRUE = contains invalid characters + BOOL fInlineTags; // TRUE = contains inline tagging + BOOL fInvalidLang; // TRUE = the language of the TUV is invalid } TMXTUV, *PTMXTUV; PTMXTUV pTuvArray; - int iCurTuv; // current TUV index - int iTuvArraySize; // current size of TUV array + int iCurTuv; // current TUV index + int iTuvArraySize; // current size of TUV array // element stack int iStackSize; @@ -1372,49 +1453,47 @@ class TMXParseHandler : public xercesc::HandlerBase TMXELEMENT CurElement; PTMXELEMENT pStack; - bool fSawErrors; - BOOL fInvalidChars; // TRUE = current TUV contains invalid characters - BOOL fInlineTags; // TRUE = current TUV contains inline tagging + bool fSawErrors; + BOOL fInvalidChars; // TRUE = current TUV contains invalid characters + BOOL fInlineTags; // TRUE = current TUV contains inline tagging // data for remove tag function PTOKENENTRY pTokBuf; int iTokBufSize; - //ULONG ulCP; - - void fillSegmentInfo( PTMXTUV pSource, PTMXTUV pTarget, PMEMEXPIMPSEG pSegment ); + // ULONG ulCP; + void fillSegmentInfo(PTMXTUV pSource, PTMXTUV pTarget, PMEMEXPIMPSEG pSegment); }; - -class StringVariants{ +class StringVariants +{ protected: std::wstring original; // full original str std::wstring originalTarget; - bool fSuccess = false; + bool fSuccess = false; - xercesc::SAXParser parser; TMXParseHandler handler; xercesc::XMLPScanToken saxToken; + public: - StringVariants(std::wstring&& w_src, std::wstring&& w_trg){ - original = std::move(w_src); - originalTarget = std::move(w_trg); - } - bool isParsed() const{ return fSuccess; } - std::wstring& getOriginalSrcStr(){ return original; } - std::wstring& getOriginalTrgStr(){ return original; } - wchar_t* getOriginalStrC() { return &original[0]; } - wchar_t* getOriginalTrgStrC() {return &originalTarget[0];} + StringVariants(std::wstring &&w_src, std::wstring &&w_trg): + original(std::move(w_src)), originalTarget(std::move(w_trg)) { } + bool isParsed() const { return fSuccess; } + std::wstring &getOriginalSrcStr() { return original; } + std::wstring &getOriginalTrgStr() { return original; } + wchar_t *getOriginalStrC() { return &original[0]; } + wchar_t *getOriginalTrgStrC() { return &originalTarget[0]; } }; -class StringTagVariants: public StringVariants{ +class StringTagVariants : public StringVariants +{ protected: - std::wstring norm; // without any tags - std::wstring npReplaced; // with generic(generated) tags but np tags is replaced with their keys(r attr) + std::wstring norm; // without any tags + std::wstring npReplaced; // with generic(generated) tags but np tags is replaced with their keys(r attr) std::wstring genericTags; // generated tags so there could be only ph or bpt/ept with id and r attributes that would map input str to common tag format - std::wstring genericTarget; + std::wstring genericTarget; std::wstring normTarget; void initParser(); @@ -1425,272 +1504,238 @@ class StringTagVariants: public StringVariants{ std::string trg; public: - StringTagVariants(std::wstring&& w_src): StringTagVariants(std::move(w_src), L""){} - StringTagVariants(std::wstring&& w_src, std::wstring&& w_trg): StringVariants(std::move(w_src), std::move(w_trg)){ + StringTagVariants(std::wstring &&w_src) : StringTagVariants(std::move(w_src), L"") {} + StringTagVariants(std::wstring &&w_src, std::wstring &&w_trg) : StringVariants(std::move(w_src), std::move(w_trg)) + { initParser(); parseSrc(); - if(fSuccess){ + if (fSuccess) + { parseTrg(); logResults(); } } bool hasTarget() const { return !genericTarget.empty(); } - bool replaceTags(std::wstring&& w_request_input); - - std::wstring& getNormStr(){ return norm; } - std::wstring& getNpReplacedStr(){ return npReplaced; } - std::wstring& getGenericTagsString() { return genericTags; } - std::wstring& getNormTargetStr(){return normTarget; } - - wchar_t* getNormStrC(){return &norm[0];} - wchar_t* getNpReplStrC(){return &npReplaced[0];} - wchar_t* getGenericTagStrC() { return &genericTags[0]; } - wchar_t* getGenericTargetStrC() {return &genericTarget[0];} - wchar_t* getNormalizedTargetStrC() { return &normTarget[0]; } + bool replaceTags(std::wstring &&w_request_input); + + std::wstring &getNormStr() { return norm; } + std::wstring &getNpReplacedStr() { return npReplaced; } + std::wstring &getGenericTagsString() { return genericTags; } + std::wstring &getGenericTargetStr() { return genericTarget; } + std::wstring &getNormTargetStr() { return normTarget; } + + wchar_t *getNormStrC() { return norm.data(); } + wchar_t *getNpReplStrC() { return npReplaced.data(); } + wchar_t *getGenericTagStrC() { return genericTags.data(); } + wchar_t *getGenericTargetStrC() { return genericTarget.data(); } + wchar_t *getNormalizedTargetStrC() { return normTarget.data(); } }; -class RequestTagReplacer: public StringVariants{ +class RequestTagReplacer : public StringVariants +{ std::wstring replacedTagsSrc, replacedTagsTrg, genericTagsReq, originalReq; void parseAndReplaceTags(); - public: - RequestTagReplacer(std::wstring&& w_src, std::wstring&& w_trg, std::wstring&& w_req): StringVariants(std::move(w_src), std::move(w_trg)){ - originalReq = std::move(w_req); - parseAndReplaceTags(); +public: + RequestTagReplacer(std::wstring &&w_src, std::wstring &&w_trg, std::wstring &&w_req) : StringVariants(std::move(w_src), std::move(w_trg)) + { + originalReq = std::move(w_req); + parseAndReplaceTags(); }; - wchar_t* getSrcWithTagsFromRequestC(){ return &replacedTagsSrc[0];} - wchar_t* getTrgWithTagsFromRequestC(){ return &replacedTagsTrg[0];} - wchar_t* getReqGenericTagStrC(){return &genericTagsReq[0]; } - wchar_t* getOrigReqC(){return &originalReq[0];} + wchar_t *getSrcWithTagsFromRequestC() { return &replacedTagsSrc[0]; } + wchar_t *getTrgWithTagsFromRequestC() { return &replacedTagsTrg[0]; } + wchar_t *getReqGenericTagStrC() { return &genericTagsReq[0]; } + wchar_t *getOrigReqC() { return &originalReq[0]; } - std::wstring& getSrcWithTagsFromRequest(){ return replacedTagsSrc;} - std::wstring& getTrgWithTagsFromRequest(){ return replacedTagsTrg;} - std::wstring& getReqGenericTagStr(){return genericTagsReq; } - std::wstring& getOrigReq(){return originalReq;} + std::wstring &getSrcWithTagsFromRequest() { return replacedTagsSrc; } + std::wstring &getTrgWithTagsFromRequest() { return replacedTagsTrg; } + std::wstring &getReqGenericTagStr() { return genericTagsReq; } + std::wstring &getOrigReq() { return originalReq; } }; struct TMX_SENTENCE { - std::unique_ptr pStrings = nullptr; - //StringTagVariants* pStrings = nullptr; - - PSZ_W pAddString = nullptr; - //USHORT usNormLen = 0; - PTMX_TERM_TOKEN pTermTokens = nullptr; - LONG lTermAlloc = 0; - PTMX_TAGTABLE_RECORD pTagRecord = nullptr; - LONG lTagAlloc = 0; - PTMX_TAGENTRY pTagEntryList = nullptr; - USHORT usActVote = 0; - PULONG pulVotes = nullptr; - std::unique_ptr pPropString = nullptr; - //StringTagVariants * pPropString = nullptr; - - PTMX_TERM_TOKEN pPropTermTokens = nullptr; // buffer for Termtokens - - //ADDINFO - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language name of source - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_FILESPEC]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - CHAR szTagTable[MAX_FNAME]; //tag table name - TIME_L lTime; //time stamp - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import - //end ADDINFO + // ADDINFO + std::array pAddString; // segment context + std::array szAddInfo; // additional segment information + std::array szContext; // additional segment information + std::array szAuthorName; // author name of target' + std::array szFileName; // where source comes from name+ext + std::array szTagTable; // tag table name + std::array szSourceLanguage; // language name of source + std::array szTargetLanguage; // language name of target + + std::unique_ptr pStrings, pPropString; + std::vector pTermTokens; + std::vector pTagRecord;//TMX_TAGTABLE_RECORD + USHORT usActVote = 0; + std::vector pulVotes; + std::vector pPropTermTokens; // buffer for Termtokens - std::vector tokens; + + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis + TIME_L lTime; // time stamp + BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import + // end ADDINFO + std::vector tokens; - TMX_SENTENCE(std::wstring&& w_src, std::wstring &&w_trg){ - if(w_trg.empty()){ - pStrings = std::make_unique(std::move(w_src));//new StringTagVariants(std::move(w_src)); //;std::make_unique (std::move(w_src)); - }else{ - pStrings = std::make_unique(std::move(w_src),std::move(w_trg));//new StringTagVariants(std::move(w_src), std::move(w_trg)); // std::make_unique (std::move(w_src), std::move(w_trg)); + TMX_SENTENCE(std::wstring &&w_src, std::wstring &&w_trg) + { + if (w_trg.empty()) + { + pStrings = std::make_unique(std::move(w_src)); } - bool fOK = true; - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pulVotes), 0L, (LONG)(ABS_VOTES * sizeof(ULONG)), NOMSG ); - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG)(2*TOK_SIZE), NOMSG); - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pTermTokens), 0L, (LONG)TOK_SIZE, NOMSG ); - - if ( fOK ) + else { - lTermAlloc = (LONG)TOK_SIZE; - lTagAlloc = (LONG)(2*TOK_SIZE); - } /* endif */ - if(!fOK){ - throw ;//new Exception(); + pStrings = std::make_unique(std::move(w_src),std::move(w_trg)); } - //pStrings = input; - //usNormLen = pStrings->getNormStr().length(); - } - - TMX_SENTENCE(wchar_t* w_src): TMX_SENTENCE(std::move(w_src), L""){} - - ~TMX_SENTENCE(){ - //if(pStrings){ - // delete pStrings; - // pStrings = nullptr; - //} - //if(pPropString){ - // delete pPropString; - // pPropString = nullptr; - //} - //if(pPropString) pPropString.reset(); - //if(pStrings) pStrings.reset(); - UtlAlloc( (PVOID *) &pTermTokens, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTagRecord, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pulVotes, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pAddString, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTermTokens, 0L, 0L, NOMSG ); + bool fOK = true; + pulVotes.reserve(ABS_VOTES / sizeof(pulVotes[0])); + pTagRecord.reserve(2 * TOK_SIZE); + pTagRecord.resize(sizeof(TMX_TAGTABLE_RECORD)); + pTermTokens.reserve(TOK_SIZE / sizeof(pTermTokens[0])); } - bool wasParsedSuccessfully()const {return pStrings && pStrings->isParsed(); } -}; -using PTMX_SENTENCE = TMX_SENTENCE *; + explicit TMX_SENTENCE(const wchar_t* w_src) : TMX_SENTENCE(std::wstring(w_src), L"") {} -//#pragma pack(pop) -//#pragma pack() + bool wasParsedSuccessfully() const { return pStrings && pStrings->isParsed(); } +}; +using PTMX_SENTENCE = TMX_SENTENCE *; +// #pragma pack(pop) +// #pragma pack() -//#pragma pack(pop) -//#pragma pack() +// #pragma pack(pop) +// #pragma pack() typedef struct _TMX_REPLTAGPAIR { - PBYTE pSrcTok; // token for paired toks - PBYTE pPropTok; - BOOL fUsed; //TRUE = Tag pair has been used in replacement already -} TMX_REPLTAGPAIR, * PTMX_REPLTAGPAIR; + PBYTE pSrcTok; // token for paired toks + PBYTE pPropTok; + BOOL fUsed; // TRUE = Tag pair has been used in replacement already +} TMX_REPLTAGPAIR, *PTMX_REPLTAGPAIR; typedef struct _TMX_SUBSTPROP { - CHAR_W szSource [ MAX_SEGMENT_SIZE ]; // source segment - CHAR_W szPropSource[ MAX_SEGMENT_SIZE ]; // source proposal - CHAR_W szPropTarget[ MAX_SEGMENT_SIZE ]; // translation of proposal - char szSourceLanguage[MAX_LANG_LENGTH];// source language - char szSourceTagTable[MAX_FNAME]; // source tagtable - char szTargetLanguage[MAX_LANG_LENGTH];// target language - char szPropTagTable[MAX_FNAME]; // tagtable for proposal - PBYTE pTokSource; // token buffer for source tokens - PBYTE pTokPropSource; // token buffer for proposal tokens - PBYTE pTokPropTarget; // token buffer for proposal target .. - //PTMX_TAGTABLE_RECORD pTagsSource; // tag table record for source .. - //PTMX_TAGTABLE_RECORD pTagsPropSource; // tag table record for prop source - //PTMX_TAGTABLE_RECORD pTagsPropTarget; // tag table record for prop target - CHAR_W chBuffer[ MAX_SEGMENT_SIZE * 2 ]; // generic token buffer + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source segment + CHAR_W szPropSource[MAX_SEGMENT_SIZE]; // source proposal + CHAR_W szPropTarget[MAX_SEGMENT_SIZE]; // translation of proposal + char szSourceLanguage[MAX_LANG_LENGTH]; // source language + char szSourceTagTable[MAX_FNAME]; // source tagtable + char szTargetLanguage[MAX_LANG_LENGTH]; // target language + char szPropTagTable[MAX_FNAME]; // tagtable for proposal + CHAR_W chBuffer[MAX_SEGMENT_SIZE * 2]; // generic token buffer USHORT usTokenSource; // number of source tokens USHORT usTokenPropSource; // number of prop source tokens USHORT usTokenPropTarget; // number of prop target tokens - ULONG ulRandom[MAX_RANDOM]; // random sequence number - PTMX_REPLTAGPAIR pTagPairs; // tags to be replaced by each other - PTMX_REPLTAGPAIR pDelTagPairs; // tags to be deleted in prop -} TMX_SUBSTPROP, * PTMX_SUBSTPROP; - - + ULONG ulRandom[MAX_RANDOM]; // random sequence number +} TMX_SUBSTPROP, *PTMX_SUBSTPROP; //======================================================================= typedef struct _TMX_PREFIX_IN { - USHORT usLengthInput; //length of complete input structure - USHORT usTmCommand; //TM command id -} TMX_PREFIX_IN, * PTMX_PREFIX_IN, XIN, * PXIN; + USHORT usLengthInput; // length of complete input structure + USHORT usTmCommand; // TM command id +} TMX_PREFIX_IN, *PTMX_PREFIX_IN, XIN, *PXIN; typedef struct _TMX_PUT_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language name of source - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_LANG_LENGTH]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - CHAR szTagTable[MAX_FNAME]; //tag table name - TIME_L lTime; //time stamp - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import -} TMX_PUT_W, * PTMX_PUT_W; - - + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence + CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language name of source + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target + CHAR szAuthorName[MAX_LANG_LENGTH]; // author name of target + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis + CHAR szTagTable[MAX_FNAME]; // tag table name + TIME_L lTime; // time stamp + CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import +} TMX_PUT_W, *PTMX_PUT_W; typedef struct _TMX_GET_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR szTagTable[MAX_FNAME]; //tag table name of source - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language of source - CHAR szFileName[MAX_FILESPEC]; //file name the source comes from - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; //segment number from analysis - CHAR szAuthorName[MAX_LANG_LENGTH]; //author name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language of translation - USHORT usRequestedMatches; //number of requested matches - USHORT usMatchThreshold; //threshold for match level - USHORT usConvert; //how the output should be converted - ULONG ulParm; //for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - ULONG ulSrcOemCP; - ULONG ulTgtOemCP; - PVOID pvReplacementList; // ptr to a SGML-DITA replacement list or NULL - PVOID pvGMOptList; // ptr to a global memory option list or NULL - bool fSourceLangIsPrefered; - bool fTargetLangIsPrefered; -} TMX_GET_W, * PTMX_GET_W; - + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence + CHAR szTagTable[MAX_FNAME]{}; // tag table name of source + CHAR szSourceLanguage[MAX_LANG_LENGTH]{}; // language of source + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSegmentId = 0; // segment number from analysis + CHAR szAuthorName[MAX_LANG_LENGTH]{}; // author name + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language of translation + USHORT usRequestedMatches = 0; // number of requested matches + USHORT usMatchThreshold = 0; // threshold for match level + USHORT usConvert = 0; // how the output should be converted + ULONG ulParm = 0; // for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) + CHAR_W szContext[MAX_SEGMENT_SIZE]{}; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]{}; // additional segment information + PVOID pvReplacementList = nullptr; // ptr to a SGML-DITA replacement list or NULL + PVOID pvGMOptList = nullptr; // ptr to a global memory option list or NULL + bool fSourceLangIsPrefered = false; + bool fTargetLangIsPrefered = false; + bool fForceFuzzySearch = false; +} TMX_GET_W, *PTMX_GET_W; //======================================================================= // structure TMX_GET_OUT -typedef struct _TMX_MATCH_TABLE_W +struct TMX_MATCH_TABLE_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence with tags - CHAR szFileName[MAX_FILESPEC]; //file name where the source comes from. - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; //segment number from analysis - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence with tags - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language of translation - CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]; //language of src of translation - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; //author name of target - TIME_L lTargetTime; //time stamp of target - USHORT usMatchLevel; //similarity of the source - int iWords; - int iDiffs; - USHORT usOverlaps; //temp field - nr of overlapping triples - CHAR szTagTable[MAX_FNAME]; //tag table name of source - ULONG ulKey; // key of match - USHORT usTargetNum; // number of target - USHORT usDBIndex; // number of memory in current hierarchy - CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context info - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - USHORT usContextRanking; // context ranking from user exit context processing - USHORT usMatchInfo; // for future use: special info for match -} TMX_MATCH_TABLE_W, * PTMX_MATCH_TABLE_W; - -using TMX_GET_OUT_W = struct _TMX_GET_OUT_W : public TMX_EXT_OUT_W + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence with tags + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSegmentId{}; // segment number from analysis + CHAR_W szTarget[MAX_SEGMENT_SIZE]{}; // target sentence with tags + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language of translation + CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]{}; // language of src of translation + USHORT usTranslationFlag{}; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]{}; // author name of target + TIME_L lTargetTime{}; // time stamp of target + USHORT usMatchLevel{}; // similarity of the source + int iWords{}; + int iDiffs{}; + USHORT usOverlaps{}; // temp field - nr of overlapping triples + CHAR szTagTable[MAX_FNAME]{}; // tag table name of source + LONG lKey{}; // key of match + USHORT usTargetNum{}; // number of target + USHORT usDBIndex{}; // number of memory in current hierarchy + CHAR_W szContext[MAX_SEGMENT_SIZE]{}; // segment context info + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]{}; // additional segment information + USHORT usContextRanking{}; // context ranking from user exit context processing + USHORT usMatchInfo{}; // for future use: special info for match + + bool sameTMLocation(LONG recordKey, USHORT targetKey)const{ + return recordKey == lKey && targetKey == usTargetNum; + } + + bool isLocationAlreadyAdded(USHORT elementsAdded, LONG recordKey, USHORT targetKey){//use this function only with the pointer to the first element + bool matchExists = std::any_of( + this, this + elementsAdded, + [&](const TMX_MATCH_TABLE_W& entry) { + return entry.sameTMLocation(recordKey, targetKey); + }); + return matchExists; + } +}; + +using PTMX_MATCH_TABLE_W = TMX_MATCH_TABLE_W*; + +struct TMX_GET_OUT_W : public TMX_EXT_OUT_W { - //TMX_PREFIX_OUT stPrefixOut; //prefix of Output buffer - USHORT usNumMatchesFound; //number of matches found - TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]; //match structure - USHORT fsAvailFlags; // additional flags (more exact/fuzzy matches avail) + USHORT usNumMatchesFound = 0; // number of matches found + TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]{}; // match structure + USHORT fsAvailFlags = 0; // additional flags (more exact/fuzzy matches avail) std::vector tokens; + TMX_GET_OUT_W(){} }; using PTMX_GET_OUT_W = TMX_GET_OUT_W *; - - - - - //======================================================================= /**********************************************************************/ /* Attention: for special names mode of TMExtract, the TMX_EXT_IN */ @@ -1706,13 +1751,13 @@ using PTMX_GET_OUT_W = TMX_GET_OUT_W *; /* names are available than fit into the output buffers. */ /* The remining fields are unused. */ /**********************************************************************/ -//structure TMX_EXT_IN +// structure TMX_EXT_IN typedef struct _TMX_EXT_IN { - ULONG ulTmKey; //tm get to get - USHORT usConvert; //how the output is to appear - USHORT usNextTarget; //which target record to address next -} TMX_EXT_IN, * PTMX_EXT_IN; + LONG lTmKey{}; // tm get to get + USHORT usConvert{}; // how the output is to appear + USHORT usNextTarget{}; // which target record to address next +} TMX_EXT_IN, *PTMX_EXT_IN; typedef TMX_EXT_IN TMX_EXT_IN_W, *PTMX_EXT_IN_W; //======================================================================= @@ -1730,656 +1775,516 @@ typedef TMX_EXT_IN TMX_EXT_IN_W, *PTMX_EXT_IN_W; // structure TMX_EXT_OUT class EqfMemoryPlugin; -class Dummy{}; +class Dummy +{ +}; class RequestData; - -#define ASD_ORGANIZE 0x100 // work in organize mode +#define ASD_ORGANIZE 0x100 // work in organize mode std::string StatusToString(int eStatus); #include -class EqfMemory //: public TMX_CLB -/*! \brief This class implements the standard translation memory (EQF) for OpenTM2. -*/ -{ - - USHORT LoadMem(); +namespace EqfMemoryError { + constexpr int ERROR_INTERNALKEY_MISSING = 1001; + constexpr int ERROR_MEMORYOBJECTISNULL = 8003; + constexpr int INFO_ENDREACHED = 8008; + constexpr int ERROR_ENTRYISCORRUPTED = 8009; +} -public: - std::recursive_timed_mutex tmMutex; - //bool fOpen = false; +struct EqfMemoryState{ BTREE TmBtree; BTREE InBtree; TMX_TABLE Languages; - TMX_TABLE FileNames; TMX_TABLE Authors; TMX_TABLE TagTables; + TMX_TABLE LangGroups; // table containing language group names + std::vector psLangIdToGroupTable; // language ID to group ID table + + size_t expectedSizeInRam = 0; + bool fCorrupted = false; USHORT usAccessMode = 0; - USHORT usThreshold = 0; TMX_SIGN stTmSign; - BYTE bCompact[MAX_COMPACT_SIZE-1]; - BYTE bCompactChanged = 0; - LONG alUpdCounter[MAX_UPD_COUNTERS]; - PTMX_LONGNAME_TABLE pLongNames = nullptr; - TMX_TABLE LangGroups; // table containing language group names - PSHORT psLangIdToGroupTable = nullptr; // language ID to group ID table - LONG lLangIdToGroupTableSize = 0; // size of table (alloc size) - LONG lLangIdToGroupTableUsed = 0; // size of table (bytes in use) - PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) + std::vector bCompact; + BYTE bCompactChanged = 0; + PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) // copy of long name table sorted ignoring the case of the file names // Note: only the stTableEntry array is filled in this area, for all other // information use the entries in the pLongNames structure - PTMX_LONGNAME_TABLE pLongNamesCaseIgnore = nullptr; + // std::vector pLongNamesCaseIgnore; + TMX_LONGNAME_TABLE LongNames, LongNamesCaseIgnore; // fields for work area pointers of various subfunctions which are allocated // only once for performance reasons - PVOID pvTempMatchList = nullptr; // matchlist of FillMatchEntry function - PVOID pvIndexRecord = nullptr; // index record area of FillMatchEntry function - PVOID pvTmRecord = nullptr; // buffer for memory record used by GetFuzzy and GetExact - ULONG ulRecBufSize = 0; // current size of pvTMRecord; - - // fields for time measurements and logging - BOOL fTimeLogging = 0; // TRUE = Time logging is active - LONG64 lAllocTime = 0; // time for memory allocation - LONG64 lTokenizeTime = 0; // time for tokenization - LONG64 lGetExactTime = 0; // time for GetExact - LONG64 lOtherTime = 0; // time for other activities - LONG64 lGetFuzzyTime = 0; // time for GetFuzzy - LONG64 lFuzzyOtherTime = 0; // other time spent in GetFuzzy - LONG64 lFuzzyTestTime = 0; // FuzzyTest time spent in GetFuzzy - LONG64 lFuzzyGetTime = 0; // NTMGet time spent in GetFuzzy - LONG64 lFuzzyFillMatchEntry = 0; // FillMatchEntry time spent in GetFuzzy - LONG64 lFillMatchAllocTime = 0; // FillMatchEntry: allocation time - LONG64 lFillMatchOtherTime = 0; // FillMatchEntry: other times - LONG64 lFillMatchReadTime = 0; // FillMatchEntry: read index DB time - LONG64 lFillMatchFillTime = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchCleanupTime = 0; // FillMatchEntry: cleanup match list time - LONG64 lFillMatchFill1Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill2Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill3Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill4Time = 0; // FillMatchEntry: fill match list time - - //std::shared_ptr readOnlyPtr; - //std::shared_ptr writePtr; + std::vector pvTempMatchList; // matchlist of FillMatchEntry function + std::vector pvTmRecord; // buffer for memory record used by GetFuzzy and GetExact + +}; + +class EqfMemory //: public TMX_CLB +/*! \brief This class implements the standard translation memory (EQF) for OpenTM2. + */ + +{ +public: + std::recursive_timed_mutex tmMutex; + + std::unique_ptr state; + std::shared_ptr readOnlyCnt; std::shared_ptr writeCnt; + RequestData *pActiveRequest; + COMMAND activeCommand = UNKNOWN_COMMAND; + time_t tLastAccessTime = 0; // last time memory has been used + MEMORY_STATUS eStatus; // status of the memory + MEMORY_STATUS eImportStatus; // status of the current/last memory import + std::unique_ptr importDetails; - //std::weak_ptr pActiveRequest; - RequestData* pActiveRequest; - int lastErrorCode = 0; + std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) - size_t expectedSizeInRam = 0; + std::string szName; // name of the memory + std::string szDescription; // description of the memory + std::string szSourceLanguage; // memory source language - int getErrorCode()const; - int setErrorCode(int rc); - void reserErrorCode(); + +//methods + + /*! \brief Constructors + */ + + /*! \brief Destructor + */ + ~EqfMemory(); + + USHORT Load(); + + int sign(){ + return state->TmBtree.EQFNTMSign(state->stTmSign); + } COMMAND getActiveRequestCommand(); void setActiveRequest(std::weak_ptr request); - void setActiveRequest(RequestData* request); + void setActiveRequest(RequestData *request); void resetActiveRequest(); - - COMMAND activeCommand = UNKNOWN_COMMAND; + void setActiveRequestCommand(COMMAND command); void resetActiveRequestCommand(); - void setExpecedSizeInRAM(size_t newSize){ - expectedSizeInRam = newSize; + void setExpecedSizeInRAM(size_t newSize) + { + state->expectedSizeInRam = newSize; } - size_t getExpectedSizeInRAM()const{ - return expectedSizeInRam; + size_t getExpectedSizeInRAM() const + { + return state->expectedSizeInRam; } - - std::string getStatusString()const; - std::string getActiveRequestString()const; - bool isWaitingToBeLoaded()const{ + std::string getStatusString() const; + std::string getActiveRequestString() const; + + bool isWaitingToBeLoaded() const + { return WAITING_FOR_LOADING_STATUS == eStatus; } - bool isLoading()const{ + bool isLoading() const + { return LOADING_STATUS == eStatus; } - bool isLoaded()const{ + bool isLoaded() const + { return OPEN_STATUS == eStatus; } - bool isFailedToLoad()const{ + bool isFailedToLoad() const + { return FAILED_TO_OPEN_STATUS == eStatus; } - bool isReorganizeRunning()const{ + bool isCorrupted()const{ + return state->fCorrupted; + } + + bool isReorganizeRunning() const + { return REORGANIZE_RUNNING_STATUS == eStatus; } - bool isImportRunning()const{ + bool isImportRunning() const + { return IMPORT_RUNNING_STATUS == eStatus; } - //bool fReorganizeOnly = false; - bool isOpenedOnlyForReorganize()const{ - return usAccessMode & ASD_ORGANIZE; + // bool fReorganizeOnly = false; + bool isOpenedOnlyForReorganize() const + { + return state->usAccessMode & ASD_ORGANIZE; } - void resetOpenedOnlyForReorganize(){ + void resetOpenedOnlyForReorganize() + { unsigned int mask = ~ASD_ORGANIZE; - usAccessMode &= mask; + state->usAccessMode &= mask; + state->fCorrupted = false; } - void updateLastUseTime(){ + void updateLastUseTime() + { tLastAccessTime = time(0); } -/*! \brief OtmMemory related return codes - -*/ - static const int ERROR_INTERNALKEY_MISSING = 1001; - -/*! \brief Constructors -*/ - + BYTE getMajorVersion()const{ + return state->stTmSign.bMajorVersion; + } -/*! \brief Destructor -*/ - ~EqfMemory(); + ushort TmtXDelSegm( + OtmProposal &pTmDelIn, // ptr to input struct + TMX_EXT_OUT_W *pTmDelOut // ptr to output struct + ); - USHORT Load(); - -/*! \brief Info structure for an open translation memory -*/ - //typedef struct _OPENEDMEMORY - //{ - //char szName[260]; // name of the memory - time_t tLastAccessTime = 0; // last time memory has been used - long lHandle = 0; // memory handle - MEMORY_STATUS eStatus; // status of the memory - MEMORY_STATUS eImportStatus; // status of the current/last memory import - //std::atomic dImportProcess; - //ushort * pusImportPersent = nullptr; - ImportStatusDetails* importDetails = nullptr; - std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) - - PPROP_NTM memoryProperties; - /*! \brief structure for memory information */ - // typedef struct _MEMORYINFO - // { - - // } MEMORYINFO, *PMEMORYINFO; -//} OPENEDMEMORY ; - -/*! \brief Error code definition -*/ - static const int ERROR_MEMORYOBJECTISNULL = 8003; - static const int ERROR_BUFFERTOOSMALL = 8004; - static const int ERROR_INVALIDOBJNAME = 8005; - static const int ERROR_MEMORYEXISTS = 8006; - static const int INFO_ENDREACHED = 8008; - static const int ERROR_ENTRYISCORRUPTED = 8009; + USHORT TmtXDelSegmByKey( + OtmProposal &TmDelIn, // ptr to input struct + TMX_EXT_OUT_W *pTmDelOut // ptr to output struct + ); + USHORT FindTargetAndDelete( + BytesRef TmRecord, + OtmProposal &TmDel, + TMX_EXT_OUT_W *pTmExtOut, + LONG& lKey); - /*! \brief Flags for the update of proposals */ + USHORT FindTargetByKeyAndDelete( + BytesRef pTmRecord, + OtmProposal &pTmDel, + PTMX_SENTENCE pSentence, + TMX_EXT_OUT_W *pTmExtOut, + LONG& lKey); - static const int UPDATE_MARKUP = 0x01; // update markup/tag table - static const int UPDATE_MTFLAG = 0x02; // update machine translation flag - static const int UPDATE_TARGLANG = 0x04; // update target language - static const int UPDATE_DATE = 0x08; // update proposal update time + USHORT TokenizeSource(PTMX_SENTENCE, PSZ, PSZ); - //void setInternalCursor(ULONG recordKey, USHORT targetKey); - //void resetInternalCursor(); + + bool areLangsMatching(const ushort langId1, const ushort langId2, const char* langStr1);//, bool fPrefferedLang1); - ushort TmtXDelSegm -( - OtmProposal& pTmDelIn, //ptr to input struct - TMX_EXT_OUT_W* pTmDelOut //ptr to output struct -); + USHORT NTMGetIDFromName(PSZ, USHORT, PUSHORT, + LONG options = 0); -USHORT TmtXDelSegmByKey -( - OtmProposal& TmDelIn, //ptr to input struct - TMX_EXT_OUT_W* pTmDelOut //ptr to output struct -); + USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, SHORT maxBuffLen = -1, bool fSuppressLogsAndResetValueIfNotFound = false); + USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, + PTMX_TABLE_ENTRY *); + USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT, bool fUseProvidedId = false); + + USHORT FillClb(BytesRef, OtmProposal &); + USHORT ComparePutData( + BytesRef ppTmRecord, // ptr to ptr of tm record data buffer + OtmProposal &TmProposal, // pointer to get in data + LONG& lKey // tm key + ); + USHORT AddTmTarget( + OtmProposal &TmProposal, // pointer to get in data + BytesRef pTmRecord, // pointer to tm record data pointer + LONG& lKey); // tm key -USHORT FindTargetAndDelete( PTMX_RECORD pTmRecord, - OtmProposal& TmDel, - TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ); - -USHORT FindTargetByKeyAndDelete( - PTMX_RECORD pTmRecord, - OtmProposal& pTmDel, - PTMX_SENTENCE pSentence, - TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ); - - -USHORT TokenizeSource( PTMX_SENTENCE, PSZ, PSZ ); - -USHORT NTMGetIDFromName( PSZ, PSZ, USHORT, PUSHORT ); -USHORT NTMGetIDFromNameEx -( - PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input, type of table to use - PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID -); -USHORT NTMGetNameFromID( PUSHORT, USHORT, PSZ, PSZ ); -USHORT NTMGetPointersToTable( USHORT, PTMX_TABLE *, - PTMX_TABLE_ENTRY * ); -USHORT NTMAddNameToTable( PSZ, USHORT, PUSHORT ); -PSZ NTMFindNameForID( - PUSHORT pusID, //intput - USHORT usTableType ); - -USHORT FillClb( PTMX_TARGET_CLB, OtmProposal& ); -USHORT ComparePutData -( - PTMX_RECORD *ppTmRecord, // ptr to ptr of tm record data buffer - PULONG pulRecBufSize, // current size of record buffer - OtmProposal& TmProposal, // pointer to get in data - PULONG pulKey // tm key -); -USHORT AddTmTarget( - OtmProposal& TmProposal, //pointer to get in data - PTMX_RECORD *ppTmRecord, //pointer to tm record data pointer - PULONG pulRecBufSize, //ptr to current size of TM record buffer - PULONG pulKey ); //tm key + /*! \brief Store the supplied proposal in the memory + When the proposal aready exists it will be overwritten with the supplied data -/*! \brief Store the supplied proposal in the memory - When the proposal aready exists it will be overwritten with the supplied data + \param pProposal pointer to a OtmProposal object - \param pProposal pointer to a OtmProposal object + \returns 0 or error code in case of errors + */ + int putProposal( + OtmProposal &Proposal); - \returns 0 or error code in case of errors -*/ - int putProposal - ( - OtmProposal &Proposal - ); - - //fuzzy search - USHORT TmtXGet - ( - PTMX_GET_W pTmGetIn, //ptr to input struct - PTMX_GET_OUT_W pTmGetOut //ptr to output struct + // fuzzy search + USHORT TmtXGet( + PTMX_GET_W pTmGetIn, // ptr to input struct + PTMX_GET_OUT_W pTmGetOut // ptr to output struct ); - // TM segment update prototypes - USHORT TmtXUpdSeg - ( - OtmProposal* pTmPutIn, // ptr to put input data - PTMX_EXT_OUT_W pTmPutOut, //ptr to output struct - USHORT usFlags // flags controlling the updated fields - ); - USHORT TmtXExtract - ( - PTMX_EXT_IN_W pTmExtIn, //ptr to input struct - TMX_EXT_OUT_W * pTmExtOut //ptr to output struct - ); - //tm extract prototypes - USHORT FillExtStructure( PTMX_TARGET_RECORD, - PTMX_TARGET_CLB, - PSZ_W, PLONG, PTMX_EXT_OUT_W ); + USHORT TmtXExtract( + PTMX_EXT_IN_W pTmExtIn, // ptr to input struct + TMX_EXT_OUT_W *pTmExtOut // ptr to output struct + ); + // tm extract prototypes + USHORT FillExtStructure(PTMX_TARGET_RECORD, + PTMX_TARGET_CLB, + std::wstring&, PLONG, PTMX_EXT_OUT_W, PBYTE); int RewriteCompactTable(); - -/*! \brief Get the next proposal from the memory (with progress info) - \param lHandle the hande returned by GetFirstProposal - \param Proposal reference to a OtmProposal object which will be filled with the proposal data - \param piProgress pointer to buffer for progress indicator, this indicator goes from 0 up to 100 - \returns 0 or error code in case of errors -*/ - int getNextProposal - ( - OtmProposal &Proposal, - int *piProgress = nullptr - ); - - /*! \brief Get the current sequential access key (the key for the next proposal in the memory) - \param pszKeyBuffer pointer to the buffer to store the sequential access key - \param iKeyBufferSize size of the key buffer in number of characters - \returns 0 or error code in case of errors + + /*! \brief Get the next proposal from the memory (with progress info) + \param lHandle the hande returned by GetFirstProposal + \param Proposal reference to a OtmProposal object which will be filled with the proposal data + \param piProgress pointer to buffer for progress indicator, this indicator goes from 0 up to 100 + \returns 0 or error code in case of errors */ - int getSequentialAccessKey - ( - char *pszKeyBuffer, - int iKeyBufferSize - ); + int getNextProposal( + OtmProposal &Proposal, + int *piProgress = nullptr); + + /*! \brief Get the current sequential access key (the key for the next proposal in the memory) + \param pszKeyBuffer pointer to the buffer to store the sequential access key + \param iKeyBufferSize size of the key buffer in number of characters + \returns 0 or error code in case of errors +*/ + int getSequentialAccessKey( + char *pszKeyBuffer, + int iKeyBufferSize); + USHORT AddToTm(OtmProposal &, LONG&); + USHORT UpdateTmIndex(PTMX_SENTENCE, ULONG); - USHORT AddToTm( OtmProposal&, PULONG ); - USHORT UpdateTmIndex( PTMX_SENTENCE, ULONG ); + void importDone(int iRC, char *pszError); + void reorganizeDone(int iRC, char *pszError); - void importDone(int iRC, char *pszError ); - void reorganizeDone(int iRC, char *pszError ); - /*! \brief Set the current sequential access key to resume the sequential access at the given position \param pszKey a sequential access key previously returned by getSequentialAccessKey - \returns 0 or error code in case of errors + \returns 0 or error code in case of errors */ - int setSequentialAccessKey - ( - char *pszKey - ); - - size_t GetRAMSize()const; - size_t GetExpectedRAMSize()const; - - USHORT ExtractRecordV6 - ( - PTMX_RECORD pTmRecord, - PTMX_EXT_IN_W pTmExtIn, - TMX_EXT_OUT_W * pTmExtOut - ); + int setSequentialAccessKey( + char *pszKey); + size_t GetRAMSize() const; + size_t GetExpectedRAMSize() const; - USHORT TmtXReplace - ( - OtmProposal& TmProposal, //ptr to input struct - PTMX_EXT_OUT_W pTmPutOut //ptr to output struct - ); - -USHORT NTMLoadNameTable -( - ULONG ulTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data -); - -/*! \brief Get a list of memory proposals matching the given search key + USHORT ExtractRecord( + BytesRef pTmRecord, + PTMX_EXT_IN_W pTmExtIn, + TMX_EXT_OUT_W *pTmExtOut); - This method uses the search data contained in the search key to find one or more - matching proposals in the memory. At least the szSource and the szTargetLang members of the - search key have to be filled by the caller. - The caller provides a list of OtmProposals which will be filled with the data of the matching - proposals. The number of requested proposals is determined by the number - of proposals in the list. - - \param SearchKey proposal containing search string and meta data - \param FoundProposals refernce to vector with OtmProposal objects - \param ulOptions options for the lookup - - \returns 0 or error code in case of errors -*/ - int SearchProposal - ( - OtmProposal &SearchKey, - std::vector &FoundProposals, - unsigned long ulOptions - ); - - USHORT UpdateTmRecord( OtmProposal& ); - - USHORT UpdateTmRecordByInternalKey - ( - OtmProposal& + USHORT TmtXReplace( + OtmProposal &TmProposal, // ptr to input struct + PTMX_EXT_OUT_W pTmPutOut // ptr to output struct ); - - int getNumOfMarkupNames(); -/*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] - \param iPos position of markup table name - \param pszBuffer pointer to a buffer for the markup table name - \param iSize size of buffer in number of characters - \returns number of characters copied to buffer -*/ - int getMarkupName - ( - int iPos, - char *pszBuffer, - int iSize + USHORT NTMLoadNameTable( + LONG lTableKey, // key of table record + PTMX_TABLE pTMTable // ptr to table data pointer ); + /*! \brief Get a list of memory proposals matching the given search key -/*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] - \param iPos position of language - \param pszBuffer pointer to a buffer for the document name - \param iSize size of buffer in number of characters - \returns number of characters copied to buffer -*/ - int getLanguage - ( - int iPos, - char *pszBuffer, - int iSize - ); + This method uses the search data contained in the search key to find one or more + matching proposals in the memory. At least the szSource and the szTargetLang members of the + search key have to be filled by the caller. + The caller provides a list of OtmProposals which will be filled with the data of the matching + proposals. The number of requested proposals is determined by the number + of proposals in the list. + \param SearchKey proposal containing search string and meta data + \param FoundProposals refernce to vector with OtmProposal objects + \param ulOptions options for the lookup + \returns 0 or error code in case of errors + */ + int SearchProposal( + OtmProposal &SearchKey, + std::vector &FoundProposals, + unsigned long ulOptions); -/*! \brief Get source language of the memory - \param pszBuffer pointer to a buffer for the source language name - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getSourceLanguage - ( - char *pszBuffer, - int iSize - ); + USHORT UpdateTmRecord(OtmProposal &); -/*! \brief Get the name of the memory - \param pszBuffer pointer to a buffer for the name - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getName - ( - char *pszBuffer, - int iSize - ); + int getNumOfMarkupNames(); + + /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] + \param iPos position of markup table name + \param pszBuffer pointer to a buffer for the markup table name + \param iSize size of buffer in number of characters + \returns number of characters copied to buffer + */ + int getMarkupName( + int iPos, + char *pszBuffer, + int iSize); + + /*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] + \param iPos position of language + \param pszBuffer pointer to a buffer for the document name + \param iSize size of buffer in number of characters + \returns number of characters copied to buffer + */ + int getLanguage( + int iPos, + char *pszBuffer, + int iSize); + + /*! \brief Get source language of the memory + \param pszBuffer pointer to a buffer for the source language name + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getSourceLanguage( + char *pszBuffer, + int iSize); + + /*! \brief Get the name of the memory + \param pszBuffer pointer to a buffer for the name + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getName( + char *pszBuffer, + int iSize); /*! \brief Get the name of the memory \param strName reference of a string receiving the memory name */ - int getName - ( - std::string &strName - ); - + int getName( + std::string &strName); -/*! \brief Get description of the memory - \param pszBuffer pointer to a buffer for the description - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getDescription - ( - char *pszBuffer, - int iSize - ); + /*! \brief Get description of the memory + \param pszBuffer pointer to a buffer for the description + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getDescription( + char *pszBuffer, + int iSize); /*! \brief Set the description of the memory \param pszBuffer pointer to the description text */ - void setDescription - ( - char *pszBuffer - ); + void setDescription( + char *pszBuffer); -/*! \brief Get plugin responsible for this memory - \returns pointer to memory plugin object -*/ + /*! \brief Get plugin responsible for this memory + \returns pointer to memory plugin object + */ void *getPlugin(); - -/*! \brief Get overall file size of this memory - \returns size of memory in number of bytes -*/ - unsigned long getFileSize(); - -/*! \brief Get the error message for the last error occured - - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ - int getLastError - ( - std::string &strError - ); -/*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ - int getLastError - ( - char *pszError, - int iBufSize - ); - -/*! \brief Set or clear the pointer to a loaded global memory option file + /*! \brief Get overall file size of this memory + \returns size of memory in number of bytes + */ + unsigned long getFileSize(); - This method sets a pointer to a loaded global memory option file. - When set the option file will be used to decide how global memory proposals will be processed. + /*! \brief Get the error message for the last error occured - \param pvGlobalMemoryOptions pointer to a loaded global memory option file or NULL to clear the current option file pointer + \param strError reference to a string receiving the error mesage text + \returns last error code + */ + int getLastError( + std::string &strError); + /*! \brief Get the error message for the last error occured - \returns 0 or error code in case of errors -*/ - int setGlobalMemoryOptions - ( - void *pvGlobalMemoryOptions - ); + \param pszError pointer to a buffer for the error text + \param iBufSize size of error text buffer in number of characters + \returns last error code + */ + int getLastError( + char *pszError, + int iBufSize); - std::string strLastError; - int iLastError = 0; - void *pvGlobalMemoryOptions = nullptr; // pointert to global memory options to be used for global memory proposals -public: - char szPlugin[256]; // name of the plugin controlling this memory - char szName[256]; // name of the memory - char szDescription[256]; // description of the memory - char szFullPath[256]; // full path to memory file(s) (if applicable only) - char szFullDataFilePath[256]; // full path to memory file(s) (if applicable only) - char szFullIndexFilePath[256]; // full path to memory file(s) (if applicable only) - char szSourceLanguage[MAX_LANG_LENGTH+1]; // memory source language - char szOwner[256]; // ID of the memory owner - char szDescrMemoryType[256]; // descriptive name of the memory type - unsigned long ulSize = 0; // size of the memory - BOOL fEnabled = 0; // memory-is-enabled flag - - std::string getName()const { return szName; } + std::string getName() const { return szName; } // functions dealing with long document tables USHORT NTMCreateLongNameTable(); USHORT NTMReadLongNameTable(); USHORT NTMWriteLongNameTable(); USHORT NTMDestroyLongNameTable(); - USHORT NTMCreateLangGroupTable(); - USHORT NTMAddLangGroup( PSZ pszLang, USHORT sLangID ); - USHORT NTMOrganizeIndexFile(); -//private: -/*! \brief Fill OtmProposal from TMX_GET_OUT_W structure - \param ulKey key of record containing the proposal - \param usTargetNum number of target within record - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int SetProposalKey -( - ULONG ulKey, - USHORT usTargetNum, - OtmProposal *pProposal -); + USHORT AddToLongNameTables(const char* pszLongName,PUSHORT pusID, bool fGenerateId =true, PSZ positionInReadBuffer = nullptr); + USHORT WriteLongNameTablesToTheTM(); + USHORT SortLongNameTables(); -/*! \brief Fill OtmProposal from TMX_GET_OUT_W structure - \param pExtOut pointer to the TMX_GET_OUT_W structure - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int ExtOutToOtmProposal -( - TMX_EXT_OUT_W* pExtOut, - OtmProposal &Proposal -); - -/*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure - \param pMatch pointer to the TMX_MATCH_TABLE_W structure - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int MatchToOtmProposal -( - PTMX_MATCH_TABLE_W pMatch, - OtmProposal *pProposal -); + USHORT NTMCreateLangGroupTable(); + USHORT NTMAddLangGroup(PSZ pszLang, USHORT sLangID); + USHORT NTMOrganizeIndexFile(); + // private: + /*! \brief Fill OtmProposal from TMX_GET_OUT_W structure + \param ulKey key of record containing the proposal + \param usTargetNum number of target within record + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int SetProposalKey( + LONG lKey, + USHORT usTargetNum, + OtmProposal *pProposal); + + /*! \brief Fill OtmProposal from TMX_GET_OUT_W structure + \param pExtOut pointer to the TMX_GET_OUT_W structure + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int ExtOutToOtmProposal( + TMX_EXT_OUT_W *pExtOut, + OtmProposal &Proposal); + + /*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure + \param pMatch pointer to the TMX_MATCH_TABLE_W structure + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int MatchToOtmProposal( + PTMX_MATCH_TABLE_W pMatch, + OtmProposal *pProposal); + + /*! \brief Fill TMX_GET_W structure with OtmProposal data + \param Proposal reference to the OtmProposal containing the data + \param pGetIn pointer to the TMX_GET_W structure + \returns 0 or error code in case of errors + */ + int OtmProposalToGetIn( + OtmProposal &Proposal, + PTMX_GET_W pGetIn); -/*! \brief Fill TMX_GET_W structure with OtmProposal data - \param Proposal reference to the OtmProposal containing the data - \param pGetIn pointer to the TMX_GET_W structure - \returns 0 or error code in case of errors -*/ -int OtmProposalToGetIn -( - OtmProposal &Proposal, - PTMX_GET_W pGetIn -); + int OtmProposalToPutIn( + OtmProposal &Proposal, + PTMX_PUT_W pPutIn); -int OtmProposalToPutIn -( - OtmProposal &Proposal, - PTMX_PUT_W pPutIn -); + /// newTM code + EqfMemory(const std::string &tmName); -/// newTM code -public: - EqfMemory(const std::string& tmName); bool IsLoadedInRAM(); - int ReloadFromDisk(); + int Reload(bool lockFromDisk); int UnloadFromRAM(); - int FlushFilebuffers(MutexTimeout& tmLockTimeout); + int FlushFilebuffers(MutexTimeout &tmLockTimeout); -private: }; - -OtmProposal::eProposalType FlagToProposalType( USHORT usTranslationFlag ); +OtmProposal::eProposalType FlagToProposalType(USHORT usTranslationFlag); BYTE ProposalTypeToFlag(OtmProposal::eProposalType t); -USHORT TmGetServerDrives( PDRIVES_IN pDrivesIn, // Pointer to DRIVES_IN struct - PDRIVES_OUT pDrivesOut, // Pointer to DRIVES_OUT struct - USHORT usMsgHandling );// Message handling parameter -USHORT TmGetServerDrivesHwnd( PDRIVES_IN, PDRIVES_OUT, USHORT, HWND ); -USHORT TmGetServerTMs( PFILE_LIST_IN pTmListIn, // Pointer to FILE_LIST_IN struct !!! CHM - PFILE_LIST_OUT pTmListOut, // Pointer to FILE_LIST_OUT struct !!! CHM - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetTMPart( HTM htm, // Memory database handle - PSZ pszMemPat, // Full translation memory path - PGETPART_IN pGetPartIn, // Pointer to GETPART_IN struct - PGETPART_OUT pGetPartOut, // Pointer to GETPART_OUT struct - USHORT usMsgHandling ); // Message handling parameter -USHORT TmSendTMProp( SERVERNAME szServer, // Servername - PSZ pszMemName, // Translation memory name - USHORT usMsgHandling ); // Message handling parameter -USHORT TmReceiveTMProp( SERVERNAME szServer, // Servername - PSZ pszMemName, // Translation memory name - USHORT usMsgHandling ); // Message handling parameter - -//use this define as usMsgHandling for special handling of greyedout TMs +USHORT TmGetServerDrives(PDRIVES_IN pDrivesIn, // Pointer to DRIVES_IN struct + PDRIVES_OUT pDrivesOut, // Pointer to DRIVES_OUT struct + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerDrivesHwnd(PDRIVES_IN, PDRIVES_OUT, USHORT, HWND); +USHORT TmGetServerTMs(PFILE_LIST_IN pTmListIn, // Pointer to FILE_LIST_IN struct !!! CHM + PFILE_LIST_OUT pTmListOut, // Pointer to FILE_LIST_OUT struct !!! CHM + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetTMPart(HTM htm, // Memory database handle + PSZ pszMemPat, // Full translation memory path + PGETPART_IN pGetPartIn, // Pointer to GETPART_IN struct + PGETPART_OUT pGetPartOut, // Pointer to GETPART_OUT struct + USHORT usMsgHandling); // Message handling parameter +USHORT TmSendTMProp(SERVERNAME szServer, // Servername + PSZ pszMemName, // Translation memory name + USHORT usMsgHandling); // Message handling parameter +USHORT TmReceiveTMProp(SERVERNAME szServer, // Servername + PSZ pszMemName, // Translation memory name + USHORT usMsgHandling); // Message handling parameter + +// use this define as usMsgHandling for special handling of greyedout TMs #define DELETE_GREYEDOUT 2 -USHORT TmDeleteFile( SERVERNAME szServer, // Servername - PSZ pszFilePath, // Full file path - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetServerFileInfo( SERVERNAME szServer, //Servername - PSZ pszFilePath, //Full file path - PFILEFINDBUF pstFile, // File info - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetServerFileInfoHwnd( SERVERNAME, PSZ, PFILEFINDBUF, USHORT, HWND ); -USHORT TmCompareLocalRemoteProperties( SERVERNAME, PSZ, USHORT ); -USHORT MemRcHandling( USHORT, PSZ, HTM *, PSZ ); -USHORT MemRcHandlingHwnd( USHORT, PSZ, HTM *, PSZ, HWND ); +USHORT TmDeleteFile(SERVERNAME szServer, // Servername + PSZ pszFilePath, // Full file path + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerFileInfo(SERVERNAME szServer, // Servername + PSZ pszFilePath, // Full file path + PFILEFINDBUF pstFile, // File info + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerFileInfoHwnd(SERVERNAME, PSZ, PFILEFINDBUF, USHORT, HWND); +USHORT TmCompareLocalRemoteProperties(SERVERNAME, PSZ, USHORT); +USHORT MemRcHandling(USHORT, PSZ, HTM *, PSZ); +USHORT MemRcHandlingHwnd(USHORT, PSZ, HTM *, PSZ, HWND); /**********************************************************************/ /* MemReadWriteSegment */ @@ -2392,89 +2297,87 @@ USHORT MemRcHandlingHwnd( USHORT, PSZ, HTM *, PSZ, HWND ); // TMRWS_NOMSG: show no error message at all // TMRWS_ALLMSG: show all error messages // TMRWS_WRITEMSG: show only error messages for write segment (for TM organize!) -#define TMRWS_NOMSG 0 -#define TMRWS_ALLMSG 1 +#define TMRWS_NOMSG 0 +#define TMRWS_ALLMSG 1 #define TMRWS_WRITEMSG 2 -USHORT NTMReadWriteSegmentW( HTM, //Handle of Input TM - PSZ, //Full path to Input TM - HTM, //Handle of Output TM - PSZ, //Full path to Output TM - PTMX_EXT_IN_W,//Pointer to the EXTRACT_IN structure - TMX_EXT_OUT_W *, //Pointer to the EXTRACT_OUT structure - PTMX_PUT_W, //Pointer to the REPLACE_OUT structure - PTMX_EXT_OUT_W, //Pointer to the REPLACE_OUT structure - ULONG *, //Pointer to a segment counter - ULONG *, //Pointer to the invalid segment counter - PSZ, //pointer to TM source language - USHORT ); //message handling flags -USHORT NTMReadWriteSegmentHwndW( HTM, //Handle of Input TM - PSZ, //Full path to Input TM - HTM, //Handle of Output TM - PSZ, //Full path to Output TM - PTMX_EXT_IN_W, //Pointer to the EXTRACT_IN structure - TMX_EXT_OUT_W *, //Pointer to the EXTRACT_OUT structure - PTMX_PUT_W, //Pointer to the REPLACE_OUT structure - PTMX_EXT_OUT_W, //Pointer to the REPLACE_OUT structure - ULONG *, //Pointer to a segment counter - ULONG *, //Pointer to the invalid segment counter - PSZ, //pointer to TM source language - USHORT, //message handling flags - HWND ); //handle for error messages parent - - - -BOOL fMemIsAvail( PSZ, PSZ, PSZ ); // memory is available and accessible -USHORT MemFillCBNames( HWND, PSZ, PSZ ); -USHORT MemFillTableLB( HWND hListBox, USHORT usBoxType, PSZ pszLastUsed ); +USHORT NTMReadWriteSegmentW(HTM, // Handle of Input TM + PSZ, // Full path to Input TM + HTM, // Handle of Output TM + PSZ, // Full path to Output TM + PTMX_EXT_IN_W, // Pointer to the EXTRACT_IN structure + TMX_EXT_OUT_W *, // Pointer to the EXTRACT_OUT structure + PTMX_PUT_W, // Pointer to the REPLACE_OUT structure + PTMX_EXT_OUT_W, // Pointer to the REPLACE_OUT structure + ULONG *, // Pointer to a segment counter + ULONG *, // Pointer to the invalid segment counter + PSZ, // pointer to TM source language + USHORT); // message handling flags +USHORT NTMReadWriteSegmentHwndW(HTM, // Handle of Input TM + PSZ, // Full path to Input TM + HTM, // Handle of Output TM + PSZ, // Full path to Output TM + PTMX_EXT_IN_W, // Pointer to the EXTRACT_IN structure + TMX_EXT_OUT_W *, // Pointer to the EXTRACT_OUT structure + PTMX_PUT_W, // Pointer to the REPLACE_OUT structure + PTMX_EXT_OUT_W, // Pointer to the REPLACE_OUT structure + ULONG *, // Pointer to a segment counter + ULONG *, // Pointer to the invalid segment counter + PSZ, // pointer to TM source language + USHORT, // message handling flags + HWND); // handle for error messages parent + +BOOL fMemIsAvail(PSZ, PSZ, PSZ); // memory is available and accessible +USHORT MemFillCBNames(HWND, PSZ, PSZ); +USHORT MemFillTableLB(HWND hListBox, USHORT usBoxType, PSZ pszLastUsed); /**********************************************************************/ /* TmOpen */ /**********************************************************************/ -USHORT TmOpen( PSZ, HTM *, USHORT, USHORT, //(in) location: TM_LOCAL - // TM_REMOTE - // TM_LOCAL_REMOTE - USHORT, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND ); //(in) window handle for error messages +USHORT TmOpen(PSZ, HTM *, USHORT, USHORT, //(in) location: TM_LOCAL + // TM_REMOTE + // TM_LOCAL_REMOTE + USHORT, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND); //(in) window handle for error messages /**********************************************************************/ /* TmClose */ /**********************************************************************/ USHORT -TmClose( HTM, //(in) TM handle returned from open - PSZ, //(in) full TM name x:\eqf\mem\mem.mem - USHORT, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND ); //(in) window handle for error messages +TmClose(HTM, //(in) TM handle returned from open + PSZ, //(in) full TM name x:\eqf\mem\mem.mem + USHORT, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND); //(in) window handle for error messages /**********************************************************************/ /* TmReplace */ /**********************************************************************/ USHORT -TmReplace( HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT, HWND ); +TmReplace(HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT, HWND); /**********************************************************************/ /* TmGet */ /**********************************************************************/ USHORT -TmGetW(EqfMemory*, //(in) TM handle - PSZ, //(in) full TM name x:\eqf\mem\mem.tmd +TmGetW(EqfMemory *, //(in) TM handle + PSZ, //(in) full TM name x:\eqf\mem\mem.tmd PTMX_GET_W, //(in) pointer to get input structure - PTMX_GET_OUT_W, //(out) pointer to get output structure - USHORT ); //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message + PTMX_GET_OUT_W, //(out) pointer to get output structure + USHORT); //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message /**********************************************************************/ /* TmExtract */ /**********************************************************************/ USHORT -TmExtractW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT ); +TmExtractW(HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT); USHORT -TmExtractHwndW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND ); +TmExtractHwndW(HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND); /**********************************************************************/ /* information levels for TmInfo function call */ @@ -2485,319 +2388,282 @@ TmExtractHwndW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND ); /**********************************************************************/ /* TmDeleteTM */ /**********************************************************************/ -USHORT TmDeleteTM( PSZ, USHORT, HWND, PUSHORT ); +USHORT TmDeleteTM(PSZ, USHORT, HWND, PUSHORT); /**********************************************************************/ /* TmDeleteSegment */ /**********************************************************************/ USHORT -TmDeleteSegmentW( HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT ); - - +TmDeleteSegmentW(HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT); /**********************************************************************/ /* NTMConvertCRLF */ /**********************************************************************/ VOID -NTMConvertCRLF( PSZ, - PSZ, - USHORT ); -VOID NTMConvertCRLFW( PSZ_W, PSZ_W, USHORT ); - + NTMConvertCRLF(PSZ, + PSZ, + USHORT); +VOID NTMConvertCRLFW(PSZ_W, PSZ_W, USHORT); /**********************************************************************/ /* NTMOpenProperties */ /**********************************************************************/ USHORT -NTMOpenProperties( HPROP *, - PVOID *, - PSZ, - PSZ, - USHORT, - BOOL ); - - - +NTMOpenProperties(HPROP *, + PVOID *, + PSZ, + PSZ, + USHORT, + BOOL); // Update specific parts of segment // flags controlling the updated segment parts -#define TMUPDSEG_MARKUP 0x01 // update markup/tag table -#define TMUPDSEG_MTFLAG 0x02 // update MT flag -#define TMUPDSEG_TARGLANG 0x04 // update target language -#define TMUPDSEG_DATE 0x08 // update segment time - - -USHORT TmUpdSegW -( - HTM htm, //(in) TM handle - PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem - PTMX_PUT_W pstPutIn, //(in) pointer to put input structure - ULONG ulUpdKey, //(in) key of record being updated - USHORT usUpdTarget, //(in) number of target being updated - USHORT usFlags, //(in) flags controlling the updated fields - USHORT usMsgHandling //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message +#define TMUPDSEG_MARKUP 0x01 // update markup/tag table +#define TMUPDSEG_MTFLAG 0x02 // update MT flag +#define TMUPDSEG_TARGLANG 0x04 // update target language +#define TMUPDSEG_DATE 0x08 // update segment time + +USHORT TmUpdSegW( + HTM htm, //(in) TM handle + PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem + PTMX_PUT_W pstPutIn, //(in) pointer to put input structure + ULONG ulUpdKey, //(in) key of record being updated + USHORT usUpdTarget, //(in) number of target being updated + USHORT usFlags, //(in) flags controlling the updated fields + USHORT usMsgHandling //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message ); -USHORT TmUpdSegHwndW -( - HTM htm, //(in) TM handle - PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem - PTMX_PUT_W pstPutIn, //(in) pointer to put input structure - ULONG ulUpdKey, //(in) key of record being updated - USHORT usUpdTarget, //(in) number of target being updated - USHORT usFlags, //(in) flags controlling the updated fields - USHORT usMsgHandling, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND hwnd //(in) handle for error messages -); - - - -// structure for MT_TMMERGE pointer -typedef struct _MT_TMMERGE -{ - CHAR chMemory[MAX_EQF_PATH]; // TM name to be merged into - CHAR chSGMLFile[MAX_EQF_PATH]; // external memory - OBJNAME szObjName; // document object name - HWND hwndNotify; // parent to be notified - CHAR szTargetLang[MAX_LANG_LENGTH]; - OBJNAME szFolObjName; // object name of folder -} MT_TMMERGE, *PMT_TMMERGE; - -BOOL MTTMMergeStart -( - PLISTCOMMAREA pCommArea, - PMT_TMMERGE pTMMerge +USHORT TmUpdSegHwndW( + HTM htm, //(in) TM handle + PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem + PTMX_PUT_W pstPutIn, //(in) pointer to put input structure + ULONG ulUpdKey, //(in) key of record being updated + USHORT usUpdTarget, //(in) number of target being updated + USHORT usFlags, //(in) flags controlling the updated fields + USHORT usMsgHandling, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND hwnd //(in) handle for error messages ); - // return codes of function MemConvertMem // memory converted successfully -#define MEM_CONVERTMEM_SUCCESS 0 +#define MEM_CONVERTMEM_SUCCESS 0 // memory conversion not required, memory is in new format already #define MEM_CONVERTMEM_ALREADYNEWFORMAT 7001 - // do a cleanup of temporary memories -void TMCleanupTempMem -( - PSZ pszPrefix // ptr to memory prefix +void TMCleanupTempMem( + PSZ pszPrefix // ptr to memory prefix ); // create a temporary memory -USHORT TMCreateTempMem -( - PSZ pszPrefix, // short prefix to be used for memory name (should start with a dollar sign) - PSZ pszMemPath, // ptr to buffer for memory path - HTM *pHtm, // ptr to buffer for memory handle - HTM htm, // htm of similar memory - PSZ pszSourceLang, // memory source language - HWND hwnd // window handle for error messages +USHORT TMCreateTempMem( + PSZ pszPrefix, // short prefix to be used for memory name (should start with a dollar sign) + PSZ pszMemPath, // ptr to buffer for memory path + HTM *pHtm, // ptr to buffer for memory handle + HTM htm, // htm of similar memory + PSZ pszSourceLang, // memory source language + HWND hwnd // window handle for error messages ); // delete a temporary memory -void TMDeleteTempMem -( - PSZ pszMemPath // ptr to memory path +void TMDeleteTempMem( + PSZ pszMemPath // ptr to memory path ); -BOOL TMFuzzynessEx -( - PSZ pszMarkup, // markup table name - PSZ_W pszSource, // original string - PSZ_W pszMatch, // found match - SHORT sLanguageId, // language id to be used - PUSHORT pusFuzzy, // fuzzyness - ULONG ulOemCP, - PUSHORT pusWords, // number of words in segment - PUSHORT pusDiffs // number of diffs in segment +BOOL TMFuzzynessEx( + PSZ pszMarkup, // markup table name + PSZ_W pszSource, // original string + PSZ_W pszMatch, // found match + SHORT sLanguageId, // language id to be used + PUSHORT pusFuzzy, // fuzzyness + PUSHORT pusWords, // number of words in segment + PUSHORT pusDiffs // number of diffs in segment ); - -USHORT NTMMorphTokenizeW( SHORT, PSZ_W, PULONG, PVOID*, USHORT); - -BOOL NTMTagSubst // generic inline tagging for TM -( - PTMX_SUBSTPROP pSubstProp, - ULONG ulSrcOemCP, - ULONG ulTgtOemCP +/**********************************************************************/ +/* Structure to access MORPH_FLAG_OFFSLIST type term lists */ +/**********************************************************************/ +typedef struct _FLAGOFFSLIST +{ + LONG lFlags; // flags of term + USHORT usLen; // length of term + USHORT usOffs; // offset of term +} FLAGOFFSLIST, *PFLAGOFFSLIST; + +USHORT NTMMorphTokenizeW(SHORT, PSZ_W, std::vector &); + +USHORT MorphAddTermToList2W( + std::vector &pList, // ptr to term list pointer + PSZ_W pszTerm, // ptr to new term being added to list + USHORT usLength, // length of term + USHORT usOffs, // offset of term + ULONG ulFlags // flags for the ter ); -BOOL NTMDocMatch( PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2 ); - - -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); +BOOL NTMDocMatch(PSZ pszLong1, PSZ pszLong2); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! tm.h Internal header file for Translation Memory functions - Copyright (c) 1990-2016, International Business Machines Corporation and others. All rights reserved. + Copyright (c) 1990-2016, International Business Machines Corporation and others. All rights reserved. */ -//#ifndef _EQFTMI_H_ -//#define _EQFTMI_H_ +// #ifndef _EQFTMI_H_ +// #define _EQFTMI_H_ /**********************************************************************/ /* This include file requires EQFDDE.H!!! */ /**********************************************************************/ #ifndef EQFDDE_INCLUDED - #include "EQFDDE.H" +#include "EQFDDE.H" #endif // include external memory export/import interface #include "EQFMEMIE.H" - #ifndef FIELDOFFSET - #define FIELDOFFSET(type, field) (LOWORD(&(((type *)0)->field))) +#define FIELDOFFSET(type, field) (LOWORD(&(((type *)0)->field))) #endif // file extension used for shared memory (LAN based) property files in property directory #define LANSHARED_MEM_PROP ".SLM" - /*---------------------------------------------------------------------*\ #Define \*---------------------------------------------------------------------*/ // memory export and import formats -#define MEM_SGMLFORMAT_ANSI 1 -#define MEM_SGMLFORMAT_ASCII 2 -#define MEM_SGMLFORMAT_UNICODE 3 -#define MEM_FORMAT_TMX 4 +#define MEM_SGMLFORMAT_ANSI 1 +#define MEM_SGMLFORMAT_ASCII 2 +#define MEM_SGMLFORMAT_UNICODE 3 +#define MEM_FORMAT_TMX 4 // the same define value is used for the import as TMX (Trados) and for export as TMX (UTF-8) -#define MEM_FORMAT_TMX_UTF8 5 -#define MEM_FORMAT_TMX_TRADOS 5 +#define MEM_FORMAT_TMX_UTF8 5 +#define MEM_FORMAT_TMX_TRADOS 5 -#define MEM_FORMAT_TMX_NOCRLF 6 +#define MEM_FORMAT_TMX_NOCRLF 6 #define MEM_FORMAT_TMX_UTF8_NOCRLF 7 -#define MEM_FORMAT_XLIFF_MT 8 - +#define MEM_FORMAT_XLIFF_MT 8 // filter for mem SGML formats (same sequence as memory export and import formats required) -//#define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" +// #define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" #define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML UTF-16\0*.*\0TMX\0*.*\0TMX (Trados)\0*.*\0XLIFF (MT)\0*.*\0\0\0" // filter for mem SGML formats (same sequence as memory export and import formats required) -//#define MEM_FORMAT_FILTERS_EXP "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" +// #define MEM_FORMAT_FILTERS_EXP "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" #define MEM_FORMAT_FILTERS_EXP "SGML ANSI (*.EXP)\0*.EXP\0SGML ASCII (*.EXP)\0*.EXP\0SGML UTF-16 (*.EXP)\0*.EXP\0TMX (UTF-16) (*.TMX)\0*.TMX\0TMX (UTF-8) (*.TMX)\0*.TMX\0TMX (UTF-16) (remove CRLF) (*.TMX)\0*.TMX\0TMX (UTF-8) (remove CRLF) (*.TMX)\0*.TMX\0\0\0" // defines for retries in case of BTREE_IN_USE conditions #define MAX_RETRY_COUNT 30 -#define MAX_WAIT_TIME 100 +#define MAX_WAIT_TIME 100 -#define TMT_CODE_VERSION 5 // Code version -#define TM_PREFIX "EQFTMT$$" // TM DB identifier -#define HUNDRED 100 // +#define TMT_CODE_VERSION 5 // Code version +#define TM_PREFIX "EQFTMT$$" // TM DB identifier +#define HUNDRED 100 // // ID to be used if a table overflow occurs -#define OVERFLOW_ID 32000 +#define OVERFLOW_ID 32000 // name to be used if a table overflow occurs -#define OVERFLOW_NAME "$$$$$$$$" +#define OVERFLOW_NAME "$$$$$$$$" -#define LAST_CLUSTER_NUMBER 4095 +#define LAST_CLUSTER_NUMBER 4095 //--- size of the message buffer for the Error message /*@1108A*/ //--- ERROR_MEM_SEGMENT_TOO_LARGE_CON /*@1108A*/ -#define MSG_BUFFER 35 /*@1108A*/ - +#define MSG_BUFFER 35 /*@1108A*/ -#define MIN_SPOOL_SIZE 20480 // 20k bytes for preformatted TM blocks -#define SECTOR_SIZE 4096 // Disk sector. Fixed now, dynamic later -#define BLOCK_HEADER_SIZE 8 // size of block header -#define MAX_TM_HEADER_SIZE 47600 // max size of TM header -#define MAX_BLOCK_SIZE 4096 // Maximum block size +#define MIN_SPOOL_SIZE 20480 // 20k bytes for preformatted TM blocks +#define SECTOR_SIZE 4096 // Disk sector. Fixed now, dynamic later +#define BLOCK_HEADER_SIZE 8 // size of block header +#define MAX_TM_HEADER_SIZE 47600 // max size of TM header +#define MAX_BLOCK_SIZE 4096 // Maximum block size #define MAX_TEXT_TAB_SIZE (MAX_TAGS_TAB_ENTRIES * sizeof(TOKENENTRY)) #define MAX_WORDS_TAB_SIZE (MAX_WORDS_TAB_ENTRIES * sizeof(TOKENENTRY)) -#define MAX_TM_WORK_AREA ( MAX_TGT_LENGTH + 300 ) - /* Work areas to convert LF into CRFL */ - /* and the other way round */ +#define MAX_TM_WORK_AREA (MAX_TGT_LENGTH + 300) +/* Work areas to convert LF into CRFL */ +/* and the other way round */ // The following define should not exceed 65535 Bytes -#define MAX_TMT_GLOBALS_SIZE (sizeof(TMT_GLOBALS) +\ - MAX_TM_HEADER_SIZE +\ - MAX_BLOCK_SIZE+\ - MAX_TEXT_TAB_SIZE+\ - MAX_WORDS_TAB_SIZE) +#define MAX_TMT_GLOBALS_SIZE (sizeof(TMT_GLOBALS) + \ + MAX_TM_HEADER_SIZE + \ + MAX_BLOCK_SIZE + \ + MAX_TEXT_TAB_SIZE + \ + MAX_WORDS_TAB_SIZE) - -#define MAX_WORDS_TAB_ENTRIES 200 /* Number of entries in WordsTable */ -#define MAX_TAGS_TAB_ENTRIES 100 /* number of entries in TextTable */ -#define KEY_DIR_ENTRIES_NUM 4096 /* number of entries key directory */ +#define MAX_WORDS_TAB_ENTRIES 200 /* Number of entries in WordsTable */ +#define MAX_TAGS_TAB_ENTRIES 100 /* number of entries in TextTable */ +#define KEY_DIR_ENTRIES_NUM 4096 /* number of entries key directory */ #define TOP_BLOCK_NUMBER (USHORT)65535 - /* 2**16 - 1, the max number of a block */ -#define MAX_BLOCKS_NUM (USHORT)65535 - /* 2**16 - 1, maximal number of blocks */ -#define GROUP_NUM 8 /* number of distribution groups */ - -#define TAG_STARTING_VALUE 0 /* the lowest attribute to tags in text */ - /* table */ -#ifndef TM_ADM -#define MAX_SERVER_NAME 15 /* length of server name */ +/* 2**16 - 1, the max number of a block */ +#define MAX_BLOCKS_NUM (USHORT)65535 +/* 2**16 - 1, maximal number of blocks */ +#define GROUP_NUM 8 /* number of distribution groups */ + +#define TAG_STARTING_VALUE 0 /* the lowest attribute to tags in text */ + /* table */ +#ifndef TM_ADM +#define MAX_SERVER_NAME 15 /* length of server name */ #endif /* The two defines below should not be used. Displacement can be */ /* computed in runtime from the structure. */ -#define SEG_LENGTH_DISP 5 /* disp of length field in a SEGMENT */ -#define DEL_FLAG_DISP 6 /* disp of del flag from segment start */ +#define SEG_LENGTH_DISP 5 /* disp of length field in a SEGMENT */ +#define DEL_FLAG_DISP 6 /* disp of del flag from segment start */ // data compress flag -#define BOCU_COMPRESS 0x01 - +#define BOCU_COMPRESS 0x01 /* define for write Tm Header function */ -#define WF_ALL 0x0000 /* Write the whole header............*/ -#define WF_CORRUPT 0x0001 -#define WF_TMBLOCKS 0x0002 -#define WF_FREEBLOCKS 0x0004 -#define WF_FIRSTAVAIL 0x0008 -#define WF_UPDATE 0x4000 /* Write all but the key directory...*/ -#define WF_KEYDIR 0x8000 /* Write the key directory...........*/ - +#define WF_ALL 0x0000 /* Write the whole header............*/ +#define WF_CORRUPT 0x0001 +#define WF_TMBLOCKS 0x0002 +#define WF_FREEBLOCKS 0x0004 +#define WF_FIRSTAVAIL 0x0008 +#define WF_UPDATE 0x4000 /* Write all but the key directory...*/ +#define WF_KEYDIR 0x8000 /* Write the key directory...........*/ /*---------------------------------------------------------------------*\ match type definitions \*---------------------------------------------------------------------*/ -#define USE_CRLF 1 -#define ALIGN_CRLF 2 -#define IGNORE_CRLF 3 +#define USE_CRLF 1 +#define ALIGN_CRLF 2 +#define IGNORE_CRLF 3 /* return codes for the Tmt Commands */ -#define OK NO_ERROR /* for Tmt commands */ - - +#define OK NO_ERROR /* for Tmt commands */ /*----------------------------------------------------------------------------*\ System wide lengthes for correction of the thresholds in dependency of the length of the input segment \*----------------------------------------------------------------------------*/ //@@@ // segments, defined thru LENGTH_SHORTER_VALUE -#define LENGTH_SHORTER_VALUE 2 // Length for shorter segment +#define LENGTH_SHORTER_VALUE 2 // Length for shorter segment //@@@ // segments, defined thru LENGTH_SHORTER_VALUE -#define LENGTH_SHORT_VALUE 6 // Length for short segment -#define LENGTH_MEDIUM_VALUE 10 // Length for medium segment -#define LENGTH_LONG_VALUE 20 // Length for long segment +#define LENGTH_SHORT_VALUE 6 // Length for short segment +#define LENGTH_MEDIUM_VALUE 10 // Length for medium segment +#define LENGTH_LONG_VALUE 20 // Length for long segment /*----------------------------------------------------------------------------*\ System wide percentages for correction of the MATCH threshold \*----------------------------------------------------------------------------*/ -#define MATCH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment -#define MATCH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment -#define MATCH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment -#define MATCH_PERCENTAGE_LONGER_VALUE 40L // percentage for longer segment +#define MATCH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment +#define MATCH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment +#define MATCH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment +#define MATCH_PERCENTAGE_LONGER_VALUE 40L // percentage for longer segment /*----------------------------------------------------------------------------*\ System wide percentages for correction of the LENGTH threshold \*----------------------------------------------------------------------------*/ -#define LENGTH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment -#define LENGTH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment -#define LENGTH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment -#define LENGTH_PERCENTAGE_LONGER_VALUE 30L // percentage for longer segment +#define LENGTH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment +#define LENGTH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment +#define LENGTH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment +#define LENGTH_PERCENTAGE_LONGER_VALUE 30L // percentage for longer segment /*---------------------------------------------------------------------*\ Macro Definition @@ -2807,16 +2673,15 @@ void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); /**********************************************************************/ /* macro to build index name from fully qualified data name */ /**********************************************************************/ -#define INDEXNAMEFROMMEMPATH( mem, ind ) \ - { \ - Utlstrccpy( ind, mem, DOT ); \ - if ( strcmp( strrchr( mem, DOT ), EXT_OF_SHARED_MEM ) == 0 ) \ - strcat( ind, EXT_OF_SHARED_MEMINDEX ); \ - else \ - strcat( ind, EXT_OF_TMINDEX ); \ +#define INDEXNAMEFROMMEMPATH(mem, ind) \ + { \ + Utlstrccpy(ind, mem, DOT); \ + if (strcmp(strrchr(mem, DOT), EXT_OF_SHARED_MEM) == 0) \ + strcat(ind, EXT_OF_SHARED_MEMINDEX); \ + else \ + strcat(ind, EXT_OF_TMINDEX); \ } - #include "EQFTAG.H" /*---------------------------------------------------------------------*\ Typedefs Definition @@ -2827,66 +2692,66 @@ void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); CHAR buf[] \*---------------------------------------------------------------------*/ - -typedef struct _TM_HEADER_UPDATE { /* tmhu */ - BOOL fCorruption ; - USHORT usNumTMBlocks, /* number of blocks in the TM */ - usNumFreeBlocks, /* # of free, pre-formatted blocks */ - usFirstAvailBlock;/* points to first available block */ +typedef struct _TM_HEADER_UPDATE +{ /* tmhu */ + BOOL fCorruption; + USHORT usNumTMBlocks, /* number of blocks in the TM */ + usNumFreeBlocks, /* # of free, pre-formatted blocks */ + usFirstAvailBlock; /* points to first available block */ } TM_HEADER_UPDATE; -typedef union _UNIONIN +typedef union _UNIONIN { - PREFIX_IN in; - ADD_IN ain; - DEL_IN din; - REP_IN rin; - EXT_IN ein; - GET_IN gin; - CLOSE_IN clin; - OPEN_IN oin; - CREATE_IN crin; - INFO_IN infoin; - CLOSEHANDLER_IN chin; - EXIT_IN xin; - DRIVES_IN drivesin; - FILE_LIST_IN filelistin; // !!! CHM - GETPART_IN getpartin; - PUTPROP_IN putpropin; - PUTDICTPROP_IN PutDictPropIn; // !!! CHM added - PUTPROPCOMBINED_IN PutPropCombinedIn; // !!! CHM added - GETPROP_IN getpropin; - ENDORG_IN endorgin; - DELTM_IN deltmin; - DELFILE_IN delfilein; - FILEINFO_IN fileinfoin; - RENFILE_IN renfilein; -} UNIONIN, * PUNIONIN; /* uin */ - -typedef union _UNIONOUT + PREFIX_IN in; + ADD_IN ain; + DEL_IN din; + REP_IN rin; + EXT_IN ein; + GET_IN gin; + CLOSE_IN clin; + OPEN_IN oin; + CREATE_IN crin; + INFO_IN infoin; + CLOSEHANDLER_IN chin; + EXIT_IN xin; + DRIVES_IN drivesin; + FILE_LIST_IN filelistin; // !!! CHM + GETPART_IN getpartin; + PUTPROP_IN putpropin; + PUTDICTPROP_IN PutDictPropIn; // !!! CHM added + PUTPROPCOMBINED_IN PutPropCombinedIn; // !!! CHM added + GETPROP_IN getpropin; + ENDORG_IN endorgin; + DELTM_IN deltmin; + DELFILE_IN delfilein; + FILEINFO_IN fileinfoin; + RENFILE_IN renfilein; +} UNIONIN, *PUNIONIN; /* uin */ + +typedef union _UNIONOUT { - PREFIX_OUT out; - ADD_OUT aout; - DEL_OUT dout; - REP_OUT rout; - EXT_OUT eout; - GET_OUT gout; - CLOSE_OUT clout; - OPEN_OUT oout; - CREATE_OUT crout; - INFO_OUT infoout; - EXIT_OUT xout; - DRIVES_OUT drivesout; - FILE_LIST_OUT filelistout; // !!! CHM - GETPART_OUT getpartout; - PUTPROP_OUT putpropout; - GETPROP_OUT getpropout; - ENDORG_OUT endorgout; - DELTM_OUT deltmout; - DELFILE_OUT delfileout; - FILEINFO_OUT fileinfoout; - RENFILE_OUT renfileout; -} UNIONOUT, * PUNIONOUT; /* uout */ + PREFIX_OUT out; + ADD_OUT aout; + DEL_OUT dout; + REP_OUT rout; + EXT_OUT eout; + GET_OUT gout; + CLOSE_OUT clout; + OPEN_OUT oout; + CREATE_OUT crout; + INFO_OUT infoout; + EXIT_OUT xout; + DRIVES_OUT drivesout; + FILE_LIST_OUT filelistout; // !!! CHM + GETPART_OUT getpartout; + PUTPROP_OUT putpropout; + GETPROP_OUT getpropout; + ENDORG_OUT endorgout; + DELTM_OUT deltmout; + DELFILE_OUT delfileout; + FILEINFO_OUT fileinfoout; + RENFILE_OUT renfileout; +} UNIONOUT, *PUNIONOUT; /* uout */ /* +---------------------------------------------------------------------------+ @@ -2901,117 +2766,111 @@ typedef union _UNIONOUT Samples: fRc = fTmComInit (); +---------------------------------------------------------------------------+ */ -BOOL fTmComInit (VOID); - - - -typedef struct _TMT_GLOBALS { /* tmtg */ - /* this structure keeps data and pointers to variables that exist from*/ - /* the point that the TM was created or opened, until close command */ - /* BlockImage, TextTable, WordsTable and TmHeader are allocated */ - /* dynamically during Create/Open and are freed at Close time. */ - HFILE hfTM; /* handle to TM db (from Dos call) */ - PUCHAR pchBlockImage; /* pointer to block image */ - PTOKENENTRY pteTextTable, /* pointer to TextTable */ - pteWordsTable, /* pointer to WordsTable */ - pteFirstSigWord, /* pointer to first significant word */ - pteSecondSigWord; /* pointer to second significant word*/ - PPREFIX_OUT pPrefixOut; /* pointer to the current PrefixOut */ - /* to be used by FormatMore */ - SEGMENT seg; /* placed here instead of allocating */ - /* this space on stack (point 276) */ - MATCH mtch; /* placed here instead of allocating */ - /* this space on stack (point 276) */ - PCHAR pWorkArea1; /* Pointer to work area 1 */ - PCHAR pWorkArea2; /* Pointer to work area 2 */ - PTM_HEADER pTmHeader; /* pointer to Tm header structure */ - LONG lActLengthThr; /* length corrected length threshold values*/ - LONG lActMatchThr; /* length corrected match threshold values */ - PLOADEDTABLE pLoadedTagTable; /* pointer to loaded tagtable */ - PVOID pstLoadedTagTable; /* pointer to loaded tagtable, used for */ - /* function TATagTokenize */ -} TMT_GLOBALS, * PTMT_GLOBALS, ** PPTMT_GLOBALS; +BOOL fTmComInit(VOID); + +typedef struct _TMT_GLOBALS +{ /* tmtg */ + /* this structure keeps data and pointers to variables that exist from*/ + /* the point that the TM was created or opened, until close command */ + /* BlockImage, TextTable, WordsTable and TmHeader are allocated */ + /* dynamically during Create/Open and are freed at Close time. */ + HFILE hfTM; /* handle to TM db (from Dos call) */ + PUCHAR pchBlockImage; /* pointer to block image */ + PTOKENENTRY pteTextTable, /* pointer to TextTable */ + pteWordsTable, /* pointer to WordsTable */ + pteFirstSigWord, /* pointer to first significant word */ + pteSecondSigWord; /* pointer to second significant word*/ + PPREFIX_OUT pPrefixOut; /* pointer to the current PrefixOut */ + /* to be used by FormatMore */ + SEGMENT seg; /* placed here instead of allocating */ + /* this space on stack (point 276) */ + MATCH mtch; /* placed here instead of allocating */ + /* this space on stack (point 276) */ + PCHAR pWorkArea1; /* Pointer to work area 1 */ + PCHAR pWorkArea2; /* Pointer to work area 2 */ + PTM_HEADER pTmHeader; /* pointer to Tm header structure */ + LONG lActLengthThr; /* length corrected length threshold values*/ + LONG lActMatchThr; /* length corrected match threshold values */ + PLOADEDTABLE pLoadedTagTable; /* pointer to loaded tagtable */ + PVOID pstLoadedTagTable; /* pointer to loaded tagtable, used for */ + /* function TATagTokenize */ +} TMT_GLOBALS, *PTMT_GLOBALS, **PPTMT_GLOBALS; // ***************** Work Constants global ************************** -#define INIT_NUMB_OF_ENTRIES 5 // Initial number of entries in a table or List -#define SEG_NUMB 6 // Segment number in char. -#define ASCII_FLAG 1 // Ascii flag converted to char. -#define MACH_TRANS 1 // Machine trans.flag converted to char. -#define TIME_STAMP 16 // Time integer converted to character -#define IND_CODES 20 // External industry codes space to hold a maximum of 6 codes -#define ASCII_IND_CODES_LENGTH 3 // Length of Industry codes in external format -#define NUMB_OF_IND_CODES 6 // Number of Industry codes in external format -#define MEM_NAME 8 // Length of memory database name without extension -#define DRIVE_NAME 12 // Special drive name -#define PTR_MEMHANDLER_IDA 0 // Relative position in the extra bytes for handler -#define PTR_MEM_IDA 0 // Relative position in the extra bytes for instance -#define PTR_CRT_IDA 0 // Relative position in the extra bytes for create dialog -#define PTR_DLG_IDA 0 // Relative position in the extra bytes for dialogs -#define TEMP 40 // Size of temporaty work area - // of WinRegisterClass call. -#define TEXT_100_CHAR 100 // string length for temporary strings -//#define NUMB_OF_TOKENS 1000 // Number of tokens in the token list -#define NUMB_OF_TOKENS 8000 // Number of tokens in the toklist:RJ increase nec. - // if 65520 bytes are read-in at once during MemImport -#define MEM_PROP_SIZE 2048 // Global size of all memory database properties -#define MEM_BLOCK_SIZE 1024 // Translation Memory block size -#define PROCESS_NUMB_OF_MESSAGES 10 // Number of messages to be process till next message is issued -#define MEM_DBCS 0 // 0 = No support, 1 = DBCS support -#define MEM_LOAD_PATTERN_NAME "*" // Default names to be shown for load dialog -#define MEM_LOAD_PATTERN_EXT ".*" // Default extension to be shown for load dialog -#define EQF_IDENTIFICATION "EQF" // EQF identification -#define CLBCOL_TITLE_STRING " " // Placeholder for of CLBCOL titles - +#define INIT_NUMB_OF_ENTRIES 5 // Initial number of entries in a table or List +#define SEG_NUMB 6 // Segment number in char. +#define ASCII_FLAG 1 // Ascii flag converted to char. +#define MACH_TRANS 1 // Machine trans.flag converted to char. +#define TIME_STAMP 16 // Time integer converted to character +#define IND_CODES 20 // External industry codes space to hold a maximum of 6 codes +#define ASCII_IND_CODES_LENGTH 3 // Length of Industry codes in external format +#define NUMB_OF_IND_CODES 6 // Number of Industry codes in external format +#define MEM_NAME 8 // Length of memory database name without extension +#define DRIVE_NAME 12 // Special drive name +#define PTR_MEMHANDLER_IDA 0 // Relative position in the extra bytes for handler +#define PTR_MEM_IDA 0 // Relative position in the extra bytes for instance +#define PTR_CRT_IDA 0 // Relative position in the extra bytes for create dialog +#define PTR_DLG_IDA 0 // Relative position in the extra bytes for dialogs +#define TEMP 40 // Size of temporaty work area + // of WinRegisterClass call. +#define TEXT_100_CHAR 100 // string length for temporary strings +// #define NUMB_OF_TOKENS 1000 // Number of tokens in the token list +#define NUMB_OF_TOKENS 8000 // Number of tokens in the toklist:RJ increase nec. + // if 65520 bytes are read-in at once during MemImport +#define MEM_PROP_SIZE 2048 // Global size of all memory database properties +#define MEM_BLOCK_SIZE 1024 // Translation Memory block size +#define PROCESS_NUMB_OF_MESSAGES 10 // Number of messages to be process till next message is issued +#define MEM_DBCS 0 // 0 = No support, 1 = DBCS support +#define MEM_LOAD_PATTERN_NAME "*" // Default names to be shown for load dialog +#define MEM_LOAD_PATTERN_EXT ".*" // Default extension to be shown for load dialog +#define EQF_IDENTIFICATION "EQF" // EQF identification +#define CLBCOL_TITLE_STRING " " // Placeholder for of CLBCOL titles // ***************** Memory database load ************************** -//#define MEM_CONTEXT_TOKEN_END 0 // -//#define MEM_CONTROL_TOKEN_END 1 // UNUSED! -//#define NTM_DESCRIPTION_TOKEN_END 2 // -//#define MEM_MEMORYDB_TOKEN_END 3 // -//#define NTM_MEMORYDB_TOKEN_END 4 // -//#define MEM_SEGMENT_TOKEN_END 5 // -//#define MEM_SOURCE_TOKEN_END 6 // -//#define MEM_TARGET_TOKEN_END 7 // -//#define MEM_CONTEXT_TOKEN 8 // -//#define MEM_CONTROL_TOKEN 9 // -//#define NTM_DESCRIPTION_TOKEN 10 // -//#define MEM_MEMORYDB_TOKEN 11 // -//#define NTM_MEMORYDB_TOKEN 12 // -//#define MEM_SEGMENT_TOKEN 13 // -//#define MEM_SOURCE_TOKEN 14 // -//#define MEM_TARGET_TOKEN 15 // +// #define MEM_CONTEXT_TOKEN_END 0 // +// #define MEM_CONTROL_TOKEN_END 1 // UNUSED! +// #define NTM_DESCRIPTION_TOKEN_END 2 // +// #define MEM_MEMORYDB_TOKEN_END 3 // +// #define NTM_MEMORYDB_TOKEN_END 4 // +// #define MEM_SEGMENT_TOKEN_END 5 // +// #define MEM_SOURCE_TOKEN_END 6 // +// #define MEM_TARGET_TOKEN_END 7 // +// #define MEM_CONTEXT_TOKEN 8 // +// #define MEM_CONTROL_TOKEN 9 // +// #define NTM_DESCRIPTION_TOKEN 10 // +// #define MEM_MEMORYDB_TOKEN 11 // +// #define NTM_MEMORYDB_TOKEN 12 // +// #define MEM_SEGMENT_TOKEN 13 // +// #define MEM_SOURCE_TOKEN 14 // +// #define MEM_TARGET_TOKEN 15 // // strings to create the token IDs for the tags dynamically using TATagTokenize -#define MEM_CONTEXT_TOKEN_END L"" -#define MEM_CONTROL_TOKEN_END L"" -#define NTM_DESCRIPTION_TOKEN_END L"" -#define MEM_MEMORYDB_TOKEN_END L"" -#define NTM_MEMORYDB_TOKEN_END L"" -#define MEM_SEGMENT_TOKEN_END L"" -#define MEM_SOURCE_TOKEN_END L"" -#define MEM_TARGET_TOKEN_END L"" -#define MEM_CONTEXT_TOKEN L"" -#define MEM_CONTROL_TOKEN L"" -#define NTM_DESCRIPTION_TOKEN L"" -#define MEM_MEMORYDB_TOKEN L"" -#define NTM_MEMORYDB_TOKEN L"" -#define MEM_SEGMENT_TOKEN L"" -#define MEM_SOURCE_TOKEN L"" -#define MEM_TARGET_TOKEN L"" -#define MEM_CODEPAGE_TOKEN L"" -#define MEM_CODEPAGE_TOKEN_END L"" -#define MEM_ADDDATA_TOKEN_END L"" -#define MEM_ADDDATA_TOKEN L"" - - - - -#define TM_IMPORT_OK 0 // Translation memory import OK -#define TM_IMPORT_FAILED 1 // Translation memory import failed -#define TM_IMPORT_FORCED 2 // Translation memory import was forced - - -//used for input from create folder dialog +#define MEM_CONTEXT_TOKEN_END L"" +#define MEM_CONTROL_TOKEN_END L"" +#define NTM_DESCRIPTION_TOKEN_END L"" +#define MEM_MEMORYDB_TOKEN_END L"" +#define NTM_MEMORYDB_TOKEN_END L"" +#define MEM_SEGMENT_TOKEN_END L"" +#define MEM_SOURCE_TOKEN_END L"" +#define MEM_TARGET_TOKEN_END L"" +#define MEM_CONTEXT_TOKEN L"" +#define MEM_CONTROL_TOKEN L"" +#define NTM_DESCRIPTION_TOKEN L"" +#define MEM_MEMORYDB_TOKEN L"" +#define NTM_MEMORYDB_TOKEN L"" +#define MEM_SEGMENT_TOKEN L"" +#define MEM_SOURCE_TOKEN L"" +#define MEM_TARGET_TOKEN L"" +#define MEM_CODEPAGE_TOKEN L"" +#define MEM_CODEPAGE_TOKEN_END L"" +#define MEM_ADDDATA_TOKEN_END L"" +#define MEM_ADDDATA_TOKEN L"" + +#define TM_IMPORT_OK 0 // Translation memory import OK +#define TM_IMPORT_FAILED 1 // Translation memory import failed +#define TM_IMPORT_FORCED 2 // Translation memory import was forced + +// used for input from create folder dialog typedef enum { MEM_CREATE_NAME_IND, @@ -3020,8 +2879,7 @@ typedef enum MEM_CREATE_TARGETLANG_IND } MEM_CREATEINDEX; -#define MEM_TEXT_BUFFER 65520 // Length of the text buffer - +#define MEM_TEXT_BUFFER 65520 // Length of the text buffer // ************ Definitions for TM list box display ********************* typedef enum @@ -3042,14 +2900,14 @@ typedef enum /* string IDs to parse the CONTROL string of the TM exported format */ /* using UtlParseX15 */ /**********************************************************************/ -//#define NTM_SEGNR_ID 0 //seg nr from analysis -//#define NTM_MTFLAG_ID 1 //machine translation flag -//#define NTM_TIME_ID 2 //time stamp -//#define NTM_SOURCELNG_ID 3 //source language -//#define NTM_TARGETLNG_ID 4 //target language -//#define NTM_AUTHORNAME_ID 5 //author name -//#define NTM_TAGTABLENAME_ID 6 //tag table name -//#define NTM_FILENAME_ID 7 //file name +// #define NTM_SEGNR_ID 0 //seg nr from analysis +// #define NTM_MTFLAG_ID 1 //machine translation flag +// #define NTM_TIME_ID 2 //time stamp +// #define NTM_SOURCELNG_ID 3 //source language +// #define NTM_TARGETLNG_ID 4 //target language +// #define NTM_AUTHORNAME_ID 5 //author name +// #define NTM_TAGTABLENAME_ID 6 //tag table name +// #define NTM_FILENAME_ID 7 //file name typedef enum { @@ -3064,15 +2922,15 @@ typedef enum NTM_LONGNAME_ID } NTM_IMPORTINDEX; -#define START_KEY 0xFFFFFF +#define START_KEY 0xFFFFFF -#define EXT_OF_RENAMED_TMDATA ".TRD" +#define EXT_OF_RENAMED_TMDATA ".TRD" #define EXT_OF_RENAMED_TMINDEX ".TRI" -#define EXT_OF_TEMP_TMPROP ".TMR" -#define EXT_OF_RENAMED_TMPROP ".RMR" -#define EXT_OF_TEMP_TMDATA ".TTD" -#define EXT_OF_TEMP_TMINDEX ".TTI" -#define EXT_OF_TMPROP EXT_OF_MEM +#define EXT_OF_TEMP_TMPROP ".TMR" +#define EXT_OF_RENAMED_TMPROP ".RMR" +#define EXT_OF_TEMP_TMDATA ".TTD" +#define EXT_OF_TEMP_TMINDEX ".TTI" +#define EXT_OF_TMPROP EXT_OF_MEM #include class ProposalFilter; @@ -3081,163 +2939,147 @@ class ProposalFilter; typedef struct _SEG_CTRL_DATA { - CHAR chSegmentNumber[SEG_NUMB]; /* Segment number converted to character*/ - CHAR chAsciiFlag[ASCII_FLAG]; /* Ascii flag converted to char. */ - CHAR chMachineTrans[MACH_TRANS]; /* Machine trans.flag converted to char.*/ - CHAR chTimeStamp[TIME_STAMP]; /* Time integer converted to character */ - CHAR chIndustryCodes[IND_CODES]; /* Industry codes */ - CHAR chFileName[MAX_FILESPEC-1]; /* Segment origin. from that file name */ -}SEG_CTRL_DATA, * PSEG_CTRL_DATA; - - + CHAR chSegmentNumber[SEG_NUMB]; /* Segment number converted to character*/ + CHAR chAsciiFlag[ASCII_FLAG]; /* Ascii flag converted to char. */ + CHAR chMachineTrans[MACH_TRANS]; /* Machine trans.flag converted to char.*/ + CHAR chTimeStamp[TIME_STAMP]; /* Time integer converted to character */ + CHAR chIndustryCodes[IND_CODES]; /* Industry codes */ + CHAR chFileName[MAX_FILESPEC - 1]; /* Segment origin. from that file name */ +} SEG_CTRL_DATA, *PSEG_CTRL_DATA; /*! \brief search a string in a proposal - \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pProposal pointer to the proposal + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ -BOOL searchInProposal -( - OtmProposal *pProposal, - PSZ_W pszSearch, - LONG lSearchOptions -); - +BOOL searchInProposal( + OtmProposal *pProposal, + PSZ_W pszSearch, + LONG lSearchOptions); /*! \brief search a string in a proposal - \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pProposal pointer to the proposal + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ -BOOL searchExtendedInProposal -( - OtmProposal *pProposal, - std::vector& filters, - LONG lSearchOptions -); - - - +BOOL searchExtendedInProposal( + OtmProposal *pProposal, + std::vector &filters, + LONG lSearchOptions); /*! \brief check if search string matches current data \param pData pointer to current position in data area \param pSearch pointer to search string \returns 0 if search string matches data */ -SHORT compareString -( - PSZ_W pData, - PSZ_W pSearch -); +SHORT compareString( + PSZ_W pData, + PSZ_W pSearch); /*! \brief find the given string in the provided data \param pszData pointer to the data being searched \param pszSearch pointer to the search string \returns TRUE if the data contains the searched string otherwise FALSE is returned */ -BOOL findString -( - PSZ_W pszData, - PSZ_W pszSearch -); +BOOL findString( + PSZ_W pszData, + PSZ_W pszSearch); typedef struct _MEM_MERGE_IDA { - CHAR szPathMergeMem[MAX_LONGPATH]; // Full path and name of TM to be merged - CHAR szPathMergeProp[MAX_LONGPATH]; // Full path and name of Properties to be imported - CHAR szDriveMergeMem[MAX_DRIVE]; // Drive letter of TM to be merged w o : ( d ) - CHAR szNameMergeMem[MAX_LONGFILESPEC];// Name of TM to be merged w o ext ( heller ) - CHAR szShortNameMergeMem[MAX_FILESPEC];// Short name of TM being merged - CHAR szDirMergeMem[MAX_LONGPATH]; // Path of directories fo TM to be merged ( \eqf\import\ ) - CHAR szExtMergeMem[MAX_FEXT]; // Ext of TM to be merged ( MIP or MEM or ? ) - EqfMemory *pMergeMem; // TM handle of TM to be merged - CHAR szInvokingHandler[40]; // ??? 40 ???? to be in Gerds stuff. Name of handler which invoked the process - CHAR szSystemPath[MAX_EQF_PATH]; // Path to the EQF system - CHAR szTemp[MAX_EQF_PATH]; // Temporary path area - CHAR szPropName[MAX_FILESPEC]; // TM property name ( xxxx.MEM ) - CHAR szPathMem[MAX_LONGPATH]; // Full name and Path to the TM ( c:\EQF\MEM\xxxx.MEM ) - CHAR szPathProp[MAX_LONGPATH]; // Full name and Path to the Properties - CHAR szDriveMem[MAX_DRIVE]; // Drive letter of TM w o : ( h ) - CHAR szNameMem[MAX_LONGFILESPEC]; // Long name of TM w o ext ( gallus ) - CHAR szShortNameMem[MAX_FILESPEC]; // Short name of TM w o ext ( gallus ) - EqfMemory *pOutputMem; // TM handle - BOOL fPropExist; // Existence of properties 0=No 1=Yes - BOOL fPropCreated; // Tm properties have been created - BOOL fMsg; // A message has been issued already - BOOL fImport; // If set the merge is started via an import - USHORT usImportRc; // Import function return code - ULONG ulSegmentCounter; // Number of segments merged - ULONG ulInvSegmentCounter; // Invalid Segment counter - EXT_IN stOldExtIn; // input for TmOldExtract - EXT_OUT stOldExtOut; // output for TmOldExtract - TMX_EXT_IN_W stExtIn; // TMX_EXTRACT_IN structure - TMX_EXT_OUT_W stExtOut; // TMX_EXTRACT_OUT structure - TMX_PUT_W stPutIn; // TMX_PUT_IN structure - TMX_EXT_OUT_W stPutOut; // TMX_PUT_OUT structure - TIME_L tStamp; // Time stamp of merge start time - HWND hwndMemLb; // Handle to the TM listbox - HWND hProgressWindow; // Handle of progress indicator window - ULONG ulProgressPos; // position of progress indicator - CHAR szServer[MAX_LONGPATH]; // Server Name of TM or \0 if TM is local - BOOL fPropReceived; // Property file got via Receive ? - BOOL fOrganizeInvoked; // TRUE if Organize is invoked - BOOL fImportAnyway; // FALSE if import of TM with - // different IDs is canceled - CHAR szSourceMemSourceLang[MAX_LANG_LENGTH]; - CHAR szSourceMemTargetLang[MAX_LANG_LENGTH]; - CHAR szSourceMemMarkup[MAX_FNAME]; /*@1276A*/ - CHAR szTargetMemSourceLang[MAX_LANG_LENGTH]; - CHAR szTargetMemTargetLang[MAX_LANG_LENGTH]; - CHAR szTargetMemMarkup[MAX_FNAME]; /*@1276A*/ - BOOL fDisplayMsg; // message flag if segment not /*@47A*/ - // not replaced message should /*@47A*/ - // be displayed or not /*@47A*/ - BOOL fOldPropFile; // used for folder import - CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM - CHAR szPathMergeIndex[MAX_LONGPATH];// Full path and name of index to be merged - CHAR szPathIndex[MAX_LONGPATH]; // name and path to index C:\EQF\MEM\xxxx.TMI ) - BOOL fBatch; // TRUE = we are in batch mode - HWND hwndErrMsg; // parent handle for error messages - CHAR szLongName[MAX_LONGFILESPEC]; // buffer for long TM names - USHORT usTask; // current taskl to do (batch mode) - OtmProposal *pProposal; // buffer for proposal data - BOOL fFirstExtract; // TRUE = this ist the first extract call - int iComplete; // process completion rate -}MEM_MERGE_IDA, * PMEM_MERGE_IDA; + CHAR szPathMergeMem[MAX_LONGPATH]; // Full path and name of TM to be merged + CHAR szPathMergeProp[MAX_LONGPATH]; // Full path and name of Properties to be imported + CHAR szDriveMergeMem[MAX_DRIVE]; // Drive letter of TM to be merged w o : ( d ) + CHAR szNameMergeMem[MAX_LONGFILESPEC]; // Name of TM to be merged w o ext ( heller ) + CHAR szShortNameMergeMem[MAX_FILESPEC]; // Short name of TM being merged + CHAR szDirMergeMem[MAX_LONGPATH]; // Path of directories fo TM to be merged ( \eqf\import\ ) + CHAR szExtMergeMem[MAX_FEXT]; // Ext of TM to be merged ( MIP or MEM or ? ) + EqfMemory *pMergeMem; // TM handle of TM to be merged + CHAR szInvokingHandler[40]; // ??? 40 ???? to be in Gerds stuff. Name of handler which invoked the process + CHAR szSystemPath[MAX_EQF_PATH]; // Path to the EQF system + CHAR szTemp[MAX_EQF_PATH]; // Temporary path area + CHAR szPropName[MAX_FILESPEC]; // TM property name ( xxxx.MEM ) + CHAR szPathMem[MAX_LONGPATH]; // Full name and Path to the TM ( c:\EQF\MEM\xxxx.MEM ) + CHAR szPathProp[MAX_LONGPATH]; // Full name and Path to the Properties + CHAR szDriveMem[MAX_DRIVE]; // Drive letter of TM w o : ( h ) + CHAR szNameMem[MAX_LONGFILESPEC]; // Long name of TM w o ext ( gallus ) + CHAR szShortNameMem[MAX_FILESPEC]; // Short name of TM w o ext ( gallus ) + EqfMemory *pOutputMem; // TM handle + BOOL fPropExist; // Existence of properties 0=No 1=Yes + BOOL fPropCreated; // Tm properties have been created + BOOL fMsg; // A message has been issued already + BOOL fImport; // If set the merge is started via an import + USHORT usImportRc; // Import function return code + ULONG ulSegmentCounter; // Number of segments merged + ULONG ulInvSegmentCounter; // Invalid Segment counter + EXT_IN stOldExtIn; // input for TmOldExtract + EXT_OUT stOldExtOut; // output for TmOldExtract + TMX_EXT_IN_W stExtIn; // TMX_EXTRACT_IN structure + TMX_EXT_OUT_W stExtOut; // TMX_EXTRACT_OUT structure + TMX_PUT_W stPutIn; // TMX_PUT_IN structure + TMX_EXT_OUT_W stPutOut; // TMX_PUT_OUT structure + TIME_L tStamp; // Time stamp of merge start time + HWND hwndMemLb; // Handle to the TM listbox + HWND hProgressWindow; // Handle of progress indicator window + ULONG ulProgressPos; // position of progress indicator + CHAR szServer[MAX_LONGPATH]; // Server Name of TM or \0 if TM is local + BOOL fPropReceived; // Property file got via Receive ? + BOOL fOrganizeInvoked; // TRUE if Organize is invoked + BOOL fImportAnyway; // FALSE if import of TM with + // different IDs is canceled + CHAR szSourceMemSourceLang[MAX_LANG_LENGTH]; + CHAR szSourceMemTargetLang[MAX_LANG_LENGTH]; + CHAR szSourceMemMarkup[MAX_FNAME]; /*@1276A*/ + CHAR szTargetMemSourceLang[MAX_LANG_LENGTH]; + CHAR szTargetMemTargetLang[MAX_LANG_LENGTH]; + CHAR szTargetMemMarkup[MAX_FNAME]; /*@1276A*/ + BOOL fDisplayMsg; // message flag if segment not /*@47A*/ + // not replaced message should /*@47A*/ + // be displayed or not /*@47A*/ + BOOL fOldPropFile; // used for folder import + CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM + CHAR szPathMergeIndex[MAX_LONGPATH]; // Full path and name of index to be merged + CHAR szPathIndex[MAX_LONGPATH]; // name and path to index C:\EQF\MEM\xxxx.TMI ) + BOOL fBatch; // TRUE = we are in batch mode + HWND hwndErrMsg; // parent handle for error messages + CHAR szLongName[MAX_LONGFILESPEC]; // buffer for long TM names + USHORT usTask; // current taskl to do (batch mode) + OtmProposal *pProposal; // buffer for proposal data + BOOL fFirstExtract; // TRUE = this ist the first extract call + int iComplete; // process completion rate +} MEM_MERGE_IDA, *PMEM_MERGE_IDA; + /**********************************************************************/ /* Dialog IDA for TM property dialog */ /**********************************************************************/ typedef struct _MEM_PROP_IDA { - CHAR szMemName[MAX_LONGFILESPEC];// Memory database name without extension - EqfMemory *pMem; // Handle of memory database - //HPROP hPropMem; // Memory database property handle - //PPROP_NTM pPropMem; // pointer to TM properties - CHAR szPropName[MAX_FILESPEC]; // buffer for property name - CHAR szPropPath[MAX_EQF_PATH]; // buffer for property path - CHAR szTempPath[MAX_EQF_PATH]; // buffer for path names -}MEM_PROP_IDA, * PMEM_PROP_IDA; - - -BOOL EqfMemPropsToHtml( HWND hwndParent, PMEM_IDA pIDA, EqfMemory *pMem ); - -VOID EQFMemImportTrojaEnd( PMEM_MERGE_IDA ); -USHORT CloseMergeTmAndTm( PMEM_MERGE_IDA, BOOL ); /*@1139A*/ -USHORT MemFuncMergeTM -( - PMEM_MERGE_IDA pMIDA // Pointer to the merge IDA + CHAR szMemName[MAX_LONGFILESPEC]; // Memory database name without extension + EqfMemory *pMem; // Handle of memory database + // HPROP hPropMem; // Memory database property handle + // PPROP_NTM pPropMem; // pointer to TM properties + CHAR szPropName[MAX_FILESPEC]; // buffer for property name + CHAR szPropPath[MAX_EQF_PATH]; // buffer for property path + CHAR szTempPath[MAX_EQF_PATH]; // buffer for path names +} MEM_PROP_IDA, *PMEM_PROP_IDA; + +BOOL EqfMemPropsToHtml(HWND hwndParent, PMEM_IDA pIDA, EqfMemory *pMem); + +VOID EQFMemImportTrojaEnd(PMEM_MERGE_IDA); +USHORT CloseMergeTmAndTm(PMEM_MERGE_IDA, BOOL); /*@1139A*/ +USHORT MemFuncMergeTM( + PMEM_MERGE_IDA pMIDA // Pointer to the merge IDA ); -//#endif +// #endif typedef struct _MEMORY_HANDLER_DATA { - IDA_HEAD stIdaHead; // Standard Ida head -}MEMORY_HANDLER_DATA, * PMEMORY_HANDLER_DATA; - + IDA_HEAD stIdaHead; // Standard Ida head +} MEMORY_HANDLER_DATA, *PMEMORY_HANDLER_DATA; /**********************************************************************/ /* TMX_ENDORG_IN, TMX_ENDORG_OUT used by NTMCloseOrganize */ @@ -3249,8 +3091,6 @@ typedef struct _MEMORY_HANDLER_DATA /* The temporary file has to be closed previously. */ /**********************************************************************/ - - /**********************************************************************/ /* The following structures for dialog IDAs are defined only */ /* if INCL_EQFMEM_DLGIDAS has been defined */ @@ -3259,374 +3099,328 @@ typedef struct _MEMORY_HANDLER_DATA typedef struct _MEM_INCL_DLG_IDA { - BOOL fInitErrorOccurred; // Important init error occurred -}MEM_INCL_DLG_IDA, * PMEM_INCL_DLG_IDA; + BOOL fInitErrorOccurred; // Important init error occurred +} MEM_INCL_DLG_IDA, *PMEM_INCL_DLG_IDA; #endif - typedef struct _SLIDER_DATA { - CHAR szLine [5] [TEXT_100_CHAR+MAX_PATH144]; // 5 text lines of 100 characters - // + max path length -} SLIDER_DATA, * PSLIDER_DATA; - - + CHAR szLine[5][TEXT_100_CHAR + MAX_PATH144]; // 5 text lines of 100 characters + // + max path length +} SLIDER_DATA, *PSLIDER_DATA; /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - OpenTmFile (PTMT_GLOBALS ptmtg, /*.The TM Globals Area.............*/ - PSZ pszFileName,/*.The TM Full File Name...........*/ - USHORT idCommand), /* TMC_CREATE / TMC_OPEN...........*/ +OpenTmFile(PTMT_GLOBALS ptmtg, /*.The TM Globals Area.............*/ + PSZ pszFileName, /*.The TM Full File Name...........*/ + USHORT idCommand), /* TMC_CREATE / TMC_OPEN...........*/ - ReadTmHeader (PTMT_GLOBALS ptmtg), /* Tmt Globals area ................*/ + ReadTmHeader(PTMT_GLOBALS ptmtg), /* Tmt Globals area ................*/ - AllocTmtGlobals (USHORT usTmtGlobalsSize, /* The allocateion size.......*/ - PPTMT_GLOBALS pptmtg), /* The TM Globals Area........*/ + AllocTmtGlobals(USHORT usTmtGlobalsSize, /* The allocateion size.......*/ + PPTMT_GLOBALS pptmtg), /* The TM Globals Area........*/ - SetTmtWorkPointers (PTMT_GLOBALS ptmtg); /* The TM Globals Area........*/ + SetTmtWorkPointers(PTMT_GLOBALS ptmtg); /* The TM Globals Area........*/ /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ -VOID -GetFirstSegAddress -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSource, // pointer to source segment - PTM_ADDRESS pAddr // pointer to address +VOID GetFirstSegAddress( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSource, // pointer to source segment + PTM_ADDRESS pAddr // pointer to address ); -VOID -CleanSource -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSource // Pointer to string to be tokenized +VOID CleanSource( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSource // Pointer to string to be tokenized ); -VOID -CalcPrimaryKey -( - PTMT_GLOBALS ptmtg, // Pointer to Globals structure - PUCHAR pchPrimaryKey // Pointer to primary key +VOID CalcPrimaryKey( + PTMT_GLOBALS ptmtg, // Pointer to Globals structure + PUCHAR pchPrimaryKey // Pointer to primary key ); -VOID -CalcSecondaryKey -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSortedSecKey // Ptr to sorted secondary key +VOID CalcSecondaryKey( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSortedSecKey // Ptr to sorted secondary key ); -VOID -Get4Chars -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - SHORT sTokenId, // WORD/USELESS/NOISE/TEXT/TAG - PTOKENENTRY pteFirst, // Ptr to Text or Word Table - PUCHAR pchPrimaryKey // 4 chars for primary key +VOID Get4Chars( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + SHORT sTokenId, // WORD/USELESS/NOISE/TEXT/TAG + PTOKENENTRY pteFirst, // Ptr to Text or Word Table + PUCHAR pchPrimaryKey // 4 chars for primary key ); -VOID -WordsTokenize -( - PTMT_GLOBALS ptmtg // Pointer to Tmt globals +VOID WordsTokenize( + PTMT_GLOBALS ptmtg // Pointer to Tmt globals ); -VOID -UselessFiltering -( - PTOKENENTRY pteWord // Pointer to word in WordsTable +VOID UselessFiltering( + PTOKENENTRY pteWord // Pointer to word in WordsTable ); USHORT -Tmt -( - HTM htm, // Pointer to TmtGlobals - PIN pIn, // Pointer to input buffer - POUT pOut // Pointer to output buffer +Tmt( + HTM htm, // Pointer to TmtGlobals + PIN pIn, // Pointer to input buffer + POUT pOut // Pointer to output buffer ); - /**********************************************************************/ /* TmtX */ /**********************************************************************/ USHORT -TmtX ( HTM htm, // Pointer to TmtGlobals - PXIN pIn, // Pointer to input buffer - PXOUT pOut ); // Pointer to output buffer - +TmtX(HTM htm, // Pointer to TmtGlobals + PXIN pIn, // Pointer to input buffer + PXOUT pOut); // Pointer to output buffer USHORT -CalcEntryInKeyDir -( - PTM_HEADER ptmh, // Pointer to Tm Header - PUCHAR pchPrimaryKey // Primary Key +CalcEntryInKeyDir( + PTM_HEADER ptmh, // Pointer to Tm Header + PUCHAR pchPrimaryKey // Primary Key ); USHORT -WriteTmhToDisk -( - PTMT_GLOBALS ptmtg, // Pointer to globals - USHORT fsWrite // Write control flags word +WriteTmhToDisk( + PTMT_GLOBALS ptmtg, // Pointer to globals + USHORT fsWrite // Write control flags word ); USHORT -WriteToDisk -( - PTMT_GLOBALS ptmtg, // TMT globals - ULONG ldispPtr, // New file pointer location - PVOID pvWrite, // Pointer to write buffer - USHORT numWrite // Number of bytes to write +WriteToDisk( + PTMT_GLOBALS ptmtg, // TMT globals + ULONG ldispPtr, // New file pointer location + PVOID pvWrite, // Pointer to write buffer + USHORT numWrite // Number of bytes to write ); USHORT -FormatMore -( - PTMT_GLOBALS ptmtg // Pointer to Tmt globals +FormatMore( + PTMT_GLOBALS ptmtg // Pointer to Tmt globals ); USHORT -ReadStringFromDisk -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PCHAR bufRead, // Read buffer - USHORT lenString, // String length - PTM_ADDRESS paddr // Pointer to string buffer +ReadStringFromDisk( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PCHAR bufRead, // Read buffer + USHORT lenString, // String length + PTM_ADDRESS paddr // Pointer to string buffer ); USHORT -ReadSegmentFromDisk -( - PTMT_GLOBALS ptmtg, // pointer to Tmt globals - PTM_ADDRESS pAddr, // Pointer to an address - PSEGMENT pSegment, // Pointer to seg. buffer - PBOOL pfFirstSeg, // Pointer to 1st seg. flag - PBOOL pfLastSeg // Pointer to last seg. flag +ReadSegmentFromDisk( + PTMT_GLOBALS ptmtg, // pointer to Tmt globals + PTM_ADDRESS pAddr, // Pointer to an address + PSEGMENT pSegment, // Pointer to seg. buffer + PBOOL pfFirstSeg, // Pointer to 1st seg. flag + PBOOL pfLastSeg // Pointer to last seg. flag ); USHORT -ReadBlock -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PTM_ADDRESS paddr, // Pointer to an address - BOOL fAddressInParameter // Address in pAddr +ReadBlock( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PTM_ADDRESS paddr, // Pointer to an address + BOOL fAddressInParameter // Address in pAddr ); -INT -CharCompare -( - const void * arg1, // First comparand - const void * arg2 // Second comparand +INT CharCompare( + const void *arg1, // First comparand + const void *arg2 // Second comparand ); -BOOL -fStrcmpCRLF -( - PSZ pszStr1, // pointer to first string to be compared - PSZ pszStr2, // pointer to second string to be compared - PUSHORT pusEqualChars, // (return value) pointer to USHORT where the - // number of equal characters (first string) - USHORT fCompareType); // flag indicating whether CRLF and LF are ignored - // ALIGN_CRLF -> CRLF and LF are equal equal - // USE_CRLF -> exact matches required s - // IGNORE_CRL -> characters are equal - +BOOL fStrcmpCRLF( + PSZ pszStr1, // pointer to first string to be compared + PSZ pszStr2, // pointer to second string to be compared + PUSHORT pusEqualChars, // (return value) pointer to USHORT where the + // number of equal characters (first string) + USHORT fCompareType); // flag indicating whether CRLF and LF are ignored + // ALIGN_CRLF -> CRLF and LF are equal equal + // USE_CRLF -> exact matches required s + // IGNORE_CRL -> characters are equal /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - TmtExtract (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PEXT_IN pExtIn, /* pointer to input buffer...*/ - PEXT_OUT pExtOut); /* pointer to output buffer..*/ +TmtExtract(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PEXT_IN pExtIn, /* pointer to input buffer...*/ + PEXT_OUT pExtOut); /* pointer to output buffer..*/ USHORT - TmtGet (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PGET_IN pGetIn, /* pointer to input buffer...*/ - PGET_OUT pGetOut), /* pointer to output buffer..*/ - - TmtGetTMPart( PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PGETPART_IN pGetPartIn, /* pointer to input buffer...*/ - PGETPART_OUT pGetPartOut), /* pointer to output buffer..*/ - - GetSegByAddr (PTMT_GLOBALS ptmtg, /* pointer to TmtGlobals.....*/ - PTM_ADDRESS pAddr, /* pointer to input address..*/ - PTM_ADDRESS pAddrNext, /* pointer to next address...*/ - PSEGMENT pseg), /* pointer to segment........*/ - - - TmtInfo (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PINFO_OUT pInfoOut); /* pointer to output buffer..*/ - - -LONG - CheckSimilar (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals.....*/ - PSEGMENT pCurrentSegment,/* pointer to current segmnt .*/ - PTOKENENTRY pTokenEntry, /* points to 1'st & 2'nd word */ - PGET_IN pGetIn, /* pointer to input buffer....*/ - PSZ pszInputKey, /* pointer to sortedkey of....*/ - /* input segment..............*/ - PMATCH pmtch), /* pointer to stMatch.........*/ - - CheckExact (PGET_IN pGetIn, /* pointer to input segment....*/ - PSEGMENT pseg, /* pointer to current seg......*/ - BOOL fSimilarMod, /* Similar= TRUE, Exact=FALSE..*/ - PMATCH pmtch), /* pointer to stMatch..........*/ - - CalcExactIndustry (PGET_IN pGetIn, /* pointer to input segment....*/ - PSEGMENT pseg); /* pointer to current seg......*/ - -VOID - RankNewMatch (USHORT numMatchesReq, // number of matches required - USHORT usConvert, // Conversion indicator - PMATCH pmtchCurrent, // pointer to stMatch - PSEGMENT pseg, // pointer to current seg - PGET_OUT pGetOut); // pointer to GET_OUT struct - -BOOL - CheckWord (PTOKENENTRY pteFirst, /*Point to first word ..........*/ - PTOKENENTRY pteSecond) ; /*Point to second word .........*/ - -VOID -LengthCorrectThresholds( PTMT_GLOBALS ptmtg, //pointer to Tmt globals - USHORT usLenInput, //number of words in inp. seg. - PGET_IN pGetIn ); //pointer to input buffer +TmtGet(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PGET_IN pGetIn, /* pointer to input buffer...*/ + PGET_OUT pGetOut), /* pointer to output buffer..*/ + + TmtGetTMPart(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PGETPART_IN pGetPartIn, /* pointer to input buffer...*/ + PGETPART_OUT pGetPartOut), /* pointer to output buffer..*/ + + GetSegByAddr(PTMT_GLOBALS ptmtg, /* pointer to TmtGlobals.....*/ + PTM_ADDRESS pAddr, /* pointer to input address..*/ + PTM_ADDRESS pAddrNext, /* pointer to next address...*/ + PSEGMENT pseg), /* pointer to segment........*/ + + TmtInfo(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PINFO_OUT pInfoOut); /* pointer to output buffer..*/ + +LONG CheckSimilar(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals.....*/ + PSEGMENT pCurrentSegment, /* pointer to current segmnt .*/ + PTOKENENTRY pTokenEntry, /* points to 1'st & 2'nd word */ + PGET_IN pGetIn, /* pointer to input buffer....*/ + PSZ pszInputKey, /* pointer to sortedkey of....*/ + /* input segment..............*/ + PMATCH pmtch), /* pointer to stMatch.........*/ + + CheckExact(PGET_IN pGetIn, /* pointer to input segment....*/ + PSEGMENT pseg, /* pointer to current seg......*/ + BOOL fSimilarMod, /* Similar= TRUE, Exact=FALSE..*/ + PMATCH pmtch), /* pointer to stMatch..........*/ + + CalcExactIndustry(PGET_IN pGetIn, /* pointer to input segment....*/ + PSEGMENT pseg); /* pointer to current seg......*/ + +VOID RankNewMatch(USHORT numMatchesReq, // number of matches required + USHORT usConvert, // Conversion indicator + PMATCH pmtchCurrent, // pointer to stMatch + PSEGMENT pseg, // pointer to current seg + PGET_OUT pGetOut); // pointer to GET_OUT struct + +BOOL CheckWord(PTOKENENTRY pteFirst, /*Point to first word ..........*/ + PTOKENENTRY pteSecond); /*Point to second word .........*/ + +VOID LengthCorrectThresholds(PTMT_GLOBALS ptmtg, // pointer to Tmt globals + USHORT usLenInput, // number of words in inp. seg. + PGET_IN pGetIn); // pointer to input buffer USHORT - TmtAdd (PTMT_GLOBALS pTmtGlobals, /* Pointer to Tmt Globals........*/ - PADD_IN pAddIn, /* Pointer to input buffer.......*/ - PADD_OUT pAddOut), /* Pointer to output buffer......*/ - - TmtDelete (PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ - PDEL_IN pDeleteIn, /* Pointer to input buffer...*/ - PDEL_OUT pDeleteOut), /* Pointer to output buffer..*/ - - TmtReplace (PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ - PREP_IN pReplaceIn, /* Pointer to input buffer...*/ - PREP_OUT pReplaceOut), /* Pointer to output buffer..*/ - - AddSegToCluster (PTMT_GLOBALS ptmtg, /* Pointer to globals area..*/ - PTM_ADDRESS pAddr, /* Ptr to initial address...*/ - PSEGMENT pseg), /* Pointer to a segment.....*/ +TmtAdd(PTMT_GLOBALS pTmtGlobals, /* Pointer to Tmt Globals........*/ + PADD_IN pAddIn, /* Pointer to input buffer.......*/ + PADD_OUT pAddOut), /* Pointer to output buffer......*/ - FindFreshAddrInCluster (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.*/ - PTM_ADDRESS pAddr, /* Pointer to address.....*/ - BOOL fReadFirstBlock), /* Read block flag........*/ + TmtDelete(PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ + PDEL_IN pDeleteIn, /* Pointer to input buffer...*/ + PDEL_OUT pDeleteOut), /* Pointer to output buffer..*/ - TakeBlockFromSpool (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ - USHORT rcPrevious, /* RC of the previous call....*/ - PTM_ADDRESS pAddr), /* Pointer to an address......*/ + TmtReplace(PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ + PREP_IN pReplaceIn, /* Pointer to input buffer...*/ + PREP_OUT pReplaceOut), /* Pointer to output buffer..*/ - WriteStringToDisk (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ - PCHAR bufWrite, /* Pointer to the string......*/ - USHORT lenString, /* String length..............*/ - PTM_ADDRESS pAddr), /* Pointer to string address..*/ + AddSegToCluster(PTMT_GLOBALS ptmtg, /* Pointer to globals area..*/ + PTM_ADDRESS pAddr, /* Ptr to initial address...*/ + PSEGMENT pseg), /* Pointer to a segment.....*/ - WriteBlock (PTMT_GLOBALS ptmtg), /* Pointer to Tmt globals.....*/ + FindFreshAddrInCluster(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.*/ + PTM_ADDRESS pAddr, /* Pointer to address.....*/ + BOOL fReadFirstBlock), /* Read block flag........*/ - FindMatchSegInCluster (PTMT_GLOBALS ptmtg, /* Ptr to Tmt globals...*/ - PTM_ADDRESS pAddr, /* Pointer to address...*/ - PBOOL pfLastSeg, /* Last segment flag....*/ - PSEGMENT psegIn, /* Ptr to input buffer..*/ - PSEGMENT pseg), /* Pointer to segment...*/ + TakeBlockFromSpool(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ + USHORT rcPrevious, /* RC of the previous call....*/ + PTM_ADDRESS pAddr), /* Pointer to an address......*/ - DeleteSegment (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals........*/ - PTM_ADDRESS pAddr, /* Address of segment to delete..*/ - BOOL fLastSeg, /* Last segment in cluster flag..*/ - PSEGMENT pseg); /* Pointer to the segment........*/ + WriteStringToDisk(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ + PCHAR bufWrite, /* Pointer to the string......*/ + USHORT lenString, /* String length..............*/ + PTM_ADDRESS pAddr), /* Pointer to string address..*/ -BOOL - CheckMatchForDelete (PSEGMENT psegIn, /* pointer to input buffer.......*/ - PSEGMENT pseg); /* currently, output buffer......*/ + WriteBlock(PTMT_GLOBALS ptmtg), /* Pointer to Tmt globals.....*/ -USHORT TmtDeleteTM( HTM hMem, // TM handle - PDELTM_IN pDelTmIn, // Input structure - PDELTM_OUT pDelTmOut ); // Output structure + FindMatchSegInCluster(PTMT_GLOBALS ptmtg, /* Ptr to Tmt globals...*/ + PTM_ADDRESS pAddr, /* Pointer to address...*/ + PBOOL pfLastSeg, /* Last segment flag....*/ + PSEGMENT psegIn, /* Ptr to input buffer..*/ + PSEGMENT pseg), /* Pointer to segment...*/ + DeleteSegment(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals........*/ + PTM_ADDRESS pAddr, /* Address of segment to delete..*/ + BOOL fLastSeg, /* Last segment in cluster flag..*/ + PSEGMENT pseg); /* Pointer to the segment........*/ -USHORT TmtCloseOrganize( HTM hMem, // TM handle - PENDORG_IN pEndOrgIn, // Input structure - PENDORG_OUT pEndOrgOut ); // Output structure +BOOL CheckMatchForDelete(PSEGMENT psegIn, /* pointer to input buffer.......*/ + PSEGMENT pseg); /* currently, output buffer......*/ +USHORT TmtDeleteTM(HTM hMem, // TM handle + PDELTM_IN pDelTmIn, // Input structure + PDELTM_OUT pDelTmOut); // Output structure +USHORT TmtCloseOrganize(HTM hMem, // TM handle + PENDORG_IN pEndOrgIn, // Input structure + PENDORG_OUT pEndOrgOut); // Output structure -VOID TmtDeleteFile( PDELFILE_IN pDelFileIn, // Input structure - PDELFILE_OUT pDelFileOut ); // Output structure +VOID TmtDeleteFile(PDELFILE_IN pDelFileIn, // Input structure + PDELFILE_OUT pDelFileOut); // Output structure -VOID GetFileInfo( PFILEINFO_IN pFileInfoIn, // Input structure - PFILEINFO_OUT pFileInfoOut ); // Output structure +VOID GetFileInfo(PFILEINFO_IN pFileInfoIn, // Input structure + PFILEINFO_OUT pFileInfoOut); // Output structure -VOID RenameFile( PRENFILE_IN pRenFileIn, // Input structure - PRENFILE_OUT pRenFileOut ); // Output structure - -BOOL fCheckFileClosed ( PSZ ); //--- pointer to file name +VOID RenameFile(PRENFILE_IN pRenFileIn, // Input structure + PRENFILE_OUT pRenFileOut); // Output structure +BOOL fCheckFileClosed(PSZ); //--- pointer to file name /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - TmtOpen (POPEN_IN pOpenIn, /* Pointer to input structure....*/ - POPEN_OUT pOpenOut), /* Pointer to output structure...*/ - - TmtCreate (PCREATE_IN pCreateIn, /* Pointer to input structure....*/ - PCREATE_OUT pCreateOut), /* Pointer to output structure...*/ - - TmtClose (PTMT_GLOBALS ptmtg, /* Pointer to Globals structure.....*/ - PCLOSE_OUT pCloseOut); /* Pointer to output buffer.........*/ - -INT_PTR /*CALLBACK*/ MEMCREATEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMLOADDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMEXPORTDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMMERGEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMPROPDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MemCreateName( HWND, USHORT, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ UTLSERVERLISTDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ UTLSERVERLISTNAMEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMINCLUDEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ SERVWAITDLG( HWND, WINMSG, WPARAM, LPARAM ); -USHORT MemCreateProcess( PMEM_IDA, PSZ, USHORT ); -VOID MemDestroyProcess( PMEM_IDA, USHORT * ); -USHORT MemGetAddressOfProcessIDA( PMEM_IDA, WPARAM, USHORT *, PVOID * ); -VOID MemRcHandlingErrorUndefined( USHORT, PSZ ); -VOID MemRcHandlingErrorUndefinedHwnd( USHORT, PSZ, HWND ); -USHORT EQFMemOrganizeStart( PPROCESSCOMMAREA ); -VOID EQFMemOrganizeProcess( PPROCESSCOMMAREA ); -VOID EQFMemOrganizeEnd( PPROCESSCOMMAREA ); -VOID EQFMemLoadStart( PPROCESSCOMMAREA, HWND ); -VOID EQFMemLoadProcess( PPROCESSCOMMAREA, HWND ); -VOID EQFMemLoadEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -USHORT EQFMemExportStart( PPROCESSCOMMAREA, HWND ); -USHORT EQFMemExportProcess( PPROCESSCOMMAREA, HWND ); -USHORT EQFMemExportEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -VOID EQFMemMergeStart( PPROCESSCOMMAREA, HWND ); -VOID EQFMemMergeProcess( PPROCESSCOMMAREA, HWND ); -VOID EQFMemMergeEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -USHORT EQFMemImportTrojaStart( HWND, LPARAM, HWND ); -USHORT MemInitSlider( HWND, USHORT, PSZ, USHORT, PSZ, PHWND, USHORT, USHORT); -USHORT ReadABGrouping( PSZ, PSZ, ABGROUP ); -MRESULT MemOrganizeCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemExportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemMergeCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemImportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); - - +TmtOpen(POPEN_IN pOpenIn, /* Pointer to input structure....*/ + POPEN_OUT pOpenOut), /* Pointer to output structure...*/ + + TmtCreate(PCREATE_IN pCreateIn, /* Pointer to input structure....*/ + PCREATE_OUT pCreateOut), /* Pointer to output structure...*/ + + TmtClose(PTMT_GLOBALS ptmtg, /* Pointer to Globals structure.....*/ + PCLOSE_OUT pCloseOut); /* Pointer to output buffer.........*/ + +INT_PTR /*CALLBACK*/ MEMCREATEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMLOADDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMEXPORTDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMMERGEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMPROPDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MemCreateName(HWND, USHORT, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ UTLSERVERLISTDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ UTLSERVERLISTNAMEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMINCLUDEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ SERVWAITDLG(HWND, WINMSG, WPARAM, LPARAM); +USHORT MemCreateProcess(PMEM_IDA, PSZ, USHORT); +VOID MemDestroyProcess(PMEM_IDA, USHORT *); +USHORT MemGetAddressOfProcessIDA(PMEM_IDA, WPARAM, USHORT *, PVOID *); +VOID MemRcHandlingErrorUndefined(USHORT, PSZ); +VOID MemRcHandlingErrorUndefinedHwnd(USHORT, PSZ, HWND); +USHORT EQFMemOrganizeStart(PPROCESSCOMMAREA); +VOID EQFMemOrganizeProcess(PPROCESSCOMMAREA); +VOID EQFMemOrganizeEnd(PPROCESSCOMMAREA); +VOID EQFMemLoadStart(PPROCESSCOMMAREA, HWND); +VOID EQFMemLoadProcess(PPROCESSCOMMAREA, HWND); +VOID EQFMemLoadEnd(PPROCESSCOMMAREA, HWND, LPARAM); +USHORT EQFMemExportStart(PPROCESSCOMMAREA, HWND); +USHORT EQFMemExportProcess(PPROCESSCOMMAREA, HWND); +USHORT EQFMemExportEnd(PPROCESSCOMMAREA, HWND, LPARAM); +VOID EQFMemMergeStart(PPROCESSCOMMAREA, HWND); +VOID EQFMemMergeProcess(PPROCESSCOMMAREA, HWND); +VOID EQFMemMergeEnd(PPROCESSCOMMAREA, HWND, LPARAM); +USHORT EQFMemImportTrojaStart(HWND, LPARAM, HWND); +USHORT MemInitSlider(HWND, USHORT, PSZ, USHORT, PSZ, PHWND, USHORT, USHORT); +USHORT ReadABGrouping(PSZ, PSZ, ABGROUP); +MRESULT MemOrganizeCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemExportCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemMergeCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemImportCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); // ************* Memory macro definitions ******************************* //======================================================================= -//prototypes and definitions for TM utilities +// prototypes and definitions for TM utilities #define SIZE_32K 32768 #define ERROR_TABLE_FULL 1111 -#define ID_NOT_FOUND 2222 - +#define ID_NOT_FOUND 2222 // adjust pointger for a new location -#define ADJUSTPTR( new, old, offsptr ) \ - ((PBYTE)new + ((PBYTE)offsptr - (PBYTE)old)) +#define ADJUSTPTR(new, old, offsptr) \ + ((PBYTE) new + ((PBYTE)offsptr - (PBYTE)old)) // do not update name tables when name is not contained in the table #define NTMGETID_NOUPDATE_OPT 0x00000001 @@ -3635,49 +3429,42 @@ MRESULT MemImportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); #define NTMGETID_NOTFOUND_ID 0xFFFF -USHORT TmtXGet( EqfMemory*, PTMX_GET_W, PTMX_GET_OUT_W ); +// tm put prototypes +VOID HashSentence(PTMX_SENTENCE); +USHORT HashTupel(PBYTE, USHORT, USHORT); +USHORT HashTupelW(PSZ_W, USHORT); +VOID BuildVotes(PTMX_SENTENCE); +VOID Vote(PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT); +USHORT CheckCompactArea(PTMX_SENTENCE, EqfMemory *); -//tm put prototypes -VOID HashSentence( PTMX_SENTENCE ); -USHORT HashTupel( PBYTE, USHORT, USHORT ); -USHORT HashTupelW( PSZ_W, USHORT ); -static VOID BuildVotes( PTMX_SENTENCE ); -static VOID Vote( PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT ); -USHORT CheckCompactArea( PTMX_SENTENCE, EqfMemory* ); +VOID FillTmRecord(PTMX_SENTENCE, + BytesRef TmRecord, // filled tm record returned + BytesRef, USHORT); -USHORT TokenizeTarget( StringTagVariants*, PTMX_TAGTABLE_RECORD*, PLONG, PSZ, EqfMemory* ); +USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); -VOID FillTmRecord( PTMX_SENTENCE, - PTMX_RECORD, PTMX_TARGET_CLB, USHORT ); +VOID FillTargetRecord(PTMX_SENTENCE, PTMX_TARGET_RECORD, PTMX_TARGET_CLB); -USHORT DetermineTmRecord( EqfMemory*, PTMX_SENTENCE, PULONG ); +VOID DeleteOldestRecord(PTMX_RECORD, PULONG); -VOID FillTargetRecord( PTMX_SENTENCE, //PTMX_TAGTABLE_RECORD, - PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB ); +// tm get prototypes +USHORT GetExactMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, + PUSHORT, PTMX_GET_OUT_W); +USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, + PTMX_MATCH_TABLE_W, PUSHORT, LONG, PBYTE); -VOID DeleteOldestRecord( PTMX_RECORD, PULONG ); -//tm get prototypes -USHORT GetExactMatch( EqfMemory*, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, - PUSHORT, PTMX_GET_OUT_W ); -USHORT ExactTest( EqfMemory*, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, - PTMX_MATCH_TABLE_W, PUSHORT, ULONG ); +INT CompCount(const void *, const void *); +INT CompCountVotes(const void *, const void *); +VOID CleanupTempMatch(PTMX_MATCHENTRY, std::vector&, PUSHORT, PUSHORT); +USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, std::vector&, PUSHORT); +USHORT FuzzyTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, + PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, LONG, PBYTE); +USHORT GetFuzzyMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT); -BOOL AddTagsToString( PSZ, PULONG, PTMX_TAGTABLE_RECORD, PSZ ); -BOOL AddTagsToStringW( PSZ_W, PLONG, PTMX_TAGTABLE_RECORD, PSZ_W ); +// tm delete segment prototypes +USHORT NTMCheckForUpdates(EqfMemory *); -INT CompCount( const void *, const void * ); -INT CompCountVotes( const void *, const void * ); -VOID CleanupTempMatch( PTMX_MATCHENTRY, PTMX_MATCHENTRY *, PUSHORT, PUSHORT ); -USHORT FillMatchEntry( EqfMemory*, PTMX_SENTENCE, PTMX_MATCHENTRY, PUSHORT ); -USHORT FuzzyTest( EqfMemory*, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, - PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, ULONG ); -USHORT GetFuzzyMatch( EqfMemory*, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT ); - - -//tm delete segment prototypes -USHORT NTMCheckForUpdates( EqfMemory* ); -USHORT NTMLockTM( EqfMemory*, BOOL, PBOOL ); // definitions for terse of TM name tables // max size of in-memory name table @@ -3686,40 +3473,27 @@ USHORT NTMLockTM( EqfMemory*, BOOL, PBOOL ); // magic word for recognition of tersed name tables // Note: this ULONG value has to ensure that it will never been used // as a name so it is prefixed and suffixed by a 0x00 -#define TERSEMAGICWORD 0x0019FF00L -// structure of header for tersed name tables -typedef struct _TERSEHEADER -{ - USHORT usAllocSize; // alloc size in untersed tables - USHORT usMaxEntries; // max entries in untersed tables - ULONG ulMagicWord; // magic word for tersed name - USHORT usDataSize; // size of data area when expanded - USHORT usCompression; // type of compression used for data - // (values as defined in EQFCMPR.H) -} TERSEHEADER, *PTERSEHEADER; - -BOOL TMDelTargetClb -( - PTMX_RECORD pTmRecord, // ptr to TM record - PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record - PTMX_TARGET_CLB pTargetClb // ptr to target control record + +BOOL TMDelTargetClb( + BytesRef TmRecord, // ptr to TM record + PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record + PTMX_TARGET_CLB pTargetClb // ptr to target control record ); -ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG usLenChar ); -LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG usLenComp ); +ULONG EQFUnicode2Compress(PBYTE pTarget, PSZ_W pInput, ULONG usLenChar); +LONG EQFCompress2Unicode(std::wstring &pOutput, PBYTE pTarget, ULONG usLenComp); #include "EncodingHelper.h" - /* class FilterParam{ public: - FilterParam(std::string& search, FilterField field, FilterType type): m_field(field), m_type(type), + FilterParam(std::string& search, FilterField field, FilterType type): m_field(field), m_type(type), m_searchString(search), m_searchStringW(convertStrToWstr(search)){}; - + FilterParam(long t1, long t2, FilterField field = TIMESTAMP): m_timestamp1(t1), m_timestamp2(t2), m_field{field}{} private: @@ -3729,8 +3503,6 @@ class FilterParam{ long m_timestamp1=0, m_timestamp2=0; };//*/ - - /* class StringFilterParam{ std::string m_searchString; @@ -3740,234 +3512,221 @@ class StringFilterParam{ StringFilterParam(std::string& search, ProposalFilter::FilterType type): FilterParam(type, field), {} }//*/ -typedef struct _MEM_ORGANIZE_IDA +struct MEM_ORGANIZE_IDA { - std::shared_ptr memRef; - TMX_PUT_W stPutIn; // input for TmReplace - TMX_EXT_OUT_W stPutOut; // The REPLACE_OUT structure - TMX_EXT_IN_W stExtIn; // input for TmExtract - TMX_EXT_OUT_W stExtOut; // output for TmExtract - CHAR szPathOrganizeMem[2048]; // Full path and name of TM to be organized - CHAR szDrive[MAX_DRIVE]; // Drive letter without colum - CHAR szMemName[MAX_LONGFILESPEC]; // Translation memory name - CHAR szTempMemName[MAX_LONGFILESPEC];// name of temporary translation memory - CHAR szPluginName[MAX_LONGFILESPEC]; // name of plugin used for the memory - CHAR szPathTempMem[2048]; // Full path of temporary transl. mem. - OtmProposal *pProposal; // buffer for memory proposal - std::shared_ptr pMem; // Handle of transl. memory - std::shared_ptr pMemTemp; // Handle of tmporary transl. memory - BOOL fMsg; // A message has been issued already - //ULONG ulSegmentCounter; // Number of segments organized - //ULONG ulInvSegmentCounter; // Invalid Segment counter - TIME_L tStamp; // Time stamp of organize start time - HWND hProgressWindow; // Handle of progress indicator window - ULONG ulProgressPos; // position of progress indicator - CHAR szPathTempIndex[MAX_EQF_PATH]; //full temporary index name - - LANGUAGE szSourceLanguage; // source language of org TM - LANGUAGE szTargetLanguage; // target language of org TM - CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM - - CHAR szPropertyName[MAX_FILESPEC]; // property name with ext. - CHAR szTempPropertyName[MAX_FILESPEC]; // property name temp TM with ext - - CHAR szOrgProp[MAX_EQF_PATH]; // full name of original property file - CHAR szTmpProp[MAX_EQF_PATH]; // full name of temporary property file - - CHAR szEqfPath[MAX_EQF_PATH]; // system path D:\EQF - USHORT usOrgType; // type ORGANIZE, CONVERT - BOOL fBatch; // TRUE if organizing in batch mode - HWND hwndErrMsg; // handle of window to be used for error msgs - PDDEMEMORG pDDEMemOrg; // ptr to batch memory organize data - USHORT usRC; // return code / error code - LONG NextTask; // next task in non-DDE batch mode - PSZ pszNameList; // pointer to list of TMs being organized - PSZ pszActiveName; // points to current name in pszNameList - CHAR szBuffer[2048]; // general purpose buffer - BOOL fFirstGet; // TRUE = this is the first get access - std::vector m_reorganizeFilters; - MutexTimeout tmListTimeout ; - MutexTimeout tmLockTimeout ; -}MEM_ORGANIZE_IDA, * PMEM_ORGANIZE_IDA; - -USHORT NTMConvertProperties( PPROPTRANSLMEM, PMEM_ORGANIZE_IDA ); + std::shared_ptr memRef; + TMX_PUT_W stPutIn; // input for TmReplace + TMX_EXT_OUT_W stPutOut; // The REPLACE_OUT structure + TMX_EXT_IN_W stExtIn; // input for TmExtract + TMX_EXT_OUT_W stExtOut; // output for TmExtract + CHAR szPathOrganizeMem[2048]; // Full path and name of TM to be organized + CHAR szDrive[MAX_DRIVE]; // Drive letter without colum + CHAR szMemName[MAX_LONGFILESPEC]; // Translation memory name + CHAR szTempMemName[MAX_LONGFILESPEC]; // name of temporary translation memory + CHAR szPluginName[MAX_LONGFILESPEC]; // name of plugin used for the memory + CHAR szPathTempMem[2048]; // Full path of temporary transl. mem. + OtmProposal *pProposal; // buffer for memory proposal + std::shared_ptr pMem; // Handle of transl. memory + std::shared_ptr pMemTemp; // Handle of tmporary transl. memory + BOOL fMsg; // A message has been issued already + // ULONG ulSegmentCounter; // Number of segments organized + // ULONG ulInvSegmentCounter; // Invalid Segment counter + TIME_L tStamp; // Time stamp of organize start time + HWND hProgressWindow; // Handle of progress indicator window + ULONG ulProgressPos; // position of progress indicator + CHAR szPathTempIndex[MAX_EQF_PATH]; // full temporary index name + + LANGUAGE szSourceLanguage; // source language of org TM + LANGUAGE szTargetLanguage; // target language of org TM + CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM + + CHAR szPropertyName[MAX_FILESPEC]; // property name with ext. + CHAR szTempPropertyName[MAX_FILESPEC]; // property name temp TM with ext + + CHAR szOrgProp[MAX_EQF_PATH]; // full name of original property file + CHAR szTmpProp[MAX_EQF_PATH]; // full name of temporary property file + + CHAR szEqfPath[MAX_EQF_PATH]; // system path D:\EQF + USHORT usOrgType; // type ORGANIZE, CONVERT + BOOL fBatch; // TRUE if organizing in batch mode + HWND hwndErrMsg; // handle of window to be used for error msgs + PDDEMEMORG pDDEMemOrg; // ptr to batch memory organize data + USHORT usRC; // return code / error code + LONG NextTask; // next task in non-DDE batch mode + PSZ pszActiveName; // points to current name in pszNameList + CHAR szBuffer[2048]; // general purpose buffer + BOOL fFirstGet; // TRUE = this is the first get access + std::vector m_reorganizeFilters; + MutexTimeout tmListTimeout; + MutexTimeout tmLockTimeout; +}; +using PMEM_ORGANIZE_IDA = MEM_ORGANIZE_IDA*; + +USHORT TmCloseOrganize(PMEM_ORGANIZE_IDA, USHORT); + +USHORT NTMCloseOrganize(PMEM_ORGANIZE_IDA, USHORT); +USHORT NTMConvertProperties(PPROPTRANSLMEM, PMEM_ORGANIZE_IDA); /*! \brief constant defining the maximum number of opened memories -*/ + */ #define OTMMEMSERVICE_MAX_NUMBER_OF_OPEN_MEMORIES 40 -//5MB reserved for service +// 5MB reserved for service #define MEMORY_RESERVED_FOR_SERVICE 5000000 - class CreateMemRequestData; class EqfMemoryPlugin : public OtmPlugin /*! \brief This class implements the standard translation memory plugin (EQF) for OpenTM2. -*/ + */ { - //static EqfMemoryPlugin* _instance; + // static EqfMemoryPlugin* _instance; public: -/*! \brief Constructor -*/ - EqfMemoryPlugin(); -/*! \brief Destructor -*/ - ~EqfMemoryPlugin(); + /*! \brief Constructor + */ + EqfMemoryPlugin(); + /*! \brief Destructor + */ + ~EqfMemoryPlugin(); - static EqfMemoryPlugin* GetInstance(); + static EqfMemoryPlugin *GetInstance(); - virtual bool isUsable() - {return OtmPlugin::isUsable();}; + virtual bool isUsable() + { + return OtmPlugin::isUsable(); + }; - -/*! \enum eRegRc - Possible return values of EqfMemory and EqfMemoryPlugin methods -*/ - enum eRc - { - eSuccess = 0, /*!< method completed successfully */ - eUnknownPlugin, /*!< the specified memory plugin is not available */ -// eInvalidName, /*!< plugin-name is invalid */ -// eAlreadyRegistered, /*!< plugin with same name was already registered before */ -// eInvalidRequest, /*!< method may only be called from within registerPlugins call */ - eMemoryNotFound, /*!< the specified memory does not exist or is not controlled by this memory plugin*/ - eUnknown, /*!< plugin with that name was not registered before */ - eNotSupported, /*!< method is not supported by this plugin */ - eBufferTooSmall, /*!< the provided buffer is too small */ + /*! \enum eRegRc + Possible return values of EqfMemory and EqfMemoryPlugin methods + */ + enum eRc + { + eSuccess = 0, /*!< method completed successfully */ + eUnknownPlugin, /*!< the specified memory plugin is not available */ + // eInvalidName, /*!< plugin-name is invalid */ + // eAlreadyRegistered, /*!< plugin with same name was already registered before */ + // eInvalidRequest, /*!< method may only be called from within registerPlugins call */ + eMemoryNotFound, /*!< the specified memory does not exist or is not controlled by this memory plugin*/ + eUnknown, /*!< plugin with that name was not registered before */ + eNotSupported, /*!< method is not supported by this plugin */ + eBufferTooSmall, /*!< the provided buffer is too small */ eNotSupportedMemoryType, - eNotEnoughMemory, /*!< not enough system memory to process the request */ - eRepeat /*!< repeat calling this method until processing has been completed*/ - }; - - -// options for the importFromMemoryFiles method -static const int IMPORTFROMMEMFILES_COMPLETEINONECALL_OPT = 1; // complete the import in one call, do not divide the processing into smaller steps + eNotEnoughMemory, /*!< not enough system memory to process the request */ + eRepeat /*!< repeat calling this method until processing has been completed*/ + }; + // options for the importFromMemoryFiles method + static const int IMPORTFROMMEMFILES_COMPLETEINONECALL_OPT = 1; // complete the import in one call, do not divide the processing into smaller steps + + /*! \brief Returns the name of the plugin + */ + const char *getName(); + /*! \brief Returns a short plugin-Description + */ + const char *getShortDescription(); + /*! \brief Returns a verbose plugin-Description + */ + const char *getLongDescription(); + /*! \brief Returns the version of the plugin + */ + const char *getVersion(); + /*! \brief Returns the name of the plugin-supplier + */ + const char *getSupplier(); + + EqfMemory *initTM(const std::string &memName, + size_t requiredMemory, + unsigned short usAccessMode, + bool fReorganizeOnly); + + + + /*! \brief set description of a memory + \param pszName name of the memory + \param pszDesc description information + \returns 0 if successful or error return code + */ + int setDescription( + const char *pszName, + const char *pszDesc); -/*! \brief Returns the name of the plugin -*/ - const char* getName(); -/*! \brief Returns a short plugin-Description -*/ - const char* getShortDescription(); -/*! \brief Returns a verbose plugin-Description -*/ - const char* getLongDescription(); -/*! \brief Returns the version of the plugin -*/ - const char* getVersion(); -/*! \brief Returns the name of the plugin-supplier -*/ - const char* getSupplier(); + /*! \brief Get the error message for the last error occured - EqfMemory* initTM(const std::string& memName, - size_t requiredMemory, - unsigned short usAccessMode, - bool fReorganizeOnly); + \param strError reference to a string receiving the error mesage text + \returns last error code + */ + int getLastError( + std::string &strError); + /*! \brief Get the error message for the last error occured -/*! \brief Close a memory - \param pMemory pointer to memory object -*/ - int closeMemory( - EqfMemory *pMemory - ); + \param pszError pointer to a buffer for the error text + \param iBufSize size of error text buffer in number of characters + \returns last error code + */ + int getLastError( + char *pszError, + int iBufSize); -/*! \brief set description of a memory - \param pszName name of the memory - \param pszDesc description information - \returns 0 if successful or error return code -*/ - int setDescription( - const char* pszName, - const char* pszDesc); + /*! \brief Stops the plugin. + Terminating-function for the plugin, will be called directly before + the DLL containing the plugin will be unloaded.\n + The method should call PluginManager::deregisterPlugin() to tell the PluginManager + that the plugin is not active anymore. + Warning: THIS METHOD SHOULD BE CALLED BY THE PLUGINMANAGER ONLY! + \param fForce, TRUE = force stop of the plugin even if functions are active, FALSE = only stop plugin when it is inactive + \returns TRUE when successful */ + bool stopPlugin(bool fForce = false); -/*! \brief Get the error message for the last error occured - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ - int getLastError - ( - std::string &strError - ); -/*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ - int getLastError - ( - char *pszError, - int iBufSize - ); - -/*! \brief Stops the plugin. - Terminating-function for the plugin, will be called directly before - the DLL containing the plugin will be unloaded.\n - The method should call PluginManager::deregisterPlugin() to tell the PluginManager - that the plugin is not active anymore. - Warning: THIS METHOD SHOULD BE CALLED BY THE PLUGINMANAGER ONLY! - \param fForce, TRUE = force stop of the plugin even if functions are active, FALSE = only stop plugin when it is inactive - \returns TRUE when successful */ - bool stopPlugin( bool fForce = false ); - -/*! \brief Handle a return code from the memory functions and create the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name or NULL if not available - \param pszMemPath fully qualified memory path name or NULL if not available - \param strLastError reference to string object receiving the message text - \param iLastError reference to a integer variable receiving the error code - \returns original or modified error return code -*/ -static int handleError( int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError ); - - int iLastError; - std::string strLastError; - std::vector< std::shared_ptr > m_MemInfoVector; + int iLastError; + std::string strLastError; private: - std::shared_ptr findMemory( const char *pszName ); - int findMemoryIndex(const char *pszName); - - std::string name; - std::string shortDesc; - std::string longDesc; - std::string version; - std::string supplier; - std::string descrType; - char szBuffer[4000]; // general purpose buffer area - char szSupportedDrives[27]; // list of supported drives + std::string name; + std::string shortDesc; + std::string longDesc; + std::string version; + std::string supplier; + std::string descrType; + char szBuffer[4000]; // general purpose buffer area +}; +// enum TIMED_BOOL{ +// FALSE = 0, +// TRUE = 1, +// TIMEOUT = -2 +// }; + +enum TMManagarState{ + INIT=0, + READY=1, + SHUTDOWN_CALLED, + WAITING_FOR_IMPORTS, + OFF }; -//enum TIMED_BOOL{ -// FALSE = 0, -// TRUE = 1, -// TIMEOUT = -2 -//}; -class TMManager{ +class TMManager +{ - public: - - - std::atomic_bool fWriteRequestsAllowed{0}; - std::atomic_bool fServiceIsRunning{1}; - /*! \brief Pointer to the list of opened memories - */ - //std::vector vMemoryList; - typedef std::map > TMMap; +public: + inline static std::mutex mutex_tmManager; + inline static bool fWriteRequestsAllowed{0}; + inline static bool fServiceIsRunning{1}; + inline static bool fLoadingTmIsAllowed{0}; + /*! \brief Pointer to the list of opened memories + */ + // std::vector vMemoryList; + typedef std::map> TMMap; TMMap tms; std::recursive_timed_mutex mutex_requestTM; std::recursive_timed_mutex mutex_access_tms; - enum TMManagerCodes{ + enum TMManagerCodes + { TMM_NO_ERROR = 0, TMM_TMD_NOT_FOUND = 16, @@ -3975,233 +3734,228 @@ class TMManager{ TMM_TM_NOT_FOUND = TMM_TMD_NOT_FOUND + TMM_TMI_NOT_FOUND, }; - bool IsMemoryLoaded(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryInList(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryLoading(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryFailedToLoad(const std::string& strMemName, MutexTimeout& tmListTimeout); - int TMExistsOnDisk(const std::string& tmName, bool logErrorIfNotExists = true); + bool IsMemoryLoaded(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryInList(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryLoading(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryFailedToLoad(const std::string &strMemName, MutexTimeout &tmListTimeout); + int TMExistsOnDisk(const std::string &tmName, bool logErrorIfNotExists = true); - int AddMem(const std::shared_ptr NewMem, MutexTimeout& tmListTimeout); - int OpenTM(const std::string& strMemName, MutexTimeout& tmListTimeout, COMMAND command); - int CloseTM(const std::string& strMemName, MutexTimeout& tmListTimeout); - int DeleteTM(const std::string& strMemName, std::string& outputMsg); + int AddMem(const std::shared_ptr NewMem, MutexTimeout &tmListTimeout); + int OpenTM(const std::string &strMemName, MutexTimeout &tmListTimeout, COMMAND command); + int CloseTM(const std::string &strMemName, MutexTimeout &tmListTimeout); + int DeleteTM(const std::string &strMemName, std::string &outputMsg); - int RenameTM(const std::string& oldMemName, const std::string& newMemName, std::string& outputMsg,MutexTimeout& tmListTimeout); + int RenameTM(const std::string &oldMemName, const std::string &newMemName, std::string &outputMsg, MutexTimeout &tmListTimeout); int MoveTM( - const std::string& oldMemoryName, - const std::string& newMemoryName, - std::string &strError, - MutexTimeout& tmListTimeout - ); - - std::shared_ptr requestServicePointer(const std::string& strMemName, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - std::shared_ptr requestReadOnlyTMPointer(const std::string& strMemName, std::shared_ptr& refBack, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - std::shared_ptr requestWriteTMPointer(const std::string& strMemName, std::shared_ptr& refBack, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - - std::shared_ptr CreateNewEmptyTM(const std::string& strMemName, const std::string& strSrcLang, - const std::string& strMemDescription, int& _rc_, bool keepInRamOnly = false); - - - /*! \brief OpenTM2 API session handle - */ - LONG hSession = 0; - /*! \brief close a memory and remove it from the open list - \param iIndex index of memory in the open list - \returns 0 - */ - - /*! \brief find a memory in our list of active memories - \param pszMemory name of the memory - \returns index in the memory table if successful, -1 if memory is not contained in the list - */ - int findMemoryInList( const std::string& memName ); - - /*! \brief close any memories which haven't been used for a long time - \returns 0 - */ - size_t CleanupMemoryList(size_t memoryRequested, MutexTimeout& tmListTimeout); - - - /*! \brief close any memories which haven't been used for a long time - \returns size of closed tms - */ - size_t CloseTmsThatWasNotUsedForTooLong(MutexTimeout& tmListTimeout, const std::string& tmToOpen); - - /*! \brief calcuate total amount of RAM occupied by opened memory files - \returns 0 - */ - size_t CalculateOccupiedRAM(MutexTimeout& tmListTimeout); - - std::shared_ptr findOpenedMemory( const std::string& memName); - - int GetMemImportInProcessCount(MutexTimeout& tmListTimeout); - - /*! \brief Close all open memories - \returns http return code0 if successful or an error code in case of failures - */ - int closeAll(); - -///MEMORY FACTORY REGION - /*! \brief This class provides factory methods for EqfMemory objects + const std::string &oldMemoryName, + const std::string &newMemoryName, + std::string &strError, + MutexTimeout &tmListTimeout); + + std::shared_ptr requestServicePointer(const std::string &strMemName, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + std::shared_ptr requestReadOnlyTMPointer(const std::string &strMemName, std::shared_ptr &refBack, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + std::shared_ptr requestWriteTMPointer(const std::string &strMemName, std::shared_ptr &refBack, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + + std::shared_ptr CreateNewEmptyTM(const std::string &strMemName, const std::string &strSrcLang, + const std::string &strMemDescription, int &_rc_, bool keepInRamOnly = false); + + /*! \brief OpenTM2 API session handle + */ + LONG hSession = 0; + /*! \brief close a memory and remove it from the open list + \param iIndex index of memory in the open list + \returns 0 + */ -*/ + /*! \brief find a memory in our list of active memories + \param pszMemory name of the memory + \returns index in the memory table if successful, -1 if memory is not contained in the list + */ + int findMemoryInList(const std::string &memName); -//public: + /*! \brief close any memories which haven't been used for a long time + \returns 0 + */ + size_t CleanupMemoryList(size_t memoryRequested, MutexTimeout &tmListTimeout); + /*! \brief close any memories which haven't been used for a long time + \returns size of closed tms + */ + size_t CloseTmsThatWasNotUsedForTooLong(MutexTimeout &tmListTimeout, const std::string &tmToOpen); -/*! \brief Error code definition -*/ - static const int ERROR_PLUGINNOTAVAILABLE = 1002; - static const int ERROR_MEMORYOBJECTISNULL = 1003; - static const int ERROR_BUFFERTOOSMALL = 1004; - static const int ERROR_INVALIDOBJNAME = 1005; - static const int ERROR_MISSINGPARAMETER = 1006; + /*! \brief calcuate total amount of RAM occupied by opened memory files + \returns 0 + */ + size_t CalculateOccupiedRAM(MutexTimeout &tmListTimeout); -/*! \brief Options for proposal sorting -*/ - static const LONG LATESTPROPOSAL_FIRST = 0x00000001; + std::shared_ptr findOpenedMemory(const std::string &memName); - /*! \brief This static method returns a pointer to the TMManager object. - The first call of the method creates the TMManager instance. -*/ - static TMManager* GetInstance(); + int GetMemImportInProcessCount(MutexTimeout &tmListTimeout); - TMManager(){ - T5LOG( T5DEBUG) << "::Ctor of TMManager"; - pluginList = NULL; - pHandleToMemoryList = new std::vector; - refreshPluginList(); - } + /*! \brief Close all open memories + \returns http return code0 if successful or an error code in case of failures + */ + int closeAll(); + + /// MEMORY FACTORY REGION + /*! \brief This class provides factory methods for EqfMemory objects -/* \brief Close a memory - Close the memory object and free all memory related resources. - The memory object is not valid anymore. - \param pMemory pointer to the memory object being closes - \returns 0 when successful or a error return code */ -int closeMemory -( - EqfMemory *pMemory -); + // public: -/*! \brief get error message for the last error - \param pMemory pointer to existing memory object or NULL if not available - \param iLastError the last error number - \param strError the error string returned with - \returns the last error string -*/ -std::string& getLastError( - EqfMemory *pMemory, - int& iLastError, - std::string& strError); - -/*! \brief Copy best matches from one proposal vector into another - and sort the proposals - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced -*/ -void copyBestMatches( - std::vector &SourceProposals, - std::vector &TargetProposals, - int iMaxProposals -); + /*! \brief Error code definition + */ + static const int ERROR_PLUGINNOTAVAILABLE = 1002; + static const int ERROR_MEMORYOBJECTISNULL = 1003; + static const int ERROR_BUFFERTOOSMALL = 1004; + static const int ERROR_INVALIDOBJNAME = 1005; + static const int ERROR_MISSINGPARAMETER = 1006; -/*! \brief Copy best matches from one proposal vector into another - and sort the proposals - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced - \param iMTDisplayFactor factor for the placement of machine matches within the table - \param fExactAndFuzzies switch to control the handling of fuzzy matches when exact matches exist, TRUE = keep fuzzy matches even when exact matches exist - \param lOptions options for the sorting of the proposals -*/ -void copyBestMatches( - std::vector &SourceProposals, - std::vector &TargetProposals, - int iMaxProposals, - int iMTDisplayFactor, - BOOL fExactAndFuzzies, - LONG lOptions = 0 -); + /*! \brief Options for proposal sorting + */ + static const LONG LATESTPROPOSAL_FIRST = 0x00000001; -/*! \brief Insert proposal into proposal vector at the correct position and - remove a proposal with lesser relevance when iMaxPropoals have already been filled - \param NewProposal pointer to proposal being inserted - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced - \param fLastEntry true = this is the last entry in the table - \param iMTDisplayFactor factor for the placement of machine matches within the table - \param lOptions options for the sorting of the proposals + /*! \brief This static method returns a pointer to the TMManager object. + The first call of the method creates the TMManager instance. */ -void insertProposalData( - OtmProposal *newProposal, - std::vector &Proposals, - int iMaxProposals, - BOOL fLastEntry, - int iMTDisplayFactor = -1, - LONG lOptions = 0 -); + static TMManager *GetInstance(); -/*! \brief Get the object name for the memory - \param pMemory pointer to the memory object - \param pszObjName pointer to a buffer for the object name - \param iBufSize size of the object name buffer - \returns 0 when successful or the error code -*/ -int getObjectName( EqfMemory *pMemory, char *pszObjName, int iBufSize ); - -/*! \brief Get the plugin name and the memory name from a memory object name - \param pszObjName pointer to the memory object name - \param pszPluginName pointer to the buffer for the plugin name or - NULL if no plugin name is requested - \param iPluginBufSize size of the buffer for the plugin name - \param pszluginName pointer to the buffer for the plugin name or - NULL if no memory name is requested - \param iNameBufSize size of the buffer for the memory name - \returns 0 when successful or the error code -*/ -int splitObjName( char *pszObjName, char *pszPluginName, int iPluginBufSize, char *pszMemoryName, int iNameBufSize ); - - /*! \brief Get the sort order key for a memory match - \param Proposal reference to a proposal for which the sort key is evaluated - \param iMTDisplayFactor the machine translation display factor, -1 to ignore the factor - \param usContextRanking the context ranking for the proposal - \param fEndIfTable TRUE when this proposal is the last in a proposal table - When there are more proposals available proposals with lesser relevance will be replaced - \returns the proposal sort key -*/ - int getProposalSortKey( OtmProposal &Proposal, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0 ); + TMManager() + { + T5LOG(T5DEBUG) << "::Ctor of TMManager"; + pluginList = NULL; + pHandleToMemoryList = new std::vector; + refreshPluginList(); + } - /*! \brief Get the sort order key for a memory match - \param Proposal reference to a proposal for which the sort key is evaluated - \returns the proposal sort key -*/ -int getProposalSortKey( OtmProposal &Proposal ); - - /*! \brief Get the sort order key for a memory match - \param MatchType match type of the match - \param ProposalType type of the proposal - \param iFuzziness fuzziness of the proposal - \param iMTDisplayFactor the machine translation display factor, -1 to use the system MT display factor - \param usContextRanking the context ranking for the proposal - \param fEndIfTable TRUE when this proposal is the last in a proposal table - When there are more proposals available proposals with lesser relevance will be replaced - \returns the proposal sort key -*/ -int getProposalSortKey( OtmProposal::eMatchType MatchType, OtmProposal::eProposalType ProposalType, int iFuzzyness, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0 ); + /*! \brief Show error message for the last error + \param pszPlugin plugin-name or NULL if not available or memory object name is used + \param pszMemoryName name of the memory causing the problem + memory object name (pluginname + colon + memoryname) + \param pMemory pointer to existing memory object or NULL if not available + \param hwndErrMsg handle of parent window message box + */ + void showLastError( + char *pszPluginName, + char *pszMemoryName, + EqfMemory *pMemory, + HWND hwndErrMsg); + + /*! \brief get error message for the last error + \param pMemory pointer to existing memory object or NULL if not available + \param iLastError the last error number + \param strError the error string returned with + \returns the last error string + */ + std::string &getLastError( + EqfMemory *pMemory, + int &iLastError, + std::string &strError); + + /*! \brief Copy best matches from one proposal vector into another + and sort the proposals + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + */ + void copyBestMatches( + std::vector &SourceProposals, + std::vector &TargetProposals, + int iMaxProposals); + + /*! \brief Copy best matches from one proposal vector into another + and sort the proposals + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + \param iMTDisplayFactor factor for the placement of machine matches within the table + \param fExactAndFuzzies switch to control the handling of fuzzy matches when exact matches exist, TRUE = keep fuzzy matches even when exact matches exist + \param lOptions options for the sorting of the proposals + */ + void copyBestMatches( + std::vector &SourceProposals, + std::vector &TargetProposals, + int iMaxProposals, + int iMTDisplayFactor, + BOOL fExactAndFuzzies, + LONG lOptions = 0); + + /*! \brief Insert proposal into proposal vector at the correct position and + remove a proposal with lesser relevance when iMaxPropoals have already been filled + \param NewProposal pointer to proposal being inserted + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + \param fLastEntry true = this is the last entry in the table + \param iMTDisplayFactor factor for the placement of machine matches within the table + \param lOptions options for the sorting of the proposals + */ + void insertProposalData( + OtmProposal *newProposal, + std::vector &Proposals, + int iMaxProposals, + BOOL fLastEntry, + int iMTDisplayFactor = -1, + LONG lOptions = 0); + + /*! \brief Get the object name for the memory + \param pMemory pointer to the memory object + \param pszObjName pointer to a buffer for the object name + \param iBufSize size of the object name buffer + \returns 0 when successful or the error code + */ + int getObjectName(EqfMemory *pMemory, char *pszObjName, int iBufSize); + + /*! \brief Get the plugin name and the memory name from a memory object name + \param pszObjName pointer to the memory object name + \param pszPluginName pointer to the buffer for the plugin name or + NULL if no plugin name is requested + \param iPluginBufSize size of the buffer for the plugin name + \param pszluginName pointer to the buffer for the plugin name or + NULL if no memory name is requested + \param iNameBufSize size of the buffer for the memory name + \returns 0 when successful or the error code + */ + int splitObjName(char *pszObjName, char *pszPluginName, int iPluginBufSize, char *pszMemoryName, int iNameBufSize); + + /*! \brief Get the sort order key for a memory match + \param Proposal reference to a proposal for which the sort key is evaluated + \param iMTDisplayFactor the machine translation display factor, -1 to ignore the factor + \param usContextRanking the context ranking for the proposal + \param fEndIfTable TRUE when this proposal is the last in a proposal table + When there are more proposals available proposals with lesser relevance will be replaced + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal &Proposal, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0); + + /*! \brief Get the sort order key for a memory match + \param Proposal reference to a proposal for which the sort key is evaluated + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal &Proposal); + + /*! \brief Get the sort order key for a memory match + \param MatchType match type of the match + \param ProposalType type of the proposal + \param iFuzziness fuzziness of the proposal + \param iMTDisplayFactor the machine translation display factor, -1 to use the system MT display factor + \param usContextRanking the context ranking for the proposal + \param fEndIfTable TRUE when this proposal is the last in a proposal table + When there are more proposals available proposals with lesser relevance will be replaced + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal::eMatchType MatchType, OtmProposal::eProposalType ProposalType, int iFuzzyness, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0); /*! \brief Replace a memory with the data from another memory This method bevaves like deleting the replace memory and renaming the @@ -4210,95 +3964,86 @@ int getProposalSortKey( OtmProposal::eMatchType MatchType, OtmProposal::ePropos \param pszPluginName name of plugin of the memory \param pszReplace name of the memory being replaced \param pszReplaceWith name of the memory replacing the pszReplace memory - returns 0 if successful or error return code + returns 0 if successful or error return code */ - int ReplaceMemory - ( - const std::string& strReplace, - const std::string& strReplaceWith, - std::string& outputMsg, - MutexTimeout& tmListTimeout - ); - + int ReplaceMemory( + const std::string &strReplace, + const std::string &strReplaceWith, + std::string &outputMsg, + MutexTimeout &tmListTimeout); private: - /* \brief Get memory plugin with the given plugin name \param pszPlugin plugin-name \returns pointer to plugin or NULL if no memory pluging with the given name is specified */ -OtmPlugin *getPlugin -( - const char *pszPluginName -); + OtmPlugin *getPlugin( + const char *pszPluginName); -/* \brief Refresh internal list of memory plugins -*/ -void refreshPluginList(); + /* \brief Refresh internal list of memory plugins + */ + void refreshPluginList(); /*! \brief search option flags -*/ -#define SEARCH_SOURCE 1 -#define SEARCH_TARGET 2 + */ +#define SEARCH_SOURCE 1 +#define SEARCH_TARGET 2 #define SEARCH_CASEINSENSITIVE 4 #define SEARCH_WHITESPACETOLERANT 8 + /*! \brief Pointer to the list of installed memory plugins + */ + std::vector *pluginList; -/*! \brief Pointer to the list of installed memory plugins -*/ - std::vector *pluginList; - - /*! \brief error code of last error occured -*/ - int iLastError; + /*! \brief error code of last error occured + */ + int iLastError; - /*! \brief error text for last error occured -*/ - std::string strLastError; + /*! \brief error text for last error occured + */ + std::string strLastError; /*! \brief buffer for name of default memory plugin */ char szDefaultMemoryPlugin[512]; -/*! \brief buffer for name of default shared memory plugin -*/ - char szDefaultSharedMemoryPlugin[512]; + /*! \brief buffer for name of default shared memory plugin + */ + char szDefaultSharedMemoryPlugin[512]; -/*! \brief array containing the memory objects referred to by a handle -*/ -std::vector *pHandleToMemoryList; + /*! \brief array containing the memory objects referred to by a handle + */ + std::vector *pHandleToMemoryList; -/*! \brief Buffer for segment text -*/ - CHAR_W m_szSegmentText[MAX_SEGMENT_SIZE+1]; + /*! \brief Buffer for segment text + */ + CHAR_W m_szSegmentText[MAX_SEGMENT_SIZE + 1]; /*! \brief Buffer for search string - */ + */ CHAR_W m_szSearchString[MAX_SEGMENT_SIZE + 1]; - -//end of MEMORY FACTORY region + // end of MEMORY FACTORY region }; - /*! \brief copy the data of a MEMPROPOSAL structure to a OtmProposal object - \param pMemProposal pointer to MEMPROPOSAL structure + \param pMemProposal pointer to MEMPROPOSAL structure \param pOtmProposal pointer to OtmProposal object \returns 0 in any case */ -void copyMemProposalToOtmProposal( PMEMPROPOSAL pProposal, OtmProposal *pOtmProposal ); +void copyMemProposalToOtmProposal(PMEMPROPOSAL pProposal, OtmProposal *pOtmProposal); ////OTMFUNC region /** -* \file tm.h -* -* \brief External header file for OpenTM2 API functions -* -* Copyright Notice: -* -* Copyright (C) 1990-2017, International Business Machines -* Corporation and others. All rights reserved -**/ + * \file tm.h + * + * \brief External header file for OpenTM2 API functions + * + * Copyright Notice: + * + * Copyright (C) 1990-2017, International Business Machines + * Corporation and others. All rights reserved + **/ #ifndef _OTMFUNC_H_INCLUDED #define _OTMFUNC_H_INCLUDED @@ -4308,9 +4053,8 @@ void copyMemProposalToOtmProposal( PMEMPROPOSAL pProposal, OtmProposal *pOtmProp // include EQFPAPI.H for segment definitions #ifndef _EQFPAPI_H_INCLUDED - #include "EQFPAPI.H" -#endif - +#include "EQFPAPI.H" +#endif typedef LONG HSESSION; typedef HSESSION *PHSESSION; @@ -4319,46 +4063,42 @@ typedef int BOOL; typedef struct _REPORTTYPE { - PSZ pszReport; + PSZ pszReport; LONG lRepType; - PSZ pszDescription; + PSZ pszDescription; } REPORTTYPE, *PREPORTTYPE; - typedef struct _REPORTSETTINGS { - PSZ pszCountType; + PSZ pszCountType; BOOL bShow; BOOL bSummary; - PSZ pszRepLayout; + PSZ pszRepLayout; BOOL bShrink; - PSZ pszStatisticType; + PSZ pszStatisticType; BOOL bExProposal; } REPORTSETTINGS, *PREPORTSETTINGS; - typedef struct _FACTSHEET { float lComplexity; float lPayFactor; -} FACTSHEET,*PFACTSHEET; - +} FACTSHEET, *PFACTSHEET; typedef struct _FINALFACTORS { - LONG lUnit; + LONG lUnit; float lCurrFactor; - PSZ pszLocalCurrency; -} FINALFACTORS,*PFINALFACTORS; - + PSZ pszLocalCurrency; +} FINALFACTORS, *PFINALFACTORS; typedef struct _EXTFOLPROP { - CHAR chDrive; - CHAR szTargetLang[MAX_LANG_LENGTH]; - CHAR szRWMemory[MAX_LONGFILESPEC]; - CHAR szROMemTbl[MAX_NUM_OF_READONLY_MDB][MAX_LONGFILESPEC]; - CHAR szDicTbl[ NUM_OF_FOLDER_DICS][MAX_FILESPEC]; + CHAR chDrive; + CHAR szTargetLang[MAX_LANG_LENGTH]; + CHAR szRWMemory[MAX_LONGFILESPEC]; + CHAR szROMemTbl[MAX_NUM_OF_READONLY_MDB][MAX_LONGFILESPEC]; + CHAR szDicTbl[NUM_OF_FOLDER_DICS][MAX_FILESPEC]; } EXTFOLPROP, *PEXTFOLPROP; // // option flags used by the function call interface @@ -4367,54 +4107,53 @@ typedef struct _EXTFOLPROP #define NO_ERROR 0 // location flags for TMs and dictionaries -#define LOCAL_OPT 0x00000001L -#define SHARED_OPT 0x00000002L - -// analysis flags -#define TMMATCH_OPT 0x00000010L -#define ADDTOMEM_OPT 0x00000020L -#define AUTOSUBST_OPT 0x00000040L -#define UNTRANSLATED_OPT 0x00000080L -#define AUTOLAST_OPT 0x00000100L -#define AUTOJOIN_OPT 0x00000200L -#define AUTOCONTEXT_OPT 0x00000400L -#define REDUNDCOUNT_OPT 0x00000800L -#define IGNOREPATH_OPT 0x00001000L -#define ADJUSTLEADWS_OPT 0x00002000L -#define ADJUSTTRAILWS_OPT 0x00004000L -#define RESPECTCRLF_OPT 0x00008000L -#define NOBLANKATSEGEND_OPT 0x00010000L +#define LOCAL_OPT 0x00000001L +#define SHARED_OPT 0x00000002L + +// analysis flags +#define TMMATCH_OPT 0x00000010L +#define ADDTOMEM_OPT 0x00000020L +#define AUTOSUBST_OPT 0x00000040L +#define UNTRANSLATED_OPT 0x00000080L +#define AUTOLAST_OPT 0x00000100L +#define AUTOJOIN_OPT 0x00000200L +#define AUTOCONTEXT_OPT 0x00000400L +#define REDUNDCOUNT_OPT 0x00000800L +#define IGNOREPATH_OPT 0x00001000L +#define ADJUSTLEADWS_OPT 0x00002000L +#define ADJUSTTRAILWS_OPT 0x00004000L +#define RESPECTCRLF_OPT 0x00008000L +#define NOBLANKATSEGEND_OPT 0x00010000L #define NOSUBSTIFIDENTICAL_OPT 0x00020000L -#define SENDTOMT_OPT 0x00080000L -#define STOPATFIRSTEXACT_OPT 0x00100000L +#define SENDTOMT_OPT 0x00080000L +#define STOPATFIRSTEXACT_OPT 0x00100000L // ignore memory proposals which are flagged with a comment -#define IGNORECOMMENTED_OPT 0x00200000L +#define IGNORECOMMENTED_OPT 0x00200000L /** @brief protect text inside XMP sections, MsgNum sections, Meta sections, screen sections, and code block sections */ -#define PROTECTXMPSCREEN_OPT 0x00040000L +#define PROTECTXMPSCREEN_OPT 0x00040000L /** @brief protect text inside XMP sections */ -#define PROTECTXMP_OPT 0x00400000L +#define PROTECTXMP_OPT 0x00400000L /** @brief protect text inside MsgNum sections */ -#define PROTECTMSGNUM_OPT 0x00800000L +#define PROTECTMSGNUM_OPT 0x00800000L /** @brief protect text inside Meta sections */ -#define PROTECTMETA_OPT 0x01000000L +#define PROTECTMETA_OPT 0x01000000L /** @brief protect text inside screen sections */ -#define PROTECTSCREEN_OPT 0x02000000L +#define PROTECTSCREEN_OPT 0x02000000L /** @brief protect text inside CodeBlock sections */ -#define PROTECTCODEBLOCK_OPT 0x04000000L - +#define PROTECTCODEBLOCK_OPT 0x04000000L // folder export and folder import flags -#define WITHDICT_OPT 0x00001000L -#define WITHMEM_OPT 0x00002000L -#define DELETE_OPT 0x00004000L -#define WITHREADONLYMEM_OPT 0x00008000L -#define WITHDOCMEM_OPT 0x00010000L -#define MASTERFOLDER_OPT 0x00020000L -#define XLIFF_OPT 0x00040000L -#define NOMARKUP_UPDATE_OPT 0x00080000L +#define WITHDICT_OPT 0x00001000L +#define WITHMEM_OPT 0x00002000L +#define DELETE_OPT 0x00004000L +#define WITHREADONLYMEM_OPT 0x00008000L +#define WITHDOCMEM_OPT 0x00010000L +#define MASTERFOLDER_OPT 0x00020000L +#define XLIFF_OPT 0x00040000L +#define NOMARKUP_UPDATE_OPT 0x00080000L #define WO_REDUNDANCY_DATA_OPT 0x00100000L // @@ -4422,85 +4161,85 @@ typedef struct _EXTFOLPROP // // do not search in sub-directories for specified document names -#define NOSUBDIRSEARCH_OPT 0x00001000L +#define NOSUBDIRSEARCH_OPT 0x00001000L // // flags for document export and wordcount // // Count replace matches separately option (can only be used in conjunction with TMMATCH_OPT) -#define SEPERATEREPLMATCH_OPT 0x02000000L +#define SEPERATEREPLMATCH_OPT 0x02000000L // create word count without duplicates (R009560) -#define DUPMEMMATCH_OPT 0x01000000L +#define DUPMEMMATCH_OPT 0x01000000L // create word count without duplicates (R009560) -#define DUPLICATE_OPT 0x00010000L +#define DUPLICATE_OPT 0x00010000L // create source word count / export source document -#define SOURCE_OPT 0x00020000L +#define SOURCE_OPT 0x00020000L // create translated/untranslated word count / export target document -#define TARGET_OPT 0x00040000L +#define TARGET_OPT 0x00040000L // export SNOMATCH document -#define SNOMATCH_OPT 0x00080000L +#define SNOMATCH_OPT 0x00080000L // count fuzzy matches (only EqfCountWords) -#define FUZZYMATCH_OPT 0x00080000L +#define FUZZYMATCH_OPT 0x00080000L // // flags for dictionary import merge options // (IGNORE_OPT also used in EqfImportMem to skip invalid segments) // -#define IGNORE_OPT 0x00000020L -#define REPLACE_OPT 0x00000040L -#define COMBINE_OPT 0x00000080L +#define IGNORE_OPT 0x00000020L +#define REPLACE_OPT 0x00000040L +#define COMBINE_OPT 0x00000080L // flags for dictionary import/export in DXT UTF8 format -#define DXT_UTF8_OPT 0x00200000L +#define DXT_UTF8_OPT 0x00200000L // external/internal flags -#define EXTERNAL_OPT 0x01000000L -#define INTERNAL_OPT 0x02000000L +#define EXTERNAL_OPT 0x01000000L +#define INTERNAL_OPT 0x02000000L // flags for build archive TM function EqfArchiveTM -#define USEASFOLDERTM_OPT 0x20000000L +#define USEASFOLDERTM_OPT 0x20000000L #define SOURCESOURCEMEM_OPT 0x40000000L -#define SETMFLAG_OPT 0x80000000L +#define SETMFLAG_OPT 0x80000000L // general flags -#define OVERWRITE_OPT 0x10000000L +#define OVERWRITE_OPT 0x10000000L // characterset flags for dictionary/memory import and export -#define ASCII_OPT 0x00020000L -#define ANSI_OPT 0x00040000L -#define UTF16_OPT 0x00080000L +#define ASCII_OPT 0x00020000L +#define ANSI_OPT 0x00040000L +#define UTF16_OPT 0x00080000L // import in TMX format (UTF-16 and UTF-8) -#define TMX_OPT 0x00100000L +#define TMX_OPT 0x00100000L // export in TMX format -#define TMX_UTF16_OPT 0x00100000L -#define TMX_UTF8_OPT 0x00200000L +#define TMX_UTF16_OPT 0x00100000L +#define TMX_UTF8_OPT 0x00200000L // clean/remove RTF tags -#define CLEANRTF_OPT 0x00400000L +#define CLEANRTF_OPT 0x00400000L // restrict RTF tag removal to tags enclosed in curly braces only -#define INCURLYBRACE_OPT 0x00800000L +#define INCURLYBRACE_OPT 0x00800000L // remove linebreaks in text of translation units -#define TMX_NOCRLF_OPT 0x01000000L +#define TMX_NOCRLF_OPT 0x01000000L // import in XLIFF machine translation (MT) format -#define XLIFF_MT_OPT 0x02000000L +#define XLIFF_MT_OPT 0x02000000L // how to handle unknown markup table when importing memory -#define CANCEL_UNKNOWN_MARKUP_OPT 0x10000000L -#define SKIP_UNKNOWN_MARKUP_OPT 0x20000000L -#define GENERIC_UNKNOWN_MARKUP_OPT 0x40000000L +#define CANCEL_UNKNOWN_MARKUP_OPT 0x10000000L +#define SKIP_UNKNOWN_MARKUP_OPT 0x20000000L +#define GENERIC_UNKNOWN_MARKUP_OPT 0x40000000L // option for EqfImportMemEx only: create a new segment match ID even if the match has one already #define FORCENEWMATCHID_OPT 0x00000004L @@ -4509,29 +4248,29 @@ typedef struct _EXTFOLPROP // export documents in validation format (cannot be used together with the other export options) // export validation document in XML format -#define VALFORMAT_XML_OPT 0x00200000L +#define VALFORMAT_XML_OPT 0x00200000L // export validation document in HML format -#define VALFORMAT_HTML_OPT 0x00400000L -//export validation document in DOC format -#define VALFORMAT_DOC_OPT 0x00800000L -//export validation document in ODT format -#define VALFORMAT_ODT_OPT 0x00100000L -//export validation document in DOCX format -#define VALFORMAT_DOCX_OPT 0x08000000L +#define VALFORMAT_HTML_OPT 0x00400000L +// export validation document in DOC format +#define VALFORMAT_DOC_OPT 0x00800000L +// export validation document in ODT format +#define VALFORMAT_ODT_OPT 0x00100000L +// export validation document in DOCX format +#define VALFORMAT_DOCX_OPT 0x08000000L // flags which can be used together with the export options above -// combine documents into a single file -#define VALFORMAT_COMBINE_OPT 0x01000000L +// combine documents into a single file +#define VALFORMAT_COMBINE_OPT 0x01000000L // include protected segments in the validation document -#define VALFORMAT_PROTSEGS_OPT 0x02000000L +#define VALFORMAT_PROTSEGS_OPT 0x02000000L // export documents as plain XML -#define PLAINXML_OPT 0x04000000L +#define PLAINXML_OPT 0x04000000L // relative path document export options -#define WITHRELATIVEPATH_OPT 0x20000000L -#define WITHOUTRELATIVEPATH_OPT 0x40000000L +#define WITHRELATIVEPATH_OPT 0x20000000L +#define WITHOUTRELATIVEPATH_OPT 0x40000000L // document export/import in internal format #define OPENTM2FORMAT_OPT 0x80000000L @@ -4542,59 +4281,57 @@ typedef struct _EXTFOLPROP // // output type options for EqfCountWords and EqfCreateCountReport // -#define TEXT_OUTPUT_OPT 0x00100000L -#define XML_OUTPUT_OPT 0x00200000L -#define HTML_OUTPUT_OPT 0x00400000L +#define TEXT_OUTPUT_OPT 0x00100000L +#define XML_OUTPUT_OPT 0x00200000L +#define HTML_OUTPUT_OPT 0x00400000L // report IDs for function EqfCreateCountReport -#define HISTORY_REP 1 -#define COUNTING_REP 2 -#define CALCULATING_REP 3 -#define PREANALYSIS_REP 4 -#define REDUNDANCY_REP 5 -#define REDUNDANCYSEGMENT_REP 6 +#define HISTORY_REP 1 +#define COUNTING_REP 2 +#define CALCULATING_REP 3 +#define PREANALYSIS_REP 4 +#define REDUNDANCY_REP 5 +#define REDUNDANCYSEGMENT_REP 6 // report types for function EqfCreateCountReport // types for HISTORY_REPORT -#define BRIEF_SORTBYDATE_REPTYPE 1 -#define BRIEF_SORTBYDOC_REPTYPE 2 -#define DETAIL_REPTYPE 3 +#define BRIEF_SORTBYDATE_REPTYPE 1 +#define BRIEF_SORTBYDOC_REPTYPE 2 +#define DETAIL_REPTYPE 3 // types for COUNTING_REPORT -#define WITHTOTALS_REPTYPE 4 -#define WITHOUTTOTALS_REPTYPE 5 +#define WITHTOTALS_REPTYPE 4 +#define WITHOUTTOTALS_REPTYPE 5 // types for CALCULATING_REPORT, PREANALYSIS_REPORT, REDUNDANCY_REPORT, // and REDUNDANCYSEGMENT_REPORT -#define BASE_REPTYPE 6 -#define BASE_SUMMARY_REPTYPE 7 -#define BASE_SUMMARY_FACTSHEET_REPTYPE 8 -#define SUMMARY_FACTSHEET_REPTYPE 9 -#define FACTSHEET_REPTYPE 10 +#define BASE_REPTYPE 6 +#define BASE_SUMMARY_REPTYPE 7 +#define BASE_SUMMARY_FACTSHEET_REPTYPE 8 +#define SUMMARY_FACTSHEET_REPTYPE 9 +#define FACTSHEET_REPTYPE 10 +// options for EqfCreateITM lType +#define NOANA_TYP 0x1 +#define NOTM_TYP 0x2 +#define PREPARE_TYP 0x4 -//options for EqfCreateITM lType -#define NOANA_TYP 0x1 -#define NOTM_TYP 0x2 -#define PREPARE_TYP 0x4 - -//options for EqfChangeMFlag lAction -#define CLEAR_MMOPT 0 -#define SET_MMOPT 1 +// options for EqfChangeMFlag lAction +#define CLEAR_MMOPT 0 +#define SET_MMOPT 1 // options for non-dde interface (calculating report) -#define BASE_TYP 1 -#define FACT_TYP 2 -#define SUM_TYP 4 -#define BRIEF_SORTBYDATE_TYP 8 -#define BRIEF_SORTBYDOC_TYP 16 -#define DETAIL_TYP 32 - +#define BASE_TYP 1 +#define FACT_TYP 2 +#define SUM_TYP 4 +#define BRIEF_SORTBYDATE_TYP 8 +#define BRIEF_SORTBYDOC_TYP 16 +#define DETAIL_TYP 32 -#define PLAUS_OPT 1 // (Plausibilty check) -#define LOST_OPT 2 // (Lost Data: Force new shipment) -#define LIST_OPT 4 // (List of Documents) +#define PLAUS_OPT 1 // (Plausibilty check) +#define LOST_OPT 2 // (Lost Data: Force new shipment) +#define LIST_OPT 4 // (List of Documents) // option for fuzzy segment search to mark the differences in the output file #define MARKDIFFERENCES_OPT 0x08000000L @@ -4604,75 +4341,73 @@ typedef struct _EXTFOLPROP #define SELECTEDCLASSANDHIGHER_MODE 1 #define ONLYSELECTEDCLASS_MODE 2 - // return when all activitiy has been completed (instead of iterative processing) #define COMPLETE_IN_ONE_CALL_OPT 0x01000000 // non-DDE function interface specific return codes // Info: return code for functions not implemented yet -#define NOTIMPLEMENTED_RC 10001 +#define NOTIMPLEMENTED_RC 10001 // Error: passed session handle is invalid -#define ERROR_INVALID_SESSION_HANDLE 10002 +#define ERROR_INVALID_SESSION_HANDLE 10002 // Notification: current function has not been completed yet // re-call function until a non-CONTINUE_RC // return code is retuned -#define CONTINUE_RC 10003 +#define CONTINUE_RC 10003 // Error: Wrong or conflicting options specified -#define WRONG_OPTIONS_RC 10004 +#define WRONG_OPTIONS_RC 10004 // Error: different function has been called although // current task has not been completed yet -#define LASTTASK_INCOMPLETE_RC 10005 +#define LASTTASK_INCOMPLETE_RC 10005 // Error: Segment has been joined to a previous one and is not // part of the visible document. It cannot be used // in function EqfGetSourceLine -#define SEGMENTISJOINED_RC 10006 +#define SEGMENTISJOINED_RC 10006 // Error: Given segment number is invalid // (probably not contained in document) -#define INVALIDSEGMENT_RC 10007 +#define INVALIDSEGMENT_RC 10007 // Error: Given handle of loaded file is invalid -#define INVALIDFILEHANDLE_RC 10008 +#define INVALIDFILEHANDLE_RC 10008 // Error: There is no segment matching the given position -#define NOMATCHINGSEGMENT_RC 10009 +#define NOMATCHINGSEGMENT_RC 10009 // Info: End if data reached -#define ENDREACHED_RC 10010 +#define ENDREACHED_RC 10010 // Info: Time out occurend (eceeded given search time) -#define TIMEOUT_RC 10011 - -#define VAL_COMBINE_OPT 0x00000001L -#define VAL_PRESERVE_LINKS_OPT 0x00000002L -#define VAL_MAN_EXACTMATCH_OPT 0x00000004L -#define VAL_REMOVE_INLINE_OPT 0x00000008L -#define VAL_TRANSTEXT_ONLY_OPT 0x00000010L -#define VAL_INCLUDE_COUNT_OPT 0x00000020L -#define VAL_INCLUDE_MATCH_OPT 0x00000040L -#define VAL_MISMATCHES_ONLY_OPT 0x00000080L - -#define VAL_AUTOSUBST_OPT 0x00000100L -#define VAL_MOD_AUTOSUBST_OPT 0x00000200L -#define VAL_EXACT_OPT 0x00000400L -#define VAL_MOD_EXACT_OPT 0x00000800L -#define VAL_GLOBAL_MEM_OPT 0x00001000L -#define VAL_MACHINE_OPT 0x00002000L -#define VAL_FUZZY_OPT 0x00004000L -#define VAL_NEW_OPT 0x00008000L -#define VAL_NOT_TRANS_OPT 0x00010000L -#define VAL_PROTECTED_OPT 0x00020000L -#define VAL_REPLACE_OPT 0x00040000L - -#define VAL_VALIDATION_OPT 0x01000000L -#define VAL_PROOFREAD_OPT 0x02000000L - +#define TIMEOUT_RC 10011 + +#define VAL_COMBINE_OPT 0x00000001L +#define VAL_PRESERVE_LINKS_OPT 0x00000002L +#define VAL_MAN_EXACTMATCH_OPT 0x00000004L +#define VAL_REMOVE_INLINE_OPT 0x00000008L +#define VAL_TRANSTEXT_ONLY_OPT 0x00000010L +#define VAL_INCLUDE_COUNT_OPT 0x00000020L +#define VAL_INCLUDE_MATCH_OPT 0x00000040L +#define VAL_MISMATCHES_ONLY_OPT 0x00000080L + +#define VAL_AUTOSUBST_OPT 0x00000100L +#define VAL_MOD_AUTOSUBST_OPT 0x00000200L +#define VAL_EXACT_OPT 0x00000400L +#define VAL_MOD_EXACT_OPT 0x00000800L +#define VAL_GLOBAL_MEM_OPT 0x00001000L +#define VAL_MACHINE_OPT 0x00002000L +#define VAL_FUZZY_OPT 0x00004000L +#define VAL_NEW_OPT 0x00008000L +#define VAL_NOT_TRANS_OPT 0x00010000L +#define VAL_PROTECTED_OPT 0x00020000L +#define VAL_REPLACE_OPT 0x00040000L + +#define VAL_VALIDATION_OPT 0x01000000L +#define VAL_PROOFREAD_OPT 0x02000000L class IMPORTMEMORYDATA; /*! \brief Import a Translation Memory @@ -4680,7 +4415,7 @@ class IMPORTMEMORYDATA; \param pszMemName name of the Translation Memory \param pszInFile fully qualified name of the input file \param lOptions options for Translation Memory import - one of the format options + one of the format options - TMX_OPT import in TMX format - XLIFF_MT_OPT import in XLIFF format - UTF16_OPT import in the EXP format (Unicode UTF-16 encoded) @@ -4694,22 +4429,20 @@ class IMPORTMEMORYDATA; . additional options which can be used - CLEANRTF_OPT to remove RTF inline tags (only in combination with TMX_OPT) - - IGNORE_OPT ignore invalid segments and continue the import + - IGNORE_OPT ignore invalid segments and continue the import \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfImportMem -( - IMPORTMEMORYDATA* pData, - //std::shared_ptr hSession, // mand: Eqf session handle - //PSZ pszInFile, // mand: fully qualified name of input file - LONG lOptions // opt: options for Translation Memory import - // @Import Mode: TMX_OPT{CLEANRTF_OPT}, XLIFF_MT_OPT,UTF16_OPT,ANSI_OPT,ASCII_OPT(default) - // @Markup Table Handling: CANCEL_UNKNOWN_MARKUP_OPT(default), SKIP_UNKNOWN_MARKUP_OPT, GENERIC_UNKNOWN_MARKUP_OPT - // @Other: {CLEANRTF_OPT,IGNORE_OPT} - -); +USHORT EqfImportMem( + IMPORTMEMORYDATA *pData, + // std::shared_ptr hSession, // mand: Eqf session handle + // PSZ pszInFile, // mand: fully qualified name of input file + LONG lOptions // opt: options for Translation Memory import + // @Import Mode: TMX_OPT{CLEANRTF_OPT}, XLIFF_MT_OPT,UTF16_OPT,ANSI_OPT,ASCII_OPT(default) + // @Markup Table Handling: CANCEL_UNKNOWN_MARKUP_OPT(default), SKIP_UNKNOWN_MARKUP_OPT, GENERIC_UNKNOWN_MARKUP_OPT + // @Other: {CLEANRTF_OPT,IGNORE_OPT} +); /*! \brief Organize a Translation Memory \param hSession the session handle returned by the EqfStartSession call @@ -4717,38 +4450,33 @@ USHORT EqfImportMem \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfOrganizeMem -( - HSESSION hSession, // mand: Eqf session handle - std::shared_ptr pMem, // mand: name of Translation Memory - long& reorgSegCount, - long& invSegCount -); +USHORT EqfOrganizeMem( + HSESSION hSession, // mand: Eqf session handle + std::shared_ptr pMem, // mand: name of Translation Memory + long &reorgSegCount, + long &invSegCount); /*! \brief Start a OpenTM2 API call session \param phSession pointer to a buffer receiving the session handle \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfStartSession -( - PHSESSION phSession // mand: ptr to callers Eqf session handle variable +USHORT EqfStartSession( + PHSESSION phSession // mand: ptr to callers Eqf session handle variable ); - /*! \brief Get information about the last occured error \param hSession the session handle returned by the EqfStartSession call \param pusRC pointer to the buffer for last return code - \param pszMsgBuffer pointer to a buffer receiving the error message text + \param pszMsgBuffer pointer to a buffer receiving the error message text \param usBufSize size of message buffer in number of bytes \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetLastError -( - HSESSION hSession, // Eqf session handle - PUSHORT pusRC, // ptr to buffer for last return code - PSZ pszMsgBuffer, // ptr to buffer receiving the message - USHORT usBufSize // size of message buffer in bytes +USHORT EqfGetLastError( + HSESSION hSession, // Eqf session handle + PUSHORT pusRC, // ptr to buffer for last return code + PSZ pszMsgBuffer, // ptr to buffer receiving the message + USHORT usBufSize // size of message buffer in bytes ); /*! \brief Get information about the last occured error (in UTF16 encoding) \param hSession the session handle returned by the EqfStartSession call @@ -4757,94 +4485,92 @@ USHORT EqfGetLastError \param usBufSize size of message buffer in number of characters \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetLastErrorW -( - HSESSION hSession, // Eqf session handle - PUSHORT pusRC, // ptr to buffer for last return code - wchar_t *pszMsgBuffer, // ptr to buffer receiving the message - USHORT usBufSize // size of message buffer in bytes +USHORT EqfGetLastErrorW( + HSESSION hSession, // Eqf session handle + PUSHORT pusRC, // ptr to buffer for last return code + wchar_t *pszMsgBuffer, // ptr to buffer receiving the message + USHORT usBufSize // size of message buffer in bytes ); /*! \brief Structure containing the segment or memory match information This structure is used by the EqfGetMatchLevel API for the segment from the document and for the memory proposal */ -//#pragma pack( push, TM2StructPacking, 1 ) +// #pragma pack( push, TM2StructPacking, 1 ) typedef struct _EQFSEGINFO { - WCHAR szSource[EQF_SEGLEN]; // segment source text (UTF-16 encoded) - WCHAR szTarget[EQF_SEGLEN]; // segment target text (UTF-16 encoded) (empty for document segment) - LONG lSegNumber; // segment number - CHAR szDocument[MAX_LONGFILESPEC]; // name of document - CHAR szSourceLanguage[MAX_LANG_LENGTH]; // source language of segment - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // target language of segment - CHAR szMarkup[MAX_FILESPEC]; // markup of segment + WCHAR szSource[EQF_SEGLEN]; // segment source text (UTF-16 encoded) + WCHAR szTarget[EQF_SEGLEN]; // segment target text (UTF-16 encoded) (empty for document segment) + LONG lSegNumber; // segment number + CHAR szDocument[MAX_LONGFILESPEC]; // name of document + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // source language of segment + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // target language of segment + CHAR szMarkup[MAX_FILESPEC]; // markup of segment } EQFSEGINFO, *PEQFSEGINFO; -//#pragma pack( pop, TM2StructPacking ) +// #pragma pack( pop, TM2StructPacking ) // // options for EqfGetMatchLevel API // // suppress generic inline tag replacement -#define NO_GENERIC_INLINETAG_REPL_OPT 0x0001 +#define NO_GENERIC_INLINETAG_REPL_OPT 0x0001 // use generic inline tag replacement function -#define USE_GENERIC_INLINETAG_REPL_OPT 0x0002 +#define USE_GENERIC_INLINETAG_REPL_OPT 0x0002 /*! \brief State/type of matches used by EqfGetMatchLevel API -*/ + */ typedef enum _EQFMATCHSTATE { - REPLACE_MATCHSTATE, // replace match - FUZZYREPLACE_MATCHSTATE, // fuzzy replace match - FUZZY_MATCHSTATE, // fuzzy match - NONE_MATCHSTATE, // no match - EXACT_MATCHSTATE, // exact match - EXACTEXACT_MATCHSTATE // exact-exact match (match from same document and same segment) -} EQFMATCHSTATE, *PEQFMATCHSTATE; + REPLACE_MATCHSTATE, // replace match + FUZZYREPLACE_MATCHSTATE, // fuzzy replace match + FUZZY_MATCHSTATE, // fuzzy match + NONE_MATCHSTATE, // no match + EXACT_MATCHSTATE, // exact match + EXACTEXACT_MATCHSTATE // exact-exact match (match from same document and same segment) +} EQFMATCHSTATE, + *PEQFMATCHSTATE; // options fpr EqfCleanMatch API call // create internal memory -#define CLEANMEM_INTERNAL_MEMORY_OPT 0x00000001 +#define CLEANMEM_INTERNAL_MEMORY_OPT 0x00000001 // create external memory -#define CLEANMEM_EXTERNAL_MEMORY_OPT 0x00000002 +#define CLEANMEM_EXTERNAL_MEMORY_OPT 0x00000002 // return when all activitiy has been completed (instead of iterative processing) #define CLEANMEM_COMPLETE_IN_ONE_CALL_OPT 0x01000000 #define COMPLETE_IN_ONE_CALL_OPT 0x01000000 - // activate logging (will append log info to the log file \EQF\LOGS\CLEANMEMORY.LOG) -#define CLEANMEM_LOGGING_OPT 0x00010000 +#define CLEANMEM_LOGGING_OPT 0x00010000 // if specified only the best match will be written to the output memory // if not specified the best 3 fuzzy matches will be written to the output memory -#define CLEANMEM_BESTMATCH_OPT 0x00020000 +#define CLEANMEM_BESTMATCH_OPT 0x00020000 // merge into existing internal memory -#define CLEANMEM_MERGE_OPT 0x20000000 +#define CLEANMEM_MERGE_OPT 0x20000000 // keep duplicate exact matches and fuzzy matches in the memory -#define CLEANMEM_KEEP_DUPS_OPT 0x40000000 +#define CLEANMEM_KEEP_DUPS_OPT 0x40000000 // rename modes -#define RENAME_FOLDER 1 -#define RENAME_MEMORY 2 -#define RENAME_DICTIONARY 4 +#define RENAME_FOLDER 1 +#define RENAME_MEMORY 2 +#define RENAME_DICTIONARY 4 // options -#define ADJUSTREFERENCES_OPT 0x04000000 +#define ADJUSTREFERENCES_OPT 0x04000000 // constants for API call EqfGetShortName -#define FOLDER_OBJ 1 -#define MEMORY_OBJ 2 -#define DICT_OBJ 3 -#define DOCUMENT_OBJ 4 - +#define FOLDER_OBJ 1 +#define MEMORY_OBJ 2 +#define DICT_OBJ 3 +#define DOCUMENT_OBJ 4 /*! \brief Search segments having fuzzy memory proposals \param hSession the session handle returned by the EqfStartSession call @@ -4866,35 +4592,29 @@ typedef enum _EQFMATCHSTATE \param pSearchKey pointer to a MemProposal structure containing the searched criteria \param *piNumOfProposals pointer to the number of requested memory proposals, will be changed on return to the number of proposals found \param pProposals pointer to a array of MemProposal structures receiving the search results - \param lOptions processing options + \param lOptions processing options \returns 0 if successful or an error code in case of failures */ -USHORT EqfQueryMem -( - HSESSION hSession, - EqfMemory* lHandle, - PMEMPROPOSAL pSearchKey, - int *piNumOfProposals, - PMEMPROPOSAL pProposals, - LONG lOptions -); - +USHORT EqfQueryMem( + HSESSION hSession, + EqfMemory *lHandle, + PMEMPROPOSAL pSearchKey, + int *piNumOfProposals, + PMEMPROPOSAL pProposals, + LONG lOptions); /*! \brief Update a segment in the memory \param hSession the session handle returned by the EqfStartSession call \param lHandle handle of a previously opened memory \param pNewProposal pointer to an MemProposal structure containing the segment data - \param lOptions processing options + \param lOptions processing options \returns 0 if successful or an error code in case of failures */ -USHORT EqfUpdateMem -( - HSESSION hSession, - EqfMemory* lHandle, - PMEMPROPOSAL pNewProposal, - LONG lOptions -); - +USHORT EqfUpdateMem( + HSESSION hSession, + EqfMemory *lHandle, + PMEMPROPOSAL pNewProposal, + LONG lOptions); /*! \brief Get the OpenTM2 language name for a ISO language identifier \param hSession the session handle returned by the EqfStartSession call @@ -4902,14 +4622,11 @@ USHORT EqfUpdateMem \param pszOpenTM2Lang buffer for the OpenTM2 language name \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetOpenTM2Lang -( - HSESSION hSession, - PSZ pszISOLang, - PSZ pszOpenTM2Lang, - bool* pfPrefered = nullptr -); - +USHORT EqfGetOpenTM2Lang( + HSESSION hSession, + PSZ pszISOLang, + PSZ pszOpenTM2Lang, + bool *pfPrefered = nullptr); /*! \brief Get the ISO language identifier for a OpenTM2 language name \param hSession the session handle returned by the EqfStartSession call @@ -4917,16 +4634,13 @@ USHORT EqfGetOpenTM2Lang \param pszISOLang pointer to a buffer for the ISO language identifier \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetIsoLang -( - HSESSION hSession, - PSZ pszOpenTM2Lang, - PSZ pszISOLang -); +USHORT EqfGetIsoLang( + HSESSION hSession, + PSZ pszOpenTM2Lang, + PSZ pszISOLang); #endif - -///end of OTMFUNC region +/// end of OTMFUNC region #endif //_tm_h_included_ \ No newline at end of file diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index eeb58903..a801c01e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,17 +1,29 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) -SET(APP_VERSION_MAJOR 6) -SET(APP_VERSION_MINOR 107) +SET(APP_VERSION_MAJOR 7) +SET(APP_VERSION_MINOR 29) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) + +#set(CMAKE_C_COMPILER "clang") +#set(CMAKE_CXX_COMPILER "clang++") + SET(APP_VERSION "${APP_VERSION_GLOBAL}.${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}") project(T5MEMORY VERSION ${APP_VERSION}) +#set(CMAKE_BUILD_TYPE RelWithDebInfo) +set(CMAKE_BUILD_TYPE Debug) + #add_compile_options(-Wall -Wextra -Wpedantic) add_compile_options( +# -O3 +# -W +# -Wunused-function +# -Wno-attributes +# -Wno-write-strings # -Wall # -Wundef # -Wuninitialized @@ -31,9 +43,9 @@ add_compile_options( add_definitions(-w) #add_definitions(-Wall -Wextra) -# Set the compilers to gcc-10 and g++-10 -#set(CMAKE_C_COMPILER "/usr/bin/gcc-10") -#set(CMAKE_CXX_COMPILER "/usr/bin/g++-10") +# Set the compilers to gcc-11 and g++-11 +#set(CMAKE_C_COMPILER "/usr/bin/gcc-11") +#set(CMAKE_CXX_COMPILER "/usr/bin/g++-11") # Define an option for selecting the sanitizer # Define an option to enable AddressSanitizer @@ -41,6 +53,8 @@ option(ENABLE_ASAN "Enable AddressSanitizer" OFF) option(ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF) # Sanitizer options +#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") # Optionally enable ASan @@ -52,6 +66,8 @@ if(ENABLE_UBSAN) add_definitions(-DUBSAN_ENABLED) endif() + + # Apply the chosen sanitizer flags if ENABLE_ASAN is ON if(ENABLE_ASAN) message("AddressSanitizer is enabled") @@ -73,7 +89,18 @@ endif() -set(CMAKE_BUILD_TYPE Debug) +if(CMAKE_CXX_FLAGS MATCHES "-O0") + add_definitions(-DOPT_LEVEL=0) +elseif(CMAKE_CXX_FLAGS MATCHES "-O1") + add_definitions(-DOPT_LEVEL=1) +elseif(CMAKE_CXX_FLAGS MATCHES "-O2") + add_definitions(-DOPT_LEVEL=2) +elseif(CMAKE_CXX_FLAGS MATCHES "-O3") + add_definitions(-DOPT_LEVEL=3) +else() + add_definitions(-DOPT_LEVEL=-1) # unknown or other +endif() + #include(GNUInstallDirs) message(STATUS "T5MEMORY CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}") @@ -145,7 +172,6 @@ set(T5MEMORY_Source_Files "${CORE_DIR}/utilities/EqfPluginWrapper.cpp" "${CORE_DIR}/utilities/EQFPRO00.cpp" "${CORE_DIR}/utilities/EQFSETUP.cpp" - "${CORE_DIR}/utilities/EQFTADIT.cpp" "${CORE_DIR}/utilities/EQFUTDOS.cpp" "${CORE_DIR}/utilities/EQFUTERR.cpp" "${CORE_DIR}/utilities/EQFUTFIL.cpp" diff --git a/source/RestAPI/OTMMSJSONFactory.cpp b/source/RestAPI/OTMMSJSONFactory.cpp index 33fa9004..2332c266 100755 --- a/source/RestAPI/OTMMSJSONFactory.cpp +++ b/source/RestAPI/OTMMSJSONFactory.cpp @@ -13,6 +13,7 @@ #include "opentm2/core/utilities/EncodingHelper.h" #include +#include // Function to extract JSON objects from a JSON array string std::vector JSONFactory::extractJsonObjects(const std::string& jsonArray, const std::string& arrayKey, std::string& errMsg) { @@ -172,13 +173,27 @@ JSONFactory* JSONFactory::getInstance() return( 0 ); } +/* + std::wstring convertToWCharSafe( const std::string& str ) + { + std::wstring_convert, wchar_t> safe_converter; + try { + std::wstring w = safe_converter.from_bytes(str); + } catch (const std::range_error&) { + std::cerr << "Invalid UTF-8 detected. Replacing with placeholder.\n"; + std::wstring w = L"[INVALID_UTF8]"; + } + return w; +}//*/ + + /*! \brief convert a ASCII std::string to a UTF16 std::wstring \param strUTF8String string in UTF8 encoding \returns string converted to UTF16 */ std::wstring convertToWChar( const std::string& strASCII ) { - std::wstring strUTF16 = EncodingHelper::convertToWChar(strASCII.c_str()); + std::wstring strUTF16 = EncodingHelper::convertToWChar(strASCII); return strUTF16; } @@ -825,11 +840,7 @@ int JSONFactory::extractString } // convert to UTF8 - //int iUTF8Len; - //iUTF8Len = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, 0, 0, 0, 0 ); - //std::string strUTF8( iUTF8Len, '\0' ); - //WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, &strUTF8[0], iUTF8Len, 0, 0 ); - std::string strUTF8 = EncodingHelper::convertToUTF8(strUTF16.c_str()); + std::string strUTF8 = EncodingHelper::convertToUTF8(strUTF16); // add UTF8 character to target string for ( int i = 0; i < strUTF8.length(); i++ ) string[iTargetPos++] = strUTF8[i]; } diff --git a/source/RestAPI/OTMMSJSONFactory.h b/source/RestAPI/OTMMSJSONFactory.h index d9bbfd1d..bebaae33 100755 --- a/source/RestAPI/OTMMSJSONFactory.h +++ b/source/RestAPI/OTMMSJSONFactory.h @@ -44,7 +44,7 @@ class JSONFactory typedef struct _JSONPARSECONTROL { - wchar_t szName[40]; // name of the parameter + wchar_t szName[240]; // name of the parameter PARMTYPE type; // type of the parameter void *pvValue; // pointer to a buffer receiving the value int iBufferLen; // size of the buffer in number of characters diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index b93f8843..6342f312 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -56,46 +56,14 @@ void importMemoryProcess( void *pvData ); OtmMemoryServiceWorker* OtmMemoryServiceWorker::getInstance() { static OtmMemoryServiceWorker _instance; - return &_instance; } -/*! \brief Data area for the processing of the openMemory function -*/ -typedef struct _OPENMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} OPENMEMORYDATA, *POPENMEMORYDATA; - -/*! \brief Data area for the processing of the createMemory function -*/ -typedef struct _CLOSEMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} CLOSEMEMORYDATA, *PCLOSEMEMORYDATA; - - - - - -int OtmMemoryServiceWorker::init(){ - return verifyAPISession(); -} - - /*! \brief Verify OpenTM2 API session - \returns 0 if successful or an error code in case of failures - */ -int OtmMemoryServiceWorker::verifyAPISession() -{ - if ( this->hSession != 0 ){ - return( 0 ); - } +OtmMemoryServiceWorker::OtmMemoryServiceWorker(){ T5LOG( T5DEBUG) << "Initializing API Session"; this->iLastRC = EqfStartSession( &(this->hSession) ); - T5LOG( T5DEBUG) << "[verifyAPISession] EqfStartSession ret: " << this->iLastRC; + T5LOG( T5DEBUG) << "EqfStartSession ret: " << this->iLastRC; if ( this->iLastRC != 0 ) { T5LOG(T5ERROR) << "OpenTM2 API session could not be started, the return code is" << this->iLastRC ; @@ -105,18 +73,15 @@ int OtmMemoryServiceWorker::verifyAPISession() xercesc::XMLPlatformUtils::Initialize(); } catch (const xercesc::XMLException& toCatch) { - toCatch; + //toCatch; T5LOG(T5ERROR) << ":: cant init Xercesc"; - return( ERROR_NOT_READY ); + throw; + //return( ERROR_NOT_READY ); } } - - return( this->iLastRC ); } - - /*! \brief convert a long time value to a date time string in the form YYYY-MM-DD HH:MM:SS \param lTime long time value \param pszDateTime buffer receiving the converted date and time @@ -189,10 +154,6 @@ int convertDateTimeStringToLong( char *pszDateTime, PLONG plTime ) return( 0 ); } - - - - template void AddObjToJson(std::stringstream& ss, const char* key, T value, bool fAddSeparator){ ss << "{ \"" << key << "\" : "; @@ -211,15 +172,6 @@ void AddObjToJson(std::stringstream& ss, const char* key, T value, bool fAddSepa ss << "\n"; } -/*! \brief Saves all open and modified memories - \returns http return code0 if successful or an error code in case of failures - -int OtmMemoryServiceWorker::closeAll(){ - int rc = TMManager::GetInstance()->closeAll(); - return rc; -}//*/ - - std::string printTime(time_t time){ char buff[255]; @@ -241,39 +193,6 @@ it is up to the caller to free this area using free() \param strError string receiving any error message text \returns 0 is sucessfull or a return code */ -int FilesystemHelper::EncodeFileInBase64( char *pszFile, char **ppStringData, std::string &strError ) -{ - int iRC = 0; - - auto file = FilesystemHelper::OpenFile(pszFile, "r", false); - size_t fSize = FilesystemHelper::GetFileSize(file); - int bytesRead = 0; - if(fSize == 0){ - - }else{ - unsigned char* pData = new unsigned char[fSize]; - FilesystemHelper::ReadFile(file, pData, fSize, bytesRead); - FilesystemHelper::CloseFile(file); - - if(bytesRead != fSize){ - - }else{ - std::string encodedData; - EncodingHelper::Base64Encode(pData, fSize, encodedData); - if(encodedData.empty()) - { - iRC = GetLastError(); - strError = "encoding of BASE64 data failed"; - T5LOG(T5ERROR) << "encodeBase64ToFile()::ecnoding of BASE64 data failed, iRC = " << iRC; - return( iRC ); - } - } - - // cleanup - delete[] pData ; - } - return( iRC ); -} /*! \brief convert a BASE64 encoded string to a binary file \param pStringData pointer to the BASE64 data @@ -286,7 +205,6 @@ int FilesystemHelper::DecodeBase64ToFile( const char *pStringData, const char *p int iRC = 0; // get decoded length of data - DWORD dwDecodedLength = 0; std::string sData(pStringData); unsigned char* pData = NULL; int pDataSize = 0; @@ -319,7 +237,7 @@ int FilesystemHelper::WriteToFile( const std::string& pszFile, const std::string return( -1); } - auto file = FilesystemHelper::OpenFile(pszFile.c_str(), "w+", false); + auto file = FilesystemHelper::OpenFile(pszFile, "w+", false); FilesystemHelper::WriteToFile(file, &Data[0], Data.size()); FilesystemHelper::CloseFile(file); @@ -355,9 +273,13 @@ void importMemoryProcess( void* pvData ) pData->szError[0] = 0; if(!pData->mem->importDetails){ - pData->mem->importDetails = new ImportStatusDetails; + pData->mem->importDetails = std::make_unique(); + } + + { + std::scoped_lock l(pData->mem->importDetails->importDetailsMutex); + pData->mem->importDetails->inclosingTagsBehaviour = pData->inclosingTagsBehaviour; } - pData->mem->importDetails->inclosingTagsBehaviour = pData->inclosingTagsBehaviour; int iRC =(int)EqfImportMem( pData, TMX_OPT | COMPLETE_IN_ONE_CALL_OPT); @@ -375,9 +297,9 @@ void importMemoryProcess( void* pvData ) // cleanup if(pData->fDeleteTmx && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG) == false){ //for DEBUG and DEVELOP modes leave file in fs std::string tempFile = pData->szInFile; - FilesystemHelper::DeleteFile( tempFile.c_str(), true ); + FilesystemHelper::DeleteFile( tempFile, true ); tempFile += ".Temp"; - FilesystemHelper::DeleteFile(tempFile.c_str(), true); + FilesystemHelper::DeleteFile(tempFile, true); } #ifdef TIME_MEASURES @@ -424,7 +346,7 @@ void reorganizeMemoryProcess( void* pvData ) pRIDA->pMem->setActiveRequestCommand(COMMAND::REORGANIZE_MEM); int _rc_ = 0; - pRIDA->pMem->usAccessMode |= ASD_ORGANIZE; + pRIDA->pMem->state->usAccessMode |= ASD_ORGANIZE; while((_rc_ == NO_ERROR || _rc_ == CONTINUE_RC) && !pData->fComplete ) { // continue current organize process @@ -441,7 +363,6 @@ void reorganizeMemoryProcess( void* pvData ) case MEM_END_ORGANIZE: pData->fComplete = TRUE; EQFMemOrganizeEnd( pCommArea); - if ( pRIDA->pszNameList ) UtlAlloc( (PVOID *)pRIDA->pszNameList, 0L, 0L, NOMSG ); break; } /* endswitch */ _rc_ = pRIDA->usRC; @@ -456,7 +377,6 @@ void reorganizeMemoryProcess( void* pvData ) } // update memory status if(pRIDA && pRIDA->pMem){ - std::string strError; pRIDA->pMem->reorganizeDone(_rc_, pData->szError ); pRIDA->pMem.reset(); pRIDA->memRef.reset(); diff --git a/source/RestAPI/OtmMemoryServiceWorker.h b/source/RestAPI/OtmMemoryServiceWorker.h index 9280c0cc..811b7460 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.h +++ b/source/RestAPI/OtmMemoryServiceWorker.h @@ -16,9 +16,7 @@ #include #include #include "ProxygenStats.h" -//#include "EQFFUNCI.H" #include "tm.h" -#include "requestdata.h" @@ -30,36 +28,22 @@ #define SHUTDOWN_CALLED_FROM_MAIN -2 class OtmMemoryServiceWorker /*! \brief This class provides translation memory related services*/ -{ +{ + OtmMemoryServiceWorker(); public: /*! \brief This static method returns a pointer to the OtmMemoryService object. The first call of the method creates the OtmMemoryService instance. */ static OtmMemoryServiceWorker* getInstance(); - - int init(); - - - /* change the status of a memory in our memory list at the end of an import - \param pszMemory name of the memory - \param - */ - void importDone( char *pszMemory, int iRC, char *pszError ); - - //std::unique_ptr replaceString(std::wstring&& src_data, std::wstring&& trg_data, std::wstring&& req_data, int* rc); - /*! \brief Verify OpenTM2 API session - \returns 0 if successful or an error code in case of failures - */ - int verifyAPISession(); + /*! \brief OpenTM2 API session handle*/ HSESSION hSession = 0; + /*! \brief Last return code*/ int iLastRC; -/*! \brief Buffer for last error message*/ - wchar_t szLastError[4000]; }; diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index 5b0f4e03..e5f55c79 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -14,7 +14,6 @@ #include "../opentm2/core/utilities/Property.h" #include "ProxygenStats.h" -#include "OtmMemoryServiceWorker.h" #include #include "LogWrapper.h" #include "lowlevelotmdatastructs.h" @@ -33,7 +32,6 @@ DEFINE_bool(request_number, namespace ProxygenService { ProxygenHandler::ProxygenHandler() { - TMManager::GetInstance()->fWriteRequestsAllowed = true; } @@ -45,17 +43,29 @@ void ProxygenHandler::onRequest(std::unique_ptr req) noexcept { //time(&startingTime); #endif - builder = new ResponseBuilder(downstream_); + builder = std::make_unique(downstream_); auto methodStr = req->getMethodString (); - auto method = req->getMethod (); + auto method = req->getMethod (); + + // Client address + auto& addr = req->getClientAddress(); + auto ipStr = addr.getAddressStr(); // returns std::string with IP + auto port = addr.getPort(); // client port (uint16_t) + + if(!pRequest){ - pRequest = new UnknownRequestData(); + pRequest = std::make_unique(); + } + if(COMMAND::UNKNOWN_COMMAND == pRequest->command){ + UnknownRequestData* unknownRequest = (UnknownRequestData*)pRequest.get(); + unknownRequest->methodStr = methodStr; + unknownRequest->srcAddress = ipStr + ":"s + toStr(port); } + pRequest->strUrl = req->getURL () ; auto path = req->getPath () ; auto queryString = req->getQueryString () ; auto headers = req->getHeaders(); - pMemService = OtmMemoryServiceWorker::getInstance(); auto command = pRequest->command; headers_ = std::move(req); @@ -235,7 +245,6 @@ void ProxygenHandler::onEOM() noexcept { if(!pRequest->_rest_rc_){ std::string truncatedInput = pRequest->strBody.size() > 3000 ? pRequest->strBody.substr(0, 3000) : pRequest->strBody; T5Logger::GetInstance()->SetBodyBuffer(", with body = \"" + truncatedInput +"\""); - } } @@ -250,30 +259,16 @@ void ProxygenHandler::onUpgrade(UpgradeProtocol /*protocol*/) noexcept { } void ProxygenHandler::requestComplete() noexcept { - //ResetLogBuffer(); - if(pRequest){ - //TimedMutexGuard l(pRequest->request_mutex); - //while(pRequest->fRunning){}; - delete pRequest; - pRequest = nullptr; - } + pRequest.reset(); delete this; } void ProxygenHandler::onError(ProxygenError /*err*/) noexcept { - if(pRequest){ - //while(pRequest->fRunning){}; - //TimedMutexGuard l(pRequest->request_mutex); - delete pRequest; - pRequest = nullptr; - } + + pRequest.reset(); delete this; } - - - - //#define LOG_SEND_RESPONSE 1 void ProxygenHandler::sendResponse()noexcept{ @@ -301,7 +296,9 @@ void ProxygenHandler::sendResponse()noexcept{ case 423: { //responseText - pRequest->outputMessage = "{\n\"msg\": \"WRITE REQUESTS ARE NOT ALLOWED\",\n\"rc\" = 423\n}\n" ; + if(pRequest->outputMessage.empty()){ + pRequest->outputMessage = "{\n\"msg\": \"WRITE REQUESTS ARE NOT ALLOWED\",\n\"rc\" = 423\n}\n" ; + } responseText = "Locked"; break; } @@ -352,7 +349,7 @@ void ProxygenHandler::sendResponse()noexcept{ } if(COMMAND::EXPORT_MEM_TMX_STREAM == pRequest->command && pRequest->isSuccessful()){ // Add headers to the response std::stringstream contDisposition; - ExportRequestData* exp_request = (ExportRequestData*) pRequest; + ExportRequestData* exp_request = (ExportRequestData*) pRequest.get(); builder->header("Content-Type", "application/octet-stream"); contDisposition << "attachment; filename=\"" << pRequest->strMemName << ".tmx\""; @@ -379,7 +376,7 @@ void ProxygenHandler::sendResponse()noexcept{ ) ){ // Respond with OK status - pRequest->sendStreamFile(builder); + pRequest->sendStreamFile(builder.get()); }else if(pRequest->outputMessage.size()){ builder->body(pRequest->outputMessage); builder->sendWithEOM(); @@ -389,6 +386,7 @@ void ProxygenHandler::sendResponse()noexcept{ } } pRequest->fRunning = false; + pRequest.reset(); T5Logger::GetInstance()->ResetLogBuffer(); if(FLAGS_limit_num_of_active_requests) diff --git a/source/RestAPI/ProxygenHandler.h b/source/RestAPI/ProxygenHandler.h index c5ec29cb..4970cf50 100755 --- a/source/RestAPI/ProxygenHandler.h +++ b/source/RestAPI/ProxygenHandler.h @@ -16,9 +16,9 @@ #include #include #include -#include "OtmMemoryServiceWorker.h" #include "ProxygenStats.h" #include +#include "requestdata.h" using namespace std::chrono; @@ -43,7 +43,7 @@ std::string iobufToString(const std::unique_ptr& buf); #define TIME_MEASURES class ProxygenHandler : public proxygen::RequestHandler { public: - RequestData *pRequest = nullptr; + std::unique_ptr pRequest; size_t bodyPartsReceived = 0; std::string bodyPart; @@ -68,8 +68,7 @@ class ProxygenHandler : public proxygen::RequestHandler { std::string errorStr; std::string responseText = "WRONG_REQUEST"; - proxygen::ResponseBuilder* builder = nullptr; - OtmMemoryServiceWorker* pMemService = nullptr; + std::unique_ptr builder; std::ofstream fileStream; private: diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index d9c1f8d5..772757bc 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -23,7 +23,6 @@ #include "ProxygenStats.h" #include "EQF.H" -#include "OtmMemoryServiceWorker.h" #include "../opentm2/core/utilities/LogWrapper.h" #include "../opentm2/core/utilities/Property.h" #include "../opentm2/core/utilities/FilesystemWrapper.h" @@ -31,8 +30,8 @@ #include "ProxygenHTTPAcceptor.h" #include "../cmake/git_version.h" #include "config.h" -#include "OtmMemoryServiceWorker.h" #include "ProxygenHTTPAcceptor.h" +#include "OtmMemoryServiceWorker.h" DECLARE_string(servicename); DECLARE_string(t5_ip); @@ -149,7 +148,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if (FLAGS_limit_num_of_active_requests && (++ProxygenStats::getInstance()->activeRequestsCounter >= FLAGS_servicethreads)) { - requestHandler->pRequest = new ErrorRequestData; + requestHandler->pRequest = std::make_unique(); requestHandler->pRequest->buildErrorReturn(503, "Server overloaded. Try again later.", 503); //ProxygenStats::getInstance()->activeRequestsCounter.fetch_sub(1); return requestHandler; @@ -207,12 +206,12 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if(urlService != serviceName && urlService != additionalServiceName ){ T5LOG(T5ERROR) <<":: Wrong url \'" << urlService << "\', should be \'"<< serviceName << "\' or \'" << additionalServiceName <<"\'"; - requestHandler->pRequest = new UnknownRequestData; + requestHandler->pRequest = std::make_unique(); //ProxygenStats::getInstance()->activeRequestsCounter.fetch_sub(1) return requestHandler; } - requestHandler->pRequest = nullptr; + requestHandler->pRequest.reset(); url = url.size() > urlSeparator ? url.substr(urlSeparator + 1) : ""; @@ -250,21 +249,21 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if( urlCommand.empty() ){ if( urlMemName.empty()){ if( methodStr == "GET"){ - requestHandler->pRequest = new ListTMRequestData(); + requestHandler->pRequest = std::make_unique(); }else if( methodStr == "POST" ){ std::string contentTypeHeader = headers.getSingleOrEmpty("Content-Type"); bool isMultipart = contentTypeHeader.find("multipart/form-data") != std::string::npos; - requestHandler->pRequest = new CreateMemRequestData(!isMultipart); + requestHandler->pRequest = std::make_unique(!isMultipart); } }else{// mem name is not empty, but command is empty if(methodStr == "POST"){ - //requestHandler->pRequest = new ImportRequestData(); + //requestHandler->pRequest = std::make_uniquepRequest = new ExportRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(methodStr == "DELETE"){ - requestHandler->pRequest = new DeleteMemRequestData(); + requestHandler->pRequest = std::make_unique(); } } }else{ @@ -273,61 +272,64 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if ( urlService == additionalServiceName ){//for calls %serviceName%_service if( methodStr == "GET"){ if(urlCommand == "shutdown"){ - requestHandler->pRequest = new ShutdownRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "resources"){ - auto request = new ResourceInfoRequestData(); - requestHandler->pRequest = request; + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "flags"){ - requestHandler->pRequest = new FlagsRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "savetms"){ - requestHandler->pRequest = new SaveAllTMsToDiskRequestData(); + requestHandler->pRequest = std::make_unique(); } }else if( methodStr == "POST"){ if(urlCommand == "tagreplacement"){ - requestHandler->pRequest = new TagRemplacementRequestData(); + requestHandler->pRequest = std::make_unique(); } } } if(!urlMemName.empty()){//for command memName should always exists if(methodStr == "POST"){ if(urlCommand == "multifuzzysearch"){ - requestHandler->pRequest = new MultiFuzzySearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "multiupdate"){ - requestHandler->pRequest = new MultiUpdateRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "fuzzysearch"){ - requestHandler->pRequest = new FuzzySearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "concordancesearch"){ - requestHandler->pRequest = new ConcordanceSearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "search"){ - requestHandler->pRequest = new ConcordanceExtendedSearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entry"){ // update - requestHandler->pRequest = new UpdateEntryRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entrydelete"){ - requestHandler->pRequest = new DeleteEntryRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entriesdelete"){ - requestHandler->pRequest = new DeleteEntriesReorganizeRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "getentry"){ - requestHandler->pRequest = new GetEntryRequestData(); - }else if(urlCommand == "import"){ // update - requestHandler->pRequest = new ImportRequestData(); + requestHandler->pRequest = std::make_unique(); + }else if(urlCommand == "import"){ + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "importtmx"){ - //requestHandler->pRequest = new ImportStreamRequestData(); - requestHandler->pRequest = new ImportRequestData(false); + //requestHandler->pRequest = std::make_unique(); + requestHandler->pRequest = std::make_unique(false); }else if(urlCommand == "clone"){ - requestHandler->pRequest = new CloneTMRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "importlocal"){ - requestHandler->pRequest = new ImportLocalRequestData(); + requestHandler->pRequest = std::make_unique(); + }else if(urlCommand == "addtotable"){ + requestHandler->pRequest = std::make_unique(); } }else if(methodStr == "GET"){ - if(urlCommand == "status"){ // update - requestHandler->pRequest = new StatusMemRequestData(); + if(urlCommand == "status"){ + requestHandler->pRequest = std::make_unique(); + }else if(urlCommand == "tables"){ + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "reorganize"){ - requestHandler->pRequest = new ReorganizeRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "download.tm"){ - requestHandler->pRequest = new ExportRequestData(EXPORT_MEM_INTERNAL_FORMAT_STREAM); + requestHandler->pRequest = std::make_unique(EXPORT_MEM_INTERNAL_FORMAT_STREAM); }else if(urlCommand == "download.tmx"){ - requestHandler->pRequest = new ExportRequestData(EXPORT_MEM_TMX_STREAM); + requestHandler->pRequest = std::make_unique(EXPORT_MEM_TMX_STREAM); }else if(urlCommand == "flush"){ - requestHandler->pRequest = new FlushMemRequestData(); + requestHandler->pRequest = std::make_unique(); } } } @@ -338,7 +340,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } if(!requestHandler->pRequest){ - requestHandler->pRequest = new UnknownRequestData; + requestHandler->pRequest = std::make_unique(); requestHandler->pRequest->strUrl = url; } @@ -357,12 +359,11 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { uint32_t receiveSessionWindowSize = 10 * (1 << 20) * 10; - T5LOG( T5TRANSACTION) << "Trying to prepare t5memory"; - auto pMemService = OtmMemoryServiceWorker::getInstance(); + T5LOG( T5INFO) << "Trying to prepare t5memory"; char szServiceName[100] = "t5memory"; - char szOtmDirPath[255] =""; + char szOtmDirPath[255]{}; unsigned int uiPort = 4040; int iWorkerThreads = 0; unsigned int uiTimeOut = 60000; @@ -403,7 +404,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } Properties::GetInstance()->set_anyway(KEY_SERVICE_URL, szServiceName); - Properties::GetInstance()->set_anyway(KEY_OTM_DIR, FilesystemHelper::GetOtmDir().c_str()); + Properties::GetInstance()->set_anyway(KEY_OTM_DIR, FilesystemHelper::GetOtmDir()); Properties::GetInstance()->set_anyway(KEY_ALLOWED_RAM, uiAllowedRAM);// saving in megabytes to avoid int overflow // Properties::GetInstance()->set_anyway(KEY_TRIPLES_THRESHOLD, uiThreshold); Properties::GetInstance()->set_anyway(KEY_NUM_OF_THREADS, iWorkerThreads); @@ -413,9 +414,8 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { //From here we have logging in file turned on T5Logger::GetInstance()->DesuppressLoggingInFile(); - T5Logger::GetInstance()->SetLogLevel(uiLogLevel); - char szValue[150]; + char szValue[150]{}; std::string host; @@ -436,7 +436,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } - T5LOG(T5TRANSACTION) << ":: binding to socket, " << addr.getAddressStr() << "; port = " <SetLogLevel(uiLogLevel); + + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fWriteRequestsAllowed = true; + TMManager::fLoadingTmIsAllowed = true; + } if( true /*printIninMsg*/){ - T5LOG(T5TRANSACTION) << "Service details: Service name = " << szServiceName << + T5INITLOG << "Service details: Service name = " << szServiceName << "\n Additional service name: "<< additionalServiceName << "\n Address :" << host << ":" << uiPort << "\n Build date: " << buildDate << @@ -499,7 +491,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { "\n Log level: " << uiLogLevel << "\n Triples threshold: " << uiThreshold << "\n Localhost only: " << fLocalHostOnly << - "\n Flags: " << listFlags(false) << + "\n Flags(*=non-default, !=custom flag from t5memory(and not from the libraries)): " << listFlags(false) << "\n\n\n\ |==================================================================|\n\ |-------------Setup is done -> waiting for requests...-------------|\n\ @@ -514,6 +506,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { }; int proxygen_server_init(){ + OtmMemoryServiceWorker::getInstance(); return ProxygenHandlerFactory::startService(); } diff --git a/source/RestAPI/ProxygenStats.cpp b/source/RestAPI/ProxygenStats.cpp index 2353fc95..7599d90d 100644 --- a/source/RestAPI/ProxygenStats.cpp +++ b/source/RestAPI/ProxygenStats.cpp @@ -36,6 +36,16 @@ int ProxygenStats::recordRequest(int command) { statusMemReqCount_++; break; } + case COMMAND::GET_TABLES: + { + getTablesMemReqCount_++; + break; + } + case COMMAND::ADD_TO_TABLE: + { + addToTableMemReqCount_++; + break; + } case COMMAND::REORGANIZE_MEM: { reorganizeMemRequestCount_++; @@ -158,6 +168,16 @@ int ProxygenStats::addRequestTime(int command,milliseconds time) { { statusMemSumTime_ += time; break; + } + case COMMAND::GET_TABLES: + { + getTablesMemSumTime_ += time; + break; + } + case COMMAND::ADD_TO_TABLE: + { + addToTableMemSumTime_ += time; + break; } case COMMAND::REORGANIZE_MEM: { diff --git a/source/RestAPI/ProxygenStats.h b/source/RestAPI/ProxygenStats.h index ca0939f2..45e80682 100755 --- a/source/RestAPI/ProxygenStats.h +++ b/source/RestAPI/ProxygenStats.h @@ -81,6 +81,20 @@ class ProxygenStats { return statusMemSumTime_; } + virtual uint64_t getGetTablesMemRequestCount() { + return getTablesMemReqCount_; + } + virtual milliseconds getGetTablesMemSumTime() { + return getTablesMemSumTime_; + } + + virtual uint64_t getAddToTableMemRequestCount() { + return addToTableMemReqCount_; + } + virtual milliseconds getAddToTableMemSumTime() { + return addToTableMemSumTime_; + } + virtual uint64_t getFuzzyRequestCount() { return fuzzyReqCount_; } @@ -213,6 +227,8 @@ class ProxygenStats { atomic_uint64_t importMemReqCount_{0}; atomic_uint64_t importLocalReqCount_{0}; atomic_uint64_t statusMemReqCount_{0}; + atomic_uint64_t getTablesMemReqCount_{0}; + atomic_uint64_t addToTableMemReqCount_{0}; atomic_uint64_t flushMemReqCount_{0}; atomic_uint64_t fuzzyReqCount_{0}; @@ -249,6 +265,8 @@ class ProxygenStats { milliseconds multiUpdateReqSumTime_{0}; milliseconds deleteEntrySumTime_{0}; milliseconds deleteEntryReorganizeSumTime_{0}; + milliseconds getTablesMemSumTime_{0}; + milliseconds addToTableMemSumTime_{0}; milliseconds saveAllTmsSumTime_{0}; milliseconds getListOfMemoriesSumTime_{0}; diff --git a/source/opentm2/OTMFUNC.cpp b/source/opentm2/OTMFUNC.cpp index a6e6b077..94eb851c 100755 --- a/source/opentm2/OTMFUNC.cpp +++ b/source/opentm2/OTMFUNC.cpp @@ -127,7 +127,6 @@ USHORT EqfImportMem ( IMPORTMEMORYDATA* pData, - //std::shared_ptr mem, // Eqf session handle LONG lOptions // options for Translation Memory import ) { @@ -154,11 +153,10 @@ USHORT EqfStartSession ) { USHORT usRC = NO_ERROR; // function return code - PFCTDATA pData = NULL; // ptr to function data area CreateSystemProperties(); // allocate internal data area - UtlAlloc( (PVOID *)&pData, 0L, sizeof(FCTDATA), NOMSG ); + PFCTDATA pData = new FCTDATA; // ptr to function data area if ( pData != NULL ) { pData->lMagicWord = FCTDATA_IDENTIFIER; @@ -195,12 +193,6 @@ USHORT EqfStartSession // Initialize error handler to allow handling of error messages UtlInitError( NULLHANDLE, HWND_FUNCIF, (HWND)&(pData->LastMessage), pData->szMsgFile ); - - //if ( !fContinue ) - { - // T5LOG(T5ERROR) << "EqfStartSession():: fContinue is false"; - // usRC = 1; - } /* endif */ } /* endif */ // set directory strings and main drive @@ -272,13 +264,6 @@ USHORT EqfStartSession UtlSetString( QST_ORGEQFDRIVES, pPropSys->szDriveList ); UtlSetString( QST_VALIDEQFDRIVES, pPropSys->szDriveList ); - - // force a refresh of QST_VALIDEQFDRIVES string - { - CHAR szDriveList[MAX_DRIVELIST]; - UtlGetCheckedEqfDrives( szDriveList ); - } - // set fuzziness limits if ( !pPropSys->lSmallLkupFuzzLevel ) pPropSys->lSmallLkupFuzzLevel = 3300; @@ -320,13 +305,7 @@ USHORT EqfStartSession // set caller's session handle if ( usRC == NO_ERROR ) { - - T5LOG( T5DEBUG) << "==EQFSTARTSESSION== Starting plugins..." ; - - - // initialie plugins - if ( usRC == NO_ERROR ) - { + T5LOG( T5DEBUG) << "==EQFSTARTSESSION== Starting plugins..." ; std::string strPluginPath = FilesystemHelper::GetOtmDir(); usRC = InitializePlugins( (PSZ)strPluginPath.c_str() ); // add return value for P402974 @@ -340,16 +319,14 @@ USHORT EqfStartSession T5LOG(T5ERROR) << "EqfSessioStart()::usRC = ERROR_PLUGIN_EXPIRED"; usRC = ERROR_PLUGIN_EXPIRED; } - // Add end - } - T5LOG( T5INFO) << " ...Plugins have been started" ; - { - char szBuf[10]; - int i = _getpid(); - sprintf( szBuf, "%ld", i ); - T5LOG( T5INFO) << "EqfStartSession: Process ID is " << szBuf; - } + T5LOG( T5INFO) << " ...Plugins have been started" ; + + char szBuf[10]; + int i = _getpid(); + sprintf( szBuf, "%ld", i ); + T5LOG( T5INFO) << "EqfStartSession: Process ID is " << szBuf; + *phSession = (LONG)pData; } else diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index e4ddb411..924bc24c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -154,48 +154,39 @@ struct random{ //| allocate memory for start/stop table and copy list into | //| return success indicator | //+----------------------------------------------------------------------------+ -//static BOOL PrepareTokens ( PLOADEDTABLE pTagTable, - PBYTE pInBuf, - PBYTE pTokBuf, PSZ_W pString, // pointer to string to be tokenized SHORT sLanguageId, // language ID - PFUZZYTOK * ppTransTokList, // resulting list of tokens - ULONG ulOemCP + std::vector& pTransTokList // resulting list of tokens ) { + std::vector pTokBuf; PCHAR_W pRest; // ptr to start of not-processed bytes USHORT usColPos = 0; // column pos used by EQFTagTokenize PTOKENENTRY pTok; // ptr for token table processing int iCurOffs = 0; // current offset into segment ULONG ulLength; // length of start/stop table - PFUZZYTOK pstCurrent; // ptr to entries of start/stop table CHAR_W chTemp; // temp character BOOL fOK = TRUE; // success indicator USHORT usRandomIndex; // index in random sequence - USHORT usListSize; // size of buffer - TERMLENOFFS* pTermList; // ptr to created term list + std::vector pTermList; // ptr to created term list TERMLENOFFS* pActTerm; // actual term USHORT usRC; // return code SHORT sNumTags; // number of tags /* KIT0857A */ BOOL fTag; // currently in tagging /* KIT0857A */ - PFUZZYTOK pstAct; // ptr start/stop table /* KIT0857A */ PSZ_W pStart; - //initializeRandom(); /********************************************************************/ /* run TATagTokenize to find tokens .... */ /********************************************************************/ TATagTokenizeW(pString, pTagTable, - TRUE, &pRest, &usColPos, - (PTOKENENTRY) pTokBuf, - TOK_BUFFER_SIZE / sizeof(TOKENENTRY) ); + pTokBuf ); /********************************************************************/ /* build tokenlist, i.e. */ @@ -204,69 +195,61 @@ PrepareTokens /* Rational: use input buffer for temporary list .... */ /* this is large enough, we can avoid any checking... */ /********************************************************************/ - pstCurrent = (PFUZZYTOK) pInBuf; // use input buffer for table - pTok = (PTOKENENTRY) pTokBuf; + pTok = (PTOKENENTRY) pTokBuf.data(); pStart = pTok->pDataStringW; while ( (pTok->sTokenid != ENDOFLIST) ) { if ( pTok->sTokenid == TEXT_TOKEN ) { - usListSize = 0; - pTermList = NULL; - - chTemp = *(pTok->pDataStringW+pTok->iLength); - *(pTok->pDataStringW+pTok->iLength) = EOS; + chTemp = pTok->pDataStringW[pTok->iLength]; + pTok->pDataStringW[pTok->iLength] = EOS; - usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, - &usListSize, &pTermList, - MORPH_OFFSLIST, ulOemCP ); + pTermList.clear(); + usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, + pTermList ); - *(pTok->pDataStringW+pTok->iLength) = chTemp; + pTok->pDataStringW[pTok->iLength] = chTemp; - if ( pTermList ) + if ( pTermList.size() ) { - pActTerm = pTermList; - if ( pTermList->iLength ) + pActTerm = pTermList.data(); + if ( pActTerm->iLength ) { while ( pActTerm->iLength ) { - pString = pTok->pDataStringW + pActTerm->iOffset; + pString = &pTok->pDataStringW[pActTerm->iOffset]; /**********************************************************/ /* ignore the linefeeds and tabs in the matching */ /**********************************************************/ if ( (*pString != LF) && (*pString != 0x09)) { - pstCurrent = SplitTokens(pstCurrent, - (iCurOffs + pActTerm->iOffset), - TEXT_TOKEN, - pActTerm->iLength, - pString); + SplitTokens(pTransTokList, + (iCurOffs + pActTerm->iOffset), + TEXT_TOKEN, + pActTerm->iLength, + pString); } /* endif */ pActTerm++; } /* endwhile */ } else { - pstCurrent = SplitTokens(pstCurrent, - iCurOffs, - pTok->sTokenid, - pTok->iLength, - pTok->pDataStringW); + SplitTokens(pTransTokList, + iCurOffs, + pTok->sTokenid, + pTok->iLength, + pTok->pDataStringW); } /* endif */ } /* endif */ - /****************************************************************/ - /* free allocated resource ... */ - /****************************************************************/ - UtlAlloc( (PVOID *)&pTermList, 0L, 0L, NOMSG ); } else { - pstCurrent = SplitTokens(pstCurrent, - iCurOffs, - pTok->sTokenid, - pTok->iLength, - pTok->pDataStringW); + SplitTokens(pTransTokList, + iCurOffs, + pTok->sTokenid, + pTok->iLength, + pTok->pDataStringW); } /* endif */ /****************************************************************/ /* adjust current offset to point to new offset in string... */ @@ -276,8 +259,7 @@ PrepareTokens } /* endwhile */ // terminate start/stop table - memset( pstCurrent, 0, sizeof( FUZZYTOK )); - + pTransTokList.push_back(FUZZYTOK{}); /********************************************************************/ /* get number of tags ... */ @@ -289,8 +271,8 @@ PrepareTokens /* pstAct -- points to active one */ /* pstCurrent -- points to next one... */ /********************************************************************/ - pstCurrent = (PFUZZYTOK) pInBuf; // use input buffer for table - pstAct = pstCurrent; + PFUZZYTOK pstCurrent = pTransTokList.data(); // use input buffer for table + PFUZZYTOK pstAct = pstCurrent; /********************************************************************/ /* set tag indication */ /********************************************************************/ @@ -304,7 +286,7 @@ PrepareTokens { fTag = ( pstCurrent->sType >= 0 ); pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; pstCurrent++; } else // it is an attribute @@ -321,7 +303,7 @@ PrepareTokens else { pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; pstCurrent++; } /* endif */ } /* endif */ @@ -330,24 +312,12 @@ PrepareTokens /* put in end indication */ /********************************************************************/ pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; /********************************************************************/ /* have to point to next one, because we use it as length parameter */ /********************************************************************/ pstAct++; - /********************************************************************/ - /* allocate memory for start/stop table and copy list into it */ - /********************************************************************/ - *ppTransTokList = NULL; // init token list.. - ulLength = (ULONG)(((PBYTE) pstAct) - pInBuf); - - fOK = UtlAlloc( (PVOID *)ppTransTokList, 0L, ulLength, ERROR_STORAGE ); - if ( fOK ) - { - memcpy( *ppTransTokList, pInBuf, ulLength ); - } /* endif */ - return( fOK ); } /* end of function */ @@ -372,30 +342,30 @@ PrepareTokens //| now deleted again (July 93) | //+----------------------------------------------------------------------------+ -//static -PFUZZYTOK SplitTokens - ( - PFUZZYTOK pstCurrent, +void SplitTokens +( + std::vector& FuzzyTokVec, USHORT usStart, SHORT sType, int usLength, PSZ_W pString - ) +) { CHAR_W chTemp; - pstCurrent->pData = pString; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStart + usLength - 1; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + usLength); - *(pString + usLength) = EOS; - MakeHashValue ( random::getRandomVec(), MAX_RANDOM, - pString , &pstCurrent->ulHash ); - *(pString + usLength) = chTemp; - pstCurrent++; - return (pstCurrent); + FuzzyTokVec.push_back(FUZZYTOK{}); + auto& current = FuzzyTokVec.back(); + + current.pData = pString; + current.usStart = usStart; + current.sType = sType; + current.usStop = usStart + usLength - 1; + current.fConnected = FALSE; + chTemp = pString[usLength]; + pString[usLength] = EOS; + MakeHashValue ( random::getRandomVec(), MAX_RANDOM, + pString , ¤t.ulHash ); + pString[usLength] = chTemp; } /* end of function SplitTokens */ //+----------------------------------------------------------------------------+ @@ -419,7 +389,6 @@ PFUZZYTOK SplitTokens //| Characters not in the alphanumeric range are ignored.. | //| (Check in case of DBCS might be necessary...) | //+----------------------------------------------------------------------------+ -//static VOID MakeHashValue ( PULONG pulRandom, // array of random numbers for hashing @@ -490,7 +459,6 @@ VOID MakeHashValue //| ment string is of different length | //| endwhile | //+----------------------------------------------------------------------------+ -//static BOOL FuzzyReplace ( PSZ_W pSource, // source string @@ -681,7 +649,7 @@ BOOL FuzzyReplace /**************************************************************/ /* source and proposal different, so do a replace .... */ /**************************************************************/ - if ( UTF16strcmp(chSource, chProp) ) + if ( wcscmp(chSource, chProp) ) { TransferSource( chSource, chProp, chTrans ); fReplaced = TRUE; @@ -813,8 +781,6 @@ static BOOL isNumber( PSZ_W pszText ) //| differences like in the representation of numbers ) | //| if something found to be edited do it during copying | //+----------------------------------------------------------------------------+ - -//static VOID TransferSource ( @@ -955,8 +921,6 @@ TransferSource //| memcmp both data strings | //| endif | //+----------------------------------------------------------------------------+ - -//static SHORT TokStrCompare ( @@ -1115,56 +1079,18 @@ TokStrCompare //| free forward and backward token lists | //| return success indicator | //+----------------------------------------------------------------------------+ -BOOL -EQFBFindDiff -( - PTBDOCUMENT pDoc, // pointer to document - PSZ_W pString1, // first string passed - PSZ_W pString2, // second string - SHORT sLanguageId, // language id to be used - PFUZZYTOK * ppFuzzyTok, // returned token list - PUSHORT pusModWords, // num of differences - PUSHORT pusTokens, - ULONG ulOemCP -) -{ - BOOL fOK; - PFUZZYTOK pFuzzyTgt = NULL; - - fOK = EQFBFindDiffEx( pDoc->pDocTagTable, - pDoc->pInBuf, - pDoc->pTokBuf, - pString1, - pString2, - sLanguageId, - (PVOID *)ppFuzzyTok, - (PVOID *)&pFuzzyTgt, ulOemCP ); - /**********************************************************************/ - /* pFuzzyTgt is not needed, so free space */ - /**********************************************************************/ - if (pFuzzyTgt ) - { - UtlAlloc ( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); - } /* endif */ - - if ( fOK ) - { - EQFBCountDiff( *ppFuzzyTok, pusTokens, pusModWords ); - } /* endif */ - return fOK; -} // counts the number of tokens and the number of differences // in a fuzzy token list created using EQFBFindDiffEx BOOL EQFBCountDiff ( - PVOID pvFuzzyTok, + std::vector& pFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ) { - PFUZZYTOK pTest = (PFUZZYTOK)pvFuzzyTok; + PFUZZYTOK pTest = pFuzzyTok.data(); PFUZZYTOK pTestToken; // pointer to token lists PFUZZYTOK pToken; // pointer to token lists USHORT usLen = 0; @@ -1176,8 +1102,8 @@ BOOL EQFBCountDiff /* MODIFIED DELETED is changed to MODIFIED */ /* MODIFIED INSERTED is changed to MODIFIED MODIFIED */ /******************************************************************/ - pTestToken = (PFUZZYTOK)pvFuzzyTok; //resulting pointers - pToken = (PFUZZYTOK)pvFuzzyTok; //current pointers + pTestToken = pFuzzyTok.data(); //resulting pointers + pToken = pFuzzyTok.data(); //current pointers usPrevType = MARK_EQUAL; while ( pToken->ulHash ) @@ -1237,63 +1163,44 @@ BOOL EQFBCountDiff BOOL EQFBFindDiffEx ( - PVOID pTable, // pointer to loaded tagtable - PBYTE pInBuf, // pointer to input buffer - PBYTE pTokBuf, // pointer to temp token buffer - PSZ_W pString1, // first string passed - PSZ_W pString2, // second string + PLOADEDTABLE pTagTable, // pointer to loaded tagtable + PSZ_W pStringSrc, // first string passed + PSZ_W pStringMatch, // second string SHORT sLanguageId, // language id to be used for pString1 - PVOID * ppFuzzyTok, // returned token list of pString2 - PVOID * ppFuzzyTgt, // returned token list of pString1 - ULONG ulOemCP + std::vector& pFuzzyTok, // returned token list of pString2 + std::vector& pFuzzyTgt // returned token list of pString1 ) { - BOOL fOK = TRUE; // success indicator - USHORT usLenStr1 = 0; // length of string 1 (in num. of tokens) - USHORT usLenStr2 = 0; // length of string 2 (in num. of tokens) - PFUZZYTOK pTokenList2 = NULL; // pointer to token lists - PFUZZYTOK pTokenList1 = NULL; // pointer to token lists + USHORT usLenStrSrc = 0; // length of string 1 (in num. of tokens) + USHORT usLenStrMatch = 0; // length of string 2 (in num. of tokens) + std::vector pTokenListSrc, pTokenListMatch; // pointer to token lists PFUZZYTOK pToken; // pointer to token lists - PLOADEDTABLE pTagTable = (PLOADEDTABLE) pTable; /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ - T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for first string: "; - //std::wstring ws1 = removeTagsFromString(pString1); - fOK = PrepareTokens( //pDoc, - pTagTable, - pInBuf, - pTokBuf, - //(wchar_t*)ws1.c_str(), - pString1, - sLanguageId, &pTokenList1, ulOemCP ); + BOOL fOK = PrepareTokens( pTagTable, + pStringSrc, + sLanguageId, pTokenListSrc ); if ( fOK ) { - - T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for second string: "; - //std::wstring ws2 = removeTagsFromString(pString2); - fOK = PrepareTokens( // pDoc, - pTagTable, - pInBuf, - pTokBuf, - //(wchar_t*)ws2.c_str(), - pString2, - sLanguageId, &pTokenList2, ulOemCP ); + fOK = PrepareTokens( pTagTable, + pStringMatch, + sLanguageId, pTokenListMatch ); } /* endif */ if (fOK ) { /********************************************************************/ /* get number of tokens in strings */ /********************************************************************/ - NUMBEROFTOKENS(usLenStr1, pTokenList1); - NUMBEROFTOKENS(usLenStr2, pTokenList2); + NUMBEROFTOKENS(usLenStrSrc, pTokenListSrc.data()); + NUMBEROFTOKENS(usLenStrMatch, pTokenListMatch.data()); /******************************************************************/ /* call LCS and compare tokens with strncmp */ /******************************************************************/ - fOK = EQFBCallLCS(pTokenList1, pTokenList2, - usLenStr1, usLenStr2, TRUE); + fOK = EQFBCallLCS(pTokenListSrc, pTokenListMatch, + usLenStrSrc, usLenStrMatch, TRUE); } /* endif */ /********************************************************************/ /* get number of tokens in strings */ @@ -1303,7 +1210,7 @@ EQFBFindDiffEx /****************************************************************/ /* adjust length to avoid non-marked spaces between marked tokens*/ /****************************************************************/ - pToken = pTokenList1; //current pointers + pToken = pTokenListSrc.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1313,7 +1220,7 @@ EQFBFindDiffEx pToken++; } /* endwhile */ - pToken = pTokenList2; //current pointers + pToken = pTokenListMatch.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1326,16 +1233,11 @@ EQFBFindDiffEx /* allocate space for fuzzy tokens ,.,,, */ /* this area MUST be freed by the calling application */ /****************************************************************/ - fOK = EQFBMarkModDelIns( pTokenList1, pTokenList2, - (PFUZZYTOK *)ppFuzzyTgt, (PFUZZYTOK *)ppFuzzyTok, - usLenStr1, usLenStr2); + fOK = EQFBMarkModDelIns( pTokenListSrc, pTokenListMatch, + pFuzzyTgt, pFuzzyTok, + usLenStrSrc, usLenStrMatch); } /* endif */ - /********************************************************************/ - /* free any allocated resources */ - /********************************************************************/ - UtlAlloc( (PVOID *)&pTokenList1, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *)&pTokenList2, 0L, 0L, NOMSG ); return( fOK ); } /* end of function EQFBFindDiffEx */ @@ -1369,18 +1271,16 @@ EQFBFindDiffEx //| MARK_MODIFIED, MARK_DELETED, MARK_INSERTED | //| return success indicator | //+----------------------------------------------------------------------------+ -//static BOOL EQFBMarkModDelIns ( - PFUZZYTOK pTokenList1, - PFUZZYTOK pTokenList2, - PFUZZYTOK * ppFuzzy1, - PFUZZYTOK * ppFuzzy2, + std::vector& pTokenList1, + std::vector& pTokenList2, + std::vector& pFuzzy1, + std::vector& pFuzzy2, USHORT usLenStr1, USHORT usLenStr2 ) - { BOOL fOK; PFUZZYTOK pTestToken; // pointer to token lists 2 @@ -1388,94 +1288,89 @@ EQFBMarkModDelIns PFUZZYTOK pT1Token; // pointer to token lists 1 LONG lTokBufLen = (LONG)(usLenStr1 + usLenStr2 + 1); - LONG lTokBufSize = lTokBufLen * sizeof(FUZZYTOK); + pFuzzy2.resize(lTokBufLen); + pFuzzy1.resize(lTokBufLen); LONG lTokBufUsed = 0; - fOK = UtlAlloc( (PVOID *)ppFuzzy2, 0L, lTokBufSize, ERROR_STORAGE ); - if (fOK ) fOK = UtlAlloc( (PVOID *)ppFuzzy1, 0L, lTokBufSize, ERROR_STORAGE ); - - if ( fOK ) + pTestToken = pFuzzy2.data(); + pT1Token = pFuzzy1.data(); + usStringA = usStringB = 0; + while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) { - pTestToken = *ppFuzzy2; - pT1Token = *ppFuzzy1; - usStringA = usStringB = 0; - while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) + if ( (pTokenList2[usStringB].sType == MARK_EQUAL) + && (pTokenList1[usStringA].sType == MARK_EQUAL) ) { - if ( ((pTokenList2+usStringB)->sType == MARK_EQUAL) - && ((pTokenList1+usStringA)->sType == MARK_EQUAL) ) - { - *pTestToken = *(pTokenList2+usStringB); - pTestToken++; - lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); - pT1Token++; - - usStringA++; usStringB++; - } - else if ((pTokenList1+usStringA)->sType == MARK_EQUAL) - { - *pTestToken = *(pTokenList2+usStringB); - pTestToken->sType = MARK_INSERTED; + *pTestToken = pTokenList2[usStringB]; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList2+usStringB); - pT1Token->sType = MARK_DELETED; + *pT1Token = pTokenList1[usStringA]; pT1Token++; - usStringB++; - } - else if ((pTokenList2+usStringB)->sType == MARK_EQUAL) - { - *pTestToken = *(pTokenList1+usStringA); - pTestToken->sType = MARK_DELETED; - pTestToken++; - lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); - pT1Token->sType = MARK_INSERTED; - pT1Token++; - - usStringA++; - } - else - { - *pTestToken = *(pTokenList2+usStringB); - pTestToken->sType = MARK_MODIFIED; - pTestToken++; - lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); - pT1Token->sType = MARK_MODIFIED; - pT1Token++; - - usStringA++; - usStringB++; - } /* endif */ - } /* endwhile */ + usStringA++; usStringB++; + } + else if (pTokenList1[usStringA].sType == MARK_EQUAL) + { + *pTestToken = pTokenList2[usStringB]; + pTestToken->sType = MARK_INSERTED; + pTestToken++; + lTokBufUsed++; + *pT1Token = pTokenList2[usStringB]; + pT1Token->sType = MARK_DELETED; + pT1Token++; - while ( usStringA < usLenStr1 ) + usStringB++; + } + else if (pTokenList2[usStringB].sType == MARK_EQUAL) { - *pTestToken = *(pTokenList1+usStringA); + *pTestToken = pTokenList1[usStringA]; pTestToken->sType = MARK_DELETED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); + *pT1Token = pTokenList1[usStringA]; pT1Token->sType = MARK_INSERTED; pT1Token++; usStringA++; - } /* endwhile */ - while ( usStringB < usLenStr2 ) + } + else { - *pTestToken = *(pTokenList2+usStringB); - pTestToken->sType = MARK_INSERTED; + *pTestToken = pTokenList2[usStringB]; + pTestToken->sType = MARK_MODIFIED; pTestToken++; lTokBufUsed++; - - *pT1Token = *(pTokenList2+usStringB); - pT1Token->sType = MARK_DELETED; + *pT1Token = pTokenList1[usStringA]; + pT1Token->sType = MARK_MODIFIED; pT1Token++; + + usStringA++; usStringB++; - } /* endwhile */ - } /* endif */ + } /* endif */ + } /* endwhile */ + + while ( usStringA < usLenStr1 ) + { + *pTestToken = pTokenList1[usStringA]; + pTestToken->sType = MARK_DELETED; + pTestToken++; + lTokBufUsed++; + *pT1Token = pTokenList1[usStringA]; + pT1Token->sType = MARK_INSERTED; + pT1Token++; + + usStringA++; + } /* endwhile */ + while ( usStringB < usLenStr2 ) + { + *pTestToken = pTokenList2[usStringB]; + pTestToken->sType = MARK_INSERTED; + pTestToken++; + lTokBufUsed++; + + *pT1Token = pTokenList2[usStringB]; + pT1Token->sType = MARK_DELETED; + pT1Token++; + usStringB++; + } /* endwhile */ return( fOK ); } /* end of function EQFBMarkModDelIns */ @@ -1536,8 +1431,6 @@ EQFBMarkModDelIns //| endif | //| return length of shortest edit script | //+----------------------------------------------------------------------------+ - -//static SHORT FindMiddleSnake ( @@ -1876,10 +1769,7 @@ static long CompareTokenWS( PSZ_W pTok1, USHORT len1, PSZ_W pTok2, USHORT len2 ) //| both strings are identical. | //| Return the compounded value | //+----------------------------------------------------------------------------+ - -//static -SHORT -Snake +SHORT Snake ( PFUZZYTOK pFuzzyStr1, PFUZZYTOK pFuzzyStr2, @@ -1973,10 +1863,7 @@ Snake //| endif | //| endif | //+----------------------------------------------------------------------------+ - -//static -VOID -LCS +VOID LCS ( LCSTOKEN LCSStringA, LCSTOKEN LCSStringB, @@ -2067,129 +1954,6 @@ LCS } /* end of function LCS */ -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: FuzzyLCSReplList | -//+----------------------------------------------------------------------------+ -//|Function call: FuzzyLCSReplList( PFUZZYTOK,PFUZZYTOK,PREPLLIST) | -//+----------------------------------------------------------------------------+ -//|Description: Will return a list of possible tokens to be replaced | -//| in a fuzzy match | -//+----------------------------------------------------------------------------+ -//|Parameters: PFUZZYTOK pPropTokList, pointer to token list | -//| PFUZZYTOK pTransTokList, pointer to token list | -//| PREPLLIST * ppReplaceList pointer to replace list | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 1 .. n number of equal tokens | -//| 0 error during allocation | -//+----------------------------------------------------------------------------+ -//|Function flow: set up pointer array | -//| return success indicator | -//+----------------------------------------------------------------------------+ - -USHORT -FuzzyLCSReplList -( - PFUZZYTOK pTokList1, // ptr to token list->TgtTok of ReplList - PFUZZYTOK pTokList2, // ptr to token list->SrcTOk ofReplList - PREPLLIST * ppReplaceList, // pointer to replace list - USHORT usLenStr1, // num of tokens in TokList1 - USHORT usLenStr2, // num of tokens in TokList2 - PUSHORT pusFuzzy, - BOOL fCompareAll -) -{ - PREPLLIST pReplaceList = NULL; // pointer to replace list - BOOL fOK = TRUE; // success indicator - USHORT usMinEqual = 0; // no of equal but not-connected tokens - USHORT usI = 0; - USHORT usStringA; - USHORT usStringB; - - /********************************************************************/ - /* call LCS and use TOkStrCompare so that tokens are compared */ - /* without regard of punctuation */ - /********************************************************************/ - fOK = EQFBCallLCS(pTokList1, pTokList2, usLenStr1, usLenStr2, fCompareAll); - - /********************************************************************/ - /* count equal and different words; nether change pTokList1/2 */ - /* since it is needed later on too */ - /********************************************************************/ - if (fOK && pusFuzzy ) - { - fOK = EQFBTokCountDiff(pTokList1, pTokList2, usLenStr1, usLenStr2, pusFuzzy); - } /* endif */ - /*****************************************************************/ - /* build ReplaceList */ - /*****************************************************************/ - if (fOK ) - { - if ( !pReplaceList ) - { - if ( ! UtlAlloc( (PVOID *)ppReplaceList, - 0L, (LONG)( (MAX_REPL+1) * sizeof(REPLLIST) ), - ERROR_STORAGE ) ) - { /************************************************************/ - /* error happened - reset pointer to stop loop */ - /************************************************************/ - usI = MAX_REPL; - fOK = FALSE; - } - else - { - pReplaceList = *ppReplaceList; - } /* endif */ - } /* endif */ - - usStringA = usStringB = 0; - while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) - { - if ( ((pTokList2+usStringB)->sType == MARK_EQUAL) - && ((pTokList1+usStringA)->sType == MARK_EQUAL) ) - { - if ( usMinEqual < MAX_REPL-1 ) - { - pReplaceList->pSrcTok = pTokList1 + usStringA; // store values - pReplaceList->pTgtTok = pTokList2 + usStringB; - (pTokList1+usStringA)->fConnected = TRUE; - (pTokList2+usStringB)->fConnected = TRUE; - pReplaceList++; - usMinEqual++; - } - else - { - usMinEqual++; - } /* endif */ - - usStringA++; usStringB++; - } - else if ((pTokList1+usStringA)->sType == MARK_EQUAL) - { - usStringB++; - } - else if ((pTokList2+usStringB)->sType == MARK_EQUAL) - { - usStringA++; - } - else - { - usStringA++; - usStringB++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - if ( !fOK ) - { - usMinEqual = 0; - } /* endif */ // tokens - return usMinEqual; -} /* end of function FuzzyLCSReplList */ - //+----------------------------------------------------------------------------+ //|Internal function | @@ -2215,8 +1979,8 @@ FuzzyLCSReplList //+----------------------------------------------------------------------------+ BOOL EQFBCallLCS ( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 + std::vector& pTokenList1, // ptr to token list1 + std::vector& pTokenList2, // ptr to token list2 USHORT usLenStr1, USHORT usLenStr2, BOOL fCompareAll @@ -2226,61 +1990,47 @@ BOOL EQFBCallLCS PFUZZYTOK pTestToken; LCSTOKEN LCSString1; LCSTOKEN LCSString2; - PFUZZYTOK pBackList1; - PFUZZYTOK pBackList2 = NULL; + std::vector pBackList1; + std::vector pBackList2; SHORT k; - - - fOK = UtlAlloc( (PVOID *)&pBackList1, - 0L, - (LONG) (usLenStr1 +1) * sizeof(FUZZYTOK), - ERROR_STORAGE ); - if (fOK ) - { - fOK = UtlAlloc( (PVOID *)&pBackList2, - 0L, - (LONG) (usLenStr2 +1) * sizeof(FUZZYTOK), - ERROR_STORAGE ); - } /* endif */ - if ( fOK ) + pBackList1.resize(usLenStr1+1); + pBackList2.resize(usLenStr2+1); + { /******************************************************************/ /* fill backward tokenlists */ /******************************************************************/ - pTestToken = pTokenList1; + pTestToken = pTokenList1.data(); for ( k=(usLenStr1-1); k >= 0 ; k-- ) { - *(pBackList1+k) = *pTestToken; + pBackList1[k] = *pTestToken; pTestToken++; } /* endfor */ - pTestToken = pTokenList2; + pTestToken = pTokenList2.data(); for ( k = (usLenStr2-1); k >= 0 ; k-- ) { - *(pBackList2+k) = *pTestToken; + pBackList2[k] = *pTestToken; pTestToken++; } /* endfor */ /******************************************************************/ /* call recursive function to find shortest edit script */ /******************************************************************/ - LCSString1.pTokenList = pTokenList1; - LCSString1.pBackList = pBackList1; + LCSString1.pTokenList = pTokenList1.data(); + LCSString1.pBackList = pBackList1.data(); LCSString1.sStart = 0; LCSString1.sStop = usLenStr1; LCSString1.sTotalLen = usLenStr1; - LCSString2.pTokenList = pTokenList2; - LCSString2.pBackList = pBackList2; + LCSString2.pTokenList = pTokenList2.data(); + LCSString2.pBackList = pBackList2.data(); LCSString2.sStart = 0; LCSString2.sStop = usLenStr2; LCSString2.sTotalLen = usLenStr2; LCS ( LCSString1,LCSString2, fCompareAll); - UtlAlloc( (PVOID *)&pBackList1, 0L, 0L, NOMSG); - UtlAlloc( (PVOID *)&pBackList2, 0L, 0L, NOMSG); - } /* endif */ return fOK; } /* end of function EQFBCallLCS */ @@ -2310,49 +2060,6 @@ BOOL EQFBCallLCS //| return success indicator | //+----------------------------------------------------------------------------+ //static -BOOL EQFBTokCountDiff -( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 - USHORT usLenStr1, - USHORT usLenStr2, - PUSHORT pusFuzzy -) -{ - PFUZZYTOK pFuzzy1 = NULL; // ret. tokenlist of Strin2 - PFUZZYTOK pFuzzy2 = NULL; // ret. tokenlist of string 1 - USHORT usDiff = 0; // different words in String2 - USHORT usWords = 0; // total words in String2 - BOOL fOK; - - fOK = EQFBMarkModDelIns( pTokenList1, pTokenList2, - &pFuzzy1, &pFuzzy2,usLenStr1, usLenStr2); - - if (fOK ) - { - /******************************************************************/ - /* usWords : total words in combined string */ - /* usDiff: # of words not MARK_EQUAL in combined string */ - /******************************************************************/ - EQFBSimplifyAndCountMarks(pFuzzy2, &usDiff, &usWords); - } /* endif */ - - - UtlAlloc ((PVOID *) &pFuzzy1, 0L, 0L, NOMSG); - UtlAlloc ((PVOID *) &pFuzzy2, 0L, 0L, NOMSG); - - if (usDiff < usWords ) - { - *pusFuzzy = (usWords != 0) ? ((usWords - usDiff)*100 / usWords) : 100; - } - else - { - *pusFuzzy = 0; - } /* endif */ - - return (fOK); -} /* end of function EQFBTokCountDiff */ - //+----------------------------------------------------------------------------+ //|Internal function | @@ -2377,10 +2084,9 @@ BOOL EQFBTokCountDiff //|Function flow: set up pointer array | //| return success indicator | //+----------------------------------------------------------------------------+ -//static void EQFBSimplifyAndCountMarks ( - PFUZZYTOK pFuzzy, // ptr to token + std::vector& pFuzzy, // ptr to token PUSHORT pusDiff, // # of different words PUSHORT pusWords // total # of words ) @@ -2400,8 +2106,8 @@ void EQFBSimplifyAndCountMarks /* count all words and count all words which are not equal */ /*******************************************************************/ usDiff = 0; - pTestToken = pFuzzy; //resulting pointers - pToken = pFuzzy; //current pointers + pTestToken = pFuzzy.data(); //resulting pointers + pToken = pFuzzy.data(); //current pointers usPrevType = MARK_EQUAL; while ( pToken->ulHash ) @@ -2487,22 +2193,18 @@ void EQFBSimplifyAndCountMarks //|Function flow: set up pointer array | //| return success indicator | //+----------------------------------------------------------------------------+ - -BOOL -NTMCompareBetweenTokens +BOOL NTMCompareBetweenTokens ( PSZ_W pD1, PSZ_W pD2, PSZ pszMarkup, SHORT sLangID, - ULONG ulSrcCP, PBOOL pfStringEqual ) { PLOADEDTABLE pTable = NULL; // ptr to loaded tag table BOOL fOK = TRUE; // success indicator - PFUZZYTOK pTokenList2 = NULL; // pointer to token lists - PFUZZYTOK pTokenList1 = NULL; // pointer to token lists + std::vector pTokenList1, pTokenList2; // pointer to token lists PFUZZYTOK pstTok1; // pointer to token lists PFUZZYTOK pstTok2; // pointer to token lists PFUZZYTOK pstNextTok1; @@ -2510,8 +2212,6 @@ NTMCompareBetweenTokens SHORT sNumTags = 0; BOOL fStringEqual = TRUE; CHAR_W chW; - PBYTE pInBuf = NULL; - PBYTE pTokBuf = NULL; USHORT usI = 0; BOOL fTokIsTag = FALSE; BOOL fTok2IsTag = FALSE; @@ -2520,17 +2220,11 @@ NTMCompareBetweenTokens ULONG ulLen2 = 0; PSZ_W pTemp = NULL; - // allocate required buffers - fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 50000L, NOMSG ); - if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, (LONG)TOK_BUFFER_SIZE, NOMSG ); - // load tag table - if ( fOK ) - { - fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, - TALOADUSEREXIT | TALOADPROTTABLEFUNC, - FALSE, NULLHANDLE ) == 0 /*NO_ERROR*/); - } /* endif */ + fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, + TALOADUSEREXIT | TALOADPROTTABLEFUNC, + FALSE, NULLHANDLE ) == 0 /*NO_ERROR*/); + if (fOK && (pTable->pfnProtTableW || pTable->pfnProtTable)) { // protect-unprotect cannot be detected via TATagTokenize in PrepareTokens @@ -2543,17 +2237,11 @@ NTMCompareBetweenTokens /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ - fOK = PrepareTokens( pTable, - pInBuf, - pTokBuf, - pD1, sLangID, &pTokenList1, ulSrcCP ); + fOK = PrepareTokens( pTable, pD1, sLangID, pTokenList1 ); } if ( fOK) { - fOK = PrepareTokens( pTable, - pInBuf, - pTokBuf, - pD2, sLangID, &pTokenList2, ulSrcCP ); + fOK = PrepareTokens( pTable, pD2, sLangID, pTokenList2 ); } // prereq is FUzzyness= 100%, i.e. all texttokens have been MARK_EQUAL in TMFuzzyness!! // check whether all tokens ( text + tag) are in both lists! @@ -2561,8 +2249,8 @@ NTMCompareBetweenTokens if (fOK) { sNumTags = (SHORT) pTable->pTagTable->uNumTags; - pstTok1 = pTokenList1; - pstTok2 = pTokenList2; + pstTok1 = pTokenList1.data(); + pstTok2 = pTokenList2.data(); while (fStringEqual && pstTok1->ulHash && pstTok2->ulHash) { @@ -2626,8 +2314,8 @@ NTMCompareBetweenTokens // Now we have to take care of the spaces between the texttokens and TagTokens! if ( fStringEqual ) { - pstTok1 = pTokenList1; - pstTok2 = pTokenList2; + pstTok1 = pTokenList1.data(); + pstTok2 = pTokenList2.data(); pstNextTok1 = pstTok1 + 1; pstNextTok2 = pstTok2 + 1; while ( pstTok1->ulHash && pstNextTok1->ulHash && @@ -2685,11 +2373,6 @@ NTMCompareBetweenTokens } /* endwhile */ } /* endif */ } /* endif if fOK */ - - if (pTokenList1) UtlAlloc( (PVOID *)&pTokenList1, 0L, 0L, NOMSG ); - if (pTokenList2) UtlAlloc( (PVOID *)&pTokenList2, 0L, 0L, NOMSG ); - if ( pInBuf ) UtlAlloc( (PVOID *) &pInBuf, 0L, 0L, NOMSG ); - if ( pTokBuf ) UtlAlloc( (PVOID *) &pTokBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable); *pfStringEqual = fStringEqual; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index bbeaf857..6cab1feb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -18,27 +18,9 @@ #include "LogWrapper.h" #include "tm.h" -// EQF.H is included by otmmemory.h -// #include // General Translation Manager include file EQF:H #include // batch mode definitions #define INCL_EQFMEM_DLGIDAS -//#include "EQFMEM.ID" // Translation Memory IDs -//#include "EQFCOLW.ID" // column width IDs - -BOOL EQFTMMaintain( PSZ ); - - - -//#define EqfRemoveObject( flg, hwnd) (BOOL)WinSendMsg( EqfQueryObjectManager(),\ -// WM_EQF_REMOVEOBJECT, \ -// MP1FROMSHORT( flg ), \ -// MP2FROMHWND(hwnd) ) - -bool EqfRemoveObject(auto flg, auto hwnd){ - LOG_UNIMPLEMENTED_FUNCTION; - return true; -} // ================ Handle the message WM_EQF_MEMORGANIZE_START ======================= @@ -98,7 +80,6 @@ USHORT EQFMemOrganizeStart // Close the Translation memory which should be organized. if ( pRIDA->pMemTemp != NULL ) { - TMManager::GetInstance()->closeMemory( pRIDA->pMemTemp.get() ); pRIDA->pMemTemp = NULL; std::string strMsg; @@ -121,11 +102,6 @@ USHORT EQFMemOrganizeStart else { pRIDA->NextTask = MEM_ORGANIZE_TASK; - //if(pRIDA->pMem->importDetails == nullptr){ - // pRIDA->pMem->importDetails = new ImportStatusDetails; - //} - //pRIDA->pMem->importDetails->reset(); - //pRIDA->pMem->importDetails->fReorganize = true; } return iRC; } /* end of function EQFMemOrganizeStart */ @@ -134,8 +110,8 @@ USHORT EQFMemOrganizeStart // ================ Handle the message WM_EQF_MEMORGANIZE_PROCESS ===================== bool IsValidXml(std::wstring&& sentence); +DECLARE_int32(maxBadSegmentsIdsSaved); -DEFINE_int32(maxBadSegmentsIdsSaved, 20, "Defines how much segments(it's id's) that failed during import or reorganize would be saved to the status response"); VOID EQFMemOrganizeProcess ( @@ -169,15 +145,17 @@ VOID EQFMemOrganizeProcess }catch(...) { pRIDA->pProposal->nextInternalKey.moveToNextRecord(); - //pRIDA->pMem->ulNextKey++; - //pRIDA->pMem->usNextTarget = 1; - iRC = EqfMemory::ERROR_ENTRYISCORRUPTED; + iRC = EqfMemoryError::ERROR_ENTRYISCORRUPTED; } if ( iRC == NO_ERROR ) { pCommArea->usComplete = (USHORT)iProgress; - pRIDA->pMem->importDetails->usProgress = (USHORT) iProgress; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + pRIDA->pMem->importDetails->usProgress = (USHORT) iProgress; + pRIDA->pProposal->fSaveDifferentTargetsForSameSource = pRIDA->pMem->importDetails->fSaveDifferentTargetsForSameSource; + } // do some consistency checking pRIDA->pProposal->getMarkup( pRIDA->szTagTable, sizeof(pRIDA->szTagTable) ); @@ -205,16 +183,18 @@ VOID EQFMemOrganizeProcess if (!fValidXml || (sourceLen == 0) || (sourceLen >= OTMPROPOSAL_MAXSEGLEN) || (targetLen == 0) || (targetLen>=OTMPROPOSAL_MAXSEGLEN) || (pRIDA->szTargetLanguage[0] == EOS) || (pRIDA->szTagTable[0] == EOS) ) { + + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); // ignore invalid proposal pRIDA->pMem->importDetails->invalidSegments++; if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), -8)); } } else { // write proposal to output memory - //bool fFilterPassed = true; bool fSkipSegment = !pRIDA->m_reorganizeFilters.empty(); // if there are no filters - no need to filter out segment //if no filters - import segment @@ -223,47 +203,56 @@ VOID EQFMemOrganizeProcess bool result = filter.check(*pRIDA->pProposal); if(!result){ fSkipSegment = false; - //fFilterPassed = false; break; } } - - //if(fFilterPassed == false){ - if( fSkipSegment ){ - pRIDA->pMem->importDetails->filteredSegments++; - }else{ - pRIDA->pProposal->pInputSentence = std::make_unique(pRIDA->pProposal->szSource, pRIDA->pProposal->szTarget); - iRC = pRIDA->pMemTemp->putProposal( *(pRIDA->pProposal) ); - if ( iRC != 0 ) - { - T5LOG(T5ERROR) << "segment in reorganize was skipped! segment: "<< *pRIDA->pProposal; - pRIDA->pMem->importDetails->invalidSegments++; - if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ - pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), iRC)); + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + if( fSkipSegment ){ + pRIDA->pMem->importDetails->filteredSegments++; + }else{ + pRIDA->pProposal->pInputSentence = std::make_unique(pRIDA->pProposal->szSource, pRIDA->pProposal->szTarget); + iRC = pRIDA->pMemTemp->putProposal( *(pRIDA->pProposal) ); + if ( iRC != 0 ) + { + T5LOG(T5ERROR) << "segment in reorganize was skipped! segment: "<< *pRIDA->pProposal; + pRIDA->pMem->importDetails->invalidSegments++; + if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); + pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), iRC)); + } + { + //std::lock_guard lk(pRIDA->pMem->importDetails->invalidSegmentsRCsMutex); + pRIDA->pMem->importDetails->invalidSegmentsRCs[iRC] ++; + } } - pRIDA->pMem->importDetails->invalidSegmentsRCs[iRC] ++; + else + { + pRIDA->pMem->importDetails->segmentsImported++; + } /* endif */ } - else - { - pRIDA->pMem->importDetails->segmentsImported++; - } /* endif */ } } /* endif */ } - else if ( iRC == EqfMemory::ERROR_ENTRYISCORRUPTED// || iRC == BTREE_BUFFER_SMALL + else if ( iRC == EqfMemoryError::ERROR_ENTRYISCORRUPTED// || iRC == BTREE_BUFFER_SMALL ) { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); pCommArea->usComplete = (USHORT)iProgress; pRIDA->pMem->importDetails->invalidSegments++; - if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), -9)); } - T5LOG(T5ERROR) << "Skipping proposal iRC == EqfMemory::ERROR_ENTRYISCORRUPTED ; proposal :" << *pRIDA->pProposal; + T5LOG(T5ERROR) << "Skipping proposal iRC == EqfMemoryError::ERROR_ENTRYISCORRUPTED ; proposal :" << *pRIDA->pProposal; } - else if ( iRC == EqfMemory::INFO_ENDREACHED ) + else if ( iRC == EqfMemoryError::INFO_ENDREACHED ) { pRIDA->pMemTemp->NTMOrganizeIndexFile(); - T5LOG(T5WARNING) << "reorganize is done, segCount = " << pRIDA->pMem->importDetails->segmentsImported << "; invSegCount = " << pRIDA->pMem->importDetails->invalidSegments; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + T5LOG(T5WARNING) << "reorganize is done, segCount = " << pRIDA->pMem->importDetails->segmentsImported << "; invSegCount = " << pRIDA->pMem->importDetails->invalidSegments; + } int use_count = pRIDA->pMemTemp.use_count(); if(use_count != 1){ T5LOG(T5WARNING) << ":: use_count for temporary tm is not 1, but " << use_count; @@ -282,7 +271,7 @@ VOID EQFMemOrganizeProcess // everything in the termination process was ok // end message only in GUI single organize mode - if ( !pRIDA->fBatch && !pRIDA->pszNameList) + if ( !pRIDA->fBatch) { CHAR szNumber1[10]; // buffer for character string CHAR szNumber2[10]; // buffer for character string @@ -308,7 +297,10 @@ VOID EQFMemOrganizeProcess } /* endif */ } /* endif */ - pRIDA->pMem->importDetails->usProgress = pCommArea->usComplete + 1; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + pRIDA->pMem->importDetails->usProgress = pCommArea->usComplete + 1; + } // ----------------------------------------------------- // Issue message WM_EQF_MEMORGANIZE_END pRIDA->NextTask = MEM_END_ORGANIZE; @@ -345,35 +337,37 @@ VOID EQFMemOrganizeEnd PMEM_ORGANIZE_IDA pRIDA = (PMEM_ORGANIZE_IDA)pCommArea->pUserIDA; if(nullptr != pRIDA->pMemTemp){ - pRIDA->pMem->TmBtree.fb.data = std::move(pRIDA->pMemTemp->TmBtree.fb.data); + auto TMDFileLock = std::move(pRIDA->pMem->state->TmBtree.fb.file); + auto TMIFileLock = std::move(pRIDA->pMem->state->InBtree.fb.file); + pRIDA->pMem->state = std::move(pRIDA->pMemTemp->state); + pRIDA->pMem->state->TmBtree.fb.file = std::move(TMDFileLock); + pRIDA->pMem->state->InBtree.fb.file = std::move(TMIFileLock); + + //get signature record and add to control block - pRIDA->pMem->TmBtree.resetLookupTable(); - USHORT usLen = sizeof( TMX_SIGN ); - USHORT usRc = pRIDA->pMem->TmBtree.EQFNTMSign((PCHAR) &(pRIDA->pMem->stTmSign), &usLen ); - pRIDA->pMem->TmBtree.fb.Flush(true); - - pRIDA->pMem->InBtree.fb.data = std::move(pRIDA->pMemTemp->InBtree.fb.data); - pRIDA->pMem->InBtree.resetLookupTable(); - usLen = sizeof( TMX_SIGN ); - pRIDA->pMem->InBtree.fb.Flush(true); - pRIDA->pMem->ReloadFromDisk(); + USHORT usRc = pRIDA->pMem->sign(); + pRIDA->pMem->state->TmBtree.fb.Flush(true); + pRIDA->pMem->state->InBtree.fb.Flush(true); } LONG lCurTime = 0; time( &lCurTime ); - - if ( pRIDA->pMem->importDetails->lImportStartTime ) { - LONG lDiff = lCurTime - pRIDA->pMem->importDetails->lImportStartTime; - char buff[256]; - sprintf( buff, "Overall reorganize time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, - (lDiff - (lDiff / 3600 * 3600)) / 60, - (lDiff - (lDiff / 3600 * 3600)) % 60 ); - - pRIDA->pMem->importDetails->importTimestamp = buff; - } + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + if ( pRIDA->pMem->importDetails->lImportStartTime ) + { + LONG lDiff = lCurTime - pRIDA->pMem->importDetails->lImportStartTime; + char buff[256]; + sprintf( buff, "Overall reorganize time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, + (lDiff - (lDiff / 3600 * 3600)) / 60, + (lDiff - (lDiff / 3600 * 3600)) % 60 ); + + //std::lock_guard l(pRIDA->pMem->importDetails->importTimestampMutex); + pRIDA->pMem->importDetails->importTimestamp = buff; + } - pRIDA->pMem->importDetails->usProgress = 100; + pRIDA->pMem->importDetails->usProgress = 100; + } } /* end of function EQFMemOrganizeEnd */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index f25a6762..abd03721 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -19,8 +19,6 @@ //| "Q"=Quick-and-dirty ) | //+----------------------------------------------------------------------------+ //|Externals: CloseFile | -//| UtlDelete | -//| UtlAlloc | //| UtlDispatch | //| MemGetAddressOfProcessIDA | //| MemCreateProcess | @@ -134,7 +132,6 @@ USHORT EQFMemExportStart( PPROCESSCOMMAREA pCommArea, //--- Return codes are for testing purposes only if ( pExportIDA->pMem != NULL) { - TMManager::GetInstance()->closeMemory( pExportIDA->pMem ); pExportIDA->pMem = NULL; } /* endif */ pExportIDA->invalidXmlSegments = 0; @@ -264,11 +261,8 @@ USHORT EQFMemExportProcess ( PPROCESSCOMMAREA pCommArea, //--- set the progress indicator to 100 percent pCommArea->usComplete = 100; - if ( !pExportIDA->pszNameList ) - { - pReplString[2] = strcpy(szNumber, std::to_string( pExportIDA->ulSegmentCounter ).c_str()); - T5LOG( T5INFO) <<"EQFMemExportProcess::MESSAGE_MEM_EXPORT_COMPLETED"; - } /* endif */ + pReplString[2] = strcpy(szNumber, std::to_string( pExportIDA->ulSegmentCounter ).c_str()); + T5LOG( T5INFO) <<"EQFMemExportProcess::MESSAGE_MEM_EXPORT_COMPLETED"; //--- Issue message WM_EQF_MEMEXPORT_END pExportIDA->NextTask = MEM_END_EXPORT; @@ -361,12 +355,7 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) { - #ifdef TO_BE_REMOVED - if ( pExportIDA->pfnMemExpEnd ) - { - pExportIDA->pfnMemExpEnd( pExportIDA->lExternalExportHandle ); - } /* endif */ - #endif + if(pExportIDA->invalidXmlSegments){ T5LOG(T5ERROR) << pExportIDA->invalidXmlSegments <<" segments was not exported because of invalid xml"; } @@ -376,10 +365,6 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, delete pExportIDA->lExternalExportHandle ; } - - if ( pExportIDA->pstMemInfo ) UtlAlloc( (PVOID *)&pExportIDA->pstMemInfo, 0L, 0L, NOMSG ); - if ( pExportIDA->pstSegment ) UtlAlloc( (PVOID *)&pExportIDA->pstSegment, 0L, 0L, NOMSG ); - //if ( pExportIDA->hmodMemExtExport ) DosFreeModule( &(pExportIDA->hmodMemExtExport) ); } //--- Check if the termination was due to a @@ -405,8 +390,6 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, //--- Close MemoryDb and input file. Return codes are for testing purposes only if ( pExportIDA->pMem != NULL) { - TMManager *pFactory = TMManager::GetInstance(); - //pFactory->closeMemory( pExportIDA->pMem ); pExportIDA->pMem = NULL; } /* endif */ if ( pExportIDA->hFile ) CloseFile( &(pExportIDA->hFile) ); @@ -418,214 +401,16 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, REMOVESYMBOL( ObjName ); } - - // continue with next TM if any - if ( pExportIDA->pszNameList ) + if ( (hwndErrMsg == HWND_FUNCIF) && pExportIDA->fDataCorrupted ) { - pExportIDA->pszActiveName += strlen(pExportIDA->pszActiveName) + 1; - - if ( *pExportIDA->pszActiveName ) - { - BOOL fStartExport = FALSE; - TMManager *pFactory = TMManager::GetInstance(); - - do - { - pFactory->splitObjName( pExportIDA->pszActiveName, pExportIDA->szPlugin, sizeof(pExportIDA->szPlugin), pExportIDA->szMemName, sizeof(pExportIDA->szMemName) ); - - // build outfile name and get overwrite permission for existing files - UtlSplitFnameFromPath( pExportIDA->ControlsIda.szPathContent ); - strcat( pExportIDA->ControlsIda.szPathContent, BACKSLASH_STR ); - strcat( pExportIDA->ControlsIda.szPathContent, pExportIDA->szMemName ); - strcat( pExportIDA->ControlsIda.szPathContent, ".TMX" ); - - - if ( UtlFileExist( pExportIDA->ControlsIda.szPathContent ) ) - { - if ( pExportIDA->usYesToAllMode == MBID_EQF_YESTOALL ) - { - // we are already in yes-to-all mode so do nothing and continue export - fStartExport = TRUE; - } - else - { - USHORT usMBCode; - PSZ pszParm = pExportIDA->ControlsIda.szPathContent; - usMBCode = UtlError( ERROR_FILE_EXISTS_ALREADY, MB_EQF_YESTOALL, 1, - &pszParm, EQF_QUERY ); - if ( usMBCode == MBID_EQF_YESTOALL ) - { - // switch to yes-to-all mode and start export - pExportIDA->usYesToAllMode = MBID_EQF_YESTOALL; - fStartExport = TRUE; - } - else if ( usMBCode == MBID_CANCEL ) - { - // stop all further processing - *pExportIDA->pszActiveName = EOS; - } - else if ( usMBCode == MBID_NO ) - { - // skip this file by removing it from the name list - PSZ pszCurrent = pExportIDA->pszActiveName; - PSZ pszNext = pszCurrent + strlen(pszCurrent) + 1; - while ( *pszNext ) - { - while ( *pszNext ) *pszCurrent++ = *pszNext++; // copy string - *pszCurrent++ = *pszNext++; // copy string enddelimiter - } /* endwhile */ - *pszCurrent = EOS; // add end-of-list delimiter - } - else - { - // continue export - fStartExport = TRUE; - } /* endif */ - } /* endif */ - } - else - { - // output file does not exist, continue export - fStartExport = TRUE; - } /* endif */ - } while ( *pExportIDA->pszActiveName && !fStartExport ); - } /* endif */ - - - if ( pExportIDA->pszActiveName[0] ) - { - // prepare export of next TM - } - else - { - // show termination message - if ( SHORT1FROMMP2( mp2 ) == 0 ) - { - PSZ pszTMNames = NULL; // buffer for TM name list - - // compute size of required buffer - PSZ pszNext = pExportIDA->pszNameList; - int iSize = 5; - while ( *pszNext ) - { - int iLen = strlen(pszNext) + 1; - iSize += iLen + 5; - pszNext += iLen; - } /* endwhile */ - - // allocate buffer - UtlAlloc( (PVOID *)&pszTMNames, 0L, iSize, ERROR_STORAGE ); - - // fill buffer with TM names - if ( pszTMNames ) - { - PSZ pszNext = pExportIDA->pszNameList; - PSZ pszCurPos = pszTMNames; - while ( *pszNext ) - { - int iNameLen = 0; - - // skip any plugin name in the memory name - PSZ pszNamePos = strchr( pszNext, ':' ); - pszNamePos = (pszNamePos == NULL) ? pszNext : pszNamePos + 1; - - iNameLen = strlen( pszNamePos ); - *pszCurPos++ = '\"'; - strcpy( pszCurPos, pszNamePos ); - pszNext = pszNamePos + iNameLen + 1; - pszCurPos += iNameLen; - *pszCurPos++ = '\"'; - if ( *pszNext ) - { - // add delimiters for following TM name - *pszCurPos++ = ','; - *pszCurPos++ = ' '; - } /* endif */ - } /* endwhile */ - } /* endif */ - UtlError ( MESSAGE_MEM_EXP_COMPLETED_MUL, MB_OK, 1, &pszTMNames, EQF_INFO ); - if ( pszTMNames ) UtlAlloc( (PVOID *)&pszTMNames, 0L, 0L, NOMSG ); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( pExportIDA->pszNameList && pExportIDA->pszActiveName[0] ) - { - BOOL fIsNew = FALSE; - BOOL fOK = TRUE; - int iRC = 0; - - // restart export with active TM - pExportIDA->fFirstExtract = TRUE; - pExportIDA->fEOF = FALSE; - pExportIDA->ulSegmentCounter = 0L; - pExportIDA->hFile = NULLHANDLE; - - TMManager *pFactory = TMManager::GetInstance(); - T5LOG(T5FATAL) << "TEMPORARY_COMMENTED CODE pExportIDA->pMem = pFactory->openMemory( pExportIDA->szPlugin, pExportIDA->szMemName, EXCLUSIVE, &iRC );"; - #ifdef TEMPORARY_COMMENTED - pExportIDA->pMem = pFactory->openMemory( pExportIDA->szPlugin, pExportIDA->szMemName, EXCLUSIVE, &iRC ); - #endif - if ( pExportIDA->pMem == NULL ) - { - T5LOG(T5ERROR) << "(pExportIDA->pMem == NULL ), memName = \""<szMemName <<"\""; - fOK = FALSE; - } - - if ( fOK ) - { - USHORT usDosRc, usAction; - - - usDosRc = UtlOpen( pExportIDA->ControlsIda.szPathContent, &(pExportIDA->hFile), &usAction, - 0L, FILE_NORMAL, FILE_TRUNCATE | FILE_CREATE, - (USHORT)(OPEN_ACCESS_WRITEONLY | OPEN_SHARE_DENYWRITE | OPEN_FLAGS_WRITE_THROUGH), - 0L, TRUE ); - if ( usDosRc ) - { - pExportIDA->hFile = NULLHANDLE; - fOK = FALSE; - if ( pExportIDA->pMem != NULL) - { - TMManager::GetInstance()->closeMemory( pExportIDA->pMem ); - pExportIDA->pMem = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !fOK ) - { - // stop processing - pExportIDA->pszActiveName[0] = EOS; - } /* endif */ + PSZ pszParm = "ANSI/ASCII"; + T5LOG(T5ERROR) << "::ERROR_MEM_EXPORT_DATACORRUPT::" << pszParm; + LOG_AND_SET_RC(usRC, T5INFO, ERROR_MEM_EXPORT_DATACORRUPT); } /* endif */ - if ( pExportIDA->pszNameList && pExportIDA->pszActiveName[0] ) - { - // restart TM export - } - else - { - //--- Free data areas - if ( pExportIDA->pszNameList ) UtlAlloc( (PVOID *)&pExportIDA->pszNameList, 0L, 0L, NOMSG ); - - if ( (hwndErrMsg == HWND_FUNCIF) && pExportIDA->fDataCorrupted ) - { - PSZ pszParm = "ANSI/ASCII"; - T5LOG(T5ERROR) << "::ERROR_MEM_EXPORT_DATACORRUPT::" << pszParm; - LOG_AND_SET_RC(usRC, T5INFO, ERROR_MEM_EXPORT_DATACORRUPT); - } /* endif */ - - if ( pExportIDA->pProposal != NULL ) delete(pExportIDA->pProposal); - UtlAlloc( (PVOID *) &pExportIDA, 0L, 0L, NOMSG ); - pCommArea->pUserIDA = NULL; - - // Dismiss the slider window if it had been created - if ( hwndErrMsg != HWND_FUNCIF ) - { - //EqfRemoveObject( TWBFORCE, hWnd ); - } /* endif */ - } + if ( pExportIDA->pProposal != NULL ) delete(pExportIDA->pProposal); + UtlAlloc( (PVOID *) &pExportIDA, 0L, 0L, NOMSG ); + pCommArea->pUserIDA = NULL; return( usRC ); } /* end of function EQFMemExportEnd */ @@ -706,15 +491,6 @@ USHORT MemExportStart( PPROCESSCOMMAREA pCommArea, if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) { - // allocate memory and segment data area - if ( !UtlAlloc( (PVOID *)&pExportIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING,ERROR_NOT_ENOUGH_MEMORY); - } - else if ( !UtlAlloc( (PVOID *)&pExportIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ // load external memory export module EqfPluginWrapper::init(); @@ -722,30 +498,30 @@ USHORT MemExportStart( PPROCESSCOMMAREA pCommArea, // call start entry point if ( !usRc ) { - memset( pExportIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + memset( &pExportIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); // get name of first markup table if ( pExportIDA->pMem->getNumOfMarkupNames() >= 1 ) { - pExportIDA->pMem->getMarkupName( 0, pExportIDA->pstMemInfo->szFormat, sizeof(pExportIDA->pstMemInfo->szFormat) ); + pExportIDA->pMem->getMarkupName( 0, pExportIDA->pstMemInfo.szFormat, sizeof(pExportIDA->pstMemInfo.szFormat) ); } /* end */ - strcpy( pExportIDA->pstMemInfo->szName, pExportIDA->szMemName ); - pExportIDA->pstMemInfo->responseHandler = pCommArea->responseHandler; - pExportIDA->pMem->getDescription( pExportIDA->pstMemInfo->szDescription, sizeof(pExportIDA->pstMemInfo->szDescription) ); - pExportIDA->pMem->getSourceLanguage( pExportIDA->pstMemInfo->szSourceLang, sizeof(pExportIDA->pstMemInfo->szSourceLang) ); - pExportIDA->pstMemInfo->fUTF16 = (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) ; - pExportIDA->pstMemInfo->fNoCRLF = (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF); + strcpy( pExportIDA->pstMemInfo.szName, pExportIDA->szMemName ); + pExportIDA->pstMemInfo.responseHandler = pCommArea->responseHandler; + pExportIDA->pMem->getDescription( pExportIDA->pstMemInfo.szDescription, sizeof(pExportIDA->pstMemInfo.szDescription) ); + pExportIDA->pMem->getSourceLanguage( pExportIDA->pstMemInfo.szSourceLang, sizeof(pExportIDA->pstMemInfo.szSourceLang) ); + pExportIDA->pstMemInfo.fUTF16 = (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) ; + pExportIDA->pstMemInfo.fNoCRLF = (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF); - T5LOG( T5INFO) <<"MemExportStart::calling external function, mem name = " << pExportIDA->pstMemInfo->szName << "; source lang = "<< pExportIDA->pstMemInfo->szSourceLang<< - "; markup = " << pExportIDA->pstMemInfo->szFormat; + T5LOG( T5INFO) <<"MemExportStart::calling external function, mem name = " << pExportIDA->pstMemInfo.szName << "; source lang = "<< pExportIDA->pstMemInfo.szSourceLang<< + "; markup = " << pExportIDA->pstMemInfo.szFormat; //usRc = EXTMEMEXPORTSTART( ); { pExportIDA->lExternalExportHandle = new CTMXExportImport; pExportIDA->lExternalExportHandle->m_xw.bufQueue = pCommArea->pBufQueue; - pExportIDA->lExternalExportHandle->WriteHeader( pExportIDA->ControlsIda.szPathContent, pExportIDA->pstMemInfo ); + pExportIDA->lExternalExportHandle->WriteHeader( pExportIDA->ControlsIda.szPathContent, &pExportIDA->pstMemInfo ); } } /* endif */ } @@ -843,9 +619,9 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export } int iTmRC = pExportIDA->pMem->getNextProposal( *(pExportIDA->pProposal), &(pExportIDA->iComplete) ); - if ( (iTmRC == NO_ERROR) || iTmRC == EqfMemory::INFO_ENDREACHED ) + if ( (iTmRC == NO_ERROR) || iTmRC == EqfMemoryError::INFO_ENDREACHED ) { - if ( iTmRC == EqfMemory::INFO_ENDREACHED ) + if ( iTmRC == EqfMemoryError::INFO_ENDREACHED ) { //--- Stop Address has been reached before END_OF_TM pExportIDA->fEOF = TRUE; @@ -859,14 +635,12 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export { usRc = MemExportWriteFile( pExportIDA ); } /* endif */ - } else { //--- Increase the segment counter by one pExportIDA->ulSegmentCounter++; - //--- Prepare the segment and write it to the export file if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) @@ -874,37 +648,37 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export // reset first extract flag (in non-TMX mode this is done in MemExportWriteFile) pExportIDA->fFirstExtract = FALSE; - memset( pExportIDA->pstSegment, 0, sizeof(MEMEXPIMPSEG) ); + memset( &pExportIDA->pstSegment, 0, sizeof(MEMEXPIMPSEG) ); - pExportIDA->pstSegment->fValid = TRUE; - pExportIDA->pstSegment->usTranslationFlag = 0; - if ( pExportIDA->pProposal->getType() == OtmProposal::eptMachine ) pExportIDA->pstSegment->usTranslationFlag = 1; - if ( pExportIDA->pProposal->getType() == OtmProposal::eptGlobalMemory ) pExportIDA->pstSegment->usTranslationFlag = 2; + pExportIDA->pstSegment.fValid = TRUE; + pExportIDA->pstSegment.usTranslationFlag = 0; + if ( pExportIDA->pProposal->getType() == OtmProposal::eptMachine ) pExportIDA->pstSegment.usTranslationFlag = 1; + if ( pExportIDA->pProposal->getType() == OtmProposal::eptGlobalMemory ) pExportIDA->pstSegment.usTranslationFlag = 2; - pExportIDA->pstSegment->lSegNum = pExportIDA->pProposal->getSegmentId(); - pExportIDA->pstSegment->lTime = pExportIDA->pProposal->getUpdateTime(); - //pExportIDA->pMem->getSourceLanguage( pExportIDA->pstSegment->szSourceLang, sizeof(pExportIDA->pstSegment->szSourceLang) ); - pExportIDA->pProposal->getOriginalSourceLanguage( pExportIDA->pstSegment->szSourceLang, sizeof(pExportIDA->pstSegment->szSourceLang) ); + pExportIDA->pstSegment.lSegNum = pExportIDA->pProposal->getSegmentId(); + pExportIDA->pstSegment.lTime = pExportIDA->pProposal->getUpdateTime(); + //pExportIDA->pMem->getSourceLanguage( pExportIDA->pstSegment.szSourceLang, sizeof(pExportIDA->pstSegment.szSourceLang) ); + pExportIDA->pProposal->getOriginalSourceLanguage( pExportIDA->pstSegment.szSourceLang, sizeof(pExportIDA->pstSegment.szSourceLang) ); - pExportIDA->pProposal->getTargetLanguage( pExportIDA->pstSegment->szTargetLang, sizeof(pExportIDA->pstSegment->szTargetLang) ); - pExportIDA->pProposal->getAuthor( pExportIDA->pstSegment->szAuthor, sizeof(pExportIDA->pstSegment->szAuthor) ); - pExportIDA->pProposal->getMarkup( pExportIDA->pstSegment->szFormat, sizeof(pExportIDA->pstSegment->szFormat) ); - pExportIDA->pProposal->getDocName( pExportIDA->pstSegment->szDocument, sizeof(pExportIDA->pstSegment->szDocument) ); + pExportIDA->pProposal->getTargetLanguage( pExportIDA->pstSegment.szTargetLang, sizeof(pExportIDA->pstSegment.szTargetLang) ); + pExportIDA->pProposal->getAuthor( pExportIDA->pstSegment.szAuthor, sizeof(pExportIDA->pstSegment.szAuthor) ); + pExportIDA->pProposal->getMarkup( pExportIDA->pstSegment.szFormat, sizeof(pExportIDA->pstSegment.szFormat) ); + pExportIDA->pProposal->getDocName( pExportIDA->pstSegment.szDocument, sizeof(pExportIDA->pstSegment.szDocument) ); - pExportIDA->pProposal->getSource( pExportIDA->pstSegment->szSource, sizeof(pExportIDA->pstSegment->szSource) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getTarget( pExportIDA->pstSegment->szTarget, sizeof(pExportIDA->pstSegment->szTarget) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getContext( pExportIDA->pstSegment->szContext, sizeof(pExportIDA->pstSegment->szContext) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getAddInfo( pExportIDA->pstSegment->szAddInfo, sizeof(pExportIDA->pstSegment->szAddInfo) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getInternalKey( pExportIDA->pstSegment->szInternalKey, sizeof(pExportIDA->pstSegment->szInternalKey) ); + pExportIDA->pProposal->getSource( pExportIDA->pstSegment.szSource, sizeof(pExportIDA->pstSegment.szSource) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getTarget( pExportIDA->pstSegment.szTarget, sizeof(pExportIDA->pstSegment.szTarget) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getContext( pExportIDA->pstSegment.szContext, sizeof(pExportIDA->pstSegment.szContext) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getAddInfo( pExportIDA->pstSegment.szAddInfo, sizeof(pExportIDA->pstSegment.szAddInfo) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getInternalKey( pExportIDA->pstSegment.szInternalKey, sizeof(pExportIDA->pstSegment.szInternalKey) ); auto ll = T5Logger::GetInstance()->suppressLogging(); - bool fValidXml = IsValidXml( pExportIDA->pstSegment->szSource); + bool fValidXml = IsValidXml( pExportIDA->pstSegment.szSource); if(fValidXml){ - fValidXml = IsValidXml( pExportIDA->pstSegment->szTarget); + fValidXml = IsValidXml( pExportIDA->pstSegment.szTarget); T5Logger::GetInstance()->desuppressLogging(ll); if(fValidXml){ - usRc = pExportIDA->lExternalExportHandle->WriteSegment( pExportIDA->pstSegment ); + usRc = pExportIDA->lExternalExportHandle->WriteSegment( &pExportIDA->pstSegment ); if(!usRc){ pExportIDA->segmentsExported++; if(pExportIDA->numOfRequestedSegmentsForExport > 0 && @@ -1554,7 +1328,6 @@ USHORT MemExportWrite // prepare the function I/F TM import USHORT FCTDATA::MemFuncPrepExport ( - //PFCTDATA pData, // function I/F session data PSZ pszMemName, // name of Translation Memory PSZ pszOutFile, // fully qualified name of output file LONG lOptions, // options for Translation Memory export @@ -1576,15 +1349,15 @@ USHORT FCTDATA::MemFuncPrepExport UtlSetUShort( QS_LASTERRORMSGID, NO_ERROR ); // allocate storage for the organize process communication area - fOK = UtlAllocHwnd( (PVOID *)&pCommArea, 0L, + fOK = UtlAlloc( (PVOID *)&pCommArea, 0L, (LONG)sizeof(PROCESSCOMMAREA), - ERROR_STORAGE, HWND_FUNCIF ); + ERROR_STORAGE ); // allocate storage for the MEM_EXPORT_IDA if ( fOK ) { - fOK = UtlAllocHwnd( (PVOID *)&pIDA, 0L, (LONG)sizeof(MEM_EXPORT_IDA), - ERROR_STORAGE, HWND_FUNCIF ); + fOK = UtlAlloc( (PVOID *)&pIDA, 0L, (LONG)sizeof(MEM_EXPORT_IDA), + ERROR_STORAGE ); } /* endif */ if (fOK) @@ -1698,8 +1471,6 @@ USHORT FCTDATA::MemFuncPrepExport { if ( pIDA->pMem != NULL) { - TMManager *pFactory = TMManager::GetInstance(); - pFactory->closeMemory( pIDA->pMem ); pIDA->pMem = NULL; } /* endif */ if ( pIDA->hFile ) @@ -1769,8 +1540,6 @@ USHORT FCTDATA::MemFuncExportProcess() break; } /* endswitch */ this->usExportProgress = pCommArea->usComplete; - //this->recordKey = pIDA->pMem->ulNextKey; - //this->targetKey = pIDA->pMem->usNextTarget; return( usRC ); } /* end of function MemFuncExportProcess */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index b1289af0..32a5fc11 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -20,8 +20,6 @@ //| "Q"=Quick-and-dirty ) | //+----------------------------------------------------------------------------+ //|Externals: | -//| | -//| UtlAlloc | //| UtlQFileInfo | //| UtlMakeEQFPath | //| UtlLoadFile | @@ -88,7 +86,7 @@ //--- declaration of internal functions static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pIda, HWND, ImportStatusDetails* pImportData); -static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA ); + USHORT /*APIENTRY*/ MEMINSERTSEGMENT @@ -98,7 +96,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT ); -static void GetElapsedTime( long long *plTime ) +void GetElapsedTime( long long *plTime ) { if(plTime) *plTime = 0; //struct timespec now; @@ -106,12 +104,6 @@ static void GetElapsedTime( long long *plTime ) //*plTime = now.tv_sec + now.tv_nsec / 1000000000.0; } -USHORT MemHandleCodePageValue -( - MEM_LOAD_DLG_IDA* pLIDA, // ptr to memory load IDA - PSZ_W pszCodePage, // ptr to specified code page value - PBOOL pfRestartImport // caller's restart the import flag -); // import logging @@ -122,7 +114,6 @@ USHORT MemHandleCodePageValue //#endif -USHORT MemLoadAndConvert( MEM_LOAD_DLG_IDA* pLIDA, PSZ_W pszBuffer, ULONG usSize, PULONG pusBytesRead ); INT_PTR /*CALLBACK*/ NTMTagLangDlg( HWND, // Dialog window handle WINMSG, // Message ID @@ -249,18 +240,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, { fOK = FALSE; } - else - { - /* - // Create full path to the memory database format table - Properties::GetInstance()->get_value( KEY_OTM_DIR, pLIDA->szFullFtabPath, MAX_EQF_PATH ); - strcat(pLIDA->szFullFtabPath, "/TABLE/"); - strcat(pLIDA->szFullFtabPath, MEM_FORMAT_TABLE); - - // Load the format table and allocate storage for it - fOK = ( TALoadTagTableHwnd( MEM_FORMAT_TABLE, &pLIDA->pFormatTable, - TRUE, TRUE, pLIDA->hwndErrMsg ) == NO_ERROR);//*/ - } /* endif */ + } /* endif */ // If fOK. Allocate storage for the tokenlist and move address in @@ -268,9 +248,9 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // If an error occurred set fOK to FALSE and issue appropriate message. if ( fOK ) { - fOK = UtlAllocHwnd( (PVOID *)&pLIDA->pTokenList, 0L, + fOK = UtlAlloc( (PVOID *)&pLIDA->pTokenList, 0L, (LONG)(NUMB_OF_TOKENS * sizeof(TOKENENTRY)), - ERROR_STORAGE, pLIDA->hwndErrMsg ); + ERROR_STORAGE ); if ( fOK ) { // Initialize pTokenEntry and pTokenEntryWork @@ -286,79 +266,13 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // use the default token values if ( fOK ) { - PWCHAR pRest = NULL; - USHORT usLastColPos = 0; - /* - pRest = NULL; - TATagTokenizeW( MEM_CONTEXT_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTROL_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_DESCRIPTION_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_MEMORYDB_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_MEMORYDB_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SEGMENT_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SOURCE_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_TARGET_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTEXT_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTROL_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_DESCRIPTION_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_MEMORYDB_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_MEMORYDB_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SEGMENT_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SOURCE_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_TARGET_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CODEPAGE_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CODEPAGE_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_ADDDATA_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_ADDDATA_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - //*/ pLIDA->pTokenList->sTokenid = ENDOFLIST; // enable correct start of import process } /* endif */ + // special handling for import in TMX format if ( fOK && ((pLIDA->usImpMode == MEM_FORMAT_TMX) || (pLIDA->usImpMode == MEM_FORMAT_TMX_TRADOS)) ) { - // allocate memory and segment data area - fOK = UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ); - if ( fOK ) fOK = UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ); - // call start entry point if ( fOK ) { @@ -371,23 +285,23 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, pLIDA->hFile = NULL; } /* endif */ - memset( pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); - strcpy( pLIDA->pstMemInfo->szName, pLIDA->szMemName ); - pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo->szSourceLang, sizeof(pLIDA->pstMemInfo->szSourceLang) ); + memset( &pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + strcpy( pLIDA->pstMemInfo.szName, pLIDA->szMemName ); + pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo.szSourceLang, sizeof(pLIDA->pstMemInfo.szSourceLang) ); if ( pLIDA->usImpMode == MEM_FORMAT_TMX_TRADOS ) { - pLIDA->pstMemInfo->fCleanRTF = TRUE; - pLIDA->pstMemInfo->fTagsInCurlyBracesOnly = TRUE; + pLIDA->pstMemInfo.fCleanRTF = TRUE; + pLIDA->pstMemInfo.fTagsInCurlyBracesOnly = TRUE; } memset(&(pLIDA->TMXImport), 0 ,sizeof(pLIDA->TMXImport)); - fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, pLIDA->pstMemInfo, pImportData ) == 0); + fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, &pLIDA->pstMemInfo, pImportData ) == 0); pLIDA->mem->getSourceLanguage( szMemSourceLang, sizeof(szMemSourceLang) ); // check if memory source lanuage matchs source language of imported file - if ( pLIDA->pstMemInfo->szSourceLang[0] && (strcmp( pLIDA->pstMemInfo->szSourceLang, szMemSourceLang ) != 0) ) + if ( pLIDA->pstMemInfo.szSourceLang[0] && (strcmp( pLIDA->pstMemInfo.szSourceLang, szMemSourceLang ) != 0) ) { - pReplAddr[0] = pLIDA->pstMemInfo->szSourceLang; + pReplAddr[0] = pLIDA->pstMemInfo.szSourceLang; pReplAddr[1] = szMemSourceLang; T5LOG(T5INFO) << "::ERROR_MEM_DIFFERENT_SOURCE_LANG_IMPORT::" << pReplAddr[0] << "; szMemSourceLang = " << pReplAddr[1]; @@ -397,7 +311,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // update memory description with description of imported memory if ( fOK ) { - pLIDA->mem->setDescription( pLIDA->pstMemInfo->szDescription ); + pLIDA->mem->setDescription( pLIDA->pstMemInfo.szDescription ); } /* endif */ } /* endif */ @@ -408,30 +322,25 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // special handling for import in XLIFF MT format if ( fOK && ((pLIDA->usImpMode == MEM_FORMAT_XLIFF_MT)) ) { - // allocate memory and segment data area - fOK = UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ); - if ( fOK ) fOK = UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ); - - // call start entry point if ( fOK ) { char szMemSourceLang[MAX_LANG_LENGTH]; - memset( pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); - strcpy( pLIDA->pstMemInfo->szName, pLIDA->szMemName ); - pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo->szSourceLang, sizeof(pLIDA->pstMemInfo->szSourceLang) ); + memset( &pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + strcpy( pLIDA->pstMemInfo.szName, pLIDA->szMemName ); + pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo.szSourceLang, sizeof(pLIDA->pstMemInfo.szSourceLang) ); - fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, pLIDA->pstMemInfo, pImportData ) == 0); + fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, &pLIDA->pstMemInfo, pImportData ) == 0); pLIDA->mem->getSourceLanguage( szMemSourceLang, sizeof(szMemSourceLang) ); // check if memory source lanuage matchs source language of imported file - if ( pLIDA->pstMemInfo->szSourceLang[0] && (strcmp( pLIDA->pstMemInfo->szSourceLang, szMemSourceLang ) != 0) ) + if ( pLIDA->pstMemInfo.szSourceLang[0] && (strcmp( pLIDA->pstMemInfo.szSourceLang, szMemSourceLang ) != 0) ) { USHORT usMBCode; - pReplAddr[0] = pLIDA->pstMemInfo->szSourceLang; + pReplAddr[0] = pLIDA->pstMemInfo.szSourceLang; pReplAddr[1] = szMemSourceLang; T5LOG(T5INFO) << "::ERROR_MEM_DIFFERENT_SOURCE_LANG_IMPORT::", pReplAddr[0] ; @@ -440,7 +349,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // update memory description with description of imported memory if ( fOK ) { - pLIDA->mem->setDescription( pLIDA->pstMemInfo->szDescription ); + pLIDA->mem->setDescription( pLIDA->pstMemInfo.szDescription ); } /* endif */ } /* endif */ @@ -475,6 +384,8 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, if ( pLIDA->pProposal != NULL ) delete(pLIDA->pProposal); } /* endif */ }else{ + + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); time( &pLIDA->mem->importDetails->lImportStartTime ); T5LOG(T5DEBUG) << "************ Memory Import Log ********************* Memory import started at : "<< asctime( localtime( &pLIDA->mem->importDetails->lImportStartTime ) )<< "\tMemory name : " << pLIDA->szMemName ; @@ -490,7 +401,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, //+----------------------------------------------------------------------------+ //|Function name: MemLoadProcess | //+----------------------------------------------------------------------------+ -//|Function call: static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA ) | +//|Function call: static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA )| //+----------------------------------------------------------------------------+ //|Description: Import a segment to TM | //| read a block of data from a file if needed and add a | @@ -573,9 +484,9 @@ static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA, ImportStatusDetails* usRc = pLIDA->TMXImport.ImportNext( MEMINSERTSEGMENT, (LONG)pLIDA, pImportData); // handle any error conditions - if ( pLIDA->pstMemInfo->fError ) + if ( pLIDA->pstMemInfo.fError ) { - PSZ pszMsg = pLIDA->pstMemInfo->szError; + PSZ pszMsg = pLIDA->pstMemInfo.szError; T5LOG(T5ERROR) << "::ERROR_MEMIMP_ERROR::MEM_SEG_SYN_ERR::" << pszMsg ; LOG_AND_SET_RC(usRc, T5INFO, MEM_SEG_SYN_ERR); } /* endif */ @@ -659,18 +570,13 @@ USHORT MemFuncPrepImport /* it will be freed here. */ /*******************************************************************/ memset(pLoadIDA, 0, sizeof(*pLoadIDA)); - //fOK = UtlAllocHwnd( (PVOID *)&pLoadIDA, 0L, (LONG)sizeof(MEM_LOAD_DLG_IDA), ERROR_STORAGE, HWND_FUNCIF ); - if ( fOK ) - { // use def.target lang. from system preferences of TM - pLoadIDA->ulOemCP = 1; - pLoadIDA->ulAnsiCP = 1; - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport:: memory allocation fails"; - } - + //if ( fOK ) + { // use def.target lang. from system preferences of TM + pLoadIDA->ulOemCP = 1; + pLoadIDA->ulAnsiCP = 1; + } + // set options for import - if ( fOK ) { pLoadIDA->fSkipInvalidSegments = ( lOptions & IGNORE_OPT ); pLoadIDA->fAscii = TRUE; // TM in external format @@ -720,57 +626,21 @@ USHORT MemFuncPrepImport } /* endif */ // prepare segment match ID prefix - if ( fOK ){ - pLoadIDA->szMatchIDPrefix[0] = '\0'; - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::set options for import fails"; - } /* endif */ - + pLoadIDA->szMatchIDPrefix[0] = '\0'; + /*******************************************************************/ /* Check if there is a TM with the given name */ /*******************************************************************/ - if ( fOK ) - { - strcpy( pLoadIDA->szMemName, pData->mem.get()->szName ); - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::prepare segment match ID prefix fails"; - } /* endif */ - - /*******************************************************************/ - /* Open the TM */ - /*******************************************************************/ - // - //if ( fOK ) - //{ - // int iRC = 0; - // TMManager *pFactory = TMManager::GetInstance(); - // pLoadIDA->pMem = pFactory->openMemory( NULL, pszMemName, EXCLUSIVE, &iRC ); - // if ( iRC != 0 || pLoadIDA->pMem == nullptr ) - // { - // fOK = FALSE; - // pFactory->showLastError( NULL, pszMemName, pLoadIDA->pMem, (HWND)HWND_FUNCIF ); - // } /* end */ - //}else if(!fOkFailHandled){ - // fOkFailHandled = true; - // T5LOG(T5ERROR) << "MemFuncPrepImport::Check if there is a TM with the given name fails"; - //} /* endif */ + strcpy( pLoadIDA->szMemName, pData->mem->szName.c_str() ); /*******************************************************************/ /* Check if input file exists */ /*******************************************************************/ - if ( fOK ) - { - if ( pData->fileData.empty() && !UtlFileExist( pszInFile ) ) - { - pszParm = pszInFile; - T5LOG(T5ERROR) << "::FILE_NOT_EXISTS::" << pszParm; - fOK = FALSE; - } /* endif */ - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::Open the TM fails"; + if ( pData->fileData.empty() && !UtlFileExist( pszInFile ) ) + { + pszParm = pszInFile; + T5LOG(T5ERROR) << "::FILE_NOT_EXISTS::" << pszParm; + fOK = FALSE; } /* endif */ /*******************************************************************/ @@ -778,7 +648,6 @@ USHORT MemFuncPrepImport /*******************************************************************/ if ( fOK ) { - //USHORT usDosRc; // Return code from Dos functions USHORT usAction; // Action for DosOpen if(pData->fileData.empty()){ strcpy( pLoadIDA->szFilePath, pszInFile ); @@ -790,12 +659,6 @@ USHORT MemFuncPrepImport }else{ pLoadIDA->hFile = nullptr; } - //if ( usDosRc ) //--- error from open - //{ - // //--- Reset file handle to NULL and set fOk to FALSE - // pLoadIDA->hFile = NULLHANDLE; - // fOK = FALSE; - //}/* endif */ }else if(!fOkFailHandled){ fOkFailHandled = true; T5LOG(T5ERROR) << "MemFuncPrepImport::Check if input file exists fails"; @@ -821,17 +684,10 @@ USHORT MemFuncPrepImport T5LOG(T5ERROR) << "MemFuncPrepImport::fOk in false, making Cleanup"; if ( pLoadIDA ) { - //if ( pLoadIDA->pMem ) - //{ - // TMManager *pFactory = TMManager::GetInstance(); - // pFactory->closeMemory( pLoadIDA->pMem ); - // pLoadIDA->pMem = NULL; - //} /* endif */ - if ( pLoadIDA->hFile && pLoadIDA->hFile ) + if ( pLoadIDA->hFile ) { UtlClose( pLoadIDA->hFile, FALSE ); } /* endif */ - //UtlAlloc( (PVOID *) &pLoadIDA, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ @@ -869,42 +725,46 @@ USHORT MemFuncImportProcess { case MEM_START_IMPORT: T5LOG( T5INFO) << "::MEM_START_IMPORT"; - pData->mem->importDetails->usProgress = 0; - - if(!pData->fileData.empty()){ - pLIDA->fileData = std::move(pData->fileData); + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pData->mem->importDetails->usProgress = 0; } - if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails ) ) - { - //TMManager *pFactory = TMManager::GetInstance(); - //pFactory->closeMemory( pLIDA->mem ); - if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); - pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); - if ( pData->usMemLoadRC == 0 ) pData->usMemLoadRC = ERROR_MEMIMP_ERROR; - usPhase = 0; + if(!pData->fileData.empty()){ + pLIDA->fileData = std::move(pData->fileData); } - else { - usPhase = MEM_IMPORT_TASK; - } /* endif */ - //UtlAlloc( (PVOID *) &pLIDA, 0L, 0L, NOMSG ); + if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails.get() ) ) + { + if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); + pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); + if ( pData->usMemLoadRC == 0 ) pData->usMemLoadRC = ERROR_MEMIMP_ERROR; + usPhase = 0; + } + else + { + usPhase = MEM_IMPORT_TASK; + } /* endif */ + } break; case MEM_IMPORT_TASK: { T5LOG( T5DEVELOP) << "::MEM_IMPORT_TASK, progress = " << pData->mem->importDetails->usProgress; - USHORT usRc = MemLoadProcess( pLIDA, pData->mem->importDetails ); + USHORT usRc = MemLoadProcess( pLIDA, pData->mem->importDetails.get() ); switch ( usRc ) { case MEM_PROCESS_OK: usPhase = MEM_IMPORT_TASK; break; case MEM_PROCESS_END: + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); if(!pData->mem->importDetails->importRc){ pData->mem->importDetails->usProgress = 100; } + } if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); if ( pLIDA->mem ) { @@ -920,7 +780,7 @@ USHORT MemFuncImportProcess pLIDA = (MEM_LOAD_DLG_IDA*)&pData->MemLoadIda; if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); pData->usMemLoadRC = usRC = usRc? usRc : UtlQueryUShort( QS_LASTERRORMSGID ); - strncpy(pData->szError, pLIDA->pstMemInfo->szError, sizeof(pData->szError)); + strncpy(pData->szError, pLIDA->pstMemInfo.szError, sizeof(pData->szError)); usPhase = MEM_END_IMPORT; break; } /* end switch */ @@ -935,9 +795,6 @@ USHORT MemFuncImportProcess ( pLIDA->usImpMode == MEM_FORMAT_XLIFF_MT ) ) { pLIDA->TMXImport.WriteEnd(); - - if ( pLIDA->pstMemInfo ) UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, 0L, NOMSG ); - if ( pLIDA->pstSegment) UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, 0L, NOMSG ); } /* endif */ @@ -950,10 +807,11 @@ USHORT MemFuncImportProcess { LONG lCurTime = 0; - time( &lCurTime ); - + time( &lCurTime ); + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); if ( pData->mem->importDetails->lImportStartTime ) { + //std::scoped_lock l(pData->mem->importDetails->importTimestampMutex); LONG lDiff = lCurTime - pData->mem->importDetails->lImportStartTime; char buff[256]; sprintf( buff, "Overall import time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, @@ -964,6 +822,7 @@ USHORT MemFuncImportProcess } if(VLOG_IS_ON(1)){ + //std::scoped_lock l(pData->mem->importDetails->importTimestampMutex); T5LOG( T5TRANSACTION) << "::Memory import for \""<< pData->mem->szName << "\" is finished at : " << asctime( localtime( &lCurTime )) << "\tNumber of segments imported : " << pData->mem->importDetails->segmentsImported << "\tNumber of invalid segments : " << pData->mem->importDetails->invalidSegments << @@ -975,9 +834,12 @@ USHORT MemFuncImportProcess if ( pLIDA->pProposal != NULL ) delete(pLIDA->pProposal); } /* endif */ usPhase = 0;; - if(!pData->mem->importDetails->importRc){ + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + if(!pData->mem->importDetails->importRc){ pData->mem->importDetails->usProgress = 100; } + } break; } /* endswitch */ @@ -993,378 +855,6 @@ USHORT MemFuncImportProcess } /* end of function MemFuncImportProcess */ -// load some data from file and convert it if necessary -// usSize is size of pszBuffer(filled as output!) in number of CHAR_W's - -USHORT MemLoadAndConvert( MEM_LOAD_DLG_IDA* pLIDA, PSZ_W pszBuffer, ULONG ulSize, PULONG pulBytesRead ) -{ - USHORT usDosRc = NO_ERROR; - ULONG ulBytesRead = 0; - -#ifdef IMPORT_LOGGING - GetElapsedTime( &lOtherTime ); -#endif - - switch ( pLIDA->usImpMode ) - { - case SGMLFORMAT_ANSI : - { - PSZ pConvBuffer = NULL; - LONG lBytesLeft = 0L; - LONG lRc = 0; - ULONG ulUnicodeChars = 0; - - if ( UtlAlloc( (PVOID *)&pConvBuffer, 0L, ulSize+1, ERROR_STORAGE ) ) - { - usDosRc = UtlReadHwnd( pLIDA->hFile, pConvBuffer, ulSize, &ulBytesRead, - TRUE, pLIDA->hwndErrMsg ); - if (ulBytesRead != ulSize ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // ensure that there are no DBCS characters splitted at the end of - // the data just read into memory - if ( ulBytesRead && !pLIDA->fEOF ) - { - // non DBCS case --- no problem at all - ulUnicodeChars = UtlDirectAnsi2UnicodeBuf( pConvBuffer, pszBuffer, - ulBytesRead, pLIDA->ulAnsiCP, FALSE, &lRc, - &lBytesLeft ); - usDosRc = (USHORT)lRc; - if ( lBytesLeft ) - { // undo the last character read... - // reposition file pointer - ULONG ulCurrentPos = 0; - UtlChgFilePtr( pLIDA->hFile, 0L, FILE_CURRENT, &ulCurrentPos, FALSE); - ulCurrentPos -= lBytesLeft; - UtlChgFilePtr( pLIDA->hFile, ulCurrentPos, FILE_BEGIN, &ulCurrentPos, FALSE); - } /* endif */ - // GQ: use number of Unicode characters (which may differ from number of chars read in) for processing - ulBytesRead = ulUnicodeChars; - } - else - { - // either at end or nothing read in... - ulUnicodeChars = UtlDirectAnsi2UnicodeBuf( pConvBuffer, pszBuffer, - ulBytesRead, pLIDA->ulAnsiCP, FALSE, &lRc, - &lBytesLeft ); - ulBytesRead = ulUnicodeChars; - usDosRc = (USHORT)lRc; - } /* endif */ - ulBytesRead *= sizeof(CHAR_W); - - if ( lRc ) - { - // find character causing the conversion error and show error message - UtlFindAndShowConversionError( pConvBuffer, ulBytesRead, pLIDA->ulAnsiCP ); - } /* endif */ - - UtlAlloc( (PVOID *)&pConvBuffer, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usDosRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - break; - - case SGMLFORMAT_UNICODE : - { - // read data into conversion buffer - usDosRc = UtlReadHwnd( pLIDA->hFile, pszBuffer, (ulSize * sizeof(CHAR_W)), - &ulBytesRead, TRUE, pLIDA->hwndErrMsg ); - - // set EOF condition - if ( (usDosRc == NO_ERROR ) && (ulBytesRead != ulSize * sizeof(CHAR_W) ) ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // if first read skip any unicode text prefix - if ( (usDosRc == NO_ERROR) // && (pLIDA->fFirstRead == FALSE)//seems like not used field - ) - { - PSZ pszPrefix = UNICODEFILEPREFIX; - int iLen = strlen(pszPrefix); - if ( memcmp( pszBuffer, pszPrefix, iLen ) == 0 ) - { - // skip prefix ... - ulBytesRead -= iLen; - memmove( (PBYTE)pszBuffer, (PBYTE)pszBuffer+iLen, ulBytesRead ); - } - else - { - PSZ pszImportFile = pLIDA->szFilePath; - usDosRc = UtlError( NO_VALID_UNICODEFORMAT, MB_YESNO, 1, &pszImportFile, EQF_WARNING ); - - //usDosRc = (usDosRc == MBID_YES ) ? NO_ERROR : ERROR_INVALID_DATA; - } /* endif */ - } /* endif */ - - } - break; - - case SGMLFORMAT_ASCII : - default: - { - PSZ pConvBuffer = NULL; - if ( UtlAlloc( (PVOID *)&pConvBuffer, 0L, ulSize+3, ERROR_STORAGE ) ) - { - usDosRc = UtlReadHwnd( pLIDA->hFile, pConvBuffer, (ulSize), &ulBytesRead, - TRUE, pLIDA->hwndErrMsg ); - if (ulBytesRead != ulSize ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // ensure that there are no DBCS characters splitted at the end of - // the data just read into memory - if ( ulBytesRead && !pLIDA->fEOF ) - { - BYTE bTest = (BYTE)pConvBuffer[ulBytesRead-1]; -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = x if ( IsDBCSLeadByteEx( pLIDA->ulOemCP, bTest ) )"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( IsDBCSLeadByteEx( pLIDA->ulOemCP, bTest ) ) - { - int iTry = 5; - ULONG ulUnicodeChars = 0; - - while ( iTry > 0 && ulBytesRead) - { - // undo the last character read... - // reposition file pointer - ULONG ulCurrentPos = 0; - UtlChgFilePtr( pLIDA->hFile, 0L, FILE_CURRENT, &ulCurrentPos, FALSE); - ulCurrentPos--; - UtlChgFilePtr( pLIDA->hFile, ulCurrentPos, FILE_BEGIN, &ulCurrentPos, FALSE); - - // adjust counters - ulBytesRead--; - // return is number of wide chars written to pszBuffer - ulUnicodeChars = ASCII2UnicodeBuf( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP ); - if ( ulUnicodeChars && pszBuffer[ulUnicodeChars-1] == 0) - { - // try again, we probably found a 2nd DBCSbyte which might be in the range of a 1st byte, too - iTry--; - } - else - { - // leave loop and set ulBytesRead - ulBytesRead = ulUnicodeChars; - iTry = 0; - } /* endif */ - } /* endwhile */ - - if ( iTry == 0 && ulUnicodeChars && pszBuffer[ulUnicodeChars-1] == 0) - { - // something went totally wrong - ulBytesRead = ulUnicodeChars-1; - } - } - else - { - LONG lBytesLeft = 0; - ULONG ulOutPut = 0; - LONG lRC = 0; - - // non DBCS case --- no problem at all - ulOutPut = ASCII2UnicodeBufEx( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP, - FALSE, &lRC, &lBytesLeft ); - // For consistent behaviour with old ASCII2UnicodeBuf function where dwFlags was 0: - // Within TM : FileRead expects to get EOS at position ulOutPut - if ( lRC ) - { - // find character causing the conversion error and show error message - UtlFindAndShowConversionError( pConvBuffer, ulBytesRead, pLIDA->ulOemCP ); - } /* endif */ - - if (lBytesLeft && (ulOutPut < ulBytesRead)) - { - *(pszBuffer+ulOutPut) = EOS; - ulOutPut++; - } - ulBytesRead = ulOutPut; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - } - else - { - // either at end or nothing read in... - ulBytesRead = ASCII2UnicodeBuf( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP ); - } /* endif */ - // return is number of wide chars written to pszBuffer - ulBytesRead *= sizeof(CHAR_W); - - UtlAlloc( (PVOID *)&pConvBuffer, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usDosRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - break; - } /* endswitch */ - - // adjust read bytes and add EOS - if ( usDosRc == NO_ERROR ) - { - PBYTE pByte = (PBYTE) pszBuffer; - *((PSZ_W)(pByte+ulBytesRead)) = EOS; - - - *pulBytesRead = ulBytesRead; - } /* endif */ - -#ifdef IMPORT_LOGGING - GetElapsedTime( &lFileReadTime ); -#endif - - - return( usDosRc ); -} /* end of function MemLoadAndConvert */ - -USHORT MemHandleCodePageValue -( - MEM_LOAD_DLG_IDA* pLIDA, // ptr to memory load IDA - PSZ_W pszCodePage, // ptr to specified code page value - PBOOL pfRestartImport // caller's restart the import flag -) -{ - ULONG ulCP = 0; - PSZ_W pMsgError[2]; // ptr to error array - USHORT usRc = 0; - USHORT usImpMode = 0; - - *pfRestartImport = FALSE; - - // skip leading whitespace - while ( iswspace( *pszCodePage ) ) pszCodePage++; - - if ( !usRc ) - { - if ( UTF16strnicmp( pszCodePage, L"ANSI=", 5 ) == 0) - { - usImpMode = SGMLFORMAT_ANSI; - ulCP = wcstol(pszCodePage + 5, 0, 10); - - if ( usImpMode != pLIDA->usImpMode ) - { - if ( pLIDA->usImpMode == SGMLFORMAT_UNICODE ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSIFORMAT); - pMsgError[0] = L"SGML ANSI"; - T5LOG(T5ERROR) << "::NO_VALID_ASCIIANSIFORMAT::" << EncodingHelper::convertToUTF8(pMsgError[0]); - } - else // usImpMode == SGMLFORMAT_ASCII - { - if ( (pLIDA->hwndErrMsg == HWND_FUNCIF) || pLIDA->fBatch ) - { - LOG_AND_SET_RC(usRc, T5INFO, MBID_OK); - } - else - { - pMsgError[1] = L"SGML ASCII"; - pMsgError[0] = pszCodePage; - T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT_SGML ASCII::"<< EncodingHelper::convertToUTF8(pMsgError[0]); - LOG_AND_SET_RC(usRc, T5INFO, WARNING_DIMP_FORMAT); - } /* endif */ - - if ( usRc == MBID_OK ) - { - *pfRestartImport = TRUE; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !usRc && (ulCP != pLIDA->ulAnsiCP) ) - { - if ( !UtlIsValidCP( ulCP ) ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSICP); - pMsgError[0] = pszCodePage + 5; - T5LOG(T5ERROR) << "::rc = " << usRc << ":: " << EncodingHelper::convertToUTF8(pMsgError[0]); - } /* endif */ - } - - if ( !usRc ) - { - pLIDA->ulAnsiCP = ADJUSTCP(ulCP); - pLIDA->usImpMode = usImpMode; - } /* endif */ - } - else if ( UTF16strnicmp( pszCodePage, L"ASCII=", 6 ) == 0) - { - usImpMode = SGMLFORMAT_ASCII; - ulCP = wcstol(pszCodePage + 6, 0, 10); - - if( usImpMode != pLIDA->usImpMode ) - { - if ( pLIDA->usImpMode == SGMLFORMAT_UNICODE ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSIFORMAT); - pMsgError[0] = L"SGML ASCII"; - T5LOG(T5ERROR) << "::NO_VALID_ASCIIANSIFORMAT::SGML ASCII"; - } - else - { - if ( ( pLIDA->hwndErrMsg == HWND_FUNCIF) || pLIDA->fBatch ) - { - LOG_AND_SET_RC(usRc, T5INFO, MBID_OK); - } - else - { - pMsgError[1] = L"SGML ANSI"; - pMsgError[0] = pszCodePage; - T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT::SGML ANSI"; - //usRc = T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT::SGML ANSI"; - } /* endif */ - - if (usRc == MBID_OK) - { - *pfRestartImport = TRUE; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !usRc && (ulCP != pLIDA->ulOemCP) ) - { - if ( !UtlIsValidCP( ulCP ) ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSICP); - pMsgError[0] = pszCodePage + 6; - T5LOG(T5ERROR) << ":: " << usRc << "::" << EncodingHelper::convertToUTF8(pMsgError[0]); - } /* endif */ - } /* endif */ - - if ( !usRc ) - { - pLIDA->ulOemCP = ADJUSTCP(ulCP); - pLIDA->usImpMode = usImpMode; - } /* endif */ - } - else if ( UTF16strnicmp( pszCodePage, L"UTF16", 5 ) == 0) - { - usImpMode = SGMLFORMAT_UNICODE; - - if ( usImpMode != pLIDA->usImpMode ) - { - LOG_AND_SET_RC(usRc, T5INFO, ERROR_DIMP_UTF16WRONG); - T5LOG(T5ERROR) << "::rc = ERROR_DIMP_UTF16WRONG = " << usRc; - } /* endif */ - } - else - { // error - this should not occur... - LOG_AND_SET_RC(usRc, T5INFO, ERROR_MEM_INVALID_SGML); - T5LOG(T5ERROR) << "::rc = ERROR_MEM_INVALID_SGML" ; - } /* endif */ - } /* endif */ - - return(usRc); -} /* end of function MemHandleCodePageValue */ - #include DECLARE_int32(maxBadSegmentsIdsSaved); @@ -1383,8 +873,10 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT MEM_LOAD_DLG_IDA* pLIDA = (MEM_LOAD_DLG_IDA*)lMemHandle; - pLIDA->mem->importDetails->segmentsCount++; // Increase segment sequence number in any case - + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pLIDA->mem->importDetails->segmentsCount++; // Increase segment sequence number in any case + } // prepare segment data if ( pSegment->fValid ) { @@ -1408,7 +900,13 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setAddInfo( pSegment->szAddInfo ); pLIDA->pProposal->setContext( pSegment->szContext ); pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); - + pLIDA->pProposal->pInputSentence.reset(); + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; + pLIDA->pProposal->fSaveDifferentTargetsForSameSource = pLIDA->mem->importDetails->fSaveDifferentTargetsForSameSource; + pLIDA->pProposal->fIgnoreSegIdOnUpdate = pLIDA->mem->importDetails->fIgnoreSegIdOnUpdate; + } pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); if(pLIDA->pProposal->pInputSentence->wasParsedSuccessfully()){ @@ -1420,38 +918,48 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT usRC = -2; } - if ( !usRC ) - { - pLIDA->mem->importDetails->segmentsImported++; // Increase the segment counter - } - else { - pLIDA->mem->importDetails->invalidSegmentsRCs[usRC] ++; - pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter - //if( pLIDA->mem->importDetails->invalidSegments < 100){ - if(pLIDA->mem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ - pLIDA->mem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pSegment->lSegNo, usRC) ); - } - if(usRC == BTREE_LOOKUPTABLE_CORRUPTED || usRC == BTREE_LOOKUPTABLE_TOO_SMALL || usRC == TMD_SIZE_IS_BIGGER_THAN_ALLOWED) - { - pLIDA->mem->importDetails->importRc = usRC; - std::string msg = "TM is reached it's size limit, please create another one and import segments there, rc = " - + toStr(usRC) + "; aciveSegment = " + toStr(pSegment->lSegNo) + "\n\nSegment " + toStr(pSegment->lSegNo) + " not imported\r\n" - +"\nReason = " + pSegment->szReason - +"\nDocument = " + pSegment->szDocument - +"\nSourceLanguage = " + pSegment->szSourceLang - +"\nTargetLanguage = " + pSegment->szTargetLang - +"\nMarkup = " + pSegment->szFormat - +"\nSource = " + EncodingHelper::convertToUTF8(pSegment->szSource) - +"\nTarget = " + EncodingHelper::convertToUTF8(pSegment->szTarget) ; - throw xercesc::SAXException(msg.c_str()); + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + if ( !usRC ) + { + pLIDA->mem->importDetails->segmentsImported++; // Increase the segment counter } - } /* endif */ + else + { + { + //std::lock_guard lk(pLIDA->mem->importDetails->invalidSegmentsRCsMutex); + pLIDA->mem->importDetails->invalidSegmentsRCs[usRC] ++; + } + pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + if(pLIDA->mem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pLIDA->mem->importDetails->firstInvalidSegmentsSegNumsMutex); + pLIDA->mem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pSegment->lSegNo, usRC) ); + } + if(usRC == BTREE_LOOKUPTABLE_CORRUPTED || usRC == BTREE_LOOKUPTABLE_TOO_SMALL || usRC == TMD_SIZE_IS_BIGGER_THAN_ALLOWED) + { + pLIDA->mem->importDetails->importRc = usRC; + std::string msg = "TM is reached it's size limit, please create another one and import segments there, rc = " + + toStr(usRC) + "; aciveSegment = " + toStr(pLIDA->mem->importDetails->segmentsCount)/*toStr(pSegment->lSegNo)*/ + "\n\ lSegNo = " + toStr(pSegment->lSegNo) + " not imported\r\n" + +"\nReason = " + pSegment->szReason + +"\nDocument = " + pSegment->szDocument + +"\nSourceLanguage = " + pSegment->szSourceLang + +"\nTargetLanguage = " + pSegment->szTargetLang + +"\nMarkup = " + pSegment->szFormat + +"\nSource = " + EncodingHelper::convertToUTF8(pSegment->szSource) + +"\nTarget = " + EncodingHelper::convertToUTF8(pSegment->szTarget) ; + throw xercesc::SAXException(msg.c_str()); + } + } /* endif */ + }//std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); } else { - pLIDA->mem->importDetails->invalidSegmentsRCs[-1]++; - pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + //std::lock_guard lk(pLIDA->mem->importDetails->invalidSegmentsRCsMutex); + pLIDA->mem->importDetails->invalidSegmentsRCs[-1]++; + pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + } #ifdef IMPORT_LOGGING // write segment to log file @@ -1466,7 +974,6 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT } /* endif */ #endif } /* endif */ - return( usRC ); } /* end of function MEMINSERTSEGMENT */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 3cfc9f13..9bfe35ee 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -59,33 +59,14 @@ USHORT EqfMemory::TmtXDelSegm TMX_EXT_OUT_W* pTmDelOut //ptr to output struct ) { - //TMX_SENTENCE Sentence(std::make_unique(pTmDelIn->szSource)); // ptr to sentence structure - ULONG ulKey; // tm record key - BOOL fOK; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits - ULONG ulLen = 0; // length indication - PTMX_RECORD pTmRecord = NULL; // pointer to tm record - CHAR szString[MAX_EQF_PATH]; // character string - PULONG pulSids = NULL; // ptr to sentence ids + CHAR szString[MAX_EQF_PATH]{}; // character string + std::vector plSids; // ptr to sentence ids PULONG pulSidStart = NULL; // ptr to sentence ids - ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer - - //allocate pSentence - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, (LONG)(MAX_INDEX_LEN * sizeof(ULONG)), - NOMSG ); - if ( fOK ) - pulSidStart = pulSids; - } /* endif */ - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + std::vector pTmRecord; // space for user data + //pTmRecord.resize(TMX_REC_SIZE); if ( !usRc ) { @@ -95,16 +76,12 @@ USHORT EqfMemory::TmtXDelSegm strcat( szString, TmDelIn.szMarkup ); strcat( szString, EXT_OF_FORMAT ); - //remember start of norm string - why? - //Sentence.pNormStringStart = Sentence.pStrings->getNormStrC(); - - //tokenize source segment, resuting in normalized string and tag table record usRc = TokenizeSource( TmDelIn.pInputSentence.get(), szString, TmDelIn.szSourceLanguage); // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) - TmDelIn.pInputSentence->pTagRecord->usTagTableId = 1; + toTmxTagtableRecord(TmDelIn.pInputSentence->pTagRecord)->usTagTableId = 1; } @@ -112,7 +89,6 @@ USHORT EqfMemory::TmtXDelSegm if ( !usRc ) { //set pNormString to beginning of string - //Sentence.pNormString = Sentence.pStrings->getNormStrC(); HashSentence( TmDelIn.pInputSentence.get() ); } /* endif */ @@ -121,53 +97,26 @@ USHORT EqfMemory::TmtXDelSegm if ( !usRc ) { usMatchesFound = CheckCompactArea( TmDelIn.pInputSentence.get(), this ); - if ( usMatchesFound == TmDelIn.pInputSentence->usActVote ) //all hash triples found + if ( usMatchesFound == TmDelIn.pInputSentence->pulVotes.size() ) //all hash triples found { - usRc = DetermineTmRecord( this, TmDelIn.pInputSentence.get(), pulSids ); + usRc = DetermineTmRecord( this, TmDelIn.pInputSentence.get(), plSids ); if ( !usRc ) { - while ( *pulSids ) + for (LONG lKey : plSids ) { - ulKey = *pulSids; - ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - ulKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = state->TmBtree.EQFNTMGet(lKey, //tm record key + pTmRecord ); //length if ( usRc == NO_ERROR ) { //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, &ulKey ); - if ( usRc == SEG_NOT_FOUND ) - { - //get next tm record - pulSids++; - } - else + usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, lKey ); + if ( usRc != SEG_NOT_FOUND ) { //target record was found and deleted or an error occured //so don't try other sids - *pulSids = 0; + break; } /* endif */ } /* endif */ } /* endwhile */ @@ -179,9 +128,6 @@ USHORT EqfMemory::TmtXDelSegm } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); pTmDelOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmDelOut->stPrefixOut.usTmtXRc = usRc; @@ -196,66 +142,29 @@ USHORT EqfMemory::TmtXDelSegmByKey ) { PTMX_SENTENCE pSentence = NULL; // ptr to sentence structure - ULONG ulKey=TmDelIn.nextInternalKey.getRecordKey(); // tm record key + LONG lKey=TmDelIn.nextInternalKey.getRecordKey(); // tm record key BOOL fOK; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits - ULONG ulLen = 0; // length indication - PTMX_RECORD pTmRecord = NULL; // pointer to tm record - CHAR szString[MAX_EQF_PATH]; // character string - //PULONG pulSids = NULL; // ptr to sentence ids - //PULONG pulSidStart = NULL; // ptr to sentence ids + CHAR szString[MAX_EQF_PATH]; // character strin ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer - - //allocate pSentence - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + std::vector pTmRecord; // space for user data // update TM databse if ( !usRc ) { - ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - ulKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = state->TmBtree.EQFNTMGet( lKey, //tm record key + pTmRecord ); //pointer to tm record data if ( usRc == NO_ERROR ) { - //find target record and delete, if the target record was the //only target in the tm record, delete the entire record usRc = FindTargetByKeyAndDelete( pTmRecord, - TmDelIn, pSentence, pTmDelOut, &ulKey ); + TmDelIn, pSentence, pTmDelOut, lKey ); } /* endif */ } /* endif */ - //release memory - - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); - pTmDelOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmDelOut->stPrefixOut.usTmtXRc = usRc; return( usRc ); @@ -317,262 +226,198 @@ USHORT EqfMemory::TmtXDelSegmByKey //| get next sentence key | // ----------------------------------------------------------------------------+ USHORT EqfMemory::FindTargetAndDelete( - PTMX_RECORD pTmRecord, + BytesRef TmRecord, OtmProposal& TmDel, TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ) + LONG& lKey ) { - BOOL fOK = FALSE; //success indicator + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); BOOL fStop = FALSE; //indicates whether to leave loop or not PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr PTMX_SOURCE_RECORD pTMXSourceRecord = NULL; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord = NULL; //ptr to target record PTMX_TARGET_CLB pClb = NULL; //ptr to target control block - LONG lTagAlloc; //allocate length ULONG ulLen = 0; //length indicator LONG lSrcLen = 0; - PSZ_W pString = NULL; //pointer to character string + std::wstring pString; //pointer to character string USHORT usId = 0; //returned id from function USHORT usRc = NO_ERROR; //returned value from function USHORT usTarget = 0; //initialize counter - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - //allocate 4k for pTagRecord - if ( fOK ) - { - //fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG) TOK_SIZE, NOMSG ); - if ( fOK ) - lTagAlloc = (LONG)TOK_SIZE; - } + //move pointer to source string + pByte = (PBYTE)(pTmRecord+1); + pByte += pTMXSourceRecord->usSource; - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + //calculate length of source string + ulLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); + + //copy source string for later compare function + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + lSrcLen = ulLen; - //move pointer to source string + //compare source strings + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) + { + ULONG ulLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + + //source strings equal + //position at first target record pByte = (PBYTE)(pTmRecord+1); - pByte += pTMXSourceRecord->usSource; + pByte += RECLEN(pTMXSourceRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pStartTarget = (PBYTE)pTMXTargetRecord; - //calculate length of source string - ulLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); + //source strings are identical so loop through target records + while ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) && !fStop ) + { + usTarget++; + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); + ulLeftTgtLen -= RECLEN(pTMXTargetRecord); + + //next check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pClb = (PTMX_TARGET_CLB)pByte; + + //get id of target language in the put structure + usRc = NTMGetIDFromName( TmDel.szTargetLanguage, + (USHORT)LANG_KEY, &usId ); + //compare target language ids + if ( (pClb->usLangId == usId) && !usRc ) + { + //compare source tag table records + //position at source tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //clear pString for later use - memset( pString, 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W) ); + //source tag tables are identical + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pByte += pTMXTargetRecord->usTarget; - //copy source string for later compare function -// memcpy( pString, pByte, ulLen ); - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - lSrcLen = ulLen; + //calculate length of target string + ulLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - //compare source strings - //if ( !UTF16strcmp( pString, pSentence->pNormString ) ) - if ( !UTF16strcmp( pString, TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) - { - ULONG ulLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - //source strings equal - //position at first target record - pByte = (PBYTE)(pTmRecord+1); - pByte += RECLEN(pTMXSourceRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pStartTarget = (PBYTE)pTMXTargetRecord; + //copy target string for later compare function + pString.clear(); + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - //source strings are identical so loop through target records - while ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) && !fStop ) - { - usTarget++; - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); - ulLeftTgtLen -= RECLEN(pTMXTargetRecord); - - //next check the target language - //position at target control block - pByte += pTMXTargetRecord->usClb; - pClb = (PTMX_TARGET_CLB)pByte; - - //get id of target language in the put structure - usRc = NTMGetIDFromName( TmDel.szTargetLanguage, - NULL, - (USHORT)LANG_KEY, &usId ); - //compare target language ids - if ( (pClb->usLangId == usId) && !usRc ) + //compare target strings + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) { - //compare source tag table records - //position at source tag table record + //compare tag table records + //identical target tag table as in del structure so + //check segment id and file name in control block pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //position at source tag table - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; + //position at target control block + pByte += pTMXTargetRecord->usClb; + pClb = (PTMX_TARGET_CLB)pByte; - //compare tag table records - if(//UtlCompIgnWhiteSpaceW(pSentence->pInputString, (wchar_t*)pStringWNormalizedTags.c_str(), pStringWNormalizedTags.size() ) == 0 - true) // we use the same tag tables in t5? + //get id of filename in the put structure + usRc = NTMGetIDFromName( TmDel.szDocName, + (USHORT)FILE_KEY, &usId ); + if ( !usRc ) { - //source tag tables are identical - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pByte += pTMXTargetRecord->usTarget; - - //calculate length of target string - ulLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - - //clear pString for later use - memset( pString, 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); - - //copy target string for later compare function -// memcpy( pString, pByte, ulLen ); - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - - //tokenize target string in del structure - //usRc = TokenizeTarget( pTmDel->szTarget, pNormString, &pTagRecord, &lTagAlloc, pTmDel->szTagTable, &usNormLen, this ); - - if ( !usRc ) + ULONG ulLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + while ( ulLeftClbLen && !fStop ) { - //compare target strings - //if ( !UTF16strcmp( pString, pNormString ) ) - if ( !UTF16strcmp( pString, TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) + if ( + (pClb->usFileId == usId) && + (pClb->ulSegmId == TmDel.lSegmentId) && + (pClb->bTranslationFlag == (BYTE)ProposalTypeToFlag(TmDel.eType)) + ) { - //target strings are equal so compare target tag records - //position at target tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTargetTagTable = (PTMX_TAGTABLE_RECORD)pByte; - - //compare tag table records - if (true - //|| //we use the same tag tables - // !memcmp( pTMXTargetTagTable, pTagRecord, - // RECLEN(pTMXTargetTagTable) ) - ) - { - //identical target tag table as in del structure so - //check segment id and file name in control block - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //position at target control block - pByte += pTMXTargetRecord->usClb; - pClb = (PTMX_TARGET_CLB)pByte; - - //get id of filename in the put structure - usRc = NTMGetIDFromName( TmDel.szDocName, - TmDel.szDocName, - (USHORT)FILE_KEY, &usId ); - if ( !usRc ) - { - ULONG ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - while ( ulLeftClbLen && !fStop ) - { - if ( - (pClb->usFileId == usId) && - (pClb->ulSegmId == TmDel.lSegmentId) && - (pClb->bTranslationFlag == (BYTE)ProposalTypeToFlag(TmDel.eType)) - ) - { - //if segment id and filename are equal then delete - //target CLB and any empty target record - - //check that multiple flag isn't on - //if on leave while loop as though delete was carried out - if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime //|| true - ) - { - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pClb, - TmDel.pInputSentence->pStrings->getGenericTargetStrC(), &lSrcLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - pTmExtOut->ulRecKey = *pulKey; - TmDel.currentInternalKey.setInternalKey(*pulKey, usTarget); - pTmExtOut->usTargetKey = usTarget; - - TMDelTargetClb( pTmRecord, pTMXTargetRecord, pClb ); - - //add updated tm record to database - /**********************************************/ - /* we usually should delete the record here */ - /* (if no translation is available for it) */ - /* BUT: since usually an untranslate is follow*/ - /* by a new translation, we will not remove */ - /* the key (only get rid of any target data) */ - /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( - *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord), true ); - } /* endif */ - - //leave while loop - fStop = TRUE; - } - else - { - usTarget++; - // try next target CLB - ulLeftClbLen -= TARGETCLBLEN(pClb); - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } /* endwhile */ - } /* endif */ + //if segment id and filename are equal then delete + //target CLB and any empty target record + + //check that multiple flag isn't on + //if on leave while loop as though delete was carried out + if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime //|| true + ) + { + PBYTE endOfMemRegion = TmRecord.data() + TmRecord.size(); + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pClb, + TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, + pTmExtOut, endOfMemRegion ); + auto langKeyRc = NTMGetNameFromID(pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage ); + pTmExtOut->ulRecKey = lKey; + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); + pTmExtOut->usTargetKey = usTarget; + + TMDelTargetClb( TmRecord, pTMXTargetRecord, pClb ); + + //add updated tm record to database + /**********************************************/ + /* we usually should delete the record here */ + /* (if no translation is available for it) */ + /* BUT: since usually an untranslate is follow*/ + /* by a new translation, we will not remove */ + /* the key (only get rid of any target data) */ + /**********************************************/ + usRc = state->TmBtree.EQFNTMUpdate( lKey, TmRecord, true); } /* endif */ + + //leave while loop + fStop = TRUE; + } + else + { + usTarget++; + // try next target CLB + ulLeftClbLen -= TARGETCLBLEN(pClb); + pClb = NEXTTARGETCLB(pClb); } /* endif */ - } /* endif */ + } /* endwhile */ } /* endif */ } /* endif */ + } /* endif */ - //position at next target - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - pByte += RECLEN(pTMXTargetRecord); - //remember the end/beginning of record - pStartTarget = pByte; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endwhile */ - } /* endif */ - - if ( !fStop ) - { - LOG_AND_SET_RC(usRc, T5INFO, SEG_NOT_FOUND); - } /* endif */ + //position at next target + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + pByte += RECLEN(pTMXTargetRecord); + //remember the end/beginning of record + pStartTarget = pByte; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } /* endwhile */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); - //UtlAlloc( (PVOID *) &(pTagRecord), 0L, 0L, NOMSG ); + if ( !fStop ) + { + LOG_AND_SET_RC(usRc, T5INFO, SEG_NOT_FOUND); + } /* endif */ return( usRc ); } USHORT EqfMemory::FindTargetByKeyAndDelete( - PTMX_RECORD pTmRecord, + BytesRef TmRecord, OtmProposal& TmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ) + LONG& lKey ) { USHORT usRc = 0; + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record PTMX_TARGET_CLB pTargetClb; //ptr to target CLB @@ -580,91 +425,118 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( PBYTE pByte; //position ptr LONG lSourceLen = 0; //length of source string USHORT usTarget = 0; //nr of target records in tm record - PSZ_W pSourceString = NULL; //pointer to source string + std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr + //check if record is empty + if ( ( RECLEN(pTmRecord) != 0) && + (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) + { + LONG lLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + //extract source string + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - UtlAlloc( (PVOID *) &(pSourceString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); + //move pointer to corresponding position + pSource = (PBYTE)(pTmRecord+1); + pSource += pTMXSourceRecord->usSource; - if ( !pSourceString ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + //calculate length of source string + lSourceLen = (RECLEN(pTMXSourceRecord) - + sizeof(TMX_SOURCE_RECORD) ); + //copy source string for fill matchtable + if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + { + lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); + if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { + pSourceString[lSourceLen] = EOS; + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSoureLen = " << lSourceLen;} + } + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< " ((RECLEN(pTmRecord) == 0) || (pTmRecord->usFirstTargetRecord >= RECLEN(pTmRecord)) ); reclen = " << RECLEN(pTmRecord) << ";usFirstTargetRecord="<usFirstTargetRecord;} + } /* endif */ - if ( !usRc ) - { - //check if record is empty - if ( ( RECLEN(pTmRecord) != 0) && - (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) + if ( !usRc ) { - LONG lLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //extract source string - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - lSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD) ); - - //copy source string for fill matchtable - if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + //find target record specified in ExtIn structure + + //move pointer to first target + pByte = (PBYTE)(pTmRecord); + pByte += pTmRecord->usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) { - lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); - if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { - pSourceString[lSourceLen] = EOS; - } - else - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSoureLen = " << lSourceLen;} - } + // target record is corrupted, continue with next TM record + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb = " << pTMXTargetRecord->usClb<<"; reclen = " << RECLEN(pTMXTargetRecord);} } else { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< " ((RECLEN(pTmRecord) == 0) || (pTmRecord->usFirstTargetRecord >= RECLEN(pTmRecord)) ); reclen = " << RECLEN(pTmRecord) << ";usFirstTargetRecord="<usFirstTargetRecord;} + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB } /* endif */ + usTarget = 1; //initialize counter - if ( !usRc ) + //loop until correct target is found + while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && (lLeftTgtLen > 0) && !usRc ) { - //find target record specified in ExtIn structure - - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; + // position to first target CLB pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { - // target record is corrupted, continue with next TM record - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb = " << pTMXTargetRecord->usClb<<"; reclen = " << RECLEN(pTMXTargetRecord);} + lLeftClbLen -= TARGETCLBLEN(pTargetClb); } else { - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - } /* endif */ - usTarget = 1; //initialize counter + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ -//NEW CODE - //loop until correct target is found - while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && (lLeftTgtLen > 0) && !usRc ) + // loop over all target CLBs + while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && ( lLeftClbLen > 0 ) ) { - // position to first target CLB + usTarget++; + pTargetClb = NEXTTARGETCLB(pTargetClb); + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } /* endwhile */ + + // continue with next target if no match yet + if ( usTarget < TmDel.nextInternalKey.getTargetKey() ) + { + // position at next target + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) + { + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + } + else + { + lLeftTgtLen = 0; + } /* endif */ + pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - + pTMXTargetRecord->usClb; // subtract size of current CLB if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { @@ -676,21 +548,52 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( lLeftClbLen = 0; LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ + usTarget++; + } /* endif */ + } /* endwhile */ - // loop over all target CLBs - while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && ( lLeftClbLen > 0 ) ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ + if ( !usRc && (usTarget == TmDel.nextInternalKey.getTargetKey()) ) + { + //position at start of target record + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - // continue with next target if no match yet - if ( usTarget < TmDel.nextInternalKey.getTargetKey() ) + //if target record exists + if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) + { + //check that multiple flag isn't on + //if on leave while loop as though delete was carried out + + PBYTE endOfMemRegion = TmRecord.data() + TmRecord.size(); + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pTargetClb, + pSourceString, &lSourceLen, + pTmExtOut, endOfMemRegion ); + auto langKeyRc = NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage ); + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); + pTmExtOut->ulRecKey = lKey; + pTmExtOut->usTargetKey = usTarget; + + //if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime ) + if(pTargetClb->ulSegmId != TmDel.lSegmentId){ + return SEGMENT_ID_NOT_EQUAL; + } else { + TMDelTargetClb( TmRecord, pTMXTargetRecord, pTargetClb ); + + //add updated tm record to database + /**********************************************/ + /* we usually should delete the record here */ + /* (if no translation is available for it) */ + /* BUT: since usually an untranslate is follow*/ + /* by a new translation, we will not remove */ + /* the key (only get rid of any target data) */ + /**********************************************/ + usRc = state->TmBtree.EQFNTMUpdate( lKey, TmRecord, true); + } /* endif */ + if ( ! usRc ) { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target + //check for another target if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) { lLeftTgtLen -= RECLEN(pTMXTargetRecord); @@ -701,84 +604,6 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( } /* endif */ pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) - { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } - else - { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - usTarget++; - } /* endif */ - } /* endwhile */ - - if ( !usRc && (usTarget == TmDel.nextInternalKey.getTargetKey()) ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target record exists - if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) - { - //check that multiple flag isn't on - //if on leave while loop as though delete was carried out - - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pTargetClb, - pSourceString, &lSourceLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - TmDel.currentInternalKey.setInternalKey(*pulKey, usTarget); - pTmExtOut->ulRecKey = *pulKey; - pTmExtOut->usTargetKey = usTarget; - - //if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime ) - if(pTargetClb->ulSegmId != TmDel.lSegmentId){ - return SEGMENT_ID_NOT_EQUAL; - } else { - TMDelTargetClb( pTmRecord, pTMXTargetRecord, pTargetClb ); - - //add updated tm record to database - /**********************************************/ - /* we usually should delete the record here */ - /* (if no translation is available for it) */ - /* BUT: since usually an untranslate is follow*/ - /* by a new translation, we will not remove */ - /* the key (only get rid of any target data) */ - /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( - *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord), true ); - } /* endif */ - if ( ! usRc ) - { - //check for another target - if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - lLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endif */ - } - else - { - //no more target so get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } else @@ -787,11 +612,13 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } - } - - } /* endif */ - - //release memory + else + { + //no more target so get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ + } + } return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index fb27a5c0..d3c10237 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -64,284 +64,59 @@ USHORT EqfMemory::TmtXExtract TMX_EXT_OUT_W * pTmExtOut //ptr to output struct ) { - USHORT usRc = NO_ERROR; //return code - BOOL fOK; //success indicator - PTMX_RECORD pTmRecord = NULL; //pointer to tm record - ULONG ulRecBufSize = 0; // current size of record buffer - ULONG ulLen = 0; //length indicator + USHORT usRc = BTREE_NOT_FOUND; //return code PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - ULONG ulStartKey; //start tm key - ULONG ulNextKey; //last tm key - BOOL fSpecialMode = FALSE; //special mode flag - ULONG ulOemCP = 1L; + LONG lStartKey = state->TmBtree.chCollate.lStartKey; //start tm key + LONG lNextKey = state->TmBtree.chCollate.lNextKey; //last tm key - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) (2 * TMX_REC_SIZE), NOMSG ); - ulRecBufSize = 2 * TMX_REC_SIZE; - //allocate pString - - if ( !fOK ) + std::vector pTmRecord; // space for user data + pTmExtOut->lMaxEntries = (lNextKey - lStartKey); + if(state->fCorrupted && lStartKey != FIRST_KEY){ + lNextKey = std::max(lNextKey, LASTALLOWEDKEY) ; //some big value to go + } + /******************************************************************/ + /* return one matching entry (if any available) */ + /******************************************************************/ + while ( (pTmExtIn->lTmKey < lNextKey) && (usRc == BTREE_NOT_FOUND) ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + usRc = state->TmBtree.EQFNTMGet(pTmExtIn->lTmKey, pTmRecord ); - if ( !usRc ) - { - SHORT sRetries = MAX_RETRY_COUNT; + if ( usRc == NO_ERROR ) { - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); // reset return code - /********************************************************************/ - /* Set special mode flag */ - /********************************************************************/ - if ( !usRc ) - { - if ( (pTmExtIn->usConvert == MEM_OUTPUT_TAGTABLES) || - (pTmExtIn->usConvert == MEM_OUTPUT_AUTHORS) || - (pTmExtIn->usConvert == MEM_OUTPUT_LONGNAMES) || - (pTmExtIn->usConvert == MEM_OUTPUT_ALLDOCS) || - (pTmExtIn->usConvert == MEM_OUTPUT_DOCUMENTS) || - (pTmExtIn->usConvert == MEM_OUTPUT_LANGUAGES) ) - { - fSpecialMode = TRUE; - } - else - { - fSpecialMode = FALSE; - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* Special mode to get list of tag tables, authors, documents or */ - /* languages */ - /********************************************************************/ - if ( !usRc && fSpecialMode) + usRc = ExtractRecord(pTmRecord, pTmExtIn, pTmExtOut ); + } + /****************************************************************/ + /* setup new starting point (do this even in the case we are */ + /* dealing with a corrupted TM ) */ + /****************************************************************/ + if ( usRc != NO_ERROR ) + { + if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((state->usAccessMode & ASD_ORGANIZE) != 0) ) { - PTMX_TABLE pTable = NULL; // table containing the requested info - ULONG ulMaxEntries = 0; // last + 1 entry in table - - /******************************************************************/ - /* Address table containg the requested names */ - /******************************************************************/ - switch ( pTmExtIn->usConvert ) - { - case MEM_OUTPUT_TAGTABLES : - pTable = &TagTables; - ulMaxEntries = pTable->ulMaxEntries; - break; - case MEM_OUTPUT_AUTHORS : - pTable = &Authors; - ulMaxEntries = pTable->ulMaxEntries; - break; - case MEM_OUTPUT_DOCUMENTS : - pTable = &FileNames; - ulMaxEntries = pTable->ulMaxEntries; - break; - case MEM_OUTPUT_LANGUAGES : - pTable = &Languages; - ulMaxEntries = pTable->ulMaxEntries; - break; - case MEM_OUTPUT_LONGNAMES : - // different handling for long names, no setting of pTable - ulMaxEntries = pLongNames->ulEntries; - break; - case MEM_OUTPUT_ALLDOCS : - // different handling for long names, no setting of pTable - pTable = &FileNames; - ulMaxEntries = pTable->ulMaxEntries; - break; - default : - LOG_AND_SET_RC(usRc, T5INFO, BTREE_INVALID); - break; - } /* endswitch */ - - /******************************************************************/ - /* Extract requested names */ - /******************************************************************/ - if ( !usRc ) - { - PTMX_TABLE_ENTRY pEntry; // ptr to current entry - PSZ_W pszBuffer; // ptr to next free space in buffer - LONG lLen; // lenght of current name - LONG lRoomLeft; // space left in current buffer - int iteration; // iteration counter - PSZ pszName; // ptr to current name - - /****************************************************************/ - /* Initialize our output buffers */ - /****************************************************************/ - memset( pTmExtOut->szSource, NULC, - sizeof(pTmExtOut->szSource) ); - memset( pTmExtOut->szTarget, NULC, - sizeof(pTmExtOut->szTarget) ); - - /****************************************************************/ - /* Loop two times: first time to fill szSource, the second */ - /* time to fill the szTarget buffer */ - /****************************************************************/ - iteration = 0; - do - { - // set buffer and lRoomLeft variables - if ( iteration == 0 ) - { - // use szSource buffer - pszBuffer = pTmExtOut->szSource; - // leave room for NULC and ASCII expansion - lRoomLeft = (sizeof(pTmExtOut->szSource)/sizeof(CHAR_W))/2 - 1; - } - else - { - // use szTarget buffer - pszBuffer = pTmExtOut->szTarget; - // leave room for NULC and ASCII expansion - lRoomLeft = (sizeof(pTmExtOut->szTarget)/sizeof(CHAR_W))/2 - 1; - } /* endif */ - - while ( lRoomLeft && (pTmExtIn->usNextTarget < ulMaxEntries) ) - { - if ( pTmExtIn->usConvert == MEM_OUTPUT_LONGNAMES ) - { - pszName = pLongNames-> - stTableEntry[pTmExtIn->usNextTarget].pszLongName; - } - else - { - pEntry = &pTable->stTmTableEntry[pTmExtIn->usNextTarget]; - pszName = pEntry->szName; - // check if there is a long name for the current document - if ( pTmExtIn->usConvert == MEM_OUTPUT_ALLDOCS ) - { - ULONG ulEntry = 0; - while ( (ulEntry < pLongNames->ulEntries) && - (pEntry->usId != pLongNames-> - stTableEntry[ulEntry].usId ) ) - { - ulEntry++; - } /* endwhile */ - if ( ulEntry < pLongNames->ulEntries ) - { - pszName = pLongNames->stTableEntry[ulEntry].pszLongName; - } /* endif */ - } /* endif */ - } /* endif */ - lLen = strlen(pszName); - - if ( lRoomLeft > (lLen + 1)) - { - ASCII2Unicode( pszName, pszBuffer, ulOemCP ); - pszBuffer += lLen; - *pszBuffer++ = X15; - lRoomLeft -= lLen + 1; - pTmExtIn->usNextTarget++; - } - else - { - lRoomLeft = 0; - } /* endif */ - } /* endwhile */ - - *pszBuffer = NULC; // terminate current buffer - iteration++; // continue with next buffer - } while ( (iteration <= 1) && (pTmExtIn->usNextTarget < ulMaxEntries) ); /* enddo */ - - /****************************************************************/ - /* Set flags in output structure */ - /****************************************************************/ - if ( pTmExtIn->usNextTarget < ulMaxEntries ) - { - // more names available for extract - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; - pTmExtOut->usTranslationFlag = TRANSLFLAG_MACHINE; - } - else - { - // all names have been extracted - pTmExtOut->usNextTarget = 0; - pTmExtOut->usTranslationFlag = TRANSLFLAG_NORMAL; - } /* endif */ - } /* endif */ + pTmExtIn->lTmKey ++; + pTmExtIn->usNextTarget = 1; + pTmExtOut->lTmKey = pTmExtIn->lTmKey; + pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; + usRc = BTREE_NOT_FOUND; // reset error condition } /* endif */ + } /* endif */ + } /* endwhile */ - /********************************************************************/ - /* Normal mode of TMExtract */ - /********************************************************************/ - if ( !usRc && !fSpecialMode ) - { - TmBtree.EQFNTMGetNextNumber( &ulStartKey, &ulNextKey); - pTmExtOut->ulMaxEntries = (ulNextKey - ulStartKey); - /******************************************************************/ - /* return one matching entry (if any available) */ - /******************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - while ( (pTmExtIn->ulTmKey < ulNextKey) && (usRc == BTREE_NOT_FOUND) ) - { - ulLen = ulRecBufSize; - usRc = TmBtree.EQFNTMGet( - pTmExtIn->ulTmKey, - (PCHAR)pTmRecord, - &ulLen ); - - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - pTmExtIn->ulTmKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - if ( usRc == NO_ERROR ) - { - usRc = ExtractRecordV6( pTmRecord, pTmExtIn, pTmExtOut ); - } - /****************************************************************/ - /* setup new starting point (do this even in the case we are */ - /* dealing with a corrupted TM ) */ - /****************************************************************/ - if ( usRc != NO_ERROR ) - { - if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((usAccessMode & ASD_ORGANIZE) != 0) ) - { - pTmExtIn->ulTmKey ++; - pTmExtIn->usNextTarget = 1; - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey; - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; - usRc = BTREE_NOT_FOUND; // reset error condition - } /* endif */ - } /* endif */ - } /* endwhile */ - - /******************************************************************/ - /* set end of file condition ... */ - /******************************************************************/ - if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->ulTmKey == ulNextKey) ) - { - //arrived at last tm record - LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); - } /* endif */ - } /* endif */ - } + /******************************************************************/ + /* set end of file condition ... */ + /******************************************************************/ + if ((usRc == BTREE_NOT_FOUND) && (pTmExtIn->lTmKey == lNextKey) ) + { + //arrived at last tm record + LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); } /* endif */ pTmExtOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmExtOut->stPrefixOut.usTmtXRc = usRc; - //release memory - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); - // in organize mode only: change any error code (except BTREE_EOF_REACHED) to BTREE_CORRUPTED - if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((usAccessMode & ASD_ORGANIZE) != 0) ) + if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((state->usAccessMode & ASD_ORGANIZE) != 0) ) { {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((usAccessMode & ASD_ORGANIZE) != 0) ) usRc =" << usRc;} } /* endif */ @@ -349,14 +124,15 @@ USHORT EqfMemory::TmtXExtract return( usRc ); } -USHORT EqfMemory::ExtractRecordV6 +USHORT EqfMemory::ExtractRecord ( - PTMX_RECORD pTmRecord, + BytesRef pTmRecordData, PTMX_EXT_IN_W pTmExtIn, TMX_EXT_OUT_W * pTmExtOut ) { USHORT usRc = 0; + PTMX_RECORD pTmRecord = toTmxRecord(pTmRecordData); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record PTMX_TARGET_CLB pTargetClb; //ptr to target CLB @@ -364,130 +140,121 @@ USHORT EqfMemory::ExtractRecordV6 PBYTE pByte; //position ptr LONG lSourceLen = 0; //length of source string USHORT usTarget; //nr of target records in tm record - PSZ_W pSourceString = NULL; //pointer to source string + std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr - - UtlAlloc( (PVOID *) &(pSourceString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - - if ( !pSourceString ) + //check if record is empty + if ( (!pTmRecordData.empty()) && ( RECLEN(pTmRecord) != 0) && + (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - + LONG lLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + //extract source string + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + + //move pointer to corresponding position + pSource = (PBYTE)(pTmRecord+1); + pSource += pTMXSourceRecord->usSource; + + //calculate length of source string + lSourceLen = (RECLEN(pTMXSourceRecord) - + sizeof(TMX_SOURCE_RECORD) ); + + //copy source string for fill matchtable + if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + { + lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); + if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { + pSourceString[lSourceLen] = EOS; + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen >= MAX_SEGMENT_SIZE || lSourceLen<0 ); lSourceLen" << lSourceLen;} + } + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( lSourceLen >= MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSourceLen=" <usFirstTargetRecord < RECLEN(pTmRecord)) ) + if ( !usRc ) { - LONG lLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //extract source string - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - lSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD) ); - - //copy source string for fill matchtable - if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + //find target record specified in ExtIn structure + + //move pointer to first target + pByte = (PBYTE)(pTmRecord); + pByte += pTmRecord->usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) { - lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); - if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { -//// memcpy( pSourceString, pSource, ulSourceLen ); - pSourceString[lSourceLen] = EOS; - } - else - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen >= MAX_SEGMENT_SIZE || lSourceLen<0 ); lSourceLen" << lSourceLen;} - } + // target record is corrupted, continue with next TM record + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb=" << pTMXTargetRecord->usClb <<"; reclen="<= MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSourceLen=" <usClb; + lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB } /* endif */ + usTarget = 1; //initialize counter - if ( !usRc ) + //loop until correct target is found + while ( (usTarget < pTmExtIn->usNextTarget) && (lLeftTgtLen > 0) && !usRc ) { - //find target record specified in ExtIn structure - - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; + // position to first target CLB pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { - // target record is corrupted, continue with next TM record - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb=" << pTMXTargetRecord->usClb <<"; reclen="<usClb; - lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - } /* endif */ - usTarget = 1; //initialize counter + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ - //loop until correct target is found - while ( (usTarget < pTmExtIn->usNextTarget) && (lLeftTgtLen > 0) && !usRc ) + // loop over all target CLBs + while ( (usTarget < pTmExtIn->usNextTarget) && ( lLeftClbLen > 0 ) ) { - // position to first target CLB - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; + usTarget++; + pTargetClb = NEXTTARGETCLB(pTargetClb); + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } /* endwhile */ - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) + // continue with next target if no match yet + if ( usTarget < pTmExtIn->usNextTarget ) + { + // position at next target + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); + lLeftTgtLen -= RECLEN(pTMXTargetRecord); } else { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - - // loop over all target CLBs - while ( (usTarget < pTmExtIn->usNextTarget) && ( lLeftClbLen > 0 ) ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ + lLeftTgtLen = 0; + } /* endif */ - // continue with next target if no match yet - if ( usTarget < pTmExtIn->usNextTarget ) + //if(lLeftClbLen > 0)//need to add this check in 0.7? { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - lLeftTgtLen = 0; - } /* endif */ pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; + pTMXTargetRecord->usClb; // subtract size of current CLB if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { @@ -499,61 +266,58 @@ USHORT EqfMemory::ExtractRecordV6 lLeftClbLen = 0; LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ - usTarget++; - } /* endif */ - } /* endwhile */ + } + + usTarget++; + } /* endif */ + } /* endwhile */ - if ( !usRc && (usTarget == pTmExtIn->usNextTarget) ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + if ( !usRc && (usTarget == pTmExtIn->usNextTarget) ) + { + //position at start of target record + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //if target record exists - if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) + //if target record exists + if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) + { + PBYTE endOfMemRegion = pTmRecordData.data() + pTmRecordData.size(); + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pTargetClb, + pSourceString, &lSourceLen, + pTmExtOut, endOfMemRegion); + auto srcLangKeyRc = NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage ); + if ( ! usRc ) { - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pTargetClb, - pSourceString, &lSourceLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - if ( ! usRc ) + //check for another target + if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) + { + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + } + else + { + lLeftTgtLen = 0; + } /* endif */ + pByte += RECLEN(pTMXTargetRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + + //if target exists + if ( ((lLeftTgtLen > 0 ) && (RECLEN(pTMXTargetRecord) > 0)) + || + (lLeftClbLen > 0) + ) { - //check for another target - if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - lLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target exists - if ( ((RECLEN(pTMXTargetRecord) > 0) && (lLeftTgtLen > 0 ) ) - || - (lLeftClbLen > 0) - ) - { - //increase target count and leave tm record key number as before - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey; - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget + 1; - } - else - { - //no more target so get next tm record and initialize target count - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey +1; - pTmExtOut->usNextTarget = 1; - } /* endif */ + //increase target count and leave tm record key number as before + pTmExtOut->lTmKey = pTmExtIn->lTmKey; + pTmExtOut->usNextTarget = pTmExtIn->usNextTarget + 1; + } + else + { + //no more target so get next tm record and initialize target count + pTmExtOut->lTmKey = pTmExtIn->lTmKey +1; + pTmExtOut->usNextTarget = 1; } /* endif */ - } - else - { - //no more target so get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } else @@ -561,20 +325,19 @@ USHORT EqfMemory::ExtractRecordV6 //no more target so get next tm record and initialize target count LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ + } + else + { + //no more target so get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ - } - else - { - //if record is empty, get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ - } - - // release memory - UtlAlloc( (PVOID *) &pSourceString, 0L, 0L, NOMSG ); - - + else + { + //if record is empty, get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ return usRc; } @@ -613,147 +376,130 @@ USHORT EqfMemory::FillExtStructure ( PTMX_TARGET_RECORD pTMXTargetRecord, //ptr to tm target PTMX_TARGET_CLB pTargetClb, // ptr to current target CLB - PSZ_W pSourceString, //ptr to source string + std::wstring& pSourceString, //ptr to source string PLONG plSourceLen, //length of source string - PTMX_EXT_OUT_W pstExt //extout ext struct + PTMX_EXT_OUT_W pstExt, //extout ext struct + PBYTE endOfMemRegion ) { PTMX_TARGET_CLB pTMXTargetClb = NULL; //ptr to target control block PTMX_TAGTABLE_RECORD pTMXTagTableRecord = NULL; //ptr to tag table record - PBYTE pByte; //position pointer - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen = 0; //length indicator - PSZ_W pTargetString = NULL; //pointer to target string + std::wstring pTargetString; //pointer to target string - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pTargetString), 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); + PBYTE pByte = (PBYTE)pTMXTargetRecord; - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + if(*plSourceLen > 0) { - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; - - if(*plSourceLen > 0) - { - //else copy string - memcpy( pstExt->szSource, pSourceString, *plSourceLen * sizeof(CHAR_W) ); - pstExt->szSource[*plSourceLen] = EOS; - } /* endif */ + //else copy string + memcpy( pstExt->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W) ); + pstExt->szSource[*plSourceLen] = EOS; + } /* endif */ - if ( fOK ) - { - //position at target string - pByte = (PBYTE)pTMXTargetRecord; - pByte += pTMXTargetRecord->usTarget; + //position at target string + pByte = (PBYTE)pTMXTargetRecord; + pByte += pTMXTargetRecord->usTarget; - //calculate length of target string - lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - fOK = ( lTargetLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); - } /* endif */ + //calculate length of target string + lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + fOK = ( lTargetLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); - // GQ 2008/06/18 Check if target string data is valid - if ( fOK ) + // GQ 2008/06/18 Check if target string data is valid + if ( fOK ) + { + BYTE b = *pByte; + if ( (b != 0) && (b != BOCU_COMPRESS) ) { - BYTE b = *pByte; - - if ( (b != 0) && (b != BOCU_COMPRESS) ) - { - fOK = FALSE; // invalid compression ID - } /* endif */ + fOK = FALSE; // invalid compression ID } /* endif */ + } /* endif */ - if ( fOK ) - { - //copy target string - lTargetLen = EQFCompress2Unicode( pTargetString, pByte, lTargetLen ); - if ( lTargetLen < MAX_SEGMENT_SIZE ){ - if( lTargetLen >= 0 ) - pTargetString[lTargetLen] = EOS; - } else { - fOK = FALSE; // invalid string length - } + if ( fOK ) + { + //copy target string + lTargetLen = EQFCompress2Unicode( pTargetString, pByte, lTargetLen ); + if ( lTargetLen < MAX_SEGMENT_SIZE ){ + if( lTargetLen >= 0 ) + pTargetString[lTargetLen] = EOS; + } else { + fOK = FALSE; // invalid string length + } - //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; + //position at target tag record + pByte = (PBYTE)pTMXTargetRecord; - //fill in the tag table name - strcpy(pstExt->szTagTable, "OTMXUXLF"); + //fill in the tag table name + strcpy(pstExt->szTagTable, "OTMXUXLF"); - - if(lTargetLen >= 0){ - memcpy( pstExt->szTarget, pTargetString, lTargetLen * sizeof(CHAR_W)); - pstExt->szTarget[lTargetLen] = EOS; - } - /* endif */ + if(lTargetLen >= 0){ + memcpy( pstExt->szTarget, pTargetString.c_str(), lTargetLen * sizeof(CHAR_W)); + pstExt->szTarget[lTargetLen] = EOS; + } + /* endif */ - //position at target control block - pTMXTargetClb = pTargetClb; + //position at target control block + pTMXTargetClb = pTargetClb; - // in organize mode preset author and file name field with default - // values as here the corresponding tables may be corrupted - if ( usAccessMode & ASD_ORGANIZE ) - { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); - pstExt->szLongName[0] = EOS; - strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); - } /* endif */ + // in organize mode preset author and file name field with default + // values as here the corresponding tables may be corrupted + if ( state->usAccessMode & ASD_ORGANIZE ) + { + strcpy( pstExt->szLongName, OVERFLOW_NAME ); + pstExt->szLongName[0] = EOS; + strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); + } /* endif */ + { //fill in the target file name - NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstExt->szFileName, pstExt->szLongName ); - //use overflow name if no document name available - if ( pstExt->szFileName[0] == EOS ) - { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); - } /* endif */ + auto fileKeyRc = NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, + pstExt->szLongName, sizeof(pstExt->szLongName) ); //fill in the target author - NTMGetNameFromID( &pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, - pstExt->szAuthorName, NULL ); + auto authorKeyRc = NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, + pstExt->szAuthorName ); //fill in the target language - NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pstExt->szTargetLanguage, NULL ); - - - //fill in the segment id - pstExt->ulSourceSegmentId = pTMXTargetClb->ulSegmId; - //state whether machine translation - pstExt->usTranslationFlag = pTMXTargetClb->bTranslationFlag ; - //fill in target time - pstExt->lTargetTime = pTMXTargetClb->lTime; - - // fill in any segment context info - pstExt->szContext[0] = 0; - pstExt->szAddInfo[0] = 0; - if ( pTMXTargetClb->usAddDataLen >= MAX_ADD_DATA_LEN ) - { - // target CLB info seems to be corrupted - fOK = FALSE; + auto langKeyRc = NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, + pstExt->szTargetLanguage ); + if( ID_NOT_FOUND == fileKeyRc + || ID_NOT_FOUND == authorKeyRc + //|| ID_NOT_FOUND == langKeyRc //skip lang check because reorganize would delete that records + ){ + state->fCorrupted = true;//reorganize requested } - else if ( pTMXTargetClb->usAddDataLen ) - { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstExt->szContext, sizeof(pstExt->szContext) / sizeof(CHAR_W) ); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstExt->szAddInfo, sizeof(pstExt->szAddInfo) / sizeof(CHAR_W) ); - } /* endif */ - } /* endif */ - if(fOK){ - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - }else{ - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<<"fOk is false";} } - } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pTargetString, 0L, 0L, NOMSG ); + //fill in the segment id + pstExt->ulSourceSegmentId = pTMXTargetClb->ulSegmId; + //state whether machine translation + pstExt->usTranslationFlag = pTMXTargetClb->bTranslationFlag ; + //fill in target time + pstExt->lTargetTime = pTMXTargetClb->lTime; + + // fill in any segment context info + if ( pTMXTargetClb->usAddDataLen >= MAX_ADD_DATA_LEN ) + { + // target CLB info seems to be corrupted + fOK = FALSE; + } + else if ( pTMXTargetClb->usAddDataLen ) + { + wchar_t buff[MAX_SEGMENT_SIZE]; + buff[0]='\0'; + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, buff, endOfMemRegion); + pstExt->szContext = buff; + buff[0]='\0'; + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, buff, endOfMemRegion ); + pstExt->szAddInfo = buff; + } /* endif */ + } /* endif */ + if(fOK){ + LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); + }else{ + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<<"fOk is false";} + } return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 51de843c..a5a82cd9 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -29,157 +29,34 @@ #include "../utilities/LogWrapper.h" #include "../utilities/Property.h" #include "../utilities/EncodingHelper.h" +#include "../utilities/LanguageFactory.H" // import logging -//#ifdef MEASURETIME static LARGE_INTEGER liLast = { 0 }; static LARGE_INTEGER liFrequency = { 0 }; -static int QueryPerformanceFrequency(LARGE_INTEGER* li){ - //int res = clock_gettime(CLOCK_REALTIME, li->QuadPart); - return 0; -} - //#endif #define ACTIVATE_NTMGenericDelete - -#ifdef MEASURETIME - void LogWritePerfTime( FILE *hLog, PLARGE_INTEGER pliStart, PLARGE_INTEGER pliEnd, char *pszText ); - - void LogWritePerfTime( FILE *hLog, PLARGE_INTEGER pliStart, PLARGE_INTEGER pliEnd, char *pszText ) - { - LARGE_INTEGER liFrequency; - - QueryPerformanceFrequency( &liFrequency ); - - if ( liFrequency.QuadPart != 0 ) - { - LONGLONG ldwDiff = pliEnd->QuadPart - pliStart->QuadPart; - DWORD dwTime = (DWORD)((ldwDiff * 1000) / liFrequency.QuadPart); - - if ( *pszText ) - { - fprintf( hLog, "Time for %-20s : %ld ms (%I64d ticks)\n", pszText, dwTime, ldwDiff ); - } - else - { - fprintf( hLog, "Total time : %ld ms (%I64d ticks)\n", dwTime, ldwDiff ); - } /* endif */ - } - } -#endif - void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); -USHORT FillMatchTable( EqfMemory*, PSZ_W, PLONG, PTMX_TARGET_RECORD, +USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, - PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, - ULONG, USHORT, PTMX_GET_W, PTMX_TAGTABLE_RECORD, USHORT, USHORT, int, int, USHORT); + PTMX_MATCH_TABLE_W, PUSHORT, PUSHORT, PUSHORT, + LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT, PBYTE); /**********************************************************************/ /* Prototypes for Static functions... */ /**********************************************************************/ //For R012645 begin static std::map Name2FamilyMap; -static std::string& GetAndCacheFamilyName -( - char* pszMarkupName -); - -//For R012645 end - -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -); - - -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusTokens, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP, - PLOADEDTABLE pTagTable -); - -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); - -#ifdef ACTIVATE_NTMGenericDelete -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); -#endif - -static BOOL NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList, // list of tokens to be replaced - BOOL fRespectLFs -); - - -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs -); -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -); - -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W * ppNewData, - PLONG plNewLen -); - +//For R012645 end -static PFUZZYTOK -NTMSplitAndAddTokens ( PTMX_SUBSTPROP, - PFUZZYTOK, PUSHORT, USHORT, SHORT, - USHORT, PSZ_W, SHORT, ULONG, PLOADEDTABLE ); USHORT NTMCompareContext ( @@ -190,16 +67,6 @@ USHORT NTMCompareContext ); -//USHORT FillMatchEntryEx -//( -// EqfMemory* pTmClb, -// PTMX_SENTENCE pSentence, -// PTMX_MATCHENTRY pMatchEntry, -// PUSHORT pusMatchThreshold, -// PTMX_INDEX_RECORD pIndexRecord -//); - - //typedef USHORT (APIENTRY *PFNCOMPCONTEXT)( PSZ_W, PSZ_W, PUSHORT ); typedef USHORT ( *PFNCOMPCONTEXT)( PSZ_W, PSZ_W, PUSHORT ); @@ -244,21 +111,6 @@ static OtmProposal::eProposalType GetProposalTypeFromFlag( USHORT usTranslationF return ( OtmProposal::eptUndefined ); } - -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); -void NTMMakeTagTablename( PSZ pszMarkup, PSZ pszTagTableName ); - -BOOL NTMAlignTags( PFUZZYTOK pTokSource, PFUZZYTOK pTokTarget, PREPLLIST *ppReplaceList ); - -static BOOL NTMCheckAndDeleteTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplaceSourceList, // list of same tokens in source and prop - PREPLLIST pReplaceList, // toklist to be replaced in prop src+tgt - BOOL fRespectLFs -); - - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -312,177 +164,99 @@ USHORT EqfMemory::TmtXGet CHAR szString[MAX_EQF_PATH]; // character string szString[0] = '\0'; - ULONG ulStrippedParm = pTmGetIn->ulParm & - ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); - -#ifdef MEASURETIME - LONG64 lDummy = 0; - fTimeLogging = TRUE; - GetElapsedTime( &lDummy ); -#endif - - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); - T5LOG(T5DEBUG) << "== Lookup in memory ==" << stTmSign.szName <<" == ;\tLookupSource = >>>" <szSource); - T5LOG(T5INFO) << "-------------- Looking up:" << str << "" ; + auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); + T5LOG(T5DEBUG) << "== Lookup in memory ==" << state->stTmSign.szName <<" == ;\tLookupSource = >>>" <szSource); + T5LOG(T5INFO) << "-------------- Looking up:" << str << "" ; } -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - - -#ifdef MEASURETIME - GetElapsedTime( &(lAllocTime) ); -#endif + //build tag table path + Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); + strcat( szString, "/TABLE/"); + strcat( szString, pTmGetIn->szTagTable ); + strcat( szString, EXT_OF_FORMAT ); + + //tokenize source segment, resuting in normalized string and + //tag table record + usRc = TokenizeSource( &Sentence, szString, pTmGetIn->szSourceLanguage); - if ( !usRc ) + // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) + if ( usRc == NO_ERROR ) { - //build tag table path - Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); - strcat( szString, "/TABLE/"); - strcat( szString, pTmGetIn->szTagTable ); - strcat( szString, EXT_OF_FORMAT ); - - //remember start of norm string - //pSentence->pNormStringStart = pSentence->pNormString; - - - //tokenize source segment, resuting in normalized string and - //tag table record - usRc = TokenizeSource( &Sentence, szString, pTmGetIn->szSourceLanguage); + toTmxTagtableRecord(Sentence.pTagRecord)->usTagTableId = 1; - // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) - if ( usRc == NO_ERROR ) - { - //usRc = NTMGetIDFromName( pTmGetIn->szTagTable, NULL, (USHORT)TAGTABLE_KEY, &Sentence.pTagRecord->usTagTableId ); - Sentence.pTagRecord->usTagTableId = 1; - } - } /* endif */ + HashSentence( &Sentence ); + usOverlaps = CheckCompactArea( &Sentence, this ); + T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; -#ifdef MEASURETIME - GetElapsedTime( &(lTokenizeTime) ); -#endif - - if ( !usRc ) - { - SHORT sRetries = MAX_RETRY_COUNT; + if ( usOverlaps == Sentence.usActVote ) //all hash triples found { - usRc = NO_ERROR; // reset return code - - if ( !usRc ) - { - HashSentence( &Sentence ); - -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - - usOverlaps = CheckCompactArea( &Sentence, this ); - T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; - if ( usOverlaps == Sentence.usActVote ) //all hash triples found - { -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - if ( ulStrippedParm & GET_EXACT ) - { - T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; - //get exact matches only - usRc = GetExactMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); - -#ifdef MEASURETIME - GetElapsedTime( &(lGetExactTime) ); -#endif - //if usNumMatchesFound is zero then try for fuzzies - if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) - { - T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } - else - { - T5LOG( T5INFO) << "TmtXGet: Calling GetFuzzyMatch" ; - - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } /* endif */ - } - else - { - BOOL fCheckForFuzzyMatches = FALSE; - - // check if the overlap is good enough... - + T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; + //get exact matches only + if(!pTmGetIn->fForceFuzzySearch){ + usRc = GetExactMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); + } - // GQ 2016/04/05: new approach: (tentative fix for P403177) - // sentences with up to 8 (and incl) votes: always check fuzzyness - // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching - // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching + //if usNumMatchesFound is zero then try for fuzzies + if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) + { + T5LOG( T5INFO) << "TmtXGet: No exact matches found or forced fuzzy match, trying GetFuzzyMatch" ; + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; + } /* endif */ + } + else + { + BOOL fCheckForFuzzyMatches = FALSE; - // old code: - // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) + // check if the overlap is good enough... + // GQ 2016/04/05: new approach: (tentative fix for P403177) + // sentences with up to 8 (and incl) votes: always check fuzzyness + // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching + // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching - // GQ 2016/05/12: disabled new approach as it causes a 12% decrease in the overall analysis performance - if ( Sentence.usActVote <= 8 ) - { - fCheckForFuzzyMatches = TRUE; - } - //else if ( pSentence->usActVote < 18 ) - //{ - // fCheckForFuzzyMatches = (usOverlaps * 100) >= (pSentence->usActVote * 20); - //} - else - { - fCheckForFuzzyMatches = (usOverlaps * 100) >= (Sentence.usActVote * pTmGetIn->usMatchThreshold); - } /* endif */ + // old code: + // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) - if ( fCheckForFuzzyMatches ) - { - T5LOG(T5INFO) << "TmtXGet: usOverlaps= "<stMatchTable, &pTmGetOut->usNumMatchesFound ); + if ( fCheckForFuzzyMatches ) + { + T5LOG(T5INFO) << "TmtXGet: Calling GetFuzzyMatcusOverlaps= "<usNumMatchesFound<<" matches" ; - -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } /* endif */ - } /* endif */ + //not all triples on in compact area so fuzzy match + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned "<usNumMatchesFound<<" matches" ; } /* endif */ - } + } /* endif */ } /* endif */ + // store indication if more matches are available if ( pTmGetOut->usNumMatchesFound > 1 ) { - SetAvailFlags( pTmGetOut, pTmGetOut->usNumMatchesFound); + SetAvailFlags( pTmGetOut, pTmGetOut->usNumMatchesFound); } // if we have more than one exact match check if we have an exact match where the source // matchs exactly the source segment data - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) + if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && !(pTmGetIn->fForceFuzzySearch) && (pTmGetIn->ulParm & GET_EXACT) ) { PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); int iNumExactSourceMatches = 0; // number of matches having exact the same source @@ -501,7 +275,7 @@ USHORT EqfMemory::TmtXGet // if we have exactly one exact match which exact matches the source segment we can discard // the other exact matches - if ( iNumExactSourceMatches == 1 ) + if ( iNumExactSourceMatches == 1 && !pTmGetIn->fForceFuzzySearch) { for ( i = 0; i < pTmGetOut->usNumMatchesFound; i++ ) { @@ -521,36 +295,6 @@ USHORT EqfMemory::TmtXGet } /* endif */ } /* endif */ - /********************************************************************/ - /* check if we have more than one equal tgt match in our proposals */ - /********************************************************************/ - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT )) - { - PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); - PTMX_MATCH_TABLE_W pstActMatch; - USHORT i, j; // index values - INT wsDiff; - - for ( j = 0 ; j < pTmGetOut->usNumMatchesFound - 1 ;j++ ) - { - pstActMatch = &(pTmGetOut->stMatchTable[j]); - for ( i=j+1; i< pTmGetOut->usNumMatchesFound ; i++) - { - //if (UtlCompIgnWhiteSpaceW(pstActMatch->szTarget, (pstMatchTable+i)->szTarget, 0, &wsDiff) == 0L && wsDiff == 0) - if(wcscmp(pstActMatch->szTarget, (pstMatchTable+i)->szTarget) == 0) - { - /*************************************************************/ - /* delete this match... */ - /*************************************************************/ - ULONG ulLen = ( MAX_MATCHES - i - 1 ) * sizeof(TMX_MATCH_TABLE_W); - memmove( pstMatchTable+i, pstMatchTable+1+i, ulLen ); - pTmGetOut->usNumMatchesFound--; - i--; // reset i because we deleted the i-th one - } /* endif */ - } /* endfor */ - } /* endfor */ - } /* endif */ - /********************************************************************/ /* restrict number of matches found to the maximum number requested */ /* by user */ @@ -558,28 +302,19 @@ USHORT EqfMemory::TmtXGet pTmGetOut->usNumMatchesFound = pTmGetOut->usNumMatchesFound < pTmGetIn->usRequestedMatches ? pTmGetOut->usNumMatchesFound : pTmGetIn->usRequestedMatches; - { - int iTemp = sizeof( TMX_GET_OUT_W); - pTmGetOut->stPrefixOut.usLengthOutput = (USHORT)(iTemp); - } + int iTemp = sizeof( TMX_GET_OUT_W); + pTmGetOut->stPrefixOut.usLengthOutput = (USHORT)(iTemp); pTmGetOut->stPrefixOut.usTmtXRc = usRc; - if ( usRc ) ERREVENT2( TMTXGET_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - // release access to any tag table loaded for segment markup // (may have been loaded for context handling) - if ( pTagTable ) + if ( state->pTagTable ) { - TAFreeTagTable( (PLOADEDTABLE)(pTagTable) ); - pTagTable = NULL; + TAFreeTagTable( (PLOADEDTABLE)(state->pTagTable) ); + state->pTagTable = NULL; } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - - T5LOG( T5INFO) <<"TmtXGet::Lookup complete, found "<usNumMatchesFound<<" matches" ; if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)) @@ -588,12 +323,8 @@ USHORT EqfMemory::TmtXGet { PTMX_MATCH_TABLE_W pMatch = &(pTmGetOut->stMatchTable[i]); T5LOG( T5DEVELOP) << "TmtXGet:: Match "<< i << " , MatchLevel = "<< pMatch->usMatchLevel<< - " , MFlag = " << pMatch->usTranslationFlag << " , segment = " << pMatch->ulSegmentId; - if(pMatch->szLongName[0] != '0') { - T5LOG( T5DEVELOP)<<"TmtXGet:: origin = "<szFileName; - }else{ - T5LOG( T5DEVELOP)<<"TmtXGet:: origin = "<szLongName; - } + " , MFlag = " << pMatch->usTranslationFlag << " , segment = " << pMatch->ulSegmentId<< "; origin = "<szLongName; + auto strSource = EncodingHelper::convertToUTF8(pMatch->szSource); auto strTarget = EncodingHelper::convertToUTF8(pMatch->szTarget); @@ -602,7 +333,6 @@ USHORT EqfMemory::TmtXGet } /* endif */ pTmGetOut->tokens = std::move(Sentence.tokens); - return( usRc ); } @@ -642,141 +372,82 @@ USHORT GetExactMatch PTMX_GET_OUT_W pTmGetOut // pointer to output struct ) { - BOOL fOK = TRUE; //success indicator - PULONG pulSids = NULL; //ptr to sentence ids - PULONG pulSidStart = NULL; //ptr to sentence ids + std::vector lSids; //ptr to sentence ids USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - ULONG ulKey; //tm record key - PTMX_RECORD pTmRecord = NULL; //pointer to tm record USHORT usMatchEntries = 0; //nr of found matches - ULONG ulRecBufSize = 0; // current size of record buffer - - ULONG ulStrippedParm = pGetIn->ulParm & - ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); - - //allocate 32K for tm record - if ( pTmClb->pvTmRecord ) - { - pTmRecord = (PTMX_RECORD)pTmClb->pvTmRecord; - ulRecBufSize = pTmClb->ulRecBufSize; -// memset( pTmRecord, 0, ulRecBufSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvTmRecord = pTmRecord; - if ( fOK ) pTmClb->ulRecBufSize = ulRecBufSize = TMX_REC_SIZE; - } /* endif */ - + BytesRef pTmRecord = pTmClb->state->pvTmRecord; - //allocate for sentence ids - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, - (LONG)((MAX_INDEX_LEN + 5) * sizeof(ULONG)), - NOMSG ); + lSids.reserve(MAX_INDEX_LEN+5); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + usRc = DetermineTmRecord( pTmClb, pSentence, lSids ); + if ( usRc == NO_ERROR ) { - pulSidStart = pulSids; - usRc = DetermineTmRecord( pTmClb, pSentence, pulSids ); - if ( usRc == NO_ERROR ) + //get tm record(s) + for(LONG lKey: lSids) { - //get tm record(s) - while ( (*pulSids) && (usRc == NO_ERROR) ) - { - ulKey = *pulSids; - ulLen = TMX_REC_SIZE; - T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( ulKey, (PCHAR)pTmRecord, &ulLen ); + T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <state->TmBtree.EQFNTMGet( lKey, pTmRecord ); - if ( usRc == BTREE_BUFFER_SMALL) + if ( usRc == NO_ERROR ) + { + //compare tm record data with data passed in the get in structure + T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << lKey; + + PBYTE endOfMemRegion = pTmRecord.data() + pTmRecord.size(); + usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, + pstMatchTable, &usMatchEntries, lKey, endOfMemRegion ); + T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; + } /* endif */ + if ( usRc == NO_ERROR ) + { + //nr of matches found + (*pusMatchesFound) = usMatchEntries; + }else{ + break; + } + } /* endwhile */ + /****************************************************************/ + /* check if we are only looking for an exact match and we really*/ + /* found more than one... */ + /* store the availability of additional flags for later use */ + /****************************************************************/ + SetAvailFlags( pTmGetOut, *pusMatchesFound); + + if ( !usRc && (*pusMatchesFound > 1 ) && (pGetIn->ulParm & GET_EXACT) ) + { + /*************************************************************/ + /* there is one exact exact match available - forget about */ + /* any 'normal' matches */ + /*************************************************************/ + if ( pstMatchTable->usMatchLevel >= EQUAL_EQUAL ) { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) + if ( pstMatchTable->usMatchLevel == (EQUAL_EQUAL + 2) ) { - pTmClb->ulRecBufSize = ulRecBufSize = ulLen; - pTmClb->pvTmRecord = pTmRecord; - memset( pTmRecord, 0, ulLen ); - - usRc = pTmClb->TmBtree.EQFNTMGet( ulKey, (PCHAR)pTmRecord, - &ulLen ); + /*********************************************************/ + /* we have an exact exact ... */ + /*********************************************************/ + *pusMatchesFound = 1; } else { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - if ( usRc == NO_ERROR ) - { - //compare tm record data with data passed in the get in structure - T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << ulKey; - usRc = ExactTest( pTmClb, pTmRecord, pGetIn, pSentence, - pstMatchTable, &usMatchEntries, ulKey ); - T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; - - if ( !usRc ) - { - //nr of matches found - (*pusMatchesFound) = usMatchEntries; - - //get next tm record - pulSids++; + /*********************************************************/ + /* we have two or more exact one's. */ + /*********************************************************/ + USHORT i; // index value + for ( i=1; i< *pusMatchesFound ;i++ ) + { + if ((pstMatchTable+i)->usMatchLevel < EQUAL_EQUAL) + { + *pusMatchesFound = i; + break; + } /* endif */ + } /* endfor */ } /* endif */ } /* endif */ - } /* endwhile */ - /****************************************************************/ - /* check if we are only looking for an exact match and we really*/ - /* found more than one... */ - /* store the availability of additional flags for later use */ - /****************************************************************/ - SetAvailFlags( pTmGetOut, *pusMatchesFound); - - if ( !usRc && (*pusMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) - { - /*************************************************************/ - /* there is one exact exact match available - forget about */ - /* any 'normal' matches */ - /*************************************************************/ - if ( pstMatchTable->usMatchLevel >= EQUAL_EQUAL ) - { - if ( pstMatchTable->usMatchLevel == (EQUAL_EQUAL + 2) ) - { - /*********************************************************/ - /* we have an exact exact ... */ - /*********************************************************/ - *pusMatchesFound = 1; - } - else - { - /*********************************************************/ - /* we have two or more exact one's. */ - /*********************************************************/ - USHORT i; // index value - for ( i=1; i< *pusMatchesFound ;i++ ) - { - if ((pstMatchTable+i)->usMatchLevel < EQUAL_EQUAL) - { - *pusMatchesFound = i; - break; - } /* endif */ - } /* endfor */ - } /* endif */ - } /* endif */ - } /* endif */ } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); - - if ( usRc ) - { - ERREVENT2( GETEXACTMATCH_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ return( usRc ); } @@ -963,10 +634,10 @@ USHORT ExactTest PTMX_SENTENCE pSentence, //pointer to sentence structure PTMX_MATCH_TABLE_W pstMatchTable, //get out structure to be filled PUSHORT pusMatchEntries, //nr of entries in match entry structure - ULONG ulKeyNum // number of key record + LONG lKeyNum, // number of key record + PBYTE endOfMemRegion ) { - BOOL fOK; //success indicator PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record @@ -974,656 +645,483 @@ USHORT ExactTest PTMX_TARGET_CLB pTMXTargetClb; //ptr to target control block PTMX_TAGTABLE_RECORD pTMXSourceTagTable; //ptr to source tag info ULONG ulLen; //length indicator - PSZ_W pString = NULL; //pointer to character string USHORT usRc = NO_ERROR; //returned value from function USHORT usMatchLevel; //pointer to ushort value BOOL fStringEqual = FALSE; // indicator for string equal BOOL fEqualFound = FALSE; // tagging equal ?? USHORT usEqual = 0; - PSZ_W pContextBuffer = NULL; //pointer to buffer for context processing + std::wstring pString; //pointer to buffer for context processing + wchar_t pContextBuffer[MAX_SEGMENT_SIZE]; + pContextBuffer[0] = '\0'; USHORT usTargetTranslationFlag = (USHORT)-1; // translation flag of target CLB (processed) PSZ pszDocName = NULL; // document name to check for equal document names in GET_IGNORE_PATH mode + pString.reserve(MAX_SEGMENT_SIZE); - //allocate pString and buffer for context processing (when necessary) - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - if ( fOK ) - { - if ( pGetIn->szContext[0] || (pGetIn->pvGMOptList != NULL) ) - { - fOK = UtlAlloc( (PVOID *) &(pContextBuffer), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - } /* endif */ - - } /* endif */ + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - if ( !fOK ) + //move pointer to corresponding position + pByte = (PBYTE)(pTmRecord+1); + pByte += pTMXSourceRecord->usSource; + //get id of target language in the getin structure + USHORT usSrcLangId = 0; + if (pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, + (USHORT)LANG_KEY, &usSrcLangId)) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); + usSrcLangId = 0; + } /* endif */ + + if(!pTmClb->areLangsMatching(usSrcLangId, pTMXSourceRecord->usLangId, pGetIn->szSourceLanguage)){ + return SOURCE_LANG_DIFFERENT; } - else - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - //move pointer to corresponding position - pByte = (PBYTE)(pTmRecord+1); - pByte += pTMXSourceRecord->usSource; + //calculate length of source string + ulLen = (RECLEN(pTMXSourceRecord) - sizeof(TMX_SOURCE_RECORD)); - //calculate length of source string - ulLen = (RECLEN(pTMXSourceRecord) - sizeof(TMX_SOURCE_RECORD)); + //copy source string for later compare function + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + auto savedSrcStringVariants = std::make_unique(pString.c_str()); + //compare source strings + fStringEqual = UtlCompIgnWhiteSpaceW(savedSrcStringVariants->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; - //copy source string for later compare function - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - auto normalizedTmStr = std::make_unique(pString); - //compare source strings - fStringEqual = UtlCompIgnWhiteSpaceW(normalizedTmStr->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; + if ( fStringEqual ) + { + USHORT usTgtNum = 0; // init target number + LONG lLeftTgtLen; // remaining target length + USHORT usGetLang; // id of target language + USHORT usGetFile; // id of target file - if ( fStringEqual ) + USHORT usOldMatches = *pusMatchEntries; + + //get id of target language in the getin structure + if (pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, + (USHORT)LANG_KEY, &usGetLang)) + { + usGetLang = 0; + } /* endif */ + + //get file name id of file name in the getin structure + if ( pTmClb->NTMGetIDFromName( pGetIn->szLongName, + (USHORT)FILE_KEY, &usGetFile, + NTMGETID_NOUPDATE_OPT )) { - USHORT usTgtNum = 0; // init target number - LONG lLeftTgtLen; // remaining target length - USHORT usGetLang; // id of target language - USHORT usGetFile; // id of target file - USHORT usAlternateGetFile; // alternate ID of target file + usGetFile = 0; + } /* endif */ - USHORT usOldMatches = *pusMatchEntries; + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - //get id of target language in the getin structure - if (pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, - NULL, - (USHORT)LANG_KEY, &usGetLang)) - { - usGetLang = 1; - } /* endif */ + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + + //position at first target record + pByte = (PBYTE)(pTmRecord+1); + pByte += RECLEN(pTMXSourceRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pStartTarget = (PBYTE)pTMXTargetRecord; + + //source strings are identical so loop through target records + while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) + ( lLeftTgtLen > 0 ) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && ( RECLEN(pTMXTargetRecord) > 0)) + { + BOOL fTestCLB = TRUE; + BOOL fMatchingDocName = FALSE; + USHORT usContextRanking = 0; // context ranking of this match + + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + usTgtNum++; // target number + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + + //next check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + + //compare target language group IDs + if(!pTmClb->areLangsMatching(usGetLang, pTMXTargetClb->usLangId, pGetIn->szTargetLanguage)){ + //T5LOG( T5WARNING) <<":: source langs is different in record(" << pTMXSourceRecord->usLangId << ") and request (" << pGetIn->szSourceLanguage << ")"; + fTestCLB = FALSE; + } - //get file name id of file name in the getin structure - if ( pTmClb->NTMGetIDFromNameEx( pGetIn->szFileName, - pGetIn->szLongName, - (USHORT)FILE_KEY, &usGetFile, - NTMGETID_NOUPDATE_OPT,//0, - &usAlternateGetFile )) + if ( pTmClb->state->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->state->psLangIdToGroupTable[usGetLang] ) { - usGetFile = 1; - } /* endif */ + fTestCLB = FALSE; + } - if ( pGetIn->ulParm & GET_IGNORE_PATH ) + // compare target table IDs + if ( fTestCLB ) { - // get pointer to document name without path - if ( pGetIn->szLongName[0] != EOS ) + //position at source tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + + /**********************************************************/ + /* if tagging record is unequal than we have some */ + /* (slight) differences */ + /* i.e. we will create a fully qualified string and try */ + /* another compare... */ + /**********************************************************/ + fStringEqual = (UtlCompIgnWhiteSpaceW( + pSentence->pStrings->getNpReplStrC(), + savedSrcStringVariants->getNpReplStrC(), + 0 ) == 0 ); + + if (fStringEqual) { - pszDocName = UtlGetFnameFromPath( pGetIn->szLongName ); - if ( pszDocName == NULL ) - { - pszDocName = pGetIn->szLongName; - } /* endif */ - } + fEqualFound = TRUE; + usEqual = EQUAL_EQUAL; + } else { - pszDocName = pGetIn->szFileName; - } /* endif */ - } /* endif */ - - - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - //position at first target record - pByte = (PBYTE)(pTmRecord+1); - pByte += RECLEN(pTMXSourceRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pStartTarget = (PBYTE)pTMXTargetRecord; - - //source strings are identical so loop through target records - while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) - - ( lLeftTgtLen > 0 ) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && ( RECLEN(pTMXTargetRecord) > 0)) - { - BOOL fTestCLB = TRUE; - BOOL fMatchingDocName = FALSE; - USHORT usContextRanking = 0; // context ranking of this match - - - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - usTgtNum++; // target number - - //assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); - - lLeftTgtLen -= RECLEN(pTMXTargetRecord); + usEqual = EQUAL_EQUAL - 3; + } /* endif */ - //next check the target language - //position at target control block + //extract data from tm record + //target with tags(TRUE) + //compare file name and segment id in control block + //position at control block + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pByte += pTMXTargetRecord->usClb; pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - - //compare target language group IDs - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usGetLang] ) + usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; + + // loop over CLBs and look for best matching entry + LONG lLeftClbLen; // left CLB entries in CLB list + PTMX_TARGET_CLB pClb; // pointer for CLB list processing + #define SEG_DOC_AND_CONTEXT_MATCH 8 + #define DOC_AND_CONTEXT_MATCH 7 + #define CONTEXT_MATCH 6 + #define SAME_SEG_AND_DOC_MATCH 5 + #define SAME_DOC_MATCH 4 + #define MULT_DOC_MATCH 3 + #define NORMAL_MATCH 2 + #define IGNORE_MATCH 1 + SHORT sCurMatch = 0; + + // loop over all target CLBs + pClb = pTMXTargetClb; + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + while ( ( lLeftClbLen > 0 ) && (sCurMatch < SAME_SEG_AND_DOC_MATCH) ) { - fTestCLB = FALSE; - } - - // check for comments when requested - if ( pGetIn->ulParm & GET_IGNORE_COMMENT ) - { - if ( NtmFindInAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, L"" ); - pszTarget += 4; - } - else if ( *pszSource == L'\r' ) + /**************************************************************/ + /* use only the best one... */ + /**************************************************************/ + if (*pusMatchEntries > usOldMatches ) { - wcscpy( pszTarget, L"" ); - pszTarget += 4; - } - else - { - *pszTarget++ = *pszSource; + *pusMatchEntries = usOldMatches + 1; } /* endif */ - pszSource++; - } /*endwhile */ - *pszTarget = 0; - } - - - void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ) - { - static CHAR_W szSegBuf[4096]; - static CHAR_W szLineBuf[4096]; - int iLen; - - NTMMarkCRLF( pszSegData, szSegBuf ); - //iLen = swprintf( szLineBuf, pszForm, szSegBuf ); - //fwrite( szLineBuf, 2, iLen, hfLog ); - } - -// removed superfluos whitespace at the beggining or end of the proposal -void NTMRemoveAdditionalWhitepace -( - PSZ_W pszSource, - PSZ_W pszProposal -) -{ - int iLen; - - // only remove leading whitespace if source has none - if ( (*pszSource != L'\n') && (*pszSource != L'\r') && (*pszSource != L' ') ) - { - PSZ_W pszProp = pszProposal; - PSZ_W pszTemp = pszProposal; - - while ( (*pszTemp == L'\n') || (*pszTemp == L'\r') || (*pszTemp == L' ') ) - { - pszTemp++; - } /*endwhile */ - - if ( pszTemp != pszProp ) - { - while ( *pszTemp ) - { - *pszProp++ = *pszTemp++; - } /*endwhile */ - *pszProp = 0; } /* endif */ } /* endif */ - // check trailing whitespace - iLen = wcslen( pszSource ); - if ( iLen ) - { - iLen--; - if ( (pszSource[iLen] != L'\n') && (pszSource[iLen] != L'\r') && (pszSource[iLen] != L' ') ) - { - iLen = wcslen( pszProposal ); - while ( iLen && ( (pszProposal[iLen-1] == L'\n') || (pszProposal[iLen-1] == L'\r') || (pszProposal[iLen-1] == L' ') ) ) - { - iLen--; - pszProposal[iLen] = 0; - } /*endwhile */ - } /* endif */ - } /* endif */ -} /* end of function NTMRemoveAdditionalWhitepace */ - -#endif - + return( usRc ); +} BOOL TMFuzzynessEx @@ -1633,19 +1131,13 @@ BOOL TMFuzzynessEx PSZ_W pszMatch, // found match SHORT sLanguageId, // language id to be used PUSHORT pusFuzzy, // fuzzyness - ULONG ulOemCP, PUSHORT pusWords, // number of words in segment PUSHORT pusDiffs // number of diffs in segment ) { - BOOL fOK; - PFUZZYTOK pFuzzyTgt = NULL; - PFUZZYTOK pFuzzyTok = NULL; // returned token list - PSZ pInBuf = NULL; // buffer for EQFBFindDiffEx function - PSZ pTokBuf = NULL; // buffer for EQFBFindDiffEx function + std::vector pFuzzyTok,pFuzzyTgt; // returned token list PLOADEDTABLE pTable = NULL; // ptr to loaded tag table - // fast exit if one or both strings are empty... if ( (*pszSource == EOS) || (*pszMatch == EOS) ) { @@ -1664,24 +1156,19 @@ BOOL TMFuzzynessEx return( TRUE ); } /* endif */ - // allocate required buffers - fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 64000L, NOMSG ); - if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, 64000 /*(LONG)TOK_BUFFER_SIZE*/, NOMSG ); - // load tag table - if ( fOK ) - { - fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, + BOOL fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC, FALSE, NULLHANDLE ) == NO_ERROR); - } /* endif */ + + // call function to evaluate the differences if ( fOK ) { - fOK = EQFBFindDiffEx( pTable, (PBYTE)pInBuf, (PBYTE)pTokBuf, pszSource, - pszMatch, sLanguageId, (PVOID *)&pFuzzyTok, - (PVOID *)&pFuzzyTgt, ulOemCP ); + fOK = EQFBFindDiffEx( pTable, pszSource, + pszMatch, sLanguageId, pFuzzyTok, + pFuzzyTgt ); } /* endif */ if ( fOK ) @@ -1690,10 +1177,6 @@ BOOL TMFuzzynessEx } /* endif */ // free allocated buffers and lists - if ( pFuzzyTgt ) UtlAlloc( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); - if ( pFuzzyTok ) UtlAlloc( (PVOID *)&pFuzzyTok, 0L, 0L, NOMSG ); - if ( pInBuf ) UtlAlloc( (PVOID *)&pInBuf, 0L, 0L, NOMSG ); - if ( pTokBuf ) UtlAlloc( (PVOID *)&pTokBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable ); if(*pusWords > *pusDiffs){ @@ -1741,153 +1224,75 @@ BOOL TMFuzzynessEx // ----------------------------------------------------------------------------+ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct - PSZ_W pSourceString, //pointer to normalized source string + std::wstring& pSourceString, //pointer to normalized source string PLONG plSourceLen, //length of source string PTMX_TARGET_RECORD pTMXTargetRecord, //pointer to tm target PTMX_TARGET_CLB pTMXTargetClb, //ptr to target control block PTMX_MATCH_TABLE_W pstMatchTable, //getout match table struct PUSHORT pusMatchLevel, //how good is match - BOOL fTag, //target with/without subst. tags PUSHORT pusMatchEntries, //nr of entries in match table - PUSHORT pusMaxVotes, //nr of tm source string votes PUSHORT pusOverlaps, //nr of overlapping triples - ULONG ulKeyNum, // key number + LONG lKeyNum, // key number USHORT usTgtNum, // target number PTMX_GET_W pGetIn, // input structure - PTMX_TAGTABLE_RECORD pTagRecord, + BytesRef pTagRecord, USHORT usTranslationFlag, USHORT usContextRanking, - int iWords, int iDiffs, USHORT usSrcLangId ) + int iWords, int iDiffs, USHORT usSrcLangId, PBYTE endOfMemRegion) { - PSZ pSrcFileName = pGetIn->szFileName; // source file name PSZ pSrcLongName = pGetIn->szLongName; // long source file name PBYTE pByte; //position pointer - BOOL fOK; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen; //length indicator LONG ulLen; //length indicator - PSZ_W pString = NULL; //pointer to temp string + std::wstring pString; //pointer to temp string USHORT usCurrentEntry; //counter BOOL fFound = FALSE; //indicates if correct pos found - PTMX_SUBSTPROP pSubstProp = NULL; // tag substitution array - BOOL fSubstAll = FALSE; + TMX_SUBSTPROP SubstProp; // tag substitution array PTMX_MATCH_TABLE_W pstMatchTableStart = pstMatchTable; //point to start of match table - ULONG ulSrcOemCP = 0L; - ULONG ulTgtOemCP = 0L; TMManager *pFactory = TMManager::GetInstance(); SHORT sLangID = 0; - - // force fTag flag if proposal with inline tagging is requested - if ( pGetIn->ulParm & GET_ALWAYS_WITH_TAGS ) + /****************************************************************/ + /* add current tagging to source and translation of proposal */ + /****************************************************************/ + strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); + //strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); + //strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); + UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); + pByte = (PBYTE)pTMXTargetRecord; + + MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); + + wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), pSourceString.size());// *plSourceLen); + if ( usRc == NO_ERROR ) { - fTag = TRUE; - } /* endif */ + //position at target string + pByte = (PBYTE)pTMXTargetRecord; + pByte += pTMXTargetRecord->usTarget; - /********************************************************************/ - /* if tagtables are different, call substitution function */ - /********************************************************************/ - if ( !(pGetIn->ulParm & GET_NO_GENERICREPLACE) ) - { - fOK = UtlAlloc( (PVOID*)&pSubstProp, 0L, (LONG)sizeof(TMX_SUBSTPROP), NOMSG); + //calculate length of target string + lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - if ( !fOK ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); + std::wstring output; + lTargetLen = EQFCompress2Unicode(output, pByte, lTargetLen); + wcsncpy(SubstProp.szPropTarget, output.c_str(), sizeof(SubstProp.szPropTarget)/sizeof(SubstProp.szPropTarget[0])); + //position at target tag record + pByte = (PBYTE)pTMXTargetRecord; } - else + if ( usRc == NO_ERROR ) { - /****************************************************************/ - /* add current tagging to source and translation of proposal */ - /****************************************************************/ - ulSrcOemCP = 1; - strcpy( pSubstProp->szSourceTagTable, pGetIn->szTagTable ); - strcpy( pSubstProp->szSourceLanguage, pGetIn->szSourceLanguage); - strcpy( pSubstProp->szTargetLanguage, pGetIn->szTargetLanguage ); - UTF16strcpy( pSubstProp->szSource, pGetIn->szSource ); - //pSubstProp->pTagsSource = pTagRecord; - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - - MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); - - //pSubstProp->pTagsPropSource = (PTMX_TAGTABLE_RECORD)pByte; - //usRc = (AddTagsToStringW( pSourceString, - // plSourceLen, // in # of w's - // (PTMX_TAGTABLE_RECORD)pByte, - // pSubstProp->szPropSource )) ? usRc : BTREE_CORRUPTED; - //instead of lines above we now save not normalize string to btree - wcsncpy(pSubstProp->szPropSource, pSourceString, *plSourceLen); - if ( usRc == NO_ERROR ) - { - PSZ_W pTarget; - - //position at target string - pByte = (PBYTE)pTMXTargetRecord; - pByte += pTMXTargetRecord->usTarget; - - //calculate length of target string - lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - - if ( UtlAlloc( (PVOID *) &pTarget, 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG)) - { - lTargetLen = EQFCompress2Unicode(pTarget, pByte, lTargetLen); - //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - - //pSubstProp->pTagsPropTarget = (PTMX_TAGTABLE_RECORD)pByte; - - //usRc = (AddTagsToStringW( pTarget, - // &lTargetLen, // in # of w's - // (PTMX_TAGTABLE_RECORD)pByte, - // pSubstProp->szPropTarget )) ? usRc : BTREE_CORRUPTED; - //instead of lines above we now save not normalize string to btree - wcsncpy(pSubstProp->szPropTarget, pTarget, lTargetLen); - UtlAlloc( (PVOID *) &pTarget, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - if ( usRc == NO_ERROR ) - { - //fill in the markup table - //PBYTE p = ((PBYTE)pTMXTargetRecord)+pTMXTargetRecord->usTargetTagTable; - //pTmClb->NTMGetNameFromID( - // &(((PTMX_TAGTABLE_RECORD)p)->usTagTableId), - // (USHORT)TAGTABLE_KEY, - // pSubstProp->szPropTagTable, NULL ); - strcpy(pSubstProp->szPropTagTable, "OTMXUXLF"); - - ulTgtOemCP = 1; - - //fill in the target language - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - pSubstProp->szTargetLanguage, NULL ); - //pTmClb->NTMGetNameFromID( &usSrcLangId, - // (USHORT)LANG_KEY, - // pSubstProp->szOriginalSrcLanguage, NULL ); - } /* endif */ - - } /* endif */ + //fill in the markup table + strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); + //fill in the target language + auto targetKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + SubstProp.szTargetLanguage ); } /* endif */ - } - else - { - } /* endif */ - - /********************************************************************/ - /* if src of prop is empty ( i.e. only inline tagging), add tagging */ - /********************************************************************/ - if (*plSourceLen <= 0 ) - { - fTag = TRUE; } /* endif */ if ( usRc == NO_ERROR ) { - //locate correct position in matchtable array usCurrentEntry = 0; while ( !fFound && usCurrentEntry < MAX_MATCHES ) @@ -1906,11 +1311,10 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } else if ( iMatchSortKey == iNewMatchSortKey ) { - CHAR szTgtFileName[ MAX_FILESPEC ]; // target file name - static CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name + CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name //fill in the target file name - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtFileName, szTgtLongName ); + auto targetKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtLongName, sizeof(szTgtLongName) ); /****************************************************************/ /* as the matching levels are the same determine the most recent*/ @@ -1918,15 +1322,14 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* higher priority */ /****************************************************************/ - if ( NTMDocMatch( szTgtFileName, szTgtLongName, pSrcFileName, pSrcLongName ) ) + if ( NTMDocMatch( szTgtLongName, pSrcLongName ) ) { /**************************************************************/ /* if compared match is from same file, check the update time */ /* because now we deal with two matches from same source doc. */ /**************************************************************/ if ( (pstMatchTable->lTargetTime > pTMXTargetClb->lTime) && - NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, - pSrcFileName, pSrcLongName ) ) + NTMDocMatch( pstMatchTable->szLongName, pSrcLongName ) ) { pstMatchTable++; usCurrentEntry++; @@ -1934,10 +1337,17 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct else { //found the right position + fFound = TRUE; + //OLD CODE ULONG ulLen = ( MAX_MATCHES - usCurrentEntry - 1 ) * sizeof(TMX_MATCH_TABLE_W); memmove( pstMatchTable+1, pstMatchTable, ulLen ); memset( pstMatchTable, 0, sizeof(TMX_MATCH_TABLE_W) ); - fFound = TRUE; + //NEW CODE + //for (int i = MAX_MATCHES - 2; i >= usCurrentEntry; --i) { + // pstMatchTable[i + 1] = pstMatchTable[i]; // uses copy assignment operator + //} + //pstMatchTable[usCurrentEntry] = TMX_MATCH_TABLE_W{}; // reset the entry + } /* endif */ } else @@ -1947,7 +1357,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* otherwise put in our match, it's coming from same source */ /* document */ /**************************************************************/ - if ( NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, pSrcFileName, pSrcLongName ) ) + if ( NTMDocMatch( pstMatchTable->szLongName, pSrcLongName ) ) { // match from match table is from same document, so skip to next position pstMatchTable++; @@ -1982,35 +1392,14 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( fFound ) { - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - - if ( !fOK ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - if (fSubstAll ) - { - /**************************************************************/ - /* do a substitution of the tags... */ - /**************************************************************/ - *plSourceLen = UTF16strlenCHAR( pSubstProp->szPropSource ); - if( *plSourceLen ) { - memcpy( pstMatchTable->szSource, pSubstProp->szPropSource, - *plSourceLen * sizeof(CHAR_W)); - } - } - else - { - // copy normalized propsource string - if(*plSourceLen > 0 ){ - memcpy( pstMatchTable->szSource, pSourceString, *plSourceLen * sizeof(CHAR_W)); - } - } /* endif */ + pByte = (PBYTE)pTMXTargetRecord; + // copy normalized propsource string + //if(*plSourceLen > 0 ){ + //T5LOG(T5TRANSACTION) << "memcpy string, len = " << *plSourceLen <<"; pSourceString.size()=" << pSourceString.size()<<"; str=\'" << EncodingHelper::convertToUTF8(pSourceString) <<"\'; position in match table = " << pstMatchTable - pstMatchTableStart; + memcpy( pstMatchTable->szSource, pSourceString.c_str(), pSourceString.size() * sizeof(CHAR_W)); + pstMatchTable->szSource[pSourceString.size()] = L'\0'; + //} if ( usRc == NO_ERROR ) { @@ -2025,101 +1414,42 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct lTargetLen = EQFCompress2Unicode( pString, pByte, lTargetLen ); // now ulTargetLen is # of w's //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - - if (fSubstAll ) - { - /**************************************************************/ - /* do a substitution of the replaced target string and */ - /* adjust match level */ - /**************************************************************/ - lTargetLen = UTF16strlenCHAR( pSubstProp->szPropTarget ); - memcpy( pstMatchTable->szTarget, pSubstProp->szPropTarget, - lTargetLen * sizeof(CHAR_W)); - - ulLen = UTF16strlenCHAR(pSubstProp->szSource); - *plSourceLen = UTF16strlenCHAR(pSubstProp->szPropSource); - if (*plSourceLen > ulLen ) - { - ulLen = *plSourceLen; - } /* endif */ - - { - BOOL fEqual; - USHORT usFuzzy = 0, usWords = 0, usDiffs = 0; - BOOL fFuzzynessOK; - -/*Q!*/ fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, - pSubstProp->szPropSource, pSubstProp->szSource, - sLangID, &usFuzzy, ulSrcOemCP, &usWords, &usDiffs ); - - if ( pGetIn->ulParm & GET_RESPECTCRLF ) - { - fEqual = (UtlCompIgnSpaceW( pSubstProp->szPropSource, - pSubstProp->szSource, 0 ) == 0); - } - else - { - fEqual = (usFuzzy == 100); -// (UtlCompIgnWhiteSpace( pSubstProp->szPropSource, -// pSubstProp->szSource, -// ulLen) == 0); + pByte = (PBYTE)pTMXTargetRecord; + //memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); + memcpy( pstMatchTable->szTarget, pString.data(), pString.size() * sizeof(CHAR_W)); + pstMatchTable->szTarget[pString.size()] = '\0'; - } /* endif */ - - // GQ 2004/08/04: do not set match level to 100 for machine translations!! - // RJ 2004/08/16: assure that *pusMatchLevel is filled independently - // of bMT flag set or not (P020111) - if ( fEqual && (usFuzzy == 100) /*&& !pTMXTargetClb->bMT */) - { - *pusMatchLevel = (usTranslationFlag == TRANSLFLAG_MACHINE) ? 99 : 100; - } /* endif */ -#ifdef SGMLDITA_LOGGING - T5LOG(T5INFO) << "new match level is "<< *pusMatchLevel <<", fuzziness is " << usFuzzy ; -#endif - } - } - else - { - memcpy( pstMatchTable->szTarget, pString, lTargetLen * sizeof(CHAR_W)); - } /* endif */ + //fill in the markup table + strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); if ( usRc == NO_ERROR ) { - //position at target control block - // should already be done!!! - // pByte = (PBYTE)pTMXTargetRecord; - // pByte += pTMXTargetRecord->usClb; - // pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - - //fill in the target file name - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usFileId, - (USHORT)FILE_KEY, - pstMatchTable->szFileName, - pstMatchTable->szLongName ); - //fill in the target author - don't care about author name.... - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usAuthorId, - (USHORT)AUTHOR_KEY, - pstMatchTable->szTargetAuthor, NULL ); - - //fill in the markup table { - //pTmClb->NTMGetNameFromID( - // &(((PTMX_TAGTABLE_RECORD)p)->usTagTableId), - // (USHORT)TAGTABLE_KEY, - // pstMatchTable->szTagTable, NULL ); - strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); + //fill in the target file name + auto fileKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, + (USHORT)FILE_KEY, + pstMatchTable->szLongName, sizeof(pstMatchTable->szLongName)); + //fill in the target author - don't care about author name.... + auto authorKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usAuthorId, + (USHORT)AUTHOR_KEY, + pstMatchTable->szTargetAuthor ); + + //fill in the target and original src language + auto targetLangKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + pstMatchTable->szTargetLanguage ); + + auto srcLangKeyRc = pTmClb->NTMGetNameFromID( usSrcLangId, + (USHORT)LANG_KEY, + pstMatchTable->szOriginalSrcLanguage ); + if( ID_NOT_FOUND == fileKeyRc + || ID_NOT_FOUND == authorKeyRc + //|| ID_NOT_FOUND == langKeyRc //skip lang check because reorganize would delete that records + ){ + pTmClb->state->fCorrupted = true;//reorganize requested + } + } - - //fill in the target and original src language - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - pstMatchTable->szTargetLanguage, NULL ); - - pTmClb->NTMGetNameFromID( &usSrcLangId, - (USHORT)LANG_KEY, - pstMatchTable->szOriginalSrcLanguage, NULL ); //fill in the segment id pstMatchTable->ulSegmentId = pTMXTargetClb->ulSegmId; //state whether machine translation @@ -2134,19 +1464,16 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct pstMatchTable->usOverlaps = *pusOverlaps; pstMatchTable->usContextRanking = usContextRanking; - - pstMatchTable->szContext[0] = 0; - pstMatchTable->szAddInfo[0] = 0; if ( pTMXTargetClb->usAddDataLen ) { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext, sizeof(pstMatchTable->szContext) / sizeof(CHAR_W) ); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo, sizeof(pstMatchTable->szAddInfo) / sizeof(CHAR_W) ); + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext, endOfMemRegion ); + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo, endOfMemRegion ); } /* endif */ /****************************************************************/ /* fill in key and target number ... */ /****************************************************************/ - pstMatchTable->ulKey = ulKeyNum; + pstMatchTable->lKey = lKeyNum; pstMatchTable->usTargetNum = usTgtNum; @@ -2166,8 +1493,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ((iMatchSortKeyTemp < iMatchSortKey) || ((iMatchSortKeyTemp == iMatchSortKey) && (pstMatchTableTemp->lTargetTime < pstMatchTable->lTargetTime) && - NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, - pstMatchTableTemp->szFileName, pstMatchTableTemp->szLongName)) ) + NTMDocMatch( pstMatchTable->szLongName, pstMatchTableTemp->szLongName)) ) { // interchange entries pstMatchTable moved on position pstMatchTableTemp TMX_MATCH_TABLE_W stMatchEntry; @@ -2191,22 +1517,9 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } /* endif */ } /* endif */ } /* endif */ - - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ - if ( usRc ) - { - ERREVENT2( FILLMATCHTABLE_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - - /********************************************************************/ - /* free any allocated resource */ - /********************************************************************/ - NTMFreeSubstProp( pSubstProp ); - return( usRc ); } @@ -2214,341 +1527,99 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //+----------------------------------------------------------------------------+ //|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: AddTagsToString | -//+----------------------------------------------------------------------------+ -//|Description: Add tags to correct position of the normalized string | -//+----------------------------------------------------------------------------+ -//|Parameters: PSZ pInString | -//| PUSHORT ousInStringLen | -//| PTMX_TAGTABLE_RECORD pTMXTagTable | -//| PSZ pOutString | -//+----------------------------------------------------------------------------+ -//|Output parameter: PSZ pOutString | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Returncodes: | -//+----------------------------------------------------------------------------+ -//|Function flow: add tags at the given offsets and output string with tags| -// ----------------------------------------------------------------------------+ - -BOOL AddTagsToStringW( PSZ_W pInString, //character string - PLONG plInStringLen, //length of pInString in # of w's - PTMX_TAGTABLE_RECORD pTMXTagTable, //ptr to tag table record - PSZ_W pOutString ) //output string -{ - PBYTE pByte; //record position pointer - USHORT usStringPos, j; //positioning counters - PTMX_TAGENTRY pTagEntry = NULL; //ptr to tag entries in tag table record - ULONG ulTagLen = 0; //length of all tags added to string - ULONG ulEnd = 0; //end length for for loop - USHORT usTagRecordPos = 0; //current length processed in tag record - BOOL fNoTagsLeft = FALSE; //indication of existence of tags - ULONG ulInputStringLen = *plInStringLen; - PSZ_W pOutStart = pOutString; // anchar start pointers - PSZ_W pInStart = pInString; // - BOOL fOK = TRUE; - //position at beginning of target record - pByte = (PBYTE)pTMXTagTable; - pByte += pTMXTagTable->usFirstTagEntry; - - fOK = (*plInStringLen < MAX_SEGMENT_SIZE ); - - if ( fOK && ( RECLEN(pTMXTagTable) > sizeof(TMX_TAGTABLE_RECORD)) ) - { - //there is tag info so go through all tag entries and add tags to outstring - pTagEntry = (PTMX_TAGENTRY)pByte; - - //start of for loop - usStringPos = 0; - //initial positioning st beginning of first tag entry - usTagRecordPos = sizeof(TMX_TAGTABLE_RECORD); - //initial stop criterium for end of for loop - ulEnd = pTagEntry->usOffset; - - while ( fOK && (*pInString || !fNoTagsLeft) ) - { - if ( ulEnd <= ulInputStringLen ) - { - for ( j = usStringPos; j < ulEnd; j++, pOutString++, pInString++ ) - { - *pOutString = *pInString; - } /* endfor */ - } - else - { - for ( j = usStringPos; j < ulEnd; j++, pOutString++, pInString++ ) - { - if ( j < ulInputStringLen ) - { - *pOutString = *pInString; - } - else - { - *pOutString = ' '; - --pInString; // we've gone 1 too far.. - } /* endif */ - - if ( pOutString >= - pOutStart + MAX_SEGMENT_SIZE + usStringPos - ulEnd ) - { - *pOutStart = EOS; - fOK = FALSE; - break; - } /* endif */ - } /* endfor */ - } /* endif */ - - if ( fOK && (j == ulEnd) && (!fNoTagsLeft) ) - { - /**************************************************************/ - /* Add check that tag length does not jeopardize our */ - /* output string... */ - /**************************************************************/ - if ( (pOutString+pTagEntry->usTagLen) < (pOutStart+MAX_SEGMENT_SIZE) - && (pTagEntry->usTagLen < MAX_SEGMENT_SIZE) ) - { - USHORT usTagLenCHAR = pTagEntry->usTagLen; - memcpy( pOutString, &pTagEntry->bData, usTagLenCHAR * sizeof(CHAR_W)); - pOutString += usTagLenCHAR; - ulInputStringLen += usTagLenCHAR; - //reset for loop start - usStringPos = j + usTagLenCHAR; - //update tag table record position counter - usTagRecordPos += sizeof(TMX_TAGENTRY) + usTagLenCHAR * sizeof(CHAR_W); - //remember length of tags added to string - ulTagLen += pTagEntry->usTagLen; - - //move to next tag entry only if there are still tags to add to string - if ( usTagRecordPos < RECLEN(pTMXTagTable) ) - { - USHORT usOldOffs = pTagEntry->usOffset; - pByte += sizeof(TMX_TAGENTRY) + pTagEntry->usTagLen * sizeof(CHAR_W); - pTagEntry = (PTMX_TAGENTRY)pByte; - //new end criterium for for loop - if ( usOldOffs == pTagEntry->usOffset ) - { - // something went wrong: two tag entries for the same offset - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W)); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } - else if ( pTagEntry->usOffset < ulEnd ) - { - // tag data seems to be corrupted, ignore the tags - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W)); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } - else - { - ulEnd = pTagEntry->usOffset; - } /* endif */ - } - else - { - fNoTagsLeft = TRUE; - //new end criterium for for loop, in this case end of string - ulEnd = *plInStringLen + ulTagLen; - } /* endif */ - } - else - { - // something got corrupted -- ignore the tags ... - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W) ); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } /* endif */ - } /* endif */ - - if ( pOutString >= pOutStart + MAX_SEGMENT_SIZE ) - { - *pOutStart = EOS; - fOK = FALSE; - break; - } /* endif */ - } /* endwhile */ - - if ( *pOutStart ) - { - // Add the end of string - *pOutString = EOS; - } - } - else - { - //there are no tags so copy character string - if ( *plInStringLen < MAX_SEGMENT_SIZE ) - { - memcpy( pOutString, pInString, *plInStringLen * sizeof(CHAR_W)); - pOutString[*plInStringLen] = EOS; - } - else - { - fOK = FALSE; - *pOutStart = EOS; - } /* endif */ - } /* endif */ - - return fOK; +//+----------------------------------------------------------------------------+ +//|Function name: AddTagsToString | +//+----------------------------------------------------------------------------+ +//|Description: Add tags to correct position of the normalized string | +//+----------------------------------------------------------------------------+ +//|Parameters: PSZ pInString | +//| PUSHORT ousInStringLen | +//| PTMX_TAGTABLE_RECORD pTMXTagTable | +//| PSZ pOutString | +//+----------------------------------------------------------------------------+ +//|Output parameter: PSZ pOutString | +//+----------------------------------------------------------------------------+ +//|Returncode type: BOOL | +//+----------------------------------------------------------------------------+ +//|Returncodes: | +//+----------------------------------------------------------------------------+ +//|Function flow: add tags at the given offsets and output string with tags| +// ----------------------------------------------------------------------------+ -} USHORT GetFuzzyMatch ( - EqfMemory* pTmClb, //ptr to ctl block struct + EqfMemory* pTmClb, //ptr to ctl block struct PTMX_SENTENCE pSentence, //ptr to sentence structure PTMX_GET_W pGetIn, //ptr to data in get in structure PTMX_MATCH_TABLE_W pstMatchTable, //get out output structure PUSHORT pusMatchesFound //number of matches found ) { - BOOL fOK = TRUE; //success indicator USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - PTMX_RECORD pTmRecord = NULL; //pointer to tm record + std::vector pMatchEntries; //pointer to match entry structure PTMX_MATCHENTRY pMatchEntry = NULL; //pointer to match entry structure - PTMX_MATCHENTRY pMatchStart = NULL; //pointer to match entry structure USHORT usMatchEntries = 0; //nr of matches found USHORT usOverlaps = 0; //nr of overlapping triples - ULONG ulRecBufSize = 0L; // current size of record buffer - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lOtherTime) ); -#endif - - //allocate 32K for tm record if not done yet - if ( pTmClb->pvTmRecord ) - { - pTmRecord = (PTMX_RECORD)pTmClb->pvTmRecord; - ulRecBufSize = pTmClb->ulRecBufSize; -// memset( pTmRecord, 0, ulRecBufSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvTmRecord = pTmRecord; - pTmClb->ulRecBufSize = ulRecBufSize = TMX_REC_SIZE; - } /* endif */ - - //allocate for match entry - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pMatchEntry), 0L, (LONG)(ABS_VOTES * sizeof(TMX_MATCHENTRY)), NOMSG ); + + BytesRef pTmRecord = pTmClb->state->pvTmRecord; + + //fill pMatchEntry with the tm keys with the highest frequency + usRc = FillMatchEntry( pTmClb, pSentence, pMatchEntries, &pGetIn->usMatchThreshold ); + pMatchEntry = pMatchEntries.data(); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + if ( usRc == NO_ERROR ) { - pMatchStart = pMatchEntry; - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - //fill pMatchEntry with the tm keys with the highest frequency - usRc = FillMatchEntry( pTmClb, pSentence, pMatchEntry, &pGetIn->usMatchThreshold ); -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyFillMatchEntry) ); -#endif - if ( usRc == NO_ERROR ) + //get tm record(s) + while ( (pMatchEntry->lKey) && (usRc == NO_ERROR)) // && + //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) { - //get tm record(s) - while ( (pMatchEntry->ulKey) && (usRc == NO_ERROR)) // && - //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) - { -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - ulLen = ulRecBufSize; - - T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->ulKey; + T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->lKey; + usRc = pTmClb->state->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, (PCHAR)pTmRecord, &ulLen ); - - if ( usRc == BTREE_BUFFER_SMALL) + if ( usRc == NO_ERROR ) + { + T5LOG( T5INFO) << "GetFuzzyMatch: MaxVotes=" << pMatchEntry->usMaxVotes << ", ActVote=" << pSentence->usActVote << " EQFNTMGET of record "; + + //compare tm record data with data passed in the get in structure + usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); + PBYTE endOfMemRegion = pTmRecord.data() + pTmRecord.size(); + usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, + &usMatchEntries, &usOverlaps, + &pMatchEntry->usMaxVotes, &pSentence->usActVote, + pSentence, pMatchEntry->lKey, endOfMemRegion ); + if(usRc == SOURCE_LANG_DIFFERENT) { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - pTmClb->ulRecBufSize = ulRecBufSize = ulLen; - pTmClb->pvTmRecord = pTmRecord; - memset( pTmRecord, 0, ulLen ); - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, (PCHAR)pTmRecord, &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyGetTime) ); -#endif - if ( usRc == NO_ERROR ) + usRc = NO_ERROR; + } + if ( !usRc ) { - T5LOG( T5INFO) << "GetFuzzyMatch: MaxVotes=" << pMatchEntry->usMaxVotes << ", ActVote=" << pSentence->usActVote << " EQFNTMGET of record "; - - //compare tm record data with data passed in the get in structure - usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); - usRc = FuzzyTest( pTmClb, pTmRecord, pGetIn, pstMatchTable, - &usMatchEntries, &usOverlaps, - &pMatchEntry->usMaxVotes, &pSentence->usActVote, - pSentence, pMatchEntry->ulKey ); - if(usRc == SOURCE_LANG_DIFFERENT) - { - usRc = NO_ERROR; - } - if ( !usRc ) - { - //nr of matches found - (*pusMatchesFound) = usMatchEntries; - - //get next tm record - pMatchEntry++; - }/* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyTestTime) ); -#endif + //nr of matches found + (*pusMatchesFound) = usMatchEntries; - } /* endif */ - } /* endwhile */ + //get next tm record + pMatchEntry++; + }/* endif */ + } /* endif */ + } /* endwhile */ - // limit matches to number of matches requested - if (*pusMatchesFound > pGetIn->usRequestedMatches) + // limit matches to number of matches requested + if (*pusMatchesFound > pGetIn->usRequestedMatches) + { + // special handling for machine matches: ensure that at least one machine match is left in the truncated list + if ( (pstMatchTable[(*pusMatchesFound)-1].usTranslationFlag == TRANSLFLAG_MACHINE) && + (pstMatchTable[pGetIn->usRequestedMatches-1].usMatchLevel < 100)) { - // special handling for machine matches: ensure that at least one machine match is left in the truncated list - if ( (pstMatchTable[(*pusMatchesFound)-1].usTranslationFlag == TRANSLFLAG_MACHINE) && - (pstMatchTable[pGetIn->usRequestedMatches-1].usMatchLevel < 100)) - { - memcpy( pstMatchTable + (pGetIn->usRequestedMatches-1), pstMatchTable + (*pusMatchesFound - 1), sizeof(TMX_MATCH_TABLE_W) ); - } /* endif */ - - memset( &pstMatchTable[pGetIn->usRequestedMatches], 0, sizeof( TMX_MATCH_TABLE_W )); - *pusMatchesFound = pGetIn->usRequestedMatches; - } - - } /* endif */ - } /* endif */ - - //release memory - UtlAlloc( (PVOID *) &pMatchStart, 0L, 0L, NOMSG ); + memcpy( pstMatchTable + (pGetIn->usRequestedMatches-1), pstMatchTable + (*pusMatchesFound - 1), sizeof(TMX_MATCH_TABLE_W) ); + } /* endif */ - if ( usRc ) - { - ERREVENT2( GETFUZZYMATCH_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); + memset( &pstMatchTable[pGetIn->usRequestedMatches], 0, sizeof( TMX_MATCH_TABLE_W )); + *pusMatchesFound = pGetIn->usRequestedMatches; + } } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - return( usRc ); } @@ -2587,6 +1658,31 @@ std::wstring removeTagsFromString(std::wstring input){ return std::wstring(pOutStart); } +bool EqfMemory::areLangsMatching(const ushort langId1, const ushort langId2, + const char* langStr1 + ){ + if(!langId1 || !langId2){// if any of them has id=0 - no match + return false; + } + + //lang matching + //if they are equal + if(langId1 == langId2){ + return true; + }//or if they both are from the same langGroup and first or second are preffered(or both)// are not sublocale langs + else if (( state->psLangIdToGroupTable[langId1] == state->psLangIdToGroupTable[langId2] )){ + if(!LanguageFactory::getInstance()->hasSubLocale(langStr1)){ + return true; + } + char langStr2[MAX_LANG_LENGTH]{}; + NTMGetNameFromID( langId2, (USHORT)LANG_KEY, langStr2 ); + return !LanguageFactory::getInstance()->hasSubLocale(langStr2); + } + return false; + // +} + + @@ -2629,7 +1725,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block PUSHORT pusTmMaxVotes, //nr of tm source triple PUSHORT pusGetMaxVotes, //nr of get source triples PTMX_SENTENCE pSentence, //ptr to sentence structure - ULONG ulKeyNum ) // record number + LONG lKeyNum, PBYTE endOfMemRegion ) // record number { PBYTE pByte; //position ptr PBYTE pSource; //position ptr @@ -2638,81 +1734,54 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block PTMX_TAGTABLE_RECORD pTMXTargetTagTable = NULL; //ptr to target tag record PTMX_SOURCE_RECORD pTMXSourceRecord = NULL; //ptr to source record PTMX_TARGET_CLB pTMXTargetClb = NULL; //ptr to target control block - PSZ_W pString = NULL; //ptr to source string + std::wstring pString; //ptr to source string BOOL fOK = TRUE; //success indicator ULONG ulSourceLen = 0; //length of normalized source string USHORT usRc = NO_ERROR; //return code USHORT usFuzzy = 0, usWords = 0, usDiffs = 0; //fuzzy match value - BOOL fStringEqual; // strings are equal ??? + BOOL fStringEqual = false; // strings are equal ??? BOOL fNormStringEqual; // normalized strings are equal ??? BOOL fRespectCRLFStringEqual = 0L; + T5LOG( T5INFO) << "FuzzyTest for record " <szSourceLanguage, &sLangID ); - if (!pGetIn->ulSrcOemCP) + USHORT srcLangId = 0; + // we have to update the memory language table to keep the language group table up-to-date... + if ( pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, (USHORT)LANG_KEY, &srcLangId)) { - pGetIn->ulSrcOemCP = 1; + srcLangId = 0; // set default.. + } /* endif */ + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + + if(!pTmClb->areLangsMatching(srcLangId, pTMXSourceRecord->usLangId, pGetIn->szSourceLanguage)){ + //T5LOG( T5WARNING) <<":: source langs is different in record(" << pTMXSourceRecord->usLangId << ") and request (" << pGetIn->szSourceLanguage << ")"; + return SOURCE_LANG_DIFFERENT; } - T5LOG( T5INFO) << "FuzzyTest for record " <usSource; - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - if(pGetIn->fSourceLangIsPrefered == false){ - char recordSrcLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXSourceRecord->usLangId, - (USHORT)LANG_KEY, - recordSrcLang, NULL ); - if(strcasecmp(recordSrcLang, pGetIn->szSourceLanguage)){ - T5LOG( T5WARNING) <<":: source langs is different in record(" << recordSrcLang << ") and request (" << pGetIn->szSourceLanguage << ")"; - return SOURCE_LANG_DIFFERENT; - } - } - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - ulSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD)); - - //copy source string for fill matchtable - //T5LOG(T5TRANSACTION) << "pTmRecord: lRecordLen="<< pTmRecord->lRecordLen <<"; usFirstTargetRecord=" << pTmRecord->usFirstTargetRecord <<"; usSourceRecord=" << pTmRecord->usSourceRecord; - ulSourceLen = EQFCompress2Unicode( pString, pSource, ulSourceLen ); - //if(pSentence->pPropString) delete pSentence->pPropString; - pSentence->pPropString = std::make_unique(pString);//new StringTagVariants(pString);//std::make_unique(pString); - - if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ - auto str = EncodingHelper::convertToUTF8(pString); - T5LOG( T5INFO) << "::FuzzyTest:" << str << "" ; - } + //calculate length of source string + ulSourceLen = (RECLEN(pTMXSourceRecord) - + sizeof(TMX_SOURCE_RECORD)); - if (pGetIn->ulParm & GET_RESPECTCRLF ) // if-else nec for P018279 - { - fRespectCRLFStringEqual = (UtlCompIgnSpaceW( pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 )== 0L); + //copy source string for fill matchtable + ulSourceLen = EQFCompress2Unicode( pString, pSource, ulSourceLen ); + pSentence->pPropString = std::make_unique(pString.c_str()); - fStringEqual = (UtlCompIgnWhiteSpaceW( pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L); - if (fStringEqual && !fRespectCRLFStringEqual) - { // there is a LF difference! - fStringEqual = fRespectCRLFStringEqual; - } - } - else - { //compare source strings - fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L - || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; - } /* endif*/ + if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ + auto str = EncodingHelper::convertToUTF8(pString); + T5LOG( T5INFO) << "::FuzzyTest:" << str << "" ; + } + //if(!fSkipExactTest) + { + //compare source strings + fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L + || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " < usNumMatches ); T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " <NTMGetIDFromNameEx( pGetIn->szTagTable, - // NULL, - // (USHORT)TAGTABLE_KEY, &usTagId, - // NTMGETID_NOUPDATE_OPT, NULL )) - { - usTagId = 1; // set default... - } /* endif */ - //get id of target language in the get structure - - // we have to update the memory language table to keep the language group table up-to-date... -// if ( NTMGetIDFromNameEx( pTmClb, pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId, NTMGETID_NOUPDATE_OPT, NULL )) - if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) - { - usTargetId = 1; // set default.. - } /* endif */ - - - //strings are not equal, assume fuzzy - //position at the first target record - pByte = (PBYTE)pTmRecord; - pByte += pTmRecord->usFirstTargetRecord; - pTMXTargetRecord = (PTMX_TARGET_RECORD)pByte; - pStartTarget = (PBYTE)pTMXTargetRecord; + if ( (pGetIn->fForceFuzzySearch || !fStringEqual) && !usRc ) + { + USHORT usTgtNum = 0; // first target number + LONG lLeftTgtLen = 0; // target length record... + USHORT usTargetId = 0; // target language id + LONG lTempSrcLen; // temp. source length - //loop through target records - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + // we have to update the memory language table to keep the language group table up-to-date... + if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, (USHORT)LANG_KEY, &usTargetId)) + { + usTargetId = 0; // set default.. + } /* endif */ - T5LOG( T5INFO) << "FuzzyTest: Checking targets, lLeftTgtLen = "< 0) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && (RECLEN(pTMXTargetRecord) != 0) ) - { - BOOL fTestCLB = TRUE; - USHORT usModifiedTranslationFlag = 0; + //strings are not equal, assume fuzzy + //position at the first target record + pByte = (PBYTE)pTmRecord; + pByte += pTmRecord->usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)pByte; + pStartTarget = (PBYTE)pTMXTargetRecord; - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - usTgtNum ++; // we are dealing with next target + //loop through target records + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); + T5LOG( T5INFO) << "FuzzyTest: Checking targets, lLeftTgtLen = "< 0) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && (RECLEN(pTMXTargetRecord) != 0) ) + { + USHORT usModifiedTranslationFlag = 0; - lLeftTgtLen -= RECLEN(pTMXTargetRecord); + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + usTgtNum ++; // we are dealing with next target + lLeftTgtLen -= RECLEN(pTMXTargetRecord); - //check the target language - //position at target control block - pByte += pTMXTargetRecord->usClb; - pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + //check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + bool fTestCLB = (false == pstMatchTable->isLocationAlreadyAdded(*pusMatchesFound, lKeyNum, usTgtNum)); + if(fTestCLB){ //compare target language group IDs - if(pGetIn->fTargetLangIsPrefered){ - //T5LOG(T5TRANSACTION) << "pTMXTargetClb->usLangId=" << pTMXTargetClb->usLangId <<"; usTargetId=" << usTargetId; - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] ) - { - T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; - fTestCLB = FALSE; - } - } else { - char recordTrgLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - recordTrgLang, NULL ); - if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ - T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; - //return SOURCE_LANG_DIFFERENT; - fTestCLB = FALSE; - } - } - - - // check for comments when requested - if ( pGetIn->ulParm & GET_IGNORE_COMMENT ) - { - if ( NtmFindInAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, L"areLangsMatching(usTargetId, pTMXTargetClb->usLangId ,pGetIn->szTargetLanguage); // preprocessing for global memory processing usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; - if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) + } + if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) + { + USHORT usAddDataLen = 0; + //std::wstring addDataOut; + wchar_t addDataOut[MAX_SEGMENT_SIZE]{}; + usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, addDataOut); + wcsncpy(pSentence->szAddInfo.data(), addDataOut, MAX_SEGMENT_SIZE); + if ( usAddDataLen ) { - USHORT usAddDataLen = 0; - if ( !pSentence->pAddString ) UtlAlloc( (PVOID *) &pSentence->pAddString, 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pSentence->pAddString, MAX_SEGMENT_SIZE ); - if ( usAddDataLen ) + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString.data() ); + switch ( GobMemOpt ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString ); - switch ( GobMemOpt ) - { - case GM_SUBSTITUTE_OPT: usModifiedTranslationFlag = TRANSLFLAG_NORMAL; break; - case GM_HFLAG_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEM; break; - case GM_HFLAGSTAR_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; - case GM_EXCLUDE_OPT : fTestCLB = FALSE;; break; - } /* endswitch */ - } /* endif */ - } /* endif */ - - T5LOG( T5INFO) << "FuzzyTest: fTestCLB="<<( fTestCLB ? "Yes" : "NO") ; - - // compare target table IDs - if ( fTestCLB ) + case GM_SUBSTITUTE_OPT: usModifiedTranslationFlag = TRANSLFLAG_NORMAL; break; + case GM_HFLAG_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEM; break; + case GM_HFLAGSTAR_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; + case GM_EXCLUDE_OPT : fTestCLB = FALSE;; break; + } /* endswitch */ + } /* endif */ + } /* endif */ + + T5LOG( T5INFO) << "FuzzyTest: fTestCLB="<<( fTestCLB ? "Yes" : "NO") ; + + // compare target table IDs + if ( fTestCLB ) + { + //compare target tag table ids + //position at target tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + +// as the strings doe not contain the inline tagging, we have + // to add the inline tags first + // if the tag tables are different this is omitted as the + // inline tags are suppressed in the proposal in such a case { - //compare target tag table ids - //position at target tag table record pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } /* endif */ - //position at target tag table - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTargetTagTable = (PTMX_TAGTABLE_RECORD)pByte; + //TODO: remove punctuation here + BOOL fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, + pSentence->pStrings->getNormStrC(), + pSentence->pPropString->getNormStrC(), + sLangID, &usFuzzy, + &usWords, &usDiffs); - // GQ: only compare tag tables if segment and proposal have inline tagging - /* - if ( (RECLEN(pTMXTargetTagTable) > sizeof(TMX_TAGTABLE_RECORD)) || (RECLEN(pSentence->pTagRecord) > sizeof(TMX_TAGTABLE_RECORD)) ) - { - fTagTableEqual =true;//(pTMXTargetTagTable->usTagTableId == usTagId); - // For R012645 begin - if(!fTagTableEqual) - { - std::string familyNameIn = GetAndCacheFamilyName(pGetIn->szTagTable); + /**********************************************************/ + /* additional comparison if normalized strings are equal */ + /**********************************************************/ +// if ( usFuzzy >= 100 ) + if ( fNormStringEqual ) + { + //compare source tag table records + //position at source tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - if( !familyNameIn.empty() ) + fStringEqual = (UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), + pSentence->pStrings->getGenericTagStrC(), + 0 ) == 0 ); + if (!fStringEqual && (usFuzzy >= 100) ) + { + // compare with respect to protected/unprotected parts + // We can assume all Text tokens to be equal ( usFuzzy = 100), + // but we do not know about tag tokens + BOOL fTempStringEqual = FALSE; + fOK = NTMCompareBetweenTokens( + pSentence->pPropString->getGenericTagStrC(), + pSentence->pStrings->getGenericTagStrC(), + pGetIn->szTagTable, + sLangID, + &fTempStringEqual); + if (fOK) { - // get markup name firstly, also with cache - char szMarkup[MAX_FNAME]; - memset(szMarkup,0,sizeof(szMarkup)); - //pTmClb->NTMGetNameFromID( &(pTMXTargetTagTable->usTagTableId), (USHORT)TAGTABLE_KEY,szMarkup, NULL ); - strcpy(szMarkup, "OTMXUXLF"); - std::string familyNameTgt = GetAndCacheFamilyName(szMarkup); - if( !familyNameTgt.empty() && familyNameIn==familyNameTgt) - fTagTableEqual = TRUE; - } - } - // For R012645 end + fStringEqual = fTempStringEqual; + } } - else - //*/ - { - fTagTableEqual = TRUE; - } /* endif */ - - /**********************************************************/ - /* TAGS_EQUAL == base 98, TAGS_UNEQUAL == base 95 */ - /**********************************************************/ - usTagThreshold = (fTagTableEqual) ? TAGS_EQUAL : TAGS_UNEQUAL; - - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + /********************************************************/ + /* ensure that we are dealing with a fuzzy match */ + /********************************************************/ + if ( !fStringEqual ) { - BOOL fFuzzynessOK; - -// as the strings doe not contain the inline tagging, we have - // to add the inline tags first - // if the tag tables are different this is omitted as the - // inline tags are suppressed in the proposal in such a case - - - if ( fOK ) + if ( usFuzzy > 3 ) { - LONG lLenTmp = ulSourceLen; - //PTMX_TAGTABLE_RECORD pTMXSourceTagTable; //ptr to source tag info - //position at source tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; - if ( !fOK ) - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "fOk is false";} - } /* endif */ - } /* endif */ - - //TODO: remove punctuation here - fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, - pSentence->pStrings->getNormStrC(), - pSentence->pPropString->getNormStrC(), - sLangID, &usFuzzy, - pGetIn->ulSrcOemCP, &usWords, &usDiffs); - - /**********************************************************/ - /* additional comparison if normalized strings are equal */ - /**********************************************************/ -// if ( usFuzzy >= 100 ) - if ( fNormStringEqual ) + usFuzzy -= 3; + } + else { - //PTMX_TAGTABLE_RECORD pTMXSourceTagTable; //ptr to source tag info - BOOL fStringEqual = FALSE; - //compare source tag table records - //position at source tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //position at source tag table - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; - - //compare tag table records - //fStringEqual = memcmp( pTMXSourceTagTable, pSentence->pTagRecord, - // RECLEN(pTMXSourceTagTable) ) == 0; - //if ( !fStringEqual ) - //{ - /**********************************************************/ - /* if tagging record is unequal than we have some */ - /* (slight) differences */ - /* i.e. we will create a fully qualified string and try */ - /* another compare... */ - /**********************************************************/ - //allocate pString - - if ( fOK ) - { - //calculate length of source string - LONG lLenTmp = ulSourceLen; - - if ( !fOK ) - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "fOK is false";} - } - else - { - fStringEqual = FALSE; - - fStringEqual = (UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), - pSentence->pStrings->getGenericTagStrC(), - 0 ) == 0 ); - if (!fStringEqual && (usFuzzy >= 100) ) - { - // compare with respect to protected/unprotected parts - // We can assume all Text tokens to be equal ( usFuzzy = 100), - // but we do not know about tag tokens - BOOL fTempStringEqual = FALSE; - BOOL fOK = FALSE; - fOK = NTMCompareBetweenTokens( - pSentence->pPropString->getGenericTagStrC(), - pSentence->pStrings->getGenericTagStrC(), - pGetIn->szTagTable, - sLangID, - pGetIn->ulSrcOemCP, &fTempStringEqual); - if (fOK) - { - fStringEqual = fTempStringEqual; - } - } - } /* endif */ - } /* endif */ - //} /* endif */ - /********************************************************/ - /* ensure that we are dealing with a fuzzy match */ - /********************************************************/ - if ( !fStringEqual ) - { - if ( usFuzzy > 3 ) - { - usFuzzy -= 3; - } - else - { - usFuzzy = 0; - } /* endif */ - - usFuzzy = std::min( (USHORT)99, usFuzzy ); - } /* endif */ - + usFuzzy = 0; } /* endif */ - } - //fill get output structure - T5LOG( T5INFO) << "FuzzyTest: usFuzzy="<& pMatchEntry, PUSHORT pusMatchThreshold ) { - PTMX_INDEX_RECORD pIndexRecord = NULL; //pointer to index record + std::vector pIndexRecord; //pointer to index record BOOL fOK = TRUE; //success indicator PTMX_MATCHENTRY pTempMatch = NULL; //ptr to working match entry structure PTMX_MATCHENTRY pTempStart = NULL; //ptr to working match entry structure USHORT usRc = NO_ERROR; //return code PTMX_INDEX_ENTRY pIndexEntry = NULL; //pointer to index entry PULONG pulVotes; // pointer to votes - ULONG ulLen; // length paramter + //ULONG ulLen; // length paramter USHORT i, j; // index in for loop USHORT usMaxEntries; // nr of index entries in index record ULONG lMatchEntries = 0; // nr of entries in pTempMatch - ULONG ulKey; // index key + LONG lKey; // index key USHORT usMaxSentences = MAX_INDEX_LEN * 4; LONG lMatchListSize = (LONG)((usMaxSentences+1) * sizeof(TMX_MATCHENTRY)); - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lOtherTime) ); -#endif - - //allocate index record buffer if not done yet - if ( pTmClb->pvIndexRecord ) - { - pIndexRecord = (PTMX_INDEX_RECORD)pTmClb->pvIndexRecord; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvIndexRecord = pIndexRecord; - } /* endif */ - // allocate match list array if not done yet - if ( fOK ) - { - if ( pTmClb->pvTempMatchList ) - { - pTempMatch = (PTMX_MATCHENTRY)pTmClb->pvTempMatchList; - memset( pTempMatch , 0, lMatchListSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTempMatch), 0L, lMatchListSize, NOMSG ); - pTmClb->pvTempMatchList = pTempMatch; - } /* endif */ - } /* endif */ - - if ( fOK ) + if ( pTmClb->state->pvTempMatchList.empty() ) { - pTempStart = pTempMatch; + pTmClb->state->pvTempMatchList.resize(lMatchListSize/sizeof(pTmClb->state->pvTempMatchList[0])); } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchAllocTime) ); -#endif - if ( fOK ) + pTempStart = pTempMatch = pTmClb->state->pvTempMatchList.data(); + memset( pTempMatch , 0, lMatchListSize ); + { - pulVotes = pSentence->pulVotes; + pulVotes = pSentence->pulVotes.data(); lMatchEntries = 0; for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchOtherTime) ); -#endif - usRc = pTmClb->InBtree.EQFNTMGet( ulKey, (PCHAR)pIndexRecord, &ulLen ); - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchReadTime) ); -#endif + lKey = (*pulVotes) & START_KEY; + + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, pIndexRecord ); + if ( usRc == NO_ERROR ) { //calculate number of index entries in index record - T5LOG(T5INFO) <<"Processing index record "<usRecordLen; + ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - pIndexEntry = &pIndexRecord->stIndexEntry; + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; pTempMatch = pTempStart; //end criteria are all sentence ids in index key or only one - //sentence id left in pulSids + //sentence id left in ulSids for ( j = 0; (j < usMaxEntries) && (lMatchEntries < usMaxSentences); j++, pIndexEntry++ ) { - ULONG ulIndexKey = NTMKEY(*pIndexEntry); + LONG lIndexKey = NTMKEY(*pIndexEntry); // reset to begin of matching table pTempMatch = pTempStart; - //before adding sentence id check if already in pulsids as the + //before adding sentence id check if already in ulSids as the //respective tm record need only be checked once -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill1Time) ); -#endif // GQ: do a binary search for the key ( bsearch() cannot be used as we need the insert position) and // a linear search is far too slow @@ -3210,7 +2086,7 @@ USHORT FillMatchEntry while (high - low > 1) { probe = (high + low) / 2; - if (pTempStart[probe].ulKey > ulIndexKey) + if (pTempStart[probe].lKey > lIndexKey) high = probe; else low = probe; @@ -3223,27 +2099,18 @@ USHORT FillMatchEntry else { pTempMatch = pTempStart + low; - if ( pTempMatch->ulKey < ulIndexKey ) pTempMatch++; + if ( pTempMatch->lKey < lIndexKey ) pTempMatch++; } /* endif */ } - // below is the old search code - //while ( (ulIndexKey > pTempMatch->ulKey) && pTempMatch->ulKey ) - //{ - // pTempMatch++; - //} /* endwhile */ - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill2Time) ); -#endif - if ( ulIndexKey == pTempMatch->ulKey ) + if ( lIndexKey == pTempMatch->lKey ) { pTempMatch->cCount++; } - else if ( !pTempMatch->ulKey ) + else if ( !pTempMatch->lKey ) { //add new match entry - pTempMatch->ulKey = NTMKEY(*pIndexEntry); + pTempMatch->lKey = NTMKEY(*pIndexEntry); pTempMatch->usMaxVotes = NTMVOTES(*pIndexEntry); pTempMatch->cCount = 1; pTempMatch->usMatchVotes = pSentence->usActVote; @@ -3256,7 +2123,7 @@ USHORT FillMatchEntry { /*Fix*/ memmove( pTempMatch+1, pTempMatch, (lMatchEntries - (pTempMatch - pTempStart) ) * sizeof(TMX_MATCHENTRY) ); } /* enidf */ - pTempMatch->ulKey = ulIndexKey; + pTempMatch->lKey = lIndexKey; pTempMatch->usMaxVotes = NTMVOTES(*pIndexEntry); pTempMatch->cCount = 1; lMatchEntries++; @@ -3266,22 +2133,17 @@ USHORT FillMatchEntry /**************************************************************/ /* make clean up if we are too near at the limit .. */ /**************************************************************/ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill3Time) ); -#endif if (lMatchEntries >= 2 * MAX_INDEX_LEN ) { - PTMX_MATCHENTRY pTemp1Start; + std::vector pTemp1Start; + pTemp1Start.resize(usMaxSentences+1); T5LOG( T5INFO) <<"Cleaning up..." ; - - fOK = UtlAlloc( (PVOID *) &(pTemp1Start), 0L, - (LONG)((usMaxSentences+1)* sizeof(TMX_MATCHENTRY)), NOMSG ); if ( fOK ) { USHORT k = 0; PTMX_MATCHENTRY pTemp1Match; - pTemp1Match = pTemp1Start; + pTemp1Match = pTemp1Start.data(); pTempMatch = pTempStart; for (k=0; klFillMatchFill4Time) ); -#endif - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFillTime) ); -#endif - } /* endif */ } /* endfor */ @@ -3325,13 +2178,9 @@ USHORT FillMatchEntry //sort the sentence keys by frequency discard those below the given //threshold - CleanupTempMatch( pTempMatch, &pMatchEntry, &pSentence->usActVote, pusMatchThreshold ); + CleanupTempMatch( pTempMatch, pMatchEntry, &pSentence->usActVote, pusMatchThreshold ); } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchCleanupTime) ); -#endif - //if the entry does not exist in the index file it is not to be regarded as //an error if ( usRc == BTREE_NOT_FOUND ) @@ -3339,14 +2188,6 @@ USHORT FillMatchEntry LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchAllocTime) ); -#endif - if ( usRc ) - { - ERREVENT2( FILLMATCHENTRY_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - T5LOG( T5INFO) << "FillMatchEntry: lMatchEntries=" <getFamilyName(); - else - Name2FamilyMap[pszMarkupName] = ""; - } - return Name2FamilyMap[pszMarkupName]; -} - diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index 1ae4642d..82495adf 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -258,54 +258,6 @@ QDAMCheckDict //| return Rc | // ----------------------------------------------------------------------------+ -SHORT BTREE::EQFNTMOpen -( - USHORT usOpenFlags // Read Only or Read/Write -) -{ - //PBTREE pBTIda; // pointer to BTRee structure - SHORT sRc = 0; // return code - - DEBUGEVENT( EQFNTMOPEN_LOC, FUNCENTRY_EVENT, 0 ); - - { - /*****************************************************************/ - /* check if same dictionary is already open else return index */ - /* of next free slot... */ - /*****************************************************************/ - sRc = QDAMCheckDict( (PSZ)fb.fileName .c_str()); - if ( !sRc ) - { - /***************************************************************/ - /* check if dictionary is locked */ - /***************************************************************/ - if ( ! usDictNum ) - { - SHORT RetryCount; // retry counter for in-use condition - RetryCount = MAX_RETRY_COUNT; - do - { - sRc = QDAMDictOpenLocal( 20, usOpenFlags ); - - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - } - else - { - sRc = ( usOpenFlags & ASD_LOCKED ) ? BTREE_DICT_LOCKED : sRc; - } /* endif */ - } /* endif */ - } /* endif */ - - if ( sRc != NO_ERROR ) - { - ERREVENT( EQFNTMOPEN_LOC, ERROR_EVENT, sRc ); - } /* endif */ - - DEBUGEVENT( EQFNTMOPEN_LOC, FUNCEXIT_EVENT, 0 ); - - return ( sRc ); -} //+----------------------------------------------------------------------------+ //|Internal function | @@ -400,17 +352,17 @@ SHORT BTREE::EQFNTMOpen SHORT BTREE::EQFNTMSign ( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data + TMX_SIGN& sign // pointer to user data ) { SHORT sRc; // function return code SHORT RetryCount; // retry counter for in-use condition - + long signSize = sizeof(sign) ; RetryCount = MAX_RETRY_COUNT; do { - sRc = QDAMDictSignLocal( pUserData, pusLen ); + USHORT signSize = sizeof(sign) ; + sRc = QDAMDictSignLocal((PCHAR)&sign, &signSize); if ( sRc == BTREE_IN_USE ) { RetryCount--; @@ -437,16 +389,7 @@ ULONG BTREE::GetNumOfSavedRecords()const{ int BTREE::initLookupTable(){ - LookupTable_V3.reserve(MIN_NUMBER_OF_LOOKUP_ENTRIES); - return 0; -} - -int BTREE::checkLookupTableAndRealocate(int number){ - if ( number >= LookupTable_V3.size() ) - { - T5LOG(T5DEBUG) << "resizing LookupTable number = " << number; - LookupTable_V3.resize(number+10); - } + //LookupTable.reserve(MIN_NUMBER_OF_LOOKUP_ENTRIES); return 0; } @@ -459,12 +402,12 @@ void BTREE::freeLookupTable(){ //+----------------------------------------------------------------------------+ //|Function name: EQFNTMInsert | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMInsert( pBTIda, &ulKey, pData, usLen ); | +//|Function call: sRc = EQFNTMInsert( pBTIda, &lKey, pData, usLen ); | //+----------------------------------------------------------------------------+ //|Description: insert a new key (ULONG) with data | //+----------------------------------------------------------------------------+ //|Parameters: PBTREE pBTIda, pointer to binary tree struct | -//| PULONG pulKey, pointer to key | +//| PULONG plKey, pointer to key | //| PBYTE pData, pointer to user data | //| USHORT usLen length of user data | //+----------------------------------------------------------------------------+ @@ -489,9 +432,8 @@ void BTREE::freeLookupTable(){ SHORT BTREE::EQFNTMInsert ( - PULONG pulKey, // pointer to key - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + LONG& lKey, // pointer to key + BytesRef pData // pointer to user data ) { SHORT sRc = 0; // return code @@ -499,67 +441,46 @@ BTREE::EQFNTMInsert /********************************************************************/ /* if user wants that we find an appropriate key, we have to do so..*/ /********************************************************************/ - if ( !sRc ) + if ( lKey == NTMREQUESTNEWKEY ) { - if ( *pulKey == NTMREQUESTNEWKEY ) + /******************************************************************/ + /* find next free key and anchor new value in file ... */ + /******************************************************************/ + ULONG Key = lKey = (chCollate.lNextKey)++; + if ( Key > 0xFFFFFF ) { - /******************************************************************/ - /* find next free key and anchor new value in file ... */ - /******************************************************************/ - ULONG ulKey; - //ulKey = *pulKey = ++(pvi->ulNextKey); - ulKey = *pulKey = (chCollate.ulNextKey)++; - if ( ulKey > 0xFFFFFF ) - { - sRc = BTREE_NUMBER_RANGE; - } - else - { - /**************************************************************/ - /* force update of header (only from time to time to avoid */ - /* too much performance degration)... */ - /**************************************************************/ - if ( (ulKey & 0x020) ) - { - sRc = QDAMWriteHeader(); - } /* endif */ - } /* endif */ + (chCollate.lNextKey)--; + sRc = BTREE_NUMBER_RANGE; } else { - /****************************************************************/ - /* check if key is in valid range ... */ - /****************************************************************/ - if ( *pulKey > NTMSTARTKEY( this ) ) + /**************************************************************/ + /* force update of header (only from time to time to avoid */ + /* too much performance degration)... */ + /**************************************************************/ + //if ( (Key & 0x020) ) { - sRc = BTREE_NUMBER_RANGE; + sRc = QDAMWriteHeader(); } /* endif */ } /* endif */ + } + else + { + /****************************************************************/ + /* check if key is in valid range ... */ + /****************************************************************/ + if ( lKey > chCollate.lStartKey ) + { + sRc = BTREE_NUMBER_RANGE; + } /* endif */ } /* endif */ /********************************************************************/ /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - if ( !sRc ) - { - SHORT RetryCount; // retry counter for in-use condition - RetryCount = MAX_RETRY_COUNT; - do - { - sRc = QDAMDictInsertLocal( (PWCHAR)pulKey, pData, ulLen ); - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - } /* endif */ - - if ( sRc ) - { - ERREVENT( EQFNTMINSERT_LOC, INTFUNCFAILED_EVENT, sRc ); - } /* endif */ + if(!sRc){ + sRc = QDAMDictInsertLocal( lKey, pData); + } return sRc; } /* end of function EQFNTMInsert */ @@ -615,7 +536,7 @@ BTREE::EQFNTMInsert //------------------------------------------------------------------------------ VOID BTREE::QDAMUpdateList_V3 ( - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { USHORT i; // index @@ -725,7 +646,7 @@ VOID BTREE::QDAMUpdateList_V3 VOID BTREE::QDAMFreeFromList_V3 ( PRECPARAM pRecTemp, - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { USHORT i; // index @@ -795,7 +716,7 @@ VOID BTREE::QDAMFreeFromList_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMFreeRecord_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, RECTYPE recType // data or key record ) { @@ -876,7 +797,7 @@ SHORT BTREE::QDAMFreeRecord_V3 //------------------------------------------------------------------------------ ULONG BTREE::QDAMGetrecDataLen_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ) { @@ -926,11 +847,10 @@ ULONG BTREE::QDAMGetrecDataLen_V3 //------------------------------------------------------------------------------ VOID QDAMCopyDataTo_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT i, - PBTREEBUFFER_V3 pNew, - SHORT j, - USHORT usVersion // version of BTREE + std::shared_ptr& pNew, + SHORT j ) { //static int nCalled = 0; @@ -1057,15 +977,15 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 SHORT sRc = 0; PUSHORT pusOffset; // offset pointer SHORT i; // index - PBTREEBUFFER_V3 pRecord; - PBTREEBUFFER_V3 pNew; // temporary record + std::shared_ptr pRecord; + std::shared_ptr pNew; // temporary record PBTREEHEADER pHeader; // pointer to header ULONG ulDataLen; // length of data USHORT usNum; // number of record // read in record recParam.usNum = recParam.usNum + usFirstDataBuffer; // adjust for offset - sRc = QDAMReadRecord_V3( recParam.usNum, &pRecord, FALSE ); + sRc = QDAMReadRecord_V3( recParam.usNum, pRecord, FALSE ); if ( ! sRc ) { BTREELOCKRECORD( pRecord ); // lock eecord. @@ -1086,7 +1006,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 usNum = NEXT( pRecord ); while ( !sRc && usNum ) { - sRc = QDAMReadRecord_V3( usNum, &pNew, FALSE ); + sRc = QDAMReadRecord_V3( usNum, pNew, FALSE ); if ( !sRc ) { if ( TYPE(pNew) != DATA_NEXTNODE ) @@ -1133,7 +1053,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 if ( !NEXT(pRecord) && (pRecord->contents.header.usWasteSize > MAXWASTESIZE) ) { // get temp record - pNew = &BTreeTempBuffer_V3; + pNew = BTreeTempBuffer_V3; pHeader = &pNew->contents.header; pHeader->usOccupied = 0; pHeader->usFilled = sizeof(BTREEHEADER ); @@ -1145,7 +1065,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 T5LOG(T5DEVELOP) << "while(isUsed = " << pRecord->sUsed; while ( i < (SHORT) OCCUPIED( pRecord )) { - QDAMCopyDataTo_V3( pRecord, i, pNew, i, usVersion ); + QDAMCopyDataTo_V3( pRecord, i, pNew, i ); i++; } /* endwhile */ // copy record data back to avoid misplacement in file @@ -1200,10 +1120,9 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 //------------------------------------------------------------------------------ VOID QDAMSetrecData_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid, // key number - RECPARAM recData, // data pointer - USHORT usVersion // version of database + RECPARAM recData // data pointer ) { PCHAR pData = NULL; @@ -1274,7 +1193,7 @@ VOID QDAMSetrecData_V3 //------------------------------------------------------------------------------ RECPARAM QDAMGetrecKey_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ) { @@ -1337,11 +1256,10 @@ RECPARAM QDAMGetrecKey_V3 // return sRc // //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictUpdateLocal +SHORT BTREE::EQFNTMUpdate ( - PCHAR_W pKey, // key string - PBYTE pUserData, // user data - ULONG ulLen, // user data length + LONG lKey, // key string + BytesRef pData, bool fSuppressTmdSizeCheck ) { @@ -1363,25 +1281,24 @@ SHORT BTREE::QDAMDictUpdateLocal /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( QDAMDictLockStatus( pKey ) ) + if ( QDAMDictLockStatus( lKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ { - PBTREEBUFFER_V3 pRecord = NULL; // pointer to record + std::shared_ptr pRecord; // pointer to record if ( !sRc ) { - if ( ulLen == 0 ) + //if ( ulLen == 0 ) + if(pData.empty()) { sRc = BTREE_DATA_RANGE; } else - { - //memcpy( chHeadTerm, pKey, sizeof(ULONG)); // save data - + { QDAMDictUpdStatus(); - sRc = QDAMFindRecord_V3( pKey, &pRecord ); + sRc = QDAMFindRecord_V3( lKey, pRecord ); } /* endif */ } /* endif */ @@ -1389,7 +1306,7 @@ SHORT BTREE::QDAMDictUpdateLocal if ( !sRc ) { // find the key - sRc = QDAMLocateKey_V3( pRecord, pKey, &i, FEXACT, &sNearKey ) ; + sRc = QDAMLocateKey_V3( pRecord, lKey, &i, FEXACT, &sNearKey ) ; if ( !sRc ) { if ( i != -1) @@ -1400,18 +1317,18 @@ SHORT BTREE::QDAMDictUpdateLocal usCurrentRecord = RECORDNUM( pRecord ); // get data value associated with key recOldKey = QDAMGetrecKey_V3( pRecord,i ); - recOldData = QDAMGetrecData_V3( pRecord, i, usVersion ); + recOldData = QDAMGetrecData_V3( pRecord, i ); // set new data value if ( recOldKey.usNum && recOldData.usNum ) { - sRc = QDAMAddToBuffer_V3( pUserData, ulLen, &recData, fSuppressTmdSizeCheck ); + sRc = QDAMAddToBuffer_V3( pData, &recData, fSuppressTmdSizeCheck ); if ( !sRc ) { - if( (ulLen > TMX_REC_SIZE) && (T5Logger::GetInstance()->CheckLogLevel(T5DEBUG))){ - T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << ulLen; + if( (pData.size() > TMX_REC_SIZE) && (T5Logger::GetInstance()->CheckLogLevel(T5DEBUG))){ + T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } - recData.ulLen = ulLen; - QDAMSetrecData_V3( pRecord, i, recData, usVersion ); + recData.ulLen = pData.size(); + QDAMSetrecData_V3( pRecord, i, recData ); sRc = QDAMWriteRecord_V3( pRecord ); } /* endif */ /****************************************************************/ @@ -1466,70 +1383,52 @@ SHORT BTREE::QDAMDictUpdateLocal int BTREE::resetLookupTable(){ int rc = 0; - for (int i=0; i < LookupTable_V3.size(); i++ ) - { - if ( LookupTable_V3[i].pBuffer ) - { - UtlAlloc( (PVOID *)&(LookupTable_V3[i].pBuffer), 0L, 0L, NOMSG ); - (usNumberOfAllocatedBuffers)--; - } /* endif */ - } /* endfor */ - LookupTable_V3.resize(0); + LookupTable.clear(); usNumberOfAllocatedBuffers = 0; return rc; } -int BTREE::allocateNewLookupTableBuffer(int number, PLOOKUPENTRY_V3& pLEntry, PBTREEBUFFER_V3& pBuffer){ +int BTREE::allocateNewLookupTableBuffer(int number, std::shared_ptr& pLEntry){ /********************************************************************/ /* Allocate space for a new buffer and let pBuffer point to it */ /********************************************************************/ int sRc = 0; - //if ( !LookupTable_V3 || ( number >= usNumberOfLookupEntries )) - if(number >= LookupTable_V3.size()) + if(LookupTable.count(number)) { LOG_AND_SET_RC(sRc,T5WARNING, BTREE_LOOKUPTABLE_CORRUPTED); } else { - //pLEntry = LookupTable_V3 + number; - pLEntry = &LookupTable_V3[number]; - - /* Safety-Check: is the lookup table entry (ptr. to buffer) for the - record to read NULL ? */ - if ( pLEntry->pBuffer ) - { - /* ptr. isn't NULL: this should never occur */ - LOG_AND_SET_RC(sRc, T5WARNING,BTREE_LOOKUPTABLE_CORRUPTED); - } - else - { - /* Allocate memory for a buffer */ - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, (LONG) BTREE_BUFFER_V3 , NOMSG ); - if ( pLEntry->pBuffer ) - { - (usNumberOfAllocatedBuffers)++; - pBuffer = pLEntry->pBuffer; - } - else - { - sRc = BTREE_NO_ROOM; - } /* endif */ - } /* endif */ + LookupTable[number] = std::make_shared(); + pLEntry = LookupTable[number]; + (usNumberOfAllocatedBuffers)++; } /* endif */ return sRc; } + +int BTREE::deallocateLookupTableBuffer(int number){ + if(LookupTable.count(number)) + { + usNumberOfAllocatedBuffers--; + LookupTable.erase(number); + return 0; + } + + return 1; +} + //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ //|Function name: EQFNTMUpdate | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMUpdate( pBTIda, ulKey, pData, usLen ); | +//|Function call: sRc = EQFNTMUpdate( pBTIda, lKey, pData, usLen ); | //+----------------------------------------------------------------------------+ //|Description: update the data of an already inserted key | //+----------------------------------------------------------------------------+ //|Parameters: PBTREE pBTIda, pointer to binary tree struct | -//| ULONG ulKey, key value | +//| ULONG lKey, key value | //| PBYTE pData, pointer to user data | //| USHORT usLen length of user data | //+----------------------------------------------------------------------------+ @@ -1544,32 +1443,6 @@ int BTREE::allocateNewLookupTableBuffer(int number, PLOOKUPENTRY_V3& pLEntry, PB //|Function flow: call QDAMDictUpdateLocal | //| return success indicator.. | // ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMUpdate -( - ULONG ulKey, // key value - PBYTE pData, // pointer to user data - ULONG ulLen, // length of user data - bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed -) -{ - SHORT sRc = 0; // success indicator - - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - do - { - sRc = QDAMDictUpdateLocal( (PSZ_W) &ulKey, pData, ulLen, fSuppressTmdSizeCheck); - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - - - return sRc; -} /* end of function EQFNTMUpdate */ //------------------------------------------------------------------------------ @@ -1623,23 +1496,20 @@ BTREE::EQFNTMUpdate // //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictExactLocal +SHORT BTREE::EQFNTMGet ( - PWCHAR pKey, // key to be searched for - PBYTE pchBuffer, // space for user data - PULONG pulLength, // in/out length of returned user data - USHORT usSearchSubType // special hyphenation lookup flag + LONG& lKey, // key to be searched for + BytesRef pchBuffer // space for user data ) { SHORT i; SHORT sNearKey; // nearest key position SHORT sRc = 0; // return code RECPARAM recData; // point to data structure + + pchBuffer.clear(); { - PBTREEBUFFER_V3 pRecord = NULL; - SHORT sRetries = MAX_RETRY_COUNT; - do - { + std::shared_ptr pRecord; /*******************************************************************/ /* For shared databases: discard all in-memory pages if database */ /* has been changed since last access */ @@ -1647,12 +1517,12 @@ SHORT BTREE::QDAMDictExactLocal if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindRecord_V3( pKey, &pRecord ); + sRc = QDAMFindRecord_V3( lKey, pRecord ); } /* endif */ if ( !sRc ) { - sRc = QDAMLocateKey_V3(pRecord, pKey, &i, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3(pRecord, lKey, &i, FEXACT, &sNearKey); if ( !sRc ) { if ( i != -1 ) @@ -1661,25 +1531,9 @@ SHORT BTREE::QDAMDictExactLocal sCurrentIndex = i; usCurrentRecord = RECORDNUM( pRecord ); - //memcpy( chHeadTerm, pKey, sizeof(ULONG) ); // save data - - recData = QDAMGetrecData_V3( pRecord, i, usVersion ); - if ( *pulLength == 0 || ! pchBuffer ) - { - *pulLength = recData.ulLen; - } - else if ( *pulLength < recData.ulLen ) - { - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG(T5ERROR) << "::BTREE_BUFFER_SMALL, pulLength = " << *pulLength << "; recData.ulLen = " << recData.ulLen; - } - *pulLength = recData.ulLen; - sRc = BTREE_BUFFER_SMALL; - } - else - { - sRc = QDAMGetszData_V3( this, recData, pchBuffer, pulLength, DATA_NODE ); - } /* endif */ + recData = QDAMGetrecData_V3( pRecord, i ); + sRc = QDAMGetszData_V3( recData, pchBuffer ); + } else { @@ -1687,121 +1541,16 @@ SHORT BTREE::QDAMDictExactLocal // set new current position sCurrentIndex = sNearKey; usCurrentRecord = RECORDNUM( pRecord ); - *pulLength = 0; // init returned length } /* endif */ } /* endif */ - } /* endif */ - - if ( (sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED) ) - { - UtlWait( MAX_WAIT_TIME ); - sRetries--; - } /* endif */ - } - while( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && (sRetries > 0)); + } /* endif */ } /* endif */ - - if ( sRc && (sRc != BTREE_NOT_FOUND)) - { - ERREVENT2( QDAMDICTEXACTLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - return ( sRc ); } -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: EQFNTMGet | -//+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGet( pBTIda, ulKey, chData, &usLen ); | -//+----------------------------------------------------------------------------+ -//|Description: get the data string for the passed key | -//+----------------------------------------------------------------------------+ -//|Parameters: PBTREE pBTIda, pointer to btree struct | -//| ULONG ulKey, key to be searched for | -//| PCHAR pchBuffer, space for user data | -//| PUSHORT pusLength in/out length of returned user data | -//+----------------------------------------------------------------------------+ -//|Returncode type: SHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: same as for QDAMDictExactLocal... | -//+----------------------------------------------------------------------------+ -//|Function flow: call QDAMDictExactLocal to retrieve the data | -// ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMGet -( - ULONG ulKey, // key to be searched for - PCHAR pchBuffer, // space for user data - PULONG pulLength // in/out length of returned user data -) -{ - SHORT sRc; // return code - DEBUGEVENT( EQFNTMGET_LOC, FUNCENTRY_EVENT, 0 ); - - CHECKPBTREE(this, sRc ); - if ( !sRc ) - { - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - ULONG ulLength; - do - { - ulLength = *pulLength; - - sRc = QDAMDictExactLocal( (PWCHAR) &ulKey, (PBYTE)pchBuffer, &ulLength, FEXACT ); - - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - *pulLength = ulLength; - } /* endif */ - - if ( sRc && (sRc != BTREE_NOT_FOUND) ) - { - ERREVENT( EQFNTMGET_LOC, INTFUNCFAILED_EVENT, sRc ); - } /* endif */ - - return sRc; -} /* end of function EQFNTMGet */ -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: EQFNTMGetMaxNumber | -//+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGetNextNumber( pBTIda, &ulKey, &ulNextFree );| -//+----------------------------------------------------------------------------+ -//|Description: get the start key and the next free key ... | -//+----------------------------------------------------------------------------+ -//|Parameters: PBTREE pBTIda, pointer to btree struct | -//| PULONG pulStartKey first key | -//| PULONG pulNextKey next key to be assigned | -//+----------------------------------------------------------------------------+ -//|Returncode type: SHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 0 always | -//+----------------------------------------------------------------------------+ -//|Function flow: access data from internal structure | -// ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMGetNextNumber -( - PULONG pulStartKey, // return start key number - PULONG pulNextKey // return next key data -) -{ - *pulStartKey = chCollate.ulStartKey; - *pulNextKey = chCollate.ulNextKey; - - return 0; -} /* end of function EQFNTMGetNextNumber */ //+----------------------------------------------------------------------------+ //|External function | diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 012a3a85..d477dc93 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -24,17 +24,6 @@ #include "requestdata.h" -int EqfMemory::getErrorCode()const{ - return lastErrorCode; -} - -int EqfMemory::setErrorCode(int rc){ - return lastErrorCode = rc; -} - -void EqfMemory::reserErrorCode(){ - lastErrorCode = 0; -} std::string StatusToString(int eStatus) { @@ -156,16 +145,10 @@ std::string EqfMemory::getActiveRequestString()const //| return open out structure | // ----------------------------------------------------------------------------+ DECLARE_bool(allowLoadingMultipleTmsSimultaneously); -USHORT EqfMemory::Load(){ - USHORT usRC = LoadMem(); - // create memory object if create function completed successfully - if ( (usRC != 0) && (usRC != BTREE_CORRUPTED) /*&& (usAccessMode == FOR_ORGANIZE)*/){ - T5LOG(T5ERROR) << "EqfMemoryPlugin::openMemory:: TmOpen fails, fName = "<< szName<< "; error = "<< this->strLastError<<"; iLastError = "<< - this->iLastError << "; rc = " << usRC; - } - return usRC; -} + + +DECLARE_int32(triplesthreshold); DECLARE_bool(suppressTmVersionCheck); DECLARE_int32(t5globversion_min_supported); DECLARE_int32(t5majversion_min_supported); @@ -174,8 +157,7 @@ DECLARE_int32(t5globversion_max_supported); DECLARE_int32(t5majversion_max_supported); DECLARE_int32(t5minversion_max_supported); - -USHORT EqfMemory::LoadMem() +USHORT EqfMemory::Load() { //TimedMutexGuard l(tmMutex); if(isLoaded()) @@ -202,39 +184,33 @@ USHORT EqfMemory::LoadMem() BOOL fOK; //success indicator USHORT usRc = NO_ERROR; //return value USHORT usRc1 = NO_ERROR; //return value - ULONG ulLen; //length indicator - //if(!usRc) - { - //TmBtree.fb.fileName = pTmOpenIn->stTmOpen.szDataName; - //InBtree.fb.fileName = pTmOpenIn->stTmOpen.szIndexName; - //call open function for data file - - usRc1 = TmBtree.EQFNTMOpen((USHORT)(usAccessMode | ASD_FORCE_WRITE) ); + { + usRc1 = state->TmBtree.EQFNTMOpen((state->usAccessMode) ); if ( (usRc1 == NO_ERROR) || (usRc1 == BTREE_CORRUPTED) ) { //get signature record and add to control block USHORT usLen = sizeof( TMX_SIGN ); - usRc = TmBtree.EQFNTMSign((PCHAR) &(stTmSign), &usLen ); + usRc = sign(); if ( usRc == NO_ERROR ) { std::stringstream msg; - if ( stTmSign.bGlobVersion > FLAGS_t5globversion_max_supported - || stTmSign.bMajorVersion > FLAGS_t5majversion_max_supported - //|| (stTmSign.bMajorVersion == T5MAJVERSION_MAX_SUPPORTED && stTmSign.bMinorVersion > T5MINVERSION_MAX_SUPPORTED ) + if ( state->stTmSign.bGlobVersion > FLAGS_t5globversion_max_supported + || state->stTmSign.bMajorVersion > FLAGS_t5majversion_max_supported + //|| (state->stTmSign.bMajorVersion == T5MAJVERSION_MAX_SUPPORTED && state->stTmSign.bMinorVersion > T5MINVERSION_MAX_SUPPORTED ) ) { - msg << "TM was created in newer vertions of t5memory, v" << toStr(stTmSign.bGlobVersion) << "." << toStr(stTmSign.bMajorVersion) << "." << toStr(stTmSign.bMinorVersion); + msg << "TM was created in newer vertions of t5memory, v" << toStr(state->stTmSign.bGlobVersion) << "." << toStr(state->stTmSign.bMajorVersion) << "." << toStr(state->stTmSign.bMinorVersion); usRc = VERSION_MISMATCH; } - else if (stTmSign.bGlobVersion < FLAGS_t5globversion_min_supported - || stTmSign.bMajorVersion < FLAGS_t5majversion_min_supported - || (stTmSign.bMajorVersion == FLAGS_t5majversion_min_supported && stTmSign.bMinorVersion < FLAGS_t5minversion_min_supported ) ) + else if (state->stTmSign.bGlobVersion < FLAGS_t5globversion_min_supported + || state->stTmSign.bMajorVersion < FLAGS_t5majversion_min_supported + || (state->stTmSign.bMajorVersion == FLAGS_t5majversion_min_supported && state->stTmSign.bMinorVersion < FLAGS_t5minversion_min_supported ) ) { - msg << "TM was created in older vertions of t5memory, v"<< toStr(stTmSign.bGlobVersion)<<"."<< toStr(stTmSign.bMajorVersion)<<"."<< toStr(stTmSign.bMinorVersion); + msg << "TM was created in older vertions of t5memory, v"<< toStr(state->stTmSign.bGlobVersion)<<"."<< toStr(state->stTmSign.bMajorVersion)<<"."<< toStr(state->stTmSign.bMinorVersion); usRc = VERSION_MISMATCH; } /* endif */ @@ -257,25 +233,27 @@ USHORT EqfMemory::LoadMem() usRc = NO_ERROR; } /* endif */ - if ( (usRc == NO_ERROR) || (usRc == VERSION_MISMATCH) ) + if(usRc == NO_ERROR){ + usRc = usRc1; + } + + if ( (usRc == NO_ERROR) || (usRc == BTREE_CORRUPTED) ) { - if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) { - ulLen = MAX_COMPACT_SIZE-1; //get compact area and add to control block - USHORT usTempRc = TmBtree.EQFNTMGet( COMPACT_KEY, (PCHAR)bCompact, &ulLen ); + LONG lKey = COMPACT_KEY; + USHORT usTempRc = state->TmBtree.EQFNTMGet( lKey, state->bCompact ); // in organize mode allow continue if compact area is corrupted - if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ) + if ( usTempRc != NO_ERROR)//&& (usTempRc != VERSION_MISMATCH) ) { - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED) << "if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ); usTempRc= " <Authors ); // in organize mode allow continue if author name area is corrupted if ( usTempRc == BTREE_READ_ERROR ){ - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} + {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} } if ( usTempRc != NO_ERROR ) { @@ -346,14 +317,13 @@ USHORT EqfMemory::LoadMem() //get tag tables and add to control block if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) + (usRc == BTREE_CORRUPTED) ) { USHORT usTempRc = NTMLoadNameTable( TAGTABLE_KEY, - &TagTables, &ulLen ); + &state->TagTables ); if ( usTempRc == BTREE_READ_ERROR ){ - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} + {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} } if ( usTempRc != NO_ERROR ) { @@ -363,20 +333,13 @@ USHORT EqfMemory::LoadMem() //get long document name table if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) + (usRc == BTREE_CORRUPTED)) { - // error in memory allocations will force end of open! - USHORT usTempRc = NTMCreateLongNameTable(); - if ( usTempRc == BTREE_READ_ERROR ){ - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<< "if ( usTempRc == BTREE_READ_ERROR )";} - } // now read any long name table from database, if there is // none (=older TM before TOP97) write our empty one to the // Translation Memory - if ( usTempRc == NO_ERROR ) { - usTempRc = NTMReadLongNameTable(); + USHORT usTempRc = NTMReadLongNameTable(); switch ( usTempRc) { @@ -401,35 +364,25 @@ USHORT EqfMemory::LoadMem() {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<<"usTempRc = " << usTempRc;} break; } /* endswitch */ - } /* endif */ - - // just for debugging purposes: check if short and long name table have the same numbe rof entries - if ( usTempRc == NO_ERROR ) - { - if ( pLongNames->ulEntries != FileNames.ulMaxEntries ) + + if ( usTempRc != NO_ERROR ) { - // this is strange... - int iSetBreakPointHere = 1; - } + usRc = usTempRc; + } /* endif */ } /* endif */ - if ( usTempRc != NO_ERROR ) - { - usRc = usTempRc; - } /* endif */ } /* endif */ } /* endif */ //add threshold to control block - usThreshold = 33;//pTmOpenIn->stTmOpen.usThreshold; + //usThreshold = FLAGS_triplesthreshold;//pTmOpenIn->stTmOpen.usThreshold; if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) + (usRc == BTREE_CORRUPTED) ) { //call open function for index file - USHORT usIndexRc = InBtree.EQFNTMOpen(usAccessMode ); + USHORT usIndexRc = state->InBtree.EQFNTMOpen(state->usAccessMode ); if ( usIndexRc != NO_ERROR && !isOpenedOnlyForReorganize()) { usRc = usIndexRc; @@ -457,16 +410,21 @@ USHORT EqfMemory::LoadMem() } /* endif */ - if ( (usRc != NO_ERROR) && - (usRc != BTREE_CORRUPTED) && - (usRc != VERSION_MISMATCH) ) + /********************************************************************/ + /* Ensure that all required data areas have been loaded and the */ + /* database files have been opened */ + /********************************************************************/ + if((usRc == BTREE_CORRUPTED) && isOpenedOnlyForReorganize()){ + state->fCorrupted = true;//could be opened only for exportTMX or reorganize, or mass deletion(also reorganize) + usRc = NO_ERROR; + } + + if (usRc != NO_ERROR) { - TmBtree.QDAMDictClose(); - InBtree.QDAMDictClose(); + state->TmBtree.QDAMDictClose(); + state->InBtree.QDAMDictClose(); - NTMDestroyLongNameTable( ); T5LOG(T5ERROR) << "Failed to open tm \""<< this->szName << "\" with rc = " << usRc; - setErrorCode(usRc); eStatus = FAILED_TO_OPEN_STATUS; }else{ eStatus = OPEN_STATUS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 6e1eb3d2..aaa51aa8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -45,24 +45,23 @@ PBYTE NTRecPos(PBYTE pStart, int iType); USHORT NTMAdjustAddDataInTgtCLB ( - PTMX_RECORD *ppTmRecordStart, - PULONG pulRecBufSize, + BytesRef pTmRecordStart, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, - PTMX_RECORD *ppCurTmRecord, PTMX_TARGET_RECORD *ppTMXTargetRecord, PLONG pulLeftClbLen, PBOOL pfUpdate); USHORT TMLoopAndDelTargetClb ( - PTMX_RECORD pTmRecord, + BytesRef pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, USHORT usPutAuthor, PBOOL fNewerTargetExists, - PINT pTargetKey + PINT pTargetKey, + PBOOL pfTargetMatch ); /**********************************************************************/ /* activate this define to get a faster memory */ @@ -92,67 +91,27 @@ USHORT TMLoopAndDelTargetClb bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) { - wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]; + wchar_t szBuff[MAX_SEGMENT_SIZE]{}; USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - if(Clb.usAddDataLen != usAddDataLen) + if(TmProposal.fNewEntryIfOnlyContextAndTimestampDifferent && Clb.usAddDataLen != usAddDataLen) return false; if (Clb.usAddDataLen) { - NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szAddInfo, sizeof(szAddInfo) / sizeof(CHAR_W) ); - if(wcscmp(szAddInfo,TmProposal.szContext)) - return false; - - NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szAddInfo, sizeof(szAddInfo) / sizeof(CHAR_W) ); - if(wcscmp(szAddInfo,TmProposal.szAddInfo)) + if(TmProposal.fNewEntryIfOnlyContextAndTimestampDifferent){ + NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szBuff ); + if(wcscmp(szBuff,TmProposal.szContext)) + return false; + } + szBuff[0]='\0'; + NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szBuff ); + if(wcscmp(szBuff,TmProposal.szAddInfo)) return false; } return true; } -static -BOOL CheckForAlloc -( - PTMX_SENTENCE pSentence, - PTMX_TERM_TOKEN * ppTermTokens, - USHORT usEntries -) -{ - LONG lFilled = 0L; - USHORT usNewAlloc = 0; - BOOL fOK = TRUE; - PTMX_TERM_TOKEN pTermTokens; - - pTermTokens = *ppTermTokens; - lFilled = ( (PBYTE)pTermTokens - (PBYTE)pSentence->pTermTokens); - if ( (pSentence->lTermAlloc - lFilled) <= (LONG)(usEntries * sizeof(TMX_TERM_TOKEN)) ) - { - //remember offset of pTagEntry - lFilled = pTermTokens - pSentence->pTermTokens; - if (usEntries == 1) - { - usNewAlloc = TOK_SIZE; - } - else - { - usNewAlloc = usEntries * sizeof(TMX_TERM_TOKEN); - } - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTermTokens, pSentence->lTermAlloc, - pSentence->lTermAlloc + (LONG)usNewAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTermAlloc += (LONG)usNewAlloc; - - //set new position of pTagEntry - pTermTokens = pSentence->pTermTokens + lFilled; - } /* endif */ - } /* endif */ - *ppTermTokens= pTermTokens; - return(fOK); -} - //------------------------------------------------------------------------------ @@ -209,59 +168,43 @@ USHORT EqfMemory::TmtXReplace PTMX_EXT_OUT_W pTmPutOut //ptr to output struct ) { - ULONG ulNewKey = 0; // sid of newly added tm record + LONG lNewKey = -1; // sid of newly added tm record BOOL fOK = TRUE; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound; // compact area hits - CHAR szString[MAX_EQF_PATH]; // character string - szString[0] = 0; - BOOL fLocked = FALSE; // TM-database-has-been-locked flag + CHAR szString[MAX_EQF_PATH]{}; // character string BOOL fUpdateOfIndexFailed = FALSE; // TRUE = update of index failed if(TmProposal.lSegmentId > 0) { - if(TmProposal.lSegmentId >= stTmSign.segmentIndex - 1){ - stTmSign.segmentIndex = TmProposal.lSegmentId; + if(TmProposal.lSegmentId >= state->stTmSign.segmentIndex - 1){ + state->stTmSign.segmentIndex = TmProposal.lSegmentId; } //keep id that was send }else{ // generate new id - TmProposal.setSegmentId( ++stTmSign.segmentIndex); + TmProposal.setSegmentId( ++state->stTmSign.segmentIndex); + TmProposal.fSegmentIdWasGenerated = true; } - if ( !usRc ) - { - //build tag table path - Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); - strcat( szString, "/TABLE/"); - strcat( szString, TmProposal.szMarkup ); - strcat( szString, EXT_OF_FORMAT ); - - //tokenize source segment, resulting in norm. string and tag table record - usRc = TokenizeSource( TmProposal.pInputSentence.get(), szString, - TmProposal.szSourceLanguage); - - if ( strstr( szString, "OTMUTF8" ) ) { - strcpy( TmProposal.szMarkup, "OTMUTF8" ); - TmProposal.fMarkupChanged = TRUE ; - } - } /* endif */ + //build tag table path + Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); + strcat( szString, "/TABLE/"); + strcat( szString, TmProposal.szMarkup ); + strcat( szString, EXT_OF_FORMAT ); + + //tokenize source segment, resulting in norm. string and tag table record + usRc = TokenizeSource( TmProposal.pInputSentence.get(), szString, + TmProposal.szSourceLanguage); + + if ( strstr( szString, "OTMUTF8" ) ) { + strcpy( TmProposal.szMarkup, "OTMUTF8" ); + TmProposal.fMarkupChanged = TRUE ; + } if ( !usRc ) { HashSentence( TmProposal.pInputSentence.get() ); - TmProposal.pInputSentence->pTagRecord->usTagTableId = 1; - - //usRc = NTMGetIDFromName( TmProposal.szMarkup, NULL, (USHORT)TAGTABLE_KEY, - // &TmProposal.pInputSentence->pTagRecord->usTagTableId ); - TmProposal.pInputSentence->pTagRecord->usTagTableId = 1; - if(usRc){ - T5LOG( T5WARNING) << ":: NTMGetIDFromName( tagtable ) returned " << usRc; - } - - - } /* endif */ + toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; - if ( !usRc ) - { usMatchesFound = CheckCompactArea( TmProposal.pInputSentence.get(), this ); if ( usMatchesFound == TmProposal.pInputSentence->usActVote ) //all hash triples found { @@ -270,17 +213,17 @@ USHORT EqfMemory::TmtXReplace if(usRc == NO_ERROR) { - //ulNewKey = TmProposal.recordKey; - ulNewKey = TmProposal.currentInternalKey.getRecordKey(); + //lNewKey = TmProposal.recordKey; + lNewKey = TmProposal.currentInternalKey.getRecordKey(); } //if no tm record fitted for update assume new and add to tm else if ( usRc == ERROR_ADD_TO_TM ) { - usRc = AddToTm(TmProposal, &ulNewKey ); + usRc = AddToTm(TmProposal, lNewKey ); //update index if ( !usRc ) { - usRc = UpdateTmIndex( TmProposal.pInputSentence.get(), ulNewKey ); + usRc = UpdateTmIndex( TmProposal.pInputSentence.get(), lNewKey ); if ( usRc ) fUpdateOfIndexFailed = TRUE; } /* endif */ @@ -289,12 +232,12 @@ USHORT EqfMemory::TmtXReplace else { //add new tm record to tm database - usRc = AddToTm(TmProposal, &ulNewKey ); + usRc = AddToTm(TmProposal, lNewKey ); //update index if ( !usRc ) { - usRc = UpdateTmIndex( TmProposal.pInputSentence.get(), ulNewKey ); + usRc = UpdateTmIndex( TmProposal.pInputSentence.get(), lNewKey ); if ( usRc ) fUpdateOfIndexFailed = TRUE; } /* endif */ @@ -302,19 +245,13 @@ USHORT EqfMemory::TmtXReplace } /* endif */ if(!usRc){ - pTmPutOut->ulTmKey = TmProposal.currentInternalKey.getRecordKey(); + pTmPutOut->lTmKey = TmProposal.currentInternalKey.getRecordKey(); pTmPutOut->usTargetKey = TmProposal.currentInternalKey.getTargetKey(); RewriteCompactTable(); - usRc = TmBtree.QDAMDictUpdSignLocal(&stTmSign); + usRc = state->TmBtree.QDAMDictUpdSignLocal(&state->stTmSign); } - // unlock TM database if database has been locked - if ( fLocked ) - { - NTMLockTM( this, FALSE, &fLocked ); - } /* endif */ - pTmPutOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmPutOut->stPrefixOut.usTmtXRc = usRc; @@ -326,15 +263,8 @@ USHORT EqfMemory::TmtXReplace { // try to delete the segment from the memory - PTMX_EXT_OUT_W pstDelOut = NULL; - UtlAlloc( (PVOID *)&pstDelOut, 0L, sizeof (TMX_EXT_OUT_W), NOMSG ); - - if ( pstDelOut ) - { - TmtXDelSegm( TmProposal, pstDelOut ); - - UtlAlloc( (PVOID *)&pstDelOut, 0L, 0L, NOMSG ); - } /* endif */ + TMX_EXT_OUT_W DelOut; + TmtXDelSegm( TmProposal, &DelOut ); } /* endif */ return( usRc ); @@ -384,31 +314,33 @@ VOID HashSentence ) { PSZ_W pNormOffset; // pointer to start of normalized string - PTMX_TERM_TOKEN pTermTokens; // pointer to term token structure USHORT usCount = 0; // counter - pTermTokens = pSentence->pTermTokens; - while ( pTermTokens->usLength ) + //while ( pTermTokens->usLength ) + for(auto& token: pSentence->pTermTokens) { - pNormOffset = pSentence->pStrings->getNormStrC() + pTermTokens->usOffset; - pTermTokens->usHash = HashTupelW( pNormOffset, pTermTokens->usLength ); + if(token.usLength == 0 && token.usOffset == 0){ + break;//end-of-list-token + } + pNormOffset = pSentence->pStrings->getNormStrC() + token.usOffset; + token.usHash = HashTupelW( pNormOffset, token.usLength ); + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP) || FLAGS_log_hashes_in_hash_sentence || FLAGS_add_tokens_to_fuzzy){ - auto str = EncodingHelper::convertToUTF8(pNormOffset).substr(0, pTermTokens->usLength); + + auto str = EncodingHelper::convertToUTF8(pNormOffset).substr(0, token.usLength); if(FLAGS_add_tokens_to_fuzzy){ pSentence->tokens.push_back(str); } if(FLAGS_log_hashes_in_hash_sentence){ - T5LOG( T5TRANSACTION) <<"HashSentence:: pNormOffset = \"" << str << "\"; len = " << pTermTokens->usLength <<"; hash = " <usHash; + T5LOG( T5TRANSACTION) <<"HashSentence:: pNormOffset = \"" << str << "\"; len = " << token.usLength <<"; hash = " <pulVotes.size = " << pSentence->pulVotes.size() <<"; actVote = " << pSentence->usActVote; } /* endfor */ } /* endif */ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(pSentence->pStrings->getOriginalStrC()); + auto str = EncodingHelper::convertToUTF8(pSentence->pStrings->getOriginalSrcStr()); T5LOG( T5DEVELOP) <<"HashSentence:: inputString =\"" << str << "\"; count = " << usCount; } @@ -510,43 +444,46 @@ USHORT HashTupelW // with 7 words A B C D E F G // PREREQ: it is ensured that sentence has at least 3 tokens //------------------------------------------------------------------------------ -static VOID +VOID BuildVotes ( PTMX_SENTENCE pSentence // pointer to sentence structure ) { - PTMX_TERM_TOKEN pTermTokens; // pointer to active token - USHORT usIndex = 0; - PTMX_TERM_TOKEN pLastTerm; - + PTMX_TERM_TOKEN pTermTokens = pSentence->pTermTokens.data(); // pointer to active token //run through list of tokens and build tuples - pTermTokens = pSentence->pTermTokens; - - while ( pTermTokens->usLength && (pSentence->usActVote < ABS_VOTES) ) + int index = 0 , listLen = pSentence->pTermTokens.size(); + while (// (indexusLength && (pSentence->usActVote < ABS_VOTES) ) { Vote( pTermTokens, pSentence, 0 ); // ABC, BCD, CDE, DEF, EFG pTermTokens++; + index++; } /* endwhile */ - pLastTerm = pTermTokens; - pLastTerm --; - - pTermTokens = pSentence->pTermTokens; - - while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) + + PTMX_TERM_TOKEN pLastTerm = pSentence->pTermTokens.data() + pSentence->pTermTokens.size()-2; + pTermTokens = pSentence->pTermTokens.data(); + index = 0; + while ( //(indexusLength && (pSentence->usActVote < MAX_VOTES) ) { Vote( pTermTokens, pSentence, 1 ); // ABD, BCE, CDF, DEG pTermTokens++; + index++; } /* endwhile */ - pTermTokens = pSentence->pTermTokens; - while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) + pTermTokens = pSentence->pTermTokens.data(); + index = 0; + while ( //indexusLength && (pSentence->usActVote < MAX_VOTES) ) { Vote( pTermTokens, pSentence, 2 ); // ACD, BDE, CEF, DFG pTermTokens++; + index++; } /* endwhile */ - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); + index = 0; if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote (pTermTokens, pSentence, 4 ); // 1 * AAC @@ -561,7 +498,8 @@ BuildVotes /********************************************************************/ /* use 1st token as triple */ /********************************************************************/ - pTermTokens = pSentence->pTermTokens; + + pTermTokens = pSentence->pTermTokens.data(); if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote (pTermTokens, pSentence, 7 ); // 1 * AAA @@ -576,10 +514,12 @@ BuildVotes Vote (pTermTokens, pSentence, 7 ); // 1 * GGG } /* endif */ - usIndex = 0; - pTermTokens = pSentence->pTermTokens; - while ((usIndex < 3) && pTermTokens->usLength - && (pSentence->usActVote < MAX_VOTES) ) + index = 0; + pTermTokens = pSentence->pTermTokens.data(); + while ((index < 3)// && index < listLen) + && pTermTokens->usLength + && (pSentence->usActVote < MAX_VOTES) + ) { Vote (pTermTokens, pSentence, 3 ); // 3 * AAB if ((pLastTerm-1)->usLength @@ -587,7 +527,7 @@ BuildVotes { Vote (pLastTerm-1, pSentence, 5 ); // 3 * FGG } /* endif */ - usIndex++; + index++; } /* endwhile */ } /* end of function BuildVotes */ @@ -606,7 +546,7 @@ BuildVotes // Function flow: build all possible 3 tupels out of a sequence of 4 consec // utive tokens. (base is the FLASH algorithm). //------------------------------------------------------------------------------ -static VOID +VOID Vote ( PTMX_TERM_TOKEN pTermTokens, //ptr to term tokens @@ -617,16 +557,17 @@ Vote ULONG ulVote; //actual vote USHORT i; //index values BOOL fGo = FALSE; + int indexInList = pTermTokens - pSentence->pTermTokens.data(); // create all possible 3 tupels out of a sequence of 4 consecutive tokens //for dealing correctly with the tail... if ( usTuple == 0 ) { - fGo = (pTermTokens+2)->usLength; + fGo = (pSentence->pTermTokens.size() > indexInList+ 2)? (pTermTokens+2)->usLength: false; } else if ((usTuple == 2) || (usTuple == 1) ) { - fGo = (pTermTokens+3)->usLength; + fGo = (pSentence->pTermTokens.size() > indexInList+ 3)?(pTermTokens+3)->usLength: false; } else { @@ -640,10 +581,18 @@ Vote { ulVote = (ulVote*131) + pTermTokens[usTLookUp[i+3*usTuple]].usHash; } /* endfor */ - - pSentence->pulVotes[pSentence->usActVote] = ulVote; - pSentence->usActVote++; + //if(pSentence->pulVotes.back()!= ulVote){ + pSentence->pulVotes.push_back(ulVote); + //pSentence->pulVotes[pSentence->usActVote] = ulVote; + pSentence->usActVote++; + //} } /* endif */ + + #ifdef TEST_VOTES + T5LOG(T5TRANSACTION) << "called Vote, tuple = "<< usTuple<<"; fGo="<< fGo <<"; indexInList=" + << indexInList<<"; pSentence->usActVote=" << pSentence->usActVote + << ";pSentence->pulVotes.size()=" << pSentence->pulVotes.size()<< "; ulVote ="<pulVotes; // pointer to begin of votes + PULONG pulVotes = pSentence->pulVotes.data(); // pointer to begin of votes USHORT usMatch = 0; // number of matches for (USHORT i = 0; ( i < pSentence->usActVote) ; i++, pulVotes++ ) @@ -678,7 +627,7 @@ USHORT CheckCompactArea BYTE bTuple = (BYTE) ulVote; // active byte BYTE bRest = bTuple & 0x7; // relevant bit of byte - PBYTE pByte = ((PBYTE)pTmClb->bCompact) + (ulVote >> 3); // byte pointer + PBYTE pByte = pTmClb->state->bCompact.data() + (ulVote >> 3); // byte pointer if ( *pByte & (1 << bRest) ) { usMatch++; @@ -729,201 +678,6 @@ USHORT CheckCompactArea // build normalized string // //------------------------------------------------------------------------------ -USHORT TokenizeTarget -( - StringTagVariants* pStrings, // ptr to target string - //PSZ_W pNormString, // ptr to normalized string - PTMX_TAGTABLE_RECORD *ppTagRecord, // ptr to tag record structure - PLONG pulTagAlloc, // size of allocated area for tag record - PSZ pTagTableName, // name of tag table - //PUSHORT pusNormLen, // length of normalized string - EqfMemory* pClb // pointer to control block -) -{ - PVOID pTokenList = NULL; // ptr to token table - BOOL fOK; // success indicator - PBYTE pTagEntry; // pointer to tag entries - PLOADEDTABLE pTable = NULL; // pointer to tagtable -// USHORT usI; // offset - USHORT usFilled = 0; // counter - USHORT usRc = NO_ERROR; // returned value - int iTagEntryLen; // length indicator - PTMX_TAGTABLE_RECORD pTagRecord; // ptr to tag record structure - PSTARTSTOP pStartStop = NULL; // ptr to start/stop table - int iIterations = 0; - USHORT usAddEntries = 0; - PSZ_W pStringToTokenize = pStrings->getNormalizedTargetStrC();//pStrings->getGenericTargetStrC(); - - pTagRecord = (*ppTagRecord); //get contents of pointer - - //allocate 4K pTokenlist for TaTagTokenize - fOK = UtlAlloc( (PVOID *) &(pTokenList), 0L, (LONG) TOK_SIZE, NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pTagEntry = (PBYTE)pTagRecord; - pTagEntry += sizeof(TMX_TAGTABLE_RECORD); - RECLEN(pTagRecord) = 0; - pTagRecord->usFirstTagEntry = (USHORT)(pTagEntry - (PBYTE)pTagRecord); - - pTagRecord->usTagTableId = 1; - if ( !usRc ) - { - //tokenize target segment with correct tag table - //load tag table for tokenize function - - //build tag table path - usRc = TALoadTagTableExHwnd( pTagTableName, &pTable, FALSE, - TALOADUSEREXIT | TALOADPROTTABLEFUNC | - TALOADCOMPCONTEXTFUNC, - FALSE, NULLHANDLE ); - if ( usRc ) - { - LOG_AND_SET_RC(usRc, T5INFO, ERROR_TA_ACC_TAGTABLE); - } /* endif */ - } /* endif */ - - if ( !usRc ) - { - // build protect start/stop table for tag recognition - usRc = TACreateProtectTableW( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, - TOK_SIZE, &pStartStop, - pTable->pfnProtTable, pTable->pfnProtTableW, 0L ); - - while ((iIterations < 10) && (usRc == EQFRS_AREA_TOO_SMALL)) - { - // (re)allocate token buffer - LONG lOldSize = (usAddEntries * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - LONG lNewSize = ((usAddEntries+128) * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - - if (UtlAlloc((PVOID *) &pTokenList, lOldSize, lNewSize, NOMSG) ) - { - usAddEntries += 128; - iIterations++; - } - else - { - iIterations = 10; // force end of loop - } /* endif */ - // retry tokenization - if (iIterations < 10 ) - { - usRc = TACreateProtectTableW( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, - (USHORT)lNewSize, &pStartStop, - pTable->pfnProtTable, pTable->pfnProtTableW, 0L ); - } /* endif */ - - } /* endwhile */ - } /* endif */ - - if ( !usRc ) - { - PSTARTSTOP pEntry = pStartStop; - while ( (pEntry->usStart != 0) || - (pEntry->usStop != 0) || - (pEntry->usType != 0) ) - { - switch ( pEntry->usType ) - { - case UNPROTECTED_CHAR : - // handle translatable text - { - USHORT usLength = pEntry->usStop - pEntry->usStart + 1; - } /* end case UNPROTECTED_CHAR */ - break; - default : - // handle not-translatable data - { - // if next start/stop-entry is a protected one ... - if ( ((pEntry+1)->usStart != 0) && - ((pEntry+1)->usType != UNPROTECTED_CHAR) ) - { - // enlarge next entry and ignore current one - (pEntry+1)->usStart = pEntry->usStart; - } - else - { - // add tag data - iTagEntryLen = sizeof(TMX_TAGENTRY) + - (pEntry->usStop - pEntry->usStart + 1)*sizeof(CHAR_W); - if ( ((LONG)*pulTagAlloc - (LONG)(pTagEntry - (PBYTE)pTagRecord)) - <= (LONG)iTagEntryLen ) - { - //remember offset of pTagEntry - usFilled = (USHORT)(pTagEntry - (PBYTE)pTagRecord); - - LONG addedSpace = TOK_SIZE;//how many 4k pages we need for new alloc - while(((LONG)*pulTagAlloc + addedSpace - (LONG)(pTagEntry - (PBYTE)pTagRecord)) - <= (LONG)iTagEntryLen ){ - addedSpace += TOK_SIZE; - } - //allocate another 4k for pTagRecord - int i=0; - do{ - fOK = UtlAlloc( (PVOID *) &pTagRecord, *pulTagAlloc, - *pulTagAlloc + addedSpace, NOMSG ); - i++; - }while(i<10 && fOK == false); - - if ( fOK ) - { - *pulTagAlloc += (LONG)TOK_SIZE; - - //set new position of pTagEntry - pTagEntry = ((PBYTE)pTagRecord) + usFilled; - }else{ - T5LOG(T5ERROR) << "::ERROR: Segment not saved. It was tried to allocate " << *pulTagAlloc + addedSpace << - " bytes to save the tags of the segment, but this was still not enough. Since this should never happen, we did not try to allocate even more bytes and did not save the segment. This was the segment, that was not saved: " - << EncodingHelper::convertToUTF8(pStringToTokenize); - } - - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - ((PTMX_TAGENTRY)pTagEntry)->usOffset = pEntry->usStart; - ((PTMX_TAGENTRY)pTagEntry)->usTagLen = - (pEntry->usStop - pEntry->usStart + 1);// * sizeof(CHAR_W); - memcpy( &(((PTMX_TAGENTRY)pTagEntry)->bData), - pStringToTokenize + pEntry->usStart, - ((PTMX_TAGENTRY)pTagEntry)->usTagLen * sizeof(CHAR_W)); - pTagEntry += iTagEntryLen; - } /* endif */ - } /* endif */ - } /* end default */ - break; - } /* endswitch */ - pEntry++; - } /* endwhile */ - RECLEN(pTagRecord) = pTagEntry - (PBYTE)pTagRecord; - } /* endif */ - } /* endif */ - - - //release memory - if ( pStartStop ) UtlAlloc( (PVOID *) &pStartStop, 0L, 0L, NOMSG ); - if ( pTokenList ) UtlAlloc( (PVOID *) &pTokenList, 0L, 0L, NOMSG ); - - //free tag table / decrement use count - if ( pTable != NULL ) - { - TAFreeTagTable( pTable ); - } /* endif */ - - *ppTagRecord = pTagRecord; - - return( usRc ); -} //------------------------------------------------------------------------------ // External function @@ -941,7 +695,7 @@ USHORT TokenizeTarget // PTMX_SENTENCE pSentence // EqfMemory* pTmClb // PTMX_PUT pTmPut -// PULONG pulNewKey +// PULONG plNewKey //------------------------------------------------------------------------------ // Output parameter: // @@ -969,62 +723,41 @@ USHORT TokenizeTarget USHORT EqfMemory::AddToTm ( OtmProposal& TmProposal, // ptr to sentence structure - PULONG pulNewKey // sid of newly added tm record + LONG& lNewKey // sid of newly added tm record ) { - PTMX_RECORD pTmRecord = NULL; // ptr to tm record - PTMX_TARGET_CLB pTargetClb = NULL; // ptr to target ctl block + std::vector TmRecord; // ptr to tm record + std::vector TargetClb; // ptr to target ctl block USHORT usRc = NO_ERROR; // return code - BOOL fOK; // success indicator - USHORT usAddDataLen = 0; - - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - - //allocate target control block record - if ( fOK ) - { - usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - - fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+usAddDataLen), NOMSG ); - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - if ( usRc == NO_ERROR ) - { - usRc = FillClb( pTargetClb, TmProposal ); - if ( usRc == NO_ERROR ) - { - USHORT usSrcLang = 0; - usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); - //fill tm record to add to database - FillTmRecord ( TmProposal.pInputSentence.get(), // ptr to sentence struct for source info - pTmRecord, // filled tm record returned - pTargetClb, usSrcLang ); - - //add new tm record to database - *pulNewKey = NTMREQUESTNEWKEY; - - usRc = TmBtree.EQFNTMInsert(//ptr to tm structure - pulNewKey, //to be allocated in funct - (PBYTE)pTmRecord, //pointer to tm record - pTmRecord->lRecordLen); //length - if(!usRc){ - TmProposal.currentInternalKey.setInternalKey(*pulNewKey, 1); - } - } /* endif */ - } /* endif */ + USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); + size_t clbSize = sizeof(TMX_TARGET_CLB)+usAddDataLen + 1; + T5LOG(T5DEBUG)<<"Adding new proposal to the tm, size = " << clbSize; + TargetClb.resize(clbSize); + + usRc = FillClb( TargetClb, TmProposal ); + if ( usRc == NO_ERROR ) + { + USHORT usSrcLang = 0; + usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, (USHORT)LANG_KEY, &usSrcLang ); + TmRecord.resize(TMX_REC_SIZE); + //fill tm record to add to database + FillTmRecord ( TmProposal.pInputSentence.get(), // ptr to sentence struct for source info + TmRecord, // filled tm record returned + TargetClb, usSrcLang ); + + //add new tm record to database + lNewKey = NTMREQUESTNEWKEY; + size_t size = toTmxRecord(TmRecord)->lRecordLen; + if(size > 0){ + TmRecord.resize(size); + } + usRc = state->TmBtree.EQFNTMInsert(lNewKey, //to be allocated in funct + TmRecord); //pointer to tm record + if(!usRc){ + TmProposal.currentInternalKey.setInternalKey(lNewKey, 1); + } } /* endif */ - //release memory - UtlAlloc( (PVOID *) &(pTmRecord), 0L, 0L, NOMSG); - UtlAlloc( (PVOID *) &(pTargetClb), 0L, 0L, NOMSG); - return( usRc ); } @@ -1060,22 +793,21 @@ USHORT EqfMemory::AddToTm VOID FillTmRecord ( PTMX_SENTENCE pSentence, // ptr to sentence struct for source info - //PTMX_TAGTABLE_RECORD pTagRecord, // ptr to target string tag table - //PSZ_W pNormString, // ptr to target normalized string - //USHORT usNormLen, // length of target normalized string - PTMX_RECORD pTmRecord, // filled tm record returned - PTMX_TARGET_CLB pTargetClb, // ptr to target control block + BytesRef TmRecord, // filled tm record returned + BytesRef TargetClb, // ptr to target control block USHORT usSrcLangId ) { + PTMX_TARGET_CLB pTargetClb = toTmxTargetClb(TargetClb); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to start of source structure PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to start of target structure PBYTE pTarget; //ptr to target record + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); ULONG ulSrcLen = wcslen(pSentence->pStrings->getGenericTagStrC()); ULONG ulTrgLen = wcslen(pSentence->pStrings->getGenericTargetStrC()); //position source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(TmRecord.data() + sizeof(TMX_RECORD)); pTMXSourceRecord->reset(); //source structure offset in tm record pTmRecord->usSourceRecord = (USHORT)((PBYTE)pTMXSourceRecord - (PBYTE)pTmRecord); @@ -1091,8 +823,7 @@ VOID FillTmRecord RECLEN(pTMXSourceRecord) = sizeof( TMX_SOURCE_RECORD ) + ulSrcLen; //position target structure in tm record - pTarget = (PBYTE)(pTmRecord+1); - pTarget += RECLEN(pTMXSourceRecord); + pTarget = TmRecord.data() + sizeof(TMX_RECORD) + RECLEN(pTMXSourceRecord); //first target offset in tm record pTmRecord->usFirstTargetRecord = (USHORT)(pTarget - (PBYTE)pTmRecord); @@ -1100,39 +831,10 @@ VOID FillTmRecord //position to start of target structure pTMXTargetRecord = ((PTMX_TARGET_RECORD)pTarget); pTMXTargetRecord->reset(); - pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); + pTMXTargetRecord->usTarget = sizeof(TMX_TARGET_RECORD); pTarget += pTMXTargetRecord->usTarget; - - //set source tag table offset - //pTMXTargetRecord->usSourceTagTable = sizeof(TMX_TARGET_RECORD); - - //position pointer for source tag table - //pTarget += pTMXTargetRecord->usSourceTagTable; - - //copy source tag table record to correct position - //memcpy( pTarget, pSentence->pTagRecord, RECLEN(pSentence->pTagRecord) ); - - //set target tag table start offset - //pTMXTargetRecord->usTargetTagTable = (USHORT)(pTMXTargetRecord->usSourceTagTable + - // RECLEN(pSentence->pTagRecord)); - //adjust target pointer for target tag table - //pTarget += RECLEN(pSentence->pTagRecord); - - //copy target tag table record to correct position - //memcpy( pTarget, pTagRecord, RECLEN(pTagRecord) ); - - //set target string start offset - //pTMXTargetRecord->usTarget = //(USHORT)(pTMXTargetRecord->usTargetTagTable + - //RECLEN(pTagRecord); - //adjust target pointer for target string - //pTarget += RECLEN(pTagRecord); - - //copy target string to correct position -//@@ memcpy( pTarget, pNormString, usNormLen ); - { - ulTrgLen = EQFUnicode2Compress( pTarget, pSentence->pStrings->getGenericTargetStrC(), ulTrgLen ); - } + ulTrgLen = EQFUnicode2Compress( pTarget, pSentence->pStrings->getGenericTargetStrC(), ulTrgLen ); //set target string control block start offset pTMXTargetRecord->usClb = pTMXTargetRecord->usTarget + @@ -1146,10 +848,7 @@ VOID FillTmRecord //size of target record RECLEN(pTMXTargetRecord) = sizeof(TMX_TARGET_RECORD) + - //RECLEN(pSentence->pTagRecord) + - //RECLEN(pTagRecord) + ulTrgLen + - //ulSrcLen + TARGETCLBLEN(pTargetClb); //size of entire tm record RECLEN(pTmRecord) = sizeof( TMX_RECORD ) + @@ -1197,33 +896,29 @@ VOID FillTmRecord //------------------------------------------------------------------------------ USHORT EqfMemory::FillClb ( - PTMX_TARGET_CLB pTargetClb, // ptr to target control block + BytesRef TargetClb, // ptr to target control block OtmProposal& TmProposal // ptr to put input structure ) { + PTMX_TARGET_CLB pTargetClb = (PTMX_TARGET_CLB) TargetClb.data(); USHORT usTrgLang,usSrcLang; USHORT usFile = 0; USHORT usAuthor = 0; // ids USHORT usRc = NO_ERROR; // returned value //get id of target language, call - usRc = NTMGetIDFromName( TmProposal.szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTrgLang ); - - //if ( !usRc ) - //{ - // usRc = NTMGetIDFromName( pTmClb, TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); - //} /* endif */ + usRc = NTMGetIDFromName( TmProposal.szTargetLanguage, (USHORT)LANG_KEY, &usTrgLang ); //get id of file name, call if ( !usRc ) { - usRc = NTMGetIDFromName( TmProposal.szDocName, TmProposal.szDocName, (USHORT)FILE_KEY, &usFile ); + usRc = NTMGetIDFromName( TmProposal.szDocName,(USHORT)FILE_KEY, &usFile ); } /* endif */ //get id of target author if ( !usRc ) { - usRc = NTMGetIDFromName( TmProposal.szTargetAuthor, NULL, (USHORT)AUTHOR_KEY, &usAuthor ); + usRc = NTMGetIDFromName( TmProposal.szTargetAuthor,(USHORT)AUTHOR_KEY, &usAuthor ); } /* endif */ if ( !usRc ) @@ -1304,64 +999,54 @@ USHORT EqfMemory::UpdateTmIndex USHORT usIndexEntries; // nr of entries in index record PULONG pulVotes = NULL; // pointer to votes USHORT i; // index in for loop - ULONG ulKey; // index key - PTMX_INDEX_RECORD pIndexRecord = NULL; // pointer to index structure - BOOL fOK = FALSE; // success indicator + LONG lKey; // index key + std::vector pIndexRecord; // pointer to index structure + BOOL fOK = TRUE; // success indicator PBYTE pIndex; // position pointer //for all votes add the index to the corresponding list - pulVotes = pSentence->pulVotes; - - //allocate 32K for tm index record - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); + pulVotes = pSentence->pulVotes.data(); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { - for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) + for ( i = 0; i < pSentence->pulVotes.size(); i++, pulVotes++ ) { if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - usRc = InBtree.EQFNTMGet( ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record - &ulLen ); //length + lKey = (*pulVotes) & START_KEY; + pIndexRecord.clear(); + usRc = state->InBtree.EQFNTMGet( lKey, //index key + pIndexRecord); //pointer to index record if ( usRc == BTREE_NOT_FOUND ) { + pIndexRecord.resize(sizeof( TMX_INDEX_RECORD )); //key is not in index file; add a new index entry - pIndexRecord->usRecordLen = sizeof( TMX_INDEX_RECORD ); + toIndexRecord(pIndexRecord)->usRecordLen = sizeof( TMX_INDEX_RECORD ); - pIndexRecord->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); + toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->pulVotes.size(),ulSidKey); - usRc = InBtree.EQFNTMInsert( &ulKey, - (PBYTE)pIndexRecord, //pointer to index - pIndexRecord->usRecordLen ); //length + usRc = state->InBtree.EQFNTMInsert( lKey, pIndexRecord); //pointer to index + //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file - if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) + if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { - usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRc = EQFNTMOrganizeIndex( &(state->InBtree), state->usAccessMode, START_KEY ); if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(&ulKey, (PBYTE)pIndexRecord, pIndexRecord->usRecordLen ); + usRc = state->InBtree.EQFNTMInsert(lKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area - ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact + (ulKey >> 3)) |= - 1 << ((BYTE)ulKey & 0x07); -// 1 << (USHORT)(ulKey & 0x07); @01M - bCompactChanged = TRUE; + lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); + *(state->bCompact.data() + (lKey >> 3)) |= + 1 << ((BYTE)lKey & 0x07); +// 1 << (USHORT)(lKey & 0x07); @01M + state->bCompactChanged = TRUE; } /* endif */ } else @@ -1371,33 +1056,36 @@ USHORT EqfMemory::UpdateTmIndex BOOL fFound = FALSE; //key is in index file; update index entry with new sid - ulLen = pIndexRecord->usRecordLen; + ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + if( toIndexRecord(pIndexRecord)->usRecordLen > 0){ + pIndexRecord.resize( toIndexRecord(pIndexRecord)->usRecordLen + sizeof(ULONG)); + } //calculate number of entries in index record usIndexEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); //// check if SID is already contained in list.. - //{ - // int i = (int)usIndexEntries; - // PULONG pulIndex = (PULONG)&(pIndexRecord->stIndexEntry); - // while ( i ) - // { - // if ( *pulIndex == ulNewSID ) - // { - // fFound = TRUE; - // break; - // } /* endif */ - // pulIndex++; - // i--; - // } /*endwhile */ - //} + { + int i = (int)usIndexEntries; + PULONG pulIndex = (PULONG)&(toIndexRecord(pIndexRecord)->stIndexEntry); + while ( i ) + { + if ( *pulIndex == ulSidKey/*ulNewSID*/ ) + { + fFound = TRUE; + break; + } /* endif */ + pulIndex++; + i--; + } /*endwhile */ + } if ( !fFound ) { if ( usIndexEntries >= (MAX_INDEX_LEN -1)) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord; + pIndex = pIndexRecord.data(); memmove( pIndex, pIndex + sizeof(ULONG), ulLen - sizeof(ULONG) ); ulLen -= sizeof(ULONG); usIndexEntries--; @@ -1407,7 +1095,7 @@ USHORT EqfMemory::UpdateTmIndex && ((ulLen + sizeof( TMX_INDEX_ENTRY )) <= TMX_REC_SIZE) ) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord; + pIndex = pIndexRecord.data(); //move pointer to end of index record pIndex += ulLen; @@ -1417,35 +1105,35 @@ USHORT EqfMemory::UpdateTmIndex NTMINDEX(pSentence->usActVote,ulSidKey); //update index record size - pIndexRecord->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); + toIndexRecord(pIndexRecord)->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); + pIndexRecord.resize(toIndexRecord(pIndexRecord)->usRecordLen); - usRc = InBtree.EQFNTMUpdate( - ulKey, - (PBYTE)pIndexRecord, //pointer to index - pIndexRecord->usRecordLen ); //length + usRc = state->InBtree.EQFNTMUpdate( + lKey, + pIndexRecord);//.data(), //pointer to index + //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file - if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) + if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { - usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRc = EQFNTMOrganizeIndex( &(state->InBtree), state->usAccessMode, START_KEY ); if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(ulKey, (PBYTE)pIndexRecord, pIndexRecord->usRecordLen ); + usRc = state->InBtree.EQFNTMUpdate(lKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area - ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact + (ulKey >> 3)) |= - 1 << ((BYTE)ulKey & 0x07); - bCompactChanged = TRUE; + lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); + *((PBYTE)state->bCompact.data() + (lKey >> 3)) |= + 1 << ((BYTE)lKey & 0x07); + state->bCompactChanged = TRUE; } /* endif */ } else { usIndexEntries = usIndexEntries; - } /* endif */ } /* endif */ @@ -1460,9 +1148,6 @@ USHORT EqfMemory::UpdateTmIndex } /* endfor */ } /* endif */ - //release allocated memory - UtlAlloc( (PVOID *) &(pIndexRecord), 0L, 0L, NOMSG); - return( usRc ); } @@ -1503,179 +1188,213 @@ USHORT DetermineTmRecord ( EqfMemory* pTmClb, // ptr to tm control block PTMX_SENTENCE pSentence, // ptr to sentence structure - PULONG pulSids // ptr to tm record + std::vector& lSids // ptr to tm record ) { USHORT usRc = NO_ERROR; // return code - ULONG ulLen; // length paramter - USHORT usSid = 0; // number of sentence ids found + ULONG ulLen = 0; // length paramter USHORT usPos; // position in pulSids PULONG pulVotes; // pointer to votes - PULONG pulSidStart; // po ter to votes USHORT i, j; // index in for loop USHORT usMaxEntries = 0; // nr of index entries in index record - ULONG ulKey; // index key - PTMX_INDEX_RECORD pIndexRecord = NULL; // pointer to index structure - BOOL fOK; // success indicator + LONG lKey; // index key + std::vector pIndexRecord; // pointer to index structure + BOOL fOk = true; // success indicator PTMX_INDEX_ENTRY pIndexEntry; // pointer to index entry structure - pulSidStart = pulSids; - //for all votes add the index to the corresponding list - pulVotes = pSentence->pulVotes; + pulVotes = pSentence->pulVotes.data(); - //allocate 32K for tm index record - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + lKey = (*pulVotes) & START_KEY; + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, //index key + pIndexRecord); //pointer to index record + if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - usRc = pTmClb->InBtree.EQFNTMGet( - ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record - &ulLen ); //length - - if ( usRc == NO_ERROR ) - { - //calculate number of index entries in index record - ulLen = pIndexRecord->usRecordLen; - usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - + //calculate number of index entries in index record + ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + + if(ulLen<(sizeof(USHORT))){ + T5LOG(T5ERROR) << "Record len in tm is too small, rec=" << lKey << ", len="<CheckLogLevel(T5DEVELOP)){ std::string msg = __func__ + std::string(":: Number Entries: ") + toStr(usMaxEntries).c_str() +"; Entries:"; - pIndexEntry = &pIndexRecord->stIndexEntry; + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; for (j=0 ; jstIndexEntry; + if ( usRc == NO_ERROR ) + { + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; - for ( j = 0; j < usMaxEntries; j++, pIndexEntry++ ) + for ( j = 0; j < usMaxEntries; j++, pIndexEntry++ ) + { + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) - { - *pulSids = NTMKEY(*pIndexEntry); - usSid++; - pulSids++; - } /* endif */ - } /* endfor */ + lSids.push_back(NTMKEY(*pIndexEntry)); + } /* endif */ + } /* endfor */ - if ( (usSid > 1) ) + if ( (lSids.size() > 1) ) + { + //there is more than one sentence to check so go through other + //indices until all have been tried or usSid has been reduced to 1 + pulVotes++; + for ( i = 0; (i < pSentence->usActVote-1) && (lSids.size() > 1); + i++, pulVotes++ ) { - //there is more than one sentence to check so go through other - //indices until all have been tried or usSid has been reduced to 1 - pulVotes++; - for ( i = 0; (i < pSentence->usActVote-1) && (usSid > 1); - i++, pulVotes++ ) + if ( usRc == NO_ERROR ) { + lKey = (*pulVotes) & START_KEY; + pIndexRecord.clear(); + //memset( pIndexRecord.data(), 0, pIndexRecord.size() ); + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, //index key + pIndexRecord); //pointer to index record + if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - usRc = pTmClb->InBtree.EQFNTMGet( - ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record - &ulLen ); //length - - if ( usRc == NO_ERROR ) + //calculate number of index entries in index record + ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + usMaxEntries = (USHORT)((pIndexRecord.size() - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); + + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; + +#ifdef OLD_CODE + PULONG pulSids = ulSidStart; + usPos = 0; + + //end criteria are all sentence ids in index key or only one + //sentence id left in pulSids + for ( j = 0; (j < usMaxEntries) && (usSid > 1); + j++, pIndexEntry++ ) { - //calculate number of index entries in index record - ulLen = pIndexRecord->usRecordLen; - usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - - pIndexEntry = &pIndexRecord->stIndexEntry; - pulSids = pulSidStart; - usPos = 0; - - //end criteria are all sentence ids in index key or only one - //sentence id left in pulSids - for ( j = 0; (j < usMaxEntries) && (usSid > 1); - j++, pIndexEntry++ ) + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) + //before adding sentence id check if already in pulsids as the + //respective tm record need only be checked once + while ( (NTMKEY(*pIndexEntry) > *pulSids) && (usSid > 1) ) { - //before adding sentence id check if already in pulsids as the - //respective tm record need only be checked once - while ( (NTMKEY(*pIndexEntry) > *pulSids) && (usSid > 1) ) + //remove sid from pulSids and decrease sid counter + if ( usSid > usPos ) { - //remove sid from pulSids and decrease sid counter - if ( usSid > usPos ) - { - ulLen = (usSid - usPos) * sizeof(ULONG); - } - else - { - ulLen = sizeof(ULONG); - } /* endif */ - memmove( (PBYTE) pulSids, (PBYTE)(pulSids+1), ulLen ); - usSid--; - } /* endwhile */ - - if ( NTMKEY(*pIndexEntry) == *pulSids ) + ulLen = (usSid - usPos) * sizeof(ULONG); + } + else { - //move on one position in pulSids - pulSids++; - usPos++; + ulLen = sizeof(ULONG); } /* endif */ + memmove( (PBYTE) pulSids, (PBYTE)(pulSids+1), ulLen ); + usSid--; + } /* endwhile */ + + if ( NTMKEY(*pIndexEntry) == *pulSids ) + { + //move on one position in pulSids + pulSids++; + usPos++; } /* endif */ - } /* endfor */ + } /* endif */ + } /* endfor */ +#elif NEW_CODE_1 + // End criteria: all sentence IDs in index key or only one sentence ID left in pulSids + for (j = 0; (j < usMaxEntries) && (lSids.size() > 1); j++, pIndexEntry++) { + if (NTMVOTES(*pIndexEntry) == static_cast(pSentence->usActVote)) { + // Before adding sentence ID, check if already in pulSids, as the respective TM record needs only be checked once + while ((NTMKEY(*pIndexEntry) > lSids.front()) && (lSids.size() > 1)) { + // Remove SID from pulSids and decrease SID counter + lSids.erase(lSids.begin()); + } + + if (NTMKEY(*pIndexEntry) == lSids.front()) { + // Move on to the next position in pulSids + lSids.erase(lSids.begin()); + usPos++; + } + } } - else +#elif NEW_CODE_2 // new code more based on old code + //PULONG pulSids = lSidsv; + //usPos = 0; + + //end criteria are all sentence ids in index key or only one + //sentence id left in pulSids + int k = 0;//sid index; + for ( j = 0; (j < usMaxEntries) && (k < lSids.size()); + j++, pIndexEntry++ ) { - //in case the index record doesn't exist exit function and - //set return code to 0 - if ( usRc == BTREE_NOT_FOUND ) + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - LOG_AND_SET_RC(usRc, T5DEVELOP, NO_ERROR); + //before adding sentence id check if already in pulsids as the + //respective tm record need only be checked once + while (k < lSids.size() && NTMKEY(*pIndexEntry) > lSids[k]) { + T5LOG(T5TRANSACTION) << "lKey=" <stIndexEntry; - while ( *pulSidStart ) + for(auto sid: lSids) { - msg += std::to_string(*pulSidStart) + "; "; - pulSidStart++; + msg += std::to_string(sid) + "; "; } T5LOG( T5DEBUG) << msg; } - UtlAlloc( (PVOID *) &(pIndexRecord), 0L, 0L, NOMSG ); - - if ( usRc ) - { - ERREVENT2( DETERMINETMRECORD_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - return( usRc ); } @@ -1718,104 +1437,66 @@ USHORT EqfMemory::UpdateTmRecord OtmProposal& TmProposal //pointer to get in data ) { - BOOL fOK; //success indicator BOOL fStop = FALSE; //indication to leave while loop - PULONG pulSids = NULL; //ptr to sentence ids - PULONG pulSidStart = NULL; //ptr to sentence ids + std::vector lSids; //ptr to sentence ids + int iSidCount=0; USHORT usRc = NO_ERROR; //return code - ULONG ulLen; //length indicator - ULONG ulKey; //tm record key - PTMX_RECORD pTmRecord = NULL; //pointer to tm record - ULONG ulRecBufSize = 0L; // current size of record buffer - - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - if ( fOK ) - { - ulRecBufSize = TMX_REC_SIZE; - } /* endif */ + std::vector TmRecord; //pointer to tm record - //allocate for sentence ids - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, - (LONG)((MAX_INDEX_LEN + 5) * sizeof(ULONG)), - NOMSG ); - } /* endif */ + lSids.reserve(MAX_INDEX_LEN + 5); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pulSidStart = pulSids; - usRc = DetermineTmRecord( this, TmProposal.pInputSentence.get(), pulSids ); - if ( usRc == NO_ERROR ) + usRc = DetermineTmRecord( this, TmProposal.pInputSentence.get(), lSids ); + + if ( usRc == NO_ERROR ) + { + #ifdef TEST_SIDS + std::string sids; int i =0; + for(auto sid:lSids){ + sids += std::to_string(i++) + ":" + std::to_string(sid) + "; "; + } + T5LOG(T5TRANSACTION) << "0_7_sids : "<< sids; + #endif//TEST_SIDS + + //get tm record(s) + //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) + for(LONG lKey: lSids) { - //get tm record(s) - while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) - { - ulKey = *pulSids; - ulLen = TMX_REC_SIZE; - memset( pTmRecord, 0, ulLen ); - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - memset( pTmRecord, 0, ulLen ); + iSidCount ++; + TmRecord.clear(); + usRc = state->TmBtree.EQFNTMGet( lKey, //tm record key + TmRecord); //pointer to tm record data - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + if ( usRc == NO_ERROR ) + { + //compare tm record data with data passed in the get in structure + usRc = ComparePutData( TmRecord, TmProposal, lKey ); + + // 5LOG(T5TRANSACTION) << "lKey = " << lKey <<"; usRc = " << usRc<<";sid n=" << iSidCount; - if ( usRc == NO_ERROR ) + if ( usRc == SOURCE_STRING_ERROR ) { - //compare tm record data with data passed in the get in structure - usRc = ComparePutData( &pTmRecord, &ulRecBufSize, - TmProposal, &ulKey ); - - if ( usRc == SOURCE_STRING_ERROR ) - { - //get next tm record - pulSids++; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } - else if ( usRc == NO_ERROR ) - { - //TmProposal.recordKey = ulKey; - //new target record was added or an existing one was successfully - //replaced so don't try other sids - fStop = TRUE; - } /* endif */ + //T5LOG(T5TRANSACTION) << "lKey lead to source_string_error, key =" <= TMX_REC_SIZE){ + usRc = BTREE_NODE_IS_FULL; + fStop = true; + fOK = false; + }else{ + auto pClbOffset = (PBYTE)pClb - pTmRecord.data(); + auto newSize = pClbOffset+ lNewClbLen + size;//RECLENB(pTmRecord) + lNewClbLen; + if(FLAGS_log_memmove_in_compareputdata) { - size_t occupiedSize = RECLEN(pTmRecord) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD); - if(lNewClbLen + occupiedSize >= TMX_REC_SIZE){ - usRc = BTREE_NODE_IS_FULL; - fStop = true; - fOK = false; - }else{ - if(FLAGS_log_memmove_in_compareputdata) - { - T5LOG(T5ERROR) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " - << RECLEN(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize - << "; ulKey = " << *pulKey ; - } - memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); - RECLEN(pTmRecord) += lNewClbLen; - RECLEN(pTMXTargetRecord) += lNewClbLen; - } + T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " + << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize + << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size() << "; newSize = " << newSize << "; pClbOffset = " << pClbOffset;// <<"; size before = " << size_before; } - else - //}catch(...) - { - T5LOG(T5ERROR) << "::memmove size is less or equal to 0, size = " << size << "; lNewClbLen = " << lNewClbLen << "; segment was not saved";//; memmove crash"; - usRc = ERROR_ADD_TO_TM; - fOK = false; + if(newSize > pTmRecord.size()){ + T5LOG(T5INFO) << "resizing pTmRecord, old size = "<< pTmRecord.size() <<"; new size = " < vTargetClb, vTargetRecord;//for managing memory USHORT usRc = NO_ERROR; // return code - BOOL fOK; // success indicator - //LONG lTagAlloc; // alloc size - PBYTE pByte; // position pointer - PTMX_RECORD pTmRecord = *ppTmRecord; //pointer to tm record data - ULONG ulAddDataLen = 0; //allocate target control block record - ulAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+ulAddDataLen), NOMSG ); - - //allocate target record - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pTargetRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - } - - //allocate 4k for pTagRecord - //if ( fOK ) - //{ - //fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG) TOK_SIZE, NOMSG ); - //if ( fOK ) - // lTagAlloc = (LONG)TOK_SIZE; - //} - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - //usRc = TokenizeTarget( TmProposal.pInputSentence->pStrings.get(), &pTagRecord, - // &lTagAlloc, TmProposal.szMarkup, this ); - if ( usRc == NO_ERROR ) + ULONG ulAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); + vTargetClb.resize(sizeof(TMX_TARGET_CLB)+ulAddDataLen); + vTargetRecord.resize(TMX_REC_SIZE); + pTargetClb = (PTMX_TARGET_CLB)vTargetClb.data(); + pTargetRecord = (PTMX_TARGET_RECORD)vTargetRecord.data(); + + usRc = FillClb( vTargetClb, TmProposal ); + if ( usRc == NO_ERROR ) + { + //fill target record + FillTargetRecord( TmProposal.pInputSentence.get(), //ptr to sentence structure + pTargetRecord, //filled tm record returned + pTargetClb ); //tm target control block + + //check space requirements + // re-alloc record buffer if too small + ULONG ulNewSize = RECLENB(pTmRecord) + sizeof(TMX_TARGET_RECORD) + + RECLEN(pTargetRecord); + if ( ulNewSize > pTmRecord.size()) { - usRc = FillClb( pTargetClb, TmProposal ); - if ( usRc == NO_ERROR ) - { - //fill target record - FillTargetRecord( TmProposal.pInputSentence.get(), //ptr to sentence structure - // pTagRecord, //ptr to target string tag table - TmProposal.pInputSentence->pStrings->getGenericTargetStrC(), - wcslen(TmProposal.pInputSentence->pStrings->getGenericTargetStrC()), - //pNormString,//ptr to target normalized string - //usNormLen, //length of target normalized string - &pTargetRecord, //filled tm record returned - pTargetClb ); //tm target control block - - //check space requirements - { - // re-alloc record buffer if too small - ULONG ulNewSize = RECLEN(pTmRecord) + sizeof(TMX_TARGET_RECORD) + - RECLEN(pTargetRecord); - if ( ulNewSize > *pulRecBufSize) - { - fOK = UtlAlloc( (PVOID *)ppTmRecord, *pulRecBufSize, ulNewSize, NOMSG ); - if ( fOK ) - { - *pulRecBufSize = ulNewSize; - pTmRecord = *ppTmRecord; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } - - if ( fOK ) - { - //postion at end on tm record - pByte = (PBYTE)pTmRecord; - pByte += RECLEN(pTmRecord); - - //add new target record to end - memcpy( pByte, pTargetRecord, RECLEN(pTargetRecord) ); - - //update overall length of tm record - RECLEN(pTmRecord) += RECLEN(pTargetRecord); - - //add updated tm record to database - usRc = TmBtree.EQFNTMUpdate( - *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord) ); - } /* endif */ - } /* endif */ + pTmRecord.resize(ulNewSize); } /* endif */ - } /* endif */ + + //postion at end on tm record + PBYTE pByte = pTmRecord.data(); + pByte += RECLENB(pTmRecord); - //release memory - UtlAlloc( (PVOID *) &(pTargetClb), 0L, 0L, NOMSG); - UtlAlloc( (PVOID *) &(pTargetRecord), 0L, 0L, NOMSG); - //UtlAlloc( (PVOID *) &(pTagRecord), 0L, 0L, NOMSG); + //add new target record to end + memcpy( pByte, pTargetRecord, RECLEN(pTargetRecord) ); - if ( usRc ) - { - ERREVENT2( ADDTMTARGET_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ + //update overall length of tm record + RECLENB(pTmRecord) += RECLEN(pTargetRecord); + //add updated tm record to database + usRc = state->TmBtree.EQFNTMUpdate(lKey, + pTmRecord); + } /* endif */ return( usRc ); } @@ -2495,53 +1993,25 @@ USHORT EqfMemory::AddTmTarget( VOID FillTargetRecord ( PTMX_SENTENCE pSentence, //ptr to sentence structure -// PTMX_TAGTABLE_RECORD pTagRecord, //ptr to target string tag table - PSZ_W pNormString, //ptr to target normalized string - USHORT usNormLen, //length of target normalized string - PTMX_TARGET_RECORD *ppTMXTargetRecord, //ptr to target record + PTMX_TARGET_RECORD pTMXTargetRecord, //ptr to target record PTMX_TARGET_CLB pTargetClb //ptr to target control block ) -{ - PBYTE pTarget; //ptr to target record - PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - - pTMXTargetRecord = *ppTMXTargetRecord; +{ + PSZ_W pNormString = pSentence->pStrings->getGenericTargetStrC(); //ptr to target normalized string + USHORT usNormLen = pSentence->pStrings->getGenericTargetStr().size(); //length of target normalized string //position to start of target structure - pTarget = (PBYTE)pTMXTargetRecord; - - //set source tag table offset - //pTMXTargetRecord->usSourceTagTable = sizeof(TMX_TARGET_RECORD); - - //position pointer for source tag table - //pTarget += pTMXTargetRecord->usSourceTagTable; - - //copy source tag table record to correct position - //memcpy( pTarget, pSentence->pTagRecord, RECLEN(pSentence->pTagRecord) ); - - //set target tag table start offset - //pTMXTargetRecord->usTargetTagTable = (USHORT)(pTMXTargetRecord->usSourceTagTable + - // RECLEN(pSentence->pTagRecord)); - - //adjust target pointer for target tag table - //pTarget += RECLEN(pSentence->pTagRecord); - - //copy target tag table record to correct position - //memcpy( pTarget, pTagRecord, RECLEN(pTagRecord) ); + PBYTE pTarget = (PBYTE)pTMXTargetRecord; //ptr to target record //set target string start offset pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); pTarget += pTMXTargetRecord->usTarget; - //(USHORT)(pTMXTargetRecord->usTargetTagTable + RECLEN(pTagRecord); - //adjust target pointer for target string - //pTarget += RECLEN(pTagRecord); //copy target string to correct position { ULONG ulTempLen = usNormLen; ulTempLen = EQFUnicode2Compress( pTarget, pNormString, ulTempLen ); usNormLen = (USHORT)ulTempLen; } -// memcpy( pTarget, pNormString, usNormLen ); //set target string control block start offset pTMXTargetRecord->usClb = pTMXTargetRecord->usTarget + usNormLen; @@ -2554,11 +2024,8 @@ VOID FillTargetRecord //size of target record RECLEN(pTMXTargetRecord) = sizeof(TMX_TARGET_RECORD) + - //RECLEN(pSentence->pTagRecord) + - //RECLEN(pTagRecord) + usNormLen + TARGETCLBLEN(pTargetClb); - *ppTMXTargetRecord = pTMXTargetRecord; } @@ -2583,238 +2050,6 @@ VOID FillTargetRecord // USHORT usUpdTarget number of target being updated // USHORT usFlags flags controlling the update //------------------------------------------------------------------------------ -USHORT EqfMemory::TmtXUpdSeg -( - OtmProposal* pTmPutIn, // ptr to put input data - PTMX_EXT_OUT_W pTmPutOut, //ptr to output struct - USHORT usFlags // flags controlling the updated fields -) -{ - BOOL fOK; // success indicator - USHORT usRc = NO_ERROR; // return code - PTMX_RECORD pTmRecord = NULL; // pointer to tm record - BOOL fLocked = FALSE; // TM-database-has-been-locked flag - ULONG ulLen = 0; //length indicator - PBYTE pByte; //position ptr - USHORT usTarget; //nr of target records in tm record - PTMX_TARGET_RECORD pTMXTargetRecord; // ptr to target record - PTMX_TARGET_CLB pTargetClb; // ptr to target CLB - ULONG ulLeftClbLen; // remaining length of CLB area - ULONG ulRecBufSize = 0L; // current size of record buffer - - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - ulRecBufSize = TMX_REC_SIZE; - } /* endif */ - - // get TM record being modified and update the record - if ( !usRc ) - { - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - - ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( pTmPutIn->currentInternalKey.getRecordKey(), (PCHAR)pTmRecord, - &ulLen ); - - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - memset( pTmRecord, 0, ulLen ); - - usRc = TmBtree.EQFNTMGet( - pTmPutIn->currentInternalKey.getRecordKey(), //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - if ( usRc == NO_ERROR ) - { - //check if record is empty - if ( ( RECLEN(pTmRecord) != 0) && - (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) - { - ULONG ulLeftTgtLen; // remaining target length - ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - if ( !usRc ) - { - //find target record specified in usUpdTarget - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - usTarget = 1; //initialize counter - - //loop until correct target is found - while ( (usTarget < pTmPutIn->currentInternalKey.getTargetKey()) //pTmPutIn->targetKey) - && ulLeftTgtLen ) - { - // position to first target CLB - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - - // loop over all target CLBs - while ( (usTarget < pTmPutIn->currentInternalKey.getTargetKey()) && ulLeftClbLen ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ - - // continue with next target if no match yet - if ( usTarget < pTmPutIn->currentInternalKey.getTargetKey() ) - { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - if (ulLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - ulLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - ulLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - usTarget++; - } /* endif */ - } /* endwhile */ - - if ( usTarget == pTmPutIn->currentInternalKey.getTargetKey())//pTmPutIn->targetKey ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target record exists - if ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) ) - { - // update requested fields - - // change markup/tag table if requested - if ( //(usFlags & TMUPDSEG_MARKUP) && - (pTmPutIn->szMarkup[0] != EOS) ) - { - PTMX_TAGTABLE_RECORD pTMXTagTableRecord; - USHORT usNewId; - PBYTE pByte; - - // get ID for new tag table - //usRc = NTMGetIDFromName( pTmPutIn->szMarkup, - // NULL, - // (USHORT)TAGTABLE_KEY, - // &usNewId ); - usNewId = 1; - - // update source tag table record - //pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; - //pTMXTagTableRecord->usTagTableId = 1;//usNewId; - - // update target tag table record - //pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; - //pTMXTagTableRecord->usTagTableId = 1;//usNewId; - } /* endif */ - - // update MT flag if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_MTFLAG) - ) - { - // set type of translation flag - pTargetClb->bTranslationFlag = (BYTE)pTmPutIn->eType; - } /* endif */ - - // update target language if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_TARGLANG) - ) - { - // set target language - usRc = NTMGetIDFromName( pTmPutIn->szTargetLanguage, - NULL, - (USHORT)LANG_KEY, - &pTargetClb->usLangId ); - } /* endif */ - - // update segment date if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_DATE) - ) - { - pTargetClb->lTime = pTmPutIn->lTargetTime; - } /* endif */ - - - // rewrite TM record - if ( usRc == NO_ERROR ) - { - usRc = TmBtree.EQFNTMUpdate( pTmPutIn->currentInternalKey.getRecordKey(), - (PBYTE)pTmRecord, RECLEN(pTmRecord) ); - } /* endif */ - - if(usRc == NO_ERROR) - { - usRc = RewriteCompactTable(); - } - } - else - { - // target not found - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } - else - { - // record not found - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } - else - { - // record is empty and should not be updated - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } /* endif */ - - // unlock TM database if database has been locked - if ( fLocked ) - { - NTMLockTM( this, FALSE, &fLocked ); - } /* endif */ - - //release allocated memory - UtlAlloc( (PVOID *)&pTmRecord, 0L, 0L, NOMSG ); - - return( usRc ); -} /* end of function TmtXUpdSeg */ /**********************************************************************/ @@ -2826,17 +2061,16 @@ USHORT EqfMemory::TmtXUpdSeg /**********************************************************************/ BOOL TMDelTargetClb ( - PTMX_RECORD pTmRecord, // ptr to TM record + BytesRef TmRecord, // ptr to TM record PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record PTMX_TARGET_CLB pTargetClb // ptr to target control record ) { BOOL fTargetRemoved = FALSE; - PBYTE pTmEnd; + PBYTE pTmEnd = TmRecord.data(); PBYTE pEndTarget; USHORT usTargetCLBLen; - pTmEnd = (PBYTE)pTmRecord; - pTmEnd += RECLEN(pTmRecord); + pTmEnd += RECLEN(toTmxRecord(TmRecord)); // check if there are more than one CLBs in current target record usTargetCLBLen = TARGETCLBLEN(pTargetClb); @@ -2849,7 +2083,7 @@ BOOL TMDelTargetClb pEndTarget = (PBYTE)pTargetRecord + RECLEN(pTargetRecord); //calculate new length of TM record - RECLEN(pTmRecord) -= RECLEN(pTargetRecord); + RECLEN(toTmxRecord(TmRecord)) -= RECLEN(pTargetRecord); //remove the current target record memmove( pTargetRecord, pEndTarget, pTmEnd - pEndTarget ); @@ -2858,8 +2092,8 @@ BOOL TMDelTargetClb fTargetRemoved = TRUE; // initialize last target record - pTmEnd = (PBYTE)pTmRecord; - pTmEnd += RECLEN(pTmRecord); + pTmEnd = TmRecord.data(); + pTmEnd += RECLEN(toTmxRecord(TmRecord)); pTargetRecord = (PTMX_TARGET_RECORD)pTmEnd; RECLEN(pTargetRecord) = 0; } @@ -2868,7 +2102,7 @@ BOOL TMDelTargetClb // delete only current target CLB //calculate new length of tm record - RECLEN(pTmRecord) -= usTargetCLBLen; + RECLEN(toTmxRecord(TmRecord)) -= usTargetCLBLen; RECLEN(pTargetRecord) -= usTargetCLBLen; //remove the current target CLB @@ -2888,13 +2122,10 @@ static int byteCount[6] = { 3, 2, 1, 1, 2, static long lOffset[6] = { 0x010ffff, 0x04040, 0x080, 0x080, 0x0bfc0, 0x0ef0000 }; static long lCompress[6] = { 0x10, 0x03f, 0x07f, 0x0bf, 0x0ee, 0x0ff }; - - ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) { ULONG ulLen = ulLenChar * sizeof(CHAR_W); - if ( ulLen < 20 ) // don't care about too short strings { *pTarget++ = 0; @@ -2909,18 +2140,15 @@ ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) int oldCodePoint = 0x80; int i, bCount, bCountTemp, cp; - *pTarget++ = BOCU_COMPRESS; pTemp = pTarget; - for ( ulLen = 0; ulLen < ulLenChar; ulLen++ ) { cp = *pInput++; delta = cp - oldCodePoint; oldCodePoint = cp; - for (i=0; ; ++i) // will always break { if ( delta <= lBoundary[i]) @@ -2960,26 +2188,27 @@ ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) } -LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG ulLenComp ) +LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp ) { LONG lLen = ulLenComp-1; + pOutput.clear(); - //T5LOG(T5TRANSACTION) <<"; lLen = " << lLen; if(lLen <= 0 ) return 0; + + pOutput.reserve(ulLenComp * 1.5); BYTE b = *pTarget++; - //T5LOG(T5TRANSACTION) << "b = " << b <<"; lLen = " << lLen; if ( b == 0 ) // no compression { - memcpy( pOutput, pTarget, lLen ); - lLen = ulLenComp / sizeof(CHAR_W); - pOutput[ lLen ] = EOS; + lLen = ulLenComp / sizeof(CHAR_W); + PSZ_W pWCharTarget = (PSZ_W) pTarget; + pOutput.insert(pOutput.begin(), pWCharTarget, pWCharTarget + lLen); + pOutput.push_back(EOS); // EOS should be L'\0' + //pOutput[ lLen ] = EOS; } - //* else if (b == BOCU_COMPRESS) { - PSZ_W pTemp = pOutput; long delta = 0; USHORT iLen = 0; int oldCodePoint = 0x80; @@ -3003,24 +2232,24 @@ LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG ulLenComp ) delta = delta + oldCodePoint - lOffset[i]; - *pOutput++ = (unsigned short) delta; + pOutput.insert(pOutput.end(), (unsigned short) delta); oldCodePoint = delta; break; } } } - *pOutput = 0; - lLen = (pOutput - pTemp); + pOutput.push_back(EOS); + lLen = pOutput.length();//(pOutput - pTemp); } else { //FilesystemHelper::FlushAllBuffers(); assert( 0 == 1); } - //*/ return lLen; } + #endif @@ -3029,11 +2258,9 @@ LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG ulLenComp ) USHORT NTMAdjustAddDataInTgtCLB ( - PTMX_RECORD *ppTmRecordStart, - PULONG pulRecBufSize, + BytesRef pTmRecord, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, - PTMX_RECORD *ppCurTmRecord, PTMX_TARGET_RECORD *ppTMXTargetRecord, PLONG pulLeftClbLen, PBOOL pfUpdate) @@ -3048,7 +2275,7 @@ USHORT NTMAdjustAddDataInTgtCLB PTMX_TARGET_CLB pNextClb = NULL; pTMXTargetRecord = *ppTMXTargetRecord; - pCurTmRecord = *ppCurTmRecord; + pCurTmRecord = toTmxRecord(pTmRecord); pCurClb = *ppClb; usOldLen = pCurClb->usAddDataLen; @@ -3079,23 +2306,16 @@ USHORT NTMAdjustAddDataInTgtCLB // re-alloc record buffer if too small ULONG ulNewSize = RECLEN(pCurTmRecord) + (ULONG)ulDiff; - if ( ulNewSize > *pulRecBufSize) + if ( ulNewSize > pTmRecord.size()) { - fOK = UtlAlloc( (PVOID *)ppTmRecordStart, *pulRecBufSize, - ulNewSize, NOMSG ); - if ( fOK ) - { - *pulRecBufSize = ulNewSize; - pCurClb = (PTMX_TARGET_CLB) ADJUSTPTR( *ppTmRecordStart, pCurTmRecord, pCurClb ); - // *ppTMRecordStart + (pClb-ppCurTmRecord) - pTMXTargetRecord = (PTMX_TARGET_RECORD) ADJUSTPTR( *ppTmRecordStart, pCurTmRecord, pTMXTargetRecord ); - pCurTmRecord = *ppTmRecordStart; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + size_t pCurClbOffs = (PBYTE)pCurClb - pTmRecord.data(); + size_t pTMXTargetRecordOffs = (PBYTE) pTMXTargetRecord - pTmRecord.data(); + pTmRecord.resize(ulNewSize); + + pCurTmRecord = toTmxRecord(pTmRecord); + pCurClb = (PTMX_TARGET_CLB) pTmRecord.data() + pCurClbOffs; + pTMXTargetRecord = (PTMX_TARGET_RECORD) pTmRecord.data() + pTMXTargetRecordOffs; + } // enlarge current control block and adjust TM record // length and target record length @@ -3128,7 +2348,7 @@ USHORT NTMAdjustAddDataInTgtCLB } /* endif */ *ppTMXTargetRecord = pTMXTargetRecord; - *ppCurTmRecord = pCurTmRecord; + //*ppCurTmRecord = pCurTmRecord; *ppClb = pCurClb; return (usRc); } // end of context handling @@ -3175,92 +2395,100 @@ USHORT NTMAdjustAddDataInTgtCLB // position at next target record // endloop //------------------------------------------------------------------------------ - USHORT TMLoopAndDelTargetClb ( - PTMX_RECORD pTmRecord, + BytesRef pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, USHORT usPutAuthor, PBOOL pfNewerTargetExists, - PINT pTargetKey + PINT pTargetKey, + PBOOL pfTargetMatch ) { USHORT usRc = NO_ERROR; PTMX_TARGET_CLB pClb = NULL; //ptr to target control block - //PTMX_TAGTABLE_RECORD pTMXSrcTTable = NULL; //ptr to source tag info LONG lLeftClbLen; LONG lLeftTgtLen; BOOL fTgtRemoved = FALSE; PTMX_TARGET_RECORD pTMXTgtRec = NULL; BOOL fDel = FALSE; - + + PBYTE pByte = NULL; //position ptr + PBYTE pStartTarget = NULL; //position ptr + // preset callers flag *pfNewerTargetExists = FALSE; // loop until either end of record or one occ. found&deleted - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; //source strings equal, position at first target record - pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(pTmRecord+1), REC_FIRSTTARGET); + pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); if(pTargetKey) *pTargetKey = 0; - while ( ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) + + while ( lLeftTgtLen>0 && ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) { - NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); - lLeftTgtLen -= RECLEN(pTMXTgtRec); - - // pos at source tag info - //pTMXSrcTTable = (PTMX_TAGTABLE_RECORD)NTRecPos((PBYTE)pTMXTgtRec, - // REC_SRCTAGTABLE); - // compare source tag record - //if ( (memcmp( pTMXSrcTTable, TmProposal.pInputSentence->pTagRecord, - // RECLEN(pTMXSrcTTable)) == 0) ) - { + NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); + lLeftTgtLen -= RECLEN(pTMXTgtRec); + pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTgtRec, REC_CLB); lLeftClbLen = RECLEN(pTMXTgtRec) - pTMXTgtRec->usClb; + + if(TmProposal.fSaveDifferentTargetsForSameSource){// default pfTargetMatch is true, but if flag is set, we compare saved target to new + std::wstring pStringTrg; + // check if target string and target tag record are identical + pByte = NTRecPos((PBYTE)pTMXTgtRec, REC_TGTSTRING); + LONG lLen = pTMXTgtRec->usClb - pTMXTgtRec->usTarget; + lLen = EQFCompress2Unicode( pStringTrg, pByte, lLen ); + pStringTrg.data()[lLen] = EOS; + + *pfTargetMatch = (pStringTrg == TmProposal.pInputSentence->pStrings->getGenericTargetStr()); + } - while ( ( lLeftClbLen > 0 ) && !fDel ) - { - (*pTargetKey)++; - if ( (pClb->usLangId == usPutLang) && - (pClb->usAuthorId == usPutAuthor) && - //(pClb->ulSegmId == TmProposal.lSegmentId) && - (pClb->usFileId == usPutFile) && - //&& !pClb->bMultiple - isAddDataIsTheSame(TmProposal, *pClb) - ) - { // remove target CLB and target record (if only 1 CLB) - // as the segment is putted with a new value - if ( (pClb->lTime < TmProposal.lTargetTime) || !TmProposal.lTargetTime ) - { - lLeftClbLen -= TARGETCLBLEN(pClb); - // loop over all CLBs of this target record and remove - // any CLB for the current segment - fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); - fDel = TRUE; - } - else - { // existing match is newer, continue with next one - lLeftClbLen -= TARGETCLBLEN(pClb); - *pfNewerTargetExists = TRUE; - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } - else - { // no match, try next one - lLeftClbLen -= TARGETCLBLEN(pClb); - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } /* endwhile */ - } /* endif */ - //position at next target - if ( !fDel ) - { - pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)pTMXTgtRec, REC_NEXTTARGET); - } /* endif */ - } /* endwhile */ + if(*pfTargetMatch){ + while ( ( lLeftClbLen > 0 ) && !fDel ) + { + (*pTargetKey)++; + + if ( (pClb->usLangId == usPutLang) && + (pClb->usAuthorId == usPutAuthor) && + (pClb->usFileId == usPutFile) && + isAddDataIsTheSame(TmProposal, *pClb) + ) + { // remove target CLB and target record (if only 1 CLB) + // as the segment is putted with a new value + if ( (pClb->lTime < TmProposal.lTargetTime) || !TmProposal.lTargetTime ) + { + lLeftClbLen -= TARGETCLBLEN(pClb); + // loop over all CLBs of this target record and remove + // any CLB for the current segment + fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); + fDel = TRUE; + } + else + { // existing match is newer, continue with next one + lLeftClbLen -= TARGETCLBLEN(pClb); + *pfNewerTargetExists = TRUE; + pClb = NEXTTARGETCLB(pClb); + } /* endif */ + } + else + { // no match, try next one + lLeftClbLen -= TARGETCLBLEN(pClb); + pClb = NEXTTARGETCLB(pClb); + } /* endif */ + } /* endwhile */ + } + + //position at next target + if ( !fDel ) + { + pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)pTMXTgtRec, REC_NEXTTARGET); + } /* endif */ + } /* endwhile */ return(usRc); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index f61e8952..f9db7ff1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -53,7 +53,47 @@ #include "LogWrapper.h" #include "OSWrapper.h" -//#include "EQFCMPR.H" +#include // for std::transform +//#include // for std::tolower + +void toLower(BytesRef vec) { + std::transform(vec.begin(), vec.end(), vec.begin(), [](unsigned char c) { + return std::tolower(c); + }); +} +void toUpper(BytesRef vec) { + std::transform(vec.begin(), vec.end(), vec.begin(), [](unsigned char c) { + return std::toupper(c); + }); +} + +#include +#include + +std::vector toLowerVector(const std::vector& input) { + std::vector result; + result.reserve(input.size()); // Optimize memory allocation + + for (unsigned char c : input) { + result.push_back(c == '\0' ? '\0' : std::tolower(c)); + } + + return result; +} + + +std::vector toUpperVector(const std::vector& input) { + std::vector result; + result.reserve(input.size()); // Optimize memory allocation + + for (unsigned char c : input) { + result.push_back(c == '\0' ? '\0' : std::toupper(c)); + } + + return result; +} + +// #include "EQFCMPR.H" /**********************************************************************/ /* prototypes for internal functions */ /**********************************************************************/ @@ -121,7 +161,7 @@ int NTMLongNameTableCompCaseIgnore( const void *, const void * ); //|+---------------+ | +--------------+ | +-------+ +-------+ | //|| pLanguages +-+ |usMaxEntries | | | usID | ........ | usID | | //|+---------------+ +--------------+ | -------+ -------+ | -//|| pAuthors +-+ |stTmTableEntry+-+ | +//|| pAuthors +-+ |table+-+ | //|+---------------+ | --------------+ | //|| pFiles | | TM_TABLE_ENTRY | //|+---------------+ | +--------------+ +-------+ +-------+ | @@ -129,7 +169,7 @@ int NTMLongNameTableCompCaseIgnore( const void *, const void * ); //|+---------------+ +--------------+ | +-------+ +-------+ | //|| ............. | |usMaxEntries | | | usID | ........ | usID | | //| ---------------+ +--------------+ | -------+ -------+ | -//| |stTmTableEntry+-+ | +//| |table+-+ | //| --------------+ | //+----------------------------------------------------------------------------+ //| | @@ -167,377 +207,141 @@ int NTMLongNameTableCompCaseIgnore( const void *, const void * ); //| | //| return usRc; | // ----------------------------------------------------------------------------+ -USHORT -EqfMemory::NTMGetIDFromName( - PSZ pszName, // input - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, //input - PUSHORT pusID ) //output -{ - return( NTMGetIDFromNameEx( pszName, pszLongName, usTableType, - pusID, 0, NULL ) ); -} /* end of function NTMGetIDFromName */ -// NTMGetIDFromNameEx -// this is an enhanced version of NTMGetIDFromName +// NTMGetIDFromName // using the new option NTMGETID_NOUPDATE_OPT the update of the name table can // suppressed for new names (important for get operations which should // not change the database) // the pusAlternativeID is filled with the ID of the name in the long name -// table if no pszLongName has been specified and the pszName is found in +// table if no pszLongName has been specified and the pszName is found in // the short name table as well -USHORT EqfMemory::NTMGetIDFromNameEx -( - PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input, type of table to use - PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID +USHORT EqfMemory::NTMGetIDFromName( + PSZ pszName, // input, name being looked up + USHORT usTableType, // input, type of table to use + PUSHORT pusID, // output, ID for name being looked up + LONG lOptions ) { - USHORT usRc = NO_ERROR; //function return coed - PTMX_TABLE_ENTRY pstTMTableEntry = NULL; //ptr to a table entry for bsearch - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - BOOL fLongName = FALSE; + USHORT usRc = NO_ERROR; // function return coed + PTMX_TABLE_ENTRY pstTMTableEntry = NULL; // ptr to a table entry for bsearch + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure // initialize ID - - // initialize ID - if ( lOptions & NTMGETID_NOUPDATE_OPT ) + if (lOptions & NTMGETID_NOUPDATE_OPT) { - *pusID = NTMGETID_NOTFOUND_ID; // use as not-found indicator + *pusID = NTMGETID_NOTFOUND_ID; // use as not-found indicator } else { *pusID = 0; } /* endif */ - if ( pusAlternativeID ) *pusAlternativeID = NTMGETID_NOTFOUND_ID; - *pusID = 0; - if ( pusAlternativeID ) *pusAlternativeID = NTMGETID_NOTFOUND_ID; - - if ( pszName[0] != EOS) + //--- if input parameters OK + if (pszName[0] != EOS) { - //--- if input parameters OK - if ( pszName[0] != EOS ) + if (usTableType == FILE_KEY) { - //--- capitalize input string - if(usTableType != FILE_KEY) - { - strupr( pszName ); - } + /**************************************************************/ + /* ... check first against our long name table */ + /**************************************************************/ + PTMX_LONGNAME_TABLE_ENTRY pEntry = nullptr; // ptr to entry found + TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); - /****************************************************************/ - /* If we are looking for documents and a long document name is */ - /* available ... */ - /****************************************************************/ + // prepare search entry + SearchEntry.pszLongName = pszName; + + if(state->LongNames.stTableEntry.size()){ + // do the actual search + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + state->LongNames.stTableEntry.data(), + state->LongNames.stTableEntry.size()-1,// last elem is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); + } - // skip any leading backslash (added by bug in ITM code ...) - if ( pszLongName != NULL ) + // if search fails try again using case insenstive search + if (pEntry == NULL && state->LongNamesCaseIgnore.stTableEntry.size()) { - if ( *pszLongName == BACKSLASH ) pszLongName++; + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + state->LongNamesCaseIgnore.stTableEntry.data(), + state->LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); } /* endif */ - fLongName = (usTableType == FILE_KEY) && - (pszLongName != NULL) && - (pszLongName[0] != EOS) - //&& (strcmp( pszLongName, pszName ) != 0) - ; - - if ( (usRc == NO_ERROR) && fLongName ) + if (pEntry != nullptr) { - /**************************************************************/ - /* ... check first against our long name table */ - /**************************************************************/ - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - - // prepare search entry - SearchEntry.pszLongName = pszLongName; - - // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - - - // if search fails try again using case insenstive search - if ( pEntry == NULL ) + // return ID of found entry + *pusID = pEntry->usId; + } + else if (!(lOptions & NTMGETID_NOUPDATE_OPT)) + { + // add a new entry for the given short name + if (usRc == NO_ERROR) { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); + usRc = NTMAddNameToTable(pszName, usTableType, pusID); } /* endif */ + } /* endif */ + }else{// (usTableType != FILE_KEY) + + //--- capitalize input string + strupr(pszName); - if ( pEntry != NULL ) + /******************************************************************/ + /* get pointer to table and table entries in dependency of the */ + /* table type */ + /******************************************************************/ + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); + if (usRc == NO_ERROR) + { + pszName[MAX_LANG_LENGTH-1] = '\0'; + /********************************************************************/ + /* search name passed in pszName in passed table usTableType */ + /********************************************************************/ + if(pstTMTable->table.size()){ + pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, + pstTMTable->table.data(), + pstTMTable->table.size()-1,// last elem is always empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } + //--- if passed name found in table + if (pstTMTableEntry) { - // return ID of found entry - *pusID = pEntry->usId; + //-- get ID of entry and set output parameter + *pusID = pstTMTableEntry->usId; } - else if ( !(lOptions & NTMGETID_NOUPDATE_OPT) ) + else { - ULONG ulNameLen = strlen(pszLongName) + 1; - ULONG ulAddLen = ulNameLen + sizeof(USHORT); - - - // add a new entry for the given short name - if ( usRc == NO_ERROR ) + // name is not contained in name table + if (!(lOptions & NTMGETID_NOUPDATE_OPT)) { - // for some (yet unknown) reasons, there are memories which have fewer entries in - // the short name table when in the long name table, using the ID generated when - // adding the short name will lead to incorrect IDs for the long name table. - // In order to circumvent this we call NTMAddNameToTable for the short name table - // until the returned ID is larger than or equal to pTmClb->pLongNames->ulEntries - do - { - DEBUGEVENT2( NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName ); - usRc = NTMAddNameToTable( pszName, usTableType, pusID ); - } while ( (usRc == NO_ERROR) && (*pusID < pLongNames->ulEntries) ); /* enddo */ - } /* endif */ + usRc = NTMAddNameToTable(pszName, usTableType, + pusID); - // add a new entry to the long name table - if ( usRc == NO_ERROR ) - { - // enlarge array if necessary - if ( pLongNames->ulEntries >= pLongNames->ulTableSize) + // for new languages update our language group table + if ((usRc == NO_ERROR) && + (usTableType == LANG_KEY)) { - ULONG ulOldSize = (ULONG)sizeof(TMX_LONGNAMETABLE) + - (ULONG)(sizeof(TMX_LONGNAME_TABLE_ENTRY) * - pLongNames->ulTableSize); - ULONG ulNewSize = ulOldSize + (ULONG) - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES); - - if ( UtlAlloc( (PVOID *)&pLongNames, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNames->ulTableSize += LONGNAMETABLE_ENTRIES; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // enlarge case ignore table as well - if ( usRc == NO_ERROR ) - { - if ( !UtlAlloc( (PVOID *)&pLongNamesCaseIgnore, - ulOldSize, ulNewSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - - // enlarge buffer if necessary - if ( usRc == NO_ERROR ) - { - if ( (pLongNames->ulBufUsed + ulAddLen ) >= - pLongNames->ulBufSize) - { - PSZ pszOldBuffer; // ptr to old buffer area - - ULONG ulNewLen = pLongNames->ulBufUsed + - ulAddLen + 256; - pszOldBuffer = pLongNames->pszBuffer; - if ( UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), - pLongNames->ulBufSize, - ulNewLen, NOMSG ) ) - { - // remember new buffer size - pLongNames->ulBufSize = ulNewLen; - - // adjust pointers in our table - { - ULONG ulI; - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to table entry - - pEntry = pLongNames->stTableEntry; - - for ( ulI = 0; ulI < pLongNames->ulEntries; ulI++ ) - { - pEntry->pszLongName = pLongNames->pszBuffer + - (pEntry->pszLongName - pszOldBuffer); - pEntry++; - } /* endfor */ - } - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - - // add new entry to buffer - if ( usRc == NO_ERROR ) - { - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to table entry - PSZ pszTarget; // ptr into buffer - - // position to table entry - pEntry = pLongNames->stTableEntry + - pLongNames->ulEntries; - - // position to free area in buffer (overwrite end delimiter!) - pszTarget = pLongNames->pszBuffer + - (pLongNames->ulBufUsed - sizeof(USHORT)); - - // add ID to buffer and table - *((PUSHORT)pszTarget) = *pusID; - pszTarget += sizeof(USHORT); - pEntry->usId = *pusID; - - // add long name to buffer and table - strcpy( pszTarget, pszLongName ); - pEntry->pszLongName = pszTarget; - pszTarget += ulNameLen; - - // add new end delimiter - *((PUSHORT)pszTarget) = 0; // end delimiter - - // adjust entry count and buffer used size - pLongNames->ulEntries++; - pLongNames->ulBufUsed += ulAddLen; - } /* endif */ - - // sort long name array - if ( usRc == NO_ERROR ) - { - qsort( pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - } /* endif */ - - // copy to case ignore table and sort it - if ( usRc == NO_ERROR ) - { - memcpy( pLongNamesCaseIgnore->stTableEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries * - sizeof(TMX_LONGNAME_TABLE_ENTRY) ); - - qsort( pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); - } /* endif */ - - // update TM record for long names - if ( usRc == NO_ERROR ) - { - DEBUGEVENT2( NTMGETIDFROMNAME_LOC, INFO_EVENT, 2, TM_GROUP, pszLongName ); - - usRc = TmBtree.EQFNTMUpdate( LONGNAME_KEY, - (PBYTE)pLongNames->pszBuffer, - pLongNames->ulBufUsed ); + usRc = NTMAddLangGroup(pszName, *pusID); } /* endif */ } /* endif */ } /* endif */ - } - else - { - if ( usRc == NO_ERROR ) - { - /********************************************************************/ - /* search name passed in pszName in passed table usTableType */ - /********************************************************************/ - pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch( pszName, - pstTMTableEntries, - pstTMTable->ulMaxEntries, - sizeof(TMX_TABLE_ENTRY), - NTMCompNames ); - //--- if passed name found in table - if ( pstTMTableEntry ) - { - //-- get ID of entry and set output parameter - *pusID = pstTMTableEntry->usId; - } - else - { - // name is not contained in name table - if ( !(lOptions & NTMGETID_NOUPDATE_OPT) ) - { - usRc = NTMAddNameToTable( pszName, usTableType, - pusID ); - - // for new languages update our language group table - if ( (usRc == NO_ERROR) && - (usTableType == LANG_KEY) ) - { - usRc = NTMAddLangGroup( pszName, *pusID ); - } /* endif */ - } /* endif */ - } /* endif */ - } /* endif */ } /* endif */ - - // file name table only: - // look for given short name in long name table if not found yet or - // an alternative file ID is requested - if ( (usRc == NO_ERROR) && (usTableType == FILE_KEY) && !fLongName && - ((*pusID == NTMGETID_NOTFOUND_ID) || (pusAlternativeID != NULL)) ) - { - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - - // prepare search entry - SearchEntry.pszLongName = pszName; - - // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - - if ( pEntry != NULL ) - { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); - } /* endif */ - - if ( pEntry != NULL ) - { - // return ID of found entry - if ( *pusID == NTMGETID_NOTFOUND_ID ) - { - *pusID = pEntry->usId; - } - else if ( pusAlternativeID != NULL ) - { - *pusAlternativeID = pEntry->usId; - } /* endif */ - } - } /* endif */ - } - else //--- pTmClb is NULL pointer or pszName is empty - { - //--- wrong function paramters - LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); - } /* endif */ + } /* endif */ + } + else //--- pTmClb is NULL pointer or pszName is empty + { + //--- wrong function paramters + LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); } /* endif */ return usRc; -} /* end of function NTMGetIDFromNameEx */ - +} /* end of function NTMGetIDFromName */ //+----------------------------------------------------------------------------+ //|External function | @@ -608,171 +412,97 @@ USHORT EqfMemory::NTMGetIDFromNameEx //| | //| return usRc; | // ----------------------------------------------------------------------------+ -USHORT -EqfMemory::NTMGetNameFromID( PUSHORT pusID, //intput - USHORT usTableType, //input - PSZ pszName, //output - PSZ pszLongName ) //output, long name (only for FILE_KEY) - +USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput + USHORT usTableType, // input + PSZ pszName, // output + SHORT maxBuffLen, bool fSuppressLogsAndResetValueIfNotFound) // output, long name (only for FILE_KEY) { - USHORT usRc = NO_ERROR; - BOOL fFound = FALSE; - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - - - if ( pszLongName != NULL ) + USHORT usRc = NO_ERROR; + BOOL fFound = FALSE; + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure + if (pszName != NULL) { - pszLongName[0] = EOS; + pszName[0] = EOS; } /* endif */ - - //--- if input parameters OK - //if ( pTmClb ) - { - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); - /****************************************************************/ - /* search ID in passed table usTableTye and get name of ID */ - /****************************************************************/ - if ( usRc == NO_ERROR ) - { - if ( *pusID == OVERFLOW_ID ) - { - strcpy( pszName, OVERFLOW_NAME ); - fFound = TRUE; - } - else - { - ULONG ulI; - for ( ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++ ) - { - if ( *pusID == pstTMTableEntries[ulI].usId ) - { - fFound = TRUE; - strcpy( pszName, pstTMTableEntries[ulI].szName ); - } /* endif */ - } /* endfor */ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - if ( fFound && (usTableType == FILE_KEY) && (pszLongName != NULL) ) + + if(usID){ + if(usTableType == FILE_KEY){ + /**************************************************************/ + /* Get any long document name for this ID */ + /**************************************************************/ + for (ULONG ulI = 0; + ulI < state->LongNames.stTableEntry.size() && !fFound; + ulI++){ + if (usID == state->LongNames.stTableEntry[ulI].usId) { - BOOL fLongFound = FALSE; - for ( ulI = 0; - ulI < pLongNames->ulEntries && !fLongFound; - ulI++ ) - { - if ( *pusID == pLongNames->stTableEntry[ulI].usId ) - { - fLongFound = TRUE; - strcpy( pszLongName, - pLongNames->stTableEntry[ulI].pszLongName ); - } /* endif */ - } /* endfor */ - - if(!fLongFound){ - strcpy(pszLongName, pszName); + fFound = TRUE; + if(maxBuffLen>0){ + strncpy(pszName, state->LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); + pszName[maxBuffLen-1] = '\0'; + }else{ + strcpy(pszName, + state->LongNames.stTableEntry[ulI].pszLongName); } } /* endif */ - - - if ( !fFound ) - { - /************************************************************/ - /* the ID was not found in the table */ - /* set usRc and reset pszName */ - /************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, ID_NOT_FOUND); - pszName[0] = EOS; - } /* endif */ - } /* endif */ - } /* endif */ - } - return usRc; -} /* end of function NTMGetNameFromID */ - - -PSZ EqfMemory::NTMFindNameForID( - PUSHORT pusID, //intput - USHORT usTableType ) // input - -{ - USHORT usRc = NO_ERROR; - BOOL fFound = FALSE; - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - PSZ pszFoundName = NULL; - - //--- if input parameters OK - //if ( pTmClb ) - { - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, + } /* endfor */ + } else{//not file_key + /******************************************************************/ + /* get pointer to table and table entries in dependency of the */ + /* table type */ + /******************************************************************/ + usRc = NTMGetPointersToTable(usTableType, &pstTMTable, - &pstTMTableEntries ); - /****************************************************************/ - /* search ID in passed table usTableTye and get name of ID */ - /****************************************************************/ - if ( usRc == NO_ERROR ) - { - if ( *pusID == OVERFLOW_ID ) - { - pszFoundName = OVERFLOW_NAME; - fFound = TRUE; - } - else + &pstTMTableEntries); + /****************************************************************/ + /* search ID in passed table usTableTye and get name of ID */ + /****************************************************************/ + if (usRc == NO_ERROR) { - ULONG ulI; - for ( ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++ ) + if ( usID == OVERFLOW_ID) { - if ( *pusID == pstTMTableEntries[ulI].usId ) - { - fFound = TRUE; - pszFoundName = pstTMTableEntries[ulI].szName; - } /* endif */ - } /* endfor */ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - if ( fFound && (usTableType == FILE_KEY) ) + strcpy(pszName, OVERFLOW_NAME); + fFound = TRUE; + } + else { - BOOL fLongFound = FALSE; - for ( ulI = 0; - ulI < pLongNames->ulEntries && !fLongFound; - ulI++ ) + for (ULONG ulI = 0; ulI < pstTMTable->table.size() && !fFound; ulI++) { - if ( *pusID == pLongNames->stTableEntry[ulI].usId ) + if (usID == pstTMTableEntries[ulI].usId) { - fLongFound = TRUE; - pszFoundName = pLongNames->stTableEntry[ulI].pszLongName; + fFound = TRUE; + constexpr int sizeOfField = MAX_LANG_LENGTH ; + strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); + pszName[sizeOfField-1] = '\0'; } /* endif */ } /* endfor */ - } /* endif */ + } + } + } + } + if (!fFound) + { + /************************************************************/ + /* the ID was not found in the table */ + /* set usRc and reset pszName */ + /************************************************************/ + if(usID){ + if(fSuppressLogsAndResetValueIfNotFound){ + usRc = ID_NOT_FOUND; + pszName[0] = EOS; + }else{ + LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID << "; table = " << usTableType; + snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short + } + }else{ + pszName[0] = EOS; + } + } /* endif */ - if ( !fFound ) - { - /************************************************************/ - /* the ID was not found in the table */ - /* set usRc and reset pszName */ - /************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, ID_NOT_FOUND); - pszFoundName = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - } - return( pszFoundName ); -} /* end of function NTMFindNameForID */ + return usRc; +} /* end of function NTMGetNameFromID */ //+----------------------------------------------------------------------------+ @@ -814,15 +544,15 @@ PSZ EqfMemory::NTMFindNameForID( //| compare the names of the passed tabele entries using strcmp | //| return compare result | // ----------------------------------------------------------------------------+ -int -NTMCompNames( const void * pstTMTableEntry1, //input - const void * pstTMTableEntry2 ) //input +int NTMCompNames(const void *pstTMTableEntry1, // input + const void *pstTMTableEntry2) // input { /********************************************************************/ /* compare the names of the passed tabele entries using strcmp */ /********************************************************************/ - return( strcmp( ((PTMX_TABLE_ENTRY) pstTMTableEntry1)->szName, - ((PTMX_TABLE_ENTRY) pstTMTableEntry2)->szName )); + const char* l = ((PTMX_TABLE_ENTRY)pstTMTableEntry1)->szName; + const char* r = ((PTMX_TABLE_ENTRY)pstTMTableEntry2)->szName; + return strcmp(l,r); } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ @@ -870,244 +600,176 @@ NTMCompNames( const void * pstTMTableEntry1, //input //| return usRc | // ----------------------------------------------------------------------------+ USHORT -EqfMemory::NTMGetPointersToTable( - USHORT usTableType, //input - PTMX_TABLE *ppstTMTable, //output - PTMX_TABLE_ENTRY *ppstTMTableEntries ) //output +EqfMemory::NTMGetPointersToTable( + USHORT usTableType, // input + PTMX_TABLE *ppstTMTable, // output + PTMX_TABLE_ENTRY *ppstTMTableEntries) // output { - USHORT usRc = NO_ERROR; //function return code - + USHORT usRc = NO_ERROR; // function return code + /********************************************************************/ /* get pointer to table in dependency of passed usTableType */ /********************************************************************/ - switch ( usTableType ) + switch (usTableType) { - //----------------------------------------------------------------------- - case LANG_KEY : - *ppstTMTable = (PTMX_TABLE)&Languages; - break; - //----------------------------------------------------------------------- - case FILE_KEY : - *ppstTMTable = (PTMX_TABLE)&FileNames; - break; - //----------------------------------------------------------------------- - case AUTHOR_KEY : - *ppstTMTable = (PTMX_TABLE)&Authors; - break; - //----------------------------------------------------------------------- - case TAGTABLE_KEY : - *ppstTMTable = (PTMX_TABLE)&TagTables; - break; - //----------------------------------------------------------------------- - case LANGGROUP_KEY : - *ppstTMTable = (PTMX_TABLE)&LangGroups; - break; - //----------------------------------------------------------------------- - default : - LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); - *ppstTMTable = NULL; - *ppstTMTableEntries = NULL; - break; + //----------------------------------------------------------------------- + case LANG_KEY: + *ppstTMTable = &state->Languages; + break; + //----------------------------------------------------------------------- + case AUTHOR_KEY: + *ppstTMTable = &state->Authors; + break; + //----------------------------------------------------------------------- + case TAGTABLE_KEY: + *ppstTMTable = &state->TagTables; + break; + //----------------------------------------------------------------------- + case LANGGROUP_KEY: + *ppstTMTable = &state->LangGroups; + break; + //----------------------------------------------------------------------- + default: + LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); + *ppstTMTable = NULL; + *ppstTMTableEntries = NULL; + break; } /* end switch */ - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { /******************************************************************/ /* get pointer to table entries */ /******************************************************************/ - *ppstTMTableEntries = &((*ppstTMTable)->stTmTableEntry[0]); + *ppstTMTableEntries = &((*ppstTMTable)->table[0]); } /* endif */ return usRc; } /* end of function NTMGetPointersToTable */ - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ -//|Function name: NTMLockTM | +//|Function name: NTMCreateLongNameTable | //+----------------------------------------------------------------------------+ -//|Function call: NTMLockTM( clb, fLock, pfLocked ); | +//|Function call: usRC = NTMCreateLongnameTable( pTmClb ); | //+----------------------------------------------------------------------------+ -//|Description: Physically lock or unlock the data and the index file | -//| of the given TM | +//|Description: Creates an empty table for long document names. | //+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb, pointer to control block | -//| BOOL fLock TRUE = Lock, FALSE = Unlock | -//| PBOOL pfLocked set to TRUE if TM has been | -//| locked | +//|Parameters: EqfMemory* pTmClb pointer to control block | //+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | +//|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ -USHORT NTMLockTM -( - EqfMemory* pTmClb, // pointer to control block - BOOL fLock, // TRUE = Lock, FALSE = Unlock - PBOOL pfLocked // set to TRUE if TM has been locked -) -{ - USHORT usRc = 0; // function return code - BOOL fLockedData = FALSE; // data-file-has-been-locked flag - BOOL fLockedIndex = FALSE; // index-file-has-been-locked flagf - DEBUGEVENT( NTMLOCKTM_LOC, FUNCENTRY_EVENT, usRc ); - if ( fLock ) - { - // Lock data file of TM - usRc = pTmClb->TmBtree.EQFNTMPhysLock( TRUE, &fLockedData ); +USHORT EqfMemory::SortLongNameTables() +{ + // sort long name array + if( state->LongNames.stTableEntry.size() > 1){ + qsort(state->LongNames.stTableEntry.data(), + state->LongNames.stTableEntry.size()-1,//last element is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); + state->LongNames.restoreStrPointersAfterRealloc(); + } - // Lock Index file of TM - if ( !usRc ) - { - usRc = pTmClb->InBtree.EQFNTMPhysLock( TRUE, &fLockedIndex ); - } /* endif */ - - // Set caller's lock flag - *pfLocked = fLockedData && fLockedIndex; + if( state->LongNamesCaseIgnore.stTableEntry.size() > 1){ + qsort(state->LongNamesCaseIgnore.stTableEntry.data(), + state->LongNamesCaseIgnore.stTableEntry.size()-1,//last element is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); + state->LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); + } + return 0; +} - // Unlock data file if lock of index failed - if ( !fLockedIndex && fLockedData ) - { - usRc = pTmClb->TmBtree.EQFNTMPhysLock( FALSE, &fLockedData ); - } /* endif */ - } - else - { - // Rewrite compact area if compact area has been changed - pTmClb->RewriteCompactTable(); +void appendLongNameEntry( + TMX_LONGNAME_TABLE& table, + USHORT id, + const std::vector& nameData, + PSZ positionInBuffer = nullptr + ) +{ + TMX_LONGNAME_TABLE_ENTRY entry; + entry.usId = id; - // Unlock index file of TM - usRc = pTmClb->InBtree.EQFNTMPhysLock( FALSE, &fLockedIndex ); + if(table.pszBuffer.size() < 4){//if in buffer we have only bytes for one id and one delimiter- then erase buff before adding new data + table.pszBuffer.clear(); + } - // Unlock data file of TM - usRc = pTmClb->TmBtree.EQFNTMPhysLock( FALSE, &fLockedData ); - } /* endif */ + //table.pszBuffer.push_back(id); + if(nullptr == positionInBuffer){//we need to add data to the buffer + table.pszBuffer.push_back(static_cast(id & 0xFF)); // low byte + table.pszBuffer.push_back(static_cast((id >> 8) & 0xFF)); // high byte + entry.longNameStartOffset = table.pszBuffer.size(); + + table.pszBuffer.insert(table.pszBuffer.end(), nameData.begin(), nameData.end()); + table.pszBuffer.push_back(0); // null-terminator + //table.pszBuffer.push_back(0); + }else{// we have data in the buffer, + entry.longNameStartOffset = positionInBuffer - (PSZ)table.pszBuffer.data(); + } - if ( usRc != NO_ERROR ) - { - ERREVENT( NTMLOCKTM_LOC, ERROR_EVENT, usRc ); - } /* endif */ + entry.pszLongName = reinterpret_cast(table.pszBuffer.data() + entry.longNameStartOffset); + table.stTableEntry.back() = entry; - return( usRc ); -} /* end of function NTMLockTM */ + table.stTableEntry.emplace_back(); // prepare next slot + table.restoreStrPointersAfterRealloc(); +} -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMCreateLongNameTable | -//+----------------------------------------------------------------------------+ -//|Function call: usRC = NTMCreateLongnameTable( pTmClb ); | -//+----------------------------------------------------------------------------+ -//|Description: Creates an empty table for long document names. | -//+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb pointer to control block | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT error return code or NO_ERROR if O.K. | -// ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMCreateLongNameTable() +USHORT EqfMemory::WriteLongNameTablesToTheTM() { - USHORT usRC = NO_ERROR; // function return code - - // allocate initial long name pointer array - if ( UtlAlloc( (PVOID *)&(pLongNames), 0L, (ULONG) - (sizeof(TMX_LONGNAMETABLE) + - sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES), - NOMSG ) ) + //if(getMajorVersion() < 7)//for compatibility { - pLongNames->ulTableSize = LONGNAMETABLE_ENTRIES; - pLongNames->ulEntries = 0; + state->LongNames.pszBuffer.push_back(0); + state->LongNames.restoreStrPointersAfterRealloc(); } - else + USHORT usRc = state->TmBtree.EQFNTMUpdate(LONGNAME_KEY, + state->LongNames.pszBuffer); + + //if(getMajorVersion() < 7)//for compatibility { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // allocate initial long name pointer array for case ignore search - if ( UtlAlloc( (PVOID *)&(pLongNamesCaseIgnore), 0L, (ULONG) - (sizeof(TMX_LONGNAMETABLE) + - sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES), - NOMSG ) ) - { - pLongNamesCaseIgnore->ulTableSize = LONGNAMETABLE_ENTRIES; - pLongNamesCaseIgnore->ulEntries = 0; + state->LongNames.pszBuffer.pop_back(); } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // allocate initial long name buffer area - if ( usRC == NO_ERROR ) - { - if ( UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), 0L, - (ULONG) LONGNAMEBUFFER_SIZE, NOMSG ) ) - { - pLongNames->ulBufSize = (ULONG)LONGNAMEBUFFER_SIZE; - pLongNames->ulBufUsed = (ULONG)sizeof(USHORT); // end-of-table delimiter - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - // cleanup in case of errors - if ( usRC != NO_ERROR ) - { - NTMDestroyLongNameTable( ); - } /* endif */ - - // return to caller - return( usRC ); -} /* end of function NTMCreateLongNameTable */ - + return usRc; +} -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMDestroyLongNameTable | -//+----------------------------------------------------------------------------+ -//|Function call: usRC = NTMDestroyLongnameTable(); | -//+----------------------------------------------------------------------------+ -//|Description: Destroys a long name table and frees the memory occupied | -//| by it. | -//+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb pointer to control block | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT error return code or NO_ERROR if O.K. | -// ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMDestroyLongNameTable() +USHORT EqfMemory::AddToLongNameTables(const char* pszLongName, PUSHORT pusID, bool fGenerateId, PSZ positionInReadBuffer) { - USHORT usRC = NO_ERROR; // function return code - - // if there is a long name table ... - if ( pLongNames != NULL ) + if(state->LongNames.stTableEntry.empty()) { - // free any buffer area - if ( pLongNames->pszBuffer != NULL ) - { - UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), 0L, 0L, NOMSG ); - } /* endif */ + state->LongNames.pszBuffer.clear(); + state->LongNames.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + if(state->LongNamesCaseIgnore.stTableEntry.empty()) + { + state->LongNamesCaseIgnore.pszBuffer.clear(); + state->LongNamesCaseIgnore.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + if(fGenerateId){ + *pusID = state->LongNames.stTableEntry.size(); + } + std::vector longName(reinterpret_cast(pszLongName), + reinterpret_cast(pszLongName + strlen(pszLongName))); + auto longNameCaseIgnore = toUpperVector(longName); - // free table - UtlAlloc( (PVOID *)&(pLongNames), 0L, 0L, NOMSG ); - pLongNames = NULL; - } /* endif */ + appendLongNameEntry(state->LongNames, *pusID, longName, positionInReadBuffer); + appendLongNameEntry(state->LongNamesCaseIgnore, *pusID, longNameCaseIgnore); - if ( pLongNamesCaseIgnore != NULL ) - { - // free table - UtlAlloc( (PVOID *)&(pLongNamesCaseIgnore), 0L, 0L, NOMSG ); - pLongNamesCaseIgnore = NULL; - } /* endif */ + SortLongNameTables(); + WriteLongNameTablesToTheTM(); + return 0; +} - // return to caller - return( usRC ); -} /* end of function NTMDestroyLongNameTable */ +bool endsWithTripleNull(const std::vector& buffer) { + return buffer.size() >= 3 && + buffer[buffer.size() - 3] == 0 && + buffer[buffer.size() - 2] == 0 && + buffer[buffer.size() - 1] == 0; +} //+----------------------------------------------------------------------------+ //|External function | @@ -1118,201 +780,94 @@ USHORT EqfMemory::NTMDestroyLongNameTable() //+----------------------------------------------------------------------------+ //|Description: Reads the data of a long name table from the database. | //+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb pointer to control block | +//|Parameters: EqfMemory* pTmClb pointer to control block | //+----------------------------------------------------------------------------+ //|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMReadLongNameTable(){ - USHORT usRC = NO_ERROR; // function return code - ULONG ulLen = 0; // record length +DECLARE_bool(skip_0s_at_the_table_start); +USHORT EqfMemory::NTMReadLongNameTable() +{ + if(state->LongNames.stTableEntry.empty()) + { + state->LongNames.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); + state->LongNames.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); + state->LongNames.pszBuffer.clear(); + state->LongNames.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + if(state->LongNamesCaseIgnore.stTableEntry.empty()) + { + state->LongNamesCaseIgnore.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); + state->LongNamesCaseIgnore.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); + state->LongNamesCaseIgnore.pszBuffer.clear(); + state->LongNamesCaseIgnore.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } // call to obtain exact length of record - ulLen = 0; - usRC = TmBtree.EQFNTMGet( LONGNAME_KEY, 0, &ulLen ); - - if ( usRC == NO_ERROR ) + LONG lKey = LONGNAME_KEY; + USHORT usRC = state->TmBtree.EQFNTMGet(lKey, state->LongNames.pszBuffer); + + // setup pointer array for long names + if (usRC == NO_ERROR) { - // allocate buffer area if it is too small - if ( pLongNames->ulBufSize < ulLen ) - { - if ( UtlAlloc( (PVOID *)&pLongNames->pszBuffer, 0L, - ulLen, NOMSG )) - { - pLongNames->ulBufSize = ulLen; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + // fill pointer array + PSZ pszTemp = (PSZ)state->LongNames.pszBuffer.data(); - // read long name table from database - if ( usRC == NO_ERROR ) + if(endsWithTripleNull(state->LongNames.pszBuffer)); { - usRC = TmBtree.EQFNTMGet( LONGNAME_KEY, - (PCHAR)pLongNames->pszBuffer, &ulLen ); - } /* endif */ - - - // unterse data if table data is tersed - if ( usRC == NO_ERROR ) - { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)pLongNames->pszBuffer; - PBYTE pNewArea = NULL; // ptr to unterse data area - - if ( pTerseHeader->ulMagicWord == TERSEMAGICWORD ) - { - // table is tersed... - - // allocate buffer for untersed data - if ( !UtlAlloc( (PVOID *)&pNewArea, 0L, - (LONG)pTerseHeader->usDataSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // unterse data - if ( usRC == NO_ERROR ) - { - ULONG ulNewLen = 0; - - memcpy( pNewArea, - (PBYTE)pLongNames->pszBuffer + sizeof(TERSEHEADER), - ulLen - sizeof(TERSEHEADER) ); - T5LOG(T5ERROR) << "::TEMPORARY_COMMENTED in NTMReadLongNameTable, fUtlHuffmanExpand"; -#ifdef TEMPORARY_COMMENTED - if ( !fUtlHuffmanExpand( (PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage - } /* endif */ - #endif - } /* endif */ - - // set table data and cleanup - if ( usRC == NO_ERROR ) - { - // set size of buffer area - pLongNames->ulBufSize = (ULONG)pTerseHeader->usDataSize; - - // free tersed data area - UtlAlloc( (PVOID *)&pLongNames->pszBuffer, 0L, 0L, NOMSG ); - - // anchor untersed data area - pLongNames->pszBuffer = (PSZ)pNewArea; - - // avoid free of untersed data area - pNewArea = NULL; - } /* endif */ - if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); - } /* endif */ - } /* endif */ - - // setup pointer array for long names - if ( usRC == NO_ERROR ) + state->LongNames.pszBuffer.pop_back(); + } + ssize_t buffLen = state->LongNames.pszBuffer.size(); + + if(FLAGS_skip_0s_at_the_table_start){ + while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen-1) && *(pszTemp) == 0){//skip 0s at the start of the table; + pszTemp++; + } + auto lenOf0s = pszTemp - (PSZ)state->LongNames.pszBuffer.data(); + if (lenOf0s > 0) {//erase 0s at the start + state->LongNames.pszBuffer.erase( + state->LongNames.pszBuffer.begin(), + state->LongNames.pszBuffer.begin() + lenOf0s + ); + } + } + + buffLen = state->LongNames.pszBuffer.size() - sizeof(USHORT); //we need to have more than 3 bytes just to parse ID and endDelimiter + pszTemp = (PSZ)state->LongNames.pszBuffer.data(); + auto startOfBuff = (PSZ)state->LongNames.pszBuffer.data(); + while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) { - ULONG ulEntries = 0; // number of entries in buffer - PSZ pszTemp; // ptr for buffer processing - - // remember used space in buffer area - pLongNames->ulBufUsed = ulLen; - - // count number of entries in long name buffer - pszTemp = pLongNames->pszBuffer; - while ( *((PUSHORT)pszTemp) != 0 ) - { - ulEntries++; - pszTemp += sizeof(USHORT); // skip ID - pszTemp += strlen(pszTemp)+1;// skip long name - } /* endwhile */ - - // enlarge pointer array if necessary - if ( pLongNames->ulTableSize < ulEntries ) - { - ULONG ulOldSize = sizeof(TMX_LONGNAMETABLE) + - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * - pLongNames->ulTableSize); - ULONG ulNewSize = sizeof(TMX_LONGNAMETABLE) + - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * ulEntries); - if ( UtlAlloc( (PVOID *)&pLongNames, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNames->ulTableSize = ulEntries; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // enlarge pointer array for case ignore search as well - if ( usRC == NO_ERROR ) - { - if ( UtlAlloc( (PVOID *)&pLongNamesCaseIgnore, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNamesCaseIgnore->ulTableSize = ulEntries; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - - // fill pointer array - if ( usRC == NO_ERROR ) - { - PTMX_LONGNAME_TABLE_ENTRY pEntry = pLongNames->stTableEntry; - pLongNames->ulEntries = 0; - pszTemp = pLongNames->pszBuffer; - while ( *((PUSHORT)pszTemp) != 0 ) - { - - // get ID of string - pEntry->usId = *((PUSHORT)pszTemp); - pszTemp += sizeof(USHORT); // skip ID - - // set pointer to long document name - pEntry->pszLongName = pszTemp; - - // continue with next entry - pEntry++; - pLongNames->ulEntries++; - pszTemp += strlen(pszTemp)+1;// skip long name - } /* endwhile */ - } /* endif */ - - // sort long name array - if ( usRC == NO_ERROR ) - { - qsort( pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - } /* endif */ - - // make copy of long name array for case ignore search - if ( usRC == NO_ERROR ) - { - memcpy( pLongNamesCaseIgnore->stTableEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries * - sizeof(TMX_LONGNAME_TABLE_ENTRY) ); - - qsort( pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); - } /* endif */ - - } /* endif */ + // get ID of string + USHORT usId = *((PUSHORT)pszTemp); + pszTemp += sizeof(USHORT); // skip ID + auto len = strlen(pszTemp); + AddToLongNameTables(pszTemp, &usId, false, pszTemp); + + pszTemp += len; + pszTemp += 1; // skip delimiter + if(!*pszTemp){// skip delimiter in tms from 0.7.0-0.7.22 + pszTemp++; + } + } /* endwhile */ + SortLongNameTables(); } /* endif */ - // return to caller - return( usRC ); + return (usRC); } /* end of function NTMReadLongNameTable */ +int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff){ + buff.clear(); + int occupSize = sizeof(LONG) + (pTable->table.size() + 1) * sizeof(TMX_TABLE_ENTRY); + buff.resize(occupSize); + TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)buff.data(); + //pOldTable = pstTMTable->table; + table->ulMaxEntries = pTable->table.size();//pTable->ulMaxEntries; + if (table->ulMaxEntries) + { + memcpy(&table->table, + &pTable->table[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); + } + return 0; +} //+----------------------------------------------------------------------------+ //|External function | @@ -1349,163 +904,89 @@ USHORT EqfMemory::NTMReadLongNameTable(){ //| ERROR_NOT_ENOUGH_MEMORY - reallocation of table failed | //| others - return codes from QDAM | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMAddNameToTable -( - PSZ pszName, // input - USHORT usTableType, //input - PUSHORT pusID //output +USHORT EqfMemory::NTMAddNameToTable( + PSZ pszName, // input + USHORT usTableType, // input + PUSHORT pusID, // output + bool fUseProvidedId ) { - USHORT usRc = NO_ERROR; //function return coed - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - ULONG ulNewSize; //for size calculation - ULONG ulReallocSize; //size for realloc of the table + USHORT usRc = NO_ERROR; // function return coed + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure + ULONG ulNewSize; // for size calculation + ULONG ulReallocSize; // size for realloc of the table /********************************************************************/ /* initialize ID, that in error case a 0 - ID is returned */ /********************************************************************/ - *pusID = 0; + if(!fUseProvidedId){ + *pusID = 0; + } //--- if input parameters OK - if ( pszName[0] != EOS ) + if (pszName[0] != EOS) { /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); - - if ( usRc == NO_ERROR ) - { - /*************************************************************/ - /* Check if space is available in table so that new entry */ - /* will fit into the table. */ - /* Check if the allocated size can hold the new entry. */ - /* If not reallocate the table and check that the table */ - /* will not exceed the maximum size of a QDAM record (32K) */ - /*************************************************************/ - #ifdef TEMPORARY_COMMENTED - ulNewSize = (ULONG)sizeof(TMX_TABLE) + - (pstTMTable->ulMaxEntries * (ULONG)sizeof(TMX_TABLE_ENTRY) ); - if ( ulNewSize >= pstTMTable->ulAllocSize ) + if(usTableType == FILE_KEY || usTableType == LONGNAME_KEY){ + AddToLongNameTables(pszName, pusID, !fUseProvidedId);// !fUseProvidedId=> generateNewId + }else{ + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); + if (usRc == NO_ERROR) { - ulReallocSize = (ULONG)pstTMTable->ulAllocSize + (ULONG)TMX_TABLE_SIZE; - { - if ( UtlAlloc( (PVOID *) &pstTMTable, - (LONG)pstTMTable->ulAllocSize, - (LONG)ulReallocSize, - NOMSG ) ) - { - /*****************************************************/ - /* no error from UtlAlloc */ - /* reset the pointer in TM_CLB because UtlAlloc */ - /* reallocates the storage on another place */ - /*****************************************************/ - switch ( usTableType ) - { - //--------------------------------------------------------- - case LANG_KEY : - pTmClb->Languages = *pstTMTable; - pstTMTableEntries = pTmClb->Languages.stTmTableEntry; - break; - //--------------------------------------------------------- - case FILE_KEY : - pTmClb->FileNames = *pstTMTable; - pstTMTableEntries = pTmClb->FileNames.stTmTableEntry; - break; - //--------------------------------------------------------- - case AUTHOR_KEY : - pTmClb->Authors = *pstTMTable; - pstTMTableEntries = pTmClb->Authors.stTmTableEntry; - break; - //--------------------------------------------------------- - case TAGTABLE_KEY : - pTmClb->TagTables = *pstTMTable; - pstTMTableEntries = pTmClb->TagTables.stTmTableEntry; - break; - //--------------------------------------------------------- - case LANGGROUP_KEY : - pTmClb->LangGroups = *pstTMTable; - pstTMTableEntries = pTmClb->LangGroups.stTmTableEntry; - break; - } /* end switch */ - //pstTMTable->ulAllocSize = ulReallocSize; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - } /* endif */ - #endif - //if( (pstTMTable->ulMaxEntries+1) * sizeof(pstTMTableEntries[0]) + sizeof(ULONG) > TMX_REC_SIZE){ - // T5LOG(T5ERROR) << ":: cannot increase table size, it reached it's limit ; maxEntries="<< pstTMTable->ulMaxEntries << "; tagTableType=" << usTableType; - //usRc = ERROR_NOT_ENOUGH_MEMORY; - //} - const size_t currentCap = pstTMTable->stTmTableEntry.size(); - if(pstTMTable->ulMaxEntries >= currentCap){ - size_t newCap = currentCap + 10; - if(newCap >= pstTMTable->stTmTableEntry.max_size()){ - T5LOG(T5FATAL) << "tried to set too big capacity for table " << usTableType <<"; newCap = " << newCap; + /*************************************************************/ + /* Check if space is available in table so that new entry */ + /* will fit into the table. */ + /* Check if the allocated size can hold the new entry. */ + /* If not reallocate the table and check that the table */ + /* will not exceed the maximum size of a QDAM record (32K) */ + /*************************************************************/ + if(pstTMTable->table.empty()){// in 0.7.shouldn't be empty, but could be when reading from 0.6 + T5LOG(T5ERROR) << "tried to write to empty tm table #" << usTableType<<"; error was already catched and table was resized to 1"; + pstTMTable->table.push_back(TMX_TABLE_ENTRY{}); } - //T5LOG(T5ERROR)<<"remove this log later: tableType = " << usTableType <<" changing capacity to "<< newCap; - pstTMTable->stTmTableEntry.resize(newCap); - pstTMTableEntries = &pstTMTable->stTmTableEntry[0]; - } - if ( usRc == NO_ERROR ) - { + TMX_TABLE_ENTRY& newEntry = pstTMTable->table.back(); + if(*pusID == 0){ + *pusID = pstTMTable->table.size(); + } + newEntry.usId = *pusID; + strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); + pstTMTable->table.push_back(TMX_TABLE_ENTRY()); + /*************************************************************/ /* insert name and id to table and sort table */ - /*************************************************************/ - int pos = pstTMTable->ulMaxEntries; - *pusID = ++pstTMTable->ulMaxEntries; - strncpy( pstTMTableEntries[pos].szName, - pszName, MAX_LANG_LENGTH-1 ); - pstTMTableEntries[pos].usId = *pusID; - qsort( pstTMTableEntries, - pstTMTable->ulMaxEntries, - sizeof(TMX_TABLE_ENTRY), - NTMCompNames ); - - // update table record in TM QDAM file (if not in read-only mode) - if ( !(usAccessMode & ASD_READONLY) ) + /*************************************************************/ + if( pstTMTable->table.size() > 1){ + qsort(pstTMTable->table.data(), + pstTMTable->table.size()-1,//keep last entry empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } + + // update table record in TM QDAM file + if (usTableType != LANGGROUP_KEY) { - if ( usTableType != LANGGROUP_KEY ) - { - int occupSize = (pstTMTable->ulMaxEntries+1) * sizeof(TMX_TABLE_ENTRY); - PBYTE pOldTable = new BYTE[occupSize]; - TMX_TABLE_OLD* table = (TMX_TABLE_OLD*) pOldTable; - table->ulMaxEntries = pstTMTable->ulMaxEntries; - if(table->ulMaxEntries){ - memcpy( &table->stTmTableEntry, - &pstTMTable->stTmTableEntry[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); - } - usRc = TmBtree.EQFNTMUpdate( - (ULONG)usTableType, - //(PBYTE)pstTMTable, - (PBYTE)table, - //pstTMTable->ulAllocSize - //BTREE_REC_SIZE_V3 - occupSize - ); - delete[] pOldTable; - } /* endif */ + std::vector pOldTable; + tmxTableToBuffer(pstTMTable, pOldTable); + usRc = state->TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); + //(PBYTE)table, occupSize); } /* endif */ } /* endif */ - } /* endif */ + } } - else //--- pTmClb is NULL pointer or pszName is empty + else //--- pTmClb is NULL pointer or pszName is empty { //--- wrong function paramters LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); } /* endif */ - if ( usRc == ERROR_TABLE_FULL ) + if (usRc == ERROR_TABLE_FULL) { usRc = 0; *pusID = OVERFLOW_ID; @@ -1514,7 +995,6 @@ USHORT EqfMemory::NTMAddNameToTable return usRc; } /* end of function NTMAddNameToTable */ - //+----------------------------------------------------------------------------+ //|Internal function | //+----------------------------------------------------------------------------+ @@ -1524,134 +1004,46 @@ USHORT EqfMemory::NTMAddNameToTable //| of the passed table entries are compared and the | //| compare result is returned. | // ----------------------------------------------------------------------------+ -int NTMLongNameTableComp -( - const void * pEntry1, //input - const void * pEntry2 //input +int NTMLongNameTableComp( + const void *pEntry1, // input + const void *pEntry2 // input ) { /********************************************************************/ /* compare the long names of the passed table entries using strcmp */ /********************************************************************/ - PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; - PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; - return( strcmp( lname1, lname2 ) ); + auto entry1 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry1; + auto entry2 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry2; + return (strcmp(entry1->pszLongName, entry2->pszLongName)); } /* end of function NTMCompNames */ -int NTMLongNameTableCompCaseIgnore -( - const void * pEntry1, //input - const void * pEntry2 //input +int NTMLongNameTableCompCaseIgnore( + const void *pEntry1, // input + const void *pEntry2 // input ) { - /********************************************************************/ - /* compare the long names of the passed table entries using strcmp */ - /********************************************************************/ - //return( _stricmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - // ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); - return( strcmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); + auto entry1 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry1; + auto entry2 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry2; + return strcasecmp(entry1->pszLongName,entry2->pszLongName);//string should be in lower case here } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMLoadNameTable | -//+----------------------------------------------------------------------------+ -//|Description: This function loads a TM name table from the database. | -//| If the table is tersed it is uncompressed. | -// ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMLoadNameTable -( - ULONG ulTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data -) +USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTable) { - USHORT usRc = NO_ERROR; // function return code - - // call to obtain exact length of record - *pulSize = 0; - usRc = TmBtree.EQFNTMGet( ulTableKey, 0, pulSize ); - - PBYTE pOldTable = nullptr; - // read table data - if ( usRc == NO_ERROR ) - { - ULONG ulLen = *pulSize; - pOldTable = new BYTE[ulLen + 1]; - usRc = TmBtree.EQFNTMGet( ulTableKey, (PCHAR)(pOldTable), &ulLen ); - } /* endif */ - - // handle tersed name tables - if ( usRc == NO_ERROR ) - { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)pOldTable; - PBYTE pNewArea = NULL; // ptr to unterse data area - - if ( pTerseHeader->ulMagicWord == TERSEMAGICWORD ) - { - // table is tersed... - - // allocate buffer for untersed data - if ( !UtlAlloc( (PVOID *)&pNewArea, 0L, - (LONG)pTerseHeader->usDataSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // unterse data - if ( usRc == NO_ERROR ) - { - ULONG ulNewLen = 0; - - memcpy( pNewArea, (PBYTE)pOldTable + sizeof(TERSEHEADER), - *pulSize - sizeof(TERSEHEADER) ); - - T5LOG(T5ERROR) << ":: TEMPORARY_COMMENTED temcom_id = 51 in NTMLoadNameTable"; -#ifdef TEMPORARY_COMMENTED - if ( !fUtlHuffmanExpand( (PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage - } /* endif */ - #endif - } /* endif */ - - // set table data and cleanup - if ( usRc == NO_ERROR ) - { - // set size of name table - *pulSize = pTerseHeader->usDataSize; - - // free tersed data area - //UtlAlloc( (PVOID *)pTMTable, 0L, 0L, NOMSG ); - - // anchor untersed data area - //*pTMTable = pNewArea; - - // avoid free of untersed data area - pNewArea = NULL; - } /* endif */ - if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); - } /* endif */ - } /* endif */ - + USHORT usRc = 0; // special handling for language name table: // correct word delimiter in "Other Languages" which is // 0xA0 under Windows (due to OemToAnsi) but should be 0xFF - if ( (usRc == NO_ERROR) && (ulTableKey == LANG_KEY) ) + if (ulTableKey == LANG_KEY) { - LONG lRest = *pulSize; - PBYTE pbTemp = (PBYTE)pOldTable; - while ( lRest ) + LONG lRest = oldTmxTable.size();; + PBYTE pbTemp = (PBYTE)oldTmxTable.data(); + while (lRest) { - if ( *pbTemp == 0xA0 ) + if (*pbTemp == 0xA0) { *pbTemp = 0xFF; } /* endif */ @@ -1660,38 +1052,64 @@ USHORT EqfMemory::NTMLoadNameTable } /* endwhile */ } /* endif */ - if(usRc == NO_ERROR){ - pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)pOldTable)->ulMaxEntries; - pTMTable->stTmTableEntry.clear(); - pTMTable->stTmTableEntry.resize(pTMTable->ulMaxEntries+1); - if(pTMTable->ulMaxEntries){ - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)pOldTable)->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); + if (usRc == NO_ERROR) + { + pTMTable->table.clear(); + unsigned long numOfEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; + unsigned long numOfEntriesCapacity = oldTmxTable.size()/sizeof(TMX_TABLE_ENTRY); + if(numOfEntries > numOfEntriesCapacity){ + T5LOG(T5ERROR) << "in the saved table num of entries"<< numOfEntries + <<" is greated than how much was saved(tableBytes.size()/sizeof(TMX_TABLE_ENTRY))=" <table.resize(numOfEntries); + if(numOfEntries > 0){ + memcpy(&pTMTable->table[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->table, numOfEntries/*(pTMTable->table.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); } - } - if(pOldTable){ - delete[] pOldTable; - pOldTable = nullptr; + if(pTMTable->table.empty() || (pTMTable->table.back().usId && pTMTable->table.back().szName[0])){ + pTMTable->table.push_back(TMX_TABLE_ENTRY{}); + } } - // return to caller - return( usRc ); -} /* end of function NTMLoadNameTable */ + return usRc; +} -USHORT EqfMemory::NTMAddLangGroup -( - PSZ pszLang, // ptr to language name - USHORT sLangID // ID of language in our tables +//+----------------------------------------------------------------------------+ +//|Internal function | +//+----------------------------------------------------------------------------+ +//|Function name: NTMLoadNameTable | +//+----------------------------------------------------------------------------+ +//|Description: This function loads a TM name table from the database. | +//| If the table is tersed it is uncompressed. | +// ----------------------------------------------------------------------------+ +USHORT EqfMemory::NTMLoadNameTable( + LONG lTableKey, // key of table record + PTMX_TABLE pTMTable // ptr to table data pointer ) { - SHORT sGroupID = 0; - USHORT usRC = NO_ERROR; // function return code - CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]; // buffer for language group name + std::vector pOldTable; + USHORT usRc = state->TmBtree.EQFNTMGet(lTableKey, pOldTable); + + if(!usRc) usRc = bytesToOldTable(lTableKey, pOldTable, pTMTable); + return (usRc); +} /* end of function NTMLoadNameTable */ + +USHORT EqfMemory::NTMAddLangGroup( + PSZ pszLang, // ptr to language name + USHORT sLangID // ID of language in our tables +) +{ + SHORT sGroupID = 0; + USHORT usRC = NO_ERROR; // function return code + CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]{}; // buffer for language group name // get language properties (incl. group name) - if ( GetLanguageGroup( pszLang, szLangGroup ) ) + if (GetLanguageGroup(pszLang, szLangGroup)) { - if ( szLangGroup[0] == EOS ) + if (szLangGroup[0] == EOS) { // no language group for language, so use language ID as // language group ID but multiply it with -1 to distinguish @@ -1704,84 +1122,58 @@ USHORT EqfMemory::NTMAddLangGroup // no language properties for language, so use language ID as // language group ID but multiply it with -1 to distinguish // language group IDs from language IDs - sGroupID = sLangID * -1; + sGroupID = sLangID * -1; } /* endif */ // get ID of group name - if ( (usRC == NO_ERROR) && (sGroupID == 0) ) + if ((usRC == NO_ERROR) && (sGroupID == 0)) { - usRC = NTMGetIDFromName( szLangGroup, NULL, LANGGROUP_KEY, (PUSHORT)&sGroupID ); + usRC = NTMGetIDFromName(szLangGroup, LANGGROUP_KEY, (PUSHORT)&sGroupID); } /* endif */ // enlarge language-ID-to-group-ID table if necessary - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { - LONG lRequiredSize = (sLangID + 1) * sizeof(SHORT); - if ( lLangIdToGroupTableSize < lRequiredSize ) + if (state->psLangIdToGroupTable.size() < sLangID + 1) { - lRequiredSize += (10 * sizeof(SHORT)); - if ( UtlAlloc( (PVOID *)&(psLangIdToGroupTable), - lLangIdToGroupTableSize, - lRequiredSize, NOMSG ) ) - { - lLangIdToGroupTableSize = lRequiredSize; - lLangIdToGroupTableUsed = lRequiredSize; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + state->psLangIdToGroupTable.resize(sLangID + 1); } /* endif */ + // update language ID to group ID table + state->psLangIdToGroupTable[sLangID] = sGroupID; } /* endif */ - // update language ID to group ID table - if ( usRC == NO_ERROR ) - { - psLangIdToGroupTable[sLangID] = sGroupID; - } /* endif */ - return( usRC ); + return (usRC); } /* end of function NTMAddLangGroup */ USHORT EqfMemory::NTMCreateLangGroupTable() { USHORT usRC = NO_ERROR; - //pTmClb->LangGroups.ulAllocSize = TMX_TABLE_SIZE; - // allocate language-ID-to-group-ID-table - - LONG lSize = Languages.ulMaxEntries > 100L ? Languages.ulMaxEntries : 100L; - if(lSize <=0) lSize = 1; - lSize *= sizeof(SHORT); - if( UtlAlloc( (PVOID *)&(psLangIdToGroupTable), - 0L, lSize, NOMSG ) ) - { - lLangIdToGroupTableSize = lSize; - lLangIdToGroupTableUsed = 0L; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + LONG lSize = state->Languages.table.size();// > 100L ? Languages.table.size() : 100L; + if (lSize <= 0) + lSize = 1; + state->psLangIdToGroupTable.resize(lSize); // get group IDs for all languages and fill map table - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { int i = 0; - while ( (usRC == NO_ERROR) && - (i < (int)Languages.ulMaxEntries) ) + while ((usRC == NO_ERROR) && (i < state->Languages.table.size()) + ) { - usRC = NTMAddLangGroup( Languages.stTmTableEntry[i].szName, - Languages.stTmTableEntry[i].usId ); + if(state->Languages.table[i].szName[0]){ + usRC = NTMAddLangGroup(state->Languages.table[i].szName, + state->Languages.table[i].usId); + } i++; } /* endwhile */ } /* endif */ - return( usRC ); + return (usRC); } /* end of function NTMCreateLangGroupTable */ - -// +// // function NTMOrganizeIndexFile // // compact/organize the index part of a memory @@ -1789,236 +1181,224 @@ USHORT EqfMemory::NTMCreateLangGroupTable() USHORT EqfMemory::NTMOrganizeIndexFile() { USHORT usRC = NO_ERROR; - usRC = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRC = EQFNTMOrganizeIndex(&(state->InBtree), state->usAccessMode, START_KEY); - return( usRC ); + return (usRC); } /* end of function NTMOrganizeIndexFile */ /////////////////////////////////////////////////////////////////////////////////////////////// /// functions for working with the additional data area following the target CLB /// /////////////////////////////////////////////////////////////////////////////////////////////// -// compute the size of the additional data for the given input -USHORT NTMComputeAddDataSize( PSZ_W pszContext, PSZ_W pszAddInfo ) +// compute the size of the additional data for the given input +USHORT NTMComputeAddDataSize(PSZ_W pszContext, PSZ_W pszAddInfo) { USHORT usLength = 0; - if ( (pszContext != NULL) && (*pszContext != 0) ) + if ((pszContext != NULL) && (*pszContext != 0)) { // length = characters in context + end delimiter + size field + IF field - usLength = usLength + (USHORT)((wcslen( pszContext) + 2) * sizeof(CHAR_W)); - } /* endif */ + usLength = usLength + (USHORT)((wcslen(pszContext) + 2) * sizeof(CHAR_W)); + } /* endif */ - if ( (pszAddInfo != NULL) && (*pszAddInfo != 0) ) + if ((pszAddInfo != NULL) && (*pszAddInfo != 0)) { // length = characters in add.info + end delimiter + size field + IF field - usLength = usLength + (USHORT)((wcslen( pszAddInfo) + 2) * sizeof(CHAR_W)); - } /* endif */ + usLength = usLength + (USHORT)((wcslen(pszAddInfo) + 2) * sizeof(CHAR_W)); + } /* endif */ - if ( usLength != 0 ) + if (usLength != 0) { // if data is available we need an area end delimiter usLength += sizeof(CHAR_W); - } /* endif */ + } /* endif */ - return( usLength ); + return (usLength); } - // check if the data is old context data or new additional data -BOOL NTMIsAddData( PSZ_W pData ) +BOOL NTMIsAddData(PSZ_W pData) { BOOL fAddData = FALSE; - PUSHORT pDataShort = (PUSHORT) pData; - if ( (*pDataShort == ADDDATA_ADDINFO_ID) || (*pDataShort == ADDDATA_CONTEXT_ID) || (*pDataShort == ADDDATA_ENDOFDATA_ID) ) + PUSHORT pDataShort = (PUSHORT)pData; + if ((*pDataShort == ADDDATA_ADDINFO_ID) || (*pDataShort == ADDDATA_CONTEXT_ID) || (*pDataShort == ADDDATA_ENDOFDATA_ID)) { fAddData = TRUE; - } /* endif */ - return( fAddData ); -} /* end of function NTMIsAddData */ + } /* endif */ + return (fAddData); +} /* end of function NTMIsAddData */ // find specified data area in additional data, returns ptr to start of area or NULL -PSZ_W NTMFindData( PSZ_W pData, USHORT usDataID ) +PSZ_W NTMFindData(PSZ_W pData, USHORT usDataID) { - PUSHORT pDataShort = (PUSHORT) pData; - while ( (*pDataShort != 0) && (*pDataShort != ADDDATA_ENDOFDATA_ID) && (*pDataShort != usDataID) ) + PUSHORT pDataShort = (PUSHORT)pData; + while ((*pDataShort != 0) && (*pDataShort != ADDDATA_ENDOFDATA_ID) && (*pDataShort != usDataID)) { USHORT usLen = pDataShort[1]; pData += (usLen + 1); - pDataShort = (PUSHORT) pData; - } /* endwhile */ + pDataShort = (PUSHORT)pData; + } /* endwhile */ - return( (*pDataShort == usDataID) ? pData : NULL ); -} /* end of function NTMFindData */ + return ((*pDataShort == usDataID) ? pData : NULL); +} /* end of function NTMFindData */ // get length of specific data in the combined data area, returns length of data area in number of CHAR_Ws -USHORT NtmGetAddDataLen( PTMX_TARGET_CLB pCLB, USHORT usDataID ) +USHORT NtmGetAddDataLen(PTMX_TARGET_CLB pCLB, USHORT usDataID) { USHORT usLength = 0; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); - if ( NTMIsAddData( pData ) ) + if (NTMIsAddData(pData)) { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { - PUSHORT pDataPShort = (PUSHORT) pData; + PUSHORT pDataPShort = (PUSHORT)pData; usLength = pDataPShort[1]; - } /* endif */ - } - } /* endif */ + } /* endif */ + } + } /* endif */ - return( usLength ); + return (usLength); } /* end of function NtmGetAddDataLen */ // store/combine additional data in the combined area, returns new size of combined data area or 0 in case of errors -USHORT NtmStoreAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData ) +USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData) { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); PSZ_W pDataStart = pData; - //PSZ_W pNewDataStr = nullptr; + // PSZ_W pNewDataStr = nullptr; PUSHORT pDataShort = (PUSHORT)pData; - USHORT usNewLength = (USHORT)(wcslen( pszNewData )); + USHORT usNewLength = (USHORT)(wcslen(pszNewData)); - if ( usNewLength != 0 ) usNewLength += 1; + if (usNewLength != 0) + usNewLength += 1; - if ( pCLB->usAddDataLen != 0 ) + if (pCLB->usAddDataLen != 0) { // remove any old data for this ID - PSZ_W pOldData = NTMFindData( pData, usDataID ); - if ( pOldData != NULL ) + PSZ_W pOldData = NTMFindData(pData, usDataID); + if (pOldData != NULL) { - PUSHORT pOldDataPShort = (PUSHORT) pOldData; - PSZ_W pEndOfData = (PSZ_W)(((PBYTE)pOldData ) + pCLB->usAddDataLen - 2); - USHORT usOldLen = pOldDataPShort [1]; + PUSHORT pOldDataPShort = (PUSHORT)pOldData; + PSZ_W pEndOfData = (PSZ_W)(((PBYTE)pOldData) + pCLB->usAddDataLen - 2); + USHORT usOldLen = pOldDataPShort[1]; PSZ_W pSource = pOldData + usOldLen + 2; - PSZ_W pTarget = pOldData ; + PSZ_W pTarget = pOldData; - while ( pSource <= pEndOfData ) + while (pSource <= pEndOfData) { *pTarget++ = *pSource++; - } /* endwhile */ + } /* endwhile */ - pCLB->usAddDataLen = pCLB->usAddDataLen - ((usOldLen + 2)*sizeof(PSZ_W)); - } /* endif */ + pCLB->usAddDataLen = pCLB->usAddDataLen - ((usOldLen + 2) * sizeof(PSZ_W)); + } /* endif */ // add new data to end of data area - if ( usNewLength != 0 ) + if (usNewLength != 0) { - PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 1 ); + PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 4); PUSHORT pTargetPShort = (PUSHORT)pTarget; *pTargetPShort++ = usDataID; *pTargetPShort++ = usNewLength; pTarget++; - //pNewDataStr = pTarget; + // pNewDataStr = pTarget; pszNewData[usNewLength] = 0; - //wcsncpy( pTarget, pszNewData, usNewLength); - memcpy( pTarget, pszNewData, sizeof(pData[0])*(usNewLength) ); + // wcsncpy( pTarget, pszNewData, usNewLength); + memcpy(pTarget, pszNewData, sizeof(pData[0]) * (usNewLength)); pTarget += usNewLength; - pTargetPShort = (PUSHORT) pTarget; + pTargetPShort = (PUSHORT)pTarget; *pTargetPShort++ = ADDDATA_ENDOFDATA_ID; *pTargetPShort = 0; - pCLB->usAddDataLen += ((usNewLength + 1 ) * sizeof(pData[0])); - } /* endif */ + pCLB->usAddDataLen += ((usNewLength + 1) * sizeof(pData[0])); + } /* endif */ } - else if ( usNewLength != 0 ) + else if (usNewLength != 0) { // no additional data yet, copy new data *pDataShort++ = usDataID; *pDataShort = usNewLength; pData++; pszNewData[usNewLength] = 0; - //wcsncpy( pData, pszNewData, usNewLength); - memcpy( pData, pszNewData, sizeof(pData[0])*(usNewLength) ); - //pNewDataStr = pData; + // wcsncpy( pData, pszNewData, usNewLength); + memcpy(pData, pszNewData, sizeof(pData[0]) * (usNewLength)); + // pNewDataStr = pData; pData += usNewLength; - pDataShort = (PUSHORT) pData; + pDataShort = (PUSHORT)pData; *pDataShort++ = ADDDATA_ENDOFDATA_ID; *pDataShort = 0; - pCLB->usAddDataLen = (usNewLength + 2) * sizeof(pData[0]);// addinfo_metadana(len+type - 1 wchar) + (usNewLength = strlen + endDelim(1 wchar)) + endAddinfoDelim(1wchar) - } /* endif */ + pCLB->usAddDataLen = (usNewLength + 2) * sizeof(pData[0]); // addinfo_metadana(len+type - 1 wchar) + (usNewLength = strlen + endDelim(1 wchar)) + endAddinfoDelim(1wchar) + } /* endif */ - return( pCLB->usAddDataLen ); + return (pCLB->usAddDataLen); } /* end of function NtmStoreAddData */ // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszBuffer, USHORT usBufSize ) +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, PBYTE endOfMemRegion) { USHORT usLength = 0; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { - PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); + PSZ_W pAddDataStart = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); + PSZ_W pData = pAddDataStart; - if ( NTMIsAddData( pData ) ) // is it additional data? + if (NTMIsAddData(pData)) // is it additional data? { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { PUSHORT pDataPUShort = (PUSHORT)pData; usLength = pDataPUShort[1]; pData++; - if ( usLength < usBufSize ) - { - pszBuffer[0] = 0; - //wcsncpy(pszBuffer, pData, usLength+1 ); - memcpy(pszBuffer, pData, usLength*sizeof(pData[0])); - pszBuffer[usLength+1] = 0; - //wcscpy( pszBuffer, (wchar_t *) pusData ); - } - else - { - usLength = 0; - } /* endif */ - } /* endif */ + wcsncpy(pszBuffer, pData, usLength); + } /* endif */ } else { - T5LOG(T5WARNING) <<"Add info was not saved properly"; - } /* endif */ - } /* endif */ + T5LOG(T5WARNING) << "Add info was not saved properly"; + } /* endif */ + } /* endif */ - return( usLength ); + return (usLength); } /* end of function NtmGetAddData */ - // find a string in a specific data area -BOOL NtmFindInAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch ) +BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch) { - USHORT usLength = 0; BOOL fFound = FALSE; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); - if ( NTMIsAddData( pData ) ) // is it additional data? + if (NTMIsAddData(pData)) // is it additional data? { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { - PUSHORT pDataUShort = (PUSHORT) pData; - usLength = pDataUShort[1]; - if ( usLength > 0 ) + PUSHORT pDataUShort = (PUSHORT)pData; + USHORT usLength = pDataUShort[1]; + if (usLength > 0) { - if ( wcsstr( (pData + 1), pszSearch ) != NULL ) + if (wcsstr((pData + 1), pszSearch) != NULL) { fFound = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ + } /* endif */ + } /* endif */ + } /* endif */ } else { - T5LOG(T5WARNING) <<"Add info was not saved properly"; - } /* endif */ - } /* endif */ + T5LOG(T5WARNING) << "Add info was not saved properly"; + } /* endif */ + } /* endif */ - return( fFound ); + return (fFound); } /* end of function NtmFindInAddData */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 50c73260..a9c7a2de 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -13,8 +13,7 @@ Copyright Notice: #define INCL_EQF_TP #define INCL_EQF_TM #define INCL_EQF_DAM -//#define INCL_EQF_ANALYSIS // analysis functions -//#include // General Translation Manager include file + #include "ThreadingWrapper.h" #include "tm.h" #include "../../core/pluginmanager/PluginManager.h" @@ -24,7 +23,6 @@ Copyright Notice: #include "../../core/utilities/FilesystemHelper.h" #include // Private header file of QDAM -//#include "EQFRDICS.H" // remote dictionary functions #include "LanguageFactory.H" // activate the folllowing define to activate logging @@ -38,8 +36,6 @@ EqfMemory::~EqfMemory() T5LOG(T5TRANSACTION) << "Closing memory " << szName; } UnloadFromRAM(); - //free allocated memory - NTMDestroyLongNameTable(); } /*! \brief Get number of markups used for the proposals in this mmoryProvides a part of the memory in binary format @@ -47,7 +43,7 @@ EqfMemory::~EqfMemory() */ int EqfMemory::getNumOfMarkupNames() { - return( (int)(TagTables.ulMaxEntries) ); + return( (int)(state->TagTables.table.size()) ); } /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] @@ -63,9 +59,9 @@ int EqfMemory::getMarkupName int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(TagTables.ulMaxEntries)) ) + if ( (iPos >= 0) && (iPos < (int)(state->TagTables.table.size())) ) { - return( CopyToBuffer( TagTables.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); + return( CopyToBuffer( state->TagTables.table[iPos].szName, pszBuffer, iSize ) ); } else { @@ -84,7 +80,7 @@ int EqfMemory::getSourceLanguage int iSize ) { - return( CopyToBuffer( stTmSign.szSourceLanguage, pszBuffer, iSize ) ); + return( CopyToBuffer( state->stTmSign.szSourceLanguage, pszBuffer, iSize ) ); } /*! \brief Get description of the memory @@ -98,7 +94,7 @@ int EqfMemory::getDescription int iSize ) { - return( CopyToBuffer(stTmSign.szDescription, pszBuffer, iSize ) ); + return( CopyToBuffer(state->stTmSign.szDescription, pszBuffer, iSize ) ); } /*! \brief Set the description of the memory @@ -112,32 +108,28 @@ void EqfMemory::setDescription //if ( ) { BOOL fOK = TRUE; - PTMX_SIGN pTmSign = new(TMX_SIGN); + TMX_SIGN pTmSign ; // get current signature record USHORT usSignLen = sizeof(TMX_SIGN); - USHORT usRc = TmBtree.EQFNTMSign((PCHAR)pTmSign, &usSignLen ); + USHORT usRc = sign(); fOK = (usRc == NO_ERROR); // update description field if ( fOK ) { - strncpy( pTmSign->szDescription, pszBuffer, sizeof(pTmSign->szDescription)-1 ); - pTmSign->szDescription[sizeof(pTmSign->szDescription)-1] = EOS; + strncpy( state->stTmSign.szDescription, pszBuffer, sizeof(state->stTmSign.szDescription)-1 ); + state->stTmSign.szDescription[sizeof(state->stTmSign.szDescription)-1] = EOS; - //trigger plugin to set description - //pMemoryPlugin->setDescription(szName,pTmSign->szDescription); } /* endif */ // re-write signature record if ( fOK ) { - usRc = TmBtree.QDAMDictUpdSignLocal(pTmSign); + usRc = state->TmBtree.QDAMDictUpdSignLocal(&state->stTmSign); fOK = (usRc == NO_ERROR); } /* endif */ - // free any allocated buffer - delete ( pTmSign ); } /* endif */ return; } @@ -157,7 +149,9 @@ int EqfMemory::getName int iSize ) { - return( CopyToBuffer( this->szName, pszBuffer, iSize ) ); + //this->szName.assign( pszBuffer, pszBuffer+iSize ); + szName = std::string(pszBuffer, iSize); + return 0; } /*! \brief Get the name of the memory @@ -182,8 +176,8 @@ unsigned long EqfMemory::getFileSize() //if ( pTmClb != NULL ) { - unsigned long ulDataSize = FilesystemHelper::GetFileSize( TmBtree.fb.file ); - unsigned long ulIndexSize = FilesystemHelper::GetFileSize( InBtree.fb.file ); + unsigned long ulDataSize = state->TmBtree.fb.file.GetFileSize(); + unsigned long ulIndexSize = state->InBtree.fb.file.GetFileSize(); ulFileSize = ulDataSize + ulIndexSize; } @@ -205,11 +199,10 @@ int EqfMemory::putProposal ) { int iRC = 0; - TMX_EXT_OUT_W TmPutOut; - memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); + TMX_EXT_OUT_W TmPutOut{}; if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string author = Proposal.szTargetAuthor; - std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalStrC()); + std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalSrcStr()); T5LOG( T5INFO) <<"EqfMemory::putProposal, source = " << source << "; author = " << author; } @@ -230,16 +223,12 @@ int EqfMemory::putProposal int EqfMemory::RewriteCompactTable(){ USHORT usRc = 0; - if ( bCompactChanged ) + if ( state->bCompactChanged ) { - usRc = TmBtree.EQFNTMUpdate( COMPACT_KEY, - bCompact, MAX_COMPACT_SIZE-1 ); + usRc = state->TmBtree.EQFNTMUpdate( COMPACT_KEY, state->bCompact ); if ( !usRc ) { - bCompactChanged = FALSE; - - //usRc = TmBtree.EQFNTMIncrUpdCounter( COMPACTAREA_UPD_COUNTER, - // &(alUpdCounter[COMPACTAREA_UPD_COUNTER]) ); + state->bCompactChanged = FALSE; } /* endif */ } /* endif */ @@ -249,15 +238,15 @@ int EqfMemory::RewriteCompactTable(){ size_t EqfMemory::GetRAMSize()const{ size_t size = sizeof(*this); - size += TmBtree.fb.data.capacity(); - size += InBtree.fb.data.capacity(); + size += state->TmBtree.fb.data.capacity(); + size += state->InBtree.fb.data.capacity(); return size; }; size_t EqfMemory::GetExpectedRAMSize()const{ size_t size = sizeof(*this); - size += TmBtree.GetFileSize(); - size += InBtree.GetFileSize(); + size += state->TmBtree.GetFileSize(); + size += state->InBtree.GetFileSize(); return size; }; @@ -274,24 +263,20 @@ int EqfMemory::getNextProposal ) { int iRC = 0; - TMX_EXT_IN_W TmExtIn; - TMX_EXT_OUT_W TmExtOut; - - memset( &TmExtIn, 0, sizeof(TMX_EXT_IN_W) ); - memset( &TmExtOut, 0, sizeof(TMX_EXT_OUT_W) ); + TMX_EXT_IN_W TmExtIn{}; + TMX_EXT_OUT_W TmExtOut{}; - TmExtIn.ulTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; - TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey();//this->usNextTarget; - TmExtIn.usConvert = MEM_OUTPUT_ASIS; + TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey(); + TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey(); Proposal.clear(); iRC = (int)TmtXExtract( &TmExtIn, &TmExtOut); if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) ) { - if ( (piProgress != NULL) && (TmExtOut.ulMaxEntries != 0) ) + if ( (piProgress != NULL) && (TmExtOut.lMaxEntries > 0) ) { - *piProgress = (int)(( TmExtIn.ulTmKey - FIRST_KEY) * 100) / (int)TmExtOut.ulMaxEntries; + *piProgress = (int)(( TmExtIn.lTmKey - FIRST_KEY) * 100) / (int)TmExtOut.lMaxEntries; } /* endif */ } /* endif */ @@ -300,14 +285,14 @@ int EqfMemory::getNextProposal ExtOutToOtmProposal( &TmExtOut, Proposal ); // set current proposal internal key ,which is used in updateProposal - Proposal.currentInternalKey.setInternalKey( TmExtIn.ulTmKey, TmExtIn.usNextTarget ); + Proposal.currentInternalKey.setInternalKey( TmExtIn.lTmKey, TmExtIn.usNextTarget ); }else{ Proposal.currentInternalKey.reset(); } /* endif */ - if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) || iRC == BTREE_BUFFER_SMALL || (usAccessMode & ASD_ORGANIZE) ) + if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) || iRC == BTREE_BUFFER_SMALL || (state->usAccessMode & ASD_ORGANIZE) ) { - Proposal.nextInternalKey.setInternalKey(TmExtOut.ulTmKey, TmExtOut.usNextTarget); + Proposal.nextInternalKey.setInternalKey(TmExtOut.lTmKey, TmExtOut.usNextTarget); } /* endif */ if ( iRC == 0 ) @@ -316,12 +301,12 @@ int EqfMemory::getNextProposal } else if ( iRC == BTREE_EOF_REACHED ) { - iRC = INFO_ENDREACHED; + iRC = EqfMemoryError::INFO_ENDREACHED; } else { T5LOG_RC(iRC, this->szName); - if ( iRC == BTREE_CORRUPTED ) iRC = ERROR_ENTRYISCORRUPTED; + if ( iRC == BTREE_CORRUPTED ) iRC = EqfMemoryError::ERROR_ENTRYISCORRUPTED; } return( iRC ); @@ -365,37 +350,6 @@ void *EqfMemory::getPlugin() } - -/*! \brief Get the error message for the last error occured - - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ -int EqfMemory::getLastError -( - std::string &strError -) -{ - strError = this->strLastError; - return( this->iLastError ); -} - - /*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ -int EqfMemory::getLastError -( - char *pszError, - int iBufSize -) -{ - strError = strLastError.c_str(); - return( this->iLastError ); -} - /*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] \param iPos position of language \param pszBuffer pointer to a buffer for the document name @@ -409,9 +363,9 @@ int EqfMemory::getLanguage int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(Languages.ulMaxEntries)) ) + if ( (iPos >= 0) && (iPos < (int)(state->Languages.table.size())) ) { - return( CopyToBuffer( Languages.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); + return( CopyToBuffer( state->Languages.table[iPos].szName, pszBuffer, iSize ) ); } else { @@ -431,13 +385,11 @@ int EqfMemory::getLanguage */ int OtmProposal::SetProposalKey ( - ULONG ulKey, - USHORT usTargetNum + const long& lKey, + const USHORT& usTargetNum ) { - currentInternalKey.setInternalKey(ulKey, usTargetNum); - //recordKey = ulKey; - //targetKey = usTargetNum; + currentInternalKey.setInternalKey(lKey, usTargetNum); return( 0 ); } @@ -455,7 +407,7 @@ std::string OtmProposal::getInternalKey()const{ int TMCursor::SplitProposalKeyIntoRecordAndTarget ( const char *pszKey, - ULONG *pulKey, + LONG *plKey, USHORT *pusTargetNum ) { @@ -464,12 +416,12 @@ int TMCursor::SplitProposalKeyIntoRecordAndTarget if ( pszTarget != NULL ) { *pszTarget = '\0'; - *pulKey = atol( pszKey ); + *plKey = atol( pszKey ); *pusTargetNum = (USHORT)atoi( pszTarget + 1 ); } else { - iRC = EqfMemory::ERROR_INTERNALKEY_MISSING; + iRC = EqfMemoryError::ERROR_INTERNALKEY_MISSING; } /* end */ return( iRC ); @@ -497,22 +449,39 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setAuthor( pExtOut->szAuthorName ); Proposal.setMarkup( pExtOut->szTagTable ); Proposal.setTargetLanguage( pExtOut->szTargetLanguage ); - Proposal.setSourceLanguage( stTmSign.szSourceLanguage); - Proposal.setAddInfo( pExtOut->szAddInfo ); - Proposal.setContext( pExtOut->szContext ); + Proposal.setSourceLanguage( state->stTmSign.szSourceLanguage); + Proposal.setAddInfo( pExtOut->szAddInfo.c_str() ); + Proposal.setContext( pExtOut->szContext.c_str() ); Proposal.setDocName( pExtOut->szLongName ); Proposal.setSegmentId( pExtOut->ulSourceSegmentId ); Proposal.setType( FlagToProposalType( pExtOut->usTranslationFlag ) ); Proposal.setUpdateTime( pExtOut->lTargetTime ); Proposal.setContextRanking( 0 ); Proposal.setOriginalSourceLanguage( pExtOut->szOriginalSourceLanguage ); - //Proposal.recordKey = pExtOut->ulRecKey; - //Proposal.targetKey = pExtOut->usTargetKey Proposal.currentInternalKey.setInternalKey(pExtOut->ulRecKey, pExtOut->usTargetKey); return( iRC ); } + +#include // for std::tolower, std::toupper +#include // for std::strlen + +void convertLangCode(const char* src, char* trg) { + bool afterDash = false; + for (size_t i = 0; src[i] != '\0'; ++i) { + if (src[i] == '-') { + trg[i] = '-'; + afterDash = true; + } else if (afterDash) { + trg[i] = std::toupper(static_cast(src[i])); + } else { + trg[i] = std::tolower(static_cast(src[i])); + } + } + trg[std::strlen(src)] = '\0'; // null-terminate +} + /*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure \param pMatch pointer to the TMX_MATCH_TABLE_W structure \param Proposal reference to the OtmProposal being filled @@ -531,21 +500,21 @@ int EqfMemory::MatchToOtmProposal pProposal->setTarget( pMatch->szTarget ); pProposal->setAuthor( pMatch->szTargetAuthor ); pProposal->setMarkup( pMatch->szTagTable ); - pProposal->setTargetLanguage( pMatch->szTargetLanguage ); - pProposal->setSourceLanguage( stTmSign.szSourceLanguage); + convertLangCode(pMatch->szOriginalSrcLanguage, pProposal->szSourceLanguage); + convertLangCode(pMatch->szTargetLanguage, pProposal->szTargetLanguage); pProposal->setAddInfo( pMatch->szAddInfo ); pProposal->setContext( pMatch->szContext ); pProposal->setDocName( pMatch->szLongName ); pProposal->setSegmentId( pMatch->ulSegmentId ); pProposal->setType( FlagToProposalType( pMatch->usTranslationFlag ) ); pProposal->setUpdateTime( pMatch->lTargetTime ); - pProposal->SetProposalKey( pMatch->ulKey, pMatch->usTargetNum ); + pProposal->SetProposalKey( pMatch->lKey, pMatch->usTargetNum ); pProposal->setContextRanking( (int)pMatch->usContextRanking ); pProposal->setMemoryIndex( pMatch->usDBIndex ); pProposal->setWords( pMatch->iWords ); pProposal->setDiffs( pMatch->iDiffs ); pProposal->setOriginalSourceLanguage( pMatch->szOriginalSrcLanguage ); - pProposal->SetProposalKey(pMatch->ulKey, pMatch->usTargetNum); + pProposal->SetProposalKey(pMatch->lKey, pMatch->usTargetNum); switch ( pMatch->usMatchLevel ) { @@ -601,7 +570,7 @@ int EqfMemory::OtmProposalToGetIn { int iRC = 0; - memset( pGetIn, 0, sizeof(pGetIn) ); + *pGetIn = TMX_GET_W{}; Proposal.getSource( pGetIn->szSource, sizeof(pGetIn->szSource) ); Proposal.getAuthor( pGetIn->szAuthorName, sizeof(pGetIn->szAuthorName) ); Proposal.getMarkup( pGetIn->szTagTable, sizeof(pGetIn->szTagTable) ); @@ -610,15 +579,10 @@ int EqfMemory::OtmProposalToGetIn Proposal.getAddInfo( pGetIn->szAddInfo, sizeof(pGetIn->szAddInfo) ); Proposal.getContext( pGetIn->szContext, sizeof(pGetIn->szContext) ); Proposal.getDocName( pGetIn->szLongName, sizeof(pGetIn->szLongName) ); - Proposal.getDocName( pGetIn->szFileName, sizeof(pGetIn->szFileName) ); - //pGetIn->usMatchThreshold = TM_DEFAULT_THRESHOLD; - int threshold = FLAGS_triplesthreshold; - //Properties::GetInstance()->get_value_or_default(KEY_TRIPLES_THRESHOLD, threshold, threshold); - pGetIn->usMatchThreshold = threshold; + pGetIn->usMatchThreshold = FLAGS_triplesthreshold; pGetIn->ulSegmentId = Proposal.getSegmentId(); pGetIn->pvReplacementList = (PVOID)Proposal.getReplacementList(); - //pGetIn->fSourceLangIsPrefered = Proposal.isSourceLangIsPrefered(); pGetIn->fSourceLangIsPrefered = LanguageFactory::getInstance()->isPreferedLang( (pGetIn->szSourceLanguage) ); pGetIn->fTargetLangIsPrefered = LanguageFactory::getInstance()->isPreferedLang( (pGetIn->szTargetLanguage) ); return( iRC ); @@ -647,8 +611,6 @@ OtmProposal::eProposalType FlagToProposalType( USHORT usTranslationFlag ) return( OtmProposal::eptUndefined ); } - - /*! \brief Set or clear the pointer to a loaded global memory option file This method sets a pointer to a loaded global memory option file. @@ -658,14 +620,6 @@ OtmProposal::eProposalType FlagToProposalType( USHORT usTranslationFlag ) \returns 0 or error code in case of errors */ -int EqfMemory::setGlobalMemoryOptions -( - void *pvGlobalMemoryOptionsIn -) -{ - this->pvGlobalMemoryOptions = pvGlobalMemoryOptionsIn; - return( 0 ); -} @@ -674,58 +628,63 @@ EqfMemory::EqfMemory(const std::string& tmName){ if(FLAGS_log_tm_lifetime){ T5LOG(T5TRANSACTION) << "Creating memory " << tmName; } - + readOnlyCnt = std::make_shared(0); writeCnt = std::make_shared(0); + state = std::make_unique(); - - pvGlobalMemoryOptions = NULL; - int rc = NTMCreateLongNameTable(); - if(rc) T5LOG(T5ERROR) << "NTMCreateLongNameTable returned non-zero, but " << rc; - - TmBtree.fb.fileName = FilesystemHelper::GetTmdPath(tmName); - InBtree.fb.fileName = FilesystemHelper::GetTmiPath(tmName); - strcpy(szName, tmName.c_str()); + state->TmBtree.fb.file.fileName = FilesystemHelper::GetTmdPath(tmName); + state->InBtree.fb.file.fileName = FilesystemHelper::GetTmiPath(tmName); + szName = tmName; } -int EqfMemory::ReloadFromDisk(){ +int EqfMemory::Reload(bool fReadFromDisk){ int rc = 0; + /* TmBtree.resetLookupTable(); InBtree.resetLookupTable(); - rc = TmBtree.fb.ReadFromFile(); - rc = InBtree.fb.ReadFromFile(); - eStatus = WAITING_FOR_LOADING_STATUS; - LoadMem(); + + LongNames.pszBuffer.clear(); + LongNames.stTableEntry.clear(); + LongNamesCaseIgnore.pszBuffer.clear(); + LongNamesCaseIgnore.stTableEntry.clear(); + Authors.table.clear(); + LangGroups.table.clear(); + Languages.table.clear(); + psLangIdToGroupTable.clear(); + + + //datashould be already in filebuffers in case of reorganize, otherwise - read from disk + if(fReadFromDisk){ + state->TmBtree.fb.file = LockedFile{};//reset fileLock + state->InBtree.fb.file = LockedFile{};//reset fileLock + rc = TmBtree.fb.ReadFromFile(); + if(!rc){ + rc = InBtree.fb.ReadFromFile(); + } + } + + if(rc){ + eStatus = FAILED_TO_OPEN_STATUS; + }else{ + eStatus = WAITING_FOR_LOADING_STATUS; + rc = Load(); + }//*/ return rc; } int EqfMemory::FlushFilebuffers(MutexTimeout& tmLockTimeout){ + if(state->fCorrupted){ + return 0; + } TimedMutexGuard l(tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); - int rc = TmBtree.fb.Flush(); - rc = rc? rc : InBtree.fb.Flush(); + int rc = state->TmBtree.fb.Flush(); + rc = rc? rc : state->InBtree.fb.Flush(); return rc; } int EqfMemory::UnloadFromRAM(){ - //if(TmBtree.fb.file == nullptr){ - //file is in mem - // return -1; - //} - //if(InBtree.fb.file == nullptr){ - //file is in mem - // return -1; - //} - //TmBtree.fb.WriteToFile(); - //InBtree.fb.WriteToFile(); - - //FlushFilebuffers(); - //TmBtree.fb.data.clear(); - //TmBtree.fb.data.shrink_to_fit(); - - //InBtree.fb.data.clear(); - //InBtree.fb.data.shrink_to_fit(); - //fOpen = false; return 0; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 88b410a9..d70438c3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -40,7 +40,6 @@ EqfMemoryPlugin::EqfMemoryPlugin() iLastError = 0; pluginType = OtmPlugin::eTranslationMemoryType; usableState = OtmPlugin::eUsable; - UtlGetCheckedEqfDrives( szSupportedDrives ); } EqfMemoryPlugin::~EqfMemoryPlugin() @@ -99,7 +98,9 @@ EqfMemory* EqfMemoryPlugin::initTM( pMemory->setExpecedSizeInRAM(requiredMemory); pMemory->updateLastUseTime(); pMemory->eStatus = WAITING_FOR_LOADING_STATUS; - pMemory->usAccessMode |= ASD_ORGANIZE; + if(fReorganizeOnly){ + pMemory->state->usAccessMode |= ASD_ORGANIZE; + } //pMemory->fReorganizeOnly = fReorganizeOnly; }else{ LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); @@ -108,17 +109,6 @@ EqfMemory* EqfMemoryPlugin::initTM( } /* end */ -/*! \brief Close a memory - \param pMemory pointer to memory object -*/ -int EqfMemoryPlugin::closeMemory( - EqfMemory *pMemory -) -{ - T5LOG(T5ERROR) << "unimplemented funtion"; - return( 0 ); -} - ///*! \brief set description of a memory // \param pszName name of the memory @@ -174,10 +164,6 @@ int tryStrCpy(char* dest, const char* src, const char* def){ } - - - - bool EqfMemoryPlugin::stopPlugin( bool fForce ) { @@ -211,179 +197,6 @@ USHORT registerPlugins() return usRC; } -/*! \brief Private data area for the memory import from data files */ -typedef struct _MEMIMPORTFROMFILEDATA -{ - EqfMemory *pInputMemory; // input memory - EqfMemory *pOutputMemory; // output memory - OtmProposal Proposal; // currently processed proposal - std::string strPropFile; // property file of the imorted memory - std::string strDataFile; // data file of the imorted memory - std::string strIndexFile; // index file of the imorted memory - char szBuffer[1014]; // general purpose buffer - -} MEMIMPORTFROMFILEDATA, *PMEMIMPORTFROMFILEDATA; - - -/*! \brief Handle a return code from the memory functions and create the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name or NULL if not available - \param pszMemPath fully qualified memory path name or NULL if not available - \param strLastError reference to string object receiving the message text - \param iLastError reference to a integer variable receiving the error code - \returns original or modified error return code -*/ -int EqfMemoryPlugin::handleError( int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError ) -{ - char* pReplAddr[2]; - - if ( iRC == 0 ) return( iRC ); - - char *pszErrorTextBuffer = (char *)malloc(8096); - if ( pszErrorTextBuffer == NULL ) return( iRC ); - pszErrorTextBuffer[0] = '\0'; - - pReplAddr[0] = pszMemName; - - switch ( iRC ) - { - case ERROR_TA_ACC_TAGTABLE: - if (pszMarkup != NULL) - pReplAddr[0] = pszMarkup; - else - strcpy(pReplAddr[0], "n/a"); - UtlGetMsgTxt( ERROR_TA_ACC_TAGTABLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_SHARING_VIOLATION: - case BTREE_DICT_LOCKED: - case BTREE_ENTRY_LOCKED: - case BTREE_ACCESS_ERROR: - UtlGetMsgTxt( ERROR_MEM_NOT_ACCESSIBLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_NO_MORE_MEMORY_AVAILABLE: - case ERROR_NOT_ENOUGH_MEMORY: - case BTREE_NO_ROOM: - case BTREE_NO_BUFFER: - UtlGetMsgTxt( ERROR_STORAGE, pszErrorTextBuffer, 0, NULL ); - break; - - case VERSION_MISMATCH: - case CORRUPT_VERSION_MISMATCH: - case ERROR_VERSION_NOT_SUPPORTED: - UtlGetMsgTxt( ERROR_MEM_VERSION_NOT_SUPPORTED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case FILE_MIGHT_BE_CORRUPTED: - case BTREE_CORRUPTED: - case BTREE_USERDATA: - case ERROR_OLD_PROPERTY_FILE: - UtlGetMsgTxt( ITM_TM_NEEDS_ORGANIZE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case BTREE_DISK_FULL: - case BTREE_WRITE_ERROR : - case DISK_FULL: - { - char szDrive[2]; - if ( pszMemPath != NULL ) - { - szDrive[0] = *pszMemPath; - szDrive[1] = '\0'; - } - if (pszMemPath != NULL) - pReplAddr[0] = szDrive; - else - strcpy(pReplAddr[0], "n/a"); - UtlGetMsgTxt( ERROR_DISK_FULL_MSG, pszErrorTextBuffer, 1, pReplAddr ); - } - break; - - case DB_FULL: - case BTREE_LOOKUPTABLE_TOO_SMALL: - UtlGetMsgTxt( ERROR_MEM_DB_FULL, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case FILE_ALREADY_EXISTS: - UtlGetMsgTxt( ERROR_MEM_NAME_INVALID, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case NOT_REPLACED_OLD_SEGMENT: - UtlGetMsgTxt( ERROR_MEM_NOT_REPLACED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_FILE_SCREWED_UP: - case NOT_A_MEMORY_DATABASE: - case BTREE_ILLEGAL_FILE: - UtlGetMsgTxt( ERROR_MEM_DESTROYED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_FILE_NOT_FOUND: - case BTREE_FILE_NOTFOUND: - UtlGetMsgTxt( ERROR_TM_FILE_NOT_FOUND, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_TM_OPENED_EXCLUSIVELY: - UtlGetMsgTxt( ERROR_TM_OPENED_EXCLUSIVELY, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_PROP_EXIST: - UtlGetMsgTxt( ERROR_PROP_EXIST, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_PROP_WRITE_ERROR: - UtlGetMsgTxt( ERROR_PROP_WRITE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_PROPERTIES_NOT_OPENED: - UtlGetMsgTxt( ERROR_OPEN_TM_PROPERTIES, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_NETWORK_ACCESS_DENIED : - case BTREE_NETWORK_ACCESS_DENIED: - UtlGetMsgTxt( ERROR_ACCESS_DENIED_MSG, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case SEGMENT_BUFFER_FULL: - case BTREE_BUFFER_SMALL: - UtlGetMsgTxt( ERROR_MEM_SEGMENT_TOO_LARGE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_INTERNAL : - UtlGetMsgTxt( ERROR_INTERNAL, pszErrorTextBuffer, 0, NULL ); - break; - - case ERROR_INTERNAL_PARM: - UtlGetMsgTxt( ERROR_INTERNAL_PARM, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case BTREE_IN_USE: - UtlGetMsgTxt( ERROR_MEM_NOT_ACCESSIBLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_TM_OPENED_SHARED: - case ERROR_PATH_NOT_FOUND: - case ERROR_INVALID_DRIVE: - case BTREE_INVALID_DRIVE : - default: - { - char szError[20]; - sprintf( szError, "%ld", iRC ); - pReplAddr[1] = szError; - T5LOG(T5ERROR) << ":: error " << iRC; - UtlGetMsgTxt( ERROR_MEM_UNDEFINED, pszErrorTextBuffer, 2, pReplAddr ); - } - } - - strLastError = pszErrorTextBuffer; - iLastError = iRC; - free( pszErrorTextBuffer ); - - return( iRC ); -} - unsigned short getPluginInfo( POTMPLUGININFO pPluginInfo ) { strcpy( pPluginInfo->szName, pszPluginName ); diff --git a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp index acd9da66..a890f4a8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp @@ -26,96 +26,6 @@ #include // event logging #include "../utilities/LogWrapper.h" -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -); - - -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusTokens, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP -); - -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); - -#ifdef ACTIVATE_NTMGenericDelete -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ pSource, // source string - PSZ pProp, // proposal string - PSZ pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); -#endif - - -static BOOL NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList, // list of tokens to be replaced - BOOL fRespectLFs -); - - -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs -); - - -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -); - -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W * ppNewData, - PLONG plNewLen -); - - -static PFUZZYTOK -NTMSplitAndAddTokens ( PTMX_SUBSTPROP, - PFUZZYTOK, PUSHORT, USHORT, SHORT, - USHORT, PSZ_W, SHORT, ULONG); - -BOOL NTMAlignTags( PFUZZYTOK pTokSource, PFUZZYTOK pTokTarget, PREPLLIST *ppReplaceList ); // helper function for token logging #ifdef INLINE_TAG_REPL_LOGGING @@ -168,936 +78,8 @@ void NTMListTokens void NTMMarkCRLF( PSZ_W pszSource, PSZ_W pszTarget ); #endif -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMTagSubst | -//+----------------------------------------------------------------------------+ -//|Function call: NTMTagSubst( pSubstProp ); | -//+----------------------------------------------------------------------------+ -//|Description: GENERIC TM: substitute tags if substitution of all | -//| is possible | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp ptr to substitution struct | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 1 all tagging in proposal replaced | -//| 0 nothing replaced | -//+----------------------------------------------------------------------------+ -//|Function flow: Prepare a token list for current src, src of prop and | -//| tgt of prop | -//| call LCS for data pair src - src of prop | -//| call LCS for data pair src of prop - tgt of prop | -//| determine which tags can be replaced | -//| if so, | -//| if all tags can be replaced | -//| replace them | -//+----------------------------------------------------------------------------+ -BOOL NTMTagSubst -( - PTMX_SUBSTPROP pSubstProp, - ULONG ulSrcOemCP, - ULONG ulTgtOemCP -) -{ - BOOL fOK; - PREPLLIST pReplaceList = NULL; - PREPLLIST pReplaceSourceList = NULL; - PFUZZYTOK pCopyTokList1 = NULL; - PFUZZYTOK pCopyTokList2 = NULL; - PFUZZYTOK pTempList; - PBYTE pTokBuf = NULL; - USHORT k; //index variable - BOOL fConnected; // temp variable for connection state - BOOL fReplaced = FALSE; - BOOL fDelete = FALSE; - SHORT sSrcLangID = 0; - SHORT sTgtLangID = 0; - PSZ_W pInBuf; -#ifdef INLINE_TAG_REPL_LOGGING - static CHAR_W szSegBuf[4096]; -#endif - - // ignore LFs in inline tagging only for IBMDITA markup - BOOL fRespectLFs = (_stricmp( pSubstProp->szSourceTagTable, "IBMDITA" ) != 0); - - UtlAlloc((PVOID *) &pInBuf, 0L, (LONG) IO_BUFFER_SIZE * sizeof(CHAR_W), ERROR_STORAGE ); - UtlAlloc((PVOID *) &pTokBuf, 0L, (LONG) TOK_BUFFER_SIZE, ERROR_STORAGE ); - fOK = ( pInBuf && pTokBuf ) ; - - /******************************************************************/ - /* prepare token structure */ - /******************************************************************/ - if (fOK ) - { - fOK = (MorphGetLanguageID(pSubstProp->szSourceLanguage, &sSrcLangID )==MORPH_OK); - } /* endif */ - - if (fOK && (MorphGetLanguageID( pSubstProp->szTargetLanguage, - &sTgtLangID ) != MORPH_OK) ) - { - sTgtLangID = sSrcLangID ; - } /* endif */ - - if (fOK ) - { - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szSource, - (PFUZZYTOK *)&pSubstProp->pTokSource, - &pSubstProp->usTokenSource, - nullptr, //pSubstProp->pTagsSource, - sSrcLangID, ulSrcOemCP ); - #ifdef INLINE_TAG_REPL_LOGGING - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)) - if ( fOK ) NTMListTokens( "Source", (PFUZZYTOK)pSubstProp->pTokSource ); - #endif - - } /* endif */ - - if ( fOK ) - { - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szPropSource, - (PFUZZYTOK *)&pSubstProp->pTokPropSource, - &pSubstProp->usTokenPropSource, - nullptr, //pSubstProp->pTagsPropSource, - sSrcLangID, ulSrcOemCP ); -#ifdef INLINE_TAG_REPL_LOGGING - if ( fOK ) NTMListTokens( "PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); -#endif - } /* endif */ - - if ( fOK ) - { // P018276: use TgtOemCP!! (must fit to sTgtLangID) - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szPropTarget, - (PFUZZYTOK *)&pSubstProp->pTokPropTarget, - &pSubstProp->usTokenPropTarget, - nullptr,//pSubstProp->pTagsPropTarget, - sTgtLangID, ulTgtOemCP ); -#ifdef INLINE_TAG_REPL_LOGGING - if ( fOK ) NTMListTokens( "PropTarget", (PFUZZYTOK)pSubstProp->pTokPropTarget ); -#endif - } /* endif */ - - if ( fOK ) - { - /******************************************************************/ - /* temporary store values */ - /******************************************************************/ - UtlAlloc( (PVOID *) &pCopyTokList1, 0L, (LONG)(pSubstProp->usTokenPropSource+1)*sizeof(FUZZYTOK), NOMSG ); - UtlAlloc( (PVOID *) &pCopyTokList2, 0L, (LONG)(pSubstProp->usTokenPropTarget+1)*sizeof(FUZZYTOK), NOMSG ); - fOK = ( pCopyTokList1 && pCopyTokList2 ); - } /* endif */ - - - if ( fOK ) - { - memcpy( pCopyTokList1, pSubstProp->pTokPropSource, pSubstProp->usTokenPropSource * sizeof(FUZZYTOK)); - memcpy( pCopyTokList2, pSubstProp->pTokPropTarget, pSubstProp->usTokenPropTarget * sizeof(FUZZYTOK)); - - /******************************************************************/ - /* now do the alignments.... */ - /******************************************************************/ -//#define USEOLDALIGN - -#ifdef USEOLDALIGN - fOK = (BOOL)FuzzyLCSReplList( (PFUZZYTOK)pSubstProp->pTokPropSource, (PFUZZYTOK)pSubstProp->pTokPropTarget, &pReplaceList, - pSubstProp->usTokenPropSource, pSubstProp->usTokenPropTarget, NULL, TRUE ); -#else - fOK = NTMAlignTags( (PFUZZYTOK)pSubstProp->pTokPropSource, (PFUZZYTOK)pSubstProp->pTokPropTarget, &pReplaceList ); -#endif - /******************************************************************/ - /* restore original content, but keep fConnected... */ - /* The original content is modified in the FuzzyLCSReplList func. */ - /******************************************************************/ - pTempList = (PFUZZYTOK) pSubstProp->pTokPropSource; - for ( k=0; k < pSubstProp->usTokenPropSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList1+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - - pTempList = (PFUZZYTOK) pSubstProp->pTokPropTarget; - for ( k=0; k < pSubstProp->usTokenPropTarget+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList2+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - #ifdef INLINE_TAG_REPL_LOGGING - NTMListTokens( "Connected PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); - NTMListTokens( "Connected PropTarget", (PFUZZYTOK)pSubstProp->pTokPropTarget ); - #endif - - /******************************************************************/ - /* free CopyTokList2 */ - /******************************************************************/ - UtlAlloc( (PVOID*)&pCopyTokList2, 0L, 0L, NOMSG); - } /* endif */ - - if ( fOK && pReplaceList ) - { - /******************************************************************/ - /* temporary store values */ - /******************************************************************/ - UtlAlloc( (PVOID *) &pCopyTokList2, 0L, (LONG)(pSubstProp->usTokenSource+1)*sizeof(FUZZYTOK), NOMSG ); - fOK = ( pCopyTokList2 != NULL ); - } /* endif */ - - if ( fOK && pReplaceList ) - { - memcpy( pCopyTokList2, pSubstProp->pTokSource, pSubstProp->usTokenSource * sizeof(FUZZYTOK)); - fOK = (BOOL)FuzzyLCSReplList( (PFUZZYTOK)pSubstProp->pTokPropSource, - (PFUZZYTOK)pSubstProp->pTokSource, - &pReplaceSourceList, - pSubstProp->usTokenPropSource, - pSubstProp->usTokenSource, - NULL, TRUE ); - // #endif - /******************************************************************/ - /* restore original content, but keep fConnected... */ - /******************************************************************/ - pTempList = (PFUZZYTOK) pSubstProp->pTokPropSource; - for ( k=0; k < pSubstProp->usTokenPropSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList1+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - - pTempList = (PFUZZYTOK) pSubstProp->pTokSource; - for ( k=0; k < pSubstProp->usTokenSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList2+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - #ifdef INLINE_TAG_REPL_LOGGING - T5LOG(T5DEBUG) << "after FuzzyLCSReplList on source and proposal source" ; - NTMListTokens( "Connected Source", (PFUZZYTOK)pSubstProp->pTokSource ); - NTMListTokens( "Connected PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); - #endif - - } /* endif */ - - /********************************************************************/ - /* find out which tags can be replaced by each other */ - /********************************************************************/ - if ( fOK && pReplaceSourceList ) - { - T5LOG(T5DEBUG) << "replace source list exists" ; - - /*************************************************************/ - /* insert a dummy token at the beginning of the pReplaceSourcList */ - /* list to have an anchor where to start from... */ - /*************************************************************/ - memmove( pReplaceSourceList+1, pReplaceSourceList, sizeof( REPLLIST ) * MAX_REPL); - - pReplaceSourceList->pSrcTok = (PFUZZYTOK)pSubstProp->pTokPropSource; - pReplaceSourceList->pTgtTok = (PFUZZYTOK)pSubstProp->pTokSource; - - fReplaced = NTMFuzzyReplace ( pSubstProp, pSubstProp->szSource, - pSubstProp->szPropSource, - pSubstProp->szPropTarget, - pReplaceSourceList, pReplaceList - ); - - T5LOG(T5DEBUG) << "After NTMFuzzyReplace: fReplaced = " << fReplaced ; - -#ifdef ACTIVATE_NTMGenericDelete - fDelete = NTMGenericDelete ( pSubstProp, pSubstProp->szSource, - pSubstProp->szPropSource, - pSubstProp->szPropTarget, - pReplaceSourceList, pReplaceList ); -#endif - - if (fReplaced || fDelete) - { - /****************************************************************/ - /* check whether all tags can be replaced */ - /****************************************************************/ - - if (NTMCheckTagPairs(pSubstProp, pReplaceSourceList, pReplaceList, fRespectLFs )) - { - /**************************************************************/ - /* replace all tags possible */ - /**************************************************************/ - fReplaced = NTMReplaceTags(pSubstProp, fRespectLFs ); - } - else - { - fReplaced = FALSE; - T5LOG(T5DEBUG) << "After NTMCheckTagPairs: fReplaced = " << fReplaced ; - - } /* endif */ - } /* endif */ - } /* endif */ - - // add tagging function of generic inline tag replacement - if ( fOK && !fReplaced ) - { - BOOL fAddTags = TRUE; - PSZ_W pSegmentEndTags = NULL; // ptr to segment end tagging - PSZ_W pSegmentText = NULL; // ptr to segment text portion - - #define TOKEN_TYPE_TAG -7 - #define TOKEN_TYPE_TEXT -1 - - // 1. check that proposal does not have any inline tagging - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokPropSource; - while ( pToken->ulHash ) - { - if ( pToken->sType != TOKEN_TYPE_TEXT ) - { - fAddTags = FALSE; - } /* endif */ - pToken++; - } /*endwhile */ - } /* endif */ - - // 2. check that source has tagging only at the beginning and at the end of the segment - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - - // handle tags at segment start - if ( pToken->sType == TOKEN_TYPE_TAG ) - { - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - } - else - { - // no tags at segment start - fAddTags = FALSE; - } /* endif */ - - // handle text part of segment - if ( pToken->ulHash &&(pToken->sType == TOKEN_TYPE_TEXT) ) - { - pSegmentText = pToken->pData; // remember start of text portion - - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TEXT) ) - { - pToken++; - } /*endwhile */ - } - else - { - // no text portion following tags - fAddTags = FALSE; - } /* endif */ - - // handle tags at segment end - if ( pToken->ulHash &&(pToken->sType == TOKEN_TYPE_TAG) ) - { - pSegmentEndTags = pToken->pData; // remember start of tagging at segment end - - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - - if ( pToken->ulHash && (pToken->pData[0] != 0) ) - { - // somethig is following tagging at segment end - fAddTags = FALSE; - } /* endif */ - } - else - { - // no tagging at segment end - fAddTags = FALSE; - } /* endif */ - } /* endif */ - - // 3. check that proposal text and segment text is identical - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - PFUZZYTOK pPropToken = (PFUZZYTOK)pSubstProp->pTokPropSource; - while ( pToken->ulHash && (pToken->pData[0] != 0) && pPropToken->ulHash && (pPropToken->pData[0] != 0)) - { - // skip any tags - while ( (pToken->ulHash && (pToken->pData[0] != 0)) && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - - if ( !pToken->ulHash || (pToken->pData[0] == 0)) - { - // end of source segment - fAddTags = FALSE; - } - else if ( (pToken->sType == TOKEN_TYPE_TEXT) && (pPropToken->sType == TOKEN_TYPE_TEXT ) ) - { - if ( wcsncmp( pToken->pData, pPropToken->pData, pToken->usStop - pToken->usStart + 1 ) != 0 ) - { - // text does not match - fAddTags = FALSE; - } /* endif */ - } - else - { - // not same number of text tokens - fAddTags = FALSE; - } /* endif */ - - // next token - if ( pToken->ulHash ) pToken++; - if ( pPropToken->ulHash ) pPropToken++; - } /*endwhile */ - } /* endif */ - - // 4. check that source segment tagging is of correct type - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - - if ( (pToken->pData[0] != L'<') || (pSegmentEndTags[0] != L'<') || (pSegmentEndTags[1] != L'/') ) - { - // incorrect type of tagging - fAddTags = FALSE; - } - else - { - // test characters of start and end tag - PSZ_W pszStartTag = pToken->pData + 1; - PSZ_W pszEndTag = pSegmentEndTags + 2; - while ( fAddTags && isalpha(*pszEndTag) ) - //while ( fAddTags && iswalpha(*pszEndTag) ) - { - if ( *pszStartTag == *pszEndTag ) - { - pszStartTag++; - pszEndTag++; - } - else - { - // tag names do not match - fAddTags = FALSE; - } /* endif */ - } /*endwhile */ - } /* endif */ - } /* endif */ - - - // add the tags when every test succeeded - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - PSZ_W pStartTag = pToken->pData; - int iStartTagLen = pSegmentText - pStartTag; - int iEndTagLen = wcslen( pSegmentEndTags ); - int iSourceTextLen = wcslen(pSubstProp->szPropSource); - int iTargetTextLen = wcslen(pSubstProp->szPropTarget); - - if ( ( (iStartTagLen + iSourceTextLen + iEndTagLen) < MAX_SEGMENT_SIZE) && - ( (iStartTagLen + iTargetTextLen + iEndTagLen) < MAX_SEGMENT_SIZE) ) - { - // change proposal source - memmove( pSubstProp->szPropSource + iStartTagLen, pSubstProp->szPropSource, (iSourceTextLen + 1) * sizeof(CHAR_W) ); - memcpy( pSubstProp->szPropSource, pStartTag, iStartTagLen * sizeof(CHAR_W) ); - wcscpy( pSubstProp->szPropSource + wcslen(pSubstProp->szPropSource), pSegmentEndTags ); - - // change proposal target - memmove( pSubstProp->szPropTarget + iStartTagLen, pSubstProp->szPropTarget, (iTargetTextLen+1) * sizeof(CHAR_W) ); - memcpy( pSubstProp->szPropTarget, pStartTag, iStartTagLen * sizeof(CHAR_W) ); - wcscpy( pSubstProp->szPropTarget + wcslen(pSubstProp->szPropTarget), pSegmentEndTags ); - - fReplaced = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* free allocated resources */ - /********************************************************************/ - if ( pCopyTokList1 ) - UtlAlloc( (PVOID*)&pCopyTokList1, 0L, 0L, NOMSG); - if ( pCopyTokList2 ) - UtlAlloc( (PVOID*)&pCopyTokList2, 0L, 0L, NOMSG); - if ( pSubstProp->pTokSource ) - UtlAlloc( (PVOID*)&pSubstProp->pTokSource, 0L, 0L, NOMSG); - if ( pSubstProp->pTokPropSource ) - UtlAlloc( (PVOID*)&pSubstProp->pTokPropSource, 0L, 0L, NOMSG); - if ( pSubstProp->pTokPropTarget ) - UtlAlloc( (PVOID*)&pSubstProp->pTokPropTarget, 0L, 0L, NOMSG); - if (pInBuf ) - UtlAlloc((PVOID *) &pInBuf, 0L, 0L, NOMSG ); - if (pTokBuf ) - UtlAlloc((PVOID *) &pTokBuf, 0L, 0L, NOMSG ); - if (pReplaceList ) - UtlAlloc((PVOID *) &pReplaceList, 0L, 0L, NOMSG); - if (pReplaceSourceList ) - UtlAlloc((PVOID *) &pReplaceSourceList, 0L, 0L, NOMSG); - return (fReplaced); - -} - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMPrepareTokens | -//+----------------------------------------------------------------------------+ -//|Function call: BOOL NTMPrepareTokens | -//| ( PTMX_SUBSTPROP pSubstProp, | -//| PSZ pData, | -//| PFUZZYTOK *ppTokData, | -//| PUSHORT pusToken, | -//| PTMX_TAGTABLE_RECORD pTagRecord, | -//| SHORT sLangID ) | -//+----------------------------------------------------------------------------+ -//|Description: | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp, | -//| PSZ pData, | -//| PFUZZYTOK *ppTokData, | -//| PUSHORT pusToken, | -//| PTMX_TAGTABLE_RECORD pTagRecord, | -//| SHORT sLangID | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Function flow: | -//+----------------------------------------------------------------------------+ #define TAG_TOKEN -7 -static -BOOL isWhiteSpaceW( PSZ_W pData ) -{ - PSZ_W pTemp = pData; - CHAR_W c; - //while ( ((c = *pTemp)!= NULC) && iswspace( c ) ) - while ( ((c = *pTemp)!= NULC) && isspace( c ) ) - { - pTemp++; - } /* endwhile */ - return (*pTemp == EOS); -} - - -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusToken, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP -) -{ - BOOL fOK = TRUE; - USHORT usStringPos; - USHORT usTagRecordPos; // in # of bytes - - PSZ_W pTokBuf = pSubstProp->chBuffer; - PFUZZYTOK pCurrent = (PFUZZYTOK)pTokBuf; - ULONG ulCurStop; - BOOL fEndReached = FALSE; - USHORT usRandomIndex; // index in random sequence - PTMX_TAGENTRY pTagEntry = NULL; //ptr to tag entries in tag table record - PBYTE pByte = (PBYTE)pTagRecord; - USHORT usMaxToksInBuffer = (sizeof(pSubstProp->chBuffer) / - sizeof(FUZZYTOK)) - 2; - - - if ( !pSubstProp->ulRandom[0] ) - { - /********************************************************************/ - /* random sequences, see e.g. the book of Wirth... */ - /********************************************************************/ - pSubstProp->ulRandom[0] = 0xABCDEF01; - for (usRandomIndex = 1; usRandomIndex < MAX_RANDOM; usRandomIndex++) - { - pSubstProp->ulRandom[usRandomIndex] = - pSubstProp->ulRandom[usRandomIndex - 1] * 5 + 0xABCDEF01; - } /* endfor*/ - } /* endif */ - - // usFirstTagEntry is in number of bytes! - pByte += pTagRecord->usFirstTagEntry; - - //there is tag info so go through all tag entries and add tags to outstring - pTagEntry = (PTMX_TAGENTRY)pByte; - - //start of for loop - usStringPos = 0; - //initial positioning st beginning of first tag entry - // in # of bytes - usTagRecordPos = sizeof(TMX_TAGTABLE_RECORD); - - /********************************************************************/ - /* add first entry */ - /********************************************************************/ - memset( pTokBuf, 0, sizeof( pSubstProp->chBuffer )); - - if ( usTagRecordPos < RECLEN(pTagRecord) ) - { - if (pTagEntry->usOffset ) - { - ulCurStop = pTagEntry->usOffset - 1; - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - 0, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - } - else - { - ulCurStop = UTF16strlenCHAR(pData); - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - 0, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - - - - while ( !fEndReached && (usTagRecordPos < RECLEN(pTagRecord) ) - && (usMaxToksInBuffer > 0) ) - { - ULONG ulNewStart; - /******************************************************************/ - /* fill in tag record */ - /******************************************************************/ - ulNewStart = pTagEntry->usOffset; - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - (USHORT)ulNewStart, - (SHORT)TAG_TOKEN, - (USHORT) (ulNewStart + pTagEntry->usTagLen-1), - pData, sLangID, ulOemCP); - - ulNewStart += pTagEntry->usTagLen; - - // pTagEntry->usTagLen is in # of w's, but usTagRecordPos counts bytes - // also pByte is a byte ptr - - usTagRecordPos += sizeof(TMX_TAGENTRY) - + sizeof(CHAR_W) * (pTagEntry->usTagLen); - - pByte += sizeof(TMX_TAGENTRY) + sizeof(CHAR_W) * (pTagEntry->usTagLen); - pTagEntry = (PTMX_TAGENTRY)pByte; - - if ( usTagRecordPos < RECLEN(pTagRecord) ) - { - ulCurStop = pTagEntry->usOffset -1; - } - else - { - ulCurStop = UTF16strlenCHAR( pData ); // we reached the end - fEndReached = TRUE; - } /* endif */ - /******************************************************************/ - /* split text record into word tokens */ - /* and fill into pCurrent struct */ - /******************************************************************/ - if (usMaxToksInBuffer > 0 ) - { - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - (USHORT)ulNewStart, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - } /* endwhile */ - /********************************************************************/ - /* terminate start-stop table */ - /********************************************************************/ - if (usMaxToksInBuffer > 0 ) - { - memset(pCurrent, 0,sizeof(FUZZYTOK)); - - *pusToken = (USHORT)(pCurrent - (PFUZZYTOK) pTokBuf); - - pCurrent++; - - fOK = UtlAlloc( (PVOID *)ppTokData, 0L, (PBYTE)pCurrent - (PBYTE)pTokBuf, NOMSG); - if ( fOK ) - { - memcpy( *ppTokData, pTokBuf, ((PBYTE)pCurrent-(PBYTE)pTokBuf)); - } /* endif */ - } - else - { - fOK = FALSE; - *ppTokData = NULL; - *pusToken = 0; - } /* endif */ - - /********************************************************************/ - /* clean list of tokens to combine consecutive tags only intercepted*/ - /* by whitespaces returned as TEXT_TOKENS (KBT0492) */ - /********************************************************************/ - if ( fOK && (*pusToken > 2) ) - { - USHORT k = 0; - USHORT usNum = *pusToken; - PFUZZYTOK pNew; - CHAR_W chTemp; - pNew = pCurrent = (PFUZZYTOK) *ppTokData; - - while ( k <= usNum ) - { - switch ( pCurrent->sType ) - { - case TAG_TOKEN: - if ( k < usNum - 2 ) - { - PFUZZYTOK pNext = pCurrent+1; - USHORT usTextLen = (pNext->usStop - pNext->usStart + 1); - chTemp = *(pNext->pData + usTextLen); - *(pNext->pData + usTextLen) = EOS; - - if ( (pNext->sType == TEXT_TOKEN) && - ((pCurrent+2)->sType == TAG_TOKEN) && - isWhiteSpaceW( pNext->pData ) ) - { - /**********************************************************/ - /* combine tag - whitespace - tag sequence into one token */ - /**********************************************************/ - *(pNext->pData + usTextLen) = chTemp; - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - - pNext = pCurrent + 2; - usTextLen = (pNext->usStop - pNext->usStart + 1); - chTemp = *(pNext->pData + usTextLen); - *(pNext->pData + usTextLen) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pNew->pData, &pNew->ulHash ); - *(pNext->pData + usTextLen) = chTemp; - - pNew->usStop = (pCurrent+2)->usStop; - pNew++; pCurrent+=3; k+=3; - } - else - { - /**********************************************************/ - /* no special handling necessary */ - /**********************************************************/ - *(pNext->pData + usTextLen) = chTemp; - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - } /* endif */ - } - else - { - /**********************************************************/ - /* no special handling necessary */ - /**********************************************************/ - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - } /* endif */ - break; - case TEXT_TOKEN: - default: - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - break; - } /* endswitch */ - } /* endwhile */ - - /******************************************************************/ - /* adjust length */ - /******************************************************************/ - *pusToken = (USHORT)( pNew- (PFUZZYTOK) *ppTokData ) -1; - - while (pNew < pCurrent ) - { - memset(pNew, 0,sizeof(FUZZYTOK)); - pNew++; - } /* endwhile */ - } /* endif */ - - return fOK; -} - -static PFUZZYTOK -NTMSplitAndAddTokens -( - PTMX_SUBSTPROP pSubstProp, - PFUZZYTOK pstCurrent, - PUSHORT pusMaxFreeToksInBuffer, - USHORT usStart, - SHORT sType, - USHORT usStop, - PSZ_W pString, - SHORT sLangID, - ULONG ulOemCP -) -{ - CHAR_W chTemp; - USHORT usListSize = 0; - TERMLENOFFS* pTermList = NULL; // ptr to created term list - TERMLENOFFS* pActTerm; // actual term - PSZ_W pCurWord; - USHORT usCurStart; - ULONG ulCurStop; - USHORT usRC; - - /********************************************************************/ - /* if token is text token, make one token for each word */ - /********************************************************************/ - if (*pusMaxFreeToksInBuffer > 0) - { - if ( sType == TEXT_TOKEN ) - { - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - - usRC = MorphTokenizeW( sLangID, pString+usStart, - &usListSize, &pTermList, - MORPH_OFFSLIST, ulOemCP ); - - *(pString + (usStop+(USHORT)1)) = chTemp; - - if ( pTermList ) - { - pActTerm = pTermList; - if ( pTermList->iLength>0 && (*pusMaxFreeToksInBuffer > 0)) - { - while ( pActTerm->iLength>0 && (*pusMaxFreeToksInBuffer > 0)) - { - pCurWord = pString + usStart + pActTerm->iOffset; - /**********************************************************/ - /* ignore the linefeeds in the matching */ - /**********************************************************/ - if ( *pCurWord != LF ) - { - usCurStart = (USHORT)(usStart + pActTerm->iOffset); - ulCurStop = usCurStart + pActTerm->iLength - 1; - - pstCurrent->pData = pCurWord; - pstCurrent->usStart = usCurStart; - pstCurrent->sType = sType; - pstCurrent->usStop = (USHORT)ulCurStop; - pstCurrent->fConnected = FALSE; - - chTemp = *(pString + (ulCurStop+(USHORT)1)); - *(pString + (ulCurStop+(USHORT)1)) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pString + usCurStart, &pstCurrent->ulHash ); - *(pString + (ulCurStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - } /* endif */ - pActTerm++; - } /* endwhile */ - } - else - { - pstCurrent->pData = pString+usStart; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStop; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pString + usStart, &pstCurrent->ulHash ); - *(pString + (usStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - - } /* endif */ - } /* endif */ - /****************************************************************/ - /* free allocated resource ... */ - /****************************************************************/ - UtlAlloc( (PVOID *)&pTermList, 0L, 0L, NOMSG ); - } - else - { - pstCurrent->pData = pString+usStart; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStop; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - MakeHashValueW( pSubstProp->ulRandom, MAX_RANDOM, - pString+usStart , &pstCurrent->ulHash ); - *(pString + (usStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - } /* endif */ - } /* endif */ - return (pstCurrent); - -} - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: MakeHashValues | -//+----------------------------------------------------------------------------+ -//|Function call: MakeHashValues( pulRandom, usMaxNum, pszString, pulHash);| -//+----------------------------------------------------------------------------+ -//|Description: build a quasi hash value of the passed string | -//+----------------------------------------------------------------------------+ -//|Parameters: PULONG pulRandom, array of random numbers | -//| USHORT usMaxRandom, maximum random numbers | -//| PSZ pData, ptr to data to be hashed | -//| PULONG pulHashVal resulting hash value | -//+----------------------------------------------------------------------------+ -//|Returncode type: VOID | -//+----------------------------------------------------------------------------+ -//|Function flow: go through the passed string and build hashvalue | -//| of all characters up to the specified max value | -//| Characters not in the alphanumeric range are ignored.. | -//| (Check in case of DBCS might be necessary...) | -//+----------------------------------------------------------------------------+ -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -) -{ - USHORT usRandomIndex = 0; - ULONG ulHashVal = 0; - CHAR_W c; - UCHAR cSingle; - - while ( ((c = *pData++)!= NULC) && (usRandomIndex < usMaxRandom)) - { - cSingle = (UCHAR) c; - if ( !isspace( cSingle ) ) - { - ulHashVal += pulRandom[usRandomIndex++] * c; - } /* endif */ - } /* endwhile */ - /********************************************************************/ - /* normalize hashvalue, just in case it is 0 and we may look at it */ - /* as the end of our further processing.... */ - /********************************************************************/ - if ( ulHashVal == 0L ) - { - ulHashVal = 1L; - } /* endif */ - - *pulHashVal = ulHashVal; - - return; -} /* end of function MakeHashValue */ //+----------------------------------------------------------------------------+ //|Internal function | @@ -1128,886 +110,8 @@ MakeHashValueW //| if we found a fuzzy match, we have to replace it now..| //| endwhile | //+----------------------------------------------------------------------------+ -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -) -{ - PFUZZYTOK pSrcTok; // source token - PFUZZYTOK pEndSrcTok; // search until you detect this token - PFUZZYTOK pEndTgtTok; // search until you detect this token - PFUZZYTOK pPropTok; // proposal token - ULONG ulTgtLen; // target length - PREPLLIST pTempRepl; // pointer to temp repl element - BOOL fFound; // match found - BOOL fReplaced = FALSE; // nothing replaced yet - BOOL fFirstInLoop = TRUE; // sep. handling 1st time - USHORT usNumOfPairs = 0; - BOOL fOK = TRUE; - - pSource; - pProp; - ulTgtLen = UTF16strlenCHAR( pTrans ); - - T5LOG(T5DEBUG) << "NTMFuzzyReplace" ; - pSubstProp->pTagPairs = (PTMX_REPLTAGPAIR) (pSubstProp->chBuffer); - /********************************************************************/ - /* replace found tokens in translation with original ones.... */ - /********************************************************************/ - while ( pReplPropSrc->pSrcTok ) - { - pSrcTok = pReplPropSrc->pTgtTok; - pPropTok = pReplPropSrc->pSrcTok; - -#ifdef INLINE_TAG_REPL_LOGGING - T5LOG(T5DEBUG) << "Testing token"; - NTMListToken( "SourceToken", pReplPropSrc->pSrcTok ); - NTMListToken( "TargetToken", pReplPropSrc->pTgtTok ); -#endif - - /******************************************************************/ - /* if next proposal token is part of the possible replacement list*/ - /* we've found one match ... */ - /* we have to compare until the next match between Source and */ - /* Proposal is detected or the end of the list .... */ - /******************************************************************/ - pEndSrcTok = (pReplPropSrc+1)->pTgtTok; - pEndTgtTok = (pReplPropSrc+1)->pSrcTok; - - if ( !pEndSrcTok ) - { - pEndSrcTok = pSrcTok; - while ( pEndSrcTok->ulHash ) - { - pEndSrcTok++; - } /* endwhile */ - } /* endif */ - - if ( !pEndTgtTok ) - { - pEndTgtTok = pPropTok; - while ( pEndTgtTok->ulHash ) - { - pEndTgtTok++; - } /* endwhile */ - } /* endif */ - /******************************************************************/ - /* adjustment for 1st time, earlier decreasing may affect */ - /* pEndSrcTok/pEndTgtTok, if only 1 token in segment */ - /******************************************************************/ - if ( fFirstInLoop ) /* @KIT1171A */ - { /* @KIT1171A */ - fFirstInLoop = FALSE; /* @KIT1171A */ - pSrcTok--; /* @KIT1171A */ - pPropTok--; /* @KIT1171A */ - } /* endif */ /* @KIT1171A */ - - while ( ++pSrcTok && ++pPropTok && - (pSrcTok < pEndSrcTok ) && (pPropTok < pEndTgtTok) ) - { - pTempRepl = pReplaceList; - fFound = FALSE; - /****************************************************************/ - /* only replace of tagging done here */ - /****************************************************************/ - if ((pSrcTok->sType == TAG_TOKEN ) && (pPropTok->sType == TAG_TOKEN)) - { - while ( pTempRepl->pSrcTok && !fFound ) - { - if ( pTempRepl->pSrcTok == pPropTok ) - { - fFound = TRUE; - pSubstProp->pTagPairs->pSrcTok = (PBYTE)pSrcTok; - pSubstProp->pTagPairs->pPropTok = (PBYTE)pPropTok; - pSubstProp->pTagPairs ++; - usNumOfPairs ++; - } - else - { - pTempRepl++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - } /* endwhile */ - pReplPropSrc++; // point to next one - } /* endwhile */ - if (usNumOfPairs) - { - fReplaced = TRUE; - //allocate for pairs - fOK = UtlAlloc( (PVOID *) &(pSubstProp->pTagPairs), 0L, - (LONG)((usNumOfPairs+1) * sizeof(TMX_REPLTAGPAIR)), NOMSG ); - - if (fOK ) - { - //copy source string for later compare function - memcpy( pSubstProp->pTagPairs, pSubstProp->chBuffer, - (USHORT)( usNumOfPairs * sizeof(TMX_REPLTAGPAIR))); - } /* endif */ - } - else - { - pSubstProp->pTagPairs = NULL; - } /* endif */ - return( fReplaced ); -} /* end of function NTMFuzzyReplace */ - -#ifdef ACTIVATE_NTMGenericDelete - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMGenericDelete | -//+----------------------------------------------------------------------------+ -//|Function call: NTMGenericDelete( pSource, pProp, pTrans, | -//| pReplPropSrc, pReplaceList ); | -//+----------------------------------------------------------------------------+ -//|Description: delete tags | -//+----------------------------------------------------------------------------+ -//|Parameters: PSZ pSource, source string | -//| PSZ pProp, proposal string | -//| PSZ pTrans, translation string | -//| PREPLLIST pReplPropSrc, list of same tokens in source | -//| PREPLLIST pReplaceList list of tokens to be replaced | -//+----------------------------------------------------------------------------+ -//|Returncode type: VOID | -//+----------------------------------------------------------------------------+ -//|Prerequesits: pTrans is large enough to hold a total segment in length | -//| of MAX_SEGMENT size | -//+----------------------------------------------------------------------------+ -//|Function flow: loop thru the list of candidates | -//| endwhile | -//+----------------------------------------------------------------------------+ -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ pSource, // source string - PSZ pProp, // proposal string - PSZ pTrans, // translation string - PREPLLIST pReplaceSource, // list of same toks in src & srcofprop - PREPLLIST pReplaceList // eq. toks in src & tgt of prop -) -{ - PFUZZYTOK pSrcTok; // source token - PFUZZYTOK pEndSrcTok; // search until you detect this token - PFUZZYTOK pEndTgtTok; // search until you detect this token - PFUZZYTOK pSrcOfPropTok; // proposal token - USHORT ulTgtLen; // target length - PREPLLIST pTempRepl; // pointer to temp repl element - BOOL fFound; // match found - BOOL fReplaced = FALSE; // nothing replaced yet - BOOL fFirstInLoop = TRUE; // sep. handling 1st time - USHORT usNumOfPairs = 0; - BOOL fOK = TRUE; - - ulTgtLen = strlen( pTrans ); - pSubstProp->pDelTagPairs = (PTMX_REPLTAGPAIR) (pSubstProp->chBuffer); - /********************************************************************/ - /* replace found tokens in translation with original ones.... */ - /********************************************************************/ - while ( pReplaceSource->pSrcTok ) - { - pSrcTok = pReplaceSource->pTgtTok; // cur source - pSrcOfPropTok = pReplaceSource->pSrcTok; // src of prop - /******************************************************************/ - /* if next proposal token is part of the possible replacement list*/ - /* we've found one match ... */ - /* we have to compare until the next match between Source and */ - /* Proposal is detected or the end of the list .... */ - /******************************************************************/ - pEndSrcTok = (pReplaceSource+1)->pTgtTok; - pEndTgtTok = (pReplaceSource+1)->pSrcTok; - - if ( !pEndSrcTok ) - { - pEndSrcTok = pSrcTok; - while ( pEndSrcTok->ulHash ) - { - pEndSrcTok++; - } /* endwhile */ - pEndSrcTok--; - /*****************************************************************/ - /* force that tags to be deleted at end of proposal are detected */ - /*****************************************************************/ - if (!fFirstInLoop ) - { - pSrcTok = pEndSrcTok; - pSrcTok--; - } /* endif */ - } /* endif */ - - if ( !pEndTgtTok ) - { - pEndTgtTok = pSrcOfPropTok; - while ( pEndTgtTok->ulHash ) - { - pEndTgtTok++; - } /* endwhile */ - pEndTgtTok--; - } /* endif */ - /******************************************************************/ - /* adjustment for 1st time, earlier decreasing may affect */ - /* pEndSrcTok/pEndTgtTok, if only 1 token in segment */ - /******************************************************************/ - if ( fFirstInLoop ) - { - fFirstInLoop = FALSE; - pSrcTok = pEndSrcTok; // force that they are equal - pSrcTok--; - pSrcOfPropTok = (PFUZZYTOK) pSubstProp->pTokPropSource; // nec if tag at begin - pSrcOfPropTok--; - } /* endif */ - - pSrcTok++; - fFound = TRUE; - - while ( pSrcTok && ++pSrcOfPropTok - && (pSrcTok == pEndSrcTok ) && (pSrcOfPropTok < pEndTgtTok) - && fFound ) - { - if (pSrcOfPropTok->sType == TAG_TOKEN ) - { - pTempRepl = pReplaceList; // eq toks in src&tgt of prop - fFound = FALSE; - /****************************************************************/ - /* only delete of tagging in src/tgt of proposal checked here */ - /****************************************************************/ - while ( pTempRepl->pSrcTok && !fFound ) - { - if ( pTempRepl->pSrcTok == pSrcOfPropTok ) - { - /************************************************************/ - /* tgt of proposal is eq to src of prop !! */ - /************************************************************/ - fFound = TRUE; - pSubstProp->pDelTagPairs->pSrcTok = (PBYTE)pTempRepl->pTgtTok; - pSubstProp->pDelTagPairs->pPropTok = (PBYTE)pSrcOfPropTok; - - pSubstProp->pDelTagPairs ++; - usNumOfPairs ++; - } - else - { - pTempRepl++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - } /* endwhile */ - pReplaceSource++; // point to next one - } /* endwhile */ - if (usNumOfPairs) - { - fReplaced = TRUE; - //allocate for pairs - fOK = UtlAlloc( (PVOID *) &(pSubstProp->pDelTagPairs), 0L, - (LONG)((usNumOfPairs+1) * sizeof(TMX_REPLTAGPAIR)), NOMSG ); - - if (fOK ) - { - //copy source string for later compare function - memcpy( pSubstProp->pDelTagPairs, pSubstProp->chBuffer, - (USHORT)( usNumOfPairs * sizeof(TMX_REPLTAGPAIR))); - } /* endif */ - } - else - { - pSubstProp->pDelTagPairs = NULL; - } /* endif */ - return( fReplaced ); -} /* end of function NTMGenericDelete */ -#endif - -//reset replace tag pair used flags -static void NTMResetUsedFlag( PTMX_REPLTAGPAIR pCurTagPair ) -{ - while ( pCurTagPair ) - { - if ( pCurTagPair->pPropTok ) - { - pCurTagPair->fUsed = FALSE; - pCurTagPair++; - } - else - { - pCurTagPair = NULL; - } /* endif */ - } /*endwhile */ -} /* end of function NTMResetUsedFlag */ - - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMCheckTagPairs | -//+----------------------------------------------------------------------------+ -//|Function call: NTMCheckTagPairs(pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: check whether all tags in src and target of proposal | -//| can be replaced | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Prerequesits: - | -//+----------------------------------------------------------------------------+ -//|Function flow: | -//+----------------------------------------------------------------------------+ -static BOOL -NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplaceSourceList, // list of same tokens in source and prop - PREPLLIST pReplaceList, // toklist to be replaced in prop src+tgt - BOOL fRespectLFs -) -{ - BOOL fAllReplace = TRUE; - PTMX_REPLTAGPAIR pCurTagPair = pSubstProp->pTagPairs; - PFUZZYTOK pTempTok; - USHORT k; - PREPLLIST pReplTest = NULL; - PREPLLIST pReplProp = NULL; - USHORT usStart= 0; - - - /********************************************************************/ - /* tags which are same in both srces and in propsrc-tgt, need not */ - /* to be replaced. Hence for ease of use change sType to TEXT_TOKEN */ - /********************************************************************/ - - pReplTest = pReplaceSourceList; - pReplTest++; // 1st entry is dummy - pReplProp = pReplaceList; - while ( pReplTest->pSrcTok) - { - pTempTok = pReplTest->pSrcTok; // toks of src of prop - if (pTempTok->sType == TAG_TOKEN ) - { - usStart = pTempTok->usStart; - /****************************************************************/ - /* tag in src of prop which is equal to a tag in current src */ - /****************************************************************/ - while ( pReplProp->pSrcTok && - (pReplProp->pSrcTok->usStart <= usStart )) - { - if (pReplProp->pSrcTok == pTempTok ) - { - /************************************************************/ - /* tag equal in all three sentence!!! no replace nec. */ - /************************************************************/ - pTempTok->sType = TEXT_TOKEN; - pReplTest->pTgtTok->sType = TEXT_TOKEN; - pReplProp->pTgtTok->sType = TEXT_TOKEN; - } /* endif */ - pReplProp++; - } /* endwhile */ - - } /* endif */ - pReplTest++; - } /* endwhile */ - -#ifdef INLINE_TAG_REPL_LOGGING - PREPLLIST pRepl = pReplaceSourceList; - T5LOG(T5DEBUG) << "Replace source list" ; - while ( pRepl->pSrcTok) - { - NTMListToken( "SourceToken", pRepl->pSrcTok ); - NTMListToken( "TargetToken", pRepl->pTgtTok ); - pRepl++; - } /*endwhile */ - - T5LOG(T5DEBUG) << "Replace list" ; - pRepl = pReplaceList; - while ( pRepl->pSrcTok) - { - NTMListToken( "SourceToken", pRepl->pSrcTok ); - NTMListToken( "TargetToken", pRepl->pTgtTok ); - pRepl++; - } /*endwhile */ -#endif - - /********************************************************************/ - /* check all remaining tags in source of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropSource; - k = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropSource) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - - if (pCurTagPair == NULL ) - { - pCurTagPair = NTMFindTagPair (pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; -#ifdef INLINE_TAG_REPL_LOGGING - - T5LOG(T5DEBUG) << "No tag pair found for" ; - NTMListToken( "Temp token", pTempTok ); -#endif - } /* endif */ - } - else - { - if ( pCurTagPair->fUsed ) - { - // this tag pair has been used already - fAllReplace = FALSE; - } - else - { - pCurTagPair->fUsed = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - /********************************************************************/ - /* check all tags in target of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropTarget; - k = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropTarget) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, pSubstProp->szPropTarget, pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - pCurTagPair = NTMFindTagPair (pTempTok, pSubstProp->szPropTarget, pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { -#ifdef INLINE_TAG_REPL_LOGGING - - T5LOG(T5DEBUG) << "No tag pair found in target" ; - NTMListToken( "Temp token", pTempTok ); -#endif - fAllReplace = FALSE; - } /* endif */ - } - else - { - if ( pCurTagPair->fUsed ) - { - // this tag pair has been used already - fAllReplace = FALSE; - } - else - { - pCurTagPair->fUsed = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - - return(fAllReplace); -} /* end of function NTMCheckTagPairs */ - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMReplaceTags | -//+----------------------------------------------------------------------------+ -//|Function call: NTMReplaceTags (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: build up new src and tgt of proposal with replaced | -//| tagging | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Function flow: replace tags in target of proposal | -//| replace tags in source of proposal | -//+----------------------------------------------------------------------------+ -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -) -{ - BOOL fAllReplace = TRUE; - PTMX_REPLTAGPAIR pCurTagPair = pSubstProp->pTagPairs; - PFUZZYTOK pTempTok; - USHORT k; - PFUZZYTOK pCurPairSrcTok; - SHORT sLen; - PSZ_W pTempData; - LONG lNewLen = 0; - PSZ_W pNewData; - - /********************************************************************/ - /* replace all tags in target of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropTarget; - k = 0; - lNewLen = 0; - pNewData = pSubstProp->chBuffer; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropTarget) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropTarget, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair != NULL ) - { - /****************************************************************/ - /* pTempTok points to a tag token; current pCurTagPair */ - /* contains tag from current active source segment */ - /* which replaces tag from TranslationMemory */ - /****************************************************************/ - pCurTagPair->fUsed = TRUE; - pCurPairSrcTok = (PFUZZYTOK) pCurTagPair->pSrcTok; - pTempData = pSubstProp->szSource + pCurPairSrcTok->usStart; - sLen = pCurPairSrcTok->usStop - pCurPairSrcTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - - if (fAllReplace ) - { - pCurPairSrcTok->fConnected = TRUE; - } /* endif */ - - /****************************************************************/ - /* copy "space" between tagtoken in prop and next tok in prop */ - /****************************************************************/ - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop + 1 < (pTempTok+1)->usStart) ) - { - sLen = (pTempTok+1)->usStart - pTempTok->usStop - 1; - pTempData = pSubstProp->szPropTarget + pTempTok->usStop+1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, - &pNewData, &lNewLen); - } /* endif */ - } /* endif */ - } - else // check whether tag should be deleted - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropTarget, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; - } /* endif */ - } /*endif */ - - } - else - { - /****************************************************************/ - /* pTempTok points to a text token; copy it into target */ - /****************************************************************/ - pTempData = pSubstProp->szPropTarget + pTempTok->usStart; - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop < (pTempTok+1)->usStart) ) - { - /************************************************************/ - /* adjust so that spaces between words are copied too */ - /************************************************************/ - pTempTok->usStop = (pTempTok+1)->usStart - 1; - } /* endif */ - } - else - { - // we are at the end of the token list, ensure that we do not miss any trailing blanks - pTempTok->usStop = (USHORT)(pTempTok->usStart + wcslen( pTempData ) - 1); - } /* endif */ - sLen = pTempTok->usStop - pTempTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - if (fAllReplace ) - { - *pNewData = EOS; - lNewLen++; - UTF16strcpy ( pSubstProp->szPropTarget, pSubstProp->chBuffer); - } /* endif */ - - /********************************************************************/ - /* replace all tags in source of proposal */ - /* build new string in chBuffer and copy it at the end in szPropSource*/ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropSource; - k = 0; - pNewData = pSubstProp->chBuffer; - lNewLen = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropSource) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair != NULL ) - { - /****************************************************************/ - /* pTempTok points to a tag token; current pCurTagPair */ - /* contains tag from current active source segment */ - /* which replaces tag from TranslationMemory */ - /****************************************************************/ - pCurTagPair->fUsed = TRUE; - pCurPairSrcTok = (PFUZZYTOK) pCurTagPair->pSrcTok; - pTempData = pSubstProp->szSource + pCurPairSrcTok->usStart; - sLen = pCurPairSrcTok->usStop - pCurPairSrcTok->usStart +1; - - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - /****************************************************************/ - /* copy "space" between tagtoken in prop and next tok in prop */ - /****************************************************************/ - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop + 1 < (pTempTok+1)->usStart) ) - { - sLen = (pTempTok+1)->usStart - pTempTok->usStop - 1; - pTempData = pSubstProp->szPropSource + pTempTok->usStop+1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, - &pNewData, &lNewLen); - } /* endif */ - } /* endif */ - } - else - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; - } /* endif */ - } /*endif */ - } - else - { - /****************************************************************/ - /* pTempTok points to a text token; copy it into target */ - /****************************************************************/ - pTempData = pSubstProp->szPropSource + pTempTok->usStart; - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop < (pTempTok+1)->usStart) ) - { - /************************************************************/ - /* adjust so that spaces between words are copied too */ - /************************************************************/ - pTempTok->usStop = (pTempTok+1)->usStart - 1; - } /* endif */ - } - else - { - // we are at the end of the token list, ensure that we do not miss any trailing blanks - pTempTok->usStop = (USHORT)(pTempTok->usStart + wcslen( pTempData ) - 1); - } /* endif */ - sLen = pTempTok->usStop - pTempTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - if (fAllReplace ) - { - *pNewData = EOS; - lNewLen++; - UTF16strcpy ( pSubstProp->szPropSource, pSubstProp->chBuffer); - /******************************************************************/ - /* after this strcpy ALL pCurTagPair->pSrcTok and ALL pSrcTok */ - /* which point on szPropSource are INVALID */ - /* Therefore replace in szPropTarget takes place before */ - /* replace in szPropSource !!! */ - /******************************************************************/ - } /* endif */ - return(fAllReplace); -} /* end of function NTMReplaceTags */ - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMFindTagPair | -//+----------------------------------------------------------------------------+ -//|Function call: NTMFindTagPair (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: find tag pair which has to be replaced by each other | -//+----------------------------------------------------------------------------+ -//|Parameters: PFUZZYTOK pTempTok, | -//| PSZ pTempTokData, | -//| PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: PTMX_REPLTAGPAIR | -//+----------------------------------------------------------------------------+ -//|Function flow: find tag pair | -//+----------------------------------------------------------------------------+ -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs - -) -{ - PTMX_REPLTAGPAIR pFoundTagPair = NULL; - BOOL fFound = FALSE; - SHORT sLen; - PFUZZYTOK pCurPairPropTok; - PSZ_W pPairData; - PSZ_W pTempData; - - - pTempData = pTempTokData + pTempTok->usStart; // prop src or tgt - sLen = pTempTok->usStop - pTempTok->usStart +1; - if (pCurTagPair ) - { - pCurPairPropTok = (PFUZZYTOK) pCurTagPair->pPropTok; //src of proposal - - while (!fFound && pCurPairPropTok ) - { - if ( pCurTagPair->fUsed ) - { - // ignore this entry - } - else if (pCurPairPropTok == pTempTok ) - { - fFound = TRUE; - } - else - { - pPairData = pSubstProp->szPropSource + pCurPairPropTok->usStart; - if (sLen == (pCurPairPropTok->usStop -pCurPairPropTok->usStart + 1 )) - { - if ( fRespectLFs ) - { - if ( memcmp( (PBYTE)pTempData,(PBYTE)pPairData, sLen * sizeof(CHAR_W)) == 0) - { - fFound = TRUE; - } /* endif */ - } - else - { - CHAR_W chTemp1 = pTempTokData[pTempTok->usStop+1]; - CHAR_W chTemp2 = pSubstProp->szPropSource[pCurPairPropTok->usStop+1]; - pTempTokData[pTempTok->usStop+1] = 0; - pSubstProp->szPropSource[pCurPairPropTok->usStop+1] = 0; - - T5LOG(T5FATAL) << "TEMPORARY_COMMENTED in NTMFindTagPair::UtlCompIgnWhiteSpaceW"; - #ifdef TEMPORARY_COMMENTED - fFound = (UtlCompIgnWhiteSpaceW( pTempData, pPairData, 0 ) == 0 ); - #endif - pTempTokData[pTempTok->usStop+1] = chTemp1; - pSubstProp->szPropSource[pCurPairPropTok->usStop+1] = chTemp2; - } /* endif */ - } /* endif */ - } /* endif */ - if (!fFound ) - { - pCurTagPair++; - if (pCurTagPair ) - { - pCurPairPropTok = (PFUZZYTOK)pCurTagPair->pPropTok; - } - else - { - pCurPairPropTok = NULL; - } /* endif */ - } - else - { - pFoundTagPair = pCurTagPair; - } /* endif */ - } /* endwhile */ - - } /* endif */ - return(pFoundTagPair); -} /* end of function NTMFindTagPair */ - - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMCopyTokData | -//+----------------------------------------------------------------------------+ -//|Function call: NTMCopyTokdata (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: copy data into the new buffer | -//+----------------------------------------------------------------------------+ -//|Parameters: PSZ pTempData, | -//| SHORT sLen, | -//| PSZ * ppNewData, | -//| PSHORT pulNewLen | -//+----------------------------------------------------------------------------+ -//|Returncode type: PTMX_REPLTAGPAIR | -//+----------------------------------------------------------------------------+ -//|Function flow: copy data | -//+----------------------------------------------------------------------------+ -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W *ppNewData, - PLONG plNewLen - -) -{ - BOOL fAllReplace = TRUE; - PSZ_W pData; - - pData = *ppNewData; - - if (*plNewLen + sLen < MAX_SEGMENT_SIZE ) - { - memcpy((PBYTE)pData, (PBYTE)pTempData, sLen*sizeof(CHAR_W)); - pData += sLen; - *ppNewData = pData; - *(plNewLen) += sLen; - } - else - { - fAllReplace = FALSE; - } /*endif */ - - return (fAllReplace); -} typedef struct _NTMGETMATCHLEVELDATA { @@ -2029,101 +133,3 @@ static ULONG ulLastSrcOemCP = 0; static ULONG ulLastTgtOemCP = 0; - - - -// free tag substitution data area -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ) -{ - if ( pSubstProp ) - { - if ( pSubstProp->pTokSource ) - UtlAlloc( (PVOID*) &pSubstProp->pTokSource, 0L, 0L, NOMSG ); - if ( pSubstProp->pTokPropSource ) - UtlAlloc( (PVOID*) &pSubstProp->pTokPropSource, 0L, 0L, NOMSG ); - if ( pSubstProp->pTokPropTarget ) - UtlAlloc( (PVOID*) &pSubstProp->pTokPropTarget, 0L, 0L, NOMSG ); - if ( pSubstProp->pTagPairs ) - UtlAlloc( (PVOID*) &pSubstProp->pTagPairs, 0L, 0L, NOMSG ); - if ( pSubstProp->pDelTagPairs ) - UtlAlloc( (PVOID*) &pSubstProp->pDelTagPairs, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID*)&pSubstProp, 0L, 0L, NOMSG); - } /* endif */ - return; -} /* end of function NTMFreeSubstProp */ - - -// function aligning tags between source and target of a proposal -BOOL NTMAlignTags -( - PFUZZYTOK pTokListSource, // list of proposal source token - PFUZZYTOK pTokListTarget, // list of proposal target token - PREPLLIST *ppReplaceList // adress of caller's replacement list pointer -) -{ - PREPLLIST pReplaceList = NULL; // pointer to replace list - int iSourceTags = 0; // number of tags in source token list - PFUZZYTOK pTokSource = pTokListSource; // pointer for source token processing - PFUZZYTOK pTokTarget = pTokListTarget; // pointer for target token processing - BOOL fOK = TRUE; // function return code - - // count number of tags in source of proposal - while ( pTokSource->ulHash ) - { - if ( pTokSource->sType == TAG_TOKEN ) - { - iSourceTags++; - } /* endif */ - pTokSource++; - } /*endwhile */ - - // allocate replacement list - int iSize = MAX_REPL > iSourceTags? MAX_REPL : iSourceTags; - iSize++; - fOK = UtlAlloc( (PVOID *)ppReplaceList, 0L, iSize * sizeof(REPLLIST), ERROR_STORAGE ); - pReplaceList = *ppReplaceList; - - // look for source tags in proposal target - if ( fOK ) - { - PREPLLIST pReplaceEntry = pReplaceList; - - pTokSource = pTokListSource; - pTokTarget = pTokListTarget; - while ( pTokSource->ulHash ) - { - if ( pTokSource->sType == TAG_TOKEN ) - { - BOOL fFound = FALSE; - PFUZZYTOK pTargetStart = pTokTarget; - do - { - if ( (pTokTarget->sType == TAG_TOKEN) && (pTokTarget->ulHash == pTokSource->ulHash) && !pTokTarget->fConnected ) - { - pReplaceEntry->pSrcTok = pTokSource; - pReplaceEntry->pTgtTok = pTokTarget; - pTokSource->fConnected = TRUE; - pTokTarget->fConnected = TRUE; - pReplaceEntry++; - fFound = TRUE; - } - else - { - // try next target token, wrap-around at end of list - pTokTarget++; - if ( !pTokTarget->ulHash ) - { - pTokTarget = pTokListTarget; - } /* endif */ - } /* endif */ - } while ( !fFound && (pTargetStart != pTokTarget) ); // while not found and not through list - } /* endif */ - pTokSource++; - } /*endwhile */ - - pTokTarget = pTokListTarget; - pReplaceList = NULL; // pointer to replace list - } /* endif */ - - return( fOK ); -} /* end of function NTMAlignTags */ diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index f2d50579..04354d51 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -299,7 +299,7 @@ size_t str16len(const char16_t* source) // helper function for the access to the name mapping tables BOOL FindName( PNAMETABLE pTable, const char *pszName, char *pszValue, int iBufSize ); -void EscapeXMLChars( PSZ_W pszText, PSZ_W pszBuffer ); +void EscapeXMLChars( const wchar_t* pszText, wchar_t* pszBuffer ); BOOL CheckEntityW( PSZ_W pszText, int *piValue, int *piLen ); BOOL CheckEntity( PSZ pszText, int *piValue, int *piLen ); void CopyTextAndProcessMaskedEntitites( PSZ_W pszTarget, PSZ_W pszSource ); @@ -434,7 +434,7 @@ std::wstring TagReplacer::PrintTag(TagInfo& tag){ res += '/'; res += ">"; T5LOG( T5DEVELOP) << ":: generated tag = " << res; - return EncodingHelper::convertToWChar(res.c_str()); + return EncodingHelper::convertToWChar(res); } bool is_number(std::u16string s) @@ -1321,6 +1321,7 @@ USHORT CTMXExportImport::ImportNext errorCount = m_parser->getErrorCount() - m_handler->getInvalidCharacterErrorCount(); if(pImportData != nullptr){ + std::scoped_lock l(pImportData->importDetailsMutex); pImportData->invalidSymbolErrors = m_handler->getInvalidCharacterErrorCount(); if(pImportData->importRc) errorCount++; @@ -1369,6 +1370,7 @@ USHORT CTMXExportImport::ImportNext //m_handler->GetErrorText( m_pMemInfo->szError, sizeof(m_pMemInfo->szError) ); m_pMemInfo->fError = TRUE; + std::scoped_lock l(pImportData->importDetailsMutex); pImportData->importMsg << buff <<";;; "; }else if(pImportData->importRc){ //std::string errormsg = "Error occured with rc = " + std::to_string(m_handler->iStopImportWRc); @@ -1726,13 +1728,25 @@ void StringTagVariants::initParser(){ parser.setExitOnFirstFatalError( false ); } +inline void removeTrailingNull(std::string& str) { + if (!str.empty() && str.back() == '\0') { + str.pop_back(); + } +} + void StringTagVariants::parseSrc(){ - src = std::string("") + EncodingHelper::convertToUTF8(original) + std::string(""); + std::string originalSrc = EncodingHelper::convertToUTF8(original) ; + //std::string sanitizedOriginalSrc; + //std::remove_copy_if(originalSrc.begin(), originalSrc.end(), std::back_inserter(sanitizedOriginalSrc), + // [](char c) { return c == '\0'; }); + removeTrailingNull(originalSrc); + src = std::string("") + /*sanitizedOriginalSrc*/ originalSrc + std::string(""); + T5LOG( T5DEBUG) << ":: parsing request str = \'" << src << "\'"; xercesc::MemBufInputSource req_buff((const XMLByte *)src.c_str(), src.size(), "req_buff (in memory)"); parser.parse(req_buff); - if(parser.getErrorCount()){ + if(int errCount = parser.getErrorCount()){ char buff[512]; handler.GetErrorText(buff, sizeof(buff)); T5LOG(T5ERROR) << ":: error during parsing req("<< src <<") : " << buff; @@ -1755,7 +1769,12 @@ void StringTagVariants::parseTrg(){ }else{ handler.tagReplacer.activeSegment = TARGET_SEGMENT; //handler.fCreateNormalizedStr = false;//not needed for target - trg = std::string("") + EncodingHelper::convertToUTF8(originalTarget) + std::string(""); + std::string originalTrg = EncodingHelper::convertToUTF8(originalTarget); + //std::string sanitizedOriginalTrg; + //std::remove_copy_if(originalTrg.begin(), originalTrg.end(), std::back_inserter(sanitizedOriginalTrg), + // [](char c) { return c == '\0'; }); + removeTrailingNull(originalTrg); + trg = std::string("") + /*sanitizedOriginalTrg*/ originalTrg + std::string(""); xercesc::MemBufInputSource trg_buff((const XMLByte *)trg.c_str(), trg.size(), "trg_buff (in memory)"); parser.parse(trg_buff); @@ -2868,7 +2887,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) if( len >= startTextOffset + tagStr.length() ){ if( wcsncmp(pStartText, tagStr.c_str(), tagStr.length()) == 0 ){ - T5LOG(T5INFO) << "found inclosing tag at the start \"" << EncodingHelper::convertToUTF8(tagStr.c_str()) << "\" \"" + T5LOG(T5INFO) << "found inclosing tag at the start \"" << EncodingHelper::convertToUTF8(tagStr) << "\" \"" <<"\" - > erasing it and " << (startTextOffset) << " whitespaces"; //skip begin and end tags int newLen = len- tagStr.length() - startTextOffset; @@ -2901,7 +2920,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) wchar_t* segEnd = pEndText-tagStr.size(); if( wcsncmp(segEnd, tagStr.c_str(), tagStr.length()) == 0 ){ - T5LOG(T5INFO) << "found inclosing tag at the end \"" << EncodingHelper::convertToUTF8(tagStr.c_str()) << "\" \"" + T5LOG(T5INFO) << "found inclosing tag at the end \"" << EncodingHelper::convertToUTF8(tagStr) << "\" \"" <<"\" - > erasing it and " << (whistespacesAtTheEnd) << " whitespaces"; //skip end tags segEnd[0] = 0; @@ -2961,8 +2980,8 @@ void TMXParseHandler::endElement(const XMLCh* const name ) if( wcsncmp(pStartText, begPairTag.c_str(), begPairTag.size()) == 0 && wcsncmp (segEnd, endPairTag.c_str(), endPairTag.size()) == 0 ){ T5LOG(T5INFO) << "found paired inclosing tags \"" - << EncodingHelper::convertToUTF8(begPairTag.c_str()) << "\": \"" - << EncodingHelper::convertToUTF8(endPairTag.c_str()) + << EncodingHelper::convertToUTF8(begPairTag) << "\": \"" + << EncodingHelper::convertToUTF8(endPairTag) <<"\" - > erasing them and whitespaces: " << (whitespacesAtTheStart + whistespacesAtTheEnd); int newLen = len - inclosingTagsLenSum - whitespacesAtTheStart - whistespacesAtTheEnd; //skip begin and end tags @@ -2983,7 +3002,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) result = EncodingHelper::convertToUTF8(pBuf->szData); T5LOG( T5DEVELOP) << ":: parsed end of TMX sentence, result = \'" << result << "\' -> fSkipped inclosed tag pair = " << inclosingTagsSkipped <<"; Generated tags first = \'" << - EncodingHelper::convertToUTF8(begPairTag) << "last tag =\'" << EncodingHelper::convertToUTF8(endPairTag.c_str()), "\'"; + EncodingHelper::convertToUTF8(begPairTag) << "last tag =\'" << EncodingHelper::convertToUTF8(endPairTag), "\'"; } } @@ -3183,18 +3202,15 @@ void TMXParseHandler::endElement(const XMLCh* const name ) void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t length) { - char16_t* pzChars = (char16_t*)chars; - std::string sChars = EncodingHelper::convertToUTF8(pzChars); - std::wstring wChars = EncodingHelper::convertToWChar(sChars.c_str()); - char* c_chars = (char*) sChars.c_str(); - wchar_t* w_chars = (wchar_t*) wChars.c_str(); - int iLength = length; + char* utf8_chars = xercesc::XMLString::transcode(chars); + std::wstring wChars = EncodingHelper::convertToWChar(utf8_chars); int iwLength = wChars.length(); + int iCurLen = 0; if ( this->fTMXTagStarted ) { // close open TMX inline tag - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + 2) < DATABUFFERSIZE) { pBuf->szData[iCurLen++] = '>'; @@ -3203,7 +3219,7 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt this->fTMXTagStarted = FALSE; } /* endif */ - T5LOG( T5DEVELOP)<< "TMXParseHandler::characters " < iwFree ) iwLength = iwFree; - wcsncpy( pBuf->szPropW + iwCurLength, w_chars, iwLength ); + wcsncpy( pBuf->szPropW + iwCurLength, wChars.c_str(), iwLength ); pBuf->szPropW[iwCurLength+iwLength] = 0; } else if ( fCatchData && (CurElement.ID == INVCHAR_ELEMENT) ) { // add invalid char - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + 1 + 1) < DATABUFFERSIZE) { - LONG lChar = _wtol( w_chars ); + LONG lChar = _wtol( wChars.c_str() ); if ( lChar != 0 ) { USHORT usChar = (USHORT)lChar; @@ -3234,11 +3250,11 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt else if ( fCatchData && (fWithTagging || !CurElement.fInlineTagging) ) { // add data to current data buffer - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + length + 1) < DATABUFFERSIZE) { // escape any special characters in inline tags - EscapeXMLChars( w_chars, pBuf->szData + iCurLen ); + EscapeXMLChars( wChars.c_str(), pBuf->szData + iCurLen ); if(fCreateNormalizedStr){ wcscat(pBuf->szReplacedNpData, pBuf->szData + iCurLen); wcscat(pBuf->szNormalizedData, pBuf->szData + iCurLen); @@ -3247,6 +3263,8 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt if ( CurElement.fInlineTagging ) this->fInlineTags = TRUE; } /* endif */ + + xercesc::XMLString::release(&utf8_chars); } void TMXParseHandler::fatalError(const SAXParseException& exception) @@ -3260,10 +3278,12 @@ void TMXParseHandler::fatalError(const SAXParseException& exception) T5LOG(T5ERROR) << "Fatal Error: " << message <<" at column " << col <<" in line "<< line<<""; sprintf( this->pBuf->szErrorMessage, "Fatal Error: %s at column %ld in line %ld", message, col, line ); if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: " << this->pBuf->szErrorMessage << ";"; } }else{ if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: INVCHAR: " << message <<" at column " << col <<" in line "<< line<<";"; } @@ -3297,10 +3317,12 @@ void TMXParseHandler::fatalInternalError(const SAXException& exception) col = m_locator->getColumnNumber(); } this->fError = TRUE; - sprintf( this->pBuf->szErrorMessage, " Fatal internal Error at column %ld in line %ld, import stopped at progress = %i%, errorMsg: %s ", + { + std::scoped_lock l(pImportDetails->importDetailsMutex); + sprintf( this->pBuf->szErrorMessage, " Fatal internal Error at column %ld in line %ld, import stopped at progress = %i%, errorMsg: %s ", col, line, (int)pImportDetails->usProgress, msg.c_str() ); - msg = std::string(pBuf->szErrorMessage); - if(pImportDetails){ + msg = std::string(pBuf->szErrorMessage); + pImportDetails->importMsg << msg; } @@ -3315,7 +3337,8 @@ void TMXParseHandler::error(const SAXParseException& exception) long col = (long)exception.getColumnNumber(); this->fError = TRUE; sprintf( this->pBuf->szErrorMessage, "Error: %s at column %ld in line %ld", message, col, line ); - if(pImportDetails){ + if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: " << this->pBuf->szErrorMessage << ";"; } T5LOG(T5ERROR) << ": "<< this->pBuf->szErrorMessage; @@ -3328,7 +3351,8 @@ void TMXParseHandler::warning(const SAXParseException& exception) long line = (long)exception.getLineNumber(); long col = (long)exception.getColumnNumber(); T5LOG(T5WARNING) << "Warning: "<< message << " at column "<< col << " in line " << line << ""; - if(pImportDetails){ + if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "Warning: "<< message << " at column "<< col << " in line " << line << ";"; } XMLString::release( &message ); @@ -3803,7 +3827,7 @@ USHORT APIENTRY WRITEEXPSEGMENT( LONG lMemHandle, PMEMEXPIMPSEG pSegment ) } // helper function: is character a list delimiter or space -static BOOL isDelimiter( CHAR chTest ) +BOOL isDelimiter( CHAR chTest ) { return ( (chTest == '(') || (chTest == ')') || (chTest == ',') || (chTest == ' ') || (chTest == EOS) ); } @@ -4967,8 +4991,9 @@ PSZ_W MADSkipWhitespace( PSZ_W pszData ) } // escape characters in segment data to form valid XML -void EscapeXMLChars( PSZ_W pszText, PSZ_W pszBuffer) +void EscapeXMLChars(const wchar_t* pszInput, wchar_t* pszBuffer) { + wchar_t* pszText = (wchar_t*)pszInput; while ( *pszText ) { if ( *pszText == L'\n' ) diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index f95d298e..76e54574 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -79,100 +79,6 @@ PSZ_W MADSkipName( PSZ_W pszName ); PSZ_W MADSkipWhitespace( PSZ_W pszData ); // -// Delete a key with all its atttributes and data -// -BOOL MADDeleteKey( HADDDATAKEY pKey ) -{ - PSZ_W pszCurPos = pKey; - BOOL fAttrAvailable = FALSE; - PSZ_W pszTagName = NULL; - PSZ_W pszTagNameEnd = NULL; - - // check key parameter - if ( (pszCurPos == NULL) || (*pszCurPos == 0) ) - return( FALSE ); - - // skip tag name - if ( *pszCurPos == L'<' ) pszCurPos++; - pszTagName = pszCurPos; - pszTagNameEnd = pszCurPos = MADSkipName( pszCurPos ); - do - { - fAttrAvailable = MADNextAttr( &pszCurPos ); - } while ( fAttrAvailable ); /* enddo */ - - // find end of key data - if ( (*pszCurPos == L'/') && (*(pszCurPos+1) == L'>') ) - { - // a selfcontained tag... so remove everything up to end of the tag - PSZ_W pszSource = pszCurPos + 2; - PSZ_W pszTarget = pKey; - while ( *pszSource ) *pszTarget++ = *pszSource++; - *pszTarget = 0; - return( TRUE ); - } - - // find ending tag - // note: this is a very simple approach which only looks for the ending tag and does not interpret the data in any way... - while ( *pszCurPos != 0 ) - { - // move to begin of next tag - while ( (*pszCurPos != 0) && (*pszCurPos != L'<') ) *pszCurPos++; - - // check for end tag start - if ( (*pszCurPos == L'<') && (*(pszCurPos+1) == L'/') ) - { - // check if we have the correct end tag - if ( MADCompareKey( pszCurPos + 2, pszTagName ) ) - { - // end tag found, now skip end tag name and closing pointy brace - pszCurPos = MADSkipName( pszCurPos + 2 ); - pszCurPos = MADSkipWhitespace( pszCurPos ); - if ( *pszCurPos == L'>' ) pszCurPos++; - - // remove all data between (and including) start tag and end tag - PSZ_W pszSource = pszCurPos; - PSZ_W pszTarget = pKey; - while ( *pszSource ) *pszTarget++ = *pszSource++; - *pszTarget = 0; - - // removal of tag has been completed - return( TRUE ); - } /* endif */ - } /* endif */ - pszCurPos++; - } /* endwhile */ - - return( FALSE ); -} - - -// Add a match segment ID to the additional data section -BOOL MADAddMatchSegID( PSZ_W pszAddData, PSZ_W pszMatchIDPrefix, ULONG ulNum, BOOL fForce ) -{ - HADDDATAKEY hKey; - BOOL fMatchIDAdded = FALSE; - - // find any existing match segment ID - hKey = MADSearchKey( pszAddData, MATCHSEGID_KEY ); - if ( (hKey != NULL) && fForce ) - { - MADDeleteKey( hKey ); - hKey = NULL; - } /* endif */ - - // add match segment ID if non exists or the existing has been deleted and there is enough room left in the AddInfo buffer - if ( hKey == NULL ) - { - int iCurLen = wcslen(pszAddData); - if ( (wcslen(pszMatchIDPrefix) + iCurLen + 23) < MAX_SEGMENT_SIZE ) - { - swprintf( pszAddData + iCurLen, wcslen(pszAddData) - iCurLen, L"", pszMatchIDPrefix, ulNum ); - fMatchIDAdded = TRUE; - } /* endif */ - } /* endif */ - return( fMatchIDAdded ); -} //+----------------------------------------------------------------------------+ @@ -198,9 +104,7 @@ BOOL MADAddMatchSegID( PSZ_W pszAddData, PSZ_W pszMatchIDPrefix, ULONG ulNum, BO //+----------------------------------------------------------------------------+ BOOL NTMDocMatch ( - PSZ pszDoc1ShortName, // ptr to short name of 1st doc PSZ pszDoc1LongName, // ptr to long name of 1st doc - PSZ pszDoc2ShortName, // ptr to short name of 2nd doc PSZ pszDoc2LongName // ptr to long name of 2nd doc ) { @@ -213,95 +117,9 @@ BOOL NTMDocMatch // base the match on the long names fMatch = (strcmp( pszDoc1LongName, pszDoc2LongName ) == 0 ); } - else - { - // use the short names for the match - fMatch = (strcmp( pszDoc1ShortName, pszDoc2ShortName ) == 0 ); - } /* endif */ return( fMatch ); } /* end of function NTMDocMatch */ - -BOOL -PrepareTokens -( - PLOADEDTABLE pTagTable, - PBYTE pInBuf, - PBYTE pTokBuf, - PSZ_W pString, // pointer to string to be tokenized - SHORT sLanguageId, // language ID - PFUZZYTOK * ppTransTokList, // resulting list of tokens - ULONG ulOemCP -); - - - - -BOOL EQFBCallLCS -( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 - USHORT usLenStr1, - USHORT usLenStr2, - BOOL fCompareAll -); - -BOOL EQFBMarkModDelIns -( - PFUZZYTOK pTokenList1, - PFUZZYTOK pTokenList2, - PFUZZYTOK * ppFuzzy1, - PFUZZYTOK * ppFuzzy2, - USHORT usLenStr1, - USHORT usLenStr2 -); - -static -BOOL CheckForAlloc( PTMX_SENTENCE pSentence, PTMX_TERM_TOKEN * ppTermTokens, USHORT usEntries ); - -static -BOOL CheckForAlloc -( - PTMX_SENTENCE pSentence, - PTMX_TERM_TOKEN * ppTermTokens, - USHORT usEntries -) -{ - LONG lFilled = 0L; - USHORT usNewAlloc = 0; - BOOL fOK = TRUE; - PTMX_TERM_TOKEN pTermTokens; - - pTermTokens = *ppTermTokens; - lFilled = ( (PBYTE)pTermTokens - (PBYTE)pSentence->pTermTokens); - if ( (pSentence->lTermAlloc - lFilled) <= (LONG)(usEntries * sizeof(TMX_TERM_TOKEN)) ) - { - //remember offset of pTagEntry - lFilled = pTermTokens - pSentence->pTermTokens; - if (usEntries == 1) - { - usNewAlloc = TOK_SIZE; - } - else - { - usNewAlloc = usEntries * sizeof(TMX_TERM_TOKEN); - } - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTermTokens, pSentence->lTermAlloc, - pSentence->lTermAlloc + (LONG)usNewAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTermAlloc += (LONG)usNewAlloc; - - //set new position of pTagEntry - pTermTokens = pSentence->pTermTokens + lFilled; - } /* endif */ - } /* endif */ - *ppTermTokens= pTermTokens; - return(fOK); -} - - //------------------------------------------------------------------------------ // External function //------------------------------------------------------------------------------ @@ -352,22 +170,22 @@ USHORT EqfMemory::TokenizeSource PSZ pSourceLang // source language ) { - PVOID pTokenList = NULL; // ptr to token table - BOOL fOK; // success indicator - PBYTE pTagEntry; // pointer to tag entries - PTMX_TERM_TOKEN pTermTokens; // pointer to term tokens + std::vector pTokenList; // ptr to token table + BOOL fOK = true; // success indicator + TMX_TAGENTRY TagEntry ; // pointer to tag entries + //PTMX_TERM_TOKEN pTermTokens = nullptr; // pointer to term tokens PLOADEDTABLE pTable = NULL; // pointer to tagtable - PTMX_TAGTABLE_RECORD pTagRecord; // pointer to record USHORT usLangId; // language id USHORT usRc = NO_ERROR; // returned value - USHORT usFilled; // counter + size_t sFilled; // counter USHORT usTagEntryLen; // length indicator - CHAR szString[MAX_FNAME]; // name without extension + std::string szString; // name without extension USHORT usStart; // position counter - PSTARTSTOP pStartStop = NULL; // ptr to start/stop table + std::vector pStartStop; // ptr to start/stop table int iIterations = 0; USHORT usAddEntries = 0; PSZ_W pStringToTokenize = pSentence->pStrings->getNormStrC();//getNormStrC();//getNpReplStrC() + szString.resize(MAX_FNAME); /********************************************************************/ /* normalize \r\n combinations in input string .. */ /********************************************************************/ @@ -403,33 +221,27 @@ USHORT EqfMemory::TokenizeSource } //allocate 4K pTokenlist for TaTagTokenize - fOK = UtlAlloc( (PVOID *) &(pTokenList), 0L, (LONG)TOK_SIZE, NOMSG ); + pTokenList.reserve(TOK_SIZE/sizeof(pTokenList[0])); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { - pTagRecord = pSentence->pTagRecord; - pTermTokens = pSentence->pTermTokens; - pTagEntry = (PBYTE)pTagRecord; - pTagEntry += sizeof(TMX_TAGTABLE_RECORD); + + //pTermTokens = pSentence->pTermTokens.data(); + //pTagEntry = (PTMX_TAGENTRY) ((PBYTE)pTagRecord + sizeof(TMX_TAGTABLE_RECORD)); - RECLEN(pTagRecord) = 0; - pTagRecord->usFirstTagEntry = (USHORT)(pTagEntry - (PBYTE)pTagRecord); + RECLEN(toTmxTagtableRecord(pSentence->pTagRecord)) = 0; + toTmxTagtableRecord(pSentence->pTagRecord)->usFirstTagEntry = pSentence->pTagRecord.size();// (USHORT)((PBYTE)pTagEntry - (PBYTE)pTagRecord); //get id of tag table, call - Utlstrccpy( szString, UtlGetFnameFromPath( pTagTableName ), DOT ); + Utlstrccpy( szString.data(), UtlGetFnameFromPath( pTagTableName ), DOT ); - pTagRecord->usTagTableId = 1; + toTmxTagtableRecord(pSentence->pTagRecord)->usTagTableId = 1; //get lang id of source lang for morphtokenize call usLangId = 0; if ( !usRc ) { //load tag table for tokenize function - usRc = TALoadTagTableExHwnd( szString, &pTable, FALSE, + usRc = TALoadTagTableExHwnd( szString.data(), &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC | TALOADCOMPCONTEXTFUNC, FALSE, NULLHANDLE ); @@ -439,11 +251,11 @@ USHORT EqfMemory::TokenizeSource LOG_AND_SET_RC(usRc, T5INFO, ERROR_TA_ACC_TAGTABLE); if ( usAction == MEMIMP_MRKUP_ACTION_RESET ) { CHAR *ptrMarkup ; - ptrMarkup = strstr( pTagTableName, szString ) ; + ptrMarkup = strstr( pTagTableName, szString.data() ) ; if ( ptrMarkup ) { strcpy( ptrMarkup, "OTMUTF8.TBL" ) ; - strcpy( szString, "OTMUTF8" ) ; - usRc = TALoadTagTableExHwnd( szString, &pTable, FALSE, + strcpy( szString.data(), "OTMUTF8" ) ; + usRc = TALoadTagTableExHwnd( szString.data(), &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC | TALOADCOMPCONTEXTFUNC, FALSE, NULLHANDLE ); @@ -463,46 +275,17 @@ USHORT EqfMemory::TokenizeSource // build protect start/stop table for tag recognition usRc = TACreateProtectTableWEx( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, - TOK_SIZE, &pStartStop, - pTable->pfnProtTable, + pTokenList, + TOK_SIZE, pStartStop, + //pTable->pfnProtTable, pTable->pfnProtTableW, 1, 0); - - - while ((iIterations < 10) && (usRc == EQFRS_AREA_TOO_SMALL)) - { - // (re)allocate token buffer - LONG lOldSize = (usAddEntries * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - LONG lNewSize = ((usAddEntries+128) * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - - if (UtlAlloc((PVOID *) &pTokenList, lOldSize, lNewSize, NOMSG) ) - { - usAddEntries += 128; - iIterations++; - } - else - { - iIterations = 10; // force end of loop - } /* endif */ - // retry tokenization - if (iIterations < 10 ) - { - usRc = TACreateProtectTableWEx( pStringToTokenize, - pTable, 0, - (PTOKENENTRY)pTokenList, - (USHORT)lNewSize, &pStartStop, - pTable->pfnProtTable, - pTable->pfnProtTableW, 1, 0 ); - } /* endif */ - - } /* endwhile */ } /* endif */ if ( !usRc ) { USHORT usEntries = 0; // number of entries in start stop table - PSTARTSTOP pEntry = pStartStop; + PSTARTSTOP pEntry = (PSTARTSTOP)pStartStop.data(); while ( (pEntry->usStart != 0) || (pEntry->usStop != 0) || (pEntry->usType != 0) ) @@ -513,8 +296,8 @@ USHORT EqfMemory::TokenizeSource // handle translatable text { PFLAGOFFSLIST pTerm; // pointer to terms list - PFLAGOFFSLIST pTermList = NULL; // pointer to offset/length term list - ULONG ulListSize = 0; + std::vector pTermList; // pointer to offset/length term list + //ULONG ulListSize = 0; USHORT usTokLen = pEntry->usStop - pEntry->usStart + 1; CHAR_W chTemp = pStringToTokenize[pEntry->usStop+1]; // buffer for character values @@ -522,47 +305,35 @@ USHORT EqfMemory::TokenizeSource usRc = NTMMorphTokenizeW( usLangId, pStringToTokenize + pEntry->usStart, - &ulListSize, (PVOID *)&pTermList, - MORPH_FLAG_OFFSLIST ); + pTermList ); pStringToTokenize[pEntry->usStop+1] = chTemp; if ( usRc == MORPH_OK ) { - pTerm = pTermList; - fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + pTerm = pTermList.data(); + fOK = true; + //fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); + + usStart = pEntry->usStart; + if ( pTerm ) { - usStart = pEntry->usStart; - if ( pTerm ) - { - while ( pTerm->usLen && !usRc ) - { - if ( !(pTerm->lFlags & TF_NEWSENTENCE) ) - { - fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pTermTokens->usOffset = pTerm->usOffs + usStart; - pTermTokens->usLength = pTerm->usLen; - pTermTokens->usHash = 0; - pTermTokens++; - } - } /* endif */ - pTerm++; - } /* endwhile */ + while ( pTerm->usLen && !usRc ) + { + if ( !(pTerm->lFlags & TF_NEWSENTENCE) ) + { + TMX_TERM_TOKEN TermToken; + + TermToken.usOffset = pTerm->usOffs + usStart; + TermToken.usLength = pTerm->usLen; + TermToken.usHash = 0; + + pSentence->pTermTokens.push_back(TermToken); + } /* endif */ + pTerm++; + } /* endwhile */ } /* endif */ - } /* endif */ } /* endif */ - UtlAlloc( (PVOID *) &pTermList, 0L, 0L, NOMSG ); } /* end case UNPROTECTED_CHAR */ break; default : @@ -578,42 +349,34 @@ USHORT EqfMemory::TokenizeSource else { // add tag data - usTagEntryLen = sizeof(TMX_TAGENTRY) + - (pEntry->usStop - pEntry->usStart + 1) * sizeof(CHAR_W); - if ( (pSentence->lTagAlloc - (pTagEntry - (PBYTE)pSentence->pTagRecord)) - <= (SHORT)usTagEntryLen ) - { - LONG lBytesToAlloc = get_max( ((LONG)TOK_SIZE), ((LONG)usTagEntryLen) ); + usTagEntryLen = sizeof(TMX_TAGENTRY) + (pEntry->usStop - pEntry->usStart + 1) * sizeof(CHAR_W); + sFilled = pSentence->pTagRecord.size(); + //sFilled = (PBYTE)pTagEntry - (PBYTE)pSentence->pTagRecord.data(); + //if ( std::ceil((sFilled + usTagEntryLen) / static_cast(sizeof(TMX_TAGTABLE_RECORD))) // space in bytes/size of entry + // <= pSentence->pTagRecord.size()) + { //remember offset of pTagEntry - usFilled = (USHORT)(pTagEntry - (PBYTE)pSentence->pTagRecord); - - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTagRecord, pSentence->lTagAlloc, - pSentence->lTagAlloc + lBytesToAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTagAlloc += lBytesToAlloc; - - //set new position of pTagEntry - pTagEntry = (PBYTE)(pSentence->pTagRecord) + usFilled; - pTagRecord = pSentence->pTagRecord; - } /* endif */ - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); + //pSentence->pTagRecord.resize(pSentence->pTagRecord.size() + usTagEntryLen); + //pTagEntry = (PTMX_TAGENTRY)((PBYTE)pSentence->pTagRecord.data() + sFilled); } - else + { - ((PTMX_TAGENTRY)pTagEntry)->usOffset = pEntry->usStart; - ((PTMX_TAGENTRY)pTagEntry)->usTagLen = - (pEntry->usStop - pEntry->usStart + 1); - memcpy( &(((PTMX_TAGENTRY)pTagEntry)->bData), - pStringToTokenize + pEntry->usStart, - ((PTMX_TAGENTRY)pTagEntry)->usTagLen * sizeof(CHAR_W)); - pTagEntry += usTagEntryLen; + //TagEntry.reset(); + //TagEntry.usOffset = pEntry->usStart; + pushBytes(pSentence->pTagRecord, pEntry->usStart); //offset + USHORT usTagLen = pEntry->usStop - pEntry->usStart + 1; + pushBytes(pSentence->pTagRecord, usTagLen);//TagEntry.usTagLen + PBYTE data = (PBYTE) &pStringToTokenize[pEntry->usStart]; + pushBytes(pSentence->pTagRecord, data, data + usTagLen * sizeof(CHAR_W)); + + //TagEntry.usTagLen = pEntry->usStop - pEntry->usStart + 1; + + //memcpy( &(TagEntry.bData), + // pStringToTokenize + pEntry->usStart, + // TagEntry.usTagLen * sizeof(CHAR_W)); + //pSentence->pTagRecord.insert(pSentence->pTagRecord.end(), TagEntry, sizeof(TagEntry)); + //pTagEntry += usTagEntryLen; } /* endif */ } /* endif */ } /* end default */ @@ -627,29 +390,24 @@ USHORT EqfMemory::TokenizeSource /* check if we filled at least one term token -- if not */ /* use a dummy token - just to get an exact match ... */ /********************************************************/ - if ( pTermTokens == pSentence->pTermTokens ) + TMX_TERM_TOKEN TermToken; + TermToken.reset(); + if (pSentence->pTermTokens.empty()) //pTermTokens == pSentence->pTermTokens.data() ) { - pTermTokens->usOffset = 0; - pTermTokens->usLength = (USHORT)UTF16strlenCHAR( pStringToTokenize ); - pTermTokens->usHash = 0; - pTermTokens++; + TermToken.usOffset = 0; + TermToken.usLength = (USHORT)UTF16strlenCHAR( pStringToTokenize ); + TermToken.usHash = 0; + pSentence->pTermTokens.push_back(TermToken); + TermToken.reset(); } /* endif */ - fOK = CheckForAlloc( pSentence, &pTermTokens, 3 ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } + pSentence->pTermTokens.push_back(TermToken); //set total tag record length - RECLEN(pTagRecord) = pTagEntry - (PBYTE)pTagRecord; + RECLEN(toTmxTagtableRecord(pSentence->pTagRecord)) = pSentence->pTagRecord.size();//(PBYTE)pTagEntry - (PBYTE)pTagRecord; } /* endif */ } /* endif */ - //release allocated memory - if ( pStartStop ) UtlAlloc( (PVOID *) &pStartStop, 0L, 0L, NOMSG ); - if ( pTokenList ) UtlAlloc( (PVOID *) &pTokenList, 0L, 0L, NOMSG ); - //free tag table / decrement use count if ( pTable != NULL ) { @@ -663,47 +421,26 @@ USHORT EqfMemory::TokenizeSource USHORT NTMTokenizeW ( PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - PVOID *ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& ppTermList // IN/OUT: address of term list pointer ); USHORT NTMMorphTokenizeW ( SHORT sLanguageID, // language ID PSZ_W pszInData, // pointer to input segment - PULONG pulBufferSize, // address of variable containing size of - // term list buffer - PVOID *ppTermList, // address of caller's term list pointer - USHORT usListType // type of term list MORPH_ZTERMLIST or - // MORPH_OFFSLIST + std::vector& pTermList // address of caller's term list pointer ) { USHORT usRC = MORPH_OK; // function return code - ULONG ulTermBufUsed = 0; // number of bytes used in term buffer /********************************************************************/ /* Check input data */ /********************************************************************/ - if ( (pszInData == NULL) || - (pulBufferSize == NULL) || - (ppTermList == NULL) || - ((*ppTermList == NULL) && (*pulBufferSize != 0) ) ) + if ( (pszInData == NULL) ) { LOG_AND_SET_RC(usRC, T5INFO, MORPH_INV_PARMS); } /* endif */ -///********************************************************************/ -///* Get language control block pointer -- not needed */ -///********************************************************************/ -//if ( usRC == MORPH_OK ) -//{ -// usRC = MorphGetLCB( sLanguageID, &pLCB ); -//} /* endif */ - /********************************************************************/ /* call language exit to tokenize the input data */ @@ -713,21 +450,17 @@ USHORT NTMMorphTokenizeW if ( *pszInData != EOS ) { usRC = NTMTokenizeW(pszInData, - pulBufferSize, - ppTermList, - usListType - ); + pTermList ); } else { - usRC = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulBufferSize, - &ulTermBufUsed, - (PSZ_W)EMPTY_STRING, - 0, - 0, - 0L, - usListType ); + + usRC = MorphAddTermToList2W( pTermList, + (PSZ_W)EMPTY_STRING, //term + 0, // Length + 0, //Offset + 0L //Flags + ); } /* endif */ } /* endif */ @@ -738,43 +471,28 @@ USHORT NTMMorphTokenizeW USHORT NTMTokenizeW ( PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - PVOID *ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer ) { USHORT usReturn = 0; // returncode ULONG ulTermBufUsed = 0; // amount of space used in term buffer PSZ_W pTerm; // pointer to current token string PSZ_W pszCurPos = pszInData; // current position in input data - BOOL fOffsList; // TRUE = return a offset/length list LONG lFlags; // flags for current term/token BOOL fAllCaps, fAlNum, fNumber;// character classification flags BOOL fSkip, fNewSentence, // token processing flags fSingleToken, fEndOfToken; CHAR_W c, d; - fOffsList = (usListType == MORPH_OFFSLIST) || - (usListType == MORPH_FLAG_OFFSLIST); /********************************************************************/ /* Always assume start of a new sentence */ /********************************************************************/ - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, - L" ", - 1, - 0, // no offset possible - TF_NEWSENTENCE, - usListType ); - } /* endif */ + usReturn = MorphAddTermToList2W( pTermList, + L" ", + 1, + 0, // no offset possible + TF_NEWSENTENCE); /********************************************************************/ /* Initialize processing flags */ @@ -946,14 +664,13 @@ USHORT NTMTokenizeW if ( fAllCaps ) lFlags |= TF_ALLCAPS; if ( !fAlNum ) lFlags |= TF_NOLOOKUP; if ( fNumber ) lFlags |= TF_NUMBER; - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, - pTerm, + + usReturn = MorphAddTermToList2W( pTermList, + pTerm, (USHORT)(pszCurPos - pTerm), - (USHORT)(fOffsList ? (pTerm-pszInData) : 0), - lFlags, - usListType ); + pTerm-pszInData, + lFlags ); + pTerm = pszCurPos; fAllCaps = TRUE; fAlNum = TRUE; @@ -965,14 +682,11 @@ USHORT NTMTokenizeW /******************************************************************/ if ( fSingleToken && !usReturn ) { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, pszCurPos, 1, - (USHORT)(fOffsList ? (pszCurPos-pszInData) : 0), - TF_NOLOOKUP, - usListType ); + pszCurPos-pszInData, + TF_NOLOOKUP ); pTerm = pszCurPos + 1; } /* endif */ @@ -981,18 +695,11 @@ USHORT NTMTokenizeW /******************************************************************/ if ( fNewSentence && !usReturn ) { - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, L" ", 1, 0, // no offset possible - TF_NEWSENTENCE, - usListType ); - } /* endif */ + TF_NEWSENTENCE); } /* endif */ @@ -1022,14 +729,11 @@ USHORT NTMTokenizeW if ( fAllCaps ) lFlags |= TF_ALLCAPS; if ( !fAlNum ) lFlags |= TF_NOLOOKUP; if ( fNumber ) lFlags |= TF_NUMBER; - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, pTerm, (USHORT)(pszCurPos - pTerm), - (USHORT)(fOffsList ? (pTerm-pszInData) : 0), - lFlags, - usListType ); + pTerm - pszInData, + lFlags); } /* endif */ /*****************************************************************/ @@ -1037,14 +741,11 @@ USHORT NTMTokenizeW /*****************************************************************/ if ( !usReturn ) { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, (PSZ_W)EMPTY_STRING, 0, 0, - 0L, - usListType ); + 0L ); } /* endif */ return (usReturn); diff --git a/source/opentm2/core/ProposalFilter.cpp b/source/opentm2/core/ProposalFilter.cpp index d0303e26..7067b6bb 100644 --- a/source/opentm2/core/ProposalFilter.cpp +++ b/source/opentm2/core/ProposalFilter.cpp @@ -48,8 +48,6 @@ std::string convertWstrToStr(std::wstring& wstr) return EncodingHelper::convertToUTF8(wstr); } - - const std::map FilterFieldToStrMap{ {ProposalFilter::UNKNOWN_FIELD,"UNKNOWN_FIELD"}, {ProposalFilter::TIMESTAMP,"TIMESTAMP"}, diff --git a/source/opentm2/core/document/EQFMAIN.cpp b/source/opentm2/core/document/EQFMAIN.cpp index ce26c536..5f747b24 100755 --- a/source/opentm2/core/document/EQFMAIN.cpp +++ b/source/opentm2/core/document/EQFMAIN.cpp @@ -105,11 +105,11 @@ BOOL TwbGetCheckProfileData( PSZ pEqfSystemMsgFile, PSZ pEqfSystemPropPath, { T5LOG( T5DEVELOP) << "TwbGetCheckProfileData( pEqfSystemMsgFile= "<< pEqfSystemMsgFile<< ", pEqfSystemPropPath= "<< pEqfSystemPropPath << ", pEqfResFile = " << pEqfResFile << ")" ; - CHAR szDrive[MAX_DRIVE]; // buffer for drive list - CHAR szLanDrive[MAX_DRIVE]; // buffer for lan drive list - CHAR szSysPath[MAX_EQF_PATH]; // buffer for system path + CHAR szDrive[MAX_DRIVE]{}; // buffer for drive list + CHAR szLanDrive[MAX_DRIVE]{}; // buffer for lan drive list + CHAR szSysPath[MAX_EQF_PATH]{}; // buffer for system path BOOL fOK = TRUE; // internal OK flag - CHAR szOTMPath[MAX_EQF_PATH]; + CHAR szOTMPath[MAX_EQF_PATH]{}; // Get system drive and system path Properties::GetInstance()->get_value(KEY_OTM_DIR, szOTMPath, MAX_EQF_PATH); @@ -118,7 +118,6 @@ BOOL TwbGetCheckProfileData( PSZ pEqfSystemMsgFile, PSZ pEqfSystemPropPath, fOK = UtlQuerySysLangFile( szEqfSysLanguage, szEqfResFile, EqfSystemHlpFile, EqfSystemMsgFile ); - /******************************************************************/ /* return strings to caller if requested... */ /******************************************************************/ diff --git a/source/opentm2/core/document/EQFTATAG.cpp b/source/opentm2/core/document/EQFTATAG.cpp index 9f7a59c9..31ba22fd 100755 --- a/source/opentm2/core/document/EQFTATAG.cpp +++ b/source/opentm2/core/document/EQFTATAG.cpp @@ -64,7 +64,7 @@ USHORT TALoadTagTableExHwnd // loads/accesses a tag table HWND hwnd // owner handle for error messages ) { - std::lock_guard l(mutexLoadTagTable); + //std::lock_guard l(mutexLoadTagTable); CHAR szTagTableFileName[MAX_EQF_PATH]; // buffer for file name szTagTableFileName[0]='\0'; PLOADEDTABLE pTable = NULL; // ptr for processing of loaded tables @@ -604,7 +604,7 @@ BOOL TATagsToUnicode USHORT i,j = 0; PSZ pCurName; - PWCHAR pTmpW; + std::wstring& pTmpW = pTable->pTagNamesW; PBYTE pByte; PTAG pTag; PSZ pTagNames; @@ -618,16 +618,16 @@ BOOL TATagsToUnicode usTagCount = pTagTable->uNumTags; usAttrCount = pTagTable->stAttribute.uNumber; pAttr = (PATTRIBUTE) (pByte + pTagTable->stAttribute.uOffset); + pTmpW.resize(usSize+1); - if ( UtlAlloc( (PVOID *)&pTmpW, 0L, usSize * sizeof(TMWCHAR), ERROR_STORAGE ) ) { - pTable->pTagTableW = pTmpW; + pTable->pTagTableW = pTmpW.data(); for ( i = 0; i < usTagCount; i++ ) { pCurName = pTag[i].uTagnameOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pTag[i].uTagnameOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pTag[i].uTagnameOffs, ulCP ); } } /* endfor */ for ( i= 0; i < usTagCount; i++ ) @@ -635,7 +635,7 @@ BOOL TATagsToUnicode pCurName = pTag[i].uEndDelimOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pTag[i].uEndDelimOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pTag[i].uEndDelimOffs, ulCP ); } } /* endfor */ @@ -644,7 +644,7 @@ BOOL TATagsToUnicode pCurName = pTagNames + pAttr[i].uStringOffs; if (pCurName) { - ASCII2Unicode(pCurName, pTmpW + pAttr[i].uStringOffs, ulCP ); + ASCII2Unicode(pCurName, pTmpW.data() + pAttr[i].uStringOffs, ulCP ); } } /* endfor */ for ( i= 0; i < usAttrCount; i++ ) @@ -652,7 +652,7 @@ BOOL TATagsToUnicode pCurName = pAttr[i].uEndDelimOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr[i].uEndDelimOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr[i].uEndDelimOffs, ulCP ); } } /* endfor */ for ( i = 0; i < 27; i++ ) @@ -663,22 +663,17 @@ BOOL TATagsToUnicode pCurName = pTagNames + pAttr->uStringOffs; if ( pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr->uStringOffs, ulCP); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr->uStringOffs, ulCP); } pCurName = pTagNames + pAttr->uEndDelimOffs; if ( pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr->uEndDelimOffs, ulCP); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr->uEndDelimOffs, ulCP); } pAttr++; } /* endfor */ } /* endfor */ - pTable->pTagNamesW = pTmpW; - } - else - { - fOK = FALSE; } return fOK; diff --git a/source/opentm2/core/morph/MorphFactory.cpp b/source/opentm2/core/morph/MorphFactory.cpp index 55ba0517..c36836cb 100755 --- a/source/opentm2/core/morph/MorphFactory.cpp +++ b/source/opentm2/core/morph/MorphFactory.cpp @@ -156,6 +156,7 @@ int MorphFactory::getLanguageList( return pOtmMorphPlugin->getLanguageList(vLanguageList); } } + return 0; } /* \brief check if the given language is supported by any of the available Morph plugins diff --git a/source/opentm2/core/morph/OtmMorphICU.cpp b/source/opentm2/core/morph/OtmMorphICU.cpp index d906cc28..6d9fe8f5 100755 --- a/source/opentm2/core/morph/OtmMorphICU.cpp +++ b/source/opentm2/core/morph/OtmMorphICU.cpp @@ -97,31 +97,26 @@ int OtmMorphICU::tokenizeByTerm(const wchar_t* pText, TERMLIST& vResult) return OtmMorph::ERROR_PARAMETAR; } - { - std::lock_guard guard(this->MorphICUMutex); - - - - std::u16string vSection_u16 = EncodingHelper::toUtf16(pText); - UnicodeString text(vSection_u16.c_str()); + std::lock_guard guard(this->MorphICUMutex); + std::u16string vSection_u16 = EncodingHelper::toUtf16(pText); + UnicodeString text(vSection_u16.c_str()); - pWordBoundary->setText(text); - getResultFromIterator(pWordBoundary, vResult); + pWordBoundary->setText(text); + getResultFromIterator(pWordBoundary, vResult); - /* Force maximum length to 511, since temp buffers are 512 10-6-16 */ - for (auto it = begin (vResult); it != end (vResult); ++it) - { + /* Force maximum length to 511, since temp buffers are 512 10-6-16 */ + for (auto it = begin (vResult); it != end (vResult); ++it) + { if ( it->iLength > 511 ) it->iLength=511; - } + } #ifdef TOKENIZELOG - T5LOG(T5DEBUG) << "results:" ; - for (auto it = begin (vResult); it != end (vResult); ++it) - { - T5LOG(T5DEBUG) << " Offset="<iStartOffset <<" Len="<iLength<<" Flag=" << it->iTermType ; - } -#endif + T5LOG(T5DEBUG) << "results:" ; + for (auto it = begin (vResult); it != end (vResult); ++it) + { + T5LOG(T5DEBUG) << " Offset="<iStartOffset <<" Len="<iLength<<" Flag=" << it->iTermType ; } +#endif return OtmMorph::SUCCESS_RETURN; } @@ -191,18 +186,8 @@ int OtmMorphICU::tokenizeByTerm( const char* pszSection, STRINGLIST& vResult ) { std::lock_guard guard(this->MorphICUMutex); - - pWordBoundary->setText(UnicodeString(pszSection)); getResultFromIterator(pWordBoundary, vResult); - -#ifdef TOKENIZELOG - T5LOG(T5DEBUG) << "results:" ; - for (auto it = begin (vResult); it != end (vResult); ++it) - { - T5LOG(T5DEBUG) << "\"" << it->c_str()<< "\""; - } -#endif } return OtmMorph::SUCCESS_RETURN; } diff --git a/source/opentm2/core/pluginmanager/PluginManagerImpl.h b/source/opentm2/core/pluginmanager/PluginManagerImpl.h index e26e8eea..2b4e2625 100755 --- a/source/opentm2/core/pluginmanager/PluginManagerImpl.h +++ b/source/opentm2/core/pluginmanager/PluginManagerImpl.h @@ -19,10 +19,6 @@ class PluginListener; */ struct LoadedPluginDLL { - #ifdef TO_BE_REMOVED - HMODULE hMod; // DLL module handle - #endif - char strDll[MAX_PATH]; // Add for P402792, dll path std::vector vPluginList; // list of plugins contained in DLL }; @@ -134,7 +130,7 @@ int m_iNextCommandID; PluginManagerImpl(const PluginManager&) {}; /*! \brief Assignment-operator is private */ -PluginManagerImpl& operator=(const PluginManagerImpl&) {}; +PluginManagerImpl& operator=(const PluginManagerImpl&) { return *this; }; /*! \brief A set containing all registered plugins. */ PLUGINSET* pluginSet; diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 0c96d582..076ca178 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1,5 +1,11 @@ #include + +#include +#include +#include +#include // for std::iota + #include "requestdata.h" #include "ThreadingWrapper.h" @@ -187,6 +193,8 @@ int convertUTCTimeToLong( char *pszDateTime, PLONG plTime ) #include +DECLARE_int32(t5loglevel); + /*! \brief build return JSON string in case of errors \param _rc_ error return code @@ -220,7 +228,9 @@ int RequestData::buildErrorReturn json_factory.addParmToJSON( outputMessage, "ErrorMsg", pszErrorMsg ); json_factory.terminateJSON( outputMessage ); _rest_rc_ = rest_rc; - T5LOG_RC(_rc_, outputMessage); + if(!fSuppressBuildErrorLog){ + T5LOG_RC(_rc_, outputMessage); + } return( rc ); } @@ -255,6 +265,10 @@ DECLARE_int64(tmRequestLockDefaultTimeout); DECLARE_int64(tmListLockDefaultTimeout); DECLARE_int64(tmLockDefaultTimeout); +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(saveDifferentTargetsForSameSource); +//DECLARE_bool(ignoreSegIdOnUpdate); + int RequestData::setMutexTimeouts(){ //set default timeouts if they were not set in request tmLockTimeout.setTimeout_ms(tmMutexTimeoutMs == -1? FLAGS_tmLockDefaultTimeout : tmMutexTimeoutMs); @@ -274,72 +288,116 @@ int RequestData::requestTM(){ fValid = true; return 0; } + + if(isReadOnlyRequest() || isWriteRequest()){//shortcut when app is closing state + bool fWriteRequestsAllowed = false, fLoadingTmIsAllowed = false; + { + std::scoped_lock(TMManager::mutex_tmManager); + fWriteRequestsAllowed = TMManager::fWriteRequestsAllowed; + fLoadingTmIsAllowed = TMManager::fLoadingTmIsAllowed; + } + + if(!fLoadingTmIsAllowed){ + if(!TMManager::GetInstance()->IsMemoryInList(strMemName, tmListTimeout)){ + return buildErrorReturn(ERR_LOADING_TM_ARE_NOT_ALLOWED, "Memory is not loaded and loading tm are not allowed, because shutdown of the service is scheduled or service is not initialized", 423); + } + //if(TMManager::GetInstance()->IsMemoryLoading()) + } + if(isWriteRequest() && !fWriteRequestsAllowed){ + return buildErrorReturn(ERR_WRITE_REQUESTS_NOT_ALLOWED, "Write requests are not allowed, because shutdown of the service is scheduled or service is not initialized", 423); + } + } + //check if memory is loaded to tmmanager if(isReadOnlyRequest()) { mem = TMManager::GetInstance()->requestReadOnlyTMPointer(strMemName, memRef, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to requestTm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } }else if(isWriteRequest()) { + // mem = TMManager::GetInstance()->requestWriteTMPointer(strMemName, memRef, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to lock requestTm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } }else if(STATUS_MEM == command){ mem = TMManager::GetInstance()->requestServicePointer(strMemName, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } fValid = true;//for status we don't care about tm pointer here return 0; } + bool isServiceReq = isServiceRequest(); if(mem.get()== nullptr){ - fValid = isServiceRequest(); - return 0; + fValid = isServiceReq; }else if(mem->isReorganizeRunning()){ - return buildErrorReturn(505, "Reorganize is running for requested tm", _rc_); + return buildErrorReturn(505, "Reorganize is running for requested tm", 505); }else if(mem->isImportRunning()){ - return buildErrorReturn(505, "Import is running for requested tm", _rc_); - } - - { + return buildErrorReturn(505, "Import is running for requested tm", 505); + }else{ TimedMutexGuard l(mem->tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ + USHORT accessModeBeforeOpenning = mem->state->usAccessMode; + + if(REORGANIZE_MEM == command || DELETE_ENTRIES_REORGANIZE == command + || EXPORT_MEM_TMX == command || EXPORT_MEM_TMX_STREAM == command){ + mem->state->usAccessMode |= ASD_ORGANIZE; + } + _rc_ = mem->Load(); + + //if(!mem->isCorrupted()){ + mem->state->usAccessMode=accessModeBeforeOpenning; + //} + if(_rc_){ - //T5LOG(T5ERROR) << "Failed to open tm, rc = " << _rc_; - return buildErrorReturn(504, "Failed to load tm", _rc_); + std::string msg = "Failed to load tm, rc = "; + msg += toStr(_rc_); + if(11==_rc_){ + msg += "; file on the disk is locked;"; + } + return buildErrorReturn(512, msg.c_str(), 512); } } } - if(mem->isLoaded()) + if(mem && mem->isLoaded()) { fValid = true; + if(mem->isCorrupted() && + ( EXPORT_MEM_TMX != command + && EXPORT_MEM_TMX_STREAM != command + && REORGANIZE_MEM != command + && GET_TABLES != command) + ) + { + return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized; BTREE_CORRUPTED, rc = 5008; ", 504); // 5008 = BTREE_CORRUPTED- call reorganize + } if( command == EXPORT_MEM_TMX || command == EXPORT_MEM_INTERNAL_FORMAT @@ -353,11 +411,11 @@ int RequestData::requestTM(){ mem->FlushFilebuffers(tmLockTimeout); if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } } - if(command != STATUS_MEM) + if(command != STATUS_MEM && command != GET_TABLES) { mem->updateLastUseTime(); } @@ -383,6 +441,7 @@ bool RequestData::isWriteRequest(){ || command == COMMAND::IMPORT_LOCAL_MEM || command == COMMAND::FLUSH_MEM || command == COMMAND::MULTIUPDATE + || command == ADD_TO_TABLE ; } @@ -396,6 +455,7 @@ bool RequestData::isReadOnlyRequest(){ || command == COMMAND::EXPORT_MEM_TMX_STREAM || command == COMMAND::EXPORT_MEM_INTERNAL_FORMAT_STREAM || command == COMMAND::GET_ENTRY + || command == COMMAND::GET_TABLES ; } @@ -463,8 +523,8 @@ int RequestData::run(){ return 0; } - int res = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if(!res) res = parseJSON(); + int res = 0; + res = parseJSON(); if(!res) res = checkData(); if(!res) res = prepareData(); @@ -481,7 +541,7 @@ int RequestData::run(){ TimedMutexGuard l(mem->tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__) ; if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } mem->setActiveRequestCommand(command); @@ -502,7 +562,7 @@ int RequestData::run(){ memRef.reset(); } - if(COMMAND::EXPORT_MEM_TMX == command || COMMAND::EXPORT_MEM_TMX_STREAM == command){ + if(!res && (COMMAND::EXPORT_MEM_TMX == command || COMMAND::EXPORT_MEM_TMX_STREAM == command)){ auto exportRequest = (ExportRequestData*) this; if(exportRequest->nextInternalKey.empty()){ T5LOG(T5ERROR) << "nextInternalKey is empty for" << "\t req body = \"" <strBody << "\"; " << "\t nextKey =\"" << exportRequest->nextInternalKey <<"\"; "; @@ -602,9 +662,6 @@ int CreateMemRequestData::createNewEmptyMemory(){ return TM_LIST_MUTEX_TIMEOUT_FAILED; } } - //if(mem != nullptr){ - // mem.reset(); - //} return 0; } @@ -634,10 +691,10 @@ int CreateMemRequestData::importInInternalFomat(){ if(binTmData.size() > 0){ auto filesize = FilesystemHelper::WriteToFile(strTempFile, &binTmData[0], binTmData.size()); if(0 == filesize){ - return buildErrorReturn( _rc_, "cant write tm data to temp file", 500); + return buildErrorReturn( 500, "cant write tm data to temp file", 500); } }else{ - return buildErrorReturn( _rc_, "binary tm file data not found", 400); + return buildErrorReturn( 400, "binary tm file data not found", 400); } } } @@ -825,13 +882,13 @@ int CreateMemRequestData::execute(){ if(_rc_ == ERROR_MEM_NAME_EXISTS){ outputMessage = "{\"" + strMemName + "\": \"ERROR_MEM_NAME_EXISTS\" }" ; - T5LOG(T5ERROR) << "OtmMemoryServiceWorker::createMemo()::usRC = " << _rc_ << " _rc_ = ERROR_MEM_NAME_EXISTS"; + T5LOG(T5ERROR) << "::createMemo()::usRC = " << _rc_ << " _rc_ = ERROR_MEM_NAME_EXISTS"; return _rc_; }else if ( _rc_ != 0 ) { char szError[PATH_MAX]; unsigned short usRC = 0; - EqfGetLastError( NULL /*OtmMemoryServiceWorker::getInstance()->hSession*/, &usRC, szError, sizeof( szError ) ); + EqfGetLastError( NULL /*::getInstance()->hSession*/, &usRC, szError, sizeof( szError ) ); //buildErrorReturn( _rc_, szError, outputMessage ); outputMessage = "{\"" + strMemName + "\": \"" ; @@ -941,6 +998,10 @@ int ImportRequestData::parseJSON(){ std::string name; std::string value; + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; + //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + try{ while ( _rc_ == 0 ) { @@ -965,9 +1026,15 @@ int ImportRequestData::parseJSON(){ } else if(strcasecmp(name.c_str(), "timeout") == 0){ timeout = std::stol(value); + }else if(strcasecmp(name.c_str(), "newEntryIfOnlyContextAndTimestampDifferent") == 0){ + fNewEntryIfOnlyContextAndTimestampDifferent = std::stol(value); + }else if(strcasecmp(name.c_str(), "ignoreSegIdOnUpdate") == 0){ + fIgnoreSegIdOnUpdate = std::stol(value); + }else if(strcasecmp(name.c_str(), "saveDifferentTargetsForSameSource") == 0){ + fSaveDifferentTargetsForSameSource = std::stol(value); }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else if(strcasecmp(name.c_str(), "framingTags") == 0){ std::string strInclosingTagsBehaviour = value; @@ -1073,138 +1140,6 @@ int ImportRequestData::checkData(){ return 0; } - -int ImportStreamRequestData::parseJSON(){ - T5LOG( T5INFO) << "+POST " << strMemName << "/import"; - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "parseJSON, rc is not 0 at the start of function", INTERNAL_SERVER_ERROR ); - } /* endif */ - - if ( strMemName.empty() ) - { - return buildErrorReturn( _rc_, "import::Missing name of memory", BAD_REQUEST ); - } /* endif */ - - // check if memory exists - if(int existscode = TMManager::GetInstance()->TMExistsOnDisk(strMemName)) - { - std::string msg = "import::Missing tm files on disk, code="; - msg += std::to_string(existscode); - return buildErrorReturn( existscode, msg.c_str(), NOT_FOUND); - } - - // find the memory to our memory list - // extract TMX data - int loggingThreshold = -1; //0-develop(show all logs), 1-debug+, 2-info+, 3-warnings+, 4-errors+, 5-fatals only - - void *parseHandle = json_factory.parseJSONStart( strBody, &_rc_ ); - if ( parseHandle == NULL ) - { - return buildErrorReturn( _rc_, "import::Missing or incorrect JSON data in request body", BAD_REQUEST ); - } /* end */ - - std::string name; - std::string value; - try{ - while ( _rc_ == 0 ) - { - _rc_ = json_factory.parseJSONGetNext( parseHandle, name, value ); - if ( _rc_ == 0 ) - { - if(strcasecmp(name.c_str(), "timeout") == 0){ - timeout = std::stol(value); - }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ - loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; - T5Logger::GetInstance()->SetLogLevel(loggingThreshold); - }else if ( strcasecmp( name.c_str(), "tmMutexTimeout" ) == 0 ) - { - tmMutexTimeoutMs = std::stoll(value.c_str()); - } - else if ( strcasecmp( name.c_str(), "tmListMutexTimeout" ) == 0 ) - { - tmListMutexTimeoutMs = std::stoll(value.c_str()); - } - else if ( strcasecmp( name.c_str(), "requestTMMutexTimeout" ) == 0 ) - { - requestTMMutexTimeoutMs = std::stoll(value.c_str()); - } - else if(strcasecmp(name.c_str(), "framingTags") == 0){ - std::string strInclosingTagsBehaviour = value; - if(strcasecmp(value.c_str(), "saveAll") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::saveAll; - }else if(strcasecmp(value.c_str(), "skipAll") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::skipAll; - }else if(strcasecmp(value.c_str(), "skipPaired") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::skipPaired; - } - }else{ - T5LOG( T5WARNING) << "JSON parsed unexpected name, " << name; - } - }else if(_rc_ != 2002){// _rc_ != INFO_ENDOFPARAMETERLISTREACHED - std::string msg = "failed to parse JSON, _rc_ = " + std::to_string(_rc_); - return buildErrorReturn(_rc_, msg.c_str(), BAD_REQUEST); - } - } /* endwhile */ - json_factory.parseJSONStop( parseHandle ); - } - catch(...) - { - return buildErrorReturn(444, "Json parsing failed", 400); - } - if(_rc_ == 2002) _rc_ = 0; - return _rc_; -} - -int ImportStreamRequestData::checkData() -{ - return 0; -} - -int ImportStreamRequestData::execute(){ - if ( mem == nullptr ) - { - return buildErrorReturn( 404, "mem not found or can't be opened" ); - } - // close the memory - when open - if ( false == memIsAvailableToOperate(mem.get()) ) - { - std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); - return buildErrorReturn( 500, msg.c_str() ); - } - - mem->eStatus = IMPORT_RUNNING_STATUS; - mem->eImportStatus = IMPORT_RUNNING_STATUS; - - T5LOG( T5DEBUG) << "status for " << strMemName << " was changed to import"; - // start the import background process - pData = new( IMPORTMEMORYDATA ); - pData->inclosingTagsBehaviour = inclosingTagsBehaviour; - strcpy( pData->szMemory, strMemName.c_str() ); - //pData->fileData = std::move(fileData); - - if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; - } - - mem->importDetails->reset(); - - LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->lImportTimeoutSec = timeout; - pData->mem = mem; - pData->tmLockTimeout = tmLockTimeout; - - //importMemoryProcess(pData);//to do in same thread - std::thread worker_thread(importMemoryProcess, pData); - worker_thread.detach(); - - return( CREATED ); -} - - int SaveAllTMsToDiskRequestData::execute(){ if(_rc_) { outputMessage = "Error: Can't save tm files on disk"; @@ -1221,7 +1156,7 @@ int SaveAllTMsToDiskRequestData::execute(){ if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg(".Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), TM_LIST_MUTEX_TIMEOUT_FAILED); + return buildErrorReturn(TM_LIST_MUTEX_TIMEOUT_FAILED, tmListTimeout.getErrMsg().c_str(), 506); } for(const auto& tm: TMManager::GetInstance()->tms){ @@ -1233,7 +1168,7 @@ int SaveAllTMsToDiskRequestData::execute(){ errMsg += tm.second->getName() + ":" + tmLockTimeout.getErrMsg() + "; "; tmLockTimeout.reset(); tmLockTimeout.setTimeout_ms(tmLockTimeoutms); - //return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + //return buildErrorReturn(_rc_, tmLockTimeout.getErrMsg().c_str(), 506); } tms += tm.first; } @@ -1260,13 +1195,13 @@ std::string RequestData::reserveFilename(){ int ImportRequestData::execute(){ if ( mem == nullptr ) { - return buildErrorReturn( 404, "mem not found or can't be opened" ); + return buildErrorReturn( 404, "mem not found or can't be opened",404 ); } // close the memory - when open if ( false == memIsAvailableToOperate(mem.get()) ) { std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); - return buildErrorReturn( 500, msg.c_str() ); + return buildErrorReturn( 500, msg.c_str(),500 ); } if(strTempFile.empty()){ @@ -1288,15 +1223,22 @@ int ImportRequestData::execute(){ strcpy( pData->szMemory, strMemName.c_str() ); if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; - LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->lImportTimeoutSec = timeout; + + LONG lCurTime = 0; + time( &lCurTime ); + mem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->lImportTimeoutSec = timeout; + } pData->mem = mem; //importMemoryProcess(pData);//to do in same thread @@ -1361,7 +1303,7 @@ int ImportLocalRequestData::parseJSON(){ } else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else{ T5LOG( T5WARNING) << "JSON parsed unexpected name, " << name; @@ -1413,13 +1355,16 @@ int ImportLocalRequestData::execute(){ strcpy( pData->szMemory, strMemName.c_str() ); if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; + time( &lCurTime ); + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->lImportStartTime = lCurTime; + } pData->mem = mem; pData->tmLockTimeout = tmLockTimeout; @@ -1441,20 +1386,6 @@ int ReorganizeRequestData::checkData(){ return 0; } -/* -int ParseSearchType(std::string& inputStr, SearchType& param){ - if(!strcasecmp(inputStr.c_str(),"concordance")) - { - param = SearchType::CONCORDANCE; - }else if(!strcasecmp(inputStr.c_str(),"EXACT")){ - param = SearchType::EXACT; - }else{ - T5LOG(T5ERROR) << ""; - return -1; - } - return 0; -}//*/ - int ProposalFilter::StrToFilterType(const char* str, FilterType &filterType, FilterOptions& options){ filterType = FilterType::UNKNOWN; if(!str){ @@ -1547,7 +1478,7 @@ int SearchFilterFactory::parseJSONNameAndValue(const std::string& name, const st else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else { @@ -1673,6 +1604,31 @@ USHORT MemFuncOrganizeProcess PFCTDATA pData // function I/F session data ); +int ReorganizeRequestData::parseJSON(){ + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + + if(strBody.empty()){ + return 0; + } + // parse input parameters + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + + JSONFactory::JSONPARSECONTROL parseControl[] = { + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource }, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, + { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, + { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, + { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, + { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } + }; + _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + return _rc_; +} + + int ReorganizeRequestData::execute(){ if ( mem == nullptr ) { @@ -1682,10 +1638,17 @@ int ReorganizeRequestData::execute(){ // close the memory - when open if ( false == memIsAvailableToOperate(mem.get()) ) { - std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); + std::string msg = "mem is not avaiflable to operate, status= " + mem->getStatusString(); return buildErrorReturn( 500, msg.c_str() ); } + const auto numOfLangs = mem->state->Languages.table.size(); + if (numOfLangs <3)// if there are less than 3 langs-reorganize shouldn'b be allowed(we neew srcLang, trgLang and empty lang) + { + std::string msg = "mem can't be reorganized without losing all the data, because there are " + std::to_string(numOfLangs) + + " language entries in the table(min is 3 - srcLang, trgLang and emptyLang)"; + return buildErrorReturn( TMERR_LANG_TOO_SMALL_FOR_REORGANIZE, msg.c_str(), 500 ); + } mem->eStatus = REORGANIZE_RUNNING_STATUS; mem->eImportStatus = REORGANIZE_RUNNING_STATUS; @@ -1695,7 +1658,7 @@ int ReorganizeRequestData::execute(){ PMEM_ORGANIZE_IDA pRIDA = new MEM_ORGANIZE_IDA; // pointer to instance area // validate session handle - //_rc_ = FctValidateSession( OtmMemoryServiceWorker::getInstance()->hSession, &pData ); + //_rc_ = FctValidateSession( ::getInstance()->hSession, &pData ); // reorganize the memory if ( !_rc_ ) @@ -1706,7 +1669,7 @@ int ReorganizeRequestData::execute(){ pRIDA->pMem = mem; pRIDA->memRef = memRef; pRIDA->usRC = NO_ERROR; - strcpy( pRIDA->szMemName, mem->szName ); + strcpy( pRIDA->szMemName, mem->szName.c_str() ); pRIDA->fBatch = TRUE; pRIDA->hwndErrMsg = HWND_FUNCIF; pRIDA->NextTask = MEM_START_ORGANIZE; @@ -1715,21 +1678,26 @@ int ReorganizeRequestData::execute(){ pData->fComplete = FALSE; pData->sLastFunction = FCT_EQFORGANIZEMEM; pData->pvMemOrganizeCommArea = pCommArea; - pRIDA->pszNameList = 0; if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + } + LONG lCurTime = 0; time( &lCurTime ); - - pRIDA->pMem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->fReorganize = true; + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->fReorganize = true; + } pRIDA->tmLockTimeout = tmLockTimeout; - - //mem.reset(); - //memRef.reset(); } //worker thread @@ -1757,11 +1725,6 @@ int ReorganizeRequestData::execute(){ int DeleteEntriesReorganizeRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ if ( strMemName.empty() ) { @@ -1769,7 +1732,11 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; JSONFactory::JSONPARSECONTROL parseControl[] = { //searchFilterFactory @@ -1779,6 +1746,9 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ { L"context", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.context ), 0 }, { L"author", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.author ), 0 }, { L"document", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.document ), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent}, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource}, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, @@ -1849,7 +1819,7 @@ int DeleteEntriesReorganizeRequestData::execute(){ pRIDA->pMem = mem; pRIDA->memRef = memRef; pRIDA->usRC = NO_ERROR; - strcpy( pRIDA->szMemName, mem->szName ); + strcpy( pRIDA->szMemName, mem->szName.c_str() ); pRIDA->fBatch = TRUE; pRIDA->hwndErrMsg = HWND_FUNCIF; pRIDA->NextTask = MEM_START_ORGANIZE; @@ -1857,25 +1827,27 @@ int DeleteEntriesReorganizeRequestData::execute(){ // enable organize process if OK pData->fComplete = FALSE; pData->sLastFunction = FCT_EQFORGANIZEMEM; - pData->pvMemOrganizeCommArea = pCommArea; - pRIDA->pszNameList = 0; + pData->pvMemOrganizeCommArea = pCommArea; if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); + LONG lCurTime = 0; time( &lCurTime ); - - pRIDA->pMem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->fReorganize = true; - - //mem.reset(); - //memRef.reset(); + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + + pRIDA->pMem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->fReorganize = true; + } pRIDA->m_reorganizeFilters = searchFilterFactory.getListOfFilters(); - pRIDA->tmLockTimeout = tmLockTimeout; - + pRIDA->tmLockTimeout = tmLockTimeout; } //worker thread @@ -1944,6 +1916,22 @@ int DeleteMemRequestData::execute(){ } +std::wstring json_escape_quotes(const std::wstring& input) { + std::wstring output; + output.reserve(input.size()); // reserve space to avoid many reallocations + + for (wchar_t ch : input) { + if (ch == L'"') { + output.push_back(L'\\'); // add backslash + output.push_back(L'"'); // add quote + } else { + output.push_back(ch); + } + } + + return output; +} + int TagRemplacementRequestData::execute(){ _rc_ = 0; std::string strSrcData, strTrgData, strReqData; @@ -1984,20 +1972,24 @@ int TagRemplacementRequestData::execute(){ json_factory.parseJSONStop( parseHandle ); } - auto result = std::make_unique(std::move(EncodingHelper::convertToWChar(strSrcData.c_str())), - std::move(EncodingHelper::convertToWChar(strTrgData.c_str())), - std::move(EncodingHelper::convertToWChar(strReqData.c_str()))); - - //auto = replaceString( &_rc_); - - wstr = L"{\n "; - std::wstring segmentLocations[] = {L"source", L"target", L"request"}; - wstr += L"\'source\' :\'" + result->getSrcWithTagsFromRequest() + L"\',\n "; - wstr += L"\'target\' :\'" + result->getTrgWithTagsFromRequest() + L"\',\n "; - wstr += L"\'request\' :\'" + result->getReqGenericTagStr() + L"\',\n "; + if(strReqData.size()){//request + auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), + EncodingHelper::convertToWChar(strTrgData), + EncodingHelper::convertToWChar(strReqData)); + + wstr += L"\"source\" :\"" + json_escape_quotes(result->getSrcWithTagsFromRequest()) + L"\",\n "; + wstr += L"\"target\" :\"" + json_escape_quotes(result->getTrgWithTagsFromRequest()) + L"\",\n "; + wstr += L"\"request\" :\"" + json_escape_quotes(result->getReqGenericTagStr()) + L"\"\n "; + }else{ + auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), + EncodingHelper::convertToWChar(strTrgData)); + wstr += L"\"source\" :\"" + json_escape_quotes(result->getGenericTagsString()) + L"\",\n "; + wstr += L"\"target\" :\"" + json_escape_quotes(result->getGenericTargetStr()) + L"\"\n "; + + } wstr += L"\n}"; outputMessage = EncodingHelper::convertToUTF8(wstr); return _rest_rc_ = 200; @@ -2068,12 +2060,7 @@ int CloneTMRequestData::checkData(){ // check mem if is not in import state if(!_rc_){ - if(mem == nullptr){ - // tm is probably not opened, buf files presence was checked before, so it should be "AVAILABLE" status - // outputMessage = "\'newName\' " + strMemName +" was not found in memory list"; - // T5LOG(T5ERROR) << outputMessage << "; for request for mem "<< strMemName <<"; with body = ", strBody ; - // _rc_ = 500; - }else{ + if(mem){ // close the memory - if open if(mem->eImportStatus == IMPORT_RUNNING_STATUS){ std::string msg = "src tm \'" + strMemName +"\' is in import running status. Repeat request later; for request for mem " @@ -2130,24 +2117,9 @@ int CloneTMRequestData::execute(){ if(FilesystemHelper::FileExists(dstTmdPath.c_str())) FilesystemHelper::DeleteFile(dstTmdPath.c_str()); _rc_ = 500; } - if(!_rc_){ - - //EqfMemoryPlugin::GetInstance()->addMemoryToList(newName.c_str()); - }else{ + if(_rc_){ return buildErrorReturn(_rc_, msg.c_str()); } - if( mem != nullptr ) - { - //fClose = true; - //lHandle = pMem->lHandle; - //lastStatus = pMem->eStatus; - //lastImportStatus = pMem->eImportStatus; - - //pMem->lHandle = 0; - //pMem->eStatus = AVAILABLE_STATUS; - //pMem->eImportStatus = AVAILABLE_STATUS; //IMPORT_RUNNING_STATUS; - //pMem->dImportProcess = 0; - } if(_rc_ == 0 ){ outputMessage = newName + " was cloned successfully"; @@ -2192,7 +2164,7 @@ void AddRequestDataToJson(std::stringstream& ss, std::string reqType, millisecon -std::string StatusMemRequestData::prepareStatusString(std::shared_ptr mem) +std::string StatusMemRequestData::prepareStatusString(std::shared_ptr& mem) { std::string outputMessage; // set status value @@ -2208,31 +2180,45 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr // return the status of the memory json_factory.startJSON( outputMessage ); json_factory.addParmToJSON( outputMessage, "status", mem->getStatusString() ); + json_factory.addParmToJSON( outputMessage, "isCorrupted", mem->isCorrupted()); //json_factory.addParmToJSON( outputMessage, "status", "open" ); json_factory.addParmToJSON( outputMessage, "sizeInRAM", mem->GetRAMSize()); json_factory.addParmToJSON( outputMessage, "activeRequest", mem->getActiveRequestString()); if(mem->importDetails != nullptr){ - + std::scoped_lock l(mem->importDetails->importDetailsMutex); json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeStatus":"tmxImportStatus", pszStatus ); json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeProgress":"importProgress", mem->importDetails->usProgress ); - json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeTime":"importTime", mem->importDetails->importTimestamp ); + { + //std::scoped_lock l(mem->importDetails->importTimestampMutex); + json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeTime":"importTime", mem->importDetails->importTimestamp ); + } json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "segmentsReorganized":"segmentsImported", mem->importDetails->segmentsImported ); if(mem->importDetails->fReorganize) json_factory.addParmToJSON( outputMessage, "segmentsFilteredOut", mem->importDetails->filteredSegments ); json_factory.addParmToJSON( outputMessage, "invalidSegments", mem->importDetails->invalidSegments ); json_factory.addParmToJSON( outputMessage, "tmxSegmentCount", mem->importDetails->segmentsCount ); json_factory.addParmToJSON( outputMessage, "importRuntimeSec", mem->importDetails->lImportRunTimeSec ); json_factory.addParmToJSON( outputMessage, "importTimeoutSec", mem->importDetails->lImportTimeoutSec ); + json_factory.addParmToJSON( outputMessage, "newEntryIfOnlyContextAndTimestampDifferent", mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent ); + json_factory.addParmToJSON( outputMessage, "saveDifferentTargetsForSameSource", mem->importDetails->fSaveDifferentTargetsForSameSource ); + //json_factory.addParmToJSON( outputMessage, "ignoreSegIdOnUpdate", mem->importDetails->fIgnoreSegIdOnUpdate); + std::string invalidSegmRCs; - for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ - invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; + { + //std::lock_guard lk(mem->importDetails->invalidSegmentsRCsMutex); + for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ + invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; + } } std::string firstInvalidSegments; int i=0; - for(auto [segNum, invSegErrCode] : mem->importDetails->firstInvalidSegmentsSegNums){ - firstInvalidSegments += std::to_string(++i) + ":" + std::to_string(segNum) + ":" + std::to_string(invSegErrCode) + "; "; + { + //std::lock_guard l(mem->importDetails->firstInvalidSegmentsSegNumsMutex); + for(auto [segNum, invSegErrCode] : mem->importDetails->firstInvalidSegmentsSegNums){ + firstInvalidSegments += std::to_string(++i) + ":" + std::to_string(segNum) + ":" + std::to_string(invSegErrCode) + "; "; + } } json_factory.addParmToJSON( outputMessage, "invalidSegmentsRCs", invalidSegmRCs); json_factory.addParmToJSON( outputMessage, "firstInvalidSegments(num:segId:errCode)", firstInvalidSegments); @@ -2241,17 +2227,33 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "rc", mem->importDetails->importRc); } json_factory.addParmToJSON( outputMessage, "lastAccessTime", printTime(mem->tLastAccessTime) ); - json_factory.addParmToJSON( outputMessage, "creationTime", printTime(mem->stTmSign.creationTime) ); - std::string creationT5MVersion = std::to_string(mem->stTmSign.bGlobVersion) - + ":"+ std::to_string(mem->stTmSign.bMajorVersion) - + ":" + std::to_string(mem->stTmSign.bMinorVersion); + json_factory.addParmToJSON( outputMessage, "creationTime", printTime(mem->state->stTmSign.creationTime) ); + std::string creationT5MVersion = std::to_string(mem->state->stTmSign.bGlobVersion) + + ":"+ std::to_string(mem->state->stTmSign.bMajorVersion) + + ":" + std::to_string(mem->state->stTmSign.bMinorVersion); json_factory.addParmToJSON( outputMessage, "tmCreatedInT5M_version", creationT5MVersion.c_str() ); - json_factory.addParmToJSON( outputMessage, "segmentIndex", mem->stTmSign.segmentIndex ); - json_factory.addParmToJSON( outputMessage, "sourceLang", mem->stTmSign.szSourceLanguage ); - json_factory.addParmToJSON( outputMessage, "internalDescription", mem->stTmSign.szDescription ); - json_factory.addParmToJSON( outputMessage, "tmd fbuffer was modified", mem->TmBtree.wasModified()); - json_factory.addParmToJSON( outputMessage, "tmi fbuffer was modified", mem->InBtree.wasModified()); - //json_factory.addParmToJSON( outputMessage, "internalName", mem->stTmSign.szName ); + json_factory.addParmToJSON( outputMessage, "segmentIndex", mem->state->stTmSign.segmentIndex ); + json_factory.addParmToJSON( outputMessage, "sourceLang", mem->state->stTmSign.szSourceLanguage ); + json_factory.addParmToJSON( outputMessage, "internalDescription", mem->state->stTmSign.szDescription ); + json_factory.addParmToJSON( outputMessage, "tmd fbuffer was modified", mem->state->TmBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmd lStartKey", mem->state->TmBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmd lNextKey", mem->state->TmBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmd usNextFreeRecord", mem->state->TmBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmd usFirstLeaf", mem->state->TmBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmd usFirstNode", mem->state->TmBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmd usFirstDataBuffer", mem->state->TmBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmd usCurrentRecord", mem->state->TmBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmd NumOfSavedRecords", mem->state->TmBtree.GetNumOfSavedRecords()); + json_factory.addParmToJSON( outputMessage, "tmi fbuffer was modified", mem->state->InBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmi lStartKey", mem->state->InBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmi lNextKey", mem->state->InBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmi usNextFreeRecord", mem->state->InBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmi usFirstLeaf", mem->state->InBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmi usFirstNode", mem->state->InBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmi usFirstDataBuffer", mem->state->InBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmi usCurrentRecord", mem->state->InBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmi NumOfSavedRecords", mem->state->InBtree.GetNumOfSavedRecords()); + //json_factory.addParmToJSON( outputMessage, "internalName", mem->state->stTmSign.szName ); if ( ( (mem->eImportStatus == IMPORT_FAILED_STATUS) || (mem->eImportStatus == REORGANIZE_FAILED_STATUS)) && ( !mem->strError.empty() ) ) { @@ -2287,6 +2289,201 @@ int StatusMemRequestData::execute() { return( OK ); }; +int GetTablesMemRequestData::execute() { + json_factory.startJSON( outputMessage ); + std::string longNames, longNamesCaseIgnore, langTable, langGroupTable, authorsTable, filenamesOldTable, tagTablesTable, langIdToGroupTable; + int i = 0; + for(const auto& entry: mem->state->LongNames.stTableEntry){ + longNames += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; + } + + i = 0; + for(const auto& entry: mem->state->LongNamesCaseIgnore.stTableEntry){ + longNamesCaseIgnore += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; + } + + i = 0; + for(const auto& entry: mem->state->Authors.table){ + authorsTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->state->Languages.table){ + langTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->state->LangGroups.table){ + langGroupTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->state->TagTables.table){ + tagTablesTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + i = 0; + for(const auto& entry: mem->state->psLangIdToGroupTable){ + langIdToGroupTable += toStr(i++) + ")"+ toStr(entry) + "; "; + } + + json_factory.addParmToJSON( outputMessage, "tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)", "position)idInTable: value;"); + std::string tableName = "langTable(" + toStr(LANG_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, langTable); + + tableName = "filenames(long)(" + toStr(LONGNAME_KEY) +"-physical, (" + toStr(FILE_KEY)+ "-use this for commands))"; + json_factory.addParmToJSON( outputMessage, tableName, longNames); + + tableName = "filenames(long, caseign, virtual)(" + toStr(LONGNAME_KEY)+", (" + toStr(FILE_KEY) + "))"; + json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); + + tableName = "authors(" + toStr(AUTHOR_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, authorsTable); + + tableName = "tagTablesTable(" + toStr(TAGTABLE_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, tagTablesTable); + + tableName = "langGroupTable(virtual)(" + toStr(LANGGROUP_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, langGroupTable); + + tableName = "langIdToGroupTable"; + json_factory.addParmToJSON( outputMessage, tableName, langIdToGroupTable); + + json_factory.terminateJSON( outputMessage ); + return( OK ); +}; + + +int AddToTableMemRequestData::parseJSON(){ + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + // parse input parameters + auto loggingThreshold = -1; + + JSONFactory::JSONPARSECONTROL parseControl[] = { + { L"loggingThreshold", JSONFactory::INT_PARM_TYPE , &(loggingThreshold), -1}, + { L"tableKey", JSONFactory::INT_PARM_TYPE , &(tableKey), 0 }, + { L"entryId", JSONFactory::INT_PARM_TYPE , &(entryId), 0 }, + { L"value", JSONFactory::ASCII_STRING_PARM_TYPE, &(value), sizeof(value) }, + { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } }; + _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + + if(_rc_){ + return buildErrorReturn( _rc_, "::json parsing failed", BAD_REQUEST); + } + + return _rc_; +} + +int AddToTableMemRequestData::checkData(){ + //check tableKey + switch(tableKey){ + case LANG_KEY: + case AUTHOR_KEY: + case FILE_KEY: + //case TAGTABLE_KEY: + //case LONGNAME_KEY: + { + //if(tableKey == LONGNAME_KEY){ + // tableKey = FILE_KEY; + //} + //okay, go to the next check + break; + }; + + default:{ + std::string errMsg = "Invalid table key(" + std::to_string(tableKey) + "); available keys = LANG_KEY(" + std::to_string(LANG_KEY) + + "), FILE_KEY(" + std::to_string(FILE_KEY) + "), AUTHOR_KEY(" + std::to_string(AUTHOR_KEY) + ")";///, TAGTABLE_KEY(" + std::to_string(TAGTABLE_KEY) + ")"; + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + }; + } + + if(entryId<=0 || entryId >=NTMGETID_NOTFOUND_ID){ + std::string errMsg = "Value for the entryId should be between " + std::to_string(1) + " and " + std::to_string(NTMGETID_NOTFOUND_ID-1); + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + + } + + //if(!value[0]){ + // return buildErrorReturn(BAD_REQUEST, "Value field is empty", BAD_REQUEST); + //}else + { + const int len = strlen(value); + const int maxSize = tableKey == FILE_KEY? 255 : (sizeof(TMX_TABLE_ENTRY::szName) - 1); + if( len > maxSize){ + std::string errMsg = "Value for the entry is too long(" + std::to_string(len) + ") for selected table, max supported len = " + std::to_string(maxSize); + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + } + } + + return 0; +} + +int AddToTableMemRequestData::execute(){ + //if tm is loaded + if(!mem || !mem->isLoaded()){ + return buildErrorReturn(BAD_REQUEST, "Tm is not loaded", BAD_REQUEST); + } + //if table have that id already + //bool fEntryExists = false; + USHORT usRc = 0; + fSuppressBuildErrorLog = true; + + { + char buffer[256]{}; + usRc = mem->NTMGetNameFromID( entryId, (USHORT) tableKey, + buffer, sizeof(buffer), true ); + if(usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else if(buffer[0]){ + std::string msg = "Id match in Table " + std::to_string(tableKey) +" already have entry with matching id = " + + std::to_string(entryId) + " and value = \""s + buffer +"\";"s; + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + } + + + //check value + if(false /*fCheckValueMatch*/){ + USHORT idFromTable = 0; + usRc = mem->NTMGetIDFromName( value, + (USHORT)tableKey, &idFromTable, + NTMGETID_NOUPDATE_OPT); + if (usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else if(idFromTable){ + char buffer[256]{}; + usRc = mem->NTMGetNameFromID( idFromTable, (USHORT) tableKey, + buffer, sizeof(buffer) ); + if(usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else{ + std::string msg = "Value match Table " + std::to_string(tableKey) +" already have entry with id = " + + std::to_string(idFromTable) + " and value = \""s + buffer +"\";"s; + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + } + } + + if(!value[0]){ + return buildErrorReturn(BAD_REQUEST, "Value field is empty", BAD_REQUEST); + } + + //if present-> check overwrite flag + USHORT id = entryId; + //if(!fEntryExists || fOverwrite){ + usRc = mem->NTMAddNameToTable(value, tableKey, &id, true); + //} + if(usRc){ + std::string msg = "AddNameToTable returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + std::string msg = "Added entry to the table"+ std::to_string(tableKey) + ", id = " + std::to_string(id) + "; value = \\\""s + value + "\\\""; + outputMessage = "{\n\t\"msg\": \"" + msg + "\"\n}"; + return _rc_; +} + #include //#include // Include the flags library @@ -2410,8 +2607,8 @@ bool jsonAddFlagToList(std::stringstream &ss, const google::CommandLineFlagInfo if (fMoreInfo) { ss << "{\t\"name\": \"" << flag.name << "\",\t"; - ss << "\"value\": \"" << flag.current_value << "\"\t"; - ss << "\",\tfrom\": \"" << flag.filename << "\",\t "; + ss << "\"value\": \"" << flag.current_value << "\",\t"; + ss << "\"from\": \"" << flag.filename << "\",\t "; ss << "\"description\": \"" << flag.description << "\",\t"; ss << "\"type\": \"" << flag.type << "\",\t"; ss << "\"default\": \"" << flag.default_value << "\"\t}"; @@ -2468,24 +2665,27 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown2(int signal, size_t tmL void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeout& tmListTimeout, bool flushTmsToDisk, bool waitForImportAndReorganizeProcesses) { - TMManager::GetInstance()->fWriteRequestsAllowed = false; - TMManager::GetInstance()->fServiceIsRunning = true; - //pMemService->closeAll(); - T5Logger::GetInstance()->LogStop(); - + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fWriteRequestsAllowed = false; + TMManager::fServiceIsRunning = true; + TMManager::fLoadingTmIsAllowed = false; + } + if(waitForImportAndReorganizeProcesses){ int j= 3; - while(int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)){ + while(int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)) { if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg(".Failed to lock tm list:", __func__, __LINE__); - return ; + //return ; + tmListTimeout.reset(); } if( ++j % 15 == 0){ - T5LOG(T5WARNING) << "SHUTDOWN:: memory still in import..waiting 15 sec more... shutdown request was = "<< j* 15; + T5LOG(T5WARNING) << "SHUTDOWN:: tms still in import..waiting 15 sec more... shutdown request was = "<< j <<" seconds ago"; } - T5LOG(T5DEBUG) << "SHUTDOWN:: memory still in import..waiting 1 sec more... mem in import = "<< i; + //T5LOG(T5DEBUG) << "SHUTDOWN:: tms still in import..waiting 1 sec more... # of tm in import = "<< i; - //sleep(1); + sleep(1); } } @@ -2501,8 +2701,15 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeou T5LOG(T5ERROR) << "saveTm returned rest code not 200, but " << saveTmRD._rest_rc_ << "; rc = " << saveTmRD._rc_; } } + + FLAGS_v = 2; FLAGS_t5loglevel=0; + T5LOG(T5TRANSACTION) << "Closing t5memory"; + T5Logger::GetInstance()->LogStop(); if(signal != SHUTDOWN_CALLED_FROM_MAIN){ - TMManager::GetInstance()->fServiceIsRunning = false; + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fServiceIsRunning = false; + } //sleep(1); exit(signal); } @@ -2559,7 +2766,7 @@ int addProposalToJSONString pJsonFactory->addParmToJSONW( strJSON, L"context", Data.szContext); pJsonFactory->addParmToJSONW( strJSON, L"additionalInfo", Data.szAddInfo ); Data.getInternalKey(buff, OTMPROPOSAL_MAXSEGLEN); - pJsonFactory->addParmToJSONW( strJSON, L"internalKey", EncodingHelper::convertToWChar(buff).c_str() ); + pJsonFactory->addParmToJSONW( strJSON, L"internalKey", EncodingHelper::convertToWChar(buff) ); if(fAddFuzzyInfo){ wbuff[0] = '\0'; @@ -2658,7 +2865,6 @@ int ResourceInfoRequestData::execute(){ std::string memInfo = StatusMemRequestData::prepareStatusString(tm.second); ssOutput << "\"StatusInfo\": " << memInfo << "\n"; //AddToJson(ssOutput, "StatusInfo", memInfo, false ); - ssOutput << " }"; total += fSize; @@ -2785,11 +2991,6 @@ int ResourceInfoRequestData::execute(){ sumTime = ProxygenStats::getInstance()->getDeleteEntrySumTime(); AddRequestDataToJson(ssOutput, "DeleteEntry", sumTime, requestCount); ssOutput <<","; - - //requestCount = ProxygenStats::getInstance()->getReorganizeRequestCount(); - //sumTime = ProxygenStats::getInstance()->getReorganizeExecutionTime(); - //AddRequestDataToJson(ssOutput, "Reorganize", sumTime, requestCount); - //ssOutput <<","; requestCount = ProxygenStats::getInstance()->getSaveAllTmsRequestCount(); sumTime = ProxygenStats::getInstance()->getSaveAllTmsSumTime(); @@ -2918,8 +3119,6 @@ int ExportRequestData::parseJSON(){ T5LOG( T5WARNING) <<"::ExportRequestData::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); } - //std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); - //_rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); T5Logger::GetInstance()->AddToLogBuffer(", with startingKey = \"" + sKey +"\""); return 0; @@ -2927,11 +3126,6 @@ int ExportRequestData::parseJSON(){ int ExportRequestData::checkData(){ T5LOG( T5INFO) <<"::getMem::=== getMem request, memory = " << strMemName << "; format = " << requestAcceptHeader; - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn(_rc_ , "::getMem::Error: no valid API session" ); - } /* endif */ if ( strMemName.empty() ) { @@ -2952,7 +3146,6 @@ int ExportRequestData::checkData(){ return _rc_; } - DECLARE_int32(t5loglevel); int ExportRequestData::execute(){ if(command != EXPORT_MEM_TMX_STREAM && command != EXPORT_MEM_TMX){ // get a temporary file name for the memory package file or TMX file @@ -3099,19 +3292,13 @@ int ExportRequestData::ExportTmx(){ } int UpdateEntryRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( BAD_REQUEST, "can't verifyAPISession", BAD_REQUEST); - } /* endif */ - if ( strMemName.empty() ) { return buildErrorReturn( BAD_REQUEST, "Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters ulong recordKey = 0, targetKey = 0; @@ -3135,6 +3322,9 @@ int UpdateEntryRequestData::parseJSON(){ { L"customId", JSONFactory::ASCII_STRING_PARM_TYPE, &(Data.szCustomId), sizeof( Data.szCustomId) / sizeof( Data.szCustomId[0] ) }, { L"recordKey", JSONFactory::INT_PARM_TYPE , &(recordKey), 0 }, { L"targetKey", JSONFactory::INT_PARM_TYPE , &(targetKey), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(Data.fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(Data.fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource }, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(Data.fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate }, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, @@ -3232,7 +3422,6 @@ int UpdateEntryRequestData::execute(){ } // update the memory TMX_EXT_OUT_W TmPutOut; // ptr to TMX_EXT_OUT_W structure - memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string source = EncodingHelper::convertToUTF8(Data.szSource); @@ -3273,7 +3462,7 @@ int UpdateEntryRequestData::execute(){ std::wstring outputMessageW; addProposalToJSONString(outputMessageW, Data); - outputMessage = EncodingHelper::convertToUTF8(outputMessageW.c_str()); + outputMessage = EncodingHelper::convertToUTF8(outputMessageW); return( _rc_ ); } @@ -3286,7 +3475,7 @@ int GetEntryRequestData::parseJSON(){ return buildErrorReturn( _rc_, "Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters Data.clear(); @@ -3334,10 +3523,10 @@ int GetEntryRequestData::execute() "Error: memptr is null", 500); } - auto accMode=mem->usAccessMode;//temp. reset access mode - mem->usAccessMode=0; + auto accMode=mem->state->usAccessMode;//temp. reset access mode + mem->state->usAccessMode=0; _rc_ = mem->getNextProposal(Data); - mem->usAccessMode = accMode; + mem->state->usAccessMode = accMode; if((BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD!=_rc_) && (Data.currentInternalKey.getTargetKey() != targetKey || Data.currentInternalKey.getRecordKey() != recordKey)){ if(recordKey == 7 && targetKey == 1 //requested first record @@ -3359,7 +3548,7 @@ int GetEntryRequestData::execute() return buildErrorReturn(BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD,"BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD: tm needs to be reorganized"); } //json_factory.terminateJSONW( outputMessageW ); - outputMessage = EncodingHelper::convertToUTF8(outputMessageW.c_str()); + outputMessage = EncodingHelper::convertToUTF8(outputMessageW); return 0; } @@ -3370,7 +3559,7 @@ int DeleteEntryRequestData::parseJSON(){ return buildErrorReturn( _rc_, "Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters Data.clear(); @@ -3453,7 +3642,6 @@ int DeleteEntryRequestData::execute(){ std::string errorStr; TMX_EXT_OUT_W TmPutOut; - memset( &TmPutOut, 0, sizeof(TmPutOut) ); if(Data.nextInternalKey.getRecordKey() && Data.nextInternalKey.getTargetKey() && Data.lSegmentId){ //internalKey = true; @@ -3537,11 +3725,6 @@ int FuzzySearchRequestData::reset(){ } int FuzzySearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", BAD_REQUEST); - } /* endif */ if ( strMemName.empty() ) { @@ -3549,7 +3732,7 @@ int FuzzySearchRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); Data.clear(); int loggingThreshold = -1; @@ -3569,10 +3752,15 @@ int FuzzySearchRequestData::parseJSON(){ { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, + { L"forceFuzzy", JSONFactory::INT_PARM_TYPE, &fForceFuzzy, 0}, { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } }; _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + if(_rc_){ + return buildErrorReturn(_rc_, "error in parseJson"); + } + if(loggingThreshold >=0) T5Logger::GetInstance()->SetLogLevel(loggingThreshold); return _rc_; @@ -3631,11 +3819,21 @@ int addProposalsToJSONString ) { JSONFactory *pJsonFactory = JSONFactory::getInstance(); + // Create index view [0, 1, 2, ..., n-1] + std::vector indices(iNumOfProposals); + std::iota(indices.begin(), indices.end(), 0); + + + // Sort indices by comparing the actual proposals + std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { + return vProposals[a].getFuzziness() > vProposals[b].getFuzziness(); + }); pJsonFactory->addArrayStartToJSONW( strJSON ); - for ( int i = 0; i < iNumOfProposals; i++ ) + //for ( int i = 0; i < iNumOfProposals; i++ ) + for (size_t idx : indices)//use indices to have sorted output { - addProposalToJSONString( strJSON, vProposals[i], fAddFuzzyInfo); + addProposalToJSONString( strJSON, vProposals[idx], fAddFuzzyInfo); } /* endfor */ pJsonFactory->addArrayEndToJSONW( strJSON ); @@ -3660,16 +3858,14 @@ int FuzzySearchRequestData::execute(){ std::vector vProposals(Data.iNumOfProposals); if ( _rc_ == NO_ERROR ) { - TMX_GET_W GetIn; - TMX_GET_OUT_W GetOut; - memset( &GetIn, 0, sizeof(TMX_GET_W) ); - memset( &GetOut, 0, sizeof(TMX_GET_OUT_W) ); + TMX_GET_W GetIn{}; + TMX_GET_OUT_W GetOut{}; mem->OtmProposalToGetIn( Data, &GetIn ); GetIn.usConvert = MEM_OUTPUT_ASIS; GetIn.usRequestedMatches = (USHORT)vProposals.size(); GetIn.ulParm = GET_EXACT; - GetIn.pvGMOptList = mem->pvGlobalMemoryOptions; + GetIn.fForceFuzzySearch = fForceFuzzy; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ auto str = EncodingHelper::convertToUTF8(GetIn.szSource); @@ -3689,38 +3885,64 @@ int FuzzySearchRequestData::execute(){ } tokens += "\n}]"; } + + std::vector addedRecords; if ( _rc_ == 0 ) { T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: lookup complete, found " << GetOut.usNumMatchesFound << " proposals" ; + //sort GetOut.stMatchTable[j]; + std::sort( + GetOut.stMatchTable, + GetOut.stMatchTable + GetOut.usNumMatchesFound, + [](const TMX_MATCH_TABLE_W& a, const TMX_MATCH_TABLE_W& b) { + return a.usMatchLevel > b.usMatchLevel; + } + ); wchar_t szRequestedString[2049]; Data.getSource( szRequestedString, sizeof(szRequestedString) ); + - for ( int i = 0; i < (int)vProposals.size(); i++ ) + for ( int i = 0, j = 0; i < (int)vProposals.size(); j++ ) { - if ( i >= GetOut.usNumMatchesFound ) + if ( j >= GetOut.usNumMatchesFound ) { - vProposals[i].clear(); + vProposals[i++].clear(); } else { + auto& match = GetOut.stMatchTable[j]; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - auto strSource = EncodingHelper::convertToUTF8(GetOut.stMatchTable[i].szSource ); - T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: proposal " << i << ": match=" << GetOut.stMatchTable[i].usMatchLevel << ", source=", strSource; + auto strSource = EncodingHelper::convertToUTF8(match.szSource ); + T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: proposal " << i << ": match=" << match.usMatchLevel << ", source=", strSource; } - //replace tags for proposal - auto result = std::make_unique(GetOut.stMatchTable[i].szSource, GetOut.stMatchTable[i].szTarget, - szRequestedString); - wcscpy(GetOut.stMatchTable[i].szSource, result->getSrcWithTagsFromRequestC()); - wcscpy(GetOut.stMatchTable[i].szTarget, result->getTrgWithTagsFromRequestC()); - - if( GetOut.stMatchTable[i].usMatchLevel >= 100){ - //correct match rate for exact match based on whitespace difference - int wsDiff = 0; - UtlCompIgnWhiteSpaceW(szRequestedString, GetOut.stMatchTable[i].szSource, 0, &wsDiff); - GetOut.stMatchTable[i].usMatchLevel -= wsDiff; + + bool proposalAlreadyAdded = std::any_of(addedRecords.begin(), addedRecords.end(), + [&](const TMCursor& cursor) { + return cursor.getRecordKey() == match.lKey && + cursor.getTargetKey() == match.usTargetNum; + }); + + if(!proposalAlreadyAdded){ + //replace tags for proposal + auto result = std::make_unique(match.szSource, match.szTarget, + szRequestedString); + wcscpy(match.szSource, result->getSrcWithTagsFromRequestC()); + wcscpy(match.szTarget, result->getTrgWithTagsFromRequestC()); + + if( match.usMatchLevel >= 100){ + //correct match rate for exact match based on whitespace difference + int wsDiff = 0; + UtlCompIgnWhiteSpaceW(szRequestedString, match.szSource, 0, &wsDiff); + match.usMatchLevel -= wsDiff; + } + + TMCursor location; + location.setInternalKey(match.lKey, match.usTargetNum); + addedRecords.push_back(location); + mem->MatchToOtmProposal( &match, &vProposals[i] ); + i++; } - mem->MatchToOtmProposal( GetOut.stMatchTable + i, &vProposals[i] ); } /* end */ } /* end */ } @@ -3742,6 +3964,7 @@ int FuzzySearchRequestData::execute(){ json_factory.addParmToJSONW( strOutputParmsW, L"NumOfFoundProposals", Data.iNumOfProposals ); json_factory.addParmToJSONW( strOutputParmsW, L"searchedSrc", Data.szSource ); json_factory.addParmToJSONW( strOutputParmsW, L"customId", Data.szCustomId ); + json_factory.addParmToJSONW( strOutputParmsW, L"forceFuzzy", fForceFuzzy ); if(FLAGS_add_tokens_to_fuzzy){ json_factory.addParmToJSONW( strOutputParmsW, L"tokens", tokens ); @@ -3749,6 +3972,14 @@ int FuzzySearchRequestData::execute(){ if ( Data.iNumOfProposals > 0 ) { + //std::sort( + // vProposals.begin(), + // vProposals.end(), + // [](const OtmProposal& a, const OtmProposal& b) { + // return a.getFuzziness() > b.getFuzziness(); + // } + //); + json_factory.addNameToJSONW( strOutputParmsW, L"results" ); addProposalsToJSONString( strOutputParmsW, vProposals, Data.iNumOfProposals, true ); } /* endif */ @@ -3835,11 +4066,6 @@ int parseAndRunBatchRequest(RequestData* pRequest,RequestData& subRequest){ } int ConcordanceExtendedSearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ if ( strMemName.empty() ) { @@ -3847,7 +4073,7 @@ int ConcordanceExtendedSearchRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); int loggingThreshold = -1; JSONFactory::JSONPARSECONTROL parseControl[] = { @@ -4070,7 +4296,7 @@ int ConcordanceExtendedSearchRequestData::execute() { strncpy(Data.szSearchPos, OProposal.nextInternalKey.getStr().c_str(), 20); } /* endif */ - else if ( _rc_ == EqfMemory::INFO_ENDREACHED ) + else if ( _rc_ == EqfMemoryError::INFO_ENDREACHED ) { _rc_ = ENDREACHED_RC; } @@ -4164,19 +4390,14 @@ int ConcordanceExtendedSearchRequestData::execute() int ConcordanceSearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ - if ( strMemName.empty() ) + if ( strMemName.empty()) { return buildErrorReturn( _rc_, "::concordanceSearch::Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); int loggingThreshold = -1; JSONFactory::JSONPARSECONTROL parseControl[] = { @@ -4391,7 +4612,7 @@ int ConcordanceSearchRequestData::execute(){ { strncpy(Data.szSearchPos, OProposal.nextInternalKey.getStr().c_str(), 20); } /* endif */ - else if ( _rc_ == EqfMemory::INFO_ENDREACHED ) + else if ( _rc_ == EqfMemoryError::INFO_ENDREACHED ) { _rc_ = ENDREACHED_RC; } diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index 8b432e8c..5f0c1380 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -109,37 +109,21 @@ static CHAR_W chInputW[ MAX_CHINPUTW ]; #define TAG_END_CHAR "." // tag end character #define TAG_END_CHARW L"." -//PNODE TAFastCreateNodeTree( PTBTAGENTRY pTags, ULONG ulNoOfTags ); -//PNODE TAFastCreateTagTree( PLOADEDTABLE pLoadedTable, PTAGTABLE pTagTable ); -//PNODE TAFastCreateAttrTree( PLOADEDTABLE pLoadedTable, PTAGTABLE pTagTable ); -//BOOL TAFastAddNode ( PNODE *, PNODE, PULONG, PULONG ); -//int TATastTagCompare( const void *, const void * ); - -VOID TASetStartStopType(PSTARTSTOP pCurrent, USHORT usStart, USHORT usStop, USHORT usType ); USHORT TAFindQuote ( PTOKENENTRY pTok, USHORT usStartSearch, PCHAR_W pchQuote ); USHORT TAProtectedPartsInQuotedText ( PTOKENENTRY pTok, int iAttrStartOffset, - PSTARTSTOP * pStartStop, - PSTARTSTOP * pCurrent, - PTOKENENTRY pAttrTokBuffer, + BytesRef pStartStop, + std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, - PULONG pulTableAlloc, - PULONG pulTableUsed, PUSHORT pusColPos, CHAR_W chQuote, LONG lAttrSize, BOOL fSpecialMode ); -USHORT TAGotoNextStartStopEntry -( - PSTARTSTOP * ppStartStop, - PSTARTSTOP * ppCurrent, - PULONG pulTableUsed, - PULONG pulTableAlloc -); + static void TATagAddWSpace ( @@ -433,28 +417,23 @@ CHAR fWhiteSpace[256] = BOOL TATagTokenizeW ( PWCHAR pszInput, // pointer to input data - PLOADEDTABLE pVoidTable, // pointer to loaded tag table - BOOL fComplete, // TRUE = no more buffers to follow + PLOADEDTABLE pTable, // pointer to loaded tag table PWCHAR *ppRest, // pointer to not processed data in buffer - PUSHORT pusColPos, // column position - PTOKENENTRY pTokBuf, // buffer for created tokens - USHORT usTokens // max entries in token buffer + PUSHORT pusColPos, // column position + std::vector& pTokBuf // buffer for created tokens ) { PWCHAR pszData; // pointer for input data processing PWCHAR pszDataStart; // pointer for end of data block - PTOKENENTRY pToken; // pointer to current token + TOKENENTRY Token; // pointer to current token PWCHAR pszEnd; // pointer to end of data BOOL fFound = FALSE; // flag used in setting pRest - PLOADEDTABLE pTable = (PLOADEDTABLE)pVoidTable; // pointer to loaded tag table TOKSTATUS TokStatus; // tokenize status area TOKENENTRY NewToken; // buffer for new tokens MSSTACK MSStack[10]; PTAG pTag; // pointer to first tag in tagtable - PTOKENENTRY pLastTag = NULL; // pointer to last tag token + int iLastTagPosInBuf = -1; USHORT usTagColPos = 0; // column position of last tag - USHORT usCurColPos; // current column position - BOOL fProcessingAttributes; // TRUE = attributes are processed PATTRIBUTE pAttr; // ptr to attribute array of tag table PBYTE pByte; // helper pointer SHORT sTags; // number of tags in tag table @@ -473,10 +452,9 @@ BOOL TATagTokenizeW } else { - pTagAddInfo = NULL; + pTagAddInfo = NULL; } /* endif */ - /*******************************************************************/ /* Initialize tokenize status area */ /*******************************************************************/ @@ -516,50 +494,46 @@ BOOL TATagTokenizeW pTag = (PTAG)((PBYTE)pTable->pTagTable + pTable->pTagTable->stFixTag.uOffset ); pszData = pszInput; // start at begin of input buffer - pToken = pTokBuf; // start at begin of token buffer - pToken->sTokenid = TEXT_TOKEN; // assume text data - pToken->pDataStringW = pszData; // remember start of data - usTokens--; // reserve space for end of list token - fProcessingAttributes = FALSE; // we are waiting for tags ... - memset(&NewToken, 0, sizeof(NewToken)); - while( fOK && *pszData && usTokens )// while not end of data and free tokens - { - USHORT usProcessFlag; // processing flag /* @KWT0010A */ - + Token.reset();// = pTokBuf.data(); // start at begin of token buffer + + Token.sTokenid = TEXT_TOKEN; // assume text data + Token.pDataStringW = pszData; // remember start of data + //usTokens--; // reserve space for end of list token + NewToken.reset(); + while( fOK && *pszData)// && usTokens )// while not end of data and free tokens + { // the following statement has been disabled to fix PTM KAT0330 -// fProcessingAttributes = FALSE; // we are waiting for tags ... pszDataStart = pszData; - usCurColPos = TokStatus.usColPos; - usProcessFlag = (fComplete ) ? /* @KWT0010A */ - ( MATCH_SKIP_DATA|MATCH_DATA_COMPLETE) : MATCH_SKIP_DATA; + //usProcessFlag = (fComplete ) ? /* @KWT0010A */ + // ( MATCH_SKIP_DATA|MATCH_DATA_COMPLETE) : MATCH_SKIP_DATA; fFound = TAMatchTag( &TokStatus, &pszData, pTable->pNodeArea->pRootNode, &NewToken, - usProcessFlag); /* @KWT0010M */ + MATCH_SKIP_DATA|MATCH_DATA_COMPLETE); /* @KWT0010M */ if ( fFound ) { - fProcessingAttributes = FALSE; - /*************************************************************/ /* Build token for data up to start of recognized tag */ /*************************************************************/ if ( pszDataStart != NewToken.pDataStringW ) { - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszDataStart; // set start of data - pToken->iLength = (USHORT)(NewToken.pDataStringW - pszDataStart); - pToken->sAddInfo = 0; - pToken->ClassId = CLS_DEFAULT; - pToken->usOrgId = 0; - usTokens--; - pToken++; + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszDataStart; // set start of data + Token.iLength = (USHORT)(NewToken.pDataStringW - pszDataStart); + Token.sAddInfo = 0; + Token.ClassId = CLS_DEFAULT; + Token.usOrgId = 0; + //usTokens--; + pTokBuf.push_back(Token); + Token.reset(); + //Token++; } /* endif */ /*************************************************************/ /* Add tag token */ /*************************************************************/ - if ( usTokens ) + //if ( usTokens ) { // special handling for tags which may be followed by attributes: // check if tag end character is a blank, when it is one look-ahead @@ -572,7 +546,7 @@ BOOL TATagTokenizeW // only for blank end delimiters... if ( *pszTest == L' ') { - PSZ_W pszEndDelim = pTable->pTagNamesW + pTag[NewToken.sTokenid].uEndDelimOffs; + PSZ_W pszEndDelim = pTable->pTagNamesW.data() + pTag[NewToken.sTokenid].uEndDelimOffs; // find first non-blank character while ( *pszTest == L' ' ) @@ -599,17 +573,20 @@ BOOL TATagTokenizeW } /* endif */ // add tag token to token list - pLastTag = pToken; + iLastTagPosInBuf = pTokBuf.size();//pToken; usTagColPos = TokStatus.usStartColPos; - memcpy( pToken, &NewToken, sizeof(TOKENENTRY) ); - pToken->sAddInfo = pTag[NewToken.sTokenid].BitFlags.AddInfo; + memcpy( &Token, &NewToken, sizeof(TOKENENTRY) ); + Token.sAddInfo = pTag[NewToken.sTokenid].BitFlags.AddInfo; if ( pTagAddInfo ) { - pToken->ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; - pToken->usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; + Token.ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; + Token.usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; } /* endif */ - pToken++; - usTokens--; + + //pToken++; + pTokBuf.push_back(Token); + Token.reset(); + //usTokens--; } /* endif */ /**********************************************************/ @@ -617,7 +594,6 @@ BOOL TATagTokenizeW /**********************************************************/ if ( (NewToken.sTokenid >= 0) && pTag[NewToken.sTokenid].fAttr ) { - fProcessingAttributes = TRUE; // attributes may follow ... do { /***********************************************************/ @@ -633,7 +609,6 @@ BOOL TATagTokenizeW if ( !fWhiteSpace[NewToken.pDataStringW[NewToken.iLength-1]] ) { fFound = FALSE; - fProcessingAttributes = FALSE; // no more attributes } else { @@ -645,39 +620,31 @@ BOOL TATagTokenizeW /***********************************************************/ if ( fFound ) { - usCurColPos = TokStatus.usColPos; fFound = TAMatchTag( &TokStatus, &pszData, pTable->pAttrNodeArea->pRootNode, &NewToken, MATCH_SKIP_WHITESPACE | MATCH_QUOTED_STRINGS ); - // the following code has been disabled to fix KAT0330 - // if no attribute has been found the pszData pointer still - // points to the begin of the maybe attribute ... - // if ( !fFound && *pszData ) - // { - // /*******************************************************/ - // /* Attribute mismatch, but not end of text block */ - // /*******************************************************/ - // fProcessingAttributes = FALSE; // no more attributes - // } /* endif */ } /* endif */ - if ( fFound && usTokens ) + if ( fFound)// && usTokens ) { - memcpy( pToken, &NewToken, sizeof(TOKENENTRY) ); - pToken->sAddInfo = pAttr[NewToken.sTokenid-sTags].BitFlags.AddInfo; + memcpy( &Token, &NewToken, sizeof(TOKENENTRY) ); + Token.sAddInfo = pAttr[NewToken.sTokenid-sTags].BitFlags.AddInfo; if ( pTagAddInfo ) { - pToken->ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; - pToken->usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; + Token.ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; + Token.usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; } /* endif */ - pToken++; - usTokens--; + //pToken++; + + pTokBuf.push_back(Token); + Token.reset(); + //usTokens--; } /* endif */ - } while ( fFound && usTokens ); /* enddo */ + } while ( fFound ); //&& usTokens ); /* enddo */ } /* endif */ } else @@ -685,145 +652,56 @@ BOOL TATagTokenizeW /*************************************************************/ /* No tag in remaining data found */ /*************************************************************/ - if ( fComplete ) - { - /***********************************************************/ - /* Add remaining data as TEXT token */ - /***********************************************************/ - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszDataStart; // set start of data - pToken->iLength = (USHORT)UTF16strlenCHAR(pszDataStart); - pszData = pszDataStart + pToken->iLength; - usTokens--; - pToken++; - } - else - { - /*************************************************************/ - /* Process remaining data the next time ... */ - /*************************************************************/ - pszData = pszDataStart + UTF16strlenCHAR(pszDataStart); - } /* endif */ + /***********************************************************/ + /* Add remaining data as TEXT token */ + /***********************************************************/ + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszDataStart; // set start of data + Token.iLength = (USHORT)UTF16strlenCHAR(pszDataStart); + pszData = pszDataStart + Token.iLength; + + pTokBuf.push_back(Token); + Token.reset(); } /* endif */ } /* endwhile */ // if not processed completely or not last part of file // set pRest pointer to begin of last tag or (if no tags // are contained in data) to last linebreak. - if ( !fOK ) - { - // nothing was processed due to errors (memory allocation?) - *ppRest = pszInput; - } - else if ( !usTokens || // end of token table reached - !fComplete ) // or not last block of data ??? - { - // at least two tokens in buffer .... - if ( (pToken != pTokBuf) && ( pToken != (pTokBuf+1)) ) // if tokens in buffer - { - /**************************************************************/ - /* There are tokens in the buffer, let's see how we have to */ - /* deal with them ... */ - /**************************************************************/ - - // go back to the last tag to allow tag attribute and whitespace - // processing in the next tokenization block (unless this tag - // is the first token in the token buffer or the remaining data - // is longer than twice the max segment size ) - int iRemainLength = UTF16strlenCHAR(pToken[-1].pDataStringW + pToken[-1].iLength); - if ( ( (pLastTag != pTokBuf) && // more than one tag in buffer - (iRemainLength < (2*MAX_SEGMENT_SIZE)) ) || // rest is smaller than 2 segments - (usTokens == 0) ) // token buffer overflow - { - pToken = pLastTag; - TokStatus.usColPos = usTagColPos; - *ppRest = pToken->pDataStringW; - } - else - { - // continue right behind the last recognized token - *ppRest = pToken[-1].pDataStringW + pToken[-1].iLength; - if ( (*pszData == EOS) && (*ppRest >= pszData) ) - { - // no more data to follow, segment has been processed completely - *ppRest = NULL; - } /* endif */ - } /* endif */ - } - else - { - /**************************************************************/ - /* The token buffer is empty, no tags were recognized ... */ - /* In addition we came here if only one tag is recognized... */ - /* */ - /* Go back to the last LF and add that data as a TEXT token */ - /**************************************************************/ - PSZ_W pszAreaStart = NULL; - pszEnd = pszData; - pszData--; - if ( pToken == pTokBuf ) - { - // no token in buffer - pszAreaStart = pszInput; // set start of data - } - else - { - // one token in buffer - pszAreaStart = pToken[-1].pDataStringW + pToken[-1].iLength; - } /* endif */ - - while ( pszData > pszAreaStart ) - { - if ( *pszData == LF ) - { - break; // leave while loop - } - else - { - pszData--; // test previous character - } /* endif */ - } /* endwhile */ - - if ( pszData == pszInput ) // no linefeed found ??? - { - *ppRest = pszEnd - 1; // set pRest to end of data CHECK in DEBUG - } - else - { - *ppRest = pszData; // set pRest to last linefeed - } /* endif */ - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszAreaStart; // set start of data - pToken->iLength = (USHORT)(*ppRest - pszAreaStart); - - usTokens--; - pToken++; - } /* endif */ - } - else - { - *ppRest = NULL; - } /* endif */ - - // add end-of-list token - if ( fOK ) - { - memset( pToken, 0, sizeof(TOKENENTRY) ); - pToken->iLength = 0; - pToken->sTokenid = ENDOFLIST; + if ( fOK ) + { + *ppRest = NULL; + + // add end-of-list token + Token.reset(); + Token.iLength = 0; + Token.sTokenid = ENDOFLIST; + pTokBuf.push_back(Token); + Token.reset(); + *pusColPos = TokStatus.usColPos; + } + else + { + // nothing was processed due to errors (memory allocation?) + *ppRest = pszInput; + } /* endif */ - *pusColPos = TokStatus.usColPos; - } /* endif */ + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ + for(auto& tok: pTokBuf){ + T5LOG(T5DEVELOP) << "pTokBuf[" << &tok - pTokBuf.data() <<"]: ClassId=" << tok.ClassId << "; iLength=" << tok.iLength << "; pDataStringW=" << (tok.pDataStringW? EncodingHelper::convertToUTF8(tok.pDataStringW): "" ) + << "; sAddInfo=" << tok.sAddInfo << "; sTokenid=" << tok.sTokenid << "; usOrgId=" << tok.usOrgId; + } + } - /*******************************************************************/ - /* cleanup */ - /*******************************************************************/ - if ( TokStatus.pRetryStack != NULL ) - { - UtlAlloc( (PVOID *)&TokStatus.pRetryStack, 0L, 0L, NOMSG ); - } /* endif */ + /*******************************************************************/ + /* cleanup */ + /*******************************************************************/ + if ( TokStatus.pRetryStack != NULL ) + { + UtlAlloc( (PVOID *)&TokStatus.pRetryStack, 0L, 0L, NOMSG ); + } /* endif */ - return( fOK ); + return( fOK ); } /* end of function TATagTokenize */ //------------------------------------------------------------------------------ @@ -1435,7 +1313,7 @@ PNODEAREA TACreateTagTree pByte = (PBYTE) pTagTable; pTag = (PTAG) ( pByte + pTagTable->stFixTag.uOffset); //pTagNames = (PSZ)( pByte + pTagTable-> uTagNames); - pTagNamesW = pLoadedTable->pTagNamesW; + pTagNamesW = pLoadedTable->pTagNamesW.data(); ulNoOfTags = pTagTable->stFixTag.uNumber; for ( usI = 0; usI < 27; usI++ ) { @@ -1551,7 +1429,7 @@ PNODEAREA TACreateAttrTree pByte = (PBYTE) pTagTable; // pTagNames = (PSZ)( pByte + pTagTable-> uTagNames); - pTagNamesW = pLoadedTable->pTagNamesW; + pTagNamesW = pLoadedTable->pTagNamesW.data(); ulNoOfTags = pTagTable->stAttribute.uNumber; for ( usI = 0; usI < 27; usI++ ) { @@ -2058,12 +1936,11 @@ PNODEAREA TACreateNodeTree USHORT TASoSiProtectTable ( PSZ_W pszSegment, // ptr to text of segment being processed - PSTARTSTOP *ppStartStop // ptr to caller's start/stop table ptr + BytesRef pStartStop // ptr to caller's start/stop table ptr ) { USHORT usRC = NO_ERROR; // function return code - PSTARTSTOP pStartStop = *ppStartStop; // ptr to new start/stop table - PSTARTSTOP pCurrent = NULL; // ptr for start/stop table processing + std::vector pCurrent; // ptr for start/stop table processing PSTARTSTOP pAct, pActNew; // ptr for start/stop table processing PSZ_W pText = pszSegment; // ptr to segment USHORT usCountSOSI = 0; @@ -2086,7 +1963,7 @@ USHORT TASoSiProtectTable if ( usCountSOSI ) { - pAct = pStartStop; + pAct = (PSTARTSTOP)pStartStop.data(); while ( pAct->usStop ) { pAct++; @@ -2095,16 +1972,12 @@ USHORT TASoSiProtectTable usCountSOSI++; LONG size = (usCountSOSI * sizeof(STARTSTOP)) > MIN_ALLOC ? (usCountSOSI * sizeof(STARTSTOP)) : MIN_ALLOC; - if ( !UtlAlloc( (PVOID *)&pCurrent, 0L, size, NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - + pCurrent.resize(size); if ( !usRC ) { - pAct = pStartStop; - pActNew = pCurrent; + pAct = (PSTARTSTOP)pStartStop.data(); + pActNew = (PSTARTSTOP)pCurrent.data(); while ( pAct->usStop ) { memcpy( pActNew, pAct, sizeof(STARTSTOP)); @@ -2154,8 +2027,7 @@ USHORT TASoSiProtectTable // Add end element .. memcpy( pActNew, pAct, sizeof(STARTSTOP)); - UtlAlloc( (PVOID *) ppStartStop, 0L, 0L, NOMSG ); - *ppStartStop = pCurrent; + pStartStop = pCurrent; } /* endif */ } /* endif */ @@ -2166,7 +2038,7 @@ USHORT TASoSiProtectTable typedef EQF_BOOL (// /*APIENTRY*/ *PFNSTARTSTOPEXIT) ( PSZ, PSTARTSTOP *); typedef EQF_BOOL (// /*APIENTRY*/ -*PFNSTARTSTOPEXITW) ( PSZ_W, PSTARTSTOP *); +*PFNSTARTSTOPEXITW) ( PSZ_W, BytesRef); //------------------------------------------------------------------------------ // External function @@ -2192,22 +2064,6 @@ typedef EQF_BOOL (// /*APIENTRY*/ // error // ERROR_NOT_ENOUGH_MEMORY if memory allocation failed //------------------------------------------------------------------------------ -USHORT TACreateProtectTableW -( - PSZ_W pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - PFN pvUserExitW, - ULONG ulCP // ASCII cp fitting to Segmenttext -) -{ - return( TACreateProtectTableWEx( pszSegment, pVoidTable, usColPos, pTokBuffer, usTokBufferSize, - ppStartStop, pvUserExit, pvUserExitW, ulCP, 0 ) ); -} int memicmp(const void *buf1, const void *buf2, size_t count){ return memcmp(buf1, buf2, count); @@ -2218,22 +2074,19 @@ USHORT TACreateProtectTableWEx PSZ_W pszSegment, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function + BytesRef pStartStop, // ptr to caller's start/stop table ptr + //PFN pvUserExit, // ptr to user exit function PFN pvUserExitW, ULONG ulCP, // ASCII cp fitting to Segmenttext int iMode // mode for function ) { USHORT usRC = NO_ERROR; // function return code - PSTARTSTOP pStartStop = NULL; // ptr to new start/stop table - PSTARTSTOP pCurrent; // ptr for start/stop table processing + //PSTARTSTOP pCurrent; // ptr for start/stop table processing PSTARTSTOP pAct; // ptr for start/stop table processing BOOL fTag; // we are in tag processing mode - ULONG ulTableUsed = 0; - ULONG ulTableAlloc = 0; PLOADEDTABLE pTagTable; // ptr to loaded tag table PTOKENENTRY pTok; // ptr for token list processing PSZ_W pRest; // ptr to not tokenized data @@ -2249,9 +2102,11 @@ USHORT TACreateProtectTableWEx BOOL fTRNoteFound = 0; // true if TRNOTE found in curr segment BOOL fTRTag= FALSE; // true if inside of a trnote CHAR chAsciiSeg[MAX_SEGMENT_SIZE]; - PTOKENENTRY pAttrTokBuffer = NULL; // used temp.for translatable attributetokens + std::vector pAttrTokBuffer; // used temp.for translatable attributetokens LONG lAttrSize = 0L; PTOKENENTRY pNextTok; // ptr for next token in token list + + STARTSTOP current; /********************************************************************/ /* Check input parameters */ @@ -2260,9 +2115,7 @@ USHORT TACreateProtectTableWEx if ( (pszSegment == NULL) || (pTagTable == NULL) || - (pTokBuffer == NULL) || - (usTokBufferSize == 0) || - (ppStartStop == NULL) ) + (usTokBufferSize == 0)) { LOG_AND_SET_RC(usRC, T5INFO, ERROR_INVALID_DATA); } /* endif */ @@ -2271,484 +2124,354 @@ USHORT TACreateProtectTableWEx /* Use user exit for start/stop table if there is one */ /********************************************************************/ if ( usRC == NO_ERROR ) + { + /******************************************************************/ + /* Create start/stop table using tokenizer */ + /* Set tag table variables */ + /********************************************************************/ + sNumTags = (SHORT)pTagTable->pTagTable->uNumTags; + pByte = (PBYTE)pTagTable->pTagTable; + pTag = OFFSETTOPOINTER(PTAG, pTagTable->pTagTable->stFixTag.uOffset ); + pAttribute = OFFSETTOPOINTER(PATTRIBUTE, + pTagTable->pTagTable->stAttribute.uOffset ); + + + /********************************************************************/ + /* Tokenize input segment */ + /********************************************************************/ + pTagTable = (PLOADEDTABLE)pVoidTable; + pRest = NULL; + TATagTokenizeW( pszSegment, + pTagTable, + &pRest, + &usColPos, + pTokBuffer); + usRC = ( pRest == NULL ) ? NO_ERROR : EQFRS_AREA_TOO_SMALL; + + TATagAddWSpace(pszSegment, pTokBuffer.data(), sNumTags); + + } + + /********************************************************************/ + /* Fill start/stop table */ + /********************************************************************/ + if ( usRC == NO_ERROR ) { - if (pvUserExitW ) - { - pfnUserExitW = (PFNSTARTSTOPEXITW) pvUserExitW; - if ( !pfnUserExitW( pszSegment, &pStartStop ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - else if ( pvUserExit ) + current.reset(); + pTok = pTokBuffer.data(); + while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { /****************************************************************/ - /* Let user exit create the start/stop table */ + /* Check for attributes or tags with translatable info */ /****************************************************************/ - pfnUserExit = (PFNSTARTSTOPEXIT)pvUserExit; - Unicode2ASCII( pszSegment, &chAsciiSeg[0], ulCP); - if ( !pfnUserExit( &chAsciiSeg[0], &pStartStop ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - else - { - - /******************************************************************/ - /* Create start/stop table using tokenizer */ - /******************************************************************/ - - /********************************************************************/ - /* Set tag table variables */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - sNumTags = (SHORT)pTagTable->pTagTable->uNumTags; - pByte = (PBYTE)pTagTable->pTagTable; - pTag = OFFSETTOPOINTER(PTAG, pTagTable->pTagTable->stFixTag.uOffset ); - pAttribute = OFFSETTOPOINTER(PATTRIBUTE, - pTagTable->pTagTable->stAttribute.uOffset ); - } /* endif */ - - /********************************************************************/ - /* Tokenize input segment */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + if ( ( (pTok->sTokenid >= sNumTags) && + (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || + ( (pTok->sTokenid >= 0) && + (pTok->sTokenid < sNumTags) && + (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) { - pTagTable = (PLOADEDTABLE)pVoidTable; - pRest = NULL; - TATagTokenizeW( pszSegment, - pTagTable, - TRUE, - &pRest, - &usColPos, - pTokBuffer, - (USHORT)(usTokBufferSize / sizeof(TOKENENTRY)) ); - usRC = ( pRest == NULL ) ? NO_ERROR : EQFRS_AREA_TOO_SMALL; - } /* endif */ + /**************************************************************/ + /* Process data up to first quote or up to end of token */ + /* Complete protected start/stop entry in front of */ + /* quoted string */ + /**************************************************************/ + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); + current.usStop = TAFindQuote( pTok, (USHORT) (current.usStart-1), &chQuote); + current.usType = + (pTok->sTokenid >= sNumTags) ? PROTECTED_CHAR : TAGPROT_CHAR; + pushBytes(pStartStop, current); + current.reset(); - /********************************************************************/ - /* Count number of tokens in list to get start/stop list size */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - pTok = pTokBuffer; + /**************************************************************/ + /* Process quoted strings if any */ + /**************************************************************/ + while ( pTok->iLength && (usRC == NO_ERROR) ) + { // P016238: check whether quoted text contains protected parts! + pAttrTokBuffer.reserve(MAX_ATTR_TOKENS); + + if ( usRC == NO_ERROR) + { + usRC = TAProtectedPartsInQuotedText( pTok, + pTok->pDataStringW - pszSegment, + pStartStop, + pAttrTokBuffer, + pTagTable, + &usColPos, + chQuote, + lAttrSize / sizeof(TOKENENTRY), + ((iMode & CREATEPROTTABLE_MARKATTR) != 0) ); + } - while ( pTok->sTokenid != ENDOFLIST ) - { - /****************************************************************/ - /* Check for attributes and tags with translatable info */ - /****************************************************************/ - if ( ( (pTok->sTokenid >= sNumTags) && - (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || - ( (pTok->sTokenid >= 0) && - (pTok->sTokenid < sNumTags) && - (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) + /************************************************************/ + /* Add entry for protected data up to next quote or up to */ + /* end of token */ + /************************************************************/ + if ( pTok->iLength && (usRC == NO_ERROR)) { - /************************************************************/ - /* for any quote in the token text leave room for one more */ - /* start/stop entry */ - /************************************************************/ - pszTemp = pTok->pDataStringW; - for ( usI = 0; usI < pTok->iLength; usI++, pszTemp++ ) - { - if ( (*pszTemp == QUOTE) || (*pszTemp == DBLQUOTE) ) - { - ulTableAlloc++; // leave room for one more entry - // P016238: leave room for 4 more entries assuming 2 inline tags - // per each translatable part of an attribute - ulTableAlloc = ulTableAlloc + 4; - } /* endif */ - } /* endfor */ + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); + current.usType = PROTECTED_CHAR; + + pTok->pDataStringW++; // assure quote is in pCurrent + pTok->iLength--; + current.usStop = TAFindQuote( pTok, (USHORT) (current.usStart), &chQuote); + pushBytes(pStartStop, current); + current.reset(); } /* endif */ - ulTableAlloc++; // leave room for token - pTok++; } /* endwhile */ - ulTableAlloc++; // leave room for table end delimiter - } /* endif */ - - // assure that ulTableAlloc * sizeof(STARTSTOP) is at least MIN_ALLOC - if (ulTableAlloc < 4 ) - { - ulTableAlloc = 4; } - - /********************************************************************/ - /* Allocate start/stop list */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + else { - if ( !UtlAlloc( (PVOID *)&pStartStop, 0L, - (LONG) (ulTableAlloc * sizeof(STARTSTOP)), - NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - ulTableUsed = 0; - } /* endif */ - - TATagAddWSpace(pszSegment, pTokBuffer, sNumTags); + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); - /********************************************************************/ - /* Fill start/stop table */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - pCurrent = pStartStop; - pTok = pTokBuffer; - while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) + if ( pTok->sTokenid == TEXT_TOKEN ) { - /****************************************************************/ - /* Check for attributes or tags with translatable info */ - /****************************************************************/ - if ( ( (pTok->sTokenid >= sNumTags) && - (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || - ( (pTok->sTokenid >= 0) && - (pTok->sTokenid < sNumTags) && - (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) + current.usType = UNPROTECTED_CHAR; + } + else if (pTok->sTokenid >= sNumTags ) + { + // do not include translatable variables in protected text when working in CREATEPROTECTTABLE_NOTRANSLVAR mode + if ( (pTok->ClassId == CLS_TRANSLVAR) && (iMode & CREATEPROTTABLE_NOTRANSLVAR) ) { - /**************************************************************/ - /* Process data up to first quote or up to end of token */ - /* Complete protected start/stop entry in front of */ - /* quoted string */ - /**************************************************************/ - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - pCurrent->usStop = TAFindQuote( pTok, (USHORT) (pCurrent->usStart-1), &chQuote); - pCurrent->usType = - (pTok->sTokenid >= sNumTags) ? PROTECTED_CHAR : TAGPROT_CHAR; - - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - - /**************************************************************/ - /* Process quoted strings if any */ - /**************************************************************/ - while ( pTok->iLength && (usRC == NO_ERROR) ) - { // P016238: check whether quoted text contains protected parts! - if (!pAttrTokBuffer) - { - lAttrSize = MAX_ATTR_TOKENS * sizeof(TOKENENTRY); - if (!UtlAlloc((PVOID *) &pAttrTokBuffer, 0L, lAttrSize, NOMSG) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - if ( usRC == NO_ERROR) - { - usRC = TAProtectedPartsInQuotedText( pTok, - pTok->pDataStringW - pszSegment, - &pStartStop, - &pCurrent, - pAttrTokBuffer, - pTagTable, - &ulTableAlloc, - &ulTableUsed, - &usColPos, - chQuote, - lAttrSize / sizeof(TOKENENTRY), - ((iMode & CREATEPROTTABLE_MARKATTR) != 0) ); - } - - /************************************************************/ - /* Add entry for protected data up to next quote or up to */ - /* end of token */ - /************************************************************/ - if ( pTok->iLength && (usRC == NO_ERROR)) - { - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - pCurrent->usType = PROTECTED_CHAR; - - pTok->pDataStringW++; // assure quote is in pCurrent - pTok->iLength--; - pCurrent->usStop = TAFindQuote( pTok, (USHORT) (pCurrent->usStart), &chQuote); - - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - - } /* endif */ - } /* endwhile */ + current.usType = UNPROTECTED_CHAR; } else { - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - - if ( pTok->sTokenid == TEXT_TOKEN ) + current.usType = PROTECTED_CHAR; + } /* endif */ + } + else + { + current.usType = TAGPROT_CHAR; + if (pTag[pTok->sTokenid].BitFlags.fTRNote ) + { + if (fTRNoteFound ) { - pCurrent->usType = UNPROTECTED_CHAR; + /****************************************************/ + /* this approach requires that a TRNote is within */ + /* one segment; also nesting of notes is not allowed*/ + /****************************************************/ + current.usType = TRNOTE_CHAR; } - else if (pTok->sTokenid >= sNumTags ) + else { - // do not include translatable variables in protected text when working in CREATEPROTECTTABLE_NOTRANSLVAR mode - if ( (pTok->ClassId == CLS_TRANSLVAR) && (iMode & CREATEPROTTABLE_NOTRANSLVAR) ) + if (memicmp((PBYTE)&pTagTable->chTrnote1TextW[0], + (PBYTE)(pTok->pDataStringW+pTok->iLength), + pTagTable->ulTRNote1Len) == 0 ) { - pCurrent->usType = UNPROTECTED_CHAR; + current.usType = TRNOTE_CHAR; + fTRNoteFound = TRUE; } - else - { - pCurrent->usType = PROTECTED_CHAR; - } /* endif */ - } - else - { - pCurrent->usType = TAGPROT_CHAR; - if (pTag[pTok->sTokenid].BitFlags.fTRNote ) + else if (memicmp((PBYTE)&pTagTable->chTrnote2TextW[0], + (PBYTE)(pTok->pDataStringW+pTok->iLength), + pTagTable->ulTRNote2Len) == 0 ) { - if (fTRNoteFound ) - { - /****************************************************/ - /* this approach requires that a TRNote is within */ - /* one segment; also nesting of notes is not allowed*/ - /****************************************************/ - pCurrent->usType = TRNOTE_CHAR; - } - else - { - if (memicmp((PBYTE)&pTagTable->chTrnote1TextW[0], - (PBYTE)(pTok->pDataStringW+pTok->iLength), - pTagTable->ulTRNote1Len) == 0 ) - { - pCurrent->usType = TRNOTE_CHAR; - fTRNoteFound = TRUE; - } - else if (memicmp((PBYTE)&pTagTable->chTrnote2TextW[0], - (PBYTE)(pTok->pDataStringW+pTok->iLength), - pTagTable->ulTRNote2Len) == 0 ) - { - pCurrent->usType = TRNOTE_CHAR; - fTRNoteFound = TRUE; - } /* endif */ - } /* endif */ + current.usType = TRNOTE_CHAR; + fTRNoteFound = TRUE; } /* endif */ } /* endif */ - pCurrent->usStop = pCurrent->usStart + pTok->iLength - 1; - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - } /* endif */ + } /* endif */ + current.usStop = current.usStart + pTok->iLength - 1; + pushBytes(pStartStop, current); + current.reset(); - pTok++; - } /* endwhile */ + } /* endif */ - if (usRC == NO_ERROR) - { - TASetStartStopType(pCurrent, 0,0,0 ); //terminate start/stoptabl. + pTok++; + } /* endwhile */ - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); + if (usRC == NO_ERROR) + { + current.reset(); //terminate start/stoptabl. + pushBytes(pStartStop, (current)); - } - } /* endif */ + } + } /* endif */ + /********************************************************************/ + /* now loop true list again and join tags with their attributes */ + /* this is necessary to avoid splitting of tags and attributes */ + /* during the insert operation in the editor... */ + /********************************************************************/ + if ( usRC == NO_ERROR ) + { + PSTARTSTOP pCurrent = pAct = (PSTARTSTOP)pStartStop.data(); + pTok = pTokBuffer.data(); + pNextTok = pTok + 1; + if ( (pAct->usType == TAGPROT_CHAR) && TA_IS_TAG(pTok->sTokenid, sNumTags)) + { + pAct->usType = PROTECTED_CHAR; + fTag = TRUE; + } + else + { + fTag = FALSE; + } /* endif */ + if ( pCurrent->usType ) + { + pCurrent++; + if (pNextTok && pNextTok->pDataStringW && + (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) + { + pTok ++; + pNextTok++; + } - /********************************************************************/ - /* now loop true list again and join tags with their attributes */ - /* this is necessary to avoid splitting of tags and attributes */ - /* during the insert operation in the editor... */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + while ( pCurrent->usType ) { - pAct = pCurrent = pStartStop; - pTok = pTokBuffer; - pNextTok = pTok + 1; - if ( (pAct->usType == TAGPROT_CHAR) && TA_IS_TAG(pTok->sTokenid, sNumTags)) + switch ( pCurrent->usType ) { - pAct->usType = PROTECTED_CHAR; - fTag = TRUE; - } - else - { - fTag = FALSE; - } /* endif */ - if ( pCurrent->usType ) + case TAGPROT_CHAR: + if ( (pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) + && TA_IS_TAG(pTok->sTokenid, sNumTags)) + { + fTag = TRUE; + } + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = PROTECTED_CHAR; + break; + case PROTECTED_CHAR: + if ( fTag && (pCurrent->usType == pAct->usType)) + { + /************************************************************/ + /* combine tag and attribute... */ + /************************************************************/ + pAct->usStop = pCurrent->usStop; + } + else + { + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = pCurrent->usType; + } /* endif */ + break; + default : + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = pCurrent->usType; + if ( ((pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) + && (pTok->sTokenid == TEXT_TOKEN)) || + (pCurrent->usType == TRNOTE_CHAR)) + { + fTag = FALSE; + } + // P016804: if text-token is from quoted text, do not reset!! + //fTag = FALSE; + break; + } /* endswitch */ + /******************************************************************/ + /* point to next one... */ + /******************************************************************/ + pCurrent++; + if (pNextTok && pNextTok->pDataStringW && + (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) { - pCurrent++; - if (pNextTok && pNextTok->pDataStringW && - (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) - { - pTok ++; - pNextTok++; - } + pTok ++; + pNextTok++; + } + } /* endwhile */ + /********************************************************************/ + /* add stopping mode.... */ + /********************************************************************/ + pAct++; + pAct->usStart = 0; + pAct->usStop = 0; + pAct->usType = 0; + } /* endif */ - while ( pCurrent->usType ) + /********************************************************************/ + /* now loop thru list and combine the tokens of a translators note */ + /* into one start-stop token */ + /********************************************************************/ + if (fTRNoteFound ) + { + pAct = pCurrent =(PSTARTSTOP) pStartStop.data(); + if ( pAct->usType == TRNOTE_CHAR ) + { + fTRTag = TRUE; + } + else + { + fTRTag = FALSE; + } /* endif */ + if ( pCurrent->usType ) + { + pCurrent++; + + while ( pCurrent->usType ) + { + switch ( pCurrent->usType ) { - switch ( pCurrent->usType ) - { - case TAGPROT_CHAR: - if ( (pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) - && TA_IS_TAG(pTok->sTokenid, sNumTags)) - { - fTag = TRUE; - } + case TRNOTE_CHAR : // end of TRNOTE of begin of next TRNOTE + if (fTRTag ) + { + pAct->usStop = pCurrent->usStop; + } + else + { pAct++; pAct->usStart = pCurrent->usStart; pAct->usStop = pCurrent->usStop; - pAct->usType = PROTECTED_CHAR; - break; - case PROTECTED_CHAR: - if ( fTag && (pCurrent->usType == pAct->usType)) - { - /************************************************************/ - /* combine tag and attribute... */ - /************************************************************/ - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - break; - default : + pAct->usType = pCurrent->usType; + } /* endif */ + fTRTag = !fTRTag; + break; + default: + if ( fTRTag ) + { + /************************************************************/ + /* combine tag and attribute... */ + /************************************************************/ + pAct->usStop = pCurrent->usStop; + } + else + { pAct++; pAct->usStart = pCurrent->usStart; pAct->usStop = pCurrent->usStop; pAct->usType = pCurrent->usType; - if ( ((pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) - && (pTok->sTokenid == TEXT_TOKEN)) || - (pCurrent->usType == TRNOTE_CHAR)) - { - fTag = FALSE; - } - // P016804: if text-token is from quoted text, do not reset!! - //fTag = FALSE; - break; - } /* endswitch */ - /******************************************************************/ - /* point to next one... */ - /******************************************************************/ - pCurrent++; - if (pNextTok && pNextTok->pDataStringW && - (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) - { - pTok ++; - pNextTok++; - } - } /* endwhile */ - /********************************************************************/ - /* add stopping mode.... */ - /********************************************************************/ - pAct++; - pAct->usStart = 0; - pAct->usStop = 0; - pAct->usType = 0; - } /* endif */ - + } /* endif */ + break; + } /* endswitch */ + /******************************************************************/ + /* point to next one... */ + /******************************************************************/ + pCurrent++; + } /* endwhile */ /********************************************************************/ - /* now loop thru list and combine the tokens of a translators note */ - /* into one start-stop token */ + /* add stopping mode.... */ /********************************************************************/ - if (fTRNoteFound ) - { - pAct = pCurrent = pStartStop; - if ( pAct->usType == TRNOTE_CHAR ) - { - fTRTag = TRUE; - } - else - { - fTRTag = FALSE; - } /* endif */ - if ( pCurrent->usType ) - { - pCurrent++; - - while ( pCurrent->usType ) - { - switch ( pCurrent->usType ) - { - case TRNOTE_CHAR : // end of TRNOTE of begin of next TRNOTE - if (fTRTag ) - { - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - fTRTag = !fTRTag; - break; - default: - if ( fTRTag ) - { - /************************************************************/ - /* combine tag and attribute... */ - /************************************************************/ - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - break; - } /* endswitch */ - /******************************************************************/ - /* point to next one... */ - /******************************************************************/ - pCurrent++; - } /* endwhile */ - /********************************************************************/ - /* add stopping mode.... */ - /********************************************************************/ - pAct++; - pAct->usStart = 0; - pAct->usStop = 0; - pAct->usType = 0; - } /* endif */ - } /* endif */ + pAct++; + pAct->usStart = 0; + pAct->usStop = 0; + pAct->usType = 0; } /* endif */ } /* endif */ } /* endif */ - if ( pAttrTokBuffer ) - { - UtlAlloc( (PVOID *)&pAttrTokBuffer, 0L, 0L, NOMSG ); - } /* endif */ - /********************************************************************/ /* Return start/stop list to caller or cleanup */ /********************************************************************/ if ( usRC == NO_ERROR ) { - *ppStartStop = pStartStop; if (IsDBCS_CP(ulCP)) { - usRC = TASoSiProtectTable( pszSegment, ppStartStop); + usRC = TASoSiProtectTable( pszSegment, pStartStop); } } else { - if ( pStartStop ) - { - UtlAlloc( (PVOID *)&pStartStop, 0L, 0L, NOMSG ); - } /* endif */ - *ppStartStop = NULL; + pStartStop.clear(); } /* endif */ return( usRC ); } -VOID TASetStartStopType -( - PSTARTSTOP pCurrent, // ptr to caller's current start/stop entry - USHORT usStart, - USHORT usStop, - USHORT usType -) -{ - pCurrent->usStart = usStart; - pCurrent->usStop = usStop; - pCurrent->usType = usType; - return; -} + USHORT TAFindQuote ( @@ -2777,368 +2500,100 @@ USHORT TAFindQuote return(usStop); } -USHORT TAGotoNextStartStopEntry -( - PSTARTSTOP * ppStartStop, - PSTARTSTOP * ppCurrent, - PULONG pulTableUsed, - PULONG pulTableAlloc -) -{ - PSTARTSTOP pCurrent = *ppCurrent; - ULONG ulTableUsed = * pulTableUsed; - ULONG ulTableAlloc = *pulTableAlloc; - USHORT usRC = NO_ERROR; // function return code - - pCurrent ++; - ulTableUsed ++; - // for tests only is pCurrent after add as above? - { PSTARTSTOP pStartStop = *ppStartStop; - pCurrent = pStartStop + ulTableUsed; - } - if ( ulTableAlloc <= ulTableUsed + 1) - { - LONG lOldLen; // old length - LONG lNewLen; // new length - PSTARTSTOP pNewStartStop = *ppStartStop; - - lOldLen = ulTableAlloc * sizeof(STARTSTOP); - ulTableAlloc = ulTableAlloc + 20; - lNewLen = ulTableAlloc * sizeof(STARTSTOP); - - // realloc nec - if ( !UtlAlloc( (PVOID *)&pNewStartStop, lOldLen, - lNewLen, NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - else - { - // if realloc ok, set pointers - pCurrent = pNewStartStop + ulTableUsed; - *ppStartStop = pNewStartStop; - *pulTableAlloc = ulTableAlloc; - } - } - if ( usRC == NO_ERROR) - { - // set return values - *ppCurrent = pCurrent; - *pulTableUsed = ulTableUsed; - } - return ( usRC ); -} - USHORT TAProtectedPartsInQuotedText ( PTOKENENTRY pTok, int iAttrStartOffset, - PSTARTSTOP * ppStartStop, - PSTARTSTOP * ppCurrent, - PTOKENENTRY pAttrTokBuffer, + BytesRef pStartStop, + std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, - PULONG pulTableAlloc, - PULONG pulTableUsed, PUSHORT pusColPos, CHAR_W chQuote, LONG lAttrSize, BOOL fSpecialMode ) { - USHORT usRC = NO_ERROR; - CHAR_W chTempBuf[MAX_SEGMENT_SIZE]; - PTOKENENTRY pAttrTok; - PSZ_W pAttrStart; - USHORT usAttrLength = 0; - PSZ_W pRest; // ptr to not tokenized data - PSZ_W pTemp; - PSTARTSTOP pCurrent = *ppCurrent; - PSTARTSTOP pStartStop = *ppStartStop; - - - memset (&chTempBuf[0], 0, sizeof(CHAR_W) * (MAX_SEGMENT_SIZE)); - - UTF16strcat(chTempBuf, pTok->pDataStringW); - pAttrStart = &chTempBuf[0]; - pTemp = &chTempBuf[0]; - while ( pTemp && (*pTemp != chQuote) ) - { - pTemp++; - usAttrLength ++; - } /* endwhile */ - - if (*pTemp == chQuote ) - { - *pTemp = EOS; //investigate only text inside quotes! - } + USHORT usRC = NO_ERROR; + CHAR_W chTempBuf[MAX_SEGMENT_SIZE]; + PTOKENENTRY pAttrTok; + PSZ_W pAttrStart; + USHORT usAttrLength = 0; + PSZ_W pRest; // ptr to not tokenized data + PSZ_W pTemp; + STARTSTOP current; - if ( usRC == NO_ERROR ) - { - TATagTokenizeW( chTempBuf, - pTagTable, - TRUE, - &pRest, - pusColPos, - pAttrTokBuffer, - (USHORT)lAttrSize ); - if(pRest == nullptr){ - LOG_AND_SET_RC(usRC, T5INFO, NO_ERROR); - }else{ - LOG_AND_SET_RC(usRC, T5INFO, EQFRS_AREA_TOO_SMALL); - } - } /* endif */ + memset (&chTempBuf[0], 0, sizeof(CHAR_W) * (MAX_SEGMENT_SIZE)); - if ( usRC == NO_ERROR ) - { - pAttrTok = pAttrTokBuffer; - while ( (pAttrTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) - { - pCurrent->usStart = (USHORT)(pAttrTok->pDataStringW - pAttrStart - + iAttrStartOffset); - pCurrent->usType = (pAttrTok->sTokenid == TEXT_TOKEN) ? - UNPROTECTED_CHAR : PROTECTED_CHAR; - if ( fSpecialMode ) - { - pCurrent->usType |= 0x8000; - } /* endif */ - pCurrent->usStop = pCurrent->usStart + pAttrTok->iLength - 1; + UTF16strcat(chTempBuf, pTok->pDataStringW); + pAttrStart = &chTempBuf[0]; + pTemp = &chTempBuf[0]; + while ( pTemp && (*pTemp != chQuote) ) + { + pTemp++; + usAttrLength ++; + } /* endwhile */ - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - pulTableUsed, pulTableAlloc); + if (*pTemp == chQuote ) + { + *pTemp = EOS; //investigate only text inside quotes! + } - pAttrTok++; - } /* endwhile */ - } /* endif */ + if ( usRC == NO_ERROR ) + { + TATagTokenizeW( chTempBuf, + pTagTable, + &pRest, + pusColPos, + pAttrTokBuffer ); + if(pRest == nullptr){ + LOG_AND_SET_RC(usRC, T5INFO, NO_ERROR); + }else{ + LOG_AND_SET_RC(usRC, T5INFO, EQFRS_AREA_TOO_SMALL); + } + } /* endif */ - /**********************************************************/ - /* Add data up to first quote */ - /**********************************************************/ - while ( pTok->iLength && (pTok->pDataStringW[0] != chQuote) ) - { - pTok->pDataStringW++; - pTok->iLength--; - usAttrLength --; - } /* endwhile */ - // CHECK: does this loop run as far as pTemp in chTempBuf? - if (usAttrLength != 0) + if ( usRC == NO_ERROR ) + { + pAttrTok = pAttrTokBuffer.data(); + while ( (pAttrTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { - // this should not happen!! - usAttrLength = 0; - } - - *ppCurrent = pCurrent; - *ppStartStop = pStartStop; - return (usRC); -} - - - -USHORT TACreateProtectTable -( - PSZ pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - ULONG ulCP -) -{ - USHORT usRC = 0; - PSZ_W pszInputW = &chInputW[0]; - - ASCII2Unicode( pszSegment, pszInputW, ulCP ); + current.usStart = (USHORT)(pAttrTok->pDataStringW - pAttrStart + + iAttrStartOffset); + current.usType = (pAttrTok->sTokenid == TEXT_TOKEN) ? + UNPROTECTED_CHAR : PROTECTED_CHAR; + if ( fSpecialMode ) + { + current.usType |= 0x8000; + } /* endif */ + current.usStop = current.usStart + pAttrTok->iLength - 1; + pushBytes(pStartStop, current); + current.reset(); - usRC = TACreateProtectTableW( pszInputW, pVoidTable, usColPos, pTokBuffer, usTokBufferSize, - ppStartStop, pvUserExit, NULL, ulCP ); + pAttrTok++; + } /* endwhile */ + } /* endif */ - // attention: - // for DBCS code pages the returned offsets and lengths do not match the actual - // position in the source data and have to be re-adjusted + /**********************************************************/ + /* Add data up to first quote */ + /**********************************************************/ + while ( pTok->iLength && (pTok->pDataStringW[0] != chQuote) ) + { + pTok->pDataStringW++; + pTok->iLength--; + usAttrLength --; + } /* endwhile */ + // CHECK: does this loop run as far as pTemp in chTempBuf? + if (usAttrLength != 0) + { + // this should not happen!! + usAttrLength = 0; + } - return usRC; + return (usRC); } -USHORT TAPrepProtectTable -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // address of ptr to segment check func - PFN *ppfnWYSIWYGExit, // address of ptr to show transl. func - PFN *ppfnTocGotoExit // address of ptr to TOC func -) -{ - return( TALoadEditUserExit( pVoidTable, "", phModule, ppfnUserExit, ppfnCheckSegExit, - ppfnWYSIWYGExit, ppfnTocGotoExit, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL ) ); -} /* end of function TAPrepProtectTable */ - -USHORT TALoadEditUserExit -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - PSZ pszTableName, // name of tag table (w/o path and ext.) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // address of ptr to segment check func - PFN *ppfnWYSIWYGExit, // address of ptr to show transl. func - PFN *ppfnTocGotoExit, // address of ptr to TOC func - PFN *ppfnGetSegContext, // address of ptr to EQFGETSEGCONTEXT function - PFN *ppfnUpdateContext, // address of ptr to EQFUPDATECONTEXT function - PFN *ppfnFormatContext, // address of ptr to EQFFORMATCONTEXT function - PFN *ppfnCompareContext, // address of ptr to EQFCOMPARECONTEXT function - PFN *ppfnUserExitW, // unicode user exit to create start-stop-tbl - PFN *ppfnCheckSegExitW, // unicode user exit to check segment - PFN *ppfnCheckSegExExitW,// unicode user exit to check segment Ex - PFN *ppfnCheckSegType // user exit to check segment type -) -{ - PTAGTABLE pTagTable; // pointer to tag table - USHORT usRC = NO_ERROR; // function return code - - /********************************************************************/ - /* initialize user's data fields */ - /********************************************************************/ - *phModule = NULLHANDLE; - if ( ppfnUserExit ) *ppfnUserExit = NULL; - if ( ppfnCheckSegExit ) *ppfnCheckSegExit = NULL; - if ( ppfnWYSIWYGExit ) *ppfnWYSIWYGExit = NULL; - if ( ppfnTocGotoExit ) *ppfnTocGotoExit = NULL; - if ( ppfnGetSegContext ) *ppfnGetSegContext = NULL; - if ( ppfnUpdateContext ) *ppfnUpdateContext = NULL; - if ( ppfnFormatContext ) *ppfnFormatContext = NULL; - if ( ppfnCompareContext ) *ppfnCompareContext = NULL; - if ( ppfnUserExitW ) *ppfnUserExitW = NULL; - if ( ppfnCheckSegExitW ) *ppfnCheckSegExitW = NULL; - if ( ppfnCheckSegExExitW ) *ppfnCheckSegExExitW = NULL; - if ( ppfnCheckSegType ) *ppfnCheckSegType = NULL; - - - /********************************************************************/ - /* Address tag table */ - /********************************************************************/ - pTagTable = ((PLOADEDTABLE)pVoidTable)->pTagTable; - - /********************************************************************/ - /* Load user exit if tag table has one */ - /********************************************************************/ - if ( pTagTable->szSegmentExit[0] != EOS ) - { - CHAR szExit[MAX_LONGFILESPEC]; // buffer for library name - CHAR *ptr ; - -// strcpy( szExit, pTagTable->szSegmentExit ); -// strcat( szExit, EXT_OF_DLL ); -// if ( MUGetUserExitFileName( pTagTable->szSegmentExit, NULL, szExit, sizeof(szExit) ) ) - if ( MUGetUserExitFileName( pszTableName, NULL, szExit, sizeof(szExit) ) ) - { - ptr = strrchr( szExit, '\\' ) ; - if ( ptr ) - strcpy( ++ptr, pTagTable->szSegmentExit ) ; - - usRC = DosLoadModule( NULL, 0, szExit, phModule ); - - if ( usRC == NO_ERROR ) - { - if ( ppfnUserExit ) - { - - usRC = DosGetProcAddr( *phModule, EQFPROTTABLE_EXIT, (PFN*)ppfnUserExit); - if ( usRC != NO_ERROR ) - { - *ppfnUserExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load SegmentCheckExit function */ - /****************************************************************/ - if ( ppfnCheckSegExit ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEG_EXIT, - (PFN*)ppfnCheckSegExit); - if ( usRC != NO_ERROR ) - { - *ppfnCheckSegExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load ShowTransl Exit function */ - /****************************************************************/ - if ( ppfnWYSIWYGExit ) - { - usRC = DosGetProcAddr( *phModule, EQFSHOWTRANS_EXIT, - (PFN*)ppfnWYSIWYGExit); - if ( usRC != NO_ERROR ) - { - *ppfnWYSIWYGExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load CheckSegType Exit function */ - /****************************************************************/ - if ( ppfnCheckSegType ) - { - *ppfnCheckSegType = NULL; - DosGetProcAddr( *phModule, EQFCHECKSEGTYPE_EXIT, (PFN*)ppfnCheckSegType); - } - /****************************************************************/ - /* try to load TOC user Exit function */ - /****************************************************************/ - if ( ppfnTocGotoExit ) - { - usRC = DosGetProcAddr( *phModule, EQFTOCGOTO_EXIT, - (PFN*)ppfnTocGotoExit); - if ( usRC != NO_ERROR ) - { - *ppfnTocGotoExit = NULL; - } /* endif */ - } - // try to load context related user exit functions - if ( ppfnGetSegContext ) - { - usRC = DosGetProcAddr( *phModule, EQFGETSEGCONTEXT_EXIT, (PFN*)ppfnGetSegContext ); - if ( usRC != NO_ERROR ) *ppfnGetSegContext = NULL; - } - if ( ppfnUpdateContext ) - { - usRC = DosGetProcAddr( *phModule, EQFUPDATECONTEXT_EXIT, (PFN*)ppfnUpdateContext ); - if ( usRC != NO_ERROR ) *ppfnUpdateContext = NULL; - } - if ( ppfnCompareContext ) - { - usRC = DosGetProcAddr( *phModule, EQFCOMPARECONTEXT_EXIT, (PFN*)ppfnCompareContext ); - if ( usRC != NO_ERROR ) *ppfnCompareContext = NULL; - } - if ( ppfnFormatContext ) - { - usRC = DosGetProcAddr( *phModule, EQFFORMATCONTEXT_EXIT, (PFN*)ppfnFormatContext ); - if ( usRC != NO_ERROR ) *ppfnFormatContext = NULL; - } - if ( ppfnUserExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFPROTTABLEW_EXIT, (PFN*)ppfnUserExitW); - if ( usRC != NO_ERROR ) *ppfnUserExitW = NULL; - } - if ( ppfnCheckSegExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEGW_EXIT, (PFN*)ppfnCheckSegExitW); - if ( usRC != NO_ERROR ) *ppfnCheckSegExitW = NULL; - } - if ( ppfnCheckSegExExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEGEXW_EXIT, (PFN*)ppfnCheckSegExExitW); - if ( usRC != NO_ERROR ) *ppfnCheckSegExExitW = NULL; - } - - } /* endif */ - } - } /* endif */ - - return( NO_ERROR ); -} /* end of function TAPrepProtectTable */ USHORT TAEndProtectTable @@ -3185,27 +2640,27 @@ bool iswspace(wchar_t ch){ //*/ // P016804: - static void - TATagAddWSpace - ( - PSZ_W pszSegment, - PTOKENENTRY pTokBuffer, - SHORT sNumTags - ) - { - PTOKENENTRY pTok; - USHORT usRC = NO_ERROR; - PTOKENENTRY pNextTok; - USHORT usStart = 0; - USHORT usNextStart = 0; - PSZ_W pTmp = NULL; - USHORT usI = 0; - USHORT usNewLen = 0; - - pTok = pTokBuffer; - pNextTok = pTok + 1; - while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR) && (pNextTok->sTokenid != ENDOFLIST)) - { +static void +TATagAddWSpace +( + PSZ_W pszSegment, + PTOKENENTRY pTokBuffer, + SHORT sNumTags +) +{ + PTOKENENTRY pTok; + USHORT usRC = NO_ERROR; + PTOKENENTRY pNextTok; + USHORT usStart = 0; + USHORT usNextStart = 0; + PSZ_W pTmp = NULL; + USHORT usI = 0; + USHORT usNewLen = 0; + + pTok = pTokBuffer; + pNextTok = pTok + 1; + while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR) && (pNextTok->sTokenid != ENDOFLIST)) + { if (TA_IS_TAG(pTok->sTokenid, sNumTags) || TA_IS_ATTRIBUTE(pTok->sTokenid, sNumTags)) { if (TA_IS_ATTRIBUTE(pNextTok->sTokenid, sNumTags)) @@ -3229,7 +2684,7 @@ bool iswspace(wchar_t ch){ } /* endif */ pTok ++; pNextTok++; - } /* endwhile */ - return; - } + } /* endwhile */ + return; +} diff --git a/source/opentm2/core/tagtable/MarkupPluginMapper.cpp b/source/opentm2/core/tagtable/MarkupPluginMapper.cpp index 418c7258..7d796e4b 100755 --- a/source/opentm2/core/tagtable/MarkupPluginMapper.cpp +++ b/source/opentm2/core/tagtable/MarkupPluginMapper.cpp @@ -339,50 +339,6 @@ BOOL MUGetMarkupTablePlugin( char *pszMarkup, char *pszBuffer, int iBufSize ) } -// update the files of a markup table - -BOOL MUUpdateMarkupTableFiles( - char *pszMarkupTableName, - char *pszPluginName, - char *pszShortDescription, - char *pszLongDescription, - char *pszMarkupTableVersion, - char *pszTableFileName, - char *pszUserExitFilename, - char *pszFileList -) -{ - static char szBuffer[MAX_LONGFILESPEC]; // buffer for markup names - BOOL bReturn = false ; - - if ( ( pszPluginName ) && - ( pszMarkupTableName ) ) { - - // loop through all markup table plugins - for( int i = 0 ; i < (int)pluginList->size() ; i++ ) { - OtmMarkupPlugin* curPlugin = (*pluginList)[i]; - if ( ( ! strcmp( curPlugin->getName(), pszPluginName ) ) || - ( ! strcmp( curPlugin->getShortName(), pszPluginName ) ) ) { - int iReturn = curPlugin->updateFiles( pszMarkupTableName, - pszShortDescription, pszLongDescription, - pszMarkupTableVersion, pszTableFileName, - pszUserExitFilename, pszFileList ) ; - if ( iReturn != UPDATE_MARKUP_ERROR ) { - bReturn = true; - // OtmMarkup *markup = curPlugin->getMarkup( pszMarkupTableName ); - // if ( markup ) { - // markup->updateInfo( pszMarkupTableName, - // pszShortDescription, pszLongDescription, - // NULL, pszUserExitFilename ) ; - // } - } - } - } - } - - return( bReturn ); -} - // delete a markup table diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 65258d27..1d8099b0 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -25,9 +25,9 @@ int TMManager::GetMemImportInProcessCount(MutexTimeout& tmListTimeout){ return TM_LIST_MUTEX_TIMEOUT_FAILED; } - for(auto& tm: tms) + for(const auto& tm: tms) { - ImportStatusDetails* pImportDetails = tm.second->importDetails; + ImportStatusDetails* pImportDetails = tm.second->importDetails.get(); if( (tm.second->eImportStatus == IMPORT_RUNNING_STATUS) || (tm.second->eImportStatus == REORGANIZE_RUNNING_STATUS) ) @@ -43,19 +43,7 @@ int TMManager::GetMemImportInProcessCount(MutexTimeout& tmListTimeout){ size_t TMManager::CalculateOccupiedRAM(MutexTimeout& tmListTimeout){ char memFolder[260]; size_t UsedMemory = 0; - #ifdef CALCULATE_ONLY_MEM_FILES - Properties::GetInstance()->get_value(KEY_MEM_DIR, memFolder, 260); - std::string path; - for(int i = 0; i < EqfMemoryPlugin::GetInstance()->m_MemInfoVector.size() ;i++){ - if(EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i].szName != 0){ - path = memFolder; - path += EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i].szName; - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".TMI")); - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".TMD")); - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".MEM")); - } - } - #else + { TimedMutexGuard l{mutex_access_tms, tmListTimeout, "tmListMutex", __func__, __LINE__};// lock tms list if(tmListTimeout.failed()){ @@ -73,7 +61,7 @@ size_t TMManager::CalculateOccupiedRAM(MutexTimeout& tmListTimeout){ } //UsedMemory += FilesystemHelper::GetTotalFilebuffersSize(); UsedMemory += MEMORY_RESERVED_FOR_SERVICE; - #endif + T5LOG( T5INFO) << ":: calculated occupied ram = " << UsedMemory/1000000 << " MB"; return UsedMemory; @@ -198,19 +186,19 @@ size_t TMManager::CleanupMemoryList(size_t memoryRequested, MutexTimeout& tmList void EqfMemory::importDone(int iRC, char *pszError ) { eStatus = OPEN_STATUS; - TmBtree.fb.Flush(); - InBtree.fb.Flush(); + state->TmBtree.fb.Flush(); + state->InBtree.fb.Flush(); if ( iRC == 0 ) { eImportStatus = OPEN_STATUS; - T5LOG( T5INFO) <<"OtmMemoryServiceWorker::importDone:: success, memName = " << szName; + T5LOG( T5INFO) <<"::importDone:: success, memName = " << szName; } else { eImportStatus = IMPORT_FAILED_STATUS; strError = pszError; - T5LOG(T5ERROR) << "OtmMemoryServiceWorker::importDone:: memName = " << szName <<", import failed: " << strError << " import details = " << importDetails->toString() ; + T5LOG(T5ERROR) << "::importDone:: memName = " << szName <<", import failed: " << strError << " import details = " << importDetails->toString() ; } resetActiveRequestCommand(); } @@ -224,17 +212,19 @@ void EqfMemory::reorganizeDone(int iRC, char *pszError ) { eStatus = OPEN_STATUS; - resetOpenedOnlyForReorganize(); + //resetOpenedOnlyForReorganize(); if ( iRC == 0 ) { eImportStatus = AVAILABLE_STATUS; + resetOpenedOnlyForReorganize(); T5LOG( T5INFO) <<":: success, memName = " << szName; } else { eImportStatus = REORGANIZE_FAILED_STATUS; strError = pszError; + state->fCorrupted = true; T5LOG(T5ERROR) << ":: memName = " << szName <<", reorganize failed: " << pszError << " import details = " << importDetails->toString() ; } @@ -246,13 +236,7 @@ void EqfMemory::reorganizeDone(int iRC, char *pszError ) */ int TMManager::closeAll() { - for ( int i = 0; i < ( int )EqfMemoryPlugin::GetInstance()->m_MemInfoVector.size(); i++ ) - { - if ( EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i]->szName[0] != 0 ) - { - //removeFromMemoryList( i ); - } - } /* endfor */ + return( 0 ); } @@ -314,7 +298,7 @@ TMManager* TMManager::GetInstance(){ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMemName, const std::string& strSrcLang, const std::string& strMemDescription, int& _rc_, bool keepInRamOnly){ std::shared_ptr NewMem; - ULONG ulKey = 0; + LONG lKey = 0; if ( _rc_ == NO_ERROR ) { T5LOG( T5DEBUG) << ":: create the memory" ; @@ -326,74 +310,79 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if(_rc_ == NO_ERROR) { - NewMem->stTmSign.bGlobVersion = T5GLOBVERSION; - NewMem->stTmSign.bMajorVersion = T5MAJVERSION; - NewMem->stTmSign.bMinorVersion = T5MINVERSION; + NewMem->state->stTmSign.bGlobVersion = T5GLOBVERSION; + NewMem->state->stTmSign.bMajorVersion = T5MAJVERSION; + NewMem->state->stTmSign.bMinorVersion = T5MINVERSION; //build name and extension of tm data file //fill signature record structure - strcpy( NewMem->stTmSign.szName, NewMem->TmBtree.fb.fileName.c_str() ); - UtlTime( &(NewMem->stTmSign.lTime) ); - UtlTime( &(NewMem->stTmSign.creationTime) ); - strcpy( NewMem->stTmSign.szSourceLanguage, + strcpy( NewMem->state->stTmSign.szName, NewMem->state->TmBtree.fb.file.fileName.c_str() ); + UtlTime( &(NewMem->state->stTmSign.lTime) ); + UtlTime( &(NewMem->state->stTmSign.creationTime) ); + strcpy( NewMem->state->stTmSign.szSourceLanguage, strSrcLang.c_str() ); - strcpy( NewMem->stTmSign.szDescription, + strcpy( NewMem->state->stTmSign.szDescription, strMemDescription.c_str() ); //call create function for data file - NewMem->usAccessMode = 1;//ASD_LOCKED; // new TMs are always in exclusive access... + NewMem->state->usAccessMode = 1;//ASD_LOCKED; // new TMs are always in exclusive access... - _rc_ = NewMem->TmBtree.QDAMDictCreateLocal( &(NewMem->stTmSign),FIRST_KEY, keepInRamOnly); + _rc_ = NewMem->state->TmBtree.QDAMDictCreateLocal( &(NewMem->state->stTmSign),FIRST_KEY, keepInRamOnly); } + + std::vector buffer; + buffer.reserve(TMX_TABLE_SIZE); + if ( _rc_ == NO_ERROR ) { //insert initialized record to tm data file - ulKey = AUTHOR_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->Authors, TMX_TABLE_SIZE ); + lKey = AUTHOR_KEY; + tmxTableToBuffer(&NewMem->state->Authors,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } if ( _rc_ == NO_ERROR ) { - ulKey = FILE_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->FileNames, TMX_TABLE_SIZE ); + lKey = FILE_KEY; + //tmxTableToBuffer(&NewMem->FileNames,buffer); + // _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { - ulKey = TAGTABLE_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->TagTables, TMX_TABLE_SIZE ); + lKey = TAGTABLE_KEY; + tmxTableToBuffer(&NewMem->state->TagTables,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { - ulKey = LANG_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->Languages, TMX_TABLE_SIZE ); + lKey = LANG_KEY; + tmxTableToBuffer(&NewMem->state->Languages,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { int size = MAX_COMPACT_SIZE-1 ; //initialize and insert compact area record - memset( NewMem->bCompact, 0, size ); + //memset( NewMem->bCompact, 0, size ); + NewMem->state->bCompact.resize(size); - ulKey = COMPACT_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - NewMem->bCompact, size); + lKey = COMPACT_KEY; + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey, + NewMem->state->bCompact); } /* endif */ // add long document table record if ( _rc_ == NO_ERROR ) { - ulKey = LONGNAME_KEY; + lKey = LONGNAME_KEY; + NewMem->state->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size // write long document name buffer area to the database - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)NewMem->pLongNames->pszBuffer, - NewMem->pLongNames->ulBufUsed ); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey, + NewMem->state->LongNames.pszBuffer ); } /* endif */ @@ -406,19 +395,21 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { //fill signature record structure - strcpy( NewMem->stTmSign.szName, NewMem->InBtree.fb.fileName.c_str() ); - _rc_ = NewMem->InBtree.QDAMDictCreateLocal(&NewMem->stTmSign, START_KEY, keepInRamOnly); + strcpy( NewMem->state->stTmSign.szName, NewMem->state->InBtree.fb.file.fileName.c_str() ); + _rc_ = NewMem->state->InBtree.QDAMDictCreateLocal(&NewMem->state->stTmSign, START_KEY, keepInRamOnly); } /* endif */ - if(_rc_ == NO_ERROR){ - NewMem->TmBtree.fb.Flush(); - NewMem->InBtree.fb.Flush(); + if(_rc_ == NO_ERROR){ + if(!keepInRamOnly){ + NewMem->state->TmBtree.fb.Flush(); + NewMem->state->InBtree.fb.Flush(); + } NewMem->eStatus = OPEN_STATUS; }else { NewMem->eStatus = FAILED_TO_OPEN_STATUS; //something went wrong during create or insert so delete data file - UtlDelete( (PSZ)NewMem->InBtree.fb.fileName.c_str(), 0L, FALSE ); - UtlDelete((PSZ) NewMem->TmBtree.fb.fileName.c_str(), 0L, FALSE); + UtlDelete( (PSZ)NewMem->state->InBtree.fb.file.fileName.c_str(), 0L, FALSE ); + UtlDelete((PSZ) NewMem->state->TmBtree.fb.file.fileName.c_str(), 0L, FALSE); } /* endif */ return NewMem; } @@ -428,37 +419,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem \param pMemory pointer to memory object beign closed \returns 0 when successful or error code */ -int TMManager::closeMemory -( - EqfMemory *pMemory -) -{ - int iRC = 0; - if ( pMemory == NULL ){ - T5LOG(T5ERROR) <<"TMManager::closeMemory, pMemory is NULL"; - return( -1 ); - } - - // build memory object name - char* pszObjName = NULL; - UtlAlloc( (PVOID *)&pszObjName, 0L, MAX_LONGFILESPEC + MAX_LONGFILESPEC + 2, NOMSG ); - strcpy( pszObjName, EqfMemoryPlugin::GetInstance()->getName() ); - strcat( pszObjName, ":" ); - pMemory->getName( pszObjName + strlen(pszObjName), MAX_LONGFILESPEC ); - - // close the memory - iRC = EqfMemoryPlugin::GetInstance()->closeMemory( pMemory ); - - // send a properties changed msg to memory handler - EqfSend2Handler( MEMORYHANDLER, WM_EQFN_PROPERTIESCHANGED, MP1FROMSHORT( PROP_CLASS_MEMORY ), MP2FROMP( pszObjName )); - - if ( pszObjName != NULL ) { - UtlAlloc( (PVOID *)&pszObjName, 0L, 0L, NOMSG ); - } - - return( iRC ); -} @@ -600,11 +561,6 @@ int TMManager::RenameTM( std::string& TMManager::getLastError( EqfMemory *pMemory, int& iLastError, std::string& strError) { - if ( pMemory != NULL ) - this->iLastError = pMemory->getLastError( this->strLastError ); - - iLastError = this->iLastError; - strError = this->strLastError; return strError; } @@ -1014,7 +970,7 @@ int TMManager::ReplaceMemory } T5LOG(T5INFO)<<"Mem is loaded, reloading filebuffers from disk..."; - iRC = TMManager::GetInstance()->tms[strReplace]->ReloadFromDisk(); + iRC = TMManager::GetInstance()->tms[strReplace]->Reload(true); } } // broadcast deleted memory name for replaceWith memory @@ -1091,7 +1047,7 @@ int AddMemToList( void *pvData, char *pszName,std::shared_ptr pInfo /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ @@ -1135,7 +1091,7 @@ BOOL searchExtendedInProposal /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ @@ -1429,7 +1385,7 @@ int TMManager::OpenTM(const std::string& strMemName, MutexTimeout& tmListTimeout //if ( pMem == nullptr ) if(memLeftAfterOpening <= 0){ - //buildErrorReturn( _rc_, "OtmMemoryServiceWorker::import::Error: too many open translation memory databases" ); + //buildErrorReturn( _rc_, "::import::Error: too many open translation memory databases" ); return( INTERNAL_SERVER_ERROR ); } /* endif */ std::shared_ptr pMem( EqfMemoryPlugin::GetInstance()->initTM(strMemName, requiredMemory, 0, fReorganizeOnly)); diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 08f286fb..7f83f7ec 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -531,9 +531,6 @@ static BTREEHEADRECORD header; // Static buffer for database header record /**********************************************************************/ -//#define NTMNEXTKEY( pBT ) ((PNTMVITALINFO)(&pBT->chCollate[0]))->ulNextKey -//#define NTMSTARTKEY( pBT ) ((PNTMVITALINFO)(&pBT->chCollate[0]))->ulStartKey - /**********************************************************************/ /* 'Magic word' for record containing locked terms */ /**********************************************************************/ @@ -687,50 +684,6 @@ static STENCODEBITS stEncodeBits[16] = { {3,0}, // 000 // get difference for last comparison and return collat.diff // return difference //------------------------------------------------------------------------------ -SHORT QDAMKeyCompare -( - PVOID pvBT, // pointer to tree structure - PVOID pKey1, // pointer to first key - PVOID pKey2 // pointer to second key -) -{ - SHORT sDiff; - CHAR_W c; - PBTREE pBTIda = (PBTREE)pvBT; // pointer to tree structure - PBTREE pBT = pBTIda; - PBYTE pCollate = nullptr;// (PBYTE)pBT->chCollate; // pointer to collating sequence - PSZ_W pbKey1 = (PSZ_W) pKey1; - PSZ_W pbKey2 = (PSZ_W) pKey2; - - while ( (c = *pbKey1) != 0 ) - { - /******************************************************************/ - /* ignore the following characters during matching: '/', '-', ' ' */ - /******************************************************************/ - while ( (c = *pbKey2) == ' ' || fIsPunctuation[(BYTE)c] ) - { - pbKey2++; - } /* endwhile */ - while ( (c = *pbKey1) == ' ' || fIsPunctuation[(BYTE)c] ) - { - pbKey1++; - } /* endwhile */ -// @@ TODO: check collating sequence and fIsPunctuation - - sDiff = *(pCollate+(BYTE)c) - *(pCollate + (BYTE)*pbKey2); - if ( !sDiff && *pbKey1 && *pbKey2 ) - { - pbKey1++; - pbKey2++; - } - else - { - return ( sDiff ); - } /* endif */ - } /* endwhile */ - return ( *(pCollate + (BYTE)c) - *(pCollate + (BYTE)*pbKey2)); -} - //+----------------------------------------------------------------------------+ @@ -809,34 +762,22 @@ VOID BTREE::QDAMTerseInit //| <> keys are unequal | //| | //+----------------------------------------------------------------------------+ -//|Function flow: return (SHORT) (*pulKey1 - *pulKey2) | +//|Function flow: return (SHORT) (*plKey1 - *plKey2) | //+----------------------------------------------------------------------------+ -SHORT NTMKeyCompare -( - PVOID pBTIda, // pointer to tree structure - PVOID pulKey1, // pointer to first key - PVOID pulKey2 // pointer to second key +SHORT compKeys(const LONG& lKey1, // pointer to first key + const LONG& lKey2 // pointer to second key ) { - TMWCHAR ulKey1 = *((PWCHAR)pulKey1); - TMWCHAR ulKey2 = *((PWCHAR)pulKey2); - SHORT sRc; - pBTIda; // avoid compiler warnings - - if (ulKey1 > ulKey2 ) + if (lKey1 > lKey2 ) { - sRc = 1; + return 1; } - else if (ulKey1 < ulKey2 ) + else if (lKey1 < lKey2 ) { - sRc = -1; + return -1; } - else - { - sRc = 0; - } /* endif */ - return sRc; + return 0; } @@ -871,35 +812,30 @@ SHORT NTMKeyCompare // //------------------------------------------------------------------------------ SHORT BTREE::QDAMWriteHeader(){ - SHORT sRc=0; - BTREEHEADRECORD HeadRecord; // header record - DEBUGEVENT2( QDAMWRITEHEADER_LOC, FUNCENTRY_EVENT, 0, DB_GROUP, "" ); // Write the initial record (describing the index) out to disk - memset( &HeadRecord, '\0', sizeof(BTREEHEADRECORD)); - memcpy(HeadRecord.chEQF,chEQF, sizeof(chEQF)); - HeadRecord.usFirstNode = usFirstNode; - HeadRecord.usFirstLeaf = usFirstLeaf; - HeadRecord.usFreeKeyBuffer = usFreeKeyBuffer; - HeadRecord.usFreeDataBuffer = usFreeDataBuffer; - HeadRecord.usFirstDataBuffer = usFirstDataBuffer; // first data buffer - HeadRecord.fOpen = true;//fOpen; // open flag set - memcpy( HeadRecord.DataRecList, DataRecList, MAX_LIST*sizeof(HeadRecord.DataRecList[0]) ); - HeadRecord.fTerse = (EQF_BOOL) fTerse; - memcpy( HeadRecord.chCollate, &chCollate, sizeof(chCollate) ); - memcpy( HeadRecord.chCaseMap, chCaseMap, COLLATE_SIZE ); - memcpy( HeadRecord.chEntryEncode, chEntryEncode, ENTRYENCODE_LEN ); + //memset( &HeadRecord, '\0', sizeof(BTREEHEADRECORD)); + memcpy(header.chEQF,chEQF, sizeof(chEQF)); + header.usFirstNode = usFirstNode; + header.usFirstLeaf = usFirstLeaf; + header.usFreeKeyBuffer = usFreeKeyBuffer; + header.usFreeDataBuffer = usFreeDataBuffer; + header.usFirstDataBuffer = usFirstDataBuffer; // first data buffer + header.fOpen = true;//fOpen; // open flag set + memcpy( header.DataRecList, DataRecList, MAX_LIST*sizeof(header.DataRecList[0]) ); + header.fTerse = (EQF_BOOL) fTerse; + memcpy( header.chCollate, &chCollate, sizeof(chCollate) ); + memcpy( header.chEntryEncode, chEntryEncode, ENTRYENCODE_LEN ); /********************************************************************/ /* Update usNextFreeRecord field of Header and indicate that the */ /* field is valid by assigning BTREE_V1 to the bVersion field. */ /********************************************************************/ - HeadRecord.usNextFreeRecord = usNextFreeRecord; - HeadRecord.Flags.f16kRec = TRUE; + header.usNextFreeRecord = usNextFreeRecord; // check if disk is full - sRc = fb.Write((PVOID) &HeadRecord, sizeof(BTREEHEADRECORD), FILE_BEGIN); + SHORT sRc = fb.Write((PVOID) &header, sizeof(BTREEHEADRECORD), FILE_BEGIN); return sRc; } @@ -962,7 +898,7 @@ SHORT QDAMDosRC2BtreeRC SHORT BTREE::QDAMWRecordToDisk_V3 ( - PBTREEBUFFER_V3 pBuffer, // pointer to buffer to write + std::shared_ptr& pBuffer, // pointer to buffer to write bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed ) { @@ -1055,8 +991,8 @@ SHORT BTREE::QDAMWRecordToDisk_V3 SHORT BTREE::QDAMReadRecordFromDisk_V3 ( USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, - BOOL fNewRec, // allow new records flag + std::shared_ptr& pBuffer, + BOOL fNewRec, // allow new records flag bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed ) { @@ -1064,13 +1000,11 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 PBTREEHEADER pHeader; // pointer to header LONG lOffset; // file offset to be set ULONG ulNewOffset; // new position - PBTREEBUFFER_V3 pBuffer = NULL; SHORT sRc = 0; // return code - PLOOKUPENTRY_V3 pLEntry = NULL; DEBUGEVENT2( QDAMREADRECORDFROMDISK_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); int resRead = 0; - sRc = allocateNewLookupTableBuffer(usNumber, pLEntry, pBuffer); + sRc = allocateNewLookupTableBuffer(usNumber, pBuffer); if ( !sRc ) { @@ -1090,7 +1024,7 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 { pBuffer->usRecordNumber = usNumber; // pBuffer->ulCheckSum = QDAMComputeCheckSum( pBuffer ); - *ppReadBuffer = pBuffer; + //*ppReadBuffer = pBuffer; if ( resRead == FilesystemHelper::FILEHELPER_WARNING_FILE_IS_SMALLER_THAN_REQUESTED) { /*************************************************************/ @@ -1131,12 +1065,8 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 /********************************************************************/ if ( sRc ) { - if ( (pLEntry != NULL) && (pLEntry->pBuffer != NULL) ) - { - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, 0L, NOMSG ); - (usNumberOfAllocatedBuffers)--; - } /* endif */ - *ppReadBuffer = nullptr; + deallocateLookupTableBuffer(usNumber); + pBuffer.reset(); } /* endif */ if ( sRc ) @@ -1251,111 +1181,35 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 SHORT BTREE::QDAMReadRecord_V3 ( USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, + std::shared_ptr& pReadBuffer, BOOL fNewRec, bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed ) { USHORT i; SHORT sRc = 0; // return code - BOOL fMemOK = FALSE; - PLOOKUPENTRY_V3 pLEntry; - //PACCESSCTRTABLEENTRY pACTEntry; DEBUGEVENT2( QDAMREADRECORD_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); - *ppReadBuffer = NULL; INC_READ_COUNT; - - sRc = checkLookupTableAndRealocate(usNumber); - - - if ( !sRc ) + if(LookupTable.count(usNumber)) { /******************************************************************/ /* If record is in memory set ppReadBuffer */ /* else call QDAMReadRecordFromDisk to read it into memory */ /******************************************************************/ - //pLEntry = LookupTable_V3 + usNumber; - pLEntry = &LookupTable_V3[usNumber]; - - if ( pLEntry->pBuffer ) - { - /* Safety-Check: is rec.number = number of rec. to read ? */ - if ( (pLEntry->pBuffer)->usRecordNumber == usNumber ) - { - /* Record is already in memory */ - *ppReadBuffer = pLEntry->pBuffer; - } - else - { - /* This should never occur ! */ - {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_LOOKUPTABLE_CORRUPTED) <<"This should never occur";} - } /* endif */ - } - else - { - /* Record isn't in memory -> read it from disk */ - sRc = QDAMReadRecordFromDisk_V3( usNumber, ppReadBuffer, fNewRec, fSuppressTmdSizeCheck); - //pACTEntry = &AccessCtrTable[usNumber]; - //pACTEntry->ulAccessCounter = 0L; - pLEntry->ulAccessCounter = 0; - } /* endif */ - } /* endif */ - - /****************************************************************************/ - /* If #calls of QDAMReadRecord is greater than MAX_READREC_CALLS: */ - /* write unlocked records with access counter < MAX_READREC_CALLS to disk */ - /* (if fNeedToWrite is set) and free the allocated memory */ - /* set #calls of QDAMReadRecord = 0 */ - /* else increment #calls of QDAMReadRecord */ - /****************************************************************************/ - if ( !sRc ) - { - if ( ulReadRecCalls >= MAX_READREC_CALLS ) - { - for ( i=0; !sRc && (i < LookupTable_V3.size()); i++ ) - { - pLEntry = &LookupTable_V3[i]; - //pACTEntry = &AccessCtrTable[i]; - if ( pLEntry->pBuffer && !((pLEntry->pBuffer)->fLocked) && (i!=usNumber) - && (pLEntry->ulAccessCounterpBuffer)->fNeedToWrite ) - { - sRc = QDAMWRecordToDisk_V3( pLEntry->pBuffer); - } /* endif */ - if ( !sRc ) - { - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, 0L , NOMSG ); - usNumberOfAllocatedBuffers--; - } /* endif */ - } /* endif */ - - if ( pLEntry->ulAccessCounterulAccessCounter = 0L; - } - else - { - pLEntry->ulAccessCounter -= MAX_READREC_CALLS; - } /* endif */ - } /* endfor */ - ulReadRecCalls = 0L; - } - else + pReadBuffer = LookupTable[usNumber]; + + /* Safety-Check: is rec.number = number of rec. to read ? */ + if ( pReadBuffer->usRecordNumber != usNumber ) { - ulReadRecCalls++; + pReadBuffer.reset(); + /* This should never occur ! */ + LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_LOOKUPTABLE_CORRUPTED) << "if ( pReadBuffer->usRecordNumber(" <usRecordNumber<<") != usNumber("< read it from disk */ + sRc = QDAMReadRecordFromDisk_V3( usNumber, pReadBuffer, fNewRec, fSuppressTmdSizeCheck ); } /* endif */ - - if ( !sRc ) - { - /* Increase the access counter for the record just read by ACCESSBONUSPOINTS */ - pLEntry = &LookupTable_V3[usNumber]; - pLEntry->ulAccessCounter += ACCESSBONUSPOINTS; - } /* endif */ - return ( sRc ); } @@ -1502,15 +1356,12 @@ QDAMAddDict SHORT BTREE::QDAMDictFlushLocal() { SHORT sRc = 0; // return code - - for (int i=0; !sRc && (i < LookupTable_V3.size()); i++ ) - { - PLOOKUPENTRY_V3 pLEntry = &LookupTable_V3[i]; - if ( pLEntry->pBuffer /*&& (pLEntry->pBuffer)->fNeedToWrite*/ ) - { - sRc = QDAMWRecordToDisk_V3( pLEntry->pBuffer); - } /* endif */ - } /* endfor */ + for(auto& entry: LookupTable){ + sRc = QDAMWRecordToDisk_V3( entry.second); + if(sRc){ + return sRc; + } + } return sRc; } @@ -1559,31 +1410,7 @@ SHORT BTREE::QDAMDictFlushLocal() SHORT BTREE::QDAMDictClose() { SHORT sRc = 0; // error return - - // reset open flag in header and force a write to disk - // open flag will only be set if opened for r/w - { - sRc = QDAMDictFlushLocal(); - //fOpen = FALSE; - // re-write header record - if ( sRc == NO_ERROR ) - sRc = QDAMWriteHeader(); - } /* endif */ - - if ( UtlClose(fb.file, FALSE) && !sRc ) - { - sRc = BTREE_CLOSE_ERROR; - } /* endif */ - - if ( fpDummy ) - { - UtlClose( fpDummy, FALSE ); - } /* endif */ - - /*******************************************************************/ - /* free the allocated buffers */ - /*******************************************************************/ - + fb.data.resize(0); freeLookupTable(); return sRc; @@ -1626,74 +1453,9 @@ SHORT BTREE::QDAMDictClose() //------------------------------------------------------------------------------ -//------------------------------------------------------------------------------ -// Internal function -//------------------------------------------------------------------------------ -// Function name: QDAMDestroy Destroy dictionary -//------------------------------------------------------------------------------ -// Function call: QDAMDestroy( PBTREE ); -// -//------------------------------------------------------------------------------ -// Description: Destroy the file if something went wrong during the -// create -// -//------------------------------------------------------------------------------ -// Parameters: PBTREE pointer to btree structure -// -//------------------------------------------------------------------------------ -// Returncode type: SHORT -//------------------------------------------------------------------------------ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -// -//------------------------------------------------------------------------------ -// Function flow: if ptr to BTree does not exist -// set RC = BTREE_INVALID -// else -// if filename exists -// call QDAMDictClose -// delete the file 'filename' -// endif -// endif -//------------------------------------------------------------------------------ - -SHORT QDAMDestroy -( - PBTREE pBT // pointer to generic structure -) -{ - SHORT sRc = 0; // return code - - CHAR chName[ MAX_EQF_PATH ]; // file name - - if ( !pBT ) - { - sRc = BTREE_INVALID; - } - else if (!pBT->fb.fileName.empty() ) - { - // - // QDAMDictClose frees up all of the memory associated with a B-tree, - // we need to save the filename as we'll be deleteing the file by name - // - strcpy(chName, pBT->fb.fileName.c_str()); - sRc = pBT->QDAMDictClose( ); - if ( chName[0] ) - { - UtlDelete( chName, 0L, FALSE); - } - } - return ( sRc ); -} - - SHORT BTREE::QDAMWriteRecord_V3 ( - PBTREEBUFFER_V3 pBuffer, + std::shared_ptr& pBuffer, bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed ) { @@ -1845,39 +1607,36 @@ SHORT BTREE::QDAMAllocKeyRecords USHORT usNum ) { - SHORT sRc = 0; - PBTREEHEADER pHeader; + SHORT sRc = 0; + PBTREEHEADER pHeader; - //if ( pBT->bRecSizeVersion == BTREE_V3 ) - { - PBTREEBUFFER_V3 pRecord; - while ( usNum-- && !sRc ) + std::shared_ptr pRecord; + while ( usNum-- && !sRc ) + { + usNextFreeRecord++; + sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE ); + + if ( !sRc ) { - usNextFreeRecord++; - sRc = QDAMReadRecord_V3( usNextFreeRecord, &pRecord, TRUE ); + // add free record to the free list + NEXT( pRecord ) = usFreeKeyBuffer; + PREV( pRecord ) = 0; + usFreeKeyBuffer = RECORDNUM( pRecord ); + + // init this record + pHeader = &pRecord->contents.header; + pHeader->usNum = usFreeKeyBuffer; + pHeader->usOccupied = 0; + pHeader->usFilled = sizeof(BTREEHEADER ); + pHeader->usLastFilled = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER ); - if ( !sRc ) - { - // add free record to the free list - NEXT( pRecord ) = usFreeKeyBuffer; - PREV( pRecord ) = 0; - usFreeKeyBuffer = RECORDNUM( pRecord ); - - // init this record - pHeader = &pRecord->contents.header; - pHeader->usNum = usFreeKeyBuffer; - pHeader->usOccupied = 0; - pHeader->usFilled = sizeof(BTREEHEADER ); - pHeader->usLastFilled = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER ); - - /*************************************************************/ - /* force write */ - /*************************************************************/ - - sRc = QDAMWRecordToDisk_V3( pRecord); - } /* endif */ - } /* endwhile */ - } + /*************************************************************/ + /* force write */ + /*************************************************************/ + + sRc = QDAMWRecordToDisk_V3( pRecord); + } /* endif */ + } /* endwhile */ return sRc; } @@ -1991,13 +1750,13 @@ SHORT BTREE::QDAMAllocKeyRecords SHORT BTREE::QDAMDictCreateLocal ( PTMX_SIGN pSign, - ULONG ulStartKey, // translation memory + LONG lStartKey, // translation memory bool keepInRamOnly ) { const SHORT sNumberOfKey = 20; NTMVITALINFO NtmVitalInfo; // structure to contain vital info for TM - NtmVitalInfo.ulStartKey = NtmVitalInfo.ulNextKey = ulStartKey; + NtmVitalInfo.lStartKey = NtmVitalInfo.lNextKey = lStartKey; SHORT i; SHORT sRc = 0; // return code USHORT usAction; // used in open of file @@ -2005,12 +1764,7 @@ SHORT BTREE::QDAMDictCreateLocal if( false == keepInRamOnly ) { // Try to create the index file - fb.file = fopen(fb.fileName.c_str(), "w+b"); - - if(!fb.file){ - sRc = -1; - T5LOG(T5ERROR) << "::Can't create file " << fb.fileName; - } + fb.file = LockedFile(fb.file.fileName, LockedFile::CalledFrom::Create); } if ( !sRc ) @@ -2020,18 +1774,14 @@ SHORT BTREE::QDAMDictCreateLocal sCurrentIndex = 0; usFirstNode=usFirstLeaf = 1; usCurrentRecord = 0; - //compare = NTMKeyCompare; usNextFreeRecord = 1; usFreeKeyBuffer = 0; usFreeDataBuffer = 0; usFirstDataBuffer = 0; // first data buffer - //fOpen = TRUE; // open flag set - fpDummy = NULLHANDLE; /******************************************************************/ /* do settings depending if we are dealing with a dict or a tm.. */ /******************************************************************/ - usVersion = NTM_VERSION3;// changed from NTM_VERSION2 strcpy(chEQF,BTREE_HEADER_VALUE_TM3); fTerse = FALSE; /******************************************************************/ @@ -2040,12 +1790,8 @@ SHORT BTREE::QDAMDictCreateLocal /* header... */ /******************************************************************/ chCollate = NtmVitalInfo; - /******************************************************************/ - /* new key compare routine .... */ - /******************************************************************/ - //compare = NTMKeyCompare; - sRc = initLookupTable(); + sRc = initLookupTable(); usNumberOfAllocatedBuffers = 0; } @@ -2057,7 +1803,7 @@ SHORT BTREE::QDAMDictCreateLocal /* write an empty record buffer to the file before writing the */ /* header */ /****************************************************************/ - BTREEBUFFER_V3 pbuffer; + BTREEBUFFER pbuffer; memset((PVOID) &pbuffer, 0, BTREE_REC_SIZE_V3); sRc = fb.Write((PVOID)&pbuffer,BTREE_REC_SIZE_V3, 0); } @@ -2076,8 +1822,8 @@ SHORT BTREE::QDAMDictCreateLocal /*******************************************************************/ /* Write out an empty root node */ /*******************************************************************/ - PBTREEBUFFER_V3 pRecord; - sRc = QDAMReadRecord_V3( usFirstNode, &pRecord, TRUE ); + std::shared_ptr pRecord; + sRc = QDAMReadRecord_V3( usFirstNode, pRecord, TRUE ); if ( !sRc ) { TYPE(pRecord) = ROOT_NODE | LEAF_NODE | DATA_KEYNODE; @@ -2111,20 +1857,6 @@ SHORT BTREE::QDAMDictCreateLocal break; } /* endswitch */ - if ( !sRc ) - { - /****************************************************************/ - /* add this dictionary to our list ... */ - /****************************************************************/ - sRc = QDAMAddDict( (PSZ)fb.fileName.c_str(), this ); - } - else - { - // leave the return code from create - // file will not be destroyed if create failed, since then - // filename is not set - QDAMDestroy( this ); - } /* endif */ return( sRc ); } @@ -2193,7 +1925,7 @@ SHORT BTREE::QDAMDictCreateLocal BOOL BTREE::QDAMDictLockStatus ( - PWCHAR pKey + const LONG& lKey ) { return( false ); @@ -2282,11 +2014,10 @@ VOID BTREE::QDAMDictUpdStatus() { } /* end of function QDAMDictUpdStatus */ -PWCHAR QDAMGetszKey_V3 +LONG QDAMGetszKey_V3 ( - PBTREEBUFFER_V3 pRecord, // active record - USHORT i, // get data term - USHORT usVersion // version of database + std::shared_ptr& pRecord, // active record + USHORT i // get data term ) { PBYTE pData = nullptr, @@ -2297,7 +2028,8 @@ PWCHAR QDAMGetszKey_V3 PUSHORT pusOffset = (PUSHORT) pRecord->contents.uchData; pusOffset += i; // point to key - int step = sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data + constexpr int step = sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data + LONG res = -1; if ( (PBYTE)pusOffset > pEndOfRec ) { @@ -2315,34 +2047,30 @@ PWCHAR QDAMGetszKey_V3 T5LOG(T5ERROR) << "QDAMGetszKey_V3:: data pointer is out of range , pusOffset = " << (long int)pusOffset << "; pEndOfRec = " << (long int)pEndOfRec; pData = nullptr; - //ERREVENT2( QDAMGETSZKEY_LOC, INTFUNCFAILED_EVENT, 2, DB_GROUP, "" ); + }else{ + res = *((PLONG)pData); } /* endif */ } /* endif */ - return ( (PWCHAR)pData ); + + return ( res ); } RECPARAM QDAMGetrecData_V3 ( - PBTREEBUFFER_V3 pRecord, - SHORT sMid, // key number - USHORT usVersion // version of database + std::shared_ptr& pRecord, + SHORT sMid // key number ) { RECPARAM recData; // data description structure - PUSHORT pusOffset; - // use record number of passed entry , read in record and pass // back pointer - pusOffset = (PUSHORT) pRecord->contents.uchData; + PUSHORT pusOffset = (PUSHORT) pRecord->contents.uchData; pusOffset += sMid; // point to key PCHAR pData = (PCHAR)(pRecord->contents.uchData + *pusOffset); pData += sizeof(USHORT ); // get pointer to datarec memcpy( &recData, (PRECPARAM) pData, sizeof( RECPARAM )); - memcpy( &recData.usNum, (PUSHORT) pData, sizeof( USHORT ) ); - memcpy( &recData.usOffset,(PUSHORT) (pData+2), sizeof( USHORT ) ); - memcpy( &recData.ulLen, (PULONG) (pData+8), sizeof( PULONG ) ); return ( recData ); } @@ -2407,8 +2135,8 @@ RECPARAM QDAMGetrecData_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMFindRecord_V3 ( - PWCHAR pKey, - PBTREEBUFFER_V3 * ppRecord + LONG& lKey, + std::shared_ptr& pRecord ) { SHORT sResult; @@ -2416,30 +2144,31 @@ SHORT BTREE::QDAMFindRecord_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PWCHAR pKey2; // pointer to search key + LONG lKey2; // pointer to search key SHORT sRc; // return code memset(&recData, 0, sizeof(recData)); - sRc = QDAMReadRecord_V3( usFirstNode, ppRecord, FALSE ); - while ( !sRc && !IS_LEAF( *ppRecord )) + sRc = QDAMReadRecord_V3( usFirstNode, pRecord, FALSE ); + while ( !sRc && !IS_LEAF( pRecord )) { - BTREELOCKRECORD( *ppRecord ); + BTREELOCKRECORD( pRecord ); sLow = 0; // start here - sHigh = (SHORT) OCCUPIED( *ppRecord) -1 ; // counting starts at zero + sHigh = (SHORT) OCCUPIED(pRecord) -1 ; // counting starts at zero while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( *ppRecord, sMid, usVersion ); - if ( pKey2 == nullptr ) + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); + //if ( lKey2 == nullptr ) + if(lKey2 == -1) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<< " pKey==nulptr";} } else { - sResult = NTMKeyCompare(this, pKey, pKey2); + sResult = compKeys(lKey, lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -2462,14 +2191,14 @@ SHORT BTREE::QDAMFindRecord_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( *ppRecord, sHigh, usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh ); } /* endif */ - BTREEUNLOCKRECORD( *ppRecord ); // unlock previous record. + BTREEUNLOCKRECORD( pRecord ); // unlock previous record. if ( !sRc ) { - sRc = QDAMReadRecord_V3( recData.usNum, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( recData.usNum, pRecord, FALSE ); } /* endif */ } /* endwhile */ return( sRc ); @@ -2524,12 +2253,14 @@ SHORT BTREE::QDAMFindRecord_V3 BOOL BTREE::QDAMTerseData ( - PUCHAR pData, // pointer to data + BytesRef pData, // pointer to data PULONG pulLen // length of the string ) { + BOOL fShorter = FALSE; // new string is not shorter + T5LOG(T5FATAL) <<"TEMPORARY COMMENTED FUNCTION"; + #ifdef TEMPORARY_COMMENTED ULONG ulLen ; // length of string - BOOL fShorter = FALSE; // new string is not shorter USHORT usCurByte = 0; // currently processed byte PUCHAR pTempData; // pointer to temp data area PUCHAR pEndData; // pointer to end data area @@ -2626,6 +2357,7 @@ BOOL BTREE::QDAMTerseData } /* endif */ #if defined(MEASURE) ulTerseEnd += (pGlobInfoSeg->msecs -ulBeg ); +#endif #endif return ( fShorter ); } @@ -2673,11 +2405,11 @@ BOOL BTREE::QDAMTerseData // update ptr to next free key buffer // endif // endif -// reset info in *ppRecord +// reset info in pRecord //------------------------------------------------------------------------------ SHORT BTREE::QDAMNewRecord_V3 ( - PBTREEBUFFER_V3 * ppRecord, + std::shared_ptr& pRecord, RECTYPE recType, // data or key record bool fSuppressTmdSizeCheck ) @@ -2687,15 +2419,15 @@ SHORT BTREE::QDAMNewRecord_V3 // Try and use a record within the file if there are any. This saves // on disk usage // - *ppRecord = NULL; // in case of error + pRecord = NULL; // in case of error if ( recType == DATAREC ) { if ( usFreeDataBuffer ) { - sRc = QDAMReadRecord_V3( usFreeDataBuffer, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( usFreeDataBuffer, pRecord, FALSE ); if ( ! sRc ) { - usFreeDataBuffer = NEXT( *ppRecord ); + usFreeDataBuffer = NEXT( pRecord ); } /* endif */ } else @@ -2707,7 +2439,7 @@ SHORT BTREE::QDAMNewRecord_V3 else { usNextFreeRecord++; - sRc = QDAMReadRecord_V3( usNextFreeRecord, ppRecord, TRUE, fSuppressTmdSizeCheck ); + sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE, fSuppressTmdSizeCheck ); } } } @@ -2720,18 +2452,18 @@ SHORT BTREE::QDAMNewRecord_V3 if ( !sRc ) { - sRc = QDAMReadRecord_V3( usFreeKeyBuffer, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( usFreeKeyBuffer, pRecord, FALSE ); if ( ! sRc ) { - usFreeKeyBuffer = NEXT( *ppRecord ); + usFreeKeyBuffer = NEXT( pRecord ); } /* endif */ } /* endif */ } /* endif */ - if ( *ppRecord ) + if ( pRecord ) { - NEXT(*ppRecord) = 0; // reset information - PREV(*ppRecord) = 0; // reset information + NEXT(pRecord) = 0; // reset information + PREV(pRecord) = 0; // reset information } /* endif */ if ( sRc ) @@ -2800,11 +2532,9 @@ SHORT BTREE::QDAMNewRecord_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMAddToBuffer_V3 ( - PBYTE pData, // pointer to data - ULONG ulDataLen, // length of data to be filled in - PRECPARAM precReturn, // pointer to return code + BytesRef Data, // pointer to data + PRECPARAM precReturn, // pointer to return code bool fSuppressTmdSizeCheck //for deletion there no need to check if size is bigger than allowed - ) { SHORT sRc = 0; // success indicator @@ -2817,8 +2547,8 @@ SHORT BTREE::QDAMAddToBuffer_V3 BOOL fFit = FALSE; // indicator if record found USHORT i; // index PBTREEHEADER pHeader; // record header - PBTREEBUFFER_V3 pRecord = NULL; // buffer record - //PBTREEBUFFER_V3 pTempRecord; // buffer record + std::shared_ptr pRecord; // buffer record + //std::shared_ptr pTempRecord; // buffer record CHAR chNodeType; // type of node BOOL fTerse = FALSE; // data are not tersed USHORT usLastPos; @@ -2830,13 +2560,17 @@ SHORT BTREE::QDAMAddToBuffer_V3 memset(&recStart, 0, sizeof(recStart)); memset(&TempRecord[0], 0, sizeof(TempRecord)); + PBYTE pData = nullptr; + ULONG ulDataLen = Data.size(); + /*******************************************************************/ /* Enlarge pTempRecord area if it is not large enough to contain */ /* the data for this record */ /*******************************************************************/ - if ( (ulDataLen + sizeof(ULONG)) > TempRecord.size() ) + + if ( (Data.size() + sizeof(ULONG)) > TempRecord.size() ) { - TempRecord.resize(ulDataLen + sizeof(ULONG)); + TempRecord.resize(Data.size() + sizeof(ULONG)); } /* endif */ usLenFieldSize = sizeof(ULONG); @@ -2847,7 +2581,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 switch ( fTerse ) { case BTREE_TERSE_HUFFMAN : - fTerse = QDAMTerseData(pData, &ulDataLen ); + fTerse = QDAMTerseData(Data, &ulDataLen ); break; default : fTerse = FALSE; @@ -2860,18 +2594,19 @@ SHORT BTREE::QDAMAddToBuffer_V3 // copy them if not yet done..... if ( !fTerse ) { - memcpy( &TempRecord[0], pData, ulDataLen ); + //memcpy( &TempRecord[0], pData, ulDataLen ); + TempRecord = Data; } /* endif */ - if ( !sRc && (ulDataLen <= BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) ) + if ( !sRc && (Data.size() <= BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) ) { pRecTemp = pRecParam; i = 0; // find slot in list - usFilled = (USHORT)(BTREE_REC_SIZE_V3 - ulDataLen - 2 * usLenFieldSize); - + usFilled = (USHORT)(BTREE_REC_SIZE_V3 - Data.size() - 2 * usLenFieldSize); const long numOfRecordsInTheFile = fb.data.size()/BTREE_REC_SIZE_V3; - while ( i < MAX_LIST ) + + while ( !fFit && i < MAX_LIST ) { if(pRecTemp->usNum > numOfRecordsInTheFile){ pRecTemp->reset(); // pRecTemp pointing to invalid position @@ -2891,7 +2626,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 recStart.usNum = pRecTemp->usNum; // get record, copy data and write them - sRc = QDAMReadRecord_V3( recStart.usNum, &pRecord, FALSE, fSuppressTmdSizeCheck ); + sRc = QDAMReadRecord_V3( recStart.usNum, pRecord, FALSE, fSuppressTmdSizeCheck ); if ( !sRc ) { BTREELOCKRECORD( pRecord ); @@ -2903,7 +2638,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 { // fill in key data usLastPos = pRecord->contents.header.usLastFilled; - usLastPos = usLastPos - (USHORT)(ulDataLen + usLenFieldSize); + usLastPos = usLastPos - (USHORT)(Data.size() + usLenFieldSize); pData = pRecord->contents.uchData + usLastPos; // insert reference *(PULONG) pData = ulDataLen; @@ -2979,7 +2714,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 // get record and store start position memset( pRecTemp, 0 , sizeof(RECPARAM)); - sRc = QDAMNewRecord_V3( &pRecord, DATAREC, fSuppressTmdSizeCheck ); + sRc = QDAMNewRecord_V3( pRecord, DATAREC, fSuppressTmdSizeCheck ); // fill it up to length or record size if ( ! sRc ) { @@ -2993,9 +2728,9 @@ SHORT BTREE::QDAMAddToBuffer_V3 /* get a new record, anchor it and fill data from end */ /* new record will be predecessor of already allocated rec.*/ /***********************************************************/ - PBTREEBUFFER_V3 pTempRecBuff; - //sRc = QDAMNewRecord_V3( (PBTREEBUFFER_V3*)&TempRecord[0], DATAREC ); - sRc = QDAMNewRecord_V3( &pTempRecBuff, DATAREC, fSuppressTmdSizeCheck ); + std::shared_ptr pTempRecBuff; + //sRc = QDAMNewRecord_V3( (PBTREEBUFFER) &TempRecord[0], DATAREC ); + sRc = QDAMNewRecord_V3( pTempRecBuff, DATAREC, fSuppressTmdSizeCheck ); if ( !sRc ) { @@ -3041,7 +2776,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 /* toggle for adressing purposes */ /* now: pRecord will be the free one for later filling */ /*********************************************************/ - //pRecord = (PBTREEBUFFER_V3)&TempRecord[0]; + //pRecord = (PBTREEBUFFER)&TempRecord[0]; pRecord = pTempRecBuff; BTREELOCKRECORD( pRecord ); } /* endif */ @@ -3120,58 +2855,10 @@ SHORT BTREE::QDAMAddToBuffer_V3 return sRc; } - -SHORT QDAMCaseCompare -( - PVOID pvBT, // pointer to tree structure - PVOID pKey1, // pointer to first key - PVOID pKey2, // pointer to second key - BOOL fIgnorePunctuation // ignore punctuation flag -) -{ - SHORT sDiff; - BYTE c; - PBTREE pBTIda = (PBTREE)pvBT; // pointer to tree structure - PBTREE pBT = pBTIda; - PBYTE pbKey1 = (PBYTE) pKey1; - PBYTE pbKey2 = (PBYTE) pKey2; - PBYTE pMap = pBT->chCaseMap; // pointer to mapping table - - while ( (c = *pbKey1) != 0 ) - { - /******************************************************************/ - /* ignore the following characters during matching: '/', '-', ' ' */ - /******************************************************************/ - if ( fIgnorePunctuation ) - { - while ( (c = *pbKey2) == ' ' || fIsPunctuation[c] ) - { - pbKey2++; - } /* endwhile */ - while ( (c = *pbKey1) == ' ' || fIsPunctuation[c] ) - { - pbKey1++; - } /* endwhile */ - } /* endif */ - - sDiff = *(pMap+c) - *(pMap + *pbKey2); - if ( !sDiff && *pbKey1 && *pbKey2 ) - { - pbKey1++; - pbKey2++; - } - else - { - return ( sDiff ); - } /* endif */ - } /* endwhile */ - return ( *(pMap + c) - *(pMap + *pbKey2)); -} - SHORT BTREE::QDAMLocateKey_V3 ( - PBTREEBUFFER_V3 pRecord, // record to be dealt with - PWCHAR pKey, // key to be searched + std::shared_ptr& pRecord, // record to be dealt with + LONG& lKey, // key to be searched PSHORT psKeyPos, // located key SEARCHTYPE searchType, // search type PSHORT psNearPos // near position @@ -3182,7 +2869,7 @@ SHORT BTREE::QDAMLocateKey_V3 SHORT sResult; SHORT sMid = 0; // SHORT sRc = 0; // return value - PWCHAR pKey2; // pointer to key string + LONG lKey2 = -1; // pointer to key string BOOL fFound = FALSE; *psKeyPos = -1; // key not found @@ -3195,11 +2882,11 @@ SHORT BTREE::QDAMLocateKey_V3 while ( !fFound && sLow <= sHigh ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pRecord, sMid, usVersion ); + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); - if ( pKey2 ) + if ( lKey2 != -1 ) { - sResult = NTMKeyCompare(this, pKey, pKey2); + sResult = compKeys(lKey,lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3218,7 +2905,7 @@ SHORT BTREE::QDAMLocateKey_V3 /* matching substring */ /*********************************************************/ { - if (*((PULONG)pKey) == *((PULONG)pKey2)) + if (lKey == lKey2) { *psKeyPos = sMid; } @@ -3227,12 +2914,12 @@ SHORT BTREE::QDAMLocateKey_V3 // try with previous if ( sMid > sLow ) { - pKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1), usVersion ); - if ( pKey2 == NULL ) + lKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1) ); + if ( lKey2 == -1 ) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<< " pKey2==nulptr";} } - else if ( *((PULONG)pKey) == *((PULONG)pKey2) ) + else if ( lKey == lKey2) { *psKeyPos = sMid-1 ; } /* endif */ @@ -3240,12 +2927,12 @@ SHORT BTREE::QDAMLocateKey_V3 // still not found if ( !sRc && *psKeyPos == -1 && sMid < sHigh ) { - pKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1), usVersion ); - if ( pKey2 == NULL ) + lKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1) ); + if ( lKey2 == -1 ) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<< " pKey2==nulptr";} } - else if ( *((PULONG)pKey) == *((PULONG)pKey2) ) + else if (lKey == lKey2) { *psKeyPos = sMid+1 ; } /* endif */ @@ -3327,7 +3014,7 @@ SHORT BTREE::QDAMLocateKey_V3 SHORT QDAMFirstEntry_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 * ppRecord // pointer to pointer of record + std::shared_ptr& pRecord // pointer to pointer of record ) { SHORT sRc = 0; @@ -3335,17 +3022,17 @@ SHORT QDAMFirstEntry_V3 PBTREE pBT = pBTIda; // read in root record - sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, ppRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pRecord, FALSE ); if ( !sRc ) { - while (!sRc && !IS_LEAF(*ppRecord)) + while (!sRc && !IS_LEAF(pRecord)) { - recData = QDAMGetrecData_V3( *ppRecord, 0, pBT->usVersion); - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, ppRecord , FALSE ); + recData = QDAMGetrecData_V3( pRecord, 0); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pRecord , FALSE ); } /* endwhile */ if ( !sRc ) { - pBTIda->usCurrentRecord = RECORDNUM(*ppRecord); + pBTIda->usCurrentRecord = RECORDNUM(pRecord); pBT->usFirstLeaf = pBTIda->usCurrentRecord; // determine first leaf pBTIda->sCurrentIndex = 0; } /* endif */ @@ -3417,7 +3104,7 @@ SHORT QDAMFirstEntry_V3 SHORT QDAMFindParent_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, PUSHORT pusParent // record number of parent ) { @@ -3426,11 +3113,12 @@ SHORT QDAMFindParent_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PCHAR_W pKey2; // pointer to search key - PCHAR_W pKey ; // pointer to search key - CHAR_W chKey[HEADTERM_SIZE]; // key to be found + LONG lKey2 = -1; // pointer to search key + LONG lKey = -1 ; // pointer to search key + //CHAR_W chKey[HEADTERM_SIZE]; // key to be found + LONG lChKey = -1; SHORT sRc; // return code - PBTREEBUFFER_V3 pTempRec; // temp. record buffer + std::shared_ptr pTempRec; // temp. record buffer USHORT usRecNum; // passed record number PBTREE pBT = pBTIda; @@ -3438,17 +3126,17 @@ SHORT QDAMFindParent_V3 *pusParent = pBT->usFirstNode; // set parent to first node usRecNum = RECORDNUM( pRecord ); - pKey = QDAMGetszKey_V3( pRecord, 0, pBT->usVersion ); - if ( pKey == NULL ) + lKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( lKey != -1 ) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<< " pKey==nulptr";} } else { - memcpy( chKey, pKey, sizeof(ULONG)); + lChKey = lKey; } /* endif */ - sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, &pTempRec, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pTempRec, FALSE ); while ( !sRc && !IS_LEAF( pTempRec ) && (usRecNum != RECORDNUM(pTempRec)) ) { @@ -3462,15 +3150,14 @@ SHORT QDAMFindParent_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pTempRec, sMid, pBT->usVersion ); - if ( pKey2 == NULL ) + lKey2 = QDAMGetszKey_V3( pTempRec, sMid ); + if ( lKey2 == -1 ) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<<" pKey2 == nullptr";} } else { - //sResult = (*pBT->compare)(pBTIda, chKey, pKey2); - sResult = NTMKeyCompare(pBTIda, chKey, pKey2); + sResult = compKeys(lChKey,lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3493,14 +3180,14 @@ SHORT QDAMFindParent_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( pTempRec, sHigh, pBT->usVersion ); + recData = QDAMGetrecData_V3( pTempRec, sHigh ); } /* endif */ BTREEUNLOCKRECORD( pTempRec ); // unlock previous record. if ( !sRc ) { - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, &pTempRec, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pTempRec, FALSE ); } /* endif */ } /* endwhile */ return( sRc ); @@ -3534,11 +3221,10 @@ SHORT QDAMFindParent_V3 VOID QDAMCopyKeyTo_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT i, - PBTREEBUFFER_V3 pNew, - SHORT j, - USHORT usVersion // version of database + std::shared_ptr& pNew, + SHORT j ) { PUSHORT pusOldOffset; // offset of data @@ -3605,18 +3291,17 @@ VOID QDAMCopyKeyTo_V3 VOID QDAMReArrangeKRec_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { - PBTREEBUFFER_V3 pNew; PBTREEHEADER pHeader; // pointer to header PUSHORT pusOffset; SHORT i; SHORT j; PBTREE pBT = pBTIda; + std::shared_ptr pNew = pBT->BTreeTempBuffer_V3; // get temp record - pNew = &pBT->BTreeTempBuffer_V3; pHeader = &pNew->contents.header; pHeader->usOccupied = 0; pHeader->usFilled = sizeof(BTREEHEADER ); @@ -3629,7 +3314,7 @@ VOID QDAMReArrangeKRec_V3 { if ( *(pusOffset+i) ) { - QDAMCopyKeyTo_V3( pRecord, i, pNew, j, pBT->usVersion ); + QDAMCopyKeyTo_V3( pRecord, i, pNew, j ); i++; j++; } @@ -3734,19 +3419,17 @@ VOID QDAMReArrangeKRec_V3 SHORT QDAMSplitNode_V3 ( PBTREE pBTIda, // pointer to generic structure - PBTREEBUFFER_V3 *record, // pointer to pointer to node - PWCHAR pKey // new key + std::shared_ptr& record, // pointer to pointer to node + LONG& lKey // new key ) { SHORT i,j; - PBTREEBUFFER_V3 newRecord; - PBTREEBUFFER_V3 child; - PBTREEBUFFER_V3 parent = NULL; + std::shared_ptr newRecord, child, parent, pRecTemp; + USHORT usParent; // number of parent - PBTREEBUFFER_V3 pRecTemp; // temporary buffer RECPARAM recKey; // position/offset for key RECPARAM recData; // position/offset for data - PWCHAR pParentKey; // new key to be inserted + LONG lParentKey = -1; // new key to be inserted PUSHORT pusOffset; // pointer to offset table BOOL fCompare; // indicator where to insert new key USHORT usFreeKeys = 0; // number of free keys required @@ -3754,39 +3437,36 @@ SHORT QDAMSplitNode_V3 SHORT sRc = 0; // success indicator - - //T5LOG(T5FATAL) << "called commented out function QDAMSplitNode_V3"; - //#ifdef TEMPORARY_COMMENTED memset( &recKey, 0, sizeof( recKey ) ); memset( &recData, 0, sizeof( recData ) ); - pRecTemp = *record; + pRecTemp = record; BTREELOCKRECORD( pRecTemp ); // if root needs to be split do it first - if (IS_ROOT(*record)) + if (IS_ROOT(record)) { - sRc = pBTIda->QDAMNewRecord_V3( &newRecord, KEYREC ); + sRc = pBTIda->QDAMNewRecord_V3( newRecord, KEYREC ); if ( newRecord ) { BTREELOCKRECORD( newRecord ); /* We can't simply split a root node, since only one is allowed */ /* so a new root is created to hold the records */ - pBT->usFirstNode = PARENT(*record) = RECORDNUM(newRecord); + pBT->usFirstNode = PARENT(record) = RECORDNUM(newRecord); TYPE(newRecord) = ROOT_NODE | INNER_NODE | DATA_KEYNODE; PARENT(newRecord) = PREV(newRecord) = NEXT(newRecord) = 0L; - TYPE(*record) &= ~ROOT_NODE; - recData.usNum = RECORDNUM(*record); + TYPE(record) &= ~ROOT_NODE; + recData.usNum = RECORDNUM(record); recData.usOffset = 0; - //pParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); + lParentKey = QDAMGetszKey_V3( record, 0 ); - if ( pParentKey ) + //if ( lParentKey ) + if(lParentKey != -1) { // store key temporarily, because record with key data is // not locked. - sRc = pBTIda->QDAMInsertKey_V3( newRecord, pParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( newRecord, lParentKey, recKey, recData); // might be freed during insert BTREELOCKRECORD( pRecTemp ); } @@ -3799,7 +3479,7 @@ SHORT QDAMSplitNode_V3 // update usFirstLeaf information if ( !sRc ) { - sRc = QDAMFirstEntry_V3( pBTIda, &newRecord ); + sRc = QDAMFirstEntry_V3( pBTIda, newRecord ); } /* endif */ if ( !sRc ) { @@ -3811,13 +3491,13 @@ SHORT QDAMSplitNode_V3 if ( !sRc ) { // allocate space for the new record - sRc = pBTIda->QDAMNewRecord_V3( &newRecord, KEYREC ); + sRc = pBTIda->QDAMNewRecord_V3( newRecord, KEYREC ); if ( newRecord ) { BTREELOCKRECORD(newRecord); - if ( NEXT(*record) ) + if ( NEXT(record) ) { - sRc = pBTIda->QDAMReadRecord_V3( NEXT(*record), &child, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( NEXT(record), child, FALSE ); if ( !sRc ) { PREV(child) = RECORDNUM(newRecord); @@ -3828,46 +3508,45 @@ SHORT QDAMSplitNode_V3 if ( !sRc ) { /* Adjust Sibling pointers */ - NEXT(newRecord) = NEXT(*record); - PREV(newRecord) = RECORDNUM(*record); - NEXT(*record) = RECORDNUM(newRecord); - PARENT(newRecord) = PARENT(*record); - TYPE(newRecord) = (CHAR)(TYPE(*record) & ~ROOT_NODE); // don't copy Root bit + NEXT(newRecord) = NEXT(record); + PREV(newRecord) = RECORDNUM(record); + NEXT(record) = RECORDNUM(newRecord); + PARENT(newRecord) = PARENT(record); + TYPE(newRecord) = (CHAR)(TYPE(record) & ~ROOT_NODE); // don't copy Root bit // Decide where to split the record - //pParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); + lParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2) ); - if ( pParentKey ) + if ( lParentKey != -1 ) { - fCompare = NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ; + fCompare = (compKeys(lParentKey, lKey) <= 0 ) ; /***********************************************************/ /* check in which part we will lay */ /***********************************************************/ if ( fCompare ) { - pParentKey = QDAMGetszKey_V3(*record, (SHORT)(OCCUPIED(*record)-MINFREEKEYS), pBT->usVersion); - fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; + lParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS)); + fCompare = (compKeys(lParentKey,lKey) <= 0 ) ; if ( fCompare ) { usFreeKeys = MINFREEKEYS; } else { - usFreeKeys = OCCUPIED( *record )/2; + usFreeKeys = OCCUPIED( record )/2; } /* endif */ } else { - pParentKey = QDAMGetszKey_V3( *record,MINFREEKEYS, pBT->usVersion); - fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; + lParentKey = QDAMGetszKey_V3( record,MINFREEKEYS); + fCompare = (compKeys(lParentKey,lKey) <= 0) ; if ( fCompare ) { - usFreeKeys = OCCUPIED( *record )/2; + usFreeKeys = OCCUPIED( record )/2; } else { - usFreeKeys = OCCUPIED( *record ) -MINFREEKEYS; + usFreeKeys = OCCUPIED( record ) -MINFREEKEYS; } /* endif */ } /* endif */ } @@ -3883,26 +3562,26 @@ SHORT QDAMSplitNode_V3 // ELSE case... if ( !sRc ) { - i = (SHORT) (OCCUPIED( *record ) - usFreeKeys); + i = (SHORT) (OCCUPIED( record ) - usFreeKeys); j = 0; - pusOffset = (PUSHORT) (*record)->contents.uchData; - while ( i < (SHORT) OCCUPIED( *record )) + pusOffset = (PUSHORT) (record)->contents.uchData; + while ( i < (SHORT) OCCUPIED( record )) { - QDAMCopyKeyTo_V3( *record, i, newRecord, j, pBT->usVersion ); + QDAMCopyKeyTo_V3( record, i, newRecord, j ); *(pusOffset+i) = 0 ; // mark it as deleted j++; i++; } /* Adjust count of keys */ - OCCUPIED(*record) = OCCUPIED(*record) - usFreeKeys; + OCCUPIED(record) = OCCUPIED(record) - usFreeKeys; OCCUPIED(newRecord) = usFreeKeys; - QDAMReArrangeKRec_V3( pBTIda, *record ); + QDAMReArrangeKRec_V3( pBTIda, record ); /* Insert pointer to new record into the parent node */ /* due to the construction parent MUST be the same */ - sRc = QDAMFindParent_V3( pBTIda, *record, &usParent ); + sRc = QDAMFindParent_V3( pBTIda, record, &usParent ); if ( !sRc && usParent ) { - sRc = pBTIda->QDAMReadRecord_V3( usParent, &parent, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( usParent, parent, FALSE ); } /* endif */ } /* endif */ @@ -3910,10 +3589,10 @@ SHORT QDAMSplitNode_V3 { recData.usNum = RECORDNUM( newRecord ); recData.usOffset = 0; - pParentKey = QDAMGetszKey_V3( newRecord,0, pBT->usVersion ); - if ( pParentKey ) + lParentKey = QDAMGetszKey_V3( newRecord,0 ); + if ( lParentKey != -1) { - sRc = pBTIda->QDAMInsertKey_V3( parent, pParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( parent, lParentKey, recKey, recData); } else { @@ -3927,13 +3606,13 @@ SHORT QDAMSplitNode_V3 { // newRecord->ulCheckSum = QDAMComputeCheckSum( newRecord ); // (*record)->ulCheckSum = QDAMComputeCheckSum( *record ); - pParentKey = QDAMGetszKey_V3( newRecord, 0, pBT->usVersion ); - if ( pParentKey ) + lParentKey = QDAMGetszKey_V3( newRecord, 0 ); + if ( lParentKey != -1) { - if ( NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0) + if ( compKeys(lParentKey,lKey) <= 0) { - sRc = pBTIda->QDAMWriteRecord_V3( *record); - *record = newRecord; // add to new record + sRc = pBTIda->QDAMWriteRecord_V3( record); + record = newRecord; // add to new record } else { @@ -4029,10 +3708,9 @@ SHORT QDAMSplitNode_V3 //+----------------------------------------------------------------------------+ SHORT QDAMUnTerseData ( - PBTREE pBTIda, // - PUCHAR pData, // pointer to data - ULONG ulDataLen, // data length (uncompressed) - PULONG pulLen // length of the string + PBTREE pBTIda, // + BytesRef pData, // pointer to data + ULONG ulDataLen // data length (uncompressed) ) { ULONG ulLen ; // length of string @@ -4041,14 +3719,17 @@ SHORT QDAMUnTerseData USHORT usStartBit = 0; // start bit looking at from high to low USHORT usDecode; // value to be decoded SHORT sRc = 0; // okay; - PSZ pInData = (PSZ)pData; // pointer to input data + PSZ pInData = (PSZ)pData.data(); // pointer to input data + PSZ ppData = (PSZ)pData.data(); PBTREE pBT = pBTIda; + long lLen = 0; #if defined(MEASURE) ulBeg = pGlobInfoSeg->msecs; #endif // get length of passed string ulLen = ulDataLen; + T5LOG(T5TRANSACTION) << "pData.size = " << pData.size() << "; ulDataLen = " << ulDataLen; // try to uncompress the passed string ( use pTempData as temporary space) @@ -4063,7 +3744,7 @@ SHORT QDAMUnTerseData /***************************************************************/ if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ @@ -4081,7 +3762,7 @@ SHORT QDAMUnTerseData case 0: // get next character if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ *pTempData++ = @@ -4090,7 +3771,7 @@ SHORT QDAMUnTerseData usStartBit -= 8; if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ break; @@ -4130,16 +3811,9 @@ SHORT QDAMUnTerseData /*******************************************************************/ if ( !sRc ) { - if ( *pulLen >= ulDataLen ) - { - memcpy( pInData, &pBT->TempRecord[0], *pulLen ); - *pulLen = ulDataLen; - } - else - { - T5LOG(T5ERROR) << ":: BufferTooSmall 1"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ + // Replace old data with new data + pData.clear(); // Remove old data + pData.insert(pData.begin(), &pBT->TempRecord[0], &pBT->TempRecord[0] + lLen); // Insert new data at the same position } /* endif */ #if defined(MEASURE) ulUnTerseEnd += (pGlobInfoSeg->msecs - ulBeg); @@ -4184,20 +3858,17 @@ SHORT QDAMUnTerseData // endif // //------------------------------------------------------------------------------ -SHORT QDAMGetszData_V3 +SHORT BTREE::QDAMGetszData_V3 ( - PBTREE pBTIda, RECPARAM recDataParam, - PBYTE pData, - PULONG pulDataLen, - CHAR chType // type of record key/data + BytesRef pData + //CHAR chType=DATA_NODE // type of record key/data ) { SHORT sRc = 0; - PBTREEBUFFER_V3 pRecord; // pointer to record + std::shared_ptr pRecord; // pointer to record PBTREEHEADER pHeader; // pointer to header PCHAR pTempData = NULL; // pointer to data pointer - PCHAR pStartData = (PCHAR)pData; // pointer to data pointer LONG lLen = 0; // length of string LONG lTerseLen = 0; // length of tersed string BOOL fTerse = FALSE; // entry tersed?? @@ -4205,7 +3876,6 @@ SHORT QDAMGetszData_V3 LONG lFitLen; // free to be filled length USHORT usNum; // record number LONG lLZSSLen; - PBTREE pBT = pBTIda; BOOL fRecLocked = FALSE; // TRUE if BTREELOCKRECORD has been done USHORT usLenFieldSize; // size of record length field @@ -4213,14 +3883,14 @@ SHORT QDAMGetszData_V3 // get size of record length field usLenFieldSize = sizeof(ULONG); - recDataParam.usNum = recDataParam.usNum + pBT->usFirstDataBuffer; + recDataParam.usNum = recDataParam.usNum + usFirstDataBuffer; // get record and copy data - sRc = pBTIda->QDAMReadRecord_V3( recDataParam.usNum, &pRecord, FALSE ); + sRc = QDAMReadRecord_V3( recDataParam.usNum, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD( pRecord ); fRecLocked = TRUE; - if ( TYPE( pRecord ) != chType ) + if ( TYPE( pRecord ) != DATA_NODE ) { {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED) << " if ( TYPE( pRecord ) != chType ), TYPE( pRecord )=" << toStr( TYPE( pRecord ));} } @@ -4261,108 +3931,83 @@ SHORT QDAMGetszData_V3 } /* endif */ } /* endif */ - if ( !sRc ) - { - if ( *pulDataLen == 0 || !pData ) - { - // give back only length - if ( fTerse ) - { - // first field contains real length - *pulDataLen = LENGTHOFDATA(pBT,pTempData); - } - else - { - *pulDataLen = lLen; // give back only length - } /* endif */ - } - else if ( *pulDataLen >= lLen ) - { - lFitLen = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER) - *pusOffset; - lFitLen = lLen < (lFitLen - usLenFieldSize) ? lLen : (lFitLen - usLenFieldSize) ; - - if ( fTerse ) - { - memcpy(pData,pTempData+usLenFieldSize,lFitLen-usLenFieldSize); - lTerseLen = LENGTHOFDATA(pBT,pTempData); - /**********************************************************/ - /* adjust pointers */ - /**********************************************************/ - lLen -= lFitLen; - pData += (lFitLen - usLenFieldSize); - } - else - { - memcpy( pData, pTempData, lFitLen ); - *pulDataLen = lLen; - /**********************************************************/ - /* adjust pointers */ - /**********************************************************/ - lLen -= lFitLen; - pData += lFitLen; - } /* endif */ + if ( !sRc ) + { + lFitLen = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER) - *pusOffset; + lFitLen = lLen < (lFitLen - usLenFieldSize) ? lLen : (lFitLen - usLenFieldSize) ; + if (pTempData && lFitLen > 0) { + if ( fTerse ) + { + long bytesToCopy = lFitLen - usLenFieldSize; + pData.insert(pData.end(), pTempData+usLenFieldSize, pTempData + bytesToCopy); + lTerseLen = LENGTHOFDATA(pBT,pTempData); /**********************************************************/ - /* copy as long as still data are available */ + /* adjust pointers */ /**********************************************************/ - while ( !sRc && lLen>0 ) - { - usNum = NEXT( pRecord ); - BTREEUNLOCKRECORD( pRecord ); - fRecLocked = FALSE; - if ( usNum ) - { - sRc = pBTIda->QDAMReadRecord_V3( usNum, &pRecord, FALSE ); - if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) - { - {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED) << "if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ), sRc = " << sRc << "; TYPE( pRecord ) " << TYPE( pRecord );} - } /* endif */ - if ( !sRc ) - { - BTREELOCKRECORD( pRecord ); - fRecLocked = TRUE; - pusOffset = (PUSHORT) pRecord->contents.uchData; - pTempData = (PCHAR)(pRecord->contents.uchData + *pusOffset); - - lFitLen = LENGTHOFDATA( pBT, pTempData ); - lFitLen = lLen < lFitLen? lLen: lFitLen ; - pTempData += usLenFieldSize; // get pointer to data - - memcpy( pData, pTempData, lFitLen ); - lLen -= lFitLen; - pData += lFitLen; - } /* endif */ - } - else - { - {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED) << "if ( usNum ) else(usNum == 0)";} - ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 4, DB_GROUP, "" ); - } /* endif */ - } /* endwhile */ - if ( !sRc && fTerse ) + lLen -= lFitLen; + } + else + { + pData.insert(pData.end(), pTempData, pTempData + lFitLen); + lLen -= lFitLen; + } /* endif */ + } + + + /**********************************************************/ + /* copy as long as still data are available */ + /**********************************************************/ + while ( !sRc && lLen>0 ) + { + usNum = NEXT( pRecord ); + BTREEUNLOCKRECORD( pRecord ); + fRecLocked = FALSE; + if ( usNum ) { - switch ( pBT->fTerse ) + sRc = QDAMReadRecord_V3( usNum, pRecord, FALSE ); + if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) + { + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 3, DB_GROUP, "" ); + } /* endif */ + if ( !sRc ) { - case BTREE_TERSE_HUFFMAN : - { - sRc = QDAMUnTerseData( pBTIda, (PUCHAR)pStartData, lTerseLen, - pulDataLen ); - } - break; - default : - {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED) << "default pBT->fTerse = " << pBT->fTerse;} - ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 5, DB_GROUP, "" ); - break; - } /* endswitch */ + BTREELOCKRECORD( pRecord ); + fRecLocked = TRUE; + pusOffset = (PUSHORT) pRecord->contents.uchData; + pTempData = (PCHAR)(pRecord->contents.uchData + *pusOffset); + + lFitLen = LENGTHOFDATA( pBT, pTempData ); + lFitLen = lLen < lFitLen? lLen: lFitLen ; + pTempData += usLenFieldSize; // get pointer to data + + pData.insert(pData.end(), pTempData, pTempData + lFitLen); + lLen -= lFitLen; + } /* endif */ + } + else + { + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 4, DB_GROUP, "" ); } /* endif */ - } - else - { - T5LOG(T5ERROR) << ":: BufferTooSmall 3"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ - + } /* endwhile */ + if ( !sRc && fTerse ) + { + switch ( fTerse ) + { + case BTREE_TERSE_HUFFMAN : + { + sRc = QDAMUnTerseData( this, pData, lTerseLen ); + } + break; + default : + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 5, DB_GROUP, "" ); + break; + } /* endswitch */ + } /* endif */ } /* endif */ if ( fRecLocked ) { @@ -4439,9 +4084,9 @@ SHORT QDAMGetszData_V3 SHORT QDAMFindChild_V3 ( PBTREE pBTIda, - PCHAR_W pKey, + LONG& lKey, USHORT usNode, - PBTREEBUFFER_V3 * ppRecord + std::shared_ptr& pRecord ) { SHORT sResult; @@ -4449,32 +4094,31 @@ SHORT QDAMFindChild_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PCHAR_W pKey2; // pointer to search key + LONG lKey2 = -1; // pointer to search key SHORT sRc; // return code PBTREE pBT = pBTIda; memset(&recData, 0, sizeof(recData)); - sRc = pBTIda->QDAMReadRecord_V3( usNode, ppRecord, FALSE ); - if ( !sRc && !IS_LEAF( *ppRecord )) + sRc = pBTIda->QDAMReadRecord_V3( usNode, pRecord, FALSE ); + if ( !sRc && !IS_LEAF( pRecord )) { - BTREELOCKRECORD( *ppRecord ); + BTREELOCKRECORD( pRecord ); sLow = 0; // start here - sHigh = (SHORT) OCCUPIED( *ppRecord) -1 ; // counting starts at zero + sHigh = (SHORT) OCCUPIED( pRecord) -1 ; // counting starts at zero while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( *ppRecord, sMid, pBT->usVersion ); - if ( pKey2 == NULL ) + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); + if ( lKey2 == -1 ) { {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED)<<" pKey2 == nullptr";} } else { - //sResult = (*pBT->compare)(pBTIda, pKey, pKey2); - sResult = NTMKeyCompare(pBTIda, pKey, pKey2); + sResult = compKeys(lKey, lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -4497,14 +4141,14 @@ SHORT QDAMFindChild_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( *ppRecord, sHigh, pBT->usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh ); } /* endif */ - BTREEUNLOCKRECORD( *ppRecord ); // unlock previous record. + BTREEUNLOCKRECORD( pRecord ); // unlock previous record. if ( !sRc ) { - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, ppRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pRecord, FALSE ); } /* endif */ } /* endif */ return( sRc ); @@ -4515,12 +4159,12 @@ SHORT QDAMChangeKey_V3 ( PBTREE pBTIda, // ptr to tree structure USHORT usNode, // start node - PWCHAR pOldKey, // find old key - PWCHAR pNewKey // find new key + LONG& lOldKey, // find old key + LONG& lNewKey // find new key ) { - PBTREEBUFFER_V3 pRecord; // buffer for record - PBTREEBUFFER_V3 pNewRecord = NULL; // buffer for new record + std::shared_ptr pRecord; // buffer for record + std::shared_ptr pNewRecord; // buffer for new record SHORT i = 0; // index SHORT sRc = 0; // return code SHORT sNearKey; @@ -4528,30 +4172,30 @@ SHORT QDAMChangeKey_V3 RECPARAM recKey; // record parameter descrip. - sRc = pBTIda->QDAMReadRecord_V3( usNode, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( usNode, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD(pRecord); if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindChild_V3( pBTIda, pNewKey, usNode, &pNewRecord ); + sRc = QDAMFindChild_V3( pBTIda, lNewKey, usNode, pNewRecord ); } /* endif */ if ( !sRc ) { // get the key description and insert the new key recKey.usNum = pNewRecord->contents.header.usNum; - sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, pNewKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, lNewKey, &i, FEXACT, &sNearKey); if ( !sRc && i != -1 ) { recKey.usOffset = i; recKey.ulLen = 0; // init length - sRc = pBTIda->QDAMInsertKey_V3( pRecord, pNewKey, recKey, recKey); + sRc = pBTIda->QDAMInsertKey_V3( pRecord, lNewKey, recKey, recKey); } /* endif */ } /* endif */ if ( !sRc ) { - sRc = pBTIda->QDAMLocateKey_V3( pRecord, pOldKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pRecord, lOldKey, &i, FEXACT, &sNearKey); } /* endif */ if (!sRc && i!= -1) @@ -4578,7 +4222,7 @@ SHORT QDAMChangeKey_V3 size_t BTREE::GetFileSize()const{ - return FilesystemHelper::GetFileSize(fb.fileName); + return FilesystemHelper::GetFileSize(fb.file.fileName); } @@ -4635,17 +4279,17 @@ size_t BTREE::GetFileSize()const{ //------------------------------------------------------------------------------ SHORT BTREE::QDAMInsertKey_V3 ( - PBTREEBUFFER_V3 pRecord, // record where key is to be inserted - PWCHAR pKey, + std::shared_ptr& pRecord, // record where key is to be inserted + LONG& lKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ) { SHORT i = 0; - PBTREEBUFFER_V3 pTempRec; - PWCHAR pCompKey = NULL; // key to be compared with - PWCHAR pOldKey; // old key at first position - PWCHAR pNewKey; // new key at first position + std::shared_ptr pTempRec; + LONG lCompKey = -1; // key to be compared with + LONG lOldKey = -1; // old key at first position + LONG lNewKey = -1; // new key at first position BOOL fFound = FALSE; SHORT sKeyFound; // key found SHORT sNearKey; // key found @@ -4662,7 +4306,7 @@ SHORT BTREE::QDAMInsertKey_V3 recKey; // get rid of compiler warnings // check if key is already there -- duplicates will not be supported - sRc = QDAMLocateKey_V3( pRecord, pKey, &sKeyFound, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3( pRecord, lKey, &sKeyFound, FEXACT, &sNearKey); BTREELOCKRECORD( pRecord ); fRecLocked = TRUE; if ( !sRc ) @@ -4683,7 +4327,7 @@ SHORT BTREE::QDAMInsertKey_V3 { BTREEUNLOCKRECORD( pRecord ); fRecLocked = FALSE; - sRc = QDAMSplitNode_V3( this, &pRecord, pKey ); + sRc = QDAMSplitNode_V3( this, pRecord, lKey ); if ( !sRc ) { // SplitNode may have passed a new record back @@ -4694,7 +4338,7 @@ SHORT BTREE::QDAMInsertKey_V3 /* inserting must be changed */ if ( !IS_LEAF( pRecord ) ) { - sRc = QDAMReadRecord_V3( recData.usNum, &pTempRec, FALSE ); + sRc = QDAMReadRecord_V3( recData.usNum,pTempRec, FALSE ); if ( ! sRc ) { PARENT( pTempRec ) = RECORDNUM( pRecord ); @@ -4715,11 +4359,10 @@ SHORT BTREE::QDAMInsertKey_V3 pusOffset = (PUSHORT) pRecord->contents.uchData; while ( i > 0 && !fFound ) { - pCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1), usVersion ); - if ( pCompKey ) + lCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1) ); + if ( lCompKey != -1 ) { - if ( NTMKeyCompare(this, pKey, pCompKey) < 0 ) - //if ( (*(compare))(this, pKey, pCompKey) < 0 ) + if ( compKeys(lKey, lCompKey) < 0 ) { *(pusOffset+i) = *(pusOffset+i-1); i--; @@ -4753,7 +4396,7 @@ SHORT BTREE::QDAMInsertKey_V3 { PBYTE pTarget; pTarget = pData + sizeof(USHORT) + sizeof(RECPARAM); - memcpy(pTarget, (PBYTE) pKey, usKeyLen ); + memcpy(pTarget, &lKey, usKeyLen ); } // set data information memcpy((PRECPARAM) (pData+sizeof(USHORT)), &recData, sizeof(RECPARAM)); @@ -4774,11 +4417,11 @@ SHORT BTREE::QDAMInsertKey_V3 { if ((i == 0) && (!IS_ROOT(pRecord))) { - pOldKey = QDAMGetszKey_V3( pRecord, 1, usVersion ); - pNewKey = QDAMGetszKey_V3( pRecord, 0, usVersion ); - if ( pOldKey && pNewKey ) + lOldKey = QDAMGetszKey_V3( pRecord, 1 ); + lNewKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( lOldKey != -1 && lNewKey != -1 ) { - if ( pCompKey && PARENT(pRecord) ) + if ( lCompKey != -1 && PARENT(pRecord) ) { /**********************************************************/ /* save old current record since ChangeKey will update it */ @@ -4786,7 +4429,7 @@ SHORT BTREE::QDAMInsertKey_V3 /**********************************************************/ _usCurrentRecord = usCurrentRecord; _sCurrentIndex = sCurrentIndex; - sRc = QDAMChangeKey_V3( this, PARENT(pRecord), pOldKey, pNewKey); + sRc = QDAMChangeKey_V3( this, PARENT(pRecord), lOldKey, lNewKey); usCurrentRecord = _usCurrentRecord; sCurrentIndex = _sCurrentIndex; @@ -4794,7 +4437,7 @@ SHORT BTREE::QDAMInsertKey_V3 } else { - {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED) << "if ( !pOldKey || !pNewKey ); pOldKey="<< (ULONG)pOldKey << "; pNewKey=" << (ULONG)pNewKey;} + {LOG_AND_SET_RC_W_INFO(sRc, T5WARNING, BTREE_CORRUPTED) << "if ( !pOldKey || !pNewKey ); lOldKey="<< lOldKey << "; lNewKey=" << lNewKey;} ERREVENT2( QDAMINSERTKEY_LOC, STATE_EVENT, 10, DB_GROUP, "" ); } /* endif */ } /* endif */ @@ -4945,9 +4588,9 @@ USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ) //------------------------------------------------------------------------------ // Internal function //------------------------------------------------------------------------------ -// Function name: QDAMDictOpenLocal Open local Dictionary +// Function name: EQFNTMOpen Open local Dictionary //------------------------------------------------------------------------------ -// Function call: QDAMDictOpenLocal( PSZ, SHORT, BOOL, PPBTREE ); +// Function call: EQFNTMOpen( PSZ, SHORT, BOOL, PPBTREE ); // //------------------------------------------------------------------------------ // Description: Open a file locally for processing @@ -5000,14 +4643,13 @@ USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ) // endif // return Rc //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictOpenLocal +SHORT BTREE::EQFNTMOpen ( - SHORT sNumberOfBuffers, // number of buffers - USHORT usOpenFlags // Read Only or Read/Write + USHORT usOpenFlags, // Read Only or Read/Write + SHORT sNumberOfBuffers // number of buffers ) { SHORT i; - BTREEHEADRECORD header; SHORT sRc = 0; // return code USHORT usFlags; // set the open flags USHORT usAction; // return code from UtlOpen @@ -5022,11 +4664,10 @@ SHORT BTREE::QDAMDictOpenLocal /****************************************************************/ /* check if immeadiate write of any changed necessary */ /****************************************************************/ - fGuard = ( usOpenFlags & ASD_FORCE_WRITE ); usOpenFlags = usOpenFlags; // open the file - fb.ReadFromFile(); + sRc = fb.ReadFromFile(); if ( !sRc ) { @@ -5049,8 +4690,6 @@ SHORT BTREE::QDAMDictOpenLocal /* (Check only first 3 characters of BTREE identifier to */ /* ignore the version number) */ /************************************************************/ - usVersion = (USHORT) header.chEQF[3]; - //compare = NTMKeyCompare; UtlTime( &(lTime) ); // set open time sCurrentIndex = 0; @@ -5060,7 +4699,6 @@ SHORT BTREE::QDAMDictOpenLocal usFreeKeyBuffer = header.usFreeKeyBuffer; usFreeDataBuffer = header.usFreeDataBuffer; usFirstDataBuffer = header.usFirstDataBuffer; // data buffer - fpDummy = NULLHANDLE; // load usNextFreeRecord either from header record of from file info { @@ -5068,6 +4706,31 @@ SHORT BTREE::QDAMDictOpenLocal } /* endif */ ASDLOG(); + memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); + //fix chCollate if neede + const long numOfRecordsInTheFile = fb.data.size()/BTREE_REC_SIZE_V3; + if(fb.isTMDFilebuffer()){ + if(chCollate.lStartKey != FIRST_KEY){ + T5LOG(T5ERROR) + << "info in the header of the tmd is corrupted, and chCollate.lStartKey=" + << chCollate.lStartKey <<"; (expected " << FIRST_KEY <<"); isCorrupted for this memory is set to true;"; + //chCollate.lStartKey = FIRST_KEY; + sRc = BTREE_CORRUPTED; + } + + if(chCollate.lNextKey >= LASTALLOWEDKEY){ + T5LOG(T5ERROR) << "info in the header of the tm is corrupted, and chCollate.lNextKey=" + << chCollate.lNextKey <<"; (expected <" << LASTALLOWEDKEY <<"); isCorrupted for this memory is set to true;"; + sRc = BTREE_CORRUPTED; + } + }else{ + if(chCollate.lStartKey != START_KEY){ + T5LOG(T5ERROR) << "info in the header of the tmi is corrupted, and chCollate.lStartKey=" + << chCollate.lStartKey <<"; (expected " << START_KEY <<"); isCorrupted for this memory is set to true;"; + sRc = BTREE_CORRUPTED; + } + } + if ( !sRc ) { memcpy(DataRecList, header.DataRecList, MAX_LIST*sizeof(DataRecList[0])); @@ -5078,9 +4741,6 @@ SHORT BTREE::QDAMDictOpenLocal { QDAMTerseInit( chEntryEncode ); // init compression } /* endif */ - - memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); - memcpy( chCaseMap, header.chCaseMap, COLLATE_SIZE ); ASDLOG(); @@ -5096,7 +4756,7 @@ SHORT BTREE::QDAMDictOpenLocal } else { - T5LOG(T5ERROR) <<"Can't understand file " << fb.fileName << ". File has illegal structure!"; + T5LOG(T5ERROR) <<"Can't understand file " << fb.file.fileName << ". File has illegal structure!"; //fOpen = FALSE; sRc = BTREE_ILLEGAL_FILE; } /* endif */ @@ -5107,43 +4767,12 @@ SHORT BTREE::QDAMDictOpenLocal } /* endif */ ASDLOG(); // close in case of error - if ( sRc ) + if ( sRc && (sRc != BTREE_CORRUPTED)) { QDAMDictClose(); } /* endif */ ASDLOG(); - - if ( !sRc && /*fWrite &&*/ !header.fOpen) - { - //fOpen = TRUE; // set open flag - } /* endif */ - - ASDLOG(); } - else - { - sRc = QDAMDosRC2BtreeRC( sRc, BTREE_OPEN_ERROR, usOpenFlags ); - } /* endif */ - - /*******************************************************************/ - /* set BTREE pointer in case it was changed or freed */ - /*******************************************************************/ - //*ppBTIda = pBTIda; - - /*******************************************************************/ - /* add the dictionary to the open list ... */ - /*******************************************************************/ - if ( !sRc || (sRc == BTREE_CORRUPTED) ) - { - QDAMAddDict( (PSZ)fb.fileName.c_str(), this ); - } /* endif */ - - if ( sRc ) - { - T5LOG(T5ERROR) << ":: sRc" << sRc; - ERREVENT2( QDAMDICTOPENLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - //fOpen = true; return ( sRc ); } @@ -5329,20 +4958,18 @@ SHORT BTREE::QDAMDictOpenLocal //------------------------------------------------------------------------------ SHORT BTREE::QDAMDictInsertLocal ( - PWCHAR pKey, // pointer to key data - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data in bytes + LONG& lKey, // pointer to key data + BytesRef pData // pointer to user data ) { SHORT sRc = 0; // return code RECPARAM recData; // offset/node of data storage RECPARAM recKey; // offset/node of key storage - USHORT usKeyLen; // length of the key memset( &recKey, 0, sizeof( recKey ) ); memset( &recData, 0, sizeof( recData ) ); - - if ( ulLen == 0) + + if ( pData.empty()) { sRc = BTREE_DATA_RANGE; } @@ -5350,32 +4977,29 @@ SHORT BTREE::QDAMDictInsertLocal /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( !sRc && QDAMDictLockStatus( pKey ) ) + if ( !sRc && QDAMDictLockStatus( lKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ - PBTREEBUFFER_V3 pNode = NULL; + std::shared_ptr pNode; if ( !sRc ) - { - usKeyLen = (USHORT) sizeof(ULONG); - //memcpy( (PBYTE)chHeadTerm, (PBYTE)pKey, usKeyLen);//+sizeof(TMWCHAR) ); // save current data - + { QDAMDictUpdStatus (); - sRc = QDAMFindRecord_V3( pKey, &pNode ); + sRc = QDAMFindRecord_V3( lKey, pNode ); } /* endif */ if ( !sRc ) { BTREELOCKRECORD( pNode ); - sRc = QDAMAddToBuffer_V3( pData, ulLen, &recData, false ); + sRc = QDAMAddToBuffer_V3( pData, &recData, false); if ( !sRc ) { - if(ulLen > TMX_REC_SIZE && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << ulLen; + if(pData.size() > TMX_REC_SIZE && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ + T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } - recData.ulLen = ulLen; - sRc = QDAMInsertKey_V3 ( pNode, pKey, recKey, recData); + recData.ulLen = pData.size(); + sRc = QDAMInsertKey_V3 ( pNode, lKey, recKey, recData); } /* endif */ BTREEUNLOCKRECORD( pNode ); /****************************************************************/ @@ -5383,13 +5007,6 @@ SHORT BTREE::QDAMDictInsertLocal /****************************************************************/ lTime ++; } - - - if ( sRc ) - { - ERREVENT2( QDAMDICTINSERTLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - return sRc; } @@ -5455,7 +5072,7 @@ SHORT BTREE::QDAMDictUpdSignLocal // let 2K at beginning as space writingPosition = (LONG) USERDATA_START; - sRc = fb.SetOffset( writingPosition, FILE_BEGIN); + sRc = fb.SetOffset( writingPosition); ulDataLen = BTREE_REC_SIZE_V3 - USERDATA_START - sizeof(USHORT); ulDataLen = ulLen < ulDataLen? ulLen : ulDataLen; @@ -5505,7 +5122,7 @@ SHORT BTREE::QDAMDictUpdSignLocal } /* endif */ if ( ! sRc ) { - fb.SetOffset(BTREE_REC_SIZE_V3, FILE_BEGIN); + fb.SetOffset(BTREE_REC_SIZE_V3); } /* endif */ } @@ -5583,7 +5200,7 @@ SHORT BTREE::QDAMDictSignLocal //sRc = SkipBytesFromBeginningInFile(pBT->fb.file, USERDATA_START); sRc = fb.Read((PVOID) &usLen, sizeof(USHORT), USERDATA_START); if(sRc){ - T5LOG(T5ERROR) << "Can't read from file \'"<fb.file, pUserData, usLen, // &usNumBytesRead, FALSE ); - sRc = fb.Read(pUserData, usLen); + sRc = fb.Read(pUserData, usLen, USERDATA_START+sizeof(USHORT)); if ( !sRc ) { *pusLen = usLen; @@ -5722,118 +5339,6 @@ BTREE::QDAMCheckDict // endif // return Rc //------------------------------------------------------------------------------ -SHORT QDAMLocSubstr_V3 -( - PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord, - PCHAR_W pKey, - PBYTE pchBuffer, // space for key data - PULONG pulLength, // in/out length of returned key data - PBYTE pchUserData, // space for user data - PULONG pulUserLen // in/out length of returned user data -) -{ - SHORT i; // index - SHORT sNearKey; // nearest key - USHORT usLen; // length of key - SHORT sRc = 0; // return code - PCHAR_W pKey2; // key to be compared with - RECPARAM recData; - PBTREE pBT = pBTIda; - - sRc = pBTIda->QDAMLocateKey_V3( pRecord, pKey, &i, FSUBSTR, &sNearKey); - if ( !sRc ) - { - if ( i != -1 ) - { - sNearKey = i; - } /* endif */ - - // set new current position - pBTIda->sCurrentIndex = sNearKey; - pBTIda->usCurrentRecord = RECORDNUM( pRecord ); - - BTREELOCKRECORD( pRecord ); - // check if the key fulfills the substring option - if ( sNearKey != -1) - { - pKey2 = QDAMGetszKey_V3( pRecord, sNearKey, pBT->usVersion ); - if ( pKey2 ) - { - if ( UTF16strnicmp(pKey, pKey2, (USHORT)UTF16strlenCHAR(pKey)) ) - { - if ( sNearKey < (SHORT)OCCUPIED(pRecord)-1 ) - { - sNearKey++; - } /* endif */ - - pKey2 = QDAMGetszKey_V3( pRecord, sNearKey, pBT->usVersion ); - if ( pKey2 ) - { - if ( UTF16strnicmp(pKey, pKey2, (USHORT)UTF16strlenCHAR(pKey)) ) - { - LOG_AND_SET_RC(sRc, T5WARNING,BTREE_NOT_FOUND); - } - else - { - pBTIda->sCurrentIndex = sNearKey; - } /* endif */ - } - else - { - LOG_AND_SET_RC(sRc, T5WARNING,BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } - else - { - {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED)<< " pKey2==nulptr";} - } /* endif */ - - if ( !sRc ) - { - usLen = (USHORT)(UTF16strlenBYTE( pKey2 ) + sizeof(CHAR_W)); - if ( !pchBuffer || *pulLength == 0) - { - *pulLength = usLen ; // give back length only - } - else - { - if ( *pulLength >= usLen ) - { - *pulLength = usLen; - memcpy( pchBuffer, pKey2, usLen ); - } - else - { - //ERREVENT2( QDAMLOCSUBSTR_LOC, STATE_EVENT, 2, DB_GROUP, "" ); - T5LOG(T5ERROR) <<":: BufferTooSmall 4"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ - } /* endif */ - if ( !sRc ) - { - recData = QDAMGetrecData_V3( pRecord, i, pBT->usVersion ); - if ( *pulUserLen == 0 || ! pchUserData ) - { - *pulUserLen = recData.ulLen; - } - else - { - sRc = QDAMGetszData_V3( pBTIda, recData, pchUserData, pulUserLen, DATA_NODE ); - } /* endif */ - } /* endif */ - } /* endif */ - } - else - { - LOG_AND_SET_RC(sRc, T5WARNING,BTREE_NOT_FOUND); - } /* endif */ - BTREEUNLOCKRECORD( pRecord ); - } /* endif */ - - return sRc; -} //------------------------------------------------------------------------------ @@ -5884,10 +5389,63 @@ SHORT QDAMLocSubstr_V3 // return Rc // //------------------------------------------------------------------------------ +SHORT QDAMValidateIndex_V3 +( + PBTREE pBTIda, + std::shared_ptr& pRecord +) +{ + SHORT sRc = 0; // set return code + USHORT usRecord; + + if ( pBTIda->sCurrentIndex < 0 || + pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(pRecord)) + { + if ( pBTIda->sCurrentIndex < 0 ) + { + if ( PREV( pRecord )) + { + usRecord = PREV(pRecord); + sRc = pBTIda->QDAMReadRecord_V3( usRecord, pRecord, FALSE ); + if (!sRc ) + { + pBTIda->usCurrentRecord = usRecord; + pBTIda->sCurrentIndex = (SHORT) (OCCUPIED(pRecord) - 1); + } /* endif */ + } + else + { + pBTIda->usCurrentRecord = 0; + pBTIda->sCurrentIndex = 0; + sRc = BTREE_EOF_REACHED; + } /* endif */ + } + else if ( pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(pRecord)) + { + if ( NEXT(pRecord) ) + { + pBTIda->usCurrentRecord = NEXT(pRecord); + sRc = pBTIda->QDAMReadRecord_V3( pBTIda->usCurrentRecord, pRecord, FALSE ); + pBTIda->sCurrentIndex = 0; + } + else + { + pBTIda->usCurrentRecord = 0; + pBTIda->sCurrentIndex = 0; + sRc = BTREE_EOF_REACHED; + } /* endif */ + } /* endif */ + } /* endif */ + if ( sRc ) + { + ERREVENT2( QDAMVALIDATEINDEX_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); + } /* endif */ + return sRc; +} RECPARAM QDAMGetrecKey_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ); @@ -5944,8 +5502,6 @@ RECPARAM QDAMGetrecKey_V3 - - //------------------------------------------------------------------------------ // Internal function //------------------------------------------------------------------------------ @@ -5982,17 +5538,15 @@ SHORT QDAMGetszKeyParam_V3 ( PBTREE pBTIda, // pointer to btree structure RECPARAM recKey, // active record - PCHAR_W pKeyData, // pointer to data - PULONG pulLen // length of data + LONG& lKeyData // pointer to data ) { PCHAR pData = NULL; - PBTREEBUFFER_V3 pRecord; // active record + std::shared_ptr pRecord; // active record SHORT sRc = 0; // return code - PBTREE pBT = pBTIda; USHORT usLen; - sRc = pBTIda->QDAMReadRecord_V3( recKey.usNum, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recKey.usNum, pRecord, FALSE ); /*******************************************************************/ /* check length */ @@ -6004,16 +5558,17 @@ SHORT QDAMGetszKeyParam_V3 // as data is now in Unicode the length of the key may be up to // HEADTERM_SIZE *2 !!! - if ( (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) && ( *pulLen > usLen ) ) + if ( (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) )//&& ( *pulLen > usLen ) ) { - *pulLen = usLen; + //*pulLen = usLen; pData += sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data - memcpy( pKeyData, pData, *pulLen ); + memcpy( &lKeyData, pData, usLen ); } else { - {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED)<< " (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) && ( *pulLen > usLen ), usLen = "<< usLen <<"; *pulLen = "<< *pulLen;} + lKeyData = -1; + {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED)<< " (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) && ( *pulLen > usLen ), usLen = "<< usLen;}// <<"; *pulLen = "<< *pulLen;} } } /* endif */ @@ -6078,7 +5633,7 @@ SHORT QDAMFirst_V3 ) { SHORT sRc = 0; - PBTREEBUFFER_V3 pRecord; + std::shared_ptr pRecord; PBTREE pBT = pBTIda; memset( precBTree, 0, sizeof(RECPARAM)); @@ -6095,7 +5650,7 @@ SHORT QDAMFirst_V3 } else { - sRc = pBTIda->QDAMReadRecord_V3(pBTIda->usCurrentRecord, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3(pBTIda->usCurrentRecord, pRecord, FALSE ); if ( !sRc ) { /***********************************************************************/ @@ -6110,7 +5665,7 @@ SHORT QDAMFirst_V3 precBTree->usOffset = pBTIda->sCurrentIndex; precBTree->usNum = pBTIda->usCurrentRecord; *precKey = QDAMGetrecKey_V3( pRecord, 0 ); - *precData = QDAMGetrecData_V3( pRecord, 0, pBT->usVersion ); + *precData = QDAMGetrecData_V3( pRecord, 0 ); } } } @@ -6175,10 +5730,8 @@ SHORT QDAMFirst_V3 SHORT QDAMDictFirstLocal ( PBTREE pBTIda, - PCHAR_W pKeyData, // pointer to space for key data - PULONG pulKeyLen, // length of space for key data - PBYTE pUserData, // pointer to space for user data - PULONG pulUserLen // length of space for user data + LONG& lKeyData, // pointer to space for key data + BytesRef pUserData // pointer to space for user data ) { SHORT sRc = 0; // return code @@ -6186,7 +5739,7 @@ SHORT QDAMDictFirstLocal RECPARAM recKey; // ... for key value RECPARAM recData; // ... for user value PBTREE pBT = pBTIda; - ULONG ulKeyLen; + LONG lKeyLen; memset(&recData, 0, sizeof(recData)); memset(&recKey, 0, sizeof(recKey)); @@ -6197,8 +5750,6 @@ SHORT QDAMDictFirstLocal do { - ulKeyLen = *pulKeyLen; - if ( !sRc ) { sRc = QDAMFirst_V3( pBTIda, &recBTree,&recKey, &recData); @@ -6206,17 +5757,12 @@ SHORT QDAMDictFirstLocal if ( !sRc ) { - sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, pKeyData, &ulKeyLen ); + sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, lKeyData ); } - if ( !sRc && ulKeyLen ) - { - ULONG ul = 0l; - // memcpy( pBTIda->chHeadTerm, &ul, sizeof(ULONG) ); - } /* endif */ - + if ( !sRc ) { - sRc = QDAMGetszData_V3( pBTIda, recData, pUserData, pulUserLen, DATA_NODE ); + sRc = pBTIda->QDAMGetszData_V3(recData, pUserData ); } /* endif */ if ( (sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED) ) @@ -6227,7 +5773,6 @@ SHORT QDAMDictFirstLocal } while( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && (sRetries > 0)); - *pulKeyLen = ulKeyLen; } /* endif */ if ( sRc ) @@ -6252,63 +5797,31 @@ USHORT EQFNTMOrganizeIndex ) { SHORT sRc = 0; // function return code - //PBTREE pbTree = (PBTREE)(*ppBTIda); - PCHAR_W pchKeyBuffer = NULL; // buffer for record keys - ULONG ulKeyBufSize = 0; // current size of key buffer (number of characters) - PBYTE pbData = NULL; // buffer for record data - ULONG ulDataBufSize = 0; // current size of record data buffer (number of bytes) + std::vector pchKeyBuffer; // buffer for record keys + ULONG lKeyBufSize = 256; // current size of key buffer (number of characters) + std::vector pbData; // buffer for record data BOOL fNewIndexCreated = FALSE; // new-index-has-been-created flag CHAR szNewIndex[MAX_LONGPATH]; // buffer for new index name - PBTREE pBtreeOut = NULL; // structure for output BTREE USHORT usSigLen = 0; // length of signature record - ULONG ulKey; + LONG lKey = -1; // allocate buffer areas - ulKeyBufSize = 256; - if ( !UtlAlloc( (PVOID *)&pchKeyBuffer, 0, ulKeyBufSize*sizeof(CHAR_W) , ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - if ( !sRc ) - { - ulDataBufSize = MAX_INDEX_LEN * sizeof(LONG) * 4; - if ( !UtlAlloc( (PVOID *)&pbData, 0, ulDataBufSize, ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - if ( !sRc ) - { - if ( !UtlAlloc( (PVOID *)&pBtreeOut, 0, sizeof(BTREE), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + pbData.resize(MAX_INDEX_LEN * sizeof(LONG) * 4); + pchKeyBuffer.resize(lKeyBufSize*sizeof(CHAR_W)); // get user data (signature record) if ( !sRc ) { - //T5LOG( T5WARNING) << "TEMPORARY HARDCODED EQFNTMOrganizeIndex:: usSigLen = (USHORT)ulDataBufSize => usSigLen = (SHORT)ulDataBufSize"); - usSigLen = (USHORT)ulDataBufSize; - sRc = pbTree->QDAMDictSignLocal( (PCHAR)pbData, &usSigLen ); + usSigLen = (USHORT)pbData.size(); + sRc = pbTree->QDAMDictSignLocal( (PCHAR)pbData.data(), &usSigLen ); } /* endif */ // check if index is empty if ( !sRc ) { - ULONG ulDataLen = ulDataBufSize; - ULONG ulKeyLen = sizeof(ULONG) + 1; // ulKeyBufSize; - - sRc = QDAMDictFirstLocal( pbTree, (PCHAR_W)&ulKey, &ulKeyLen, pbData, &ulDataLen ); + sRc = QDAMDictFirstLocal( pbTree, lKey, pbData ); } /* endif */ - // cleanup - if ( pchKeyBuffer ) UtlAlloc( (PVOID *)&pchKeyBuffer, 0, 0, NOMSG ); - if ( pbData ) UtlAlloc( (PVOID *)&pbData, 0, 0, NOMSG ); - if ( pBtreeOut ) UtlAlloc( (PVOID *)pBtreeOut, 0, 0, NOMSG ); - // re-map some return codes.. switch ( sRc ) { diff --git a/source/opentm2/core/utilities/EQFHASH.cpp b/source/opentm2/core/utilities/EQFHASH.cpp index 106cf148..6ba7ceb3 100755 --- a/source/opentm2/core/utilities/EQFHASH.cpp +++ b/source/opentm2/core/utilities/EQFHASH.cpp @@ -565,26 +565,7 @@ PPOOL PoolCreate //| add string to pool using PoolAddData | //| return pointer to added string | //+----------------------------------------------------------------------------+ -PSZ_W PoolAddStringW -( - PPOOL pPool, // pointer to a pool created using - // PoolCreate - PSZ_W pszString // string which should be stored in the - // pool -) -{ - LONG lStringLen; // length of string - PSZ_W pszStringInPool; // ptr to location of string in pool - - lStringLen = UTF16strlenCHAR( pszString) + 1;// get length of string + delimiter - - pszStringInPool = (PSZ_W)PoolAddData( pPool, - (lStringLen * sizeof(CHAR_W)), - pszString ); - return( pszStringInPool ); - -} /* end of function PoolAddStringW */ //+----------------------------------------------------------------------------+ //|External function | @@ -731,18 +712,14 @@ HATOMTBL APIENTRY WinCreateAtomTable USHORT cBuckets ) { - BOOL fOK = TRUE; // internal O.K. flag HATOMTBL pAtomTable = NULL; // ptr to new atom table - cbInitial; /********************************************************************/ /* Allocate atom table control area */ /********************************************************************/ - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *)&pAtomTable, 0L, + + BOOL fOK = UtlAlloc( (PVOID *)&pAtomTable, 0L, get_max( MIN_ALLOC, (LONG)sizeof(ATOMTBL)), ERROR_STORAGE ); - } /* endif */ /********************************************************************/ /* Create low level hash */ @@ -857,8 +834,6 @@ ULONG AtomHashKeyValue ULONG hash1 = 0; ULONG hash2 = 0; - ulSize; // avoid compiler warning - while ( *pbTerm ) { hash1 = random1[hash1 ^ *pbTerm ]; @@ -901,8 +876,6 @@ LONG AtomHashCompare PVOID pUserPtr ) { - pUserPtr; // avoid compiler warning - return strcmp( pEntry1->szAtomName, pEntry2->szAtomName ); } /* end of function AtomHashCompare */ @@ -932,9 +905,7 @@ LONG AtomHashCompare //+----------------------------------------------------------------------------+ SHORT AtomHashFree( PATOMENTRY pEntry, PVOID pUserPtr ) { - pUserPtr; UtlAlloc( (PVOID *) &pEntry, 0L, 0L, NOMSG ); - return( TRUE ); } /* end of function AtomHashFree */ diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 6fa014d9..67c96398 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -72,12 +72,8 @@ USHORT TokenizeW ( PVOID pvLangCB, // IN : ptr to language control block PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - TERMLENOFFS**ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer + // MORPH_OFFSLIST ); void MorphCheckSpellInstance( PLANGCB pLangCB ); USHORT MorphWStrings2TermList( vector &vStrings, PVOID *ppTermList, PULONG pulTermListSize ); @@ -140,14 +136,6 @@ static MORPHLANGTYPE LangTypes[] = #define min( a, b ) ( a < b ) ? (a) : (b) #endif -#if defined(MEASURETIME) - #define INITTIME( pLCB ) pLCB->ulLastTime = pLCB->pInfoSeg->msecs - #define GETTIME( pLCB, ulTime ) \ - { \ - ulTime += pLCB->pInfoSeg->msecs - pLCB->ulLastTime;\ - pLCB->ulLastTime = pLCB->pInfoSeg->msecs; \ - } -#endif /**********************************************************************/ /* indicator for DBCS initialisation */ @@ -246,20 +234,12 @@ USHORT TokenizeW ( PVOID pvLangCB, // IN : ptr to language control block PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - TERMLENOFFS** ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer + // MORPH_OFFSLIST ) { USHORT usReturn = 0; // return code ULONG ulTermBufUsed = 0; // amount of space used in term buffer - BOOL fOffsList; // TRUE = return a offset/length list - - fOffsList = (usListType == MORPH_OFFSLIST) || - (usListType == MORPH_FLAG_OFFSLIST); /********************************************************************/ /* Work on input data */ @@ -309,15 +289,6 @@ USHORT TokenizeW std::string ps_In = EncodingHelper::convertToUTF8(pszInData); for (size_t j = 0; j < vSentenceList.size(); j++) { - if (!usReturn) - { - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, pulTermListSize, &ulTermBufUsed, L" ", 1, 0, TF_NEWSENTENCE, usListType ); - } /* endif */ - } /* endif */ - // split segment/sentences into words OtmMorph::TERMLIST vTermList; PSZ_W pszStartOfSegment = pszInData + vSentenceList[j].iStartOffset; @@ -378,10 +349,8 @@ USHORT TokenizeW } else { - USHORT usPos = (USHORT)(vSentenceList[j].iStartOffset + iStartOffs); - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, pulTermListSize, &ulTermBufUsed, - pszStartOfSegment + iStartOffs, - (USHORT)iLen, usPos, vTermList[i].iTermType, usListType ); + int iPos = vSentenceList[j].iStartOffset + iStartOffs; + usReturn = MorphAddTermToList2W( pTermList, iLen, iPos ); } } } @@ -392,7 +361,7 @@ USHORT TokenizeW /*****************************************************************/ if ( !usReturn ) { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, pulTermListSize, &ulTermBufUsed, L" ", 0, 0, 0L, usListType ); + usReturn = MorphAddTermToList2W( pTermList, 0, 0); } /* endif */ return (usReturn); @@ -500,8 +469,6 @@ USHORT BuildDict { USHORT usRC = MORPH_OK; - usListType; usTermListSize; - if (ADDENDA_DICT != usDictType && ABBREV_DICT != usDictType) { return MORPH_NOT_FOUND; @@ -655,40 +622,17 @@ USHORT BuildDict //------------------------------------------------------------------------------ USHORT MorphTokenizeW ( - SHORT sLanguageID, // language ID - PSZ_W pszInData, // pointer to input segment - PUSHORT pusBufferSize, // address of variable containing size of - // term list buffer - TERMLENOFFS** ppTermList, // address of caller's term list pointer - USHORT usListType, // type of term list MORPH_ZTERMLIST or - // MORPH_OFFSLIST - ULONG ulOemCP // CP of language of sLangID!! + SHORT sLanguageID, // language ID + PSZ_W pszInData, // pointer to input segment + std::vector& pTermList // address of caller's term list pointer ) { USHORT usRC = MORPH_OK; // function return code PLCB pLCB = NULL; // pointer to language control block - ULONG ulTermBufUsed = 0; // number of bytes used in term buffer - ULONG ulBufferSize = *pusBufferSize; -#ifdef MEASURE_TOKENIZE_TIME - static int iMeasureCount = 0; - static _int64 iTotalTime = 0; - LARGE_INTEGER iStartTime; - LARGE_INTEGER iEndTime; -#endif - - -#ifdef MEASURE_TOKENIZE_TIME - QueryPerformanceCounter( &iStartTime ); -#endif - - ulOemCP; /********************************************************************/ /* Check input data */ /********************************************************************/ - if ( (pszInData == NULL) || - (pusBufferSize == NULL) || - (ppTermList == NULL) || - ((*ppTermList == NULL) && (*pusBufferSize != 0) ) ) + if (pszInData == NULL) { LOG_AND_SET_RC(usRC, T5INFO, MORPH_INV_PARMS); } /* endif */ @@ -716,84 +660,15 @@ USHORT MorphTokenizeW { if ( *pszInData != EOS ) { -#ifdef LOG_TOKENIZE - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(pszInData); - T5LOG( T5DEVELOP) << "::Tokenizing \"" << str << "\"" ; - } -#endif - - usRC = TokenizeW( pLCB->pvLangCB, pszInData, &ulBufferSize, ppTermList, usListType ); - -#ifdef LOG_TOKENIZE - - if ( usRC == MORPH_OK ) - { - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - if ( usListType == MORPH_FLAG_OFFSLIST ) - { - PINT psData = (PINT)*ppTermList; - INT sStart, sLen; - LONG lFlag; - do - { - lFlag = *(PLONG)psData; - psData += 2; - sLen = *psData++; - sStart = *psData++; - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << ":: Start=" << sStart << " Len=" << sLen << " Flag=" << lFlag; - } - } while ( (sStart != 0) || (sLen != 0) || (lFlag != 0) ); - } - else if ( usListType == MORPH_OFFSLIST ) - { - //PINT psData = (PINT)*ppTermList; - //INT sStart, sLen; - PUSHORT psData = (PUSHORT)*ppTermList; - USHORT sStart, sLen; - do - { - sLen = *psData++; - sStart = *psData++; - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << "::Start=" << sStart << ", Len=" << sLen; - } - } while ( (sStart != 0) || (sLen != 0) ); - } /* endif */ - } - } - else - { - T5LOG(T5ERROR) << ":: TokenizeW failed with rc=" << usRC ; - } /* endif */ - -#endif - + usRC = TokenizeW( pLCB->pvLangCB, pszInData, pTermList ); } else { - usRC = MorphAddTermToList2W( (PSZ_W *)ppTermList, &ulBufferSize, &ulTermBufUsed, L" ", 0, 0, 0L, usListType ); + usRC = MorphAddTermToList2W( pTermList, 0, 0 ); } /* endif */ } /* endif */ - *pusBufferSize = (USHORT)ulBufferSize; - -#ifdef MEASURE_TOKENIZE_TIME - QueryPerformanceCounter( &iEndTime ); - - iTotalTime += (iEndTime.QuadPart - iStartTime.QuadPart); - iMeasureCount++; - if ( iMeasureCount == 100000 ) - { - char szMsg[60]; - sprintf( szMsg, "Time for %ld MorphTokenizeWCalls is %I64d ticks", iMeasureCount, iTotalTime ); - MessageBox( HWND_DESKTOP, szMsg, "Info", MB_OK ); - } -#endif - return( usRC ); - } /* endof MorphTokenizeW */ //------------------------------------------------------------------------------ @@ -1111,242 +986,36 @@ USHORT MorphGetLCB } /* end of function MorphGetLCB */ - USHORT MorphAddTermToList2W ( - PSZ_W *ppList, // ptr to term list pointer - PULONG pulSize, // alloc. size of term list in # of w's - PULONG pulUsed, // used # of w's in term list + std::vector& pList, // ptr to term list pointer PSZ_W pszTerm, // ptr to new term being added to list - USHORT usLength, // length of term in # of w's - USHORT usOffs, // offset of term in # of w's??? - ULONG ulFlags, // flags describing the term - USHORT usListType // type of list + USHORT usLength, // length of term + USHORT usOffs, // offset of term + ULONG ulFlags // flags for the ter ) { - USHORT usRC = MORPH_OK; // function return code - ULONG ulDataLenBytes = 0; // length of data to add in # of bytes - ULONG ulDataLen = 0; // length of data to add in w's - BOOL fOK = TRUE; // internal OK flag - - PBYTE pList = (PBYTE)*ppList; // ptr to term list - PSZ_W *pAnchor = NULL; // ptr to anchor for list - - /********************************************************************/ - /* position to last block for large term lists */ - /********************************************************************/ - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - PSZ_W pNextList; - - // use caller's list pointer as default anchor - pAnchor = ppList; - - if ( pList != NULL ) - { - pNextList = *((PSZ_W *)pList); + FLAGOFFSLIST data; + data.lFlags = ulFlags; + data.usLen = usLength; + data.usOffs = usOffs; + pList.emplace_back(data); + return 0; +} - while ( pNextList != NULL ) - { - pAnchor = (PSZ_W *)pList; - pList = (PBYTE)pNextList; - pNextList = *((PSZ_W *)pList); - } /* endwhile */ - } /* endif */ - } /* endif */ - /********************************************************************/ - /* get required size for new entry */ - /********************************************************************/ - switch ( usListType ) - { - case MORPH_OFFSLIST : - /******************************************************************/ - /* space for offset and length of term plus end of term list */ - /* indicator which is a zero offset and length value */ - /******************************************************************/ - //ulDataLenBytes = 4 * sizeof(CHAR_W); - ulDataLenBytes = 4 * sizeof(USHORT); - //ulDataLenBytes = 4 * sizeof(int); - break; - - case MORPH_ZTERMLIST : - /******************************************************************/ - /* space for term, term delimiter and end of term list indicator */ - /******************************************************************/ - ulDataLenBytes = (usLength + 1 + 1)*sizeof(CHAR_W); - break; - - case MORPH_FLAG_OFFSLIST : - /******************************************************************/ - /* space for flag, offset and length of term plus end of term list*/ - /* indicator which is a zero flag, offset and length value */ - /******************************************************************/ - ulDataLenBytes = (2 * sizeof(ULONG)) + (4 * sizeof(CHAR_W)); - break; - - case MORPH_FLAG_ZTERMLIST : - /******************************************************************/ - /* space for flag, term, term delimiter and end of term list */ - /* indicator which is a empty flag and a term delimiter */ - /******************************************************************/ - ulDataLenBytes = sizeof(ULONG) + (usLength + 1)*sizeof(CHAR_W) - + sizeof(ULONG) + sizeof(CHAR_W); - break; - - case MORPH_LARGE_ZTERMLIST : - /******************************************************************/ - /* space for term, term delimiter and end of term list indicator */ - /******************************************************************/ - ulDataLenBytes = (usLength + 1 + 1)*sizeof(CHAR_W); - break; - - default : - fOK = FALSE; - break; - } /* endswitch */ - - /********************************************************************/ - /* enlarge term list if necessary */ - /********************************************************************/ - ulDataLen = ulDataLenBytes / sizeof(CHAR_W); // # of w's needed - if ( fOK && ((*pulUsed + ulDataLen) > *pulSize) ) - { - LONG lOldLen = 0L; - LONG max = TERMLIST_INCR > ulDataLen? TERMLIST_INCR : ulDataLen ; - LONG lNewLen = ((LONG)*pulSize) + max ; - - LONG lMaxLen = (usListType == MORPH_LARGE_ZTERMLIST) ? - MORPH_LARGE_ZTERMLIST_LEN : MAX_ALLOC; - lNewLen = lNewLen * sizeof(CHAR_W); // len in # of bytes - lOldLen = (*pulSize) * sizeof(CHAR_W); - if ( lNewLen >= lMaxLen ) - { - // for large termlists allocate another buffer else set error condition - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - lNewLen = (LONG)sizeof(PSZ_W) + - (LONG)sizeof(CHAR_W) * max; - *pulSize = 0; // no size yet - lOldLen = 0L; - pAnchor = (PSZ_W *)pList; // use current list as anchor - } - } /* endif */ - - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *)&pList, lOldLen, lNewLen, NOMSG ); - if ( fOK ) - { // set caller's termlist pointer - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - *pAnchor = (PSZ_W)pList; - if (*pulSize == 0 ) // new block??? - { - *pulUsed = sizeof(PSZ_W)/ sizeof(CHAR_W); // leave room for pointer to next block - } /* endif */ - *pulSize = lNewLen / sizeof(CHAR_W); - } - else - { - *pulSize = lNewLen / sizeof(CHAR_W); //@@+= max( TERMLIST_INCR, ulDataLength );?? - *ppList = (PSZ_W)pList; - } /* endif */ - } - else - { - LOG_AND_SET_RC(usRC, T5INFO, MORPH_NO_MEMORY); - - // free the term list in case of erros - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - pList = (PBYTE)*ppList; - while ( pList != NULL ) - { - PSZ_W pNextList = *((PSZ_W *)pList); - UtlAlloc( (PVOID *)&pList, 0L, 0L, NOMSG ); - pList = (PBYTE)pNextList; - } /* endwhile */ - } - else - { - UtlAlloc( (PVOID *)ppList, 0L, 0L, NOMSG ); - } /* endif */ - *ppList = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* add term to term list (pList is a PBYTE ptr */ - /********************************************************************/ - if ( fOK ) - { - ULONG ulByteUsedInList = (*pulUsed) * sizeof(CHAR_W); - switch ( usListType ) - { - case MORPH_OFFSLIST : - *((PUSHORT)(pList + ulByteUsedInList)) = usLength; - ulByteUsedInList += sizeof(USHORT); - - *((PUSHORT)(pList + ulByteUsedInList)) = usOffs; - *pulUsed += 1; - ulByteUsedInList += sizeof(USHORT); - break; - - case MORPH_LARGE_ZTERMLIST : - case MORPH_ZTERMLIST : - if ( usLength ) - { - memcpy( (PBYTE)(pList + ulByteUsedInList), - (PBYTE) pszTerm, usLength*sizeof(CHAR_W) ); - *pulUsed += usLength; - ulByteUsedInList += usLength * sizeof(CHAR_W); - } /* endif */ - *((PSZ_W)(pList + ((*pulUsed) * sizeof(CHAR_W)))) = EOS; /* 10-11-16 */ - *pulUsed += 1; - ulByteUsedInList += sizeof(CHAR_W); - break; - - case MORPH_FLAG_OFFSLIST : - *((PULONG)(pList + ulByteUsedInList)) = ulFlags; - *pulUsed += sizeof(ULONG)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(ULONG); - - *((PUSHORT)(pList + ulByteUsedInList)) = usLength; - *pulUsed += 1;//sizeof(USHORT)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(USHORT); - - *((PUSHORT)(pList + ulByteUsedInList)) = usOffs; - *pulUsed += 1;//sizeof(USHORT) / sizeof(CHAR_W); - ulByteUsedInList += sizeof(USHORT); - break; - - case MORPH_FLAG_ZTERMLIST : - *((PULONG)(pList + ulByteUsedInList)) = ulFlags; - *pulUsed += sizeof(ULONG)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(ULONG); - - if ( usLength ) - { - memcpy( pList + ulByteUsedInList, - (PBYTE)pszTerm, usLength*sizeof(CHAR_W) ); - *pulUsed += usLength; - ulByteUsedInList += usLength * sizeof(CHAR_W); - } /* endif */ - *((PSZ_W)(pList + ulByteUsedInList)) = EOS; /* 10-11-16 */ - *pulUsed += 1; - break; - - default : - break; - } /* endswitch */ - } /* endif */ - - /********************************************************************/ - /* return usRC to calling function */ - /********************************************************************/ - return( usRC ); +USHORT MorphAddTermToList2W +( + std::vector& pList, // ptr to term list pointer + int iLength, // length of term in # of w's + int iOffs // offset of term in # of w's??? +) +{ + TERMLENOFFS data; + data.iLength = iLength; + data.iOffset = iOffs; + pList.emplace_back(data); + return( 0 ); } /* end of function MorphAddTermToList2W */ @@ -1600,101 +1269,6 @@ BOOL MorphHashDestroy // add new element to low level hash // update recently used chain //------------------------------------------------------------------------------ -BOOL MorphHashAdd -( - PMORPHHASH pMorphHash, // pointer to our hash control strucuture - PSZ_W pszTerm, // name of term being added - ULONG ulLemmaListLen, // length of lemma list in bytes - PSZ_W pszLemmaList // lemma list for term or NULL -) -{ - HASHENTRY HashEntry; // buffer for new hash entry - PHASHENTRY pHashEntry; // pointer to found/created hash entry - ULONG ulEntry; // number (index) of new entry - - /********************************************************************/ - /* Check if term is already in cash */ - /********************************************************************/ - HashEntry.pszTerm = pszTerm; - pHashEntry = (PHASHENTRY) HashSearch( pMorphHash->pHash, &HashEntry ); - - /********************************************************************/ - /* Remove oldest entry from cache if all cache elements are in use */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - if ( pMorphHash->ulElements == pMorphHash->ulMaxElements ) - { - /****************************************************************/ - /* Cache is full: remove oldest element from cache */ - /****************************************************************/ - MorphHashDelete( pMorphHash, pMorphHash->pElements + pMorphHash->ulFirstUsed ); - pHashEntry = NULL; - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* Add term and lemma list to string pool */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - HashEntry.pszTerm = PoolAddStringW( pMorphHash->StringPool, pszTerm ); - HashEntry.ulListLength = ulLemmaListLen; - if ( pszLemmaList ) - { - HashEntry.pLemmaList = (PSZ_W) PoolAddData( pMorphHash->StringPool, - (ulLemmaListLen * sizeof(CHAR_W)), - pszLemmaList ); - } - else - { - HashEntry.pLemmaList = NULL; - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* Add new element to low level hash */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - /******************************************************************/ - /* Get entry from free chain */ - /******************************************************************/ - ulEntry = pMorphHash->ulFirstFree; - pHashEntry = pMorphHash->pElements + ulEntry; - pMorphHash->ulFirstFree = pHashEntry->ulNext; - pMorphHash->ulElements++; - - /******************************************************************/ - /* Fill-in element data */ - /******************************************************************/ - pHashEntry->pszTerm = HashEntry.pszTerm; - pHashEntry->ulListLength = HashEntry.ulListLength; - pHashEntry->pLemmaList = HashEntry.pLemmaList; - - /******************************************************************/ - /* add element to end of recently used chain */ - /******************************************************************/ - pHashEntry->ulNext = END_OF_ELEMENTS; - pHashEntry->ulPrev = pMorphHash->ulLastUsed; - if ( pMorphHash->ulLastUsed != END_OF_ELEMENTS ) - { - pMorphHash->pElements[pMorphHash->ulLastUsed].ulNext = pHashEntry->ulKey; - } /* endif */ - pMorphHash->ulLastUsed = ulEntry; - if ( pMorphHash->ulFirstUsed == END_OF_ELEMENTS ) - { - pMorphHash->ulFirstUsed = ulEntry; - } /* endif */ - - /******************************************************************/ - /* Add element to low level hash */ - /******************************************************************/ - HashAdd( pMorphHash->pHash, pHashEntry ); - } /* endif */ - - return( TRUE ); -} /* end of function MorphHashAdd */ //------------------------------------------------------------------------------ // Internal function @@ -1725,26 +1299,6 @@ BOOL MorphHashAdd //------------------------------------------------------------------------------ // Function flow: call low level hash delete function //------------------------------------------------------------------------------ -BOOL MorphHashDelete -( - PMORPHHASH pMorphHash, // pointer to our hash control strucuture - PHASHENTRY pEntry // pointer to the entry being sdeleted -) -{ - - // delete the hash entry, the callback function MorphHassFree will be called (which adjusts the recently used chain and the free entries chain) - if ( HashDelete( pMorphHash->pHash, pEntry ) ) - { - return( TRUE ); // nothing to do any more, MorphHashFree has done the house keeping - } /* endif */ - - // this part of the function is called when HashDelete failed to delete the entry, - // we will have to remove the entry from the MorphHash table ... - MorphHashFree( pEntry, (PVOID)pMorphHash ); - - return( TRUE ); -} /* end of function MorphHashDelete */ - //------------------------------------------------------------------------------ // Internal function @@ -1782,7 +1336,7 @@ SHORT MorphHashCompare { pUserPtr; // avoid compiler warning - return (SHORT)(UTF16strcmp( pEntry1->pszTerm, pEntry2->pszTerm )); + return (SHORT)(wcscmp( pEntry1->pszTerm, pEntry2->pszTerm )); } /* end of function MorphHashCompare */ //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/utilities/EQFTADIT.cpp b/source/opentm2/core/utilities/EQFTADIT.cpp deleted file mode 100755 index 14ec7f8d..00000000 --- a/source/opentm2/core/utilities/EQFTADIT.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#ifdef TO_BE_REMOVED -//+----------------------------------------------------------------------------+ -//| EQFTADIT.CPP | -//+----------------------------------------------------------------------------+ -//|Copyright Notice: | -//| | -//| Copyright (C) 1990-2012, International Business Machines | -//| Corporation and others. All rights reserved | -//+----------------------------------------------------------------------------+ -//| Author: Gerhard Queck | -//| | -//+----------------------------------------------------------------------------+ -//| Description: SGML-DITA special processing functions | -//| | -//| | -//+----------------------------------------------------------------------------+ - -//#include -#include -#include -#include -#include - -// the Win32 Xerces build requires the default structure packing... -//#pragma pack( push ) -//#pragma pack(8) - -#include -#include -#include -#include -#include -#include -#include - -//#pragma pack( pop ) - - -XERCES_CPP_NAMESPACE_USE - - -//#pragma pack( push, TM2StructPacking, 1 ) - -#define INCL_EQF_TP // public translation processor functions -#define INCL_EQF_EDITORAPI // editor API -#define INCL_EQF_TAGTABLE // tagtable defines and functions -#define INCL_EQF_ANALYSIS // analysis functions -#define INCL_EQF_TM // general Transl. Memory functions -#define INCL_EQF_TMREMOTE // remote Transl. Memory functions (U-Code) -#define INCL_EQF_DLGUTILS // dialog utilities -#define INCL_EQF_MORPH -#define INCL_EQF_DAM -#include "EQF.H" // General .H for EQF -#include "EQFMEMIE.H" - -//#pragma pack( pop, TM2StructPacking ) - - -#endif - diff --git a/source/opentm2/core/utilities/EQFUTDOS.cpp b/source/opentm2/core/utilities/EQFUTDOS.cpp index 1401c5e9..76991c04 100755 --- a/source/opentm2/core/utilities/EQFUTDOS.cpp +++ b/source/opentm2/core/utilities/EQFUTDOS.cpp @@ -740,50 +740,6 @@ USHORT UtlCloseHwnd { FilesystemHelper::CloseFile( hf ) ; return 0; - #ifdef TEMPORARY_COMMENTED - USHORT usRetCode = 0; // function return code - USHORT usMBCode = 0; // message box/UtlError return code - - /*******************************************************************/ - /* close only if file has been opened */ - /*******************************************************************/ - if ( hf != NULL ) - { - // remove handle from our handle/drive array - int i = 0; - while ( i < MAX_OPEN_FILES ) - { - if ( DriveHandles[i].hf == hf ) - { - // mark as empty and leave loop - DriveHandles[i].chDrive = BLANK; - DriveHandles[i].hf = NULLHANDLE; - break; - } - else - { - i++; - } /* endif */ - } /* endwhile */ - - do { - DosError(0); - - if ( FilesystemHelper::CloseFile( hf ) == 0 ) - { - //usRetCode = (USHORT)GetLastError(); - } /* endif */ - - DosError(1); - if ( fMsg && usRetCode ) - { - usMBCode = MB_CANCEL; - T5LOG(T5ERROR) << ":: rc = " << usRetCode; - } /* endif */ - } while ( fMsg && usRetCode && (usMBCode == MBID_RETRY) ); /* enddo */ - } /* endif */ - return( usRetCode ); - #endif } //+----------------------------------------------------------------------------+ @@ -2316,34 +2272,8 @@ T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 59 FreeModule( hMod ); //| endif | //| while fMsg and an error occured and MB code is RETRY | //+----------------------------------------------------------------------------+ -USHORT UtlQPathInfo -( - PSZ pszPath, // Path string - USHORT usInfoLevel, // level of file information required - PBYTE pInfoBuf, // storage area for returned information - USHORT cbInfoBuf, // length of storage area - ULONG ulReserved, // Reserved (must be zero) - BOOL fMsg // if TRUE handle errors in utility -) -{ - return( UtlQPathInfoHwnd( pszPath, usInfoLevel, pInfoBuf, cbInfoBuf, - ulReserved, fMsg, (HWND) NULL ) ); -} -USHORT UtlQPathInfoHwnd -( - PSZ pszPath, // Path string - USHORT usInfoLevel, // level of file information required - PBYTE pInfoBuf, // storage area for returned information - USHORT cbInfoBuf, // length of storage area - ULONG ulReserved, // Reserved (must be zero) - BOOL fMsg, // if TRUE handle errors in utility - HWND hwndParent -) -{ - T5LOG(T5ERROR) << ":: called commented function"; - return( 0 ); -} + //+----------------------------------------------------------------------------+ //|External function | @@ -2378,32 +2308,7 @@ VOID UtlWait /**********************************************************************/ /* Find First File with long file name support */ /**********************************************************************/ -USHORT UtlFindFirstLong -( - PSZ pszFSpec, // path name of files to be found - PHDIR phdir, // directory handle - USHORT usAttr, // attribute used to search for the files - PLONGFILEFIND pffb, // result buffer - BOOL fMsg // if TRUE handle errors in utility -) -{ - return( UtlFindFirstLongHwnd( pszFSpec, phdir, usAttr, pffb, fMsg, (HWND)NULL ) ); -} -USHORT UtlFindFirstLongHwnd -( - PSZ pszFSpec, // path name of files to be found - PHDIR phdir, // directory handle - USHORT usAttr, // attribute used to search for the files - PLONGFILEFIND pffb, // result buffer - BOOL fMsg, // if TRUE handle errors in utility - HWND hwndParent -) -{ - memset( pffb, 0, sizeof(LONGFILEFIND) ); - LOG_UNIMPLEMENTED_FUNCTION; - return( 0 ); -} /**********************************************************************/ /* Get the drive letter /2 DosCopy as this function fails when copying */ @@ -2498,10 +2403,8 @@ USHORT UtlGetFileSizeHwnd { USHORT usRC = NO_ERROR; // function return code DWORD dwSize; - - fMsg; hwndParent; DosError(0); - dwSize = GetFileSize( hf ); + dwSize = FilesystemHelper::GetFileSize( hf ); DosError(1); if ( dwSize == -1 ) { diff --git a/source/opentm2/core/utilities/EQFUTERR.cpp b/source/opentm2/core/utilities/EQFUTERR.cpp index 4395040e..c46eff0c 100755 --- a/source/opentm2/core/utilities/EQFUTERR.cpp +++ b/source/opentm2/core/utilities/EQFUTERR.cpp @@ -73,9 +73,6 @@ CHAR INTERNAL_ERROR_TXT [30]; // "INTERNAL ERROR" /* Protoypes for internal functions */ /**********************************************************************/ SHORT UtlQdamMsgTxt ( SHORT ); -static SHORT UtlDosMsgTxt ( SHORT, PSZ, PUSHORT ); -static SHORT UtlQldbMsgTxt ( SHORT ); -static SHORT UtlPropMsgTxt ( SHORT ); USHORT UtlErrorHwnd @@ -136,99 +133,8 @@ USHORT UtlErrorHwnd //------------------------------------------------------------------------------ // Function flow: Call UtlErrorHwnd with a NULL window handle //------------------------------------------------------------------------------ -USHORT UtlErrorHwndW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - HWND hwnd, // window handle - BOOL fUnicode // parameter table contains unicode strings -); -USHORT UtlErrorW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - BOOL fUnicode // parameter table contains unicode strings -) -{ - return UtlErrorHwndW( sErrorNumber, // number of message - usMsgType, // type of message - usNoOfParms, // number of message parameters - pParmTableW, // pointer to parameter table - ErrorType, // type of error - NULLHANDLE, // window handle - fUnicode ); // parameter table contains unicode strings -} -USHORT UtlErrorHwndW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - HWND hwnd, // window handle - BOOL fUnicode // parameter table contains unicode strings -) -{ - USHORT usRc = 0; - PSZ pParmTable[5]; - int i = 0; - - - if (!fUnicode) - { - for ( i=0; i= 128) - { - CHAR_W chW = pParmTableW[i][128]; - pParmTableW[i][128] = EOS; - UtlDirectUnicode2Ansi( pParmTableW[i], pParmTable[i], 0L ); - pParmTableW[i][128] = chW; - } - else - { - UtlDirectUnicode2Ansi( pParmTableW[i], pParmTable[i], 0L ); - } - } - - usRc = UtlErrorHwnd( sErrorNumber, - usMsgType, - usNoOfParms, - pParmTable, - ErrorType, - hwnd ); - UtlAlloc((PVOID *) &pData, 0L, 0L, NOMSG ); - } - - } - return( usRc ); -} + USHORT UtlError ( diff --git a/source/opentm2/core/utilities/EQFUTFIL.cpp b/source/opentm2/core/utilities/EQFUTFIL.cpp index 28ef7789..0a38bd80 100755 --- a/source/opentm2/core/utilities/EQFUTFIL.cpp +++ b/source/opentm2/core/utilities/EQFUTFIL.cpp @@ -50,22 +50,6 @@ static USHORT usLastDosRc; // buffer for last DOS return code //| return to drive letter converted return code | //+----------------------------------------------------------------------------+ -BYTE UtlQCurDisk() -{ - static CHAR szCurDir[MAX_PATH+100]; - - DosError(0); -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 65 if ( GetCurrentDirectory( sizeof(szCurDir), szCurDir ) == 0 ) { szCurDir[0] = EOS; }"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( GetCurrentDirectory( sizeof(szCurDir), szCurDir ) == 0 ) - { - szCurDir[0] = EOS; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - DosError(1); - - return (BYTE)szCurDir[0]; -} //+----------------------------------------------------------------------------+ //|External Function | @@ -183,7 +167,7 @@ USHORT UtlGetLANDriveList( PBYTE pszList ) //| return ok flag | //+----------------------------------------------------------------------------+ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file USHORT *pusBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -199,7 +183,7 @@ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded } BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -212,7 +196,7 @@ BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded BOOL UtlLoadFileHwnd ( PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -260,15 +244,9 @@ BOOL UtlLoadFileHwnd // If previous call OK allocate storage if required if ( !usDosRc && (ulSize > 0L) ) { - if ( *ppLoadedFile == NULL ) - { - usMessage = ( fMsg ) ? ERROR_STORAGE : NOMSG; - fOK = UtlAllocHwnd( (PVOID *)ppLoadedFile, // Allocate storage for input data - 0L, - get_max( (LONG) MIN_ALLOC, ulSize ), - usMessage, hwnd ); - fStorage = TRUE; - } /* endif */ + usMessage = ( fMsg ) ? ERROR_STORAGE : NOMSG; + ppLoadedFile.resize( get_max( (LONG) MIN_ALLOC, ulSize )); + fStorage = TRUE; } else { @@ -280,7 +258,7 @@ BOOL UtlLoadFileHwnd // If fOK. Load the input data in storage and close it if load was OK if (fOK) { - usDosRc = UtlReadL( hInputfile , *ppLoadedFile, + usDosRc = UtlReadL( hInputfile , ppLoadedFile.data(), ulSize, pulBytesRead, FALSE ); fOK = (usDosRc == 0 ); @@ -300,7 +278,7 @@ BOOL UtlLoadFileHwnd // Free storage allocation if allocated and something went wrong if ( !fOK && fStorage ) { - UtlAlloc( (PVOID *)ppLoadedFile, 0L, 0L, NOMSG ); + ppLoadedFile.clear(); } /* endif */ if ( !fOK ) // display error message if not ok @@ -615,86 +593,6 @@ USHORT UtlWriteFile /*@86C*/ //| endif | //| return isEqfDrive flag | //+----------------------------------------------------------------------------+ -BOOL UtlIsEqfDrive -( - CHAR chDrive // drive letter being checked -) -{ - BOOL fIsEqfDrive = TRUE; // function return value - union - { - CHAR szDrives[MAX_DRIVELIST]; // buffer for drive letters - CHAR szPath[MAX_EQF_PATH+1]; // buffer for path names - } Buffers; - - // - // get string containing all known EQF drives - // - UtlQueryString( QST_ORGEQFDRIVES, Buffers.szDrives, - sizeof(Buffers.szDrives) ); - - // - // check if given drive is contained in EQF drive list - // - if ( !strchr( Buffers.szDrives, chDrive ) ) - { - fIsEqfDrive = FALSE; - } /* endif */ - - // - // check if given drive is a valid drive for this system - // - if ( fIsEqfDrive ) - { - UtlGetDriveList( (PBYTE)Buffers.szDrives ); // get list of attached drives - if ( !strchr( Buffers.szDrives, chDrive ) ) - { - fIsEqfDrive = FALSE; - } /* endif */ - } /* endif */ - - // - // check if required directories exist on drive - // - if ( fIsEqfDrive ) - { - UtlMakeEQFPath( Buffers.szPath, chDrive, MEM_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, DIC_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, TABLE_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, LIST_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - // everything seems to be ok with the given drive - } /* endif */ - } /* endif */ - } /* endif */ - } /* endif */ - } /* endif */ - - return( fIsEqfDrive ); -} /* end of UtlIsEqfDrive */ //+----------------------------------------------------------------------------+ //|External function | @@ -727,46 +625,6 @@ BOOL UtlIsEqfDrive //| copy checked EQF drives to supplied buffer | //| return ok flag | //+----------------------------------------------------------------------------+ -BOOL UtlGetCheckedEqfDrives -( - PSZ pszDrives // ptr to buffer for EQF drives -) -{ - BOOL fOK = TRUE; // function return code - CHAR szOrgDrives[MAX_DRIVELIST]; // buffer for original EQF drive letters - PSZ pszOrgDrive = szOrgDrives; // pointer for drive string processing - CHAR szValidDrives[MAX_DRIVELIST]; // buffer for valid EQF drive letters - PSZ pszValidDrive = szValidDrives;// pointer for drive string processing - - /*******************************************************************/ - /* Get list of MAT drives (source: system properties) */ - /*******************************************************************/ - UtlQueryString( QST_ORGEQFDRIVES, szOrgDrives, sizeof(szOrgDrives) ); - - /*******************************************************************/ - /* Check each of the drives in the drive list */ - /*******************************************************************/ - while ( *pszOrgDrive) - { - if ( UtlIsEqfDrive( *pszOrgDrive ) ) - { - *pszValidDrive++ = *pszOrgDrive; // add to valid drives - } /* endif */ - pszOrgDrive++; // test next drive - } /* endwhile */ - *pszValidDrive = EOS; // terminate valid drives list - - /*******************************************************************/ - /* Set list of checked MAT drives */ - /*******************************************************************/ - UtlReplString( QST_VALIDEQFDRIVES, szValidDrives ); - if ( pszDrives ) - { - strcpy( pszDrives, szValidDrives ); - } /* endif */ - - return( fOK ); -} /* end of UtlGetCheckedEqfDrives */ //+----------------------------------------------------------------------------+ //|Internal function | @@ -800,113 +658,7 @@ BOOL UtlGetCheckedEqfDrives //| open the file | //| initialize control block variables | //+----------------------------------------------------------------------------+ -USHORT UtlBufOpen -( - PBUFCB *ppBufCB, - PSZ pszFile, - ULONG ulBufSize, - ULONG ulReadWrite, - BOOL fMsg -) -{ - return( UtlBufOpenHwnd( ppBufCB, pszFile, ulBufSize, ulReadWrite, fMsg, - NULLHANDLE ) ); -} - -USHORT UtlBufOpenHwnd -( - PBUFCB *ppBufCB, - PSZ pszFile, - ULONG ulBufSize, - ULONG ulReadWrite, - BOOL fMsg, - HWND hwnd -) -{ - USHORT usOpenAction; // action performed by UtlOpen - USHORT usRC = 0; // return code of function - USHORT usOpenFlag; // File open flag - USHORT usOpenMode; // File open mode - PBUFCB pBufCB; // buffered input/output control block - ULONG ulFilePos; // new file position - - /********************************************************************/ - /* Allocate control block / input output buffer */ - /********************************************************************/ - UtlAllocHwnd( (PVOID *)&pBufCB, 0L, (sizeof(BUFCB) + ulBufSize ), - ERROR_STORAGE, hwnd ); - if ( !pBufCB ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - /********************************************************************/ - /* Open the output file */ - /********************************************************************/ - if ( !usRC ) - { - switch (ulReadWrite) - { - case FILE_CREATE: - { - usOpenFlag = FILE_TRUNCATE | FILE_CREATE ; - usOpenMode = OPEN_ACCESS_READWRITE | OPEN_SHARE_DENYWRITE; - } - break; - case FILE_APPEND: - { - usOpenFlag = FILE_OPEN; - usOpenMode = OPEN_ACCESS_READWRITE | OPEN_SHARE_DENYWRITE; - } - break; - default: - { - usOpenFlag = FILE_TRUNCATE | FILE_OPEN; - usOpenMode = OPEN_ACCESS_READONLY | OPEN_SHARE_DENYWRITE; - } - break; - } /* endswitch */ - - usRC = UtlOpenHwnd( pszFile, - &pBufCB->hFile, - &usOpenAction, 0L, - FILE_NORMAL, - usOpenFlag, - usOpenMode, - 0L, - fMsg, hwnd ); - } /* endif */ - if ( !usRC && (ulReadWrite == FILE_APPEND) ) - { - usRC = UtlChgFilePtrHwnd( pBufCB->hFile, 0L, FILE_END, &ulFilePos, fMsg, hwnd ); - } /* endif */ - - - /********************************************************************/ - /* Initialize control block variables */ - /********************************************************************/ - if ( !usRC ) - { - pBufCB->ulUsed = 0; - pBufCB->ulSize = ulBufSize; - pBufCB->ulProcessed = 0; - strcpy( pBufCB->szFileName, pszFile ); - if ( ulReadWrite != FILE_CREATE ) - { - usRC = UtlGetFileSizeHwnd( pBufCB->hFile, - &(pBufCB->ulRemaining), - fMsg, hwnd ); - pBufCB->fWrite = ( ulReadWrite == FILE_APPEND ); - } - else - { - pBufCB->fWrite = TRUE; - } /* endif */ - *ppBufCB = pBufCB; - } /* endif */ - return( usRC ); -} /* end of function UtlBufOpen */ //+----------------------------------------------------------------------------+ //|Internal function | diff --git a/source/opentm2/core/utilities/EQFUTLNG.cpp b/source/opentm2/core/utilities/EQFUTLNG.cpp index 02127214..36bdc94d 100755 --- a/source/opentm2/core/utilities/EQFUTLNG.cpp +++ b/source/opentm2/core/utilities/EQFUTLNG.cpp @@ -46,16 +46,16 @@ BOOL UtlQuerySysLangFile PSZ pszMsgFile // buffer for name of message file ) { - CHAR szFileName[MAX_EQF_PATH]; // buffer for file name + CHAR szFileName[MAX_EQF_PATH]{}; // buffer for file name BOOL fOK = TRUE; // function return code - PSZ pszLangFileBuffer = NULL; // buffer for in-memory copy of language file + std::vector pszLangFileBuffer; // buffer for in-memory copy of language file PSZ pszCurLine; // ptr to current line in file BOOL fFound = FALSE; ULONG ulLength; // length of language file USHORT usI; // loop index - CHAR szDrive[MAX_DRIVE]; // buffer for drive list - CHAR szSysPath[MAX_EQF_PATH]; // global system path - CHAR EqfSystemPath[MAX_EQF_PATH];// global system path + CHAR szDrive[MAX_DRIVE]{}; // buffer for drive list + CHAR szSysPath[MAX_EQF_PATH]{}; // global system path + CHAR EqfSystemPath[MAX_EQF_PATH]{};// global system path /********************************************************************/ /* initialize result fields */ @@ -76,7 +76,7 @@ BOOL UtlQuerySysLangFile /********************************************************************/ /* load system language file into memory */ /********************************************************************/ - fOK = UtlLoadFileL( szFileName, (PVOID *)&pszLangFileBuffer, + fOK = UtlLoadFileL( szFileName, pszLangFileBuffer, &ulLength, FALSE, FALSE ); /********************************************************************/ @@ -98,7 +98,7 @@ BOOL UtlQuerySysLangFile /********************************************************************/ if ( fOK ) { - pszCurLine = pszLangFileBuffer; + pszCurLine = (PSZ) pszLangFileBuffer.data(); pszLangFileBuffer[ulLength-1] = EOS; // force end string delimiter @@ -255,11 +255,6 @@ BOOL UtlQuerySysLangFile } /* endwhile */ } /* endif */ - /********************************************************************/ - /* cleanup */ - /********************************************************************/ - if ( pszLangFileBuffer ) - UtlAlloc( (PVOID *)&pszLangFileBuffer, 0L, 0L, NOMSG ); return( fOK && fFound ); } /* end of function TWBQuerySysLangFile */ diff --git a/source/opentm2/core/utilities/EncodingHelper.cpp b/source/opentm2/core/utilities/EncodingHelper.cpp index d2ab2219..1b14da9f 100755 --- a/source/opentm2/core/utilities/EncodingHelper.cpp +++ b/source/opentm2/core/utilities/EncodingHelper.cpp @@ -23,11 +23,11 @@ struct deletable_facet : Facet{ ~deletable_facet(){}; }; -void EscapeXMLChars( wchar_t* pszText, wchar_t* pszBuffer ); +void EscapeXMLChars( const wchar_t* pszText, wchar_t* pszBuffer ); std::wstring EncodingHelper::EscapeXML( std::wstring input ){ wchar_t buff[4096]; - EscapeXMLChars((wchar_t*)input.c_str(), buff); + EscapeXMLChars(input.c_str(), buff); return buff; } @@ -170,33 +170,27 @@ std::wstring read_with_bom(std::istream & src) } } -std::wstring EncodingHelper::convertToUTF32(std::string inputStr ){ - std::stringstream ss; - ss<> convert; std::wstring wstr = convert.from_bytes( strUTF8String ); return wstr; } -std::u16string EncodingHelper::toUtf16(std::wstring wstr){ +std::u16string EncodingHelper::toUtf16(const std::wstring& wstr){ std::string u8str = EncodingHelper::convertToUTF8(wstr); return std::wstring_convert, char16_t>{}.from_bytes(u8str); } -std::wstring EncodingHelper::toWChar(std::u16string u16str){ +std::wstring EncodingHelper::toWChar(const std::u16string& u16str){ auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); - return convertToWChar(u8str.c_str()); + return convertToWChar(u8str); } @@ -204,7 +198,7 @@ std::wstring EncodingHelper::toWChar(std::u16string u16str){ // std::locale(), // new std::codecvt_utf16); -std::string EncodingHelper::toChar(std::u16string u16str){ +std::string EncodingHelper::toChar(const std::u16string& u16str){ auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); //auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); return u8str; @@ -235,47 +229,6 @@ std::string EncodingHelper::convertToUTF8( const std::u16string& strUTF16String } -/*! \brief convert a UTF8 std::string to a ASCII std::string (on spot conversion) -\param strText on input string in UTF8 encoding, on output string in ASCII encoding -\returns string converted to UTF16 -*/ -void EncodingHelper::convertUTF8ToASCII( std::string& strText ) -{ - //T5LOG( T5WARNING) <<"EncodingHelper::convertUTF8ToASCII( std::string& strText ) is not implemented, strText = ", strText.c_str()); - int iUTF16Len; - int iASCIILen; - int iUTF8Len = (int)strText.length() + 1; - char *pszNewData = NULL; - iUTF16Len = MultiByteToWideChar( CP_UTF8, 0, strText.c_str(), iUTF8Len, 0, 0 ); - std::wstring strUTF16( iUTF16Len, L'\0' ); - MultiByteToWideChar( CP_UTF8, 0, strText.c_str(), iUTF8Len, &strUTF16[0], iUTF16Len ); - iASCIILen = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, 0, 0, 0, 0 ); - pszNewData = (char *)malloc( iASCIILen + 1 ); - WideCharToMultiByte( CP_OEMCP, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, pszNewData, iASCIILen, 0, 0 ); - strText = pszNewData; - free( pszNewData ); -} - -/*! \brief convert a ASCII std::string to UTF8 std::string (on spot conversion) -\param strText on input string in ASCII encoding, on output string in UTF8 encoding -*/ -void EncodingHelper::convertASCIIToUTF8( std::string& strText ) -{ - int iUTF16Len; - int iUTF8Len; - int iASCIILen = (int)strText.length() + 1; - char *pszNewData = NULL; - iUTF16Len = MultiByteToWideChar( CP_OEMCP, 0, strText.c_str(), iASCIILen, 0, 0 ); - std::wstring strUTF16( iUTF16Len, L'\0' ); - MultiByteToWideChar( CP_OEMCP, 0, strText.c_str(), iASCIILen, &strUTF16[0], iUTF16Len ); - iUTF8Len = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, 0, 0, 0, 0 ); - pszNewData = (char *)malloc( iUTF8Len + 1); - WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, pszNewData, iUTF8Len, 0, 0 ); - - strText = pszNewData; - free( pszNewData ); -} - std::u16string EncodingHelper::toLower(const std::u16string& strText){ std::u16string ldata; diff --git a/source/opentm2/core/utilities/EncodingHelper.h b/source/opentm2/core/utilities/EncodingHelper.h index 8f8477c3..03eba106 100755 --- a/source/opentm2/core/utilities/EncodingHelper.h +++ b/source/opentm2/core/utilities/EncodingHelper.h @@ -7,18 +7,28 @@ #include +template +std::string str(T t) +{ + if constexpr(std::is_constructible_v){ + return t; + }else{ + return std::to_string(t); + } +} + + + class EncodingHelper{ public: /*! \brief convert a UTF8 std::string to a UTF16 std::wstring \param strUTF8String string in UTF8 encoding \returns string converted to UTF16 */ - std::wstring static convertToWChar(std::string strUTF8String); - std::u16string static toUtf16(std::wstring wstr); - std::wstring static toWChar(std::u16string u16str); - std::string static toChar(std::u16string u16str); - - std::wstring static convertToUTF32(std::string str); + std::wstring static convertToWChar(const std::string& strUTF8String); + std::u16string static toUtf16(const std::wstring& wstr); + std::wstring static toWChar(const std::u16string& u16str); + std::string static toChar(const std::u16string& u16str); /*! \brief convert a UTF8 std::string to a UTF16 std::wstring \param strUTF8String string in UTF8 encoding @@ -33,16 +43,6 @@ class EncodingHelper{ std::string static convertToUTF8( const std::u16string& strUTF16String ); std::string static convertToUTF8( const wchar_t* UTF32String); - /*! \brief convert a UTF8 std::string to a ASCII std::string (on spot conversion) - \param strText on input string in UTF8 encoding, on output string in ASCII encoding - */ - void static convertUTF8ToASCII( std::string& strText ); - - /*! \brief convert a ASCII std::string to UTF8 std::string (on spot conversion) - \param strText on input string in ASCII encoding, on output string in UTF8 encoding - */ - void static convertASCIIToUTF8( std::string& strText ); - std::u16string static toLower(const std::u16string& strText); void static Base64Encode (const unsigned char* input, int inSize, std::string& output); diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index 808fa707..517e6654 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -57,48 +57,49 @@ std::recursive_mutex FilesystemHelper::loadingTm; DECLARE_bool(keep_tm_backups); int FileBuffer::atomicWriteWithBackup() { try { - // Step 1: Create a temporary file - std::string tempFilename = fileName + ".tmp"; + std::string tempFilename = file.fileName + ".tmp"; + // Step 1: Write to temp file with lock { - std::ofstream tempFile(tempFilename, std::ios::out | std::ios::trunc | std::ios::binary); - if (!tempFile) { - T5LOG(T5ERROR) << "Error: Unable to open temporary file for writing"; - return 1; - } - //tempFile.write(data, data.size()); - if (!tempFile) { - T5LOG(T5ERROR) << "Error: Write operation to temporary file failed"; + LockedFile tempFile(tempFilename, LockedFile::CalledFrom::Write); //, LockedFile::Mode::Write); + FILE* f = tempFile.handle(); + + size_t written = fwrite(data.data(), 1, data.size(), f); + fflush(f); + + if (written != data.size()) { + T5LOG(T5ERROR) << "Write to temp file failed: " << written + << " of " << data.size(); return 2; } - - tempFile.write(reinterpret_cast(data.data()), data.size()); - tempFile.flush(); } - // Step 2: Rename original file to backup (if it exists) - std::string backupFilename = fileName + ".old"; + // Step 2: Backup original file + std::string backupFilename = file.fileName + ".old"; - if(std::filesystem::exists(backupFilename)) { + if (std::filesystem::exists(backupFilename)) { FilesystemHelper::DeleteFile(backupFilename, true); } - if (std::filesystem::exists(fileName)) { - std::filesystem::rename(fileName, backupFilename); + std::string filename = file.fileName; + file = LockedFile{};//unlock our file + if (std::filesystem::exists(filename)) { + std::filesystem::rename(filename, backupFilename); } - // Step 3: Rename the temporary file to the original filename - std::filesystem::rename(tempFilename, fileName); + // Step 3: Rename temp to original + std::filesystem::rename(tempFilename, filename); - // Step 4: Delete the backup file + // Step 4: Optionally delete backup if (std::filesystem::exists(backupFilename) && !FLAGS_keep_tm_backups) { FilesystemHelper::DeleteFile(backupFilename, true); - //std::filesystem::remove(backupFilename); } + file = LockedFile(filename, LockedFile::CalledFrom::Write); + return 0; } catch (const std::exception& e) { - T5LOG(T5ERROR) << "Error: " << e.what() << ""; - return 3; + T5LOG(T5ERROR) << "atomicWriteWithBackup error: " << e.what(); + return -3; } } @@ -121,13 +122,13 @@ bool handleMissingTMI(const std::string& tmdFilename, const std::string& tmiFile ///////////////////////////////////// -std::string FilesystemHelper::parseDirectory(const std::string path){ +std::string FilesystemHelper::parseDirectory(const std::string& path){ std::size_t found = path.rfind('/'); if (found!=std::string::npos) return path.substr(0,found); return path; } -std::string FilesystemHelper::parseFilename(const std::string path){ +std::string FilesystemHelper::parseFilename(const std::string& path){ std::size_t found = path.rfind('/'); if (found!=std::string::npos) @@ -135,7 +136,7 @@ std::string FilesystemHelper::parseFilename(const std::string path){ return path; } -std::string FilesystemHelper::RemovePathAndExtention(const std::string path){ +std::string FilesystemHelper::RemovePathAndExtention(const std::string& path){ std::string filename = parseFilename(path); std::size_t found = filename.rfind('.'); if (found!=std::string::npos) @@ -304,11 +305,11 @@ int FilesystemHelper::RenameFile(std::string oldPath, std::string newPath){ errno = 0; bool fOk = true; //Copy file instead - if(FileExists(newPath.c_str())){ + if(FileExists(newPath)){ T5LOG(T5ERROR) << ":: trg file \'" << newPath << "\' already exists!"; return FILE_EXISTS; } - if(FileExists(oldPath.c_str()) == false){ + if(FileExists(oldPath) == false){ T5LOG(T5ERROR) << ":: src file \'" << oldPath << "\' is missing!"; return FILE_NOT_EXISTS; } @@ -336,11 +337,11 @@ int FilesystemHelper::MoveFile(std::string oldPath, std::string newPath){ bool fOk = true; //fOk = rename(fixedOldPath.c_str(), fixedNewPath.c_str()) != -1; //Copy file instead - if(FileExists(newPath.c_str())){ + if(FileExists(newPath)){ T5LOG(T5ERROR) << ":: trg file \'" << newPath << "\' already exists!"; return FILE_EXISTS; } - if(FileExists(oldPath.c_str()) == false){ + if(FileExists(oldPath) == false){ T5LOG(T5ERROR) << ":: src file \'" << oldPath << "\' is missing!"; return FILE_NOT_EXISTS; } @@ -349,7 +350,8 @@ int FilesystemHelper::MoveFile(std::string oldPath, std::string newPath){ DeleteFile(oldPath); //remove(oldPath); - if(!fOk){ + if(!fOk) + { T5LOG(T5ERROR) << "MoveFile:: cannot move "<< fixedOldPath << " to " << fixedNewPath << ", error = " << strerror(errno); return errno; }else{ @@ -527,45 +529,51 @@ int FilesystemHelper::RemoveDirWithFiles(const std::string& path){ } } -int FileBuffer::ReadFromFile(){ +int FileBuffer::ReadFromFile(){ #ifdef LOG_FILE_WRITE - T5LOG(T5TRANSACTION) <<"called read file " << fileName <<" from disk;";// << " , stacktrace:" << GET_STACKTRACE_EXPL; - #endif + T5LOG(T5TRANSACTION) <<"called read file " << file.filename <<" from disk;";// << " , stacktrace:" << GET_STACKTRACE_EXPL; + #endif - if(fileName.empty()){ + if (file.fileName.empty()) { return FilesystemHelper::FILEHELPER_ERROR_WRONG_FILENAME; } - if(FilesystemHelper::FileExists(fileName) == false){ + + if (!FilesystemHelper::FileExists(file.fileName)) { return FilesystemHelper::FILEHELPER_ERROR_NO_FILES_FOUND; } - originalFileSize = FilesystemHelper::GetFileSize(fileName); - int readed = 0; - if( originalFileSize > 0 && originalFileSize != -1){ - data.resize(originalFileSize); - - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size >0 -> Filebuffer resized to filesize(" << originalFileSize << "), fname = " << fileName; - } - file = fopen(fileName.c_str(), "rb"); - if(file == nullptr){ - T5LOG(T5ERROR) << "Can't open the file under the path=" << fileName; - }else - { - readed = fread(&data[0], originalFileSize, 1, file); - fclose(file); - file = nullptr; - } - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size >0 -> Filebuffer reading to buffer, size = " << originalFileSize << - "), fname = " << fileName << "; readed = " << readed; + + try { + if(nullptr ==file.handle()){ + file = LockedFile(file.fileName, LockedFile::CalledFrom::Read); //, LockedFile::Mode::Read); // lock + open } - }else{ - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size <=0 -> Filebuffer resized to default value, fname = " << fileName ; + + originalFileSize = file.GetFileSize(); + if (originalFileSize > 0) { + data.resize(originalFileSize); + + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "FileBuffer resized to " << originalFileSize + << ", fname = " << file.fileName; + } + + fseek(file.handle(), 0, SEEK_SET); + size_t read = fread(data.data(), 1, originalFileSize, file.handle()); + + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "Read " << read << " bytes from " << file.fileName; + } + } else { + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "File size <= 0, resizing buffer to default"; + } + data.resize(16384); } - data.resize(16384); + + return 0; + } catch (const std::exception& e) { + T5LOG(T5ERROR) << "ReadFromFile error: " << e.what(); + return FilesystemHelper::FILEHELPER_ERROR_CANT_OPEN_FILE; } - return 0; } int FileBuffer::SetOffset(size_t newOffset, int fileAnchor){ @@ -585,7 +593,7 @@ int FileBuffer::SetOffset(size_t newOffset, int fileAnchor){ return 0; } -bool FileBuffer::wasModified() const{ +bool FileBuffer::WasModified() const{ return status & FileBufferStatus::MODIFIED; } @@ -594,44 +602,23 @@ int FileBuffer::Flush(bool forced){ T5LOG(T5TRANSACTION) <<"called flush filebuffer " << fileName <<" to disk;";// << " , stacktrace:" << GET_STACKTRACE_EXPL; #endif - bool fileWasOpened = file != nullptr; + //bool fileWasOpened = file != nullptr; int rc = 0; size_t writenBytes = 0; - if(!FilesystemHelper::FileExists(fileName) && !forced){ - T5LOG(T5WARNING) << "File is not exists on disk so flushing would be skipped, fName = " << fileName; - }else if(forced || wasModified()){ + //if(!FilesystemHelper::FileExists(fileName) && !forced){//since we are locking files, that always should be true + // T5LOG(T5WARNING) << "File is not exists on disk so flushing would be skipped, fName = " << fileName; + //}else + if(forced || WasModified()){ if(VLOG_IS_ON(1)){ T5LOG( T5INFO) << "Flush:: writing files from buffer"; } int oldSize = 0; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - oldSize = FilesystemHelper::GetFileSize(fileName); + oldSize = FilesystemHelper::GetFileSize(file.fileName); } - #ifdef TO_BE_REMOVED - PUCHAR bufStart = &data[0]; - int size = data.size(); - //if(fileWasOpened && file) fclose(file); - if(!fileWasOpened) file = fopen(fileName.c_str(),"w+b"); - - if(file==nullptr){ - T5LOG(T5ERROR) <<"Cant open the file " << fileName; - return FilesystemHelper::FILEHELPER_FILE_PTR_IS_NULL; - } - - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - oldSize = FilesystemHelper::GetFileSize(file); - } - - fseek(file, 0, SEEK_SET); - //if(!fileWasOpened) - { - fclose(file); - file = nullptr; - } - #endif //writenBytes = fwrite(bufStart, size, 1, file) * size; rc = atomicWriteWithBackup(); if ( rc ){ @@ -640,29 +627,33 @@ int FileBuffer::Flush(bool forced){ } if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG( T5DEBUG) "::Flush(" << (long int) file << ") buff = " << "void" << - ", buffsize = " << data.size() << ", path = " << fileName << ", file size = " << - FilesystemHelper::GetFileSize(file) << ", oldSize = " << oldSize; + T5LOG( T5DEBUG) "::Flush(" << (long int) file.handle() << ") buff = " << "void" << + ", buffsize = " << data.size() << ", path = " << file.path() << ", file size = " << + file.GetFileSize() << ", oldSize = " << oldSize; } }else{ if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) <<"Flush:: buffer not modified, so no need to overwrite file, fName = " << fileName; + T5LOG( T5INFO) <<"Flush:: buffer not modified, so no need to overwrite file, fName = " << file.fileName; } } return 0; } -int FileBuffer::Read(void* buff, size_t buffSize){ +int FileBuffer::Read(void* buff, size_t buffSize,ssize_t startingPos){ + if(startingPos>=0){ + offset = startingPos; + } + if(data.size()< offset + buffSize){ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG(T5ERROR) << "ReadBuffer:: Trying to read not existing bytes from buffer, fName = " << fileName; + T5LOG(T5ERROR) << "ReadBuffer:: Trying to read not existing bytes from buffer, fName = " << file.fileName; } return FilesystemHelper::FILEHELPER_WARNING_FILE_IS_SMALLER_THAN_REQUESTED; } if(VLOG_IS_ON(1) && T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << ":: fName = " << fileName << "; buff size = " << buffSize << "; data.size = " << data.size() << "; offset = " << offset << ";"; + T5LOG( T5DEVELOP) << ":: fName = " << file.fileName << "; buff size = " << buffSize << "; data.size = " << data.size() << "; offset = " << offset << ";"; } memcpy(buff, &(data[offset]), buffSize); @@ -670,15 +661,8 @@ int FileBuffer::Read(void* buff, size_t buffSize){ return 0; } -int FileBuffer::Read(void* buff, size_t buffSize, size_t startingPos){ - offset = startingPos; - return Read(buff, buffSize); -} - - - -bool FileBuffer::isTMDFilebuffer(){ - return EncodingHelper::endsWithIgnoreCase(fileName, ".tmd" ); +bool FileBuffer::isTMDFilebuffer()const{ + return EncodingHelper::endsWithIgnoreCase(file.fileName, ".tmd" ); } DECLARE_int64(allowedtmdsize); @@ -687,14 +671,14 @@ int FileBuffer::Write(const void* buff, size_t buffSize, size_t startingPosition status |= MODIFIED; if(startingPosition + buffSize > data.size()){ if(VLOG_IS_ON(1)){ - T5LOG( T5DEBUG) << "::Resizing filebuffer "< allowedSize){ - T5LOG(T5INFO) << "filebuffer vector for " << fileName + T5LOG(T5INFO) << "filebuffer vector for " << file.fileName << " resizing in unsuccessfull, because it's bigger than allowed size " << allowedSize <<" bytes" ; //return FilesystemHelper::FILEHELPER_ERROR_DATA_SIZE_BIGGER_THAN_ALLOWED_FILEBUFFER_SIZE; return TMD_SIZE_IS_BIGGER_THAN_ALLOWED; @@ -703,7 +687,7 @@ int FileBuffer::Write(const void* buff, size_t buffSize, size_t startingPosition data.resize(offset + buffSize); if( offset + buffSize > data.size() ){ - T5LOG(T5ERROR) << "filebuffer vector resizing in unsuccessfull for " << fileName ; + T5LOG(T5ERROR) << "filebuffer vector resizing in unsuccessfull for " << file.fileName ; return FilesystemHelper::FILEHELPER_ERROR_DATA_SIZE_BIGGER_THAN_RECORD_SIZE; } } @@ -799,7 +783,6 @@ int FilesystemHelper::CreateFilebuffer(std::string name){ if(pfb->find(name) != pfb->end()) return FILEHELPER_WARNING_FILEBUFFER_EXISTS; (*pfb)[name].offset=0; - return FILEHELPER_NO_ERROR; } @@ -809,7 +792,7 @@ int FilesystemHelper::FlushAllBuffers(std::string * modifiedFiles){ return -1; } - for(auto file : *pFileBuffers){ + for(auto& file : *pFileBuffers){ if(WriteBuffToFile(file.first) == FILEHELPER_NO_ERROR){ if(modifiedFiles){ *modifiedFiles += file.first + "; "; @@ -1151,25 +1134,6 @@ int FilesystemHelper::ReadFile(const std::string& path, char* buff, return __last_error_code = errcode; } -/* -int FilesystemHelper::ReadFile(FILE*& ptr, char* buff, const int buffSize, int& bytesRead){ - int errCode = FILEHELPER_NO_ERROR; - if(!ptr){ - T5LOG(T5ERROR) <<"FilesystemHelper::ReadFile():: FILEHELPER_FILE_PTR_IS_NULL"); - errCode = FILEHELPER_FILE_PTR_IS_NULL; - }else{ - bytesRead = fread(buff, buffSize, 1, ptr); - if(!bytesRead){ - T5LOG( T5WARNING) << "FilesystemHelper::ReadFile():: bytes not readed from ", toStr((long int)ptr).c_str()); - errCode = FILEHELPER_ERROR_FILE_NOT_READ; - }else{ - T5LOG( T5DEBUG) << "FilesystemHelper::ReadFile():: readed ", toStr(bytesRead).c_str(), "; data = ", buff); - } - } - return __last_error_code = errCode; -} -//*/ - int FilesystemHelper::ReadFile(FILE*& ptr, void* buff, const int buffSize, int& bytesRead){ int errCode = FILEHELPER_NO_ERROR; @@ -1429,7 +1393,6 @@ int FilesystemHelper::CreateDir(const std::string& dir, int rights) { } bool FilesystemHelper::DirExists(const std::string& path){ - bool bExists = false; DIR *pDir = opendir (path.c_str()); if (pDir != NULL) @@ -1442,3 +1405,85 @@ bool FilesystemHelper::DirExists(const std::string& path){ } return bExists; } + +size_t LockedFile::GetFileSize() const { + if (!file_) return 0; + + int fd = fileno(file_); + struct stat st{}; + if (fstat(fd, &st) == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to get file size"); + } + return static_cast(st.st_size); +} + +DECLARE_bool(log_file_locks); +void LockedFile::openAndLock() { + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "locking \"" << path_ << "\" file, locked from " << CalledFromToString(location_) << " function"; + } + int flags = O_RDWR | O_CREAT; + //if (mode_ == Mode::Append) flags |= O_APPEND; + + fd_ = ::open(path_.c_str(), flags, 0666); + if (fd_ == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to open file"); + } + + fl_.l_type = F_WRLCK;//(mode_ == Mode::Read) ? F_RDLCK : F_WRLCK; + fl_.l_whence = SEEK_SET; + fl_.l_start = 0; + fl_.l_len = 0; + + if (fcntl(fd_, F_SETLK, &fl_) == -1) { + ::close(fd_); + throw std::system_error(errno, std::generic_category(), + "Failed to lock file"); + } + + // Only truncate in Write mode + //if (mode_ == Mode::Write) { + // if (ftruncate(fd, 0) == -1) { + // ::close(fd); + // throw std::system_error(errno, std::generic_category(), + // "Failed to truncate file"); + // } + //} + + const char* fmode = "r+b"; + //const char* fmode = "r+"; + //if (mode_ == Mode::Append) fmode = "a+"; + //else if (mode_ == Mode::Read) fmode = "rb"; + + file_ = fdopen(fd_, fmode); + if (!file_) { + ::close(fd_); + throw std::system_error(errno, std::generic_category(), + "Failed to create file stream"); + } + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "locked \"" << path_ << "\" file locked from " << CalledFromToString(location_) << " function"; + } +} + +void LockedFile::closeFile() { + if(fd_>=1){ + fl_.l_type = F_UNLCK; + fl_.l_whence = SEEK_SET; + fl_.l_start = 0; + fl_.l_len = 0; // 0 = unlock entire file + fcntl(fd_, F_SETLK, &fl_); // Unlock the file + } + + if (file_) { + fclose(file_); // Releases lock and closes file + file_ = nullptr; + fd_ = -1; // Prevent stale fd reuse + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "unlocked \"" << path_ << "\" file locked from " << CalledFromToString(location_) << " function"; + } + path_.clear(); + } +} \ No newline at end of file diff --git a/source/opentm2/core/utilities/FilesystemHelper.h b/source/opentm2/core/utilities/FilesystemHelper.h index 1c1f963d..f888c854 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.h +++ b/source/opentm2/core/utilities/FilesystemHelper.h @@ -29,9 +29,9 @@ class FilesystemHelper{ static std::mutex fsLock; static std::recursive_mutex loadingTm; static bool checkFileName(const std::string& path); - static std::string parseFilename(const std::string path); - static std::string parseDirectory(const std::string path); - static std::string RemovePathAndExtention(const std::string path); + static std::string parseFilename(const std::string& path); + static std::string parseDirectory(const std::string& path); + static std::string RemovePathAndExtention(const std::string& path); static PFileBufferMap getFileBufferInstance(); static std::string FixPath(std::string& path); @@ -131,6 +131,7 @@ class FilesystemHelper{ FILEHELPER_ERROR_NO_FILES_FOUND, FILEHELPER_ERROR_WRONG_FILENAME, + FILEHELPER_ERROR_CANT_OPEN_FILE, FILEHELPER_ERROR_CANT_OPEN_DIR, diff --git a/source/opentm2/core/utilities/FilesystemWrapper.cpp b/source/opentm2/core/utilities/FilesystemWrapper.cpp index e89ddf7d..180b1430 100755 --- a/source/opentm2/core/utilities/FilesystemWrapper.cpp +++ b/source/opentm2/core/utilities/FilesystemWrapper.cpp @@ -13,20 +13,6 @@ std::string filesystem_get_home_dir() { return FilesystemHelper::GetHomeDir(); } -int filesystem_open_file(const char* path, FILE*& ptr, const char* mode){ - T5LOG( T5WARNING) <<"TEMPORARY HARDCODED useBuffer= true in filesystem_open_file, fname = " << path; - bool useBuffer = true;//false; - if( (strcasestr(path, ".TMI") ) - || (strcasestr(path, ".TMD") ) - //|| (strcasestr(path, ".MEM") ) - ){ - T5LOG( T5INFO) << "filesystem_open_file::Openning data file(with ext. .TMI, .TMD, .MEM => forcing to use filebuffers, fName = "<< - path<< ", useFilebuffer before = " <findIfPreferedLanguage( lang ) != -1); } +bool hasDash(const char* langCode) { + return std::strchr(langCode, '-') != nullptr; +} + +bool LanguageFactory::hasSubLocale(const char* lang){ + return hasDash(lang); +} + bool LanguageFactory::isMatchingLangs( const char* strMemLang,//can be prefered const char* strSegLang @@ -382,7 +390,6 @@ int LanguageFactory::findIfPreferedLanguage const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vpPreferedLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -403,7 +410,6 @@ int LanguageFactory::findPreferedLangForThisLang const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vpPreferedLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -428,7 +434,6 @@ int LanguageFactory::findLanguage const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -522,14 +527,11 @@ unsigned short LanguageFactory::loadLanguageList( const char *pszLangList ) unsigned short usRC = 0; // parse the XML file containing the language properties - if ( !usRC ) - { SAXParser parser; // Create a SAX parser object // create an instance of our handler LangParseHandler *handler = new LangParseHandler(); - // install our SAX handler as the document and error handler. parser.setDocumentHandler( handler ); parser.setErrorHandler( handler ); @@ -571,9 +573,7 @@ unsigned short LanguageFactory::loadLanguageList( const char *pszLangList ) } } } - } /* endif */ - - + return( usRC ); } @@ -617,7 +617,6 @@ void LangParseHandler::startElement(const XMLCh* const name, AttributeList& attr case UNKNOWN_ELEMENT: default: if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - std::string sName = EncodingHelper::convertToUTF8(u16name); T5LOG( T5WARNING) << "LangParseHandler::startElement::unknown_element found: "<< EncodingHelper::convertToUTF8(u16name) << "; id = " << CurID << "; skipping..."; } @@ -673,7 +672,7 @@ void LangParseHandler::endElement(const XMLCh* const name ) void LangParseHandler::characters(const XMLCh* const chars, const XMLSize_t length) { - auto wstr = EncodingHelper::toWChar(std::u16string((char16_t*)chars).c_str()); + auto wstr = EncodingHelper::toWChar(std::u16string((char16_t*)chars)); // add data to current data buffer int iCurLen = wcslen( szDataW ); @@ -683,7 +682,7 @@ void LangParseHandler::characters(const XMLCh* const chars, const XMLSize_t leng szDataW[iCurLen+wstr.size()] = 0; }else{ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(wstr.c_str()); + auto str = EncodingHelper::convertToUTF8(wstr); T5LOG(T5ERROR) << ":: (iCurLen + length + 1) >= (sizeof(szData)/sizeof(szData[0])), characters = " << str; } } /* endif */ diff --git a/source/opentm2/core/utilities/LogWrapper.cpp b/source/opentm2/core/utilities/LogWrapper.cpp index a0565468..58d1bfa2 100755 --- a/source/opentm2/core/utilities/LogWrapper.cpp +++ b/source/opentm2/core/utilities/LogWrapper.cpp @@ -178,9 +178,11 @@ std::string T5Logger::FlushBodyBuffer(){ return str; } -int T5Logger::SetLogLevel(int level){ +int T5Logger::SetLogLevel(int level, bool fAddMessage){ if(level <= T5FATAL && level>= T5DEVELOP){ - T5LOG(T5INFO) << "SetLogLevel::Log level treshold was set to " << level; + if(fAddMessage){ + T5LOG(T5INFO) << "SetLogLevel::Log level treshold was set to " << level; + } logLevelTreshold = level; return level; }else{ diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index a27d573c..a6ae7f1c 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -11,7 +11,6 @@ #ifdef _DEBUG_LOGS #define TOKENIZELOG #define SPELLFACTORY_LOGGING - //#define MEASURETIME #define SGMLDITA_LOGGING #define INLINE_TAG_REPL_LOGGING #define _NTGET_LOG @@ -53,7 +52,21 @@ std::basic_ostream& operator<<(std::basic_ostream +//std::string toStr(T i) { +// if constexpr (std::is_pointer::value) { +// std::ostringstream oss; +// oss << static_cast(i); // print pointer address +// return oss.str(); +// } else { +// return std::to_string(i); +// } +//} enum LOGLEVEL{ T5DEVELOP=0, @@ -132,7 +145,7 @@ class T5Logger{ std::string FlushLogBuffer(); std::string FlushBodyBuffer(); - int SetLogLevel(int level); + int SetLogLevel(int level, bool fAddMessage = true); int GetLogLevel(); bool CheckLogLevel(int level); void SetLogFilter(bool filterOn); @@ -140,7 +153,7 @@ class T5Logger{ std::string FlushBuffers(int severity); }; -static void GetElapsedTime( long long *plTime ); +void GetElapsedTime( long long *plTime ); std::string getTimeStr(); int getBuffIdForLog(int severity); @@ -188,10 +201,10 @@ int getBuffIdForLog(int severity); : id == -2? google::LogMessageVoidify() & T5Logger::__getLogBuffer() \ : google::LogMessageVoidify() & _T5_LOG_##severity +#define T5INITLOG LOG(INFO)<< "::" << __func__ << ": " + #define T5LOG(severity) T5LOG_FROM_ID(getBuffIdForLog(severity), severity) << \ "::[" << #severity << "]::" << __func__ << ": " - - #endif //CUSTOM_LOG_MESSAGE #define LOG_AND_SET_RC_W_INFO(rcVar, logLevel, rc) int ___prev_rc = rcVar; rcVar = rc; T5LOG(logLevel) << "rc was set to " <<#rc << "(" << rcVar <<"), prev state = " <<___prev_rc <<"; " @@ -302,4 +315,4 @@ bool rcNeedToBeLoggedAsError(int rc); else \ T5LOG(T5WARNING) << outputMessage<<";rc = "< #include "EncodingHelper.h" -int GetLocaleInfo( - LCID Locale, - LCTYPE LCType, - LPSTR lpLCData, - int cchData - ){ - LOG_UNIMPLEMENTED_FUNCTION; - return 0; - } - -UINT GetOEMCP(){ - return 0; - -} - unsigned long int _ttol(const char* source){ return atoi(source); @@ -34,7 +19,6 @@ unsigned long int _ttol(const char* source){ int MultiByteToWideChar( UINT CodePage, DWORD dwFlags, - //_In_NLS_string_(cbMultiByte)LPCCH lpMultiByteStr, LPCSTR lpMultiByteStr, int cbMultiByte, LPWSTR lpWideCharStr, @@ -43,7 +27,6 @@ int MultiByteToWideChar( mbstate_t state; int cnt = 0; memset (&state, '\0', sizeof (state)); - //return mbsnrtowcs(lpMultiByteStr, lpWideCharStr, cchWideChar, cbMultiByte, &state ); if(cbMultiByte<=0){ cbMultiByte = strlen(lpMultiByteStr); } @@ -51,34 +34,24 @@ int MultiByteToWideChar( lpWideCharStr[cbMultiByte] = 0; return mbsnrtowcs(lpWideCharStr, (const char**)&lpMultiByteStr, cbMultiByte, cchWideChar, &state ); - //EncodingHelper::convertToWChar(lpMultiByteStr); - //return mbtowc(lpWideCharStr, lpMultiByteStr, cchWideChar); } int WideCharToMultiByte( UINT CodePage, DWORD dwFlags, - //_In_NLS_string_(cchWideChar)LPCWCH lpWideCharStr, LPWSTR lpWideCharStr, int cchWideChar, LPSTR lpMultiByteStr, int cbMultiByte, - //LPCCH char* lpDefaultChar, LPBOOL lpUsedDefaultChar ){ - //mbstate_t state; - //int cnt = 0; - //memset (&state, '\0', sizeof (state)); - if(cchWideChar<=0){ cchWideChar = wcslen(lpWideCharStr); } if(cchWideChar > cbMultiByte-1) cchWideChar = cbMultiByte-1; lpMultiByteStr[cchWideChar] = 0; - return wcstombs(lpMultiByteStr, (const wchar_t*) &lpWideCharStr, cchWideChar); - //return wcsnrtombs(lpMultiByteStr, (const wchar_t**)&lpWideCharStr, cchWideChar, cbMultiByte, &state ); - + return wcstombs(lpMultiByteStr, (const wchar_t*) &lpWideCharStr, cchWideChar); } @@ -90,22 +63,6 @@ DWORD WaitForSingleObject( return 0; } -HANDLE CreateMutex( - LPSECURITY_ATTRIBUTES lpMutexAttributes, - BOOL bInitialOwner, - LPCSTR lpName - ){ - LOG_UNIMPLEMENTED_FUNCTION; - return NULL; - } - -BOOL ReleaseMutex( - HANDLE hMutex -){ - LOG_UNIMPLEMENTED_FUNCTION; - return true; -} - BOOL FindClose( HANDLE hFindFile ){ diff --git a/source/opentm2/core/utilities/OSWrapper.h b/source/opentm2/core/utilities/OSWrapper.h index d47fa87c..b194a5dc 100755 --- a/source/opentm2/core/utilities/OSWrapper.h +++ b/source/opentm2/core/utilities/OSWrapper.h @@ -148,15 +148,6 @@ enum LCTYPES }; -int GetLocaleInfo( - LCID Locale, - LCTYPE LCType, - LPSTR lpLCData, - int cchData -); - -UINT GetOEMCP(); - unsigned long int _ttol(const char* source); int _strcmp(const char* a, const char* b); diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index 00000455..b11d79fe 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -25,7 +25,6 @@ int CopyToBuffer(const char *pszSource, char *pszBuffer, int iBufSize ); /*! \brief Constructors */ OtmProposal::OtmProposal() { - this->clear(); fFilled = 0; }; @@ -36,16 +35,18 @@ OtmProposal::~OtmProposal() }; /* operations */ +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(saveDifferentTargetsForSameSource); +//DECLARE_bool(ignoreSegIdOnUpdate); void OtmProposal::clear()//bool clearKeys) { - //if(this->pInputSentence) delete pInputSentence; pInputSentence.reset(); TMCursor currentKey, nextKey; - //if(false == clearKeys){ } - memset( this, 0, sizeof(OtmProposal) ); - eType = OtmProposal::eptUndefined; - eMatch = OtmProposal::emtUndefined; + *this = OtmProposal{}; + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent;//default value + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; } OtmProposal::eProposalType getMemProposalType( char *pszType ) @@ -337,7 +338,7 @@ void OtmProposal::setUpdateTime( long lTime ) /* \brief get proposal fuzziness \returns proposal fuzziness */ -int OtmProposal::getFuzziness() +int OtmProposal::getFuzziness()const { return( iFuzziness ); } @@ -527,10 +528,14 @@ int OtmProposal::getContext( wchar_t *pszBuffer, int iBufSize ) /* \brief set the proposal context \param pszBuffer Pointer to buffer containing the proposal context */ -void OtmProposal::setContext( wchar_t *pszBuffer ) +void OtmProposal::setContext(const wchar_t *pszBuffer ) { - wcsncpy( szContext, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); - fFilled = 1; + szContext[0] = '\0'; + if(pszBuffer){ + wcsncpy( szContext, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); + fFilled = 1; + } + szContext[OTMPROPOSAL_MAXSEGLEN] = '\0'; } /* \brief get proposal context ranking @@ -572,10 +577,14 @@ int OtmProposal::getAddInfo( wchar_t *pszBuffer, int iBufSize ) /* \brief set the proposal additional information \param pszBuffer Pointer to buffer containing the additional info */ -void OtmProposal::setAddInfo( wchar_t *pszBuffer ) -{ - wcsncpy( szAddInfo, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); - fFilled = 1; +void OtmProposal::setAddInfo(const wchar_t *pszBuffer ) +{ + szAddInfo[0] = '\0'; + if(pszBuffer){ + wcsncpy( szAddInfo, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); + fFilled = 1; + } + szAddInfo[OTMPROPOSAL_MAXSEGLEN] = '\0'; } diff --git a/source/opentm2/core/utilities/Stopwatch.hpp b/source/opentm2/core/utilities/Stopwatch.hpp index 6b32f9be..a32e5b08 100644 --- a/source/opentm2/core/utilities/Stopwatch.hpp +++ b/source/opentm2/core/utilities/Stopwatch.hpp @@ -49,6 +49,7 @@ namespace stopwatch{ const std::uint64_t ns_count = std::chrono::duration_cast(duration).count(); std::uint64_t up = ((ns_count/100000)%10 >= 5) ? 1 : 0; const auto ms_count = (ns_count /1000000) + up; + return ms_count; } std::string getString(){ diff --git a/source/opentm2/core/utilities/ThreadingWrapper.cpp b/source/opentm2/core/utilities/ThreadingWrapper.cpp index 586c75d3..1e7e1f8a 100755 --- a/source/opentm2/core/utilities/ThreadingWrapper.cpp +++ b/source/opentm2/core/utilities/ThreadingWrapper.cpp @@ -25,7 +25,7 @@ int _getpid(){ #include "sys/times.h" -#include "sys/vtimes.h" +// #include "sys/vtimes.h" static clock_t lastCPU, lastSysCPU, lastUserCPU; static int numProcessors; diff --git a/source/opentm2/core/utilities/ThreadingWrapper.h b/source/opentm2/core/utilities/ThreadingWrapper.h index 62b032a8..05d99bcb 100755 --- a/source/opentm2/core/utilities/ThreadingWrapper.h +++ b/source/opentm2/core/utilities/ThreadingWrapper.h @@ -45,7 +45,7 @@ class MutexTimeout{ return m_timeout_failed; } - bool reset(){ + void reset(){ m_timeout_failed = false; m_errMsg = ""; m_timeout_ms = 0; diff --git a/source/opentm2/core/utilities/Utility.h b/source/opentm2/core/utilities/Utility.h index 140c2167..88d6dc2c 100755 --- a/source/opentm2/core/utilities/Utility.h +++ b/source/opentm2/core/utilities/Utility.h @@ -5,7 +5,6 @@ //| Corporation and others. All rights reserved | //+----------------------------------------------------------------------------+ #define MAX_MEM_TABLES 8 // max. no of segment tables -#define MAX_MEM_SEGMENTS 8162 // max. no of segments for UtlAlloc #define SEG_TABLE_EMPTY (-1) // there are no segments in the table #define DATEFORMATSTRING_YMD "%4.4d%c%2.2d%c%2.2d" diff --git a/source/opentm2/core/utilities/UtlAlloc.cpp b/source/opentm2/core/utilities/UtlAlloc.cpp index 4bc22c4b..b5ade408 100755 --- a/source/opentm2/core/utilities/UtlAlloc.cpp +++ b/source/opentm2/core/utilities/UtlAlloc.cpp @@ -74,31 +74,6 @@ BOOL UtlAlloc return fSuccess; } /* end of function UtlAlloc */ -//+----------------------------------------------------------------------------+ -//|External Function | -//+----------------------------------------------------------------------------+ -//|Function Name: UtlAllocHwnd Interface for UtlAlloc | -//+----------------------------------------------------------------------------+ -//|Description: Call UtlAlloc and use given window handle for error | -//| messages. | -//+----------------------------------------------------------------------------+ -//|Function Call: BOOL UtlAllocHwnd( PVOID *ppMemory | -//| LONG lOldLength, | -//| LONG lNewLength, | -//| USHORT usMsgNo, | -//| HWND hwndOwner ); | -//+----------------------------------------------------------------------------+ -BOOL UtlAllocHwnd -( - PVOID *ppStorage, // pointer to allocated memory area - LONG lOldLength, // old length of storage area - LONG lNewLength, // length of area to be allocated - USHORT usMessageNo, // message to be displayed when error - HWND hwnd ) // handle for UtlError Hwnd call -{ - return UtlAlloc( ppStorage, lOldLength, lNewLength, NOMSG ); -} /* end of UtlAllocHwnd */ - /**********************************************************************/ /* get the correct task id ... */ diff --git a/source/opentm2/core/utilities/UtlMATVal.cpp b/source/opentm2/core/utilities/UtlMATVal.cpp index 3e6a757a..468b1147 100755 --- a/source/opentm2/core/utilities/UtlMATVal.cpp +++ b/source/opentm2/core/utilities/UtlMATVal.cpp @@ -234,8 +234,7 @@ PSZ UtlMakeEQFPath USHORT usPathID, // ID of path PSZ pszFolder // folder name or NULL ) -{ - +{ pszPath = new char[MAX_EQF_PATH]; // no path info yet auto otmDir = filesystem_get_otm_dir(); diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index 2a392805..47a0dc6b 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -45,7 +45,6 @@ #define MAX_DISPATCH_MSGS 30 // number of msgs processed per dispatch static HWND hwndDlgs[MAX_DIALOGS]; // array for handles of modeless dialogs static SHORT sRegisteredDlgs = 0; // number of registered dialogs -static FILE *hFFSTFile = NULL; // UtlAlloc log file //static FARPROC lpfnOldButtonProc = NULL;// ptr to original button proc LONG FAR PASCAL EqfHelpButtonProc (HWND, UINT, WPARAM, LPARAM); @@ -363,128 +362,10 @@ LONG UtlTime( PLONG plTime ) // Returncodes: 0 function completed successfully // other DOS error return codes //+----------------------------------------------------------------------------+ -USHORT UtlCompFDates -( - PSZ pszFile1, // fully qualified path first file - PSZ pszFile2, // fully qualified path of second file - PSHORT psResult, // address of result buffer - BOOL fMsg // error handling flag -) -{ - USHORT usRC = NO_ERROR; // function return code - FILESTATUS stStatus1; // File status information of 1st file - FILESTATUS stStatus2; // File status information of 2nd file - LONG lResult = 0L; // buffer for result value - - /********************************************************************/ - /* Get date and time of files */ - /********************************************************************/ - usRC = UtlQPathInfo( pszFile1, 1, (PBYTE)&stStatus1, sizeof(stStatus1), - 0L, fMsg ); - if ( usRC == NO_ERROR ) - { - usRC = UtlQPathInfo( pszFile2, 1, (PBYTE)&stStatus2, sizeof(stStatus2), - 0L, fMsg ); - } /* endif */ - - /********************************************************************/ - /* Compare file dates */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - lResult = UtlCompareDate( &(stStatus1.fdateLastWrite), - &(stStatus2.fdateLastWrite) ); - if ( lResult == 0L ) - { - lResult = UtlCompareTime( &(stStatus1.ftimeLastWrite), - &(stStatus2.ftimeLastWrite) ); - } /* endif */ - *psResult = (SHORT)lResult; - } /* endif */ - - return usRC; -} /* end of function UtlCompFDates */ - // function searching first character causing a conversion error and showing an // approbriate error message const int MB_ERR_INVALID_CHARS = 1234;//just to compile -USHORT UtlFindAndShowConversionError -( - PSZ pszBuffer, - ULONG ulLen, - ULONG ulCP -) -{ - CHAR_W chBufOut[10]; - USHORT usRC = NO_ERROR; - - ULONG ulOutPut; - - // always use 932 when 943 is specified - if ( ulCP == 943 ) ulCP = 932; - - while ( !usRC && ulLen ) - { - ulOutPut = MultiByteToWideChar( ulCP, MB_ERR_INVALID_CHARS, pszBuffer, 1, chBufOut, 1 ); - - // try conversion together with next byte, might be a DBCS character - if ( !ulOutPut && ulLen ) - { - ulOutPut = MultiByteToWideChar( ulCP, MB_ERR_INVALID_CHARS, pszBuffer, 2, chBufOut, 2 ); - if ( ulOutPut ) - { - pszBuffer++; - ulLen--; - } /* endif */ - } /* endif */ - - if ( !ulOutPut ) - { - PSZ pszParms[3]; - CHAR szRC[10], szCP[10], szCharacter[20]; - int iCh = (UCHAR)*pszBuffer; -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 74 usRC = (USHORT)GetLastError();"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - usRC = (USHORT)GetLastError(); -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - - sprintf( szCharacter, "%d (hex %2.2X)", iCh, iCh ); - sprintf( szCP, "%lu", ulCP ); - sprintf( szRC, "%u", usRC ); - - pszParms[0] = szCharacter; - pszParms[1] = szCP; - pszParms[2] = szRC; - - UtlError( ERROR_DATA_CONVERSION_EXINFO, MB_CANCEL, 3, pszParms, EQF_ERROR ); - } - else - { - pszBuffer++; - ulLen--; - } /* endif */ - } /* endwhile */ - return( usRC ); -} /* end of function UtlFindAndShowConversionError */ - -// function checking a given codepage value by doing a test conversion -BOOL UtlIsValidCP( ULONG ulCP ) -{ - // do a test conversion using the supplied codepage - CHAR_W c_W[10]; - LONG lBytesLeft = 0L; - LONG lRc = 0; - ULONG ulOutPut = 0; - - ulCP = ADJUSTCP( ulCP ); - - ulOutPut = UtlDirectAnsi2UnicodeBuf( "abcdefgh", c_W, 8, ulCP, FALSE, &lRc, &lBytesLeft); - - return( (ulOutPut != 0) && (ulCP != 0) ); -} /* end of function UtlIsValidCP */ - - /*! UtlInitUtils Initialization of utilities Description: General utilities initialization routine @@ -580,201 +461,10 @@ BOOL UtlInitUtils( HAB hab ) //| FALSE if no change is required (in this | //| case no new target segment is allocated!) | //+----------------------------------------------------------------------------+ -//|Returncode type: BOOL FALSE in case of errors (=UtlAlloc failed) | +//|Returncode type: BOOL FALSE in case of errors | //| TRUE everything O.K. | //+----------------------------------------------------------------------------+ #define TAISWHITESPACE( c ) \ ( (c == LF) || (c == CR) || (c == SPACE) || (c == 0x09) ) - -static BOOL TACheckForWS -( - CHAR_W c, - PSZ_W pWSList -) -{ - BOOL fIsWS = FALSE; - - fIsWS = TAISWHITESPACE( c ) ; - - if (pWSList && !fIsWS) - { - while (*pWSList && !fIsWS) - { - if (c == *pWSList) - { - fIsWS = TRUE; - } - else - { - pWSList++; - } - } /* endwhile */ - } /* endif */ - - - return(fIsWS); -} - - -BOOL TAAdjustWhiteSpace -( - PSZ_W pszSourceSeg, // ptr to source segment data - PSZ_W pszTargetSeg, // ptr to target segment - PSZ_W *ppszNewTargetSeg, // ptr to ptr of output buffer - BOOL fLeadingWS, // process leading whitespace flag - BOOL fTrailingWS, // process trailing whitespace flag - PBOOL pfChanged, // TRUE if target has been changed - PSZ_W pWSList -) -{ - BOOL fOK = TRUE; // function return code - PSZ_W pszTargetWhiteSpace=NULL;// points to last whitespace in target - PSZ_W pszSourceWhiteSpace=NULL;// points to last whitespace in source - PSZ_W pszTempSegW = NULL; - int iAdjustLen = 0; // length of adjusted leading whitespace - - // initialize callers change flag - *pfChanged = FALSE; - - // allocate buffer for temporary segment - fOK = UtlAlloc( (PVOID *)&pszTempSegW, 0L, (2L*(LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W)), - ERROR_STORAGE ); - - // handle leading whitespace - if ( fOK && fLeadingWS ) - { - CHAR_W chFirstNonWSSource; - CHAR_W chFirstNonWSTarget; - PSZ_W pszNonWSSource = pszSourceSeg; - PSZ_W pszNonWSTarget = pszTargetSeg; - - - // position to first non-whitespace character in source and target - while ( TACheckForWS(*pszNonWSSource, pWSList) ) pszNonWSSource++; - while ( TACheckForWS(*pszNonWSTarget, pWSList) ) pszNonWSTarget++; - - // temporarly terminate segment at found locations - chFirstNonWSSource = *pszNonWSSource; - *pszNonWSSource = EOS; - chFirstNonWSTarget = *pszNonWSTarget; - *pszNonWSTarget = EOS; - - // compare leading whitespace and adjust if necessary - if ( UTF16strcmp( pszSourceSeg, pszTargetSeg ) != 0 ) - { - // copy whitespace of source segment to our temp seg buffer - UTF16strcpy( pszTempSegW, pszSourceSeg ); - - iAdjustLen = UTF16strlenCHAR( pszTempSegW ); - - // restore end of leading whitespace - *pszNonWSSource = chFirstNonWSSource; - *pszNonWSTarget = chFirstNonWSTarget; - - // copy data of target segment to our temp seg buffer - UTF16strcat( pszTempSegW, pszNonWSTarget ); - - // set caller's changed flag - *pfChanged = TRUE; - } - else - { - iAdjustLen = UTF16strlenCHAR( pszTargetSeg ); - - // restore end of leading whitespace - *pszNonWSSource = chFirstNonWSSource; - *pszNonWSTarget = chFirstNonWSTarget; - } /* endif */ - } /* endif */ - - // if nothing has been changed copy complete target segment to - // our temp seg buffer - if ( fOK && !*pfChanged ) - { - UTF16strcpy( pszTempSegW, pszTargetSeg ); - } /* endif */ - - // locate trailing whitespace in target segment - if ( fOK && fTrailingWS ) - { - int iLen; - - iLen = UTF16strlenCHAR(pszTempSegW); - if ( iLen > iAdjustLen ) - { - pszTargetWhiteSpace = pszTempSegW + iLen - 1; - while ( (iLen > iAdjustLen) && TACheckForWS(*pszTargetWhiteSpace, pWSList) ) - { - pszTargetWhiteSpace--; - iLen--; - } /* endwhile */ - pszTargetWhiteSpace++; // point to first byte of whitespace - } /* endif */ - } /* endif */ - - // locate trailing whitespace in source segment - if ( fOK && fTrailingWS ) - { - int iLen; - - iLen = UTF16strlenCHAR(pszSourceSeg); - if ( iLen > iAdjustLen ) - { - pszSourceWhiteSpace = pszSourceSeg + iLen - 1; - while ( (iLen > iAdjustLen) && TACheckForWS(*pszSourceWhiteSpace, pWSList) ) - { - pszSourceWhiteSpace--; - iLen--; - } /* endwhile */ - pszSourceWhiteSpace++; // point to first byte of whitespace - } /* endif */ - } /* endif */ - - // setup new target if trailing whitespace differs - if ( fOK && fTrailingWS && pszSourceWhiteSpace && pszTargetWhiteSpace && - UTF16strcmp( pszSourceWhiteSpace, pszTargetWhiteSpace ) != 0 ) - { - // build new target segment - *pszTargetWhiteSpace = EOS; - UTF16strcat( pszTempSegW, pszSourceWhiteSpace ); - - // set caller's changed flag - *pfChanged = TRUE; - } /* endif */ - - // return modified temp segment to caller - if ( fOK && *pfChanged ) - { - LONG lNewLen = UTF16strlenCHAR(pszTempSegW) + 1; - - // if changed segment exceeds maximum segment size... - if ( lNewLen > MAX_SEGMENT_SIZE ) - { - // .. ignore the changes - *pfChanged = FALSE; - } - else - { - if ( *ppszNewTargetSeg == NULL ) - { - LONG lNewLen = UTF16strlenCHAR(pszTempSegW) + 1; - // GQ: allocate segment at least large enough for the :NONE. tag! - fOK = UtlAlloc( (PVOID *)ppszNewTargetSeg, 0L, - get_max( lNewLen * sizeof(CHAR_W), (sizeof(EMPTY_TAG)+4)), ERROR_STORAGE ); - } /* endif */ - - if ( fOK ) - { - UTF16strcpy( *ppszNewTargetSeg, pszTempSegW ); - } /* endif */ - } /* endif */ - } /* endif */ - - // cleanup - if ( pszTempSegW ) UtlAlloc( (PVOID *)&pszTempSegW, 0L, 0L, NOMSG ); - - return( fOK ); -} /* end of function TAAdjustWhiteSpace */ - diff --git a/source/opentm2/core/utilities/UtlRegistry.cpp b/source/opentm2/core/utilities/UtlRegistry.cpp index 003cb8b8..8880944b 100755 --- a/source/opentm2/core/utilities/UtlRegistry.cpp +++ b/source/opentm2/core/utilities/UtlRegistry.cpp @@ -28,7 +28,6 @@ int GetIntFromRegistry( PSZ pszAppl, PSZ pszKey, int iDefault ) BOOL WriteStringToRegistry( PSZ pszAppl, PSZ pszKey, PSZ pszValue ) { int res = Properties::GetInstance()->set_anyway(pszKey, pszValue); - return res == PROPERTY_NO_ERRORS; } /* end of function WriteStringToRegistry */ @@ -36,7 +35,6 @@ BOOL WriteStringToRegistry( PSZ pszAppl, PSZ pszKey, PSZ pszValue ) BOOL WriteIntToRegistry( PSZ pszAppl, PSZ pszKey, int iValue ) { int res = Properties::GetInstance()->set_anyway(pszKey, iValue); - return res == PROPERTY_NO_ERRORS; - } /* end of function WriteIntToRegistry */ + diff --git a/source/opentm2/core/utilities/UtlString.cpp b/source/opentm2/core/utilities/UtlString.cpp index 3bb1231a..a55f7c20 100755 --- a/source/opentm2/core/utilities/UtlString.cpp +++ b/source/opentm2/core/utilities/UtlString.cpp @@ -230,14 +230,6 @@ PSZ_W UTF16strcat( PSZ_W pszTarget, PSZ_W pszSource ) return( pszTarget ); } -/////////////////////////////////////////////////////////////////////////////// -/// UTF16strcmp compare two Unicode UTF16 strings /// -/////////////////////////////////////////////////////////////////////////////// - -int UTF16strcmp( PSZ_W pszString1, PSZ_W pszString2 ) -{ - return wcscmp(pszString1, pszString2); -} /////////////////////////////////////////////////////////////////////////////// /// UTF16strncmp compare two Unicode UTF16 strings up to usLen chars /// @@ -783,17 +775,7 @@ USHORT UtlQueryCharTableEx case 813 : pTable = chAnsiToPC813; break; case 737 : T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 78 if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869)) { // fix for sev1 Greek: Win NT problem (01/09/23)"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869)) - { // fix for sev1 Greek: Win NT problem (01/09/23) - usCP = 869; - pTable = chAnsiToPC869; - } - else - { - pTable = chAnsiToPC737; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE + break; case 775 : pTable = chAnsiToPC775; break; case 864 : pTable = chAnsiToPC864; break; // Arabic OS/2 @@ -848,19 +830,7 @@ T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 78 if ( (GetOEMCP() = case 813 : pTable = chAnsiToPC813; pInvTable = chPC813ToAnsi; break; case 737 : T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 79 if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869) ) { // fix for sev1 Greek: Win NT problem (01/09/23)"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869) ) - { // fix for sev1 Greek: Win NT problem (01/09/23) - usCP = 869; - pTable = chAnsiToPC869; - pInvTable = chPC869ToAnsi; - } - else - { - pTable = chAnsiToPC737; - pInvTable = chPC737ToAnsi; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE + break; case 775 : pTable = chAnsiToPC775; pInvTable = chPC775ToAnsi; break; case 864 : pTable = chAnsiToPC864; pInvTable = chPC864ToAnsi; break; // Arabic OS/2 @@ -929,28 +899,7 @@ T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 79 if ( (GetOEMCP() = // store copy of string in internal buffers // endif //+----------------------------------------------------------------------------+ -VOID UtlReplString( SHORT sID, PSZ pszString ) -{ - PSZ pszBuffer; // buffer for new string value - - if ( (sID > QST_FIRST) && (sID < QST_LAST) ) - { - if ( UtlAlloc( (PVOID *)&pszBuffer, 0L, - (LONG) get_max( MIN_ALLOC, strlen(pszString)+1 ), - ERROR_STORAGE ) ) - { - USHORT usTask = UtlGetTask(); - strcpy( pszBuffer, pszString ); - if ( UtiVar[usTask].pszQueryArea[sID] ) - { - UtlAlloc( (PVOID *)&(UtiVar[usTask].pszQueryArea[sID]), 0L, 0L, NOMSG ); - } /* endif */ - - UtiVar[usTask].pszQueryArea[sID] = pszBuffer; - } /* endif */ - } /* endif */ -} //ulLen in number of CHAR_W's which are in pszUni! ULONG UtlDirectUnicode2AnsiBuf( PSZ_W pszUni, PSZ pszAnsi, ULONG ulLen, LONG lBufLen, diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt deleted file mode 100755 index 41f81704..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required (VERSION 3.10) -project(OpenTM2_Markup_OtmPlugins) - -add_subdirectory(common) - -set(OpenTM2_Markup_OtmPlugins_Source_Files - OtmMarkupTable.cpp - OtmMarkupTablePlugin.cpp) - - -add_library(OtmPlugins ${OpenTM2_Markup_OtmPlugins_Source_Files}) - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR deleted file mode 100755 index 7ebd20f5..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR +++ /dev/null @@ -1,26 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMBMRI Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ANSI=YES to enable detection of strings enclosed in double quotes -* ANSINO=NO to disable detection of strings enclosed in double quotes -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ANSI=YES diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL deleted file mode 100755 index 13f36d23..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX deleted file mode 100755 index c5831af3..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX +++ /dev/null @@ -1,873 +0,0 @@ - - - - - -OTMBMRI - Windows resource files - OTMAMRI - * - ? - ASCII - NO - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - ACCELTABLE - ' ,|\r\n' - - - ASSOCTABLE - ' ,|\r\n' - - - AUTOCHECKBOX - ' ,|\r\n' - - - AUTORADIOBUTTON - ' ,|\r\n' - - - BS_3STATE - ' ,|\r\n' - - - BS_AUTO3STATE - ' ,|\r\n' - - - BS_AUTOCHECKBOX - ' ,|\r\n' - - - BS_AUTORADIOBUTTON - ' ,|\r\n' - - - BS_AUTOSIZE - ' ,|\r\n' - - - BS_BITMAP - ' ,|\r\n' - - - BS_CHECKBOX - ' ,|\r\n' - - - BS_DEFAULT - ' ,|\r\n' - - - BS_HELP - ' ,|\r\n' - - - BS_ICON - ' ,|\r\n' - - - BS_MINIICON - ' ,|\r\n' - - - BS_NOBORDER - ' ,|\r\n' - - - BS_NOCURSORSELECT - ' ,|\r\n' - - - BS_NOPOINTERFOCUS - ' ,|\r\n' - - - BS_PRIMARYSTYLES - ' ,|\r\n' - - - BS_PUSHBUTTON - ' ,|\r\n' - - - BS_RADIOBUTTON - ' ,|\r\n' - - - BS_SYSCOMMAND - ' ,|\r\n' - - - BS_TEXT - ' ,|\r\n' - - - BS_USERBUTTON - ' ,|\r\n' - - - CBS_COMPATIBLE - ' ,|\r\n' - - - CBS_DROPDOWN - ' ,|\r\n' - - - CBS_DROPDOWNLIST - ' ,|\r\n' - - - CBS_SIMPLE - ' ,|\r\n' - - - CHECKBOX - ' ,|\r\n' - - - COMBOBOX - ' ,|\r\n' - - - CONTAINER - ' ,|\r\n' - - - CS_CLIPCHILDREN - ' ,|\r\n' - - - CS_CLIPSIBLINGS - ' ,|\r\n' - - - CS_FRAME - ' ,|\r\n' - - - CS_HITTEST - ' ,|\r\n' - - - CS_MOVENOTIFY - ' ,|\r\n' - - - CS_PARENTCLIP - ' ,|\r\n' - - - CS_PUBLIC - ' ,|\r\n' - - - CS_SAVEBITS - ' ,|\r\n' - - - CS_SIZEREDRAW - ' ,|\r\n' - - - CS_SYNCPAINT - ' ,|\r\n' - - - CTEXT - ' ,|\r\n' - - - CTLDATA - ' ,|\r\n' - - - DEFAULTICON - ' ,|\r\n' - - - DEFPUSHBUTTON - ' ,|\r\n' - - - DLGINCLUDE - ' ,|\r\n' - - - DLGTEMPLATE - ' ,|\r\n' - - - EDITTEXT - ' ,|\r\n' - - - ENTRYFIELD - ' ,|\r\n' - - - ES_ANY - ' ,|\r\n' - - - ES_AUTOSCROLL - ' ,|\r\n' - - - ES_AUTOSIZE - ' ,|\r\n' - - - ES_AUTOTAB - ' ,|\r\n' - - - ES_CENTER - ' ,|\r\n' - - - ES_COMMAND - ' ,|\r\n' - - - ES_DBCS - ' ,|\r\n' - - - ES_LEFT - ' ,|\r\n' - - - ES_MARGIN - ' ,|\r\n' - - - ES_MIXED - ' ,|\r\n' - - - ES_READONLY - ' ,|\r\n' - - - ES_RIGHT - ' ,|\r\n' - - - ES_SBCS - ' ,|\r\n' - - - ES_UNREADABLE - ' ,|\r\n' - - - FCF_ACCELTABLE - ' ,|\r\n' - - - FCF_AUTOICON - ' ,|\r\n' - - - FCF_BORDER - ' ,|\r\n' - - - FCF_DBE_APPSTAT - ' ,|\r\n' - - - FCF_DLGBORDER - ' ,|\r\n' - - - FCF_HIDEBUTTON - ' ,|\r\n' - - - FCF_HIDEMAX - ' ,|\r\n' - - - FCF_HORZSCROLL - ' ,|\r\n' - - - FCF_ICON - ' ,|\r\n' - - - FCF_MAXBUTTON - ' ,|\r\n' - - - FCF_MENU - ' ,|\r\n' - - - FCF_MINBUTTON - ' ,|\r\n' - - - FCF_MINMAX - ' ,|\r\n' - - - FCF_MOUSEALIGN - ' ,|\r\n' - - - FCF_NOBYTEALIGN - ' ,|\r\n' - - - FCF_NOMOVEWITHOWNER - ' ,|\r\n' - - - FCF_SCREENALIGN - ' ,|\r\n' - - - FCF_SHELLPOSITION - ' ,|\r\n' - - - FCF_SIZEBORDER - ' ,|\r\n' - - - FCF_STANDARD - ' ,|\r\n' - - - FCF_SYSMENU - ' ,|\r\n' - - - FCF_SYSMODAL - ' ,|\r\n' - - - FCF_TASKLIST - ' ,|\r\n' - - - FCF_TITLEBAR - ' ,|\r\n' - - - FCF_VERTSCROLL - ' ,|\r\n' - - - FS_ACCELTABLE - ' ,|\r\n' - - - FS_AUTOICON - ' ,|\r\n' - - - FS_BORDER - ' ,|\r\n' - - - FS_DBE_APPSTAT - ' ,|\r\n' - - - FS_DLGBORDER - ' ,|\r\n' - - - FS_ICON - ' ,|\r\n' - - - FS_MOUSEALIGN - ' ,|\r\n' - - - FS_NOBYTEALIGN - ' ,|\r\n' - - - FS_NOMOVEWITHOWNER - ' ,|\r\n' - - - FS_SCREENALIGN - ' ,|\r\n' - - - FS_SHELLPOSITION - ' ,|\r\n' - - - FS_SIZEBORDER - ' ,|\r\n' - - - FS_STANDARD - ' ,|\r\n' - - - FS_SYSMODAL - ' ,|\r\n' - - - FS_TASKLIST - ' ,|\r\n' - - - GROUPBOX - ' ,|\r\n' - - - HELPTABLE - ' ,|\r\n' - - - LISTBOX - ' ,|\r\n' - - - LS_EXTENDEDSEL - ' ,|\r\n' - - - LS_HORZSCROLL - ' ,|\r\n' - - - LS_MULTIPLESEL - ' ,|\r\n' - - - LS_NOADJUSTPOS - ' ,|\r\n' - - - LS_OWNERDRAW - ' ,|\r\n' - - - LTEXT - ' ,|\r\n' - - - MESSAGETABLE - ' ,|\r\n' - - - MIA_CHECKED - ' ,|\r\n' - - - MIA_DISABLED - ' ,|\r\n' - - - MIA_FRAMED - ' ,|\r\n' - - - MIA_HILITED - ' ,|\r\n' - - - MIA_NODISMISS - ' ,|\r\n' - - - MIS_BITMAP - ' ,|\r\n' - - - MIS_BREAK - ' ,|\r\n' - - - MIS_BREAKSEPARATOR - ' ,|\r\n' - - - MIS_BUTTONSEPARATOR - ' ,|\r\n' - - - MIS_GROUP - ' ,|\r\n' - - - MIS_HELP - ' ,|\r\n' - - - MIS_MULTMENU - ' ,|\r\n' - - - MIS_OWNERDRAW - ' ,|\r\n' - - - MIS_SEPARATOR - ' ,|\r\n' - - - MIS_SINGLE - ' ,|\r\n' - - - MIS_STATIC - ' ,|\r\n' - - - MIS_SUBMENU - ' ,|\r\n' - - - MIS_SYSCOMMAND - ' ,|\r\n' - - - MIS_TEXT - ' ,|\r\n' - - - MLE - ' ,|\r\n' - - - MS_ACTIONBAR - ' ,|\r\n' - - - MS_CONDITIONALCASCADE - ' ,|\r\n' - - - MS_TITLEBUTTON - ' ,|\r\n' - - - MS_VERTICALFLIP - ' ,|\r\n' - - - NOTEBOOK - ' ,|\r\n' - - - POINTER - ' ,|\r\n' - - - PUSHBUTTON - ' ,|\r\n' - - - RADIOBUTTON - ' ,|\r\n' - - - RCDATA - ' ,|\r\n' - - - RTEXT - ' ,|\r\n' - - - SBS_AUTOSIZE - ' ,|\r\n' - - - SBS_AUTOTRACK - ' ,|\r\n' - - - SBS_HORZ - ' ,|\r\n' - - - SBS_THUMBSIZE - ' ,|\r\n' - - - SBS_VERT - ' ,|\r\n' - - - SLIDER - ' ,|\r\n' - - - SPINBUTTON - ' ,|\r\n' - - - SS_AUTOSIZE - ' ,|\r\n' - - - SS_BITMAP - ' ,|\r\n' - - - SS_BKGNDFRAME - ' ,|\r\n' - - - SS_BKGNDRECT - ' ,|\r\n' - - - SS_FGNDFRAME - ' ,|\r\n' - - - SS_FGNDRECT - ' ,|\r\n' - - - SS_GROUPBOX - ' ,|\r\n' - - - SS_HALFTONEFRAME - ' ,|\r\n' - - - SS_HALFTONERECT - ' ,|\r\n' - - - SS_ICON - ' ,|\r\n' - - - SS_SYSICON - ' ,|\r\n' - - - SS_TEXT - ' ,|\r\n' - - - STRINGTABLE - ' ,|\r\n' - - - WC_APPSTAT - ' ,|\r\n' - - - WC_BUTTON - ' ,|\r\n' - - - WC_CIRCULARSLIDER - ' ,|\r\n' - - - WC_COMBOBOX - ' ,|\r\n' - - - WC_CONTAINER - ' ,|\r\n' - - - WC_DBE_KKPOPUP - ' ,|\r\n' - - - WC_ENTRYFIELD - ' ,|\r\n' - - - WC_FRAME - ' ,|\r\n' - - - WC_KBDSTAT - ' ,|\r\n' - - - WC_LISTBOX - ' ,|\r\n' - - - WC_MENU - ' ,|\r\n' - - - WC_MLE - ' ,|\r\n' - - - WC_MMPMFIRST - ' ,|\r\n' - - - WC_MMPMLAST - ' ,|\r\n' - - - WC_NOTEBOOK - ' ,|\r\n' - - - WC_PECIC - ' ,|\r\n' - - - WC_SCROLLBAR - ' ,|\r\n' - - - WC_SLIDER - ' ,|\r\n' - - - WC_SPINBUTTON - ' ,|\r\n' - - - WC_STATIC - ' ,|\r\n' - - - WC_TITLEBAR - ' ,|\r\n' - - - WC_VALUESET - ' ,|\r\n' - - - WS_ANIMATE - ' ,|\r\n' - - - WS_CLIPCHILDREN - ' ,|\r\n' - - - WS_CLIPSIBLINGS - ' ,|\r\n' - - - WS_DISABLED - ' ,|\r\n' - - - WS_GROUP - ' ,|\r\n' - - - WS_MAXIMIZED - ' ,|\r\n' - - - WS_MINIMIZED - ' ,|\r\n' - - - WS_MULTISELECT - ' ,|\r\n' - - - WS_PARENTCLIP - ' ,|\r\n' - - - WS_SAVEBITS - ' ,|\r\n' - - - WS_SYNCPAINT - ' ,|\r\n' - - - WS_TABSTOP - ' ,|\r\n' - - - WS_VISIBLE - ' ,|\r\n' - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak deleted file mode 100755 index cd0d709d..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak +++ /dev/null @@ -1,55 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright (C) 1998-2017, International Business Machines -# Corporation and others. All rights reserved -# --------------------------------------------------------------------------- -SRCPATH = $(_OTMMARKUP_SRC)\otmamri -CMNPATH = $(_OTMMARKUP_SRC)\common -INCLUDEOPT = /I$(SRCPATH) /I$(CMNPATH) - - -#CFLAGS = $(_OTMMARKUP_CFLAGS) $(INCLUDEOPT) -CFLAGS = $(_CL_CPP_OPTIONS_DLL) $(INCLUDEOPT) -LINKFLAGS = $(_OTMMARKUP_LINKFLAGS) - - -.all: \ - $(_OTMMARKUP_OBJ)\otmbmri.obj \ - $(_OBJ)\eqfparse.obj \ - $(_OTMMARKUP_DLL)\otmbmri.dll - -#--------------------------------------- -# Compile OBJs -#--------------------------------------- -$(_OTMMARKUP_OBJ)\otmbmri.obj: \ - $(SRCPATH)\otmbmri.c \ - $(_INC)\eqfparse.h \ - $(CMNPATH)\usrcalls.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\otmbmri $(SRCPATH)\otmbmri.c /Fd$(_BIN)\EQFSTART.PDB /Fe$(_BIN)\ - - -#--------------------------------------- -# Link DLL -#--------------------------------------- -$(_OTMMARKUP_DLL)\otmbmri.dll: \ - $(_OTMMARKUP_OBJ)\otmbmri.obj \ - $(_OTMMARKUP_OBJ)\eqfcalls.obj \ - $(_OTMMARKUP_OBJ)\usrcalls.obj \ - $(_OBJ)\eqfparse.obj - @if exist $(_OTMMARKUP_DLL)\otmbmri.dll erase $(_OTMMARKUP_DLL)\otmbmri.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmbmri.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmbmri.dll >>$(_ERR) - @$(_LINKER) >>$(_ERR) @<< -$(_OTMMARKUP_OBJ)\otmbmri.obj -$(_OTMMARKUP_OBJ)\eqfcalls.obj -$(_OTMMARKUP_OBJ)\usrcalls.obj -$(_OBJ)\eqfparse.obj -/OUT:$(_OTMMARKUP_DLL)\otmbmri.dll /nologo /MACHINE:IX86 /ALIGN:0X1000 /DRIVER /DLL /NOD -/MAP:$(_OTMMARKUP_MAP)\otmbmri.map -$(_LINK_LIB_CRT) $(_LIB)\otmbase.lib $(_LIB)\OTMLinguistic.lib -<< - @copy $(_OTMMARKUP_DLL)\otmbmri.dll $(_OTMMARKUP_RELEASE_DIR)\BIN /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmamri\otmamri.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmamri\otmamri.chr $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL deleted file mode 100755 index 38b32b3d..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBX deleted file mode 100755 index 62334d64..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBX +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - Plain text (ANSI) - OTMANSI - * - ? - ANSI - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL deleted file mode 100755 index acd9f6e1..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBX deleted file mode 100755 index 599ef61e..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBX +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - Plain text (ASCII) - OTMASCII - % - ? - ASCII - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL deleted file mode 100755 index 8d33573f..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBX deleted file mode 100755 index f936cbbd..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBX +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - Plain text (UTF-16) - Plain text (UTF-16) - YES - UNICODE - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL deleted file mode 100755 index 7026dec1..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBX deleted file mode 100755 index f653eb89..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBX +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - Plain text (UTF-8) - OTMUTF8 - % - ? - UTF8 - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR deleted file mode 100755 index 7ebd20f5..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR +++ /dev/null @@ -1,26 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMBMRI Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ANSI=YES to enable detection of strings enclosed in double quotes -* ANSINO=NO to disable detection of strings enclosed in double quotes -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ANSI=YES diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL deleted file mode 100755 index 7fbb7797..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX deleted file mode 100755 index 05a293e2..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX +++ /dev/null @@ -1,157 +0,0 @@ - - - - - -OTMBMRI - Double Quoted text (ANSI) - OTMADQUO - * - ? - ASCII - YES - - "" - 2 - - - ... - 3 - - - /P0 - 3 - - - /P01 - 4 - - - /P02 - 4 - - - /P03 - 4 - - - /P04 - 4 - - - /P05 - 4 - - - /P06 - 4 - - - /P07 - 4 - - - /P08 - 4 - - - /P09 - 4 - - - /P1 - 3 - - - /P2 - 3 - - - /P3 - 3 - - - /P4 - 3 - - - /P5 - 3 - - - /P6 - 3 - - - /P7 - 3 - - - /P8 - 3 - - - /P9 - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL deleted file mode 100755 index 2293a277..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX deleted file mode 100755 index 06283f7c..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX +++ /dev/null @@ -1,81 +0,0 @@ - - - - - -OTMBMRI - Double Quoted text (ASCII) - OTMDQUOT - * - ? - ASCII - YES - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR deleted file mode 100755 index bd66e8ae..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR +++ /dev/null @@ -1,27 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMQUOTE Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ESCAPECHARS=YES to enable escape character handling -* ESCAPECHARS=NO to disable escape character handling -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ESCAPECHARS=YES - \ No newline at end of file diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL deleted file mode 100755 index d89b2096..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX deleted file mode 100755 index 2e16443b..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX +++ /dev/null @@ -1,81 +0,0 @@ - - - - - -OTMQUOTE - Double Quoted text (UTF-8) - OTMUDQUO - * - ? - UTF8 - YES - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL deleted file mode 100755 index cf62acb4..00000000 Binary files a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL and /dev/null differ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX deleted file mode 100755 index a9635681..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX +++ /dev/null @@ -1,6246 +0,0 @@ - - - - - -HTML 3.2/4.01 -OTMHTM32 -OTMHTM32 -~ - - #INCLUDE - TSL - 8 - 1 - - - #IF - TSL - 3 - 1 - - - #IFDEF - TSL - 6 - 1 - - - #IFNDEF - TSL - 7 - 1 - - - #ELIF - TSL - 5 - 1 - - - #ELSE - TSL - 5 - 1 - - - #ENDIF - TSL - 6 - 1 - - - #UNDEF - TSL - 6 - 1 - - - $ - 1 - ENDNEUTRAL - - - $$~ - ' $.)\r\n\t>' - ENDNEUTRAL - CLS_TRANSLVAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 5 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - - ENDDEL - 5 - - - - - - - -HTML 3.2/4.01 (UTF-8) -OTMUHTM3 -OTMHTM32 -~ - - #INCLUDE - TSL - 8 - 1 - - - #IF - TSL - 3 - 1 - - - #IFDEF - TSL - 6 - 1 - - - #IFNDEF - TSL - 7 - 1 - - - #ELIF - TSL - 5 - 1 - - - #ELSE - TSL - 5 - 1 - - - #ENDIF - TSL - 6 - 1 - - - #UNDEF - TSL - 6 - 1 - - - $ - 1 - ENDNEUTRAL - - - $$~ - ' $.)\r\n\t>' - ENDNEUTRAL - CLS_TRANSLVAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 5 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - - ENDDEL - 5 - - - " ) ; - if ( str2 ) { - for( i=0 ; str1", 8 ) ; - } - } - if ( ! wcsicmp( temp_str, L"STARTNON-TRANSLATABLEVALUE" ) ) { - *bInNonTransAttrValue = TRUE ; - } else { - if ( ! wcsicmp( temp_str, L"ENDNON-TRANSLATABLEVALUE" ) ) { - *bInNonTransAttrValue = FALSE ; - } - } - } - } - - if ( ( *bInNonTrans ) && - ( ! bSkipStmt ) ) { - for( str1=sub_str ; *str1 ; ++str1 ) { - if ( IsDBCS( *str1 ) ) { - ++str1 ; - continue ; - } - if ( ! wcsncmp( str1, L"", 8 ) ; - str1 += 8 ; - } else - if ( ! wcsncmp( str1, L"-->", 3 ) ) { - str1 += 3 ; - memmove( str1+9, str1, (wcslen(str1)+1)*sizeof(wchar_t) ) ; - wcsncpy( str1, L"", 3 ) ) { - sTagState = TAG_NONE ; - sPrevTagState = TAG_NORMAL ; - if ( bInCommentCheck ) { /* Change '--->' to '- -->' */ - bInCommentCheck = FALSE ; - if ( ( ptrChar > Text ) && - ( *(ptrChar-1) == '-' ) ) { - memmove( ptrChar+1, ptrChar, (wcslen(ptrChar)+1)*sizeof(wchar_t) ) ; - *ptrChar = ' ' ; - } - } - ptrChar += 2 ; - } else { - if ( ! wcsncmp( ptrChar, L" ", 6 ) ) { - ptrChar += 5 ; - } else { - if ( ! wcsncmp( ptrChar, L"[TWB", 4 ) ) { /* JSP internal tags */ - ptrChar2 = wcschr( ptrChar, ']' ) ; - if ( ptrChar2 ) { - if ( sTagState == TAG_SCRIPT ) { - if ( ! wcsncmp( ptrChar, L"[TWBSTART] [TWBLT]%", 19 ) ) { - memmove( ptrChar+7, ptrChar-1, (wcslen(ptrChar-1)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar-1, L"{TWBSCR}", 8 ) ; - ptrChar += 8 ; - sTagState = TAG_JSP ; - sPrevTagState = TAG_SCRIPT ; - } - } else { - if ( sTagState == TAG_JSP ) { - if ( ! wcsncmp( ptrChar, L"[TWBGT] [TWBSTOP] ", 18 ) ) { - memmove( ptrChar+18+10, ptrChar+18, (wcslen(ptrChar+18)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar+18, L" MAXENTITY_READ-2 ) || // (1024-2 ) - ( bIn2K ) ) && - ( !wcschr( Text, '\n' ) ) ) { - if ( !wcschr( Text, '>' ) ) { - ptrChar = wcschr( Text, '<' ) ; - if ( ptrChar ) - ptrChar2 = wcschr( ptrChar+1, '<' ) ; - else - ptrChar2 = 0 ; - if ( ( bIn2K ) || - ( ( ptrChar ) && - ( !ptrChar2 ) && - ( ( iswalpha( *(ptrChar+1) ) ) || - ( ( *(ptrChar+1) == '/' ) && - ( iswalpha( *(ptrChar+2) ) ) ) || - ( ( *(ptrChar+1) == '\\' ) && /* <\/xxxx> */ - ( *(ptrChar+2) == '/' ) && - ( iswalpha( *(ptrChar+3) ) ) ) ) ) ) { - if ( ! bIn2K ) - wcscpy( szQuote, L" " ) ; - bIn2K = TRUE ; - for( ptrChar=Text ; *ptrChar ; ++ptrChar ) { - if ( ( *(ptrChar) == '\\' ) && /* Skip escaped quotes */ - ( ( *(ptrChar+1) == '\"' ) || - ( *(ptrChar+1) == '\'' ) ) ) { - ++ptrChar ; - continue ; - } - if ( ( *ptrChar == '\"' ) || /* Process single/double quote */ - ( *ptrChar == '\'' ) ) { - if ( *(ptrChar+1) == *ptrChar ) { /* Skip consecutive quotes */ - ++ptrChar ; - continue ; - } - if ( szQuote[1] == *ptrChar ) {/* If 1st quote is a match, */ - if ( szQuote[2] == ' ' ) /* If no 2nd quote char, */ - szQuote[1] = ' ' ; /* then end of 1st quote */ - } else { - if ( szQuote[2] == *ptrChar ) /* If 2nd quote is a match,*/ - szQuote[2] = ' ' ; /* then end of 2nd quote */ - else - if ( szQuote[1] == ' ' ) /* If no 1st quote yet, */ - szQuote[1] = *ptrChar ;/* Set 1st quote char */ - else - szQuote[2] = *ptrChar ;/* Set 2nd quote char */ - } - } - } - wcscat( Text, L"{TWB2K}" ) ; - memset( szTemp, 0, 4*sizeof(wchar_t) ) ; - if ( szQuote[2] != ' ' ) { - szTemp[2] = szQuote[2] ; - wcscat( Text, &szTemp[2] ) ; - } - if ( szQuote[1] != ' ' ) { - szTemp[0] = szQuote[1] ; - wcscat( Text, szTemp ) ; - } - wcscat( Text, L">\n Text ) && - ( ( iswalpha( *(ptrChar+1) ) ) || /* */ - ( ( *(ptrChar+1) == '/' ) && /* */ - ( iswalpha( *(ptrChar+2) ) ) ) || - ( ( *(ptrChar+1) == '\\' ) && /* <\/xxxx> */ - ( *(ptrChar+2) == '/' ) && - ( iswalpha( *(ptrChar+3) ) ) ) ) ) { - wcsncpy( szTagName2, ptrChar+1, sizeof(szTagName2)/sizeof(wchar_t) ) ; - szTagName2[sizeof(szTagName2)/sizeof(wchar_t)-1] = 0 ; - wcstok( szTagName2, L" >\n\t\r" ) ; - wcsupr( szTagName2 ) ; - if ( szTagName2[0] == '/' ) - swprintf( szTempTag, L" %s ", &szTagName2[1] ) ; - else - if ( szTagName2[0] == '\\' ) - swprintf( szTempTag, L" %s ", &szTagName2[2] ) ; - else - swprintf( szTempTag, L" %s ", szTagName2 ) ; - if ( wcsstr( TAGLIST_NEUTRAL , szTempTag ) ) { - memmove( ptrChar+15, ptrChar, (wcslen(ptrChar)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar, L"{TWB2K}\n", 15 ) ; - } - } - } - } else { - bIn2K = FALSE ; - } -} - -//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -/***************************************************************************/ -/* Handle , - 8 - - - - 6 - ENDNEUTRAL - - - - 6 - ENDNEUTRAL - - - - 8 - - - - 8 - - - - 5 - - - - 11 - - - - 8 - - - - 5 - - - - 8 - - - - 8 - - - - 5 - - - - 5 - ENDNEUTRAL - - - - 4 - ENDNEUTRAL - - - - 5 - - - - 6 - ENDNEUTRAL - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c deleted file mode 100755 index 64178b75..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c +++ /dev/null @@ -1,691 +0,0 @@ -/* -* -* Copyright (C) 1998-2013 International Business Machines -* Corporation and others. All rights reserved -* -*/ -/****************************************************************************/ -/* */ -/* otmjdk11.c */ -/* */ -/****************************************************************************/ -/* */ -/* Description: */ -/* This function will examine a Java Resource Bundle and do the */ -/* segmentation processing. This DLL will have 4 entry points. */ -/* These entry points are listed below and are defined by TM2. */ -/* */ -/* */ -/* EQF_BOOL APIENTRY16 EQFPRESEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSource, */ -/* PSZ pTempSource, */ -/* PEQF_BOOL pfNoSegment) */ -/* EQF_BOOL APIENTRY16 EQFPOSTSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSource, */ -/* PSZ pSegTarget, */ -/* PTATAG pTATAG) */ -/* EQF_BOOL APIENTRY16 EQFPREUNSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSegTarget, */ -/* PSZ pTemp, */ -/* PTATAG pTATAG, */ -/* PEQF_BOOL pfNoUnseg) */ -/* EQF_BOOL APIENTRY16 EQFPOSTUNSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pTarget, */ -/* PTATAG pTATAG) */ -/* */ -/*==========================================================================*/ -/* Date : Who : Description */ -/*==========================================================================*/ -/* 7/08/2010: IBM : Original Source */ -/*==========================================================================*/ -/****************************************************************************/ - -#include "otmjdk11.h" - - -extern char szDocTargetLanguage[80]; /* From USRCALLS.C */ -extern char szDocSourceLanguage[80]; /* From USRCALLS.C */ -extern short sTPVersion ; /* From USRCALLS.C */ - BOOL bDBCSSrc ; - BOOL bDBCSTgt ; - BOOL bDBCSInfo ; - BOOL bJapanese ; - BOOL bKorean ; - BOOL bSChinese ; - BOOL bTChinese ; - BOOL bNoDBCS ; - - - -/****************************************************************************/ -/* */ -/* EQFPRESEG */ -/* */ -/****************************************************************************/ -/* */ -/* Description: */ -/* called after text segmentation is invoked, to change the */ -/* segmented source and target file before translation takes place. */ -/* */ -/* Arguments: PSZ ... pointer to markup table name. */ -/* */ -/* PSZ ... pointer to editor name. */ -/* */ -/* PSZ ... pointer to program path. */ -/* */ -/* PSZ ... pointer to segmented source file name. */ -/* */ -/* PSZ ... pointer to the segmented target file name. */ -/* */ -/* PTATAG ... pointer to tags inserted by text segmentation,*/ -/* ( see layout of tag structure ). */ -/* */ -/* Return: BOOL ... TRUE:processing was OK. */ -/* FALSE: an error occured during processing. */ -/* */ -/* Note: It is vital that the name of the segmented source and target */ -/* file is not changed! */ -/* */ -/* */ -/****************************************************************************/ - - -/*************************************************************************/ -/* Pre-segmentation, including access to progress window. */ -/*************************************************************************/ - - -EQF_BOOL /*APIENTRY*/ EQFPOSTSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pSegSource, - PSZ pSegTarget, - PTATAG_W pTATAGW, - HWND hSlider, - PEQF_BOOL pfKill ) - -{ - - return(TRUE); - -} - - -/*************************************************************************/ -/* Pre-unsegmentation, including access to progress window. */ -/*************************************************************************/ - -EQF_BOOL /*APIENTRY*/ EQFPREUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pSegTarget, - PSZ pTemp, - PTATAG_W pTATagW, - PEQF_BOOL pfNoUnseg, - PEQF_BOOL pfKill ) -{ - UCHAR tmpfilename[1024]; - UCHAR tmpfilename2[1024]; - char *szAltTempExt3 = ".$$3"; - char *szAltTempExt4 = ".$$4"; - short sType ; - BOOL bReturn = TRUE ; - BOOL bDBCS ; - - - PrepDocLanguageInfo( pSegTarget ) ; /* Set language unqiue processing, like DBCS */ - bDBCSSrc = SetDBCSLang( szDocSourceLanguage ) ; - bDBCSTgt = SetDBCSLang( szDocTargetLanguage ) ; - bDBCS = IsDBCS( '\x90' ) ; - - *pfNoUnseg = FALSE ; /* Unsegmentation is required. */ - - TM2ConvertShortToLong( pSegTarget, tmpfilename ); - - if (tmpfilename[0] == 0) { - strcpy(tmpfilename, pProgPath); - } - - // Check if LRB or Property file and process as required. - - CreateTempFileName2( pTemp, pSegTarget, szAltTempExt3, TEMPNAME_STARGET ) ; - bReturn = ConvertImport( pSegTarget, pTemp, EQF_UTF162UTF8 ) ; /* UTF-16 -> UTF-8 */ - - - if ( bReturn ) { - CreateTempFileName2( tmpfilename2, pSegTarget, szAltTempExt4, TEMPNAME_STARGET ) ; - - if ( ! chk_LRB_File( tmpfilename, pTemp, &sType, pTagTable ) ) { - - bReturn = ! ExportFile( pTemp, tmpfilename2 ); - - if ( bReturn ) { - bReturn = ConvertImport( tmpfilename2, pTemp, EQF_UTF82UTF16 ) ; /* UTF-8 to UTF-16 */ - } - - } else { - - bReturn = ExportLRBFile( pTemp, tmpfilename2, pTemp ) ; - - if ( bReturn ) { - bReturn = ConvertImport( tmpfilename2, pTemp, EQF_UTF82UTF16 ) ; /* UTF-8 to UTF-16 */ - } - - } - remove( tmpfilename2 ) ; - } - - - return( bReturn ) ; - -} /* EQFPREUNSEG2 */ - - - - - - -/*************************************************************************/ -/* Post-unsegmentation, including access to progress window. */ -/*************************************************************************/ - -EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pTarget, - PTATAG pTATag, - PEQF_BOOL pfKill ) -{ - char szTempFile[256] ; - char szMarkupTable[80] ; - UCHAR tmpfilename[1024]; - char *szAltTempExt5 = ".$$5"; - short sType ; - USHORT usCPType = EXPORT_CP_NONE ; - BOOL bReturn = TRUE ; - - - PrepDocLanguageInfo( pTarget ) ; /* Set language unqiue processing, like DBCS */ - bDBCSSrc = SetDBCSLang( szDocSourceLanguage ) ; - bDBCSTgt = SetDBCSLang( szDocTargetLanguage ) ; - - TM2ConvertShortToLong( pTarget, tmpfilename ); - - if (tmpfilename[0] == 0) - strcpy(tmpfilename, pProgPath); - - /*************************************************************************/ - /* Convert ASCII file into UTF-8 for export. */ - /*************************************************************************/ - strcpy( szMarkupTable, pTagTable ) ; - strupr( szMarkupTable ) ; - if ( ! strncmp( szMarkupTable, "OTMU", 4 ) ) { - usCPType = EXPORT_CP_UTF8 ; - bReturn = ConvertExport( pTarget, EQF_UTF162UTF8 ) ; /* UTF-16 -> UTF-8 */ - } else { - if ( ! strncmp( szMarkupTable, "OTMA", 4 ) ) { /* ANSI markup table */ - usCPType = EXPORT_CP_DBCS_ASCII ; - bReturn = ConvertExport( pTarget, EQF_UTF162ANSI ) ; /* UTF-16 -> ANSI */ - if ( ! strcmp( szDocTargetLanguage, "Kazakh" ) ) /* Kazakh */ - usCPType = EXPORT_CP_UTF8 ; - } else { - if ( ! strncmp( szMarkupTable, "OTMNJDK", 7 ) ) { - bReturn = ConvertToEscapedUnicode( pTarget ) ; /* UTF-16 -> Escaped Unicode */ - } else { - usCPType = EXPORT_CP_DBCS_ASCII ; - bReturn = ConvertExport( pTarget, EQF_UTF162ASCII ) ;/* UTF-16 -> ASCII */ - if ( ! strcmp( szDocTargetLanguage, "Kazakh" ) ) /* Kazakh */ - usCPType = EXPORT_CP_UTF8 ; - } - } - } - - - // Check if Property file, then process as required. - - if ( ! chk_LRB_File(tmpfilename, pTarget, &sType, pTagTable ) ) { - - CreateTempFileName2( szTempFile, pTarget, szAltTempExt5, TEMPNAME_TARGET ) ; - - ExportFile2( pTarget, szTempFile, usCPType ); - - - remove( szTempFile ) ; - - } - - return( bReturn ); -} - - -/*******************************************************************/ -/* */ -/* Check if file is a ListResourceBundle Type file */ -/* */ -/*******************************************************************/ - - -BOOL chk_LRB_File(char *sourceName, char *shortName, short *sType, - char* MarkupName ) -{ - FILE *sourceFile_ptr = NULL; - char line[MAX_LEN]; - char upperCaseBuff[MAX_LEN+1]; - PSZ pName, pDot ; - BOOL bLRB = FALSE ; - - *sType = LRB_NONE ; - memset(upperCaseBuff, 0, sizeof(upperCaseBuff)); - - strcpy(upperCaseBuff, sourceName); - StrUpr(upperCaseBuff); - - pName = strrchr( upperCaseBuff, '\\' ); /* Find file name */ - pDot = strrchr( upperCaseBuff, '.' ); /* Find file extension */ - - if (pName == NULL) { - pName = upperCaseBuff; - } - - if ( ( pDot > pName ) && /* 10-25-01 */ - ( ( ! strcmp( pDot, ".JAV" ) ) || - ( ! strcmp( pDot, ".JAVA" ) ) ) ) { - bLRB = TRUE; - } - - sourceFile_ptr = fopen (shortName,"rb"); - while (fgets(line, MAX_LEN, sourceFile_ptr)) { - memset(upperCaseBuff, 0, sizeof(upperCaseBuff)); - strcpy(upperCaseBuff, line); - StrUpr(upperCaseBuff); - if ( ( strstr( upperCaseBuff, "IMPORT JAVA." ) != NULL ) && - ( strchr( upperCaseBuff, ';' ) != NULL ) ) { /* 4-3-08 */ - bLRB = TRUE; - } - if ( strstr( upperCaseBuff, "EXTENDS" ) != NULL ) { - if ( strstr( upperCaseBuff, "RESOURCEBUNDLE" ) != NULL ) { - bLRB = TRUE; - } - break ; - } - } - fclose(sourceFile_ptr); - - - if ( *sType == LRB_NONE ) { - if ( bLRB ) { - *sType = LRB_STANDARD ; - } else { - *sType = PRB_STANDARD ; - } - } - return( bLRB ) ; -} - - - -/****************************************************************************/ -/* */ -/* TM2ConvertShortToLong */ -/* */ -/* Convert the TM/2 short file name to the long file name. */ -/* */ -/* szPath - Complete short name path, including TM/2 folder, file */ -/* szFileName - pointer to Empty string */ -/* */ -/* */ -/****************************************************************************/ -USHORT TM2ConvertShortToLong( UCHAR *szPath, UCHAR *szLong ) -{ - UCHAR szTempPath[MAX_LEN]; - UCHAR szShortName[MAX_LEN]; - UCHAR *path_ptr ; - UCHAR *short_ptr ; - UCHAR *temp_ptr ; - USHORT rc ; - - strcpy( szTempPath, szPath ) ; - strcpy( szShortName, szPath ) ; - - /* Pull folder path out of the full path name name FOLDER_DRIVE:\EQF\FOLDER_NAME.F00 */ - - path_ptr = (strstr( szTempPath, szFOLDER_EXT )); - if ( path_ptr ) { - *(path_ptr+2)= 0 ; - } else { - rc=10; // Return fail no folder path - return rc; - } - - -/* Pull short name out of the full path name name NAME.000 */ - - short_ptr = strrchr(szShortName, '\\'); - - if (short_ptr ) { - short_ptr += 1 ; - } else { - rc = 11; // Return fail no short filename - return rc; - } - temp_ptr = strrchr( short_ptr, '.' ) ; - if ( ( temp_ptr ) && - ( strlen(temp_ptr) == 5 ) ) { - *(temp_ptr+4) = 0 ; // Remove trailing letter from name NAME.000A - } - szLong[0] = 0 ; // Clear the long filename buff TM/2 requires to know which conversion to do - - -//kml : P007994 do conversion to appropriate filename - rc = EQFCONVERTFILENAMES( szTempPath, szLong, short_ptr ) ; - if ( (rc == 0) && (szLong[0] != 0) ){ - strcpy( short_ptr, szLong ) ; - } else { - strcpy( szLong, short_ptr ) ; - } - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* ConvertFromEscapedUnicode */ -/* */ -/* Convert the Java escaped Unicode data into UTF-16. */ -/* */ -/* szInFile - Input file (escaped Unicode). */ -/* szOutFile - Output file (UTF-16). */ -/* */ -/****************************************************************************/ -USHORT ConvertFromEscapedUnicode( UCHAR *szInFile, UCHAR *szOutFile ) -{ - FILE *fInput ; - FILE *fOutput ; - CHAR szInChars[10] ; - CHAR cChar ; - LONG lFilePos ; - USHORT usChar ; - USHORT usChar1 ; - USHORT usChar2 ; - USHORT i ; - USHORT rc = 1 ; - BOOL bSkipRead = FALSE ; - BOOL bLoop = TRUE ; - - - fInput = fopen( szInFile, "rb" ) ; - fOutput = fopen( szOutFile, "wb" ) ; - fprintf( fOutput, "\xFF\xFE" ) ; /* Write BOM x'FFFE" for UTF-16 file */ - - /*------------------------------------------------------------------------*/ - /* Process all characters in this file. */ - /*------------------------------------------------------------------------*/ - while( bLoop==TRUE && rc==1 ) { - if ( ! bSkipRead ) { - usChar1 = fgetc( fInput ) ; - if ( feof(fInput) ) - break ; - } - bSkipRead = FALSE ; - - if ( usChar1 == '\\' ) { /* Escaped character */ - usChar2 = fgetc( fInput ) ; /* Get next character */ - if ( usChar2 != 'u' ) { /* If not Unicode, */ - fprintf( fOutput, "%c", usChar1 ) ; /* then write out '\' */ - fputc( '\x00', fOutput ) ; - if ( usChar2 == '\\' ) { /* If escaped backslash, */ - fprintf( fOutput, "%c", usChar2 ) ; - fputc( '\x00', fOutput ) ; - } else { - usChar1 = usChar2 ; - bSkipRead = TRUE ; /* Next char already read */ - } - continue ; - } - lFilePos = ftell( fInput ) ; - for( i=0 ; i<4 ; ++i ) { /* Escaped Unicode \u____ */ - cChar = (UCHAR)fgetc(fInput) ; /* Get next character */ - if ( isxdigit(cChar) ) { /* If valid hex digit */ - if ( isdigit(cChar) ) { /* Set hex digit value */ - usChar = cChar - '0' ; /* 0 - 9 */ - } else - if (cChar <= 'F' ) { - usChar = cChar - 'A' + 10 ; /* A - F */ - } else { - usChar = cChar - 'a' + 10 ; /* a - f */ - } - szInChars[i] = usChar ; /* Save value */ - } else { - rc = 0 ; - fseek( fInput, lFilePos, 0 ) ; /* Reset file reading pos */ - fputc( '\\', fOutput ) ; - fputc( '\x00', fOutput ) ; - fputc( 'u', fOutput ) ; - fputc( '\x00', fOutput ) ; - break ; - } - } - - if ( rc != 0 ) { /* Write UTF-16 value */ - if ( ( szInChars[2] == 5 ) && - ( szInChars[3] == 12 ) && - ( szInChars[0] == 0 ) && - ( szInChars[1] == 0 ) ) { - fwprintf( fOutput, L"\\u005%c",cChar ) ; - } else - if ( ( ( szInChars[3] == 0 ) || - ( szInChars[3] == 10 ) || - ( szInChars[3] == 13 ) ) && - ( szInChars[0] == 0 ) && - ( szInChars[1] == 0 ) && - ( szInChars[2] == 0 ) ) { - fwprintf( fOutput, L"\\u000%c",cChar ) ; - } else - if ( ( szInChars[0] == 15 ) && - ( szInChars[1] == 15 ) && - ( szInChars[2] == 15 ) && - ( ( szInChars[3] == 12 ) || - ( szInChars[3] == 13 ) ) ) { - fwprintf( fOutput, L"\\uFFF%c",cChar ) ; - } else { - fprintf( fOutput, "%c", (UCHAR)(szInChars[2]*16+szInChars[3]) ) ; - fprintf( fOutput, "%c", (UCHAR)(szInChars[0]*16+szInChars[1]) ) ; - } - } else { - rc = 1 ; - } - } else { - fprintf( fOutput, "%c", usChar1 ) ; /* Write non-escaped char */ - fputc( '\x00', fOutput ) ; - } - } - fclose( fInput ) ; - fclose( fOutput ) ; - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* ConvertToEscapedUnicode */ -/* */ -/* Convert the UTF-16 data into Java escaped Unicode. */ -/* */ -/* szFile - Input file (UTF-16). */ -/* */ -/****************************************************************************/ -USHORT ConvertToEscapedUnicode( UCHAR *szFile ) -{ - FILE *fInput ; - FILE *fOutput ; - UCHAR szTempFile[256]; - char *szAltTempExt6 = ".$$6"; - CHAR szOutChars[10] ; - USHORT usChar1 ; - USHORT usChar2 ; - USHORT rc = 1 ; - BOOL bLoop = TRUE ; - - CreateTempFileName2( szTempFile, szFile, szAltTempExt6, TEMPNAME_TARGET ) ; - - fInput = fopen( szFile, "rb" ) ; - fOutput = fopen( szTempFile, "wb" ) ; - usChar1 = fgetc( fInput ) ; /* Skip BOM x'FFFE' for UTF-16 file */ - usChar2 = fgetc( fInput ) ; - - /*----------------------------------------------------------------------*/ - /* Process all characters in this file. */ - /*----------------------------------------------------------------------*/ - while( bLoop == TRUE ){ - usChar1 = fgetc( fInput ) ; /* Get next UTF-16 char */ - usChar2 = fgetc( fInput ) ; - if ( feof(fInput) ) - break ; - - if ( ( usChar1 < 128 ) && /* If non-escaped char, */ - ( usChar2 == 0 ) ) { - szOutChars[0] = usChar1 ; /* Write out 1 character */ - szOutChars[1] = 0 ; - } else { /* Write escaped Unicode */ - sprintf(szOutChars, "\\u%2.2x%2.2x", usChar2, usChar1 ) ; - } - fputs( szOutChars,fOutput ) ; - - } - fclose( fInput ) ; - fclose( fOutput ) ; - - CopyFile( szTempFile, szFile, FALSE); - remove( szTempFile ) ; - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* GetCharLength */ -/* */ -/* Determine the length of this character. */ -/* */ -/* Check for differences between ANSI/ASCII and UTF-8. */ -/* Unicode UTF-8 */ -/* Min Max Byte Sequence in Binary */ -/* --------------- ----------------------------------- */ -/* 0000 007F 0vvvvvvv */ -/* 0080 07FF 110vvvvv 10vvvvvv */ -/* 0800 FFFF 1110vvvv 10vvvvvv 10vvvvvv */ -/* 10000 10FFFF 11110vvv 10vvvvvv 10vvvvvv 10vvvvvv */ -/* */ -/* Input: szInput - Input string to check. */ -/* Return: - If UTF-8 char, then return the length of char. */ -/* - If note UTF-8, return 1. */ -/* */ -/****************************************************************************/ - -SHORT GetCharLength( UCHAR * szCheck ) -{ - SHORT sLength = 1 ; - - if ( szCheck[0] >= (UCHAR)'\x80' ) { /* Possible UTF-8 */ - if ( ( szCheck[1] >= (UCHAR)'\x80' ) && - ( szCheck[1] <= (UCHAR)'\xBF' ) ) { - sLength = 2 ; /* Valid 2-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xE0' ) { - if ( ( szCheck[2] >= (UCHAR)'\x80' ) && - ( szCheck[2] <= (UCHAR)'\xBF' ) ) { - sLength = 3 ; /* Valid 3-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xF0' ) { - if ( ( szCheck[3] >= (UCHAR)'\x80' ) && - ( szCheck[3] <= (UCHAR)'\xBF' ) ) { - sLength = 4 ; /* Valid 4-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xF8' ) { - sLength = 1 ; /* Not UTF-8 character */ - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } -//printf("CC: %d ##%s##\n",sLength,szCheck); - - return( sLength ) ; -} - -/****************************************************************************/ -/* */ -/* SetDBCSLang */ -/* */ -/* Determine if language is DBCS. */ -/* */ -/* Input: szLangauge - Language to chekc. */ -/* Return: - TRUE. Language is DBCS. */ -/* - FALSE. Language is not DBCS. */ -/* */ -/****************************************************************************/ - -BOOL SetDBCSLang( UCHAR * szLanguage ) -{ - if ( ( ! stricmp( szLanguage, "Japanese" ) ) || - ( ! stricmp( szLanguage, "Korean" ) ) || - ( ! stricmp( szLanguage, "Chinese(simpl.)" ) ) || - ( ! stricmp( szLanguage, "Chinese(trad.)" ) ) ) - return( TRUE ) ; - - return( FALSE ) ; -} - - -/******************************************************************************* -* -* function: EQFQUERYEXITINFO -* -* ----------------------------------------------------------------------------- -* Description: -* Determine the files which are required for this markup table. -* Parameters: -* PSZ // name of the markup table, e.g. "OTMHTM32" -* USHORT // type of information being queried -* PSZ // buffer area receiving the information returned by the exit -* USHORT // length of buffer area -* Return: -* EQF_BOOL // 0=Successful -*******************************************************************************/ - - -USHORT /*APIENTRY*/ EQFQUERYEXITINFO( PSZ pszTagTable, // name of the markup table, e.g. "OTMHTM32" - USHORT usMode, // type of information being queried - PSZ pszBuffer, // buffer area receiving the information returned by the exit - USHORT usBufLen // length of buffer area -) -{ - - if( usMode == QUERYEXIT_ADDFILES) { - QueryExportFiles(pszTagTable, pszBuffer, usBufLen, FALSE); - } - - return 0; -} diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h deleted file mode 100755 index a39fb220..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -* -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -* -*/ -/********************************************************************* - * - * FUNCTION: - * - * Segmentation routine to segment JDK 1.1 Property files - * for Translation in TM/2 - * - ********************************************************************/ - -/*==========================================================================*/ -/* Date : Who : Description */ -/*==========================================================================*/ -/* 7/08/2010: IBM : Original Source */ -/*==========================================================================*/ - -/* Header file for otmjdk11.dll */ - -#ifndef _OTMJDK11_H_INCLUDE_ - #define _OTMJDK11_H_INCLUDE_ - - #define INCL_DOSNLS - #define INCL_WINWINDOWMGR - #define INCL_WININPUT - #define INCL_DOSFILEMGR - #define INCL_DOSERRORS - #define INCL_BASE - #define MAX_SIZE 144 - #define MAX_LEN 1024 - #define MAX_SEGMENT_SIZE 2048 - #define MAX_CONTEXT_SIZE 2048 - #define szFOLDER_EXT "00\\" - - #define LRB_NONE 0 - #define LRB_STANDARD 1 - - #define PRB_STANDARD 1 - - #define EXPORT_CP_NONE 0 - #define EXPORT_CP_DBCS_ASCII 1 - #define EXPORT_CP_UTF8 2 - - #include - #include - #include - #include - #include - #include - #include "usrcalls.h" - #include "reseq.h" - #include "javaseg.h" - #include "javapro.h" - - #include - #include - - - -BOOL chk_LRB_File(char *sourceName, char *shortName, short *Type, char*MarkupName ); -USHORT TM2ConvertShortToLong( UCHAR *szPath, UCHAR *szFileName ); -USHORT ConvertFromEscapedUnicode( UCHAR *szInFile, UCHAR *szOutFile ); -USHORT ConvertToEscapedUnicode( UCHAR *szFile ); -SHORT GetCharLength( UCHAR *szText ); -BOOL SetDBCSLang( UCHAR *szLanguage ); -BOOL IsHTMLTag( UCHAR *, USHORT * ) ; - -#endif - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak deleted file mode 100755 index 4fa7435b..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak +++ /dev/null @@ -1,81 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright (C) 1998-2013, International Business Machines -# Corporation and others. All rights reserved -# --------------------------------------------------------------------------- -SRCPATH = $(_OTMMARKUP_SRC)\otmjdk21 -CMNPATH = $(_OTMMARKUP_SRC)\common -INCLUDEOPT = /I$(SRCPATH) - - -CFLAGS = $(_OTMMARKUP_CFLAGS) $(INCLUDEOPT) -LINKFLAGS = $(_OTMMARKUP_LINKFLAGS) - - -.all: \ - $(_OTMMARKUP_OBJ)\otmjdk11.obj \ - $(_OTMMARKUP_OBJ)\javapro.obj \ - $(_OTMMARKUP_OBJ)\javaseg.obj \ - $(_OTMMARKUP_OBJ)\jcontext.obj \ - $(_OTMMARKUP_DLL)\otmjdk11.dll - -#--------------------------------------- -# Compile OBJs -#--------------------------------------- -$(_OTMMARKUP_OBJ)\otmjdk11.obj: \ - $(SRCPATH)\otmjdk11.c \ - $(SRCPATH)\otmjdk11.h \ - $(SRCPATH)\javapro.h \ - $(SRCPATH)\javaseg.h \ - $(CMNPATH)\usrcalls.h \ - $(CMNPATH)\reseq.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\otmjdk11 $(SRCPATH)\otmjdk11.c - -$(_OTMMARKUP_OBJ)\javapro.obj: \ - $(SRCPATH)\javapro.c \ - $(SRCPATH)\javapro.h \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\javapro $(SRCPATH)\javapro.c - -$(_OTMMARKUP_OBJ)\javaseg.obj: \ - $(SRCPATH)\javaseg.c \ - $(SRCPATH)\javaseg.h \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\javaseg $(SRCPATH)\javaseg.c - -$(_OTMMARKUP_OBJ)\jcontext.obj: \ - $(SRCPATH)\context.c \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\jcontext $(SRCPATH)\context.c - -#--------------------------------------- -# Link DLL -#--------------------------------------- -$(_OTMMARKUP_DLL)\otmjdk11.dll: \ - $(_OTMMARKUP_OBJ)\otmjdk11.obj \ - $(_OTMMARKUP_OBJ)\javapro.obj \ - $(_OTMMARKUP_OBJ)\javaseg.obj \ - $(_OTMMARKUP_OBJ)\jcontext.obj \ - $(_OTMMARKUP_OBJ)\usrcalls.obj \ - $(_OTMMARKUP_OBJ)\reseq.obj \ - $(_OTMMARKUP_OBJ)\eqfcalls.obj - @if exist $(_OTMMARKUP_DLL)\otmjdk11.dll erase $(_OTMMARKUP_DLL)\otmjdk11.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmjdk11.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmjdk11.dll >>$(_ERR) - @$(_LINKER) >>$(_ERR) @<< -$(_OTMMARKUP_OBJ)\otmjdk11.obj -$(_OTMMARKUP_OBJ)\javapro.obj -$(_OTMMARKUP_OBJ)\javaseg.obj -$(_OTMMARKUP_OBJ)\jcontext.obj -$(_OTMMARKUP_OBJ)\usrcalls.obj -$(_OTMMARKUP_OBJ)\reseq.obj -$(_OTMMARKUP_OBJ)\eqfcalls.obj -/OUT:$(_OTMMARKUP_DLL)\otmjdk11.dll $(LINKFLAGS) -/MAP:$(_OTMMARKUP_MAP)\otmjdk11.map -user32.lib -<< - @copy $(_OTMMARKUP_DLL)\otmjdk11.dll $(_OTMMARKUP_RELEASE_DIR)\BIN /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmjdk21.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmajdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmnjdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmujdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx deleted file mode 100755 index 864d4c4d..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx +++ /dev/null @@ -1,3247 +0,0 @@ - - - - - -Java List/Prop. ResourceBundle (ASCII) -OTMJDK21 -OTMJDK11 -~ - - - > - STNEUTRAL - - - &TWBQ; - 6 - STNEUTRAL - - - &TWBQ;{ - 7 - STNEUTRAL - - - &TWBEQ; - 7 - STNEUTRAL - - - \\\\ - STNEUTRAL - 2 - - - \\B - STNEUTRAL - 2 - - - \\F - STNEUTRAL - 2 - - - \\N - STNEUTRAL - 2 - - - \\R - STNEUTRAL - 2 - - - \\T - STNEUTRAL - 2 - - - \\V - STNEUTRAL - 2 - - - '\\ ' - STNEUTRAL - 2 - - - \\' - STNEUTRAL - 2 - - - \\" - STNEUTRAL - 2 - - - \\: - STNEUTRAL - 2 - - - \\= - STNEUTRAL - 2 - - - \\# - STNEUTRAL - 2 - - - \\! - STNEUTRAL - 2 - - - \\U000A - STNEUTRAL - 6 - - - \\U000D - STNEUTRAL - 6 - - - {0~ - } - STNEUTRAL - - - {1~ - } - STNEUTRAL - - - {2~ - } - STNEUTRAL - - - {3~ - } - STNEUTRAL - - - {4~ - } - STNEUTRAL - - - {5~ - } - STNEUTRAL - - - {6~ - } - STNEUTRAL - - - {7~ - } - STNEUTRAL - - - {8~ - } - STNEUTRAL - - - {9~ - } - STNEUTRAL - - - ${~ - } - STNEUTRAL - - - {TWB~ - } - STNEUTRAL - - - {TWBP}~$ - } - STNEUTRAL - - - $$?~$ - $ - STNEUTRAL - - - @@?~@ - @ - STNEUTRAL - - - &1; - 3 - STNEUTRAL - CLS_VAR - - - &2; - 3 - STNEUTRAL - CLS_VAR - - - &3; - 3 - STNEUTRAL - CLS_VAR - - - &4; - 3 - STNEUTRAL - CLS_VAR - - - &5; - 3 - STNEUTRAL - CLS_VAR - - - &6; - 3 - STNEUTRAL - CLS_VAR - - - &7; - 3 - STNEUTRAL - CLS_VAR - - - &8; - 3 - STNEUTRAL - CLS_VAR - - - &9; - 3 - STNEUTRAL - CLS_VAR - - - <%=~% - > - STNEUTRAL - CLS_VAR - - - <%= ~% - > - STNEUTRAL - CLS_VAR - - - <%=\n~% - > - STNEUTRAL - CLS_VAR - - - <%=\r~% - > - STNEUTRAL - CLS_VAR - - - <%=\t~% - > - STNEUTRAL - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - &#~ - STNEUTRAL - ;\n\r\t> - CLS_VAR - - - - 3 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - 6 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - 9 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - -
- 9 - - -
- > - - - - > - - - - > - - - - > - - - - 8 - - - - > - - - - > - - - - > - - - - > - - - - 6 - - - - 7 - - - - > - - - - > - - - - > - - - - > - - - - 3 - STNEUTRAL - - - - 6 - - - - 7 - - - - > - - - - > - - - - > - - - - > - - - - > - - - - > - - - - 5 - STNEUTRAL - - - - > - - - - 6 - - - - > - - - - > - - - - > - - - - > - - -
- 4 - - -
- 5 - - -
- > - - - - > - - - - > - - - - > - - -