fix bug whereby multi-byte utf8 at end of string incorrectly treated …

…as malformed
liquidaty · Dec 26, 2023 · e5203f8 · e5203f8
1 parent 93d349e
commit e5203f8
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 2 deletions.
diff --git a/app/test/Makefile b/app/test/Makefile
@@ -100,7 +100,7 @@ test: ${TESTS}
 test-prop:
  EXE=${BUILD_DIR}/bin/zsv_prop${EXE} make -C prop test
 
-test-echo : test-echo1 test-echo-overwrite test-echo-eol test-echo-overwrite-csv
+test-echo : test-echo1 test-echo-overwrite test-echo-eol test-echo-overwrite-csv test-echo-chars
 
 test-echo1: ${BUILD_DIR}/bin/zsv_echo${EXE}
  @${TEST_INIT}
@@ -114,6 +114,11 @@ test-echo-eol-%: ${BUILD_DIR}/bin/zsv_echo${EXE}
  @${PREFIX} $< ${TEST_DATA_DIR}/test/no-eol-$*.csv ${REDIRECT} ${TMP_DIR}/$@.out
  @${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}
 
+test-echo-chars: ${BUILD_DIR}/bin/zsv_echo${EXE}
+ @${TEST_INIT}
+ @${PREFIX} echo '東京都' | $< -u '?' ${REDIRECT} ${TMP_DIR}/$@.out
+ @${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}
+
 test-echo-overwrite: ${BUILD_DIR}/bin/zsv_echo${EXE}
  @${TEST_INIT}
  @${PREFIX} $< ${TEST_DATA_DIR}/loans_1.csv --overwrite 'sqlite3://${TEST_DATA_DIR}/loans_1-overwrite.db?sql=select row,col,value from overwrites order by row,col' ${REDIRECT} ${TMP_DIR}/$@.out

diff --git a/app/test/expected/test-echo-chars.out b/app/test/expected/test-echo-chars.out
@@ -0,0 +1 @@
+東京都
diff --git a/app/zsv_main.h b/app/zsv_main.h
@@ -13,6 +13,7 @@
 #define ZSV_MAIN_NO_OPTIONS_FUNC1(x) zsv_ ## x ## _main_no_options
 
 struct zsv_opts;
+struct zsv_prop_handler;
 
 /* macros for commands that use common zsv parsing */
 #define ZSV_MAIN_FUNC(x) ZSV_MAIN_FUNC1(x)

diff --git a/src/zsv_strencode.c b/src/zsv_strencode.c
@@ -20,7 +20,7 @@ size_t zsv_strencode(unsigned char *s, size_t n, unsigned char replace,
  clen = ZSV_UTF8_CHARLEN(s[i2]);
  if(LIKELY(clen == 1))
  s[new_len++] = s[i2];
- else if(UNLIKELY(clen < 0) || UNLIKELY(i2 + clen >= n)) {
+ else if(UNLIKELY(clen < 0) || UNLIKELY(i2 + clen > n)) {
  if(malformed_handler)
  malformed_handler(handler_ctx, s, n, new_len);
  if(replace)