Skip to content

pack: fix token count regression in JSON tokenizer and remove parser reset #10733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edsiper merged 2 commits into from
Aug 14, 2025
Merged

pack: fix token count regression in JSON tokenizer and remove parser reset #10733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
653b1c7
pack: fix token count regression in JSON tokenizer and remove parser …
edsiper Aug 14, 2025
834eb00
tests: internal: pack: add test_json_pack_token_count_overflow test
edsiper Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/flb_pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ int flb_json_tokenise(const char *js, size_t len,

ret = jsmn_parse(&state->parser, js, len,
state->tokens, state->tokens_size);

while (ret == JSMN_ERROR_NOMEM) {
/* Get current size of the array in bytes */
old_size = state->tokens_size * sizeof(jsmntok_t);
Expand All @@ -79,13 +80,11 @@ int flb_json_tokenise(const char *js, size_t len,
state->tokens = tmp;
state->tokens_size += new_tokens;

/* Reset parser to reprocess the JSON data from the beginning */
jsmn_init(&state->parser);

ret = jsmn_parse(&state->parser, js, len,
state->tokens, state->tokens_size);
}


if (ret == JSMN_ERROR_INVAL) {
return FLB_ERR_JSON_INVAL;
}
Expand All @@ -96,7 +95,8 @@ int flb_json_tokenise(const char *js, size_t len,
return FLB_ERR_JSON_PART;
}

state->tokens_count += ret;
/* always use jsmn_parser.toknext to count tokens */
state->tokens_count = state->parser.toknext;
return 0;
}

Expand Down
53 changes: 49 additions & 4 deletions tests/internal/pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ void test_json_dup_keys()
flb_free(data_out);
}

/* https://github.com/fluent/fluent-bit/issues/342 */
void test_json_pack_bug342()
{
int i = 0;
Expand Down Expand Up @@ -989,7 +990,7 @@ void test_json_pack_empty_array()

/* unpack just to validate the msgpack buffer */
msgpack_unpacked result;
msgpack_object obj;

size_t off = 0;
msgpack_unpacked_init(&result);
off = 0;
Expand Down Expand Up @@ -1103,7 +1104,7 @@ void test_json_pack_large_uint64()
};

for (i = 0; i < sizeof(test_cases) / sizeof(test_cases[0]); i++) {
p_in = test_cases[i].json_str;
p_in = (char *) test_cases[i].json_str;
Copy link

@coderabbitai coderabbitai bot Aug 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Avoid casting away const; make p_in const char.*

Casting a const string to char* can hide bugs and violates const-correctness. flb_pack_json accepts const char*, so keep p_in const.

Apply this diff to the assignment:

-        p_in = (char *) test_cases[i].json_str;
+        p_in = test_cases[i].json_str;

Additionally, adjust the declaration of p_in (outside this hunk) from:

char *p_in;

to:

const char *p_in;
🤖 Prompt for AI Agents 
In tests/internal/pack.c around line 1107, the code casts a const JSON string to
char* with "p_in = (char *) test_cases[i].json_str;"; change the assignment to
keep the const qualifier (p_in = test_cases[i].json_str;) and update p_in's
declaration (earlier in the file) from "char *p_in;" to "const char *p_in;" so
p_in matches flb_pack_json's const char* parameter and preserves
const-correctness.

len_in = strlen(p_in);
expected = test_cases[i].expected_val;

Expand Down Expand Up @@ -1137,6 +1138,50 @@ void test_json_pack_large_uint64()
}
}

void test_json_pack_token_count_overflow()
{
int i;
flb_sds_t json = NULL;
struct flb_pack_state state;
int ret;

flb_pack_state_init(&state);

/* Create a JSON array big enough to trigger realloc in flb_json_tokenise */
json = flb_sds_create("[");
for (i = 0; i < 300; i++) {
if (i > 0) {
flb_sds_cat_safe(&json, ",", 1);
}
json = flb_sds_printf(&json, "%d", i);
}
flb_sds_cat_safe(&json, "]", 1);

if (!TEST_CHECK(json != NULL)) {
TEST_MSG("Failed to allocate JSON string");
exit(1);
}

/* First parse: forces realloc at least once (because by default we have space for 256 tokens) */
ret = flb_json_tokenise(json, flb_sds_len(json), &state);
TEST_CHECK(ret == 0);
printf("\nFirst parse: tokens_count=%d\n", state.tokens_count);

/* Second parse with the same JSON and same state — should be ~301, but will be doubled if bug exists */
ret = flb_json_tokenise(json, flb_sds_len(json), &state);
TEST_CHECK(ret == 0);
printf("Second parse: tokens_count=%d (BUG if > ~301)\n", state.tokens_count);

TEST_CHECK(state.tokens_count == 301);
if (state.tokens_count != 301) {
TEST_MSG("tokens_count=%d (BUG if > ~301)\n", state.tokens_count);
exit(1);
}

flb_sds_destroy(json);
flb_pack_state_reset(&state);
}

TEST_LIST = {
/* JSON maps iteration */
{ "json_pack" , test_json_pack },
Expand All @@ -1161,7 +1206,7 @@ TEST_LIST = {
/* Mixed bytes, check JSON encoding */
{ "utf8_to_json", test_utf8_to_json},
{ "json_pack_surrogate_pairs", test_json_pack_surrogate_pairs},
{ "json_pack_surrogate_pairs_with_replacement",
test_json_pack_surrogate_pairs_with_replacement},
{ "json_pack_surrogate_pairs_with_replacement", test_json_pack_surrogate_pairs_with_replacement},
{ "json_pack_token_count_overflow", test_json_pack_token_count_overflow},
{ 0 }
};
Loading