From 7d2ef3574bd4e94a3866d1d6d7353b859a30c2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20=C3=96hrstr=C3=B6m?= Date: Tue, 4 Feb 2025 22:07:10 +0100 Subject: [PATCH] Make sure --lines works as expected. --- Makefile | 4 +- grammars/measurement/metar.ixml | 35 ++++-- grammars/misc/testmeter.ixml | 52 ++++++++ src/main/c/parts/ixml.c | 15 ++- src/main/c/parts/text.c | 12 ++ src/main/c/parts/text.h | 1 + src/main/c/parts/xmq_internals.c | 110 +++++++++++++++++ src/main/c/parts/xmq_internals.h | 3 + src/main/c/xmq-cli.c | 206 ++++++++++++++++++++----------- src/main/c/xmq.c | 114 +---------------- src/main/c/xmq.h | 9 +- tests/format_057_jsonl.test | 13 ++ tests/format_058_xmll.test | 13 ++ tests/format_059_xmql.test | 16 +++ tests/test_ixml_grammar.sh | 3 +- 15 files changed, 408 insertions(+), 198 deletions(-) create mode 100644 grammars/misc/testmeter.ixml create mode 100644 tests/format_057_jsonl.test create mode 100644 tests/format_058_xmll.test create mode 100644 tests/format_059_xmql.test diff --git a/Makefile b/Makefile index a66db5b7..c88c6302 100644 --- a/Makefile +++ b/Makefile @@ -163,13 +163,13 @@ test_release: test_debug: @echo "Running debug tests" @for x in $(BUILDDIRS); do if [ ! -f $${x}debug/testinternals ]; then echo "Run make first. $${x}debug/testinternals not found."; exit 1; fi ; $${x}debug/testinternals $(SILENCER) ; done - @for x in $(BUILDDIRS); do if [ ! -f $${x}debug/parts/testinternals ]; then echo "Run make first. $${x}debug/parts/testinternals not found."; exit 1; fi ; $${x}release/debug/testinternals $(SILENCER) ; ./tests/test.sh $${x}debug $${x}debug/test_output $(FILTER) $(SILENCER) ; done + @for x in $(BUILDDIRS); do if [ ! -f $${x}debug/parts/testinternals ]; then echo "Run make first. $${x}debug/parts/testinternals not found."; exit 1; fi ; $${x}debug/parts/testinternals $(SILENCER) ; ./tests/test.sh $${x}debug $${x}debug/test_output $(FILTER) $(SILENCER) ; done test_asan: @echo "Running asan tests" @if [ "$$(cat /proc/sys/kernel/randomize_va_space)" != "0" ]; then echo "Please disable address randomization for libasan to work! make disable_address_randomization"; exit 1; fi @for x in $(BUILDDIRS); do if [ ! -f $${x}asan/testinternals ]; then echo "Run make first. $${x}asan/testinternals not found."; exit 1; fi ; $${x}asan/testinternals $(SILENCER) ; done - @for x in $(BUILDDIRS); do if [ ! -f $${x}asan/parts/testinternals ]; then echo "Run make first. $${x}asan/parts/testinternals not found."; exit 1; fi ; $${x}release/asan/testinternals $(SILENCER) ; ./tests/test.sh $${x}asan $${x}asan/test_output $(FILTER) $(SILENCER) ; done + @for x in $(BUILDDIRS); do if [ ! -f $${x}asan/parts/testinternals ]; then echo "Run make first. $${x}asan/parts/testinternals not found."; exit 1; fi ; $${x}asan/parts/testinternals $(SILENCER) ; ./tests/test.sh $${x}asan $${x}asan/test_output $(FILTER) $(SILENCER) ; done disable_address_randomization: @echo "Now running: echo 0 | sudo tee /proc/sys/kernel/randomize_va_space" diff --git a/grammars/measurement/metar.ixml b/grammars/measurement/metar.ixml index 5c8fdeb3..a57afb50 100644 --- a/grammars/measurement/metar.ixml +++ b/grammars/measurement/metar.ixml @@ -8,9 +8,9 @@ METAR LBBG 041600Z 12012MPS 090V150 1400 R04/P1500N R22/P1500U +SN BKN022 OVC050 M04/M07 Q1020 NOSIG 8849//91= } -weather_report = (type, -' ')?, icao_airport_code, -' ', part**-' ', -'='?, -[Zs;#a;#d]*. +weather_report = (report_type, -' ')?, icao_airport_code, -' ', part**-' ', -'='?, -[Zs;#a;#d]*. -type = 'METAR' | 'SPECI'. +report_type = 'METAR' | 'SPECI'. -part = | time_of_observation @@ -23,7 +23,8 @@ type = 'METAR' | 'SPECI'. | temperature | air_pressure | trend - | cavok. + | cavok + | remark. icao_airport_code = [Lu]+. @@ -65,17 +66,26 @@ down>status = -'D', +"falling". { Precipitation } weather = observation++-' '. -observation = intensity?, characteristic?, type. +observation = intensity?, characteristic*, weather_type*. intensity = -'-', +"light". intensity = -'+', +"heavy". intensity = -'VC', +"in the vicinity". -characteristic = char_bc | char_sh. +characteristic = char_bc | char_bl | char_sh | char_up. + char_bc = code_bc, info_bc. code_bc = 'BC'. info_bc = +"patches of". +char_bl = code_bl, info_bl. +code_bl = 'BL'. +info_bl = +"blowing". + +char_up = code_up, info_up. +code_up = 'UP'. +info_up = +"unknown precipitation". + -char_sh = code_sh, info_sh. -code_sh = -'SH'. -info_sh = +"showers". @@ -84,12 +94,11 @@ info_bc = +"patches of". DR low drifting MI shallow PR partial -BL blowing FZ freezing SH showers TS thunderstorms} --type = type_br | type_ds | type_du | type_dz | type_fc | type_fg | type_ra. +-weather_type = type_br | type_ds | type_du | type_dz | type_fc | type_fg | type_fz | type_ra | type_va. -type_ra = code_ra, info_ra. code_ra>code = 'RA'. @@ -119,6 +128,14 @@ info_fc>info = +"funnel cloud". code_fg>code = 'FG'. info_fg>info = +"fog". +-type_fz = code_fz, info_fz. +code_fz>code = 'FZ'. +info_fz>info = +"freezing". + +-type_va = code_va, info_va. +code_va>code = 'VA'. +info_va>info = +"volcanic ash". + { FU smoke GR hail @@ -159,3 +176,7 @@ trend = 'NOSIG' | view_becoming. view_becoming = -'BECMG ', meters. cavok = 'CAVOK'. + +*remarks = 'RMK', ~["="]*. + +{-remark = rmk_a02 | 'A02' SLP130 } \ No newline at end of file diff --git a/grammars/misc/testmeter.ixml b/grammars/misc/testmeter.ixml new file mode 100644 index 00000000..f7696016 --- /dev/null +++ b/grammars/misc/testmeter.ixml @@ -0,0 +1,52 @@ +{4E4401062020202205077A4B0040852F2F0FE566B99390000087C0B24B732679FF75350010FCFB00004155594265086A0043B4017301DFF600006AE70000BFD5000051BC0000A0F56C2602FFFF1B1B} + +telegram = len, c, dll-mfct, dll-id, dll-version, dll-type, tpl-ci, tpl-acc, tpl-sts, tpl-cfg, '2F2F0F', intro, parts, outro, ws. + +-ws = -[Zs;#a;#d]*. + +len = byte. +*c = c_hi, c_lo. + +c_hi>set = bit_2, +'PRM'. +c_lo>set = bit_2, +'SND-NR'. + +dll-mfct = byte, byte. +dll-id = byte, byte, byte, byte. +dll-version = byte. +dll-type = byte. + +tpl-ci = byte. +tpl-acc = byte. +tpl-sts = byte. +tpl-cfg = byte, byte. + +-byte = hex, hex. +-hex = ['0'-'9';'A'-'F']. + +intro = byte, byte, byte, byte, byte, byte, byte. +outro = byte, byte, byte, byte. +parts = part*. +-part = total_consumption + | data_a + | data_b + | data_c + | data_d + | data_e + | data_f. + +total_consumption = -'10', byte, byte, byte, byte. +data_a = -'87', byte, byte, byte, byte, byte, byte, byte, byte, byte, byte. +data_b = -'41', byte, byte. +data_c = -'42', byte, byte, byte, byte. +data_d = -'43', byte, byte. +data_e = -'73', byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte. +data_f = -'A0', byte, byte, byte, byte. + +{ 0001 0011 0101 0111 1001 1011 1101 1111 } +-bit_0 = -['13579BDF']. +{ 0010 0011 0110 0111 1010 1011 1110 1111 } +-bit_1 = -['2367ABEF']. +{ 0100 0101 0110 0111 1100 1101 1110 1111 } +-bit_2 = -['4567ABCF']. +{ 1000 1001 1010 1011 1100 1101 1110 1111 } +-bit_3 = -['89ABCDEF']. diff --git a/src/main/c/parts/ixml.c b/src/main/c/parts/ixml.c index 48c35aad..b02a3e8b 100644 --- a/src/main/c/parts/ixml.c +++ b/src/main/c/parts/ixml.c @@ -1,4 +1,4 @@ -/* libxmq - Copyright (C) 2024 Fredrik Öhrström (spdx: MIT) +/* libxmq - Copyright (C) 2024-2025 Fredrik Öhrström (spdx: MIT) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -156,6 +156,8 @@ void skip_whitespace(const char **i); void allocate_yaep_tmp_terminals(XMQParseState *state); void free_yaep_tmp_terminals(XMQParseState *state); +void free_yaep_tmp_terminals_and_content(XMQParseState *state); + bool has_ixml_tmp_terminals(XMQParseState *state); bool has_more_than_one_ixml_tmp_terminals(XMQParseState *state); void add_yaep_term_to_rule(XMQParseState *state, char mark, IXMLTerminal *t, IXMLNonTerminal *nt); @@ -971,7 +973,7 @@ void parse_ixml_insertion(XMQParseState *state) UTF8Char c; encode_utf8(state->ixml_encoded, &c); add_insertion_rule(state, c.bytes); - free_yaep_tmp_terminals(state); + free_yaep_tmp_terminals_and_content(state); } else { @@ -1753,7 +1755,7 @@ void add_insertion_rule(XMQParseState *state, const char *content) void scan_content_fixup_charsets(XMQParseState *state, const char *start, const char *stop) { const char *i = start; - // We allocate a byte for every possible unicode. Yes, a bit excessive, 1 megabyte. Can be fixed though. + // We allocate a byte for every possible unicode. // 17*65536 = 0x11000 = 1 MB char *used = (char*)malloc(0x110000); memset(used, 0, 0x110000); @@ -1939,6 +1941,13 @@ void free_yaep_tmp_terminals(XMQParseState *state) state->ixml_tmp_terminals = NULL; } +void free_yaep_tmp_terminals_and_content(XMQParseState *state) +{ + // Free the tmp terminals themselves. + vector_free_and_values(state->ixml_tmp_terminals, (FreeFuncPtr)free_ixml_terminal); + state->ixml_tmp_terminals = NULL; +} + bool has_ixml_tmp_terminals(XMQParseState *state) { assert(state->ixml_tmp_terminals); diff --git a/src/main/c/parts/text.c b/src/main/c/parts/text.c index 72ce19de..f8bd987a 100644 --- a/src/main/c/parts/text.c +++ b/src/main/c/parts/text.c @@ -899,4 +899,16 @@ bool category_has_code(int code, int *cat, size_t cat_len) } } +const char *find_eol_or_stop(const char *start, const char *stop) +{ + const char *i = start; + + while (i < stop) + { + if (*i == '\n') return i; + i++; + } + return stop; +} + #endif // TEXT_MODULE diff --git a/src/main/c/parts/text.h b/src/main/c/parts/text.h index d7eabf19..d7a81423 100644 --- a/src/main/c/parts/text.h +++ b/src/main/c/parts/text.h @@ -65,6 +65,7 @@ char *xmq_quote_as_c(const char *start, const char *stop, bool add_quotes); char *xmq_unquote_as_c(const char *start, const char *stop, bool remove_quotes); char *potentially_add_leading_ending_space(const char *start, const char *stop); bool find_line_col(const char *start, const char *stop, size_t at, int *line, int *col); +const char *find_eol_or_stop(const char *start, const char *stop); // Fetch a pointer to a NULL ended array of char pointers to category parts for the given name. // For input "Lu" this function returns the array { "Lu", 0 } diff --git a/src/main/c/parts/xmq_internals.c b/src/main/c/parts/xmq_internals.c index 2e7f78f2..7e225356 100644 --- a/src/main/c/parts/xmq_internals.c +++ b/src/main/c/parts/xmq_internals.c @@ -614,5 +614,115 @@ bool is_safe_value_char(const char *i, const char *stop) c == '\r'); } +bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer) +{ + bool rc = true; + int blocksize = 1024; + char block[blocksize]; + + MemBuffer *mb = new_membuffer(); + + int fd = 0; + while (true) { + ssize_t n = read(fd, block, sizeof(block)); + if (n == 0) { + break; + } + if (n == -1) { + if (errno == EINTR) { + continue; + } + PRINT_ERROR("Could not read stdin errno=%d\n", errno); + close(fd); + + return false; + } + membuffer_append_region(mb, block, block + n); + } + close(fd); + + membuffer_append_null(mb); + + *out_fsize = mb->used_-1; + *out_buffer = free_membuffer_but_return_trimmed_content(mb); + + return rc; +} + +bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer) +{ + if (file == NULL || (file[0] == '-' && file[1] == 0)) + { + return load_stdin(doq, out_fsize, out_buffer); + } + + bool rc = false; + char *buffer = NULL; + size_t block_size = 0; + size_t n = 0; + + FILE *f = fopen(file, "rb"); + if (!f) { + doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE; + doq->error_ = build_error_message("xmq: %s: No such file or directory\n", file); + return false; + } + + fseek(f, 0, SEEK_END); + size_t fsize = ftell(f); + fseek(f, 0, SEEK_SET); + + debug("xmq=", "file size %zu", fsize); + + buffer = (char*)malloc(fsize + 1); + if (!buffer) + { + doq->errno_ = XMQ_ERROR_OOM; + doq->error_ = build_error_message("xmq: %s: File too big, out of memory\n", file); + goto exit; + } + + block_size = fsize; + if (block_size > 10000) block_size = 10000; + n = 0; + do { + if (n + block_size > fsize) block_size = fsize - n; + size_t r = fread(buffer+n, 1, block_size, f); + debug("xmq=", "read %zu bytes total %zu", r, n); + if (!r) break; + n += r; + } while (n < fsize); + + debug("xmq=", "read total %zu bytes fsize %zu bytes", n, fsize); + + if (n != fsize) { + rc = false; + doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE; + doq->error_ = build_error_message("xmq: %s: Cannot read file\n", file); + goto exit; + } + fclose(f); + buffer[fsize] = 0; + rc = true; + +exit: + + *out_fsize = fsize; + *out_buffer = buffer; + return rc; +} + +const char *build_error_message(const char* fmt, ...) +{ + char *buf = (char*)malloc(4096); + va_list args; + va_start(args, fmt); + vsnprintf(buf, 4096, fmt, args); + va_end(args); + buf[4095] = 0; + buf = (char*)realloc(buf, strlen(buf)+1); + return buf; +} + #endif // XMQ_INTERNALS_MODULE diff --git a/src/main/c/parts/xmq_internals.h b/src/main/c/parts/xmq_internals.h index 403171b4..37090560 100644 --- a/src/main/c/parts/xmq_internals.h +++ b/src/main/c/parts/xmq_internals.h @@ -695,6 +695,9 @@ XMQParseState *xmq_get_yaep_parse_state(XMQDoc *doc); void set_node_namespace(XMQParseState *state, xmlNodePtr node, const char *node_name); +bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer); +bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer); + // Multicolor terminals like gnome-term etc. #define NOCOLOR "\033[0m" diff --git a/src/main/c/xmq-cli.c b/src/main/c/xmq-cli.c index 6bac94f9..edf99991 100644 --- a/src/main/c/xmq-cli.c +++ b/src/main/c/xmq-cli.c @@ -27,6 +27,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include"parts/text.h" #include"parts/ixml.h" #include"parts/xml.h" +#include"parts/xmq_internals.h" #include"parts/yaep.h" #include @@ -171,6 +172,7 @@ struct XMQCliCommand xmlDocPtr ixml_doc; // A DOM containing the ixml grammar. const char *ixml_ixml; // IXML grammar source to be used. const char *ixml_filename; // Where the ixml grammar was read from. + XMQDoc *ixml_grammar; // The prepared ixml grammar to be reused. bool ixml_all_parses; // Print all possible parses when parse is ambiguous. bool ixml_try_to_recover; // Try to recover when parsing with an ixml grammar. bool build_xml_of_ixml; // Generate xml directly from ixml. @@ -215,7 +217,17 @@ struct XMQCliCommand bool escape_non_7bit; bool escape_tabs; const char *implicit_root; + + // If true then chip away one line at a time from the content. bool lines; + // The content to be parsed one line at a time. + const char *input_content_start; + // The byte after the last input content byte. + const char *input_content_stop; + // Current line start. + const char *input_current_line_start; + // Current line stop, points to byte after #a (newline). + const char *input_current_line_stop; // Command tok XMQCliTokenizeType tok_type; // Do not pretty print, just debug/colorize tokens. @@ -306,7 +318,7 @@ bool cmd_to(XMQCliCommand *command); bool cmd_output(XMQCliCommand *command); bool cmd_transform(XMQCliCommand *command); bool cmd_validate(XMQCliCommand *command); -void cmd_unload(XMQCliCommand *command); +bool cmd_unload(XMQCliCommand *command); bool cmd_quote_unquote(XMQCliCommand *command); const char *content_type_to_string(XMQContentType ct); const char *tokenize_type_to_string(XMQCliTokenizeType type); @@ -1830,84 +1842,99 @@ bool cmd_load(XMQCliCommand *command) command->in = NULL; } - verbose_("xmq=", "cmd-load %s", command->in); + const char *from = "stdin"; + + if (command->in_is_content) + { + from = "-i argument"; + command->input_current_line_start = command->in; + command->input_current_line_stop = command->in+strlen(command->in); + xmqSetOriginalSize(command->env->doc, command->input_current_line_stop-command->input_current_line_start); + verbose_("xmq=", "cmd-load from -i argument"); + } + else + { + if (command->in) from = command->in; + if (command->input_content_start == NULL) + { + XMQDoc *tmp = xmqNewDoc(); + size_t len = 0; + verbose_("xmq=", "cmd-load from %s", from); + bool ok = load_file(tmp, command->in, &len, &command->input_content_start); + command->input_content_stop = command->input_content_start+len; + + if (!ok) + { + printf("%s\n", tmp->error_); + exit(1); + } + xmqFreeDoc(tmp); + if (command->lines) + { + command->input_current_line_start = command->input_content_start; + command->input_current_line_stop = find_eol_or_stop(command->input_current_line_start, + command->input_content_stop); + } + else + { + command->input_current_line_start = command->input_content_start; + command->input_current_line_stop = command->input_content_stop; + } + } + else + { + command->input_current_line_start = command->input_current_line_stop+1; + command->input_current_line_stop = find_eol_or_stop(command->input_current_line_start, + command->input_content_stop); + if (command->input_current_line_start >= command->input_content_stop) + { + return false; + } + } + } command->env->load = command; if (command->ixml_ixml != NULL) { - XMQDoc *ixml_grammar = xmqNewDoc(); - xmqSetDocSourceName(ixml_grammar, command->ixml_filename); + if (command->ixml_grammar == NULL) + { + command->ixml_grammar = xmqNewDoc(); + xmqSetDocSourceName(command->ixml_grammar, command->ixml_filename); - bool ok = xmqParseBufferWithType(ixml_grammar, command->ixml_ixml, NULL, NULL, XMQ_CONTENT_IXML, 0); + bool ok = xmqParseBufferWithType(command->ixml_grammar, command->ixml_ixml, NULL, NULL, XMQ_CONTENT_IXML, 0); - if (!ok) - { - fprintf(stderr, "%s\n", xmqDocError(ixml_grammar)); - exit(1); + if (!ok) + { + fprintf(stderr, "%s\n", xmqDocError(command->ixml_grammar)); + exit(1); + } } int flags = 0; if (command->ixml_all_parses) flags |= XMQ_FLAG_IXML_ALL_PARSES; if (command->ixml_try_to_recover) flags |= XMQ_FLAG_IXML_TRY_TO_RECOVER; - if (command->in) + bool ok = xmqParseBufferWithIXML(command->env->doc, + command->input_current_line_start, + command->input_current_line_stop, + command->ixml_grammar, + flags); + + if (!ok) { - if (command->in_is_content) - { - ok = xmqParseBufferWithIXML(command->env->doc, command->in, NULL, ixml_grammar, flags); - if (!ok) - { - printf("xmq: could not parse: %s\n", command->in); - exit(1); - } - } - else - { - ok = check_file_exists(command->in); - if (!ok) - { - printf("xmq: could not read file %s\n", command->in); - exit(1); - } - ok = xmqParseFileWithIXML(command->env->doc, command->in, ixml_grammar, flags); - if (!ok) - { - exit(1); - } - } + exit(1); } - - xmqFreeDoc(ixml_grammar); - - const char *from = "stdin"; - if (command->in) from = command->in; - free((char*)command->ixml_filename); - command->ixml_filename = NULL; - free((char*)command->ixml_ixml); - command->ixml_ixml = NULL; verbose_("xmq=", "cmd-load-ixml %zu bytes from %s", xmqGetOriginalSize(command->env->doc), from); } else { - bool ok = false; - if (command->in_is_content) - { - ok = xmqParseBufferWithType(command->env->doc, - command->in, - NULL, - command->implicit_root, - command->in_format, - command->flags); - } - else - { - ok = xmqParseFileWithType(command->env->doc, - command->in, - command->implicit_root, - command->in_format, - command->flags); - } + bool ok = xmqParseBufferWithType(command->env->doc, + command->input_current_line_start, + command->input_current_line_stop, + command->implicit_root, + command->in_format, + command->flags); if (!ok) { @@ -1919,22 +1946,44 @@ bool cmd_load(XMQCliCommand *command) command->env->doc = NULL; return false; } - const char *from = "stdin"; - if (command->in) from = command->in; verbose_("xmq=", "cmd-load %zu bytes from %s", xmqGetOriginalSize(command->env->doc), from); } return true; } -void cmd_unload(XMQCliCommand *command) +bool cmd_unload(XMQCliCommand *command) { if (command && command->env && command->env->doc) { verbose_("xmq=", "cmd-unload document"); xmqFreeDoc(command->env->doc); command->env->doc = NULL; + + if (command->input_current_line_start >= command->input_content_stop) + { + // Handled all input. + if (command->ixml_grammar) + { + xmqFreeDoc(command->ixml_grammar); + command->ixml_grammar = NULL; + } + + if (command->input_content_start) free((char*)command->input_content_start); + command->input_content_start = NULL; + command->input_content_stop = NULL; + command->input_current_line_start = NULL; + command->input_current_line_stop = NULL; + + free((char*)command->ixml_filename); + command->ixml_filename = NULL; + free((char*)command->ixml_ixml); + command->ixml_ixml = NULL; + return false; + } + return true; } + return false; } bool cmd_to(XMQCliCommand *command) @@ -4071,26 +4120,33 @@ int main(int argc, const char **argv) return cmd_help(load_command->next); } - XMQCliCommand *c = load_command; + bool more_content = true; - // Execute commands. - while (c) + while (more_content) { - debug_("xmq=", "performing %s", cmd_name(c->cmd)); - bool ok = perform_command(c); - if (!ok) + // The load command will either load the whole file at once + // or chip away at it for each line. + XMQCliCommand *c = load_command; + + // Execute commands. + while (c) { - rc = 1; - break; + debug_("xmq=", "performing %s", cmd_name(c->cmd)); + bool ok = perform_command(c); + if (!ok) + { + rc = 1; + break; + } + c = c->next; } - c = c->next; + // Free document. + more_content = cmd_unload(load_command); } - // Free document. - cmd_unload(load_command); // Free commands. - c = load_command; + XMQCliCommand *c = load_command; while (c) { XMQCliCommand *tmp = c; diff --git a/src/main/c/xmq.c b/src/main/c/xmq.c index 92324f6a..590194a9 100644 --- a/src/main/c/xmq.c +++ b/src/main/c/xmq.c @@ -1,4 +1,4 @@ -/* libxmq - Copyright (C) 2023-2024 Fredrik Öhrström (spdx: MIT) +/* libxmq - Copyright (C) 2023-2025 Fredrik Öhrström (spdx: MIT) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -115,8 +115,6 @@ void handle_yaep_syntax_error(YaepParseRun *pr, bool has_leading_ending_quote(const char *start, const char *stop); bool is_safe_char(const char *i, const char *stop); size_t line_length(const char *start, const char *stop, int *numq, int *lq, int *eq); -bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer); -bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer); bool need_separation_before_entity(XMQPrintState *ps); const char *node_yaep_type_to_string(YaepTreeNodeType t); size_t num_utf8_bytes(char c); @@ -1233,18 +1231,6 @@ void xmqSetLogHumanReadable(bool e) xmq_log_line_config_.human_readable_ = e; } -const char *build_error_message(const char* fmt, ...) -{ - char *buf = (char*)malloc(4096); - va_list args; - va_start(args, fmt); - vsnprintf(buf, 4096, fmt, args); - va_end(args); - buf[4095] = 0; - buf = (char*)realloc(buf, strlen(buf)+1); - return buf; -} - /** xmq_un_quote: @indent: The number of chars before the quote starts on the first line. @@ -1734,6 +1720,11 @@ size_t xmqGetOriginalSize(XMQDoc *doq) return doq->original_size_; } +void xmqSetOriginalSize(XMQDoc *doq, size_t size) +{ + doq->original_size_ = size; +} + XMQNode *xmqGetRootNode(XMQDoc *doq) { return &doq->root_; @@ -4165,99 +4156,6 @@ bool xmqParseBufferWithType(XMQDoc *doq, return ok; } -bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer) -{ - bool rc = true; - int blocksize = 1024; - char block[blocksize]; - - MemBuffer *mb = new_membuffer(); - - int fd = 0; - while (true) { - ssize_t n = read(fd, block, sizeof(block)); - if (n == 0) { - break; - } - if (n == -1) { - if (errno == EINTR) { - continue; - } - PRINT_ERROR("Could not read stdin errno=%d\n", errno); - close(fd); - - return false; - } - membuffer_append_region(mb, block, block + n); - } - close(fd); - - membuffer_append_null(mb); - - *out_fsize = mb->used_-1; - *out_buffer = free_membuffer_but_return_trimmed_content(mb); - - return rc; -} - -bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer) -{ - bool rc = false; - char *buffer = NULL; - size_t block_size = 0; - size_t n = 0; - - FILE *f = fopen(file, "rb"); - if (!f) { - doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE; - doq->error_ = build_error_message("xmq: %s: No such file or directory\n", file); - return false; - } - - fseek(f, 0, SEEK_END); - size_t fsize = ftell(f); - fseek(f, 0, SEEK_SET); - - debug("xmq=", "file size %zu", fsize); - - buffer = (char*)malloc(fsize + 1); - if (!buffer) - { - doq->errno_ = XMQ_ERROR_OOM; - doq->error_ = build_error_message("xmq: %s: File too big, out of memory\n", file); - goto exit; - } - - block_size = fsize; - if (block_size > 10000) block_size = 10000; - n = 0; - do { - if (n + block_size > fsize) block_size = fsize - n; - size_t r = fread(buffer+n, 1, block_size, f); - debug("xmq=", "read %zu bytes total %zu", r, n); - if (!r) break; - n += r; - } while (n < fsize); - - debug("xmq=", "read total %zu bytes fsize %zu bytes", n, fsize); - - if (n != fsize) { - rc = false; - doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE; - doq->error_ = build_error_message("xmq: %s: Cannot read file\n", file); - goto exit; - } - fclose(f); - buffer[fsize] = 0; - rc = true; - -exit: - - *out_fsize = fsize; - *out_buffer = buffer; - return rc; -} - bool xmqParseFileWithType(XMQDoc *doq, const char *file, const char *implicit_root, diff --git a/src/main/c/xmq.h b/src/main/c/xmq.h index 5c2ed3f1..b9513b81 100644 --- a/src/main/c/xmq.h +++ b/src/main/c/xmq.h @@ -1,4 +1,4 @@ -/* libxmq - Copyright (C) 2023-2024 Fredrik Öhrström (spdx: MIT) +/* libxmq - Copyright (C) 2023-2025 Fredrik Öhrström (spdx: MIT) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -545,6 +545,13 @@ XMQContentType xmqGetOriginalContentType(XMQDoc *doq); */ size_t xmqGetOriginalSize(XMQDoc *doq); +/** + xmqSetOriginalSize: + + Override the original size of the document. +*/ +void xmqSetOriginalSize(XMQDoc *doq, size_t size); + /** xmqGetRootNode: diff --git a/tests/format_057_jsonl.test b/tests/format_057_jsonl.test new file mode 100644 index 00000000..b2fe4616 --- /dev/null +++ b/tests/format_057_jsonl.test @@ -0,0 +1,13 @@ +START +{"speed":123,"car":"volvo"} +{"speed":33,"car":"saab"} +{"speed":1,"car":"ford"} +COMPACT +IGNORE +XMQ +_{speed=123 car=volvo} +_{speed=33 car=saab} +_{speed=1 car=ford} +END +ARGS --lines +CMDS to-xmq --compact diff --git a/tests/format_058_xmll.test b/tests/format_058_xmll.test new file mode 100644 index 00000000..894d0ba2 --- /dev/null +++ b/tests/format_058_xmll.test @@ -0,0 +1,13 @@ +START +123volvo +33saab +1ford +COMPACT +IGNORE +XMQ +v{speed=123 car=volvo} +v{speed=33 car=saab} +v{speed=1 car=ford} +END +ARGS --lines +CMDS to-xmq --compact diff --git a/tests/format_059_xmql.test b/tests/format_059_xmql.test new file mode 100644 index 00000000..18d72f97 --- /dev/null +++ b/tests/format_059_xmql.test @@ -0,0 +1,16 @@ +START +v{speed=123 car=volvo} +v{speed=33 car=saab} +v{speed=1 car=ford} +COMPACT +IGNORE +XMQ + +123volvo + +33saab + +1ford +END +ARGS --lines +CMDS to-xml diff --git a/tests/test_ixml_grammar.sh b/tests/test_ixml_grammar.sh index 2f834e40..b76befd0 100755 --- a/tests/test_ixml_grammar.sh +++ b/tests/test_ixml_grammar.sh @@ -23,8 +23,7 @@ mkdir -p $OUTPUT sed -n '/^START$/,/^END$/p' $TEST_FILE | tail -n +2 | sed '$d' > $OUTPUT/${TEST_NAME}.ixml - -if $PROG --ixml=$OUTPUT/${TEST_NAME}.ixml > $OUTPUT/${TEST_NAME}.output 2>&1 +if $PROG --ixml=$OUTPUT/${TEST_NAME}.ixml -z > $OUTPUT/${TEST_NAME}.output 2>&1 then echo "OK: test ${TEST_NAME}" else