Make sure --lines works as expected.

Makefile

@@ -163,13 +163,13 @@ test_release:
 test_debug:
 	@echo "Running debug tests"
 	@for x in $(BUILDDIRS); do if [ ! -f $${x}debug/testinternals ]; then echo "Run make first. $${x}debug/testinternals not found."; exit 1; fi ; $${x}debug/testinternals $(SILENCER) ; done
-	@for x in $(BUILDDIRS); do if [ ! -f $${x}debug/parts/testinternals ]; then echo "Run make first. $${x}debug/parts/testinternals not found."; exit 1; fi ; $${x}release/debug/testinternals $(SILENCER) ; ./tests/test.sh $${x}debug $${x}debug/test_output $(FILTER) $(SILENCER) ; done
+	@for x in $(BUILDDIRS); do if [ ! -f $${x}debug/parts/testinternals ]; then echo "Run make first. $${x}debug/parts/testinternals not found."; exit 1; fi ; $${x}debug/parts/testinternals $(SILENCER) ; ./tests/test.sh $${x}debug $${x}debug/test_output $(FILTER) $(SILENCER) ; done
 
 test_asan:
 	@echo "Running asan tests"
 	@if [ "$$(cat /proc/sys/kernel/randomize_va_space)" != "0" ]; then echo "Please disable address randomization for libasan to work! make disable_address_randomization"; exit 1; fi
 	@for x in $(BUILDDIRS); do if [ ! -f $${x}asan/testinternals ]; then echo "Run make first. $${x}asan/testinternals not found."; exit 1; fi ; $${x}asan/testinternals $(SILENCER) ; done
-	@for x in $(BUILDDIRS); do if [ ! -f $${x}asan/parts/testinternals ]; then echo "Run make first. $${x}asan/parts/testinternals not found."; exit 1; fi ; $${x}release/asan/testinternals $(SILENCER) ; ./tests/test.sh $${x}asan $${x}asan/test_output $(FILTER) $(SILENCER) ; done
+	@for x in $(BUILDDIRS); do if [ ! -f $${x}asan/parts/testinternals ]; then echo "Run make first. $${x}asan/parts/testinternals not found."; exit 1; fi ; $${x}asan/parts/testinternals $(SILENCER) ; ./tests/test.sh $${x}asan $${x}asan/test_output $(FILTER) $(SILENCER) ; done
 
 disable_address_randomization:
 	@echo "Now running: echo 0 | sudo tee /proc/sys/kernel/randomize_va_space"

grammars/measurement/metar.ixml

@@ -8,9 +8,9 @@
     METAR LBBG 041600Z 12012MPS 090V150 1400 R04/P1500N R22/P1500U +SN BKN022 OVC050 M04/M07 Q1020 NOSIG 8849//91=
 }
 
-weather_report = (type, -' ')?, icao_airport_code, -' ', part**-' ', -'='?, -[Zs;#a;#d]*.
+weather_report = (report_type, -' ')?, icao_airport_code, -' ', part**-' ', -'='?, -[Zs;#a;#d]*.
 
-type = 'METAR' | 'SPECI'.
+report_type = 'METAR' | 'SPECI'.
 
 -part =
      | time_of_observation
@@ -23,7 +23,8 @@ type = 'METAR' | 'SPECI'.
      | temperature
      | air_pressure
      | trend
-     | cavok.
+     | cavok
+     | remark.
 
 icao_airport_code = [Lu]+.
 
@@ -65,17 +66,26 @@ down>status = -'D', +"falling".
 { Precipitation }
 weather = observation++-' '.
 
-observation = intensity?, characteristic?, type.
+observation = intensity?, characteristic*, weather_type*.
 
 intensity = -'-', +"light".
 intensity = -'+', +"heavy".
 intensity = -'VC', +"in the vicinity".
 
-characteristic = char_bc | char_sh.
+characteristic = char_bc | char_bl | char_sh | char_up.
+
 char_bc = code_bc, info_bc.
 code_bc = 'BC'.
 info_bc = +"patches of".
 
+char_bl = code_bl, info_bl.
+code_bl = 'BL'.
+info_bl = +"blowing".
+
+char_up = code_up, info_up.
+code_up = 'UP'.
+info_up = +"unknown precipitation".
+
 -char_sh = code_sh, info_sh.
 -code_sh = -'SH'.
 -info_sh = +"showers".
@@ -84,12 +94,11 @@ info_bc = +"patches of".
 DR 	low drifting
 MI 	shallow
 PR 	partial
-BL 	blowing
 FZ 	freezing
 SH 	showers
 TS 	thunderstorms}
 
--type = type_br | type_ds | type_du | type_dz | type_fc | type_fg | type_ra.
+-weather_type = type_br | type_ds | type_du | type_dz | type_fc | type_fg | type_fz | type_ra | type_va.
 
 -type_ra = code_ra, info_ra.
 code_ra>code = 'RA'.
@@ -119,6 +128,14 @@ info_fc>info = +"funnel cloud".
 code_fg>code = 'FG'.
 info_fg>info = +"fog".
 
+-type_fz = code_fz, info_fz.
+code_fz>code = 'FZ'.
+info_fz>info = +"freezing".
+
+-type_va = code_va, info_va.
+code_va>code = 'VA'.
+info_va>info = +"volcanic ash".
+
 {
 FU 	smoke
 GR 	hail
@@ -159,3 +176,7 @@ trend = 'NOSIG' | view_becoming.
 view_becoming = -'BECMG ', meters.
 
 cavok = 'CAVOK'.
+
+*remarks = 'RMK', ~["="]*.
+
+{-remark = rmk_a02 | 'A02' SLP130  }

grammars/misc/testmeter.ixml

@@ -0,0 +1,52 @@
+{4E4401062020202205077A4B0040852F2F0FE566B99390000087C0B24B732679FF75350010FCFB00004155594265086A0043B4017301DFF600006AE70000BFD5000051BC0000A0F56C2602FFFF1B1B}
+
+telegram = len, c, dll-mfct, dll-id, dll-version, dll-type, tpl-ci, tpl-acc, tpl-sts, tpl-cfg, '2F2F0F', intro, parts, outro, ws.
+
+-ws = -[Zs;#a;#d]*.
+
+len = byte.
+*c = c_hi, c_lo.
+
+c_hi>set = bit_2, +'PRM'.
+c_lo>set = bit_2, +'SND-NR'.
+
+dll-mfct = byte, byte.
+dll-id = byte, byte, byte, byte.
+dll-version = byte.
+dll-type = byte.
+
+tpl-ci = byte.
+tpl-acc = byte.
+tpl-sts = byte.
+tpl-cfg = byte, byte.
+
+-byte = hex, hex.
+-hex = ['0'-'9';'A'-'F'].
+
+intro = byte, byte, byte, byte, byte, byte, byte.
+outro = byte, byte, byte, byte.
+parts = part*.
+-part =  total_consumption
+        | data_a
+        | data_b
+        | data_c
+        | data_d
+        | data_e
+        | data_f.
+
+total_consumption = -'10', byte, byte, byte, byte.
+data_a = -'87', byte, byte, byte, byte, byte, byte, byte, byte, byte, byte.
+data_b = -'41', byte, byte.
+data_c = -'42', byte, byte, byte, byte.
+data_d = -'43', byte, byte.
+data_e = -'73', byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte, byte.
+data_f = -'A0', byte, byte, byte, byte.
+
+{ 0001 0011 0101 0111 1001 1011 1101 1111 }
+-bit_0 = -['13579BDF'].
+{ 0010 0011 0110 0111 1010 1011 1110 1111 }
+-bit_1 = -['2367ABEF'].
+{ 0100 0101 0110 0111 1100 1101 1110 1111 }
+-bit_2 = -['4567ABCF'].
+{ 1000 1001 1010 1011 1100 1101 1110 1111 }
+-bit_3 = -['89ABCDEF'].

src/main/c/parts/ixml.c

@@ -1,4 +1,4 @@
-/* libxmq - Copyright (C) 2024 Fredrik Öhrström (spdx: MIT)
+/* libxmq - Copyright (C) 2024-2025 Fredrik Öhrström (spdx: MIT)
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -156,6 +156,8 @@ void skip_whitespace(const char **i);
 
 void allocate_yaep_tmp_terminals(XMQParseState *state);
 void free_yaep_tmp_terminals(XMQParseState *state);
+void free_yaep_tmp_terminals_and_content(XMQParseState *state);
+
 bool has_ixml_tmp_terminals(XMQParseState *state);
 bool has_more_than_one_ixml_tmp_terminals(XMQParseState *state);
 void add_yaep_term_to_rule(XMQParseState *state, char mark, IXMLTerminal *t, IXMLNonTerminal *nt);
@@ -971,7 +973,7 @@ void parse_ixml_insertion(XMQParseState *state)
         UTF8Char c;
         encode_utf8(state->ixml_encoded, &c);
         add_insertion_rule(state, c.bytes);
-        free_yaep_tmp_terminals(state);
+        free_yaep_tmp_terminals_and_content(state);
     }
     else
     {
@@ -1753,7 +1755,7 @@ void add_insertion_rule(XMQParseState *state, const char *content)
 void scan_content_fixup_charsets(XMQParseState *state, const char *start, const char *stop)
 {
     const char *i = start;
-    // We allocate a byte for every possible unicode. Yes, a bit excessive, 1 megabyte. Can be fixed though.
+    // We allocate a byte for every possible unicode.
     // 17*65536 = 0x11000 = 1 MB
     char *used = (char*)malloc(0x110000);
     memset(used, 0, 0x110000);
@@ -1939,6 +1941,13 @@ void free_yaep_tmp_terminals(XMQParseState *state)
     state->ixml_tmp_terminals = NULL;
 }
 
+void free_yaep_tmp_terminals_and_content(XMQParseState *state)
+{
+    // Free the tmp terminals themselves.
+    vector_free_and_values(state->ixml_tmp_terminals, (FreeFuncPtr)free_ixml_terminal);
+    state->ixml_tmp_terminals = NULL;
+}
+
 bool has_ixml_tmp_terminals(XMQParseState *state)
 {
     assert(state->ixml_tmp_terminals);

src/main/c/parts/text.c

@@ -899,4 +899,16 @@ bool category_has_code(int code, int *cat, size_t cat_len)
     }
 }
 
+const char *find_eol_or_stop(const char *start, const char *stop)
+{
+    const char *i = start;
+
+    while (i < stop)
+    {
+        if (*i == '\n') return i;
+        i++;
+    }
+    return stop;
+}
+
 #endif // TEXT_MODULE

src/main/c/parts/text.h

@@ -65,6 +65,7 @@ char *xmq_quote_as_c(const char *start, const char *stop, bool add_quotes);
 char *xmq_unquote_as_c(const char *start, const char *stop, bool remove_quotes);
 char *potentially_add_leading_ending_space(const char *start, const char *stop);
 bool find_line_col(const char *start, const char *stop, size_t at, int *line, int *col);
+const char *find_eol_or_stop(const char *start, const char *stop);
 
 // Fetch a pointer to a NULL ended array of char pointers to category parts for the given name.
 // For input "Lu" this function returns the array { "Lu", 0 }

src/main/c/parts/xmq_internals.c

@@ -614,5 +614,115 @@ bool is_safe_value_char(const char *i, const char *stop)
              c == '\r');
 }
 
+bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer)
+{
+    bool rc = true;
+    int blocksize = 1024;
+    char block[blocksize];
+
+    MemBuffer *mb = new_membuffer();
+
+    int fd = 0;
+    while (true) {
+        ssize_t n = read(fd, block, sizeof(block));
+        if (n == 0) {
+            break;
+        }
+        if (n == -1) {
+            if (errno == EINTR) {
+                continue;
+            }
+            PRINT_ERROR("Could not read stdin errno=%d\n", errno);
+            close(fd);
+
+            return false;
+        }
+        membuffer_append_region(mb, block, block + n);
+    }
+    close(fd);
+
+    membuffer_append_null(mb);
+
+    *out_fsize = mb->used_-1;
+    *out_buffer = free_membuffer_but_return_trimmed_content(mb);
+
+    return rc;
+}
+
+bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer)
+{
+    if (file == NULL || (file[0] == '-' && file[1] == 0))
+    {
+        return load_stdin(doq, out_fsize, out_buffer);
+    }
+
+    bool rc = false;
+    char *buffer = NULL;
+    size_t block_size = 0;
+    size_t n = 0;
+
+    FILE *f = fopen(file, "rb");
+    if (!f) {
+        doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE;
+        doq->error_ = build_error_message("xmq: %s: No such file or directory\n", file);
+        return false;
+    }
+
+    fseek(f, 0, SEEK_END);
+    size_t fsize = ftell(f);
+    fseek(f, 0, SEEK_SET);
+
+    debug("xmq=", "file size %zu", fsize);
+
+    buffer = (char*)malloc(fsize + 1);
+    if (!buffer)
+    {
+        doq->errno_ = XMQ_ERROR_OOM;
+        doq->error_ = build_error_message("xmq: %s: File too big, out of memory\n", file);
+        goto exit;
+    }
+
+    block_size = fsize;
+    if (block_size > 10000) block_size = 10000;
+    n = 0;
+    do {
+        if (n + block_size > fsize) block_size = fsize - n;
+        size_t r = fread(buffer+n, 1, block_size, f);
+        debug("xmq=", "read %zu bytes total %zu", r, n);
+        if (!r) break;
+        n += r;
+    } while (n < fsize);
+
+    debug("xmq=", "read total %zu bytes fsize %zu bytes", n, fsize);
+
+    if (n != fsize) {
+        rc = false;
+        doq->errno_ = XMQ_ERROR_CANNOT_READ_FILE;
+        doq->error_ = build_error_message("xmq: %s: Cannot read file\n", file);
+        goto exit;
+    }
+    fclose(f);
+    buffer[fsize] = 0;
+    rc = true;
+
+exit:
+
+    *out_fsize = fsize;
+    *out_buffer = buffer;
+    return rc;
+}
+
+const char *build_error_message(const char* fmt, ...)
+{
+    char *buf = (char*)malloc(4096);
+    va_list args;
+    va_start(args, fmt);
+    vsnprintf(buf, 4096, fmt, args);
+    va_end(args);
+    buf[4095] = 0;
+    buf = (char*)realloc(buf, strlen(buf)+1);
+    return buf;
+}
+
 
 #endif // XMQ_INTERNALS_MODULE

src/main/c/parts/xmq_internals.h

@@ -695,6 +695,9 @@ XMQParseState *xmq_get_yaep_parse_state(XMQDoc *doc);
 
 void set_node_namespace(XMQParseState *state, xmlNodePtr node, const char *node_name);
 
+bool load_file(XMQDoc *doq, const char *file, size_t *out_fsize, const char **out_buffer);
+bool load_stdin(XMQDoc *doq, size_t *out_fsize, const char **out_buffer);
+
 // Multicolor terminals like gnome-term etc.
 
 #define NOCOLOR      "\033[0m"