From 1b6f077e0dbdf01fbc0e44aaaa27d28bbfde504a Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Tue, 14 May 2024 15:24:30 -0500 Subject: [PATCH] Make tests pass when --disable-decnum --- .github/workflows/decnum.yml | 1 + docs/content/manual/manual.yml | 49 +++++++++++++++++++++++++--------- jq.1.prebuilt | 30 ++++++++++++++------- src/builtin.c | 11 ++++++++ src/jq_test.c | 2 ++ tests/jq.test | 14 +++++----- tests/man.test | 12 ++++----- tests/shtest | 6 +++-- 8 files changed, 88 insertions(+), 37 deletions(-) diff --git a/.github/workflows/decnum.yml b/.github/workflows/decnum.yml index 209d6d4b7e..7ccdc5c6bf 100644 --- a/.github/workflows/decnum.yml +++ b/.github/workflows/decnum.yml @@ -25,6 +25,7 @@ jobs: --disable-maintainer-mode \ --disable-decnum make -j"$(nproc)" + make check file ./jq - name: Test run: | diff --git a/docs/content/manual/manual.yml b/docs/content/manual/manual.yml index 6d7597bf94..f10f072efc 100644 --- a/docs/content/manual/manual.yml +++ b/docs/content/manual/manual.yml @@ -357,10 +357,13 @@ sections: The way in which jq handles numbers has changed over time and further changes are likely within the parameters set by - the relevant JSON standards. The following remarks are - therefore offered with the understanding that they are - intended to be descriptive of the current version of jq and - should not be interpreted as being prescriptive: + the relevant JSON standards. Moreover, build configuration + options can alter how jq processes numbers. + + The following remarks are therefore offered with the + understanding that they are intended to be descriptive of the + current version of jq and should not be interpreted as being + prescriptive: (1) Any arithmetic operation on a number that has not already been converted to an IEEE754 double precision @@ -368,9 +371,10 @@ sections: representation. (2) jq will attempt to maintain the original decimal - precision of number literals, but in expressions such - `1E1234567890`, precision will be lost if the exponent is - too large. + precision of number literals (if the `--disable-decnum` + build configuration option was not used), but in expressions + such `1E1234567890`, precision will be lost if the exponent + is too large. (3) In jq programs, a leading minus sign will trigger the conversion of the number to an IEEE754 representation. @@ -379,6 +383,12 @@ sections: big decimal representation of numbers if available, as illustrated in one of the following examples. + The examples below use the builtin function `have_decnum` in + order to demonstrate the expected effects of using / not + using the `--disable-decnum` build configuration option, and + also to allow automated tests derived from these examples to + pass regardless of whether that option is used. + examples: - program: '.' input: '"Hello, world!"' @@ -388,21 +398,21 @@ sections: input: '0.12345678901234567890123456789' output: ['0.12345678901234567890123456789'] - - program: '[., tojson]' + - program: '[., tojson] | . == if have_decnum then [12345678909876543212345,"12345678909876543212345"] else [12345678909876543000000,"12345678909876543000000"] end' input: '12345678909876543212345' - output: ['[12345678909876543212345,"12345678909876543212345"]'] + output: ['true'] - program: '. < 0.12345678901234567890123456788' input: '0.12345678901234567890123456789' output: ['false'] - - program: 'map([., . == 1]) | tojson' + - program: 'map([., . == 1]) | tojson | . == if have_decnum then "[[1,true],[1.000,true],[1.0,true],[1.00,true]]" else "[[1,true],[1,true],[1,true],[1,true]]" end' input: '[1, 1.000, 1.0, 100e-2]' - output: ['"[[1,true],[1.000,true],[1.0,true],[1.00,true]]"'] + output: ['true'] - - program: '. as $big | [$big, $big + 1] | map(. > 10000000000000000000000000000000)' + - program: '. as $big | [$big, $big + 1] | map(. > 10000000000000000000000000000000) | . == if have_decnum then [true, false] else [false, false] end' input: '10000000000000000000000000000001' - output: ['[true, false]'] + output: ['true'] - title: "Object Identifier-Index: `.foo`, `.foo.bar`" body: | @@ -1973,6 +1983,19 @@ sections: output: - '[{"a":{"b":2}}]' + - title: "`have_literal_numbers`" + body: | + + This builtin returns true if jq's build configuration + includes support for preservation of input number literals. + + - title: "`have_decnum`" + body: | + + This builtin returns true if jq was built with "decnum", + which is the current literal number preserving numeric + backend implementation for jq. + - title: "`$JQ_BUILD_CONFIGURATION`" body: | diff --git a/jq.1.prebuilt b/jq.1.prebuilt index f3103fb9bf..81b897b866 100644 --- a/jq.1.prebuilt +++ b/jq.1.prebuilt @@ -1,5 +1,5 @@ . -.TH "JQ" "1" "February 2024" "" "" +.TH "JQ" "1" "May 2024" "" "" . .SH "NAME" \fBjq\fR \- Command\-line JSON processor @@ -286,13 +286,16 @@ Although the identity filter never modifies the value of its input, jq processin produces \fB1\.7976931348623157e+308\fR on at least one platform\. This is because, in the process of parsing the number, this particular version of jq has converted it to an IEEE754 double\-precision representation, losing precision\. . .P -The way in which jq handles numbers has changed over time and further changes are likely within the parameters set by the relevant JSON standards\. The following remarks are therefore offered with the understanding that they are intended to be descriptive of the current version of jq and should not be interpreted as being prescriptive: +The way in which jq handles numbers has changed over time and further changes are likely within the parameters set by the relevant JSON standards\. Moreover, build configuration options can alter how jq processes numbers\. +. +.P +The following remarks are therefore offered with the understanding that they are intended to be descriptive of the current version of jq and should not be interpreted as being prescriptive: . .P (1) Any arithmetic operation on a number that has not already been converted to an IEEE754 double precision representation will trigger a conversion to the IEEE754 representation\. . .P -(2) jq will attempt to maintain the original decimal precision of number literals, but in expressions such \fB1E1234567890\fR, precision will be lost if the exponent is too large\. +(2) jq will attempt to maintain the original decimal precision of number literals (if the \fB\-\-disable\-decnum\fR build configuration option was not used), but in expressions such \fB1E1234567890\fR, precision will be lost if the exponent is too large\. . .P (3) In jq programs, a leading minus sign will trigger the conversion of the number to an IEEE754 representation\. @@ -300,6 +303,9 @@ The way in which jq handles numbers has changed over time and further changes ar .P (4) Comparisons are carried out using the untruncated big decimal representation of numbers if available, as illustrated in one of the following examples\. . +.P +The examples below use the builtin function \fBhave_decnum\fR in order to demonstrate the expected effects of using / not using the \fB\-\-disable\-decnum\fR build configuration option, and also to allow automated tests derived from these examples to pass regardless of whether that option is used\. +. .IP "" 4 . .nf @@ -312,21 +318,21 @@ jq \'\.\' 0\.12345678901234567890123456789 => 0\.12345678901234567890123456789 -jq \'[\., tojson]\' +jq \'[\., tojson] | \. == if have_decnum then [12345678909876543212345,"12345678909876543212345"] else [12345678909876543000000,"12345678909876543000000"] end\' 12345678909876543212345 -=> [12345678909876543212345,"12345678909876543212345"] +=> true jq \'\. < 0\.12345678901234567890123456788\' 0\.12345678901234567890123456789 => false -jq \'map([\., \. == 1]) | tojson\' +jq \'map([\., \. == 1]) | tojson | \. == if have_decnum then "[[1,true],[1\.000,true],[1\.0,true],[1\.00,true]]" else "[[1,true],[1,true],[1,true],[1,true]]" end\' [1, 1\.000, 1\.0, 100e\-2] -=> "[[1,true],[1\.000,true],[1\.0,true],[1\.00,true]]" +=> true -jq \'\. as $big | [$big, $big + 1] | map(\. > 10000000000000000000000000000000)\' +jq \'\. as $big | [$big, $big + 1] | map(\. > 10000000000000000000000000000000) | \. == if have_decnum then [true, false] else [false, false] end\' 10000000000000000000000000000001 -=> [true, false] +=> true . .fi . @@ -2152,6 +2158,12 @@ jq \'walk( if type == "object" then with_entries( \.key |= sub( "^_+"; "") ) els . .IP "" 0 . +.SS "have_literal_numbers" +This builtin returns true if jq\'s build configuration includes support for preservation of input number literals\. +. +.SS "have_decnum" +This builtin returns true if jq was built with "decnum", which is the current literal number preserving numeric backend implementation for jq\. +. .SS "$JQ_BUILD_CONFIGURATION" This builtin binding shows the jq executable\'s build configuration\. Its value has no particular format, but it can be expected to be at least the \fB\./configure\fR command\-line arguments, and may be enriched in the future to include the version strings for the build tooling used\. . diff --git a/src/builtin.c b/src/builtin.c index 9aebd1f2d2..0bab2394f1 100644 --- a/src/builtin.c +++ b/src/builtin.c @@ -1670,6 +1670,15 @@ static jv f_current_line(jq_state *jq, jv a) { return jq_util_input_get_current_line(jq); } +static jv f_have_decnum(jq_state *jq, jv a) { + jv_free(a); +#ifdef USE_DECNUM + return jv_true(); +#else + return jv_false(); +#endif +} + #define LIBM_DD(name) \ {f_ ## name, #name, 1}, #define LIBM_DD_NO(name) LIBM_DD(name) @@ -1744,6 +1753,8 @@ BINOPS {f_now, "now", 1}, {f_current_filename, "input_filename", 1}, {f_current_line, "input_line_number", 1}, + {f_have_decnum, "have_decnum", 1}, + {f_have_decnum, "have_literal_numbers", 1}, }; #undef LIBM_DDDD_NO #undef LIBM_DDD_NO diff --git a/src/jq_test.c b/src/jq_test.c index 39456861a5..e73d3155b1 100644 --- a/src/jq_test.c +++ b/src/jq_test.c @@ -208,11 +208,13 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata, int skip, int printf(" for test at line number %u: %s\n", lineno, prog); pass = 0; } +#ifdef USE_DECNUM jv as_string = jv_dump_string(jv_copy(expected), rand() & ~(JV_PRINT_COLOR|JV_PRINT_REFCOUNT)); jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length_bytes(jv_copy(as_string))); assert(jv_equal(jv_copy(expected), jv_copy(reparsed))); jv_free(as_string); jv_free(reparsed); +#endif jv_free(expected); jv_free(actual); } diff --git a/tests/jq.test b/tests/jq.test index b94f29d245..b404040584 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1836,17 +1836,17 @@ map(. == 1) # When no arithmetic is involved jq should preserve the literal value -.[0] | tostring +.[0] | tostring | . == if have_decnum then "13911860366432393" else "13911860366432392" end [13911860366432393] -"13911860366432393" +true -.x | tojson +.x | tojson | . == if have_decnum then "13911860366432393" else "13911860366432392" end {"x":13911860366432393} -"13911860366432393" +true -13911860366432393 == 13911860366432392 +(13911860366432393 == 13911860366432392) | . == if have_decnum then false else true end null -false +true # Applying arithmetic to the value will truncate the result to double @@ -1939,7 +1939,7 @@ tojson | fromjson {"a":null} # also "nan with payload" #2985 -fromjson | isnan +if have_decnum then fromjson else nan end | isnan "nan1234" true diff --git a/tests/man.test b/tests/man.test index 07938cd586..3f06032619 100644 --- a/tests/man.test +++ b/tests/man.test @@ -6,21 +6,21 @@ 0.12345678901234567890123456789 0.12345678901234567890123456789 -[., tojson] +[., tojson] | . == if have_decnum then [12345678909876543212345,"12345678909876543212345"] else [12345678909876543000000,"12345678909876543000000"] end 12345678909876543212345 -[12345678909876543212345,"12345678909876543212345"] +true . < 0.12345678901234567890123456788 0.12345678901234567890123456789 false -map([., . == 1]) | tojson +map([., . == 1]) | tojson | . == if have_decnum then "[[1,true],[1.000,true],[1.0,true],[1.00,true]]" else "[[1,true],[1,true],[1,true],[1,true]]" end [1, 1.000, 1.0, 100e-2] -"[[1,true],[1.000,true],[1.0,true],[1.00,true]]" +true -. as $big | [$big, $big + 1] | map(. > 10000000000000000000000000000000) +. as $big | [$big, $big + 1] | map(. > 10000000000000000000000000000000) | . == if have_decnum then [true, false] else [false, false] end 10000000000000000000000000000001 -[true, false] +true .foo {"foo": 42, "bar": "less interesting data"} diff --git a/tests/shtest b/tests/shtest index 6cc2e1725b..03fbf665e5 100755 --- a/tests/shtest +++ b/tests/shtest @@ -668,9 +668,11 @@ if ! x=$($JQ -cn '[ fi # CVE-2023-50268: No stack overflow comparing a nan with a large payload -$VALGRIND $Q $JQ '1 != .' <<\EOF >/dev/null -Nan4000 +if $JQ -ne 'have_decnum'; then + $VALGRIND $Q $JQ '1 != .' <<\EOF >/dev/null + Nan4000 EOF +fi # Allow passing the inline jq script before -- #2919 if ! r=$($JQ --args -rn -- '$ARGS.positional[0]' bar) || [ "$r" != bar ]; then