cli: Add --decode-file VAR PATH support

doc/usage.md

@@ -7,25 +7,26 @@ Tool, language and decoders for exploring binary data.
 For more information see https://github.com/wader/fq
 
 Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
---arg NAME VALUE        Set variable $NAME to string VALUE
---argjson NAME JSON     Set variable $NAME to JSON
---compact-output,-c     Compact output
---decode,-d NAME        Decode format (probe)
---formats               Show supported formats
---from-file,-f PATH     Read EXPR from file
---help,-h               Show help
---include-path,-L PATH  Include search path
---join-output,-j        No newline between outputs
---null-input,-n         Null input (use input/0 and inputs/0 to read input)
---null-output,-0        Null byte between outputs
---option,-o KEY=VALUE   Set option, eg: color=true
---options               Show all options
---raw-input,-R          Read raw input strings (don't decode)
---raw-output,-r         Raw string output (without quotes)
---rawfile NAME PATH     Set variable $NAME to string content of file
---repl,-i               Interactive REPL
---slurp,-s              Read (slurp) all inputs into an array
---version,-v            Show version (dev)
+--arg NAME VALUE         Set variable $NAME to string VALUE
+--argjson NAME JSON      Set variable $NAME to JSON
+--compact-output,-c      Compact output
+--decode,-d NAME         Decode format (probe)
+--decode-file NAME PATH  Set variable $NAME to decode of file
+--formats                Show supported formats
+--from-file,-f PATH      Read EXPR from file
+--help,-h                Show help
+--include-path,-L PATH   Include search path
+--join-output,-j         No newline between outputs
+--null-input,-n          Null input (use input/0 and inputs/0 to read input)
+--null-output,-0         Null byte between outputs
+--option,-o KEY=VALUE    Set option, eg: color=true
+--options                Show all options
+--raw-input,-R           Read raw input strings (don't decode)
+--raw-output,-r          Raw string output (without quotes)
+--rawfile NAME PATH      Set variable $NAME to string content of file
+--repl,-i                Interactive REPL
+--slurp,-s               Read (slurp) all inputs into an array
+--version,-v             Show version (dev)
 </pre>
 
 - TODO: null input

pkg/interp/funcs.jq

@@ -1,21 +1,3 @@
-def display($opts): _display($opts);
-def display: _display({});
-def d($opts): _display($opts);
-def d: _display({});
-def full($opts): _display({arraytruncate: 0} + $opts);
-def full: full({});
-def f($opts): full($opts);
-def f: full;
-def verbose($opts): _display({verbose: true, arraytruncate: 0} + $opts);
-def verbose: verbose({});
-def v($opts): verbose($opts);
-def v: verbose;
-
-def decode($name; $opts): _decode($name; $opts);
-def decode($name): _decode($name; {});
-def decode: _decode("probe"; {});
-
-
 # TODO: figure out a saner way to force int
 def _to_int: (. % (. + 1));
 

pkg/interp/interp.jq

@@ -63,6 +63,7 @@ def _build_default_options:
         } | _obj_to_csv_kv
       ),
       compact:         false,
+      decode_file:      [],
       decode_format:   "probe",
       decode_progress: (env.NODECODEPROGRESS == null),
       depth:           0,
@@ -135,6 +136,7 @@ def _to_options:
       color:           (.color | _toboolean),
       colors:          (.colors | _tostring),
       compact:         (.compact | _toboolean),
+      decode_file:     (.decode_file | _toarray(type == "string")),
       decode_format:   (.decode_format | _tostring),
       decode_progress: (.decode_progress | _toboolean),
       depth:           (.depth | _tonumber),
@@ -270,7 +272,7 @@ def _eval($e; $filename; f; on_error; on_compile_error):
       end
   );
 
-def _repl_display: display({depth: 1});
+def _repl_display: _display({depth: 1});
 def _repl_on_error:
   ( if _eval_is_compile_error then _eval_compile_error_tostring end
   | (_error_str | println)
@@ -343,8 +345,37 @@ def _cli_expr_on_compile_error:
   | halt_error(_exit_code_compile_error)
   );
 # _cli_expr_eval halts on compile errors
-def _cli_expr_eval($e; $filename; f): _eval($e; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
-def _cli_expr_eval($e; $filename): _eval($e; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
+def _cli_expr_eval($e; $filename; f):
+  _eval($e; $filename; f; _cli_expr_on_error; _cli_expr_on_compile_error);
+def _cli_expr_eval($e; $filename):
+  _eval($e; $filename; .; _cli_expr_on_error; _cli_expr_on_compile_error);
+
+
+# TODO: introspect and show doc, reflection somehow?
+def help:
+  ( "Type jq expression to evaluate"
+  , "\\t       Auto completion"
+  , "Up/Down  History"
+  , "^C       Interrupt execution"
+  , "^D       Exit REPL"
+  ) | println;
+
+def display($opts): _display($opts);
+def display: _display({});
+def d($opts): _display($opts);
+def d: _display({});
+def full($opts): _display({arraytruncate: 0} + $opts);
+def full: full({});
+def f($opts): full($opts);
+def f: full;
+def verbose($opts): _display({verbose: true, arraytruncate: 0} + $opts);
+def verbose: verbose({});
+def v($opts): verbose($opts);
+def v: verbose;
+
+def decode($name; $opts): _decode($name; $opts);
+def decode($name): _decode($name; {});
+def decode: _decode(options.decode_format; {});
 
 # next valid input
 def input:
@@ -435,14 +466,6 @@ def var($k; f):
   );
 def var($k): . as $c | var($k; $c);
 
-# TODO: introspect and show doc, reflection somehow?
-def help:
-  ( "Type jq expression to evaluate"
-  , "\\t       Auto completion"
-  , "Up/Down  History"
-  , "^C       Interrupt execution"
-  , "^D       Exit REPL"
-  ) | println;
 
 def _main:
   def _formats_list:
@@ -484,6 +507,11 @@ def _main:
         description: "Decode format (probe)",
         string: "NAME"
       },
+      "decode_file": {
+        long: "--decode-file",
+        description: "Set variable $NAME to decode of file",
+        pairs: "NAME PATH"
+      },
       "expr_file": {
         short: "-f",
         long: "--from-file",
@@ -591,9 +619,11 @@ def _main:
   | ( try args_parse($args[1:]; _opts($version))
       catch halt_error(_exit_code_args_error)
     ) as {parsed: $parsed_args, $rest}
-  | _default_options(_build_default_options) as $_
+  | _build_default_options as $default_opts
+  | _default_options($default_opts) as $_
   # combine --args and -o key=value args
-  | ( ($parsed_args.option | _to_options)
+  | ( $default_opts
+    + ($parsed_args.option | _to_options)
     + $parsed_args
     ) as $args_opts
   | _options_stack(
@@ -611,7 +641,20 @@ def _main:
                 end
               )
             ),
+            decode_file: (
+              ( $args_opts.decode_file
+              | if . then
+                  ( map(.[1] |=
+                      try (open | decode($args_opts.decode_format))
+                      catch halt_error(_exit_code_args_error)
+                    )
+                  )
+                end
+              )
+            ),
             expr: (
+              # if -f was used, all rest non-args are filenames
+              # otherwise first is expr rest is filesnames
               ( $args_opts.expr_file
               | if . then
                   try (open | tobytes | tostring)
@@ -692,12 +735,14 @@ def _main:
     else
       # use _finally as display etc prints and results in empty
       _finally(
+        # store some globals
         ( _include_paths($opts.include_path) as $_
-        | _input_filenames($opts.filenames) as $_ # store inputs
+        | _input_filenames($opts.filenames) as $_
         | _variables(
             ( $opts.arg +
               $opts.argjson +
-              $opts.rawfile
+              $opts.rawfile +
+              $opts.decode_file
             | map({key: .[0], value: .[1]})
             | from_entries
             )

pkg/interp/testdata/args.fqtest

@@ -9,25 +9,26 @@ Tool, language and decoders for exploring binary data.
 For more information see https://github.com/wader/fq
 
 Usage: fq [OPTIONS] [--] [EXPR] [FILE...]
---arg NAME VALUE        Set variable $NAME to string VALUE
---argjson NAME JSON     Set variable $NAME to JSON
---compact-output,-c     Compact output
---decode,-d NAME        Decode format (probe)
---formats               Show supported formats
---from-file,-f PATH     Read EXPR from file
---help,-h               Show help
---include-path,-L PATH  Include search path
---join-output,-j        No newline between outputs
---null-input,-n         Null input (use input/0 and inputs/0 to read input)
---null-output,-0        Null byte between outputs
---option,-o KEY=VALUE   Set option, eg: color=true
---options               Show all options
---raw-input,-R          Read raw input strings (don't decode)
---raw-output,-r         Raw string output (without quotes)
---rawfile NAME PATH     Set variable $NAME to string content of file
---repl,-i               Interactive REPL
---slurp,-s              Read (slurp) all inputs into an array
---version,-v            Show version (dev)
+--arg NAME VALUE         Set variable $NAME to string VALUE
+--argjson NAME JSON      Set variable $NAME to JSON
+--compact-output,-c      Compact output
+--decode,-d NAME         Decode format (probe)
+--decode-file NAME PATH  Set variable $NAME to decode of file
+--formats                Show supported formats
+--from-file,-f PATH      Read EXPR from file
+--help,-h                Show help
+--include-path,-L PATH   Include search path
+--join-output,-j         No newline between outputs
+--null-input,-n          Null input (use input/0 and inputs/0 to read input)
+--null-output,-0         Null byte between outputs
+--option,-o KEY=VALUE    Set option, eg: color=true
+--options                Show all options
+--raw-input,-R           Read raw input strings (don't decode)
+--raw-output,-r          Raw string output (without quotes)
+--rawfile NAME PATH      Set variable $NAME to string content of file
+--repl,-i                Interactive REPL
+--slurp,-s               Read (slurp) all inputs into an array
+--version,-v             Show version (dev)
 $ fq -i
 null> ^D
 $ fq -i . /test.mp3
@@ -125,6 +126,7 @@ $ fq --options
   "color": false,
   "colors": "array=white,dumpaddr=yellow,dumpheader=yellow+underline,error=brightred,false=yellow,index=white,null=brightblack,number=cyan,object=white,objectkey=brightblue,string=green,true=yellow,value=white",
   "compact": false,
+  "decode_file": [],
   "decode_format": "probe",
   "decode_progress": false,
   "depth": 0,

pkg/interp/testdata/argvars.fqtest

@@ -0,0 +1,49 @@
+/a.json:
+aaa
+/b.json:
+bbb
+/test.mp3:
+$ fq -i -n --rawfile filea /a.json --rawfile fileb /b.json --arg arga aa --arg argb bb --argjson argjsona 123 --argjson argjsonb '[true,123,{},"abc"]' --decode-file decodefilea /test.mp3 --decode-file decodefileb /test.mp3
+null> $filea
+"aaa\n"
+null> $fileb
+"bbb\n"
+null> $arga
+"aa"
+null> $argb
+"bb"
+null> $argjsona
+123
+null> $argjsonb
+[
+  true,
+  123,
+  {},
+  "abc"
+]
+null> $decodefileb | format
+"mp3"
+null> $decodefileb | format
+"mp3"
+null> ^D
+$ fq -n --rawfile filea /nonexisting
+exitcode: 2
+stderr:
+error: /nonexisting: file not found
+$ fq -n --decode-file filea /nonexisting
+exitcode: 2
+stderr:
+error: /nonexisting: file not found
+$ fq -n -d mp4 --decode-file filea /test.mp3 '$filea'
+     |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|                |.: {} /test.mp3 (mp4)
+     |                                               |                |  error: mp4: failed to validate at position 0x8: no styp, ftyp, free or moov box found
+0x000|49 44 33 04 00 00 00 00 00 23 54 53 53 45 00 00|ID3......#TSSE..|  unknown0: 49443304000000000023545353450000...
+*    |until 0x283.7 (end) (644)                      |                |
+$ fq -n --argjson a '(' '$a'
+exitcode: 5
+stderr:
+error: --argjson a: invalid character '(' looking for beginning of value
+$ fq -n --argjson a
+exitcode: 2
+stderr:
+error: --argjson: needs two argument

pkg/interp/testdata/options.fqtest

@@ -9,6 +9,7 @@ $ fq -n options
   "color": false,
   "colors": "array=white,dumpaddr=yellow,dumpheader=yellow+underline,error=brightred,false=yellow,index=white,null=brightblack,number=cyan,object=white,objectkey=brightblue,string=green,true=yellow,value=white",
   "compact": false,
+  "decode_file": [],
   "decode_format": "probe",
   "decode_progress": false,
   "depth": 0,