diff --git a/JSONPath.sh b/JSONPath.sh index 69cf50c..f300379 100755 --- a/JSONPath.sh +++ b/JSONPath.sh @@ -5,7 +5,6 @@ # --------------------------------------------------------------------------- DEBUG=0 -INCLEMPTY=0 NOCASE=0 WHOLEWORD=0 FILE= @@ -14,9 +13,11 @@ NORMALIZE_SOLIDUS=0 BRIEF=0 PASSTHROUGH=0 JSON=0 -PRINT=1 MULTIPASS=0 FLATTEN=0 +COLON_SPACE=0 +ARRAY_SAME_LINE=0 +TAB_INDENT=0 STDINFILE=/var/tmp/JSONPath.$$.stdin STDINFILE2=/var/tmp/JSONPath.$$.stdin2 PASSFILE=/var/tmp/JSONPath.$$.pass1 @@ -35,26 +36,27 @@ main() { if [[ $QUERY == *'?(@'* ]]; then # This will be a multipass query - [[ -n $FILE ]] && STDINFILE=$FILE - [[ -z $FILE ]] && cat >$STDINFILE + [[ -n $FILE ]] && STDINFILE="$FILE" + [[ -z $FILE ]] && cat >"$STDINFILE" while true; do tokenize_path create_filter - cat "$STDINFILE" | tokenize | parse | filter | indexmatcher >$PASSFILE + tokenize < "$STDINFILE" | parse | filter | indexmatcher >"$PASSFILE" [[ $MULTIPASS -eq 1 ]] && { # replace filter expression with index sequence - SET=$(sed -rn 's/.*[[,"]+([0-9]+)[],].*/\1/p' $PASSFILE | tr '\n' ,) + SET=$(sed -rn 's/.*[[,"]+([0-9]+)[],].*/\1/p' "$PASSFILE" | tr '\n' ,) SET=${SET%,} - QUERY=$(echo $QUERY | sed "s/?(@[^)]\+)/$SET/") + # shellcheck disable=2001 + QUERY=$(echo "$QUERY" | sed "s/?(@[^)]\+)/$SET/") [[ $DEBUG -eq 1 ]] && echo "QUERY=$QUERY" >/dev/stderr reset continue } - cat $PASSFILE | flatten | json | brief + flatten < "$PASSFILE" | json | brief break done @@ -70,11 +72,11 @@ main() { elif [[ -z $FILE ]]; then tokenize | parse | filter | indexmatcher | flatten | json | brief else - cat "$FILE" | tokenize | parse | filter | indexmatcher | flatten | \ + tokenize < "$FILE" | parse | filter | indexmatcher | flatten | \ json | brief fi - fi + fi } # --------------------------------------------------------------------------- @@ -83,7 +85,7 @@ sanity_checks() { # Reset some vars for binary in gawk grep sed; do - if ! which $binary >& /dev/null; then + if ! command -v "$binary" >& /dev/null; then echo "ERROR: $binary binary not found in path. Aborting." exit 1 fi @@ -117,9 +119,9 @@ usage() { # --------------------------------------------------------------------------- echo - echo "Usage: JSONPath.sh [-b] [j] [-h] [-f FILE] [pattern]" + echo "Usage: JSONPath.sh [-h] [-b] [-j] [-u] [-i] [-p] [-w] [-f FILE] [-n] [-s] [-S] [-A] [-T] [pattern]" echo - echo "pattern - the JSONPath query. Defaults to '$.*' if not supplied." + echo "-h - Print this help text." echo "-b - Brief. Only show values." echo "-j - JSON output." echo "-u - Strip unnecessary leading path elements." @@ -127,7 +129,12 @@ usage() { echo "-p - Pass-through to the JSON parser." echo "-w - Match whole words only (for filter script expression)." echo "-f FILE - Read a FILE instead of stdin." - echo "-h - This help text." + echo "-n - Do not print header." + echo "-s - Normalize solidus." + echo "-S - Print spaces around :'s." + echo "-A - Start array on same line as JSON member." + echo "-T - Indent with tabs instead of 4 character spaces." + echo "pattern - the JSONPath query. Defaults to '$.*' if not supplied." echo } @@ -136,45 +143,68 @@ parse_options() { # --------------------------------------------------------------------------- set -- "$@" - local ARGN=$# - while [ "$ARGN" -ne 0 ] - do - case $1 in - -h) usage - exit 0 - ;; - -f) shift - FILE=$1 - ;; - -i) NOCASE=1 - ;; - -j) JSON=1 - ;; - -n) NO_HEAD=1 - ;; - -b) BRIEF=1 - ;; - -u) FLATTEN=1 - ;; - -p) PASSTHROUGH=1 - ;; - -w) WHOLEWORD=1 - ;; - -s) NORMALIZE_SOLIDUS=1 - ;; - ?*) QUERY=$1 - ;; + while getopts :hbjuipwf:nsSAT flag; do + case "$flag" in + h) usage + exit 0 + ;; + b) BRIEF=1 + ;; + j) JSON=1 + ;; + u) FLATTEN=1 + ;; + i) NOCASE=1 + ;; + p) PASSTHROUGH=1 + ;; + w) WHOLEWORD=1 + ;; + f) FILE="$OPTARG" + ;; + n) NO_HEAD=1 + ;; + s) NORMALIZE_SOLIDUS=1 + ;; + S) COLON_SPACE=1 + ;; + A) ARRAY_SAME_LINE=1 + ;; + T) TAB_INDENT=1 + ;; + \?) echo "$0: ERROR: invalid option: -$OPTARG" 1>&2 + usage + exit 3 + ;; + :) echo "$0: ERROR: option -$OPTARG requires an argument" 1>&2 + usage + exit 3 + ;; + *) echo "$0: ERROR: unexpected value from getopts: $flag" 1>&2 + usage + exit 3 + ;; esac - shift 1 - ARGN=$((ARGN-1)) done - [[ -z $QUERY ]] && QUERY='$.*' + # remove the options + # + shift $(( OPTIND - 1 )); + # parse optional patten + case "$#" in + 0) QUERY='$.*' + ;; + 1) QUERY="$*" + ;; + *) echo "$0: ERROR: expected 0 or 1 args, found: $#" 1>&2 + usage + ;; + esac } # --------------------------------------------------------------------------- awk_egrep() { # --------------------------------------------------------------------------- - local pattern_string=$1 + local pattern_string="$1" gawk '{ while ($0) { @@ -195,14 +225,14 @@ tokenize() { local ESCAPE local CHAR - if echo "test string" | egrep -ao --color=never "test" >/dev/null 2>&1 + if echo "test string" | grep -E -ao --color=never "test" >/dev/null 2>&1 then - GREP='egrep -ao --color=never' + GREP='grep -E -ao --color=never' else - GREP='egrep -ao' + GREP='grep -E -ao' fi - if echo "test string" | egrep -o "test" >/dev/null 2>&1 + if echo "test string" | grep -E -o "test" >/dev/null 2>&1 then ESCAPE='(\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})' CHAR='[^[:cntrl:]"\\]' @@ -218,10 +248,11 @@ tokenize() { local SPACE='[[:space:]]+' # Force zsh to expand $A into multiple words - local is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$') - if [ $is_wordsplit_disabled != 0 ]; then setopt shwordsplit; fi - $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$" - if [ $is_wordsplit_disabled != 0 ]; then unsetopt shwordsplit; fi + local is_wordsplit_disabled + is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$') + if [[ $is_wordsplit_disabled != 0 ]]; then setopt shwordsplit; fi + $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | grep -E -v "^$SPACE$" + if [[ $is_wordsplit_disabled != 0 ]]; then unsetopt shwordsplit; fi } # --------------------------------------------------------------------------- @@ -231,14 +262,14 @@ tokenize_path () { local ESCAPE local CHAR - if echo "test string" | egrep -ao --color=never "test" >/dev/null 2>&1 + if echo "test string" | grep -E -ao --color=never "test" >/dev/null 2>&1 then - GREP='egrep -ao --color=never' + GREP='grep -E -ao --color=never' else - GREP='egrep -ao' + GREP='grep -E -ao' fi - if echo "test string" | egrep -o "test" >/dev/null 2>&1 + if echo "test string" | grep -E -o "test" >/dev/null 2>&1 then CHAR='[^[:cntrl:]"\\]' else @@ -256,18 +287,19 @@ tokenize_path () { local SPACE='[[:space:]]+' # Force zsh to expand $A into multiple words - local is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$') - if [ $is_wordsplit_disabled != 0 ]; then setopt shwordsplit; fi + local is_wordsplit_disabled + is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$') + if [[ $is_wordsplit_disabled != 0 ]]; then setopt shwordsplit; fi readarray -t PATHTOKENS < <( echo "$QUERY" | \ $GREP "$INDEX|$STRING|$WORD|$WILDCARD|$FILTER|$DEEPSCAN|$SET|$INDEXALL|." | \ - egrep -v "^$SPACE$|^\\.$|^\[$|^\]$|^'$|^\\\$$|^\)$") + grep -E -v "^$SPACE$|^\\.$|^\[$|^\]$|^'$|^\\\$$|^\)$") [[ $DEBUG -eq 1 ]] && { - echo "egrep -o '$INDEX|$STRING|$WORD|$WILDCARD|$FILTER|$DEEPSCAN|$SET|$INDEXALL|.'" >/dev/stderr + echo "grep -E -o '$INDEX|$STRING|$WORD|$WILDCARD|$FILTER|$DEEPSCAN|$SET|$INDEXALL|.'" >/dev/stderr echo -n "TOKENISED QUERY="; echo "$QUERY" | \ $GREP "$INDEX|$STRING|$WORD|$WILDCARD|$FILTER|$DEEPSCAN|$SET|$INDEXALL|." | \ - egrep -v "^$SPACE$|^\\.$|^\[$|^\]$|^'$|^\\\$$|^\)$" >/dev/stderr + grep -E -v "^$SPACE$|^\\.$|^\[$|^\]$|^'$|^\\\$$|^\)$" >/dev/stderr } - if [ $is_wordsplit_disabled != 0 ]; then unsetopt shwordsplit; fi + if [[ $is_wordsplit_disabled != 0 ]]; then unsetopt shwordsplit; fi } # --------------------------------------------------------------------------- @@ -285,72 +317,77 @@ create_filter() { case "${PATHTOKENS[i]}" in '"') : ;; - '..') query+="$comma[^]]*" + '..') query+="${comma}[^]]*" comma= ;; - '[*]') query+="$comma[^,]*" + '[*]') query+="${comma}[^,]*" comma="," ;; - '*') query+="$comma(\"[^\"]*\"|[0-9]+[^],]*)" + '*') query+="${comma}(\"[^\"]*\"|[0-9]+[^],]*)" comma="," ;; '?(@'*) a=${PATHTOKENS[i]#?(@.} elem="${a%%[<>=!]*}" rhs="${a##*[<>=!]}" - a="${a#$elem}" + a="${a#"$elem"}" elem="${elem//./[\",.]+}" # Allows child node matching - operator="${a%$rhs}" + operator="${a%"${rhs}"}" [[ -z $operator ]] && { operator="=="; rhs=; } if [[ $rhs == *'"'* || $rhs == *"'"* ]]; then - case $operator in + case "$operator" in '=='|'=') OPERATOR= if [[ $elem == '?(@' ]]; then # To allow search on @.property such as: # $..book[?(@.title==".*Book 1.*")] - query+="$comma[0-9]+[],][[:space:]\"]*${rhs//\"/}" + query+="${comma}[0-9]+[],][[:space:]\"]*${rhs//\"/}" else # To allow search on @ (this node) such as: # $..reviews[?(@==".*Fant.*")] - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*${rhs//\"/}" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*${rhs//\"/}" fi FILTER="$query" ;; '>='|'>') OPERATOR=">" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" ;; '<='|'<') OPERATOR="<" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" ;; + *) + ;; esac else case $operator in '=='|'=') OPERATOR= - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*$rhs" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*$rhs" FILTER="$query" ;; '>=') OPERATOR="-ge" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" ;; '>') OPERATOR="-gt" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" ;; '<=') OPERATOR="-le" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" ;; '<') OPERATOR="-lt" RHS="$rhs" - query+="$comma[0-9]+,\"$elem\"[],][[:space:]\"]*" + query+="${comma}[0-9]+,\"$elem\"[],][[:space:]\"]*" FILTER="$query" + ;; + *) + ;; esac fi MULTIPASS=1 @@ -361,18 +398,18 @@ create_filter() { if [[ $a =~ [[:alpha:]] ]]; then # converts only one comma: s/("[^"]+),([^"]+")/\1`\2/g;s/"//g #a=$(echo $a | sed 's/\([[:alpha:]]*\)/"\1"/g') - a=$(echo $a | sed -r "s/[\"']//g;s/([^,]*)/\"\1\"/g") + a=$(echo "$a" | sed -r "s/[\"']//g;s/([^,]*)/\"\1\"/g") fi - query+="$comma(${a//,/|})" + query+="${comma}(${a//,/|})" elif [[ ${PATHTOKENS[i]} =~ : ]]; then if ! [[ ${PATHTOKENS[i]} =~ [0-9][0-9] || ${PATHTOKENS[i]} =~ :] ]] then if [[ ${PATHTOKENS[i]#*:} =~ : ]]; then INDEXMATCH_QUERY+=("${PATHTOKENS[i]}") - query+="$comma[^,]*" + query+="${comma}[^,]*" else # Index in the range of 0-9 can be handled by regex - query+="${comma}$(echo ${PATHTOKENS[i]} | \ + query+="${comma}$(echo "${PATHTOKENS[i]}" | gawk '/:/ { a=substr($0,0,index($0,":")-1); b=substr($0,index($0,":")+1,index($0,"]")-index($0,":")-1); if(b>0) { print a ":" b-1 "]" }; @@ -383,13 +420,13 @@ create_filter() { fi else INDEXMATCH_QUERY+=("${PATHTOKENS[i]}") - query+="$comma[^,]*" + query+="${comma}[^,]*" fi else a=${PATHTOKENS[i]#[} a=${a%]} if [[ $a =~ [[:alpha:]] ]]; then - a=$(echo $a | sed -r "s/[\"']//g;s/([^,]*)/\"\1\"/g") + a=$(echo "$a" | sed -r "s/[\"']//g;s/([^,]*)/\"\1\"/g") else [[ $i -gt 0 ]] && comma="," fi @@ -402,10 +439,10 @@ create_filter() { comma="," ;; esac - i=i+1 + ((++i)) done - [[ -z $FILTER ]] && FILTER="$query[],]" + [[ -z $FILTER ]] && FILTER="${query}[],]" [[ $DEBUG -eq 1 ]] && echo "FILTER=$FILTER" >/dev/stderr } @@ -425,7 +462,7 @@ parse_array () { do parse_value "$1" "$index" index=$((index+1)) - ary="$ary""$value" + ary="$ary""$value" read -r token case "$token" in ']') break ;; @@ -449,13 +486,13 @@ parse_object () { local obj='' read -r token case "$token" in - '}') + '}') ;; *) while : do case "$token" in - '"'*'"') key=$token ;; + '"'*'"') key="$token" ;; *) throw "EXPECTED string GOT ${token:-EOF}" ;; esac read -r token @@ -465,7 +502,7 @@ parse_object () { esac read -r token parse_value "$1" "$key" - obj="$obj$key:$value" + obj="$obj$key:$value" read -r token case "$token" in '}') break ;; @@ -491,18 +528,19 @@ parse_value () { '[') parse_array "$jpath" ;; # At this point, the only valid single-character tokens are digits. ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;; - *) value=$token + *) value="$token" # if asked, replace solidus ("\/") in json strings with normalized value: "/" - [ "$NORMALIZE_SOLIDUS" -eq 1 ] && value=$(echo "$value" | sed 's#\\/#/#g') + # shellcheck disable=SC2001 + [[ "$NORMALIZE_SOLIDUS" -eq 1 ]] && value=$(echo "$value" | sed 's#\\/#/#g') isleaf=1 - [ "$value" = '""' ] && isempty=1 + [[ "$value" = '""' ]] && isempty=1 ;; esac - [[ -z INCLEMPTY ]] && [ "$value" = '' ] && return - [ "$NO_HEAD" -eq 1 ] && [ -z "$jpath" ] && return + [[ "$value" = '' ]] && return + [[ "$NO_HEAD" -eq 1 && -z "$jpath" ]] && return - [ "$isleaf" -eq 1 ] && [ $isempty -eq 0 ] && print=1 - [ "$print" -eq 1 ] && printf "[%s]\t%s\n" "$jpath" "$value" + [[ "$isleaf" -eq 1 && $isempty -eq 0 ]] && print=1 + [[ "$print" -eq 1 ]] && printf "[%s]\t%s\n" "$jpath" "$value" : } @@ -515,11 +553,11 @@ flatten() { if [[ $FLATTEN -eq 1 ]]; then cat >"$STDINFILE2" - + highest=9999 - while read line; do - a=${line#[};a=${a%%]*} + while read -r line; do + a=${line#[}; a=${a%%]*} readarray -t path < <(grep -o "[^,]*"<<<"$a") [[ -z $prevpath ]] && { prevpath=("${path[@]}") @@ -529,22 +567,22 @@ flatten() { pathlen=$((${#path[*]}-1)) - for i in `seq 0 $pathlen`; do - [[ ${path[i]} != ${prevpath[i]} ]] && { - high=$i + for i in $(seq 0 "$pathlen"); do + [[ ${path[i]} != "${prevpath[i]}" ]] && { + high="$i" break } done - [[ $high -lt $highest ]] && highest=$high + [[ $high -lt $highest ]] && highest="$high" prevpath=("${path[@]}") done <"$STDINFILE2" - + if [[ $highest -gt 0 ]]; then sed -r 's/\[(([0-9]+|"[^"]+")[],]){'$((highest))'}(.*)/[\3/' \ "$STDINFILE2" - else + else cat "$STDINFILE2" fi else @@ -561,7 +599,7 @@ indexmatcher() { local a b [[ $DEBUG -eq 1 ]] && { - for i in `seq 0 $((${#INDEXMATCH_QUERY[*]}-1))`; do + for i in $(seq 0 $((${#INDEXMATCH_QUERY[*]}-1))); do echo "INDEXMATCH_QUERY[$i]=${INDEXMATCH_QUERY[i]}" >/dev/stderr done } @@ -571,7 +609,7 @@ indexmatcher() { step= if [[ ${#INDEXMATCH_QUERY[*]} -gt 0 ]]; then while read -r line; do - for i in `seq 0 $((${#INDEXMATCH_QUERY[*]}-1))`; do + for i in $(seq 0 $((${#INDEXMATCH_QUERY[*]}-1))); do [[ ${INDEXMATCH_QUERY[i]#*:} =~ : ]] && { step=${INDEXMATCH_QUERY[i]##*:} step=${step%]} @@ -581,7 +619,7 @@ indexmatcher() { a=${q%:*} # <- number before ':' b=${q#*:} # <- number after ':' [[ -z $b ]] && b=99999999999 - readarray -t num < <( (grep -Eo '[0-9]+[],]' | tr -d ,])<<<$line ) + readarray -t num < <( (grep -Eo '[0-9]+[],]' | tr -d ,])<<<"$line" ) if [[ ${num[i]} -ge $a && ${num[i]} -lt $b && matched -eq 1 ]]; then matched=1 [[ $i -eq $((${#INDEXMATCH_QUERY[*]}-1)) ]] && { @@ -615,7 +653,11 @@ brief() { if [[ $BRIEF -eq 1 ]]; then sed 's/^[^\t]*\t//;s/^"//;s/"$//;' else - cat + if [[ $TAB_INDENT == 1 ]]; then + unexpand -t 4 + else + cat + fi fi } @@ -624,73 +666,75 @@ json() { # --------------------------------------------------------------------------- # Turn output into JSON - local a tab=$(echo -e "\t") + local a tab + tab=$(echo -e "\t") local UP=1 DOWN=2 SAME=3 local prevpathlen=-1 prevpath=() path a + local QUOTE_STAR='"*' declare -a closers if [[ $JSON -eq 0 ]]; then cat - else while read -r line; do - a=${line#[};a=${a%%]*} + a=${line#[}; a=${a%%]*} readarray -t path < <(grep -o "[^,]*"<<<"$a") - value=${line#*$tab} + value=${line#*"$tab"} # Not including the object itself (last item) pathlen=$((${#path[*]}-1)) # General direction - direction=$SAME - [[ $pathlen -gt $prevpathlen ]] && direction=$DOWN - [[ $pathlen -lt $prevpathlen ]] && direction=$UP + direction="$SAME" + [[ $pathlen -gt $prevpathlen ]] && direction="$DOWN" + [[ $pathlen -lt $prevpathlen ]] && direction="$UP" # Handle jumps UP the tree (close previous paths) [[ $prevpathlen != -1 ]] && { - for i in `seq 0 $((pathlen-1))`; do - [[ ${prevpath[i]} == ${path[i]} ]] && continue + for i in $(seq 0 $((pathlen-1))); do + [[ ${prevpath[i]} == "${path[i]}" ]] && continue [[ ${path[i]} != '"'* ]] && { - a=(${!arrays[*]}) - [[ -n $a ]] && { - for k in `seq $((i+1)) ${a[-1]}`; do + a=("${!arrays[@]}") + [[ -n ${a[*]} ]] && { + for k in $(seq $((i+1)) "${a[-1]}"); do arrays[k]= done } - a=(${!comma[*]}) - [[ -n $a ]] && { - for k in `seq $((i+1)) ${a[-1]}`; do + a=("${!comma[@]}") + [[ -n ${a[*]} ]] && { + for k in $(seq $((i+1)) "${a[-1]}"); do comma[k]= done } - for j in `seq $((prevpathlen)) -1 $((i+2))` + for j in $(seq $((prevpathlen)) -1 $((i+2))) do arrays[j]= [[ -n ${closers[j]} ]] && { - let indent=j*4 + ((indent=j*4)) printf "\n%${indent}s${closers[j]}" "" - unset closers[j] + unset 'closers[j]' comma[j]= } done - direction=$DOWN + direction="$DOWN" break } - direction=$DOWN - for j in `seq $((prevpathlen)) -1 $((i+1))` + direction="$DOWN" + for j in $(seq $((prevpathlen)) -1 $((i+1))) do arrays[j]= [[ -n ${closers[j]} ]] && { - let indent=j*4 + ((indent=j*4)) printf "\n%${indent}s${closers[j]}" "" - unset closers[j] + unset 'closers[j]' comma[j]= } done - a=(${!arrays[*]}) - [[ -n $a ]] && { - for k in `seq $i ${a[-1]}`; do + a=("${!arrays[@]}") + [[ -n ${a[*]} ]] && { + for k in $(seq "$i" "${a[-1]}"); do arrays[k]= done } @@ -700,19 +744,19 @@ json() { [[ $direction -eq $UP ]] && { [[ $prevpathlen != -1 ]] && comma[prevpathlen]= - for i in `seq $((prevpathlen+1)) -1 $((pathlen+1))` + for i in $(seq $((prevpathlen+1)) -1 $((pathlen+1))) do arrays[i]= [[ -n ${closers[i]} ]] && { - let indent=i*4 + ((indent=i*4)) printf "\n%${indent}s${closers[i]}" "" - unset closers[i] + unset 'closers[i]' comma[i]= } done - a=(${!arrays[*]}) - [[ -n $a ]] && { - for k in `seq $i ${a[-1]}`; do + a=("${!arrays[@]}") + [[ -n ${a[*]} ]] && { + for k in $(seq "$i" "$a"); do arrays[k]= done } @@ -721,36 +765,46 @@ json() { # Opening braces (the path leading up to the key) broken= - for i in `seq 0 $((pathlen-1))`; do - [[ -z $broken && ${prevpath[i]} == ${path[i]} ]] && continue + for i in $(seq 0 $((pathlen-1))); do + [[ -z $broken && ${prevpath[i]} == "${path[i]}" ]] && continue [[ -z $broken ]] && { - broken=$i + broken="$i" [[ $prevpathlen -ne -1 ]] && broken=$((i+1)) } - if [[ ${path[i]} == '"'* ]]; then + # shellcheck disable=2053 + if [[ ${path[i]} == $QUOTE_STAR ]]; then # Object [[ $i -ge $broken ]] && { - let indent=i*4 + ((indent=i*4)) printf "${comma[i]}%${indent}s{\n" "" closers[i]='}' comma[i]= } - let indent=(i+1)*4 - printf "${comma[i]}%${indent}s${path[i]}:\n" "" + ((indent=(i+1)*4)) + if [[ $COLON_SPACE == 1 ]]; then + printf "${comma[i]}%${indent}s${path[i]} : " "" + else + printf "${comma[i]}%${indent}s${path[i]}:" "" + fi + if [[ $ARRAY_SAME_LINE == 0 ]]; then + echo + fi comma[i]=",\n" else # Array if [[ ${arrays[i]} != 1 ]]; then - let indent=i*4 - printf "%${indent}s" "" + ((indent=i*4)) + if [[ $ARRAY_SAME_LINE == 0 ]]; then + printf "%${indent}s" "" + fi echo "[" closers[i]=']' arrays[i]=1 comma[i]= else - let indent=(i+1)*4 + ((indent=(i+1)*4)) printf "\n%${indent}s${closers[i-1]}" "" - direction=$DOWN + direction="$DOWN" comma[i+1]=",\n" fi fi @@ -761,39 +815,43 @@ json() { if [[ ${path[-1]} == '"'* ]]; then # Object [[ $direction -eq $DOWN ]] && { - let indent=pathlen*4 + ((indent=pathlen*4)) printf "${comma[pathlen]}%${indent}s{\n" "" closers[pathlen]='}' comma[pathlen]= } - let indent=(pathlen+1)*4 + ((indent=(pathlen+1)*4)) printf "${comma[pathlen]}%${indent}s" "" - echo -n "${path[-1]}:$value" + if [[ $COLON_SPACE == 1 ]]; then + echo -n "${path[-1]} : $value" + else + echo -n "${path[-1]}:$value" + fi comma[pathlen]=",\n" else # Array [[ ${arrays[i]} != 1 ]] && { - let indent=(pathlen-0)*4 + ((indent=(pathlen-0)*4)) printf "%${indent}s[\n" "" closers[pathlen]=']' comma[pathlen]= arrays[i]=1 } - let indent=(pathlen+1)*4 + ((indent=(pathlen+1)*4)) printf "${comma[pathlen]}%${indent}s" "" echo -n "$value" comma[pathlen]=",\n" fi prevpath=("${path[@]}") - prevpathlen=$pathlen + prevpathlen="$pathlen" done # closing braces - for i in `seq $((pathlen)) -1 0` + for i in $(seq $((pathlen)) -1 0) do - let indent=i*4 + ((indent=i*4)) printf "\n%${indent}s${closers[i]}" "" done echo @@ -805,19 +863,22 @@ filter() { # --------------------------------------------------------------------------- # Apply the query filter - local a tab=$(echo -e "\t") v + local a tab v + tab=$(echo -e "\t") + unset opts + declare -ag opts - [[ $NOCASE -eq 1 ]] && opts+="-i" - [[ $WHOLEWORD -eq 1 ]] && opts+=" -w" + [[ $NOCASE -eq 1 ]] && opts+=("-i") + [[ $WHOLEWORD -eq 1 ]] && opts+=("-w") if [[ -z $OPERATOR ]]; then - [[ $MULTIPASS -eq 1 ]] && FILTER="$FILTER[\"]?$" - egrep $opts "$FILTER" + [[ $MULTIPASS -eq 1 ]] && FILTER="${FILTER}[\"]?$" + grep -E "${opts[@]}" "$FILTER" [[ $DEBUG -eq 1 ]] && echo "FILTER=$FILTER" >/dev/stderr else - egrep $opts "$FILTER" | \ - while read line; do - v=${line#*$tab} - case $OPERATOR in + grep -E "${opts[@]}" "$FILTER" | \ + while read -r line; do + v=${line#*"$tab"} + case "$OPERATOR" in '-ge') if gawk '{exit !($1>=$2)}'<<<"$v $RHS";then echo "$line"; fi ;; '-gt') if gawk '{exit !($1>$2) }'<<<"$v $RHS";then echo "$line"; fi @@ -826,14 +887,16 @@ filter() { ;; '-lt') if gawk '{exit !($1<$2) }'<<<"$v $RHS";then echo "$line"; fi ;; - '>') v=${v#\"};v=${v%\"} - RHS=${RHS#\"};RHS=${RHS%\"} + '>') v=${v#\"}; v=${v%\"} + RHS=${RHS#\"}; RHS=${RHS%\"} [[ "${v,,}" > "${RHS,,}" ]] && echo "$line" ;; - '<') v=${v#\"};v=${v%\"} - RHS=${RHS#\"};RHS=${RHS%\"} + '<') v=${v#\"}; v=${v%\"} + RHS=${RHS#\"}; RHS=${RHS%\"} [[ "${v,,}" < "${RHS,,}" ]] && echo "$line" ;; + *) + ;; esac done fi @@ -847,11 +910,9 @@ parse () { read -r token parse_value read -r token - case "$token" in - '') ;; - *) throw "EXPECTED EOF GOT $token" - exit 1;; - esac + if [[ -n $token ]]; then + throw "EXPECTED EOF GOT $token" + fi } # --------------------------------------------------------------------------- @@ -861,7 +922,7 @@ throw() { exit 1 } -if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]); +if [[ "$0" = "${BASH_SOURCE[*]}" || -z "${BASH_SOURCE[*]}" ]]; then main "$@" fi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..42fc945 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +#!/usr/bin/env make +# +# JSONPath.sh - JSONPath implementation written in Bash + +############# +# utilities # +############# + +INSTALL= install +SHELL= bash + +###################### +# target information # +###################### + +DESTDIR= /usr/local/bin + +TARGETS= JSONPath.sh + +###################################### +# all - default rule - must be first # +###################################### + +all: ${TARGETS} + +################################################# +# .PHONY list of rules that do not create files # +################################################# + +.PHONY: all configure clean clobber install + +################################### +# standard Makefile utility rules # +################################### + +configure: + +clean: + +clobber: clean + +install: all + ${INSTALL} -m 0555 ${TARGETS} ${DESTDIR} diff --git a/README.md b/README.md index e844147..1241b8e 100755 --- a/README.md +++ b/README.md @@ -12,32 +12,50 @@ try wrapping the command like `./ensure_deps.sh ./JSONPath.sh`. ## Invocation - JSONPath.sh [-b] [-i] [-j] [-h] [-p] [-u] [-f FILE] [pattern] + JSONPath.sh [-h] [-b] [-j] [-u] [-i] [-p] [-w] [-f FILE] [-n] [-s] [-S] [-A] [-T] [pattern] -pattern -> the JSONPath query. Defaults to '$.\*' if not supplied. +-h +> Show help text. -b > Brief output. Only show the values, not the path and key. --f FILE -> Read a FILE instead of reading from standard input. - --i -> Case insensitive searching. - -j > Output in JSON format, instead of JSON.sh format. -u > Strip unnecessary leading path elements. +-i +> Case insensitive searching. + -p > Pass JSON.sh formatted data through to the JSON parser only. Useful after > JSON.sh data has been manipulated. --h -> Show help text. +-w +> Match whole words only (for filter script expression). + +-f FILE +> Read a FILE instead of reading from standard input. + +-n +> Do not print header. + +-s +> Normalize solidus. + +-S +> Print spaces around :'s. + +-A +> Start array on same line as JSON member. + +-T +> Indent with tabs instead of 4 character spaces. + +pattern +> the JSONPath query. Defaults to '$.\*' if not supplied. ## Requirements @@ -55,6 +73,10 @@ Install with npm: * `sudo npm install -g jsonpath.sh` +Install with make: + +* `make install` + Or copy the `JSONPath.sh` script to your PATH, for example: ``` bash @@ -215,7 +237,7 @@ done -f test/valid/goessner.net.expanded.json \ '$.store.book[?(@.price<4.20)].[title,price]' -# The following does not work yet (TODO) +# The following does not work yet (TODO) ./JSONPath.sh \ -f test/valid/goessner.net.expanded.json \ '$.store.book[(@.length-1)].title' @@ -340,7 +362,7 @@ Show all authors, without showing duplicates and output in JSON format. All authors with duplicates: ``` -$ ./JSONPath.sh -f test/valid/goessner.net.expanded.json '$..author' +$ ./JSONPath.sh -f test/valid/goessner.net.expanded.json '$..author' ... omitted ... ["store","book",9,"author"] "James S. A. Corey" ["store","book",10,"author"] "James S. A. Corey" @@ -352,7 +374,7 @@ Use standard unix tools to remove duplicates: ``` $ ./JSONPath.sh -f test/valid/goessner.net.expanded.json '$..author' \ - | sort -k2 | uniq -f 1 + | sort -k2 | uniq -f 1 ... 11 lines of output ... ``` @@ -388,7 +410,7 @@ $ ./JSONPath.sh -f test/valid/goessner.net.expanded.json \ "book": [ { - "author":"Douglas E. Richards" + "author":"Douglas E. Richards" }, { "author":"Evelyn Waugh" diff --git a/all-docker-tests.sh b/all-docker-tests.sh index 16525a0..b0ea320 100755 --- a/all-docker-tests.sh +++ b/all-docker-tests.sh @@ -1,45 +1,46 @@ +#!/usr/bin/env bash + log=testlog.log # Fedora export IMAGE=json-path-fedora-bash cp test/docker/Dockerfile-fedora test/docker/Dockerfile -./test/docker/wrap_in_docker.sh ./all-tests.sh | tee $log +./test/docker/wrap_in_docker.sh ./all-tests.sh | tee "$log" -a=$(grep 'test(s) failed' $log) +a=$(grep 'test(s) failed' "$log") # Ubuntu export IMAGE=json-path-ubuntu-bash cp test/docker/Dockerfile-ubuntu test/docker/Dockerfile -./test/docker/wrap_in_docker.sh ./all-tests.sh | tee $log +./test/docker/wrap_in_docker.sh ./all-tests.sh | tee "$log" -b=$(grep 'test(s) failed' $log) +b=$(grep 'test(s) failed' "$log") # Centos export IMAGE=json-path-centos-bash cp test/docker/Dockerfile-centos test/docker/Dockerfile -./test/docker/wrap_in_docker.sh ./all-tests.sh | tee $log +./test/docker/wrap_in_docker.sh ./all-tests.sh | tee "$log" -c=$(grep 'test(s) failed' $log) +c=$(grep 'test(s) failed' "$log") # Debian export IMAGE=json-path-debian-bash cp test/docker/Dockerfile-debian test/docker/Dockerfile -./test/docker/wrap_in_docker.sh ./all-tests.sh | tee $log +./test/docker/wrap_in_docker.sh ./all-tests.sh | tee "$log" -d=$(grep 'test(s) failed' $log) +d=$(grep 'test(s) failed' "$log") # Cleanup -rm $log +rm -- "$log" rm test/docker/Dockerfile # Results echo echo "Fedora tests" -echo $a +echo "$a" echo "Ubuntu tests" -echo $b +echo "$b" echo "Centos tests" -echo $c +echo "$c" echo "Debian tests" -echo $c - +echo "$d" diff --git a/all-tests.sh b/all-tests.sh index 84f826f..488f9e9 100755 --- a/all-tests.sh +++ b/all-tests.sh @@ -1,33 +1,39 @@ -#!/bin/sh +#!/usr/bin/env bash -cd ${0%/*} +shopt -s lastpipe +export CD_FAILED= +cd "${0%/*}" || CD_FAILED="true" +if [[ -n $CD_FAILED ]]; then + echo "$0: ERROR: cannot cd ${0%/*}" 1>&2 + exit 1 +fi #set -e fail=0 tests=0 #all_tests=${__dirname:} #echo PLAN ${#all_tests} -for test in test/*.sh ; +find test -mindepth 1 -maxdepth 1 -name '*.sh' -print | while read -r test; do - tests=$((tests+1)) - echo TEST: $test - ./$test + ((++tests)) + echo TEST: "$test" + ./"$test" ret=$? - if [ $ret -eq 0 ] ; then - echo OK: ---- $test - passed=$((passed+1)) + if [[ $ret -eq 0 ]]; then + echo OK: ---- "$test" + ((++passed)) else - echo FAIL: $test $fail - fail=$((fail+ret)) + echo FAIL: "$test" "$fail" + ((fail=fail+ret)) fi done -if [ $fail -eq 0 ]; then +if [[ $fail -eq 0 ]]; then echo -n 'SUCCESS ' exitcode=0 else echo -n 'FAILURE ' exitcode=1 fi -echo $passed / $tests -exit $exitcode +echo "$passed" "/" "$tests" +exit "$exitcode" diff --git a/ensure_deps.sh b/ensure_deps.sh index b79b233..5059fca 100755 --- a/ensure_deps.sh +++ b/ensure_deps.sh @@ -22,18 +22,22 @@ test_gnu_compat() { >&2 echo "Make sure you have GNU sed or compatible installed" fi - if [[ $error -gt 0 && is_osx ]]; then + if [[ $error -gt 0 ]] && is_osx; then >&2 echo "With homebrew you may install necessary deps via \`brew install grep gnu-sed coreutils\`" + >&2 echo 'Put /opt/homebrew/bin/gsed early in your PATH' + >&2 echo 'Then put /opt/homebrew/bin/ggrep early in your PATH' + >&2 echo 'And put /opt/homebrew/var/homebrew/linked/coreutils/libexec/gnubin early in your PATH' + >&2 echo "For example: PATH=$PATH" fi - return $error + return "$error" } main() { if is_osx; then - export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH" - export PATH="/usr/local/opt/gnu-sed/libexec/gnubin:$PATH" + export PATH="/opt/homebrew/bin/gsed:$PATH" + export PATH="/opt/homebrew/bin/ggrep:$PATH" + export PATH="/opt/homebrew/var/homebrew/linked/coreutils/libexec/gnubin:$PATH" fi test_gnu_compat diff --git a/test/flatten-test.sh b/test/flatten-test.sh index 2038f54..4247e68 100755 --- a/test/flatten-test.sh +++ b/test/flatten-test.sh @@ -1,24 +1,30 @@ -#!/bin/sh +#!/usr/bin/env bash -cd ${0%/*} +shopt -s lastpipe +CD_FAILED= +cd "${0%/*}" || CD_FAILED="true" +if [[ -n $CD_FAILED ]]; then + echo "$0: ERROR: cannot cd ${0%/*}" 1>&2 + exit 1 +fi fails=0 i=0 -tests=`ls flatten/*.argp* | wc -l` +tests=$(find flatten -name '*.argp*' -print | wc -l) echo "1..${tests##* }" # Standard flatten tests -for argpfile in flatten/*.argp* +find flatten -name '*.argp*' -print | while read -r argpfile; do input="${argpfile%.*}.json" expected="${argpfile%.*}_${argpfile##*.}.flattened" - argp=$(cat $argpfile) - i=$((i+1)) - if ! ../JSONPath.sh "$argp" -u < "$input" | diff -u - "$expected" + argp=$(< "$argpfile") + ((++i)) + if ! ../JSONPath.sh -u -- "$argp" < "$input" | diff -u -- - "$expected" then echo "not ok $i - $argpfile" - fails=$((fails+1)) + ((++fails)) else echo "ok $i - $argpfile" fi done echo "$fails test(s) failed" -exit $fails +exit "$fails" diff --git a/test/json-encoding-test.sh b/test/json-encoding-test.sh index c58a065..84d28d4 100755 --- a/test/json-encoding-test.sh +++ b/test/json-encoding-test.sh @@ -1,23 +1,29 @@ -#!/bin/sh +#!/usr/bin/env bash -cd ${0%/*} +shopt -s lastpipe +CD_FAILED= +cd "${0%/*}" || CD_FAILED="true" +if [[ -n $CD_FAILED ]]; then + echo "$0: ERROR: cannot cd ${0%/*}" 1>&2 + exit 1 +fi fails=0 i=0 -tests=`ls valid/*.argp* | wc -l` +tests=$(find valid -name '*.argp*' -print | wc -l) echo "1..${tests##* }" # Json output tests -for argpfile in valid/*.argp* +find valid -name '*.argp*' -print | while read -r argpfile; do input="${argpfile%.*}.json" - argp=$(cat $argpfile) - i=$((i+1)) - if ! ../JSONPath.sh "$argp" -j < "$input" | python -mjson.tool >/dev/null + argp=$(< "$argpfile") + ((++i)) + if ! ../JSONPath.sh -j -- "$argp" < "$input" | python3 -mjson.tool >/dev/null then echo "not ok $i - $argpfile" - fails=$((fails+1)) + ((++fails)) else echo "ok $i - JSON validated for $argpfile" fi done echo "$fails test(s) failed" -exit $fails +exit "$fails" diff --git a/test/passthrough-test.sh b/test/passthrough-test.sh index 1504daa..f7a9c9b 100755 --- a/test/passthrough-test.sh +++ b/test/passthrough-test.sh @@ -1,34 +1,40 @@ -#!/bin/bash +#!/usr/bin/env bash -cd ${0%/*} +shopt -s lastpipe +CD_FAILED= +cd "${0%/*}" || CD_FAILED="true" +if [[ -n $CD_FAILED ]]; then + echo "$0: ERROR: cannot cd ${0%/*}" 1>&2 + exit 1 +fi fails=0 i=0 declare -i tests -tests=`ls valid/*.parsed | wc -l` -tests+=`ls flatten/*.flattened | wc -l` +tests=$(find valid -name '*.parsed' -print | wc -l) +tests+=$(find flatten -name '*.flattened' -print | wc -l) echo "1..${tests##* }" # Json output tests -for file in valid/*.parsed +find valid -name '*.parsed' -print | while read -r file; do - i=$((i+1)) - if ! ../JSONPath.sh -p < "$file" | python -mjson.tool >/dev/null + ((++i)) + if ! ../JSONPath.sh -p < "$file" | python3 -mjson.tool >/dev/null then echo "not ok $i - $file" - fails=$((fails+1)) + ((++fails)) else echo "ok $i - JSON validated for $file" fi done -for file in flatten/*.flattened +find flatten -name '*.flattened' -print | while read -r file; do - i=$((i+1)) - if ! ../JSONPath.sh -p < "$file" | python -mjson.tool >/dev/null + ((++i)) + if ! ../JSONPath.sh -p < "$file" | python3 -mjson.tool >/dev/null then echo "not ok $i - $file" - fails=$((fails+1)) + ((++fails)) else echo "ok $i - JSON validated for $file" fi done echo "$fails test(s) failed" -exit $fails +exit "$fails" diff --git a/test/valid-test.sh b/test/valid-test.sh index a4211d3..e17ceee 100755 --- a/test/valid-test.sh +++ b/test/valid-test.sh @@ -1,24 +1,30 @@ -#!/bin/sh +#!/usr/bin/env bash -cd ${0%/*} +shopt -s lastpipe +CD_FAILED= +cd "${0%/*}" || CD_FAILED="true" +if [[ -n $CD_FAILED ]]; then + echo "$0: ERROR: cannot cd ${0%/*}" 1>&2 + exit 1 +fi fails=0 i=0 -tests=`ls valid/*.argp* | wc -l` +tests=$(find valid -name '*.argp*' -print | wc -l) echo "1..${tests##* }" # Standard valid tests -for argpfile in valid/*.argp* +find valid -name '*.argp*' -print | while read -r argpfile; do input="${argpfile%.*}.json" expected="${argpfile%.*}_${argpfile##*.}.parsed" - argp=$(cat $argpfile) - i=$((i+1)) - if ! ../JSONPath.sh "$argp" < "$input" | diff -u - "$expected" + argp=$(< "$argpfile") + ((++i)) + if ! ../JSONPath.sh -- "$argp" < "$input" | diff -u -- - "$expected" then echo "not ok $i - $argpfile" - fails=$((fails+1)) + ((++fails)) else echo "ok $i - $argpfile" fi done echo "$fails test(s) failed" -exit $fails +exit "$fails"