diff --git a/Cargo.lock b/Cargo.lock index 304b2dcd6..cd1fc9595 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2324,16 +2324,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72167d68f5fce3b8655487b8038691a3c9984ee769590f93f2a631f4ad64e4f5" [[package]] -name = "jql" -version = "5.2.0" +name = "jql-parser" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766659fcc44d40b326f0116ae6998838c7031f9bf2bcc556a8cda3532d8972d1" +checksum = "767fb7da2bf204dd3afd725d9acc0555b30782e74253b2c27b4a23b90972fc84" dependencies = [ - "anyhow", - "pest", - "pest_derive", + "nom", + "thiserror", +] + +[[package]] +name = "jql-runner" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfd112dfc9e86bb1b2057ecaf0af565b1983cb98d98a75029532fb511d3de88" +dependencies = [ + "jql-parser", "rayon", "serde_json", + "thiserror", ] [[package]] @@ -2577,9 +2586,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" +checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" [[package]] name = "local-channel" @@ -2670,10 +2679,11 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matrixmultiply" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb99c395ae250e1bf9133673f03ca9f97b7e71b705436bf8f089453445d1e9fe" +checksum = "432b44fed6d9ed750739e7b719f0325555e6209d15e5ec3fd9e8a0a5509fdaf8" dependencies = [ + "autocfg", "rawpointer", ] @@ -3114,50 +3124,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pest" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" -dependencies = [ - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "pest_meta" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" -dependencies = [ - "once_cell", - "pest", - "sha2", -] - [[package]] name = "petgraph" version = "0.6.3" @@ -3547,6 +3513,15 @@ version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -3678,7 +3653,7 @@ dependencies = [ "itertools", "itoa", "jemallocator", - "jql", + "jql-runner", "jsonschema", "jsonxf", "log", @@ -3712,6 +3687,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serde_stacker", "serde_urlencoded", "serial_test", "simdutf8", @@ -4181,9 +4157,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.15" +version = "0.37.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0661814f891c57c930a610266415528da53c4933e6dea5fb350cbfe048a9ece" +checksum = "bc809f704c03a812ac71f22456c857be34185cac691a4316f27ab0f633bb9009" dependencies = [ "bitflags 1.3.2", "errno", @@ -4345,6 +4321,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_stacker" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5557f4c1103cecd0e639a17ab22d670b89912d8a506589ee627bf738a15a5d" +dependencies = [ + "serde", + "stacker", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4560,6 +4546,19 @@ dependencies = [ "lock_api", ] +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -4929,10 +4928,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.38" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if", "log", "pin-project-lite", "tracing-core", @@ -4970,12 +4970,6 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" -[[package]] -name = "ucd-trie" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" - [[package]] name = "unic-char-property" version = "0.9.0" @@ -5091,9 +5085,9 @@ checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" dependencies = [ "getrandom", ] diff --git a/Cargo.toml b/Cargo.toml index 6232bc328..86478bda9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ jsonschema = { version = "0.17", features = [ "resolve-http", ], default-features = false } jsonxf = { version = "1", optional = true } -jql = { version = "5.2", default-features = false, optional = true } +jql-runner = { version = "6.0.6", default-features = false, optional = true } log = "0.4" mimalloc = { version = "0.1", default-features = false, optional = true } mlua = { version = "0.9.0-beta.2", features = [ @@ -168,6 +168,7 @@ self_update = { version = "0.36", features = [ semver = "1" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } +serde_stacker = { version = "0.1", optional = true } serde_urlencoded = { version = "0.7", optional = true } snap = "1" strsim = { version = "0.10", optional = true } @@ -242,9 +243,10 @@ fetch = [ "flate2", "governor", "hashbrown", - "jql", + "jql-runner", "jsonxf", "redis", + "serde_stacker", "serde_urlencoded", ] foreach = [] diff --git a/resources/test/fetch_jql_multiple.jql b/resources/test/fetch_jql_multiple.jql index 175a34436..87a680c92 100644 --- a/resources/test/fetch_jql_multiple.jql +++ b/resources/test/fetch_jql_multiple.jql @@ -1 +1 @@ -"places"[0]."place name","places"[0]."state abbreviation" +"places"[0]"place name","places"[0]"state abbreviation" diff --git a/resources/test/fetch_jql_single.jql b/resources/test/fetch_jql_single.jql index f472fc7b0..6f96c9703 100644 --- a/resources/test/fetch_jql_single.jql +++ b/resources/test/fetch_jql_single.jql @@ -1 +1 @@ -."places"[0]."place name" +"places"[0]"place name" diff --git a/src/cmd/fetch.rs b/src/cmd/fetch.rs index 932878340..c4f9704f3 100644 --- a/src/cmd/fetch.rs +++ b/src/cmd/fetch.rs @@ -44,17 +44,17 @@ Given the data.csv above, fetch the JSON response. Note the output will be a JSONL file - with a minified JSON response per line, not a CSV file. Now, if we want to generate a CSV file with the parsed City and State, we use the -new-column and jql options. (See https://github.com/yamafaktory/jql#jql for more info on -how to use the jql JSON Query Language) +new-column and jql options. (See https://github.com/yamafaktory/jql#%EF%B8%8F-usage +for more info on how to use the jql JSON Query Language) -$ qsv fetch URL --new-column CityState --jql '"places"[0]."place name","places"[0]."state abbreviation"' +$ qsv fetch URL --new-column CityState --jql '"places"[0]"place name","places"[0]"state abbreviation"' data.csv > data_with_CityState.csv data_with_CityState.csv URL, CityState, - https://api.zippopotam.us/us/90210, "Beverly Hills, CA" - https://api.zippopotam.us/us/94105, "San Francisco, CA" - https://api.zippopotam.us/us/92802, "Anaheim, CA" + https://api.zippopotam.us/us/90210, "[\"Beverly Hills\",\"CA\"]" + https://api.zippopotam.us/us/94105, "[\"San Francisco\",\"CA\"]" + https://api.zippopotam.us/us/92802, "[\"Anaheim\",\"CA\"]" As you can see, entering jql selectors on the command line is error prone and can quickly become cumbersome. Alternatively, the jql selector can be saved and loaded from a file using the --jqlfile option. @@ -89,7 +89,7 @@ Geocode addresses in addresses.csv, pass the "street address" and "zip-code" fie and use jql to parse placename from the JSON response into a new column in addresses_with_placename.csv. Note how field name non-alphanumeric characters (space and hyphen) in the url-template were replaced with _. -$ qsv fetch --jql '"features"[0]."properties","name"' addresses.csv -c placename --url-template +$ qsv fetch --jql '"features"[0]"properties","name"' addresses.csv -c placename --url-template "https://api.geocode.earth/v1/search/structured?address={street_address}&postalcode={zip_code}" > addresses_with_placename.csv @@ -203,7 +203,7 @@ use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, }; use serde::{Deserialize, Serialize}; -use serde_json::json; +use serde_json::{json, Value}; use simdutf8::basic::from_utf8; use url::Url; @@ -302,7 +302,6 @@ struct FetchResponse { } static REDISCONFIG: Lazy = Lazy::new(RedisConfig::load); -static JQL_GROUPS: once_cell::sync::OnceCell> = OnceCell::new(); pub fn run(argv: &[&str]) -> CliResult<()> { let args: Args = util::get_args(USAGE, argv)?; @@ -962,11 +961,7 @@ fn get_response( error_flag = false; // apply JQL selector if provided if let Some(selectors) = flag_jql { - // instead of repeatedly parsing the jql selector, - // we compile it only once and cache it for performance using once_cell - let jql_groups = - JQL_GROUPS.get_or_init(|| jql::selectors_parser(selectors).unwrap()); - match apply_jql(&api_value, jql_groups) { + match process_jql(&api_value, selectors) { Ok(s) => { final_value = s; } @@ -1170,85 +1165,52 @@ fn get_response( } } -use jql::groups_walker; -use serde_json::{Deserializer, Value}; +impl From for CliError { + fn from(err: jql_runner::errors::JqlRunnerError) -> CliError { + CliError::Other(format!("jql runner error: {err:?}")) + } +} #[inline] -pub fn apply_jql(json: &str, groups: &[jql::Group]) -> Result { - // check if api returned valid JSON before applying JQL selector - if let Err(error) = serde_json::from_str::(json) { - return fail_format!("Invalid json: {error:?}"); - } +pub fn process_jql(json: &str, query: &str) -> CliResult { + let mut deserializer = serde_json::Deserializer::from_str(json); - let mut result: Result = Ok(String::default()); - - Deserializer::from_str(json) - .into_iter::() - .for_each(|value| match value { - Ok(valid_json) => { - // Walk through the JSON content with the provided selectors as - // input. - match groups_walker(&valid_json, groups) { - Ok(selection) => { - fn get_value_string(v: &Value) -> String { - if v.is_null() { - "null".to_string() - } else if v.is_boolean() { - v.as_bool().unwrap().to_string() - } else if v.is_f64() { - v.as_f64().unwrap().to_string() - } else if v.is_i64() { - v.as_i64().unwrap().to_string() - } else if v.is_u64() { - v.as_u64().unwrap().to_string() - } else if v.is_string() { - v.as_str().unwrap().to_string() - } else { - v.to_string() - } - } + deserializer.disable_recursion_limit(); - match &selection { - Value::Array(array) => { - let mut concat_string = String::new(); + let deserializer = serde_stacker::Deserializer::new(&mut deserializer); - let mut values = array.iter(); + let value = match serde_json::Value::deserialize(deserializer) { + Ok(valid_value) => valid_value, + Err(e) => return fail_clierror!("Failed to deserialize the JSON data: {e:?}"), + }; + let result: Value = jql_runner::runner::raw(query, &value)?; - if let Some(v) = values.next() { - let str_val = get_value_string(v); - concat_string.push_str(&str_val); - } + Ok(serde_json::to_string(&result)?) +} - for v in values { - let str_val = get_value_string(v); - concat_string.push_str(", "); - concat_string.push_str(&str_val); - } +// TODO: use this in the future to process JQL with pre-parsed tokens +// pub fn process_json_with_tokens( +// json: &str, +// tokens_vec: &Vec, +// ) -> CliResult { +// if let Err(error) = serde_json::from_str::(json) { +// return fail_clierror!("Invalid json: {error:?}"); +// } - result = Ok(concat_string); - } - Value::Object(object) => { - // if Value::Object, then just set to debug display of object - result = Ok(format!("{object:?}")); - } - _ => { - result = Ok(get_value_string(&selection)); - } - } - } - Err(error) => { - result = Err(format!("{error:?}")); - } - } - } - Err(error) => { - // shouldn't happen, but do same thing earlier when checking for invalid json - result = Err(format!("Invalid json: {error:?}")); - } - }); +// let mut deserializer = serde_json::Deserializer::from_str(json); - result -} +// deserializer.disable_recursion_limit(); + +// let deserializer = serde_stacker::Deserializer::new(&mut deserializer); + +// let value = match serde_json::Value::deserialize(deserializer) { +// Ok(valid_value) => valid_value, +// Err(e) => return fail_clierror!("Failed to deserialize the JSON data: {e:?}"), +// }; +// let result: Value = jql_runner::runner::token(tokens_vec, &value)?; + +// Ok(serde_json::to_string(&result)?) +// } #[test] fn test_apply_jql_invalid_json() { @@ -1256,11 +1218,10 @@ fn test_apply_jql_invalid_json() { r#"shortest html5"#; let selectors = r#"."places"[0]."place name""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap_err(); + let value = process_jql(json, selectors).unwrap_err().to_string(); assert_eq!( - "Invalid json: Error(\"expected value\", line: 1, column: 1)", + "Failed to deserialize the JSON data: Error(\"expected value\", line: 1, column: 1)", value ); } @@ -1270,30 +1231,41 @@ fn test_apply_jql_invalid_selector() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": "-118.4065", "state": "California", "state abbreviation": "CA", "latitude": "34.0901"}]}"#; let selectors = r#"."place"[0]."place name""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value = apply_jql(json, &jql_groups).unwrap_err(); + let value = process_jql(json, selectors).unwrap_err().to_string(); - assert_eq!(r#""Node \"place\" not found on the parent element""#, value); + assert_eq!( + r#"jql runner error: ParsingError(ParsingError { tokens: "", unparsed: ".\"place\"[0].\"place name\"" })"#, + value + ); } #[test] fn test_apply_jql_string() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": "-118.4065", "state": "California", "state abbreviation": "CA", "latitude": "34.0901"}]}"#; - let selectors = r#"."places"[0]."place name""#; + let selectors = r#""places"[0]"place name""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap(); + let value = process_jql(json, selectors).unwrap(); - assert_eq!("Beverly Hills", value); + assert_eq!(r#""Beverly Hills""#, value); } +// #[test] +// fn test_apply_jql_string_with_tokens_vec() { +// let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": +// "US", "places": [{"place name": "Beverly Hills", "longitude": "-118.4065", "state": "California", +// "state abbreviation": "CA", "latitude": "34.0901"}]}"#; let selectors = r#""places"[0]"place +// name""#; +// let token_vec = jql_parser::parser::parse(&selectors).unwrap(); +// let value = process_json_with_token(json, &token_vec).unwrap(); +// assert_eq!(r#""Beverly Hills""#, value); +// } + #[test] fn test_apply_jql_number() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": -118.4065, "state": "California", "state abbreviation": "CA", "latitude": 34.0901}]}"#; - let selectors = r#"."places"[0]."longitude""#; + let selectors = r#""places"[0]"longitude""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap(); + let value = process_jql(json, selectors).unwrap(); assert_eq!("-118.4065", value); } @@ -1301,10 +1273,9 @@ fn test_apply_jql_number() { #[test] fn test_apply_jql_bool() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": -118.4065, "state": "California", "state abbreviation": "CA", "latitude": 34.0901, "expensive": true}]}"#; - let selectors = r#"."places"[0]."expensive""#; + let selectors = r#""places"[0]"expensive""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap(); + let value = process_jql(json, selectors).unwrap(); assert_eq!("true", value); } @@ -1312,10 +1283,9 @@ fn test_apply_jql_bool() { #[test] fn test_apply_jql_null() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": -118.4065, "state": "California", "state abbreviation": "CA", "latitude": 34.0901, "university":null}]}"#; - let selectors = r#"."places"[0]."university""#; + let selectors = r#""places"[0]"university""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap(); + let value = process_jql(json, selectors).unwrap(); assert_eq!("null", value); } @@ -1323,12 +1293,11 @@ fn test_apply_jql_null() { #[test] fn test_apply_jql_array() { let json = r#"{"post code": "90210", "country": "United States", "country abbreviation": "US", "places": [{"place name": "Beverly Hills", "longitude": -118.4065, "state": "California", "state abbreviation": "CA", "latitude": 34.0901}]}"#; - let selectors = r#"."places"[0]."longitude",."places"[0]."latitude""#; + let selectors = r#""places"[0]"longitude","places"[0]"latitude""#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value: String = apply_jql(json, &jql_groups).unwrap(); + let value = process_jql(json, selectors).unwrap(); - assert_eq!("-118.4065, 34.0901", value); + assert_eq!("[-118.4065,34.0901]", value); } #[test] @@ -1336,13 +1305,12 @@ fn test_root_out_of_bounds() { // test for out_of_bounds root element handling // see https://github.com/yamafaktory/jql/issues/129 let json = r#"[{"page":1,"pages":1,"per_page":"50","total":1},[{"id":"BRA","iso2Code":"BR","name":"Brazil","region":{"id":"LCN","iso2code":"ZJ","value":"Latin America & Caribbean (all income levels)"},"adminregion":{"id":"LAC","iso2code":"XJ","value":"Latin America & Caribbean (developing only)"},"incomeLevel":{"id":"UMC","iso2code":"XT","value":"Upper middle income"},"lendingType":{"id":"IBD","iso2code":"XF","value":"IBRD"},"capitalCity":"Brasilia","longitude":"-47.9292","latitude":"-15.7801"}]]"#; - let selectors = r#"[2].[0]."incomeLevel"."value"'"#; + let selectors = r#"[2][0]"incomeLevel""value"'"#; - let jql_groups = jql::selectors_parser(selectors).unwrap(); - let value = apply_jql(json, &jql_groups).unwrap_err(); + let value = process_jql(json, selectors).unwrap_err().to_string(); assert_eq!( - r#""Index [2] is out of bound, root element has a length of 2""#, + r#"jql runner error: ParsingError(ParsingError { tokens: "Array Index Selector [Index (2)], Array Index Selector [Index (0)], Key Selector \"incomeLevel\", Key Selector \"value\"", unparsed: "'" })"#, value ); } diff --git a/src/cmd/fetchpost.rs b/src/cmd/fetchpost.rs index 1b9af161b..29c84d10f 100644 --- a/src/cmd/fetchpost.rs +++ b/src/cmd/fetchpost.rs @@ -53,8 +53,8 @@ Given the data.csv above, fetch the JSON response. Note the output will be a JSONL file - with a minified JSON response per line, not a CSV file. Now, if we want to generate a CSV file with a parsed response - getting only the "form" property, -we use the new-column and jql options. (See https://github.com/yamafaktory/jql#jql for more info -on how to use the jql JSON Query Language) +we use the new-column and jql options. (See https://github.com/yamafaktory/jql#%EF%B8%8F-usage +for more info on how to use the jql JSON Query Language) $ qsv fetchpost URL zipcode,country --new-column form --jql '"form"' data.csv > data_with_response.csv @@ -200,7 +200,7 @@ use simdutf8::basic::from_utf8; use url::Url; use crate::{ - cmd::fetch::apply_jql, + cmd::fetch::process_jql, config::{Config, Delimiter}, select::SelectColumns, util, CliError, CliResult, @@ -290,7 +290,6 @@ struct FetchResponse { } static REDISCONFIG: Lazy = Lazy::new(RedisConfig::load); -static JQL_GROUPS: once_cell::sync::OnceCell> = OnceCell::new(); pub fn run(argv: &[&str]) -> CliResult<()> { let args: Args = util::get_args(USAGE, argv)?; @@ -987,11 +986,7 @@ fn get_response( error_flag = false; // apply JQL selector if provided if let Some(selectors) = flag_jql { - // instead of repeatedly parsing the jql selector, - // we compile it only once and cache it for performance using once_cell - let jql_groups = - JQL_GROUPS.get_or_init(|| jql::selectors_parser(selectors).unwrap()); - match apply_jql(&api_value, jql_groups) { + match process_jql(&api_value, selectors) { Ok(s) => { final_value = s; } diff --git a/tests/test_fetch.rs b/tests/test_fetch.rs index 475c858d7..4776d418d 100644 --- a/tests/test_fetch.rs +++ b/tests/test_fetch.rs @@ -239,16 +239,16 @@ fn fetch_jql_single() { .arg("--new-column") .arg("City") .arg("--jql") - .arg(r#"."places"[0]."place name""#) + .arg(r#""places"[0]"place name""#) .arg("data.csv"); let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["URL", "City"], - svec!["http://api.zippopotam.us/us/90210", "Beverly Hills"], - svec!["http://api.zippopotam.us/us/94105", "San Francisco"], + svec!["http://api.zippopotam.us/us/90210", "\"Beverly Hills\""], + svec!["http://api.zippopotam.us/us/94105", "\"San Francisco\""], svec!["thisisnotaurl", ""], - svec!["https://api.zippopotam.us/us/92802", "Anaheim"], + svec!["https://api.zippopotam.us/us/92802", "\"Anaheim\""], ]; assert_eq!(got, expected); @@ -281,9 +281,9 @@ fn fetch_jql_single_file() { let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["URL", "City"], - svec!["http://api.zippopotam.us/us/90210", "Beverly Hills"], - svec!["http://api.zippopotam.us/us/94105", "San Francisco"], - svec!["https://api.zippopotam.us/us/92802", "Anaheim"], + svec!["http://api.zippopotam.us/us/90210", "\"Beverly Hills\""], + svec!["http://api.zippopotam.us/us/94105", "\"San Francisco\""], + svec!["https://api.zippopotam.us/us/92802", "\"Anaheim\""], ]; assert_eq!(got, expected); } @@ -331,7 +331,7 @@ fn fetch_jql_jqlfile_error() { "/resources/test/fetch_jql_single.jql" )) .arg("--jql") - .arg(r#"."places"[0]."place name""#) + .arg(r#""places"[0]"place name""#) .arg("data.csv"); let got: String = wrk.output_stderr(&mut cmd); @@ -358,15 +358,21 @@ fn fetch_jql_multiple() { .arg("--new-column") .arg("CityState") .arg("--jql") - .arg(r#""places"[0]."place name","places"[0]."state abbreviation""#) + .arg(r#""places"[0]"place name","places"[0]"state abbreviation""#) .arg("data.csv"); let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["URL", "CityState"], - svec!["http://api.zippopotam.us/us/90210", "Beverly Hills, CA"], - svec!["http://api.zippopotam.us/us/94105", "San Francisco, CA"], - svec!["https://api.zippopotam.us/us/92802", "Anaheim, CA"], + svec![ + "http://api.zippopotam.us/us/90210", + "[\"Beverly Hills\",\"CA\"]" + ], + svec![ + "http://api.zippopotam.us/us/94105", + "[\"San Francisco\",\"CA\"]" + ], + svec!["https://api.zippopotam.us/us/92802", "[\"Anaheim\",\"CA\"]"], ]; assert_eq!(got, expected); } @@ -398,9 +404,15 @@ fn fetch_jql_multiple_file() { let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["URL", "CityState"], - svec!["http://api.zippopotam.us/us/90210", "Beverly Hills, CA"], - svec!["http://api.zippopotam.us/us/94105", "San Francisco, CA"], - svec!["https://api.zippopotam.us/us/92802", "Anaheim, CA"], + svec![ + "http://api.zippopotam.us/us/90210", + "[\"Beverly Hills\",\"CA\"]" + ], + svec![ + "http://api.zippopotam.us/us/94105", + "[\"San Francisco\",\"CA\"]" + ], + svec!["https://api.zippopotam.us/us/92802", "[\"Anaheim\",\"CA\"]"], ]; assert_eq!(got, expected); } @@ -420,11 +432,11 @@ fn fetch_custom_header() { .arg("-H") .arg("X-Api-Secret :ABC123XYZ") .arg("--jql") - .arg(r#""headers"."X-Api-Key","headers"."X-Api-Secret""#) + .arg(r#""headers""X-Api-Key","headers""X-Api-Secret""#) .arg("data.csv"); let got = wrk.stdout::(&mut cmd); - let expected = "DEMO_KEY, ABC123XYZ"; + let expected = "[\"DEMO_KEY\",\"ABC123XYZ\"]"; assert_eq!(got, expected); } @@ -734,7 +746,7 @@ fn fetch_ratelimit() { .arg("--new-column") .arg("Fullname") .arg("--jql") - .arg(r#"."fullname""#) + .arg(r#""fullname""#) .arg("--rate-limit") .arg("4") .arg("data.csv"); @@ -742,31 +754,31 @@ fn fetch_ratelimit() { let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["URL", "Fullname"], - svec![test_url!("user/Smurfette"), "Smurfette Smurf"], - svec![test_url!("user/Papa"), "Papa Smurf"], - svec![test_url!("user/Clumsy"), "Clumsy Smurf"], - svec![test_url!("user/Brainy"), "Brainy Smurf"], - svec![test_url!("user/Grouchy"), "Grouchy Smurf"], - svec![test_url!("user/Hefty"), "Hefty Smurf"], - svec![test_url!("user/Greedy"), "Greedy Smurf"], - svec![test_url!("user/Jokey"), "Jokey Smurf"], - svec![test_url!("user/Chef"), "Chef Smurf"], - svec![test_url!("user/Vanity"), "Vanity Smurf"], - svec![test_url!("user/Handy"), "Handy Smurf"], - svec![test_url!("user/Scaredy"), "Scaredy Smurf"], - svec![test_url!("user/Tracker"), "Tracker Smurf"], - svec![test_url!("user/Sloppy"), "Sloppy Smurf"], - svec![test_url!("user/Harmony"), "Harmony Smurf"], - svec![test_url!("user/Painter"), "Painter Smurf"], - svec![test_url!("user/Poet"), "Poet Smurf"], - svec![test_url!("user/Farmer"), "Farmer Smurf"], - svec![test_url!("user/Natural"), "Natural Smurf"], - svec![test_url!("user/Snappy"), "Snappy Smurf"], + svec![test_url!("user/Smurfette"), "\"Smurfette Smurf\""], + svec![test_url!("user/Papa"), "\"Papa Smurf\""], + svec![test_url!("user/Clumsy"), "\"Clumsy Smurf\""], + svec![test_url!("user/Brainy"), "\"Brainy Smurf\""], + svec![test_url!("user/Grouchy"), "\"Grouchy Smurf\""], + svec![test_url!("user/Hefty"), "\"Hefty Smurf\""], + svec![test_url!("user/Greedy"), "\"Greedy Smurf\""], + svec![test_url!("user/Jokey"), "\"Jokey Smurf\""], + svec![test_url!("user/Chef"), "\"Chef Smurf\""], + svec![test_url!("user/Vanity"), "\"Vanity Smurf\""], + svec![test_url!("user/Handy"), "\"Handy Smurf\""], + svec![test_url!("user/Scaredy"), "\"Scaredy Smurf\""], + svec![test_url!("user/Tracker"), "\"Tracker Smurf\""], + svec![test_url!("user/Sloppy"), "\"Sloppy Smurf\""], + svec![test_url!("user/Harmony"), "\"Harmony Smurf\""], + svec![test_url!("user/Painter"), "\"Painter Smurf\""], + svec![test_url!("user/Poet"), "\"Poet Smurf\""], + svec![test_url!("user/Farmer"), "\"Farmer Smurf\""], + svec![test_url!("user/Natural"), "\"Natural Smurf\""], + svec![test_url!("user/Snappy"), "\"Snappy Smurf\""], svec![ test_url!( "user/The quick brown fox jumped over the lazy dog by the zigzag quarry site" ), - "The quick brown fox jumped over the lazy dog by the zigzag quarry site Smurf" + "\"The quick brown fox jumped over the lazy dog by the zigzag quarry site Smurf\"" ], ]; assert_eq!(got, expected); @@ -828,7 +840,7 @@ fn fetch_complex_url_template() { .arg("--new-column") .arg("Fullname") .arg("--jql") - .arg(r#"."fullname""#) + .arg(r#""fullname""#) .arg("--rate-limit") .arg("4") .arg("data.csv"); @@ -836,26 +848,26 @@ fn fetch_complex_url_template() { let got: Vec> = wrk.read_stdout(&mut cmd); let expected = vec![ svec!["first name", "color", "Fullname"], - svec!["Smurfette", "blue", "Smurfette blue Smurf"], - svec!["Papa", "blue", "Papa blue Smurf"], - svec!["Clumsy", "blue", "Clumsy blue Smurf"], - svec!["Brainy", "blue", "Brainy blue Smurf"], - svec!["Grouchy", "blue", "Grouchy blue Smurf"], - svec!["Hefty", "blue", "Hefty blue Smurf"], - svec!["Greedy", "green", "Greedy green Smurf"], - svec!["Jokey", "blue", "Jokey blue Smurf"], - svec!["Chef", "blue", "Chef blue Smurf"], - svec!["Vanity", "blue", "Vanity blue Smurf"], - svec!["Handy", "blue", "Handy blue Smurf"], - svec!["Scaredy", "black", "Scaredy black Smurf"], - svec!["Tracker", "blue", "Tracker blue Smurf"], - svec!["Sloppy", "blue", "Sloppy blue Smurf"], - svec!["Harmony", "blue", "Harmony blue Smurf"], - svec!["Painter", "multicolor", "Painter multicolor Smurf"], - svec!["Poet", "blue", "Poet blue Smurf"], - svec!["Farmer", "blue", "Farmer blue Smurf"], - svec!["Natural", "blue", "Natural blue Smurf"], - svec!["Snappy", "blue", "Snappy blue Smurf"], + svec!["Smurfette", "blue", "\"Smurfette blue Smurf\""], + svec!["Papa", "blue", "\"Papa blue Smurf\""], + svec!["Clumsy", "blue", "\"Clumsy blue Smurf\""], + svec!["Brainy", "blue", "\"Brainy blue Smurf\""], + svec!["Grouchy", "blue", "\"Grouchy blue Smurf\""], + svec!["Hefty", "blue", "\"Hefty blue Smurf\""], + svec!["Greedy", "green", "\"Greedy green Smurf\""], + svec!["Jokey", "blue", "\"Jokey blue Smurf\""], + svec!["Chef", "blue", "\"Chef blue Smurf\""], + svec!["Vanity", "blue", "\"Vanity blue Smurf\""], + svec!["Handy", "blue", "\"Handy blue Smurf\""], + svec!["Scaredy", "black", "\"Scaredy black Smurf\""], + svec!["Tracker", "blue", "\"Tracker blue Smurf\""], + svec!["Sloppy", "blue", "\"Sloppy blue Smurf\""], + svec!["Harmony", "blue", "\"Harmony blue Smurf\""], + svec!["Painter", "multicolor", "\"Painter multicolor Smurf\""], + svec!["Poet", "blue", "\"Poet blue Smurf\""], + svec!["Farmer", "blue", "\"Farmer blue Smurf\""], + svec!["Natural", "blue", "\"Natural blue Smurf\""], + svec!["Snappy", "blue", "\"Snappy blue Smurf\""], ]; assert_eq!(got, expected); @@ -910,36 +922,31 @@ fn fetchpost_simple_test() { "a", "42", "true", - "{\"bool_col\": String(\"true\"), \"col1\": String(\"a\"), \"number col\": \ - String(\"42\")}" + "{\"bool_col\":\"true\",\"col1\":\"a\",\"number col\":\"42\"}" ], svec![ "b", "3.14", "false", - "{\"bool_col\": String(\"false\"), \"col1\": String(\"b\"), \"number col\": \ - String(\"3.14\")}" + "{\"bool_col\":\"false\",\"col1\":\"b\",\"number col\":\"3.14\"}" ], svec![ "c", "666", "true", - "{\"bool_col\": String(\"true\"), \"col1\": String(\"c\"), \"number col\": \ - String(\"666\")}" + "{\"bool_col\":\"true\",\"col1\":\"c\",\"number col\":\"666\"}" ], svec![ "d", "33", "true", - "{\"bool_col\": String(\"true\"), \"col1\": String(\"d\"), \"number col\": \ - String(\"33\")}" + "{\"bool_col\":\"true\",\"col1\":\"d\",\"number col\":\"33\"}" ], svec![ "e", "0", "false", - "{\"bool_col\": String(\"false\"), \"col1\": String(\"e\"), \"number col\": \ - String(\"0\")}" + "{\"bool_col\":\"false\",\"col1\":\"e\",\"number col\":\"0\"}" ], ]; @@ -1108,22 +1115,19 @@ fn fetchpost_literalurl_test() { "a", "42", "true", - "{\"bool_col\": String(\"true\"), \"col1\": String(\"a\"), \"number col\": \ - String(\"42\")}" + "{\"bool_col\":\"true\",\"col1\":\"a\",\"number col\":\"42\"}" ], svec![ "b", "3.14", "false", - "{\"bool_col\": String(\"false\"), \"col1\": String(\"b\"), \"number col\": \ - String(\"3.14\")}" + "{\"bool_col\":\"false\",\"col1\":\"b\",\"number col\":\"3.14\"}" ], svec![ "c", "666", "true", - "{\"bool_col\": String(\"true\"), \"col1\": String(\"c\"), \"number col\": \ - String(\"666\")}" + "{\"bool_col\":\"true\",\"col1\":\"c\",\"number col\":\"666\"}" ], ];