From 3f2a5e4a6f4f66dda0723e00d2ff41eb7327c4a4 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 24 Nov 2024 17:52:12 -0500 Subject: [PATCH 1/3] Altair 5.5 --- pixi.lock | 53 +++++++++++++++++++++++++---------------------------- pixi.toml | 2 +- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/pixi.lock b/pixi.lock index 0681d929..32c5a451 100644 --- a/pixi.lock +++ b/pixi.lock @@ -13,7 +13,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.12-h4ab18f5_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.4.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.6.2.post1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anywidget-0.9.13-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-23.1.0-pyhd8ed1ab_0.conda @@ -197,7 +197,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/mypy-1.13.0-py310ha75aee5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/myst-parser-4.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.9.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.14.2-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.7.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.4-hd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda @@ -356,7 +356,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ablog-0.11.12-pyh91182bf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.4.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.6.2.post1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anywidget-0.9.13-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_0.conda @@ -503,7 +503,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/mypy-1.13.0-py310hb9d19b6_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/myst-parser-4.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.9.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.14.2-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.7.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.4-hd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda @@ -641,7 +641,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ablog-0.11.12-pyh91182bf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.4.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.6.2.post1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anywidget-0.9.13-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_0.conda @@ -817,7 +817,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/mypy-1.13.0-py310hf9df320_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/myst-parser-4.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.9.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.14.2-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.7.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.4-hd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda @@ -959,7 +959,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ablog-0.11.12-pyh91182bf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.16-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.4.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/altair-5.5.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.6.2.post1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anywidget-0.9.13-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-23.1.0-pyhd8ed1ab_0.conda @@ -1098,7 +1098,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/mypy-1.13.0-py310ha8f682b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/myst-parser-4.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.9.4-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.14.2-pyhff2d567_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbclient-0.7.4-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.4-hd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda @@ -1348,28 +1348,26 @@ packages: timestamp: 1718118368236 - kind: conda name: altair - version: 5.4.1 - build: pyhd8ed1ab_1 - build_number: 1 + version: 5.5.0 + build: pyhd8ed1ab_0 subdir: noarch noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/altair-5.4.1-pyhd8ed1ab_1.conda - sha256: adc8a0ff6052c11e9addbcf329d15180aa5481376f0d5048cfefe314354a2c9e - md5: 8431a457f055d4f89569d60583623c6e + url: https://conda.anaconda.org/conda-forge/noarch/altair-5.5.0-pyhd8ed1ab_0.conda + sha256: de9d4f1211b6f818b9c5b42b641ca7b729e92a265d442f5a471cb9a83d7e10dd + md5: a36e83a3ca4416a393be9d023f1f9c8d depends: - importlib-metadata - jinja2 - jsonschema >=3.0 - - narwhals >=1.1.0 + - narwhals >=1.14.2 - packaging - - python >=3.8 + - python >=3.9 - typing-extensions >=4.10.0 license: BSD-3-Clause - license_family: BSD purls: - pkg:pypi/altair?source=hash-mapping - size: 462954 - timestamp: 1727892152911 + size: 495342 + timestamp: 1732408526911 - kind: conda name: anyio version: 4.6.2.post1 @@ -9751,21 +9749,20 @@ packages: timestamp: 1722964397370 - kind: conda name: narwhals - version: 1.9.4 - build: pyhd8ed1ab_0 + version: 1.14.2 + build: pyhff2d567_0 subdir: noarch noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.9.4-pyhd8ed1ab_0.conda - sha256: 9ffae2d9162ba3e818bde14f04485a10a18475fb6379834b936a1ceb5b03e9e4 - md5: 0af00b268c7266ba07704c8542845116 + url: https://conda.anaconda.org/conda-forge/noarch/narwhals-1.14.2-pyhff2d567_0.conda + sha256: 0befc04516cb350cf35beb50a0b2662a6b9319e203234fc54862b9951fa33e42 + md5: c7948f818368f181dea8d4865efd8039 depends: - - python >=3.8 + - python >=3.9 license: MIT - license_family: MIT purls: - pkg:pypi/narwhals?source=hash-mapping - size: 110460 - timestamp: 1729185405948 + size: 130132 + timestamp: 1732387884925 - kind: conda name: nbclient version: 0.7.4 diff --git a/pixi.toml b/pixi.toml index 5d45f89f..ed9d4e62 100644 --- a/pixi.toml +++ b/pixi.toml @@ -150,7 +150,7 @@ pydata-sphinx-theme = ">=0.16.0,<0.17" # Dependencies are those required at runtime by the Python packages [dependencies] psutil = "5.9.5.*" -altair = "5.4.*" +altair = "5.5.*" ipywidgets = "8.1.0.*" vl-convert-python = "1.7.*" anywidget = ">=0.9.6,<0.10" From d63889618720dabf511d22eaf02a80e1a718bf8a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 24 Nov 2024 18:42:23 -0500 Subject: [PATCH 2/3] Fix custom date format parsing --- vegafusion-runtime/src/data/tasks.rs | 8 +-- vegafusion-runtime/src/transform/pipeline.rs | 2 +- .../tests/test_pre_transform_values.rs | 55 +++++++++++++++++++ vegafusion-runtime/tests/util/check.rs | 2 +- 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/vegafusion-runtime/src/data/tasks.rs b/vegafusion-runtime/src/data/tasks.rs index f1b82b3d..6fa3b482 100644 --- a/vegafusion-runtime/src/data/tasks.rs +++ b/vegafusion-runtime/src/data/tasks.rs @@ -216,7 +216,7 @@ async fn eval_sql_df( ) }; - let table_value = TaskValue::Table(transformed_df); + let table_value = TaskValue::Table(transformed_df.without_ordering()?); Ok((table_value, output_values)) } @@ -360,7 +360,7 @@ async fn process_datetimes( if fmt.starts_with("'") && fmt.ends_with("'") { (typ.to_lowercase(), Some(fmt[1..fmt.len() - 1].to_string())) } else { - (typ.to_lowercase(), None) + (typ.to_lowercase(), Some(fmt.to_string())) } } else { (datatype.to_lowercase(), None) @@ -541,7 +541,7 @@ impl TaskCall for DataValuesTask { (values_df.collect_to_table().await?, Vec::new()) }; - let table_value = TaskValue::Table(transformed_table); + let table_value = TaskValue::Table(transformed_table.without_ordering()?); Ok((table_value, output_values)) } @@ -587,7 +587,7 @@ impl TaskCall for DataSourceTask { (source_table, Vec::new()) }; - let table_value = TaskValue::Table(transformed_table); + let table_value = TaskValue::Table(transformed_table.without_ordering()?); Ok((table_value, output_values)) } } diff --git a/vegafusion-runtime/src/transform/pipeline.rs b/vegafusion-runtime/src/transform/pipeline.rs index 33a17462..de1149d6 100644 --- a/vegafusion-runtime/src/transform/pipeline.rs +++ b/vegafusion-runtime/src/transform/pipeline.rs @@ -103,7 +103,7 @@ impl TransformPipelineUtils for TransformPipeline { nulls_first: false, }])?; - let table = result_sql_df.collect_to_table().await?.without_ordering()?; + let table = result_sql_df.collect_to_table().await?; // Sort result signal value by signal name let (_, signals_values): (Vec<_>, Vec<_>) = result_outputs diff --git a/vegafusion-runtime/tests/test_pre_transform_values.rs b/vegafusion-runtime/tests/test_pre_transform_values.rs index f1c5836d..fe1e58ca 100644 --- a/vegafusion-runtime/tests/test_pre_transform_values.rs +++ b/vegafusion-runtime/tests/test_pre_transform_values.rs @@ -423,4 +423,59 @@ mod tests { assert_eq!(dataset.pretty_format(None).unwrap(), expected); } } + + #[tokio::test] + async fn test_pre_transform_dataset_date_format() { + // Load spec + let spec_path = format!("{}/tests/specs/vegalite/histogram.vg.json", crate_dir()); + let spec_str = fs::read_to_string(spec_path).unwrap(); + let spec: ChartSpec = serde_json::from_value(json!({ + "data": [ + { + "name": "data_0", + "format": {"parse": {"a": "date:%Y"}}, + "values": [ + {"a": "2020", "b": 2, "c": 3}, + {"a": "2022", "b": 22, "c": 33}, + ] + } + ] + })) + .unwrap(); + + // Initialize task graph runtime + let runtime = VegaFusionRuntime::new(None); + + let (values, warnings) = runtime + .pre_transform_values( + &spec, + &[(Variable::new_data("data_0"), vec![])], + &Default::default(), + &PreTransformValuesOpts { + row_limit: None, + local_tz: "UTC".to_string(), + default_input_tz: None, + }, + ) + .await + .unwrap(); + + // Check there are no warnings + assert!(warnings.is_empty()); + + // Check single returned dataset + assert_eq!(values.len(), 1); + + let dataset = values[0].as_table().cloned().unwrap(); + println!("{:?}", dataset); + + let expected = "\ ++----+----+------------+ +| b | c | a | ++----+----+------------+ +| 2 | 3 | 2020-01-01 | +| 22 | 33 | 2022-01-01 | ++----+----+------------+"; + assert_eq!(dataset.pretty_format(None).unwrap(), expected); + } } diff --git a/vegafusion-runtime/tests/util/check.rs b/vegafusion-runtime/tests/util/check.rs index 3021d3fb..f7534a74 100644 --- a/vegafusion-runtime/tests/util/check.rs +++ b/vegafusion-runtime/tests/util/check.rs @@ -135,5 +135,5 @@ pub fn eval_vegafusion_transforms( .map(|v| v.as_scalar().cloned()) .collect::>>() .unwrap(); - (result_data, result_signals) + (result_data.without_ordering().unwrap(), result_signals) } From a6ec68c95e29a5a0506da86670ced913bcf70b66 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 24 Nov 2024 18:49:50 -0500 Subject: [PATCH 3/3] fix warnings --- vegafusion-runtime/tests/test_pre_transform_values.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/vegafusion-runtime/tests/test_pre_transform_values.rs b/vegafusion-runtime/tests/test_pre_transform_values.rs index fe1e58ca..3129c79e 100644 --- a/vegafusion-runtime/tests/test_pre_transform_values.rs +++ b/vegafusion-runtime/tests/test_pre_transform_values.rs @@ -426,9 +426,6 @@ mod tests { #[tokio::test] async fn test_pre_transform_dataset_date_format() { - // Load spec - let spec_path = format!("{}/tests/specs/vegalite/histogram.vg.json", crate_dir()); - let spec_str = fs::read_to_string(spec_path).unwrap(); let spec: ChartSpec = serde_json::from_value(json!({ "data": [ {