From e20814037e927b6088f04f2e9ef88eca18de9a45 Mon Sep 17 00:00:00 2001 From: Saikrishna Achalla Date: Mon, 18 Sep 2023 13:02:34 -0700 Subject: [PATCH] feat(topolotree): new implementation and Hydro Deploy setup --- .github/workflows/build-cli.yml | 11 +- .github/workflows/ci.yml | 30 +- Cargo.lock | 72 +++- Cargo.toml | 1 + README.md | 12 +- benches/benches/fork_join.rs | 6 +- benches/benches/join.rs | 12 +- benches/benches/micro_ops.rs | 42 +- benches/benches/reachability.rs | 4 +- benches/benches/symmetric_hash_join.rs | 48 ++- build_docs.bash | 8 +- design_docs/2023-07_lattice_properties.md | 268 ++++++++++++ design_docs/2023-08_lattice_properties.md | 334 +++++++++++++++ docs/README.md | 35 +- docs/docs/deploy/index.md | 12 +- docs/docs/deploy/install.md | 2 +- .../architecture/{index.md => index.mdx} | 39 +- docs/docs/hydroflow/concepts/cyclic_flows.md | 49 +++ docs/docs/hydroflow/concepts/debugging.md | 2 +- .../hydroflow/concepts/distributed_time.md | 2 +- .../docs/hydroflow/concepts/error_handling.md | 2 +- .../docs/hydroflow/concepts/stratification.md | 2 +- docs/docs/hydroflow/index.mdx | 3 +- .../quickstart/example_1_simplest.mdx | 9 +- .../hydroflow/quickstart/example_2_simple.mdx | 6 + .../hydroflow/quickstart/example_3_stream.mdx | 2 +- .../quickstart/example_4_neighbors.mdx | 6 +- .../quickstart/example_5_reachability.mdx | 28 +- .../quickstart/example_6_unreachability.mdx | 12 +- .../quickstart/example_7_echo_server.mdx | 18 +- .../quickstart/example_8_chat_server.mdx | 36 +- docs/docs/hydroflow/quickstart/setup.md | 11 +- docs/docs/hydroflow/syntax/index.mdx | 2 +- docs/docs/hydroflow/syntax/surface_data.mdx | 38 +- docs/docs/hydroflow/syntax/surface_flows.md | 12 +- docs/docs/hydroflow/todo.md | 17 +- docs/docusaurus.config.js | 4 + docs/src/pages/playground.js | 42 +- docs/wasm-plugin.js | 10 +- hydro_cli/CHANGELOG.md | 69 +++- hydro_cli/Cargo.toml | 6 +- hydro_cli/hydro/_core.pyi | 2 +- hydro_cli/src/core/custom_service.rs | 4 +- hydro_cli/src/core/deployment.rs | 18 +- hydro_cli/src/core/gcp.rs | 67 +-- hydro_cli/src/core/hydroflow_crate/build.rs | 7 + hydro_cli/src/core/hydroflow_crate/mod.rs | 32 +- hydro_cli/src/core/hydroflow_crate/ports.rs | 1 - hydro_cli/src/core/localhost.rs | 44 +- hydro_cli/src/core/mod.rs | 3 +- hydro_cli/src/core/progress.rs | 87 ++-- hydro_cli/src/core/ssh.rs | 16 +- hydro_cli/src/core/terraform.rs | 27 +- hydro_cli/src/core/util.rs | 2 +- hydro_cli/src/lib.rs | 14 +- hydro_cli_examples/Cargo.toml | 12 - .../examples/topolotree/main.rs | 351 ---------------- hydro_cli_examples/topolotree.hydro.py | 181 -------- .../topolotree_latency.hydro.py | 306 -------------- hydroflow/CHANGELOG.md | 327 ++++++++++++++- hydroflow/Cargo.toml | 28 +- hydroflow/README.md | 23 +- hydroflow/examples/chat/client.rs | 10 +- hydroflow/examples/chat/main.rs | 49 +++ hydroflow/examples/chat/server.rs | 6 +- .../examples/deadlock_detector/README.md | 2 +- .../examples/deadlock_detector/helpers.rs | 10 +- hydroflow/examples/deadlock_detector/main.rs | 72 ++++ hydroflow/examples/deadlock_detector/peer.rs | 30 +- hydroflow/examples/deletable_state/README.md | 6 - hydroflow/examples/deletable_state/main.rs | 78 ---- hydroflow/examples/echo_serde_json/main.rs | 39 ++ hydroflow/examples/echoserver/main.rs | 35 ++ hydroflow/examples/example_2_simple_1.rs | 2 +- hydroflow/examples/example_5_reachability.rs | 14 +- .../examples/example_6_unreachability.rs | 16 +- hydroflow/examples/example_naturals.rs | 14 + .../examples/graph_reachability/README.md | 8 - hydroflow/examples/graph_reachability/main.rs | 80 ---- .../examples/graph_unreachability/README.md | 8 - .../examples/graph_unreachability/main.rs | 93 ----- hydroflow/examples/hello_world/main.rs | 11 +- hydroflow/examples/kvs/main.rs | 45 ++ hydroflow/examples/kvs/server.rs | 2 +- hydroflow/examples/kvs_bench/main.rs | 10 + .../serialization/lattices/map_union.rs | 4 +- .../lattices/my_last_write_wins.rs | 9 +- .../protocol/serialization/lattices/point.rs | 2 +- .../examples/kvs_bench/protocol/test/mod.rs | 20 +- hydroflow/examples/kvs_bench/server.rs | 15 +- hydroflow/examples/lamport_clock/client.rs | 7 +- hydroflow/examples/lamport_clock/main.rs | 45 ++ hydroflow/examples/lamport_clock/server.rs | 7 +- hydroflow/examples/modules/main.rs | 48 +++ .../examples/modules/triple_cross_join.hf | 15 + hydroflow/examples/python_udf/main.rs | 27 ++ hydroflow/examples/rga/README.md | 6 +- hydroflow/examples/rga/main.rs | 125 +++++- hydroflow/examples/shopping/driver.rs | 17 +- hydroflow/examples/shopping/flows/bp_flow.rs | 4 +- .../shopping/flows/client_state_flow.rs | 4 +- .../examples/shopping/flows/orig_flow.rs | 4 +- .../shopping/flows/push_group_flow.rs | 4 +- .../shopping/flows/rep_server_flow.rs | 8 +- .../shopping/flows/server_state_flow.rs | 4 +- .../examples/shopping/flows/ssiv_flow.rs | 4 +- hydroflow/examples/shopping/main.rs | 133 ++++++ hydroflow/examples/three_clique/main.rs | 21 +- hydroflow/examples/two_pc/coordinator.rs | 6 +- hydroflow/examples/two_pc/main.rs | 87 ++++ hydroflow/examples/two_pc/subordinate.rs | 4 +- hydroflow/examples/vector_clock/client.rs | 5 +- hydroflow/examples/vector_clock/main.rs | 45 ++ hydroflow/examples/vector_clock/server.rs | 5 +- hydroflow/src/compiled/pull/anti_join.rs | 66 +++ .../src/compiled/pull/half_join_state/fold.rs | 48 +++ .../pull/half_join_state/fold_from.rs | 48 +++ .../src/compiled/pull/half_join_state/mod.rs | 31 +- .../compiled/pull/half_join_state/multiset.rs | 47 ++- .../pull/half_join_state/multiset2.rs | 42 ++ .../compiled/pull/half_join_state/reduce.rs | 47 +++ .../src/compiled/pull/half_join_state/set.rs | 48 ++- hydroflow/src/compiled/pull/mod.rs | 6 +- .../src/compiled/pull/symmetric_hash_join.rs | 127 +++--- .../pull/symmetric_hash_join_lattice.rs | 231 ----------- hydroflow/src/declarative_macro.rs | 63 +++ hydroflow/src/lib.rs | 8 +- hydroflow/src/scheduled/context.rs | 8 + hydroflow/src/scheduled/graph.rs | 4 + hydroflow/src/scheduled/net/mod.rs | 2 +- hydroflow/src/scheduled/state.rs | 7 +- hydroflow/src/util/cli.rs | 2 + hydroflow/src/util/mod.rs | 100 +++++ hydroflow/src/util/monotonic.rs | 9 + hydroflow/src/util/multiset.rs | 64 +++ hydroflow/src/util/sparse_vec.rs | 2 +- hydroflow/src/util/unsync/mpsc.rs | 4 +- .../compile-fail/surface_badgeneric_type.rs | 4 +- .../surface_badgeneric_type.stderr | 8 +- .../surface_conflicting_name.stderr | 2 +- .../surface_degenerate_null.stderr | 4 +- .../surface_flow_props_cumul_warn_cumul.rs | 15 + ...surface_flow_props_cumul_warn_cumul.stderr | 25 ++ .../surface_flow_props_cumul_warn_none.rs | 16 + .../surface_flow_props_cumul_warn_none.stderr | 25 ++ .../compile-fail/surface_flow_props_upcast.rs | 13 + .../surface_flow_props_upcast.stderr | 19 + ...surface_lattice_fold_badgeneric.rs.ignore} | 2 +- ...=> surface_lattice_fold_badgeneric.stderr} | 16 +- ...c.rs => surface_lattice_fold_nogeneric.rs} | 2 +- .../surface_lattice_fold_nogeneric.stderr | 5 + ...s => surface_lattice_fold_wronggeneric.rs} | 2 +- .../surface_lattice_fold_wronggeneric.stderr | 29 ++ .../surface_lattice_merge_nogeneric.stderr | 5 - ...urface_lattice_reduce_badgeneric.rs.ignore | 10 + .../surface_lattice_reduce_badgeneric.stderr | 68 +++ .../surface_lattice_reduce_nogeneric.rs | 10 + .../surface_lattice_reduce_nogeneric.stderr | 5 + .../surface_lattice_reduce_wronggeneric.rs | 10 + ...surface_lattice_reduce_wronggeneric.stderr | 21 + ...surface_latticereduce_wronggeneric.stderr} | 8 +- .../compile-fail/surface_negative_loop.stderr | 2 +- .../tests/compile-fail/surface_null.stderr | 4 +- .../surface_syntax_eol_arrow.stderr | 2 +- .../surface_syntax_eol_indexing.stderr | 2 +- .../surface_syntax_eol_missingop.stderr | 2 +- .../surface_syntax_indexing_empty.stderr | 2 +- .../surface_syntax_paren_arrow.stderr | 2 +- .../surface_syntax_paren_indexing.stderr | 2 +- .../surface_syntax_paren_missingop.stderr | 2 +- .../tests/compile-fail/surface_use_bad1.rs | 10 + .../compile-fail/surface_use_bad1.stderr | 5 + .../tests/compile-fail/surface_use_bad2.rs | 10 + .../compile-fail/surface_use_bad2.stderr | 5 + .../tests/compile-fail/surface_use_bad3.rs | 10 + .../compile-fail/surface_use_bad3.stderr | 5 + .../tests/compile-fail/surface_use_unknown.rs | 10 + .../compile-fail/surface_use_unknown.stderr | 5 + hydroflow/tests/datalog_frontend.rs | 269 ++++++------ .../surface_async__echo@graphvis_mermaid.snap | 2 +- .../surface_batch__basic_2@graphvis_dot.snap | 37 ++ ...rface_batch__basic_2@graphvis_mermaid.snap | 30 ++ ...ook__surface_flows_1@graphvis_mermaid.snap | 16 +- ...ace_codegen__basic_2@graphvis_mermaid.snap | 2 +- ...ace_codegen__basic_3@graphvis_mermaid.snap | 4 +- ...codegen__basic_union@graphvis_mermaid.snap | 6 +- ...degen__covid_tracing@graphvis_mermaid.snap | 28 +- ...e_codegen__recv_expr@graphvis_mermaid.snap | 2 +- ...urface_codegen__sort@graphvis_mermaid.snap | 6 +- ...codegen__sort_by_key@graphvis_mermaid.snap | 6 +- ...achability_generated@graphvis_mermaid.snap | 18 +- ...__transitive_closure@graphvis_mermaid.snap | 24 +- ...ntext_current_tick_start@graphvis_dot.snap | 37 ++ ...t_current_tick_start@graphvis_mermaid.snap | 31 ++ ...count_time_between_ticks@graphvis_dot.snap | 17 + ...t_time_between_ticks@graphvis_mermaid.snap | 17 + ...time_between_ticks_async@graphvis_dot.snap | 17 + ..._between_ticks_async@graphvis_mermaid.snap | 17 + ...context__context_mut@graphvis_mermaid.snap | 8 +- ...context__context_ref@graphvis_mermaid.snap | 2 +- ...ce__diff_multiset_static@graphvis_dot.snap | 64 +++ ...diff_multiset_static@graphvis_mermaid.snap | 51 +++ ...iff_multiset_static_tick@graphvis_dot.snap | 64 +++ ...multiset_static_tick@graphvis_mermaid.snap | 51 +++ ...iff_multiset_tick_static@graphvis_dot.snap | 64 +++ ...multiset_tick_static@graphvis_mermaid.snap | 51 +++ ...ce__diff_multiset_timing@graphvis_dot.snap | 51 +++ ...diff_multiset_timing@graphvis_mermaid.snap | 42 ++ ..._difference__diff_static@graphvis_dot.snap | 53 ++- ...ference__diff_static@graphvis_mermaid.snap | 47 ++- ...ference__diff_timing@graphvis_mermaid.snap | 14 +- ...surface_examples__example_4_neighbors.snap | 12 +- ...face_examples__example_5_reachability.snap | 34 +- ...ce_examples__example_6_unreachability.snap | 56 +-- .../surface_examples__example_naturals.snap | 5 + ...urface_flow_props__basic@graphvis_dot.snap | 44 ++ ...ce_flow_props__basic@graphvis_mermaid.snap | 40 ++ ...low_props__union_warning@graphvis_dot.snap | 24 ++ ...props__union_warning@graphvis_mermaid.snap | 23 ++ .../surface_fold__fold_sort@graphvis_dot.snap | 2 +- ...face_fold__fold_sort@graphvis_mermaid.snap | 10 +- ...urface_fold__fold_static@graphvis_dot.snap | 2 +- ...ce_fold__fold_static@graphvis_mermaid.snap | 8 +- .../surface_fold__fold_tick@graphvis_dot.snap | 2 +- ...face_fold__fold_tick@graphvis_mermaid.snap | 8 +- ...rdref_basic_backward@graphvis_mermaid.snap | 2 +- ...ardref_basic_forward@graphvis_mermaid.snap | 2 +- ...wardref_basic_middle@graphvis_mermaid.snap | 4 +- ...face_join__static_static@graphvis_dot.snap | 88 ++++ ..._join__static_static@graphvis_mermaid.snap | 68 +++ ...urface_join__static_tick@graphvis_dot.snap | 88 ++++ ...ce_join__static_tick@graphvis_mermaid.snap | 68 +++ ...urface_join__tick_static@graphvis_dot.snap | 88 ++++ ...ce_join__tick_static@graphvis_mermaid.snap | 68 +++ .../surface_join__tick_tick@graphvis_dot.snap | 88 ++++ ...face_join__tick_tick@graphvis_mermaid.snap | 68 +++ ...s_blocking_rhs_streaming@graphvis_dot.snap | 97 +++++ ...ocking_rhs_streaming@graphvis_mermaid.snap | 74 ++++ ...s_streaming_rhs_blocking@graphvis_dot.snap | 97 +++++ ...reaming_rhs_blocking@graphvis_mermaid.snap | 74 ++++ ...s_blocking_rhs_streaming@graphvis_dot.snap | 97 +++++ ...ocking_rhs_streaming@graphvis_mermaid.snap | 74 ++++ ...s_streaming_rhs_blocking@graphvis_dot.snap | 97 +++++ ...reaming_rhs_blocking@graphvis_mermaid.snap | 74 ++++ ...s_blocking_rhs_streaming@graphvis_dot.snap | 97 +++++ ...ocking_rhs_streaming@graphvis_mermaid.snap | 74 ++++ ...tick_lhs_fold_rhs_reduce@graphvis_dot.snap | 104 +++++ ..._lhs_fold_rhs_reduce@graphvis_mermaid.snap | 78 ++++ ...s_streaming_rhs_blocking@graphvis_dot.snap | 97 +++++ ...reaming_rhs_blocking@graphvis_mermaid.snap | 74 ++++ ...ld_keyed_infer_basic@graphvis_mermaid.snap | 8 +- ...d__fold_keyed_static@graphvis_mermaid.snap | 6 +- ...old__fold_keyed_tick@graphvis_mermaid.snap | 6 +- ...elta__multiset_delta@graphvis_mermaid.snap | 4 +- ...ace_persist__persist@graphvis_mermaid.snap | 10 +- ...e_persist__persist_basic@graphvis_dot.snap | 2 +- ...rsist__persist_basic@graphvis_mermaid.snap | 12 +- ...persist__persist_mut@graphvis_mermaid.snap | 16 +- ...t__persist_mut_keyed@graphvis_mermaid.snap | 16 +- ...ce_persist__persist_pull@graphvis_dot.snap | 2 +- ...ersist__persist_pull@graphvis_mermaid.snap | 20 +- ...ce_persist__persist_push@graphvis_dot.snap | 2 +- ...ersist__persist_push@graphvis_mermaid.snap | 20 +- ...ist__persist_replay_join@graphvis_dot.snap | 2 +- ..._persist_replay_join@graphvis_mermaid.snap | 20 +- ...ace_python__python_basic@graphvis_dot.snap | 21 + ...python__python_basic@graphvis_mermaid.snap | 21 + ...on__python_too_many_args@graphvis_dot.snap | 21 + ...python_too_many_args@graphvis_mermaid.snap | 21 + ..._python__python_two_args@graphvis_dot.snap | 19 + ...hon__python_two_args@graphvis_mermaid.snap | 19 + .../surface_reduce__reduce@graphvis_dot.snap | 2 +- ...rface_reduce__reduce@graphvis_mermaid.snap | 24 +- ...ce_reduce__reduce_static@graphvis_dot.snap | 2 +- ...educe__reduce_static@graphvis_mermaid.snap | 8 +- ...rface_reduce__reduce_sum@graphvis_dot.snap | 2 +- ...e_reduce__reduce_sum@graphvis_mermaid.snap | 8 +- ...face_reduce__reduce_tick@graphvis_dot.snap | 2 +- ..._reduce__reduce_tick@graphvis_mermaid.snap | 8 +- ...ersist_stratum_run_async@graphvis_dot.snap | 26 ++ ...st_stratum_run_async@graphvis_mermaid.snap | 23 ++ ...st_stratum_run_available@graphvis_dot.snap | 36 ++ ...tratum_run_available@graphvis_mermaid.snap | 31 ++ ...scheduling__stratum_loop@graphvis_dot.snap | 40 ++ ...duling__stratum_loop@graphvis_mermaid.snap | 36 ++ ...ce_scheduling__tick_loop@graphvis_dot.snap | 49 +++ ...cheduling__tick_loop@graphvis_mermaid.snap | 42 ++ ...ce_stratum__difference_a@graphvis_dot.snap | 2 +- ...tratum__difference_a@graphvis_mermaid.snap | 10 +- ...ce_stratum__difference_b@graphvis_dot.snap | 6 +- ...tratum__difference_b@graphvis_mermaid.snap | 20 +- ...tratum_consolidation@graphvis_mermaid.snap | 36 +- ...graph_unreachability@graphvis_mermaid.snap | 36 +- ...ace_stratum__tick_loop_1@graphvis_dot.snap | 4 +- ...stratum__tick_loop_1@graphvis_mermaid.snap | 22 +- ...ace_stratum__tick_loop_2@graphvis_dot.snap | 8 +- ...stratum__tick_loop_2@graphvis_mermaid.snap | 30 +- ...ace_stratum__tick_loop_3@graphvis_dot.snap | 12 +- ...stratum__tick_loop_3@graphvis_mermaid.snap | 40 +- ...rface_unique__unique@graphvis_mermaid.snap | 4 +- ...__unique_static_pull@graphvis_mermaid.snap | 18 +- ...__unique_static_push@graphvis_mermaid.snap | 16 +- ...ue__unique_tick_pull@graphvis_mermaid.snap | 18 +- ...ue__unique_tick_push@graphvis_mermaid.snap | 16 +- hydroflow/tests/surface_async.rs | 2 +- hydroflow/tests/surface_batch.rs | 28 ++ hydroflow/tests/surface_codegen.rs | 391 ++++++++++-------- hydroflow/tests/surface_context.rs | 64 +++ hydroflow/tests/surface_difference.rs | 176 +++++++- hydroflow/tests/surface_flow_props.rs | 44 ++ hydroflow/tests/surface_fold.rs | 17 +- hydroflow/tests/surface_join.rs | 129 ++++++ hydroflow/tests/surface_join_fused.rs | 211 ++++++++++ hydroflow/tests/surface_lattice_batch.rs | 27 ++ ...ttice_merge.rs => surface_lattice_fold.rs} | 2 +- hydroflow/tests/surface_lattice_join.rs | 130 ++++++ hydroflow/tests/surface_lattice_reduce.rs | 13 + hydroflow/tests/surface_persist.rs | 14 +- hydroflow/tests/surface_python.rs | 62 +++ hydroflow/tests/surface_reduce.rs | 8 +- hydroflow/tests/surface_scheduling.rs | 8 +- hydroflow/tests/surface_state_scheduling.rs | 10 +- hydroflow/tests/surface_stratum.rs | 27 +- hydroflow/tests/surface_warnings.rs | 58 +-- hydroflow/tests/test.rs | 4 +- hydroflow_cli_integration/CHANGELOG.md | 30 +- hydroflow_cli_integration/Cargo.toml | 2 +- hydroflow_datalog/CHANGELOG.md | 70 +++- hydroflow_datalog/Cargo.toml | 6 +- hydroflow_datalog_core/CHANGELOG.md | 114 ++++- hydroflow_datalog_core/Cargo.toml | 6 +- hydroflow_datalog_core/src/join_plan.rs | 31 +- hydroflow_datalog_core/src/lib.rs | 75 ++-- ...egations_and_comments@datalog_program.snap | 9 +- ...gregations_and_comments@surface_graph.snap | 2 +- ...tions_fold_keyed_expr@datalog_program.snap | 3 +- ...ore__tests__anti_join@datalog_program.snap | 107 +++-- ...core__tests__expr_lhs@datalog_program.snap | 2 +- ...tests__expr_predicate@datalog_program.snap | 2 +- ...og_core__tests__index@datalog_program.snap | 154 ++++++- ...alog_core__tests__index@surface_graph.snap | 4 +- ...ests__join_with_other@datalog_program.snap | 39 +- ...tests__join_with_self@datalog_program.snap | 39 +- ...__local_constraints@datalog_program-2.snap | 2 +- ...ts__local_constraints@datalog_program.snap | 2 +- ...alog_core__tests__max@datalog_program.snap | 3 +- ..._core__tests__max_all@datalog_program.snap | 3 +- ...ests__minimal_program@datalog_program.snap | 2 +- ...multiple_contributors@datalog_program.snap | 2 +- ...s__non_copy_but_clone@datalog_program.snap | 2 +- ..._core__tests__persist@datalog_program.snap | 365 +++++++++++++--- ...og_core__tests__persist@surface_graph.snap | 6 +- ...s__persist_uniqueness@datalog_program.snap | 78 +++- ...sts__persist_uniqueness@surface_graph.snap | 2 +- ...__tests__send_to_node@datalog_program.snap | 2 +- ..._tests__simple_filter@datalog_program.snap | 2 +- ...single_column_program@datalog_program.snap | 39 +- ...s__transitive_closure@datalog_program.snap | 39 +- ..._triple_relation_join@datalog_program.snap | 76 +++- ...ests__wildcard_fields@datalog_program.snap | 39 +- ...__wildcard_join_count@datalog_program.snap | 78 +++- hydroflow_lang/CHANGELOG.md | 233 ++++++++++- hydroflow_lang/Cargo.toml | 4 +- hydroflow_lang/build.rs | 58 ++- hydroflow_lang/src/diagnostic.rs | 26 ++ hydroflow_lang/src/graph/di_mul_graph.rs | 68 +-- .../src/graph/eliminate_extra_unions_tees.rs | 3 +- .../src/graph/flat_graph_builder.rs | 215 +++++++++- .../src/graph/flat_to_partitioned.rs | 42 +- hydroflow_lang/src/graph/flow_props.rs | 29 ++ hydroflow_lang/src/graph/graph_algorithms.rs | 52 +++ hydroflow_lang/src/graph/graph_write.rs | 72 ++-- hydroflow_lang/src/graph/hydroflow_graph.rs | 235 +++++++++-- hydroflow_lang/src/graph/mod.rs | 112 ++++- .../src/graph/ops/_lattice_fold_batch.rs | 112 +++++ .../src/graph/ops/_lattice_join_fused_join.rs | 112 +++++ hydroflow_lang/src/graph/ops/anti_join.rs | 130 ++++-- .../src/graph/ops/anti_join_multiset.rs | 201 +++++++++ hydroflow_lang/src/graph/ops/assert.rs | 111 ++--- hydroflow_lang/src/graph/ops/assert_eq.rs | 96 +++++ hydroflow_lang/src/graph/ops/batch.rs | 149 ------- hydroflow_lang/src/graph/ops/cast.rs | 68 +++ hydroflow_lang/src/graph/ops/cross_join.rs | 17 +- .../src/graph/ops/cross_join_multiset.rs | 69 ++++ hydroflow_lang/src/graph/ops/defer_signal.rs | 89 ++++ hydroflow_lang/src/graph/ops/defer_tick.rs | 81 ++++ hydroflow_lang/src/graph/ops/demux.rs | 5 +- hydroflow_lang/src/graph/ops/dest_file.rs | 1 + hydroflow_lang/src/graph/ops/dest_sink.rs | 1 + .../src/graph/ops/dest_sink_serde.rs | 4 +- hydroflow_lang/src/graph/ops/difference.rs | 108 ++--- .../src/graph/ops/difference_multiset.rs | 79 ++++ hydroflow_lang/src/graph/ops/enumerate.rs | 12 +- hydroflow_lang/src/graph/ops/filter.rs | 3 +- hydroflow_lang/src/graph/ops/filter_map.rs | 3 +- hydroflow_lang/src/graph/ops/flat_map.rs | 3 +- hydroflow_lang/src/graph/ops/flatten.rs | 3 +- hydroflow_lang/src/graph/ops/fold.rs | 43 +- hydroflow_lang/src/graph/ops/fold_keyed.rs | 21 +- hydroflow_lang/src/graph/ops/for_each.rs | 1 + hydroflow_lang/src/graph/ops/identity.rs | 5 +- hydroflow_lang/src/graph/ops/initialize.rs | 3 +- hydroflow_lang/src/graph/ops/inspect.rs | 9 +- hydroflow_lang/src/graph/ops/join.rs | 58 ++- hydroflow_lang/src/graph/ops/join_fused.rs | 346 ++++++++++++++++ .../src/graph/ops/join_fused_lhs.rs | 184 +++++++++ .../src/graph/ops/join_fused_rhs.rs | 69 ++++ hydroflow_lang/src/graph/ops/join_multiset.rs | 73 ++++ hydroflow_lang/src/graph/ops/lattice_batch.rs | 162 -------- hydroflow_lang/src/graph/ops/lattice_fold.rs | 81 ++++ hydroflow_lang/src/graph/ops/lattice_join.rs | 209 ---------- .../{lattice_merge.rs => lattice_reduce.rs} | 25 +- hydroflow_lang/src/graph/ops/map.rs | 17 +- hydroflow_lang/src/graph/ops/mod.rs | 200 ++++++++- .../src/graph/ops/multiset_delta.rs | 3 +- hydroflow_lang/src/graph/ops/next_stratum.rs | 1 + hydroflow_lang/src/graph/ops/next_tick.rs | 58 --- hydroflow_lang/src/graph/ops/null.rs | 1 + hydroflow_lang/src/graph/ops/persist.rs | 9 +- hydroflow_lang/src/graph/ops/persist_mut.rs | 3 +- .../src/graph/ops/persist_mut_keyed.rs | 3 +- hydroflow_lang/src/graph/ops/py_udf.rs | 151 +++++++ hydroflow_lang/src/graph/ops/reduce.rs | 60 ++- hydroflow_lang/src/graph/ops/reduce_keyed.rs | 13 +- hydroflow_lang/src/graph/ops/sort.rs | 5 +- hydroflow_lang/src/graph/ops/sort_by_key.rs | 7 +- hydroflow_lang/src/graph/ops/source_file.rs | 3 +- .../src/graph/ops/source_interval.rs | 3 +- hydroflow_lang/src/graph/ops/source_iter.rs | 1 + .../src/graph/ops/source_iter_delta.rs | 25 ++ hydroflow_lang/src/graph/ops/source_json.rs | 1 + hydroflow_lang/src/graph/ops/source_stdin.rs | 11 +- hydroflow_lang/src/graph/ops/source_stream.rs | 1 + .../src/graph/ops/source_stream_serde.rs | 3 +- hydroflow_lang/src/graph/ops/spin.rs | 14 +- hydroflow_lang/src/graph/ops/tee.rs | 13 +- hydroflow_lang/src/graph/ops/union.rs | 39 +- hydroflow_lang/src/graph/ops/unique.rs | 7 +- hydroflow_lang/src/graph/ops/unzip.rs | 5 +- hydroflow_lang/src/graph/ops/zip.rs | 3 +- hydroflow_lang/src/graph/ops/zip_longest.rs | 3 +- .../src/graph/propegate_flow_props.rs | 83 ++++ hydroflow_lang/src/lib.rs | 3 + hydroflow_lang/src/parse.rs | 135 +++++- hydroflow_lang/src/pretty_span.rs | 20 +- hydroflow_lang/src/union_find.rs | 15 + hydroflow_macro/CHANGELOG.md | 78 +++- hydroflow_macro/Cargo.toml | 8 +- hydroflow_macro/build.rs | 28 +- hydroflow_macro/src/lib.rs | 46 ++- lattices/CHANGELOG.md | 185 ++++++++- lattices/Cargo.toml | 2 +- lattices/README.md | 24 +- lattices/src/collections.rs | 239 ++++++++++- lattices/src/conflict.rs | 36 +- lattices/src/dom_pair.rs | 29 +- lattices/src/lib.rs | 37 +- lattices/src/map_union.rs | 136 ++++-- lattices/src/ord.rs | 227 +++++++++- lattices/src/pair.rs | 18 +- lattices/src/seq.rs | 168 -------- lattices/src/set_union.rs | 56 ++- lattices/src/test.rs | 75 +++- lattices/src/unit.rs | 37 ++ lattices/src/vec_union.rs | 191 +++++++++ lattices/src/with_bot.rs | 89 +++- lattices/src/with_top.rs | 67 ++- multiplatform_test/Cargo.toml | 2 +- multiplatform_test/src/lib.rs | 4 +- pusherator/CHANGELOG.md | 70 +++- pusherator/Cargo.toml | 2 +- pusherator/src/lib.rs | 3 +- pusherator/src/null.rs | 27 ++ relalg/Cargo.toml | 2 +- rust-toolchain.toml | 4 +- topolotree/.gitignore | 1 + topolotree/Cargo.toml | 38 ++ .../src/latency_measure.rs | 37 +- topolotree/src/main.rs | 212 ++++++++++ .../main.rs => topolotree/src/pn.rs | 38 +- .../main.rs => topolotree/src/pn_delta.rs | 40 +- topolotree/src/protocol.rs | 27 ++ topolotree/src/tests.rs | 381 +++++++++++++++++ topolotree/topolotree_latency.hydro.py | 352 ++++++++++++++++ website_playground/Cargo.toml | 2 +- website_playground/src/lib.rs | 11 +- 486 files changed, 15876 insertions(+), 4611 deletions(-) create mode 100644 design_docs/2023-07_lattice_properties.md create mode 100644 design_docs/2023-08_lattice_properties.md rename docs/docs/hydroflow/architecture/{index.md => index.mdx} (81%) create mode 100644 docs/docs/hydroflow/concepts/cyclic_flows.md delete mode 100644 hydro_cli_examples/examples/topolotree/main.rs delete mode 100644 hydro_cli_examples/topolotree.hydro.py delete mode 100644 hydro_cli_examples/topolotree_latency.hydro.py delete mode 100644 hydroflow/examples/deletable_state/README.md delete mode 100644 hydroflow/examples/deletable_state/main.rs create mode 100644 hydroflow/examples/example_naturals.rs delete mode 100644 hydroflow/examples/graph_reachability/README.md delete mode 100644 hydroflow/examples/graph_reachability/main.rs delete mode 100644 hydroflow/examples/graph_unreachability/README.md delete mode 100644 hydroflow/examples/graph_unreachability/main.rs create mode 100644 hydroflow/examples/modules/main.rs create mode 100644 hydroflow/examples/modules/triple_cross_join.hf create mode 100644 hydroflow/examples/python_udf/main.rs create mode 100644 hydroflow/src/compiled/pull/anti_join.rs create mode 100644 hydroflow/src/compiled/pull/half_join_state/fold.rs create mode 100644 hydroflow/src/compiled/pull/half_join_state/fold_from.rs create mode 100644 hydroflow/src/compiled/pull/half_join_state/multiset2.rs create mode 100644 hydroflow/src/compiled/pull/half_join_state/reduce.rs delete mode 100644 hydroflow/src/compiled/pull/symmetric_hash_join_lattice.rs create mode 100644 hydroflow/src/util/monotonic.rs create mode 100644 hydroflow/src/util/multiset.rs create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.stderr create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.rs create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.stderr create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_upcast.rs create mode 100644 hydroflow/tests/compile-fail/surface_flow_props_upcast.stderr rename hydroflow/tests/compile-fail/{surface_lattice_merge_badgeneric.rs.ignore => surface_lattice_fold_badgeneric.rs.ignore} (81%) rename hydroflow/tests/compile-fail/{surface_lattice_merge_badgeneric.stderr => surface_lattice_fold_badgeneric.stderr} (85%) rename hydroflow/tests/compile-fail/{surface_lattice_merge_nogeneric.rs => surface_lattice_fold_nogeneric.rs} (83%) create mode 100644 hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.stderr rename hydroflow/tests/compile-fail/{surface_lattice_merge_wronggeneric.rs => surface_lattice_fold_wronggeneric.rs} (69%) create mode 100644 hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.stderr delete mode 100644 hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.stderr create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.rs.ignore create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.stderr create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.rs create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.stderr create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.rs create mode 100644 hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.stderr rename hydroflow/tests/compile-fail/{surface_lattice_merge_wronggeneric.stderr => surface_latticereduce_wronggeneric.stderr} (66%) create mode 100644 hydroflow/tests/compile-fail/surface_use_bad1.rs create mode 100644 hydroflow/tests/compile-fail/surface_use_bad1.stderr create mode 100644 hydroflow/tests/compile-fail/surface_use_bad2.rs create mode 100644 hydroflow/tests/compile-fail/surface_use_bad2.stderr create mode 100644 hydroflow/tests/compile-fail/surface_use_bad3.rs create mode 100644 hydroflow/tests/compile-fail/surface_use_bad3.stderr create mode 100644 hydroflow/tests/compile-fail/surface_use_unknown.rs create mode 100644 hydroflow/tests/compile-fail/surface_use_unknown.stderr create mode 100644 hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_examples__example_naturals.snap create mode 100644 hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join__static_static@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join__static_static@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join__static_tick@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join__static_tick@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join__tick_static@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join__tick_static@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_basic@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_basic@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_mermaid.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_dot.snap create mode 100644 hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_mermaid.snap create mode 100644 hydroflow/tests/surface_batch.rs create mode 100644 hydroflow/tests/surface_flow_props.rs create mode 100644 hydroflow/tests/surface_join.rs create mode 100644 hydroflow/tests/surface_join_fused.rs create mode 100644 hydroflow/tests/surface_lattice_batch.rs rename hydroflow/tests/{surface_lattice_merge.rs => surface_lattice_fold.rs} (85%) create mode 100644 hydroflow/tests/surface_lattice_join.rs create mode 100644 hydroflow/tests/surface_lattice_reduce.rs create mode 100644 hydroflow/tests/surface_python.rs create mode 100644 hydroflow_lang/src/graph/flow_props.rs create mode 100644 hydroflow_lang/src/graph/ops/_lattice_fold_batch.rs create mode 100644 hydroflow_lang/src/graph/ops/_lattice_join_fused_join.rs create mode 100644 hydroflow_lang/src/graph/ops/anti_join_multiset.rs create mode 100644 hydroflow_lang/src/graph/ops/assert_eq.rs delete mode 100644 hydroflow_lang/src/graph/ops/batch.rs create mode 100644 hydroflow_lang/src/graph/ops/cast.rs create mode 100644 hydroflow_lang/src/graph/ops/cross_join_multiset.rs create mode 100644 hydroflow_lang/src/graph/ops/defer_signal.rs create mode 100644 hydroflow_lang/src/graph/ops/defer_tick.rs create mode 100644 hydroflow_lang/src/graph/ops/difference_multiset.rs create mode 100644 hydroflow_lang/src/graph/ops/join_fused.rs create mode 100644 hydroflow_lang/src/graph/ops/join_fused_lhs.rs create mode 100644 hydroflow_lang/src/graph/ops/join_fused_rhs.rs create mode 100644 hydroflow_lang/src/graph/ops/join_multiset.rs delete mode 100644 hydroflow_lang/src/graph/ops/lattice_batch.rs create mode 100644 hydroflow_lang/src/graph/ops/lattice_fold.rs delete mode 100644 hydroflow_lang/src/graph/ops/lattice_join.rs rename hydroflow_lang/src/graph/ops/{lattice_merge.rs => lattice_reduce.rs} (74%) delete mode 100644 hydroflow_lang/src/graph/ops/next_tick.rs create mode 100644 hydroflow_lang/src/graph/ops/py_udf.rs create mode 100644 hydroflow_lang/src/graph/ops/source_iter_delta.rs create mode 100644 hydroflow_lang/src/graph/propegate_flow_props.rs delete mode 100644 lattices/src/seq.rs create mode 100644 lattices/src/unit.rs create mode 100644 lattices/src/vec_union.rs create mode 100644 pusherator/src/null.rs create mode 100644 topolotree/.gitignore create mode 100644 topolotree/Cargo.toml rename hydro_cli_examples/examples/topolotree_latency_measure/main.rs => topolotree/src/latency_measure.rs (77%) create mode 100644 topolotree/src/main.rs rename hydro_cli_examples/examples/pn_counter/main.rs => topolotree/src/pn.rs (83%) rename hydro_cli_examples/examples/pn_counter_delta/main.rs => topolotree/src/pn_delta.rs (83%) create mode 100644 topolotree/src/protocol.rs create mode 100644 topolotree/src/tests.rs create mode 100644 topolotree/topolotree_latency.hydro.py diff --git a/.github/workflows/build-cli.yml b/.github/workflows/build-cli.yml index 285a04b77b2..e4c7485fe93 100644 --- a/.github/workflows/build-cli.yml +++ b/.github/workflows/build-cli.yml @@ -105,17 +105,8 @@ jobs: with: working-directory: hydro_cli target: ${{ matrix.platform.target }} - manylinux: musllinux_1_2 + manylinux: auto args: --release --out dist - - uses: uraimo/run-on-arch-action@master - name: Install built wheel - with: - arch: ${{ matrix.platform.arch }} - distro: alpine_latest - install: | - apk add py3-pip - run: | - pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links hydro_cli/dist/ --force-reinstall - name: "Upload wheels" uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6e5cbfd1e3..fd27b28f5c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: check - args: --all-targets + args: --all-targets --features python check-wasm: name: Check WebAssembly @@ -91,7 +91,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: check - args: -p hydroflow_lang --target wasm32-unknown-unknown --no-default-features + args: -p hydroflow_lang --target wasm32-unknown-unknown test: name: Test Suite @@ -121,11 +121,17 @@ jobs: toolchain: nightly override: ${{ matrix.rust_release == 'latest-nightly' }} - - name: Run cargo test + - name: Run cargo test all targets (which does not include doctests) + uses: actions-rs/cargo@v1 + with: + command: test + args: --no-fail-fast --all-targets --features python + + - name: Run cargo doc tests uses: actions-rs/cargo@v1 with: command: test - args: --no-fail-fast + args: --no-fail-fast --doc --features python test-wasm: name: Test Suite (WebAssembly) @@ -275,7 +281,21 @@ jobs: uses: actions-rs/cargo@v1 with: command: clippy - args: --all-targets -- -D warnings + args: --all-targets --features python -- -D warnings + + build-website: + name: Build Website + if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} + timeout-minutes: 25 + needs: pre_job + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Build Website + run: bash build_docs.bash x86_64-linux-gnu-ubuntu-20.04 docs: name: Docs (rustdoc) diff --git a/Cargo.lock b/Cargo.lock index 9c288400cc9..ac363114db8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,9 +36,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -1291,7 +1291,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hydro_cli" -version = "0.2.0" +version = "0.4.0" dependencies = [ "anyhow", "async-channel", @@ -1340,7 +1340,7 @@ dependencies = [ [[package]] name = "hydroflow" -version = "0.2.0" +version = "0.4.0" dependencies = [ "bincode", "byteorder", @@ -1359,10 +1359,12 @@ dependencies = [ "hydroflow_lang", "hydroflow_macro", "insta", + "instant", "itertools", "lattices", "multiplatform_test", "pusherator", + "pyo3", "rand 0.8.5", "rand_distr", "ref-cast", @@ -1389,7 +1391,7 @@ dependencies = [ [[package]] name = "hydroflow_cli_integration" -version = "0.2.0" +version = "0.3.0" dependencies = [ "async-recursion", "async-trait", @@ -1405,7 +1407,7 @@ dependencies = [ [[package]] name = "hydroflow_datalog" -version = "0.2.0" +version = "0.4.0" dependencies = [ "hydroflow_datalog_core", "proc-macro-crate", @@ -1416,7 +1418,7 @@ dependencies = [ [[package]] name = "hydroflow_datalog_core" -version = "0.2.0" +version = "0.4.0" dependencies = [ "hydroflow_lang", "insta", @@ -1433,7 +1435,7 @@ dependencies = [ [[package]] name = "hydroflow_lang" -version = "0.2.0" +version = "0.4.0" dependencies = [ "auto_impl", "itertools", @@ -1449,7 +1451,7 @@ dependencies = [ [[package]] name = "hydroflow_macro" -version = "0.2.0" +version = "0.4.0" dependencies = [ "hydroflow_lang", "itertools", @@ -1505,11 +1507,12 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.3" +version = "0.17.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" dependencies = [ "console", + "instant", "number_prefix", "portable-atomic", "unicode-width", @@ -1541,6 +1544,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -1624,7 +1630,7 @@ dependencies = [ [[package]] name = "lattices" -version = "0.2.0" +version = "0.4.0" dependencies = [ "cc-traits", "sealed", @@ -2112,9 +2118,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.19" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "ppv-lite86" @@ -2168,9 +2174,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -2189,7 +2195,7 @@ dependencies = [ [[package]] name = "pusherator" -version = "0.0.1" +version = "0.0.3" dependencies = [ "either", "variadics", @@ -2459,13 +2465,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -2474,7 +2480,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -2483,6 +2489,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "relalg" version = "0.0.0" @@ -3141,6 +3153,22 @@ dependencies = [ "winnow", ] +[[package]] +name = "topolotree" +version = "0.0.0" +dependencies = [ + "dashmap", + "futures", + "hydroflow", + "hydroflow_datalog", + "procinfo", + "rand 0.8.5", + "serde", + "serde_json", + "tokio", + "tokio-tungstenite", +] + [[package]] name = "tracing" version = "0.1.37" @@ -3242,7 +3270,7 @@ dependencies = [ "lazy_static", "log", "regex", - "regex-syntax", + "regex-syntax 0.6.29", "rustc-hash", "semver 1.0.17", "serde", diff --git a/Cargo.toml b/Cargo.toml index 7bc13064ba0..a5b428dd328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "multiplatform_test", "pusherator", "relalg", + "topolotree", "variadics", "website_playground", ] diff --git a/README.md b/README.md index 25eb46374a1..a97230b747c 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Docs.rs

-Hydroflow is a low-latency dataflow runtime written in Rust. The [Hydro Project](https://hydro.run/docs/hydroflow/ecosystem/) -will empower developers to harness the full potential of the cloud by making distributed programs easy to specify and automatic to scale. Hydroflow serves as the lowest level in the [Hydro stack](https://hydro.run/docs/hydroflow/ecosystem/), +Hydroflow is a low-latency dataflow runtime written in Rust. The goal of the [Hydro Project](https://hydro.run) +is to empower developers to harness the full potential of the cloud by making distributed programs easy to specify and automatic to scale. Hydroflow is the lowest level in the [Hydro stack](https://hydro.run/docs/hydroflow/ecosystem/), serving as a single-node low-latency runtime with explicit networking. This allows us to support not just data processing pipelines, but distributed protocols (e.g. Paxos) and real-world long-running applications as well. @@ -17,16 +17,16 @@ Take a look at the [Hydroflow Book](https://hydro.run/docs/hydroflow/). ## The Hydroflow Surface Syntax -Hydroflow comes with a custom "surface syntax" domain-specific language which serves as a very -simple, readable IR for specifying single-node Hydroflow programs, intended to be stitched together -by the Hydro stack to create larger autoscaling distributed systems.' +Hydroflow comes with a custom "surface syntax"—a domain-specific language which serves as a very +simple, readable IR for specifying single-node Hydroflow programs. These programs are intended to be stitched together +by the Hydro stack to create larger autoscaling distributed systems. Here's a simple example of the surface syntax. Check out the [Hydroflow Playground](https://hydro.run/playground) for an interactive demo. ```rust source_iter(0..10) -> map(|n| n * n) - -> filter(|&n| n > 10) + -> filter(|n| *n > 10) -> foo; foo = map(|n| (n..=n+1)) diff --git a/benches/benches/fork_join.rs b/benches/benches/fork_join.rs index 1e06f0da331..58ca87868ca 100644 --- a/benches/benches/fork_join.rs +++ b/benches/benches/fork_join.rs @@ -54,11 +54,7 @@ fn benchmark_hydroflow(c: &mut Criterion) { send1, send2, |_ctx, recv1, recv2, send1, send2| { - for v in recv1 - .take_inner() - .into_iter() - .chain(recv2.take_inner().into_iter()) - { + for v in recv1.take_inner().into_iter().chain(recv2.take_inner()) { if v % 2 == 0 { send1.give(Some(v)); } else { diff --git a/benches/benches/join.rs b/benches/benches/join.rs index 385f60e2cdb..409820d2536 100644 --- a/benches/benches/join.rs +++ b/benches/benches/join.rs @@ -61,7 +61,7 @@ where } } - left_tab.entry(k).or_insert_with(Vec::new).push(v); + left_tab.entry(k).or_default().push(v); } }); @@ -76,7 +76,7 @@ where } } - right_tab.entry(k).or_insert_with(Vec::new).push(v); + right_tab.entry(k).or_default().push(v); } }); } @@ -98,8 +98,8 @@ where b.iter(|| { let iter_a = (0..NUM_INTS).map(|x| (x, L::new(x))); let iter_b = (0..NUM_INTS).map(|x| (x, R::new(x))); - let mut items_a = HashMap::new(); - let mut items_b = HashMap::new(); + let mut items_a = HashMap::<_, Vec<_>>::new(); + let mut items_b = HashMap::<_, Vec<_>>::new(); for (key, val_a) in iter_a { if let Some(vals_b) = items_b.get(&key) { @@ -107,7 +107,7 @@ where black_box((key, val_a.clone(), val_b)); } } - items_a.entry(key).or_insert_with(Vec::new).push(val_a); + items_a.entry(key).or_default().push(val_a); } for (key, val_b) in iter_b { if let Some(vals_a) = items_a.get(&key) { @@ -115,7 +115,7 @@ where black_box((key, val_a, val_b.clone())); } } - items_b.entry(key).or_insert_with(Vec::new).push(val_b); + items_b.entry(key).or_default().push(val_b); } }); }, diff --git a/benches/benches/micro_ops.rs b/benches/benches/micro_ops.rs index bc548bbd53e..422613750b5 100644 --- a/benches/benches/micro_ops.rs +++ b/benches/benches/micro_ops.rs @@ -188,7 +188,7 @@ fn ops(c: &mut Criterion) { #[allow(clippy::unnecessary_fold)] { hydroflow_syntax! { - source_iter(black_box(input0)) -> fold::<'tick>(0, |accum, elem| { accum + elem }) -> for_each(|x| { black_box(x); }); + source_iter(black_box(input0)) -> fold::<'tick>(0, |accum: &mut _, elem| { *accum += elem }) -> for_each(|x| { black_box(x); }); } } }, @@ -275,25 +275,25 @@ fn ops(c: &mut Criterion) { let mut df = hydroflow_syntax! { source_iter(black_box(DATA)) -> persist() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) -> for_each(|x| { black_box(x); }); }; @@ -308,25 +308,25 @@ fn ops(c: &mut Criterion) { let mut df = hydroflow_syntax! { source_iter(black_box(DATA)) -> persist() - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) - -> next_tick() + -> defer_tick() -> map(black_box) -> for_each(|x| { black_box(x); }); }; diff --git a/benches/benches/reachability.rs b/benches/benches/reachability.rs index 2737d74796e..7d68e6787b7 100644 --- a/benches/benches/reachability.rs +++ b/benches/benches/reachability.rs @@ -14,14 +14,14 @@ lazy_static::lazy_static! { let cursor = Cursor::new(include_bytes!("reachability_edges.txt")); let reader = BufReader::new(cursor); - let mut edges = HashMap::new(); + let mut edges = HashMap::<_, Vec<_>>::new(); for line in reader.lines() { let line = line.unwrap(); let mut nums = line.split_whitespace(); let a = nums.next().unwrap().parse().unwrap(); let b = nums.next().unwrap().parse().unwrap(); assert!(nums.next().is_none()); - edges.entry(a).or_insert_with(Vec::new).push(b); + edges.entry(a).or_default().push(b); } edges }; diff --git a/benches/benches/symmetric_hash_join.rs b/benches/benches/symmetric_hash_join.rs index b1221b8d9e4..1834c767cc2 100644 --- a/benches/benches/symmetric_hash_join.rs +++ b/benches/benches/symmetric_hash_join.rs @@ -1,7 +1,7 @@ use std::hint::black_box; use criterion::{criterion_group, criterion_main, Criterion}; -use hydroflow::compiled::pull::{SetJoinState, SymmetricHashJoin}; +use hydroflow::compiled::pull::{symmetric_hash_join_into_iter, HalfSetJoinState}; use rand::distributions::Distribution; use rand::rngs::StdRng; use rand::SeedableRng; @@ -14,11 +14,14 @@ fn ops(c: &mut Criterion) { let rhs: Vec<_> = (0..3000).map(|v| (v + 50000, ())).collect(); b.iter(|| { - let mut state = black_box(SetJoinState::default()); - let join = SymmetricHashJoin::new( + let (mut lhs_state, mut rhs_state) = + black_box((HalfSetJoinState::default(), HalfSetJoinState::default())); + let join = symmetric_hash_join_into_iter( black_box(lhs.iter().cloned()), black_box(rhs.iter().cloned()), - &mut state, + &mut lhs_state, + &mut rhs_state, + false, ); for v in join { @@ -32,11 +35,14 @@ fn ops(c: &mut Criterion) { let rhs: Vec<_> = (0..3000).map(|v| (v, v + 50000)).collect(); b.iter(|| { - let mut state = black_box(SetJoinState::default()); - let join = SymmetricHashJoin::new( + let (mut lhs_state, mut rhs_state) = + black_box((HalfSetJoinState::default(), HalfSetJoinState::default())); + let join = symmetric_hash_join_into_iter( black_box(lhs.iter().cloned()), black_box(rhs.iter().cloned()), - &mut state, + &mut lhs_state, + &mut rhs_state, + false, ); for v in join { @@ -50,13 +56,15 @@ fn ops(c: &mut Criterion) { let rhs: Vec<_> = (0..3000).map(|v| (v, v)).collect(); b.iter(|| { - let mut state = black_box(SetJoinState::default()); - let join = SymmetricHashJoin::new( + let (mut lhs_state, mut rhs_state) = + black_box((HalfSetJoinState::default(), HalfSetJoinState::default())); + let join = symmetric_hash_join_into_iter( black_box(lhs.iter().cloned()), black_box(rhs.iter().cloned()), - &mut state, + &mut lhs_state, + &mut rhs_state, + false, ); - for v in join { black_box(v); } @@ -77,11 +85,14 @@ fn ops(c: &mut Criterion) { .collect(); b.iter(|| { - let mut state = black_box(SetJoinState::default()); - let join = SymmetricHashJoin::new( + let (mut lhs_state, mut rhs_state) = + black_box((HalfSetJoinState::default(), HalfSetJoinState::default())); + let join = symmetric_hash_join_into_iter( black_box(lhs.iter().cloned()), black_box(rhs.iter().cloned()), - &mut state, + &mut lhs_state, + &mut rhs_state, + false, ); for v in join { @@ -105,11 +116,14 @@ fn ops(c: &mut Criterion) { .collect(); b.iter(|| { - let mut state = black_box(SetJoinState::default()); - let join = SymmetricHashJoin::new( + let (mut lhs_state, mut rhs_state) = + black_box((HalfSetJoinState::default(), HalfSetJoinState::default())); + let join = symmetric_hash_join_into_iter( black_box(lhs.iter().cloned()), black_box(rhs.iter().cloned()), - &mut state, + &mut lhs_state, + &mut rhs_state, + false, ); for v in join { diff --git a/build_docs.bash b/build_docs.bash index a004ae84698..00c5768d513 100644 --- a/build_docs.bash +++ b/build_docs.bash @@ -1,6 +1,8 @@ set -e -wget -qO- https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz | tar xJ +PLATFORM=${1:-"x86_64-linux-gnu-ubuntu-16.04"} + +wget -qO- https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-$PLATFORM.tar.xz | tar xJ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -10,10 +12,10 @@ curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh cd website_playground -CARGO_CFG_HYDROFLOW_GENERATE_DOCS="1" RUSTFLAGS="--cfg procmacro2_semver_exempt" CC="$PWD/../clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04/bin/clang" wasm-pack build +CARGO_CFG_HYDROFLOW_GENERATE_DOCS="1" RUSTFLAGS="--cfg procmacro2_semver_exempt --cfg super_unstable" CC="$PWD/../clang+llvm-13.0.0-$PLATFORM/bin/clang" wasm-pack build cd ../docs npm ci -npm run build +LOAD_PLAYGROUND=1 npm run build diff --git a/design_docs/2023-07_lattice_properties.md b/design_docs/2023-07_lattice_properties.md new file mode 100644 index 00000000000..7e7b09f0f81 --- /dev/null +++ b/design_docs/2023-07_lattice_properties.md @@ -0,0 +1,268 @@ +# Lattice Properties + +## Goals + +1. Make monotonicity the easy and default option, make non-monotonic operations the special case. +2. Reject operations that are incorrect (would violate monotonicity). + E.g. can't use a order-dependent fold on an arbitrarily ordered stream. +3. Reason about and optimize Hydroflow graphs at proc-macro time. + What portions can be parallelized, partitioned, etc. + +## Design + +Introduce _stream types_, as a layer on top of lattice types. The stream type represents sequential +information about the lattice instances, such as ordering, sorting, monotonicity, or atomization. + +* `SeqFlow<*, T>` +* `LatticeFlow` +* `DiffLatticeFlow` +* `CumuLatticeFlow` + +`SeqFlow` is a special per-element representation of the `Seq<*, T>` lattice type. + +Stream types are **NOT** automatically infered. It will be up to the user to explicitly switch +between different stream types. +An alternative, using Rust's type system to infer stream types, is too fragile and more importantly +cannot be used a proc-macro time which prevents goal #3. Having the user manually specify stream +types ensures the scaling and monotonicity of the system will be top-of-mind, and avoids the +complexity of implementing our own type inference system. + +Stream type topology. Stream types can be cast upwards: +```mermaid +flowchart BT +seq["SeqFlow<*, T>"] +lat["LatticeFlow<Lat>"] +dif["DiffLatticeFlow<Lat>"] +cum["CumuLatticeFlow<Lat>"] + +cum --> lat +dif --> lat +lat --> seq +``` + +Monotonic function topology: +```mermaid +flowchart BT +any["any "function""] +fun["deterministic function"] +mono["monotonic function"] +morph["morphism"] + +morph --> mono +mono --> fun +fun --> any +``` + +--- + +Sending bottom $\bot$ through a [lattice flow] stream should have the exact same behavior as sending nothing +through. + +
+ Note: bottom in a SeqStream is not SeqStream's bottom + +```rust +Seq = VecUnion> +Seq bottom = vec![] +vec![bottom, bottom, bottom] is not Seq's bottom +``` +
+ +## Operators + +```mermaid +flowchart BT +map_fn +seq["SeqFlow<*, T>"] +lat["LatticeFlow<Lat>"] +dif["DiffLatticeFlow<Lat>"] +cum["CumuLatticeFlow<Lat>"] + +cum --> lat +dif --> lat +lat --> seq +``` + +| Input(s) | Operator | Output(s) | Condition | +| --- | --- | --- | --- | +| `SeqFlow<*1, T>` | `map(f)` | `SeqFlow<*2, U>` | `f: Fn(T) -> U` | +| `LatticeFlow` | `map(f)` | `LatticeFlow` | `f: Fn(Lat1) -> Lat2` | +| `DiffLatticeFlow` | `map(f)` | `DiffLatticeFlow` | `f: Morphism(Lat1) -> Lat2` | +| `CumuLatticeFlow` | `map(f)` | `CumuLatticeFlow` | `f: MonotonicFn(Lat1) -> Lat2` | +| | | | +| `SeqFlow<*1, T>` | `filter(p)` | `SeqFlow<*2, T>` | `p: Fn(&T) -> bool` | +| `LatticeFlow` | `filter(p)` | `LatticeFlow` | `p: Morphism(&T) -> boolxxxx` | +| `DiffLatticeFlow` | `filter(p)` | `DiffLatticeFlow` | `p: Morphism(&T) -> boolxxxx` | +| `CumuLatticeFlow` | `filter(p)` | `CumuLatticeFlow` | `p: MonotonicFn(&T) -> Max` | +| | | | +| `SeqFlow<*1, (K, V1)>`, `SeqFlow<*2, (K, V2)>` | `join()` | `SeqFlow<*3, (K, (V1, V2))>` | | +| | | | +| `LatticeFlow` | `unmerge()` | `DiffLatticeFlow` | | +| `LatticeFlow` | `merge()` | `CumuLatticeFlow` | | +| any, $N$ times | `union()` | same out | | +| any | `tee()` | same out, $N$ times | | +| | | | +| `SeqFlow<*1, T>` | `sort()` | `SeqFlow<*SORT, T>` | | +| `LatticeFlow` | `sort()` | `SeqFlow<*SORT, Lat>` | | +| | | | + +--- + +| Input(s) | Operator | Output(s) | Condition | +| --- | --- | --- | --- | +| `SeqFlow<*1, T>` | `filter(p)` | `SeqFlow<*2, T>` | `p: Fn(&T) -> bool` | +| `CumuLatticeFlow` | `filter(p)` | `CumuLatticeFlow` | `p: Fn(&T) -> bool` | +| `[Diff]LatticeFlow` | `filter(p)` | `[Diff]LatticeFlow` | `p: Fn(&T) -> bool` | + +```rust +// filter has CumuLatticeFlow input +input + -> merge() + // Legal + // Good, monotonic fn + -> filter(|set: HashSetUnion| set.contains("hello")) + -> output + +// VS + +// filter has DiffLatticeFlow input +input + // Good, monotonic fn (??morphism??) + -> filter(|set: SingletonSetUnion| set.contains("hello")) + -> merge() + -> output + +// VS + +// filter has DiffLatticeFlow input +input + -> merge_batch() + // Non-deterministic + -> filter(|set: HashSetUnion| set.contains("hello")) + -> merge() + -> output +``` + +--- + +With a `[Diff]LatticeFlow` + +`[Diff]LatticeFlow` should be isomorphic to a cumulative lattice flow. + +--- + +`filter(P)` is equivalent to `map(|x| x.keep(P))` ? + +$$ + f(a \sqcup_S b) \quad=\quad f(a) \sqcup_T f(b) + \quad\quad\quad\mathrm{\textit{(morphism)}} +$$ + +```rust +// filter has CumuLatticeFlow input +input + -> merge() + // Legal + // Good, monotonic fn + -> filter(|set: HashSetUnion| set.contains("hello")) + -> output + +// VS + +// filter has DiffLatticeFlow input +input + // Good, monotonic fn (??morphism??) + -> filter(|set: SingletonSetUnion| set.contains("hello")) + -> merge() + -> output + +// VS + +// filter has DiffLatticeFlow input +input + -> merge_batch() + // Non-deterministic + -> filter(|set: HashSetUnion| set.contains("hello")) + -> merge() + -> output +``` + +--- + +```rust +// filter has CumuLatticeFlow input +input + -> merge() + // Good, monotonic + -> filter(|set: HashSetUnion| set.len() > 10) + -> output + +// VS + +// filter has DiffLatticeFlow input +input + // singleton, silly + -> filter(|set: SingletonSetUnion| set.len() > 10) + -> merge() + -> output + +// VS + +// filter has DiffLatticeFlow input +input + -> merge_batch() + // bad, non deterministic + -> filter(|set: HashSetUnion| set.len() > 10) + -> merge() + -> output +``` + +--- + +```rust +input + -> merge() + // flat_map? + -> map(|hash_set| hash_set.keep(|x| x.starts_with("hello"))) + -> output + +// vs + +input + -> filter(|SingletonSet(x)| x.starts_with("hello")) + -> merge() + -> output +``` + +--- + + +```rust +// filter has LatticeFlow input +input -> merge_batch() -> filter(P2) -> merge() -> output +``` + +```rust +P1 = |SingletonSet(x)| x.starts_with("hello") +P2 = |hash_set| hash_set.starts_with("hello") +``` + +```rust +P1 = |set| set.len() < 10 +P2 = |set| set.len() < 10 +``` + + + + +```rust +// filter has CumuLatticeFlow input +input -> merge() -> filter(P1) -> output + +// VS + +// filter has [Diff]LatticeFlow input +input -> filter(P2) -> merge() -> output +``` + + diff --git a/design_docs/2023-08_lattice_properties.md b/design_docs/2023-08_lattice_properties.md new file mode 100644 index 00000000000..80b4653b104 --- /dev/null +++ b/design_docs/2023-08_lattice_properties.md @@ -0,0 +1,334 @@ +# Lattice Properties + +## Goals + +1. Make monotonicity the easy and default option, make non-monotonic operations the special case. +2. Reject operations that are incorrect (would violate monotonicity/determinism). + E.g. can't use a order-dependent fold on an arbitrarily ordered stream. +3. Reason about and optimize Hydroflow graphs at proc-macro time. + What portions can be parallelized, partitioned, etc. + +## Design + +Introduce _stream types_, as a layer on top of lattice types. The stream type represents sequential +information about the lattice instances, such as ordering, sorting, monotonicity, or atomization. + +* `SeqFlow<*, T>` +* `LatticeFlow` +* `CumuLatticeFlow` + +`SeqFlow` is a special per-element representation of the `Seq<*, T>` lattice type. + +Stream types are **NOT** automatically infered. It will be up to the user to explicitly switch +between different stream types. +An alternative, using Rust's type system to infer stream types, is too fragile and more importantly +cannot be used a proc-macro time which prevents goal #3. Having the user manually specify stream +types ensures the scaling and monotonicity of the system will be top-of-mind, and avoids the +complexity of implementing our own type inference system. + + +Items flowing through lattice flows are _lattice points_, not atoms. Not all lattices are +atomizable, and we want to have a lattice-first perspective. + + + +Stream type topology. Stream types can be cast upwards: +```mermaid +flowchart BT +seq["SeqFlow<*, T>"] +del["DeltaLatticeFlow<Lat>"] +cum["CumuLatticeFlow<Lat>"] +atom["Atom lattice flow???"] + +del --> seq +cum --> seq +atom --> del +``` + +```mermaid +flowchart LR +del["DeltaLatticeFlow<Lat>"] +cum["CumuLatticeFlow<Lat>"] + +del --> merge --> cum --> delta --> del +``` + +Monotonic function topology: +```mermaid +flowchart BT +any["any "function""] +fun["deterministic function"] +mono["monotonic function"] +morph["morphism"] + +morph --> mono +mono --> fun +fun --> any +``` + +--- + +Sending bottom $\bot$ through a [lattice flow] stream should have the exact same behavior as sending nothing +through. + +
+ Note: bottom in a SeqStream is not SeqStream's bottom + +```rust +Seq = VecUnion> +Seq bottom = vec![] +vec![bottom, bottom, bottom] is not Seq's bottom +``` +
+ +## Operators + +```rust +// input: set {1, 2, 3, 4} + +// map stream +input -> random_batches() + // input: { 1 }, { 2 }, { 3 }, { 4 } + // NOT A MORPHISM ILLEGAL + // the map function is a set union morphism if it acts on the atoms. + -> map(|x: Set| if x.all(is_even) { OptionSet(x) } else { OptionSet(None) }) -> output + // { 2 }, { 4 } + +// filter stream +input -> atomize() + // input: { 1 }, { 2 }, { 3 }, { 4 } + -> filter(|x: Set| if x.all(is_even)) -> output + // { 2 }, { 4 } +``` + +## TODO: start with cumul thing + +```rust +// input: set {1, 2, 3, 4} + +// map stream +input + -> map(|x: Set| if x.all(is_even) { OptionSet(x) } else { OptionSet(None) }) -> output + +// filter stream +input + -> filter(|x: (x)| 0 == x % 2) -> output + // { 2 }, { 4 } +``` + +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| --- | --- | --- | --- | +| `SeqFlow<*1, T>` | `map(f)` | `SeqFlow<*2, U>` | `f: Fn(T) -> U` | +| `CumuLatticeFlow` | `map(f)` | `CumuLatticeFlow` | `f: MonotonicFn(Lat1) -> Lat2` | +| `LatticeFlow` | `map(f)` | `LatticeFlow` | `f: Morphism(Lat1) -> Lat2` | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `filter(p)` | `SeqFlow<*2, T>` | `p: Fn(&T) -> bool` | +| `CumuLatticeFlow` | `filter(p)` | `CumuLatticeFlow` | `f: MonotonicFn(&Lat) -> Max` | +| `LatticeFlow` | `filter(p)` | Nope | no meaningful filter morphisms exist. Use `map` (convert atoms to bot) instead. | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `filter_map(f)` | `SeqFlow<*2, U>` | `f: Fn(T) -> Option` | +| `CumuLatticeFlow` | `filter_map(f)` | `CumuLatticeFlow` | `f: MonotonicFn(&Lat1) -> WithBot` | +| `LatticeFlow` | `filter_map(f)` | Nope | see `filter(p)` for explanation | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `LatticeFlow` | `debottom()` | `LatticeFlow` | `Lat: Debottom` | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `tee()` | $N\times$ `SeqFlow<*1, T>` | | +| `LatticeFlow` | `tee()` | $N\times$ `LatticeFlow` | | +| `CumuLatticeFlow` | `tee()` | $N\times$ `CumuLatticeFlow` | | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| $N\times$ `SeqFlow<***, T>` | `union()` | `SeqFlow<*out, T>` | | +| $N\times$ `LatticeFlow` | `union()` | `LatticeFlow` | | +| $N\times$ `CumuLatticeFlow` | `union()` | `LatticeFlow` | Note: no longer `Cumu`. Mingwei: Unnatural? | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `fold(init, f)` | `SeqFlow<*2, U>` | `init: Fn() -> U`
`fold: Fn(U, T) -> U` +| `LatticeFlow` | `lattice_fold::()` | `CumuLatticeFlow` | `Lat2: Default` | +| `CumuLatticeFlow` | `lattice_fold::()` | `CumuLatticeFlow` | Silly, equivalent to just `map` with convert | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>`
`SeqFlow<*2, U>` | `cross_join()` | `SeqFlow<*3, (T, U)>` | | +| +| `CumuLatticeFlow`
`CumuLatticeFlow` | `lattice_binary_map(f)` | `CumuLatticeFlow` | `f: BinaryMonotonicFn(Lat1, Lat2) -> Lat3` | +| `LatticeFlow`
`LatticeFlow` | `lattice_cross_join(f)` | `LatticeFlow` | `f: BinaryMorphism(Lat1, Lat2) -> Lat3` | +| +| `CumuLatticeFlow`
`LatticeFlow` | `lattice_half_binary_map(f)` | `LatticeFlow` | silly?
`f: BinaryMorphismRight(Lat1, Lat2) -> Lat3` | +| `LatticeFlow`
`CumuLatticeFlow` | `lattice_half_binary_map(f)` | `LatticeFlow` | silly?
`f: BinaryMorphismLeft(Lat1, Lat2) -> Lat3` | +| +| `SeqFlow<*1, (K, V1)>`
`SeqFlow<*2, (K, V2)>` | `join()` | `SeqFlow<*3, (K, (V1, V2))>` | | +| `LatticeFlow`
`LatticeFlow` | `lattice_cross_join(keyed(f))`
`NEW_lattice_join(f)` | `LatticeFlow>` | `f: BinaryMorphism(Lat1, Lat2) -> Lat3` | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `CAST` | `SeqFlow<*1, T>` | ✅ | +| `SeqFlow<*1, Lat>` | `CAST` | `LatticeFlow` | ❌ | +| `SeqFlow<*1, Lat>` | `CAST` | `CumuLatticeFlow` | ❌ | +| `LatticeFlow` | `CAST` | `SeqFlow<*1, Lat>` | ✅ | +| `LatticeFlow` | `CAST` | `LatticeFlow` | ✅ | +| `LatticeFlow` | `CAST` | `CumuLatticeFlow` | ❌ | +| `CumuLatticeFlow` | `CAST` | `SeqFlow<*1, Lat>` | ✅ | +| `CumuLatticeFlow` | `CAST` | `LatticeFlow` | ✅ | +| `CumuLatticeFlow` | `CAST` | `CumuLatticeFlow` | ✅ | +| +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| `SeqFlow<*1, T>` | `sort()` | `SeqFlow, T>` | | +| `LatticeFlow` | `sort()` | `LatticeFlow` | | + +$ +\texttt{BinaryFn:}\quad R\times S \rightarrow T +$ + +$ + \texttt{BinaryMonotonicFn:}\quad + a \sqsubseteq_R b,\;\; x \sqsubseteq_S y + \quad\Longrightarrow\quad + f(a, x)\ \sqsubseteq_T f(b, y) +$ + + +$ + \texttt{BinaryMorphism:}\quad\\ + f(a\,\sqcup_R\,b,\; x) \quad=\quad f(a, x)\ \sqcup_T\ f(b, x) \\ + f(a,\; x\,\sqcup_S\,y) \quad=\quad f(a, x)\ \sqcup_T\ f(a, y) +$ + +Example: join + +$ + join((a \sqcup b), x) = join(a, x) \sqcup join(b, x) +$ + +$ + (A \cup B) \bowtie (X \cup Y) = (A \bowtie X) \cup (A \bowtie Y) \cup (B \bowtie X) \cup (B \bowtie Y) +$ + +Lattice cross join: + + +$ f(\bigsqcup_tA \sqcup \delta_t a,\ \bigsqcup_t X) \rightarrow (\delta_t a,\ \bigsqcup_t X) $ + +$ f(\bigsqcup_t A,\ \bigsqcup_tX \sqcup \delta_t x) \rightarrow (\bigsqcup_t A,\ \delta_t x) $ + +--- + +```rust +source_iter(...) -> map(f) -> for_each(|x| println!("{:?}", x)) + +source_iter(...) -> deltify() -> dest_stream() .... +-> source_stream() -> merge() -> map(f) -> for_each(|x| println!("{:?}", x)) +``` + +--- + +Users deal with `SeqFlow<*, T>` or `CumuLatticeFlow` + +`SeqFlow<*, T>` +`LatticeFlow` -> `SeqFlow<*, Lat>` +`DeltaLatticeFlow` is deltification of cumulative lattice flow + +```rust + --LF--> map(morph) --LF--> + = --LF--> Deltify() --DLF--> Merge() --LF--> map(morph) --LF--> + = --LF--> Deltify() --DLF--> map(morph) --DLF--> Merge() --LF--> + +``` + +```rust +--LF--> Deltify() --DLF--> map(Non_Morphism()) --SeqFlow<*, Lat2>--> merge() --LF--> +!= +--LF--> Deltify() --DLF--> merge() --LF<2>--> map(Non_Morphism()) --LF--> +``` + +```rust +source_iter([cumu]) --LF--> map(f) --LF--> for_each(|x| println!("{:?}", x)) + +source_iter([cumu] --LF--> deltify() --DLF<*1, L>--> dest_stream() .... +-> source_stream() --DLF<*2, L>--> merge() --DLF<*3, L>--> map(f) -> for_each(|x| println!("{:?}", x)) +``` + +# Postponement stack: +* `AtomLatticeFlow`, should it exist? +* `Debottom`, bottomless lattices +* dependent sort orders (nested loops join) + +| **Input(s)** | **Operator** | **Output(s)** | **Condition** | +| --- | --- | --- | --- | +| `LatticeFlow` | `unmerge()` | `LatticeFlow` | | +| `LatticeFlow` | `merge()` | `CumuLatticeFlow` | | +| | | | +| `SeqFlow<*1, T>` | `sort()` | `SeqFlow<*SORT, T>` | | +| `LatticeFlow` | `sort()` | `SeqFlow<*SORT, Lat>` | | +| | | | + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +--- + +# junk stack + +```rust +source_iter(vec![("a", 1), ("a", 2), ("b", 1), ("b", 3)]) + -> fold_keyed(MapUnion::new(), |accum, elem| { accum.merge(MapUnion::from(elem.0, elem.1))}) + // { "a": MaxInt<2>, "b": MaxInt<3> } + -> flatten() + // { ("a", MaxInt<2>), ("b", MaxInt<3>)} + -> lattice_fold(MaxInt::Bot() )//, |accum, elem| {accum.merge(elem.1)}) + -> assert_eq(MaxInt::from(3)) +``` + +```rust +source_iter(vec![("a", 1), ("a", 2), ("b", 1), ("b", 2)]) + -> fold_keyed(MapUnion::new(), |accum, elem| { accum.merge(MapUnion::from(elem.0, elem.1))}) + // { "a": 2, "b": 2 } + -> flat_map(|map_union| map_union.values()) + -> lattice_merge() +``` + + + + +```rust +SeqFlow -> filter(|x| x != "hello") -> + +LatticeFlow> + -> map(|SingletonSet(x)| if x.starts_with("hello") { + OptionSet::new(x) + } else { + OptionSet::new(None) + }) + -> +``` + +```rust +LatticeFlow> // Max + -> map(|Max(x)| if 0 == x % 2 { + WithBot::new(x) + } else { + WithBot::new(None) + }) // WithBot> + -> debottom() + // Max + -> map(|Max(x)| /* do something else */) +``` +```rust +WithBot +OptionSet /* equivalent to */ WithBot> +``` \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index d37d4f2130c..1874a63678a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,20 +4,45 @@ This website is built using [Docusaurus 2](https://docusaurus.io/), a modern sta You'll need Node installed to build the website. First, install the necessary dependencies: ```bash -$ npm install +$ npm ci ``` -Next, you'll need to build the WebAssembly components of the website. This requires Rust and [wasm-pack](https://rustwasm.github.io/wasm-pack/): +Finally, you can run the website locally: + +```bash +$ npm run start +``` + +## Building the Playground +By default, the Hydroflow / Datalog playgrounds are not loaded when launching the website. To build the playground, you'll need to follow a couple additional steps. This requires Rust and [wasm-pack](https://rustwasm.github.io/wasm-pack/): ```bash $ rustup target add wasm32-unknown-unknown $ cargo install wasm-pack $ cd ../website_playground -$ CARGO_CFG_HYDROFLOW_GENERATE_DOCS="1" wasm-pack build +$ CARGO_CFG_HYDROFLOW_GENERATE_DOCS="1" RUSTFLAGS="--cfg procmacro2_semver_exempt --cfg super_unstable" wasm-pack build ``` -Finally, you can run the website locally: +### Notes on building on macOS +If you're building on macOS, you may need to install the `llvm` package with Homebrew (because the default toolchain has WASM support missing): ```bash -$ npm run start +$ brew install llvm ``` + +Then, you'll need to set `TARGET_CC` and `TARGET_AR` environment variables when building the playground: + +```bash +$ TARGET_CC="$(brew --prefix)/opt/llvm/bin/clang" TARGET_AR="$(brew --prefix)/opt/llvm/bin/llvm-ar" CARGO_CFG_HYDROFLOW_GENERATE_DOCS="1" RUSTFLAGS="--cfg procmacro2_semver_exempt --cfg super_unstable" wasm-pack build +``` + +With the WASM portion built, we can launch the website with the playground loaded: + +```bash +$ LOAD_PLAYGROUND=1 npm run start +``` + +## Adding Papers +1. Upload the paper PDF to the `static/papers` folder. +2. Run the script `./extract-paper-thumbnails` (from this `docs` directory), which requires [ImageMagick to be installed](https://imagemagick.org/script/download.php). +3. Go to `src/pages/research.js` and add the paper to the array at the top of the file. diff --git a/docs/docs/deploy/index.md b/docs/docs/deploy/index.md index 1d2e5870cfb..6e057864a7c 100644 --- a/docs/docs/deploy/index.md +++ b/docs/docs/deploy/index.md @@ -3,6 +3,13 @@ sidebar_position: 1 --- # Introduction + +:::caution + +Hydro Deploy is currently in alpha. Although it has already been used for large-scale distributed experiments within the Hydro research group, the API is rapidly changing and may come with some rough edges, and many pieces are not yet documented. Please reach out if you run into any issues! + +::: + Hydro comes equipped with a built-in deployment system, **Hydro Deploy**, which allows you to deploy your Hydro app to a variety of platforms. With Hydro Deploy, you can spin up complex services with just a few lines of Python! This guide will walk you through the process of deploying your Hydro app in a variety of scenarios. Hydro Deploy focuses on managing the end-to-end lifecycle of networked services in the cloud. It is not a general-purpose deployment tool, and is not intended to replace systems like Docker Compose or Kubernetes. Instead, Hydro Deploy is designed to be used in conjunction with these tools to manage the lifecycle of your Hydro app. @@ -14,8 +21,3 @@ Currently, Hydro Deploy is focused on _ephemeral applications_, which can be spu - Initializing network connections based on a user-defined topology - Monitoring logs from your services -:::caution - -Hydro Deploy is currently in alpha. Although it has already been used for large-scale distributed experiments within the Hydro research group, the API is rapidly changing and may come with some rough edges. Please reach out if you run into any issues! - -::: diff --git a/docs/docs/deploy/install.md b/docs/docs/deploy/install.md index 68783075688..b62873c7ee8 100644 --- a/docs/docs/deploy/install.md +++ b/docs/docs/deploy/install.md @@ -64,5 +64,5 @@ To check that Hydro Deploy is installed correctly, you can run the following com ```console #shell-command-next-line hydro --version -hydro_cli 0.0.0 +Hydro Deploy 0.1.0 ``` diff --git a/docs/docs/hydroflow/architecture/index.md b/docs/docs/hydroflow/architecture/index.mdx similarity index 81% rename from docs/docs/hydroflow/architecture/index.md rename to docs/docs/hydroflow/architecture/index.mdx index dd7b93550b7..cbd4461d79f 100644 --- a/docs/docs/hydroflow/architecture/index.md +++ b/docs/docs/hydroflow/architecture/index.mdx @@ -1,3 +1,6 @@ +import exampleOutput from '!!raw-loader!../../../../hydroflow/tests/snapshots/surface_examples__example_5_reachability.snap'; +import { extractMermaid } from '../../../src/util'; + # Architecture Hydroflow graphs are divided into two layers: the outer _scheduled layer_ and @@ -9,39 +12,7 @@ and handoffs, while the insides of the gray boxes are compiled via the compiled here is the graph from the [reachability](../quickstart/example_5_reachability) chapter, which we will use a running example: -```mermaid -%%{init: {'theme': 'base', 'themeVariables': {'clusterBkg':'#ddd'}}}%% -flowchart TD -classDef pullClass fill:#02f,color:#fff,stroke:#000 -classDef pushClass fill:#ff0,stroke:#000 -linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; -subgraph "sg_1v1 stratum 0" - 1v1[\"(1v1) source_iter(vec! [0])"/]:::pullClass - 2v1[\"(2v1) source_stream(edges_recv)"/]:::pullClass - 3v1[\"(3v1) union()"/]:::pullClass - 7v1[\"(7v1) map(| v | (v, ()))"/]:::pullClass - 4v1[\"(4v1) join()"/]:::pullClass - 5v1[/"(5v1) flat_map(| (src, ((), dst)) | [src, dst])"\]:::pushClass - 6v1[/"(6v1) tee()"\]:::pushClass - 10v1["(10v1) handoff"]:::otherClass - 10v1--1--->3v1 - 1v1--0--->3v1 - 2v1--1--->4v1 - 3v1--->7v1 - 7v1--0--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--0--->10v1 -end -subgraph "sg_2v1 stratum 1" - 8v1[/"(8v1) unique()"\]:::pushClass - 9v1[/"(9v1) for_each(| x | println! ("Reached: {}", x))"\]:::pushClass - 8v1--->9v1 -end -6v1--1--->11v1 -11v1["(11v1) handoff"]:::otherClass -11v1===o8v1 -``` + The [Hydroflow Architecture Design Doc](https://hydro-project.github.io/hydroflow/design_docs/2021-10_architecture_design_doc.html) contains a more detailed explanation of this section. Note that some aspects of @@ -107,7 +78,7 @@ convert a graph into in-out trees. Hydroflow's _Surface Syntax_ hides the distinction between these two layers. It offers a natural `Iterator`-like chaining syntax for building -graphs that get parsed and compiled into a scheduled graph of one or more compiled subgraphs. Please see the [Surface Syntax](../syntax/index) docs for more information. +graphs that get parsed and compiled into a scheduled graph of one or more compiled subgraphs. Please see the [Surface Syntax](../syntax/) docs for more information. Alternatively, the _Core API_ allows you to interact with handoffs directly at a low level. It doesn't provide any notion of chainable operators. You can use Rust `Iterator`s diff --git a/docs/docs/hydroflow/concepts/cyclic_flows.md b/docs/docs/hydroflow/concepts/cyclic_flows.md new file mode 100644 index 00000000000..e8ac836d52a --- /dev/null +++ b/docs/docs/hydroflow/concepts/cyclic_flows.md @@ -0,0 +1,49 @@ +--- +sidebar_position: 2 +--- + +import CodeBlock from '@theme/CodeBlock'; +import exampleCode from '!!raw-loader!../../../../hydroflow/examples/example_5_reachability.rs'; +import exampleCode2 from '!!raw-loader!../../../../hydroflow/examples/example_naturals.rs'; +import { getLines, extractOutput, extractMermaid } from '../../../src/util'; + +# Dataflow Cycles and Fixpoints +Many dataflow libraries only support acyclic flow graphs (DAGs); Hydroflow goes further and supports cycles. Hydroflow's semantics for cyclic flows are based on the formal foundations of recursive queries in the [Datalog](https://en.wikipedia.org/wiki/Datalog) language, which also influenced the design of recursive query features in SQL. + +The basic pattern for cycles in Hydroflow looks something like this: +``` + base = source_() -> ... -> [base]cycle; + cycle = union() + -> ... + -> [next]cycle; +``` +That is, we can trace a cycle of operators in the graph, where one operator is a `union()` that accepts two inputs, one of which is the "back edge" that closes the cycle. + +For a concrete example, we can revisit the code in the [Graph Reachability](../quickstart/example_5_reachability.mdx) quickstart program: + +{getLines(exampleCode, 7, 22)} + +The cycle in that program matches our rough pattern as follows: +``` + origin = source_iter(vec![0]) -> [base]reached_vertices; + reached_vertices = union() -> map(...) + -> [0]my_join_tee + -> ... + -> [next]reached_vertices; +``` + +How should we think about a cycle like this? Intuitively, we can think of the cycle beginning to compute on the data from `base` that comes in via `[0]cycle`. In the Graph Reachability example, this base data corresponds to the origin vertex, `0`. By joining [0] with the `stream_of_edges`, we generate neighbors (1 hop away) and pass them back into the cycle. When one of these is joined again with `stream_of_edges` we get a neighbor of a neighbor (2 hops away). When one of *these* is joined with `stream_of_edges` we get a vertex 3 hops away, and so on. + +If you prefer to think about this program as logic, it represents [Mathematical Induction](https://en.wikipedia.org/wiki/Mathematical_induction) via dataflow: the data from `base` going into `[0]cycle` (i.e. the origin vertex, 0 hops away) is like a "base case", and the data going into `[1]cycle` represents the "induction step" (a vertex *k+1* hops away). (A graph with multiple cycles represents multiple induction, which is a relatively uncommon design pattern in both mathematics and Hydroflow!) + +When does this process end? As with most Hydroflow questions, the answer is not in terms of control flow, but rather in terms of dataflow: *the cycle terminates when it produces no new data*, a notion called a [fixpoint](https://en.wikipedia.org/wiki/Fixed_point_(mathematics)). Our graph reachability example, it terminates when there are no new vertices to visit. Note that the `[join()](../syntax/surface_ops_gen#join)` operator is defined over the *sets* of inputs on each side, and sets +by definition do not contain duplicate values. This prevents the Reachability dataflow from regenerating the same value multiple times. + +Like many looping constructs, it is possible to write a cyclic Hydroflow program that never ``terminates``, in the sense that it produces an unbounded stream of data. If we use `[join_multiset()](../syntax/surface_ops_gen#join_multiset)` instead of `[join()](../syntax/surface_ops_gen#join)` in our Reachability dataflow, the call to `flow.run_available()` never terminates, because each time the same vertex is visited, new data is generated! + +A simpler example of a non-terminating cycle is the following, which specifies the natural numbers: + +{exampleCode2} + +Like any sufficiently powerful language, Hydroflow cannot guarantee that your programs terminate. If you're debugging a non-terminating Hydroflow program, it's a good idea to identify the dataflow cycles and insert an +`[inspect()](../syntax/surface_ops_gen#inspect)` operator along an edge of the cycle to see if it's generating unbounded amounts of duplicated data. You can use the `[unique()](../syntax/surface_ops_gen#unique)` operator to ameliorate the problem. \ No newline at end of file diff --git a/docs/docs/hydroflow/concepts/debugging.md b/docs/docs/hydroflow/concepts/debugging.md index 083d1ec6ba0..1b1fc22b6d6 100644 --- a/docs/docs/hydroflow/concepts/debugging.md +++ b/docs/docs/hydroflow/concepts/debugging.md @@ -1,5 +1,5 @@ --- -sidebar_position: 4 +sidebar_position: 5 --- # Debugging diff --git a/docs/docs/hydroflow/concepts/distributed_time.md b/docs/docs/hydroflow/concepts/distributed_time.md index e410612535a..82f5e79c407 100644 --- a/docs/docs/hydroflow/concepts/distributed_time.md +++ b/docs/docs/hydroflow/concepts/distributed_time.md @@ -1,5 +1,5 @@ --- -sidebar_position: 3 +sidebar_position: 4 --- # Distributed Time diff --git a/docs/docs/hydroflow/concepts/error_handling.md b/docs/docs/hydroflow/concepts/error_handling.md index 7c28c29142e..3ec80599129 100644 --- a/docs/docs/hydroflow/concepts/error_handling.md +++ b/docs/docs/hydroflow/concepts/error_handling.md @@ -1,5 +1,5 @@ --- -sidebar_position: 5 +sidebar_position: 6 --- # Error Handling diff --git a/docs/docs/hydroflow/concepts/stratification.md b/docs/docs/hydroflow/concepts/stratification.md index 4d1042c810f..b06018b3bbb 100644 --- a/docs/docs/hydroflow/concepts/stratification.md +++ b/docs/docs/hydroflow/concepts/stratification.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 3 --- # Streaming, Blocking and Stratification diff --git a/docs/docs/hydroflow/index.mdx b/docs/docs/hydroflow/index.mdx index f66b9e51909..1cf7a673b1d 100644 --- a/docs/docs/hydroflow/index.mdx +++ b/docs/docs/hydroflow/index.mdx @@ -11,7 +11,6 @@ import HydroflowDocs from '../../../hydroflow/README.md' ## This Book This book will teach you how to set up your environment to get started with Hydroflow, and how to program in the Hydroflow surface syntax. -Keep in mind that Hydroflow is under active development and is constantly -changing. However the code in this book is tested with the Hydroflow library so should always be up-to-date. +Keep in mind that Hydroflow is under active development. However the code in this book is tested with the Hydroflow library so should always be up-to-date. If you have any questions, feel free to [create an issue on Github](https://github.com/hydro-project/hydroflow/issues/new). diff --git a/docs/docs/hydroflow/quickstart/example_1_simplest.mdx b/docs/docs/hydroflow/quickstart/example_1_simplest.mdx index e378d92e4e5..980ad8f6f62 100644 --- a/docs/docs/hydroflow/quickstart/example_1_simplest.mdx +++ b/docs/docs/hydroflow/quickstart/example_1_simplest.mdx @@ -62,12 +62,17 @@ item passed in. The Hydroflow surface syntax is merely a *specification*; it does not actually do anything until we run it. -We run the flow from within Rust via the [`run_available()` method](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Hydroflow.html#method.run_available). +We can run this flow from within Rust via the [`run_available()` method](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Hydroflow.html#method.run_available). + {getLines(exampleCode, 8)} + Note that `run_available()` runs the Hydroflow graph until no more work is immediately available. In this example flow, running the graph drains the iterator completely, so no more work will *ever* be available. In future examples we will use external inputs such as -network ingress, in which case more work might appear at any time. +network ingress, in which case more work might appear at any time. In those examples we may need a different method than `run_available()`, +e.g. the [`run_async()`](https://hydro-project.github.io/hydroflow/doc/hydroflow/scheduled/graph/struct.Hydroflow.html#method.run_async) method, +which we'll see +in [the EchoServer example](./example_7_echo_server). ### A Note on Project Structure The template project is intended to be a starting point for your own Hydroflow project, and you can add files and directories as you see fit. The only requirement is that the `src/main.rs` file exists and contains a `main()` function. diff --git a/docs/docs/hydroflow/quickstart/example_2_simple.mdx b/docs/docs/hydroflow/quickstart/example_2_simple.mdx index 9be2e31ae76..5e8cf286b71 100644 --- a/docs/docs/hydroflow/quickstart/example_2_simple.mdx +++ b/docs/docs/hydroflow/quickstart/example_2_simple.mdx @@ -61,6 +61,12 @@ Replace the contents of `src/main.rs` with the following: {exampleCode2} +Here the `filter_map` operator takes a map closure that returns a Rust [`Option`](https://doc.rust-lang.org/std/option/enum.Option.html). +If the value is `Some(...)`, it is passed to the output; if it is `None` it is filtered. + +The `flat_map` operator takes a map closure that generates a collection type (in this case a `Vec`) +which is flattened. + Results: {extractOutput(exampleOutput2)} diff --git a/docs/docs/hydroflow/quickstart/example_3_stream.mdx b/docs/docs/hydroflow/quickstart/example_3_stream.mdx index 9a806d9bd97..8ec6856c876 100644 --- a/docs/docs/hydroflow/quickstart/example_3_stream.mdx +++ b/docs/docs/hydroflow/quickstart/example_3_stream.mdx @@ -12,7 +12,7 @@ import { getLines, extractOutput } from '../../../src/util'; > - the [`source_stream`](../syntax/surface_ops_gen.md#source_stream) operator that brings channel input into Hydroflow > - Rust syntax to programmatically send data to a (local) channel -In our previous examples, data came from within the Hydroflow spec, via Rust iterators and the [`source_iter`](../syntax/surface_ops_gen.md#source_iter) operator. In most cases, however, data comes from outside the Hydroflow spec. In this example, we'll see a simple version of this idea, with data being generated on the same machine and sent into the channel programmatically via Rust. +In our previous examples, data came from within the Hydroflow spec, via Rust iterators and the [`source_iter`](../syntax/surface_ops_gen.md#source_iter) operator. In most cases, however, data comes from outside the Hydroflow spec. In this example, we'll see a simple version of this idea, with data being generated on the same thread and sent into the channel programmatically via Rust. For discussion, we start with a skeleton much like before: diff --git a/docs/docs/hydroflow/quickstart/example_4_neighbors.mdx b/docs/docs/hydroflow/quickstart/example_4_neighbors.mdx index 29c4fb6e98b..2f755c35131 100644 --- a/docs/docs/hydroflow/quickstart/example_4_neighbors.mdx +++ b/docs/docs/hydroflow/quickstart/example_4_neighbors.mdx @@ -14,7 +14,7 @@ import { getLines, extractOutput, extractMermaid } from '../../../src/util'; > * The [`unique`](../syntax/surface_ops_gen.md#unique) operator for removing duplicates from a stream > * Visualizing hydroflow code via `flow.meta_graph().expect(...).to_mermaid()` -So far all the operators we've used have one input and one output and therefore +So far, all the operators we've used have one input and one output and therefore create a linear flow of operators. Let's now take a look at a Hydroflow program containing an operator which has multiple inputs; in the following examples we'll extend this to multiple outputs. @@ -67,7 +67,7 @@ Run the program and focus on the last three lines of output, which come from `fl That looks right: the edges we "sent" into the flow that start at `0` are `(0, 1)` and `(0, 3)`, so the nodes reachable from `0` in 0 or 1 hops are `0, 1, 3`. -> Note: When you run the program you may see the lines printed out in a different order. That's OK; the flow we're defining here is producing a `set` of nodes, so the order in which they are printed out is not specified. The [`sort_by_key`](../syntax/surface_ops_gen.md#sort_by_key) operator can be used to sort the output of a flow. +> Note: When you run the program you may see the lines printed out in a different order. That's OK; the flow we're defining here uses the [`join()`](../syntax/surface_ops_gen.md#join) operator, which deals in `sets` of data items, so the order in which a `join()`'s output items are generated is not specified or guaranteed. The [`sort_by_key`](../syntax/surface_ops_gen.md#sort_by_key) operator can always be used to sort the output of a flow if needed. ## Examining the Hydroflow Code In the code, we want to start out with the origin vertex, `0`, @@ -123,7 +123,7 @@ to see the graph, which should look as follows: You may be wondering why the nodes in the graph have different colors (and shapes, for readers who cannot distinguish colors easily). The answer has nothing to do with the meaning of the program, only with the way that Hydroflow compiles operators into Rust. Simply put, blue (wide-topped) boxes _pull_ data, yellow (wide-bottomed) boxes _push_ data, and the `handoff` is a special operator that buffers pushed data for subsequent pulling. Hydroflow always places a handoff -between a push producer and a pull consumer, for reasons explained in the [Architecture](../architecture/index.md) chapter. +between a push producer and a pull consumer, for reasons explained in the [Architecture](../architecture/index.mdx) chapter. Returning to the code, if you read the `edges_send` calls carefully, you'll see that the example data has vertices (`2`, `4`) that are more than one hop away from `0`, which were diff --git a/docs/docs/hydroflow/quickstart/example_5_reachability.mdx b/docs/docs/hydroflow/quickstart/example_5_reachability.mdx index e9834ff8c8e..364ff45c4f1 100644 --- a/docs/docs/hydroflow/quickstart/example_5_reachability.mdx +++ b/docs/docs/hydroflow/quickstart/example_5_reachability.mdx @@ -11,8 +11,7 @@ import { getLines, extractOutput, extractMermaid } from '../../../src/util'; > * Implementing a recursive algorithm (graph reachability) via cyclic dataflow > * Operators to union data from multiple inputs ([`union`](../syntax/surface_ops_gen.md#union)), and send data to multiple outputs ([`tee`](../syntax/surface_ops_gen.md#tee)) > * Indexing multi-output operators by appending a bracket expression -> * An example of how a cyclic dataflow in one stratum executes to completion before starting the next stratum. - +> * A first example of a cyclic (recursive) flow and the concept of fixpoint. To expand from graph neighbors to graph reachability, we want to find vertices that are connected not just to `origin`, but also to vertices reachable *transitively* from `origin`. Said differently, a vertex is reachable from `origin` if it is @@ -67,19 +66,16 @@ We route the `origin` vertex into it as one input right away: {getLines(exampleCode, 8, 12)} -Note the square-bracket syntax for differentiating the multiple inputs to `union()` -is the same as that of `join()` (except that union can have an unbounded number of inputs, -whereas `join()` is defined to only have two.) +Note the square-bracket syntax for assigning index names to the multiple inputs to `union()`; this is similar +to the indexes for `join()`, except that (a) union can have an arbitrary number of inputs, (b) the index names can be arbitrary strings, and (c) the indexes are optional can be omitted entirely. (By contrast, recall that +`join()` is defined to take 2 required input indexes, `[0]` and `[1]`). The only reason to assign index names to the inputs of `union()` is for labeling edges in the generated (e.g. Mermaid) graphs. -Now, `join()` is defined to only have one output. In our program, we want to copy the joined -output to two places: to the original `for_each` from above to print output, and *also* -back to the `union` operator we called `reached_vertices`. We feed the `join()` output +The next group of statements lays out the join of `reached_vertices` and the `stream_of_edges`. The `join()` operator is defined to only have one output, but in our program, we need its output twice: once to feed the original `for_each` from above to print output, and also to feed +back to the `union` operator that we called `reached_vertices`. We pass the `join()` output through a `flat_map()` as before, and then we feed the result into a [`tee()`](../syntax/surface_ops_gen.md#tee) operator, which is the mirror image of `union()`: instead of merging many inputs to one output, -it copies one input to many different outputs. Each input element is _cloned_, in Rust terms, and -given to each of the outputs. The syntax for the outputs of `tee()` mirrors that of union: we *append* -an output index in square brackets to the `tee` or variable. In this example we have -`my_join_tee[0] ->` and `my_join_tee[1] ->`. +it copies one input to many different outputs. Each input element is _cloned_, in Rust terms, and separate copy is given to each of the outputs. The syntax for the outputs of `tee()` mirrors that of the inputs to union: we can (optionally) *append* +an arbitrary output index name in square brackets to the `tee` or variable. In this example we have `my_join_tee[cycle] ->` and `my_join_tee[print] ->`. Finally, we process the output of the `join` as passed through the `tee`. One branch pushes reached vertices back up into the `reached_vertices` variable (which begins with a `union`), while the other @@ -87,9 +83,6 @@ prints out all the reached vertices as in the simple program. {getLines(exampleCode, 14, 17)} -Note the syntax for differentiating the *outputs* of a `tee()` is symmetric to that of `union()`, -showing up to the right of the variable rather than the left. - Below is the diagram rendered by [mermaid](https://mermaid-js.github.io/) showing the structure of the full flow: @@ -97,4 +90,7 @@ the structure of the full flow: This is similar to the flow for graph neighbors, but has a few more operators that make it look more complex. In particular, it includes the `union` and `tee` operators, and a cycle-forming back-edge. -There is also an auto-generated `handoff` operator that enforces the rule that a push producer and a pull consumer must be separated by a `handoff`. +There is also an auto-generated `handoff` operator that enforces the rule that a push producer and a pull consumer must be separated by a `handoff` (see the [Architecture section](../architecture/handoffs)). + +# Cyclic Dataflow +Many dataflow and workflow systems are restricted to acyclic graphs (DAGs), but Hydroflow supports cycles, as we see in this example. \ No newline at end of file diff --git a/docs/docs/hydroflow/quickstart/example_6_unreachability.mdx b/docs/docs/hydroflow/quickstart/example_6_unreachability.mdx index 941543551c9..18a1cdbcd5d 100644 --- a/docs/docs/hydroflow/quickstart/example_6_unreachability.mdx +++ b/docs/docs/hydroflow/quickstart/example_6_unreachability.mdx @@ -11,6 +11,7 @@ import { getLines, extractOutput, extractMermaid } from '../../../src/util'; > * Extending a program with additional downstream logic. > * Hydroflow's ([`difference`](../syntax/surface_ops_gen.md#difference)) operator > * A first exposure to the concepts of _strata_ and _ticks_ +> * An example of how a cyclic dataflow in one stratum executes to completion before starting the next stratum. Our next example builds on the previous by finding vertices that are _not_ reachable. To do this, we need to capture the set `all_vertices`, and use a [difference](../syntax/surface_ops_gen.md#difference) operator to form the difference between that set of vertices and `reachable_vertices`. @@ -74,16 +75,17 @@ in order, one at a time, ensuring all values are computed before moving on to the next stratum. Between strata we see a _handoff_, which logically buffers the output of the first stratum, and delineates the separation of execution between the 2 strata. +If you look carefully, you'll see two subgraphs labeled with `stratum 0`. The reason that stratum 0 was broken into subgraphs has nothing to do with +correctness, but rather the way that Hydroflow graphs are compiled and scheduled (as +discussed in the section on [In-Out Trees](../architecture/in-out_trees). We need not concern ourselves with this detail other than to look carefully at the `stratum` labels on the grey boxes in our Mermaid diagrams. + All the subgraphs labeled `stratum 0` are run first to completion, and then all the subgraphs labeled `stratum 1` are run. This captures the requirements of the `difference` operator: it has to wait for its full negative input before it can start producing output. Note how the `difference` operator has two inputs (labeled `pos` and `neg`), and only the `neg` input shows up as blocking (with the bold edge ending in a ball). -Meanwhile, note stratum 0 has a recursive loop, and stratum 1 that computes `difference`, with the blocking input. This means that Hydroflow will first run the loop of stratum 0 repeatedly until all the transitive reached vertices are found, before moving on to compute the unreached vertices. +In this Mermaid graph, note that stratum 0 has a recursive loop back through `my_join`, and `tee`s off output to the `difference` operator in stratum 1 via the handoff and the blocking `neg` input. This means that Hydroflow will first run the loop of stratum 0 repeatedly until all the transitive reached vertices are passed to the handoff (a [fixpoint](../concepts/cyclic_flows)), before moving on to compute the unreached vertices via stratum 1. -After all strata are run, Hydroflow returns to the stratum 0; this begins the next _tick_. This doesn't really matter for this example, but it is important for long-running Hydroflow services that accept input from the outside world. More on this topic in the chapter on [time](../concepts/life_and_times.md). +After all strata are run, Hydroflow returns to stratum 0; this begins the next _tick_. This doesn't really matter for this example, but it is important for long-running Hydroflow services that accept input from the outside world. More on this topic in the chapter on [time](../concepts/life_and_times.md). -If you look carefully, you'll see two subgraphs labeled with `stratum 0`. The reason that stratum 0 was broken into subgraphs has nothing to do with -correctness, but rather the way that Hydroflow graphs are compiled and scheduled, as -discussed in the chapter on [Architecture](../architecture/index.md). diff --git a/docs/docs/hydroflow/quickstart/example_7_echo_server.mdx b/docs/docs/hydroflow/quickstart/example_7_echo_server.mdx index 4f100b22ef2..48ef12b3e31 100644 --- a/docs/docs/hydroflow/quickstart/example_7_echo_server.mdx +++ b/docs/docs/hydroflow/quickstart/example_7_echo_server.mdx @@ -31,23 +31,9 @@ cargo generate hydro-project/hydroflow-template ``` Then change directory into the resulting project. +The Hydroflow template project provides *this example* as its default, so there's no code for us to change. The `README.md` for the template project is a good place to start. It contains a brief overview of the project structure, and how to build and run the example. Here we'll spend more time learning from the code. -## Hydroflow project structure -The Hydroflow template project auto-generates this example for us. If you prefer, you can find the source in the `examples/echo_server` directory of the Hydroflow repository. - -The directory structure encouraged by the template is as follows: -```txt -project/README.md # documentation -project/Cargo.toml # package and dependency info -project/src/main.rs # main function -project/src/protocol.rs # message types exchanged between roles -project/src/helpers.rs # helper functions used by all roles -project/src/.rs # service definition for role A (e.g. server) -project/src/.rs # service definition for role B (e.g. client) -``` -In the default example, the roles we use are `Client` and `Server`, but you can imagine different roles depending on the structure of your service or application. - ### `main.rs` We start with a `main` function that parses command-line options, and invokes the appropriate role-specific service. @@ -61,7 +47,7 @@ Following that, we use Rust's [`clap`](https://docs.rs/clap/latest/clap/) (Comma This sets up 3 command-line flags: `role`, `addr`, and `server_addr`. Note how the `addr` and `server_addr` flags are made optional via wrapping in a Rust `Option`; by contrast, the `role` option is required. The `clap` crate will parse the command-line options and populate the `Opts` struct with the values. `clap` handles parsing the command line strings into the associated Rust types -- the `value_parser` attribute tells `clap` to use Hydroflow's `ipv4_resolve` helper function to parse a string like "127.0.0.1:6552" into a `SocketAddr`. -This brings us to the `main` function itself. It is prefaced by a `#[hydroflow::main]` attribute, which is a macro that sets up the tokio runtime for Hydroflow. This is necessary because Hydroflow uses the tokio runtime for asynchronous execution as a service. +This brings us to the `main` function itself. It is prefaced by a `#[hydroflow::main]` attribute, which is a macro that sets up the tokio runtime for Hydroflow. It is also an async function. This is necessary because Hydroflow uses the tokio runtime for asynchronous execution as a service. {getLines(main, 29, 40)} diff --git a/docs/docs/hydroflow/quickstart/example_8_chat_server.mdx b/docs/docs/hydroflow/quickstart/example_8_chat_server.mdx index 06d3c8c6c52..90dec2b370b 100644 --- a/docs/docs/hydroflow/quickstart/example_8_chat_server.mdx +++ b/docs/docs/hydroflow/quickstart/example_8_chat_server.mdx @@ -13,7 +13,7 @@ import { getLines } from '../../../src/util'; > * Multiple message types and the [`demux`](../syntax/surface_ops_gen.md#demux) operator. > * A broadcast pattern via the [`cross_join`](../syntax/surface_ops_gen.md#cross_join) operator. > * One-time bootstrapping pipelines -> * A "gated buffer" pattern via `cross_join` with a single-object input. +> * A "gated buffer" using [`defer_signal`](../syntax/surface_ops_gen.md#defer_signal) and [`persist`](../syntax/surface_ops_gen.md#persist) operators Our previous [echo server](./example_7_echo_server) example was admittedly simplistic. In this example, we'll build something a bit more useful: a simple chat server. We will again have two roles: a `Client` and a `Server`. `Clients` will register their presence with the `Server`, which maintains a list of clients. Each `Client` sends messages to the `Server`, which will then broadcast those messages to all other clients. @@ -53,18 +53,21 @@ To follow along, replace the contents of `src/server.rs` with the code below: {getLines(server, 1, 24)} -After a short prelude, we have the Hydroflow code near the top of `run_server()`. It begins by defining `outbound_chan` as a `union`d destination sink for network messages. Then we get to the +After a short prelude, we have the Hydroflow code near the top of `run_server()`. It begins by defining `outbound_chan` as a `union`ed destination sink for network messages. Then we get to the more interesting `inbound_chan` definition. The `inbound` channel is a source stream that will carry many -types of `Message`s. We use the [`demux`](../syntax/surface_ops_gen.md#demux) operator to partition the stream objects into three channels. The `clients` channel +types of `Message`s. +We first use a `map` operator to `unwrap` the Rust `Result` type that comes from deserializing the input +from `source_stream_serde`. +Then we use the [`demux`](../syntax/surface_ops_gen.md#demux) operator to partition the stream objects into three channels. The `clients` channel will carry the addresses of clients that have connected to the server. The `msgs` channel will carry the `ChatMsg` messages that clients send to the server. The `errs` channel will carry any other messages that clients send to the server. Note the structure of the `demux` operator: it takes a closure on -`(Message, SocketAddr)` pairs, and a variadic tuple (`var_args!`) of output channel names—in this case `clients`, `msgs`, and `errs`. The closure is basically a big +`(Message, SocketAddr)` pairs, and a variadic tuple (`var_args!`) of the output channel names—in this case `clients`, `msgs`, and `errs`. The closure is basically a big Rust pattern [`match`](https://doc.rust-lang.org/book/ch06-02-match.html), with one arm for each output channel name given in the variadic tuple. Note -that the different output channels can have different-typed messages! Note also that we destructure the incoming `Message` types into tuples of fields. (If we didn't we'd either have to write boilerplate code for each message type in every downstream pipeline, or face Rust's dreaded [refutable pattern](https://doc.rust-lang.org/book/ch18-02-refutability.html) error!) +that each output channel can have its own message type! Note also that we destructure the incoming `Message` types into component fields. (If we didn't we'd have to write boilerplate code to handle every possible `Message` type in every downstream pipeline!) The remainder of the server consists of two independent pipelines, the code to print out the flow graph, and the code to run the flow graph. To follow along, paste the following into the bottom of your `src/server.rs` file: @@ -139,9 +142,8 @@ end ### `client.rs` The chat client is not very different from the echo server client, with two new design patterns: - 1. a degenerate `source_iter` pipeline that runs once -as a "bootstrap" in the first tick - 2. the use of `cross_join` as a "gated buffer" to postpone sending messages. + 1. a `initialize` operator that runs once to "bootstrap" action in the first tick + 2. the use of `defer_signal` and `persist` as a "gated buffer" to postpone sending messages. We also include a Rust helper routine `pretty_print_msg` for formatting output. @@ -152,7 +154,7 @@ To follow along, start by replacing the contents of `src/client.rs` with the fol {getLines(client, 1, 27)} This brings us to the `run_client` function. As in `run_server` we begin by ensuring the server address -is supplied. We then have the hydroflow code starting with a standard pattern of a `union`d `outbound_chan`, +is supplied. We then have the hydroflow code starting with a standard pattern of a `union`ed `outbound_chan`, and a `demux`ed `inbound_chan`. The client handles only two inbound `Message` types: `Message::ConnectResponse` and `Message::ChatMsg`. Paste the following to the bottom of `src/client.rs`: @@ -165,15 +167,15 @@ bottom of your `src/client.rs` file. {getLines(client, 47, 64)} 1. The first pipeline is the "bootstrap" alluded to above. -It starts with `source_iter` operator that emits a single, opaque "unit" (`()`) value. This value is available when the client begins, which means +It starts with the `initialize` operator that emits a single, opaque "unit" (`()`) value. This value is emitted when the client begins, which means this pipeline runs once, immediately on startup, and generates a single `ConnectRequest` message which is sent to the server. -2. The second pipeline reads from `source_stdin` and sends messages to the server. It differs from our echo-server example in the use of a `cross_join` -with `inbound_chan[acks]`. This cross-join is similar to that of the server: it forms pairs between all messages and all servers that send a `ConnectResponse` ack. -In principle this means that the client is broadcasting each message to all servers. -In practice, however, the client establishes at most one connection to a server. Hence over time, this pipeline starts with zero `ConnectResponse`s and is sending no messages; -subsequently it receives a single `ConnectResponse` and starts sending messages. The `cross_join` is thus effectively a buffer for messages, and a "gate" on that buffer that opens -when the client receives its sole `ConnectResponse`. +2. The second pipeline reads from `source_stdin` and sends messages to the server. It differs from our echo-server example in the use of the [`defer_signal`](../syntax/surface_ops_gen.md#defer_signal) operator, which buffers up messages until a `ConnectResponse` is received. The flow assigned to the `lines` +variable takes chat messages from stdin and passes them to the `[input]` channel of the `defer_signal`. +The `defer_signal` operator buffers these messages until it gets an input on its `[signal]` channel. Then all `[input]` data buffered from previous ticks is passed along to the output, along with any data that streams in during the current tick. +In our chat example, we want messages to be sent to the server in *all subsequent ticks* after `ConnectResponse` is received! To enforce this, we need to send something on the `[signal]` channel of `defer_signal` every subsequent tick. We achieve this by interposing a `persist` between `inbound_chan[acks]` and `[signal]msg_send`. The [`persist`](../syntax/surface_ops_gen.md#persist) operator stores its input data in order across time, and replays its current contents +each tick. In this case it is storing `ConnectResponse` messages, of which we expect only one. The +`persist` op will replay this signal every tick after it is received, so the client will always send its messages to the server once connected. 3. The final pipeline simply pretty-prints the messages received from the server. @@ -275,7 +277,7 @@ May 31, 5:12:40 alice: Is there anyone home? Now start client "bob" in terminal 3, and notice how he instantly receives the backlog of Alice's messages from the server's `cross_join`. (The messages may not be printed in the same order as they were timestamped! The `cross_join` operator is not guaranteed to preserve order, nor -is the udp network. Fixing these issues requires extra client logic that we leave as an exercise to the reader.) +is the udp network. Fixing these issues requires extra client logic (perhaps using the [`sort()`](../syntax/surface_ops_gen#sort) operator) that we leave as an exercise to the reader.) ```console #shell-command-next-line cargo run -- --name "bob" --role client --server-addr 127.0.0.1:12347 diff --git a/docs/docs/hydroflow/quickstart/setup.md b/docs/docs/hydroflow/quickstart/setup.md index 08b76be74e0..f3a2c477688 100644 --- a/docs/docs/hydroflow/quickstart/setup.md +++ b/docs/docs/hydroflow/quickstart/setup.md @@ -47,9 +47,7 @@ cargo install cargo-generate ## VS Code Setup We recommend using VS Code with the `rust-analyzer` extension (and NOT the -`Rust` extension). To enable the pre-release version of `rust-analyzer` -(required by some new nightly syntax we use, at least until 2022-03-14), click -the "Switch to Pre-Release Version" button next to the uninstall button. +`Rust` extension). ## Setting up a Hydroflow Project The easiest way to get started with Hydroflow is to begin with a template project. @@ -122,9 +120,10 @@ will provide inline type and error messages, code completion, etc. To work with the repository, it's best to start with an "example", found in the [`hydroflow/examples` folder](https://github.com/hydro-project/hydroflow/tree/main/hydroflow/examples). -These examples are included via the [`hydroflow/Cargo.toml` file](https://github.com/hydro-project/hydroflow/blob/main/hydroflow/Cargo.toml), -so make sure to add your example there if you create a new one. The simplest -example is the [`echo server`](https://github.com/hydro-project/hydroflow/blob/main/hydroflow/examples/echoserver/main.rs). +The simplest example is the +['hello world'](https://github.com/hydro-project/hydroflow/blob/main/hydroflow/examples/hello_world/main.rs) example; +the simplest example with networking is the +[`echo server`](https://github.com/hydro-project/hydroflow/blob/main/hydroflow/examples/echoserver/main.rs). The Hydroflow repository is set up as a [workspace](https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html), i.e. a repo containing a bunch of separate packages, `hydroflow` is just the diff --git a/docs/docs/hydroflow/syntax/index.mdx b/docs/docs/hydroflow/syntax/index.mdx index bbaf37cc0e8..458a2800ff3 100644 --- a/docs/docs/hydroflow/syntax/index.mdx +++ b/docs/docs/hydroflow/syntax/index.mdx @@ -4,7 +4,7 @@ import exampleCode from '!!raw-loader!../../../../hydroflow/examples/example_syn # Hydroflow Surface Syntax The natural way to write a Hydroflow program is using the _Surface Syntax_ documented here. It is a chained `Iterator`-style syntax of operators built into Hydroflow that should be sufficient -for most uses. If you want lower-level access you can work with the `Core API` documented in the [Architecture](../architecture/index) section. +for most uses. If you want lower-level access you can work with the `Core API` documented in the [Architecture](../architecture/) section. In this chapter we go over the syntax piece by piece: how to [embed surface syntax in Rust](./surface_embedding) and how to specify [_flows_](./surface_flows), which consist of [_data sources_](./surface_data) flowing through [_operators_](./surface_ops_gen). diff --git a/docs/docs/hydroflow/syntax/surface_data.mdx b/docs/docs/hydroflow/syntax/surface_data.mdx index 6e6d3a18e42..66a27b4c99d 100644 --- a/docs/docs/hydroflow/syntax/surface_data.mdx +++ b/docs/docs/hydroflow/syntax/surface_data.mdx @@ -6,20 +6,27 @@ import exampleCode from '!!raw-loader!../../../../hydroflow/examples/example_syn # Data Sources and Sinks in Rust Any useful flow requires us to define sources of data, either generated computationally or received from -and outside environment via I/O. +an outside environment via I/O. -## `source_iter` -A flow can receive data from a Rust collection object via the `source_iter` operator, which takes the +## One-time Iterator Sources +A flow can receive data from a Rust collection object via the [`source_iter()`](./surface_ops_gen.md#source_iter) operator, which takes the iterable collection as an argument and passes the items down the flow. For example, here we iterate through a vector of `usize` items and push them down the flow: ```rust,ignore source_iter(vec![0, 1]) -> ... ``` -The Hello, World example above uses this construct. +The Hello, World example above uses this construct. -## `source_stream` +The [`source_file()`](./surface_ops_gen.md#source_file) and [`source_json()`](./surface_ops_gen.md#source_json) operators are similar, but read from a specified file. + +All of these operators output the contents of their collection only once, during the first tick. +To output every tick, consider feeding results into a [`persist()`](./surface_ops_gen.md#persist) +operator. + +## Streaming Sources More commonly, a flow should handle external data coming in asynchronously from a [_Tokio_ runtime](https://tokio.rs/tokio/tutorial). -One way to do this is with _channels_ that allow Rust code to send data into the Hydroflow inputs. + +One way to do this is with _channels_ that allow Rust code to send data into Hydroflow via the [`source_stream()`](./surface_ops_gen.md#source_stream) operator. The code below creates a channel for data of (Rust) type `(usize, usize)`: ```rust,ignore let (input_send, input_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); @@ -30,7 +37,7 @@ it explicitly as follows: ```rust,ignore input_send.send((0, 1)).unwrap() ``` -And in our Hydroflow syntax we can receive the data from the channel using the `source_stream` syntax and +And in our Hydroflow syntax we can receive the data from the channel using the [`source_stream()`](./surface_ops_gen.md#source_stream) operator and pass it along a flow: ```rust,ignore source_stream(input_recv) -> ... @@ -41,4 +48,19 @@ in from outside the flow: {exampleCode} -TODO: add the remaining sources. \ No newline at end of file +Sometimes we want to trigger activity based on timing, not data. To achieve this, we can use the [`source_interval()`](./surface_ops_gen.md#source_interval) operator, which takes a `Duration` `d` as an argument, and outputs a Tokio time Instant after every `d` units of time pass. + +## Destinations +As duals to our data source operators, we also have data destination operators. The dest operators you'll likely use +most often are [`dest_sink()`](./surface_ops_gen.md#dest_sink) and [`dest_file()`](./surface_ops_gen.md#dest_file). They are fairly +straightforward, so the best source for further information is the documentation you can find by following the links on the operator names above. + +## SerDe: Network Serialization and Deserialization +One of the mechanical annoyances of networked systems is the need to convert data to wire format ("serialization") and convert it back from wire format to data ("deserialization"), +also known as "SerDe". +This can be done with `map` functions, but we provide a convenience source/sink pair that does serde and networking for you. +The source side, [`source_serde()`](./surface_ops_gen.md#source_serde) generates tuples of the type `(T, SocketAddr)`, +where the first field is a deserialized item of type `T`, and the second field is the address of the sender of the item. +The dest side, [`source_serde()`](./surface_ops_gen.md#source_serde), takes in tuples of type `(T, SocketAddr)`, +where the first field is an item of type `T` to be serialized, and the second field is a destination address. + diff --git a/docs/docs/hydroflow/syntax/surface_flows.md b/docs/docs/hydroflow/syntax/surface_flows.md index 2f56d55a970..1d55b2c756d 100644 --- a/docs/docs/hydroflow/syntax/surface_flows.md +++ b/docs/docs/hydroflow/syntax/surface_flows.md @@ -23,15 +23,15 @@ data, making the program more understandable. ## Operators with Multiple Ports Some operators have more than one input _port_ that can be referenced by `->`. For example [`union`](./surface_ops_gen.md#union) -unions the contents of many flows, so it can have an abitrary number of input ports. Some operators have multiple outputs, notably [`tee`](./surface_ops_gen.md#tee), -which has an arbitrary number of outputs. +unions the contents of many flows, so it can have an abitrary number of input ports. Some operators have multiple outputs; [`tee`](./surface_ops_gen.md#tee) and [`demux`](./surface_ops_gen.md#demux) +have an arbitrary number of outputs. In the syntax, we optionally distinguish input ports via an _indexing prefix_ string -in square brackets before the name (e.g. `[0]my_union` and `[1]my_union`). Binary operators --- -those with two distinct input ports --- require indexing prefixes, and require them to be `0` and `1`. -Operators with arbitrary numbers of inputs ([`union`](./surface_ops_gen.md#union)) and outputs +in square brackets before the name (e.g. `[0]my_union` and `[1]my_union`). Most operators with a fixed number of input ports\ require specific indexing prefixes to +distinguish the inputs. For example, the inputs to [`join`](./surface_ops_gen.md#join) must be `[0]` and `[1]`; the inputs to [`difference`](./surface_ops_gen.md#difference) must be `[pos]` and `[neg]`. +Operators with an arbitrary number of inputs ([`union`](./surface_ops_gen.md#union)) and outputs ([`demux`](./surface_ops_gen.md#demux), [`tee`](./surface_ops_gen.md#tee)) -allow for arbitrary strings, which can make code and dataflow graphs more readable and understandable +allow you to choose arbitrary strings, which help you make your code and dataflow graphs more readable and understandable (e.g. `my_tee[print]` and `my_tee[continue]`). Here is an example that tees one flow into two, handles each separately, and then unions them to print out the contents in both lowercase and uppercase: diff --git a/docs/docs/hydroflow/todo.md b/docs/docs/hydroflow/todo.md index ddac14b7fed..b10714f83fd 100644 --- a/docs/docs/hydroflow/todo.md +++ b/docs/docs/hydroflow/todo.md @@ -56,21 +56,23 @@ sidebar_position: 8 - cargo generate for templating - Hydroflow program specs embedded in Rust - Tokio Channels and how to use them in Hydroflow - - Network sources and sinks (source_stream) - - Built-in serde (source_stream_serde, dest_sink_serde) + - Network sources and sinks (`source_stream`) + - Built-in serde (`source_stream_serde`, `dest_sink_serde`) - Hydroflow syntax: operators, ->, variables, indexing multi-input/output operators - running Hydroflow via `run_available` and `run_async` - Recursion via cyclic dataflow - Fixpoints and Strata - - Template structure: clap, message types - - source_stdin + - Template structure: `clap`, message types + - `source_stdin` - Messages and `demux` - broadcast pattern - - gated buffer pattern - - bootstrapping pipelines + - the `persist` operator to store and replay dataflow + - the `defer_signal` operator to gate a dataflow + - bootstrapping pipelines: `initialize` - Operators covered - cross_join + - defer_signal - demux - dest_sink_serde - difference @@ -79,12 +81,15 @@ sidebar_position: 8 - flatten - flat_map - for_each + - initialize - join - map + - persist - union - source_iter - source_stdin - source_stream - source_stream_serde - tee + - union - unique diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 1a1e5be7845..2b37bfc6e5d 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -27,6 +27,10 @@ const config = { onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'throw', + customFields: { + 'LOAD_PLAYGROUND': process.env.LOAD_PLAYGROUND || false, + }, + markdown: { mermaid: true }, diff --git a/docs/src/pages/playground.js b/docs/src/pages/playground.js index e519d3e24ac..684181f3b46 100644 --- a/docs/src/pages/playground.js +++ b/docs/src/pages/playground.js @@ -5,27 +5,37 @@ import Editor from "@monaco-editor/react"; import ExecutionEnvironment from '@docusaurus/ExecutionEnvironment'; +import siteConfig from '@generated/docusaurus.config'; + import * as wasm from "website_playground/website_playground_bg.wasm"; -import { __wbg_set_wasm, init, compile_hydroflow, compile_datalog } from "website_playground/website_playground_bg.js"; +import * as playgroundJS from "website_playground/website_playground_bg.js"; + +let compile_hydroflow = null; +let compile_datalog = null; -if (ExecutionEnvironment.canUseDOM) { - __wbg_set_wasm(wasm); -} else { - const wasmUri = require("website_playground/website_playground_bg.wasm"); - const wasmBuffer = Buffer.from(wasmUri.split(",")[1], 'base64'); - const wasm = new WebAssembly.Module(wasmBuffer); - const instance = new WebAssembly.Instance(wasm, { - "./website_playground_bg.js": require("website_playground/website_playground_bg.js") - }); - __wbg_set_wasm(instance.exports); +if (siteConfig.customFields.LOAD_PLAYGROUND === '1') { + compile_hydroflow = playgroundJS.compile_hydroflow; + compile_datalog = playgroundJS.compile_datalog; + + if (ExecutionEnvironment.canUseDOM) { + playgroundJS.__wbg_set_wasm(wasm); + } else { + const wasmUri = require("website_playground/website_playground_bg.wasm"); + const wasmBuffer = Buffer.from(wasmUri.split(",")[1], 'base64'); + const wasm = new WebAssembly.Module(wasmBuffer); + const instance = new WebAssembly.Instance(wasm, { + "./website_playground_bg.js": require("website_playground/website_playground_bg.js") + }); + playgroundJS.__wbg_set_wasm(instance.exports); + } + + playgroundJS.init(); } import mermaid from "mermaid"; import styles from "./playground.module.css"; -init(); - function MermaidGraph({ id, source }) { const [svg, setSvg] = useState({ __html: 'Loading Mermaid graph...' }); useEffect(() => { @@ -50,7 +60,7 @@ source_iter(0..10) -> for_each(|n| println!("Hello {}", n));`, // https://hydro.run/docs/hydroflow/quickstart/example_2_simple source_iter(0..10) -> map(|n| n * n) - -> filter(|&n| n > 10) + -> filter(|n| *n > 10) -> map(|n| (n..=n+1)) -> flatten() -> for_each(|n| println!("Howdy {}", n));`, @@ -179,6 +189,10 @@ export function EditorDemo({ compileFn, examples, mermaidId }) { const [showingMermaid, setShowingMermaid] = useState(true); const [editorAndMonaco, setEditorAndMonaco] = useState(null); + if (siteConfig.customFields.LOAD_PLAYGROUND !== '1') { + return
Please set LOAD_PLAYGROUND environment variable to 1 to enable the playground.
; + } + const { output, diagnostics } = (compileFn)(program); const numberOfLines = program.split("\n").length; diff --git a/docs/wasm-plugin.js b/docs/wasm-plugin.js index a3dd8c8b945..16181e5850e 100644 --- a/docs/wasm-plugin.js +++ b/docs/wasm-plugin.js @@ -14,7 +14,15 @@ module.exports = function (context, options) { type: "asset/inline", }, ] : [] - } + }, + ...(process.env.LOAD_PLAYGROUND !== "1" ? { + resolve: { + alias: { + "website_playground/website_playground_bg.wasm": false, + "website_playground/website_playground_bg.js": false + } + } + } : {}) }; }, }; diff --git a/hydro_cli/CHANGELOG.md b/hydro_cli/CHANGELOG.md index 387ff162005..750b836ec7f 100644 --- a/hydro_cli/CHANGELOG.md +++ b/hydro_cli/CHANGELOG.md @@ -1,7 +1,69 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## 0.4.0 (2023-08-15) + +### Chore + + - fix lints for latest nightly + +### Commit Statistics + + + + - 1 commit contributed to the release over the course of 27 calendar days. + - 42 days passed between releases. + - 1 commit was understood as [conventional](https://www.conventionalcommits.org). + - 1 unique issue was worked on: [#844](https://github.com/hydro-project/hydroflow/issues/844) + +### Commit Details + + + +
view details + + * **[#844](https://github.com/hydro-project/hydroflow/issues/844)** + - Fix lints for latest nightly ([`949db02`](https://github.com/hydro-project/hydroflow/commit/949db02e9fa9878e1a7176c180d6f44c5cddf052)) +
+ +## 0.3.0 (2023-07-04) + + + +Unchanged from previous release. + +### Chore + + - mark hydro_cli as unchanged for 0.3 release + +### Commit Statistics + + + + - 2 commits contributed to the release. + - 33 days passed between releases. + - 1 commit was understood as [conventional](https://www.conventionalcommits.org). + - 0 issues like '(#ID)' were seen in commit messages + +### Commit Details + + + +
view details + + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) + - Mark hydro_cli as unchanged for 0.3 release ([`4c2cf81`](https://github.com/hydro-project/hydroflow/commit/4c2cf81411835529b5d7daa35717834e46e28b9b)) +
## v0.2.0 (2023-05-31) + + ### Chore - manually bump versions for v0.2.0 release @@ -14,7 +76,7 @@ - - 2 commits contributed to the release. + - 3 commits contributed to the release. - 1 day passed between releases. - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - 1 unique issue was worked on: [#723](https://github.com/hydro-project/hydroflow/issues/723) @@ -28,6 +90,7 @@ * **[#723](https://github.com/hydro-project/hydroflow/issues/723)** - Add more detailed Hydro Deploy docs and rename `ConnectedBidi` => `ConnectedDirect` ([`8b2c9f0`](https://github.com/hydro-project/hydroflow/commit/8b2c9f09b1f423ac6d562c29d4ea587578f1c98a)) * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a)) @@ -41,6 +104,8 @@ + + ### Chore @@ -73,7 +138,7 @@ - initialize hydro_cli/CHANGELOG.md - publish hydro_cli - Will bump versions for python deploy. + Will bump versions for python deploy. Update build-cli.yml to publish on hydro_cli release ### Style diff --git a/hydro_cli/Cargo.toml b/hydro_cli/Cargo.toml index d1c173a2051..125aed7f3b0 100644 --- a/hydro_cli/Cargo.toml +++ b/hydro_cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydro_cli" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydro_cli/" @@ -37,8 +37,8 @@ bytes = "1.1.0" nanoid = "0.4.0" ctrlc = "3.2.5" nix = "0.26.2" -hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.2.0" } -indicatif = "0.17.3" +hydroflow_cli_integration = { path = "../hydroflow_cli_integration", version = "^0.3.0" } +indicatif = "0.17.6" cargo_metadata = "0.15.4" [dev-dependencies] diff --git a/hydro_cli/hydro/_core.pyi b/hydro_cli/hydro/_core.pyi index 7c0f491c9c7..5f561ce846a 100644 --- a/hydro_cli/hydro/_core.pyi +++ b/hydro_cli/hydro/_core.pyi @@ -20,7 +20,7 @@ class Deployment(object): def CustomService(self, on: "Host", external_ports: List[int]) -> "CustomService": ... - def HydroflowCrate(self, src: str, on: "Host", example: Optional[str] = None, profile: Optional[str] = None, features: Optional[List[str]] = None, args: Optional[List[str]] = None, display_id: Optional[str] = None, external_ports: Optional[List[int]] = None) -> "HydroflowCrate": ... + def HydroflowCrate(self, src: str, on: "Host", bin: Optional[str] = None, example: Optional[str] = None, profile: Optional[str] = None, features: Optional[List[str]] = None, args: Optional[List[str]] = None, display_id: Optional[str] = None, external_ports: Optional[List[int]] = None) -> "HydroflowCrate": ... async def deploy(self): ... diff --git a/hydro_cli/src/core/custom_service.rs b/hydro_cli/src/core/custom_service.rs index 98cf030ca25..a55eecb4d70 100644 --- a/hydro_cli/src/core/custom_service.rs +++ b/hydro_cli/src/core/custom_service.rs @@ -65,7 +65,9 @@ impl Service for CustomService { Ok(()) } - async fn start(&mut self) {} + async fn start(&mut self) -> Result<()> { + Ok(()) + } async fn stop(&mut self) -> Result<()> { Ok(()) diff --git a/hydro_cli/src/core/deployment.rs b/hydro_cli/src/core/deployment.rs index 331470c7031..11390a27467 100644 --- a/hydro_cli/src/core/deployment.rs +++ b/hydro_cli/src/core/deployment.rs @@ -17,7 +17,7 @@ pub struct Deployment { impl Deployment { pub async fn deploy(&mut self) -> Result<()> { - progress::ProgressTracker::with_group("deploy", || async { + progress::ProgressTracker::with_group("deploy", None, || async { let mut resource_batch = super::ResourceBatch::new(); let active_services = self .services @@ -41,7 +41,7 @@ impl Deployment { } let result = Arc::new( - progress::ProgressTracker::with_group("provision", || async { + progress::ProgressTracker::with_group("provision", None, || async { resource_batch .provision(&mut self.resource_pool, self.last_resource_result.clone()) .await @@ -50,7 +50,7 @@ impl Deployment { ); self.last_resource_result = Some(result.clone()); - progress::ProgressTracker::with_group("provision", || { + progress::ProgressTracker::with_group("provision", None, || { let hosts_provisioned = self.hosts .iter_mut() @@ -61,7 +61,7 @@ impl Deployment { }) .await; - progress::ProgressTracker::with_group("deploy", || { + progress::ProgressTracker::with_group("deploy", None, || { let services_future = self.services .iter_mut() @@ -79,7 +79,7 @@ impl Deployment { }) .await; - progress::ProgressTracker::with_group("ready", || { + progress::ProgressTracker::with_group("ready", None, || { let all_services_ready = self.services .iter() @@ -97,7 +97,7 @@ impl Deployment { .await } - pub async fn start(&mut self) { + pub async fn start(&mut self) -> Result<()> { let active_services = self .services .iter() @@ -110,10 +110,12 @@ impl Deployment { self.services .iter() .map(|service: &Weak>| async { - service.upgrade().unwrap().write().await.start().await; + service.upgrade().unwrap().write().await.start().await?; + Ok(()) as Result<()> }); - futures::future::join_all(all_services_start).await; + futures::future::try_join_all(all_services_start).await?; + Ok(()) } pub fn add_host T>( diff --git a/hydro_cli/src/core/gcp.rs b/hydro_cli/src/core/gcp.rs index aabbc066d9e..1dd250f838b 100644 --- a/hydro_cli/src/core/gcp.rs +++ b/hydro_cli/src/core/gcp.rs @@ -92,35 +92,44 @@ impl LaunchedSSHHost for LaunchedComputeEngine { 22, ); - let res = ProgressTracker::leaf( - format!( - "connecting to host @ {}", - self.external_ip.as_ref().unwrap() - ), - async_retry( - &|| async { - let mut config = SessionConfiguration::new(); - config.set_compress(true); - - let mut session = - AsyncSession::::connect(target_addr, Some(config)).await?; - - session.handshake().await?; - - session - .userauth_pubkey_file( - self.user.as_str(), - None, - self.ssh_key_path().as_path(), - None, - ) - .await?; - - Ok(session) - }, - 10, - Duration::from_secs(1), - ), + let mut attempt_count = 0; + + let res = async_retry( + || { + attempt_count += 1; + ProgressTracker::leaf( + format!( + "connecting to host @ {} (attempt: {})", + self.external_ip.as_ref().unwrap(), + attempt_count + ), + async { + let mut config = SessionConfiguration::new(); + config.set_compress(true); + + let mut session = + AsyncSession::::connect(target_addr, Some(config)).await?; + + tokio::time::timeout(Duration::from_secs(15), async move { + session.handshake().await?; + + session + .userauth_pubkey_file( + self.user.as_str(), + None, + self.ssh_key_path().as_path(), + None, + ) + .await?; + + Ok(session) + }) + .await? + }, + ) + }, + 10, + Duration::from_secs(1), ) .await?; diff --git a/hydro_cli/src/core/hydroflow_crate/build.rs b/hydro_cli/src/core/hydroflow_crate/build.rs index 5833299afdc..e837566c1a3 100644 --- a/hydro_cli/src/core/hydroflow_crate/build.rs +++ b/hydro_cli/src/core/hydroflow_crate/build.rs @@ -16,6 +16,7 @@ type CacheKey = ( PathBuf, Option, Option, + Option, HostTargetType, Option>, ); @@ -27,6 +28,7 @@ static BUILDS: Lazy>>>> = pub async fn build_crate( src: PathBuf, + bin: Option, example: Option, profile: Option, target_type: HostTargetType, @@ -34,6 +36,7 @@ pub async fn build_crate( ) -> Result { let key = ( src.clone(), + bin.clone(), example.clone(), profile.clone(), target_type, @@ -56,6 +59,10 @@ pub async fn build_crate( profile.unwrap_or("release".to_string()), ]); + if let Some(bin) = bin.as_ref() { + command.args(["--bin", bin]); + } + if let Some(example) = example.as_ref() { command.args(["--example", example]); } diff --git a/hydro_cli/src/core/hydroflow_crate/mod.rs b/hydro_cli/src/core/hydroflow_crate/mod.rs index 7b379c4ffaf..19f1d6f034e 100644 --- a/hydro_cli/src/core/hydroflow_crate/mod.rs +++ b/hydro_cli/src/core/hydroflow_crate/mod.rs @@ -24,6 +24,7 @@ pub struct HydroflowCrate { id: usize, src: PathBuf, on: Arc>, + bin: Option, example: Option, profile: Option, features: Option>, @@ -55,6 +56,7 @@ impl HydroflowCrate { id: usize, src: PathBuf, on: Arc>, + bin: Option, example: Option, profile: Option, features: Option>, @@ -65,6 +67,7 @@ impl HydroflowCrate { Self { id, src, + bin, on, example, profile, @@ -162,6 +165,7 @@ impl HydroflowCrate { fn build(&mut self) -> JoinHandle> { let src_cloned = self.src.canonicalize().unwrap(); + let bin_cloned = self.bin.clone(); let example_cloned = self.example.clone(); let features_cloned = self.features.clone(); let host = self.on.clone(); @@ -170,6 +174,7 @@ impl HydroflowCrate { tokio::task::spawn(build_crate( src_cloned, + bin_cloned, example_cloned, profile_cloned, target_type, @@ -213,6 +218,7 @@ impl Service for HydroflowCrate { .display_id .clone() .unwrap_or_else(|| format!("service/{}", self.id)), + None, || async { let mut host_write = self.on.write().await; let launched = host_write.provision(resource_result); @@ -232,6 +238,7 @@ impl Service for HydroflowCrate { .display_id .clone() .unwrap_or_else(|| format!("service/{}", self.id)), + None, || async { let launched_host = self.launched_host.as_ref().unwrap(); @@ -256,7 +263,7 @@ impl Service for HydroflowCrate { let formatted_bind_config = serde_json::to_string(&bind_config).unwrap(); // request stdout before sending config so we don't miss the "ready" response - let stdout_receiver = binary.write().await.stdout().await; + let stdout_receiver = binary.write().await.cli_stdout().await; binary .write() @@ -286,9 +293,9 @@ impl Service for HydroflowCrate { .await } - async fn start(&mut self) { + async fn start(&mut self) -> Result<()> { if self.started { - return; + return Ok(()); } let mut sink_ports = HashMap::new(); @@ -298,6 +305,15 @@ impl Service for HydroflowCrate { let formatted_defns = serde_json::to_string(&sink_ports).unwrap(); + let stdout_receiver = self + .launched_binary + .as_mut() + .unwrap() + .write() + .await + .cli_stdout() + .await; + self.launched_binary .as_mut() .unwrap() @@ -309,7 +325,17 @@ impl Service for HydroflowCrate { .await .unwrap(); + let start_ack_line = ProgressTracker::leaf( + "waiting for ack start".to_string(), + tokio::time::timeout(Duration::from_secs(60), stdout_receiver.recv()), + ) + .await??; + if !start_ack_line.starts_with("ack start") { + bail!("expected ack start"); + } + self.started = true; + Ok(()) } async fn stop(&mut self) -> Result<()> { diff --git a/hydro_cli/src/core/hydroflow_crate/ports.rs b/hydro_cli/src/core/hydroflow_crate/ports.rs index a937570a81c..a025933455a 100644 --- a/hydro_cli/src/core/hydroflow_crate/ports.rs +++ b/hydro_cli/src/core/hydroflow_crate/ports.rs @@ -544,7 +544,6 @@ impl ServerConfig { ServerConfig::TaggedUnwrap(underlying) => { let loaded = underlying.load_instantiated(select).await; - dbg!(&loaded); if let ServerPort::Tagged(underlying, _) = loaded { *underlying } else { diff --git a/hydro_cli/src/core/localhost.rs b/hydro_cli/src/core/localhost.rs index fa565874fe7..512d2b3211c 100644 --- a/hydro_cli/src/core/localhost.rs +++ b/hydro_cli/src/core/localhost.rs @@ -22,6 +22,7 @@ use super::{ struct LaunchedLocalhostBinary { child: RwLock, stdin_sender: Sender, + stdout_cli_receivers: Arc>>>, stdout_receivers: Arc>>>, stderr_receivers: Arc>>>, } @@ -32,6 +33,13 @@ impl LaunchedBinary for LaunchedLocalhostBinary { self.stdin_sender.clone() } + async fn cli_stdout(&self) -> Receiver { + let mut receivers = self.stdout_cli_receivers.write().await; + let (sender, receiver) = async_channel::unbounded::(); + receivers.push(sender); + receiver + } + async fn stdout(&self) -> Receiver { let mut receivers = self.stdout_receivers.write().await; let (sender, receiver) = async_channel::unbounded::(); @@ -72,14 +80,34 @@ struct LaunchedLocalhost {} pub fn create_broadcast( source: T, default: impl Fn(String) + Send + 'static, -) -> Arc>>> { +) -> ( + Arc>>>, + Arc>>>, +) { + let cli_receivers = Arc::new(RwLock::new(Vec::>::new())); let receivers = Arc::new(RwLock::new(Vec::>::new())); + + let weak_cli_receivers = Arc::downgrade(&cli_receivers); let weak_receivers = Arc::downgrade(&receivers); tokio::spawn(async move { let mut lines = BufReader::new(source).lines(); - while let Some(Result::Ok(line)) = lines.next().await { + 'line_loop: while let Some(Result::Ok(line)) = lines.next().await { + if let Some(cli_receivers) = weak_cli_receivers.upgrade() { + let mut cli_receivers = cli_receivers.write().await; + let mut successful_send = false; + for r in cli_receivers.iter() { + successful_send |= r.send(line.clone()).await.is_ok(); + } + + cli_receivers.retain(|r| !r.is_closed()); + + if successful_send { + continue 'line_loop; + } + } + if let Some(receivers) = weak_receivers.upgrade() { let mut receivers = receivers.write().await; let mut successful_send = false; @@ -98,7 +126,7 @@ pub fn create_broadcast( } }); - receivers + (cli_receivers, receivers) } #[async_trait] @@ -158,16 +186,18 @@ impl LaunchedHost for LaunchedLocalhost { }); let id_clone = id.clone(); - let stdout_receivers = create_broadcast(child.stdout.take().unwrap(), move |s| { - println!("[{id_clone}] {s}") - }); - let stderr_receivers = create_broadcast(child.stderr.take().unwrap(), move |s| { + let (stdout_cli_receivers, stdout_receivers) = + create_broadcast(child.stdout.take().unwrap(), move |s| { + println!("[{id_clone}] {s}") + }); + let (_, stderr_receivers) = create_broadcast(child.stderr.take().unwrap(), move |s| { eprintln!("[{id}] {s}") }); Ok(Arc::new(RwLock::new(LaunchedLocalhostBinary { child: RwLock::new(child), stdin_sender, + stdout_cli_receivers, stdout_receivers, stderr_receivers, }))) diff --git a/hydro_cli/src/core/mod.rs b/hydro_cli/src/core/mod.rs index c28d1f0fe43..66c82107e7c 100644 --- a/hydro_cli/src/core/mod.rs +++ b/hydro_cli/src/core/mod.rs @@ -69,6 +69,7 @@ pub struct ResourceResult { #[async_trait] pub trait LaunchedBinary: Send + Sync { async fn stdin(&self) -> Sender; + async fn cli_stdout(&self) -> Receiver; async fn stdout(&self) -> Receiver; async fn stderr(&self) -> Receiver; @@ -186,7 +187,7 @@ pub trait Service: Send + Sync { async fn ready(&mut self) -> Result<()>; /// Starts the service by having it connect to other services and start computations. - async fn start(&mut self); + async fn start(&mut self) -> Result<()>; /// Stops the service by having it disconnect from other services and stop computations. async fn stop(&mut self) -> Result<()>; diff --git a/hydro_cli/src/core/progress.rs b/hydro_cli/src/core/progress.rs index 05a7d0d8df3..153f4d1b417 100644 --- a/hydro_cli/src/core/progress.rs +++ b/hydro_cli/src/core/progress.rs @@ -12,7 +12,6 @@ tokio::task_local! { #[derive(Clone, PartialEq, Eq, Debug)] pub enum LeafStatus { - Queued, Started, Finished, } @@ -20,7 +19,12 @@ pub enum LeafStatus { #[derive(Debug)] pub enum BarTree { Root(Vec), - Group(String, Arc, Vec), + Group( + String, + Arc, + Vec, + Option, + ), Leaf(String, Arc, LeafStatus), Finished, } @@ -29,26 +33,22 @@ impl BarTree { fn get_pb(&self) -> Option<&Arc> { match self { BarTree::Root(_) => None, - BarTree::Group(_, pb, _) | BarTree::Leaf(_, pb, _) => Some(pb), + BarTree::Group(_, pb, _, _) | BarTree::Leaf(_, pb, _) => Some(pb), BarTree::Finished => None, } } fn status(&self) -> LeafStatus { match self { - BarTree::Root(children) | BarTree::Group(_, _, children) => { - if children - .iter() - .all(|child| child.status() == LeafStatus::Finished) + BarTree::Root(children) | BarTree::Group(_, _, children, _) => { + if children.len() > 0 + && children + .iter() + .all(|child| child.status() == LeafStatus::Finished) { LeafStatus::Finished - } else if children - .iter() - .any(|child| child.status() == LeafStatus::Started) - { - LeafStatus::Started } else { - LeafStatus::Queued + LeafStatus::Started } } BarTree::Leaf(_, _, status) => status.clone(), @@ -63,7 +63,7 @@ impl BarTree { child.refresh_prefix(cur_path); } } - BarTree::Group(name, pb, children) => { + BarTree::Group(name, pb, children, anticipated_total) => { let mut path_with_group = cur_path.to_vec(); path_with_group.push(name.clone()); @@ -75,18 +75,26 @@ impl BarTree { .iter() .filter(|child| child.status() == LeafStatus::Started) .count(); - let queued_count = children - .iter() - .filter(|child| child.status() == LeafStatus::Queued) - .count(); - - pb.set_prefix(format!( - "{} ({}/{}/{})", - path_with_group.join(" / "), - finished_count, - started_count, - queued_count - )); + let queued_count = + anticipated_total.map(|total| total - finished_count - started_count); + + match queued_count { + Some(queued_count) => { + pb.set_prefix(format!( + "{} ({}/{}/{})", + path_with_group.join(" / "), + finished_count, + started_count, + queued_count + )); + } + None => pb.set_prefix(format!( + "{} ({}/{})", + path_with_group.join(" / "), + finished_count, + started_count + )), + } for child in children { child.refresh_prefix(&path_with_group); } @@ -106,7 +114,7 @@ impl BarTree { } match self { - BarTree::Root(children) | BarTree::Group(_, _, children) => { + BarTree::Root(children) | BarTree::Group(_, _, children, _) => { children[path[0]].find_node(&path[1..]) } _ => panic!(), @@ -134,12 +142,13 @@ impl ProgressTracker { under_path: Vec, name: String, group: bool, + anticipated_total: Option, progress: bool, ) -> (usize, Arc) { let surrounding = self.tree.find_node(&under_path); let (surrounding_children, surrounding_pb) = match surrounding { BarTree::Root(children) => (children, None), - BarTree::Group(_, pb, children) => (children, Some(pb)), + BarTree::Group(_, pb, children, _) => (children, Some(pb)), _ => panic!(), }; @@ -161,7 +170,7 @@ impl ProgressTracker { let pb = Arc::new(created_bar); if group { - surrounding_children.push(BarTree::Group(name, pb.clone(), vec![])); + surrounding_children.push(BarTree::Group(name, pb.clone(), vec![], anticipated_total)); } else { surrounding_children.push(BarTree::Leaf(name, pb.clone(), LeafStatus::Started)); } @@ -189,7 +198,7 @@ impl ProgressTracker { pub fn end_task(&mut self, path: Vec) { match self.tree.find_node(&path[0..path.len() - 1]) { - BarTree::Root(children) | BarTree::Group(_, _, children) => { + BarTree::Root(children) | BarTree::Group(_, _, children, _) => { let removed = children[*path.last().unwrap()].get_pb().unwrap().clone(); children[*path.last().unwrap()] = BarTree::Finished; self.multi_progress.remove(&removed); @@ -213,11 +222,15 @@ impl ProgressTracker { .get_or_init(|| Mutex::new(ProgressTracker::new())) .lock() .unwrap(); - progress_bar.multi_progress.println(msg).unwrap(); + + if progress_bar.multi_progress.println(msg).is_err() { + println!("{}", msg); + } } pub fn with_group<'a, T, F: Future>( name: &str, + anticipated_total: Option, f: impl FnOnce() -> F + 'a, ) -> impl Future + 'a { let mut group = CURRENT_GROUP @@ -229,7 +242,13 @@ impl ProgressTracker { .get_or_init(|| Mutex::new(ProgressTracker::new())) .lock() .unwrap(); - progress_bar.start_task(group.clone(), name.to_string(), true, false) + progress_bar.start_task( + group.clone(), + name.to_string(), + true, + anticipated_total, + false, + ) }; group.push(group_i); @@ -255,7 +274,7 @@ impl ProgressTracker { .get_or_init(|| Mutex::new(ProgressTracker::new())) .lock() .unwrap(); - progress_bar.start_task(group.clone(), name, false, false) + progress_bar.start_task(group.clone(), name, false, None, false) }; group.push(leaf_i); @@ -284,7 +303,7 @@ impl ProgressTracker { .get_or_init(|| Mutex::new(ProgressTracker::new())) .lock() .unwrap(); - progress_bar.start_task(group.clone(), name, false, true) + progress_bar.start_task(group.clone(), name, false, None, true) }; group.push(leaf_i); diff --git a/hydro_cli/src/core/ssh.rs b/hydro_cli/src/core/ssh.rs index bde3fa06d60..b5cef3f255d 100644 --- a/hydro_cli/src/core/ssh.rs +++ b/hydro_cli/src/core/ssh.rs @@ -26,6 +26,7 @@ struct LaunchedSSHBinary { channel: AsyncChannel, stdin_sender: Sender, stdout_receivers: Arc>>>, + stdout_cli_receivers: Arc>>>, stderr_receivers: Arc>>>, } @@ -35,6 +36,13 @@ impl LaunchedBinary for LaunchedSSHBinary { self.stdin_sender.clone() } + async fn cli_stdout(&self) -> Receiver { + let mut receivers = self.stdout_cli_receivers.write().await; + let (sender, receiver) = async_channel::unbounded::(); + receivers.push(sender); + receiver + } + async fn stdout(&self) -> Receiver { let mut receivers = self.stdout_receivers.write().await; let (sender, receiver) = async_channel::unbounded::(); @@ -207,15 +215,17 @@ impl LaunchedHost for T { }); let id_clone = id.clone(); - let stdout_receivers = + let (stdout_cli_receivers, stdout_receivers) = create_broadcast(channel.stream(0), move |s| println!("[{id_clone}] {s}")); - let stderr_receivers = create_broadcast(channel.stderr(), move |s| eprintln!("[{id}] {s}")); + let (_, stderr_receivers) = + create_broadcast(channel.stderr(), move |s| eprintln!("[{id}] {s}")); Ok(Arc::new(RwLock::new(LaunchedSSHBinary { _resource_result: self.resource_result().clone(), session: Some(session), channel, stdin_sender, + stdout_cli_receivers, stdout_receivers, stderr_receivers, }))) @@ -243,7 +253,7 @@ impl LaunchedHost for T { // if anyone wants to connect to this forwarded port } - println!("[hydro] closing forwarded port"); + ProgressTracker::println("[hydro] closing forwarded port"); }); Ok(local_addr) diff --git a/hydro_cli/src/core/terraform.rs b/hydro_cli/src/core/terraform.rs index ad44be39074..f89af014df6 100644 --- a/hydro_cli/src/core/terraform.rs +++ b/hydro_cli/src/core/terraform.rs @@ -46,6 +46,7 @@ impl TerraformPool { let spawned_child = apply_command .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .spawn() .context("Failed to spawn `terraform`. Is it installed?")?; @@ -98,6 +99,11 @@ impl Default for TerraformBatch { impl TerraformBatch { pub async fn provision(self, pool: &mut TerraformPool) -> Result { + // Hack to quiet false-positive `clippy::needless_pass_by_ref_mut` on latest nightlies. + // TODO(mingwei): Remove this when it is no longer needed (current date 2023-08-30). + // https://github.com/rust-lang/rust-clippy/issues/11380 + let pool = std::convert::identity(pool); + if self.terraform.required_providers.is_empty() && self.resource.is_empty() && self.data.is_empty() @@ -109,7 +115,7 @@ impl TerraformBatch { }); } - ProgressTracker::with_group("terraform", || async { + ProgressTracker::with_group("terraform", None, || async { let dothydro_folder = std::env::current_dir().unwrap().join(".hydro"); std::fs::create_dir_all(&dothydro_folder).unwrap(); let deployment_folder = tempfile::tempdir_in(dothydro_folder).unwrap(); @@ -135,9 +141,11 @@ impl TerraformBatch { let (apply_id, apply) = pool.create_apply(deployment_folder)?; - let output = ProgressTracker::with_group("apply", || async { - apply.write().await.output().await - }) + let output = ProgressTracker::with_group( + "apply", + Some(self.resource.values().map(|r| r.len()).sum()), + || async { apply.write().await.output().await }, + ) .await; pool.drop_apply(apply_id); output @@ -221,13 +229,22 @@ impl TerraformApply { async fn output(&mut self) -> Result { let (_, child) = self.child.as_ref().unwrap().clone(); let mut stdout = child.write().unwrap().stdout.take().unwrap(); + let stderr = child.write().unwrap().stderr.take().unwrap(); let status = tokio::task::spawn_blocking(move || { // it is okay for this thread to keep running even if the future is cancelled child.write().unwrap().wait().unwrap() }); - display_apply_outputs(&mut stdout).await; + let display_apply = display_apply_outputs(&mut stdout); + let stderr_loop = tokio::task::spawn_blocking(move || { + let mut lines = BufReader::new(stderr).lines(); + while let Some(Ok(line)) = lines.next() { + ProgressTracker::println(&format!("[terraform] {}", line)); + } + }); + + let _ = futures::join!(display_apply, stderr_loop); let status = status.await; diff --git a/hydro_cli/src/core/util.rs b/hydro_cli/src/core/util.rs index d397a3d52c8..5668212bb57 100644 --- a/hydro_cli/src/core/util.rs +++ b/hydro_cli/src/core/util.rs @@ -4,7 +4,7 @@ use anyhow::Result; use futures::Future; pub async fn async_retry>>( - thunk: impl Fn() -> F, + mut thunk: impl FnMut() -> F, count: usize, delay: Duration, ) -> Result { diff --git a/hydro_cli/src/lib.rs b/hydro_cli/src/lib.rs index 30290c988b3..2dcb6b9a8fd 100644 --- a/hydro_cli/src/lib.rs +++ b/hydro_cli/src/lib.rs @@ -236,6 +236,7 @@ impl Deployment { py: Python<'_>, src: String, on: &Host, + bin: Option, example: Option, profile: Option, features: Option>, @@ -248,6 +249,7 @@ impl Deployment { id, src.into(), on.underlying.clone(), + bin, example, profile, features, @@ -286,7 +288,11 @@ impl Deployment { let underlying = self.underlying.clone(); let py_none = py.None(); interruptible_future_to_py(py, async move { - underlying.write().await.start().await; + underlying.write().await.start().await.map_err(|e| { + AnyhowError::new_err(AnyhowWrapper { + underlying: Arc::new(RwLock::new(Some(e))), + }) + })?; Ok(py_none) }) } @@ -576,7 +582,7 @@ impl HydroflowCratePort { .into_py(py)) } - fn send_to(&mut self, to: &mut HydroflowSink) { + fn send_to(&mut self, to: &HydroflowSink) { self.underlying .try_write() .unwrap() @@ -621,7 +627,7 @@ struct TaggedSource { #[pymethods] impl TaggedSource { - fn send_to(&mut self, to: &mut HydroflowSink) { + fn send_to(&mut self, to: &HydroflowSink) { self.underlying .try_write() .unwrap() @@ -648,7 +654,7 @@ struct HydroflowNull { #[pymethods] impl HydroflowNull { - fn send_to(&mut self, to: &mut HydroflowSink) { + fn send_to(&mut self, to: &HydroflowSink) { self.underlying .try_write() .unwrap() diff --git a/hydro_cli_examples/Cargo.toml b/hydro_cli_examples/Cargo.toml index 828f206ce28..8f9964914bb 100644 --- a/hydro_cli_examples/Cargo.toml +++ b/hydro_cli_examples/Cargo.toml @@ -31,18 +31,6 @@ name = "dedalus_2pc_coordinator" [[example]] name = "dedalus_2pc_participant" -[[example]] -name = "topolotree" - -[[example]] -name = "topolotree_latency_measure" - -[[example]] -name = "pn_counter" - -[[example]] -name = "pn_counter_delta" - [[example]] name = "ws_chat_server" diff --git a/hydro_cli_examples/examples/topolotree/main.rs b/hydro_cli_examples/examples/topolotree/main.rs deleted file mode 100644 index f842cbdc1a4..00000000000 --- a/hydro_cli_examples/examples/topolotree/main.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::hash::{Hash, Hasher}; - -use hydroflow::serde::{Deserialize, Serialize}; -use hydroflow::util::cli::{ConnectedDirect, ConnectedSink, ConnectedSource}; -use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; -use hydroflow::{hydroflow_syntax, tokio}; - -#[derive(Serialize, Deserialize, Clone, Debug)] -struct IncrementRequest { - tweet_id: u64, - likes: i32, -} - -#[derive(Serialize, Deserialize, Default, Copy, Clone, Debug)] -struct TimestampedValue { - pub value: T, - pub timestamp: u32, -} - -impl TimestampedValue { - pub fn merge_from(&mut self, other: TimestampedValue) -> bool { - if other.timestamp > self.timestamp { - self.value = other.value; - self.timestamp = other.timestamp; - true - } else { - false - } - } - - pub fn update(&mut self, updater: impl Fn(&T) -> T) { - self.value = updater(&self.value); - self.timestamp += 1; - } -} - -impl PartialEq for TimestampedValue { - fn eq(&self, other: &Self) -> bool { - self.timestamp == other.timestamp - } -} - -impl Eq for TimestampedValue {} - -impl Hash for TimestampedValue { - fn hash(&self, state: &mut H) { - self.timestamp.hash(state); - } -} - -#[hydroflow::main] -async fn main() { - let mut ports = hydroflow::util::cli::init().await; - let increment_requests = ports - .port("increment_requests") - .connect::() - .await - .into_source(); - - let query_responses = ports - .port("query_responses") - .connect::() - .await - .into_sink(); - - let to_parent = ports - .port("to_parent") - .connect::() - .await - .into_sink(); - - let from_parent = ports - .port("from_parent") - .connect::() - .await - .into_source(); - - let to_left = ports - .port("to_left") - .connect::() - .await - .into_sink(); - - let from_left = ports - .port("from_left") - .connect::() - .await - .into_source(); - - let to_right = ports - .port("to_right") - .connect::() - .await - .into_sink(); - - let from_right = ports - .port("from_right") - .connect::() - .await - .into_source(); - - let f1 = async move { - #[cfg(target_os = "linux")] - loop { - let x = procinfo::pid::stat_self().unwrap(); - let bytes = x.rss * 1024 * 4; - println!("memory,{}", bytes); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } - }; - - type UpdateType = (u64, i32); - - let df = hydroflow_syntax! { - from_parent = source_stream(from_parent) - -> map(|x| deserialize_from_bytes::)>>(x.unwrap()).unwrap()) - -> fold::<'static>( - (HashMap::>::new(), HashSet::new(), 0), - |(mut prev, mut modified_tweets, prev_tick), req: Vec<(u64, TimestampedValue)>| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - for (k, v) in req { - let updated = if let Some(e) = prev.get_mut(&k) { - e.merge_from(v) - } else { - prev.insert(k, v); - true - }; - - if updated { - modified_tweets.insert(k); - } - } - - (prev, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, tick)| *tick == context.current_tick()) - -> flat_map(|(state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect::>()) - -> tee(); - - from_left = source_stream(from_left) - -> map(|x| deserialize_from_bytes::)>>(x.unwrap()).unwrap()) - -> fold::<'static>( - (HashMap::>::new(), HashSet::new(), 0), - |(mut prev, mut modified_tweets, prev_tick), req: Vec<(u64, TimestampedValue)>| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - for (k, v) in req { - let updated = if let Some(e) = prev.get_mut(&k) { - e.merge_from(v) - } else { - prev.insert(k, v); - true - }; - - if updated { - modified_tweets.insert(k); - } - } - - (prev, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, tick)| *tick == context.current_tick()) - -> flat_map(|(state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect::>()) - -> tee(); - - from_right = source_stream(from_right) - -> map(|x| deserialize_from_bytes::)>>(x.unwrap()).unwrap()) - -> fold::<'static>( - (HashMap::>::new(), HashSet::new(), 0), - |(mut prev, mut modified_tweets, prev_tick), req: Vec<(u64, TimestampedValue)>| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - for (k, v) in req { - let updated = if let Some(e) = prev.get_mut(&k) { - e.merge_from(v) - } else { - prev.insert(k, v); - true - }; - - if updated { - modified_tweets.insert(k); - } - } - - (prev, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, tick)| *tick == context.current_tick()) - -> flat_map(|(state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect::>()) - -> tee(); - - from_local = source_stream(increment_requests) - -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) - -> map(|x| (x.tweet_id, x.likes)) - -> fold::<'static>( - (HashMap::>::new(), HashSet::new(), 0), - |(mut prev, mut modified_tweets, prev_tick), req: UpdateType| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - prev.entry(req.0).or_default().update(|v| v + req.1); - modified_tweets.insert(req.0); - (prev, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, tick)| *tick == context.current_tick()) - -> flat_map(|(state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect::>()) - -> tee(); - - to_right = union(); - - from_parent -> map(|v| (0, v)) -> to_right; - from_left -> map(|v| (1, v)) -> to_right; - from_local -> map(|v| (2, v)) -> to_right; - - to_right - -> fold::<'static>( - (vec![HashMap::>::new(); 3], HashMap::>::new(), HashSet::new(), 0), - |(mut each_source, mut acc_source, mut modified_tweets, prev_tick), (source_i, (key, v))| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - let updated = each_source[source_i].entry(key).or_default().merge_from(v); - - if updated { - acc_source.entry(key).or_default().update(|_| each_source.iter().map(|s| s.get(&key).map(|t| t.value).unwrap_or_default()).sum()); - modified_tweets.insert(key); - } - - (each_source, acc_source, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, _, tick)| *tick == context.current_tick()) - -> map(|(_, state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect()) - -> map(serialize_to_bytes::)>>) - -> dest_sink(to_right); - - to_left = union(); - - from_parent -> map(|v| (0, v)) -> to_left; - from_right -> map(|v| (1, v)) -> to_left; - from_local -> map(|v| (2, v)) -> to_left; - - to_left - -> fold::<'static>( - (vec![HashMap::>::new(); 3], HashMap::>::new(), HashSet::new(), 0), - |(mut each_source, mut acc_source, mut modified_tweets, prev_tick), (source_i, (key, v))| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - let updated = each_source[source_i].entry(key).or_default().merge_from(v); - - if updated { - acc_source.entry(key).or_default().update(|_| each_source.iter().map(|s| s.get(&key).map(|t| t.value).unwrap_or_default()).sum()); - modified_tweets.insert(key); - } - - (each_source, acc_source, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, _, tick)| *tick == context.current_tick()) - -> map(|(_, state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect()) - -> map(serialize_to_bytes::)>>) - -> dest_sink(to_left); - - to_parent = union(); - - from_right -> map(|v| (0, v)) -> to_parent; - from_left -> map(|v| (1, v)) -> to_parent; - from_local -> map(|v| (2, v)) -> to_parent; - - to_parent - -> fold::<'static>( - (vec![HashMap::>::new(); 3], HashMap::>::new(), HashSet::new(), 0), - |(mut each_source, mut acc_source, mut modified_tweets, prev_tick), (source_i, (key, v))| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - let updated = each_source[source_i].entry(key).or_default().merge_from(v); - - if updated { - acc_source.entry(key).or_default().update(|_| each_source.iter().map(|s| s.get(&key).map(|t| t.value).unwrap_or_default()).sum()); - modified_tweets.insert(key); - } - - (each_source, acc_source, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, _, tick)| *tick == context.current_tick()) - -> map(|(_, state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, *state.get(t).unwrap())).collect()) - -> map(serialize_to_bytes::)>>) - -> dest_sink(to_parent); - - to_query = union(); - - from_parent -> map(|v| (0, v)) -> to_query; - from_left -> map(|v| (1, v)) -> to_query; - from_right -> map(|v| (2, v)) -> to_query; - from_local -> map(|v| (3, v)) -> to_query; - - to_query - -> fold::<'static>( - (vec![HashMap::>::new(); 4], HashMap::>::new(), HashSet::new(), 0), - |(mut each_source, mut acc_source, mut modified_tweets, prev_tick), (source_i, (key, v))| { - if prev_tick != context.current_tick() { - modified_tweets.clear(); - } - - let updated = each_source[source_i].entry(key).or_default().merge_from(v); - - if updated { - acc_source.entry(key).or_default().update(|_| each_source.iter().map(|s| s.get(&key).map(|t| t.value).unwrap_or_default()).sum()); - modified_tweets.insert(key); - } - - (each_source, acc_source, modified_tweets, context.current_tick()) - } - ) - -> filter(|(_, _, _, tick)| *tick == context.current_tick()) - -> flat_map(|(_, state, modified_tweets, _)| modified_tweets.iter().map(|t| (*t, state.get(t).unwrap().value)).collect::>()) - -> map(serialize_to_bytes::<(u64, i32)>) - -> dest_sink(query_responses); - }; - - // initial memory - #[cfg(target_os = "linux")] - { - let x = procinfo::pid::stat_self().unwrap(); - let bytes = x.rss * 1024 * 4; - println!("memory,{}", bytes); - } - - let f1_handle = tokio::spawn(f1); - hydroflow::util::cli::launch_flow(df).await; - f1_handle.abort(); -} diff --git a/hydro_cli_examples/topolotree.hydro.py b/hydro_cli_examples/topolotree.hydro.py deleted file mode 100644 index 211757a7551..00000000000 --- a/hydro_cli_examples/topolotree.hydro.py +++ /dev/null @@ -1,181 +0,0 @@ -import asyncio -from codecs import decode -from typing import Optional -import hydro -import json -from pathlib import Path -from aiostream import stream - -class Tree: - def __init__(self, node, left, right): - self.node = node - self.left = left - self.right = right - - def map(self, transform): - return Tree( - transform(self.node), - self.left.map(transform) if self.left is not None else None, - self.right.map(transform) if self.right is not None else None - ) - - def flatten_with_path(self, cur_path=""): - return [(self.node, cur_path)] + \ - (self.left.flatten_with_path(cur_path + "L") if self.left is not None else []) + \ - (self.right.flatten_with_path(cur_path + "R") if self.right is not None else []) - - async def map_async(self, transform): - return Tree( - await transform(self.node), - (await self.left.map_async(transform)) if self.left is not None else None, - (await self.right.map_async(transform)) if self.right is not None else None - ) - -def create_tree(depth, deployment, create_machine) -> Optional[Tree]: - if depth == 0: - return None - else: - self_service = deployment.HydroflowCrate( - src=str(Path(__file__).parent.absolute()), - example="topolotree", - on=create_machine() - ) - - left = create_tree(depth - 1, deployment, create_machine) - right = create_tree(depth - 1, deployment, create_machine) - - if left is not None: - self_service.ports.to_left.send_to(left.node.ports.from_parent) - left.node.ports.to_parent.send_to(self_service.ports.from_left) - else: - self_service.ports.to_left.send_to(hydro.null()) - hydro.null().send_to(self_service.ports.from_left) - - if right is not None: - self_service.ports.to_right.send_to(right.node.ports.from_parent) - right.node.ports.to_parent.send_to(self_service.ports.from_right) - else: - self_service.ports.to_right.send_to(hydro.null()) - hydro.null().send_to(self_service.ports.from_right) - - return Tree( - self_service, - left, - right - ) - -# hydro deploy ../hydro_cli_examples/toplotree.hydro.py -- local/gcp DEPTH_OF_TREE -async def main(args): - tree_depth = int(args[1]) - deployment = hydro.Deployment() - - localhost_machine = deployment.Localhost() - - python_sender = deployment.CustomService( - external_ports=[], - on=localhost_machine, - ) - - gcp_vpc = hydro.GCPNetwork( - project="hydro-chrisdouglas", - ) - - def create_machine(): - if args[0] == "gcp": - return deployment.GCPComputeEngineHost( - project="hydro-chrisdouglas", - machine_type="e2-micro", - image="debian-cloud/debian-11", - region="us-west1-a", - network=gcp_vpc - ) - else: - return localhost_machine - - tree = create_tree(tree_depth, deployment, create_machine) - tree.node.ports.to_parent.send_to(hydro.null()) - hydro.null().send_to(tree.node.ports.from_parent) - - def create_increment_port(node): - port = python_sender.client_port() - port.send_to(node.ports.increment_requests) - return port - - tree_increment_ports = tree.map(create_increment_port) - - def create_query_response_port(node): - port = python_sender.client_port() - node.ports.query_responses.send_to(port) - return port - - tree_query_response_ports = tree.map(create_query_response_port) - - await deployment.deploy() - - async def get_increment_channel(port): - return await (await port.server_port()).into_sink() - tree_increment_channels = await tree_increment_ports.map_async(get_increment_channel) - - async def get_query_response_channel(port): - return await (await port.server_port()).into_source() - tree_query_response_channels = await tree_query_response_ports.map_async(get_query_response_channel) - - async def get_stdouts(node): - return await node.stdout() - tree_stdouts = await tree.map_async(get_stdouts) - - def stream_printer(path, v): - parsed = json.loads(decode(bytes(v), "utf-8")) - return f"{path}: {parsed}" - - print("deployed!") - - with_path_responses = [ - stream.map(response, lambda x,path=path: stream_printer(path, x)) - for (response, path) in tree_query_response_channels.flatten_with_path() - ] - - async def print_queries(): - try: - async with stream.merge(*with_path_responses).stream() as merged: - async for log in merged: - print(log) - except asyncio.CancelledError: - pass - - with_stdouts = [ - stream.map(stdout, lambda x,path=path: (path, x)) - for (stdout, path) in tree_stdouts.flatten_with_path() - ] - - async def print_stdouts(): - try: - async with stream.merge(*with_stdouts).stream() as merged: - async for path, log in merged: - if not log.startswith("to_query"): - return f"{path}: {log}" - except asyncio.CancelledError: - pass - - print_query_task = asyncio.create_task(print_queries()) - print_stdout_task = asyncio.create_task(print_stdouts()) - try: - await deployment.start() - print("started!") - - await asyncio.sleep(1) - - for i in range(1000000): - if i % 10000 == 0: - print(f"sending increment {i}") - await tree_increment_channels.node.send(bytes("{\"tweet_id\": " + str(i) + ", \"likes\": " + str(i % 2 * 2 - 1) + "}", "utf-8")) - finally: - print_query_task.cancel() - print_stdout_task.cancel() - await print_query_task - await print_stdout_task - -if __name__ == "__main__": - import sys - import hydro.async_wrapper - hydro.async_wrapper.run(main, sys.argv[1:]) diff --git a/hydro_cli_examples/topolotree_latency.hydro.py b/hydro_cli_examples/topolotree_latency.hydro.py deleted file mode 100644 index cf1a88c11a6..00000000000 --- a/hydro_cli_examples/topolotree_latency.hydro.py +++ /dev/null @@ -1,306 +0,0 @@ -import asyncio -from codecs import decode -from typing import Optional -from venv import create -import hydro -import json -from pathlib import Path -from aiostream import stream - -import pandas as pd -import numpy as np -import uuid - - -class Tree: - def __init__(self, node, left, right): - self.node = node - self.left = left - self.right = right - - def map(self, transform): - return Tree( - transform(self.node), - self.left.map(transform) if self.left is not None else None, - self.right.map(transform) if self.right is not None else None - ) - - def flatten_with_path(self, cur_path=""): - return [(self.node, cur_path)] + \ - (self.left.flatten_with_path(cur_path + "L") if self.left is not None else []) + \ - (self.right.flatten_with_path(cur_path + "R") if self.right is not None else []) - - async def map_async(self, transform): - return Tree( - await transform(self.node), - (await self.left.map_async(transform)) if self.left is not None else None, - (await self.right.map_async(transform)) if self.right is not None else None - ) - -def create_tree(depth, deployment, create_machine) -> Optional[Tree]: - if depth == 0: - return None - else: - self_service = deployment.HydroflowCrate( - src=str(Path(__file__).parent.absolute()), - example="topolotree", - on=create_machine() - ) - - left = create_tree(depth - 1, deployment, create_machine) - right = create_tree(depth - 1, deployment, create_machine) - - if left is not None: - self_service.ports.to_left.send_to(left.node.ports.from_parent) - left.node.ports.to_parent.send_to(self_service.ports.from_left) - else: - self_service.ports.to_left.send_to(hydro.null()) - hydro.null().send_to(self_service.ports.from_left) - - if right is not None: - self_service.ports.to_right.send_to(right.node.ports.from_parent) - right.node.ports.to_parent.send_to(self_service.ports.from_right) - else: - self_service.ports.to_right.send_to(hydro.null()) - hydro.null().send_to(self_service.ports.from_right) - - return Tree( - self_service, - left, - right - ) - -async def run_experiment(deployment, machine_pool, experiment_id, summaries_file, tree_arg, depth_arg, clients_arg, is_gcp, gcp_vpc): - tree_depth = int(depth_arg) - is_tree = tree_arg == "topolo" # or "pn" - - num_replicas = 2 ** tree_depth - 1 - - num_clients = int(clients_arg) - - localhost_machine = deployment.Localhost() - - currently_deployed = [] - def create_machine(): - if len(machine_pool) > 0: - print("Using machine from pool") - ret = machine_pool.pop() - currently_deployed.append(ret) - return ret - else: - if is_gcp: - out = deployment.GCPComputeEngineHost( - project="hydro-chrisdouglas", - machine_type="n2-standard-4", - image="debian-cloud/debian-11", - region="us-west1-a", - network=gcp_vpc - ) - else: - out = localhost_machine - currently_deployed.append(out) - return out - - all_nodes = [] - if is_tree: - tree = create_tree(tree_depth, deployment, create_machine) - tree.node.ports.to_parent.send_to(hydro.null()) - hydro.null().send_to(tree.node.ports.from_parent) - all_nodes = [tup[0] for tup in tree.flatten_with_path()] - else: - cluster = [ - deployment.HydroflowCrate( - src=str(Path(__file__).parent.absolute()), - example="pn_counter" if tree_arg == "pn" else "pn_counter_delta", - args=[json.dumps([i]), json.dumps([num_replicas])], - on=create_machine() - ) - for i in range(num_replicas) - ] - - for i in range(num_replicas): - cluster[i].ports.to_peer.send_to(hydro.demux( - { - j: cluster[j].ports.from_peer.merge() - for j in range(num_replicas) - if i != j - } - )) - - all_nodes = cluster - - if is_tree: - source = tree - while source.left is not None: - source = source.left - source = source.node - - dest = tree - while dest.right is not None: - dest = dest.right - dest = dest.node - else: - source = cluster[0] - dest = cluster[-1] - - for node in all_nodes: - if node is not dest: - node.ports.query_responses.send_to(hydro.null()) - if node is not source: - hydro.null().send_to(node.ports.increment_requests) - - latency_measurer = deployment.HydroflowCrate( - src=str(Path(__file__).parent.absolute()), - example="topolotree_latency_measure", - args=[json.dumps([num_clients])], - on=create_machine() - ) - - latency_measurer.ports.increment_start_node.send_to(source.ports.increment_requests.merge()) - dest.ports.query_responses.send_to(latency_measurer.ports.end_node_query) - - await deployment.deploy() - - print("deployed!") - - latency = [] - memory_per_node = [[] for _ in range(num_replicas)] - throughput_raw = [] - - throughput = [] - - latency_stdout = await latency_measurer.stdout() - - memories_streams_with_index = [ - stream.map( - await node.stdout(), - lambda x,i=i: (i, x) - ) - for i, node in enumerate(all_nodes) - ] - - async def memory_plotter(): - try: - async with stream.merge(*memories_streams_with_index).stream() as merged: - async for node_idx, line in merged: - line_split = line.split(",") - if line_split[0] == "memory": - memory_per_node[node_idx].append(int(line_split[1])) - except asyncio.CancelledError: - return - - memory_plotter_task = asyncio.create_task(memory_plotter()) - - async def latency_plotter(): - try: - async for line in latency_stdout: - line_split = line.split(",") - if line_split[0] == "throughput": - count = int(line_split[1]) - period = float(line_split[2]) - throughput_raw.append([count, period]) - throughput.append(count / period) - elif line_split[0] == "latency": - number = int(line_split[1]) # microseconds - latency.append(number) - except asyncio.CancelledError: - return - - latency_plotter_task = asyncio.create_task(latency_plotter()) - - await deployment.start() - print("started!") - - await asyncio.sleep(30) - - await latency_measurer.stop() - await asyncio.gather(*[node.stop() for node in all_nodes]) - - memory_plotter_task.cancel() - await memory_plotter_task - - latency_plotter_task.cancel() - await latency_plotter_task - - def summarize(v, kind): - print("mean = ", np.mean(v)) - print("std = ", np.std(v)) - print("min = ", np.min(v)) - print("max = ", np.max(v)) - print("percentile 99 = ", np.percentile(v, 99)) - print("percentile 75 = ", np.percentile(v, 75)) - print("percentile 50 = ", np.percentile(v, 50)) - print("percentile 25 = ", np.percentile(v, 25)) - print("percentile 1 = ", np.percentile(v, 1)) - - summaries_file.write("\n") - summaries_file.write(tree_arg + ",") - summaries_file.write(str(tree_depth) + ",") - summaries_file.write(str(num_clients) + ",") - summaries_file.write(kind + ",") - summaries_file.write(str(np.mean(v)) + ",") - summaries_file.write(str(np.std(v)) + ",") - summaries_file.write(str(np.min(v)) + ",") - summaries_file.write(str(np.max(v)) + ",") - summaries_file.write(str(np.percentile(v, 99)) + ",") - summaries_file.write(str(np.percentile(v, 75)) + ",") - summaries_file.write(str(np.percentile(v, 50)) + ",") - summaries_file.write(str(np.percentile(v, 25)) + ",") - summaries_file.write(str(np.percentile(v, 1))) - summaries_file.flush() - - print("latency:") - summarize(latency, "latency") - - print("throughput:") - summarize(throughput, "throughput") - - init_memory = [ - memory[0] - for memory in memory_per_node - ] - print("init memory:") - summarize(init_memory, "init_memory") - - final_memory = [ - memory[-1] - for memory in memory_per_node - ] - print("final memory:") - summarize(final_memory, "final_memory") - - pd.DataFrame(latency).to_csv("latency_" + tree_arg + "_tree_depth_" + str(tree_depth) + "_num_clients_" + str(num_clients) + "_" + experiment_id+".csv", index=False, header=["latency"]) - pd.DataFrame(throughput_raw).to_csv("throughput_" + tree_arg + "_tree_depth_" + str(tree_depth) + "_num_clients_" + str(num_clients) + "_" + experiment_id+".csv", index=False, header=["count", "period"]) - pd.DataFrame(init_memory).to_csv("init_memory_" + tree_arg + "_tree_depth_" + str(tree_depth) + "_num_clients_" + str(num_clients) + "_" + experiment_id+".csv", index=False, header=["memory"]) - pd.DataFrame(final_memory).to_csv("final_memory_" + tree_arg + "_tree_depth_" + str(tree_depth) + "_num_clients_" + str(num_clients) + "_" + experiment_id+".csv", index=False, header=["memory"]) - - for machine in currently_deployed: - machine_pool.append(machine) - -# hydro deploy ../hydro_cli_examples/toplotree_latency.hydro.py -- local/gcp DEPTH_OF_TREE -async def main(args): - # the current timestamp - import datetime - experiment_id = str(datetime.datetime.now()) - - summaries_file = open(f"summaries_{experiment_id}.csv", "w") - summaries_file.write("protocol,tree_depth,num_clients,kind,mean,std,min,max,percentile_99,percentile_75,percentile_50,percentile_25,percentile_1") - - deployment = hydro.Deployment() - pool = [] - - network = hydro.GCPNetwork( - project="hydro-chrisdouglas", - ) if args[0] == "gcp" else None - - for depth_arg in args[2].split(","): - for tree_arg in args[1].split(","): - for num_clients_arg in args[3].split(","): - await run_experiment(deployment, pool, experiment_id, summaries_file, tree_arg, depth_arg, num_clients_arg, args[0] == "gcp", network) - - summaries_file.close() - -if __name__ == "__main__": - import sys - import hydro.async_wrapper - hydro.async_wrapper.run(main, sys.argv[1:]) diff --git a/hydroflow/CHANGELOG.md b/hydroflow/CHANGELOG.md index a7650946a61..93734458223 100644 --- a/hydroflow/CHANGELOG.md +++ b/hydroflow/CHANGELOG.md @@ -5,8 +5,330 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +### Chore + + - fix lints for latest nightly + - fix lint, format errors for latest nightly version (without updated pinned) + For nightly version (d9c13cd45 2023-07-05) + +### New Features + + - add kvs_bench example test + - Add `use` statements to hydroflow syntax + And use in doc tests. + - add `iter_batches_stream` util to break up iterator into per-tick batches + * Also tightens up a bit of `assert_eq`'s code + - rename assert => assert_eq, add assert, change underlying implementation to work across ticks + - make batch take two inputs [input] and [signal] + +### Bug Fixes + + - fix shopping cart example test + - change "gated buffer" discussion from `cross_join` to `defer_signal` + shopping still erroring out for reasons unrelated to this PR + - unify antijoin and difference with set and multiset semantics + * fix: unify antijoin and difference with set and multiset semantics + + * fix: replay semantics for antijoin and difference now work + also added cross_join_multiset + + * fix: enforce sort for tests of anti_join and difference using assert_eq + + * fix: advance __borrow_ident beyond the current tick to prevent replay loops + + * fix: add modified snapshots + + * fix: temp + + * fix: spelling typo in comment + + * fix: make anti_join replay more efficient + + Also add multiset data structure, use it in some tests, make join() + replay logic more similar to anti_join's and presist's. + + * fix: ignore test that depends on order of antijoin + + * fix: really ignore test_index + + * fix: fix specific test ordering in wasm + + --------- + - joins now replay correctly + - livelock in deadlock detector #810 + - lattice_batch now takes [input] and [signal] + - make all operators 'tick by default + - stop two_pc example test from hanging sometimes + - rename next_tick -> defer, batch -> defer_signal + - remove python from default features, add it to ci + - `py_udf` operator feature gating + +### Other + + - join probe returns first item directly + * optimization: join probe returns first item directly + + * update comments + +### Test + + - add `use` compile-fail tests + - Change example tests to use localhost IP + +### New Features (BREAKING) + + - add fused joins, make lattice_join replay correctly + * feat!: add fused joins, make lattice_join replay correctly + + * address comments + + * fix clippy + +### Commit Statistics + + + + - 24 commits contributed to the release over the course of 39 calendar days. + - 42 days passed between releases. + - 22 commits were understood as [conventional](https://www.conventionalcommits.org). + - 23 unique issues were worked on: [#820](https://github.com/hydro-project/hydroflow/issues/820), [#821](https://github.com/hydro-project/hydroflow/issues/821), [#822](https://github.com/hydro-project/hydroflow/issues/822), [#823](https://github.com/hydro-project/hydroflow/issues/823), [#833](https://github.com/hydro-project/hydroflow/issues/833), [#835](https://github.com/hydro-project/hydroflow/issues/835), [#837](https://github.com/hydro-project/hydroflow/issues/837), [#840](https://github.com/hydro-project/hydroflow/issues/840), [#842](https://github.com/hydro-project/hydroflow/issues/842), [#843](https://github.com/hydro-project/hydroflow/issues/843), [#844](https://github.com/hydro-project/hydroflow/issues/844), [#845](https://github.com/hydro-project/hydroflow/issues/845), [#846](https://github.com/hydro-project/hydroflow/issues/846), [#848](https://github.com/hydro-project/hydroflow/issues/848), [#851](https://github.com/hydro-project/hydroflow/issues/851), [#853](https://github.com/hydro-project/hydroflow/issues/853), [#857](https://github.com/hydro-project/hydroflow/issues/857), [#861](https://github.com/hydro-project/hydroflow/issues/861), [#870](https://github.com/hydro-project/hydroflow/issues/870), [#872](https://github.com/hydro-project/hydroflow/issues/872), [#874](https://github.com/hydro-project/hydroflow/issues/874), [#878](https://github.com/hydro-project/hydroflow/issues/878), [#880](https://github.com/hydro-project/hydroflow/issues/880) + +### Commit Details + + + +
view details + + * **[#820](https://github.com/hydro-project/hydroflow/issues/820)** + - Make batch take two inputs [input] and [signal] ([`8710022`](https://github.com/hydro-project/hydroflow/commit/871002267e3c03da83729ecc2d028f3c7b5c18d2)) + * **[#821](https://github.com/hydro-project/hydroflow/issues/821)** + - `py_udf` operator feature gating ([`2d53110`](https://github.com/hydro-project/hydroflow/commit/2d53110336b2da5a16887c3d72101da72b2362bb)) + * **[#822](https://github.com/hydro-project/hydroflow/issues/822)** + - Fix lint, format errors for latest nightly version (without updated pinned) ([`f60053f`](https://github.com/hydro-project/hydroflow/commit/f60053f70da3071c54de4a0eabb059a143aa2ccc)) + * **[#823](https://github.com/hydro-project/hydroflow/issues/823)** + - Book/doc edits ([`4bdd556`](https://github.com/hydro-project/hydroflow/commit/4bdd5568fa0a6674f650f91a029fab302cbf14f4)) + * **[#833](https://github.com/hydro-project/hydroflow/issues/833)** + - Rename next_tick -> defer, batch -> defer_signal ([`6c98bbc`](https://github.com/hydro-project/hydroflow/commit/6c98bbc2bd3443fe6f77e0b8689b461edde1b316)) + * **[#835](https://github.com/hydro-project/hydroflow/issues/835)** + - Rename assert => assert_eq, add assert, change underlying implementation to work across ticks ([`8f306e2`](https://github.com/hydro-project/hydroflow/commit/8f306e2a36582e168417808099eedf8a9de3b419)) + * **[#837](https://github.com/hydro-project/hydroflow/issues/837)** + - Remove python from default features, add it to ci ([`c6cb86d`](https://github.com/hydro-project/hydroflow/commit/c6cb86d4549370b9a10c6f49ecbf0f160b481a1d)) + * **[#840](https://github.com/hydro-project/hydroflow/issues/840)** + - Make all operators 'tick by default ([`a55fc74`](https://github.com/hydro-project/hydroflow/commit/a55fc74dc1ebbe26b49359a104beb48d7f6cd449)) + * **[#842](https://github.com/hydro-project/hydroflow/issues/842)** + - Stop two_pc example test from hanging sometimes ([`fb517b9`](https://github.com/hydro-project/hydroflow/commit/fb517b984735309eaa9a1008b23375555438529d)) + * **[#843](https://github.com/hydro-project/hydroflow/issues/843)** + - Add `iter_batches_stream` util to break up iterator into per-tick batches ([`fe02f23`](https://github.com/hydro-project/hydroflow/commit/fe02f23649312bb64c5d0c8870edf578e516f397)) + * **[#844](https://github.com/hydro-project/hydroflow/issues/844)** + - Fix lints for latest nightly ([`949db02`](https://github.com/hydro-project/hydroflow/commit/949db02e9fa9878e1a7176c180d6f44c5cddf052)) + * **[#845](https://github.com/hydro-project/hydroflow/issues/845)** + - Add `use` compile-fail tests ([`d38b1bc`](https://github.com/hydro-project/hydroflow/commit/d38b1bce848b4672109a4f411aaa81b9d210c2c4)) + - Add `use` statements to hydroflow syntax ([`b4b9644`](https://github.com/hydro-project/hydroflow/commit/b4b9644a19e8e7e7725c9c5b88e3a6b8c2be7364)) + * **[#846](https://github.com/hydro-project/hydroflow/issues/846)** + - Change example tests to use localhost IP ([`060715a`](https://github.com/hydro-project/hydroflow/commit/060715a7549c8477362971e39d39a3da7e26ad29)) + * **[#848](https://github.com/hydro-project/hydroflow/issues/848)** + - Add kvs_bench example test ([`a7ab8ff`](https://github.com/hydro-project/hydroflow/commit/a7ab8ff09bc34ebd6fb4127460733056aee2adf7)) + * **[#851](https://github.com/hydro-project/hydroflow/issues/851)** + - Lattice_batch now takes [input] and [signal] ([`ebba382`](https://github.com/hydro-project/hydroflow/commit/ebba38230df134b04dd38c1df7c6de8712e3122e)) + * **[#853](https://github.com/hydro-project/hydroflow/issues/853)** + - Book updates ([`2e57445`](https://github.com/hydro-project/hydroflow/commit/2e574457246ac5bd231745a8ad068558859698ef)) + * **[#857](https://github.com/hydro-project/hydroflow/issues/857)** + - Livelock in deadlock detector #810 ([`0e715ae`](https://github.com/hydro-project/hydroflow/commit/0e715ae2648b8b9ce76a60a69c18e0543cf2c033)) + * **[#861](https://github.com/hydro-project/hydroflow/issues/861)** + - Add fused joins, make lattice_join replay correctly ([`7a3b4c0`](https://github.com/hydro-project/hydroflow/commit/7a3b4c04779ea38bfa06c246882fa8dfb52bc8f1)) + * **[#870](https://github.com/hydro-project/hydroflow/issues/870)** + - Joins now replay correctly ([`cc959c7`](https://github.com/hydro-project/hydroflow/commit/cc959c762c3a0e036e672801c615028cbfb95168)) + * **[#872](https://github.com/hydro-project/hydroflow/issues/872)** + - Unify antijoin and difference with set and multiset semantics ([`d378e5e`](https://github.com/hydro-project/hydroflow/commit/d378e5eada3d2bae90f98c5a33b2d055940a8c7f)) + * **[#874](https://github.com/hydro-project/hydroflow/issues/874)** + - Join probe returns first item directly ([`aa52b10`](https://github.com/hydro-project/hydroflow/commit/aa52b10b60e733a65bdd1fc0234acf7249c22f1a)) + * **[#878](https://github.com/hydro-project/hydroflow/issues/878)** + - Change "gated buffer" discussion from `cross_join` to `defer_signal` ([`d6472b9`](https://github.com/hydro-project/hydroflow/commit/d6472b90c2caf26b98c0bd753616e675b7de9769)) + * **[#880](https://github.com/hydro-project/hydroflow/issues/880)** + - Fix shopping cart example test ([`c9e7c7d`](https://github.com/hydro-project/hydroflow/commit/c9e7c7dc1c0885640b1c5d6bc15194a256eff833)) +
+ +## 0.3.0 (2023-07-04) + + + + + + + + + + + +### Documentation + + - remove pattern deref from inspect, filter examples + `*` derefs are easier for Rust beginners to comprehend. + - import doc examples from runnable code with tested output + - change mermaid colors + Use a lighter shade of blue and yellow, and dark text. + +### New Features + + + + + + + + + + + - fold and reduce take accumulated value by mutable reference + * feat: fold and reduce take accumulated value by mutable reference +* address comments +* feat: add lattice_reduce and lattice_fold +* address comments +* simplify lattice fold a bit +* address comments +* feat: add join_multiset() +* address comments +* fix assert +* feat: add assert() operator +* update: change for_each -> assert, make doctest use run_avaialble() +* don't run tests that panic in wasm +* update comments +* address comments + +### Bug Fixes + + - update proc-macro2, use new span location API where possible + requires latest* rust nightly version + + *latest = 2023-06-28 or something + - SparseVec does not need T: Default + - remove nightly feature `never_type` where unused + - removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` + - fix scheduler spinning on stateful operators across strata + - use proper tcp/udp localhost IPs to fix test on windows + +### Style + + - `warn` missing docs (instead of `deny`) to allow code before docs + - use `is_ok` for clippy latest nightly + +### Test + + - test examples outputs from docs + - add failing spinning stratum-persist bug tests + - add `test_stratum/tick_loop` tests + - ignore `surface_lattice_merge_badgeneric` `compile-fail` test due to `Seq` inconsistent messages + +### New Features (BREAKING) + + + + - Add `reveal` methods, make fields private + - Add `Provenance` generic param token to `Point`. + - Use `()` provenance for `kvs_bench` example. +* Adds `tokio` for `#[tokio::test]`. +* Adds `async_std` for `#[async_std::test]`. +* Adds `hydroflow` for `#[hydroflow::test]`. +* Adds `env_logging` for `env_logger` registering. +* Adds `env_tracing` for `EnvFilter` `FmtSubscriber` `tracing`. + +### Bug Fixes (BREAKING) + + - make join default to multiset join + +### Refactor (BREAKING) + + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` + - Rename `Bottom` -> `WithBot`, `Top` -> `WithTop`, constructors now take `Option`s 2/4 + - Rename `Immut` -> `Point` lattice. + +### Commit Statistics + + + + - 34 commits contributed to the release over the course of 31 calendar days. + - 33 days passed between releases. + - 31 commits were understood as [conventional](https://www.conventionalcommits.org). + - 25 unique issues were worked on: [#739](https://github.com/hydro-project/hydroflow/issues/739), [#743](https://github.com/hydro-project/hydroflow/issues/743), [#745](https://github.com/hydro-project/hydroflow/issues/745), [#748](https://github.com/hydro-project/hydroflow/issues/748), [#749](https://github.com/hydro-project/hydroflow/issues/749), [#755](https://github.com/hydro-project/hydroflow/issues/755), [#761](https://github.com/hydro-project/hydroflow/issues/761), [#763](https://github.com/hydro-project/hydroflow/issues/763), [#765](https://github.com/hydro-project/hydroflow/issues/765), [#772](https://github.com/hydro-project/hydroflow/issues/772), [#773](https://github.com/hydro-project/hydroflow/issues/773), [#774](https://github.com/hydro-project/hydroflow/issues/774), [#775](https://github.com/hydro-project/hydroflow/issues/775), [#778](https://github.com/hydro-project/hydroflow/issues/778), [#780](https://github.com/hydro-project/hydroflow/issues/780), [#784](https://github.com/hydro-project/hydroflow/issues/784), [#788](https://github.com/hydro-project/hydroflow/issues/788), [#789](https://github.com/hydro-project/hydroflow/issues/789), [#791](https://github.com/hydro-project/hydroflow/issues/791), [#792](https://github.com/hydro-project/hydroflow/issues/792), [#799](https://github.com/hydro-project/hydroflow/issues/799), [#801](https://github.com/hydro-project/hydroflow/issues/801), [#803](https://github.com/hydro-project/hydroflow/issues/803), [#804](https://github.com/hydro-project/hydroflow/issues/804), [#809](https://github.com/hydro-project/hydroflow/issues/809) + +### Commit Details + + + +
view details + + * **[#739](https://github.com/hydro-project/hydroflow/issues/739)** + - Add bind_tcp and connect_tcp, analogues of bind_udp ([`baf320e`](https://github.com/hydro-project/hydroflow/commit/baf320e7d31e3189adc85a98ff3824a321a60995)) + * **[#743](https://github.com/hydro-project/hydroflow/issues/743)** + - Use `is_ok` for clippy latest nightly ([`5c654f2`](https://github.com/hydro-project/hydroflow/commit/5c654f2add8ef389eefeddccc063fd26a08b5be8)) + * **[#745](https://github.com/hydro-project/hydroflow/issues/745)** + - Use proper tcp/udp localhost IPs to fix test on windows ([`7c9632a`](https://github.com/hydro-project/hydroflow/commit/7c9632a0316c29df0ee793a15b1a02f651c4ff51)) + * **[#748](https://github.com/hydro-project/hydroflow/issues/748)** + - Add `test_stratum/tick_loop` tests ([`c992423`](https://github.com/hydro-project/hydroflow/commit/c99242378caf06810fa7de94e504e36af8aeaaf4)) + * **[#749](https://github.com/hydro-project/hydroflow/issues/749)** + - Add basic tracing support, use in (some) tests. ([`a233818`](https://github.com/hydro-project/hydroflow/commit/a23381854a45f9c5791bd399dd633fee291d400a)) + * **[#755](https://github.com/hydro-project/hydroflow/issues/755)** + - `hydroflow`, `logging`/`tracing` features ([`c1b0280`](https://github.com/hydro-project/hydroflow/commit/c1b028089ea9d76ab71cd9cb4eaaaf16aa4b65a6)) + * **[#761](https://github.com/hydro-project/hydroflow/issues/761)** + - Rename `Immut` -> `Point` lattice. ([`1bdadb8`](https://github.com/hydro-project/hydroflow/commit/1bdadb82b25941d11f3fa24eaac35109927c852f)) + * **[#763](https://github.com/hydro-project/hydroflow/issues/763)** + - Rename `Bottom` -> `WithBot`, `Top` -> `WithTop`, constructors now take `Option`s 2/4 ([`5c7e4d3`](https://github.com/hydro-project/hydroflow/commit/5c7e4d3aea1dfb61d51bcb0291740281824e3090)) + * **[#765](https://github.com/hydro-project/hydroflow/issues/765)** + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` ([`4a727ec`](https://github.com/hydro-project/hydroflow/commit/4a727ecf1232e0f03f5300547282bfbe73342cfa)) + * **[#772](https://github.com/hydro-project/hydroflow/issues/772)** + - Add `Provenance` generic param token to `Point`. ([`7aec1ac`](https://github.com/hydro-project/hydroflow/commit/7aec1ac884e01a560770dfab7e0ba64d520415f6)) + * **[#773](https://github.com/hydro-project/hydroflow/issues/773)** + - `warn` missing docs (instead of `deny`) to allow code before docs ([`70c88a5`](https://github.com/hydro-project/hydroflow/commit/70c88a51c4c83a4dc2fc67a0cd344786a4ff26f7)) + * **[#774](https://github.com/hydro-project/hydroflow/issues/774)** + - Make join default to multiset join ([`6f3c536`](https://github.com/hydro-project/hydroflow/commit/6f3c536fcd4d1305d478ec3db62416aad9cf3c68)) + * **[#775](https://github.com/hydro-project/hydroflow/issues/775)** + - Add persist_mut and persist_mut_keyed for non-monitone deletions ([`8d8247f`](https://github.com/hydro-project/hydroflow/commit/8d8247f0b37d53415f5738099c0c8a021415b158)) + * **[#778](https://github.com/hydro-project/hydroflow/issues/778)** + - Import doc examples from runnable code with tested output ([`23f27e5`](https://github.com/hydro-project/hydroflow/commit/23f27e590df648ee8f6bd9ae452f2b2bec5ac652)) + - Test examples outputs from docs ([`920b2df`](https://github.com/hydro-project/hydroflow/commit/920b2dfb88243c1d4833dd8fb0b80ea626380df5)) + - Change mermaid colors ([`f55d540`](https://github.com/hydro-project/hydroflow/commit/f55d540532ba0a0970cab2bb5aef81b6a76b317a)) + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Emit `compile_error!` diagnostics for stable ([`ea65349`](https://github.com/hydro-project/hydroflow/commit/ea65349d241873f8460d7a8b024d64c63180246f)) + - Allow stable build, refactors behind `nightly` feature flag ([`22abcaf`](https://github.com/hydro-project/hydroflow/commit/22abcaff806c7de6e4a7725656bbcf201e7d9259)) + - Remove nightly feature `never_type` where unused ([`a3c1fbb`](https://github.com/hydro-project/hydroflow/commit/a3c1fbbd1e3fa7a7299878f61b4bfd12dce0052c)) + - Removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` ([`9bb5528`](https://github.com/hydro-project/hydroflow/commit/9bb5528d99e83fdae5aeca9456802379131c2f90)) + * **[#784](https://github.com/hydro-project/hydroflow/issues/784)** + - Add assert() operator ([`d83b049`](https://github.com/hydro-project/hydroflow/commit/d83b049e4d643617a2b15b3dbf1698aa79846aeb)) + * **[#788](https://github.com/hydro-project/hydroflow/issues/788)** + - SparseVec does not need T: Default ([`e628da5`](https://github.com/hydro-project/hydroflow/commit/e628da5b70543ac4001d8c4f0ef8f663f95bc17d)) + * **[#789](https://github.com/hydro-project/hydroflow/issues/789)** + - Add `reveal` methods, make fields private ([`931d938`](https://github.com/hydro-project/hydroflow/commit/931d93887c238025596cb22226e16d43e16a7425)) + * **[#791](https://github.com/hydro-project/hydroflow/issues/791)** + - Add tests for examples ([`8f67c26`](https://github.com/hydro-project/hydroflow/commit/8f67c264f5aed560fc14af70b062edf7d839afe6)) + * **[#792](https://github.com/hydro-project/hydroflow/issues/792)** + - Add `py_udf` operator [wip] ([`7dbd5e2`](https://github.com/hydro-project/hydroflow/commit/7dbd5e24d6e71cf8fab7c3ce09d5937c0f301456)) + * **[#799](https://github.com/hydro-project/hydroflow/issues/799)** + - Remove pattern deref from inspect, filter examples ([`fa5b180`](https://github.com/hydro-project/hydroflow/commit/fa5b180d96498d144f3617bba7722e8f4ac9dd0e)) + * **[#801](https://github.com/hydro-project/hydroflow/issues/801)** + - Update proc-macro2, use new span location API where possible ([`8d3494b`](https://github.com/hydro-project/hydroflow/commit/8d3494b5afee858114a602a3e23077bb6d24dd77)) + * **[#803](https://github.com/hydro-project/hydroflow/issues/803)** + - Add lattice_reduce and lattice_fold ([`6323980`](https://github.com/hydro-project/hydroflow/commit/6323980e83bee27a8233a69a35734b5970336701)) + * **[#804](https://github.com/hydro-project/hydroflow/issues/804)** + - Add join_multiset() ([`0105246`](https://github.com/hydro-project/hydroflow/commit/010524615bb78288e339e03880c4dd3b432b6d7f)) + * **[#809](https://github.com/hydro-project/hydroflow/issues/809)** + - Fold and reduce take accumulated value by mutable reference ([`b435bbb`](https://github.com/hydro-project/hydroflow/commit/b435bbb1d64d60f1248fdcd636635b15954e7325)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) + - Fix scheduler spinning on stateful operators across strata ([`0ecabc8`](https://github.com/hydro-project/hydroflow/commit/0ecabc80348093416ecde3de7b6bf0bb22ff30d6)) + - Add failing spinning stratum-persist bug tests ([`4675c2c`](https://github.com/hydro-project/hydroflow/commit/4675c2c334b6bb1550124a27614728fe29c53e12)) + - Ignore `surface_lattice_merge_badgeneric` `compile-fail` test due to `Seq` inconsistent messages ([`aabaa27`](https://github.com/hydro-project/hydroflow/commit/aabaa27fd736534a14f5414fb31328fad25984f3)) +
+ + + add lattice_reduce and lattice_fold add join_multiset()also remove documentation about HalfJoinMultiset, the way to accessthat now is to use join_multiset() add tests for examples add assert() operator emit compile_error! diagnostics for stable allow stable build, refactors behind nightly feature flag add basic tracing support, use in (some) tests. add bind_tcp and connect_tcp, analogues of bind_udp hydroflow, logging/tracing features + ## 0.2.0 (2023-05-31) + + + ### Chore - manually bump versions for v0.2.0 release @@ -23,8 +345,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 3 commits contributed to the release. - - 2 days passed between releases. + - 4 commits contributed to the release. + - 1 day passed between releases. - 3 commits were understood as [conventional](https://www.conventionalcommits.org). - 0 issues like '(#ID)' were seen in commit messages @@ -35,6 +357,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
view details * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Add `hydroflow/README.md`, integrate into book ([`6434dd8`](https://github.com/hydro-project/hydroflow/commit/6434dd8928b913370f70cb4ae68a13044d999a82)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a)) - Rename `Fake` -> `Immut` ([`10b3085`](https://github.com/hydro-project/hydroflow/commit/10b308532245db8f4480ce53b67aea050ae1918d)) diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index da4e29bfb1b..7bd21668bb1 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -1,38 +1,45 @@ [package] name = "hydroflow" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow/" description = "Hydro's low-level dataflow runtime and IR" [features] -default = [ "async", "macros" , "nightly" ] +default = [ "macros" , "nightly" ] + nightly = [ "hydroflow_macro", "hydroflow_macro/diagnostics" ] -async = [ "dep:futures" ] macros = [ "hydroflow_macro", "hydroflow_datalog" ] hydroflow_macro = [ "dep:hydroflow_macro" ] hydroflow_datalog = [ "dep:hydroflow_datalog" ] cli_integration = [ "dep:hydroflow_cli_integration" ] +python = [ "dep:pyo3" ] [[example]] name = "kvs_bench" required-features = [ "nightly" ] +[[example]] +name = "python_udf" +required-features = [ "python" ] + [dependencies] bincode = "1.3" byteorder = "1.4.3" bytes = "1.1.0" -futures = { version = "0.3", optional = true } -hydroflow_cli_integration = { optional = true, path = "../hydroflow_cli_integration", version = "^0.2.0" } -hydroflow_datalog = { optional = true, path = "../hydroflow_datalog", version = "^0.2.0" } -hydroflow_lang = { path = "../hydroflow_lang", version = "^0.2.0" } -hydroflow_macro = { optional = true, path = "../hydroflow_macro", version = "^0.2.0" } +futures = "0.3" +hydroflow_cli_integration = { optional = true, path = "../hydroflow_cli_integration", version = "^0.3.0" } +hydroflow_datalog = { optional = true, path = "../hydroflow_datalog", version = "^0.4.0" } +hydroflow_lang = { path = "../hydroflow_lang", version = "^0.4.0" } +hydroflow_macro = { optional = true, path = "../hydroflow_macro", version = "^0.4.0" } itertools = "0.10" -lattices = { path = "../lattices", version = "^0.2.0", features = [ "serde" ] } -pusherator = { path = "../pusherator", version = "^0.0.1" } +lattices = { path = "../lattices", version = "^0.4.0", features = [ "serde" ] } +pusherator = { path = "../pusherator", version = "^0.0.3" } +pyo3 = { optional = true, version = "0.18" } ref-cast = "1.0" +regex = "1.8.4" rustc-hash = "1.1.0" sealed = "0.5" serde = { version = "1", features = [ "derive" ] } @@ -42,6 +49,7 @@ smallvec = "1.10.0" tokio-stream = { version = "0.1.10", features = [ "io-util", "sync" ] } tracing = "0.1" variadics = { path = "../variadics", version = "^0.0.2" } +instant = { version = "0.1.12", features = ["wasm-bindgen"] } # Instant::now() is not supported on wasm, use this shim instead. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { version = "1.16", features = [ "full" ] } diff --git a/hydroflow/README.md b/hydroflow/README.md index 1337aa592b9..8aa98d34cd2 100644 --- a/hydroflow/README.md +++ b/hydroflow/README.md @@ -1,16 +1,23 @@ # Hydroflow -[Hydroflow](https://github.com/hydro-project/hydroflow) is a compiler for low-latency -dataflow programs, written in Rust. Hydroflow is the runtime library for the +[Hydroflow](https://github.com/hydro-project/hydroflow) is a small language and compiler for low-latency +dataflow programs, written in Rust. Hydroflow serves as the runtime library for the [Hydro language stack](https://hydro.run/docs/hydroflow/ecosystem), which is under development as a complete compiler stack for distributed programming languages. -Hydroflow is designed with two goals in mind: -- Expert developers can program Hydroflow directly to build components in a distributed system. -- Higher levels of the Hydro stack will offer friendlier languages with more abstractions, and treat Hydroflow as a compiler target. +Hydroflow is designed with two use cases in mind: + - Expert developers can program directly in the Hydroflow language to build individual components that can interoperate in a distributed program or service. + - Higher levels of the Hydro stack will offer higher-level abstractions and DSLs, and treat Hydroflow as a compiler target. -Hydroflow provides a DSL—the *surface syntax*—embedded in Rust, which compiles to high-efficiency machine code. +Hydroflow is targeted at supporting the following unique features: + 1. A type system that helps developers reason about progress and consistency guarantees in a distributed program. This includes an emphasis on [lattice types](https://hydro.run/docs/hydroflow/lattices_crate/) that can allow for consistent outcomes in the face of network messages that may be interleaved, reordered, batched, and resent. + 2. A [dataflow programming model](https://hydro.run/docs/hydroflow/syntax/surface_flows), capturing the message- and data-driven nature of many distributed services. + 3. Extremely low-latency handling of asynchronously-arriving data/messages, via aggressive exploitation of Rust's [monomorphization](https://rustc-dev-guide.rust-lang.org/backend/monomorph.html) techniques. + 4. Dataflow optimizations, both to optimize single-node Hydroflow flows, and to enable distributed optimizations across multiple flows. + +Hydroflow's language—the Hydroflow *surface syntax*—is embedded in Rust, which compiles Hydroflow code to high-efficiency machine code. As the lowest level of the Hydro stack, Hydroflow requires some knowledge of Rust to use. -Check out the [Hydroflow Playground](https://hydro.run/playground) to see Hydroflow's surface syntax in action! -Or read the [Hydroflow Book docs](https://hydro.run/docs/hydroflow/#this-book) to get started. +The most recent release of the [Hydroflow Book docs](https://hydro.run/docs/hydroflow/#this-book) are online, providing documentation and numerous annotated examples. + +You can also check out the [Hydroflow Playground](https://hydro.run/playground) to see Hydroflow's surface syntax in action! diff --git a/hydroflow/examples/chat/client.rs b/hydroflow/examples/chat/client.rs index 19a76eecf93..f0fc46891dc 100644 --- a/hydroflow/examples/chat/client.rs +++ b/hydroflow/examples/chat/client.rs @@ -45,18 +45,18 @@ pub(crate) async fn run_client(outbound: UdpSink, inbound: UdpStream, opts: Opts inbound_chan[errs] -> for_each(|m| println!("Received unexpected message type: {:?}", m)); // send a single connection request on startup - source_iter([()]) -> map(|_m| (Message::ConnectRequest, server_addr)) -> [0]outbound_chan; + initialize() -> map(|_m| (Message::ConnectRequest, server_addr)) -> [0]outbound_chan; // take stdin and send to server as a msg - // the cross_join serves to buffer msgs until the connection request is acked - msg_send = cross_join() -> map(|(msg, _)| (msg, server_addr)) -> [1]outbound_chan; + // the batch serves to buffer msgs until the connection request is acked lines = source_stdin() -> map(|l| Message::ChatMsg { nickname: opts.name.clone(), message: l.unwrap(), ts: Utc::now()}) - -> [0]msg_send; - inbound_chan[acks] -> [1]msg_send; + -> [input]msg_send; + inbound_chan[acks] -> persist() -> [signal]msg_send; + msg_send = defer_signal() -> map(|msg| (msg, server_addr)) -> [1]outbound_chan; // receive and print messages inbound_chan[msgs] -> for_each(pretty_print_msg); diff --git a/hydroflow/examples/chat/main.rs b/hydroflow/examples/chat/main.rs index 1d16be20ef3..3edfe40e39f 100644 --- a/hydroflow/examples/chat/main.rs +++ b/hydroflow/examples/chat/main.rs @@ -57,3 +57,52 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_output) = + run_cargo_example("chat", "--role server --name server --addr 127.0.0.1:11247"); + + let mut server_output_so_far = String::new(); + wait_for_process_output( + &mut server_output_so_far, + &mut server_output, + "Server live!", + ); + + let (_client1, mut client1_input, mut client1_output) = run_cargo_example( + "chat", + "--role client --name client1 --server-addr 127.0.0.1:11247", + ); + + let (_client2, _, mut client2_output) = run_cargo_example( + "chat", + "--role client --name client2 --server-addr 127.0.0.1:11247", + ); + + let mut client1_output_so_far = String::new(); + let mut client2_output_so_far = String::new(); + + wait_for_process_output( + &mut client1_output_so_far, + &mut client1_output, + "Client live!", + ); + wait_for_process_output( + &mut client2_output_so_far, + &mut client2_output, + "Client live!", + ); + + client1_input.write_all(b"Hello\n").unwrap(); + + wait_for_process_output( + &mut client2_output_so_far, + &mut client2_output, + ".*, .* client1: Hello", + ); +} diff --git a/hydroflow/examples/chat/server.rs b/hydroflow/examples/chat/server.rs index b898eb19c2b..fac7a379ddd 100644 --- a/hydroflow/examples/chat/server.rs +++ b/hydroflow/examples/chat/server.rs @@ -13,13 +13,13 @@ pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream, opts: Opts outbound_chan = union() -> dest_sink_serde(outbound); inbound_chan = source_stream_serde(inbound) -> map(Result::unwrap) - -> demux(|(msg, addr), var_args!(clients, msgs, errs)| + -> demux(|(msg, addr), var_args!(clients, msgs, errs)| match msg { Message::ConnectRequest => clients.give(addr), Message::ChatMsg {..} => msgs.give(msg), _ => errs.give(msg), } - ); + ); clients = inbound_chan[clients] -> tee(); inbound_chan[errs] -> for_each(|m| println!("Received unexpected message type: {:?}", m)); @@ -27,9 +27,9 @@ pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream, opts: Opts clients[0] -> map(|addr| (Message::ConnectResponse, addr)) -> [0]outbound_chan; // Pipeline 2: Broadcast messages to all clients - broadcast = cross_join() -> [1]outbound_chan; inbound_chan[msgs] -> [0]broadcast; clients[1] -> [1]broadcast; + broadcast = cross_join::<'static>() -> [1]outbound_chan; }; if let Some(graph) = opts.graph { diff --git a/hydroflow/examples/deadlock_detector/README.md b/hydroflow/examples/deadlock_detector/README.md index 44280665dae..db50d432045 100644 --- a/hydroflow/examples/deadlock_detector/README.md +++ b/hydroflow/examples/deadlock_detector/README.md @@ -24,7 +24,7 @@ cargo run --example deadlock_detector -- --path hydroflow/examples/deadlock_dete Now, you will be prompted to type an integer pair at `stdin`. Each pair `(x, y)` you type is a *waits-for* edge. E.g., if you type `(2, 4)` you are indicating that transaction `2` is waiting for transaction `4`. (Ordinarily this information -would come from another module like a lock manager). The edges across all peers represents the current *waits-for* relatiopn +would come from another module like a lock manager). The edges across all peers represents the current *waits-for* relation Adding the `--graph ` flag to the end of the command lines above will print out a node-and-edge diagram of the program. Supported values for `` include [`mermaid`](https://mermaid-js.github.io/), [`dot`](https://graphviz.org/doc/info/lang.html) and `json`. diff --git a/hydroflow/examples/deadlock_detector/helpers.rs b/hydroflow/examples/deadlock_detector/helpers.rs index eba3551a614..d145f7b0063 100644 --- a/hydroflow/examples/deadlock_detector/helpers.rs +++ b/hydroflow/examples/deadlock_detector/helpers.rs @@ -1,4 +1,4 @@ -use std::fmt::Display; +use std::fmt::{Display, Write}; use std::net::SocketAddr; use regex::Regex; @@ -40,6 +40,10 @@ pub fn format_cycle(cycle: Vec) -> String where T: Display, { - let sep_str: String = cycle.iter().map(|i: &T| format!("{} -> ", i)).collect(); - format!("{}{}", sep_str, cycle[0]) + let mut sep_str = cycle.iter().fold(String::new(), |mut s, i| { + write!(&mut s, "{} -> ", i).unwrap(); + s + }); + write!(&mut sep_str, "{}", cycle[0]).unwrap(); + sep_str } diff --git a/hydroflow/examples/deadlock_detector/main.rs b/hydroflow/examples/deadlock_detector/main.rs index e5261662d70..29f2d1089a2 100644 --- a/hydroflow/examples/deadlock_detector/main.rs +++ b/hydroflow/examples/deadlock_detector/main.rs @@ -56,3 +56,75 @@ async fn main() { let peers = read_addresses_from_file(path).unwrap().peers; run_detector(opts, peers).await; } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_peer1, mut peer1_stdin, mut peer1_stdout) = run_cargo_example( + "deadlock_detector", + &format!( + "--path {}/examples/deadlock_detector/peers.json --addr 127.0.0.1 --port 12346", + env!("CARGO_MANIFEST_DIR") + ), + ); + + let (_peer2, mut peer2_stdin, mut peer2_stdout) = run_cargo_example( + "deadlock_detector", + &format!( + "--path {}/examples/deadlock_detector/peers.json --addr 127.0.0.1 --port 12347", + env!("CARGO_MANIFEST_DIR") + ), + ); + + let (_peer3, mut peer3_stdin, mut peer3_stdout) = run_cargo_example( + "deadlock_detector", + &format!( + "--path {}/examples/deadlock_detector/peers.json --addr 127.0.0.1 --port 12348", + env!("CARGO_MANIFEST_DIR") + ), + ); + + let mut peer3_output = String::new(); + wait_for_process_output( + &mut peer3_output, + &mut peer3_stdout, + "Type in an edge as a tuple of two integers \\(x,y\\):", + ); + let mut peer1_output = String::new(); + wait_for_process_output( + &mut peer1_output, + &mut peer1_stdout, + "Type in an edge as a tuple of two integers \\(x,y\\):", + ); + let mut peer2_output = String::new(); + wait_for_process_output( + &mut peer2_output, + &mut peer2_stdout, + "Type in an edge as a tuple of two integers \\(x,y\\):", + ); + + peer1_stdin.write_all(b"(1, 2)\n").unwrap(); + peer2_stdin.write_all(b"(2, 3)\n").unwrap(); + peer3_stdin.write_all(b"(3, 1)\n").unwrap(); + + wait_for_process_output( + &mut peer1_output, + &mut peer1_stdout, + "path found: 1 -> 2 -> 3 -> 1", + ); + + wait_for_process_output( + &mut peer2_output, + &mut peer2_stdout, + "path found: 1 -> 2 -> 3 -> 1", + ); + + wait_for_process_output( + &mut peer3_output, + &mut peer3_stdout, + "path found: 1 -> 2 -> 3 -> 1", + ); +} diff --git a/hydroflow/examples/deadlock_detector/peer.rs b/hydroflow/examples/deadlock_detector/peer.rs index 1188bf6f08a..2ac479ebc0b 100644 --- a/hydroflow/examples/deadlock_detector/peer.rs +++ b/hydroflow/examples/deadlock_detector/peer.rs @@ -28,13 +28,26 @@ pub(crate) async fn run_detector(opts: Opts, peer_list: Vec) { // set up channels outbound_chan = map(|(m,a)| (serialize_msg(m), a)) -> dest_sink(outbound); - inbound_chan = source_stream(inbound) -> map(deserialize_msg::); + inbound_chan = source_stream(inbound) + -> filter(|msg| { + // For some reason Windows generates connection reset errors on UDP sockets, even though UDP has no sessions. + // This code filters them out. + // `Os { code: 10054, kind: ConnectionReset, message: "An existing connection was forcibly closed by the remote host."` + // https://stackoverflow.com/questions/10332630/connection-reset-on-receiving-packet-in-udp-server + // TODO(mingwei): Clean this up, figure out how to configure windows UDP sockets correctly. + if let Err(tokio_util::codec::LinesCodecError::Io(io_err)) = msg { + io_err.kind() != std::io::ErrorKind::ConnectionReset + } else { + true + } + }) + -> map(deserialize_msg::); // setup gossip channel to all peers. gen_bool chooses True with the odds passed in. - gossip_join = cross_join() + gossip_join = cross_join::<'tick>() -> filter(|_| gen_bool(0.8)) -> outbound_chan; - gossip = map(identity) -> [0]gossip_join; - peers[1] -> [1]gossip_join; + gossip = map(identity) -> persist() -> [0]gossip_join; + peers[1] -> persist() -> [1]gossip_join; peers[2] -> for_each(|s| println!("Peer: {:?}", s)); // prompt for input @@ -46,15 +59,14 @@ pub(crate) async fn run_detector(opts: Opts, peer_list: Vec) { // persist an edges set edges = union() -> tee(); - edges[0] -> next_tick() -> [1]edges; + edges[0] -> defer_tick() -> [1]edges; // add new edges locally new_edges -> [0]edges; // gossip all edges - edges[1] -> fold::<'static>(Message::new(), |mut m, edge| { + edges[1] -> fold::<'static>(Message::new(), |m: &mut Message, edge| { m.edges.insert(edge); - m }) -> gossip; @@ -68,7 +80,7 @@ pub(crate) async fn run_detector(opts: Opts, peer_list: Vec) { // Rule 2: form new_paths from the join of acyclic paths and edges // new_paths(from, to, path.append(to)) :- paths(from, mid, path), edges(mid, to), paths.cycle() == false - new_paths = join() -> map(|(_mid, ((from, mut path), to))| { + new_paths = join::<'static>() -> map(|(_mid, ((from, mut path), to))| { path.push(to); (from, to, path) }) -> tee(); @@ -80,7 +92,7 @@ pub(crate) async fn run_detector(opts: Opts, peer_list: Vec) { // stdio(from, to, path) :- new_paths(from, to, path) new_paths[0] -> filter_map(|(from, to, path): (u32, u32, SimplePath)| if from == to {Some(path.canonical())} else {None}) - -> unique() + -> unique::<'static>() -> for_each(|path: Vec| { println!("path found: {}", format_cycle(path)); }); diff --git a/hydroflow/examples/deletable_state/README.md b/hydroflow/examples/deletable_state/README.md deleted file mode 100644 index d9f1e786bad..00000000000 --- a/hydroflow/examples/deletable_state/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Hello World - -To run: -``` -cargo run -p hydroflow --example hello_world -``` diff --git a/hydroflow/examples/deletable_state/main.rs b/hydroflow/examples/deletable_state/main.rs deleted file mode 100644 index 75ff8a334fd..00000000000 --- a/hydroflow/examples/deletable_state/main.rs +++ /dev/null @@ -1,78 +0,0 @@ -use hydroflow::hydroflow_syntax; - -pub fn main() { - { - println!("persist() -> fold()"); - hydroflow_syntax! { - source_iter([2, 3, 5, 7]) - -> persist() - -> fold::<'tick>(0, |a, b| a + b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - - println!("persist() -> fold() => fold<'static>()"); - hydroflow_syntax! { - source_iter([2, 3, 5, 7]) - -> fold::<'static>(0, |a, b| a + b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - } - - { - println!("persist() -> fold_keyed()"); - hydroflow_syntax! { - source_iter([(0, 2), (0, 3), (1, 5), (0, 7)]) - -> persist() - -> fold_keyed::<'tick>(|| 0, |a: &mut i32, b| *a += b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - - println!("persist() -> fold_keyed<'static>()"); - hydroflow_syntax! { - source_iter([(0, 2), (0, 3), (1, 5), (0, 7)]) - -> fold_keyed::<'static>(|| 0, |a: &mut i32, b| *a += b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - } - - { - use hydroflow::util::Persistence::*; - - println!("persist_mut() -> fold()"); - hydroflow_syntax! { - source_iter([Delete(2), Persist(2), Persist(3), Persist(5), Persist(7), Delete(2)]) - -> persist_mut() - -> fold::<'tick>(0, |a, b| a + b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - - println!("persist_mut() -> fold() => todo!()"); - println!("cannot implement without inverses"); - } - - { - use hydroflow::util::PersistenceKeyed::*; - - println!("persist_mut_keyed() -> fold_keyed()"); - hydroflow_syntax! { - source_iter([Delete(0), Persist(0, 2), Persist(0, 3), Persist(1, 5), Persist(0, 7), Delete(0)]) - -> persist_mut_keyed() - -> fold_keyed::<'tick>(|| 0, |a: &mut i32, b| *a += b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - - println!("persist_mut_keyed() -> fold_keyed() => fold_keyed<'mutable>()"); - hydroflow_syntax! { - source_iter([Delete(0), Persist(0, 2), Persist(0, 3), Persist(1, 5), Persist(0, 7), Delete(0)]) - -> fold_keyed::<'mutable>(|| 0, |a: &mut i32, b| *a += b) - -> for_each(|string| println!("{:?}", string)); - } - .run_available(); - } -} diff --git a/hydroflow/examples/echo_serde_json/main.rs b/hydroflow/examples/echo_serde_json/main.rs index 260bc8770e9..0e5ad583326 100644 --- a/hydroflow/examples/echo_serde_json/main.rs +++ b/hydroflow/examples/echo_serde_json/main.rs @@ -56,3 +56,42 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_output) = run_cargo_example( + "echo_serde_json", + "--role server --server-addr 127.0.0.1:2049", + ); + + let (_client, mut client_input, mut client_output) = run_cargo_example( + "echo_serde_json", + "--role client --server-addr 127.0.0.1:2049", + ); + + let mut server_output_so_far = String::new(); + let mut client_output_so_far = String::new(); + + wait_for_process_output( + &mut server_output_so_far, + &mut server_output, + "Server live!\n", + ); + wait_for_process_output( + &mut client_output_so_far, + &mut client_output, + "Client live!\n", + ); + + client_input.write_all(b"Hello\n").unwrap(); + + wait_for_process_output( + &mut client_output_so_far, + &mut client_output, + "EchoMsg \\{ payload: \"Hello\", ts: .* \\}", + ); +} diff --git a/hydroflow/examples/echoserver/main.rs b/hydroflow/examples/echoserver/main.rs index f25174d90c0..88a69594bc9 100644 --- a/hydroflow/examples/echoserver/main.rs +++ b/hydroflow/examples/echoserver/main.rs @@ -48,3 +48,38 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_output) = + run_cargo_example("echoserver", "--role server --addr 127.0.0.1:2048"); + + let (_client, mut client_input, mut client_output) = + run_cargo_example("echoserver", "--role client --server-addr 127.0.0.1:2048"); + + let mut server_output_so_far = String::new(); + let mut client_output_so_far = String::new(); + + wait_for_process_output( + &mut server_output_so_far, + &mut server_output, + "Server live!\n", + ); + wait_for_process_output( + &mut client_output_so_far, + &mut client_output, + "Client live!\n", + ); + + client_input.write_all(b"Hello\n").unwrap(); + + wait_for_process_output( + &mut client_output_so_far, + &mut client_output, + "UTC: Got EchoMsg \\{ payload: \"Hello\",", + ); +} diff --git a/hydroflow/examples/example_2_simple_1.rs b/hydroflow/examples/example_2_simple_1.rs index d20cd11c797..1fe53455e99 100644 --- a/hydroflow/examples/example_2_simple_1.rs +++ b/hydroflow/examples/example_2_simple_1.rs @@ -4,7 +4,7 @@ pub fn main() { let mut flow = hydroflow_syntax! { source_iter(0..10) -> map(|n| n * n) - -> filter(|&n| n > 10) + -> filter(|n| *n > 10) -> map(|n| (n..=n+1)) -> flatten() -> for_each(|n| println!("Howdy {}", n)); diff --git a/hydroflow/examples/example_5_reachability.rs b/hydroflow/examples/example_5_reachability.rs index 749ce0a2d63..ab8be9e61c5 100644 --- a/hydroflow/examples/example_5_reachability.rs +++ b/hydroflow/examples/example_5_reachability.rs @@ -8,17 +8,20 @@ pub fn main() { // inputs: the origin vertex (vertex 0) and stream of input edges origin = source_iter(vec![0]); stream_of_edges = source_stream(edges_recv); - origin -> [0]reached_vertices; - reached_vertices = union(); // the join reached_vertices -> map(|v| (v, ())) -> [0]my_join_tee; stream_of_edges -> [1]my_join_tee; my_join_tee = join() -> flat_map(|(src, ((), dst))| [src, dst]) -> tee(); - // the loop and the output - my_join_tee[0] -> [1]reached_vertices; - my_join_tee[1] -> unique() -> for_each(|x| println!("Reached: {}", x)); + // the cycle: my_join_tee gets data from reached_vertices + // and provides data back to reached_vertices! + origin -> [base]reached_vertices; + my_join_tee -> [cycle]reached_vertices; + reached_vertices = union(); + + // the output + my_join_tee[print] -> unique() -> for_each(|x| println!("Reached: {}", x)); }; println!( @@ -33,5 +36,6 @@ pub fn main() { edges_send.send((1, 2)).unwrap(); edges_send.send((0, 3)).unwrap(); edges_send.send((0, 3)).unwrap(); + edges_send.send((4, 0)).unwrap(); flow.run_available(); } diff --git a/hydroflow/examples/example_6_unreachability.rs b/hydroflow/examples/example_6_unreachability.rs index 6fd8a2100db..ae3238fb7d1 100644 --- a/hydroflow/examples/example_6_unreachability.rs +++ b/hydroflow/examples/example_6_unreachability.rs @@ -5,25 +5,27 @@ pub fn main() { let (pairs_send, pairs_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let mut flow = hydroflow_syntax! { + // inputs: the origin vertex (vertex 0) and stream of input edges origin = source_iter(vec![0]); stream_of_edges = source_stream(pairs_recv) -> tee(); - reached_vertices = union()->tee(); - origin -> [0]reached_vertices; // the join for reachable vertices - my_join = join() -> flat_map(|(src, ((), dst))| [src, dst]); reached_vertices[0] -> map(|v| (v, ())) -> [0]my_join; stream_of_edges[1] -> [1]my_join; + my_join = join() -> flat_map(|(src, ((), dst))| [src, dst]); - // the loop - my_join -> [1]reached_vertices; + // the cycle: my_join gets data from reached_vertices + // and provides data back to reached_vertices! + origin -> [base]reached_vertices; + my_join -> [cycle]reached_vertices; + reached_vertices = union()->tee(); - // the difference all_vertices - reached_vertices + // the difference: all_vertices - reached_vertices all_vertices = stream_of_edges[0] -> flat_map(|(src, dst)| [src, dst]) -> tee(); - unreached_vertices = difference(); all_vertices[0] -> [pos]unreached_vertices; reached_vertices[1] -> [neg]unreached_vertices; + unreached_vertices = difference(); // the output all_vertices[1] -> unique() -> for_each(|v| println!("Received vertex: {}", v)); diff --git a/hydroflow/examples/example_naturals.rs b/hydroflow/examples/example_naturals.rs new file mode 100644 index 00000000000..e9d72370975 --- /dev/null +++ b/hydroflow/examples/example_naturals.rs @@ -0,0 +1,14 @@ +use hydroflow::hydroflow_syntax; + +pub fn main() { + let mut _flow = hydroflow_syntax! { + base = source_iter(vec![1]) -> cycle; + cycle = union() + -> map(|i| i + 1) + -> inspect(|i| println!("{}", i)) + -> cycle; + }; + + // Let's not run this -- it will go forever! + // flow.run_available(); +} diff --git a/hydroflow/examples/graph_reachability/README.md b/hydroflow/examples/graph_reachability/README.md deleted file mode 100644 index 5b19b7bfd5f..00000000000 --- a/hydroflow/examples/graph_reachability/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Graph Reachability - -To run: -``` -cargo run -p hydroflow --example graph_reachability -``` - -Adding the `-- --graph ` flag to the end of the command line above will print out a node-and-edge diagram of the program. Supported values for `` include [`mermaid`](https://mermaid-js.github.io/), [`dot`](https://graphviz.org/doc/info/lang.html) and `json`. \ No newline at end of file diff --git a/hydroflow/examples/graph_reachability/main.rs b/hydroflow/examples/graph_reachability/main.rs deleted file mode 100644 index 7cd4adf2345..00000000000 --- a/hydroflow/examples/graph_reachability/main.rs +++ /dev/null @@ -1,80 +0,0 @@ -use clap::{Parser, ValueEnum}; -use hydroflow::hydroflow_syntax; - -#[derive(Parser, Debug, Clone, ValueEnum)] -enum GraphType { - Mermaid, - Dot, - Json, -} -#[derive(Parser, Debug)] -struct Opts { - #[clap(value_enum, long)] - graph: Option, -} -pub fn main() { - let opts = Opts::parse(); - // An edge in the input data = a pair of `usize` vertex IDs. - let (edges_send, edges_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); - - let mut df = hydroflow_syntax! { - // inputs: the origin vertex (node 0) and stream of input edges - origin = source_iter(vec![0]); - stream_of_edges = source_stream(edges_recv); - reached_vertices = union(); - origin -> [0]reached_vertices; - - // the join - my_join_tee = join() -> flat_map(|(src, ((), dst))| [src, dst]) -> tee(); - reached_vertices -> map(|v| (v, ())) -> [0]my_join_tee; - stream_of_edges -> [1]my_join_tee; - - // the loop and the output - my_join_tee[0] -> [1]reached_vertices; - my_join_tee[1] -> unique() -> for_each(|x| println!("Reached: {}", x)); - }; - - if let Some(graph) = opts.graph { - let serde_graph = df - .meta_graph() - .expect("No graph found, maybe failed to parse."); - match graph { - GraphType::Mermaid => { - println!("{}", serde_graph.to_mermaid()); - } - GraphType::Dot => { - println!("{}", serde_graph.to_dot()) - } - GraphType::Json => { - unimplemented!(); - // println!("{}", serde_graph.to_json()) - } - } - } - - df.run_available(); - - println!("A"); - - edges_send.send((0, 1)).unwrap(); - edges_send.send((2, 4)).unwrap(); - edges_send.send((3, 4)).unwrap(); - edges_send.send((1, 2)).unwrap(); - edges_send.send((0, 3)).unwrap(); - edges_send.send((0, 3)).unwrap(); - df.run_available(); - - println!("B"); - - edges_send.send((6, 5)).unwrap(); - df.run_available(); - - // A - // Reached: 3 - // Reached: 6 - // Reached: 0 - // B - // Reached: 6 - // Reached: 5 - // Reached: 10 -} diff --git a/hydroflow/examples/graph_unreachability/README.md b/hydroflow/examples/graph_unreachability/README.md deleted file mode 100644 index c3bb3c5f772..00000000000 --- a/hydroflow/examples/graph_unreachability/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Graph UnReachability - -To run: -``` -cargo run -p hydroflow --example graph_unreachability -``` - -Adding the `-- --graph ` flag to the end of the command line above will print out a node-and-edge diagram of the program. Supported values for `` include [`mermaid`](https://mermaid-js.github.io/), [`dot`](https://graphviz.org/doc/info/lang.html) and `json`. \ No newline at end of file diff --git a/hydroflow/examples/graph_unreachability/main.rs b/hydroflow/examples/graph_unreachability/main.rs deleted file mode 100644 index eaa650a1bda..00000000000 --- a/hydroflow/examples/graph_unreachability/main.rs +++ /dev/null @@ -1,93 +0,0 @@ -use clap::{Parser, ValueEnum}; -use hydroflow::hydroflow_syntax; - -#[derive(Parser, Debug, Clone, ValueEnum)] -enum GraphType { - Mermaid, - Dot, - Json, -} -#[derive(Parser, Debug)] -struct Opts { - #[clap(value_enum, long)] - graph: Option, -} -pub fn main() { - let opts = Opts::parse(); - // An edge in the input data = a pair of `usize` vertex IDs. - let (pairs_send, pairs_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); - - let mut df = hydroflow_syntax! { - origin = source_iter(vec![0]); - stream_of_edges = source_stream(pairs_recv) -> tee(); - reached_vertices = union()->tee(); - unreached_vertices = difference(); - origin -> [0]reached_vertices; - - all_vertices = stream_of_edges[0] - -> flat_map(|(src, dst)| [src, dst]) - -> tee(); - - // the join for reachable nodes - my_join = join() -> flat_map(|(src, ((), dst))| [src, dst]); - reached_vertices[0] -> map(|v| (v, ())) -> [0]my_join; - stream_of_edges[1] -> [1]my_join; - - // the loop - my_join -> [1]reached_vertices; - - // the difference all_vertices - reached_vertices - all_vertices[0] -> [pos]unreached_vertices; - reached_vertices[1] -> [neg]unreached_vertices; - - // the output - all_vertices[1] -> for_each(|v| println!("Received vertex: {}", v)); - unreached_vertices -> for_each(|v| println!("unreached_vertices vertex: {}", v)); - }; - - if let Some(graph) = opts.graph { - let serde_graph = df - .meta_graph() - .expect("No graph found, maybe failed to parse."); - match graph { - GraphType::Mermaid => { - println!("{}", serde_graph.to_mermaid()); - } - GraphType::Dot => { - println!("{}", serde_graph.to_dot()) - } - GraphType::Json => { - unimplemented!(); - // println!("{}", serde_graph.to_json()) - } - } - } - - println!("A"); - - pairs_send.send((5, 10)).unwrap(); - pairs_send.send((0, 3)).unwrap(); - pairs_send.send((3, 6)).unwrap(); - // df.run_available(); - - println!("B"); - pairs_send.send((6, 5)).unwrap(); - pairs_send.send((11, 12)).unwrap(); - df.run_available(); - - // A - // Received vertex: 6 - // Received vertex: 10 - // Received vertex: 3 - // Received vertex: 5 - // Received vertex: 0 - // unreached_vertices vertex: 10 - // unreached_vertices vertex: 5 - // B - // Received vertex: 11 - // Received vertex: 6 - // Received vertex: 5 - // Received vertex: 12 - // unreached_vertices vertex: 11 - // unreached_vertices vertex: 12 -} diff --git a/hydroflow/examples/hello_world/main.rs b/hydroflow/examples/hello_world/main.rs index 50793a1a6f7..7b70f9daf2b 100644 --- a/hydroflow/examples/hello_world/main.rs +++ b/hydroflow/examples/hello_world/main.rs @@ -2,10 +2,13 @@ use hydroflow::hydroflow_syntax; pub fn main() { let mut df = hydroflow_syntax! { - source_iter([1,2,3,4,5]) - -> map(hydroflow::lattices::Max::new) - -> lattice_merge::<'static, hydroflow::lattices::Max>() - -> assert([hydroflow::lattices::Max(5)]); + source_iter(["Hello World"]) + -> assert_eq(["Hello World"]); }; df.run_available(); } + +#[test] +fn test() { + main(); +} diff --git a/hydroflow/examples/kvs/main.rs b/hydroflow/examples/kvs/main.rs index 4f2953af2b6..a08be5cb5e0 100644 --- a/hydroflow/examples/kvs/main.rs +++ b/hydroflow/examples/kvs/main.rs @@ -60,3 +60,48 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_stdout) = + run_cargo_example("kvs", "--role server --addr 127.0.0.1:2051"); + + let (_client, mut client_stdin, mut client_stdout) = run_cargo_example( + "kvs", + "--role client --addr 127.0.0.1:2052 --server-addr 127.0.0.1:2051", + ); + + let mut server_output = String::new(); + wait_for_process_output(&mut server_output, &mut server_stdout, "Server live!"); + + let mut client_output = String::new(); + wait_for_process_output(&mut client_output, &mut client_stdout, "Client live!"); + + client_stdin.write_all(b"PUT a,7\n").unwrap(); + + wait_for_process_output( + &mut client_output, + &mut client_stdout, + r#"Got a Response: Response \{ key: "a", value: "7" \}"#, + ); + + let (_client2, mut client2_stdin, mut client2_stdout) = run_cargo_example( + "kvs", + "--role client --addr 127.0.0.1:2053 --server-addr 127.0.0.1:2051", + ); + + let mut client2_output = String::new(); + wait_for_process_output(&mut client2_output, &mut client2_stdout, "Client live!"); + + client2_stdin.write_all(b"GET a\n").unwrap(); + + wait_for_process_output( + &mut client2_output, + &mut client2_stdout, + r#"Got a Response: Response \{ key: "a", value: "7" \}"#, + ); +} diff --git a/hydroflow/examples/kvs/server.rs b/hydroflow/examples/kvs/server.rs index b81b2efd2b1..4adb1d8e1e9 100644 --- a/hydroflow/examples/kvs/server.rs +++ b/hydroflow/examples/kvs/server.rs @@ -42,7 +42,7 @@ pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream, graph: Opt -> [0]outbound_chan; // join PUTs and GETs by key - lookup = join()->tee(); + lookup = join::<'static>()->tee(); parsed_puts[1] -> map(|(key, value, _)| (key, value)) -> [0]lookup; parsed_gets -> [1]lookup; lookup[0] -> for_each(|t| println!("Found a match: {:?}", t)); diff --git a/hydroflow/examples/kvs_bench/main.rs b/hydroflow/examples/kvs_bench/main.rs index f5e8ab148fd..729d73c05c7 100644 --- a/hydroflow/examples/kvs_bench/main.rs +++ b/hydroflow/examples/kvs_bench/main.rs @@ -166,3 +166,13 @@ fn main() { } } } + +#[test] +fn test() { + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_stdout) = run_cargo_example("kvs_bench", "bench --threads 2"); + + let mut server_output = String::new(); + wait_for_process_output(&mut server_output, &mut server_stdout, r#"[0-9]+\.[0-9]+"#); +} diff --git a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/map_union.rs b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/map_union.rs index 04bd5b35098..1be9a9c4e9b 100644 --- a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/map_union.rs +++ b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/map_union.rs @@ -22,7 +22,7 @@ impl<'a, const SIZE: usize> Serialize for MapUnionHashMapWrapper<'a, SIZE> { { use serde::ser::SerializeMap; - let inner_map = &self.0 .0; + let inner_map = self.0.as_reveal_ref(); let mut map_serializer = serializer.serialize_map(Some(inner_map.len()))?; @@ -64,7 +64,7 @@ impl<'de, const SIZE: usize> DeserializeSeed<'de> for MapUnionHashMapDeserialize let k: Option = map.next_key()?; if let Some(k) = k { - inner_map.0.insert( + inner_map.as_reveal_mut().insert( k, map.next_value_seed(MyLastWriteWinsDeserializer { collector: self.collector.clone(), diff --git a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/my_last_write_wins.rs b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/my_last_write_wins.rs index d25954741d0..91fc1b71dfb 100644 --- a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/my_last_write_wins.rs +++ b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/my_last_write_wins.rs @@ -20,8 +20,9 @@ impl<'a, const SIZE: usize> Serialize for MyLastWriteWinsWrapper<'a, SIZE> { let mut struct_serializer = serializer.serialize_struct("DomPair", 2)?; - struct_serializer.serialize_field("key", &self.0.key)?; - struct_serializer.serialize_field("val", &WithBotWrapper(&self.0.val))?; + let (key, val) = self.0.as_reveal_ref(); + struct_serializer.serialize_field("key", key)?; + struct_serializer.serialize_field("val", &WithBotWrapper(val))?; struct_serializer.end() } @@ -58,7 +59,7 @@ impl<'de, const SIZE: usize> DeserializeSeed<'de> for MyLastWriteWinsDeserialize })? .unwrap(); - Ok(Self::Value { key, val }) + Ok(Self::Value::new(key, val)) } fn visit_map(self, mut map: A) -> Result @@ -90,7 +91,7 @@ impl<'de, const SIZE: usize> DeserializeSeed<'de> for MyLastWriteWinsDeserialize let key = key.unwrap(); let val = val.unwrap(); - Ok(Self::Value { key, val }) + Ok(Self::Value::new(key, val)) } } diff --git a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/point.rs b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/point.rs index e700b20562d..997b125a5f8 100644 --- a/hydroflow/examples/kvs_bench/protocol/serialization/lattices/point.rs +++ b/hydroflow/examples/kvs_bench/protocol/serialization/lattices/point.rs @@ -15,7 +15,7 @@ impl<'a, const SIZE: usize> Serialize for PointWrapper<'a, SIZE> { where S: Serializer, { - serializer.serialize_newtype_struct("Point", &self.0) + serializer.serialize_newtype_struct("Point", &self.0.val) } } diff --git a/hydroflow/examples/kvs_bench/protocol/test/mod.rs b/hydroflow/examples/kvs_bench/protocol/test/mod.rs index 01b3645d8c9..8a36212ffe2 100644 --- a/hydroflow/examples/kvs_bench/protocol/test/mod.rs +++ b/hydroflow/examples/kvs_bench/protocol/test/mod.rs @@ -23,7 +23,7 @@ fn test_gossip() { let reg = MyLastWriteWins::new(Max::new(49), WithBot::new_from(Point::new(buffer))); let mut map = MapUnionHashMap::default(); - map.0.insert(7, reg); + map.as_reveal_mut().insert(7, reg); KvsRequest::Gossip { map } }; @@ -71,7 +71,7 @@ fn test_json() { let reg = MyLastWriteWins::new(Max::new(49), WithBot::new_from(Point::new(buffer))); let mut map = MapUnionHashMap::default(); - map.0.insert(7, reg); + map.as_reveal_mut().insert(7, reg); KvsRequest::Gossip { map } }; @@ -83,16 +83,10 @@ fn test_json() { serde::ser::Serialize::serialize(&req, &mut serializer).unwrap(); println!("serialized: {}", std::str::from_utf8(&serialized).unwrap()); - println!( - "serialized: {}", - serde_json::to_string(&MapUnionHashMap::new_from([( - 7, - MyLastWriteWins::new( - Max::new(49), - WithBot::new_from(Point::new(BufferPool::get_from_buffer_pool(&buffer_pool))) - ) - )])) - .unwrap() + + assert_eq!( + std::str::from_utf8(&serialized).unwrap(), + serde_json::to_string(&req).unwrap() ); let mut deserializer = @@ -120,7 +114,7 @@ fn test_bincode() { let reg = MyLastWriteWins::new(Max::new(49), WithBot::new_from(Point::new(buffer))); let mut map = MapUnionHashMap::default(); - map.0.insert(7, reg); + map.as_reveal_mut().insert(7, reg); KvsRequest::Gossip { map } }; diff --git a/hydroflow/examples/kvs_bench/server.rs b/hydroflow/examples/kvs_bench/server.rs index 78cfa0d7a5c..f8586df3d3c 100644 --- a/hydroflow/examples/kvs_bench/server.rs +++ b/hydroflow/examples/kvs_bench/server.rs @@ -114,10 +114,6 @@ pub fn run_server( } }); - let batch_interval_ticker = tokio_stream::wrappers::IntervalStream::new(tokio::time::interval( - Duration::from_millis(100), - )); - let mut rng = rand::rngs::SmallRng::from_entropy(); let dist = rand_distr::Zipf::new(1_000_000, dist).unwrap(); @@ -198,7 +194,7 @@ pub fn run_server( ))); }, KvsRequest::Gossip {map} => { - for (key, reg) in map.0 { + for (key, reg) in map.into_reveal() { store.give((key, reg)); } }, @@ -222,10 +218,15 @@ pub fn run_server( peers = cross_join::<'static, 'tick, HalfMultisetJoinState>(); source_iter(topology.lookup) -> [0]peers; + ticker = source_interval(Duration::from_millis(100)) + -> [signal]batcher; + // broadcast out locally generated changes to other nodes. client_input[broadcast] -> map(|(key, reg)| MapUnionSingletonMap::new_from((key, reg))) - -> lattice_batch::>(batch_interval_ticker) + -> [input]batcher; + + batcher = _lattice_fold_batch::>() -> map(|lattice| { use bincode::Options; let serialization_options = options(); @@ -243,7 +244,7 @@ pub fn run_server( -> for_each(|(node_id, serialized_req)| transducer_to_peers_tx.try_send((serialized_req, node_id)).unwrap()); // join for lookups - lookup = lattice_join::<'static, 'tick, MyLastWriteWins, MySetUnion>(); + lookup = _lattice_join_fused_join::<'static, 'tick, MyLastWriteWins, MySetUnion>(); client_input[store] // -> inspect(|x| println!("{server_id}:{:5}: stores-into-lookup: {x:?}", context.current_tick())) diff --git a/hydroflow/examples/lamport_clock/client.rs b/hydroflow/examples/lamport_clock/client.rs index b9a8f92ee47..585dbcea620 100644 --- a/hydroflow/examples/lamport_clock/client.rs +++ b/hydroflow/examples/lamport_clock/client.rs @@ -11,7 +11,7 @@ use crate::{GraphType, Opts}; pub(crate) async fn run_client(outbound: UdpSink, inbound: UdpStream, opts: Opts) { // server_addr is required for client let server_addr = opts.server_addr.expect("Client requires a server address"); - let bot: Max = Max(0); + let bot: Max = Max::new(0); println!("Client live!"); @@ -29,11 +29,10 @@ pub(crate) async fn run_client(outbound: UdpSink, inbound: UdpStream, opts: Opts inbound_chan[merge] -> map(|(msg, _sender): (EchoMsg, SocketAddr)| msg.lamport_clock) -> [net]mergevc; mergevc = union() -> fold::<'static>( bot, - |mut old: Max, lamport_clock: Max| { - let bump = Max(old.0 + 1); + |old: &mut Max, lamport_clock: Max| { + let bump = Max::new(old.into_reveal() + 1); old.merge(bump); old.merge(lamport_clock); - old } ); diff --git a/hydroflow/examples/lamport_clock/main.rs b/hydroflow/examples/lamport_clock/main.rs index 035a8282652..19d4a3c99d2 100644 --- a/hydroflow/examples/lamport_clock/main.rs +++ b/hydroflow/examples/lamport_clock/main.rs @@ -57,3 +57,48 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_stdout) = + run_cargo_example("lamport_clock", "--role server --addr 127.0.0.1:11052"); + + let (_client1, mut client1_stdin, mut client1_stdout) = run_cargo_example( + "lamport_clock", + "--role client --server-addr 127.0.0.1:11052", + ); + + let (_client2, mut client2_stdin, mut client2_stdout) = run_cargo_example( + "lamport_clock", + "--role client --server-addr 127.0.0.1:11052", + ); + + let mut server_output = String::new(); + wait_for_process_output(&mut server_output, &mut server_stdout, "Server live!"); + + let mut client1_output = String::new(); + wait_for_process_output(&mut client1_output, &mut client1_stdout, "Client live!"); + + let mut client2_output = String::new(); + wait_for_process_output(&mut client2_output, &mut client2_stdout, "Client live!"); + + client1_stdin.write_all(b"Hello1\n").unwrap(); + + wait_for_process_output( + &mut client1_output, + &mut client1_stdout, + r#"UTC: Got EchoMsg \{ payload: "Hello1", lamport_clock: Max\(1\) \} from 127.0.0.1:11052"#, + ); + + client2_stdin.write_all(b"Hello2\n").unwrap(); + + wait_for_process_output( + &mut client2_output, + &mut client2_stdout, + r#"UTC: Got EchoMsg \{ payload: "Hello2", lamport_clock: Max\(2\) \} from 127.0.0.1:11052"#, + ); +} diff --git a/hydroflow/examples/lamport_clock/server.rs b/hydroflow/examples/lamport_clock/server.rs index b800f383a93..3d6d58be152 100644 --- a/hydroflow/examples/lamport_clock/server.rs +++ b/hydroflow/examples/lamport_clock/server.rs @@ -9,7 +9,7 @@ use hydroflow::util::{UdpSink, UdpStream}; use crate::protocol::EchoMsg; pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream) { - let bot: Max = Max(0); + let bot: Max = Max::new(0); println!("Server live!"); let mut flow: Hydroflow = hydroflow_syntax! { @@ -24,11 +24,10 @@ pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream) { inbound_chan[merge] -> map(|(msg, _addr): (EchoMsg, SocketAddr)| msg.lamport_clock) -> mergevc; mergevc = fold::<'static>( bot, - |mut old: Max, lamport_clock: Max| { - let bump = Max(old.0 + 1); + |old: &mut Max, lamport_clock: Max| { + let bump = Max::new(old.into_reveal() + 1); old.merge(bump); old.merge(lamport_clock); - old } ); diff --git a/hydroflow/examples/modules/main.rs b/hydroflow/examples/modules/main.rs new file mode 100644 index 00000000000..af640f888ac --- /dev/null +++ b/hydroflow/examples/modules/main.rs @@ -0,0 +1,48 @@ +use std::cell::RefCell; +use std::rc::Rc; + +use hydroflow::hydroflow_syntax; +use hydroflow::scheduled::graph::Hydroflow; +use hydroflow::util::multiset::HashMultiSet; + +pub fn main() { + let output = Rc::new(RefCell::new( + HashMultiSet::<(usize, usize, usize)>::default(), + )); + + let mut df: Hydroflow = { + let output = output.clone(); + hydroflow_syntax! { + source_iter(0..2) -> [0]cj; + source_iter(0..2) -> [1]cj; + source_iter(0..2) -> [2]cj; + + cj = import!("triple_cross_join.hf") + -> for_each(|x| output.borrow_mut().insert(x)); + } + }; + + println!("{}", df.meta_graph().unwrap().to_mermaid()); + + df.run_available(); + + #[rustfmt::skip] + assert_eq!( + *output.borrow(), + HashMultiSet::from_iter([ + (0, 0, 0), + (0, 0, 1), + (0, 1, 0), + (0, 1, 1), + (1, 0, 0), + (1, 0, 1), + (1, 1, 0), + (1, 1, 1), + ]) + ); +} + +#[test] +fn test() { + main(); +} diff --git a/hydroflow/examples/modules/triple_cross_join.hf b/hydroflow/examples/modules/triple_cross_join.hf new file mode 100644 index 00000000000..a5126ec2e34 --- /dev/null +++ b/hydroflow/examples/modules/triple_cross_join.hf @@ -0,0 +1,15 @@ +mod[0] + -> [0]cj1; + +mod[1] + -> [1]cj1; + +cj1 = cross_join() + -> [0]cj2; + +mod[2] + -> [1]cj2; + +cj2 = cross_join() + -> map(|((a, b), c)| (a, b, c)) + -> mod; diff --git a/hydroflow/examples/python_udf/main.rs b/hydroflow/examples/python_udf/main.rs new file mode 100644 index 00000000000..c9f2a195aca --- /dev/null +++ b/hydroflow/examples/python_udf/main.rs @@ -0,0 +1,27 @@ +use hydroflow_macro::hydroflow_syntax; +use pyo3::{Py, PyAny, PyResult, Python}; + +#[hydroflow::main] +async fn main() { + eprintln!("Vec sender starting..."); + + let v = vec![1, 2, 3, 4, 5]; + + let mut df = hydroflow_syntax! { + source_iter(v) -> inspect( + |x| println!("input:\t{:?}", x) + ) + // Map to tuples + -> map(|x| (x, 1)) + -> py_udf(r#" +def add(a, b): + return a + 1 + "#, "add") + -> map(|x: PyResult>| -> i32 {Python::with_gil(|py| { + x.unwrap().extract(py).unwrap() + })}) + -> for_each(|x| println!("output:\t{:?}", x)); + }; + + df.run_available(); +} diff --git a/hydroflow/examples/rga/README.md b/hydroflow/examples/rga/README.md index dc8e61b633a..8c7cc303b09 100644 --- a/hydroflow/examples/rga/README.md +++ b/hydroflow/examples/rga/README.md @@ -6,12 +6,12 @@ It then outputs a graph in the DOT format, which can be rendered with Graphviz - There are multiple implementations to choose from, via the `--impl` flag: - A `minimal` implementation is nothing more than a set of (child, parent) edges. The Hydroflow code here does essentially nothing beyond collecting edges and outputting them. No ordering is produced. - A `datalog` implementation based on a [talk by Martin Kleppman](https://speakerdeck.com/ept/data-structures-as-queries-expressing-crdts-using-datalog). Kleppman's dataflow has been hand-compiled to Hydroflow, rule-by-rule. -- A `datalog_agg` implementation that modifies Kleppman's code for Datalog-with-aggregation, again hand-compiled to Hydroflow +- A `datalog-agg` implementation that modifies Kleppman's code for Datalog-with-aggregation, again hand-compiled to Hydroflow - An `adjacency` implementation that builds an adjacency list in Hydroflow to reduce the number of aggregation passes. *This is the default if you do not specify the `--impl` flag.* To run: ``` -cargo run -p hydroflow --example rga -- -o +cargo run -p hydroflow --example rga -- --impl datalog ``` -Optionally append `--impl ` to choose an implementation among {`minimal`, `datalog`, `datalog_agg`, `adjacency`} +Optionally append `--impl ` to choose an implementation among {`minimal`, `datalog`, `datalog-agg`, `adjacency`} and append `--graph ` to also print a graph of the hydroflow code to stdout in one of the formats {`dot`, `mermaid`, `json`}. diff --git a/hydroflow/examples/rga/main.rs b/hydroflow/examples/rga/main.rs index 8d21f7f021e..7fa3940c3d2 100644 --- a/hydroflow/examples/rga/main.rs +++ b/hydroflow/examples/rga/main.rs @@ -37,8 +37,6 @@ struct Opts { implementation: Option, #[clap(value_enum, long, short)] graph: Option, - #[clap(value_enum, long, short)] - output: String, } #[hydroflow::main] @@ -91,7 +89,7 @@ pub async fn main() { let mut output = String::new(); write_to_dot(&mut rga_recv, &mut list_recv, &mut output).await; - std::fs::write(opts.output, output).expect("write to output file failed"); + println!("{}", output); } async fn keystroke( @@ -167,3 +165,124 @@ fn format_dot_node(n: Token) -> String { n.ts, n.value, n.ts ) } + +#[test] +fn test() { + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + fn escape_regex(input: &str) -> String { + input + .replace("[", "\\[") + .replace("]", "\\]") + .replace("{", "\\{") + .replace("}", "\\}") + } + + { + let (_child, _, mut stdout) = run_cargo_example("rga", "--impl adjacency"); + + let mut output = String::new(); + for line in EXPECTED_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("rga", "--impl datalog"); + + let mut output = String::new(); + for line in EXPECTED_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("rga", "--impl minimal"); + + let mut output = String::new(); + for line in EXPECTED_OUTPUT_MINIMAL.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + // TODO: This implementation appears to be broken. + // { + // let (_, _, mut stdout) = spawn("rga", "--impl datalog-agg"); + + // let mut output = String::new(); + // for line in EXPECTED_OUTPUT_MINIMAL.split("\n") { + // wait_for_output(&mut output, &mut stdout, &escape_regex(line)); + // } + // } +} + +// Output can be re-ordered, so this will be tested line by line +#[cfg(test)] +const EXPECTED_OUTPUT: &str = r#"digraph G { +rankdir = TB +ts0_0 [label=ts0_0>] +ts4_0 -> ts5_0 [color=gray, constraint=true] +ts5_0 [label=ts5_0>] +ts9_0 -> ts10_0 [color=gray, constraint=true] +ts10_0 [label=ts10_0>] +ts6_1 -> ts7_1 [color=gray, constraint=true] +ts7_1 [label=ts7_1>] +ts0_0 -> ts1_0 [color=gray, constraint=true] +ts1_0 [label=ts1_0>] +ts0_0 -> ts8_0 [color=gray, constraint=true] +ts8_0 [label=ts8_0>] +ts1_0 -> ts2_0 [color=gray, constraint=true] +ts2_0 [label=ts2_0>] +ts8_0 -> ts9_0 [color=gray, constraint=true] +ts9_0 [label=ts9_0>] +ts2_0 -> ts3_0 [color=gray, constraint=true] +ts3_0 [label=ts3_0>] +ts10_0 -> ts11_0 [color=gray, constraint=true] +ts11_0 [label=ts11_0>] +ts2_0 -> ts6_1 [color=gray, constraint=true] +ts6_1 [label=ts6_1>] +ts3_0 -> ts4_0 [color=gray, constraint=true] +ts4_0 [label=ts4_0>] +ts0_0 -> ts8_0 [style=dashed, color=blue, constraint=false] +ts9_0 -> ts10_0 [style=dashed, color=blue, constraint=false] +ts6_1 -> ts7_1 [style=dashed, color=blue, constraint=false] +ts11_0 -> ts1_0 [style=dashed, color=blue, constraint=false] +ts7_1 -> ts3_0 [style=dashed, color=blue, constraint=false] +ts2_0 -> ts6_1 [style=dashed, color=blue, constraint=false] +ts4_0 -> ts5_0 [style=dashed, color=blue, constraint=false] +ts10_0 -> ts11_0 [style=dashed, color=blue, constraint=false] +ts3_0 -> ts4_0 [style=dashed, color=blue, constraint=false] +ts8_0 -> ts9_0 [style=dashed, color=blue, constraint=false] +ts1_0 -> ts2_0 [style=dashed, color=blue, constraint=false] +label=<Collaborate> +}"#; + +// Output can be re-ordered, so this will be tested line by line +#[cfg(test)] +const EXPECTED_OUTPUT_MINIMAL: &str = r#"digraph G { +rankdir = TB +ts0_0 [label=ts0_0>] +ts3_0 -> ts4_0 [color=gray, constraint=true] +ts4_0 [label=ts4_0>] +ts0_0 -> ts8_0 [color=gray, constraint=true] +ts8_0 [label=ts8_0>] +ts2_0 -> ts3_0 [color=gray, constraint=true] +ts3_0 [label=ts3_0>] +ts6_1 -> ts7_1 [color=gray, constraint=true] +ts7_1 [label=ts7_1>] +ts0_0 -> ts1_0 [color=gray, constraint=true] +ts1_0 [label=ts1_0>] +ts10_0 -> ts11_0 [color=gray, constraint=true] +ts11_0 [label=ts11_0>] +ts8_0 -> ts9_0 [color=gray, constraint=true] +ts9_0 [label=ts9_0>] +ts2_0 -> ts6_1 [color=gray, constraint=true] +ts6_1 [label=ts6_1>] +ts9_0 -> ts10_0 [color=gray, constraint=true] +ts10_0 [label=ts10_0>] +ts1_0 -> ts2_0 [color=gray, constraint=true] +ts2_0 [label=ts2_0>] +ts4_0 -> ts5_0 [color=gray, constraint=true] +ts5_0 [label=ts5_0>] +label=<Unknown> +}"#; diff --git a/hydroflow/examples/shopping/driver.rs b/hydroflow/examples/shopping/driver.rs index f9d5f918332..f07238b2221 100644 --- a/hydroflow/examples/shopping/driver.rs +++ b/hydroflow/examples/shopping/driver.rs @@ -123,6 +123,13 @@ pub(crate) async fn run_driver(opts: Opts) { }); }); + // The above thread spawn sets up some sockets, but we do not wait for it to do so. + // So there is a race condition where, while the thread is still spawning, we can proceed and send messages to a socket that is not bound. + // This means that the spawned thread would be waiting for a message that it has already missed and which won't get re-sent. + // There's a timeout at the bottom of this function that was presumably added to fix the issue of this test hanging. + // This sleep makes it much more likely that the thread has set up everything needed by the tiem we finish this sleep. + std::thread::sleep(std::time::Duration::from_secs(1)); + // Run client proxy in this thread match opt { 5 => { @@ -153,7 +160,7 @@ pub(crate) async fn run_driver(opts: Opts) { let addr1 = ipv4_resolve("localhost:23430").unwrap(); let addr2 = ipv4_resolve("localhost:23431").unwrap(); let addr3 = ipv4_resolve("localhost:23432").unwrap(); - let server_addrs = vec![addr1, addr2, addr3]; + let server_addrs = [addr1, addr2, addr3]; // define the server addresses for gossip let gossip_addr1 = ipv4_resolve("localhost:23440").unwrap(); @@ -200,6 +207,14 @@ pub(crate) async fn run_driver(opts: Opts) { }); }); } + + // The above thread spawn sets up some sockets, but we do not wait for it to do so. + // So there is a race condition where, while the thread is still spawning, we can proceed and send messages to a socket that is not bound. + // This means that the spawned thread would be waiting for a message that it has already missed and which won't get re-sent. + // There's a timeout at the bottom of this function that was presumably added to fix the issue of this test hanging. + // This sleep makes it much more likely that the thread has set up everything needed by the tiem we finish this sleep. + std::thread::sleep(std::time::Duration::from_secs(1)); + // Run client proxy in this thread rep_server_flow( shopping_ssiv, diff --git a/hydroflow/examples/shopping/flows/bp_flow.rs b/hydroflow/examples/shopping/flows/bp_flow.rs index 0a3e9857b1e..93e8e8a6f08 100644 --- a/hydroflow/examples/shopping/flows/bp_flow.rs +++ b/hydroflow/examples/shopping/flows/bp_flow.rs @@ -31,9 +31,9 @@ pub(crate) async fn bp_flow( // BP, two customer classes source_iter(shopping_bp) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) - -> fold_keyed(BP_BOT, bp_merge) + -> fold_keyed::<'static>(BP_BOT, bp_merge) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } } diff --git a/hydroflow/examples/shopping/flows/client_state_flow.rs b/hydroflow/examples/shopping/flows/client_state_flow.rs index ea80eef5ace..0d12ed44ed4 100644 --- a/hydroflow/examples/shopping/flows/client_state_flow.rs +++ b/hydroflow/examples/shopping/flows/client_state_flow.rs @@ -34,12 +34,12 @@ pub(crate) async fn client_state_flow( // The second transducer listens on reqs_in and runs the lookup join. hydroflow_syntax! { source_iter(shopping_ssiv) - -> fold_keyed(SSIV_BOT, ssiv_merge) + -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> map(|pair| (pair, remote_addr)) -> dest_sink_serde(carts_out); source_stream_serde(carts_in) -> map(Result::unwrap) -> map(|((client, cart), _a): ((usize, SealedSetOfIndexedValues), _)| (client, cart)) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } diff --git a/hydroflow/examples/shopping/flows/orig_flow.rs b/hydroflow/examples/shopping/flows/orig_flow.rs index c0197e5687a..d5803399619 100644 --- a/hydroflow/examples/shopping/flows/orig_flow.rs +++ b/hydroflow/examples/shopping/flows/orig_flow.rs @@ -25,9 +25,9 @@ pub(crate) async fn orig_flow( // the original flow source_iter(shopping) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) - -> fold_keyed(Vec::new, Vec::push) + -> fold_keyed::<'static>(Vec::new, Vec::push) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } } diff --git a/hydroflow/examples/shopping/flows/push_group_flow.rs b/hydroflow/examples/shopping/flows/push_group_flow.rs index c1265b3df63..4f22e3a0c96 100644 --- a/hydroflow/examples/shopping/flows/push_group_flow.rs +++ b/hydroflow/examples/shopping/flows/push_group_flow.rs @@ -31,9 +31,9 @@ pub(crate) async fn push_group_flow( // (basic or prime) via a join operator, and generate the output. hydroflow_syntax! { // push fold_keyed through join - source_iter(shopping_ssiv) -> fold_keyed(SSIV_BOT, ssiv_merge) -> [0]lookup_class; + source_iter(shopping_ssiv) -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } diff --git a/hydroflow/examples/shopping/flows/rep_server_flow.rs b/hydroflow/examples/shopping/flows/rep_server_flow.rs index 070d2012c49..26b8e446e11 100644 --- a/hydroflow/examples/shopping/flows/rep_server_flow.rs +++ b/hydroflow/examples/shopping/flows/rep_server_flow.rs @@ -39,21 +39,21 @@ pub(crate) async fn rep_server_flow( source_stream_serde(reqs_in) -> map(Result::unwrap) -> map(|((client, req), _a): ((usize, SealedSetOfIndexedValues), _)| (client, req)) - -> fold_keyed(SSIV_BOT, ssiv_merge) + -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li) ) -> tee(); lookup_class[clients] -> all_in; lookup_class[broadcast] -> [0]broadcast; source_iter(server_addrs) -> [1]broadcast; - broadcast = cross_join() -> dest_sink_serde(broadcast_out); + broadcast = cross_join::<'static>() -> dest_sink_serde(broadcast_out); source_stream_serde(broadcast_in) -> map(Result::unwrap) -> map(|(m, _a): (((usize, ClientClass), SealedSetOfIndexedValues), _)| m) -> all_in; all_in = union() - -> fold_keyed(SSIV_BOT, ssiv_merge) + -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> unique() -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } diff --git a/hydroflow/examples/shopping/flows/server_state_flow.rs b/hydroflow/examples/shopping/flows/server_state_flow.rs index dcbd8da8641..d20416b654d 100644 --- a/hydroflow/examples/shopping/flows/server_state_flow.rs +++ b/hydroflow/examples/shopping/flows/server_state_flow.rs @@ -37,9 +37,9 @@ pub(crate) async fn server_state_flow( source_iter(shopping_ssiv) -> map(|pair| (pair, remote_addr)) -> dest_sink_serde(reqs_out); source_stream_serde(reqs_in) -> map(Result::unwrap) -> map(|((client, req), _a): ((usize, SealedSetOfIndexedValues), _)| (client, req)) - -> fold_keyed(SSIV_BOT, ssiv_merge) -> [0]lookup_class; + -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } diff --git a/hydroflow/examples/shopping/flows/ssiv_flow.rs b/hydroflow/examples/shopping/flows/ssiv_flow.rs index a2200e68046..154d224c2cc 100644 --- a/hydroflow/examples/shopping/flows/ssiv_flow.rs +++ b/hydroflow/examples/shopping/flows/ssiv_flow.rs @@ -31,9 +31,9 @@ pub(crate) async fn ssiv_flow( hydroflow_syntax! { source_iter(shopping_ssiv) -> [0]lookup_class; source_iter(client_class) -> [1]lookup_class; - lookup_class = join() + lookup_class = join::<'static>() -> map(|(client, (li, class))| ((client, class), li)) - -> fold_keyed(SSIV_BOT, ssiv_merge) + -> fold_keyed::<'static>(SSIV_BOT, ssiv_merge) -> map(|m| (m, out_addr)) -> dest_sink_serde(out); } } diff --git a/hydroflow/examples/shopping/main.rs b/hydroflow/examples/shopping/main.rs index bda05d7046f..66d17c9818a 100644 --- a/hydroflow/examples/shopping/main.rs +++ b/hydroflow/examples/shopping/main.rs @@ -38,3 +38,136 @@ async fn main() { // all the interesting logic is in the driver run_driver(opts).await; } + +#[test] +fn test() { + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + fn escape_regex(input: &str) -> String { + input + .replace("(", "\\(") + .replace(")", "\\)") + .replace("{", "\\{") + .replace("}", "\\}") + .replace("[", "\\[") + .replace("]", "\\]") + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 1"); + + let mut output = String::new(); + for line in OPT1_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 2"); + + let mut output = String::new(); + for line in OPT2_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 3"); + + let mut output = String::new(); + for line in OPT3_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 4"); + + let mut output = String::new(); + for line in OPT4_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 5"); + + let mut output = String::new(); + for line in OPT5_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 6"); + + let mut output = String::new(); + for line in OPT6_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } + + { + let (_child, _, mut stdout) = run_cargo_example("shopping", "--opt 7"); + + let mut output = String::new(); + for line in OPT7_OUTPUT.split("\n") { + wait_for_process_output(&mut output, &mut stdout, &escape_regex(line)); + } + } +} + +#[cfg(test)] +const OPT1_OUTPUT: &str = r#" +((2, Basic), [LineItem { name: "apple", qty: 1 }, LineItem { name: "apple", qty: -1 }, LineItem { name: "", qty: 0 }]) +((1, Basic), [LineItem { name: "apple", qty: 1 }, LineItem { name: "banana", qty: 6 }, LineItem { name: "", qty: 0 }]) +((100, Prime), [LineItem { name: "potato", qty: 1 }, LineItem { name: "ferrari", qty: 1 }, LineItem { name: "", qty: 0 }]) +"#; + +#[cfg(test)] +const OPT2_OUTPUT: &str = r#" +((2, Basic), BoundedPrefix { vec: [ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }, Checkout { client: 2 }], len: Some(3) }) +((1, Basic), BoundedPrefix { vec: [ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }, Checkout { client: 1 }], len: Some(3) }) +((100, Prime), BoundedPrefix { vec: [ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }, Checkout { client: 100 }], len: Some(3) }) +"#; + +#[cfg(test)] +const OPT3_OUTPUT: &str = r#" +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +"#; + +#[cfg(test)] +const OPT4_OUTPUT: &str = r#" +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +"#; + +#[cfg(test)] +const OPT5_OUTPUT: &str = r#" +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +"#; + +#[cfg(test)] +const OPT6_OUTPUT: &str = r#" +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +"#; + +#[cfg(test)] +const OPT7_OUTPUT: &str = r#" +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +((2, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 2, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 2, li: LineItem { name: "apple", qty: -1 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((1, Basic), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 1, li: LineItem { name: "apple", qty: 1 } }, 1: ClLineItem { client: 1, li: LineItem { name: "banana", qty: 6 } }}, len: Some(3) }) +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +((100, Prime), SealedSetOfIndexedValues { set: {0: ClLineItem { client: 100, li: LineItem { name: "potato", qty: 1 } }, 1: ClLineItem { client: 100, li: LineItem { name: "ferrari", qty: 1 } }}, len: Some(3) }) +"#; diff --git a/hydroflow/examples/three_clique/main.rs b/hydroflow/examples/three_clique/main.rs index 78e067a8f80..9ddb7c491c8 100644 --- a/hydroflow/examples/three_clique/main.rs +++ b/hydroflow/examples/three_clique/main.rs @@ -24,9 +24,9 @@ pub fn main() { // set up the two joins // edge_pairs((z,x), y) :- edges(x,y), edges(y,z) - edge_pairs = join() -> map(|(y, (x,z))| ((z,x), y)); //Here we have found all paths from x to z that go through y. Now we need to find edges that connect z back to x. + edge_pairs = join::<'static>() -> map(|(y, (x,z))| ((z,x), y)); //Here we have found all paths from x to z that go through y. Now we need to find edges that connect z back to x. // triangle(x,y,z) :- edge_pairs((z,x), y), edges(z, x) - triangle = join() -> map(|((z,x), (y, ()))| (x, y, z)); + triangle = join::<'static>() -> map(|((z,x), (y, ()))| (x, y, z)); // wire the inputs to the joins edges[0] -> map(|(y,z)| (z,y)) -> [0]edge_pairs; @@ -36,7 +36,7 @@ pub fn main() { // post-process: sort fields of each tuple by node ID triangle -> map(|(x, y, z)| { - let mut v = vec![x, y, z]; + let mut v = [x, y, z]; v.sort(); (v[0], v[1], v[2]) }) -> for_each(|e| println!("three_clique found: {:?}", e)); @@ -85,3 +85,18 @@ pub fn main() { // three_clique found: (0, 3, 6) // three_clique found: (5, 6, 10) } + +#[test] +fn test() { + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_child, _, mut stdout) = run_cargo_example("three_clique", ""); + + let mut output = String::new(); + wait_for_process_output(&mut output, &mut stdout, r#"0, 3, 6"#); + wait_for_process_output(&mut output, &mut stdout, r#"5, 6, 10"#); + wait_for_process_output(&mut output, &mut stdout, r#"0, 3, 6"#); + wait_for_process_output(&mut output, &mut stdout, r#"5, 6, 10"#); + wait_for_process_output(&mut output, &mut stdout, r#"0, 3, 6"#); + wait_for_process_output(&mut output, &mut stdout, r#"5, 6, 10"#); +} diff --git a/hydroflow/examples/two_pc/coordinator.rs b/hydroflow/examples/two_pc/coordinator.rs index 777103074cb..3139acceebf 100644 --- a/hydroflow/examples/two_pc/coordinator.rs +++ b/hydroflow/examples/two_pc/coordinator.rs @@ -15,6 +15,8 @@ pub(crate) async fn run_coordinator( path: impl AsRef, graph: Option, ) { + println!("Coordinator live!"); + let mut df: Hydroflow = hydroflow_syntax! { // fetch subordinates from file, convert ip:port to a SocketAddr, and tee subords = source_json(path) @@ -41,7 +43,7 @@ pub(crate) async fn run_coordinator( outbound_chan[1] -> for_each(|(m, a)| println!("Sending {:?} to {:?}", m, a)); // setup broadcast channel to all subords - broadcast_join = cross_join() -> outbound_chan; + broadcast_join = cross_join::<'static>() -> outbound_chan; broadcast = union() -> [0]broadcast_join; subords[1] -> [1]broadcast_join; subords[2] -> for_each(|s| println!("Subordinate: {:?}", s)); @@ -66,7 +68,7 @@ pub(crate) async fn run_coordinator( -> fold_keyed::<'static, u16, u32>(|| 0, |acc: &mut _, val| *acc += val); // count subordinates - subord_total = subords[0] -> fold::<'tick>(0, |a,_b| a+1); // -> for_each(|n| println!("There are {} subordinates.", n)); + subord_total = subords[0] -> fold::<'tick>(0, |a: &mut _, _b| *a += 1); // -> for_each(|n| println!("There are {} subordinates.", n)); // If commit_votes for this xid is the same as all_votes, send a P2 Commit message committed = join() -> map(|(_c, (xid, ()))| xid); diff --git a/hydroflow/examples/two_pc/main.rs b/hydroflow/examples/two_pc/main.rs index 7d314a0043c..e8d0760380c 100644 --- a/hydroflow/examples/two_pc/main.rs +++ b/hydroflow/examples/two_pc/main.rs @@ -63,3 +63,90 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let members_path = format!( + "{}/examples/two_pc/members.json", + env!("CARGO_MANIFEST_DIR") + ); + + let (_coordinator, mut coordinator_stdin, mut coordinator_stdout) = run_cargo_example( + "two_pc", + &format!("--path {members_path} --role coordinator --addr 127.0.0.1:12346"), + ); + + let (_subordinate1, _, mut subordinate1_stdout) = run_cargo_example( + "two_pc", + &format!("--path {members_path} --role subordinate --addr 127.0.0.1:12347"), + ); + + let (_subordinate2, _, mut subordinate2_stdout) = run_cargo_example( + "two_pc", + &format!("--path {members_path} --role subordinate --addr 127.0.0.1:12348"), + ); + + let (_subordinate3, _, mut subordinate3_stdout) = run_cargo_example( + "two_pc", + &format!("--path {members_path} --role subordinate --addr 127.0.0.1:12349"), + ); + + let mut coordinator_output = String::new(); + wait_for_process_output( + &mut coordinator_output, + &mut coordinator_stdout, + "Coordinator live!", + ); + + let mut subordinate1_output = String::new(); + wait_for_process_output( + &mut subordinate1_output, + &mut subordinate1_stdout, + "Subordinate live!", + ); + + let mut subordinate2_output = String::new(); + wait_for_process_output( + &mut subordinate2_output, + &mut subordinate2_stdout, + "Subordinate live!", + ); + + let mut subordinate3_output = String::new(); + wait_for_process_output( + &mut subordinate3_output, + &mut subordinate3_stdout, + "Subordinate live!", + ); + + coordinator_stdin.write_all(b"1\n").unwrap(); + + let mut coordinator_output = String::new(); + wait_for_process_output( + &mut coordinator_output, + &mut coordinator_stdout, + r#"Sending CoordMsg \{ xid: 1, mtype: Prepare \} to 127.0.0.1:12347"#, + ); + wait_for_process_output( + &mut coordinator_output, + &mut coordinator_stdout, + r#"Sending CoordMsg \{ xid: 1, mtype: Prepare \} to 127.0.0.1:12348"#, + ); + wait_for_process_output( + &mut coordinator_output, + &mut coordinator_stdout, + r#"Sending CoordMsg \{ xid: 1, mtype: Prepare \} to 127.0.0.1:12349"#, + ); + + // One of two things can happen now, all 3 members commit or at least one of them aborts the transaction. + // In the case of all 3 commits, then 3 "Commit" messages will be printed, in the case of an aborted transaction then 'Ended' will get printed, so: + wait_for_process_output( + &mut coordinator_output, + &mut coordinator_stdout, + r#"(Received SubordResponse \{ xid: 1, mtype: Commit \}|Received SubordResponse \{ xid: 1, mtype: Ended \})"#, + ); +} diff --git a/hydroflow/examples/two_pc/subordinate.rs b/hydroflow/examples/two_pc/subordinate.rs index ae774c115fb..b16952b7821 100644 --- a/hydroflow/examples/two_pc/subordinate.rs +++ b/hydroflow/examples/two_pc/subordinate.rs @@ -15,13 +15,15 @@ pub(crate) async fn run_subordinate( path: impl AsRef, graph: Option, ) { + println!("Subordinate live!"); + let mut df: Hydroflow = hydroflow_syntax! { // Outbound address server_addr = source_json(path) -> map(|json: Addresses| json.coordinator) -> map(|s| s.parse::().unwrap()) -> inspect(|coordinator| println!("Coordinator: {}", coordinator)); - server_addr_join = cross_join(); + server_addr_join = cross_join::<'static>(); server_addr -> [1]server_addr_join; // set up channels diff --git a/hydroflow/examples/vector_clock/client.rs b/hydroflow/examples/vector_clock/client.rs index 63234fa2551..70317c96871 100644 --- a/hydroflow/examples/vector_clock/client.rs +++ b/hydroflow/examples/vector_clock/client.rs @@ -32,12 +32,11 @@ pub(crate) async fn run_client( // given the inbound packet, bump the local clock and merge this in inbound_chan[merge] -> map(|(msg, _sender): (EchoMsg, SocketAddr)| msg.vc) -> [net]mergevc; - mergevc = union() -> fold::<'static> (VecClock::default(), |mut old, vc| { + mergevc = union() -> fold::<'static> (VecClock::default(), |old: &mut VecClock, vc| { let my_addr = format!("{:?}", addr); - let bump = MapUnionSingletonMap::new_from((my_addr.clone(), Max::new(old.0[&my_addr].0 + 1))); + let bump = MapUnionSingletonMap::new_from((my_addr.clone(), Max::new(old.as_reveal_mut().entry(my_addr).or_insert(Max::new(0)).into_reveal() + 1))); old.merge(bump); old.merge(vc); - old } ); diff --git a/hydroflow/examples/vector_clock/main.rs b/hydroflow/examples/vector_clock/main.rs index ef99a6d85be..1802633202b 100644 --- a/hydroflow/examples/vector_clock/main.rs +++ b/hydroflow/examples/vector_clock/main.rs @@ -57,3 +57,48 @@ async fn main() { } } } + +#[test] +fn test() { + use std::io::Write; + + use hydroflow::util::{run_cargo_example, wait_for_process_output}; + + let (_server, _, mut server_stdout) = + run_cargo_example("vector_clock", "--role server --addr 127.0.0.1:11053"); + + let (_client1, mut client1_stdin, mut client1_stdout) = run_cargo_example( + "vector_clock", + "--role client --server-addr 127.0.0.1:11053", + ); + + let (_client2, mut client2_stdin, mut client2_stdout) = run_cargo_example( + "vector_clock", + "--role client --server-addr 127.0.0.1:11053", + ); + + let mut server_output = String::new(); + wait_for_process_output(&mut server_output, &mut server_stdout, "Server live!"); + + let mut client1_output = String::new(); + wait_for_process_output(&mut client1_output, &mut client1_stdout, "Client live!"); + + let mut client2_output = String::new(); + wait_for_process_output(&mut client2_output, &mut client2_stdout, "Client live!"); + + client1_stdin.write_all(b"Hello1\n").unwrap(); + + wait_for_process_output( + &mut client1_output, + &mut client1_stdout, + r#"payload: "Hello1", vc: .*"127.0.0.1:11053": Max\(1\).*from 127.0.0.1:11053"#, + ); + + client2_stdin.write_all(b"Hello2\n").unwrap(); + + wait_for_process_output( + &mut client2_output, + &mut client2_stdout, + r#"payload: "Hello2", vc: .*"127.0.0.1:11053": Max\(2\).*from 127.0.0.1:11053"#, + ); +} diff --git a/hydroflow/examples/vector_clock/server.rs b/hydroflow/examples/vector_clock/server.rs index 27c3e2e9a5b..a87d28ffbe1 100644 --- a/hydroflow/examples/vector_clock/server.rs +++ b/hydroflow/examples/vector_clock/server.rs @@ -22,12 +22,11 @@ pub(crate) async fn run_server(outbound: UdpSink, inbound: UdpStream, opts: crat // merge in the msg vc to the local vc inbound_chan[merge] -> map(|(msg, _addr): (EchoMsg, SocketAddr)| msg.vc) -> mergevc; - mergevc = fold::<'static> (VecClock::default(), |mut old, vc| { + mergevc = fold::<'static> (VecClock::default(), |old: &mut VecClock, vc| { let my_addr = format!("{:?}", opts.addr.unwrap()); - let bump = MapUnionSingletonMap::new_from((my_addr.clone(), Max::new(old.0[&my_addr].0 + 1))); + let bump = MapUnionSingletonMap::new_from((my_addr.clone(), Max::new(old.as_reveal_mut().entry(my_addr).or_insert(Max::new(0)).into_reveal() + 1))); old.merge(bump); old.merge(vc); - old } ); diff --git a/hydroflow/src/compiled/pull/anti_join.rs b/hydroflow/src/compiled/pull/anti_join.rs new file mode 100644 index 00000000000..bd8a592bee0 --- /dev/null +++ b/hydroflow/src/compiled/pull/anti_join.rs @@ -0,0 +1,66 @@ +use itertools::Either; +use rustc_hash::FxHashSet; + +pub struct AntiJoin<'a, Key, V, Ipos> +where + Key: Eq + std::hash::Hash + Clone, + V: Eq + std::hash::Hash + Clone, + Ipos: Iterator, +{ + input_pos: Ipos, + neg_state: &'a mut FxHashSet, + pos_state: &'a mut FxHashSet<(Key, V)>, +} + +impl<'a, Key, V, Ipos> Iterator for AntiJoin<'a, Key, V, Ipos> +where + Key: Eq + std::hash::Hash + Clone, + V: Eq + std::hash::Hash + Clone, + Ipos: Iterator, +{ + type Item = (Key, V); + + fn next(&mut self) -> Option { + for item in self.input_pos.by_ref() { + if !self.neg_state.contains(&item.0) && !self.pos_state.contains(&item) { + self.pos_state.insert(item.clone()); + return Some(item); + } + } + + None + } +} + +pub fn anti_join_into_iter<'a, Key, V, Ipos>( + input_pos: Ipos, + state_neg: &'a mut FxHashSet, + state_pos: &'a mut FxHashSet<(Key, V)>, + new_tick: bool, +) -> impl 'a + Iterator +where + Key: Eq + std::hash::Hash + Clone, + V: Eq + std::hash::Hash + Clone, + Ipos: 'a + Iterator, +{ + if new_tick { + for kv in input_pos { + if !state_neg.contains(&kv.0) { + state_pos.insert(kv); + } + } + + Either::Left( + state_pos + .iter() + .filter(|(k, _)| !state_neg.contains(k)) + .cloned(), + ) + } else { + Either::Right(AntiJoin { + input_pos, + neg_state: state_neg, + pos_state: state_pos, + }) + } +} diff --git a/hydroflow/src/compiled/pull/half_join_state/fold.rs b/hydroflow/src/compiled/pull/half_join_state/fold.rs new file mode 100644 index 00000000000..5241fb11061 --- /dev/null +++ b/hydroflow/src/compiled/pull/half_join_state/fold.rs @@ -0,0 +1,48 @@ +use std::collections::hash_map::Entry::*; + +use rustc_hash::FxHashMap; + +use crate::util::clear::Clear; + +pub struct HalfJoinStateFold { + pub table: FxHashMap, +} + +impl Default for HalfJoinStateFold { + fn default() -> Self { + Self { + table: Default::default(), + } + } +} + +impl Clear for HalfJoinStateFold { + fn clear(&mut self) { + self.table.clear() + } +} + +impl HalfJoinStateFold +where + K: Eq + std::hash::Hash, +{ + pub fn fold_into( + &mut self, + iter: impl Iterator, + mut fold: impl FnMut(&mut A, V) -> X, + mut default: impl FnMut() -> A, + ) { + for (k, v) in iter { + let entry = self.table.entry(k); + + match entry { + Occupied(mut e) => { + (fold)(e.get_mut(), v); + } + Vacant(e) => { + (fold)(e.insert((default)()), v); + } + } + } + } +} diff --git a/hydroflow/src/compiled/pull/half_join_state/fold_from.rs b/hydroflow/src/compiled/pull/half_join_state/fold_from.rs new file mode 100644 index 00000000000..f352345a134 --- /dev/null +++ b/hydroflow/src/compiled/pull/half_join_state/fold_from.rs @@ -0,0 +1,48 @@ +use std::collections::hash_map::Entry::*; + +use rustc_hash::FxHashMap; + +use crate::util::clear::Clear; + +pub struct HalfJoinStateFoldFrom { + pub table: FxHashMap, +} + +impl Default for HalfJoinStateFoldFrom { + fn default() -> Self { + Self { + table: Default::default(), + } + } +} + +impl Clear for HalfJoinStateFoldFrom { + fn clear(&mut self) { + self.table.clear() + } +} + +impl HalfJoinStateFoldFrom +where + K: Eq + std::hash::Hash, +{ + pub fn fold_into( + &mut self, + iter: impl Iterator, + mut fold: impl FnMut(&mut A, V) -> X, + mut from: impl FnMut(V) -> A, + ) { + for (k, v) in iter { + let entry = self.table.entry(k); + + match entry { + Occupied(mut e) => { + (fold)(e.get_mut(), v); + } + Vacant(e) => { + e.insert((from)(v)); + } + } + } + } +} diff --git a/hydroflow/src/compiled/pull/half_join_state/mod.rs b/hydroflow/src/compiled/pull/half_join_state/mod.rs index da4d6379959..9ca16fb9bfe 100644 --- a/hydroflow/src/compiled/pull/half_join_state/mod.rs +++ b/hydroflow/src/compiled/pull/half_join_state/mod.rs @@ -1,14 +1,37 @@ +mod fold; +mod fold_from; mod multiset; +mod multiset2; +mod reduce; mod set; -// pub use half_join_state_trait::HalfJoinState; -pub use multiset::HalfMultisetJoinState; +use std::collections::hash_map::Iter; +use std::slice; + +pub use fold::HalfJoinStateFold; +pub use fold_from::HalfJoinStateFoldFrom; +pub use multiset2::HalfJoinStateMultiset; +pub use reduce::HalfJoinStateReduce; pub use set::HalfSetJoinState; +use smallvec::SmallVec; -pub type SetJoinState = (HalfSetJoinState, HalfSetJoinState); +pub use self::multiset::HalfMultisetJoinState; pub trait HalfJoinState { + /// Insert a key value pair into the join state, currently this is always inserting into a hash table + /// If the key-value pair exists then it is implementation defined what happens, usually either two copies are stored or only one copy is stored. fn build(&mut self, k: Key, v: &ValBuild) -> bool; - fn probe(&mut self, k: &Key, v: &ValProbe); + + /// This function does the actual joining part of the join. It looks up a key in the local join state and creates matches + /// The first match is return directly to the caller, and any additional matches are stored internally to be retrieved later with `pop_match` + fn probe(&mut self, k: &Key, v: &ValProbe) -> Option<(Key, ValProbe, ValBuild)>; + + /// If there are any stored matches from previous calls to probe then this function will remove them one at a time and return it. fn pop_match(&mut self) -> Option<(Key, ValProbe, ValBuild)>; + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn iter(&self) -> Iter<'_, Key, SmallVec<[ValBuild; 1]>>; + fn full_probe(&self, k: &Key) -> slice::Iter<'_, ValBuild>; } diff --git a/hydroflow/src/compiled/pull/half_join_state/multiset.rs b/hydroflow/src/compiled/pull/half_join_state/multiset.rs index 624aa40ad8a..0894bb0e8c1 100644 --- a/hydroflow/src/compiled/pull/half_join_state/multiset.rs +++ b/hydroflow/src/compiled/pull/half_join_state/multiset.rs @@ -1,5 +1,6 @@ use std::collections::hash_map::Entry; use std::collections::VecDeque; +use std::slice; use super::HalfJoinState; use crate::util::clear::Clear; @@ -18,12 +19,14 @@ pub struct HalfMultisetJoinState { table: HashMap>, /// Not-yet emitted matches. current_matches: VecDeque<(Key, ValProbe, ValBuild)>, + len: usize, } impl Default for HalfMultisetJoinState { fn default() -> Self { Self { table: HashMap::default(), current_matches: VecDeque::default(), + len: 0, } } } @@ -31,6 +34,7 @@ impl Clear for HalfMultisetJoinState HalfJoinState @@ -48,29 +52,50 @@ where let vec = e.get_mut(); vec.push(v.clone()); + self.len += 1; } Entry::Vacant(e) => { e.insert(smallvec![v.clone()]); + self.len += 1; } }; true } - fn probe(&mut self, k: &Key, v: &ValProbe) { - if let Some(entry) = self.table.get(k) { - // TODO: We currently don't free/shrink the self.current_matches vecdeque to save time. - // This mean it will grow to eventually become the largest number of matches in a single probe call. - // Maybe we should clear this memory at the beginning of every tick/periodically? - self.current_matches.extend( - entry - .iter() - .map(|valbuild| (k.clone(), v.clone(), valbuild.clone())), - ); - } + fn probe(&mut self, k: &Key, v: &ValProbe) -> Option<(Key, ValProbe, ValBuild)> { + // TODO: We currently don't free/shrink the self.current_matches vecdeque to save time. + // This mean it will grow to eventually become the largest number of matches in a single probe call. + // Maybe we should clear this memory at the beginning of every tick/periodically? + let mut iter = self + .table + .get(k)? + .iter() + .map(|valbuild| (k.clone(), v.clone(), valbuild.clone())); + + let first = iter.next(); + + self.current_matches.extend(iter); + + first + } + + fn full_probe(&self, k: &Key) -> slice::Iter<'_, ValBuild> { + let Some(sv) = self.table.get(k) else { + return [].iter(); + }; + + sv.iter() } fn pop_match(&mut self) -> Option<(Key, ValProbe, ValBuild)> { self.current_matches.pop_front() } + + fn len(&self) -> usize { + self.len + } + fn iter(&self) -> ::std::collections::hash_map::Iter<'_, Key, SmallVec<[ValBuild; 1]>> { + self.table.iter() + } } diff --git a/hydroflow/src/compiled/pull/half_join_state/multiset2.rs b/hydroflow/src/compiled/pull/half_join_state/multiset2.rs new file mode 100644 index 00000000000..d84f1b7004d --- /dev/null +++ b/hydroflow/src/compiled/pull/half_join_state/multiset2.rs @@ -0,0 +1,42 @@ +use std::collections::hash_map::Entry::*; + +use rustc_hash::FxHashMap; +use smallvec::{smallvec, SmallVec}; + +use crate::util::clear::Clear; + +pub struct HalfJoinStateMultiset { + pub table: FxHashMap>, +} + +impl Default for HalfJoinStateMultiset { + fn default() -> Self { + Self { + table: Default::default(), + } + } +} + +impl Clear for HalfJoinStateMultiset { + fn clear(&mut self) { + self.table.clear() + } +} + +impl HalfJoinStateMultiset +where + K: Eq + std::hash::Hash, +{ + pub fn push(&mut self, iter: impl Iterator) { + for (k, v) in iter { + let entry = self.table.entry(k); + + match entry { + Occupied(mut e) => e.get_mut().push(v), + Vacant(e) => { + e.insert(smallvec![v]); + } + } + } + } +} diff --git a/hydroflow/src/compiled/pull/half_join_state/reduce.rs b/hydroflow/src/compiled/pull/half_join_state/reduce.rs new file mode 100644 index 00000000000..cccaa3dc249 --- /dev/null +++ b/hydroflow/src/compiled/pull/half_join_state/reduce.rs @@ -0,0 +1,47 @@ +use std::collections::hash_map::Entry::*; + +use rustc_hash::FxHashMap; + +use crate::util::clear::Clear; + +pub struct HalfJoinStateReduce { + pub table: FxHashMap, +} + +impl Default for HalfJoinStateReduce { + fn default() -> Self { + Self { + table: Default::default(), + } + } +} + +impl Clear for HalfJoinStateReduce { + fn clear(&mut self) { + self.table.clear() + } +} + +impl HalfJoinStateReduce +where + K: Eq + std::hash::Hash, +{ + pub fn reduce_into( + &mut self, + iter: impl Iterator, + mut reduce: impl FnMut(&mut A, A) -> X, + ) { + for (k, v) in iter { + let entry = self.table.entry(k); + + match entry { + Occupied(mut e) => { + (reduce)(e.get_mut(), v); + } + Vacant(e) => { + e.insert(v); + } + } + } + } +} diff --git a/hydroflow/src/compiled/pull/half_join_state/set.rs b/hydroflow/src/compiled/pull/half_join_state/set.rs index 8469d13d664..e99e0ad7e51 100644 --- a/hydroflow/src/compiled/pull/half_join_state/set.rs +++ b/hydroflow/src/compiled/pull/half_join_state/set.rs @@ -1,5 +1,6 @@ use std::collections::hash_map::Entry; use std::collections::VecDeque; +use std::slice; use super::HalfJoinState; use crate::util::clear::Clear; @@ -19,12 +20,14 @@ pub struct HalfSetJoinState { table: HashMap>, /// Not-yet emitted matches. current_matches: VecDeque<(Key, ValProbe, ValBuild)>, + len: usize, } impl Default for HalfSetJoinState { fn default() -> Self { Self { table: HashMap::default(), current_matches: VecDeque::default(), + len: 0, } } } @@ -32,6 +35,7 @@ impl Clear for HalfSetJoinState HalfJoinState @@ -50,11 +54,13 @@ where if !vec.contains(v) { vec.push(v.clone()); + self.len += 1; return true; } } Entry::Vacant(e) => { e.insert(smallvec![v.clone()]); + self.len += 1; return true; } }; @@ -62,20 +68,40 @@ where false } - fn probe(&mut self, k: &Key, v: &ValProbe) { - if let Some(entry) = self.table.get(k) { - // TODO: We currently don't free/shrink the self.current_matches vecdeque to save time. - // This mean it will grow to eventually become the largest number of matches in a single probe call. - // Maybe we should clear this memory at the beginning of every tick/periodically? - self.current_matches.extend( - entry - .iter() - .map(|valbuild| (k.clone(), v.clone(), valbuild.clone())), - ); - } + fn probe(&mut self, k: &Key, v: &ValProbe) -> Option<(Key, ValProbe, ValBuild)> { + // TODO: We currently don't free/shrink the self.current_matches vecdeque to save time. + // This mean it will grow to eventually become the largest number of matches in a single probe call. + // Maybe we should clear this memory at the beginning of every tick/periodically? + let mut iter = self + .table + .get(k)? + .iter() + .map(|valbuild| (k.clone(), v.clone(), valbuild.clone())); + + let first = iter.next(); + + self.current_matches.extend(iter); + + first + } + + fn full_probe(&self, k: &Key) -> slice::Iter<'_, ValBuild> { + let Some(sv) = self.table.get(k) else { + return [].iter(); + }; + + sv.iter() } fn pop_match(&mut self) -> Option<(Key, ValProbe, ValBuild)> { self.current_matches.pop_front() } + + fn len(&self) -> usize { + self.len + } + + fn iter(&self) -> ::std::collections::hash_map::Iter<'_, Key, SmallVec<[ValBuild; 1]>> { + self.table.iter() + } } diff --git a/hydroflow/src/compiled/pull/mod.rs b/hydroflow/src/compiled/pull/mod.rs index c63e864404d..be85b359a8b 100644 --- a/hydroflow/src/compiled/pull/mod.rs +++ b/hydroflow/src/compiled/pull/mod.rs @@ -7,8 +7,8 @@ pub use cross_join::*; mod symmetric_hash_join; pub use symmetric_hash_join::*; -mod symmetric_hash_join_lattice; -pub use symmetric_hash_join_lattice::*; - mod half_join_state; pub use half_join_state::*; + +mod anti_join; +pub use anti_join::*; diff --git a/hydroflow/src/compiled/pull/symmetric_hash_join.rs b/hydroflow/src/compiled/pull/symmetric_hash_join.rs index 4eed36661b8..d240e6a347a 100644 --- a/hydroflow/src/compiled/pull/symmetric_hash_join.rs +++ b/hydroflow/src/compiled/pull/symmetric_hash_join.rs @@ -1,3 +1,5 @@ +use itertools::Either; + use super::HalfJoinState; pub struct SymmetricHashJoin<'a, Key, I1, V1, I2, V2, LhsState, RhsState> @@ -40,13 +42,17 @@ where if let Some((k, v1)) = self.lhs.next() { if self.lhs_state.build(k.clone(), &v1) { - self.rhs_state.probe(&k, &v1); + if let Some((k, v1, v2)) = self.rhs_state.probe(&k, &v1) { + return Some((k, (v1, v2))); + } } continue; } if let Some((k, v2)) = self.rhs.next() { if self.rhs_state.build(k.clone(), &v2) { - self.lhs_state.probe(&k, &v2); + if let Some((k, v2, v1)) = self.lhs_state.probe(&k, &v2) { + return Some((k, (v1, v2))); + } } continue; } @@ -55,81 +61,104 @@ where } } } -impl<'a, Key, I1, V1, I2, V2, LhsState, RhsState> - SymmetricHashJoin<'a, Key, I1, V1, I2, V2, LhsState, RhsState> + +pub fn symmetric_hash_join_into_iter<'a, Key, I1, V1, I2, V2, LhsState, RhsState>( + mut lhs: I1, + mut rhs: I2, + lhs_state: &'a mut LhsState, + rhs_state: &'a mut RhsState, + is_new_tick: bool, +) -> impl 'a + Iterator where - Key: Eq + std::hash::Hash + Clone, - V1: Clone, - V2: Clone, - I1: Iterator, - I2: Iterator, + Key: 'a + Eq + std::hash::Hash + Clone, + V1: 'a + Clone, + V2: 'a + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, LhsState: HalfJoinState, RhsState: HalfJoinState, { - pub fn new(lhs: I1, rhs: I2, state: &'a mut (LhsState, RhsState)) -> Self { - Self { - lhs, - rhs, - lhs_state: &mut state.0, - rhs_state: &mut state.1, + if is_new_tick { + for (k, v1) in lhs.by_ref() { + lhs_state.build(k.clone(), &v1); + } + + for (k, v2) in rhs.by_ref() { + rhs_state.build(k.clone(), &v2); } - } - pub fn new_from_mut( - lhs: I1, - rhs: I2, - state_lhs: &'a mut LhsState, - state_rhs: &'a mut RhsState, - ) -> Self { - Self { + Either::Left(if lhs_state.len() < rhs_state.len() { + Either::Left(lhs_state.iter().flat_map(|(k, sv)| { + sv.iter().flat_map(|v1| { + rhs_state + .full_probe(k) + .map(|v2| (k.clone(), (v1.clone(), v2.clone()))) + }) + })) + } else { + Either::Right(rhs_state.iter().flat_map(|(k, sv)| { + sv.iter().flat_map(|v2| { + lhs_state + .full_probe(k) + .map(|v1| (k.clone(), (v1.clone(), v2.clone()))) + }) + })) + }) + } else { + Either::Right(SymmetricHashJoin { lhs, rhs, - lhs_state: state_lhs, - rhs_state: state_rhs, - } + lhs_state, + rhs_state, + }) } } #[cfg(test)] mod tests { - use super::SymmetricHashJoin; - use crate::compiled::pull::SetJoinState; + use std::collections::HashSet; + + use crate::compiled::pull::{symmetric_hash_join_into_iter, HalfSetJoinState}; #[test] fn hash_join() { let lhs = (0..10).map(|x| (x, format!("left {}", x))); let rhs = (6..15).map(|x| (x / 2, format!("right {} / 2", x))); - let mut state = SetJoinState::default(); - let join = SymmetricHashJoin::new(lhs, rhs, &mut state); - - assert_eq!( - join.collect::>(), - vec![ - (3, ("left 3".into(), "right 6 / 2".into())), - (3, ("left 3".into(), "right 7 / 2".into())), - (4, ("left 4".into(), "right 8 / 2".into())), - (4, ("left 4".into(), "right 9 / 2".into())), - (5, ("left 5".into(), "right 10 / 2".into())), - (5, ("left 5".into(), "right 11 / 2".into())), - (6, ("left 6".into(), "right 12 / 2".into())), - (6, ("left 6".into(), "right 13 / 2".into())), - (7, ("left 7".into(), "right 14 / 2".into())) - ] - ); + let (mut lhs_state, mut rhs_state) = + (HalfSetJoinState::default(), HalfSetJoinState::default()); + let join = symmetric_hash_join_into_iter(lhs, rhs, &mut lhs_state, &mut rhs_state, true); + + let joined = join.collect::>(); + + assert!(joined.contains(&(3, ("left 3".into(), "right 6 / 2".into())))); + assert!(joined.contains(&(3, ("left 3".into(), "right 7 / 2".into())))); + assert!(joined.contains(&(4, ("left 4".into(), "right 8 / 2".into())))); + assert!(joined.contains(&(4, ("left 4".into(), "right 9 / 2".into())))); + assert!(joined.contains(&(5, ("left 5".into(), "right 10 / 2".into())))); + assert!(joined.contains(&(5, ("left 5".into(), "right 11 / 2".into())))); + assert!(joined.contains(&(6, ("left 6".into(), "right 12 / 2".into())))); + assert!(joined.contains(&(7, ("left 7".into(), "right 14 / 2".into())))); } #[test] - fn hash_join_subsequent_ticks_dont_produce_if_nothing_is_changed() { + fn hash_join_subsequent_ticks_do_produce_even_if_nothing_is_changed() { let (lhs_tx, lhs_rx) = std::sync::mpsc::channel::<(usize, usize)>(); let (rhs_tx, rhs_rx) = std::sync::mpsc::channel::<(usize, usize)>(); - let mut state = SetJoinState::default(); - let mut join = SymmetricHashJoin::new(lhs_rx.try_iter(), rhs_rx.try_iter(), &mut state); - lhs_tx.send((7, 3)).unwrap(); rhs_tx.send((7, 3)).unwrap(); + let (mut lhs_state, mut rhs_state) = + (HalfSetJoinState::default(), HalfSetJoinState::default()); + let mut join = symmetric_hash_join_into_iter( + lhs_rx.try_iter(), + rhs_rx.try_iter(), + &mut lhs_state, + &mut rhs_state, + true, + ); + assert_eq!(join.next(), Some((7, (3, 3)))); assert_eq!(join.next(), None); diff --git a/hydroflow/src/compiled/pull/symmetric_hash_join_lattice.rs b/hydroflow/src/compiled/pull/symmetric_hash_join_lattice.rs deleted file mode 100644 index 9464376d890..00000000000 --- a/hydroflow/src/compiled/pull/symmetric_hash_join_lattice.rs +++ /dev/null @@ -1,231 +0,0 @@ -use std::collections::hash_map::Entry; -use std::collections::hash_set; - -use lattices::map_union::MapUnion; -use lattices::{LatticeFrom, Merge}; -use rustc_hash::{FxHashMap, FxHashSet}; - -use crate::util::clear::Clear; - -pub struct HalfJoinStateLattice { - table: MapUnion>, -} - -impl Default for HalfJoinStateLattice { - fn default() -> Self { - Self { - table: Default::default(), - } - } -} - -impl Clear for HalfJoinStateLattice { - fn clear(&mut self) { - self.table.0.clear() - } -} - -impl HalfJoinStateLattice -where - K: Clone + Eq + std::hash::Hash, -{ - fn build(&mut self, k: K, v: LatticeDelta) -> bool - where - Lattice: Merge + LatticeFrom, - { - let entry = self.table.0.entry(k); - - match entry { - Entry::Occupied(mut e) => e.get_mut().merge(v), - Entry::Vacant(e) => { - e.insert(LatticeFrom::lattice_from(v)); - true - } - } - } -} - -pub type JoinStateLatticeMut<'a, K, LhsLattice, RhsLattice> = ( - &'a mut HalfJoinStateLattice, - &'a mut HalfJoinStateLattice, -); - -pub struct SymmetricHashJoinLattice<'a, K, LhsLattice, RhsLattice> -where - K: Eq + std::hash::Hash + Clone, -{ - state: JoinStateLatticeMut<'a, K, LhsLattice, RhsLattice>, - updated_keys: hash_set::Drain<'a, K>, -} - -impl<'a, K, LhsLattice, RhsLattice> Iterator - for SymmetricHashJoinLattice<'a, K, LhsLattice, RhsLattice> -where - K: Eq + std::hash::Hash + Clone, - LhsLattice: Clone, - RhsLattice: Clone, -{ - type Item = (K, (LhsLattice, RhsLattice)); - - fn next(&mut self) -> Option { - if let Some(key) = self.updated_keys.next() { - if let Some(lhs) = self.state.0.table.0.get(&key) { - if let Some(rhs) = self.state.1.table.0.get(&key) { - return Some((key, (lhs.clone(), rhs.clone()))); - } - } - } - - None - } -} -impl<'a, K, LhsLattice, RhsLattice> SymmetricHashJoinLattice<'a, K, LhsLattice, RhsLattice> -where - K: Eq + std::hash::Hash + Clone, -{ - pub fn new_from_mut( - lhs: I1, - rhs: I2, - updated_keys: &'a mut FxHashSet, - state_lhs: &'a mut HalfJoinStateLattice, - state_rhs: &'a mut HalfJoinStateLattice, - ) -> Self - where - I1: Iterator, - I2: Iterator, - LhsLattice: Merge + LatticeFrom, - RhsLattice: Merge + LatticeFrom, - { - for (k, v1) in lhs { - if state_lhs.build(k.clone(), v1) { - updated_keys.insert(k); - } - } - - for (k, v2) in rhs { - if state_rhs.build(k.clone(), v2) { - updated_keys.insert(k); - } - } - - Self { - state: (state_lhs, state_rhs), - updated_keys: updated_keys.drain(), - } - } -} - -#[cfg(test)] -mod tests { - pub type JoinStateLattice = ( - HalfJoinStateLattice, - HalfJoinStateLattice, - ); - - use lattices::Max; - use rustc_hash::FxHashSet; - - use super::{HalfJoinStateLattice, SymmetricHashJoinLattice}; - - type MyLattice = Max; - type JoinState = JoinStateLattice; - - fn join< - Lhs: IntoIterator, - Rhs: IntoIterator, - >( - state: &mut JoinState, - lhs: Lhs, - rhs: Rhs, - ) -> Vec<(usize, (MyLattice, MyLattice))> { - let mut updated_keys = FxHashSet::default(); - SymmetricHashJoinLattice::new_from_mut( - lhs.into_iter(), - rhs.into_iter(), - &mut updated_keys, - &mut state.0, - &mut state.1, - ) - .collect::>() - } - - #[test] - fn produces_fully_merged_output() { - let mut state = JoinState::default(); - - let lhs = [(7, MyLattice::new(3)), (7, MyLattice::new(4))]; - let rhs = [(7, MyLattice::new(5)), (7, MyLattice::new(6))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(4), MyLattice::new(6)))] - ); - } - - #[test] - fn lattice_only_moves_forward() { - let mut state = JoinState::default(); - - let lhs = [(7, MyLattice::new(4)), (7, MyLattice::new(3))]; - let rhs = [(7, MyLattice::new(6)), (7, MyLattice::new(5))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(4), MyLattice::new(6)))] - ); - } - - #[test] - fn subsequent_ticks_dont_produce_if_nothing_has_changed() { - let mut state = JoinState::default(); - - let lhs = [(7, MyLattice::new(3))]; - let rhs = [(7, MyLattice::new(3))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (Max::new(3), Max::new(3)))] - ); - - let lhs = [(7, Max::new(3))]; - let rhs = [(7, Max::new(3))]; - assert_eq!(join(&mut state, lhs, rhs), vec![]); - } - - #[test] - fn subsequent_ticks_do_produce_if_something_has_changed() { - let mut state = JoinState::default(); - - let lhs = [(7, MyLattice::new(3))]; - let rhs = [(7, MyLattice::new(3))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(3), MyLattice::new(3)))] - ); - - let lhs = [(7, MyLattice::new(3))]; - let rhs = [(7, MyLattice::new(4))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(3), MyLattice::new(4)))] - ); - } - - #[test] - fn resetting_one_side_works() { - let mut state = JoinState::default(); - - let lhs = [(7, MyLattice::new(3))]; - let rhs = [(7, MyLattice::new(3))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(3), MyLattice::new(3)))] - ); - - std::mem::take(&mut state.1); - - let lhs = [(7, Max::new(3))]; - let rhs = [(7, Max::new(3))]; - assert_eq!( - join(&mut state, lhs, rhs), - vec![(7, (MyLattice::new(3), MyLattice::new(3)))] - ); - } -} diff --git a/hydroflow/src/declarative_macro.rs b/hydroflow/src/declarative_macro.rs index bdfe21b306a..7622aea17a4 100644 --- a/hydroflow/src/declarative_macro.rs +++ b/hydroflow/src/declarative_macro.rs @@ -44,6 +44,37 @@ macro_rules! assert_var_impl { }; } +/// Tests that the given warnings are emitted by the hydroflow macro invocation. +/// +/// For example usage, see `hydroflow/tests/surface_warnings.rs`. +#[macro_export] +macro_rules! hydroflow_expect_warnings { + ( + $hf:tt, + $( $msg:literal ),* + $( , )? + ) => { + { + let __file = std::file!(); + let __line = std::line!() as usize; + let __hf = hydroflow::hydroflow_syntax_noemit! $hf; + + let diagnostics = __hf.diagnostics().expect("Expected `diagnostics()` to be set."); + let expecteds = &[ + $( $msg , )* + ]; + assert_eq!(diagnostics.len(), expecteds.len(), "Wrong number of diagnostics."); + for (expected, diagnostic) in expecteds.iter().zip(diagnostics.iter()) { + let mut diagnostic = diagnostic.clone(); + diagnostic.span.line = diagnostic.span.line.saturating_sub(__line); + assert_eq!(expected.to_string(), diagnostic.to_string().replace(__file, "$FILE")); + } + + __hf + } + }; +} + /// Test helper, emits and checks snapshots for the mermaid and dot graphs. #[doc(hidden)] #[macro_export] @@ -62,3 +93,35 @@ macro_rules! assert_graphvis_snapshots { } } } + +#[doc(hidden)] +#[macro_export] +#[cfg(feature = "python")] +macro_rules! __python_feature_gate { + ( + { + $( $ypy:tt )* + }, + { + $( $npy:tt )* + } + ) => { + $( $ypy )* + }; +} + +#[doc(hidden)] +#[macro_export] +#[cfg(not(feature = "python"))] +macro_rules! __python_feature_gate { + ( + { + $( $ypy:tt )* + }, + { + $( $npy:tt )* + } + ) => { + $( $npy )* + }; +} diff --git a/hydroflow/src/lib.rs b/hydroflow/src/lib.rs index 27f7b83b5bd..bd7b4c18b17 100644 --- a/hydroflow/src/lib.rs +++ b/hydroflow/src/lib.rs @@ -26,12 +26,14 @@ pub mod props; pub mod scheduled; pub mod util; +#[cfg(feature = "python")] +pub use pyo3; #[cfg(feature = "tracing")] pub use tracing; pub use variadics::{self, var_args, var_expr, var_type}; pub use { - bincode, bytes, futures, itertools, lattices, pusherator, rustc_hash, serde, serde_json, tokio, - tokio_stream, tokio_util, + bincode, bytes, futures, instant, itertools, lattices, pusherator, rustc_hash, serde, + serde_json, tokio, tokio_stream, tokio_util, }; mod declarative_macro; @@ -41,7 +43,7 @@ pub use hydroflow_datalog::*; #[cfg(feature = "hydroflow_macro")] pub use hydroflow_macro::{ hydroflow_main as main, hydroflow_parser, hydroflow_syntax, hydroflow_syntax_noemit, - hydroflow_test as test, + hydroflow_test as test, monotonic_fn, morphism, }; #[cfg(not(nightly))] diff --git a/hydroflow/src/scheduled/context.rs b/hydroflow/src/scheduled/context.rs index c4ba6aa75f0..7996cd79529 100644 --- a/hydroflow/src/scheduled/context.rs +++ b/hydroflow/src/scheduled/context.rs @@ -4,6 +4,7 @@ use std::any::Any; use std::future::Future; use std::marker::PhantomData; +use instant::Instant; use tokio::sync::mpsc::UnboundedSender; use tokio::task::JoinHandle; @@ -28,6 +29,8 @@ pub struct Context { pub(crate) current_tick: usize, pub(crate) current_stratum: usize, + pub(crate) current_tick_start: Instant, + /// The SubgraphId of the currently running operator. When this context is /// not being forwarded to a running operator, this field is (mostly) /// meaningless. @@ -42,6 +45,11 @@ impl Context { self.current_tick } + /// Gets the timestamp of the beginning of the current tick. + pub fn current_tick_start(&self) -> Instant { + self.current_tick_start + } + /// Gets the current stratum nubmer. pub fn current_stratum(&self) -> usize { self.current_stratum diff --git a/hydroflow/src/scheduled/graph.rs b/hydroflow/src/scheduled/graph.rs index f78c00521aa..7a6e8abbd5c 100644 --- a/hydroflow/src/scheduled/graph.rs +++ b/hydroflow/src/scheduled/graph.rs @@ -9,6 +9,7 @@ use std::marker::PhantomData; use hydroflow_lang::diagnostic::{Diagnostic, SerdeSpan}; use hydroflow_lang::graph::HydroflowGraph; +use instant::Instant; use ref_cast::RefCast; use tokio::sync::mpsc::{self, UnboundedReceiver}; @@ -55,6 +56,8 @@ impl Default for Hydroflow { current_stratum: 0, current_tick: 0, + current_tick_start: Instant::now(), + subgraph_id: SubgraphId(0), task_join_handles: Vec::new(), @@ -244,6 +247,7 @@ impl Hydroflow { // Starting the tick, reset this to `false`. tracing::trace!("Starting tick, setting `can_start_tick = false`."); self.can_start_tick = false; + self.context.current_tick_start = Instant::now(); // Ensure external events are received before running the tick. if !self.events_received_tick { diff --git a/hydroflow/src/scheduled/net/mod.rs b/hydroflow/src/scheduled/net/mod.rs index c517e5afafe..ae1fbde936f 100644 --- a/hydroflow/src/scheduled/net/mod.rs +++ b/hydroflow/src/scheduled/net/mod.rs @@ -123,7 +123,7 @@ impl Hydroflow { // TODO(mingwei): queue may grow unbounded? Subtle rate matching concern. // TODO(mingwei): put into state system. - message_queue.extend(recv.take_inner().into_iter()); + message_queue.extend(recv.take_inner()); while !message_queue.is_empty() { if let std::task::Poll::Ready(Ok(())) = Pin::new(&mut writer).poll_ready(&mut cx) { let v = message_queue.pop_front().unwrap(); diff --git a/hydroflow/src/scheduled/state.rs b/hydroflow/src/scheduled/state.rs index 7c2009217ef..05c862cd06d 100644 --- a/hydroflow/src/scheduled/state.rs +++ b/hydroflow/src/scheduled/state.rs @@ -12,12 +12,9 @@ pub struct StateHandle { pub(crate) state_id: StateId, pub(crate) _phantom: PhantomData<*mut T>, } +impl Copy for StateHandle {} impl Clone for StateHandle { fn clone(&self) -> Self { - Self { - state_id: self.state_id, - _phantom: PhantomData, - } + *self } } -impl Copy for StateHandle {} diff --git a/hydroflow/src/util/cli.rs b/hydroflow/src/util/cli.rs index 90928acd8a0..1f55def45d1 100644 --- a/hydroflow/src/util/cli.rs +++ b/hydroflow/src/util/cli.rs @@ -70,6 +70,8 @@ pub async fn init() -> HydroCLI { all_connected.insert(name, ServerOrBound::Bound(defn)); } + println!("ack start"); + HydroCLI { ports: all_connected, } diff --git a/hydroflow/src/util/mod.rs b/hydroflow/src/util/mod.rs index 1b9c6b4b76a..3e3f93574f2 100644 --- a/hydroflow/src/util/mod.rs +++ b/hydroflow/src/util/mod.rs @@ -3,9 +3,13 @@ pub mod clear; pub mod monotonic_map; +pub mod multiset; pub mod sparse_vec; pub mod unsync; +mod monotonic; +pub use monotonic::*; + mod udp; #[cfg(not(target_arch = "wasm32"))] pub use udp::*; @@ -250,3 +254,99 @@ mod test { ); } } + +use std::io::Read; +use std::process::{Child, ChildStdin, ChildStdout, Stdio}; + +/// When a child process is spawned often you want to wait until the child process is ready before moving on. +/// One way to do that synchronization is by waiting for the child process to output something and match regex against that output. +/// For example, you could wait until the child process outputs "Client live!" which would indicate that it is ready to receive input now on stdin. +pub fn wait_for_process_output( + output_so_far: &mut String, + output: &mut ChildStdout, + wait_for: &str, +) { + let re = regex::Regex::new(wait_for).unwrap(); + + while !re.is_match(output_so_far) { + println!("waiting: {}", output_so_far); + let mut buffer = [0u8; 1024]; + let bytes_read = output.read(&mut buffer).unwrap(); + + if bytes_read == 0 { + panic!(); + } + + output_so_far.push_str(&String::from_utf8_lossy(&buffer[0..bytes_read])); + } +} + +/// When a `Child` is dropped normally nothing happens but in unit tests you usually want to terminate +/// the child and wait for it to terminate. `DroppableChild` does that for us. +pub struct DroppableChild(Child); + +impl Drop for DroppableChild { + fn drop(&mut self) { + #[cfg(target_family = "windows")] + let _ = self.0.kill(); // Windows throws `PermissionDenied` if the process has already exited. + #[cfg(not(target_family = "windows"))] + self.0.kill().unwrap(); + + self.0.wait().unwrap(); + } +} + +/// rust examples are meant to be run by people and have a natural interface for that. This makes unit testing them cumbersome. +/// This function wraps calling cargo run and piping the stdin/stdout of the example to easy to handle returned objects. +/// The function also returns a `DroppableChild` which will ensure that the child processes will be cleaned up appropriately. +pub fn run_cargo_example(test_name: &str, args: &str) -> (DroppableChild, ChildStdin, ChildStdout) { + let mut server = if args.is_empty() { + std::process::Command::new("cargo") + .args(["run", "-p", "hydroflow", "--example"]) + .arg(test_name) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap() + } else { + std::process::Command::new("cargo") + .args(["run", "-p", "hydroflow", "--example"]) + .arg(test_name) + .arg("--") + .args(args.split(' ')) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap() + }; + + let stdin = server.stdin.take().unwrap(); + let stdout = server.stdout.take().unwrap(); + + (DroppableChild(server), stdin, stdout) +} + +/// Returns an [`Stream`] that emits `n` items at a time from `iter` at a time, yielding in-between. +/// This is useful for breaking up a large iterator across several ticks: `source_iter(...)` always +/// releases all items in the first tick. However using `iter_batches_stream` with `source_stream(...)` +/// will cause `n` items to be released each tick. (Although more than that may be emitted if there +/// are loops in the stratum). +pub fn iter_batches_stream( + mut iter: I, + n: usize, +) -> futures::stream::PollFn) -> Poll>> +where + I: Iterator + Unpin, +{ + let mut count = 0; + futures::stream::poll_fn(move |ctx| { + count += 1; + if n < count { + count = 0; + ctx.waker().wake_by_ref(); + Poll::Pending + } else { + Poll::Ready(iter.next()) + } + }) +} diff --git a/hydroflow/src/util/monotonic.rs b/hydroflow/src/util/monotonic.rs new file mode 100644 index 00000000000..e27135bba4f --- /dev/null +++ b/hydroflow/src/util/monotonic.rs @@ -0,0 +1,9 @@ +/// A wrapper christening a closure as a [monotonic function](https://hydro.run/docs/hydroflow/lattices_crate/lattice_math#the-calm-theorem-and-monotonicity) +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct MonotonicFn(pub F); + +/// A wrapper christening a closure as a [lattice morphism](https://hydro.run/docs/hydroflow/lattices_crate/lattice_math#lattice-morphism) +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct Morphism(pub F); diff --git a/hydroflow/src/util/multiset.rs b/hydroflow/src/util/multiset.rs new file mode 100644 index 00000000000..7385e0631f4 --- /dev/null +++ b/hydroflow/src/util/multiset.rs @@ -0,0 +1,64 @@ +//! A multiset backed by a HashMap +use std::collections::HashMap; +use std::hash::Hash; + +/// A multiset backed by a HashMap +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct HashMultiSet { + items: HashMap, + len: usize, +} + +impl HashMultiSet { + /// Insert item into the multiset. see `https://doc.rust-lang.org/std/collections/struct.HashSet.html#method.insert` + pub fn insert(&mut self, value: T) { + *self.items.entry(value).or_default() += 1; + self.len += 1; + } +} + +impl Default for HashMultiSet +where + T: Hash + Eq, +{ + fn default() -> Self { + Self { + items: HashMap::default(), + len: 0, + } + } +} + +impl FromIterator for HashMultiSet +where + T: Hash + Eq, +{ + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + let mut ret = HashMultiSet::default(); + + for item in iter { + ret.insert(item); + } + + ret + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn basic() { + let mut x = HashMultiSet::default(); + + x.insert(1); + x.insert(2); + x.insert(2); + + assert_eq!(x, HashMultiSet::from_iter([2, 1, 2])); + } +} diff --git a/hydroflow/src/util/sparse_vec.rs b/hydroflow/src/util/sparse_vec.rs index 6dd706eb63a..17ae82f6bd0 100644 --- a/hydroflow/src/util/sparse_vec.rs +++ b/hydroflow/src/util/sparse_vec.rs @@ -38,7 +38,7 @@ impl SparseVec { } /// Iterate through all items in the vector in order. Deleted items will not appear in the iteration. - pub fn iter(&self) -> impl Iterator + FusedIterator + DoubleEndedIterator + Clone { + pub fn iter(&self) -> impl DoubleEndedIterator + FusedIterator + Clone { self.items.iter().filter_map(|x| x.as_ref()) } } diff --git a/hydroflow/src/util/unsync/mpsc.rs b/hydroflow/src/util/unsync/mpsc.rs index d8e9a6a4fd2..b4b8d802fb4 100644 --- a/hydroflow/src/util/unsync/mpsc.rs +++ b/hydroflow/src/util/unsync/mpsc.rs @@ -151,8 +151,8 @@ impl Receiver { } /// Poll for a value. - // NOTE: takes `&mut` to prevent multiple concurrent receives. - pub fn poll_recv(&mut self, ctx: &mut Context<'_>) -> Poll> { + /// NOTE: takes `&mut self` to prevent multiple concurrent receives. + pub fn poll_recv(&mut self, ctx: &Context<'_>) -> Poll> { let mut shared = self.strong.borrow_mut(); if let Some(value) = shared.buffer.pop_front() { shared.wake_sender(); diff --git a/hydroflow/tests/compile-fail/surface_badgeneric_type.rs b/hydroflow/tests/compile-fail/surface_badgeneric_type.rs index b05c70a380e..f7e1b1fe3a9 100644 --- a/hydroflow/tests/compile-fail/surface_badgeneric_type.rs +++ b/hydroflow/tests/compile-fail/surface_badgeneric_type.rs @@ -2,8 +2,8 @@ use hydroflow::hydroflow_syntax; fn main() { let mut df = hydroflow_syntax! { - // no generic arguments for `next_tick`. - source_iter(0..10) -> next_tick::() -> for_each(std::mem::drop); + // no generic arguments for `defer`. + source_iter(0..10) -> defer_tick::() -> for_each(std::mem::drop); }; df.run_available(); } diff --git a/hydroflow/tests/compile-fail/surface_badgeneric_type.stderr b/hydroflow/tests/compile-fail/surface_badgeneric_type.stderr index 1db88fba5b0..6a8ab9bffce 100644 --- a/hydroflow/tests/compile-fail/surface_badgeneric_type.stderr +++ b/hydroflow/tests/compile-fail/surface_badgeneric_type.stderr @@ -1,5 +1,5 @@ -error: `next_tick` should have exactly 0 generic type arguments, actually has 1. - --> tests/compile-fail/surface_badgeneric_type.rs:6:43 +error: `defer_tick` should have exactly 0 generic type arguments, actually has 1. + --> tests/compile-fail/surface_badgeneric_type.rs:6:44 | -6 | source_iter(0..10) -> next_tick::() -> for_each(std::mem::drop); - | ^^^^^ +6 | source_iter(0..10) -> defer_tick::() -> for_each(std::mem::drop); + | ^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_conflicting_name.stderr b/hydroflow/tests/compile-fail/surface_conflicting_name.stderr index c404b3bf831..5aef3f1b0fa 100644 --- a/hydroflow/tests/compile-fail/surface_conflicting_name.stderr +++ b/hydroflow/tests/compile-fail/surface_conflicting_name.stderr @@ -2,7 +2,7 @@ error: Name assignment to `a` conflicts with existing assignment: $DIR/tests/com --> tests/compile-fail/surface_conflicting_name.rs:6:9 | 6 | a = null() -> null(); - | ^^^^^^^^^^^^^^^^^^^^ + | ^^^^^^^^^^^^^^^^^^^^^ error: Existing assignment to `a` conflicts with later assignment: $DIR/tests/compile-fail/surface_conflicting_name.rs:6:9 (2/2) --> tests/compile-fail/surface_conflicting_name.rs:5:9 diff --git a/hydroflow/tests/compile-fail/surface_degenerate_null.stderr b/hydroflow/tests/compile-fail/surface_degenerate_null.stderr index c30fe49a499..c6566c394a3 100644 --- a/hydroflow/tests/compile-fail/surface_degenerate_null.stderr +++ b/hydroflow/tests/compile-fail/surface_degenerate_null.stderr @@ -7,6 +7,4 @@ error: proc macro panicked 7 | | }; | |_____^ | - = help: message: assertion failed: `(left == right)` - left: `1`, - right: `0`: If entire subgraph is pull, should have only one handoff output. Do you have a loose `null()` or other degenerate pipeline somewhere? + = help: message: Degenerate subgraph detected, is there a disconnected `null()` or other degenerate pipeline somewhere? diff --git a/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs new file mode 100644 index 00000000000..b958c140e90 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs @@ -0,0 +1,15 @@ +use hydroflow::lattices::set_union::{SetUnionHashSet, SetUnionSingletonSet, SetUnion}; +use hydroflow::lattices::collections::SingletonSet; + +pub fn main() { + let mut hf = hydroflow::hydroflow_syntax! { + source_iter_delta((0..10).map(SetUnionSingletonSet::new_from)) + -> lattice_fold::<'static, SetUnionHashSet<_>>() + -> lattice_fold::<'static, SetUnionHashSet<_>>() + -> lattice_reduce::<'static>() + -> null(); + }; + hf.run_available(); + + compile_error!("warning test"); +} diff --git a/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.stderr b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.stderr new file mode 100644 index 00000000000..7befef7c234 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_cumul.stderr @@ -0,0 +1,25 @@ +error: `lattice_reduce` should have exactly 1 generic type arguments, actually has 0. + --> tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs:9:33 + | +9 | -> lattice_reduce::<'static>() + | ^^^^^^^ + +error: warning test + --> tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs:14:5 + | +14 | compile_error!("warning test"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `SetUnionHashSet`, `SetUnionSingletonSet`, `SetUnion` + --> tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs:1:38 + | +1 | use hydroflow::lattices::set_union::{SetUnionHashSet, SetUnionSingletonSet, SetUnion}; + | ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `hydroflow::lattices::collections::SingletonSet` + --> tests/compile-fail/surface_flow_props_cumul_warn_cumul.rs:2:5 + | +2 | use hydroflow::lattices::collections::SingletonSet; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.rs b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.rs new file mode 100644 index 00000000000..35d60ac63f8 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.rs @@ -0,0 +1,16 @@ +use hydroflow::lattices::set_union::{SetUnionHashSet, SetUnionSingletonSet, SetUnion}; +use hydroflow::lattices::collections::SingletonSet; + +pub fn main() { + let mut hf = hydroflow::hydroflow_syntax! { + source_iter_delta((0..10).map(SetUnionSingletonSet::new_from)) + -> cast(None) + -> lattice_fold::<'static, SetUnionHashSet<_>>() + -> cast(None) + -> lattice_reduce::<'static>() + -> null(); + }; + hf.run_available(); + + compile_error!("warning test"); +} diff --git a/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.stderr b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.stderr new file mode 100644 index 00000000000..3399a637d5d --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_cumul_warn_none.stderr @@ -0,0 +1,25 @@ +error: `lattice_reduce` should have exactly 1 generic type arguments, actually has 0. + --> tests/compile-fail/surface_flow_props_cumul_warn_none.rs:10:33 + | +10 | -> lattice_reduce::<'static>() + | ^^^^^^^ + +error: warning test + --> tests/compile-fail/surface_flow_props_cumul_warn_none.rs:15:5 + | +15 | compile_error!("warning test"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: unused imports: `SetUnionHashSet`, `SetUnionSingletonSet`, `SetUnion` + --> tests/compile-fail/surface_flow_props_cumul_warn_none.rs:1:38 + | +1 | use hydroflow::lattices::set_union::{SetUnionHashSet, SetUnionSingletonSet, SetUnion}; + | ^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `hydroflow::lattices::collections::SingletonSet` + --> tests/compile-fail/surface_flow_props_cumul_warn_none.rs:2:5 + | +2 | use hydroflow::lattices::collections::SingletonSet; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_flow_props_upcast.rs b/hydroflow/tests/compile-fail/surface_flow_props_upcast.rs new file mode 100644 index 00000000000..ea5c7c20906 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_upcast.rs @@ -0,0 +1,13 @@ +use hydroflow::lattices::set_union::{SetUnionSingletonSet, SetUnion}; +use hydroflow::lattices::collections::SingletonSet; + +pub fn main() { + let mut hf = hydroflow::hydroflow_syntax! { + source_iter_delta((0..10).map(SetUnionSingletonSet::new_from)) + -> cast(None) + -> map(|SetUnion(SingletonSet(x))| 10 * x) + -> cast(Some(Delta)) + -> for_each(|x| println!("seq {:?}", x)); + }; + hf.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_flow_props_upcast.stderr b/hydroflow/tests/compile-fail/surface_flow_props_upcast.stderr new file mode 100644 index 00000000000..cfec3dd4fff --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_flow_props_upcast.stderr @@ -0,0 +1,19 @@ +error: Cannot ilegally up-cast from `None` to `Some(Delta)`. + --> tests/compile-fail/surface_flow_props_upcast.rs:9:16 + | +9 | -> cast(Some(Delta)) + | ^^^^^^^^^^^^^^^^^ + +warning: unused imports: `SetUnionSingletonSet`, `SetUnion` + --> tests/compile-fail/surface_flow_props_upcast.rs:1:38 + | +1 | use hydroflow::lattices::set_union::{SetUnionSingletonSet, SetUnion}; + | ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^ + | + = note: `#[warn(unused_imports)]` on by default + +warning: unused import: `hydroflow::lattices::collections::SingletonSet` + --> tests/compile-fail/surface_flow_props_upcast.rs:2:5 + | +2 | use hydroflow::lattices::collections::SingletonSet; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.rs.ignore b/hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.rs.ignore similarity index 81% rename from hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.rs.ignore rename to hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.rs.ignore index 638cdd2b244..051e7f6f6a7 100644 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.rs.ignore +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.rs.ignore @@ -3,7 +3,7 @@ use hydroflow::hydroflow_syntax; fn main() { let mut df = hydroflow_syntax! { source_iter([1,2,3,4,5]) - -> lattice_merge::<'static, usize>() + -> lattice_fold::<'static, usize>() -> for_each(|x| println!("Least upper bound: {:?}", x)); }; df.run_available(); diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.stderr similarity index 85% rename from hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.stderr rename to hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.stderr index f082da77b67..a92cdebecbb 100644 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_badgeneric.stderr +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_badgeneric.stderr @@ -1,7 +1,7 @@ error[E0277]: the trait bound `usize: Merge` is not satisfied - --> tests/compile-fail/surface_lattice_merge_badgeneric.rs:6:41 + --> tests/compile-fail/surface_lattice_fold_badgeneric.rs:6:41 | -6 | -> lattice_merge::<'static, usize>() +6 | -> lattice_fold::<'static, usize>() | ^^^^^ the trait `Merge` is not implemented for `usize` | = help: the following other types implement trait `Merge`: @@ -15,22 +15,22 @@ error[E0277]: the trait bound `usize: Merge` is not satisfied as Merge>> and $N others note: required by a bound in `check_inputs` - --> tests/compile-fail/surface_lattice_merge_badgeneric.rs:4:18 + --> tests/compile-fail/surface_lattice_fold_badgeneric.rs:4:18 | 4 | let mut df = hydroflow_syntax! { | __________________^ 5 | | source_iter([1,2,3,4,5]) -6 | | -> lattice_merge::<'static, usize>() +6 | | -> lattice_fold::<'static, usize>() 7 | | -> for_each(|x| println!("Least upper bound: {:?}", x)); 8 | | }; | |_____^ required by this bound in `check_inputs` = note: this error originates in the macro `hydroflow_syntax` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: the trait bound `usize: Merge` is not satisfied - --> tests/compile-fail/surface_lattice_merge_badgeneric.rs:5:9 + --> tests/compile-fail/surface_lattice_fold_badgeneric.rs:5:9 | 5 | / source_iter([1,2,3,4,5]) -6 | | -> lattice_merge::<'static, usize>() +6 | | -> lattice_fold::<'static, usize>() | |________________________________________________^ the trait `Merge` is not implemented for `usize` | = help: the following other types implement trait `Merge`: @@ -45,12 +45,12 @@ error[E0277]: the trait bound `usize: Merge` is not satisfied and $N others error[E0277]: the trait bound `usize: Merge` is not satisfied - --> tests/compile-fail/surface_lattice_merge_badgeneric.rs:4:18 + --> tests/compile-fail/surface_lattice_fold_badgeneric.rs:4:18 | 4 | let mut df = hydroflow_syntax! { | __________________^ 5 | | source_iter([1,2,3,4,5]) -6 | | -> lattice_merge::<'static, usize>() +6 | | -> lattice_fold::<'static, usize>() 7 | | -> for_each(|x| println!("Least upper bound: {:?}", x)); 8 | | }; | |_____^ the trait `Merge` is not implemented for `usize` diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.rs b/hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.rs similarity index 83% rename from hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.rs rename to hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.rs index 0f249cd3048..e5954bbcd18 100644 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.rs +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.rs @@ -3,7 +3,7 @@ use hydroflow::hydroflow_syntax; fn main() { let mut df = hydroflow_syntax! { source_iter([1,2,3,4,5]) - -> lattice_merge::<'static>() + -> lattice_fold::<'static>() -> for_each(|x| println!("Least upper bound: {:?}", x)); }; df.run_available(); diff --git a/hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.stderr new file mode 100644 index 00000000000..636a8c3e9cb --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_nogeneric.stderr @@ -0,0 +1,5 @@ +error: `lattice_fold` should have exactly 1 generic type arguments, actually has 0. + --> tests/compile-fail/surface_lattice_fold_nogeneric.rs:6:31 + | +6 | -> lattice_fold::<'static>() + | ^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.rs b/hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.rs similarity index 69% rename from hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.rs rename to hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.rs index 1edc48f08e5..87e82dfaa97 100644 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.rs +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.rs @@ -3,7 +3,7 @@ use hydroflow::hydroflow_syntax; fn main() { let mut df = hydroflow_syntax! { source_iter([1,2,3,4,5]) - -> lattice_merge::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + -> lattice_fold::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() -> for_each(|x| println!("Least upper bound: {:?}", x)); }; df.run_available(); diff --git a/hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.stderr new file mode 100644 index 00000000000..af7301fb80d --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_fold_wronggeneric.stderr @@ -0,0 +1,29 @@ +warning: `lattice_fold` expects lattice flow input, has sequential input. This may be an error in the future. + --> tests/compile-fail/surface_lattice_fold_wronggeneric.rs:6:16 + | +6 | -> lattice_fold::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0308]: mismatched types + --> tests/compile-fail/surface_lattice_fold_wronggeneric.rs:6:16 + | +4 | let mut df = hydroflow_syntax! { + | __________________- +5 | | source_iter([1,2,3,4,5]) +6 | | -> lattice_fold::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `SetUnion<_>`, found integer +7 | | -> for_each(|x| println!("Least upper bound: {:?}", x)); +8 | | }; + | |_____- arguments to this function are incorrect + | + = note: expected struct `SetUnion<_>` + found type `{integer}` +note: method defined here + --> $WORKSPACE/lattices/src/lib.rs + | + | fn merge(&mut self, other: Other) -> bool; + | ^^^^^ +help: try wrapping the expression in `hydroflow::lattices::set_union::SetUnion` + | +6 | -> hydroflow::lattices::set_union::SetUnion(lattice_fold::<'static, hydroflow::lattices::set_union::SetUnionHashSet>()) + | +++++++++++++++++++++++++++++++++++++++++ + diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.stderr deleted file mode 100644 index fb0b00b138c..00000000000 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_nogeneric.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: `lattice_merge` should have exactly 1 generic type arguments, actually has 0. - --> tests/compile-fail/surface_lattice_merge_nogeneric.rs:6:32 - | -6 | -> lattice_merge::<'static>() - | ^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.rs.ignore b/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.rs.ignore new file mode 100644 index 00000000000..1fad32e0685 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.rs.ignore @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + source_iter([1,2,3,4,5]) + -> lattice_reduce::<'static, usize>() + -> for_each(|x| println!("Least upper bound: {:?}", x)); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.stderr new file mode 100644 index 00000000000..29bb4627a53 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_badgeneric.stderr @@ -0,0 +1,68 @@ +error[E0277]: the trait bound `usize: Merge` is not satisfied + --> tests/compile-fail/surface_lattice_reduce_badgeneric.rs:6:41 + | +6 | -> lattice_reduce::<'static, usize>() + | ^^^^^ the trait `Merge` is not implemented for `usize` + | + = help: the following other types implement trait `Merge`: + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + and $N others +note: required by a bound in `check_inputs` + --> tests/compile-fail/surface_lattice_reduce_badgeneric.rs:4:18 + | +4 | let mut df = hydroflow_syntax! { + | __________________^ +5 | | source_iter([1,2,3,4,5]) +6 | | -> lattice_reduce::<'static, usize>() +7 | | -> for_each(|x| println!("Least upper bound: {:?}", x)); +8 | | }; + | |_____^ required by this bound in `check_inputs` + = note: this error originates in the macro `hydroflow_syntax` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `usize: Merge` is not satisfied + --> tests/compile-fail/surface_lattice_reduce_badgeneric.rs:5:9 + | +5 | / source_iter([1,2,3,4,5]) +6 | | -> lattice_reduce::<'static, usize>() + | |________________________________________________^ the trait `Merge` is not implemented for `usize` + | + = help: the following other types implement trait `Merge`: + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + and $N others + +error[E0277]: the trait bound `usize: Merge` is not satisfied + --> tests/compile-fail/surface_lattice_reduce_badgeneric.rs:4:18 + | +4 | let mut df = hydroflow_syntax! { + | __________________^ +5 | | source_iter([1,2,3,4,5]) +6 | | -> lattice_reduce::<'static, usize>() +7 | | -> for_each(|x| println!("Least upper bound: {:?}", x)); +8 | | }; + | |_____^ the trait `Merge` is not implemented for `usize` + | + = help: the following other types implement trait `Merge`: + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + as Merge>> + and $N others + = note: this error originates in the macro `hydroflow_syntax` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.rs b/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.rs new file mode 100644 index 00000000000..63fdfdeaa7a --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + source_iter([1,2,3,4,5]) + -> lattice_reduce::<'static>() + -> for_each(|x| println!("Least upper bound: {:?}", x)); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.stderr new file mode 100644 index 00000000000..2e10a7fb125 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_nogeneric.stderr @@ -0,0 +1,5 @@ +error: `lattice_reduce` should have exactly 1 generic type arguments, actually has 0. + --> tests/compile-fail/surface_lattice_reduce_nogeneric.rs:6:33 + | +6 | -> lattice_reduce::<'static>() + | ^^^^^^^ diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.rs b/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.rs new file mode 100644 index 00000000000..08306055ea8 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + source_iter([1,2,3,4,5]) + -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + -> for_each(|x| println!("Least upper bound: {:?}", x)); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.stderr b/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.stderr new file mode 100644 index 00000000000..af24760170d --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_lattice_reduce_wronggeneric.stderr @@ -0,0 +1,21 @@ +warning: `lattice_reduce` expects lattice flow input, has sequential input. This may be an error in the future. + --> tests/compile-fail/surface_lattice_reduce_wronggeneric.rs:6:16 + | +6 | -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0271]: expected `Drain<'_, {integer}>` to be an iterator that yields `SetUnion>`, but it yields `{integer}` + --> tests/compile-fail/surface_lattice_reduce_wronggeneric.rs:5:9 + | +5 | source_iter([1,2,3,4,5]) + | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `SetUnion>`, found integer +6 | -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + | ---------------------------------------------------- required by a bound introduced by this call + | + = note: expected struct `SetUnion>` + found type `{integer}` +note: required by a bound in `check_inputs` + --> tests/compile-fail/surface_lattice_reduce_wronggeneric.rs:6:42 + | +6 | -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `check_inputs` diff --git a/hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.stderr b/hydroflow/tests/compile-fail/surface_latticereduce_wronggeneric.stderr similarity index 66% rename from hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.stderr rename to hydroflow/tests/compile-fail/surface_latticereduce_wronggeneric.stderr index fed054512c1..714a5d8013f 100644 --- a/hydroflow/tests/compile-fail/surface_lattice_merge_wronggeneric.stderr +++ b/hydroflow/tests/compile-fail/surface_latticereduce_wronggeneric.stderr @@ -1,15 +1,15 @@ error[E0271]: expected `Drain<'_, {integer}>` to be an iterator that yields `SetUnion>`, but it yields `{integer}` - --> tests/compile-fail/surface_lattice_merge_wronggeneric.rs:5:9 + --> tests/compile-fail/surface_lattice_reduce_wronggeneric.rs:5:9 | 5 | source_iter([1,2,3,4,5]) | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `SetUnion>`, found integer -6 | -> lattice_merge::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() +6 | -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() | ---------------------------------------------------- required by a bound introduced by this call | = note: expected struct `SetUnion>` found type `{integer}` note: required by a bound in `check_inputs` - --> tests/compile-fail/surface_lattice_merge_wronggeneric.rs:6:41 + --> tests/compile-fail/surface_lattice_reduce_wronggeneric.rs:6:41 | -6 | -> lattice_merge::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() +6 | -> lattice_reduce::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `check_inputs` diff --git a/hydroflow/tests/compile-fail/surface_negative_loop.stderr b/hydroflow/tests/compile-fail/surface_negative_loop.stderr index b1ce8e62b49..b82fce2d061 100644 --- a/hydroflow/tests/compile-fail/surface_negative_loop.stderr +++ b/hydroflow/tests/compile-fail/surface_negative_loop.stderr @@ -1,4 +1,4 @@ -error: Negative edge creates a negative cycle which must be broken with a `next_tick()` operator. +error: Negative edge creates a negative cycle which must be broken with a `defer_tick()` operator. --> tests/compile-fail/surface_negative_loop.rs:7:18 | 7 | diff -> [neg]diff; diff --git a/hydroflow/tests/compile-fail/surface_null.stderr b/hydroflow/tests/compile-fail/surface_null.stderr index 1aa6e5414e4..1b93ceb376f 100644 --- a/hydroflow/tests/compile-fail/surface_null.stderr +++ b/hydroflow/tests/compile-fail/surface_null.stderr @@ -7,6 +7,4 @@ error: proc macro panicked 6 | | }; | |_____^ | - = help: message: assertion failed: `(left == right)` - left: `1`, - right: `0`: If entire subgraph is pull, should have only one handoff output. Do you have a loose `null()` or other degenerate pipeline somewhere? + = help: message: Degenerate subgraph detected, is there a disconnected `null()` or other degenerate pipeline somewhere? diff --git a/hydroflow/tests/compile-fail/surface_syntax_eol_arrow.stderr b/hydroflow/tests/compile-fail/surface_syntax_eol_arrow.stderr index f51dde5abac..7eaa1562265 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_eol_arrow.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_eol_arrow.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected one of: square brackets, identifier, parentheses +error: unexpected end of input, expected one of: square brackets, `mod`, identifier, parentheses --> tests/compile-fail/surface_syntax_eol_arrow.rs:4:18 | 4 | let mut df = hydroflow_syntax! { diff --git a/hydroflow/tests/compile-fail/surface_syntax_eol_indexing.stderr b/hydroflow/tests/compile-fail/surface_syntax_eol_indexing.stderr index 2c2cc183022..bbdf02f4a57 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_eol_indexing.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_eol_indexing.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected parentheses or identifier +error: unexpected end of input, expected one of: parentheses, identifier, `mod` --> tests/compile-fail/surface_syntax_eol_indexing.rs:4:18 | 4 | let mut df = hydroflow_syntax! { diff --git a/hydroflow/tests/compile-fail/surface_syntax_eol_missingop.stderr b/hydroflow/tests/compile-fail/surface_syntax_eol_missingop.stderr index 6184e119532..d1ef30a9a9d 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_eol_missingop.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_eol_missingop.stderr @@ -1,4 +1,4 @@ -error: expected one of: square brackets, identifier, parentheses +error: expected one of: square brackets, `mod`, identifier, parentheses --> tests/compile-fail/surface_syntax_eol_missingop.rs:5:31 | 5 | source_iter(0..10) -> ; diff --git a/hydroflow/tests/compile-fail/surface_syntax_indexing_empty.stderr b/hydroflow/tests/compile-fail/surface_syntax_indexing_empty.stderr index ac26499a7ce..98fa9ec95ac 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_indexing_empty.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_indexing_empty.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected one of: square brackets, identifier, parentheses +error: unexpected end of input, expected one of: square brackets, `mod`, identifier, parentheses --> tests/compile-fail/surface_syntax_indexing_empty.rs:5:35 | 5 | source_iter(0..10) -> [0](); diff --git a/hydroflow/tests/compile-fail/surface_syntax_paren_arrow.stderr b/hydroflow/tests/compile-fail/surface_syntax_paren_arrow.stderr index aa6f05075c0..4d1ba1f9a3c 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_paren_arrow.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_paren_arrow.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected one of: square brackets, identifier, parentheses +error: unexpected end of input, expected one of: square brackets, `mod`, identifier, parentheses --> tests/compile-fail/surface_syntax_paren_arrow.rs:5:31 | 5 | (source_iter(0..10) ->); diff --git a/hydroflow/tests/compile-fail/surface_syntax_paren_indexing.stderr b/hydroflow/tests/compile-fail/surface_syntax_paren_indexing.stderr index 958c91d6671..81e5b0506aa 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_paren_indexing.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_paren_indexing.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected parentheses or identifier +error: unexpected end of input, expected one of: parentheses, identifier, `mod` --> tests/compile-fail/surface_syntax_paren_indexing.rs:5:35 | 5 | (source_iter(0..10) -> [0]); diff --git a/hydroflow/tests/compile-fail/surface_syntax_paren_missingop.stderr b/hydroflow/tests/compile-fail/surface_syntax_paren_missingop.stderr index 8f594040753..9a3c4238ec0 100644 --- a/hydroflow/tests/compile-fail/surface_syntax_paren_missingop.stderr +++ b/hydroflow/tests/compile-fail/surface_syntax_paren_missingop.stderr @@ -1,4 +1,4 @@ -error: unexpected end of input, expected one of: square brackets, identifier, parentheses +error: unexpected end of input, expected one of: square brackets, `mod`, identifier, parentheses --> tests/compile-fail/surface_syntax_paren_missingop.rs:5:32 | 5 | (source_iter(0..10) -> ); diff --git a/hydroflow/tests/compile-fail/surface_use_bad1.rs b/hydroflow/tests/compile-fail/surface_use_bad1.rs new file mode 100644 index 00000000000..09a005b9ae5 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad1.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + use x::; + + source_iter(0..10) -> for_each(std::mem::drop); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_use_bad1.stderr b/hydroflow/tests/compile-fail/surface_use_bad1.stderr new file mode 100644 index 00000000000..1509a90316f --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad1.stderr @@ -0,0 +1,5 @@ +error: expected one of: identifier, `self`, `super`, `crate`, `try`, `*`, curly braces + --> tests/compile-fail/surface_use_bad1.rs:5:16 + | +5 | use x::; + | ^ diff --git a/hydroflow/tests/compile-fail/surface_use_bad2.rs b/hydroflow/tests/compile-fail/surface_use_bad2.rs new file mode 100644 index 00000000000..4cb2cf288f9 --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad2.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + use + + source_iter(0..10) -> for_each(std::mem::drop); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_use_bad2.stderr b/hydroflow/tests/compile-fail/surface_use_bad2.stderr new file mode 100644 index 00000000000..a775fed6a9c --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad2.stderr @@ -0,0 +1,5 @@ +error: expected `;` + --> tests/compile-fail/surface_use_bad2.rs:7:20 + | +7 | source_iter(0..10) -> for_each(std::mem::drop); + | ^ diff --git a/hydroflow/tests/compile-fail/surface_use_bad3.rs b/hydroflow/tests/compile-fail/surface_use_bad3.rs new file mode 100644 index 00000000000..64024c337cf --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad3.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + use; + + source_iter(0..10) -> for_each(std::mem::drop); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_use_bad3.stderr b/hydroflow/tests/compile-fail/surface_use_bad3.stderr new file mode 100644 index 00000000000..b48bbb52e8c --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_bad3.stderr @@ -0,0 +1,5 @@ +error: expected one of: identifier, `self`, `super`, `crate`, `try`, `*`, curly braces + --> tests/compile-fail/surface_use_bad3.rs:5:12 + | +5 | use; + | ^ diff --git a/hydroflow/tests/compile-fail/surface_use_unknown.rs b/hydroflow/tests/compile-fail/surface_use_unknown.rs new file mode 100644 index 00000000000..67049c34a3d --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_unknown.rs @@ -0,0 +1,10 @@ +use hydroflow::hydroflow_syntax; + +fn main() { + let mut df = hydroflow_syntax! { + use x; + + source_iter(0..10) -> for_each(std::mem::drop); + }; + df.run_available(); +} diff --git a/hydroflow/tests/compile-fail/surface_use_unknown.stderr b/hydroflow/tests/compile-fail/surface_use_unknown.stderr new file mode 100644 index 00000000000..06a0fda78ec --- /dev/null +++ b/hydroflow/tests/compile-fail/surface_use_unknown.stderr @@ -0,0 +1,5 @@ +error[E0432]: unresolved import `x` + --> tests/compile-fail/surface_use_unknown.rs:5:13 + | +5 | use x; + | ^ no external crate `x` diff --git a/hydroflow/tests/datalog_frontend.rs b/hydroflow/tests/datalog_frontend.rs index dd59e5b00f0..e651c50cb1f 100644 --- a/hydroflow/tests/datalog_frontend.rs +++ b/hydroflow/tests/datalog_frontend.rs @@ -1,5 +1,6 @@ use hydroflow::datalog; use hydroflow::util::collect_ready; +use hydroflow::util::multiset::HashMultiSet; use multiplatform_test::multiplatform_test; #[multiplatform_test] @@ -91,8 +92,8 @@ pub fn test_join_with_self() { flow.run_tick(); assert_eq!( - &*collect_ready::, _>(&mut out_recv), - &[(2, 1), (1, 2)] + collect_ready::, _>(&mut out_recv), + HashMultiSet::from_iter([(2, 1), (1, 2)]) ); } @@ -140,8 +141,8 @@ pub fn test_multi_use_intermediate() { flow.run_tick(); assert_eq!( - &*collect_ready::, _>(&mut out_recv), - &[(2, 1), (1, 2)] + collect_ready::, _>(&mut out_recv), + HashMultiSet::from_iter([(2, 1), (1, 2)]) ); } @@ -442,8 +443,8 @@ pub fn test_join_multiple_and_relation() { flow.run_tick(); assert_eq!( - &*collect_ready::, _>(&mut out_recv), - &[(1, 2, 3, 4), (1, 2, 4, 5)] + collect_ready::, _>(&mut out_recv), + HashMultiSet::from_iter([(1, 2, 3, 4), (1, 2, 4, 5)]) ); } @@ -479,13 +480,13 @@ pub fn test_join_multiple_then_relation() { flow.run_tick(); assert_eq!( - &*collect_ready::, _>(&mut out_recv), - &[(1, 2, 3, 4), (1, 2, 4, 5)] + collect_ready::, _>(&mut out_recv), + HashMultiSet::from_iter([(1, 2, 3, 4), (1, 2, 4, 5)]) ); } #[multiplatform_test] -pub fn test_next_tick() { +pub fn test_defer_tick() { let (ints_1_send, ints_1) = hydroflow::util::unbounded_channel::<(usize,)>(); let (ints_2_send, ints_2) = hydroflow::util::unbounded_channel::<(usize,)>(); let (result, mut result_recv) = hydroflow::util::unbounded_channel::<(usize,)>(); @@ -561,7 +562,7 @@ pub fn test_anti_join() { } #[multiplatform_test] -pub fn test_anti_join_next_tick() { +pub fn test_anti_join_defer_tick() { let (ints_1_send, ints_1) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let (ints_2_send, ints_2) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let (ints_3_send, ints_3) = hydroflow::util::unbounded_channel::<(usize,)>(); @@ -604,7 +605,7 @@ pub fn test_anti_join_next_tick() { } #[multiplatform_test] -pub fn test_anti_join_next_tick_cycle() { +pub fn test_anti_join_defer_cycle() { let (ints_1_send, ints_1) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let (ints_2_send, ints_2) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let (ints_3_send, ints_3) = hydroflow::util::unbounded_channel::<(usize,)>(); @@ -698,7 +699,7 @@ fn test_max_all() { } #[multiplatform_test] -fn test_max_next_tick() { +fn test_max_defer_tick() { let (ints_send, ints) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let (result, mut result_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); @@ -1039,12 +1040,21 @@ fn test_persist() { flow.run_tick(); assert_eq!( - &*collect_ready::, _>(&mut result_recv), - &[(1, 2, 6), (1, 1, 6), (1, 3, 6)] + collect_ready::, _>(&mut result_recv), + HashMultiSet::from_iter([(1, 2, 6), (1, 1, 6), (1, 3, 6)]) + ); + assert_eq!( + collect_ready::, _>(&mut result2_recv), + HashMultiSet::from_iter([]) + ); + assert_eq!( + collect_ready::, _>(&mut result3_recv), + HashMultiSet::from_iter([(1,)]) + ); + assert_eq!( + collect_ready::, _>(&mut result4_recv), + HashMultiSet::from_iter([(1,)]) ); - assert_eq!(&*collect_ready::, _>(&mut result2_recv), &[]); - assert_eq!(&*collect_ready::, _>(&mut result3_recv), &[(1,)]); - assert_eq!(&*collect_ready::, _>(&mut result4_recv), &[(1,)]); } #[multiplatform_test] @@ -1115,113 +1125,118 @@ fn test_wildcard_join_count() { assert_eq!(&*collect_ready::, _>(&mut result2_recv), &[(1,)]); } -#[multiplatform_test] -fn test_index() { - let (ints_send, ints) = hydroflow::util::unbounded_channel::<(i64, i64)>(); - let (result, mut result_recv) = hydroflow::util::unbounded_channel::<(i64, i64, i32)>(); - let (result2, mut result2_recv) = hydroflow::util::unbounded_channel::<(i64, usize, usize)>(); - - let (result3, mut result3_recv) = hydroflow::util::unbounded_channel::<(i64, i64, usize)>(); - let (result4, mut result4_recv) = hydroflow::util::unbounded_channel::<(i64, usize, usize)>(); - let (result5, mut result5_recv) = hydroflow::util::unbounded_channel::<(i64, i64, usize)>(); - - let mut flow = datalog!( - r#" - .input ints `source_stream(ints)` - - .output result `for_each(|v| result.send(v).unwrap())` - .output result2 `for_each(|v| result2.send(v).unwrap())` - .output result3 `for_each(|v| result3.send(v).unwrap())` - .output result4 `for_each(|v| result4.send(v).unwrap())` - - .persist result5 - .output result5 `for_each(|v| result5.send(v).unwrap())` - - result(a, b, index()) :- ints(a, b) - result2(a, count(b), index()) :- ints(a, b) - - .persist ints_persisted - ints_persisted(a, b) :- ints(a, b) - - result3(a, b, index()) :- ints_persisted(a, b) - result4(a, count(b), index()) :- ints_persisted(a, b) - result5(a, b, index()) :- ints_persisted(a, b) - "# - ); - - ints_send.send((1, 1)).unwrap(); - ints_send.send((1, 2)).unwrap(); - ints_send.send((2, 1)).unwrap(); - - flow.run_tick(); - - assert_eq!( - &*collect_ready::, _>(&mut result_recv), - &[(1, 1, 0), (1, 2, 1), (2, 1, 2)] - ); - - // hashing / ordering differences? - #[cfg(not(target_arch = "wasm32"))] - assert_eq!( - &*collect_ready::, _>(&mut result2_recv), - &[(1, 2, 0), (2, 1, 1)] - ); - - #[cfg(target_arch = "wasm32")] - assert_eq!( - &*collect_ready::, _>(&mut result2_recv), - &[(2, 1, 0), (1, 2, 1)] - ); - - assert_eq!( - &*collect_ready::, _>(&mut result3_recv), - &[(1, 1, 0), (1, 2, 1), (2, 1, 2)] - ); - - #[cfg(not(target_arch = "wasm32"))] - assert_eq!( - &*collect_ready::, _>(&mut result4_recv), - &[(1, 2, 0), (2, 1, 1)] - ); - #[cfg(target_arch = "wasm32")] - assert_eq!( - &*collect_ready::, _>(&mut result4_recv), - &[(2, 1, 0), (1, 2, 1)] - ); - - assert_eq!( - &*collect_ready::, _>(&mut result5_recv), - &[(1, 1, 0), (1, 2, 1), (2, 1, 2)] - ); - - ints_send.send((3, 1)).unwrap(); - - flow.run_tick(); - - assert_eq!(&*collect_ready::, _>(&mut result_recv), &[(3, 1, 0)]); - assert_eq!( - &*collect_ready::, _>(&mut result2_recv), - &[(3, 1, 0)] - ); - - assert_eq!( - &*collect_ready::, _>(&mut result3_recv), - &[(1, 1, 0), (1, 2, 1), (2, 1, 2), (3, 1, 3)] - ); - - #[cfg(not(target_arch = "wasm32"))] - assert_eq!( - &*collect_ready::, _>(&mut result4_recv), - &[(1, 2, 0), (2, 1, 1), (3, 1, 2)] - ); - #[cfg(target_arch = "wasm32")] - assert_eq!( - &*collect_ready::, _>(&mut result4_recv), - &[(2, 1, 0), (3, 1, 1), (1, 2, 2)] - ); - - assert_eq!( - &*collect_ready::, _>(&mut result5_recv), - &[(1, 1, 0), (1, 2, 1), (2, 1, 2), (3, 1, 3)] - ); -} +// #[ignore] doesn't seem to work for #[multiplatform_test] +// #[ignore] // This test depends on the ordering of specific tuples which is undefined. +// #[multiplatform_test] +// fn test_index() { +// let (ints_send, ints) = hydroflow::util::unbounded_channel::<(i64, i64)>(); +// let (result, mut result_recv) = hydroflow::util::unbounded_channel::<(i64, i64, i32)>(); +// let (result2, mut result2_recv) = hydroflow::util::unbounded_channel::<(i64, usize, usize)>(); + +// let (result3, mut result3_recv) = hydroflow::util::unbounded_channel::<(i64, i64, usize)>(); +// let (result4, mut result4_recv) = hydroflow::util::unbounded_channel::<(i64, usize, usize)>(); +// let (result5, mut result5_recv) = hydroflow::util::unbounded_channel::<(i64, i64, usize)>(); + +// let mut flow = datalog!( +// r#" +// .input ints `source_stream(ints)` + +// .output result `for_each(|v| result.send(v).unwrap())` +// .output result2 `for_each(|v| result2.send(v).unwrap())` +// .output result3 `for_each(|v| result3.send(v).unwrap())` +// .output result4 `for_each(|v| result4.send(v).unwrap())` + +// .persist result5 +// .output result5 `for_each(|v| result5.send(v).unwrap())` + +// result(a, b, index()) :- ints(a, b) +// result2(a, count(b), index()) :- ints(a, b) + +// .persist ints_persisted +// ints_persisted(a, b) :- ints(a, b) + +// result3(a, b, index()) :- ints_persisted(a, b) +// result4(a, count(b), index()) :- ints_persisted(a, b) +// result5(a, b, index()) :- ints_persisted(a, b) +// "# +// ); + +// ints_send.send((1, 1)).unwrap(); +// ints_send.send((1, 2)).unwrap(); +// ints_send.send((2, 1)).unwrap(); + +// flow.run_tick(); + +// assert_eq!( +// collect_ready::, _>(&mut result_recv), +// HashMultiSet::from_iter([(1, 1, 0), (1, 2, 1), (2, 1, 2)]), +// ); + +// // hashing / ordering differences? +// #[cfg(not(target_arch = "wasm32"))] +// assert_eq!( +// collect_ready::, _>(&mut result2_recv), +// HashMultiSet::from_iter([(1, 2, 0), (2, 1, 1)]) +// ); + +// #[cfg(target_arch = "wasm32")] +// assert_eq!( +// collect_ready::, _>(&mut result2_recv), +// HashMultiSet::from_iter([(2, 1, 0), (1, 2, 1)]) +// ); + +// assert_eq!( +// collect_ready::, _>(&mut result3_recv), +// HashMultiSet::from_iter([(1, 1, 0), (1, 2, 1), (2, 1, 2)]) +// ); + +// #[cfg(not(target_arch = "wasm32"))] +// assert_eq!( +// collect_ready::, _>(&mut result4_recv), +// HashMultiSet::from_iter([(1, 2, 0), (2, 1, 1)]) +// ); +// #[cfg(target_arch = "wasm32")] +// assert_eq!( +// collect_ready::, _>(&mut result4_recv), +// HashMultiSet::from_iter([(2, 1, 0), (1, 2, 1)]) +// ); + +// assert_eq!( +// collect_ready::, _>(&mut result5_recv), +// HashMultiSet::from_iter([(1, 1, 0), (1, 2, 1), (2, 1, 2)]) +// ); + +// ints_send.send((3, 1)).unwrap(); + +// flow.run_tick(); + +// assert_eq!( +// collect_ready::, _>(&mut result_recv), +// HashMultiSet::from_iter([(3, 1, 0)]) +// ); +// assert_eq!( +// collect_ready::, _>(&mut result2_recv), +// HashMultiSet::from_iter([(3, 1, 0)]) +// ); + +// assert_eq!( +// collect_ready::, _>(&mut result3_recv), +// HashMultiSet::from_iter([(1, 1, 0), (1, 2, 1), (2, 1, 2), (3, 1, 3)]) +// ); + +// #[cfg(not(target_arch = "wasm32"))] +// assert_eq!( +// collect_ready::, _>(&mut result4_recv), +// HashMultiSet::from_iter([(1, 2, 0), (2, 1, 1), (3, 1, 2)]) +// ); +// #[cfg(target_arch = "wasm32")] +// assert_eq!( +// collect_ready::, _>(&mut result4_recv), +// HashMultiSet::from_iter([(2, 1, 0), (3, 1, 1), (1, 2, 2)]) +// ); + +// assert_eq!( +// collect_ready::, _>(&mut result5_recv), +// HashMultiSet::from_iter([(1, 1, 0), (1, 2, 1), (2, 1, 2), (3, 1, 3)]) +// ); +// } diff --git a/hydroflow/tests/snapshots/surface_async__echo@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_async__echo@graphvis_mermaid.snap index 82cd583f338..ca901887491 100644 --- a/hydroflow/tests/snapshots/surface_async__echo@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_async__echo@graphvis_mermaid.snap @@ -10,6 +10,6 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(lines_recv)"/]:::pullClass 2v1[/"(2v1) dest_sink(stdout_lines)"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 end diff --git a/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_dot.snap new file mode 100644 index 00000000000..cd1b8fc3060 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_dot.snap @@ -0,0 +1,37 @@ +--- +source: hydroflow/tests/surface_batch.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n2v1 [label="(n2v1) source_iter([1, 2, 3])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n3v1 [label="(n3v1) source_stream(signal_rx)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 1" + n1v1 [label="(n1v1) defer_signal()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) for_each(|x| egress_tx.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n4v1 + subgraph "cluster sg_3v1_var_gate" { + label="var gate" + n1v1 + } + } + n2v1 -> n5v1 + n3v1 -> n6v1 + n5v1 [label="(n5v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n5v1 -> n1v1 [label="input", arrowhead=box, color=red] + n6v1 [label="(n6v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n6v1 -> n1v1 [label="signal", arrowhead=box, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_mermaid.snap new file mode 100644 index 00000000000..366381012a2 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_batch__basic_2@graphvis_mermaid.snap @@ -0,0 +1,30 @@ +--- +source: hydroflow/tests/surface_batch.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 2v1[\"(2v1) source_iter([1, 2, 3])"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 3v1[\"(3v1) source_stream(signal_rx)"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 1"] + 1v1[\"(1v1) defer_signal()"/]:::pullClass + 4v1[/"(4v1) for_each(|x| egress_tx.send(x).unwrap())"\]:::pushClass + 1v1-->4v1 + subgraph sg_3v1_var_gate ["var gate"] + 1v1 + end +end +2v1-->5v1 +3v1-->6v1 +5v1["(5v1) handoff"]:::otherClass +5v1--x|input|1v1 +6v1["(6v1) handoff"]:::otherClass +6v1--x|signal|1v1 + diff --git a/hydroflow/tests/snapshots/surface_book__surface_flows_1@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_book__surface_flows_1@graphvis_mermaid.snap index d6e69761b32..0ee32f19fc9 100644 --- a/hydroflow/tests/snapshots/surface_book__surface_flows_1@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_book__surface_flows_1@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(vec!["Hello", "world"])"/]:::pullClass 2v1[/"(2v1) tee()"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 subgraph sg_1v1_var_my_tee ["var my_tee"] 1v1 2v1 @@ -21,18 +21,18 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 4v1[\"(4v1) map(|x| x.to_lowercase())"/]:::pullClass 5v1[\"(5v1) union()"/]:::pullClass 6v1[/"(6v1) for_each(|x| println!("{}", x))"\]:::pushClass - 3v1--0--->5v1 - 4v1--1--->5v1 - 5v1--->6v1 + 3v1-->|0|5v1 + 4v1-->|1|5v1 + 5v1-->6v1 subgraph sg_2v1_var_my_union ["var my_union"] 5v1 6v1 end end -2v1--0--->7v1 -2v1--1--->8v1 +2v1-->|0|7v1 +2v1-->|1|8v1 7v1["(7v1) handoff"]:::otherClass -7v1--->3v1 +7v1-->3v1 8v1["(8v1) handoff"]:::otherClass -8v1--->4v1 +8v1-->4v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__basic_2@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__basic_2@graphvis_mermaid.snap index 820a406b397..aea0cfec6ad 100644 --- a/hydroflow/tests/snapshots/surface_codegen__basic_2@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__basic_2@graphvis_mermaid.snap @@ -10,6 +10,6 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter([1])"/]:::pullClass 2v1[/"(2v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 end diff --git a/hydroflow/tests/snapshots/surface_codegen__basic_3@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__basic_3@graphvis_mermaid.snap index e6d724e5c2d..b996d59f708 100644 --- a/hydroflow/tests/snapshots/surface_codegen__basic_3@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__basic_3@graphvis_mermaid.snap @@ -11,7 +11,7 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter([1])"/]:::pullClass 2v1[\"(2v1) map(|v| v + 1)"/]:::pullClass 3v1[/"(3v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 - 2v1--->3v1 + 1v1-->2v1 + 2v1-->3v1 end diff --git a/hydroflow/tests/snapshots/surface_codegen__basic_union@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__basic_union@graphvis_mermaid.snap index e4c34a33ab5..fb9b6f772ac 100644 --- a/hydroflow/tests/snapshots/surface_codegen__basic_union@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__basic_union@graphvis_mermaid.snap @@ -12,9 +12,9 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 4v1[\"(4v1) source_iter([2])"/]:::pullClass 1v1[\"(1v1) union()"/]:::pullClass 2v1[/"(2v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 3v1--0--->1v1 - 4v1--1--->1v1 - 1v1--->2v1 + 3v1-->|0|1v1 + 4v1-->|1|1v1 + 1v1-->2v1 subgraph sg_1v1_var_m ["var m"] 1v1 2v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__covid_tracing@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__covid_tracing@graphvis_mermaid.snap index 13da967d776..bf4fecd0951 100644 --- a/hydroflow/tests/snapshots/surface_codegen__covid_tracing@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__covid_tracing@graphvis_mermaid.snap @@ -18,16 +18,16 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 7v1[\"(7v1) map(|(_pid_a, (pid_b_t_contact, _t_from_to))| pid_b_t_contact)"/]:::pullClass 8v1[/"(8v1) tee()"\]:::pushClass 13v1["(13v1) handoff"]:::otherClass - 13v1--->9v1 - 1v1--->2v1 - 2v1--0--->5v1 - 4v1--0--->3v1 - 9v1--1--->3v1 - 3v1--1--->5v1 - 5v1--->6v1 - 6v1--->7v1 - 7v1--->8v1 - 8v1--0--->13v1 + 13v1-->9v1 + 1v1-->2v1 + 2v1-->|0|5v1 + 4v1-->|0|3v1 + 9v1-->|1|3v1 + 3v1-->|1|5v1 + 5v1-->6v1 + 6v1-->7v1 + 7v1-->8v1 + 8v1-->|0|13v1 subgraph sg_1v1_var_contacts ["var contacts"] 1v1 2v1 @@ -46,14 +46,14 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 12v1[\"(12v1) source_stream(people_recv)"/]:::pullClass 10v1[\"(10v1) join()"/]:::pullClass 11v1[/"
(11v1)
for_each(|(_pid, ((name, phone), exposure))| {
println!("[{}] To {}: Possible Exposure at t = {}", name, phone, exposure);
})"\]:::pushClass - 12v1--0--->10v1 - 10v1--->11v1 + 12v1-->|0|10v1 + 10v1-->11v1 subgraph sg_2v1_var_notifs ["var notifs"] 10v1 11v1 end end -8v1--1--->14v1 +8v1-->|1|14v1 14v1["(14v1) handoff"]:::otherClass -14v1--1--->10v1 +14v1-->|1|10v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__recv_expr@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__recv_expr@graphvis_mermaid.snap index a75013443b9..778353bb249 100644 --- a/hydroflow/tests/snapshots/surface_codegen__recv_expr@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__recv_expr@graphvis_mermaid.snap @@ -10,6 +10,6 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(send_recv.1)"/]:::pullClass 2v1[/"(2v1) for_each(|v| print!("{:?}", v))"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 end diff --git a/hydroflow/tests/snapshots/surface_codegen__sort@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__sort@graphvis_mermaid.snap index 4564665ad63..fe64c0f5f12 100644 --- a/hydroflow/tests/snapshots/surface_codegen__sort@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__sort@graphvis_mermaid.snap @@ -13,9 +13,9 @@ end subgraph sg_2v1 ["sg_2v1 stratum 1"] 2v1[\"(2v1) sort()"/]:::pullClass 3v1[/"(3v1) for_each(|v| print!("{:?}, ", v))"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__sort_by_key@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__sort_by_key@graphvis_mermaid.snap index bcb9d501a5f..69272d5c467 100644 --- a/hydroflow/tests/snapshots/surface_codegen__sort_by_key@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__sort_by_key@graphvis_mermaid.snap @@ -13,9 +13,9 @@ end subgraph sg_2v1 ["sg_2v1 stratum 1"] 2v1[\"(2v1) sort_by_key(|(k, _v)| k)"/]:::pullClass 3v1[/"(3v1) for_each(|v| println!("{:?}", v))"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__surface_syntax_reachability_generated@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__surface_syntax_reachability_generated@graphvis_mermaid.snap index 2ccb1e4a76b..d816f5eb37a 100644 --- a/hydroflow/tests/snapshots/surface_codegen__surface_syntax_reachability_generated@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__surface_syntax_reachability_generated@graphvis_mermaid.snap @@ -17,15 +17,15 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 6v1[/"(6v1) tee()"\]:::pushClass 8v1[/"(8v1) for_each(|x| println!("Reached: {}", x))"\]:::pushClass 9v1["(9v1) handoff"]:::otherClass - 9v1--1--->1v1 - 3v1--0--->1v1 - 1v1--->2v1 - 2v1--0--->4v1 - 7v1--1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--0--->9v1 - 6v1--1--->8v1 + 9v1-->|1|1v1 + 3v1-->|0|1v1 + 1v1-->2v1 + 2v1-->|0|4v1 + 7v1-->|1|4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->|0|9v1 + 6v1-->|1|8v1 subgraph sg_1v1_var_my_join_tee ["var my_join_tee"] 4v1 5v1 diff --git a/hydroflow/tests/snapshots/surface_codegen__transitive_closure@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_codegen__transitive_closure@graphvis_mermaid.snap index bdab21ef6e7..3a574a24c39 100644 --- a/hydroflow/tests/snapshots/surface_codegen__transitive_closure@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_codegen__transitive_closure@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 4v1[\"(4v1) source_stream(pairs_recv)"/]:::pullClass 3v1[/"(3v1) tee()"\]:::pushClass - 4v1--->3v1 + 4v1-->3v1 subgraph sg_1v1_var_link_tee ["var link_tee"] 3v1 end @@ -23,13 +23,13 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 2v1[/"(2v1) tee()"\]:::pushClass 8v1[/"(8v1) for_each(|(a, b)| println!("transitive closure: ({},{})", a, b))"\]:::pushClass 10v1["(10v1) handoff"]:::otherClass - 10v1--->6v1 - 6v1--0--->5v1 - 5v1--->7v1 - 7v1--1--->1v1 - 1v1--->2v1 - 2v1--0--->10v1 - 2v1--1--->8v1 + 10v1-->6v1 + 6v1-->|0|5v1 + 5v1-->7v1 + 7v1-->|1|1v1 + 1v1-->2v1 + 2v1-->|0|10v1 + 2v1-->|1|8v1 subgraph sg_2v1_var_edge_union_tee ["var edge_union_tee"] 1v1 2v1 @@ -38,10 +38,10 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 5v1 end end -3v1--0--->9v1 -3v1--1--->11v1 +3v1-->|0|9v1 +3v1-->|1|11v1 9v1["(9v1) handoff"]:::otherClass -9v1--0--->1v1 +9v1-->|0|1v1 11v1["(11v1) handoff"]:::otherClass -11v1--1--->5v1 +11v1-->|1|5v1 diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_dot.snap new file mode 100644 index 00000000000..794a7480323 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_dot.snap @@ -0,0 +1,37 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([()])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|_| context.current_tick_start())", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n3v1 [label="(n3v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) assert(|t: &hydroflow::instant::Instant| t.elapsed().as_nanos() > 0)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) for_each(|t: hydroflow::instant::Instant| {\l println!(\"Time between ticks: {:?}\", t.elapsed())\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n4v1 + n4v1 -> n5v1 + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 1" + n7v1 [label="(n7v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n6v1 + n6v1 [label="(n6v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n6v1 -> n7v1 + n7v1 -> n8v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n3v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_mermaid.snap new file mode 100644 index 00000000000..bc4cac24c4f --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start@graphvis_mermaid.snap @@ -0,0 +1,31 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([()])"/]:::pullClass + 2v1[\"(2v1) map(|_| context.current_tick_start())"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 3v1[\"(3v1) defer_tick()"/]:::pullClass + 4v1[\"(4v1) assert(|t: &hydroflow::instant::Instant| t.elapsed().as_nanos() > 0)"/]:::pullClass + 5v1[/"
(5v1)
for_each(|t: hydroflow::instant::Instant| {
println!("Time between ticks: {:?}", t.elapsed())
})"\]:::pushClass + 3v1-->4v1 + 4v1-->5v1 +end +subgraph sg_3v1 ["sg_3v1 stratum 1"] + 7v1[\"(7v1) identity()"/]:::pullClass +end +2v1-->6v1 +6v1["(6v1) handoff"]:::otherClass +6v1-->7v1 +7v1-->8v1 +8v1["(8v1) handoff"]:::otherClass +8v1--o3v1 + diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_dot.snap new file mode 100644 index 00000000000..7abc7548eb4 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_dot.snap @@ -0,0 +1,17 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([()])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) persist()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|_| time.set(Some(Instant::now() - context.current_tick_start())))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n2v1 -> n3v1 + } +} + diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_mermaid.snap new file mode 100644 index 00000000000..fa48851c3b4 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks@graphvis_mermaid.snap @@ -0,0 +1,17 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([()])"/]:::pullClass + 2v1[\"(2v1) persist()"/]:::pullClass + 3v1[/"(3v1) for_each(|_| time.set(Some(Instant::now() - context.current_tick_start())))"\]:::pushClass + 1v1--->2v1 + 2v1--->3v1 +end + diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_dot.snap new file mode 100644 index 00000000000..7abc7548eb4 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_dot.snap @@ -0,0 +1,17 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([()])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) persist()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|_| time.set(Some(Instant::now() - context.current_tick_start())))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n2v1 -> n3v1 + } +} + diff --git a/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_mermaid.snap new file mode 100644 index 00000000000..6bb77e36150 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_context__context_current_tick_start_does_not_count_time_between_ticks_async@graphvis_mermaid.snap @@ -0,0 +1,17 @@ +--- +source: hydroflow/tests/surface_context.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([()])"/]:::pullClass + 2v1[\"(2v1) persist()"/]:::pullClass + 3v1[/"(3v1) for_each(|_| time.set(Some(Instant::now() - context.current_tick_start())))"\]:::pushClass + 1v1-->2v1 + 2v1-->3v1 +end + diff --git a/hydroflow/tests/snapshots/surface_context__context_mut@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_context__context_mut@graphvis_mermaid.snap index 61f410cfec8..84e3b4dfacc 100644 --- a/hydroflow/tests/snapshots/surface_context__context_mut@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_context__context_mut@graphvis_mermaid.snap @@ -10,14 +10,14 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(0..10)"/]:::pullClass 2v1[\"(2v1) map(|n| context.add_state(n))"/]:::pullClass - 1v1--->2v1 + 1v1-->2v1 end subgraph sg_2v1 ["sg_2v1 stratum 1"] 3v1[\"(3v1) next_stratum()"/]:::pullClass 4v1[/"(4v1) for_each(|handle| println!("{:?}: {}", handle, context.state_ref(handle)))"\]:::pushClass - 3v1--->4v1 + 3v1-->4v1 end -2v1--->5v1 +2v1-->5v1 5v1["(5v1) handoff"]:::otherClass -5v1===o3v1 +5v1--x3v1 diff --git a/hydroflow/tests/snapshots/surface_context__context_ref@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_context__context_ref@graphvis_mermaid.snap index 94a06ce6c45..3901d6404e0 100644 --- a/hydroflow/tests/snapshots/surface_context__context_ref@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_context__context_ref@graphvis_mermaid.snap @@ -10,6 +10,6 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter([()])"/]:::pullClass 2v1[/"
(2v1)
for_each(|()| {
println!(
"Current tick: {}, stratum: {}", context.current_tick(), context
.current_stratum()
)
})"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 end diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_dot.snap new file mode 100644 index 00000000000..623a1434a21 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_dot.snap @@ -0,0 +1,64 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 1" + n1v1 [label="(n1v1) difference_multiset::<'static>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_1v1_var_diff" { + label="var diff" + n1v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 2" + n2v1 [label="(n2v1) sort()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|v| output_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n2v1 -> n3v1 + subgraph "cluster sg_2v1_var_diff" { + label="var diff" + n2v1 + n3v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n5v1 [label="(n5v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n5v1 -> n6v1 + n6v1 -> n7v1 + subgraph "cluster sg_3v1_var_negs" { + label="var negs" + n5v1 + n6v1 + } + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n4v1 [label="(n4v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_4v1_var_poss" { + label="var poss" + n4v1 + } + } + n1v1 -> n8v1 + n4v1 -> n10v1 + n6v1 -> n9v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n2v1 [arrowhead=box, color=red] + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n10v1 [label="(n10v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n10v1 -> n1v1 [label="pos"] +} + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_mermaid.snap new file mode 100644 index 00000000000..afa2d56d5e4 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static@graphvis_mermaid.snap @@ -0,0 +1,51 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 1"] + 1v1[\"(1v1) difference_multiset::<'static>()"/]:::pullClass + subgraph sg_1v1_var_diff ["var diff"] + 1v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 2"] + 2v1[\"(2v1) sort()"/]:::pullClass + 3v1[/"(3v1) for_each(|v| output_send.send(v).unwrap())"\]:::pushClass + 2v1-->3v1 + subgraph sg_2v1_var_diff ["var diff"] + 2v1 + 3v1 + end +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 5v1[\"(5v1) source_stream(neg_recv)"/]:::pullClass + 6v1[/"(6v1) tee()"\]:::pushClass + 7v1[/"(7v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass + 5v1-->6v1 + 6v1-->7v1 + subgraph sg_3v1_var_negs ["var negs"] + 5v1 + 6v1 + end +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 4v1[\"(4v1) source_stream(pos_recv)"/]:::pullClass + subgraph sg_4v1_var_poss ["var poss"] + 4v1 + end +end +1v1-->8v1 +4v1-->10v1 +6v1-->9v1 +8v1["(8v1) handoff"]:::otherClass +8v1--x2v1 +9v1["(9v1) handoff"]:::otherClass +9v1--x|neg|1v1 +10v1["(10v1) handoff"]:::otherClass +10v1-->|pos|1v1 + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_dot.snap new file mode 100644 index 00000000000..eaf1418adbd --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_dot.snap @@ -0,0 +1,64 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 1" + n1v1 [label="(n1v1) difference_multiset::<'static, 'tick>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_1v1_var_diff" { + label="var diff" + n1v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 2" + n2v1 [label="(n2v1) sort()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|v| output_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n2v1 -> n3v1 + subgraph "cluster sg_2v1_var_diff" { + label="var diff" + n2v1 + n3v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n5v1 [label="(n5v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n5v1 -> n6v1 + n6v1 -> n7v1 + subgraph "cluster sg_3v1_var_negs" { + label="var negs" + n5v1 + n6v1 + } + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n4v1 [label="(n4v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_4v1_var_poss" { + label="var poss" + n4v1 + } + } + n1v1 -> n8v1 + n4v1 -> n10v1 + n6v1 -> n9v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n2v1 [arrowhead=box, color=red] + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n10v1 [label="(n10v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n10v1 -> n1v1 [label="pos"] +} + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_mermaid.snap new file mode 100644 index 00000000000..a064599de4b --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_static_tick@graphvis_mermaid.snap @@ -0,0 +1,51 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 1"] + 1v1[\"(1v1) difference_multiset::<'static, 'tick>()"/]:::pullClass + subgraph sg_1v1_var_diff ["var diff"] + 1v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 2"] + 2v1[\"(2v1) sort()"/]:::pullClass + 3v1[/"(3v1) for_each(|v| output_send.send(v).unwrap())"\]:::pushClass + 2v1-->3v1 + subgraph sg_2v1_var_diff ["var diff"] + 2v1 + 3v1 + end +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 5v1[\"(5v1) source_stream(neg_recv)"/]:::pullClass + 6v1[/"(6v1) tee()"\]:::pushClass + 7v1[/"(7v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass + 5v1-->6v1 + 6v1-->7v1 + subgraph sg_3v1_var_negs ["var negs"] + 5v1 + 6v1 + end +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 4v1[\"(4v1) source_stream(pos_recv)"/]:::pullClass + subgraph sg_4v1_var_poss ["var poss"] + 4v1 + end +end +1v1-->8v1 +4v1-->10v1 +6v1-->9v1 +8v1["(8v1) handoff"]:::otherClass +8v1--x2v1 +9v1["(9v1) handoff"]:::otherClass +9v1--x|neg|1v1 +10v1["(10v1) handoff"]:::otherClass +10v1-->|pos|1v1 + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_dot.snap new file mode 100644 index 00000000000..6dc79a5e26e --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_dot.snap @@ -0,0 +1,64 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 1" + n1v1 [label="(n1v1) difference_multiset::<'tick, 'static>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_1v1_var_diff" { + label="var diff" + n1v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 2" + n2v1 [label="(n2v1) sort()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|v| output_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n2v1 -> n3v1 + subgraph "cluster sg_2v1_var_diff" { + label="var diff" + n2v1 + n3v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n5v1 [label="(n5v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n5v1 -> n6v1 + n6v1 -> n7v1 + subgraph "cluster sg_3v1_var_negs" { + label="var negs" + n5v1 + n6v1 + } + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n4v1 [label="(n4v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_4v1_var_poss" { + label="var poss" + n4v1 + } + } + n1v1 -> n8v1 + n4v1 -> n10v1 + n6v1 -> n9v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n2v1 [arrowhead=box, color=red] + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n10v1 [label="(n10v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n10v1 -> n1v1 [label="pos"] +} + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_mermaid.snap new file mode 100644 index 00000000000..27270533db1 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_tick_static@graphvis_mermaid.snap @@ -0,0 +1,51 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 1"] + 1v1[\"(1v1) difference_multiset::<'tick, 'static>()"/]:::pullClass + subgraph sg_1v1_var_diff ["var diff"] + 1v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 2"] + 2v1[\"(2v1) sort()"/]:::pullClass + 3v1[/"(3v1) for_each(|v| output_send.send(v).unwrap())"\]:::pushClass + 2v1-->3v1 + subgraph sg_2v1_var_diff ["var diff"] + 2v1 + 3v1 + end +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 5v1[\"(5v1) source_stream(neg_recv)"/]:::pullClass + 6v1[/"(6v1) tee()"\]:::pushClass + 7v1[/"(7v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass + 5v1-->6v1 + 6v1-->7v1 + subgraph sg_3v1_var_negs ["var negs"] + 5v1 + 6v1 + end +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 4v1[\"(4v1) source_stream(pos_recv)"/]:::pullClass + subgraph sg_4v1_var_poss ["var poss"] + 4v1 + end +end +1v1-->8v1 +4v1-->10v1 +6v1-->9v1 +8v1["(8v1) handoff"]:::otherClass +8v1--x2v1 +9v1["(9v1) handoff"]:::otherClass +9v1--x|neg|1v1 +10v1["(10v1) handoff"]:::otherClass +10v1-->|pos|1v1 + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_dot.snap new file mode 100644 index 00000000000..0eedcfcfc04 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_dot.snap @@ -0,0 +1,51 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 1" + n1v1 [label="(n1v1) difference_multiset()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) for_each(|x| println!(\"diff: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + subgraph "cluster sg_1v1_var_diff" { + label="var diff" + n1v1 + n2v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n6v1 [label="(n6v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n4v1 -> n5v1 + n5v1 -> n6v1 + subgraph "cluster sg_2v1_var_negs" { + label="var negs" + n4v1 + n5v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n3v1 [label="(n3v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_3v1_var_poss" { + label="var poss" + n3v1 + } + } + n3v1 -> n8v1 + n5v1 -> n7v1 + n7v1 [label="(n7v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n7v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n1v1 [label="pos"] +} + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_mermaid.snap new file mode 100644 index 00000000000..426af08533d --- /dev/null +++ b/hydroflow/tests/snapshots/surface_difference__diff_multiset_timing@graphvis_mermaid.snap @@ -0,0 +1,42 @@ +--- +source: hydroflow/tests/surface_difference.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 1"] + 1v1[\"(1v1) difference_multiset()"/]:::pullClass + 2v1[/"(2v1) for_each(|x| println!("diff: {:?}", x))"\]:::pushClass + 1v1-->2v1 + subgraph sg_1v1_var_diff ["var diff"] + 1v1 + 2v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_stream(neg_recv)"/]:::pullClass + 5v1[/"(5v1) tee()"\]:::pushClass + 6v1[/"(6v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass + 4v1-->5v1 + 5v1-->6v1 + subgraph sg_2v1_var_negs ["var negs"] + 4v1 + 5v1 + end +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 3v1[\"(3v1) source_stream(pos_recv)"/]:::pullClass + subgraph sg_3v1_var_poss ["var poss"] + 3v1 + end +end +3v1-->8v1 +5v1-->7v1 +7v1["(7v1) handoff"]:::otherClass +7v1--x|neg|1v1 +8v1["(8v1) handoff"]:::otherClass +8v1-->|pos|1v1 + diff --git a/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_dot.snap index 39fe0cd7a82..7afd1b7692e 100644 --- a/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_dot.snap @@ -8,44 +8,57 @@ digraph { style=filled label = "sg_1v1\nstratum 1" n1v1 [label="(n1v1) difference::<'tick, 'static>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - n2v1 [label="(n2v1) for_each(|v| output_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] - n1v1 -> n2v1 subgraph "cluster sg_1v1_var_diff" { label="var diff" n1v1 - n2v1 } } subgraph "cluster n2v1" { fillcolor="#dddddd" style=filled - label = "sg_2v1\nstratum 0" - n4v1 [label="(n4v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - n5v1 [label="(n5v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] - n6v1 [label="(n6v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] - n4v1 -> n5v1 + label = "sg_2v1\nstratum 2" + n2v1 [label="(n2v1) sort()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) for_each(|v| output_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n2v1 -> n3v1 + subgraph "cluster sg_2v1_var_diff" { + label="var diff" + n2v1 + n3v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n5v1 [label="(n5v1) source_stream(neg_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|x| println!(\"neg: {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n5v1 -> n6v1 - subgraph "cluster sg_2v1_var_negs" { + n6v1 -> n7v1 + subgraph "cluster sg_3v1_var_negs" { label="var negs" - n4v1 n5v1 + n6v1 } } - subgraph "cluster n3v1" { + subgraph "cluster n4v1" { fillcolor="#dddddd" style=filled - label = "sg_3v1\nstratum 0" - n3v1 [label="(n3v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - subgraph "cluster sg_3v1_var_poss" { + label = "sg_4v1\nstratum 0" + n4v1 [label="(n4v1) source_stream(pos_recv)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + subgraph "cluster sg_4v1_var_poss" { label="var poss" - n3v1 + n4v1 } } - n3v1 -> n8v1 - n5v1 -> n7v1 - n7v1 [label="(n7v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n7v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n1v1 -> n8v1 + n4v1 -> n10v1 + n6v1 -> n9v1 n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n8v1 -> n1v1 [label="pos"] + n8v1 -> n2v1 [arrowhead=box, color=red] + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n1v1 [label="neg", arrowhead=box, color=red] + n10v1 [label="(n10v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n10v1 -> n1v1 [label="pos"] } diff --git a/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_mermaid.snap index e3a11d85a87..822f24862b0 100644 --- a/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_difference__diff_static@graphvis_mermaid.snap @@ -9,34 +9,43 @@ classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 1"] 1v1[\"(1v1) difference::<'tick, 'static>()"/]:::pullClass - 2v1[/"(2v1) for_each(|v| output_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 subgraph sg_1v1_var_diff ["var diff"] 1v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 2"] + 2v1[\"(2v1) sort()"/]:::pullClass + 3v1[/"(3v1) for_each(|v| output_send.send(v).unwrap())"\]:::pushClass + 2v1-->3v1 + subgraph sg_2v1_var_diff ["var diff"] 2v1 + 3v1 end end -subgraph sg_2v1 ["sg_2v1 stratum 0"] - 4v1[\"(4v1) source_stream(neg_recv)"/]:::pullClass - 5v1[/"(5v1) tee()"\]:::pushClass - 6v1[/"(6v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass - 4v1--->5v1 - 5v1--->6v1 - subgraph sg_2v1_var_negs ["var negs"] - 4v1 +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 5v1[\"(5v1) source_stream(neg_recv)"/]:::pullClass + 6v1[/"(6v1) tee()"\]:::pushClass + 7v1[/"(7v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass + 5v1-->6v1 + 6v1-->7v1 + subgraph sg_3v1_var_negs ["var negs"] 5v1 + 6v1 end end -subgraph sg_3v1 ["sg_3v1 stratum 0"] - 3v1[\"(3v1) source_stream(pos_recv)"/]:::pullClass - subgraph sg_3v1_var_poss ["var poss"] - 3v1 +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 4v1[\"(4v1) source_stream(pos_recv)"/]:::pullClass + subgraph sg_4v1_var_poss ["var poss"] + 4v1 end end -3v1--->8v1 -5v1--->7v1 -7v1["(7v1) handoff"]:::otherClass -7v1==neg===o1v1 +1v1-->8v1 +4v1-->10v1 +6v1-->9v1 8v1["(8v1) handoff"]:::otherClass -8v1--pos--->1v1 +8v1--x2v1 +9v1["(9v1) handoff"]:::otherClass +9v1--x|neg|1v1 +10v1["(10v1) handoff"]:::otherClass +10v1-->|pos|1v1 diff --git a/hydroflow/tests/snapshots/surface_difference__diff_timing@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_difference__diff_timing@graphvis_mermaid.snap index 15c9709325e..3ff62474be6 100644 --- a/hydroflow/tests/snapshots/surface_difference__diff_timing@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_difference__diff_timing@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 1"] 1v1[\"(1v1) difference()"/]:::pullClass 2v1[/"(2v1) for_each(|x| println!("diff: {:?}", x))"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 subgraph sg_1v1_var_diff ["var diff"] 1v1 2v1 @@ -20,8 +20,8 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 4v1[\"(4v1) source_stream(neg_recv)"/]:::pullClass 5v1[/"(5v1) tee()"\]:::pushClass 6v1[/"(6v1) for_each(|x| println!("neg: {:?}", x))"\]:::pushClass - 4v1--->5v1 - 5v1--->6v1 + 4v1-->5v1 + 5v1-->6v1 subgraph sg_2v1_var_negs ["var negs"] 4v1 5v1 @@ -33,10 +33,10 @@ subgraph sg_3v1 ["sg_3v1 stratum 0"] 3v1 end end -3v1--->8v1 -5v1--->7v1 +3v1-->8v1 +5v1-->7v1 7v1["(7v1) handoff"]:::otherClass -7v1==neg===o1v1 +7v1--x|neg|1v1 8v1["(8v1) handoff"]:::otherClass -8v1--pos--->1v1 +8v1-->|pos|1v1 diff --git a/hydroflow/tests/snapshots/surface_examples__example_4_neighbors.snap b/hydroflow/tests/snapshots/surface_examples__example_4_neighbors.snap index f70fea3935c..4d63d19c6bb 100644 --- a/hydroflow/tests/snapshots/surface_examples__example_4_neighbors.snap +++ b/hydroflow/tests/snapshots/surface_examples__example_4_neighbors.snap @@ -15,12 +15,12 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 5v1[\"(5v1) flat_map(|(src, (_, dst))| [src, dst])"/]:::pullClass 6v1[\"(6v1) unique()"/]:::pullClass 7v1[/"(7v1) for_each(|n| println!("Reached: {}", n))"\]:::pushClass - 1v1--->3v1 - 2v1--1--->4v1 - 3v1--0--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--->7v1 + 1v1-->3v1 + 2v1-->|1|4v1 + 3v1-->|0|4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->7v1 subgraph sg_1v1_var_my_join ["var my_join"] 4v1 5v1 diff --git a/hydroflow/tests/snapshots/surface_examples__example_5_reachability.snap b/hydroflow/tests/snapshots/surface_examples__example_5_reachability.snap index c8071835181..9e5ff9e9239 100644 --- a/hydroflow/tests/snapshots/surface_examples__example_5_reachability.snap +++ b/hydroflow/tests/snapshots/surface_examples__example_5_reachability.snap @@ -10,34 +10,34 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(vec![0])"/]:::pullClass 2v1[\"(2v1) source_stream(edges_recv)"/]:::pullClass - 3v1[\"(3v1) union()"/]:::pullClass - 4v1[\"(4v1) map(|v| (v, ()))"/]:::pullClass - 5v1[\"(5v1) join()"/]:::pullClass - 6v1[\"(6v1) flat_map(|(src, ((), dst))| [src, dst])"/]:::pullClass - 7v1[/"(7v1) tee()"\]:::pushClass + 7v1[\"(7v1) union()"/]:::pullClass + 3v1[\"(3v1) map(|v| (v, ()))"/]:::pullClass + 4v1[\"(4v1) join()"/]:::pullClass + 5v1[\"(5v1) flat_map(|(src, ((), dst))| [src, dst])"/]:::pullClass + 6v1[/"(6v1) tee()"\]:::pushClass 8v1[/"(8v1) unique()"\]:::pushClass 9v1[/"(9v1) for_each(|x| println!("Reached: {}", x))"\]:::pushClass 10v1["(10v1) handoff"]:::otherClass - 10v1--1--->3v1 - 1v1--0--->3v1 - 2v1--1--->5v1 - 3v1--->4v1 - 4v1--0--->5v1 - 5v1--->6v1 - 6v1--->7v1 - 7v1--0--->10v1 - 7v1--1--->8v1 - 8v1--->9v1 + 10v1-->|cycle|7v1 + 1v1-->|base|7v1 + 2v1-->|1|4v1 + 7v1-->3v1 + 3v1-->|0|4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->10v1 + 6v1-->|print|8v1 + 8v1-->9v1 subgraph sg_1v1_var_my_join_tee ["var my_join_tee"] + 4v1 5v1 6v1 - 7v1 end subgraph sg_1v1_var_origin ["var origin"] 1v1 end subgraph sg_1v1_var_reached_vertices ["var reached_vertices"] - 3v1 + 7v1 end subgraph sg_1v1_var_stream_of_edges ["var stream_of_edges"] 2v1 diff --git a/hydroflow/tests/snapshots/surface_examples__example_6_unreachability.snap b/hydroflow/tests/snapshots/surface_examples__example_6_unreachability.snap index b1cda51e5c0..e65021118de 100644 --- a/hydroflow/tests/snapshots/surface_examples__example_6_unreachability.snap +++ b/hydroflow/tests/snapshots/surface_examples__example_6_unreachability.snap @@ -9,29 +9,29 @@ classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(vec![0])"/]:::pullClass - 8v1[\"(8v1) map(|v| (v, ()))"/]:::pullClass - 6v1[\"(6v1) join()"/]:::pullClass - 7v1[\"(7v1) flat_map(|(src, ((), dst))| [src, dst])"/]:::pullClass - 4v1[\"(4v1) union()"/]:::pullClass - 5v1[/"(5v1) tee()"\]:::pushClass + 4v1[\"(4v1) map(|v| (v, ()))"/]:::pullClass + 5v1[\"(5v1) join()"/]:::pullClass + 6v1[\"(6v1) flat_map(|(src, ((), dst))| [src, dst])"/]:::pullClass + 7v1[\"(7v1) union()"/]:::pullClass + 8v1[/"(8v1) tee()"\]:::pushClass 15v1["(15v1) handoff"]:::otherClass - 15v1--->8v1 - 1v1--0--->4v1 - 8v1--0--->6v1 - 6v1--->7v1 - 7v1--1--->4v1 - 4v1--->5v1 - 5v1--0--->15v1 + 15v1-->4v1 + 1v1-->|base|7v1 + 4v1-->|0|5v1 + 5v1-->6v1 + 6v1-->|cycle|7v1 + 7v1-->8v1 + 8v1-->|0|15v1 subgraph sg_1v1_var_my_join ["var my_join"] + 5v1 6v1 - 7v1 end subgraph sg_1v1_var_origin ["var origin"] 1v1 end subgraph sg_1v1_var_reached_vertices ["var reached_vertices"] - 4v1 - 5v1 + 7v1 + 8v1 end end subgraph sg_2v1 ["sg_2v1 stratum 0"] @@ -41,11 +41,11 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 10v1[/"(10v1) tee()"\]:::pushClass 12v1[/"(12v1) unique()"\]:::pushClass 13v1[/"(13v1) for_each(|v| println!("Received vertex: {}", v))"\]:::pushClass - 2v1--->3v1 - 3v1--0--->9v1 - 9v1--->10v1 - 10v1--1--->12v1 - 12v1--->13v1 + 2v1-->3v1 + 3v1-->|0|9v1 + 9v1-->10v1 + 10v1-->|1|12v1 + 12v1-->13v1 subgraph sg_2v1_var_all_vertices ["var all_vertices"] 9v1 10v1 @@ -58,20 +58,20 @@ end subgraph sg_3v1 ["sg_3v1 stratum 1"] 11v1[\"(11v1) difference()"/]:::pullClass 14v1[/"(14v1) for_each(|v| println!("unreached_vertices vertex: {}", v))"\]:::pushClass - 11v1--->14v1 + 11v1-->14v1 subgraph sg_3v1_var_unreached_vertices ["var unreached_vertices"] 11v1 end end -3v1--1--->16v1 -5v1--1--->18v1 -10v1--0--->17v1 +3v1-->|1|16v1 +8v1-->|1|18v1 +10v1-->|0|17v1 16v1["(16v1) handoff"]:::otherClass -16v1--1--->6v1 +16v1-->|1|5v1 17v1["(17v1) handoff"]:::otherClass -17v1--pos--->11v1 +17v1-->|pos|11v1 18v1["(18v1) handoff"]:::otherClass -18v1==neg===o11v1 +18v1--x|neg|11v1 Received vertex: 5 Received vertex: 10 @@ -80,6 +80,6 @@ Received vertex: 3 Received vertex: 6 Received vertex: 11 Received vertex: 12 -unreached_vertices vertex: 11 unreached_vertices vertex: 12 +unreached_vertices vertex: 11 diff --git a/hydroflow/tests/snapshots/surface_examples__example_naturals.snap b/hydroflow/tests/snapshots/surface_examples__example_naturals.snap new file mode 100644 index 00000000000..01a15b13afb --- /dev/null +++ b/hydroflow/tests/snapshots/surface_examples__example_naturals.snap @@ -0,0 +1,5 @@ +--- +source: hydroflow/tests/surface_examples.rs +expression: output +--- + diff --git a/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_dot.snap new file mode 100644 index 00000000000..0990a66e118 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_dot.snap @@ -0,0 +1,44 @@ +--- +source: hydroflow/tests/surface_flow_props.rs +expression: hf.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter_delta((0..10).map(SetUnionSingletonSet::new_from))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|SetUnion(SingletonSet(x))| SetUnion(SingletonSet(x + 5)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n4v1 [label="(n4v1) cast(None)", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n5v1 [label="(n5v1) map(|SetUnion(SingletonSet(x))| 10 * x)", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n6v1 [label="(n6v1) for_each(|x| println!(\"seq {:?}\", x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|s| println!(\"delta {:?}\", s))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n8v1 [label="(n8v1) map(|SetUnion(SingletonSet(x))| SetUnionHashSet::new_from([x]))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 [style=dashed] + n2v1 -> n3v1 [style=dashed] + n3v1 -> n4v1 [style=dashed] + n3v1 -> n7v1 [style=dashed] + n3v1 -> n8v1 [style=dashed] + n4v1 -> n5v1 + n5v1 -> n6v1 + subgraph "cluster sg_1v1_var_my_tee" { + label="var my_tee" + n1v1 + n2v1 + n3v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 1" + n9v1 [label="(n9v1) lattice_reduce::<'static, SetUnionHashSet<_>>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) for_each(|s| println!(\"cumul {:?}\", s))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n9v1 -> n10v1 [style=bold] + } + n8v1 -> n11v1 [style=dashed] + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n9v1 [arrowhead=box, color=red, style=dashed] +} + diff --git a/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_mermaid.snap new file mode 100644 index 00000000000..d15c9cf331b --- /dev/null +++ b/hydroflow/tests/snapshots/surface_flow_props__basic@graphvis_mermaid.snap @@ -0,0 +1,40 @@ +--- +source: hydroflow/tests/surface_flow_props.rs +expression: hf.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter_delta((0..10).map(SetUnionSingletonSet::new_from))"/]:::pullClass + 2v1[\"(2v1) map(|SetUnion(SingletonSet(x))| SetUnion(SingletonSet(x + 5)))"/]:::pullClass + 3v1[/"(3v1) tee()"\]:::pushClass + 4v1[/"(4v1) cast(None)"\]:::pushClass + 5v1[/"(5v1) map(|SetUnion(SingletonSet(x))| 10 * x)"\]:::pushClass + 6v1[/"(6v1) for_each(|x| println!("seq {:?}", x))"\]:::pushClass + 7v1[/"(7v1) for_each(|s| println!("delta {:?}", s))"\]:::pushClass + 8v1[/"(8v1) map(|SetUnion(SingletonSet(x))| SetUnionHashSet::new_from([x]))"\]:::pushClass + 1v1-.->2v1 + 2v1-.->3v1 + 3v1-.->4v1 + 3v1-.->7v1 + 3v1-.->8v1 + 4v1-->5v1 + 5v1-->6v1 + subgraph sg_1v1_var_my_tee ["var my_tee"] + 1v1 + 2v1 + 3v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 1"] + 9v1[\"(9v1) lattice_reduce::<'static, SetUnionHashSet<_>>()"/]:::pullClass + 10v1[/"(10v1) for_each(|s| println!("cumul {:?}", s))"\]:::pushClass + 9v1==>10v1 +end +8v1-.->11v1 +11v1["(11v1) handoff"]:::otherClass +11v1-.-x9v1 + diff --git a/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_dot.snap new file mode 100644 index 00000000000..52cf1cb1777 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_dot.snap @@ -0,0 +1,24 @@ +--- +source: hydroflow/tests/surface_flow_props.rs +expression: hf.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter_delta((0..10).map(SetUnionSingletonSet::new_from))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) source_iter((0..10).map(SetUnionSingletonSet::new_from))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) for_each(|s| println!(\"{:?}\", s))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n3v1 [label="0", style=dashed] + n2v1 -> n3v1 [label="1"] + n3v1 -> n4v1 + subgraph "cluster sg_1v1_var_my_union" { + label="var my_union" + n3v1 + n4v1 + } + } +} + diff --git a/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_mermaid.snap new file mode 100644 index 00000000000..0219157e00b --- /dev/null +++ b/hydroflow/tests/snapshots/surface_flow_props__union_warning@graphvis_mermaid.snap @@ -0,0 +1,23 @@ +--- +source: hydroflow/tests/surface_flow_props.rs +expression: hf.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter_delta((0..10).map(SetUnionSingletonSet::new_from))"/]:::pullClass + 2v1[\"(2v1) source_iter((0..10).map(SetUnionSingletonSet::new_from))"/]:::pullClass + 3v1[\"(3v1) union()"/]:::pullClass + 4v1[/"(4v1) for_each(|s| println!("{:?}", s))"\]:::pushClass + 1v1-.->|0|3v1 + 2v1-->|1|3v1 + 3v1-->4v1 + subgraph sg_1v1_var_my_union ["var my_union"] + 3v1 + 4v1 + end +end + diff --git a/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_dot.snap index 5704892b272..88b3679362a 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) fold::<\l 'tick,\l>(\l Vec::new(),\l |mut v, x| {\l v.push(x);\l v\l },\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) fold::<'tick>(Vec::new(), Vec::push)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) flat_map(|mut vec| {\l vec.sort();\l vec\l})\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n4v1 [label="(n4v1) for_each(|v| print!(\"{:?}, \", v))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 diff --git a/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_mermaid.snap index 49c02eeb5e1..65b8a3cfd09 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_sort@graphvis_mermaid.snap @@ -11,13 +11,13 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"
(2v1)
fold::<
'tick,
>(
Vec::new(),
|mut v, x| {
v.push(x);
v
},
)"/]:::pullClass + 2v1[\"(2v1) fold::<'tick>(Vec::new(), Vec::push)"/]:::pullClass 3v1[\"
(3v1)
flat_map(|mut vec| {
vec.sort();
vec
})"/]:::pullClass 4v1[/"(4v1) for_each(|v| print!("{:?}, ", v))"\]:::pushClass - 2v1--->3v1 - 3v1--->4v1 + 2v1-->3v1 + 3v1-->4v1 end -1v1--->5v1 +1v1-->5v1 5v1["(5v1) handoff"]:::otherClass -5v1===o2v1 +5v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_dot.snap index b28bdb9bf35..08bf05ebbf7 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) fold::<\l 'static,\l>(\l Vec::new(),\l |mut old: Vec, mut x: Vec| {\l old.append(&mut x);\l old\l },\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) fold::<\l 'static,\l>(\l Vec::new(),\l |old: &mut Vec, mut x: Vec| {\l old.append(&mut x);\l },\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) for_each(|v| result_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 } diff --git a/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_mermaid.snap index 7cddd0923c4..b357a31fd83 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_static@graphvis_mermaid.snap @@ -11,11 +11,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"
(2v1)
fold::<
'static,
>(
Vec::new(),
|mut old: Vec<u32>, mut x: Vec<u32>| {
old.append(&mut x);
old
},
)"/]:::pullClass + 2v1[\"
(2v1)
fold::<
'static,
>(
Vec::new(),
|old: &mut Vec<u32>, mut x: Vec<u32>| {
old.append(&mut x);
},
)"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_dot.snap index 852f64fa541..c1d4b2d85d6 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) fold::<\l 'tick,\l>(\l Vec::new(),\l |mut old: Vec, mut x: Vec| {\l old.append(&mut x);\l old\l },\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) fold::<\l 'tick,\l>(\l Vec::new(),\l |old: &mut Vec, mut x: Vec| {\l old.append(&mut x);\l },\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) for_each(|v| result_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 } diff --git a/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_mermaid.snap index b3f06b472ba..860cf85deb6 100644 --- a/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_fold__fold_tick@graphvis_mermaid.snap @@ -11,11 +11,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"
(2v1)
fold::<
'tick,
>(
Vec::new(),
|mut old: Vec<u32>, mut x: Vec<u32>| {
old.append(&mut x);
old
},
)"/]:::pullClass + 2v1[\"
(2v1)
fold::<
'tick,
>(
Vec::new(),
|old: &mut Vec<u32>, mut x: Vec<u32>| {
old.append(&mut x);
},
)"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_backward@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_backward@graphvis_mermaid.snap index c31891667fe..37f55f3d519 100644 --- a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_backward@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_backward@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 2v1[\"(2v1) source_iter(0..10)"/]:::pullClass 1v1[/"(1v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 2v1--->1v1 + 2v1-->1v1 subgraph sg_1v1_var_forward_ref ["var forward_ref"] 2v1 end diff --git a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_forward@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_forward@graphvis_mermaid.snap index 6c933cf37e9..ac7f42a57c2 100644 --- a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_forward@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_forward@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(0..10)"/]:::pullClass 2v1[/"(2v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 + 1v1-->2v1 subgraph sg_1v1_var_forward_ref ["var forward_ref"] 2v1 end diff --git a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_middle@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_middle@graphvis_mermaid.snap index 30947ec448a..bee73eaf0bb 100644 --- a/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_middle@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_forwardref__forwardref_basic_middle@graphvis_mermaid.snap @@ -11,8 +11,8 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter(0..10)"/]:::pullClass 3v1[\"(3v1) identity()"/]:::pullClass 2v1[/"(2v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->3v1 - 3v1--->2v1 + 1v1-->3v1 + 3v1-->2v1 subgraph sg_1v1_var_forward_ref ["var forward_ref"] 3v1 end diff --git a/hydroflow/tests/snapshots/surface_join__static_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join__static_static@graphvis_dot.snap new file mode 100644 index 00000000000..3a37a0ad93e --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__static_static@graphvis_dot.snap @@ -0,0 +1,88 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n3v1 [label="(n3v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n5v1 [label="(n5v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) join::<'static, 'static>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n9v1 [label="0"] + n2v1 -> n8v1 + n4v1 -> n8v1 + n7v1 -> n8v1 + n8v1 -> n9v1 [label="1"] + n9v1 -> n10v1 + subgraph "cluster sg_4v1_var_my_join" { + label="var my_join" + n9v1 + n10v1 + } + subgraph "cluster sg_4v1_var_unioner" { + label="var unioner" + n8v1 + } + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n14v1 [label="(n14v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 1" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 1" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n3v1 -> n11v1 + n5v1 -> n13v1 + n6v1 -> n12v1 + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n16v1 + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n18v1 + n14v1 -> n15v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n4v1 [arrowhead=dot, color=red] + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n7v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n6v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join__static_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join__static_static@graphvis_mermaid.snap new file mode 100644 index 00000000000..a25d32dc4dc --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__static_static@graphvis_mermaid.snap @@ -0,0 +1,68 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 3v1[\"(3v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 5v1[\"(5v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) defer_tick()"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) source_iter([(7, 0)])"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass + 7v1[\"(7v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) union()"/]:::pullClass + 9v1[\"(9v1) join::<'static, 'static>()"/]:::pullClass + 10v1[/"
(10v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 1v1-->|0|9v1 + 2v1-->8v1 + 4v1-->8v1 + 7v1-->8v1 + 8v1-->|1|9v1 + 9v1-->10v1 + subgraph sg_4v1_var_my_join ["var my_join"] + 9v1 + 10v1 + end + subgraph sg_4v1_var_unioner ["var unioner"] + 8v1 + end +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 14v1[\"(14v1) identity()"/]:::pullClass +end +subgraph sg_6v1 ["sg_6v1 stratum 1"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 1"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +3v1-->11v1 +5v1-->13v1 +6v1-->12v1 +11v1["(11v1) handoff"]:::otherClass +11v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1-->16v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->18v1 +14v1-->15v1 +15v1["(15v1) handoff"]:::otherClass +15v1--o4v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o7v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o6v1 + diff --git a/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_dot.snap new file mode 100644 index 00000000000..d97f66b2cbb --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_dot.snap @@ -0,0 +1,88 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n3v1 [label="(n3v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n5v1 [label="(n5v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) join::<'static, 'tick>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n9v1 [label="0"] + n2v1 -> n8v1 + n4v1 -> n8v1 + n7v1 -> n8v1 + n8v1 -> n9v1 [label="1"] + n9v1 -> n10v1 + subgraph "cluster sg_4v1_var_my_join" { + label="var my_join" + n9v1 + n10v1 + } + subgraph "cluster sg_4v1_var_unioner" { + label="var unioner" + n8v1 + } + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n14v1 [label="(n14v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 1" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 1" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n3v1 -> n11v1 + n5v1 -> n13v1 + n6v1 -> n12v1 + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n16v1 + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n18v1 + n14v1 -> n15v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n4v1 [arrowhead=dot, color=red] + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n7v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n6v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_mermaid.snap new file mode 100644 index 00000000000..a7066d0fb55 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__static_tick@graphvis_mermaid.snap @@ -0,0 +1,68 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 3v1[\"(3v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 5v1[\"(5v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) defer_tick()"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) source_iter([(7, 0)])"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass + 7v1[\"(7v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) union()"/]:::pullClass + 9v1[\"(9v1) join::<'static, 'tick>()"/]:::pullClass + 10v1[/"
(10v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 1v1-->|0|9v1 + 2v1-->8v1 + 4v1-->8v1 + 7v1-->8v1 + 8v1-->|1|9v1 + 9v1-->10v1 + subgraph sg_4v1_var_my_join ["var my_join"] + 9v1 + 10v1 + end + subgraph sg_4v1_var_unioner ["var unioner"] + 8v1 + end +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 14v1[\"(14v1) identity()"/]:::pullClass +end +subgraph sg_6v1 ["sg_6v1 stratum 1"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 1"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +3v1-->11v1 +5v1-->13v1 +6v1-->12v1 +11v1["(11v1) handoff"]:::otherClass +11v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1-->16v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->18v1 +14v1-->15v1 +15v1["(15v1) handoff"]:::otherClass +15v1--o4v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o7v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o6v1 + diff --git a/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_dot.snap new file mode 100644 index 00000000000..0b981d925cf --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_dot.snap @@ -0,0 +1,88 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n3v1 [label="(n3v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n5v1 [label="(n5v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) join::<'tick, 'static>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n9v1 [label="0"] + n2v1 -> n8v1 + n4v1 -> n8v1 + n7v1 -> n8v1 + n8v1 -> n9v1 [label="1"] + n9v1 -> n10v1 + subgraph "cluster sg_4v1_var_my_join" { + label="var my_join" + n9v1 + n10v1 + } + subgraph "cluster sg_4v1_var_unioner" { + label="var unioner" + n8v1 + } + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n14v1 [label="(n14v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 1" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 1" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n3v1 -> n11v1 + n5v1 -> n13v1 + n6v1 -> n12v1 + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n16v1 + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n18v1 + n14v1 -> n15v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n4v1 [arrowhead=dot, color=red] + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n7v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n6v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_mermaid.snap new file mode 100644 index 00000000000..3c5d592c2b6 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__tick_static@graphvis_mermaid.snap @@ -0,0 +1,68 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 3v1[\"(3v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 5v1[\"(5v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) defer_tick()"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) source_iter([(7, 0)])"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass + 7v1[\"(7v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) union()"/]:::pullClass + 9v1[\"(9v1) join::<'tick, 'static>()"/]:::pullClass + 10v1[/"
(10v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 1v1-->|0|9v1 + 2v1-->8v1 + 4v1-->8v1 + 7v1-->8v1 + 8v1-->|1|9v1 + 9v1-->10v1 + subgraph sg_4v1_var_my_join ["var my_join"] + 9v1 + 10v1 + end + subgraph sg_4v1_var_unioner ["var unioner"] + 8v1 + end +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 14v1[\"(14v1) identity()"/]:::pullClass +end +subgraph sg_6v1 ["sg_6v1 stratum 1"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 1"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +3v1-->11v1 +5v1-->13v1 +6v1-->12v1 +11v1["(11v1) handoff"]:::otherClass +11v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1-->16v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->18v1 +14v1-->15v1 +15v1["(15v1) handoff"]:::otherClass +15v1--o4v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o7v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o6v1 + diff --git a/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_dot.snap new file mode 100644 index 00000000000..279b226fba7 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_dot.snap @@ -0,0 +1,88 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n3v1 [label="(n3v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n5v1 [label="(n5v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) join::<'tick, 'tick>()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n9v1 [label="0"] + n2v1 -> n8v1 + n4v1 -> n8v1 + n7v1 -> n8v1 + n8v1 -> n9v1 [label="1"] + n9v1 -> n10v1 + subgraph "cluster sg_4v1_var_my_join" { + label="var my_join" + n9v1 + n10v1 + } + subgraph "cluster sg_4v1_var_unioner" { + label="var unioner" + n8v1 + } + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n14v1 [label="(n14v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 1" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 1" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n3v1 -> n11v1 + n5v1 -> n13v1 + n6v1 -> n12v1 + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n16v1 + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n18v1 + n14v1 -> n15v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n4v1 [arrowhead=dot, color=red] + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n7v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n6v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_mermaid.snap new file mode 100644 index 00000000000..d344a462dbf --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join__tick_tick@graphvis_mermaid.snap @@ -0,0 +1,68 @@ +--- +source: hydroflow/tests/surface_join.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 3v1[\"(3v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 5v1[\"(5v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) defer_tick()"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) source_iter([(7, 0)])"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass + 7v1[\"(7v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) union()"/]:::pullClass + 9v1[\"(9v1) join::<'tick, 'tick>()"/]:::pullClass + 10v1[/"
(10v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 1v1-->|0|9v1 + 2v1-->8v1 + 4v1-->8v1 + 7v1-->8v1 + 8v1-->|1|9v1 + 9v1-->10v1 + subgraph sg_4v1_var_my_join ["var my_join"] + 9v1 + 10v1 + end + subgraph sg_4v1_var_unioner ["var unioner"] + 8v1 + end +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 14v1[\"(14v1) identity()"/]:::pullClass +end +subgraph sg_6v1 ["sg_6v1 stratum 1"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 1"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +3v1-->11v1 +5v1-->13v1 +6v1-->12v1 +11v1["(11v1) handoff"]:::otherClass +11v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1-->16v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->18v1 +14v1-->15v1 +15v1["(15v1) handoff"]:::otherClass +15v1--o4v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o7v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o6v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_dot.snap new file mode 100644 index 00000000000..405202af2dc --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_lhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="1"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="0", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_mermaid.snap new file mode 100644 index 00000000000..e1603c70bde --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_blocking_rhs_streaming@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_lhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|1|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|0|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_dot.snap new file mode 100644 index 00000000000..276a3110401 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_rhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="0"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="1", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_mermaid.snap new file mode 100644 index 00000000000..4279e816ddf --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_static_lhs_streaming_rhs_blocking@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_rhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|0|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|1|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap new file mode 100644 index 00000000000..353956d1fa0 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_lhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="1"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="0", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap new file mode 100644 index 00000000000..d810821f30a --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_lhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|1|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|0|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap new file mode 100644 index 00000000000..c2bedc85bbc --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_rhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="0"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="1", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap new file mode 100644 index 00000000000..64d63498ef1 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__static_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_rhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|0|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|1|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap new file mode 100644 index 00000000000..2473fe8ba7f --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_lhs(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="1"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="0", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap new file mode 100644 index 00000000000..83afd14914f --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_blocking_rhs_streaming@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_lhs(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|1|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|0|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_dot.snap new file mode 100644 index 00000000000..ab65e69c7dc --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_dot.snap @@ -0,0 +1,104 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 0" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 1" + n10v1 [label="(n10v1) join_fused(\l Fold(SetUnionHashSet::default, Merge::merge),\l Reduce(std::ops::AddAssign::add_assign),\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n10v1 -> n11v1 + subgraph "cluster sg_6v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n17v1 [label="(n17v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n19v1 [label="(n19v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n9v1" { + fillcolor="#dddddd" + style=filled + label = "sg_9v1\nstratum 2" + n21v1 [label="(n21v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n9v1 -> n16v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="0", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n17v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n19v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n21v1 + n16v1 [label="(n16v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n16v1 -> n10v1 [label="1", arrowhead=box, color=red] + n17v1 -> n18v1 + n18v1 [label="(n18v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n18v1 -> n5v1 [arrowhead=dot, color=red] + n19v1 -> n20v1 + n20v1 [label="(n20v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n20v1 -> n8v1 [arrowhead=dot, color=red] + n21v1 -> n22v1 + n22v1 [label="(n22v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n22v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_mermaid.snap new file mode 100644 index 00000000000..b31e208373a --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_fold_rhs_reduce@graphvis_mermaid.snap @@ -0,0 +1,78 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 0"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 1"] + 10v1[\"
(10v1)
join_fused(
Fold(SetUnionHashSet::default, Merge::merge),
Reduce(std::ops::AddAssign::add_assign),
)"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 10v1-->11v1 + subgraph sg_6v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 17v1[\"(17v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 19v1[\"(19v1) identity()"/]:::pullClass +end +subgraph sg_9v1 ["sg_9v1 stratum 2"] + 21v1[\"(21v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +9v1-->16v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|0|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->17v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->19v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->21v1 +16v1["(16v1) handoff"]:::otherClass +16v1--x|1|10v1 +17v1-->18v1 +18v1["(18v1) handoff"]:::otherClass +18v1--o5v1 +19v1-->20v1 +20v1["(20v1) handoff"]:::otherClass +20v1--o8v1 +21v1-->22v1 +22v1["(22v1) handoff"]:::otherClass +22v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap new file mode 100644 index 00000000000..e64d9897142 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_dot.snap @@ -0,0 +1,97 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(7, 1), (7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n4v1 [label="(n4v1) source_iter([(7, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 0" + n6v1 [label="(n6v1) source_iter([(7, 2)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n4v1" { + fillcolor="#dddddd" + style=filled + label = "sg_4v1\nstratum 0" + n7v1 [label="(n7v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n5v1" { + fillcolor="#dddddd" + style=filled + label = "sg_5v1\nstratum 1" + n3v1 [label="(n3v1) source_iter([(7, 0)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n9v1 [label="(n9v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n10v1 [label="(n10v1) join_fused_rhs(Fold(SetUnionHashSet::default, Merge::merge))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n11v1 [label="(n11v1) for_each(|x| {\l results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)\l})\l", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n9v1 + n5v1 -> n9v1 + n8v1 -> n9v1 + n9v1 -> n10v1 [label="0"] + n10v1 -> n11v1 + subgraph "cluster sg_5v1_var_my_join" { + label="var my_join" + n10v1 + n11v1 + } + subgraph "cluster sg_5v1_var_unioner" { + label="var unioner" + n9v1 + } + } + subgraph "cluster n6v1" { + fillcolor="#dddddd" + style=filled + label = "sg_6v1\nstratum 2" + n16v1 [label="(n16v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n7v1" { + fillcolor="#dddddd" + style=filled + label = "sg_7v1\nstratum 2" + n18v1 [label="(n18v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n8v1" { + fillcolor="#dddddd" + style=filled + label = "sg_8v1\nstratum 2" + n20v1 [label="(n20v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n2v1 -> n12v1 + n4v1 -> n13v1 + n6v1 -> n15v1 + n7v1 -> n14v1 + n12v1 [label="(n12v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n12v1 -> n10v1 [label="1", arrowhead=box, color=red] + n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n13v1 -> n16v1 + n14v1 [label="(n14v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n14v1 -> n18v1 + n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n15v1 -> n20v1 + n16v1 -> n17v1 + n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n17v1 -> n5v1 [arrowhead=dot, color=red] + n18v1 -> n19v1 + n19v1 [label="(n19v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n19v1 -> n8v1 [arrowhead=dot, color=red] + n20v1 -> n21v1 + n21v1 [label="(n21v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n21v1 -> n7v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap new file mode 100644 index 00000000000..b7c91068374 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_join_fused__tick_tick_lhs_streaming_rhs_blocking@graphvis_mermaid.snap @@ -0,0 +1,74 @@ +--- +source: hydroflow/tests/surface_join_fused.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(7, 1), (7, 2)])"/]:::pullClass + 2v1[\"(2v1) map(|(k, v)| (k, SetUnionSingletonSet::new_from(v)))"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 4v1[\"(4v1) source_iter([(7, 1)])"/]:::pullClass +end +subgraph sg_3v1 ["sg_3v1 stratum 0"] + 6v1[\"(6v1) source_iter([(7, 2)])"/]:::pullClass +end +subgraph sg_4v1 ["sg_4v1 stratum 0"] + 7v1[\"(7v1) defer_tick()"/]:::pullClass +end +subgraph sg_5v1 ["sg_5v1 stratum 1"] + 3v1[\"(3v1) source_iter([(7, 0)])"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass + 8v1[\"(8v1) defer_tick()"/]:::pullClass + 9v1[\"(9v1) union()"/]:::pullClass + 10v1[\"(10v1) join_fused_rhs(Fold(SetUnionHashSet::default, Merge::merge))"/]:::pullClass + 11v1[/"
(11v1)
for_each(|x| {
results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)
})"\]:::pushClass + 3v1-->9v1 + 5v1-->9v1 + 8v1-->9v1 + 9v1-->|0|10v1 + 10v1-->11v1 + subgraph sg_5v1_var_my_join ["var my_join"] + 10v1 + 11v1 + end + subgraph sg_5v1_var_unioner ["var unioner"] + 9v1 + end +end +subgraph sg_6v1 ["sg_6v1 stratum 2"] + 16v1[\"(16v1) identity()"/]:::pullClass +end +subgraph sg_7v1 ["sg_7v1 stratum 2"] + 18v1[\"(18v1) identity()"/]:::pullClass +end +subgraph sg_8v1 ["sg_8v1 stratum 2"] + 20v1[\"(20v1) identity()"/]:::pullClass +end +2v1-->12v1 +4v1-->13v1 +6v1-->15v1 +7v1-->14v1 +12v1["(12v1) handoff"]:::otherClass +12v1--x|1|10v1 +13v1["(13v1) handoff"]:::otherClass +13v1-->16v1 +14v1["(14v1) handoff"]:::otherClass +14v1-->18v1 +15v1["(15v1) handoff"]:::otherClass +15v1-->20v1 +16v1-->17v1 +17v1["(17v1) handoff"]:::otherClass +17v1--o5v1 +18v1-->19v1 +19v1["(19v1) handoff"]:::otherClass +19v1--o8v1 +20v1-->21v1 +21v1["(21v1) handoff"]:::otherClass +21v1--o7v1 + diff --git a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_infer_basic@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_infer_basic@graphvis_mermaid.snap index 77bd10eda91..38f369d48e5 100644 --- a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_infer_basic@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_infer_basic@graphvis_mermaid.snap @@ -10,14 +10,14 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"
(1v1)
source_iter([
SubordResponse {
xid: "123",
mtype: 33,
},
SubordResponse {
xid: "123",
mtype: 52,
},
SubordResponse {
xid: "123",
mtype: 72,
},
SubordResponse {
xid: "123",
mtype: 83,
},
SubordResponse {
xid: "123",
mtype: 78,
},
])"/]:::pullClass 2v1[\"(2v1) map(|m: SubordResponse| (m.xid, 1))"/]:::pullClass - 1v1--->2v1 + 1v1-->2v1 end subgraph sg_2v1 ["sg_2v1 stratum 1"] 3v1[\"(3v1) fold_keyed::<'static>(|| 0, |old: &mut u32, val: u32| *old += val)"/]:::pullClass 4v1[/"(4v1) for_each(|(k, v)| println!("{}: {}", k, v))"\]:::pushClass - 3v1--->4v1 + 3v1-->4v1 end -2v1--->5v1 +2v1-->5v1 5v1["(5v1) handoff"]:::otherClass -5v1===o3v1 +5v1--x3v1 diff --git a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_static@graphvis_mermaid.snap index 337ec6802d2..09964fef7b4 100644 --- a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_static@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_static@graphvis_mermaid.snap @@ -13,9 +13,9 @@ end subgraph sg_2v1 ["sg_2v1 stratum 1"] 2v1[\"
(2v1)
fold_keyed::<
'static,
>(Vec::new, |old: &mut Vec<u32>, mut x: Vec<u32>| old.append(&mut x))"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_tick@graphvis_mermaid.snap index cce56e94412..d7a6a9f9fe4 100644 --- a/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_tick@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_keyed_fold__fold_keyed_tick@graphvis_mermaid.snap @@ -13,9 +13,9 @@ end subgraph sg_2v1 ["sg_2v1 stratum 1"] 2v1[\"
(2v1)
fold_keyed::<
'tick,
>(Vec::new, |old: &mut Vec<u32>, mut x: Vec<u32>| old.append(&mut x))"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_multiset_delta__multiset_delta@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_multiset_delta__multiset_delta@graphvis_mermaid.snap index f55fc274f68..2dbbf8993ec 100644 --- a/hydroflow/tests/snapshots/surface_multiset_delta__multiset_delta@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_multiset_delta__multiset_delta@graphvis_mermaid.snap @@ -11,7 +11,7 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(input_recv)"/]:::pullClass 2v1[\"(2v1) multiset_delta()"/]:::pullClass 3v1[/"(3v1) for_each(|x| result_send.send(x).unwrap())"\]:::pushClass - 1v1--->2v1 - 2v1--->3v1 + 1v1-->2v1 + 2v1-->3v1 end diff --git a/hydroflow/tests/snapshots/surface_persist__persist@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist@graphvis_mermaid.snap index 4274004f8d0..8f4d6efdb1a 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist@graphvis_mermaid.snap @@ -14,11 +14,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 4v1[/"(4v1) for_each(|v| pull_tx.send(v).unwrap())"\]:::pushClass 5v1[/"(5v1) persist()"\]:::pushClass 6v1[/"(6v1) for_each(|v| push_tx.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 - 2v1--->3v1 - 3v1--->4v1 - 3v1--->5v1 - 5v1--->6v1 + 1v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->5v1 + 5v1-->6v1 subgraph sg_1v1_var_my_tee ["var my_tee"] 1v1 2v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_dot.snap index 9471445045c..88b132cff2b 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_dot.snap @@ -17,7 +17,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n4v1 [label="(n4v1) fold(0, |a, b| (a + b))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) fold(0, |a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n5v1 [label="(n5v1) for_each(|x| result_send.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n4v1 -> n5v1 } diff --git a/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_mermaid.snap index 0258bc43c3b..fc108797ae6 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_basic@graphvis_mermaid.snap @@ -11,15 +11,15 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_iter([1])"/]:::pullClass 2v1[\"(2v1) persist()"/]:::pullClass 3v1[\"(3v1) persist()"/]:::pullClass - 1v1--->2v1 - 2v1--->3v1 + 1v1-->2v1 + 2v1-->3v1 end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 4v1[\"(4v1) fold(0, |a, b| (a + b))"/]:::pullClass + 4v1[\"(4v1) fold(0, |a: &mut _, b| *a += b)"/]:::pullClass 5v1[/"(5v1) for_each(|x| result_send.send(x).unwrap())"\]:::pushClass - 4v1--->5v1 + 4v1-->5v1 end -3v1--->6v1 +3v1-->6v1 6v1["(6v1) handoff"]:::otherClass -6v1===o4v1 +6v1--x4v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_mut@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_mut@graphvis_mermaid.snap index 25e63ca591f..1994d746680 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_mut@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_mut@graphvis_mermaid.snap @@ -18,9 +18,9 @@ subgraph sg_2v1 ["sg_2v1 stratum 1"] 3v1[/"(3v1) tee()"\]:::pushClass 4v1[/"(4v1) for_each(|v| pull_tx.send(v).unwrap())"\]:::pushClass 5v1[/"(5v1) flat_map(|x| if x == 3 { vec![Persist(x), Delete(x)] } else { vec![Persist(x)] })"\]:::pushClass - 2v1--->3v1 - 3v1--->4v1 - 3v1--->5v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->5v1 subgraph sg_2v1_var_my_tee ["var my_tee"] 2v1 3v1 @@ -29,12 +29,12 @@ end subgraph sg_3v1 ["sg_3v1 stratum 2"] 6v1[\"(6v1) persist_mut()"/]:::pullClass 7v1[/"(7v1) for_each(|v| push_tx.send(v).unwrap())"\]:::pushClass - 6v1--->7v1 + 6v1-->7v1 end -1v1--->8v1 -5v1--->9v1 +1v1-->8v1 +5v1-->9v1 8v1["(8v1) handoff"]:::otherClass -8v1===o2v1 +8v1--x2v1 9v1["(9v1) handoff"]:::otherClass -9v1===o6v1 +9v1--x6v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_mut_keyed@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_mut_keyed@graphvis_mermaid.snap index 9021842f1ba..d4aace81499 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_mut_keyed@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_mut_keyed@graphvis_mermaid.snap @@ -18,9 +18,9 @@ subgraph sg_2v1 ["sg_2v1 stratum 1"] 3v1[/"(3v1) tee()"\]:::pushClass 4v1[/"(4v1) for_each(|(_k, v)| pull_tx.send(v).unwrap())"\]:::pushClass 5v1[/"
(5v1)
flat_map(|(k, v)| {
if v == 3 { vec![Persist(k, v), Delete(k)] } else { vec![Persist(k, v)] }
})"\]:::pushClass - 2v1--->3v1 - 3v1--->4v1 - 3v1--->5v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->5v1 subgraph sg_2v1_var_my_tee ["var my_tee"] 2v1 3v1 @@ -29,12 +29,12 @@ end subgraph sg_3v1 ["sg_3v1 stratum 2"] 6v1[\"(6v1) persist_mut_keyed()"/]:::pullClass 7v1[/"(7v1) for_each(|(_k, v)| push_tx.send(v).unwrap())"\]:::pushClass - 6v1--->7v1 + 6v1-->7v1 end -1v1--->8v1 -5v1--->9v1 +1v1-->8v1 +5v1-->9v1 8v1["(8v1) handoff"]:::otherClass -8v1===o2v1 +8v1--x2v1 9v1["(9v1) handoff"]:::otherClass -9v1===o6v1 +9v1--x6v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_dot.snap index 060d6fea2be..913b0c0c276 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_dot.snap @@ -34,7 +34,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n8v1 [label="(n8v1) fold(0, |a, b| (a + b))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) fold(0, |a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n9v1 [label="(n9v1) for_each(|x| result_send.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n8v1 -> n9v1 subgraph "cluster sg_2v1_var_m1" { diff --git a/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_mermaid.snap index 6362c5e5139..96b5e6ac514 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_pull@graphvis_mermaid.snap @@ -15,12 +15,12 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 5v1[\"(5v1) persist()"/]:::pullClass 6v1[\"(6v1) null()"/]:::pullClass 7v1[\"(7v1) union()"/]:::pullClass - 1v1--->2v1 - 2v1--->4v1 - 3v1--->4v1 - 4v1--->5v1 - 5v1--->7v1 - 6v1--->7v1 + 1v1-->2v1 + 2v1-->4v1 + 3v1-->4v1 + 4v1-->5v1 + 5v1-->7v1 + 6v1-->7v1 subgraph sg_1v1_var_m0 ["var m0"] 4v1 5v1 @@ -30,15 +30,15 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] end end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 8v1[\"(8v1) fold(0, |a, b| (a + b))"/]:::pullClass + 8v1[\"(8v1) fold(0, |a: &mut _, b| *a += b)"/]:::pullClass 9v1[/"(9v1) for_each(|x| result_send.send(x).unwrap())"\]:::pushClass - 8v1--->9v1 + 8v1-->9v1 subgraph sg_2v1_var_m1 ["var m1"] 8v1 9v1 end end -7v1--->10v1 +7v1-->10v1 10v1["(10v1) handoff"]:::otherClass -10v1===o8v1 +10v1--x8v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_dot.snap index ef4c4cbbb6b..c73c81387d6 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_dot.snap @@ -36,7 +36,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n8v1 [label="(n8v1) fold(0, |a, b| (a + b))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) fold(0, |a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n9v1 [label="(n9v1) for_each(|x| result_send.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n8v1 -> n9v1 } diff --git a/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_mermaid.snap index 4f2795872a5..fe86384506e 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_push@graphvis_mermaid.snap @@ -15,12 +15,12 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 5v1[/"(5v1) persist()"\]:::pushClass 6v1[/"(6v1) tee()"\]:::pushClass 7v1[/"(7v1) null()"\]:::pushClass - 1v1--->2v1 - 2v1--->3v1 - 3v1--->4v1 - 3v1--->5v1 - 5v1--->6v1 - 6v1--->7v1 + 1v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->5v1 + 5v1-->6v1 + 6v1-->7v1 subgraph sg_1v1_var_t0 ["var t0"] 1v1 2v1 @@ -32,11 +32,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] end end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 8v1[\"(8v1) fold(0, |a, b| (a + b))"/]:::pullClass + 8v1[\"(8v1) fold(0, |a: &mut _, b| *a += b)"/]:::pullClass 9v1[/"(9v1) for_each(|x| result_send.send(x).unwrap())"\]:::pushClass - 8v1--->9v1 + 8v1-->9v1 end -6v1--->10v1 +6v1-->10v1 10v1["(10v1) handoff"]:::otherClass -10v1===o8v1 +10v1--x8v1 diff --git a/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_dot.snap index f61148043e2..7d940386df9 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_dot.snap @@ -15,7 +15,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n3v1 [label="(n3v1) fold::<'tick>(0, |a, b| (a + b))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) fold::<'tick>(0, |a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n3v1" { fillcolor="#dddddd" diff --git a/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_mermaid.snap index a233ecdd70a..34724559620 100644 --- a/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_persist__persist_replay_join@graphvis_mermaid.snap @@ -10,17 +10,17 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(persist_input)"/]:::pullClass 2v1[\"(2v1) persist()"/]:::pullClass - 1v1--->2v1 + 1v1-->2v1 end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 3v1[\"(3v1) fold::<'tick>(0, |a, b| (a + b))"/]:::pullClass + 3v1[\"(3v1) fold::<'tick>(0, |a: &mut _, b| *a += b)"/]:::pullClass end subgraph sg_3v1 ["sg_3v1 stratum 2"] 4v1[\"(4v1) next_stratum()"/]:::pullClass 6v1[\"(6v1) cross_join::<'tick, 'tick>()"/]:::pullClass 7v1[/"(7v1) for_each(|x| result_send.send(x).unwrap())"\]:::pushClass - 4v1--0--->6v1 - 6v1--->7v1 + 4v1-->|0|6v1 + 6v1-->7v1 subgraph sg_3v1_var_product_node ["var product_node"] 6v1 7v1 @@ -29,13 +29,13 @@ end subgraph sg_4v1 ["sg_4v1 stratum 0"] 5v1[\"(5v1) source_stream(other_input)"/]:::pullClass end -2v1--->9v1 -3v1--->8v1 -5v1--->10v1 +2v1-->9v1 +3v1-->8v1 +5v1-->10v1 8v1["(8v1) handoff"]:::otherClass -8v1===o4v1 +8v1--x4v1 9v1["(9v1) handoff"]:::otherClass -9v1===o3v1 +9v1--x3v1 10v1["(10v1) handoff"]:::otherClass -10v1--1--->6v1 +10v1-->|1|6v1 diff --git a/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_dot.snap new file mode 100644 index 00000000000..fcb1bc43b0e --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_dot.snap @@ -0,0 +1,21 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter(0..10)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) map(|x| (x,))", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) py_udf(\l r#\"\ldef fib(n):\lif n < 2:\l return n\lelse:\l return fib(n - 2) + fib(n - 1)\l \"#,\l \"fib\",\l)\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) map(|x: PyResult>| Python::with_gil(|py| {\l usize::extract(x.unwrap().as_ref(py)).unwrap()\l}))\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) assert_eq([0, 1, 1, 2, 3, 5, 8, 13, 21, 34])", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n2v1 -> n3v1 + n3v1 -> n4v1 + n4v1 -> n5v1 + } +} + diff --git a/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_mermaid.snap new file mode 100644 index 00000000000..71193ecb6f8 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_basic@graphvis_mermaid.snap @@ -0,0 +1,21 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter(0..10)"/]:::pullClass + 2v1[\"(2v1) map(|x| (x,))"/]:::pullClass + 3v1[\"
(3v1)
py_udf(
r#"
def fib(n):
if n < 2:
return n
else:
return fib(n - 2) + fib(n - 1)
"#,
"fib",
)"/]:::pullClass + 4v1[\"
(4v1)
map(|x: PyResult<Py<PyAny>>| Python::with_gil(|py| {
usize::extract(x.unwrap().as_ref(py)).unwrap()
}))"/]:::pullClass + 5v1[/"(5v1) assert_eq([0, 1, 1, 2, 3, 5, 8, 13, 21, 34])"\]:::pushClass + 1v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 4v1-->5v1 +end + diff --git a/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_dot.snap new file mode 100644 index 00000000000..fdc1558a0da --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_dot.snap @@ -0,0 +1,21 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(5,)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) py_udf(r#\"\ldef add(a, b):\lreturn a + b\l \"#, \"add\")\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) map(PyResult::>::unwrap_err)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) map(|py_err| py_err.to_string())", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) assert_eq([\"TypeError: add() missing 1 required positional argument: 'b'\"])", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n2v1 -> n3v1 + n3v1 -> n4v1 + n4v1 -> n5v1 + } +} + diff --git a/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_mermaid.snap new file mode 100644 index 00000000000..7114fae808c --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_too_many_args@graphvis_mermaid.snap @@ -0,0 +1,21 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(5,)])"/]:::pullClass + 2v1[\"
(2v1)
py_udf(r#"
def add(a, b):
return a + b
"#, "add")"/]:::pullClass + 3v1[\"(3v1) map(PyResult::<Py<PyAny>>::unwrap_err)"/]:::pullClass + 4v1[\"(4v1) map(|py_err| py_err.to_string())"/]:::pullClass + 5v1[/"(5v1) assert_eq(["TypeError: add() missing 1 required positional argument: 'b'"])"\]:::pushClass + 1v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 4v1-->5v1 +end + diff --git a/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_dot.snap new file mode 100644 index 00000000000..6c6b1cefa3d --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_dot.snap @@ -0,0 +1,19 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([(5, 1)])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) py_udf(r#\"\ldef add(a, b):\lreturn a + b\l \"#, \"add\")\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) map(|x: PyResult>| Python::with_gil(|py| {\l usize::extract(x.unwrap().as_ref(py)).unwrap()\l}))\l", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) assert_eq([6])", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n2v1 -> n3v1 + n3v1 -> n4v1 + } +} + diff --git a/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_mermaid.snap new file mode 100644 index 00000000000..58d90b4495c --- /dev/null +++ b/hydroflow/tests/snapshots/surface_python__python_two_args@graphvis_mermaid.snap @@ -0,0 +1,19 @@ +--- +source: hydroflow/tests/surface_python.rs +expression: hf.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([(5, 1)])"/]:::pullClass + 2v1[\"
(2v1)
py_udf(r#"
def add(a, b):
return a + b
"#, "add")"/]:::pullClass + 3v1[\"
(3v1)
map(|x: PyResult<Py<PyAny>>| Python::with_gil(|py| {
usize::extract(x.unwrap().as_ref(py)).unwrap()
}))"/]:::pullClass + 4v1[/"(4v1) assert_eq([6])"\]:::pushClass + 1v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 +end + diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_dot.snap index ce3f898b244..69de6946768 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_dot.snap @@ -39,7 +39,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n8v1 [label="(n8v1) reduce(|a, b| a + b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n8v1 [label="(n8v1) reduce(|a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n9v1 [label="(n9v1) for_each(|sum| println!(\"{}\", sum))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n8v1 -> n9v1 } diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_mermaid.snap index 8c57edef2b0..5cd1f3ffa0d 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce@graphvis_mermaid.snap @@ -16,14 +16,14 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 5v1[\"(5v1) map(|(_src, ((), dst))| dst)"/]:::pullClass 6v1[/"(6v1) tee()"\]:::pushClass 10v1["(10v1) handoff"]:::otherClass - 10v1--1--->1v1 - 3v1--0--->1v1 - 1v1--->2v1 - 2v1--0--->4v1 - 7v1--1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--0--->10v1 + 10v1-->|1|1v1 + 3v1-->|0|1v1 + 1v1-->2v1 + 2v1-->|0|4v1 + 7v1-->|1|4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->|0|10v1 subgraph sg_1v1_var_my_join_tee ["var my_join_tee"] 4v1 5v1 @@ -35,11 +35,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] end end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 8v1[\"(8v1) reduce(|a, b| a + b)"/]:::pullClass + 8v1[\"(8v1) reduce(|a: &mut _, b| *a += b)"/]:::pullClass 9v1[/"(9v1) for_each(|sum| println!("{}", sum))"\]:::pushClass - 8v1--->9v1 + 8v1-->9v1 end -6v1--1--->11v1 +6v1-->|1|11v1 11v1["(11v1) handoff"]:::otherClass -11v1===o8v1 +11v1--x8v1 diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_dot.snap index c6e71ffab48..9451a3311e2 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) reduce::<'static>(|acc: u32, next: u32| acc + next)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) reduce::<'static>(|acc: &mut u32, next: u32| *acc += next)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) for_each(|v| result_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 } diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_mermaid.snap index 151f1e7097c..91de614612a 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_static@graphvis_mermaid.snap @@ -11,11 +11,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"(2v1) reduce::<'static>(|acc: u32, next: u32| acc + next)"/]:::pullClass + 2v1[\"(2v1) reduce::<'static>(|acc: &mut u32, next: u32| *acc += next)"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_dot.snap index 18dc45d576f..0ddf50f4f99 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) reduce(|a, b| a + b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) reduce(|a: &mut _, b| *a += b)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) for_each(|v| print!(\"{:?}\", v))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 } diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_mermaid.snap index 846cf739d07..c5188ded7f5 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_sum@graphvis_mermaid.snap @@ -11,11 +11,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"(2v1) reduce(|a, b| a + b)"/]:::pullClass + 2v1[\"(2v1) reduce(|a: &mut _, b| *a += b)"/]:::pullClass 3v1[/"(3v1) for_each(|v| print!("{:?}", v))"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_dot.snap index d352cc70760..31643b7ad5a 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_dot.snap @@ -13,7 +13,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 1" - n2v1 [label="(n2v1) reduce::<'tick>(|acc: u32, next: u32| acc + next)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) reduce::<'tick>(|acc: &mut u32, next: u32| *acc += next)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) for_each(|v| result_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n3v1 } diff --git a/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_mermaid.snap index d464255ba3f..d832b08eba3 100644 --- a/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_reduce__reduce_tick@graphvis_mermaid.snap @@ -11,11 +11,11 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] - 2v1[\"(2v1) reduce::<'tick>(|acc: u32, next: u32| acc + next)"/]:::pullClass + 2v1[\"(2v1) reduce::<'tick>(|acc: &mut u32, next: u32| *acc += next)"/]:::pullClass 3v1[/"(3v1) for_each(|v| result_send.send(v).unwrap())"\]:::pushClass - 2v1--->3v1 + 2v1-->3v1 end -1v1--->4v1 +1v1-->4v1 4v1["(4v1) handoff"]:::otherClass -4v1===o2v1 +4v1--x2v1 diff --git a/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_dot.snap new file mode 100644 index 00000000000..123db08e386 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_dot.snap @@ -0,0 +1,26 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([0])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) persist()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 1" + n3v1 [label="(n3v1) next_stratum()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) for_each(|x| out_send.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n4v1 + } + n2v1 -> n5v1 + n5v1 [label="(n5v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n5v1 -> n3v1 [arrowhead=box, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_mermaid.snap new file mode 100644 index 00000000000..45a691f95e0 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_async@graphvis_mermaid.snap @@ -0,0 +1,23 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([0])"/]:::pullClass + 2v1[\"(2v1) persist()"/]:::pullClass + 1v1-->2v1 +end +subgraph sg_2v1 ["sg_2v1 stratum 1"] + 3v1[\"(3v1) next_stratum()"/]:::pullClass + 4v1[/"(4v1) for_each(|x| out_send.send(x).unwrap())"\]:::pushClass + 3v1-->4v1 +end +2v1-->5v1 +5v1["(5v1) handoff"]:::otherClass +5v1--x3v1 + diff --git a/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_dot.snap new file mode 100644 index 00000000000..7a9ad67afce --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_dot.snap @@ -0,0 +1,36 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([0])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) persist()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n1v1 -> n2v1 + subgraph "cluster sg_1v1_var_a" { + label="var a" + n1v1 + n2v1 + } + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 1" + n3v1 [label="(n3v1) next_stratum()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) for_each(|x| out_send.send(x).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n3v1 -> n4v1 + subgraph "cluster sg_2v1_var_a" { + label="var a" + n3v1 + n4v1 + } + } + n2v1 -> n5v1 + n5v1 [label="(n5v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n5v1 -> n3v1 [arrowhead=box, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_mermaid.snap new file mode 100644 index 00000000000..bc2553c4fdb --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__persist_stratum_run_available@graphvis_mermaid.snap @@ -0,0 +1,31 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 1v1[\"(1v1) source_iter([0])"/]:::pullClass + 2v1[\"(2v1) persist()"/]:::pullClass + 1v1-->2v1 + subgraph sg_1v1_var_a ["var a"] + 1v1 + 2v1 + end +end +subgraph sg_2v1 ["sg_2v1 stratum 1"] + 3v1[\"(3v1) next_stratum()"/]:::pullClass + 4v1[/"(4v1) for_each(|x| out_send.send(x).unwrap())"\]:::pushClass + 3v1-->4v1 + subgraph sg_2v1_var_a ["var a"] + 3v1 + 4v1 + end +end +2v1-->5v1 +5v1["(5v1) handoff"]:::otherClass +5v1--x3v1 + diff --git a/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_dot.snap new file mode 100644 index 00000000000..dc8c998ce24 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_dot.snap @@ -0,0 +1,40 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n5v1 [label="(n5v1) filter(|&n| n < 10)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 1" + n1v1 [label="(n1v1) source_iter([0])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) next_stratum()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n4v1 [label="(n4v1) map(|n| n + 1)", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|v| out_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n6v1 -> n2v1 + n2v1 -> n3v1 + n3v1 -> n4v1 + n3v1 -> n7v1 + subgraph "cluster sg_2v1_var_union_tee" { + label="var union_tee" + n2v1 + n3v1 + } + } + n4v1 -> n9v1 + n5v1 -> n8v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n6v1 [arrowhead=box, color=red] + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n5v1 +} + diff --git a/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_mermaid.snap new file mode 100644 index 00000000000..755f68e812a --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__stratum_loop@graphvis_mermaid.snap @@ -0,0 +1,36 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 5v1[\"(5v1) filter(|&n| n < 10)"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 1"] + 1v1[\"(1v1) source_iter([0])"/]:::pullClass + 6v1[\"(6v1) next_stratum()"/]:::pullClass + 2v1[\"(2v1) union()"/]:::pullClass + 3v1[/"(3v1) tee()"\]:::pushClass + 4v1[/"(4v1) map(|n| n + 1)"\]:::pushClass + 7v1[/"(7v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass + 1v1-->2v1 + 6v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->7v1 + subgraph sg_2v1_var_union_tee ["var union_tee"] + 2v1 + 3v1 + end +end +4v1-->9v1 +5v1-->8v1 +8v1["(8v1) handoff"]:::otherClass +8v1--x6v1 +9v1["(9v1) handoff"]:::otherClass +9v1-->5v1 + diff --git a/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_dot.snap new file mode 100644 index 00000000000..f50ed9fa854 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_dot.snap @@ -0,0 +1,49 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_dot() +--- +digraph { + subgraph "cluster n1v1" { + fillcolor="#dddddd" + style=filled + label = "sg_1v1\nstratum 0" + n5v1 [label="(n5v1) filter(|&n| n < 10)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + subgraph "cluster n2v1" { + fillcolor="#dddddd" + style=filled + label = "sg_2v1\nstratum 0" + n1v1 [label="(n1v1) source_iter([0])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n2v1 [label="(n2v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n3v1 [label="(n3v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n4v1 [label="(n4v1) map(|n| n + 1)", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n7v1 [label="(n7v1) for_each(|v| out_send.send(v).unwrap())", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n1v1 -> n2v1 + n6v1 -> n2v1 + n2v1 -> n3v1 + n3v1 -> n4v1 + n3v1 -> n7v1 + subgraph "cluster sg_2v1_var_union_tee" { + label="var union_tee" + n2v1 + n3v1 + } + } + subgraph "cluster n3v1" { + fillcolor="#dddddd" + style=filled + label = "sg_3v1\nstratum 1" + n10v1 [label="(n10v1) identity()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + } + n4v1 -> n9v1 + n5v1 -> n8v1 + n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n8v1 -> n10v1 + n9v1 [label="(n9v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n9v1 -> n5v1 + n10v1 -> n11v1 + n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] + n11v1 -> n6v1 [arrowhead=dot, color=red] +} + diff --git a/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_mermaid.snap new file mode 100644 index 00000000000..aa7b6594c53 --- /dev/null +++ b/hydroflow/tests/snapshots/surface_scheduling__tick_loop@graphvis_mermaid.snap @@ -0,0 +1,42 @@ +--- +source: hydroflow/tests/surface_scheduling.rs +expression: df.meta_graph().unwrap().to_mermaid() +--- +%%{init:{'theme':'base','themeVariables':{'clusterBkg':'#ddd','clusterBorder':'#888'}}}%% +flowchart TD +classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre +classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre +linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; +subgraph sg_1v1 ["sg_1v1 stratum 0"] + 5v1[\"(5v1) filter(|&n| n < 10)"/]:::pullClass +end +subgraph sg_2v1 ["sg_2v1 stratum 0"] + 1v1[\"(1v1) source_iter([0])"/]:::pullClass + 6v1[\"(6v1) defer_tick()"/]:::pullClass + 2v1[\"(2v1) union()"/]:::pullClass + 3v1[/"(3v1) tee()"\]:::pushClass + 4v1[/"(4v1) map(|n| n + 1)"\]:::pushClass + 7v1[/"(7v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass + 1v1-->2v1 + 6v1-->2v1 + 2v1-->3v1 + 3v1-->4v1 + 3v1-->7v1 + subgraph sg_2v1_var_union_tee ["var union_tee"] + 2v1 + 3v1 + end +end +subgraph sg_3v1 ["sg_3v1 stratum 1"] + 10v1[\"(10v1) identity()"/]:::pullClass +end +4v1-->9v1 +5v1-->8v1 +8v1["(8v1) handoff"]:::otherClass +8v1-->10v1 +9v1["(9v1) handoff"]:::otherClass +9v1-->5v1 +10v1-->11v1 +11v1["(11v1) handoff"]:::otherClass +11v1--o6v1 + diff --git a/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_dot.snap index e4b533c7f65..c5a1bec5481 100644 --- a/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_dot.snap @@ -15,7 +15,7 @@ digraph { label = "sg_2v1\nstratum 1" n2v1 [label="(n2v1) source_iter([1, 2, 3, 4])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n1v1 [label="(n1v1) difference()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - n4v1 [label="(n4v1) for_each(|x| output_inner.borrow_mut().push(x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n4v1 [label="(n4v1) for_each(|x| output_inner.borrow_mut().insert(x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n2v1 -> n1v1 [label="pos"] n1v1 -> n4v1 subgraph "cluster sg_2v1_var_a" { diff --git a/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_mermaid.snap index c074eff5744..c6b96c5d389 100644 --- a/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__difference_a@graphvis_mermaid.snap @@ -13,14 +13,14 @@ end subgraph sg_2v1 ["sg_2v1 stratum 1"] 2v1[\"(2v1) source_iter([1, 2, 3, 4])"/]:::pullClass 1v1[\"(1v1) difference()"/]:::pullClass - 4v1[/"(4v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 2v1--pos--->1v1 - 1v1--->4v1 + 4v1[/"(4v1) for_each(|x| output_inner.borrow_mut().insert(x))"\]:::pushClass + 2v1-->|pos|1v1 + 1v1-->4v1 subgraph sg_2v1_var_a ["var a"] 1v1 end end -3v1--->5v1 +3v1-->5v1 5v1["(5v1) handoff"]:::otherClass -5v1==neg===o1v1 +5v1--x|neg|1v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_dot.snap index 9735f082824..b7a62be17f9 100644 --- a/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_dot.snap @@ -7,7 +7,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_1v1\nstratum 0" - n4v1 [label="(n4v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n2v1" { fillcolor="#dddddd" @@ -15,7 +15,7 @@ digraph { label = "sg_2v1\nstratum 1" n1v1 [label="(n1v1) difference()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n3v1 [label="(n3v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] - n5v1 [label="(n5v1) for_each(|x| output_inner.borrow_mut().push(x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] + n5v1 [label="(n5v1) for_each(|x| output_inner.borrow_mut().insert(x))", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] n1v1 -> n3v1 n3v1 -> n5v1 [label="1"] subgraph "cluster sg_2v1_var_a" { @@ -39,7 +39,7 @@ digraph { n6v1 [label="(n6v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] n6v1 -> n1v1 [label="neg", arrowhead=box, color=red] n7v1 [label="(n7v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n7v1 -> n4v1 + n7v1 -> n4v1 [arrowhead=dot, color=red] n8v1 [label="(n8v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] n8v1 -> n1v1 [label="pos"] } diff --git a/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_mermaid.snap index acf678cddf8..5530183dac4 100644 --- a/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__difference_b@graphvis_mermaid.snap @@ -8,14 +8,14 @@ classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] - 4v1[\"(4v1) next_tick()"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 1"] 1v1[\"(1v1) difference()"/]:::pullClass 3v1[/"(3v1) tee()"\]:::pushClass - 5v1[/"(5v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 1v1--->3v1 - 3v1--1--->5v1 + 5v1[/"(5v1) for_each(|x| output_inner.borrow_mut().insert(x))"\]:::pushClass + 1v1-->3v1 + 3v1-->|1|5v1 subgraph sg_2v1_var_a ["var a"] 1v1 end @@ -26,13 +26,13 @@ end subgraph sg_3v1 ["sg_3v1 stratum 0"] 2v1[\"(2v1) source_stream(inp_recv)"/]:::pullClass end -2v1--->8v1 -3v1--0--->7v1 -4v1--->6v1 +2v1-->8v1 +3v1-->|0|7v1 +4v1-->6v1 6v1["(6v1) handoff"]:::otherClass -6v1==neg===o1v1 +6v1--x|neg|1v1 7v1["(7v1) handoff"]:::otherClass -7v1--->4v1 +7v1--o4v1 8v1["(8v1) handoff"]:::otherClass -8v1--pos--->1v1 +8v1-->|pos|1v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__subgraph_stratum_consolidation@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__subgraph_stratum_consolidation@graphvis_mermaid.snap index 0252c247605..0614be837f8 100644 --- a/hydroflow/tests/snapshots/surface_stratum__subgraph_stratum_consolidation@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__subgraph_stratum_consolidation@graphvis_mermaid.snap @@ -12,9 +12,9 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 10v1[\"(10v1) source_iter([1])"/]:::pullClass 1v1[\"(1v1) union()"/]:::pullClass 2v1[/"(2v1) tee()"\]:::pushClass - 9v1--0--->1v1 - 10v1--1--->1v1 - 1v1--->2v1 + 9v1-->|0|1v1 + 10v1-->|1|1v1 + 1v1-->2v1 subgraph sg_1v1_var_a ["var a"] 1v1 2v1 @@ -23,7 +23,7 @@ end subgraph sg_2v1 ["sg_2v1 stratum 0"] 3v1[\"(3v1) union()"/]:::pullClass 4v1[/"(4v1) tee()"\]:::pushClass - 3v1--->4v1 + 3v1-->4v1 subgraph sg_2v1_var_b ["var b"] 3v1 4v1 @@ -32,7 +32,7 @@ end subgraph sg_3v1 ["sg_3v1 stratum 0"] 5v1[\"(5v1) union()"/]:::pullClass 6v1[/"(6v1) tee()"\]:::pushClass - 5v1--->6v1 + 5v1-->6v1 subgraph sg_3v1_var_c ["var c"] 5v1 6v1 @@ -41,28 +41,28 @@ end subgraph sg_4v1 ["sg_4v1 stratum 0"] 7v1[\"(7v1) union()"/]:::pullClass 8v1[/"(8v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 7v1--->8v1 + 7v1-->8v1 subgraph sg_4v1_var_d ["var d"] 7v1 8v1 end end -2v1--0--->13v1 -2v1--1--->16v1 -4v1--0--->12v1 -4v1--1--->15v1 -6v1--0--->11v1 -6v1--1--->14v1 +2v1-->|0|13v1 +2v1-->|1|16v1 +4v1-->|0|12v1 +4v1-->|1|15v1 +6v1-->|0|11v1 +6v1-->|1|14v1 11v1["(11v1) handoff"]:::otherClass -11v1--0--->7v1 +11v1-->|0|7v1 12v1["(12v1) handoff"]:::otherClass -12v1--0--->5v1 +12v1-->|0|5v1 13v1["(13v1) handoff"]:::otherClass -13v1--0--->3v1 +13v1-->|0|3v1 14v1["(14v1) handoff"]:::otherClass -14v1--1--->7v1 +14v1-->|1|7v1 15v1["(15v1) handoff"]:::otherClass -15v1--1--->5v1 +15v1-->|1|5v1 16v1["(16v1) handoff"]:::otherClass -16v1--1--->3v1 +16v1-->|1|3v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__surface_syntax_graph_unreachability@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__surface_syntax_graph_unreachability@graphvis_mermaid.snap index 69edab11b8f..8f081331194 100644 --- a/hydroflow/tests/snapshots/surface_stratum__surface_syntax_graph_unreachability@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__surface_syntax_graph_unreachability@graphvis_mermaid.snap @@ -10,7 +10,7 @@ linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] 4v1[\"(4v1) source_stream(pairs_recv)"/]:::pullClass 5v1[/"(5v1) tee()"\]:::pushClass - 4v1--->5v1 + 4v1-->5v1 subgraph sg_1v1_var_edges ["var edges"] 4v1 5v1 @@ -26,15 +26,15 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 9v1[\"(9v1) map(|x| x)"/]:::pullClass 10v1[/"(10v1) tee()"\]:::pushClass 15v1["(15v1) handoff"]:::otherClass - 15v1--1--->1v1 - 3v1--0--->1v1 - 1v1--->2v1 - 2v1--0--->6v1 - 6v1--->7v1 - 7v1--->8v1 - 8v1--->9v1 - 9v1--->10v1 - 10v1--0--->15v1 + 15v1-->|1|1v1 + 3v1-->|0|1v1 + 1v1-->2v1 + 2v1-->|0|6v1 + 6v1-->7v1 + 7v1-->8v1 + 8v1-->9v1 + 9v1-->10v1 + 10v1-->|0|15v1 subgraph sg_2v1_var_my_join_tee ["var my_join_tee"] 6v1 7v1 @@ -51,20 +51,20 @@ subgraph sg_3v1 ["sg_3v1 stratum 1"] 13v1[\"(13v1) flat_map(|(a, b)| [a, b])"/]:::pullClass 11v1[\"(11v1) difference()"/]:::pullClass 12v1[/"(12v1) for_each(|x| println!("Not reached: {}", x))"\]:::pushClass - 13v1--pos--->11v1 - 11v1--->12v1 + 13v1-->|pos|11v1 + 11v1-->12v1 subgraph sg_3v1_var_diff ["var diff"] 11v1 12v1 end end -5v1--1--->14v1 -5v1--0--->16v1 -10v1--1--->17v1 +5v1-->|1|14v1 +5v1-->|0|16v1 +10v1-->|1|17v1 14v1["(14v1) handoff"]:::otherClass -14v1--1--->6v1 +14v1-->|1|6v1 16v1["(16v1) handoff"]:::otherClass -16v1--->13v1 +16v1-->13v1 17v1["(17v1) handoff"]:::otherClass -17v1==neg===o11v1 +17v1--x|neg|11v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_dot.snap index 440385684f1..7640c69977b 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_dot.snap @@ -7,7 +7,7 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_1v1\nstratum 0" - n4v1 [label="(n4v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n2v1" { fillcolor="#dddddd" @@ -42,6 +42,6 @@ digraph { n8v1 -> n9v1 n9v1 -> n10v1 n10v1 [label="(n10v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n10v1 -> n4v1 + n10v1 -> n4v1 [arrowhead=dot, color=red] } diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_mermaid.snap index 95b489ef0e0..ce63b4c9b13 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_1@graphvis_mermaid.snap @@ -8,7 +8,7 @@ classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] - 4v1[\"(4v1) next_tick()"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 0"] 3v1[\"(3v1) source_iter([1, 3])"/]:::pullClass @@ -16,10 +16,10 @@ subgraph sg_2v1 ["sg_2v1 stratum 0"] 1v1[\"(1v1) union()"/]:::pullClass 2v1[/"(2v1) tee()"\]:::pushClass 6v1[/"(6v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 3v1--0--->1v1 - 5v1--1--->1v1 - 1v1--->2v1 - 2v1--1--->6v1 + 3v1-->|0|1v1 + 5v1-->|1|1v1 + 1v1-->2v1 + 2v1-->|1|6v1 subgraph sg_2v1_var_a ["var a"] 1v1 2v1 @@ -28,13 +28,13 @@ end subgraph sg_3v1 ["sg_3v1 stratum 1"] 9v1[\"(9v1) identity()"/]:::pullClass end -2v1--0--->8v1 -4v1--->7v1 +2v1-->|0|8v1 +4v1-->7v1 7v1["(7v1) handoff"]:::otherClass -7v1--->5v1 +7v1-->5v1 8v1["(8v1) handoff"]:::otherClass -8v1--->9v1 -9v1--->10v1 +8v1-->9v1 +9v1-->10v1 10v1["(10v1) handoff"]:::otherClass -10v1--->4v1 +10v1--o4v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_dot.snap index 538648fb685..26b3099d66b 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_dot.snap @@ -7,14 +7,14 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_1v1\nstratum 0" - n4v1 [label="(n4v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n2v1" { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 0" n3v1 [label="(n3v1) source_iter([1, 3])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - n5v1 [label="(n5v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n6v1 [label="(n6v1) map(|x| 2 * x)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n1v1 [label="(n1v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n2v1 [label="(n2v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] @@ -50,9 +50,9 @@ digraph { n9v1 -> n12v1 n10v1 -> n11v1 n11v1 [label="(n11v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n11v1 -> n5v1 + n11v1 -> n5v1 [arrowhead=dot, color=red] n12v1 -> n13v1 n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n13v1 -> n4v1 + n13v1 -> n4v1 [arrowhead=dot, color=red] } diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_mermaid.snap index 7c7f5b11641..296e447663c 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_2@graphvis_mermaid.snap @@ -8,20 +8,20 @@ classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] - 4v1[\"(4v1) next_tick()"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 0"] 3v1[\"(3v1) source_iter([1, 3])"/]:::pullClass - 5v1[\"(5v1) next_tick()"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass 6v1[\"(6v1) map(|x| 2 * x)"/]:::pullClass 1v1[\"(1v1) union()"/]:::pullClass 2v1[/"(2v1) tee()"\]:::pushClass 7v1[/"(7v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 3v1--0--->1v1 - 5v1--->6v1 - 6v1--1--->1v1 - 1v1--->2v1 - 2v1--1--->7v1 + 3v1-->|0|1v1 + 5v1-->6v1 + 6v1-->|1|1v1 + 1v1-->2v1 + 2v1-->|1|7v1 subgraph sg_2v1_var_a ["var a"] 1v1 2v1 @@ -33,16 +33,16 @@ end subgraph sg_4v1 ["sg_4v1 stratum 1"] 12v1[\"(12v1) identity()"/]:::pullClass end -2v1--0--->9v1 -4v1--->8v1 +2v1-->|0|9v1 +4v1-->8v1 8v1["(8v1) handoff"]:::otherClass -8v1--->10v1 +8v1-->10v1 9v1["(9v1) handoff"]:::otherClass -9v1--->12v1 -10v1--->11v1 +9v1-->12v1 +10v1-->11v1 11v1["(11v1) handoff"]:::otherClass -11v1--->5v1 -12v1--->13v1 +11v1--o5v1 +12v1-->13v1 13v1["(13v1) handoff"]:::otherClass -13v1--->4v1 +13v1--o4v1 diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_dot.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_dot.snap index 677cf3dc587..0bcbd9912a8 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_dot.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_dot.snap @@ -7,20 +7,20 @@ digraph { fillcolor="#dddddd" style=filled label = "sg_1v1\nstratum 0" - n4v1 [label="(n4v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n4v1 [label="(n4v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n2v1" { fillcolor="#dddddd" style=filled label = "sg_2v1\nstratum 0" - n5v1 [label="(n5v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n5v1 [label="(n5v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] } subgraph "cluster n3v1" { fillcolor="#dddddd" style=filled label = "sg_3v1\nstratum 0" n3v1 [label="(n3v1) source_iter([1, 3])", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] - n6v1 [label="(n6v1) next_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] + n6v1 [label="(n6v1) defer_tick()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n7v1 [label="(n7v1) map(|x| 2 * x)", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n1v1 [label="(n1v1) union()", fontname=Monaco, shape=invhouse, style = filled, color = "#0022ff", fontcolor = "#ffffff"] n2v1 [label="(n2v1) tee()", fontname=Monaco, shape=house, style = filled, color = "#ffff00"] @@ -65,12 +65,12 @@ digraph { n11v1 -> n16v1 n12v1 -> n13v1 n13v1 [label="(n13v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n13v1 -> n6v1 + n13v1 -> n6v1 [arrowhead=dot, color=red] n14v1 -> n15v1 n15v1 [label="(n15v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n15v1 -> n5v1 + n15v1 -> n5v1 [arrowhead=dot, color=red] n16v1 -> n17v1 n17v1 [label="(n17v1) handoff", fontname=Monaco, shape=parallelogram, style = filled, color = "#ddddff"] - n17v1 -> n4v1 + n17v1 -> n4v1 [arrowhead=dot, color=red] } diff --git a/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_mermaid.snap index 7b66d18fbb3..efefe402617 100644 --- a/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_stratum__tick_loop_3@graphvis_mermaid.snap @@ -8,23 +8,23 @@ classDef pullClass fill:#8af,stroke:#000,text-align:left,white-space:pre classDef pushClass fill:#ff8,stroke:#000,text-align:left,white-space:pre linkStyle default stroke:#aaa,stroke-width:4px,color:red,font-size:1.5em; subgraph sg_1v1 ["sg_1v1 stratum 0"] - 4v1[\"(4v1) next_tick()"/]:::pullClass + 4v1[\"(4v1) defer_tick()"/]:::pullClass end subgraph sg_2v1 ["sg_2v1 stratum 0"] - 5v1[\"(5v1) next_tick()"/]:::pullClass + 5v1[\"(5v1) defer_tick()"/]:::pullClass end subgraph sg_3v1 ["sg_3v1 stratum 0"] 3v1[\"(3v1) source_iter([1, 3])"/]:::pullClass - 6v1[\"(6v1) next_tick()"/]:::pullClass + 6v1[\"(6v1) defer_tick()"/]:::pullClass 7v1[\"(7v1) map(|x| 2 * x)"/]:::pullClass 1v1[\"(1v1) union()"/]:::pullClass 2v1[/"(2v1) tee()"\]:::pushClass 8v1[/"(8v1) for_each(|x| output_inner.borrow_mut().push(x))"\]:::pushClass - 3v1--0--->1v1 - 6v1--->7v1 - 7v1--1--->1v1 - 1v1--->2v1 - 2v1--1--->8v1 + 3v1-->|0|1v1 + 6v1-->7v1 + 7v1-->|1|1v1 + 1v1-->2v1 + 2v1-->|1|8v1 subgraph sg_3v1_var_a ["var a"] 1v1 2v1 @@ -39,22 +39,22 @@ end subgraph sg_6v1 ["sg_6v1 stratum 1"] 16v1[\"(16v1) identity()"/]:::pullClass end -2v1--0--->11v1 -4v1--->10v1 -5v1--->9v1 +2v1-->|0|11v1 +4v1-->10v1 +5v1-->9v1 9v1["(9v1) handoff"]:::otherClass -9v1--->12v1 +9v1-->12v1 10v1["(10v1) handoff"]:::otherClass -10v1--->14v1 +10v1-->14v1 11v1["(11v1) handoff"]:::otherClass -11v1--->16v1 -12v1--->13v1 +11v1-->16v1 +12v1-->13v1 13v1["(13v1) handoff"]:::otherClass -13v1--->6v1 -14v1--->15v1 +13v1--o6v1 +14v1-->15v1 15v1["(15v1) handoff"]:::otherClass -15v1--->5v1 -16v1--->17v1 +15v1--o5v1 +16v1-->17v1 17v1["(17v1) handoff"]:::otherClass -17v1--->4v1 +17v1--o4v1 diff --git a/hydroflow/tests/snapshots/surface_unique__unique@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_unique__unique@graphvis_mermaid.snap index d4585b7917d..7347d763606 100644 --- a/hydroflow/tests/snapshots/surface_unique__unique@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_unique__unique@graphvis_mermaid.snap @@ -11,7 +11,7 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 1v1[\"(1v1) source_stream(items_recv)"/]:::pullClass 2v1[\"(2v1) unique()"/]:::pullClass 3v1[/"(3v1) for_each(|v| print!("{:?}, ", v))"\]:::pushClass - 1v1--->2v1 - 2v1--->3v1 + 1v1-->2v1 + 2v1-->3v1 end diff --git a/hydroflow/tests/snapshots/surface_unique__unique_static_pull@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_unique__unique_static_pull@graphvis_mermaid.snap index c9234ab8118..dcd5db1b000 100644 --- a/hydroflow/tests/snapshots/surface_unique__unique_static_pull@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_unique__unique_static_pull@graphvis_mermaid.snap @@ -18,15 +18,15 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 8v1[\"(8v1) persist()"/]:::pullClass 9v1[\"(9v1) union()"/]:::pullClass 10v1[/"(10v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 - 2v1--->5v1 - 3v1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--->9v1 - 7v1--->8v1 - 8v1--->9v1 - 9v1--->10v1 + 1v1-->2v1 + 2v1-->5v1 + 3v1-->4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->9v1 + 7v1-->8v1 + 8v1-->9v1 + 9v1-->10v1 subgraph sg_1v1_var_m1 ["var m1"] 5v1 6v1 diff --git a/hydroflow/tests/snapshots/surface_unique__unique_static_push@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_unique__unique_static_push@graphvis_mermaid.snap index f935682b086..84d90cc836e 100644 --- a/hydroflow/tests/snapshots/surface_unique__unique_static_push@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_unique__unique_static_push@graphvis_mermaid.snap @@ -17,14 +17,14 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 7v1[/"(7v1) unique::<'static>()"\]:::pushClass 8v1[/"(8v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass 9v1[/"(9v1) for_each(std::mem::drop)"\]:::pushClass - 1v1--->2v1 - 2v1--->5v1 - 3v1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--->7v1 - 6v1--->9v1 - 7v1--->8v1 + 1v1-->2v1 + 2v1-->5v1 + 3v1-->4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->7v1 + 6v1-->9v1 + 7v1-->8v1 subgraph sg_1v1_var_pivot ["var pivot"] 5v1 6v1 diff --git a/hydroflow/tests/snapshots/surface_unique__unique_tick_pull@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_unique__unique_tick_pull@graphvis_mermaid.snap index af17b5cd201..e65170bcd50 100644 --- a/hydroflow/tests/snapshots/surface_unique__unique_tick_pull@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_unique__unique_tick_pull@graphvis_mermaid.snap @@ -18,15 +18,15 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 8v1[\"(8v1) persist()"/]:::pullClass 9v1[\"(9v1) union()"/]:::pullClass 10v1[/"(10v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass - 1v1--->2v1 - 2v1--->5v1 - 3v1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--->9v1 - 7v1--->8v1 - 8v1--->9v1 - 9v1--->10v1 + 1v1-->2v1 + 2v1-->5v1 + 3v1-->4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->9v1 + 7v1-->8v1 + 8v1-->9v1 + 9v1-->10v1 subgraph sg_1v1_var_m1 ["var m1"] 5v1 6v1 diff --git a/hydroflow/tests/snapshots/surface_unique__unique_tick_push@graphvis_mermaid.snap b/hydroflow/tests/snapshots/surface_unique__unique_tick_push@graphvis_mermaid.snap index d0595ac04a0..bb8be2c36d8 100644 --- a/hydroflow/tests/snapshots/surface_unique__unique_tick_push@graphvis_mermaid.snap +++ b/hydroflow/tests/snapshots/surface_unique__unique_tick_push@graphvis_mermaid.snap @@ -17,14 +17,14 @@ subgraph sg_1v1 ["sg_1v1 stratum 0"] 7v1[/"(7v1) unique::<'tick>()"\]:::pushClass 8v1[/"(8v1) for_each(|v| out_send.send(v).unwrap())"\]:::pushClass 9v1[/"(9v1) for_each(std::mem::drop)"\]:::pushClass - 1v1--->2v1 - 2v1--->5v1 - 3v1--->4v1 - 4v1--->5v1 - 5v1--->6v1 - 6v1--->7v1 - 6v1--->9v1 - 7v1--->8v1 + 1v1-->2v1 + 2v1-->5v1 + 3v1-->4v1 + 4v1-->5v1 + 5v1-->6v1 + 6v1-->7v1 + 6v1-->9v1 + 7v1-->8v1 subgraph sg_1v1_var_pivot ["var pivot"] 5v1 6v1 diff --git a/hydroflow/tests/surface_async.rs b/hydroflow/tests/surface_async.rs index 39815f48ece..d6e8d3542f9 100644 --- a/hydroflow/tests/surface_async.rs +++ b/hydroflow/tests/surface_async.rs @@ -393,7 +393,7 @@ async fn asynctest_check_state_yielding() { async move { let mut hf = hydroflow_syntax! { source_stream(a_recv) - -> reduce::<'static>(|a, b| a + b) + -> reduce::<'static>(|a: &mut _, b| *a += b) -> for_each(|x| b_send.send(x).unwrap()); }; diff --git a/hydroflow/tests/surface_batch.rs b/hydroflow/tests/surface_batch.rs new file mode 100644 index 00000000000..9837ec24167 --- /dev/null +++ b/hydroflow/tests/surface_batch.rs @@ -0,0 +1,28 @@ +use hydroflow::util::collect_ready; +use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; +use multiplatform_test::multiplatform_test; + +#[multiplatform_test] +pub fn test_basic_2() { + let (signal_tx, signal_rx) = hydroflow::util::unbounded_channel::<()>(); + let (egress_tx, mut egress_rx) = hydroflow::util::unbounded_channel(); + + let mut df = hydroflow_syntax! { + gate = defer_signal(); + source_iter([1, 2, 3]) -> [input]gate; + source_stream(signal_rx) -> [signal]gate; + + gate -> for_each(|x| egress_tx.send(x).unwrap()); + }; + assert_graphvis_snapshots!(df); + + df.run_available(); + let out: Vec<_> = collect_ready(&mut egress_rx); + assert_eq!(out, [0; 0]); + + signal_tx.send(()).unwrap(); + df.run_available(); + + let out: Vec<_> = collect_ready(&mut egress_rx); + assert_eq!(out, vec![1, 2, 3]); +} diff --git a/hydroflow/tests/surface_codegen.rs b/hydroflow/tests/surface_codegen.rs index 0991ba25861..4d205426baa 100644 --- a/hydroflow/tests/surface_codegen.rs +++ b/hydroflow/tests/surface_codegen.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use hydroflow::scheduled::graph::Hydroflow; use hydroflow::util::collect_ready; +use hydroflow::util::multiset::HashMultiSet; use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; use multiplatform_test::multiplatform_test; @@ -204,105 +205,74 @@ pub fn test_cross_join() { let mut df = hydroflow_syntax! { cj = cross_join() -> for_each(|v| out_send.send(v).unwrap()); - source_iter([1, 2, 3]) -> [0]cj; - source_iter(["a", "b", "c"]) -> [1]cj; + source_iter([1, 2, 2, 3]) -> [0]cj; + source_iter(["a", "b", "c", "c"]) -> [1]cj; }; df.run_available(); - let out: HashSet<_> = collect_ready(&mut out_recv); - assert_eq!(3 * 3, out.len()); - for n in [1, 2, 3] { - for c in ["a", "b", "c"] { - assert!(out.contains(&(n, c))); - } - } + let mut out: Vec<_> = collect_ready(&mut out_recv); + out.sort(); + assert_eq!( + out, + [ + (1, "a"), + (1, "b"), + (1, "c"), + (2, "a"), + (2, "b"), + (2, "c"), + (3, "a"), + (3, "b"), + (3, "c") + ] + ); } #[multiplatform_test] -pub fn test_lattice_join() { - use hydroflow::lattices::Max; - - // 'static, 'tick. - { - let (out_tx, mut out_rx) = - hydroflow::util::unbounded_channel::<(usize, (Max, Max))>(); - let (lhs_tx, lhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); - let (rhs_tx, rhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); - - let mut df = hydroflow_syntax! { - my_join = lattice_join::<'static, 'tick, Max, Max>(); - - source_stream(lhs_rx) -> [0]my_join; - source_stream(rhs_rx) -> [1]my_join; - - my_join -> for_each(|v| out_tx.send(v).unwrap()); - }; - - // Merges forward correctly, without going backward - lhs_tx.send((7, Max::new(3))).unwrap(); - lhs_tx.send((7, Max::new(4))).unwrap(); - rhs_tx.send((7, Max::new(6))).unwrap(); - rhs_tx.send((7, Max::new(5))).unwrap(); - - df.run_tick(); - - let out: Vec<_> = collect_ready(&mut out_rx); - assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); - - // Forgets rhs state - rhs_tx.send((7, Max::new(6))).unwrap(); - rhs_tx.send((7, Max::new(5))).unwrap(); - - df.run_tick(); - - let out: Vec<_> = collect_ready(&mut out_rx); - assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); - } - - // 'static, 'static. - { - let (out_tx, mut out_rx) = - hydroflow::util::unbounded_channel::<(usize, (Max, Max))>(); - let (lhs_tx, lhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); - let (rhs_tx, rhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); - - let mut df = hydroflow_syntax! { - my_join = lattice_join::<'static, 'static, Max, Max>(); - - source_stream(lhs_rx) -> [0]my_join; - source_stream(rhs_rx) -> [1]my_join; - - my_join -> for_each(|v| out_tx.send(v).unwrap()); - }; - - lhs_tx.send((7, Max::new(3))).unwrap(); - lhs_tx.send((7, Max::new(4))).unwrap(); - rhs_tx.send((7, Max::new(6))).unwrap(); - rhs_tx.send((7, Max::new(5))).unwrap(); - - df.run_tick(); - let out: Vec<_> = collect_ready(&mut out_rx); - assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); +pub fn test_cross_join_multiset() { + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, &str)>(); - // Doesn't produce unless a lattice has changed. - lhs_tx.send((7, Max::new(4))).unwrap(); - rhs_tx.send((7, Max::new(5))).unwrap(); + let mut df = hydroflow_syntax! { + cj = cross_join_multiset() -> for_each(|v| out_send.send(v).unwrap()); + source_iter([1, 2, 2, 3]) -> [0]cj; + source_iter(["a", "b", "c", "c"]) -> [1]cj; + }; + df.run_available(); - df.run_tick(); - let out: Vec<_> = collect_ready(&mut out_rx); - assert_eq!(out, vec![]); - } + let mut out: Vec<_> = collect_ready(&mut out_recv); + out.sort(); + assert_eq!( + out, + [ + (1, "a"), + (1, "b"), + (1, "c"), + (1, "c"), + (2, "a"), + (2, "a"), + (2, "b"), + (2, "b"), + (2, "c"), + (2, "c"), + (2, "c"), + (2, "c"), + (3, "a"), + (3, "b"), + (3, "c"), + (3, "c"), + ] + ); } #[multiplatform_test] -pub fn test_next_tick() { +pub fn test_defer_tick() { let (inp_send, inp_recv) = hydroflow::util::unbounded_channel::(); let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut flow = hydroflow::hydroflow_syntax! { inp = source_stream(inp_recv) -> tee(); diff = difference() -> for_each(|x| out_send.send(x).unwrap()); inp -> [pos]diff; - inp -> next_tick() -> [neg]diff; + inp -> defer_tick() -> [neg]diff; }; for x in [1, 2, 3, 4] { @@ -316,8 +286,10 @@ pub fn test_next_tick() { flow.run_tick(); flow.run_available(); - let out: Vec<_> = collect_ready(&mut out_recv); - assert_eq!(&[1, 2, 3, 4, 5, 6], &*out); + assert_eq!( + HashMultiSet::from_iter([1, 2, 3, 4, 5, 6]), + collect_ready(&mut out_recv) + ); } #[multiplatform_test] @@ -326,12 +298,12 @@ pub fn test_anti_join() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); let mut flow = hydroflow::hydroflow_syntax! { inp = source_stream(inp_recv) -> tee(); - diff = anti_join() -> for_each(|x| out_send.send(x).unwrap()); + diff = anti_join() -> sort() -> for_each(|x| out_send.send(x).unwrap()); inp -> [pos]diff; - inp -> next_tick() -> map(|x: (usize, usize)| x.0) -> [neg]diff; + inp -> defer_tick() -> map(|x: (usize, usize)| x.0) -> [neg]diff; }; - for x in [(1, 2), (2, 3), (3, 4), (4, 5)] { + for x in [(1, 2), (1, 2), (2, 3), (3, 4), (4, 5)] { inp_send.send(x).unwrap(); } flow.run_tick(); @@ -347,118 +319,156 @@ pub fn test_anti_join() { } #[multiplatform_test] -pub fn test_batch() { - let (batch1_tx, batch1_rx) = hydroflow::util::unbounded_channel::<()>(); - let (batch2_tx, batch2_rx) = hydroflow::util::unbounded_channel::<()>(); - let (tx, mut rx) = hydroflow::util::unbounded_channel::<()>(); - let mut df = hydroflow_syntax! { - my_tee = tee(); - - source_iter([()]) - -> batch(1, batch1_rx) // pull - -> my_tee; - - my_tee -> for_each(|x| tx.send(x).unwrap()); - my_tee - -> batch(1, batch2_rx) // push - -> for_each(|x| tx.send(x).unwrap()); +pub fn test_anti_join_static() { + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let mut flow = hydroflow::hydroflow_syntax! { + pos = source_stream(pos_recv); + neg = source_stream(neg_recv); + pos -> [pos]diff_static; + neg -> [neg]diff_static; + diff_static = anti_join::<'static>() -> sort() -> for_each(|x| out_send.send(x).unwrap()); }; - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx); - assert_eq!(out, Vec::<()>::new()); + for x in [(1, 2), (1, 2), (200, 3), (300, 4), (400, 5), (5, 6)] { + pos_send.send(x).unwrap(); + } + for x in [200, 300] { + neg_send.send(x).unwrap(); + } + flow.run_tick(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (5, 6), (400, 5)], &*out); - batch1_tx.send(()).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx); - assert_eq!(out, vec![()]); + neg_send.send(400).unwrap(); - batch2_tx.send(()).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx); - assert_eq!(out, vec![()]); + flow.run_available(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (5, 6)], &*out); } #[multiplatform_test] -pub fn test_lattice_batch() { - type SetUnionHashSet = lattices::set_union::SetUnionHashSet; - type SetUnionSingletonSet = lattices::set_union::SetUnionSingletonSet; - - let (batch1_tx, batch1_rx) = hydroflow::util::unbounded_channel::<()>(); - let (batch2_tx, batch2_rx) = hydroflow::util::unbounded_channel::<()>(); - let (tx_in, rx_in) = hydroflow::util::unbounded_channel::(); - let (tx_out, mut rx_out) = hydroflow::util::unbounded_channel::(); - let mut df = hydroflow_syntax! { - my_tee = tee(); +pub fn test_anti_join_tick_static() { + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let mut flow = hydroflow::hydroflow_syntax! { + pos = source_stream(pos_recv); + neg = source_stream(neg_recv); + pos -> [pos]diff_static; + neg -> [neg]diff_static; + diff_static = anti_join::<'tick, 'static>() -> sort() -> for_each(|x| out_send.send(x).unwrap()); + }; - source_stream(rx_in) - -> lattice_batch::(batch1_rx) // pull - -> my_tee; + for x in [(1, 2), (1, 2), (200, 3), (300, 4), (400, 5), (5, 6)] { + pos_send.send(x).unwrap(); + } + for x in [200, 300] { + neg_send.send(x).unwrap(); + } + flow.run_tick(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (5, 6), (400, 5)], &*out); - my_tee - -> for_each(|x| tx_out.send(x).unwrap()); + for x in [(10, 10), (10, 10), (200, 5)] { + pos_send.send(x).unwrap(); + } - my_tee - -> lattice_batch::(batch2_rx) // push - -> for_each(|x| tx_out.send(x).unwrap()); - }; + flow.run_available(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(10, 10)], &*out); +} - tx_in.send(SetUnionSingletonSet::new_from(0)).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, Vec::::new()); +#[multiplatform_test] +pub fn test_anti_join_multiset_tick_static() { + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let mut flow = hydroflow::hydroflow_syntax! { + pos = source_stream(pos_recv); + neg = source_stream(neg_recv); + pos -> [pos]diff_static; + neg -> [neg]diff_static; + diff_static = anti_join_multiset::<'tick, 'static>() -> sort() -> for_each(|x| out_send.send(x).unwrap()); + }; - batch1_tx.send(()).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, vec![SetUnionHashSet::new_from([0])]); + for x in [(1, 2), (1, 2), (200, 3), (300, 4), (400, 5), (5, 6)] { + pos_send.send(x).unwrap(); + } + for x in [200, 300] { + neg_send.send(x).unwrap(); + } + flow.run_tick(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (1, 2), (5, 6), (400, 5),], &*out); - batch1_tx.send(()).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, Vec::::new()); + for x in [(10, 10), (10, 10), (200, 5)] { + pos_send.send(x).unwrap(); + } - batch2_tx.send(()).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, vec![SetUnionHashSet::new_from([0])]); + flow.run_available(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(10, 10), (10, 10)], &*out); } #[multiplatform_test] -pub fn test_batch_exceed_limit() { - let (_, batch1_rx) = hydroflow::util::unbounded_channel::<()>(); - let (_, batch2_rx) = hydroflow::util::unbounded_channel::<()>(); - let (tx_in, rx_in) = hydroflow::util::unbounded_channel::(); - let (tx_out, mut rx_out) = hydroflow::util::unbounded_channel::(); - let mut df = hydroflow_syntax! { - my_tee = tee(); +pub fn test_anti_join_multiset_static() { + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let mut flow = hydroflow::hydroflow_syntax! { + pos = source_stream(pos_recv); + neg = source_stream(neg_recv); + pos -> [pos]diff_static; + neg -> [neg]diff_static; + diff_static = anti_join_multiset::<'static>() -> sort() -> for_each(|x| out_send.send(x).unwrap()); + }; - source_stream(rx_in) - -> batch(1, batch1_rx) // pull - -> my_tee; + for x in [(1, 2), (1, 2), (200, 3), (300, 4), (400, 5), (5, 6)] { + pos_send.send(x).unwrap(); + } + for x in [200, 300] { + neg_send.send(x).unwrap(); + } + flow.run_tick(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (1, 2), (5, 6), (400, 5)], &*out); - my_tee - -> for_each(|x| tx_out.send(x).unwrap()); + neg_send.send(400).unwrap(); - my_tee - -> batch(1, batch2_rx) // push - -> for_each(|x| tx_out.send(x).unwrap()); + flow.run_available(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!(&[(1, 2), (1, 2), (5, 6)], &*out); +} + +#[multiplatform_test] +pub fn test_anti_join_multiset() { + let (inp_send, inp_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(usize, usize)>(); + let mut flow = hydroflow::hydroflow_syntax! { + inp = source_stream(inp_recv) -> tee(); + diff = anti_join_multiset() -> sort() -> for_each(|x| out_send.send(x).unwrap()); + inp -> [pos]diff; + inp -> defer_tick() -> map(|x: (usize, usize)| x.0) -> [neg]diff; }; - tx_in.send(0).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, Vec::::new()); + for x in [(1, 2), (1, 2), (2, 3), (3, 4), (4, 5)] { + inp_send.send(x).unwrap(); + } + flow.run_tick(); - tx_in.send(1).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, vec![0, 1, 0, 1]); + for x in [(3, 2), (4, 3), (5, 4), (6, 5)] { + inp_send.send(x).unwrap(); + } + flow.run_tick(); - tx_in.send(2).unwrap(); - df.run_available(); - let out: Vec<_> = collect_ready(&mut rx_out); - assert_eq!(out, Vec::::new()); + flow.run_available(); + let out: Vec<_> = collect_ready(&mut out_recv); + assert_eq!( + &[(1, 2), (1, 2), (2, 3), (3, 4), (4, 5), (5, 4), (6, 5)], + &*out + ); } #[multiplatform_test] @@ -809,12 +819,12 @@ pub fn test_covid_tracing() { } #[multiplatform_test] -pub fn test_assert() { +pub fn test_assert_eq() { let mut df = hydroflow_syntax! { - source_iter([1, 2, 3]) -> assert([1, 2, 3]) -> assert([1, 2, 3]); // one in pull, one in push - source_iter([1, 2, 3]) -> assert([1, 2, 3]) -> assert(vec![1, 2, 3]); - source_iter([1, 2, 3]) -> assert(vec![1, 2, 3]) -> assert([1, 2, 3]); - source_iter(vec![1, 2, 3]) -> assert([1, 2, 3]) -> assert([1, 2, 3]); + source_iter([1, 2, 3]) -> assert_eq([1, 2, 3]) -> assert_eq([1, 2, 3]); // one in pull, one in push + source_iter([1, 2, 3]) -> assert_eq([1, 2, 3]) -> assert_eq(vec![1, 2, 3]); + source_iter([1, 2, 3]) -> assert_eq(vec![1, 2, 3]) -> assert_eq([1, 2, 3]); + source_iter(vec![1, 2, 3]) -> assert_eq([1, 2, 3]) -> assert_eq([1, 2, 3]); }; df.run_available(); } @@ -823,7 +833,7 @@ pub fn test_assert() { pub fn test_assert_failures() { assert!(std::panic::catch_unwind(|| { let mut df = hydroflow_syntax! { - source_iter([0]) -> assert([1]); + source_iter([0]) -> assert_eq([1]); }; df.run_available(); @@ -832,10 +842,27 @@ pub fn test_assert_failures() { assert!(std::panic::catch_unwind(|| { let mut df = hydroflow_syntax! { - source_iter([0]) -> assert([1]) -> null(); + source_iter([0]) -> assert_eq([1]) -> null(); }; df.run_available(); }) .is_err()); } + +#[multiplatform_test] +pub fn test_iter_stream_batches() { + const ITEMS: usize = 100; + const BATCH: usize = 5; + let stream = hydroflow::util::iter_batches_stream(0..ITEMS, BATCH); + + // expect 5 items per tick. + let expected: Vec<_> = (0..ITEMS).map(|n| (n / BATCH, n)).collect(); + + let mut df = hydroflow_syntax! { + source_stream(stream) + -> map(|x| (context.current_tick(), x)) + -> assert_eq(expected); + }; + df.run_available(); +} diff --git a/hydroflow/tests/surface_context.rs b/hydroflow/tests/surface_context.rs index 8868e7d59a7..6b8942ac735 100644 --- a/hydroflow/tests/surface_context.rs +++ b/hydroflow/tests/surface_context.rs @@ -1,4 +1,9 @@ +use std::cell::Cell; +use std::rc::Rc; + use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; +use hydroflow_macro::hydroflow_test; +use instant::{Duration, Instant}; use multiplatform_test::multiplatform_test; #[multiplatform_test] @@ -24,3 +29,62 @@ pub fn test_context_mut() { assert_graphvis_snapshots!(df); df.run_available(); } + +#[multiplatform_test] +pub fn test_context_current_tick_start() { + let mut df = hydroflow_syntax! { + source_iter([()]) + -> map(|_| context.current_tick_start()) + -> defer_tick() + -> assert(|t: &hydroflow::instant::Instant| t.elapsed().as_nanos() > 0) + -> for_each(|t: hydroflow::instant::Instant| println!("Time between ticks: {:?}", t.elapsed())); + }; + assert_graphvis_snapshots!(df); + df.run_available(); +} + +#[multiplatform_test(hydroflow)] +pub async fn test_context_current_tick_start_does_not_count_time_between_ticks_async() { + let time = Rc::new(Cell::new(None)); + + let mut df = { + let time = time.clone(); + hydroflow_syntax! { + source_iter([()]) + -> persist() + -> for_each(|_| time.set(Some(Instant::now() - context.current_tick_start()))); + } + }; + assert_graphvis_snapshots!(df); + tokio::time::sleep(Duration::from_millis(100)).await; + df.run_tick(); + assert!(time.take().unwrap() < Duration::from_millis(50)); + + tokio::time::sleep(Duration::from_millis(100)).await; + df.run_tick(); + assert!(time.take().unwrap() < Duration::from_millis(50)); + + tokio::time::sleep(Duration::from_millis(100)).await; + df.run_available(); + assert!(time.take().unwrap() < Duration::from_millis(50)); + + tokio::time::sleep(Duration::from_millis(100)).await; + df.run_available_async().await; + assert!(time.take().unwrap() < Duration::from_millis(50)); +} + +#[hydroflow_test] +pub async fn test_defered_tick_and_no_io_with_run_async() { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + + let mut df = hydroflow_syntax! { + source_iter([()]) + -> defer_tick() + -> for_each(|_| tx.send(()).unwrap()); + }; + + tokio::select! { + _ = df.run_async() => {}, + _ = rx.recv() => {}, + } +} diff --git a/hydroflow/tests/surface_difference.rs b/hydroflow/tests/surface_difference.rs index b4687abc0d1..55ace654c61 100644 --- a/hydroflow/tests/surface_difference.rs +++ b/hydroflow/tests/surface_difference.rs @@ -32,6 +32,7 @@ pub fn test_diff_timing() { pos_send.send(2).unwrap(); pos_send.send(3).unwrap(); pos_send.send(4).unwrap(); + pos_send.send(4).unwrap(); neg_send.send(2).unwrap(); neg_send.send(3).unwrap(); df.run_tick(); @@ -50,7 +51,7 @@ pub fn test_diff_static() { let (output_send, mut output_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow::hydroflow_syntax! { - diff = difference::<'tick, 'static>() -> for_each(|v| output_send.send(v).unwrap()); + diff = difference::<'tick, 'static>() -> sort() -> for_each(|v| output_send.send(v).unwrap()); poss = source_stream(pos_recv); //-> tee(); poss -> [pos]diff; @@ -64,6 +65,7 @@ pub fn test_diff_static() { }; assert_graphvis_snapshots!(df); + pos_send.send(1).unwrap(); pos_send.send(1).unwrap(); pos_send.send(2).unwrap(); @@ -73,6 +75,7 @@ pub fn test_diff_static() { assert_eq!(&[1], &*collect_ready::, _>(&mut output_recv)); + pos_send.send(1).unwrap(); pos_send.send(1).unwrap(); pos_send.send(2).unwrap(); pos_send.send(3).unwrap(); @@ -81,3 +84,174 @@ pub fn test_diff_static() { assert_eq!(&[1, 3], &*collect_ready::, _>(&mut output_recv)); } + +#[multiplatform_test] +pub fn test_diff_multiset_timing() { + // An edge in the input data = a pair of `usize` vertex IDs. + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + + let mut df = hydroflow::hydroflow_syntax! { + diff = difference_multiset() -> for_each(|x| println!("diff: {:?}", x)); + + poss = source_stream(pos_recv); //-> tee(); + poss -> [pos]diff; + // if you enable the comment below it produces the right answer + // poss -> for_each(|x| println!("pos: {:?}", x)); + + negs = source_stream(neg_recv) -> tee(); + negs -> [neg]diff; + negs -> for_each(|x| println!("neg: {:?}", x)); + + }; + assert_graphvis_snapshots!(df); + + df.run_tick(); + println!("{}x{}", df.current_tick(), df.current_stratum()); + + println!("A"); + + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + pos_send.send(3).unwrap(); + pos_send.send(4).unwrap(); + pos_send.send(4).unwrap(); + neg_send.send(2).unwrap(); + neg_send.send(3).unwrap(); + df.run_tick(); + + println!("B"); + neg_send.send(1).unwrap(); + df.run_tick(); +} + +#[multiplatform_test] +pub fn test_diff_multiset_static() { + // An edge in the input data = a pair of `usize` vertex IDs. + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + + let (output_send, mut output_recv) = hydroflow::util::unbounded_channel::(); + + let mut df = hydroflow::hydroflow_syntax! { + diff = difference_multiset::<'static>() -> sort() -> for_each(|v| output_send.send(v).unwrap()); + + poss = source_stream(pos_recv); //-> tee(); + poss -> [pos]diff; + // if you enable the comment below it produces the right answer + // poss -> for_each(|x| println!("pos: {:?}", x)); + + negs = source_stream(neg_recv) -> tee(); + negs -> [neg]diff; + negs -> for_each(|x| println!("neg: {:?}", x)); + + }; + assert_graphvis_snapshots!(df); + + pos_send.send(1).unwrap(); + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + + neg_send.send(2).unwrap(); + + df.run_tick(); + + assert_eq!(&[1, 1], &*collect_ready::, _>(&mut output_recv)); + + pos_send.send(1).unwrap(); + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + pos_send.send(3).unwrap(); + + df.run_tick(); + + assert_eq!( + &[1, 1, 1, 1, 3], + &*collect_ready::, _>(&mut output_recv) + ); +} + +#[multiplatform_test] +pub fn test_diff_multiset_tick_static() { + // An edge in the input data = a pair of `usize` vertex IDs. + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + + let (output_send, mut output_recv) = hydroflow::util::unbounded_channel::(); + + let mut df = hydroflow::hydroflow_syntax! { + diff = difference_multiset::<'tick, 'static>() -> sort() -> for_each(|v| output_send.send(v).unwrap()); + + poss = source_stream(pos_recv); //-> tee(); + poss -> [pos]diff; + // if you enable the comment below it produces the right answer + // poss -> for_each(|x| println!("pos: {:?}", x)); + + negs = source_stream(neg_recv) -> tee(); + negs -> [neg]diff; + negs -> for_each(|x| println!("neg: {:?}", x)); + + }; + assert_graphvis_snapshots!(df); + + pos_send.send(1).unwrap(); + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + + neg_send.send(2).unwrap(); + + df.run_tick(); + + assert_eq!(&[1, 1], &*collect_ready::, _>(&mut output_recv)); + + pos_send.send(1).unwrap(); + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + pos_send.send(3).unwrap(); + + df.run_tick(); + + assert_eq!(&[1, 1, 3], &*collect_ready::, _>(&mut output_recv)); +} + +#[multiplatform_test] +pub fn test_diff_multiset_static_tick() { + // An edge in the input data = a pair of `usize` vertex IDs. + let (pos_send, pos_recv) = hydroflow::util::unbounded_channel::(); + let (neg_send, neg_recv) = hydroflow::util::unbounded_channel::(); + + let (output_send, mut output_recv) = hydroflow::util::unbounded_channel::(); + + let mut df = hydroflow::hydroflow_syntax! { + diff = difference_multiset::<'static, 'tick>() -> sort() -> for_each(|v| output_send.send(v).unwrap()); + + poss = source_stream(pos_recv); //-> tee(); + poss -> [pos]diff; + // if you enable the comment below it produces the right answer + // poss -> for_each(|x| println!("pos: {:?}", x)); + + negs = source_stream(neg_recv) -> tee(); + negs -> [neg]diff; + negs -> for_each(|x| println!("neg: {:?}", x)); + + }; + assert_graphvis_snapshots!(df); + + pos_send.send(1).unwrap(); + pos_send.send(1).unwrap(); + pos_send.send(2).unwrap(); + + neg_send.send(2).unwrap(); + + df.run_tick(); + + assert_eq!(&[1, 1], &*collect_ready::, _>(&mut output_recv)); + + pos_send.send(3).unwrap(); + + neg_send.send(3).unwrap(); + + df.run_tick(); + + assert_eq!(&[1, 1, 2], &*collect_ready::, _>(&mut output_recv)); +} diff --git a/hydroflow/tests/surface_flow_props.rs b/hydroflow/tests/surface_flow_props.rs new file mode 100644 index 00000000000..b6e95930d6b --- /dev/null +++ b/hydroflow/tests/surface_flow_props.rs @@ -0,0 +1,44 @@ +use hydroflow::lattices::collections::SingletonSet; +use hydroflow::lattices::set_union::{SetUnion, SetUnionHashSet, SetUnionSingletonSet}; +use hydroflow::{assert_graphvis_snapshots, hydroflow_expect_warnings, hydroflow_syntax}; + +#[test] +pub fn test_basic() { + let mut hf = hydroflow_syntax! { + my_tee = source_iter_delta((0..10).map(SetUnionSingletonSet::new_from)) + -> map(|SetUnion(SingletonSet(x))| SetUnion(SingletonSet(x + 5))) + -> tee(); + + my_tee + -> cast(None) + -> map(|SetUnion(SingletonSet(x))| 10 * x) + -> for_each(|x| println!("seq {:?}", x)); + + my_tee + -> for_each(|s| println!("delta {:?}", s)); + + my_tee + -> map(|SetUnion(SingletonSet(x))| SetUnionHashSet::new_from([x])) + -> lattice_reduce::<'static, SetUnionHashSet<_>>() + -> for_each(|s| println!("cumul {:?}", s)); + }; + hf.run_available(); + + assert_graphvis_snapshots!(hf); +} + +#[test] +pub fn test_union_warning() { + let mut hf = hydroflow_expect_warnings! { + { + source_iter_delta((0..10).map(SetUnionSingletonSet::new_from)) -> [0]my_union; + source_iter((0..10).map(SetUnionSingletonSet::new_from)) -> [1]my_union; + + my_union = union() -> for_each(|s| println!("{:?}", s)); + }, + "Warning: Input to `union()` will be downcast to `None` to match other inputs.\n --> $FILE:0:0" + }; + hf.run_available(); + + assert_graphvis_snapshots!(hf); +} diff --git a/hydroflow/tests/surface_fold.rs b/hydroflow/tests/surface_fold.rs index b542503dac6..43d57f1e595 100644 --- a/hydroflow/tests/surface_fold.rs +++ b/hydroflow/tests/surface_fold.rs @@ -11,7 +11,7 @@ pub fn test_fold_tick() { let mut df = hydroflow::hydroflow_syntax! { source_stream(items_recv) - -> fold::<'tick>(Vec::new(), |mut old: Vec, mut x: Vec| { old.append(&mut x); old }) + -> fold::<'tick>(Vec::new(), |old: &mut Vec, mut x: Vec| { old.append(&mut x); }) -> for_each(|v| result_send.send(v).unwrap()); }; assert_graphvis_snapshots!(df); @@ -48,7 +48,7 @@ pub fn test_fold_static() { let mut df = hydroflow::hydroflow_syntax! { source_stream(items_recv) - -> fold::<'static>(Vec::new(), |mut old: Vec, mut x: Vec| { old.append(&mut x); old }) + -> fold::<'static>(Vec::new(), |old: &mut Vec, mut x: Vec| { old.append(&mut x); }) -> for_each(|v| result_send.send(v).unwrap()); }; assert_graphvis_snapshots!(df); @@ -84,10 +84,10 @@ pub fn test_fold_flatten() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(u8, u8)>(); let mut df_pull = hydroflow_syntax! { source_iter([(1,1), (1,2), (2,3), (2,4)]) - -> fold::<'tick>(HashMap::::new(), |mut ht, t: (u8,u8)| { + -> fold::<'tick>(HashMap::::new(), |ht: &mut HashMap, t: (u8,u8)| { let e = ht.entry(t.0).or_insert(0); *e += t.1; - ht}) + }) -> flatten() -> for_each(|(k,v)| out_send.send((k,v)).unwrap()); }; @@ -105,10 +105,10 @@ pub fn test_fold_flatten() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::<(u8, u8)>(); let mut df_push = hydroflow_syntax! { datagen = source_iter([(1,2), (1,2), (2,4), (2,4)]) -> tee(); - datagen[0] -> fold::<'tick>(HashMap::::new(), |mut ht, t:(u8,u8)| { + datagen[0] -> fold::<'tick>(HashMap::::new(), |ht: &mut HashMap, t:(u8,u8)| { let e = ht.entry(t.0).or_insert(0); *e += t.1; - ht}) + }) -> flatten() -> for_each(|(k,v)| out_send.send((k,v)).unwrap()); datagen[1] -> null(); @@ -132,10 +132,7 @@ pub fn test_fold_sort() { let mut df = hydroflow_syntax! { source_stream(items_recv) - -> fold::<'tick>(Vec::new(), |mut v, x| { - v.push(x); - v - }) + -> fold::<'tick>(Vec::new(), Vec::push) -> flat_map(|mut vec| { vec.sort(); vec }) -> for_each(|v| print!("{:?}, ", v)); }; diff --git a/hydroflow/tests/surface_join.rs b/hydroflow/tests/surface_join.rs new file mode 100644 index 00000000000..38bdabe1b3e --- /dev/null +++ b/hydroflow/tests/surface_join.rs @@ -0,0 +1,129 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; +use multiplatform_test::multiplatform_test; + +macro_rules! assert_contains_each_by_tick { + ($results:expr, $tick:expr, &[]) => {{ + assert_eq!($results.borrow().get(&$tick), None); + }}; + ($results:expr, $tick:expr, $input:expr) => {{ + for v in $input { + assert!( + $results.borrow()[&$tick].contains(v), + "did not contain: {:?} in {:?}", + v, + $results.borrow()[&$tick] + ); + } + }}; +} + +#[multiplatform_test] +pub fn tick_tick() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join::<'tick, 'tick>() + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (1, 0)), (7, (2, 0))]); + assert_contains_each_by_tick!(results, 1, &[]); +} + +#[multiplatform_test] +pub fn tick_static() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join::<'tick, 'static>() + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (1, 0)), (7, (2, 0))]); + assert_contains_each_by_tick!(results, 1, &[]); +} + +#[multiplatform_test] +pub fn static_tick() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join::<'static, 'tick>() + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (1, 0)), (7, (2, 0))]); + assert_contains_each_by_tick!(results, 1, &[(7, (1, 1)), (7, (2, 1))]); + assert_contains_each_by_tick!(results, 2, &[(7, (1, 2)), (7, (2, 2))]); + assert_contains_each_by_tick!(results, 3, &[]); +} + +#[multiplatform_test] +pub fn static_static() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join::<'static, 'static>() + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + #[rustfmt::skip] + { + assert_contains_each_by_tick!(results, 0, &[(7, (1, 0)), (7, (2, 0))]); + assert_contains_each_by_tick!(results, 1, &[(7, (1, 0)), (7, (2, 0)), (7, (1, 1)), (7, (2, 1))]); + assert_contains_each_by_tick!(results, 2, &[(7, (1, 0)), (7, (2, 0)), (7, (1, 1)), (7, (2, 1)), (7, (1, 2)), (7, (2, 2))]); + assert_contains_each_by_tick!(results, 3, &[]); + }; +} diff --git a/hydroflow/tests/surface_join_fused.rs b/hydroflow/tests/surface_join_fused.rs new file mode 100644 index 00000000000..a1ffa3fbfd1 --- /dev/null +++ b/hydroflow/tests/surface_join_fused.rs @@ -0,0 +1,211 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use hydroflow::lattices::set_union::SetUnionSingletonSet; +use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; +use lattices::set_union::SetUnionHashSet; +use lattices::Merge; +use multiplatform_test::multiplatform_test; + +macro_rules! assert_contains_each_by_tick { + ($results:expr, $tick:expr, $input:expr) => {{ + for v in $input { + assert!( + $results.borrow()[&$tick].contains(v), + "did not contain: {:?} in {:?}", + v, + $results.borrow()[&$tick] + ); + } + }}; +} + +#[multiplatform_test] +pub fn tick_tick_lhs_blocking_rhs_streaming() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join_fused_lhs(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (SetUnionHashSet::new_from([1, 2]), 0))]); +} + +#[multiplatform_test] +pub fn static_tick_lhs_blocking_rhs_streaming() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join_fused_lhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (SetUnionHashSet::new_from([1, 2]), 0))]); + assert_contains_each_by_tick!(results, 1, &[(7, (SetUnionHashSet::new_from([1, 2]), 1))]); + assert_contains_each_by_tick!(results, 2, &[(7, (SetUnionHashSet::new_from([1, 2]), 2))]); +} + +#[multiplatform_test] +pub fn static_static_lhs_blocking_rhs_streaming() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join_fused_lhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + #[rustfmt::skip] + { + assert_contains_each_by_tick!(results, 0, &[(7, (SetUnionHashSet::new_from([1, 2]), 0))]); + assert_contains_each_by_tick!(results, 1, &[(7, (SetUnionHashSet::new_from([1, 2]), 0)), (7, (SetUnionHashSet::new_from([1, 2]), 1))]); + assert_contains_each_by_tick!(results, 2, &[(7, (SetUnionHashSet::new_from([1, 2]), 0)), (7, (SetUnionHashSet::new_from([1, 2]), 1)), (7, (SetUnionHashSet::new_from([1, 2]), 2))]); + }; +} + +#[multiplatform_test] +pub fn tick_tick_lhs_streaming_rhs_blocking() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [1]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [0]my_join; + + my_join = join_fused_rhs(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (0, (SetUnionHashSet::new_from([1, 2]))))]); +} + +#[multiplatform_test] +pub fn static_tick_lhs_streaming_rhs_blocking() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [1]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [0]my_join; + + my_join = join_fused_rhs::<'static, 'tick>(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (0, SetUnionHashSet::new_from([1, 2])))]); + assert_contains_each_by_tick!(results, 1, &[(7, (1, SetUnionHashSet::new_from([1, 2])))]); + assert_contains_each_by_tick!(results, 2, &[(7, (2, SetUnionHashSet::new_from([1, 2])))]); +} + +#[multiplatform_test] +pub fn static_static_lhs_streaming_rhs_blocking() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [1]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [0]my_join; + + my_join = join_fused_rhs::<'static, 'static>(Fold(SetUnionHashSet::default, Merge::merge)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + #[rustfmt::skip] + { + assert_contains_each_by_tick!(results, 0, &[(7, (0, SetUnionHashSet::new_from([1, 2])))]); + assert_contains_each_by_tick!(results, 1, &[(7, (0, SetUnionHashSet::new_from([1, 2]))), (7, (1, SetUnionHashSet::new_from([1, 2])))]); + assert_contains_each_by_tick!(results, 2, &[(7, (0, SetUnionHashSet::new_from([1, 2]))), (7, (1, SetUnionHashSet::new_from([1, 2]))), (7, (2, SetUnionHashSet::new_from([1, 2])))]); + }; +} + +#[multiplatform_test] +pub fn tick_tick_lhs_fold_rhs_reduce() { + let results = Rc::new(RefCell::new(HashMap::>::new())); + let results_inner = Rc::clone(&results); + + let mut df = hydroflow_syntax! { + source_iter([(7, 1), (7, 2)]) + -> map(|(k, v)| (k, SetUnionSingletonSet::new_from(v))) + -> [0]my_join; + + source_iter([(7, 0)]) -> unioner; + source_iter([(7, 1)]) -> defer_tick() -> unioner; + source_iter([(7, 2)]) -> defer_tick() -> defer_tick() -> unioner; + unioner = union() + -> [1]my_join; + + my_join = join_fused(Fold(SetUnionHashSet::default, Merge::merge), Reduce(std::ops::AddAssign::add_assign)) + -> for_each(|x| results_inner.borrow_mut().entry(context.current_tick()).or_default().push(x)); + }; + assert_graphvis_snapshots!(df); + df.run_available(); + + assert_contains_each_by_tick!(results, 0, &[(7, (SetUnionHashSet::new_from([1, 2]), 0))]); +} diff --git a/hydroflow/tests/surface_lattice_batch.rs b/hydroflow/tests/surface_lattice_batch.rs new file mode 100644 index 00000000000..c04bd0d9857 --- /dev/null +++ b/hydroflow/tests/surface_lattice_batch.rs @@ -0,0 +1,27 @@ +use hydroflow::hydroflow_syntax; +use multiplatform_test::multiplatform_test; + +#[multiplatform_test] +pub fn test_lattice_batch() { + type SetUnionHashSet = lattices::set_union::SetUnionHashSet; + type SetUnionSingletonSet = lattices::set_union::SetUnionSingletonSet; + + let mut df = hydroflow_syntax! { + // Can release in the same tick + source_iter([SetUnionSingletonSet::new_from(0), SetUnionSingletonSet::new_from(1)]) -> [input]b1; + source_iter([()]) -> defer_tick() -> [signal]b1; + b1 = _lattice_fold_batch::() -> assert_eq([SetUnionHashSet::new_from([1, 0])]); + + // Can hold the data across a tick + source_iter([SetUnionSingletonSet::new_from(0), SetUnionSingletonSet::new_from(1)]) -> [input]b2; + source_iter([()]) -> defer_tick() -> [signal]b2; + b2 = _lattice_fold_batch::() -> assert_eq([SetUnionHashSet::new_from([1, 0])]); + + // Doesn't release without a signal + source_iter([SetUnionSingletonSet::new_from(0), SetUnionSingletonSet::new_from(1)]) -> [input]b3; + source_iter([(); 0]) -> [signal]b3; + b3 = _lattice_fold_batch::() -> assert(|_| false); + }; + + df.run_available(); +} diff --git a/hydroflow/tests/surface_lattice_merge.rs b/hydroflow/tests/surface_lattice_fold.rs similarity index 85% rename from hydroflow/tests/surface_lattice_merge.rs rename to hydroflow/tests/surface_lattice_fold.rs index a307c1dc57f..d7fd6ced5b3 100644 --- a/hydroflow/tests/surface_lattice_merge.rs +++ b/hydroflow/tests/surface_lattice_fold.rs @@ -6,7 +6,7 @@ fn test_basic() { let mut df = hydroflow_syntax! { source_iter([1,2,3,4,5]) -> map(Max::new) - -> lattice_merge::<'static, Max>() + -> lattice_fold::<'static, Max>() -> for_each(|x: Max| println!("Least upper bound: {:?}", x)); }; df.run_available(); diff --git a/hydroflow/tests/surface_lattice_join.rs b/hydroflow/tests/surface_lattice_join.rs new file mode 100644 index 00000000000..abd01f36644 --- /dev/null +++ b/hydroflow/tests/surface_lattice_join.rs @@ -0,0 +1,130 @@ +use hydroflow::util::collect_ready; +use hydroflow_macro::hydroflow_syntax; +use multiplatform_test::multiplatform_test; + +#[multiplatform_test] +pub fn test_lattice_join_fused_join_reducing_behavior() { + // lattice_fold has the following particular initialization behavior. It uses LatticeFrom to convert the first element into another lattice type to create the accumulator and then it folds the remaining elements into that accumulator. + // This initialization style supports things like SetUnionSingletonSet -> SetUnionHashSet. It also helps with things like Min, where there is not obvious default value so you want reduce-like behavior. + let mut df = hydroflow_syntax! { + use lattices::Min; + use lattices::Max; + + source_iter([(7, Min::new(5)), (7, Min::new(6))]) -> [0]my_join; + source_iter([(7, Max::new(5)), (7, Max::new(6))]) -> [1]my_join; + + my_join = _lattice_join_fused_join::, Max>() + -> assert_eq([(7, (Min::new(5), Max::new(6)))]); + }; + + df.run_available(); +} + +#[multiplatform_test] +pub fn test_lattice_join_fused_join_set_union() { + let mut df = hydroflow_syntax! { + use lattices::set_union::SetUnionSingletonSet; + use lattices::set_union::SetUnionHashSet; + + source_iter([(7, SetUnionHashSet::new_from([5])), (7, SetUnionHashSet::new_from([6]))]) -> [0]my_join; + source_iter([(7, SetUnionSingletonSet::new_from(5)), (7, SetUnionSingletonSet::new_from(6))]) -> [1]my_join; + + my_join = _lattice_join_fused_join::, SetUnionHashSet>() + -> assert_eq([(7, (SetUnionHashSet::new_from([5, 6]), SetUnionHashSet::new_from([5, 6])))]); + }; + + df.run_available(); +} + +#[multiplatform_test] +pub fn test_lattice_join_fused_join_map_union() { + let mut df = hydroflow_syntax! { + use lattices::map_union::MapUnionSingletonMap; + use lattices::map_union::MapUnionHashMap; + use hydroflow::lattices::Min; + + source_iter([(7, MapUnionHashMap::new_from([(3, Min::new(4))])), (7, MapUnionHashMap::new_from([(3, Min::new(5))]))]) -> [0]my_join; + source_iter([(7, MapUnionSingletonMap::new_from((3, Min::new(5)))), (7, MapUnionSingletonMap::new_from((3, Min::new(4))))]) -> [1]my_join; + + my_join = _lattice_join_fused_join::>, MapUnionHashMap>>() + -> assert_eq([(7, (MapUnionHashMap::new_from([(3, Min::new(4))]), MapUnionHashMap::new_from([(3, Min::new(4))])))]); + }; + + df.run_available(); +} + +#[multiplatform_test] +pub fn test_lattice_join_fused_join() { + use hydroflow::lattices::Max; + + // 'static, 'tick. + { + let (out_tx, mut out_rx) = + hydroflow::util::unbounded_channel::<(usize, (Max, Max))>(); + let (lhs_tx, lhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); + let (rhs_tx, rhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); + + let mut df = hydroflow_syntax! { + my_join = _lattice_join_fused_join::<'static, 'tick, Max, Max>(); + + source_stream(lhs_rx) -> [0]my_join; + source_stream(rhs_rx) -> [1]my_join; + + my_join -> for_each(|v| out_tx.send(v).unwrap()); + }; + + // Merges forward correctly, without going backward + lhs_tx.send((7, Max::new(3))).unwrap(); + lhs_tx.send((7, Max::new(4))).unwrap(); + rhs_tx.send((7, Max::new(6))).unwrap(); + rhs_tx.send((7, Max::new(5))).unwrap(); + + df.run_tick(); + + let out: Vec<_> = collect_ready(&mut out_rx); + assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); + + // Forgets rhs state + rhs_tx.send((7, Max::new(6))).unwrap(); + rhs_tx.send((7, Max::new(5))).unwrap(); + + df.run_tick(); + + let out: Vec<_> = collect_ready(&mut out_rx); + assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); + } + + // 'static, 'static. + { + let (out_tx, mut out_rx) = + hydroflow::util::unbounded_channel::<(usize, (Max, Max))>(); + let (lhs_tx, lhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); + let (rhs_tx, rhs_rx) = hydroflow::util::unbounded_channel::<(usize, Max)>(); + + let mut df = hydroflow_syntax! { + my_join = _lattice_join_fused_join::<'static, 'static, Max, Max>(); + + source_stream(lhs_rx) -> [0]my_join; + source_stream(rhs_rx) -> [1]my_join; + + my_join -> for_each(|v| out_tx.send(v).unwrap()); + }; + + lhs_tx.send((7, Max::new(3))).unwrap(); + lhs_tx.send((7, Max::new(4))).unwrap(); + rhs_tx.send((7, Max::new(6))).unwrap(); + rhs_tx.send((7, Max::new(5))).unwrap(); + + df.run_tick(); + let out: Vec<_> = collect_ready(&mut out_rx); + assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); + + // Doesn't forget + lhs_tx.send((7, Max::new(4))).unwrap(); + rhs_tx.send((7, Max::new(5))).unwrap(); + + df.run_tick(); + let out: Vec<_> = collect_ready(&mut out_rx); + assert_eq!(out, vec![(7, (Max::new(4), Max::new(6)))]); + } +} diff --git a/hydroflow/tests/surface_lattice_reduce.rs b/hydroflow/tests/surface_lattice_reduce.rs new file mode 100644 index 00000000000..70fd4e9d312 --- /dev/null +++ b/hydroflow/tests/surface_lattice_reduce.rs @@ -0,0 +1,13 @@ +use hydroflow::hydroflow_syntax; +use hydroflow::lattices::Max; + +#[test] +fn test_basic() { + let mut df = hydroflow_syntax! { + source_iter([1,2,3,4,5]) + -> map(Max::new) + -> lattice_reduce::<'static, Max>() + -> for_each(|x: Max| println!("Least upper bound: {:?}", x)); + }; + df.run_available(); +} diff --git a/hydroflow/tests/surface_persist.rs b/hydroflow/tests/surface_persist.rs index d8ec91b7cbb..6f661418af5 100644 --- a/hydroflow/tests/surface_persist.rs +++ b/hydroflow/tests/surface_persist.rs @@ -13,7 +13,7 @@ pub fn test_persist_basic() { source_iter([1]) -> persist() -> persist() - -> fold(0, |a, b| (a + b)) + -> fold(0, |a: &mut _, b| *a += b) -> for_each(|x| result_send.send(x).unwrap()); }; assert_graphvis_snapshots!(hf); @@ -23,7 +23,7 @@ pub fn test_persist_basic() { hf.run_tick(); } assert_eq!( - &[1, 3, 6, 10, 15, 21, 28, 36, 45, 55], + &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &*collect_ready::, _>(&mut result_recv) ); } @@ -39,7 +39,7 @@ pub fn test_persist_pull() { m0 = union() -> persist() -> m1; null() -> m1; m1 = union() - -> fold(0, |a, b| (a + b)) + -> fold(0, |a: &mut _, b| *a += b) -> for_each(|x| result_send.send(x).unwrap()); }; assert_graphvis_snapshots!(hf); @@ -49,7 +49,7 @@ pub fn test_persist_pull() { hf.run_tick(); } assert_eq!( - &[1, 3, 6, 10, 15, 21, 28, 36, 45, 55], + &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &*collect_ready::, _>(&mut result_recv) ); } @@ -63,7 +63,7 @@ pub fn test_persist_push() { t0 -> null(); t1 = t0 -> persist() -> tee(); t1 -> null(); - t1 -> fold(0, |a, b| (a + b)) -> for_each(|x| result_send.send(x).unwrap()); + t1 -> fold(0, |a: &mut _, b| *a += b) -> for_each(|x| result_send.send(x).unwrap()); }; assert_graphvis_snapshots!(hf); @@ -72,7 +72,7 @@ pub fn test_persist_push() { hf.run_tick(); } assert_eq!( - &[1, 3, 6, 10, 15, 21, 28, 36, 45, 55], + &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &*collect_ready::, _>(&mut result_recv) ); } @@ -100,7 +100,7 @@ pub fn test_persist_replay_join() { let mut hf = hydroflow_syntax! { source_stream(persist_input) -> persist() - -> fold::<'tick>(0, |a, b| (a + b)) + -> fold::<'tick>(0, |a: &mut _, b| *a += b) -> next_stratum() -> [0]product_node; diff --git a/hydroflow/tests/surface_python.rs b/hydroflow/tests/surface_python.rs new file mode 100644 index 00000000000..2cf3c5942f8 --- /dev/null +++ b/hydroflow/tests/surface_python.rs @@ -0,0 +1,62 @@ +#![cfg(feature = "python")] + +use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; +use multiplatform_test::multiplatform_test; +use pyo3::prelude::*; + +#[multiplatform_test(test)] +pub fn test_python_basic() { + let mut hf = hydroflow_syntax! { + source_iter(0..10) + -> map(|x| (x,)) + -> py_udf(r#" +def fib(n): + if n < 2: + return n + else: + return fib(n - 2) + fib(n - 1) + "#, "fib") + -> map(|x: PyResult>| Python::with_gil(|py| { + usize::extract(x.unwrap().as_ref(py)).unwrap() + })) + -> assert_eq([0, 1, 1, 2, 3, 5, 8, 13, 21, 34]); + }; + assert_graphvis_snapshots!(hf); + + hf.run_available(); +} + +#[multiplatform_test(test)] +pub fn test_python_too_many_args() { + let mut hf = hydroflow_syntax! { + source_iter([(5,)]) + -> py_udf(r#" +def add(a, b): + return a + b + "#, "add") + -> map(PyResult::>::unwrap_err) + -> map(|py_err| py_err.to_string()) + -> assert_eq(["TypeError: add() missing 1 required positional argument: 'b'"]); + }; + assert_graphvis_snapshots!(hf); + + hf.run_available(); +} + +#[multiplatform_test(test)] +pub fn test_python_two_args() { + let mut hf = hydroflow_syntax! { + source_iter([(5,1)]) + -> py_udf(r#" +def add(a, b): + return a + b + "#, "add") + -> map(|x: PyResult>| Python::with_gil(|py| { + usize::extract(x.unwrap().as_ref(py)).unwrap() + })) + -> assert_eq([6]); + }; + assert_graphvis_snapshots!(hf); + + hf.run_available(); +} diff --git a/hydroflow/tests/surface_reduce.rs b/hydroflow/tests/surface_reduce.rs index c0c0598267d..90806bc3a95 100644 --- a/hydroflow/tests/surface_reduce.rs +++ b/hydroflow/tests/surface_reduce.rs @@ -8,7 +8,7 @@ pub fn test_reduce_tick() { let mut df = hydroflow::hydroflow_syntax! { source_stream(items_recv) - -> reduce::<'tick>(|acc: u32, next: u32| acc + next) + -> reduce::<'tick>(|acc: &mut u32, next: u32| *acc += next) -> for_each(|v| result_send.send(v).unwrap()); }; assert_graphvis_snapshots!(df); @@ -42,7 +42,7 @@ pub fn test_reduce_static() { let mut df = hydroflow::hydroflow_syntax! { source_stream(items_recv) - -> reduce::<'static>(|acc: u32, next: u32| acc + next) + -> reduce::<'static>(|acc: &mut u32, next: u32| *acc += next) -> for_each(|v| result_send.send(v).unwrap()); }; assert_graphvis_snapshots!(df); @@ -75,7 +75,7 @@ pub fn test_reduce_sum() { let mut df = hydroflow_syntax! { source_stream(items_recv) - -> reduce(|a, b| a + b) + -> reduce(|a: &mut _, b| *a += b) -> for_each(|v| print!("{:?}", v)); }; assert_graphvis_snapshots!(df); @@ -120,7 +120,7 @@ pub fn test_reduce() { source_stream(pairs_recv) -> [1]my_join_tee; my_join_tee[0] -> [1]reached_vertices; - my_join_tee[1] -> reduce(|a, b| a + b) -> for_each(|sum| println!("{}", sum)); + my_join_tee[1] -> reduce(|a: &mut _, b| *a += b) -> for_each(|sum| println!("{}", sum)); }; assert_graphvis_snapshots!(df); assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); diff --git a/hydroflow/tests/surface_scheduling.rs b/hydroflow/tests/surface_scheduling.rs index 273cb67f440..4638ac941d8 100644 --- a/hydroflow/tests/surface_scheduling.rs +++ b/hydroflow/tests/surface_scheduling.rs @@ -1,6 +1,6 @@ use std::error::Error; -use hydroflow::{hydroflow_syntax, rassert_eq}; +use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax, rassert_eq}; use multiplatform_test::multiplatform_test; #[multiplatform_test(test, wasm, env_tracing)] @@ -13,6 +13,7 @@ pub fn test_stratum_loop() { union_tee -> map(|n| n + 1) -> filter(|&n| n < 10) -> next_stratum() -> union_tee; union_tee -> for_each(|v| out_send.send(v).unwrap()); }; + assert_graphvis_snapshots!(df); df.run_available(); assert_eq!( @@ -29,9 +30,10 @@ pub fn test_tick_loop() { let mut df = hydroflow_syntax! { source_iter([0]) -> union_tee; union_tee = union() -> tee(); - union_tee -> map(|n| n + 1) -> filter(|&n| n < 10) -> next_tick() -> union_tee; + union_tee -> map(|n| n + 1) -> filter(|&n| n < 10) -> defer_tick() -> union_tee; union_tee -> for_each(|v| out_send.send(v).unwrap()); }; + assert_graphvis_snapshots!(df); df.run_available(); assert_eq!( @@ -52,6 +54,7 @@ async fn test_persist_stratum_run_available() -> Result<(), Box> { -> next_stratum() -> for_each(|x| out_send.send(x).unwrap()); }; + assert_graphvis_snapshots!(df); df.run_available(); }); tokio::time::sleep(std::time::Duration::from_millis(200)).await; @@ -79,6 +82,7 @@ async fn test_persist_stratum_run_async() -> Result<(), Box> { -> next_stratum() -> for_each(|x| out_send.send(x).unwrap()); }; + assert_graphvis_snapshots!(df); tokio::time::timeout(std::time::Duration::from_millis(200), df.run_async()) .await diff --git a/hydroflow/tests/surface_state_scheduling.rs b/hydroflow/tests/surface_state_scheduling.rs index 785456b00fb..2a9f867f3a1 100644 --- a/hydroflow/tests/surface_state_scheduling.rs +++ b/hydroflow/tests/surface_state_scheduling.rs @@ -27,7 +27,7 @@ pub fn test_fold_tick() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow_syntax! { - source_iter([1]) -> fold::<'tick>(0, |accum, elem| accum + elem) -> for_each(|v| out_send.send(v).unwrap()); + source_iter([1]) -> fold::<'tick>(0, |accum: &mut _, elem| *accum += elem) -> for_each(|v| out_send.send(v).unwrap()); }; assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); df.run_tick(); @@ -47,7 +47,7 @@ pub fn test_fold_static() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow_syntax! { - source_iter([1]) -> fold::<'static>(0, |accum, elem| accum + elem) -> for_each(|v| out_send.send(v).unwrap()); + source_iter([1]) -> fold::<'static>(0, |accum: &mut _, elem| *accum += elem) -> for_each(|v| out_send.send(v).unwrap()); }; assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); df.run_tick(); @@ -67,7 +67,7 @@ pub fn test_reduce_tick() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow_syntax! { - source_iter([1]) -> reduce::<'tick>(|a, b| a + b) -> for_each(|v| out_send.send(v).unwrap()); + source_iter([1]) -> reduce::<'tick>(|a: &mut _, b| *a += b) -> for_each(|v| out_send.send(v).unwrap()); }; assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); df.run_tick(); @@ -87,7 +87,7 @@ pub fn test_reduce_static() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow_syntax! { - source_iter([1]) -> reduce::<'static>(|a, b| a + b) -> for_each(|v| out_send.send(v).unwrap()); + source_iter([1]) -> reduce::<'static>(|a: &mut _, b| *a += b) -> for_each(|v| out_send.send(v).unwrap()); }; assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); df.run_tick(); @@ -153,7 +153,7 @@ pub fn test_resume_external_event() { let (out_send, mut out_recv) = hydroflow::util::unbounded_channel::(); let mut df = hydroflow_syntax! { - source_stream(in_recv) -> fold::<'static>(0, >::add) -> for_each(|v| out_send.send(v).unwrap()); + source_stream(in_recv) -> fold::<'static>(0, |a: &mut _, b| *a += b) -> for_each(|v| out_send.send(v).unwrap()); }; assert_eq!((0, 0), (df.current_tick(), df.current_stratum())); diff --git a/hydroflow/tests/surface_stratum.rs b/hydroflow/tests/surface_stratum.rs index 032808e0f87..b0bc344bd1e 100644 --- a/hydroflow/tests/surface_stratum.rs +++ b/hydroflow/tests/surface_stratum.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::rc::Rc; use hydroflow::scheduled::graph::Hydroflow; +use hydroflow::util::multiset::HashMultiSet; use hydroflow::{assert_graphvis_snapshots, hydroflow_syntax}; use multiplatform_test::multiplatform_test; use tokio::sync::mpsc::error::SendError; @@ -19,19 +20,19 @@ use tokio::sync::mpsc::error::SendError; /// Basic difference test, test difference between two one-off iterators. #[multiplatform_test] pub fn test_difference_a() { - let output = >>>::default(); + let output = >>>::default(); let output_inner = Rc::clone(&output); let mut df: Hydroflow = hydroflow_syntax! { a = difference(); source_iter([1, 2, 3, 4]) -> [pos]a; source_iter([1, 3, 5, 7]) -> [neg]a; - a -> for_each(|x| output_inner.borrow_mut().push(x)); + a -> for_each(|x| output_inner.borrow_mut().insert(x)); }; assert_graphvis_snapshots!(df); df.run_available(); - assert_eq!(&[2, 4], &*output.take()); + assert_eq!(HashMultiSet::from_iter([2, 4]), output.take()); } /// More complex different test. @@ -40,15 +41,15 @@ pub fn test_difference_a() { pub fn test_difference_b() -> Result<(), SendError<&'static str>> { let (inp_send, inp_recv) = hydroflow::util::unbounded_channel::<&'static str>(); - let output = >>>::default(); + let output = >>>::default(); let output_inner = Rc::clone(&output); let mut df: Hydroflow = hydroflow_syntax! { a = difference(); source_stream(inp_recv) -> [pos]a; b = a -> tee(); - b[0] -> next_tick() -> [neg]a; - b[1] -> for_each(|x| output_inner.borrow_mut().push(x)); + b[0] -> defer_tick() -> [neg]a; + b[1] -> for_each(|x| output_inner.borrow_mut().insert(x)); }; assert_graphvis_snapshots!(df); @@ -56,19 +57,19 @@ pub fn test_difference_b() -> Result<(), SendError<&'static str>> { inp_send.send("02")?; inp_send.send("03")?; df.run_tick(); - assert_eq!(&["01", "02", "03"], &*output.take()); + assert_eq!(HashMultiSet::from_iter(["01", "02", "03"]), output.take()); inp_send.send("02")?; inp_send.send("11")?; inp_send.send("12")?; df.run_tick(); - assert_eq!(&["11", "12"], &*output.take()); + assert_eq!(HashMultiSet::from_iter(["11", "12"]), output.take()); inp_send.send("02")?; inp_send.send("11")?; inp_send.send("12")?; df.run_tick(); - assert_eq!(&["02"], &*output.take()); + assert_eq!(HashMultiSet::from_iter(["02"]), output.take()); Ok(()) } @@ -78,12 +79,12 @@ pub fn test_tick_loop_1() { let output = >>>::default(); let output_inner = Rc::clone(&output); - // Without `next_tick()` this would be "unsafe" although legal. + // Without `defer_tick()` this would be "unsafe" although legal. // E.g. it would spin forever in a single infinite tick/tick. let mut df: Hydroflow = hydroflow_syntax! { a = union() -> tee(); source_iter([1, 3]) -> [0]a; - a[0] -> next_tick() -> map(|x| 2 * x) -> [1]a; + a[0] -> defer_tick() -> map(|x| 2 * x) -> [1]a; a[1] -> for_each(|x| output_inner.borrow_mut().push(x)); }; assert_graphvis_snapshots!(df); @@ -109,7 +110,7 @@ pub fn test_tick_loop_2() { let mut df: Hydroflow = hydroflow_syntax! { a = union() -> tee(); source_iter([1, 3]) -> [0]a; - a[0] -> next_tick() -> next_tick() -> map(|x| 2 * x) -> [1]a; + a[0] -> defer_tick() -> defer_tick() -> map(|x| 2 * x) -> [1]a; a[1] -> for_each(|x| output_inner.borrow_mut().push(x)); }; assert_graphvis_snapshots!(df); @@ -138,7 +139,7 @@ pub fn test_tick_loop_3() { let mut df: Hydroflow = hydroflow_syntax! { a = union() -> tee(); source_iter([1, 3]) -> [0]a; - a[0] -> next_tick() -> next_tick() -> next_tick() -> map(|x| 2 * x) -> [1]a; + a[0] -> defer_tick() -> defer_tick() -> defer_tick() -> map(|x| 2 * x) -> [1]a; a[1] -> for_each(|x| output_inner.borrow_mut().push(x)); }; assert_graphvis_snapshots!(df); diff --git a/hydroflow/tests/surface_warnings.rs b/hydroflow/tests/surface_warnings.rs index 351d2e2a744..ae71f41c5ae 100644 --- a/hydroflow/tests/surface_warnings.rs +++ b/hydroflow/tests/surface_warnings.rs @@ -1,45 +1,23 @@ +// TODO(mingwei): fix line numbers in tests +// https://github.com/hydro-project/hydroflow/issues/729 +// https://github.com/rust-lang/rust/pull/111571 + use std::cell::RefCell; use std::rc::Rc; +use hydroflow::hydroflow_expect_warnings; use hydroflow::scheduled::graph::Hydroflow; use hydroflow::util::collect_ready; -macro_rules! test_warnings { - ( - $hf:tt, - $( $msg:literal ),* - $( , )? - ) => { - { - let __file = std::file!(); - let __line = std::line!() as usize; - let __hf = hydroflow::hydroflow_syntax_noemit! $hf; - - let diagnostics = __hf.diagnostics().expect("Expected `diagnostics()` to be set."); - let expecteds = &[ - $( $msg , )* - ]; - assert_eq!(diagnostics.len(), expecteds.len(), "Wrong number of diagnostics."); - for (expected, diagnostic) in expecteds.iter().zip(diagnostics.iter()) { - let mut diagnostic = diagnostic.clone(); - diagnostic.span.line -= __line; - assert_eq!(expected.to_string(), diagnostic.to_string().replace(__file, "$FILE")); - } - - __hf - } - }; -} - #[test] fn test_degenerate_union() { let (result_send, mut result_recv) = hydroflow::util::unbounded_channel::(); - let mut df = test_warnings! { + let mut df = hydroflow_expect_warnings! { { source_iter([1, 2, 3]) -> union() -> for_each(|x| result_send.send(x).unwrap()); }, - "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:2:39", + "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:0:0", }; df.run_available(); @@ -48,11 +26,11 @@ fn test_degenerate_union() { #[test] fn test_empty_union() { - let mut df = test_warnings! { + let mut df = hydroflow_expect_warnings! { { union() -> for_each(|x: usize| println!("{}", x)); }, - "Warning: `union` should have at least 2 input(s), actually has 0.\n --> $FILE:2:13", + "Warning: `union` should have at least 2 input(s), actually has 0.\n --> $FILE:0:0", }; df.run_available(); } @@ -61,11 +39,11 @@ fn test_empty_union() { fn test_degenerate_tee() { let (result_send, mut result_recv) = hydroflow::util::unbounded_channel::(); - let mut df = test_warnings! { + let mut df = hydroflow_expect_warnings! { { source_iter([1, 2, 3]) -> tee() -> for_each(|x| result_send.send(x).unwrap()); }, - "Warning: `tee` should have at least 2 output(s), actually has 1.\n --> $FILE:2:39" + "Warning: `tee` should have at least 2 output(s), actually has 1.\n --> $FILE:0:0" }; df.run_available(); @@ -77,11 +55,11 @@ fn test_empty_tee() { let output = >>>::default(); let output_inner = Rc::clone(&output); - let mut df = test_warnings! { + let mut df = hydroflow_expect_warnings! { { source_iter([1, 2, 3]) -> inspect(|&x| output_inner.borrow_mut().push(x)) -> tee(); }, - "Warning: `tee` should have at least 2 output(s), actually has 0.\n --> $FILE:2:90", + "Warning: `tee` should have at least 2 output(s), actually has 0.\n --> $FILE:0:0", }; df.run_available(); @@ -92,7 +70,7 @@ fn test_empty_tee() { // But in a different edge order. #[test] pub fn test_warped_diamond() { - let mut df = test_warnings! { + let mut df = hydroflow_expect_warnings! { { // active nodes nodes = union(); @@ -108,14 +86,14 @@ pub fn test_warped_diamond() { nodes -> [0]init; new_node[1] -> map(|n| (n, 'b')) -> [1]init; }, - "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:3:21", + "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:0:0", }; df.run_available(); } #[test] pub fn test_warped_diamond_2() { - let mut hf: Hydroflow = test_warnings! { + let mut hf: Hydroflow = hydroflow_expect_warnings! { { // active nodes nodes = union(); @@ -133,8 +111,8 @@ pub fn test_warped_diamond_2() { ntwk = source_iter([4, 5, 6]) -> tee(); }, - "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:3:21", - "Warning: `tee` should have at least 2 output(s), actually has 0.\n --> $FILE:16:46", + "Warning: `union` should have at least 2 input(s), actually has 1.\n --> $FILE:0:0", + "Warning: `tee` should have at least 2 output(s), actually has 0.\n --> $FILE:0:0", }; hf.run_available(); } diff --git a/hydroflow/tests/test.rs b/hydroflow/tests/test.rs index 40d50609360..0049347334e 100644 --- a/hydroflow/tests/test.rs +++ b/hydroflow/tests/test.rs @@ -151,7 +151,7 @@ fn test_cycle() { (6, 7), (7, 8), ] { - edges.entry(from).or_insert_with(Vec::new).push(to); + edges.entry(from).or_default().push(to); } let mut df = Hydroflow::new(); @@ -181,7 +181,7 @@ fn test_cycle() { union_rhs, union_out, |_ctx, recv1, recv2, send| { - for v in (recv1.take_inner().into_iter()).chain(recv2.take_inner().into_iter()) { + for v in (recv1.take_inner().into_iter()).chain(recv2.take_inner()) { send.give(Some(v)); } }, diff --git a/hydroflow_cli_integration/CHANGELOG.md b/hydroflow_cli_integration/CHANGELOG.md index 303bb66b819..d3503458db6 100644 --- a/hydroflow_cli_integration/CHANGELOG.md +++ b/hydroflow_cli_integration/CHANGELOG.md @@ -5,8 +5,35 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.3.0 (2023-07-04) + +### Bug Fixes + + - remove nightly feature `never_type` where unused + +### Commit Statistics + + + + - 1 commit contributed to the release over the course of 13 calendar days. + - 33 days passed between releases. + - 1 commit was understood as [conventional](https://www.conventionalcommits.org). + - 1 unique issue was worked on: [#780](https://github.com/hydro-project/hydroflow/issues/780) + +### Commit Details + + + +
view details + + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Remove nightly feature `never_type` where unused ([`a3c1fbb`](https://github.com/hydro-project/hydroflow/commit/a3c1fbbd1e3fa7a7299878f61b4bfd12dce0052c)) +
+ ## 0.2.0 (2023-05-31) + + ### Chore - manually bump versions for v0.2.0 release @@ -19,7 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 2 commits contributed to the release. + - 3 commits contributed to the release. - 2 days passed between releases. - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - 1 unique issue was worked on: [#723](https://github.com/hydro-project/hydroflow/issues/723) @@ -33,6 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#723](https://github.com/hydro-project/hydroflow/issues/723)** - Add more detailed Hydro Deploy docs and rename `ConnectedBidi` => `ConnectedDirect` ([`8b2c9f0`](https://github.com/hydro-project/hydroflow/commit/8b2c9f09b1f423ac6d562c29d4ea587578f1c98a)) * **Uncategorized** + - Release hydroflow_cli_integration v0.2.0 ([`71d940b`](https://github.com/hydro-project/hydroflow/commit/71d940b1b2ca379543f28c4e827f895fc5efa4bd)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a))
diff --git a/hydroflow_cli_integration/Cargo.toml b/hydroflow_cli_integration/Cargo.toml index ce2bf441102..df89b11d0c6 100644 --- a/hydroflow_cli_integration/Cargo.toml +++ b/hydroflow_cli_integration/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydroflow_cli_integration" publish = true -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow_cli_integration/" diff --git a/hydroflow_datalog/CHANGELOG.md b/hydroflow_datalog/CHANGELOG.md index e2980858021..312cece2260 100644 --- a/hydroflow_datalog/CHANGELOG.md +++ b/hydroflow_datalog/CHANGELOG.md @@ -5,8 +5,73 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +Unchanged from previous release. + +### Chore + + - mark hydro_datalog as unchanged for 0.4 release + +### Commit Statistics + + + + - 1 commit contributed to the release. + - 42 days passed between releases. + - 1 commit was understood as [conventional](https://www.conventionalcommits.org). + - 0 issues like '(#ID)' were seen in commit messages + +### Commit Details + + + +
view details + + * **Uncategorized** + - Mark hydro_datalog as unchanged for 0.4 release ([`5faee64`](https://github.com/hydro-project/hydroflow/commit/5faee64ab82eeb7a24f62a1b55c46d72d8eb5320)) +
+ +## 0.3.0 (2023-07-04) + +### New Features + + - allow stable build, refactors behind `nightly` feature flag + +### Bug Fixes + + - update proc-macro2, use new span location API where possible + requires latest* rust nightly version + + *latest = 2023-06-28 or something + +### Commit Statistics + + + + - 3 commits contributed to the release over the course of 12 calendar days. + - 33 days passed between releases. + - 2 commits were understood as [conventional](https://www.conventionalcommits.org). + - 2 unique issues were worked on: [#780](https://github.com/hydro-project/hydroflow/issues/780), [#801](https://github.com/hydro-project/hydroflow/issues/801) + +### Commit Details + + + +
view details + + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Allow stable build, refactors behind `nightly` feature flag ([`22abcaf`](https://github.com/hydro-project/hydroflow/commit/22abcaff806c7de6e4a7725656bbcf201e7d9259)) + * **[#801](https://github.com/hydro-project/hydroflow/issues/801)** + - Update proc-macro2, use new span location API where possible ([`8d3494b`](https://github.com/hydro-project/hydroflow/commit/8d3494b5afee858114a602a3e23077bb6d24dd77)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) +
+ ## 0.2.0 (2023-05-31) + + ### Chore - manually bump versions for v0.2.0 release @@ -15,8 +80,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 1 commit contributed to the release. - - 8 days passed between releases. + - 2 commits contributed to the release. + - 7 days passed between releases. - 1 commit was understood as [conventional](https://www.conventionalcommits.org). - 0 issues like '(#ID)' were seen in commit messages @@ -27,6 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
view details * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a))
diff --git a/hydroflow_datalog/Cargo.toml b/hydroflow_datalog/Cargo.toml index 49d4cdce87b..9d9f528dab6 100644 --- a/hydroflow_datalog/Cargo.toml +++ b/hydroflow_datalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydroflow_datalog" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow_datalog/" @@ -17,9 +17,9 @@ diagnostics = [ "hydroflow_datalog_core/diagnostics" ] [dependencies] quote = "1.0.0" syn = { version = "2.0.0", features = [ "parsing", "extra-traits" ] } -proc-macro2 = "1.0.27" +proc-macro2 = "1.0.57" proc-macro-crate = "1.1.0" # Note: If we ever compile this proc macro crate to WASM (e.g., if we are # building on a WASM host), we may need to turn diagnostics off for WASM if # proc_macro2 does not support WASM at that time. -hydroflow_datalog_core = { path = "../hydroflow_datalog_core", version = "^0.2.0" } +hydroflow_datalog_core = { path = "../hydroflow_datalog_core", version = "^0.4.0" } diff --git a/hydroflow_datalog_core/CHANGELOG.md b/hydroflow_datalog_core/CHANGELOG.md index 891a2b6c6e0..1f375bcefbd 100644 --- a/hydroflow_datalog_core/CHANGELOG.md +++ b/hydroflow_datalog_core/CHANGELOG.md @@ -5,8 +5,117 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +### New Features + + - Add `use` statements to hydroflow syntax + And use in doc tests. + +### Bug Fixes + + - unify antijoin and difference with set and multiset semantics + * fix: unify antijoin and difference with set and multiset semantics + + * fix: replay semantics for antijoin and difference now work + also added cross_join_multiset + + * fix: enforce sort for tests of anti_join and difference using assert_eq + + * fix: advance __borrow_ident beyond the current tick to prevent replay loops + + * fix: add modified snapshots + + * fix: temp + + * fix: spelling typo in comment + + * fix: make anti_join replay more efficient + + Also add multiset data structure, use it in some tests, make join() + replay logic more similar to anti_join's and presist's. + + * fix: ignore test that depends on order of antijoin + + * fix: really ignore test_index + + * fix: fix specific test ordering in wasm + + --------- + - joins now replay correctly + - rename next_tick -> defer, batch -> defer_signal + +### Refactor + + - fix new clippy lints on latest nightly 1.73.0-nightly (db7ff98a7 2023-07-31) + +### Commit Statistics + + + + - 5 commits contributed to the release over the course of 32 calendar days. + - 42 days passed between releases. + - 5 commits were understood as [conventional](https://www.conventionalcommits.org). + - 4 unique issues were worked on: [#833](https://github.com/hydro-project/hydroflow/issues/833), [#845](https://github.com/hydro-project/hydroflow/issues/845), [#870](https://github.com/hydro-project/hydroflow/issues/870), [#872](https://github.com/hydro-project/hydroflow/issues/872) + +### Commit Details + + + +
view details + + * **[#833](https://github.com/hydro-project/hydroflow/issues/833)** + - Rename next_tick -> defer, batch -> defer_signal ([`6c98bbc`](https://github.com/hydro-project/hydroflow/commit/6c98bbc2bd3443fe6f77e0b8689b461edde1b316)) + * **[#845](https://github.com/hydro-project/hydroflow/issues/845)** + - Add `use` statements to hydroflow syntax ([`b4b9644`](https://github.com/hydro-project/hydroflow/commit/b4b9644a19e8e7e7725c9c5b88e3a6b8c2be7364)) + * **[#870](https://github.com/hydro-project/hydroflow/issues/870)** + - Joins now replay correctly ([`cc959c7`](https://github.com/hydro-project/hydroflow/commit/cc959c762c3a0e036e672801c615028cbfb95168)) + * **[#872](https://github.com/hydro-project/hydroflow/issues/872)** + - Unify antijoin and difference with set and multiset semantics ([`d378e5e`](https://github.com/hydro-project/hydroflow/commit/d378e5eada3d2bae90f98c5a33b2d055940a8c7f)) + * **Uncategorized** + - Fix new clippy lints on latest nightly 1.73.0-nightly (db7ff98a7 2023-07-31) ([`6a2ad6b`](https://github.com/hydro-project/hydroflow/commit/6a2ad6b770c2ccf470548320d8753025b3a66c0a)) +
+ +## 0.3.0 (2023-07-04) + +### New Features + + - allow stable build, refactors behind `nightly` feature flag + +### Bug Fixes + + - update proc-macro2, use new span location API where possible + requires latest* rust nightly version + + *latest = 2023-06-28 or something + +### Commit Statistics + + + + - 3 commits contributed to the release over the course of 12 calendar days. + - 33 days passed between releases. + - 2 commits were understood as [conventional](https://www.conventionalcommits.org). + - 2 unique issues were worked on: [#780](https://github.com/hydro-project/hydroflow/issues/780), [#801](https://github.com/hydro-project/hydroflow/issues/801) + +### Commit Details + + + +
view details + + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Allow stable build, refactors behind `nightly` feature flag ([`22abcaf`](https://github.com/hydro-project/hydroflow/commit/22abcaff806c7de6e4a7725656bbcf201e7d9259)) + * **[#801](https://github.com/hydro-project/hydroflow/issues/801)** + - Update proc-macro2, use new span location API where possible ([`8d3494b`](https://github.com/hydro-project/hydroflow/commit/8d3494b5afee858114a602a3e23077bb6d24dd77)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) +
+ ## 0.2.0 (2023-05-31) + + ### Chore - manually bump versions for v0.2.0 release @@ -15,8 +124,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 1 commit contributed to the release. - - 2 days passed between releases. + - 2 commits contributed to the release. + - 1 day passed between releases. - 1 commit was understood as [conventional](https://www.conventionalcommits.org). - 0 issues like '(#ID)' were seen in commit messages @@ -27,6 +136,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
view details * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a))
diff --git a/hydroflow_datalog_core/Cargo.toml b/hydroflow_datalog_core/Cargo.toml index e4dd23c7ba5..87e091c960b 100644 --- a/hydroflow_datalog_core/Cargo.toml +++ b/hydroflow_datalog_core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydroflow_datalog_core" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow_datalog_core/" @@ -18,10 +18,10 @@ diagnostics = [ "hydroflow_lang/diagnostics" ] quote = "1.0.0" slotmap = "1.0.6" syn = { version = "2.0.0", features = [ "parsing", "extra-traits" ] } -proc-macro2 = "1.0.27" +proc-macro2 = "1.0.57" proc-macro-crate = "1.1.0" rust-sitter = "0.3.2" -hydroflow_lang = { path = "../hydroflow_lang", version = "^0.2.0" } +hydroflow_lang = { path = "../hydroflow_lang", version = "^0.4.0" } [build-dependencies] rust-sitter-tool = "0.3.2" diff --git a/hydroflow_datalog_core/src/join_plan.rs b/hydroflow_datalog_core/src/join_plan.rs index 2c3bbdeff8c..5434c6b468a 100644 --- a/hydroflow_datalog_core/src/join_plan.rs +++ b/hydroflow_datalog_core/src/join_plan.rs @@ -131,11 +131,9 @@ fn emit_join_input_pipeline( let statement = match source_expanded.tee_idx { Some(i) => { let in_index = syn::LitInt::new(&format!("{}", i), Span::call_site()); - parse_quote_spanned!(source_expanded.span=> #source_name [#in_index] -> #rhs) - } - None => { - parse_quote_spanned!(source_expanded.span=> #source_name -> #rhs) + parse_quote_spanned! {source_expanded.span=> #source_name [#in_index] -> #rhs; } } + None => parse_quote_spanned! {source_expanded.span=> #source_name -> #rhs; }, }; flat_graph_builder.add_statement(statement); @@ -155,10 +153,10 @@ fn find_relation_local_constraints<'a>( let mut indices_grouped_by_var = BTreeMap::new(); for (i, ident) in fields.enumerate() { if let IdentOrUnderscore::Ident(ident) = ident.deref() { - let entry = indices_grouped_by_var + let entry: &mut Vec<_> = indices_grouped_by_var // TODO(shadaj): Can we avoid cloning here? .entry(ident.name.clone()) - .or_insert_with(Vec::new); + .or_default(); entry.push(i); } } @@ -286,7 +284,7 @@ pub fn expand_join_plan( let conditions = build_local_constraint_conditions(&local_constraints); flat_graph_builder.add_statement(parse_quote_spanned! {get_span(rule_span)=> - #filter_node = #relation_node [#relation_idx] -> filter(|row: &#row_type| #conditions) + #filter_node = #relation_node [#relation_idx] -> filter(|row: &#row_type| #conditions); }); IntermediateJoinNode { @@ -454,11 +452,14 @@ pub fn expand_join_plan( if is_anti { // this is always a 'tick join, so we place a persist operator in the join input pipeline - flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(rule_span)=> #join_node = anti_join() -> map(#flatten_closure))); + flat_graph_builder.add_statement(parse_quote_spanned! {get_span(rule_span)=> + #join_node = anti_join() -> map(#flatten_closure); + }); } else { flat_graph_builder.add_statement( - parse_quote_spanned!(get_span(rule_span)=> #join_node = join::<#lt_left, #lt_right, hydroflow::compiled::pull::HalfMultisetJoinState>() -> map(#flatten_closure)), + parse_quote_spanned! {get_span(rule_span)=> + #join_node = join::<#lt_left, #lt_right, hydroflow::compiled::pull::HalfMultisetJoinState>() -> map(#flatten_closure); + } ); } @@ -561,7 +562,7 @@ pub fn expand_join_plan( ); flat_graph_builder.add_statement(parse_quote_spanned! { get_span(rule_span)=> - #predicate_filter_node = #inner_name -> filter(|row: &#row_type| #conditions ) + #predicate_filter_node = #inner_name -> filter(|row: &#row_type| #conditions ); }); IntermediateJoinNode { @@ -595,7 +596,7 @@ pub fn expand_join_plan( let inner_name = inner_expanded.name.clone(); let row_type = inner_expanded.tuple_type; - if let IdentOrUnderscore::Ident(less_than) = less_than.deref() { + if let IdentOrUnderscore::Ident(less_than) = less_than { if inner_expanded .variable_mapping .contains_key(&less_than.name) @@ -604,7 +605,7 @@ pub fn expand_join_plan( } } - let threshold_name = if let IdentOrUnderscore::Ident(threshold) = threshold.deref() { + let threshold_name = if let IdentOrUnderscore::Ident(threshold) = threshold { threshold.name.clone() } else { panic!("The threshold must be a variable") @@ -632,7 +633,7 @@ pub fn expand_join_plan( flattened_elements.push(parse_quote!(row.#syn_wildcard_idx.clone())); } - if let IdentOrUnderscore::Ident(less_than) = less_than.deref() { + if let IdentOrUnderscore::Ident(less_than) = less_than { if less_than.name == threshold_name { panic!("The threshold and less_than variables must be different") } @@ -645,7 +646,7 @@ pub fn expand_join_plan( flattened_elements.push(parse_quote!(v)); flat_graph_builder.add_statement(parse_quote_spanned! {get_span(rule_span)=> - #magic_node = #inner_name -> flat_map(|row: #row_type| (0..(row.#threshold_index)).map(move |v| (#(#flattened_elements, )*)) ) + #magic_node = #inner_name -> flat_map(|row: #row_type| (0..(row.#threshold_index)).map(move |v| (#(#flattened_elements, )*)) ); }); IntermediateJoinNode { diff --git a/hydroflow_datalog_core/src/lib.rs b/hydroflow_datalog_core/src/lib.rs index 5b932ef9ea8..1215e85074d 100644 --- a/hydroflow_datalog_core/src/lib.rs +++ b/hydroflow_datalog_core/src/lib.rs @@ -3,12 +3,15 @@ use std::ops::Deref; use hydroflow_lang::diagnostic::{Diagnostic, Level}; use hydroflow_lang::graph::{ - eliminate_extra_unions_tees, partition_graph, FlatGraphBuilder, HydroflowGraph, + eliminate_extra_unions_tees, partition_graph, propegate_flow_props, FlatGraphBuilder, + HydroflowGraph, +}; +use hydroflow_lang::parse::{ + HfStatement, IndexInt, Indexing, Pipeline, PipelineLink, PipelineStatement, PortIndex, }; -use hydroflow_lang::parse::{IndexInt, Indexing, Pipeline, PipelineLink}; use proc_macro2::{Span, TokenStream}; use rust_sitter::errors::{ParseError, ParseErrorReason}; -use syn::{parse_quote, parse_quote_spanned}; +use syn::{parse_quote, parse_quote_spanned, Token}; mod grammar; mod join_plan; @@ -143,18 +146,18 @@ pub fn gen_hydroflow_graph( if persists.contains(&target_ident.value.name) { // read outputs the *new* values for this tick flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #insert_name = union() -> unique::<'tick>())); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #insert_name = union() -> unique::<'tick>(); }); flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #read_name = difference::<'tick, 'static>() -> tee())); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #read_name = difference::<'tick, 'static>() -> tee(); }); flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #insert_name -> [pos] #read_name)); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #insert_name -> [pos] #read_name; }); flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #read_name -> next_tick() -> [neg] #read_name)); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #read_name -> defer_tick() -> [neg] #read_name; }); } else { flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #insert_name = union() -> unique::<'tick>())); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #insert_name = union() -> unique::<'tick>(); }); flat_graph_builder - .add_statement(parse_quote_spanned!(get_span(target_ident.span)=> #read_name = #insert_name -> tee())); + .add_statement(parse_quote_spanned!{get_span(target_ident.span)=> #read_name = #insert_name -> tee(); }); } } } @@ -173,7 +176,7 @@ pub fn gen_hydroflow_graph( let input_pipeline: Pipeline = parse_pipeline(&hf_code.code, &get_span)?; flat_graph_builder.add_statement(parse_quote_spanned! {get_span(target.span)=> - #input_pipeline -> [#my_union_index_lit] #name + #input_pipeline -> [#my_union_index_lit] #name; }); } @@ -196,7 +199,7 @@ pub fn gen_hydroflow_graph( }; flat_graph_builder.add_statement(parse_quote_spanned! {get_span(target.span)=> - #target_ident [#my_tee_index_lit] -> #output_pipeline + #target_ident [#my_tee_index_lit] -> #output_pipeline; }); } @@ -219,11 +222,11 @@ pub fn gen_hydroflow_graph( let recv_pipeline: Pipeline = parse_pipeline(&recv_hf.code, &get_span)?; flat_graph_builder.add_statement(parse_quote_spanned! {get_span(target.span)=> - #async_send_pipeline = union() -> unique::<'tick>() -> #send_pipeline + #async_send_pipeline = union() -> unique::<'tick>() -> #send_pipeline; }); flat_graph_builder.add_statement(parse_quote_spanned! {get_span(target.span)=> - #recv_pipeline -> [#recv_union_index_lit] #target_ident + #recv_pipeline -> [#recv_union_index_lit] #target_ident; }); } @@ -241,7 +244,7 @@ pub fn gen_hydroflow_graph( let static_expression: syn::Expr = parse_static(&hf_code.code, &get_span)?; flat_graph_builder.add_statement(parse_quote_spanned! {get_span(target.span)=> - source_iter(#static_expression) -> persist() -> [#my_union_index_lit] #name + source_iter(#static_expression) -> persist() -> [#my_union_index_lit] #name; }); } @@ -263,17 +266,22 @@ pub fn gen_hydroflow_graph( } if !diagnostics.is_empty() { - Err(diagnostics) - } else { - let (mut flat_graph, mut diagnostics) = flat_graph_builder.build(); - diagnostics.retain(Diagnostic::is_error); - if !diagnostics.is_empty() { - Err(diagnostics) - } else { - eliminate_extra_unions_tees(&mut flat_graph); - Ok(flat_graph) - } + return Err(diagnostics); } + + let (mut flat_graph, _uses, mut diagnostics) = flat_graph_builder.build(); + diagnostics.retain(Diagnostic::is_error); + if !diagnostics.is_empty() { + return Err(diagnostics); + } + + if let Err(err) = flat_graph.merge_modules() { + diagnostics.push(err); + return Err(diagnostics); + } + + eliminate_extra_unions_tees(&mut flat_graph); + Ok(flat_graph) } fn handle_errors( @@ -317,11 +325,15 @@ fn handle_errors( } pub fn hydroflow_graph_to_program(flat_graph: HydroflowGraph, root: TokenStream) -> TokenStream { - let partitioned_graph = + let mut partitioned_graph = partition_graph(flat_graph).expect("Failed to partition (cycle detected)."); let mut diagnostics = Vec::new(); - let code_tokens = partitioned_graph.as_code(&root, true, &mut diagnostics); + // Propgeate flow properties throughout the graph. + // TODO(mingwei): Should this be done at a flat graph stage instead? + let _ = propegate_flow_props::propegate_flow_props(&mut partitioned_graph, &mut diagnostics); + + let code_tokens = partitioned_graph.as_code(&root, true, quote::quote!(), &mut diagnostics); assert_eq!( 0, diagnostics.len(), @@ -385,7 +397,7 @@ fn generate_rule( panic!("Rule must be async to send data to other nodes") } - parse_quote_spanned!(get_span(rule.rule_type.span)=> #after_join -> next_tick() -> [#my_union_index_lit] #target_ident) + parse_quote_spanned!(get_span(rule.rule_type.span)=> #after_join -> defer_tick() -> [#my_union_index_lit] #target_ident) } RuleType::Async(_) => { if rule.target.at_node.is_none() { @@ -423,18 +435,19 @@ fn generate_rule( // directly outputting a transformation of a single relation on the RHS. let out_indexing = out_expanded.tee_idx.map(|i| Indexing { bracket_token: syn::token::Bracket::default(), - index: hydroflow_lang::parse::PortIndex::Int(IndexInt { + index: PortIndex::Int(IndexInt { value: i, span: Span::call_site(), }), }); - flat_graph_builder.add_statement(hydroflow_lang::parse::HfStatement::Pipeline( - Pipeline::Link(PipelineLink { + flat_graph_builder.add_statement(HfStatement::Pipeline(PipelineStatement { + pipeline: Pipeline::Link(PipelineLink { lhs: Box::new(parse_quote!(#out_name #out_indexing)), // out_name[idx] arrow: parse_quote!(->), rhs: Box::new(after_join_and_send), }), - )); + semi_token: Token![;](Span::call_site()), + })); } fn compute_join_plan<'a>(sources: &'a [Atom], persisted_rules: &HashSet) -> JoinPlan<'a> { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@datalog_program.snap index 8e829596d97..c38a0a51bee 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () . 1 , g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + val . 0) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Operator\":\"identity ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":24,\"version\":1}],\"version\":5},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":7,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":3},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":2,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () . 1 , g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + val . 0) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Operator\":\"identity ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":24,\"version\":1}],\"version\":5},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":7,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":3},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":2,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"star_ord\":16,\"lattice_flow_type\":null},\"version\":1}]}", ); df.__assign_diagnostics("[]"); let (hoff_1v3_send, hoff_1v3_recv) = df @@ -360,6 +360,7 @@ fn main() { iter } for kv in check_input(hoff_9v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_1v1_node_14v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -463,7 +464,7 @@ fn main() { let op_19v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_19v1__next_tick__loc_unknown_start_8_30_end_8_32< + pub fn op_19v1__defer_tick__loc_unknown_start_8_30_end_8_32< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -489,7 +490,7 @@ fn main() { } Pull { inner: input } } - op_19v1__next_tick__loc_unknown_start_8_30_end_8_32(op_19v1) + op_19v1__defer_tick__loc_unknown_start_8_30_end_8_32(op_19v1) }; let op_4v1 = { #[allow(unused)] @@ -661,6 +662,7 @@ fn main() { iter } for kv in check_input(hoff_23v1_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_2v1_node_21v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -862,6 +864,7 @@ fn main() { iter } for kv in check_input(hoff_1v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_3v1_node_17v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@surface_graph.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@surface_graph.snap index fecd9caddc0..4cf2329abb9 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@surface_graph.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_and_comments@surface_graph.snap @@ -16,7 +16,7 @@ expression: flat_graph_ref.surface_syntax_string() 16v1 = map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,))); 17v1 = fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + val . 0) } else { Some (val . 0) } ; }); 18v1 = map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,)); -19v1 = next_tick (); +19v1 = defer_tick (); 20v1 = map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,))); 21v1 = fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev) } else { Some (val . 0) } ; }); 22v1 = map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,)); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_fold_keyed_expr@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_fold_keyed_expr@datalog_program.snap index 5ddf9998d22..03d2c9cf8e3 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_fold_keyed_expr@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__aggregations_fold_keyed_expr@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 % 2 ,) , (row . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + val . 0) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 , a . 0 . unwrap () ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 % 2 ,) , (row . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + val . 0) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 , a . 0 . unwrap () ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -219,6 +219,7 @@ fn main() { iter } for kv in check_input(hoff_6v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_1v1_node_10v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__anti_join@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__anti_join@datalog_program.snap index c4f4157d073..856c2aa76a7 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__anti_join@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__anti_join@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints_1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints_2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints_3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"anti_join ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (_ , _ ,)) | (kv . 0 . 0 , kv . 1 . 0 , kv . 1 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ , _ ,) | ((_v . 0 ,) , (_v . 1 , _v . 2 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (_v . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 1 , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":5},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints_1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints_2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints_3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"anti_join ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (_ , _ ,)) | (kv . 0 . 0 , kv . 1 . 0 , kv . 1 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ , _ ,) | ((_v . 0 ,) , (_v . 1 , _v . 2 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (_v . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 1 , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":5},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -381,15 +381,28 @@ fn main() { ), ), ); - let sg_1v1_node_21v1_diffdata_handle = df + let sg_1v1_node_17v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); + let sg_1v1_node_21v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), + ), + ); + let sg_1v1_node_21v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( hydroflow::util::monotonic_map::MonotonicMap::< _, hydroflow::rustc_hash::FxHashSet<_>, >::default(), ), ); + let sg_1v1_node_21v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_1v1_node_11v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -581,6 +594,9 @@ fn main() { let mut sg_1v1_node_17v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_17v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_17v1_persisttick_borrow = context + .state_ref(sg_1v1_node_17v1_persisttick) + .borrow_mut(); let op_17v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -596,6 +612,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -604,21 +621,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_17v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_17v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_19v1, + op_20v1, + &mut *sg_1v1_node_17v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_17v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_19v1, - op_20v1, - &mut *sg_1v1_node_17v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_17v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_17v1 = { #[allow(non_snake_case)] @@ -716,34 +746,55 @@ fn main() { } op_23v1__map__loc_unknown_start_7_28_end_7_54(op_23v1) }; - let mut sg_1v1_node_21v1_borrow = context - .state_ref(sg_1v1_node_21v1_diffdata_handle) + let mut sg_1v1_node_21v1_antijoindata_neg_borrow = context + .state_ref(sg_1v1_node_21v1_antijoindata_neg) + .borrow_mut(); + let mut sg_1v1_node_21v1_antijoindata_pos_borrow = context + .state_ref(sg_1v1_node_21v1_antijoindata_pos) .borrow_mut(); let op_21v1 = { /// Limit error propagation by bounding locally, erasing output iterator type. #[inline(always)] fn check_inputs<'a, K, I1, V, I2>( - input_pos: I1, - input_neg: I2, - borrow_state: &'a mut hydroflow::rustc_hash::FxHashSet, + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + ::std::hash::Hash + Clone, - V: Eq + Clone, - I1: 'a + Iterator, - I2: 'a + Iterator, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, { - borrow_state.extend(input_neg); - input_pos.filter(move |x| !borrow_state.contains(&x.0)) + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_1v1_node_21v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; check_inputs( - op_23v1, hoff_12v3_recv, - sg_1v1_node_21v1_borrow - .get_mut_clear(( - context.current_tick(), - context.current_stratum(), - )), + op_23v1, + &mut *sg_1v1_node_21v1_antijoindata_neg_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_21v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) }; let op_21v1 = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_lhs@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_lhs@datalog_program.snap index dba05cc60c3..18256c92405 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_lhs@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_lhs@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 + 123 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 . clone () + row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 - row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 % (row . 0 + 5) ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 * 5 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"2\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"3\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"3\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"4\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"4\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"5\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"5\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 + 123 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 . clone () + row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 - row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((123 % (row . 0 + 5) ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 * 5 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"2\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"3\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"3\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"4\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"4\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"5\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"5\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"star_ord\":21,\"lattice_flow_type\":null},\"version\":1}]}", ); df.__assign_diagnostics("[]"); let (hoff_1v3_send, hoff_1v3_recv) = df diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_predicate@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_predicate@datalog_program.snap index d47ebeef19c..65d0629501a 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_predicate@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__expr_predicate@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 == 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 != 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 - 1 == 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((3 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 - 1 == 1 - 1)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((4 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"2\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"3\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_0_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_1_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_2_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_3_filter\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 == 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 != 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 - 1 == 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((3 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ ,) | row . 0 - 1 == 1 - 1)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((4 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"2\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"3\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_0_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_1_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_2_filter\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_3_filter\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"star_ord\":19,\"lattice_flow_type\":null},\"version\":1}]}", ); df.__assign_diagnostics("[]"); let (hoff_1v3_send, hoff_1v3_recv) = df diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@datalog_program.snap index 3e9512bd006..1e1326756b3 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result3 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result4 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result5 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 ,) , ((row . 1) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ ,) , _)) | (g . 0 , a . 0 . unwrap () . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 ,) , ((row . 1) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'static , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ ,) , _)) | (g . 0 , a . 0 . unwrap () . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":35,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":25,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":39,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":45,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":49,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":3},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":52,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":15,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":29,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":31,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":34,\"version\":1},{\"idx\":35,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1},{\"idx\":34,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":33,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":38,\"version\":1},{\"idx\":39,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":37,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":36,\"version\":1},{\"idx\":21,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":36,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":3},{\"idx\":37,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":40,\"version\":1},{\"idx\":16,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":3},{\"idx\":51,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":44,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":42,\"version\":1},{\"idx\":43,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":42,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":3},{\"idx\":47,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":48,\"version\":1},{\"idx\":49,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":46,\"version\":1},{\"idx\":13,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":46,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":3},{\"idx\":41,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":50,\"version\":1},{\"idx\":10,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":25,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":33,\"version\":1},{\"idx\":34,\"version\":1},{\"idx\":35,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":36,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":37,\"version\":1},{\"idx\":38,\"version\":1},{\"idx\":39,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":42,\"version\":1},{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1},{\"idx\":45,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":29,\"version\":1},{\"idx\":46,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1},{\"idx\":49,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result4_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result5_insert\",\"version\":1},{\"value\":\"result5\",\"version\":1},{\"value\":\"result5\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_persisted_insert\",\"version\":1},{\"value\":\"ints_persisted\",\"version\":1},{\"value\":\"ints_persisted\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result3 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result4 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result5 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 ,) , ((row . 1) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ ,) , _)) | (g . 0 , a . 0 . unwrap () . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 ,) , ((row . 1) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'static , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ ,) , _)) | (g . 0 , a . 0 . unwrap () . 1 , i ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"enumerate :: < 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (i , (g , a)) : (_ , ((_ , _ ,) , _)) | (g . 0 , g . 1 , i ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":35,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":25,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":39,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":23,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":45,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":49,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":3},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":52,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":15,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":29,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":31,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":34,\"version\":1},{\"idx\":35,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1},{\"idx\":34,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":33,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":38,\"version\":1},{\"idx\":39,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":37,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":36,\"version\":1},{\"idx\":21,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":36,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":3},{\"idx\":37,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":40,\"version\":1},{\"idx\":16,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":3},{\"idx\":51,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":44,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":42,\"version\":1},{\"idx\":43,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":42,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":3},{\"idx\":47,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":48,\"version\":1},{\"idx\":49,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":46,\"version\":1},{\"idx\":13,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":46,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":3},{\"idx\":41,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":50,\"version\":1},{\"idx\":10,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":25,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":33,\"version\":1},{\"idx\":34,\"version\":1},{\"idx\":35,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":36,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":37,\"version\":1},{\"idx\":38,\"version\":1},{\"idx\":39,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":42,\"version\":1},{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1},{\"idx\":45,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":29,\"version\":1},{\"idx\":46,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1},{\"idx\":49,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":\"ints\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result4_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result5_insert\",\"version\":1},{\"value\":\"result5\",\"version\":1},{\"value\":\"result5\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_persisted_insert\",\"version\":1},{\"value\":\"ints_persisted\",\"version\":1},{\"value\":\"ints_persisted\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -518,7 +518,7 @@ fn main() { let op_20v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_20v1__next_tick__loc_unknown_start_9_21_end_9_28< + pub fn op_20v1__defer_tick__loc_unknown_start_9_21_end_9_28< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -544,7 +544,7 @@ fn main() { } Pull { inner: input } } - op_20v1__next_tick__loc_unknown_start_9_21_end_9_28(op_20v1) + op_20v1__defer_tick__loc_unknown_start_9_21_end_9_28(op_20v1) }; #[inline(always)] fn check_pivot_run< @@ -581,7 +581,7 @@ fn main() { let op_25v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_25v1__next_tick__loc_unknown_start_15_21_end_15_35< + pub fn op_25v1__defer_tick__loc_unknown_start_15_21_end_15_35< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -607,7 +607,7 @@ fn main() { } Pull { inner: input } } - op_25v1__next_tick__loc_unknown_start_15_21_end_15_35(op_25v1) + op_25v1__defer_tick__loc_unknown_start_15_21_end_15_35(op_25v1) }; #[inline(always)] fn check_pivot_run< @@ -662,6 +662,7 @@ fn main() { iter } for kv in check_input(hoff_21v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_4v1_node_37v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -900,12 +901,21 @@ fn main() { >::default(), ), ); - let sg_5v1_node_23v1_diffdata_handle = df + let sg_5v1_node_23v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_5v1_node_23v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_5v1_node_23v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_5v1_node_11v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -1028,12 +1038,58 @@ fn main() { } op_22v1__unique__loc_unknown_start_15_21_end_15_35(op_22v1) }; - let mut sg_5v1_node_23v1_negset = context - .state_ref(sg_5v1_node_23v1_diffdata_handle) + let op_22v1 = op_22v1.map(|k| (k, ())); + let mut sg_5v1_node_23v1_antijoindata_neg_borrow = context + .state_ref(sg_5v1_node_23v1_antijoindata_neg) + .borrow_mut(); + let mut sg_5v1_node_23v1_antijoindata_pos_borrow = context + .state_ref(sg_5v1_node_23v1_antijoindata_pos) .borrow_mut(); - sg_5v1_node_23v1_negset.extend(hoff_9v3_recv); - let op_23v1 = op_22v1 - .filter(move |x| !sg_5v1_node_23v1_negset.contains(x)); + let op_23v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_5v1_node_23v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_9v3_recv, + op_22v1, + &mut *sg_5v1_node_23v1_antijoindata_neg_borrow, + &mut *sg_5v1_node_23v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_23v1 = op_23v1.map(|(k, ())| k); let op_23v1 = { #[allow(non_snake_case)] #[inline(always)] @@ -1475,6 +1531,7 @@ fn main() { iter } for kv in check_input(hoff_13v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_6v1_node_47v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -1725,12 +1782,21 @@ fn main() { >::default(), ), ); - let sg_7v1_node_18v1_diffdata_handle = df + let sg_7v1_node_18v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_7v1_node_18v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_7v1_node_18v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_7v1_node_31v1_persistdata = df .add_state(::std::cell::RefCell::new((0_usize, ::std::vec::Vec::new()))); df.add_subgraph_stratified( @@ -1873,12 +1939,58 @@ fn main() { } op_17v1__unique__loc_unknown_start_9_21_end_9_28(op_17v1) }; - let mut sg_7v1_node_18v1_negset = context - .state_ref(sg_7v1_node_18v1_diffdata_handle) + let op_17v1 = op_17v1.map(|k| (k, ())); + let mut sg_7v1_node_18v1_antijoindata_neg_borrow = context + .state_ref(sg_7v1_node_18v1_antijoindata_neg) + .borrow_mut(); + let mut sg_7v1_node_18v1_antijoindata_pos_borrow = context + .state_ref(sg_7v1_node_18v1_antijoindata_pos) .borrow_mut(); - sg_7v1_node_18v1_negset.extend(hoff_15v3_recv); - let op_18v1 = op_17v1 - .filter(move |x| !sg_7v1_node_18v1_negset.contains(x)); + let op_18v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_7v1_node_18v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_15v3_recv, + op_17v1, + &mut *sg_7v1_node_18v1_antijoindata_neg_borrow, + &mut *sg_7v1_node_18v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_18v1 = op_18v1.map(|(k, ())| k); let op_18v1 = { #[allow(non_snake_case)] #[inline(always)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@surface_graph.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@surface_graph.snap index 4dddd968805..05a16e19c72 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@surface_graph.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__index@surface_graph.snap @@ -11,11 +11,11 @@ expression: flat_graph_ref.surface_syntax_string() 17v1 = unique :: < 'tick > (); 18v1 = difference :: < 'tick , 'static > (); 19v1 = tee (); -20v1 = next_tick (); +20v1 = defer_tick (); 22v1 = unique :: < 'tick > (); 23v1 = difference :: < 'tick , 'static > (); 24v1 = tee (); -25v1 = next_tick (); +25v1 = defer_tick (); 26v1 = source_stream (ints); 27v1 = for_each (| v | result . send (v) . unwrap ()); 28v1 = for_each (| v | result2 . send (v) . unwrap ()); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_other@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_other@datalog_program.snap index 899a555a8bc..02f5fb6a9c8 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_other@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_other@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ , _ ,) , (() , ())) | (kv . 0 . 0 , kv . 0 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 , _v . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 , _v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ , _ ,) , (() , ())) | (kv . 0 . 0 , kv . 0 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 , _v . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 , _v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_10v1_stream = { @@ -76,6 +76,8 @@ fn main() { ), ), ); + let sg_1v1_node_13v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_1v1_node_8v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -337,6 +339,9 @@ fn main() { let mut sg_1v1_node_13v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_13v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_13v1_persisttick_borrow = context + .state_ref(sg_1v1_node_13v1_persisttick) + .borrow_mut(); let op_13v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -352,6 +357,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -360,21 +366,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_13v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_13v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_15v1, + op_16v1, + &mut *sg_1v1_node_13v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_13v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_15v1, - op_16v1, - &mut *sg_1v1_node_13v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_13v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_13v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_self@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_self@datalog_program.snap index 126cab8872b..334bb2dfe3f 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_self@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__join_with_self@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ , _ ,) , (() , ())) | (kv . 0 . 0 , kv . 0 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 , _v . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 , _v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":\"input\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ , _ ,) , (() , ())) | (kv . 0 . 0 , kv . 0 . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 , _v . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 , _v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":\"input\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_4v3_send, hoff_4v3_recv) = df @@ -210,6 +210,8 @@ fn main() { ), ), ); + let sg_2v1_node_9v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_2v1_node_5v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -299,6 +301,9 @@ fn main() { let mut sg_2v1_node_9v1_joindata_rhs_borrow = context .state_ref(sg_2v1_node_9v1_joindata_rhs) .borrow_mut(); + let mut sg_2v1_node_9v1_persisttick_borrow = context + .state_ref(sg_2v1_node_9v1_persisttick) + .borrow_mut(); let op_9v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -314,6 +319,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -322,21 +328,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_2v1_node_9v1_persisttick_borrow + <= context.current_tick() + { + *sg_2v1_node_9v1_persisttick_borrow = context.current_tick() + + 1; + true + } else { + false + }; + check_inputs( + op_11v1, + op_12v1, + &mut *sg_2v1_node_9v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_2v1_node_9v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_11v1, - op_12v1, - &mut *sg_2v1_node_9v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_2v1_node_9v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_9v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program-2.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program-2.snap index 67aa47da38b..b40c03611f8 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program-2.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program-2.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ , _ , _ ,) | row . 0 == row . 1 && row . 2 == row . 3)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ , _ ,) | ((row . 0 . clone () , row . 0 , row . 2 . clone () , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 , g . 3 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0_filter\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ , _ , _ ,) | row . 0 == row . 1 && row . 2 == row . 3)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ , _ ,) | ((row . 0 . clone () , row . 0 , row . 2 . clone () , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 , g . 3 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0_filter\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_7v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program.snap index bb2e3806db3..4ef74d7f0cb 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__local_constraints@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ ,) | row . 0 == row . 1)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 . clone () , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0_filter\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ ,) | row . 0 == row . 1)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 . clone () , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0_filter\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_7v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max@datalog_program.snap index c7a8ced02dd..83587ca8a1f 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (std :: cmp :: max (prev , val . 0)) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , (row . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , (_ ,) , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (std :: cmp :: max (prev , val . 0)) } else { Some (val . 0) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (a . 0 . unwrap () , g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -219,6 +219,7 @@ fn main() { iter } for kv in check_input(hoff_6v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_1v1_node_10v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max_all@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max_all@datalog_program.snap index 11c330207fb..1345f723266 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max_all@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__max_all@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , (row . 0 , row . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > , Option < _ > ,) > (| | (None , None ,) , | old : & mut (Option < _ > , Option < _ > ,) , val : (_ , _ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (std :: cmp :: max (prev , val . 0)) } else { Some (val . 0) } ; old . 1 = if let Some (prev) = old . 1 . take () { Some (std :: cmp :: max (prev , val . 1)) } else { Some (val . 1) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () , a . 1 . unwrap () ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , (row . 0 , row . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > , Option < _ > ,) > (| | (None , None ,) , | old : & mut (Option < _ > , Option < _ > ,) , val : (_ , _ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (std :: cmp :: max (prev , val . 0)) } else { Some (val . 0) } ; old . 1 = if let Some (prev) = old . 1 . take () { Some (std :: cmp :: max (prev , val . 1)) } else { Some (val . 1) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () , a . 1 . unwrap () ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -222,6 +222,7 @@ fn main() { iter } for kv in check_input(hoff_6v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_1v1_node_10v1_hashtable .entry(kv.0) .or_insert_with(|| (None, None)); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__minimal_program@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__minimal_program@datalog_program.snap index 28d3e1afde5..4ded8204b79 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__minimal_program@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__minimal_program@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_7v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__multiple_contributors@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__multiple_contributors@datalog_program.snap index a7889d7b416..c69abb67c28 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__multiple_contributors@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__multiple_contributors@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":\"out_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":\"out_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"star_ord\":8,\"lattice_flow_type\":null},\"version\":1}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_10v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__non_copy_but_clone@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__non_copy_but_clone@datalog_program.snap index 5733d1df8f5..0d30d26eaa9 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__non_copy_but_clone@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__non_copy_but_clone@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (strings)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 . clone () , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"strings_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (strings)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 . clone () , row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"strings_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_7v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@datalog_program.snap index 9f36d0dd7e4..388cecba0c4 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result3 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result4 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'static , 'static , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : (() , ((_ ,) , (_ ,))) | (kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'static , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : (() , ((_ , _ ,) , (_ ,))) | (kv . 1 . 0 . 0 , kv . 1 . 0 . 1 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | (() , (_v . 0 , _v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 0 , row . 1 , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"anti_join ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ()) | (kv . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (_v . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":34,\"version\":1},{\"idx\":14,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":28,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":25,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":35,\"version\":1},{\"idx\":11,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":22,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":19,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":36,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":13,\"version\":3},{\"idx\":33,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":50,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":3},{\"idx\":31,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":58,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":19,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":63,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":22,\"version\":3},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":68,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":61,\"version\":1},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":28,\"version\":3},{\"idx\":3,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":65,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":30,\"version\":1},{\"idx\":31,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1},{\"idx\":16,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":32,\"version\":1},{\"idx\":13,\"version\":3}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":3},{\"idx\":7,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":37,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":39,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":42,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":43,\"version\":1},{\"idx\":41,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":29,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":44,\"version\":1},{\"idx\":41,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":26,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":45,\"version\":1},{\"idx\":46,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":42,\"version\":1},{\"idx\":47,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":48,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":48,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":3},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":49,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":46,\"version\":1},{\"idx\":49,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":54,\"version\":1},{\"idx\":51,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":53,\"version\":1},{\"idx\":54,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":23,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":56,\"version\":1},{\"idx\":20,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":55,\"version\":1},{\"idx\":56,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":55,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":3},{\"idx\":65,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":57,\"version\":1},{\"idx\":58,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":52,\"version\":1},{\"idx\":57,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":3},{\"idx\":44,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":60,\"version\":1},{\"idx\":61,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":59,\"version\":1},{\"idx\":60,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":59,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":3},{\"idx\":51,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":62,\"version\":1},{\"idx\":63,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":27,\"version\":1},{\"idx\":62,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":29,\"version\":3},{\"idx\":43,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":64,\"version\":1},{\"idx\":17,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":64,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":3},{\"idx\":53,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":67,\"version\":1},{\"idx\":68,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":66,\"version\":1},{\"idx\":67,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":32,\"version\":1},{\"idx\":66,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"3\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":9,\"version\":1},\"version\":1},{\"value\":{\"idx\":10,\"version\":1},\"version\":1},{\"value\":{\"idx\":11,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1},{\"idx\":41,\"version\":1},{\"idx\":42,\"version\":1},{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1},{\"idx\":45,\"version\":1},{\"idx\":46,\"version\":1},{\"idx\":49,\"version\":1},{\"idx\":50,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":37,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":53,\"version\":1},{\"idx\":54,\"version\":1},{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1},{\"idx\":57,\"version\":1},{\"idx\":58,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":59,\"version\":1},{\"idx\":60,\"version\":1},{\"idx\":61,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":62,\"version\":1},{\"idx\":63,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":39,\"version\":1},{\"idx\":64,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":65,\"version\":1},{\"idx\":30,\"version\":1},{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1},{\"idx\":66,\"version\":1},{\"idx\":67,\"version\":1},{\"idx\":68,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":55,\"version\":1},{\"idx\":56,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":34,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":35,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":36,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result4_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"intermediate_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"intermediate_persist_insert\",\"version\":1},{\"value\":\"intermediate_persist\",\"version\":1},{\"value\":\"intermediate_persist\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_2\",\"version\":1},{\"value\":\"join_2\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result3 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result4 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'static , 'static , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : (() , ((_ ,) , (_ ,))) | (kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'static , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : (() , ((_ , _ ,) , (_ ,))) | (kv . 1 . 0 . 0 , kv . 1 . 0 . 1 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | (() , (_v . 0 , _v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (() , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 0 , row . 1 , row . 2 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"anti_join ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ()) | (kv . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | (_v . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"persist ()\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":34,\"version\":1},{\"idx\":14,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":28,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":25,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":35,\"version\":1},{\"idx\":11,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":22,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":19,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":36,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":13,\"version\":3},{\"idx\":33,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":50,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":3},{\"idx\":31,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":58,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":19,\"version\":3},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":63,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":22,\"version\":3},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":68,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":61,\"version\":1},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":28,\"version\":3},{\"idx\":3,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":65,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":30,\"version\":1},{\"idx\":31,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1},{\"idx\":16,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":32,\"version\":1},{\"idx\":13,\"version\":3}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":3},{\"idx\":7,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":37,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":39,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":41,\"version\":1},{\"idx\":42,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":43,\"version\":1},{\"idx\":41,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":29,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":44,\"version\":1},{\"idx\":41,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":26,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":45,\"version\":1},{\"idx\":46,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":47,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":42,\"version\":1},{\"idx\":47,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":48,\"version\":1},{\"idx\":45,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":48,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":14,\"version\":3},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":49,\"version\":1},{\"idx\":50,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":46,\"version\":1},{\"idx\":49,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":54,\"version\":1},{\"idx\":51,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":53,\"version\":1},{\"idx\":54,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":23,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":56,\"version\":1},{\"idx\":20,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":55,\"version\":1},{\"idx\":56,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":55,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":3},{\"idx\":65,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":57,\"version\":1},{\"idx\":58,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":52,\"version\":1},{\"idx\":57,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":3},{\"idx\":44,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":60,\"version\":1},{\"idx\":61,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":59,\"version\":1},{\"idx\":60,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":59,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":20,\"version\":3},{\"idx\":51,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":62,\"version\":1},{\"idx\":63,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":27,\"version\":1},{\"idx\":62,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":29,\"version\":3},{\"idx\":43,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":64,\"version\":1},{\"idx\":17,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":64,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":3},{\"idx\":53,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":67,\"version\":1},{\"idx\":68,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":66,\"version\":1},{\"idx\":67,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":32,\"version\":1},{\"idx\":66,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"2\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"3\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":9,\"version\":1},\"version\":1},{\"value\":{\"idx\":10,\"version\":1},\"version\":1},{\"value\":{\"idx\":11,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":8,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1},{\"value\":{\"idx\":7,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":33,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":43,\"version\":1},{\"idx\":44,\"version\":1},{\"idx\":41,\"version\":1},{\"idx\":42,\"version\":1},{\"idx\":47,\"version\":1},{\"idx\":48,\"version\":1},{\"idx\":45,\"version\":1},{\"idx\":46,\"version\":1},{\"idx\":49,\"version\":1},{\"idx\":50,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":37,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":53,\"version\":1},{\"idx\":54,\"version\":1},{\"idx\":51,\"version\":1},{\"idx\":52,\"version\":1},{\"idx\":57,\"version\":1},{\"idx\":58,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":38,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":59,\"version\":1},{\"idx\":60,\"version\":1},{\"idx\":61,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":62,\"version\":1},{\"idx\":63,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":39,\"version\":1},{\"idx\":64,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":65,\"version\":1},{\"idx\":30,\"version\":1},{\"idx\":31,\"version\":1},{\"idx\":32,\"version\":1},{\"idx\":66,\"version\":1},{\"idx\":67,\"version\":1},{\"idx\":68,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":40,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":55,\"version\":1},{\"idx\":56,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":34,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":35,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":36,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result4_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"intermediate_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"intermediate_persist_insert\",\"version\":1},{\"value\":\"intermediate_persist\",\"version\":1},{\"value\":\"intermediate_persist\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_2\",\"version\":1},{\"value\":\"join_2\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -337,7 +337,7 @@ fn main() { let op_5v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_5v1__next_tick__loc_unknown_start_2_19_end_2_24< + pub fn op_5v1__defer_tick__loc_unknown_start_2_19_end_2_24< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -363,7 +363,7 @@ fn main() { } Pull { inner: input } } - op_5v1__next_tick__loc_unknown_start_2_19_end_2_24(op_5v1) + op_5v1__defer_tick__loc_unknown_start_2_19_end_2_24(op_5v1) }; #[inline(always)] fn check_pivot_run< @@ -400,7 +400,7 @@ fn main() { let op_10v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_10v1__next_tick__loc_unknown_start_5_19_end_5_24< + pub fn op_10v1__defer_tick__loc_unknown_start_5_19_end_5_24< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -426,7 +426,7 @@ fn main() { } Pull { inner: input } } - op_10v1__next_tick__loc_unknown_start_5_19_end_5_24(op_10v1) + op_10v1__defer_tick__loc_unknown_start_5_19_end_5_24(op_10v1) }; #[inline(always)] fn check_pivot_run< @@ -463,7 +463,7 @@ fn main() { let op_33v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_33v1__next_tick__loc_unknown_start_21_21_end_21_41< + pub fn op_33v1__defer_tick__loc_unknown_start_21_21_end_21_41< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -489,7 +489,7 @@ fn main() { } Pull { inner: input } } - op_33v1__next_tick__loc_unknown_start_21_21_end_21_41(op_33v1) + op_33v1__defer_tick__loc_unknown_start_21_21_end_21_41(op_33v1) }; #[inline(always)] fn check_pivot_run< @@ -523,6 +523,8 @@ fn main() { hydroflow::compiled::pull::HalfMultisetJoinState::default(), ), ); + let sg_4v1_node_41v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_4v1_node_45v1_joindata_lhs = df .add_state( std::cell::RefCell::new( @@ -537,6 +539,8 @@ fn main() { ), ), ); + let sg_4v1_node_45v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_4v1_node_15v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -680,6 +684,9 @@ fn main() { let mut sg_4v1_node_41v1_joindata_rhs_borrow = context .state_ref(sg_4v1_node_41v1_joindata_rhs) .borrow_mut(); + let mut sg_4v1_node_41v1_persisttick_borrow = context + .state_ref(sg_4v1_node_41v1_persisttick) + .borrow_mut(); let op_41v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -695,6 +702,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -703,19 +711,32 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_4v1_node_41v1_persisttick_borrow + <= context.current_tick() + { + *sg_4v1_node_41v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_43v1, + op_44v1, + &mut *sg_4v1_node_41v1_joindata_lhs_borrow, + &mut *sg_4v1_node_41v1_joindata_rhs_borrow, + __is_new_tick, ) } - check_inputs( - op_43v1, - op_44v1, - &mut *sg_4v1_node_41v1_joindata_lhs_borrow, - &mut *sg_4v1_node_41v1_joindata_rhs_borrow, - ) }; let op_41v1 = { #[allow(non_snake_case)] @@ -851,6 +872,9 @@ fn main() { let mut sg_4v1_node_45v1_joindata_rhs_borrow = context .state_ref(sg_4v1_node_45v1_joindata_rhs) .borrow_mut(); + let mut sg_4v1_node_45v1_persisttick_borrow = context + .state_ref(sg_4v1_node_45v1_persisttick) + .borrow_mut(); let op_45v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -866,6 +890,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -874,20 +899,33 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_4v1_node_45v1_persisttick_borrow + <= context.current_tick() + { + *sg_4v1_node_45v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_47v1, + op_48v1, + &mut *sg_4v1_node_45v1_joindata_lhs_borrow, + &mut *sg_4v1_node_45v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_47v1, - op_48v1, - &mut *sg_4v1_node_45v1_joindata_lhs_borrow, - &mut *sg_4v1_node_45v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_45v1 = { #[allow(non_snake_case)] @@ -1107,19 +1145,32 @@ fn main() { hydroflow::pusherator::pivot::Pivot::new(pull, push).run(); } check_pivot_run(op_15v1, op_37v1); + context.schedule_subgraph(context.current_subgraph(), false); + context.schedule_subgraph(context.current_subgraph(), false); }, ); let sg_5v1_node_53v1_persistdata = df .add_state(::std::cell::RefCell::new((0_usize, ::std::vec::Vec::new()))); - let sg_5v1_node_51v1_diffdata_handle = df + let sg_5v1_node_51v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), + ), + ); + let sg_5v1_node_51v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( hydroflow::util::monotonic_map::MonotonicMap::< _, hydroflow::rustc_hash::FxHashSet<_>, >::default(), ), ); + let sg_5v1_node_51v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_5v1_node_18v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -1220,34 +1271,55 @@ fn main() { } op_54v1__map__loc_unknown_start_16_26_end_16_34(op_54v1) }; - let mut sg_5v1_node_51v1_borrow = context - .state_ref(sg_5v1_node_51v1_diffdata_handle) + let mut sg_5v1_node_51v1_antijoindata_neg_borrow = context + .state_ref(sg_5v1_node_51v1_antijoindata_neg) + .borrow_mut(); + let mut sg_5v1_node_51v1_antijoindata_pos_borrow = context + .state_ref(sg_5v1_node_51v1_antijoindata_pos) .borrow_mut(); let op_51v1 = { /// Limit error propagation by bounding locally, erasing output iterator type. #[inline(always)] fn check_inputs<'a, K, I1, V, I2>( - input_pos: I1, - input_neg: I2, - borrow_state: &'a mut hydroflow::rustc_hash::FxHashSet, + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + ::std::hash::Hash + Clone, - V: Eq + Clone, - I1: 'a + Iterator, - I2: 'a + Iterator, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, { - borrow_state.extend(input_neg); - input_pos.filter(move |x| !borrow_state.contains(&x.0)) + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_5v1_node_51v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; check_inputs( - op_54v1, hoff_20v3_recv, - sg_5v1_node_51v1_borrow - .get_mut_clear(( - context.current_tick(), - context.current_stratum(), - )), + op_54v1, + &mut *sg_5v1_node_51v1_antijoindata_neg_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_5v1_node_51v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) }; let op_51v1 = { @@ -1478,12 +1550,21 @@ fn main() { >::default(), ), ); - let sg_6v1_node_3v1_diffdata_handle = df + let sg_6v1_node_3v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_6v1_node_3v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_6v1_node_3v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_6v1_node_21v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -1590,12 +1671,58 @@ fn main() { } op_2v1__unique__loc_unknown_start_2_19_end_2_24(op_2v1) }; - let mut sg_6v1_node_3v1_negset = context - .state_ref(sg_6v1_node_3v1_diffdata_handle) + let op_2v1 = op_2v1.map(|k| (k, ())); + let mut sg_6v1_node_3v1_antijoindata_neg_borrow = context + .state_ref(sg_6v1_node_3v1_antijoindata_neg) + .borrow_mut(); + let mut sg_6v1_node_3v1_antijoindata_pos_borrow = context + .state_ref(sg_6v1_node_3v1_antijoindata_pos) .borrow_mut(); - sg_6v1_node_3v1_negset.extend(hoff_28v3_recv); - let op_3v1 = op_2v1 - .filter(move |x| !sg_6v1_node_3v1_negset.contains(x)); + let op_3v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_6v1_node_3v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_28v3_recv, + op_2v1, + &mut *sg_6v1_node_3v1_antijoindata_neg_borrow, + &mut *sg_6v1_node_3v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_3v1 = op_3v1.map(|(k, ())| k); let op_3v1 = { #[allow(non_snake_case)] #[inline(always)] @@ -2053,12 +2180,21 @@ fn main() { >::default(), ), ); - let sg_7v1_node_31v1_diffdata_handle = df + let sg_7v1_node_31v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_7v1_node_31v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_7v1_node_31v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_7v1_node_24v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -2169,12 +2305,58 @@ fn main() { } op_30v1__unique__loc_unknown_start_21_21_end_21_41(op_30v1) }; - let mut sg_7v1_node_31v1_negset = context - .state_ref(sg_7v1_node_31v1_diffdata_handle) + let op_30v1 = op_30v1.map(|k| (k, ())); + let mut sg_7v1_node_31v1_antijoindata_neg_borrow = context + .state_ref(sg_7v1_node_31v1_antijoindata_neg) .borrow_mut(); - sg_7v1_node_31v1_negset.extend(hoff_16v3_recv); - let op_31v1 = op_30v1 - .filter(move |x| !sg_7v1_node_31v1_negset.contains(x)); + let mut sg_7v1_node_31v1_antijoindata_pos_borrow = context + .state_ref(sg_7v1_node_31v1_antijoindata_pos) + .borrow_mut(); + let op_31v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_7v1_node_31v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_16v3_recv, + op_30v1, + &mut *sg_7v1_node_31v1_antijoindata_neg_borrow, + &mut *sg_7v1_node_31v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_31v1 = op_31v1.map(|(k, ())| k); let op_31v1 = { #[allow(non_snake_case)] #[inline(always)] @@ -2476,12 +2658,21 @@ fn main() { >::default(), ), ); - let sg_8v1_node_8v1_diffdata_handle = df + let sg_8v1_node_8v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_8v1_node_8v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_8v1_node_8v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); let sg_8v1_node_55v1_persistdata = df .add_state(::std::cell::RefCell::new((0_usize, ::std::vec::Vec::new()))); df.add_subgraph_stratified( @@ -2561,12 +2752,58 @@ fn main() { } op_7v1__unique__loc_unknown_start_5_19_end_5_24(op_7v1) }; - let mut sg_8v1_node_8v1_negset = context - .state_ref(sg_8v1_node_8v1_diffdata_handle) + let op_7v1 = op_7v1.map(|k| (k, ())); + let mut sg_8v1_node_8v1_antijoindata_neg_borrow = context + .state_ref(sg_8v1_node_8v1_antijoindata_neg) .borrow_mut(); - sg_8v1_node_8v1_negset.extend(hoff_22v3_recv); - let op_8v1 = op_7v1 - .filter(move |x| !sg_8v1_node_8v1_negset.contains(x)); + let mut sg_8v1_node_8v1_antijoindata_pos_borrow = context + .state_ref(sg_8v1_node_8v1_antijoindata_pos) + .borrow_mut(); + let op_8v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_8v1_node_8v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_22v3_recv, + op_7v1, + &mut *sg_8v1_node_8v1_antijoindata_neg_borrow, + &mut *sg_8v1_node_8v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_8v1 = op_8v1.map(|(k, ())| k); let op_8v1 = { #[allow(non_snake_case)] #[inline(always)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@surface_graph.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@surface_graph.snap index 7d98ac220fb..12778a0a1e2 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@surface_graph.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist@surface_graph.snap @@ -5,11 +5,11 @@ expression: flat_graph_ref.surface_syntax_string() 2v1 = unique :: < 'tick > (); 3v1 = difference :: < 'tick , 'static > (); 4v1 = tee (); -5v1 = next_tick (); +5v1 = defer_tick (); 7v1 = unique :: < 'tick > (); 8v1 = difference :: < 'tick , 'static > (); 9v1 = tee (); -10v1 = next_tick (); +10v1 = defer_tick (); 12v1 = unique :: < 'tick > (); 15v1 = unique :: < 'tick > (); 18v1 = unique :: < 'tick > (); @@ -19,7 +19,7 @@ expression: flat_graph_ref.surface_syntax_string() 30v1 = unique :: < 'tick > (); 31v1 = difference :: < 'tick , 'static > (); 32v1 = tee (); -33v1 = next_tick (); +33v1 = defer_tick (); 34v1 = source_stream (ints1); 35v1 = source_stream (ints2); 36v1 = source_stream (ints3); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@datalog_program.snap index 6d96e00392c..8dcf7966c9b 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"next_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | (() , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'static , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":8,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":8,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":3},{\"idx\":3,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":7,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"difference :: < 'tick , 'static > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"defer_tick ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | (() , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'static , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":11,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":8,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":5},{\"value\":[{\"idx\":8,\"version\":3},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":3},{\"idx\":3,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":7,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Path\":\"pos\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":5},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Path\":\"neg\"}],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":5,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":2,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -133,7 +133,7 @@ fn main() { let op_5v1 = { #[allow(non_snake_case)] #[inline(always)] - pub fn op_5v1__next_tick__loc_unknown_start_2_21_end_2_26< + pub fn op_5v1__defer_tick__loc_unknown_start_2_21_end_2_26< Item, Input: ::std::iter::Iterator, >(input: Input) -> impl ::std::iter::Iterator { @@ -159,7 +159,7 @@ fn main() { } Pull { inner: input } } - op_5v1__next_tick__loc_unknown_start_2_21_end_2_26(op_5v1) + op_5v1__defer_tick__loc_unknown_start_2_21_end_2_26(op_5v1) }; #[inline(always)] fn check_pivot_run< @@ -208,6 +208,7 @@ fn main() { iter } for kv in check_input(hoff_9v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_2v1_node_17v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -419,12 +420,21 @@ fn main() { >::default(), ), ); - let sg_3v1_node_3v1_diffdata_handle = df + let sg_3v1_node_3v1_antijoindata_neg = df .add_state( - ::std::cell::RefCell::new( - hydroflow::rustc_hash::FxHashSet::default(), + std::cell::RefCell::new(hydroflow::rustc_hash::FxHashSet::default()), + ); + let sg_3v1_node_3v1_antijoindata_pos = df + .add_state( + std::cell::RefCell::new( + hydroflow::util::monotonic_map::MonotonicMap::< + _, + hydroflow::rustc_hash::FxHashSet<_>, + >::default(), ), ); + let sg_3v1_node_3v1_persisttick = df + .add_state(std::cell::RefCell::new(0_usize)); df.add_subgraph_stratified( "Subgraph GraphSubgraphId(3v1)", 1, @@ -609,12 +619,58 @@ fn main() { } op_2v1__unique__loc_unknown_start_2_21_end_2_26(op_2v1) }; - let mut sg_3v1_node_3v1_negset = context - .state_ref(sg_3v1_node_3v1_diffdata_handle) + let op_2v1 = op_2v1.map(|k| (k, ())); + let mut sg_3v1_node_3v1_antijoindata_neg_borrow = context + .state_ref(sg_3v1_node_3v1_antijoindata_neg) .borrow_mut(); - sg_3v1_node_3v1_negset.extend(hoff_11v3_recv); - let op_3v1 = op_2v1 - .filter(move |x| !sg_3v1_node_3v1_negset.contains(x)); + let mut sg_3v1_node_3v1_antijoindata_pos_borrow = context + .state_ref(sg_3v1_node_3v1_antijoindata_pos) + .borrow_mut(); + let op_3v1 = { + /// Limit error propagation by bounding locally, erasing output iterator type. + #[inline(always)] + fn check_inputs<'a, K, I1, V, I2>( + input_neg: I1, + input_pos: I2, + neg_state: &'a mut hydroflow::rustc_hash::FxHashSet, + pos_state: &'a mut hydroflow::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, + ) -> impl 'a + Iterator + where + K: Eq + ::std::hash::Hash + Clone, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, + { + neg_state.extend(input_neg); + hydroflow::compiled::pull::anti_join_into_iter( + input_pos, + neg_state, + pos_state, + is_new_tick, + ) + } + let __is_new_tick = { + let mut __borrow_ident = context + .state_ref(sg_3v1_node_3v1_persisttick) + .borrow_mut(); + if *__borrow_ident <= context.current_tick() { + *__borrow_ident = context.current_tick() + 1; + true + } else { + false + } + }; + check_inputs( + hoff_11v3_recv, + op_2v1, + &mut *sg_3v1_node_3v1_antijoindata_neg_borrow, + &mut *sg_3v1_node_3v1_antijoindata_pos_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, + ) + }; + let op_3v1 = op_3v1.map(|(k, ())| k); let op_3v1 = { #[allow(non_snake_case)] #[inline(always)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@surface_graph.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@surface_graph.snap index 0ed370def67..2f3c967b2a4 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@surface_graph.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__persist_uniqueness@surface_graph.snap @@ -5,7 +5,7 @@ expression: flat_graph_ref.surface_syntax_string() 2v1 = unique :: < 'tick > (); 3v1 = difference :: < 'tick , 'static > (); 4v1 = tee (); -5v1 = next_tick (); +5v1 = defer_tick (); 7v1 = unique :: < 'tick > (); 10v1 = unique :: < 'tick > (); 12v1 = source_stream (ints2); diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__send_to_node@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__send_to_node@datalog_program.snap index 8b08fa22d74..03d3ab57007 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__send_to_node@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__send_to_node@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| (node , data) | async_send_result (node , data))\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (async_receive_result)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| v : (_ , _ ,) | (v . 1 , (v . 0 ,)))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_async_send\",\"version\":1},{\"value\":\"result_async_send\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (ints)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| (node , data) | async_send_result (node , data))\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (async_receive_result)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| v : (_ , _ ,) | (v . 1 , (v . 0 ,)))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result_async_send\",\"version\":1},{\"value\":\"result_async_send\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_12v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__simple_filter@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__simple_filter@datalog_program.snap index b6b4c38f84e..c960849c6fe 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__simple_filter@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__simple_filter@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ ,) | row . 0 > row . 1 && row . 1 == row . 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_0_filter\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"filter (| row : & (_ , _ ,) | row . 0 > row . 1 && row . 1 == row . 0)\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 0 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ ,) , _) | (g . 0 , g . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"predicate_0_filter\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_7v1_stream = { diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__single_column_program@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__single_column_program@datalog_program.snap index 133b9692271..15b18942e74 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__single_column_program@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__single_column_program@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (() , ())) | (kv . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (() , ())) | (kv . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_10v1_stream = { @@ -76,6 +76,8 @@ fn main() { ), ), ); + let sg_1v1_node_13v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_1v1_node_8v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -337,6 +339,9 @@ fn main() { let mut sg_1v1_node_13v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_13v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_13v1_persisttick_borrow = context + .state_ref(sg_1v1_node_13v1_persisttick) + .borrow_mut(); let op_13v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -352,6 +357,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -360,21 +366,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_13v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_13v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_15v1, + op_16v1, + &mut *sg_1v1_node_13v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_13v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_15v1, - op_16v1, - &mut *sg_1v1_node_13v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_13v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_13v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__transitive_closure@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__transitive_closure@datalog_program.snap index 32f7ac06801..ca24afcb6e5 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__transitive_closure@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__transitive_closure@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (edges)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (seed_reachable)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | reachable . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (() , (_ ,))) | (kv . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"edges_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"seed_reachable_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"reachable_insert\",\"version\":1},{\"value\":\"reachable_insert\",\"version\":1},{\"value\":\"reachable\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"union ()\"},\"version\":1},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (edges)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (seed_reachable)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | reachable . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , (() , (_ ,))) | (kv . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | ((row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":17,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":7,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":16,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":16,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":7,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":12,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"edges_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"seed_reachable_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"reachable_insert\",\"version\":1},{\"value\":\"reachable_insert\",\"version\":1},{\"value\":\"reachable\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"star_ord\":5,\"lattice_flow_type\":null},\"version\":1}]}", ); df.__assign_diagnostics("[]"); let (hoff_6v3_send, hoff_6v3_recv) = df @@ -81,6 +81,8 @@ fn main() { ), ), ); + let sg_1v1_node_15v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_1v1_node_8v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -413,6 +415,9 @@ fn main() { let mut sg_1v1_node_15v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_15v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_15v1_persisttick_borrow = context + .state_ref(sg_1v1_node_15v1_persisttick) + .borrow_mut(); let op_15v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -428,6 +433,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -436,21 +442,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_15v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_15v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_17v1, + op_18v1, + &mut *sg_1v1_node_15v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_15v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_17v1, - op_18v1, - &mut *sg_1v1_node_15v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_15v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_15v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__triple_relation_join@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__triple_relation_join@datalog_program.snap index 91d395ae091..624fa7bb510 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__triple_relation_join@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__triple_relation_join@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ , _ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 0 . 1 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ , _ ,) | ((_v . 2 ,) , (_v . 1 , _v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ , _ ,) | ((row . 3 , row . 0 , row . 2 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 , g . 3 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"source_stream (in1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in2)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (in3)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ , _ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 0 . 1 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ , _ ,) | ((_v . 2 ,) , (_v . 1 , _v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ , _ ,) | ((row . 3 , row . 0 , row . 2 , row . 1 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ , _ , _ , _ ,) , _) | (g . 0 , g . 1 , g . 2 , g . 3 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":15,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":21,\"version\":1},{\"idx\":22,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":26,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in1_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"in3_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let mut sg_1v1_node_13v1_stream = { @@ -100,6 +100,8 @@ fn main() { ), ), ); + let sg_1v1_node_17v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_1v1_node_21v1_joindata_lhs = df .add_state( std::cell::RefCell::new( @@ -116,6 +118,8 @@ fn main() { ), ), ); + let sg_1v1_node_21v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_1v1_node_11v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -465,6 +469,9 @@ fn main() { let mut sg_1v1_node_17v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_17v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_17v1_persisttick_borrow = context + .state_ref(sg_1v1_node_17v1_persisttick) + .borrow_mut(); let op_17v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -480,6 +487,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -488,21 +496,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_17v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_17v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_19v1, + op_20v1, + &mut *sg_1v1_node_17v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_17v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_19v1, - op_20v1, - &mut *sg_1v1_node_17v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_17v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_17v1 = { #[allow(non_snake_case)] @@ -638,6 +659,9 @@ fn main() { let mut sg_1v1_node_21v1_joindata_rhs_borrow = context .state_ref(sg_1v1_node_21v1_joindata_rhs) .borrow_mut(); + let mut sg_1v1_node_21v1_persisttick_borrow = context + .state_ref(sg_1v1_node_21v1_persisttick) + .borrow_mut(); let op_21v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -653,6 +677,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -661,21 +686,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_1v1_node_21v1_persisttick_borrow + <= context.current_tick() + { + *sg_1v1_node_21v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_23v1, + op_24v1, + &mut *sg_1v1_node_21v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_1v1_node_21v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_23v1, - op_24v1, - &mut *sg_1v1_node_21v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_1v1_node_21v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_21v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_fields@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_fields@datalog_program.snap index eb4302baf46..1da92c80953 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_fields@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_fields@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":\"input\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":2},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (input)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | out . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , (_ ,))) | (kv . 0 . 0 , kv . 1 . 0 . 0 , kv . 1 . 1 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 1 ,) , (_v . 0 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ , _ ,) | ((row . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : ((_ ,) , _) | (g . 0 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":6,\"version\":3},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":null,\"version\":2},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":6,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":12,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":7,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":12,\"version\":1},{\"idx\":9,\"version\":1},{\"idx\":10,\"version\":1},{\"idx\":13,\"version\":1},{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"input_insert\",\"version\":1},{\"value\":\"input\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"out_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_4v3_send, hoff_4v3_recv) = df @@ -210,6 +210,8 @@ fn main() { ), ), ); + let sg_2v1_node_9v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); let sg_2v1_node_5v1_uniquedata = df .add_state( ::std::cell::RefCell::new( @@ -299,6 +301,9 @@ fn main() { let mut sg_2v1_node_9v1_joindata_rhs_borrow = context .state_ref(sg_2v1_node_9v1_joindata_rhs) .borrow_mut(); + let mut sg_2v1_node_9v1_persisttick_borrow = context + .state_ref(sg_2v1_node_9v1_persisttick) + .borrow_mut(); let op_9v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -314,6 +319,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -322,21 +328,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_2v1_node_9v1_persisttick_borrow + <= context.current_tick() + { + *sg_2v1_node_9v1_persisttick_borrow = context.current_tick() + + 1; + true + } else { + false + }; + check_inputs( + op_11v1, + op_12v1, + &mut *sg_2v1_node_9v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_2v1_node_9v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_11v1, - op_12v1, - &mut *sg_2v1_node_9v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_2v1_node_9v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_9v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_join_count@datalog_program.snap b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_join_count@datalog_program.snap index 6a2e7393844..2ff5178a038 100644 --- a/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_join_count@datalog_program.snap +++ b/hydroflow_datalog_core/src/snapshots/hydroflow_datalog_core__tests__wildcard_join_count@datalog_program.snap @@ -9,7 +9,7 @@ fn main() { use hydroflow::{var_expr, var_args}; let mut df = hydroflow::scheduled::graph::Hydroflow::new(); df.__assign_meta_graph( - "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , ())) | (kv . 0 . 0 , kv . 1 . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , (() ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + 1) } else { Some (1) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () ,))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , ())) | (kv . 0 . 0 , kv . 1 . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":6,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":30,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":29,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":7,\"version\":3},{\"idx\":26,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":10,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":7,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":27,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":3},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":29,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":28,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":6,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":29,\"version\":1},{\"idx\":30,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}]}", + "{\"nodes\":[{\"value\":null,\"version\":0},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Operator\":\"tee ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"unique :: < 'tick > ()\"},\"version\":1},{\"value\":{\"Handoff\":{}},\"version\":3},{\"value\":{\"Operator\":\"source_stream (ints1)\"},\"version\":1},{\"value\":{\"Operator\":\"source_stream (ints2)\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"for_each (| v | result2 . send (v) . unwrap ())\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , ())) | (kv . 0 . 0 , kv . 1 . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , (() ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some (prev + 1) } else { Some (1) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () ,))\"},\"version\":1},{\"value\":{\"Operator\":\"join :: < 'tick , 'tick , hydroflow :: compiled :: pull :: HalfMultisetJoinState > ()\"},\"version\":1},{\"value\":{\"Operator\":\"map (| kv : ((_ ,) , ((_ ,) , ())) | (kv . 0 . 0 , kv . 1 . 0 . 0 ,))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ , _ ,) | ((_v . 0 ,) , (_v . 1 ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| _v : (_ ,) | ((_v . 0 ,) , ()))\"},\"version\":1},{\"value\":{\"Operator\":\"map (| row : (_ , _ ,) | (() , ((row . 0) ,)))\"},\"version\":1},{\"value\":{\"Operator\":\"fold_keyed :: < 'tick , () , (Option < _ > ,) > (| | (None ,) , | old : & mut (Option < _ > ,) , val : (_ ,) | { old . 0 = if let Some (prev) = old . 0 . take () { Some ({ let prev : (hydroflow :: rustc_hash :: FxHashSet < _ > , _) = prev ; let mut set : hydroflow :: rustc_hash :: FxHashSet < _ > = prev . 0 ; if set . insert (val . 0) { (set , prev . 1 + 1) } else { (set , prev . 1) } }) } else { Some ({ let mut set = hydroflow :: rustc_hash :: FxHashSet :: < _ > :: default () ; set . insert (val . 0) ; (set , 1) }) } ; })\"},\"version\":1},{\"value\":{\"Operator\":\"map (| (g , a) : (() , _) | (a . 0 . unwrap () . 1 ,))\"},\"version\":1}],\"graph\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":5,\"version\":1},{\"idx\":6,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":23,\"version\":1},{\"idx\":8,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":9,\"version\":3},{\"idx\":20,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":30,\"version\":1},{\"idx\":11,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":12,\"version\":3},{\"idx\":19,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":1,\"version\":3},{\"idx\":29,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":4,\"version\":3},{\"idx\":27,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":8,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":12,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":1},{\"idx\":9,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":7,\"version\":3},{\"idx\":26,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":21,\"version\":1},{\"idx\":10,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":18,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":3,\"version\":1},{\"idx\":7,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":27,\"version\":1},{\"idx\":24,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":6,\"version\":1},{\"idx\":4,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":10,\"version\":3},{\"idx\":22,\"version\":1}],\"version\":3},{\"value\":[{\"idx\":29,\"version\":1},{\"idx\":30,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":28,\"version\":1},{\"idx\":1,\"version\":3}],\"version\":3},{\"value\":[{\"idx\":25,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1}],\"ports\":[{\"value\":null,\"version\":0},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"0\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",{\"Int\":\"0\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",{\"Int\":\"1\"}],\"version\":1},{\"value\":[{\"Int\":\"1\"},\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1},{\"value\":[\"Elided\",\"Elided\"],\"version\":3},{\"value\":[\"Elided\",\"Elided\"],\"version\":1}],\"node_subgraph\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":null,\"version\":0},{\"value\":{\"idx\":1,\"version\":1},\"version\":1},{\"value\":{\"idx\":2,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":5,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":3,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":6,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1},{\"value\":{\"idx\":4,\"version\":1},\"version\":1}],\"subgraph_nodes\":[{\"value\":null,\"version\":0},{\"value\":[{\"idx\":13,\"version\":1},{\"idx\":2,\"version\":1},{\"idx\":3,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":14,\"version\":1},{\"idx\":5,\"version\":1},{\"idx\":6,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":22,\"version\":1},{\"idx\":23,\"version\":1},{\"idx\":8,\"version\":1},{\"idx\":15,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":29,\"version\":1},{\"idx\":30,\"version\":1},{\"idx\":11,\"version\":1},{\"idx\":16,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":19,\"version\":1},{\"idx\":20,\"version\":1},{\"idx\":17,\"version\":1},{\"idx\":18,\"version\":1},{\"idx\":21,\"version\":1}],\"version\":1},{\"value\":[{\"idx\":26,\"version\":1},{\"idx\":27,\"version\":1},{\"idx\":24,\"version\":1},{\"idx\":25,\"version\":1},{\"idx\":28,\"version\":1}],\"version\":1}],\"subgraph_stratum\":[{\"value\":null,\"version\":0},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1},{\"value\":1,\"version\":1},{\"value\":1,\"version\":1},{\"value\":0,\"version\":1},{\"value\":0,\"version\":1}],\"node_varnames\":[{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"ints1_insert\",\"version\":1},{\"value\":\"ints1\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"ints2_insert\",\"version\":1},{\"value\":\"ints2\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":\"result_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"result2_insert\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_0\",\"version\":1},{\"value\":\"join_0\",\"version\":1},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":null,\"version\":0},{\"value\":\"join_1\",\"version\":1},{\"value\":\"join_1\",\"version\":1}],\"flow_props\":[{\"value\":null,\"version\":0}]}", ); df.__assign_diagnostics("[]"); let (hoff_1v3_send, hoff_1v3_recv) = df @@ -422,6 +422,7 @@ fn main() { iter } for kv in check_input(hoff_10v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_3v1_node_22v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -627,6 +628,7 @@ fn main() { iter } for kv in check_input(hoff_1v3_recv) { + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = sg_4v1_node_29v1_hashtable .entry(kv.0) .or_insert_with(|| (None,)); @@ -826,6 +828,8 @@ fn main() { ), ), ); + let sg_5v1_node_17v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); df.add_subgraph_stratified( "Subgraph GraphSubgraphId(5v1)", 0, @@ -915,6 +919,9 @@ fn main() { let mut sg_5v1_node_17v1_joindata_rhs_borrow = context .state_ref(sg_5v1_node_17v1_joindata_rhs) .borrow_mut(); + let mut sg_5v1_node_17v1_persisttick_borrow = context + .state_ref(sg_5v1_node_17v1_persisttick) + .borrow_mut(); let op_17v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -930,6 +937,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -938,21 +946,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_5v1_node_17v1_persisttick_borrow + <= context.current_tick() + { + *sg_5v1_node_17v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_19v1, + op_20v1, + &mut *sg_5v1_node_17v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_5v1_node_17v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_19v1, - op_20v1, - &mut *sg_5v1_node_17v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_5v1_node_17v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_17v1 = { #[allow(non_snake_case)] @@ -1077,6 +1098,8 @@ fn main() { ), ), ); + let sg_6v1_node_24v1_persisttick = df + .add_state(std::cell::RefCell::new(0usize)); df.add_subgraph_stratified( "Subgraph GraphSubgraphId(6v1)", 0, @@ -1166,6 +1189,9 @@ fn main() { let mut sg_6v1_node_24v1_joindata_rhs_borrow = context .state_ref(sg_6v1_node_24v1_joindata_rhs) .borrow_mut(); + let mut sg_6v1_node_24v1_persisttick_borrow = context + .state_ref(sg_6v1_node_24v1_persisttick) + .borrow_mut(); let op_24v1 = { #[inline(always)] fn check_inputs<'a, K, I1, V1, I2, V2>( @@ -1181,6 +1207,7 @@ fn main() { V2, V1, >, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -1189,21 +1216,34 @@ fn main() { I1: 'a + Iterator, I2: 'a + Iterator, { - hydroflow::compiled::pull::SymmetricHashJoin::new_from_mut( + hydroflow::compiled::pull::symmetric_hash_join_into_iter( lhs, rhs, lhs_state, rhs_state, + is_new_tick, + ) + } + { + let __is_new_tick = if *sg_6v1_node_24v1_persisttick_borrow + <= context.current_tick() + { + *sg_6v1_node_24v1_persisttick_borrow = context + .current_tick() + 1; + true + } else { + false + }; + check_inputs( + op_26v1, + op_27v1, + &mut *sg_6v1_node_24v1_joindata_lhs_borrow + .get_mut_clear(context.current_tick()), + &mut *sg_6v1_node_24v1_joindata_rhs_borrow + .get_mut_clear(context.current_tick()), + __is_new_tick, ) } - check_inputs( - op_26v1, - op_27v1, - &mut *sg_6v1_node_24v1_joindata_lhs_borrow - .get_mut_clear(context.current_tick()), - &mut *sg_6v1_node_24v1_joindata_rhs_borrow - .get_mut_clear(context.current_tick()), - ) }; let op_24v1 = { #[allow(non_snake_case)] diff --git a/hydroflow_lang/CHANGELOG.md b/hydroflow_lang/CHANGELOG.md index 153994db602..8fb4028d140 100644 --- a/hydroflow_lang/CHANGELOG.md +++ b/hydroflow_lang/CHANGELOG.md @@ -5,8 +5,236 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +### Chore + + - Allow `clippy::redundant_locals`, for latest nightlies + - fix lints for latest nightly + - fix lint, format errors for latest nightly version (without updated pinned) + For nightly version (d9c13cd45 2023-07-05) + +### New Features + + - Add `use` statements to hydroflow syntax + And use in doc tests. + - add `iter_batches_stream` util to break up iterator into per-tick batches + * Also tightens up a bit of `assert_eq`'s code + - rename assert => assert_eq, add assert, change underlying implementation to work across ticks + - make batch take two inputs [input] and [signal] + +### Bug Fixes + + - unify antijoin and difference with set and multiset semantics + * fix: unify antijoin and difference with set and multiset semantics + + * fix: replay semantics for antijoin and difference now work + also added cross_join_multiset + + * fix: enforce sort for tests of anti_join and difference using assert_eq + + * fix: advance __borrow_ident beyond the current tick to prevent replay loops + + * fix: add modified snapshots + + * fix: temp + + * fix: spelling typo in comment + + * fix: make anti_join replay more efficient + + Also add multiset data structure, use it in some tests, make join() + replay logic more similar to anti_join's and presist's. + + * fix: ignore test that depends on order of antijoin + + * fix: really ignore test_index + + * fix: fix specific test ordering in wasm + + --------- + - joins now replay correctly + - lattice_batch now takes [input] and [signal] + - make all operators 'tick by default + - rename next_tick -> defer, batch -> defer_signal + - `py_udf` operator feature gating + +### New Features (BREAKING) + + - add fused joins, make lattice_join replay correctly + * feat!: add fused joins, make lattice_join replay correctly + + * address comments + + * fix clippy + +### Commit Statistics + + + + - 16 commits contributed to the release over the course of 39 calendar days. + - 42 days passed between releases. + - 14 commits were understood as [conventional](https://www.conventionalcommits.org). + - 16 unique issues were worked on: [#820](https://github.com/hydro-project/hydroflow/issues/820), [#821](https://github.com/hydro-project/hydroflow/issues/821), [#822](https://github.com/hydro-project/hydroflow/issues/822), [#823](https://github.com/hydro-project/hydroflow/issues/823), [#833](https://github.com/hydro-project/hydroflow/issues/833), [#835](https://github.com/hydro-project/hydroflow/issues/835), [#840](https://github.com/hydro-project/hydroflow/issues/840), [#843](https://github.com/hydro-project/hydroflow/issues/843), [#844](https://github.com/hydro-project/hydroflow/issues/844), [#845](https://github.com/hydro-project/hydroflow/issues/845), [#851](https://github.com/hydro-project/hydroflow/issues/851), [#853](https://github.com/hydro-project/hydroflow/issues/853), [#861](https://github.com/hydro-project/hydroflow/issues/861), [#870](https://github.com/hydro-project/hydroflow/issues/870), [#872](https://github.com/hydro-project/hydroflow/issues/872), [#873](https://github.com/hydro-project/hydroflow/issues/873) + +### Commit Details + + + +
view details + + * **[#820](https://github.com/hydro-project/hydroflow/issues/820)** + - Make batch take two inputs [input] and [signal] ([`8710022`](https://github.com/hydro-project/hydroflow/commit/871002267e3c03da83729ecc2d028f3c7b5c18d2)) + * **[#821](https://github.com/hydro-project/hydroflow/issues/821)** + - `py_udf` operator feature gating ([`2d53110`](https://github.com/hydro-project/hydroflow/commit/2d53110336b2da5a16887c3d72101da72b2362bb)) + * **[#822](https://github.com/hydro-project/hydroflow/issues/822)** + - Fix lint, format errors for latest nightly version (without updated pinned) ([`f60053f`](https://github.com/hydro-project/hydroflow/commit/f60053f70da3071c54de4a0eabb059a143aa2ccc)) + * **[#823](https://github.com/hydro-project/hydroflow/issues/823)** + - Book/doc edits ([`4bdd556`](https://github.com/hydro-project/hydroflow/commit/4bdd5568fa0a6674f650f91a029fab302cbf14f4)) + * **[#833](https://github.com/hydro-project/hydroflow/issues/833)** + - Rename next_tick -> defer, batch -> defer_signal ([`6c98bbc`](https://github.com/hydro-project/hydroflow/commit/6c98bbc2bd3443fe6f77e0b8689b461edde1b316)) + * **[#835](https://github.com/hydro-project/hydroflow/issues/835)** + - Rename assert => assert_eq, add assert, change underlying implementation to work across ticks ([`8f306e2`](https://github.com/hydro-project/hydroflow/commit/8f306e2a36582e168417808099eedf8a9de3b419)) + * **[#840](https://github.com/hydro-project/hydroflow/issues/840)** + - Make all operators 'tick by default ([`a55fc74`](https://github.com/hydro-project/hydroflow/commit/a55fc74dc1ebbe26b49359a104beb48d7f6cd449)) + * **[#843](https://github.com/hydro-project/hydroflow/issues/843)** + - Add `iter_batches_stream` util to break up iterator into per-tick batches ([`fe02f23`](https://github.com/hydro-project/hydroflow/commit/fe02f23649312bb64c5d0c8870edf578e516f397)) + * **[#844](https://github.com/hydro-project/hydroflow/issues/844)** + - Fix lints for latest nightly ([`949db02`](https://github.com/hydro-project/hydroflow/commit/949db02e9fa9878e1a7176c180d6f44c5cddf052)) + * **[#845](https://github.com/hydro-project/hydroflow/issues/845)** + - Add `use` statements to hydroflow syntax ([`b4b9644`](https://github.com/hydro-project/hydroflow/commit/b4b9644a19e8e7e7725c9c5b88e3a6b8c2be7364)) + * **[#851](https://github.com/hydro-project/hydroflow/issues/851)** + - Lattice_batch now takes [input] and [signal] ([`ebba382`](https://github.com/hydro-project/hydroflow/commit/ebba38230df134b04dd38c1df7c6de8712e3122e)) + * **[#853](https://github.com/hydro-project/hydroflow/issues/853)** + - Book updates ([`2e57445`](https://github.com/hydro-project/hydroflow/commit/2e574457246ac5bd231745a8ad068558859698ef)) + * **[#861](https://github.com/hydro-project/hydroflow/issues/861)** + - Add fused joins, make lattice_join replay correctly ([`7a3b4c0`](https://github.com/hydro-project/hydroflow/commit/7a3b4c04779ea38bfa06c246882fa8dfb52bc8f1)) + * **[#870](https://github.com/hydro-project/hydroflow/issues/870)** + - Joins now replay correctly ([`cc959c7`](https://github.com/hydro-project/hydroflow/commit/cc959c762c3a0e036e672801c615028cbfb95168)) + * **[#872](https://github.com/hydro-project/hydroflow/issues/872)** + - Unify antijoin and difference with set and multiset semantics ([`d378e5e`](https://github.com/hydro-project/hydroflow/commit/d378e5eada3d2bae90f98c5a33b2d055940a8c7f)) + * **[#873](https://github.com/hydro-project/hydroflow/issues/873)** + - Allow `clippy::redundant_locals`, for latest nightlies ([`d6db9cd`](https://github.com/hydro-project/hydroflow/commit/d6db9cd22a3d63bcc65dafd5bc0ca663ecc553d7)) +
+ +## 0.3.0 (2023-07-04) + + + + +### Documentation + + - remove pattern deref from inspect, filter examples + `*` derefs are easier for Rust beginners to comprehend. + - change mermaid colors + Use a lighter shade of blue and yellow, and dark text. + +### New Features + + + + + + + + - fold and reduce take accumulated value by mutable reference + * feat: fold and reduce take accumulated value by mutable reference +* address comments +* feat: add lattice_reduce and lattice_fold +* address comments +* simplify lattice fold a bit +* address comments +* feat: add join_multiset() +* address comments +* fix assert +* feat: add assert() operator +* update: change for_each -> assert, make doctest use run_avaialble() +* don't run tests that panic in wasm +* update comments +* address comments + +### Bug Fixes + + - update proc-macro2, use new span location API where possible + requires latest* rust nightly version + + *latest = 2023-06-28 or something + - fix nightly removing array_zip feature, bump pinned nightly to 06-01 + +### Style + + - `warn` missing docs (instead of `deny`) to allow code before docs + +### New Features (BREAKING) + + - Add `reveal` methods, make fields private + +### Bug Fixes (BREAKING) + + - make join default to multiset join + +### Refactor (BREAKING) + + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` + +### Commit Statistics + + + + - 17 commits contributed to the release over the course of 31 calendar days. + - 33 days passed between releases. + - 14 commits were understood as [conventional](https://www.conventionalcommits.org). + - 15 unique issues were worked on: [#741](https://github.com/hydro-project/hydroflow/issues/741), [#765](https://github.com/hydro-project/hydroflow/issues/765), [#773](https://github.com/hydro-project/hydroflow/issues/773), [#774](https://github.com/hydro-project/hydroflow/issues/774), [#775](https://github.com/hydro-project/hydroflow/issues/775), [#778](https://github.com/hydro-project/hydroflow/issues/778), [#780](https://github.com/hydro-project/hydroflow/issues/780), [#784](https://github.com/hydro-project/hydroflow/issues/784), [#789](https://github.com/hydro-project/hydroflow/issues/789), [#792](https://github.com/hydro-project/hydroflow/issues/792), [#799](https://github.com/hydro-project/hydroflow/issues/799), [#801](https://github.com/hydro-project/hydroflow/issues/801), [#803](https://github.com/hydro-project/hydroflow/issues/803), [#804](https://github.com/hydro-project/hydroflow/issues/804), [#809](https://github.com/hydro-project/hydroflow/issues/809) + +### Commit Details + + + +
view details + + * **[#741](https://github.com/hydro-project/hydroflow/issues/741)** + - Fix nightly removing array_zip feature, bump pinned nightly to 06-01 ([`20cb381`](https://github.com/hydro-project/hydroflow/commit/20cb3811fc0da3ce1b36003c8823b4b242d64196)) + * **[#765](https://github.com/hydro-project/hydroflow/issues/765)** + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` ([`4a727ec`](https://github.com/hydro-project/hydroflow/commit/4a727ecf1232e0f03f5300547282bfbe73342cfa)) + * **[#773](https://github.com/hydro-project/hydroflow/issues/773)** + - `warn` missing docs (instead of `deny`) to allow code before docs ([`70c88a5`](https://github.com/hydro-project/hydroflow/commit/70c88a51c4c83a4dc2fc67a0cd344786a4ff26f7)) + * **[#774](https://github.com/hydro-project/hydroflow/issues/774)** + - Make join default to multiset join ([`6f3c536`](https://github.com/hydro-project/hydroflow/commit/6f3c536fcd4d1305d478ec3db62416aad9cf3c68)) + * **[#775](https://github.com/hydro-project/hydroflow/issues/775)** + - Add persist_mut and persist_mut_keyed for non-monitone deletions ([`8d8247f`](https://github.com/hydro-project/hydroflow/commit/8d8247f0b37d53415f5738099c0c8a021415b158)) + * **[#778](https://github.com/hydro-project/hydroflow/issues/778)** + - Change mermaid colors ([`f55d540`](https://github.com/hydro-project/hydroflow/commit/f55d540532ba0a0970cab2bb5aef81b6a76b317a)) + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Emit `compile_error!` diagnostics for stable ([`ea65349`](https://github.com/hydro-project/hydroflow/commit/ea65349d241873f8460d7a8b024d64c63180246f)) + - Allow stable build, refactors behind `nightly` feature flag ([`22abcaf`](https://github.com/hydro-project/hydroflow/commit/22abcaff806c7de6e4a7725656bbcf201e7d9259)) + * **[#784](https://github.com/hydro-project/hydroflow/issues/784)** + - Add assert() operator ([`d83b049`](https://github.com/hydro-project/hydroflow/commit/d83b049e4d643617a2b15b3dbf1698aa79846aeb)) + * **[#789](https://github.com/hydro-project/hydroflow/issues/789)** + - Add `reveal` methods, make fields private ([`931d938`](https://github.com/hydro-project/hydroflow/commit/931d93887c238025596cb22226e16d43e16a7425)) + * **[#792](https://github.com/hydro-project/hydroflow/issues/792)** + - Add `py_udf` operator [wip] ([`7dbd5e2`](https://github.com/hydro-project/hydroflow/commit/7dbd5e24d6e71cf8fab7c3ce09d5937c0f301456)) + * **[#799](https://github.com/hydro-project/hydroflow/issues/799)** + - Remove pattern deref from inspect, filter examples ([`fa5b180`](https://github.com/hydro-project/hydroflow/commit/fa5b180d96498d144f3617bba7722e8f4ac9dd0e)) + * **[#801](https://github.com/hydro-project/hydroflow/issues/801)** + - Update proc-macro2, use new span location API where possible ([`8d3494b`](https://github.com/hydro-project/hydroflow/commit/8d3494b5afee858114a602a3e23077bb6d24dd77)) + * **[#803](https://github.com/hydro-project/hydroflow/issues/803)** + - Add lattice_reduce and lattice_fold ([`6323980`](https://github.com/hydro-project/hydroflow/commit/6323980e83bee27a8233a69a35734b5970336701)) + * **[#804](https://github.com/hydro-project/hydroflow/issues/804)** + - Add join_multiset() ([`0105246`](https://github.com/hydro-project/hydroflow/commit/010524615bb78288e339e03880c4dd3b432b6d7f)) + * **[#809](https://github.com/hydro-project/hydroflow/issues/809)** + - Fold and reduce take accumulated value by mutable reference ([`b435bbb`](https://github.com/hydro-project/hydroflow/commit/b435bbb1d64d60f1248fdcd636635b15954e7325)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) +
+ + + add lattice_reduce and lattice_fold add join_multiset()also remove documentation about HalfJoinMultiset, the way to accessthat now is to use join_multiset() add assert() operator emit compile_error! diagnostics for stable allow stable build, refactors behind nightly feature flag + ## 0.2.0 (2023-05-31) + + + ### Chore - manually bump versions for v0.2.0 release @@ -27,8 +255,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 4 commits contributed to the release. - - 2 days passed between releases. + - 5 commits contributed to the release. + - 1 day passed between releases. - 4 commits were understood as [conventional](https://www.conventionalcommits.org). - 2 unique issues were worked on: [#728](https://github.com/hydro-project/hydroflow/issues/728), [#730](https://github.com/hydro-project/hydroflow/issues/730) @@ -43,6 +271,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#730](https://github.com/hydro-project/hydroflow/issues/730)** - Categorize operators, organize op docs, fix #727 ([`989adcb`](https://github.com/hydro-project/hydroflow/commit/989adcbcd304ad0890d71351d56a22977bdcf73f)) * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Make `build.rs`s infallible, log to stderr, to fix release ([`554d563`](https://github.com/hydro-project/hydroflow/commit/554d563fe53a1303c5a5c9352197365235c607e3)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a)) diff --git a/hydroflow_lang/Cargo.toml b/hydroflow_lang/Cargo.toml index 51298ce5b42..0f348d10b23 100644 --- a/hydroflow_lang/Cargo.toml +++ b/hydroflow_lang/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydroflow_lang" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow_lang/" @@ -15,7 +15,7 @@ diagnostics = [] auto_impl = "1.0.1" itertools = "0.10" # TODO(mingwei): remove when `iter_intersperse` is stabilized. prettyplease = { version = "0.2.0", features = [ "verbatim" ] } -proc-macro2 = { version = "1.0.0", features = ["span-locations"] } +proc-macro2 = { version = "1.0.57", features = ["span-locations"] } quote = "1.0.0" regex = "1.7.0" serde = "1.0.1" diff --git a/hydroflow_lang/build.rs b/hydroflow_lang/build.rs index b40c47abf9c..cb3b6935b6e 100644 --- a/hydroflow_lang/build.rs +++ b/hydroflow_lang/build.rs @@ -33,8 +33,12 @@ fn generate_op_docs() -> Result<()> { .map_err(|syn_err| Error::new(ErrorKind::InvalidData, syn_err))?; for item in op_parsed.items { - let Item::Const(item_const) = item else { continue; }; - let Expr::Struct(expr_struct) = *item_const.expr else { continue; }; + let Item::Const(item_const) = item else { + continue; + }; + let Expr::Struct(expr_struct) = *item_const.expr else { + continue; + }; if identity::(parse_quote!(OperatorConstraints)) != expr_struct.path { continue; } @@ -44,7 +48,11 @@ fn generate_op_docs() -> Result<()> { .iter() .find(|&field_value| identity::(parse_quote!(name)) == field_value.member) .expect("Expected `name` field not found."); - let Expr::Lit(ExprLit { lit: Lit::Str(op_name), .. }) = &name_field.expr else { + let Expr::Lit(ExprLit { + lit: Lit::Str(op_name), + .. + }) = &name_field.expr + else { panic!("Unexpected non-literal or non-str `name` field value.") }; let op_name = op_name.value(); @@ -60,11 +68,26 @@ fn generate_op_docs() -> Result<()> { let mut in_hf_doctest = false; for attr in item_const.attrs.iter() { - let AttrStyle::Outer = attr.style else { continue; }; - let Meta::NameValue(MetaNameValue { path, eq_token: _, value }) = &attr.meta else { continue; }; - let Some("doc") = path.get_ident().map(Ident::to_string).as_deref() else { continue; }; - let Expr::Lit(ExprLit { attrs: _, lit }) = value else { continue; }; - let Lit::Str(doc_lit_str) = lit else { continue; }; + let AttrStyle::Outer = attr.style else { + continue; + }; + let Meta::NameValue(MetaNameValue { + path, + eq_token: _, + value, + }) = &attr.meta + else { + continue; + }; + let Some("doc") = path.get_ident().map(Ident::to_string).as_deref() else { + continue; + }; + let Expr::Lit(ExprLit { attrs: _, lit }) = value else { + continue; + }; + let Lit::Str(doc_lit_str) = lit else { + continue; + }; // At this point we know we have a `#[doc = "..."]`. let doc_str = doc_lit_str.value(); let doc_str = doc_str.strip_prefix(' ').unwrap_or(&*doc_str); @@ -75,6 +98,12 @@ fn generate_op_docs() -> Result<()> { // Output `doc_str` below. } else if doc_str.trim() == "```hydroflow" { in_hf_doctest = true; + + writeln!(docgen_write, "```rust")?; + // py_udf special-cased. + if "py_udf" == op_name { + writeln!(docgen_write, "# #[cfg(feature = \"python\")]")?; + } writeln!(docgen_write, "{}", DOCTEST_HYDROFLOW_PREFIX)?; continue; } else if doc_str.trim() == "```rustbook" { @@ -92,13 +121,12 @@ fn generate_op_docs() -> Result<()> { } const DOCTEST_HYDROFLOW_PREFIX: &str = "\ -```rust -# #[allow(unused_imports)] use hydroflow::{var_args, var_expr}; -# #[allow(unused_imports)] use hydroflow::pusherator::Pusherator; -# let __rt = hydroflow::tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); -# __rt.block_on(async { hydroflow::tokio::task::LocalSet::new().run_until(async { -# let mut __hf = hydroflow::hydroflow_syntax! {"; +# { +# let __rt = ::hydroflow::tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); +# __rt.block_on(async { ::hydroflow::tokio::task::LocalSet::new().run_until(async { +# let mut __hf = ::hydroflow::hydroflow_syntax! {"; const DOCTEST_HYDROFLOW_SUFFIX: &str = "\ # }; # __hf.run_available(); -# }).await})"; +# }).await}) +# }"; diff --git a/hydroflow_lang/src/diagnostic.rs b/hydroflow_lang/src/diagnostic.rs index 84ba00f5538..44ec8c588a8 100644 --- a/hydroflow_lang/src/diagnostic.rs +++ b/hydroflow_lang/src/diagnostic.rs @@ -1,3 +1,5 @@ +//! Compatibility for `proc_macro` diagnostics, which are missing from [`proc_macro2`]. + use std::borrow::Cow; use std::hash::{Hash, Hasher}; @@ -7,6 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::pretty_span::PrettySpan; +/// Diagnostic reporting level. #[non_exhaustive] #[derive(Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Level { @@ -28,23 +31,34 @@ pub enum Level { Help, } impl Level { + /// If this level is [`Level::Error`]. pub fn is_error(&self) -> bool { self <= &Self::Error } } +/// Diagnostic. A warning or error (or lower [`Level`]) with a message and span. Shown by IDEs +/// usually as a squiggly red or yellow underline. +/// +/// Must call [`Diagnostic::emit`] or manually emit the output of [`Diagnostic::to_tokens`] for the +/// diagnostic to show up. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Diagnostic { + /// Span (source code location). pub span: S, + /// Severity level. pub level: Level, + /// Human-readable message. pub message: String, } impl Diagnostic { + /// If this diagnostic's level is [`Level::Error`]. pub fn is_error(&self) -> bool { self.level.is_error() } } impl Diagnostic { + /// Create a new diagnostic from the given span, level, and message. pub fn spanned(span: Span, level: Level, message: impl Into) -> Self { let message = message.into(); Self { @@ -54,6 +68,8 @@ impl Diagnostic { } } + /// Emit the diagnostic. Only works from the `proc_macro` context. Does not work outside of + /// that e.g. in normal runtime execution or in tests. pub fn emit(&self) { #[cfg(feature = "diagnostics")] { @@ -67,6 +83,8 @@ impl Diagnostic { } } + /// Used to emulate [`Diagnostic::emit`] by turning this diagnostic into a properly spanned [`TokenStream`] + /// that emits an error with this diagnostic's message. pub fn to_tokens(&self) -> TokenStream { let msg_lit: Literal = Literal::string(&self.message); let unique_ident = { @@ -100,6 +118,9 @@ impl Diagnostic { } } + /// Converts this into a serializable and deserializable Diagnostic. Span information is + /// converted into [`SerdeSpan`] which keeps the span info but cannot be plugged into or + /// emitted through the Rust compiler's diagnostic system. pub fn to_serde(&self) -> Diagnostic { let Self { span, @@ -133,12 +154,17 @@ impl std::fmt::Display for Diagnostic { } } +/// A serializable and deserializable version of [`Span`]. Cannot be plugged into the Rust +/// compiler's diagnostic system. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerdeSpan { + /// The source file path. // https://github.com/serde-rs/serde/issues/1852#issuecomment-904840811 #[serde(borrow)] pub path: Cow<'static, str>, + /// Line number, one-indexed. pub line: usize, + /// Column number, one-indexed. pub column: usize, } impl From for SerdeSpan { diff --git a/hydroflow_lang/src/graph/di_mul_graph.rs b/hydroflow_lang/src/graph/di_mul_graph.rs index c52b936fcfb..9e15b1f8e02 100644 --- a/hydroflow_lang/src/graph/di_mul_graph.rs +++ b/hydroflow_lang/src/graph/di_mul_graph.rs @@ -162,9 +162,13 @@ where /// Returns `None` if `vertex` is not in the graph or does not have the right degree in/out. pub fn remove_intermediate_vertex(&mut self, vertex: V) -> Option<(E, (E, E))> { let preds = self.preds.remove(vertex)?; - let &[pred_edge] = &*preds else { return None; }; + let &[pred_edge] = &*preds else { + return None; + }; let succs = self.succs.remove(vertex).unwrap(); - let &[succ_edge] = &*succs else { return None; }; + let &[succ_edge] = &*succs else { + return None; + }; let (src, _v) = self.edges.remove(pred_edge).unwrap(); let (_v, dst) = self.edges.remove(succ_edge).unwrap(); @@ -176,6 +180,27 @@ where Some((new_edge, (pred_edge, succ_edge))) } + /// Remove an edge from the graph. If the edgeId is found then the edge is removed from the graph and returned. + /// If the edgeId was not found in the graph then nothing is returned and nothing is done. + pub fn remove_edge(&mut self, e: E) -> Option<(V, V)> { + let Some((src, dst)) = self.edges.remove(e) else { + return None; + }; + + self.succs[src].retain(|x| *x != e); + self.preds[dst].retain(|x| *x != e); + + Some((src, dst)) + } + + /// Remove a vertex from the graph, it must have no edges to or from it when doing this. + pub fn remove_vertex(&mut self, v: V) { + assert!(self.preds[v].is_empty() && self.succs[v].is_empty()); + + self.preds.remove(v); + self.succs.remove(v); + } + /// Get the source and destination vertex IDs for the given edge ID. pub fn edge(&self, e: E) -> Option<(V, V)> { self.edges.get(e).copied() @@ -189,8 +214,7 @@ where /// Return an iterator over all edges in form `(E, (V, V))`. pub fn edges( &self, - ) -> impl '_ + Iterator + ExactSizeIterator + FusedIterator + Clone + Debug - { + ) -> impl '_ + ExactSizeIterator + FusedIterator + Clone + Debug { self.edges.iter().map(|(e, &(src, dst))| (e, (src, dst))) } @@ -216,13 +240,8 @@ where pub fn successor_vertices( &self, v: V, - ) -> impl '_ - + Iterator - + DoubleEndedIterator - + ExactSizeIterator - + FusedIterator - + Clone - + Debug { + ) -> impl '_ + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug + { self.successor_edges(v).map(|edge_id| self.edges[edge_id].1) } @@ -230,13 +249,8 @@ where pub fn predecessor_vertices( &self, v: V, - ) -> impl '_ - + Iterator - + DoubleEndedIterator - + ExactSizeIterator - + FusedIterator - + Clone - + Debug { + ) -> impl '_ + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug + { self.predecessor_edges(v) .map(|edge_id| self.edges[edge_id].0) } @@ -245,13 +259,8 @@ where pub fn successors( &self, v: V, - ) -> impl '_ - + Iterator - + DoubleEndedIterator - + ExactSizeIterator - + FusedIterator - + Clone - + Debug { + ) -> impl '_ + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug + { self.successor_edges(v) .map(|edge_id| (edge_id, self.edges[edge_id].1)) } @@ -260,13 +269,8 @@ where pub fn predecessors( &self, v: V, - ) -> impl '_ - + Iterator - + DoubleEndedIterator - + ExactSizeIterator - + FusedIterator - + Clone - + Debug { + ) -> impl '_ + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug + { self.predecessor_edges(v) .map(|edge_id| (edge_id, self.edges[edge_id].0)) } diff --git a/hydroflow_lang/src/graph/eliminate_extra_unions_tees.rs b/hydroflow_lang/src/graph/eliminate_extra_unions_tees.rs index 5f3b07a2739..7fc5acc68dd 100644 --- a/hydroflow_lang/src/graph/eliminate_extra_unions_tees.rs +++ b/hydroflow_lang/src/graph/eliminate_extra_unions_tees.rs @@ -20,7 +20,8 @@ fn find_unary_ops<'a>( }) } -/// Removes missing unions and tees. Must be applied BEFORE subgraph partitioning. +/// Removes missing unions and tees. Must be applied BEFORE subgraph partitioning, i.e. on a flat +/// graph. pub fn eliminate_extra_unions_tees(graph: &mut HydroflowGraph) { let extra_ops = find_unary_ops(graph, UNION.name) .chain(find_unary_ops(graph, TEE.name)) diff --git a/hydroflow_lang/src/graph/flat_graph_builder.rs b/hydroflow_lang/src/graph/flat_graph_builder.rs index 3b72f15e2ac..d0d3a1b467b 100644 --- a/hydroflow_lang/src/graph/flat_graph_builder.rs +++ b/hydroflow_lang/src/graph/flat_graph_builder.rs @@ -1,11 +1,14 @@ +//! Build a flat graph from [`HfStatement`]s. + use std::borrow::Cow; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use proc_macro2::Span; use quote::ToTokens; use syn::spanned::Spanned; -use syn::Ident; +use syn::{Error, Ident, ItemUse}; use super::ops::find_op_op_constraints; use super::{GraphNodeId, HydroflowGraph, Node, PortIndexValue}; @@ -26,6 +29,7 @@ enum GraphDet { Undetermined(Ident), } +/// Wraper around [`HydroflowGraph`] to build a flat graph from AST code. #[derive(Debug, Default)] pub struct FlatGraphBuilder { /// Spanned error/warning/etc diagnostics to emit. @@ -38,6 +42,16 @@ pub struct FlatGraphBuilder { varname_ends: BTreeMap>, /// Each (out -> inn) link inputted. links: Vec, + + /// Use statements. + uses: Vec, + + /// In order to make import!() statements relative to the current file, we need to know where the file is that is building the flat graph. + macro_invocation_path: PathBuf, + + /// If the flat graph is being loaded as a module, then two initial ModuleBoundary nodes are inserted into the graph. One + /// for the input into the module and one for the output out of the module. + module_boundary_nodes: Option<(GraphNodeId, GraphNodeId)>, } impl FlatGraphBuilder { @@ -46,8 +60,44 @@ impl FlatGraphBuilder { Default::default() } - pub fn from_hfcode(input: HfCode) -> Self { - input.into() + /// Convert the Hydroflow code AST into a graph builder. + pub fn from_hfcode(input: HfCode, macro_invocation_path: PathBuf) -> Self { + let mut builder = Self { + macro_invocation_path, + ..Default::default() + }; + builder.process_statements(input.statements); + + builder + } + + /// Convert the Hydroflow code AST into a graph builder. + pub fn from_hfmodule(input: HfCode) -> Self { + let mut builder = Self::default(); + builder.module_boundary_nodes = Some(( + builder.flat_graph.insert_node( + Node::ModuleBoundary { + input: true, + import_expr: Span::call_site(), + }, + Some(Ident::new("input", Span::call_site())), + ), + builder.flat_graph.insert_node( + Node::ModuleBoundary { + input: false, + import_expr: Span::call_site(), + }, + Some(Ident::new("output", Span::call_site())), + ), + )); + builder.process_statements(input.statements); + builder + } + + fn process_statements(&mut self, statements: impl IntoIterator) { + for stmt in statements { + self.add_statement(stmt); + } } /// Build into an unpartitioned [`HydroflowGraph`], returning a tuple of a `HydroflowGraph` and @@ -55,18 +105,21 @@ impl FlatGraphBuilder { /// /// Even if there are errors, the `HydroflowGraph` will be returned (potentially in a invalid /// state). Does not call `emit` on any diagnostics. - pub fn build(mut self) -> (HydroflowGraph, Vec) { + pub fn build(mut self) -> (HydroflowGraph, Vec, Vec) { self.connect_operator_links(); self.process_operator_errors(); - (self.flat_graph, self.diagnostics) + (self.flat_graph, self.uses, self.diagnostics) } /// Add a single [`HfStatement`] line to this `HydroflowGraph`. pub fn add_statement(&mut self, stmt: HfStatement) { - let stmt_span = stmt.span(); match stmt { + HfStatement::Use(yuse) => { + self.uses.push(yuse); + } HfStatement::Named(named) => { + let stmt_span = named.span(); let ends = self.add_pipeline(named.pipeline, Some(&named.name)); match self.varname_ends.entry(named.name) { Entry::Vacant(vacant_entry) => { @@ -95,8 +148,8 @@ impl FlatGraphBuilder { } } } - HfStatement::Pipeline(pipeline) => { - self.add_pipeline(pipeline, None); + HfStatement::Pipeline(pipeline_stmt) => { + self.add_pipeline(pipeline_stmt.pipeline, None); } } } @@ -112,6 +165,7 @@ impl FlatGraphBuilder { } Pipeline::Name(pipeline_name) => { let (inn_port, ident, out_port) = PortIndexValue::from_ported(pipeline_name); + // We could lookup non-forward references immediately, but easier to just have one // consistent code path. -mingwei Ends { @@ -119,6 +173,29 @@ impl FlatGraphBuilder { out: Some((out_port, GraphDet::Undetermined(ident))), } } + Pipeline::ModuleBoundary(pipeline_name) => { + let Some((input_node, output_node)) = self.module_boundary_nodes else { + self.diagnostics.push( + Error::new( + pipeline_name.span(), + "mod is only usable inside of a module", + ) + .into(), + ); + + return Ends { + inn: None, + out: None, + }; + }; + + let (inn_port, _, out_port) = PortIndexValue::from_ported(pipeline_name); + + Ends { + inn: Some((inn_port, GraphDet::Determined(output_node))), + out: Some((out_port, GraphDet::Determined(input_node))), + } + } Pipeline::Link(pipeline_link) => { // Add the nested LHS and RHS of this link. let lhs_ends = self.add_pipeline(*pipeline_link.lhs, current_varname); @@ -147,9 +224,114 @@ impl FlatGraphBuilder { out: Some((PortIndexValue::Elided(op_span), GraphDet::Determined(nid))), } } + Pipeline::Import(import) => { + // TODO: https://github.com/rust-lang/rfcs/pull/3200 + // this would be way better... + let mut dir = self.macro_invocation_path.clone(); + dir.pop(); + + let file_contents = match std::fs::read_to_string(dir.join(import.filename.value())) + { + Ok(contents) => contents, + Err(err) => { + self.diagnostics.push(Diagnostic::spanned( + import.filename.span(), + Level::Error, + format!("filename: {}, err: {err}", import.filename.value()), + )); + + return Ends { + inn: None, + out: None, + }; + } + }; + + let statements = match syn::parse_str::(&file_contents) { + Ok(code) => code, + Err(err) => { + self.diagnostics.push(Diagnostic::spanned( + import.span(), + Level::Error, + err.to_string(), + )); + + return Ends { + inn: None, + out: None, + }; + } + }; + + let flat_graph_builder = crate::graph::FlatGraphBuilder::from_hfmodule(statements); + let (flat_graph, _uses, diagnostics) = flat_graph_builder.build(); + diagnostics + .iter() + .for_each(crate::diagnostic::Diagnostic::emit); + + self.merge_in(flat_graph, import.span()) + } } } + /// Merge one flatgraph into the current flatgraph + /// other must be a flatgraph and not be partitioned yet. + fn merge_in(&mut self, other: HydroflowGraph, parent_span: Span) -> Ends { + assert_eq!(other.subgraphs().count(), 0); + + let mut ends = Ends { + inn: None, + out: None, + }; + + let mut node_mapping = BTreeMap::new(); + + for (nid, node) in other.nodes() { + match node { + Node::Operator(_) => { + let varname = other.node_varname(nid); + let new_id = self.flat_graph.insert_node(node.clone(), varname); + node_mapping.insert(nid, new_id); + } + Node::ModuleBoundary { input, .. } => { + let new_id = self.flat_graph.insert_node( + Node::ModuleBoundary { + input: *input, + import_expr: parent_span, + }, + Some(Ident::new( + &format!("module_{}", input.to_string()), + parent_span, + )), + ); + node_mapping.insert(nid, new_id); + + if *input { + ends.inn = + Some((PortIndexValue::Elided(None), GraphDet::Determined(new_id))); + } else { + ends.out = + Some((PortIndexValue::Elided(None), GraphDet::Determined(new_id))); + } + } + Node::Handoff { .. } => panic!("Handoff in graph that is being merged into self"), + } + } + + for (eid, (src, dst)) in other.edges() { + let (src_port, dst_port) = other.edge_ports(eid); + + self.flat_graph.insert_edge( + *node_mapping.get(&src).unwrap(), + src_port.clone(), + *node_mapping.get(&dst).unwrap(), + dst_port.clone(), + ); + } + + ends + } + /// Connects operator links as a final building step. Processes all the links stored in /// `self.links` and actually puts them into the graph. fn connect_operator_links(&mut self) { @@ -322,7 +504,9 @@ impl FlatGraphBuilder { for (node_id, node) in self.flat_graph.nodes() { match node { Node::Operator(operator) => { - let Some(op_constraints) = find_op_op_constraints(operator) else { continue }; + let Some(op_constraints) = find_op_op_constraints(operator) else { + continue; + }; // Check number of args if op_constraints.num_args != operator.args.len() { self.diagnostics.push(Diagnostic::spanned( @@ -503,6 +687,9 @@ impl FlatGraphBuilder { ); } Node::Handoff { .. } => todo!("Node::Handoff"), + Node::ModuleBoundary { .. } => { + // Module boundaries don't require any checking. + } } } } @@ -565,13 +752,3 @@ impl FlatGraphBuilder { } } } - -impl From for FlatGraphBuilder { - fn from(input: HfCode) -> Self { - let mut builder = Self::default(); - for stmt in input.statements { - builder.add_statement(stmt); - } - builder - } -} diff --git a/hydroflow_lang/src/graph/flat_to_partitioned.rs b/hydroflow_lang/src/graph/flat_to_partitioned.rs index c0af4b7fe49..dfe3b6befbf 100644 --- a/hydroflow_lang/src/graph/flat_to_partitioned.rs +++ b/hydroflow_lang/src/graph/flat_to_partitioned.rs @@ -1,3 +1,5 @@ +//! Subgraph partioning algorithm + use std::collections::{BTreeMap, BTreeSet}; use proc_macro2::Span; @@ -26,7 +28,7 @@ fn find_barrier_crossers( } fn find_subgraph_unionfind( - partitioned_graph: &mut HydroflowGraph, + partitioned_graph: &HydroflowGraph, barrier_crossers: &SecondaryMap, ) -> (UnionFind, BTreeSet) { // Modality (color) of nodes, push or pull. @@ -99,7 +101,7 @@ fn find_subgraph_unionfind( /// after handoffs have already been inserted to partition subgraphs. /// This list of nodes in each subgraph are returned in topological sort order. fn make_subgraph_collect( - partitioned_graph: &mut HydroflowGraph, + partitioned_graph: &HydroflowGraph, mut subgraph_unionfind: UnionFind, ) -> SecondaryMap> { // We want the nodes of each subgraph to be listed in topo-sort order. @@ -248,7 +250,7 @@ fn find_subgraph_strata( barrier_crossers: &SecondaryMap, ) -> Result<(), Diagnostic> { // Determine subgraphs's stratum number. - // Find SCCs ignoring `next_tick()` (`DelayType::Tick`) edges, then do TopoSort on the + // Find SCCs ignoring `defer_tick()` (`DelayType::Tick`) edges, then do TopoSort on the // resulting DAG. // Cycles thru cross-stratum negative edges (both `DelayType::Tick` and `DelayType::Stratum`) // are an error. @@ -259,7 +261,7 @@ fn find_subgraph_strata( let mut subgraph_preds: BTreeMap> = Default::default(); let mut subgraph_succs: BTreeMap> = Default::default(); - // Negative (next stratum) connections between subgraphs. (Ignore `next_tick()` connections). + // Negative (next stratum) connections between subgraphs. (Ignore `defer_tick()` connections). let mut subgraph_negative_connections: BTreeSet<(GraphSubgraphId, GraphSubgraphId)> = Default::default(); @@ -288,29 +290,14 @@ fn find_subgraph_strata( } } - let scc = graph_algorithms::scc_kosaraju( - partitioned_graph.subgraph_ids(), + // Topological sort (of strongly connected components) is how we find the (nondecreasing) + // order of strata. + let topo_sort_order = graph_algorithms::topo_sort_scc( + || partitioned_graph.subgraph_ids(), |v| subgraph_preds.get(&v).into_iter().flatten().cloned(), |u| subgraph_succs.get(&u).into_iter().flatten().cloned(), ); - // Topological sort is how we find the (nondecreasing) order of strata. - let topo_sort_order = { - // Condensed each SCC into a single node for toposort. - let mut condensed_preds: BTreeMap> = - Default::default(); - for (u, preds) in subgraph_preds.iter() { - condensed_preds - .entry(scc[u]) - .or_default() - .extend(preds.iter().map(|v| scc[v])); - } - - graph_algorithms::topo_sort(partitioned_graph.subgraph_ids(), |v| { - condensed_preds.get(&v).into_iter().flatten().cloned() - }) - }; - // Each subgraph's stratum number is the same as it's predecessors. Unless there is a negative // edge, then we increment. for sg_id in topo_sort_order { @@ -332,8 +319,8 @@ fn find_subgraph_strata( partitioned_graph.set_subgraph_stratum(sg_id, stratum); } - // Re-introduce the `next_tick()` edges, ensuring they actually go to the next tick. - let extra_stratum = partitioned_graph.max_stratum().unwrap_or(0) + 1; // Used for `next_tick()` delayer subgraphs. + // Re-introduce the `defer_tick()` edges, ensuring they actually go to the next tick. + let extra_stratum = partitioned_graph.max_stratum().unwrap_or(0) + 1; // Used for `defer_tick()` delayer subgraphs. for (edge_id, &delay_type) in barrier_crossers.iter() { let (hoff, dst) = partitioned_graph.edge(edge_id); let (_hoff_port, dst_port) = partitioned_graph.edge_ports(edge_id); @@ -385,7 +372,7 @@ fn find_subgraph_strata( // Any negative edges which go onto the same or previous stratum are bad. // Indicates an unbroken negative cycle. if dst_stratum <= src_stratum { - return Err(Diagnostic::spanned(dst_port.span(), Level::Error, "Negative edge creates a negative cycle which must be broken with a `next_tick()` operator.")); + return Err(Diagnostic::spanned(dst_port.span(), Level::Error, "Negative edge creates a negative cycle which must be broken with a `defer_tick()` operator.")); } } } @@ -439,6 +426,9 @@ fn separate_external_inputs(partitioned_graph: &mut HydroflowGraph) { } } +/// Main method for this module. Partions a flat [`HydroflowGraph`] into one with subgraphs. +/// +/// Returns an error if a negative cycle exists in the graph. Negative cycles prevent partioning. pub fn partition_graph(flat_graph: HydroflowGraph) -> Result { let mut partitioned_graph = flat_graph; let mut barrier_crossers = find_barrier_crossers(&partitioned_graph); diff --git a/hydroflow_lang/src/graph/flow_props.rs b/hydroflow_lang/src/graph/flow_props.rs new file mode 100644 index 00000000000..87985413b53 --- /dev/null +++ b/hydroflow_lang/src/graph/flow_props.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; + +/// Stream and lattice properties. Used to determine correctness for scaling transformations. +#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize)] +pub struct FlowProps { + /// An abstract token representing the "order" and provenance of a flow. + /// + /// TODO(mingwei): may have richer order info later + pub star_ord: usize, + /// The lattice flow type (for lattice flows) or `None` for sequential dataflow. + pub lattice_flow_type: Option, +} + +/// Type of lattice flow. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)] +pub enum LatticeFlowType { + /// Delta: Elements are (generally) disjoint, each new element represents incremental progress. + Delta, + /// Cumulative: Each element must be greater than or equal to the previous. Used for monotonic + /// functions such as thresholding. + Cumul, +} + +impl LatticeFlowType { + /// If it is always correct to downcast a stream flow type from `from` to `to`. + pub fn can_downcast(from: Option, to: Option) -> bool { + from >= to + } +} diff --git a/hydroflow_lang/src/graph/graph_algorithms.rs b/hydroflow_lang/src/graph/graph_algorithms.rs index 4d23f5f48fa..96ec1e357df 100644 --- a/hydroflow_lang/src/graph/graph_algorithms.rs +++ b/hydroflow_lang/src/graph/graph_algorithms.rs @@ -1,6 +1,48 @@ +//! General graph algorithm utility functions + use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet}; +/// Computers the topological sort of the nodes of a possibly cyclic graph by ordering strongly +/// connected components together. +pub fn topo_sort_scc( + mut nodes_fn: NodesFn, + mut preds_fn: PredsFn, + succs_fn: SuccsFn, +) -> Vec +where + Id: Copy + Eq + Ord, + NodesFn: FnMut() -> NodeIds, + NodeIds: IntoIterator, + PredsFn: FnMut(Id) -> PredsIter, + SuccsFn: FnMut(Id) -> SuccsIter, + PredsIter: IntoIterator, + SuccsIter: IntoIterator, +{ + let scc = scc_kosaraju((nodes_fn)(), &mut preds_fn, succs_fn); + let topo_sort_order = { + // Condensed each SCC into a single node for toposort. + let mut condensed_preds: BTreeMap> = Default::default(); + for u in (nodes_fn)() { + condensed_preds + .entry(scc[&u]) + .or_default() + .extend((preds_fn)(u).into_iter().map(|v| scc[&v])); + } + + topo_sort((nodes_fn)(), |v| { + condensed_preds.get(&v).into_iter().flatten().cloned() + }) + }; + topo_sort_order +} + +/// Topologically sorts a set of nodes. Returns a list where the order of `Id`s will agree with +/// the order of any path through the graph. +/// +/// This naturally requires a directed acyclic graph (DAG). +/// +/// pub fn topo_sort( node_ids: NodeIds, mut preds_fn: PredsFn, @@ -40,6 +82,16 @@ where order } +/// Finds the strongly connected components in the graph. A strongly connected component is a +/// subset of nodes that are all reachable by each other. +/// +/// +/// +/// Each component is represented by a specific member node. The returned `BTreeMap` maps each node +/// ID to the node ID of its "representative." Nodes with the same "representative" node are in the +/// same strongly connected component. +/// +/// This function uses [Kosaraju's algorithm](https://en.wikipedia.org/wiki/Kosaraju%27s_algorithm). pub fn scc_kosaraju( nodes: NodeIds, mut preds_fn: PredsFn, diff --git a/hydroflow_lang/src/graph/graph_write.rs b/hydroflow_lang/src/graph/graph_write.rs index 1dce9382115..42e28c4278e 100644 --- a/hydroflow_lang/src/graph/graph_write.rs +++ b/hydroflow_lang/src/graph/graph_write.rs @@ -1,12 +1,13 @@ #![warn(missing_docs)] +use std::borrow::Cow; use std::error::Error; use auto_impl::auto_impl; use slotmap::Key; use super::ops::DelayType; -use super::{Color, GraphNodeId, GraphSubgraphId}; +use super::{Color, FlowProps, GraphNodeId, GraphSubgraphId, LatticeFlowType}; /// Trait for writing textual representations of graphs, i.e. mermaid or dot graphs. #[auto_impl(&mut, Box)] @@ -37,6 +38,7 @@ pub trait GraphWrite { src_id: GraphNodeId, dst_id: GraphNodeId, delay_type: Option, + flow_props: Option, label: Option<&str>, in_subgraph: Option, ) -> Result<(), Self::Err>; @@ -166,6 +168,7 @@ where src_id: GraphNodeId, dst_id: GraphNodeId, delay_type: Option, + flow_props: Option, label: Option<&str>, in_subgraph: Option, ) -> Result<(), Self::Err> { @@ -173,22 +176,23 @@ where let dest_str = format!("{:?}", dst_id.data()); writeln!( self.write, - "{:t$}{src}{label}{delay}{dst}", + "{:t$}{src}{arrow_body}{arrow_head}{label}{dst}", "", src = src_str.trim(), - label = if let Some(label) = &label { - if Some(DelayType::Stratum) == delay_type { - format!("=={}", label.trim()) - } else { - format!("--{}", label.trim()) - } - } else { - "".to_string() + arrow_body = match flow_props.and_then(|flow_props| flow_props.lattice_flow_type) { + None => "--", + Some(LatticeFlowType::Delta) => "-.-", + Some(LatticeFlowType::Cumul) => "==", + }, + arrow_head = match delay_type { + None => ">", + Some(DelayType::Stratum) => "x", + Some(DelayType::Tick) => "o", }, - delay = if Some(DelayType::Stratum) == delay_type { - "===o" + label = if let Some(label) = &label { + Cow::Owned(format!("|{}|", label.trim())) } else { - "--->" + Cow::Borrowed("") }, dst = dest_str.trim(), t = if in_subgraph.is_some() { 4 } else { 0 }, @@ -316,29 +320,49 @@ where src_id: GraphNodeId, dst_id: GraphNodeId, delay_type: Option, + flow_props: Option, label: Option<&str>, in_subgraph: Option, ) -> Result<(), Self::Err> { - let mut properties = Vec::new(); + // TODO(mingwei): handle `flow_props`. + let mut properties = Vec::>::new(); if let Some(label) = label { - properties.push(format!("label=\"{}\"", label)); + properties.push(format!("label=\"{}\"", label).into()); }; - if Some(DelayType::Stratum) == delay_type { - properties.push("arrowhead=box, color=red".to_string()); + match delay_type { + Some(DelayType::Stratum) => { + properties.push("arrowhead=box, color=red".into()); + } + Some(DelayType::Tick) => { + properties.push("arrowhead=dot, color=red".into()); + } + None => (), }; - writeln!( + match flow_props.and_then(|flow_props| flow_props.lattice_flow_type) { + Some(LatticeFlowType::Delta) => { + properties.push("style=dashed".into()); + } + Some(LatticeFlowType::Cumul) => { + properties.push("style=bold".into()); + } + None => (), + } + write!( self.write, - "{:t$}n{:?} -> n{:?}{}", + "{:t$}n{:?} -> n{:?}", "", src_id.data(), dst_id.data(), - if !properties.is_empty() { - format!(" [{}]", properties.join(", ")) - } else { - "".to_string() - }, t = if in_subgraph.is_some() { 8 } else { 4 }, )?; + if !properties.is_empty() { + write!(self.write, " [")?; + for prop in itertools::Itertools::intersperse(properties.into_iter(), ", ".into()) { + write!(self.write, "{}", prop)?; + } + write!(self.write, "]")?; + } + writeln!(self.write)?; Ok(()) } diff --git a/hydroflow_lang/src/graph/hydroflow_graph.rs b/hydroflow_lang/src/graph/hydroflow_graph.rs index 1a8aa63fb1b..24d1c153b02 100644 --- a/hydroflow_lang/src/graph/hydroflow_graph.rs +++ b/hydroflow_lang/src/graph/hydroflow_graph.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fmt::Debug; use std::iter::FusedIterator; +use itertools::Itertools; use proc_macro2::{Ident, Literal, Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, ToTokens}; use serde::{Deserialize, Serialize}; @@ -13,15 +14,16 @@ use syn::spanned::Spanned; use super::graph_write::{Dot, GraphWrite, Mermaid}; use super::ops::{find_op_op_constraints, OperatorWriteOutput, WriteContextArgs, OPERATORS}; use super::{ - get_operator_generics, node_color, Color, DiMulGraph, GraphEdgeId, GraphNodeId, + get_operator_generics, node_color, Color, DiMulGraph, FlowProps, GraphEdgeId, GraphNodeId, GraphSubgraphId, Node, OperatorInstance, PortIndexValue, Varname, CONTEXT, HANDOFF_NODE_STR, HYDROFLOW, }; use crate::diagnostic::{Diagnostic, Level}; use crate::graph::ops::null_write_iterator_fn; +use crate::graph::MODULE_BOUNDARY_NODE_STR; use crate::pretty_span::{PrettyRowCol, PrettySpan}; -/// A graph representing a hydroflow dataflow graph (with or without subgraph partitioning, +/// A graph representing a Hydroflow dataflow graph (with or without subgraph partitioning, /// stratification, and handoff insertion). This is a "meta" graph used for generating Rust source /// code in macros from Hydroflow surface sytnax. /// @@ -51,6 +53,9 @@ pub struct HydroflowGraph { /// What variable name each graph node belongs to (if any). node_varnames: SparseSecondaryMap, + + /// Stream properties. + flow_props: SecondaryMap, } impl HydroflowGraph { /// Create a new empty `HydroflowGraph`. @@ -73,6 +78,11 @@ impl HydroflowGraph { self.operator_instances.get(node_id) } + /// Get the debug variable name attached to a graph node. + pub fn node_varname(&self, node_id: GraphNodeId) -> Option { + self.node_varnames.get(node_id).map(|x| x.0.clone()) + } + /// Get subgraph for node. pub fn node_subgraph(&self, node_id: GraphNodeId) -> Option { self.node_subgraph.get(node_id).copied() @@ -93,8 +103,7 @@ impl HydroflowGraph { &self, src: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone @@ -107,8 +116,7 @@ impl HydroflowGraph { &self, dst: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone @@ -121,8 +129,7 @@ impl HydroflowGraph { &self, src: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone @@ -133,15 +140,14 @@ impl HydroflowGraph { /// Predecessor edges, iterator of `GraphEdgeId` of incoming edges. pub fn node_predecessor_edges( &self, - src: GraphNodeId, + dst: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug { - self.graph.predecessor_edges(src) + self.graph.predecessor_edges(dst) } /// Successor nodes, iterator of `GraphNodeId`. @@ -149,8 +155,7 @@ impl HydroflowGraph { &self, src: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone @@ -161,15 +166,14 @@ impl HydroflowGraph { /// Predecessor edges, iterator of `GraphNodeId`. pub fn node_predecessor_nodes( &self, - src: GraphNodeId, + dst: GraphNodeId, ) -> impl '_ - + Iterator - + DoubleEndedIterator + + DoubleEndedIterator + ExactSizeIterator + FusedIterator + Clone + Debug { - self.graph.predecessor_vertices(src) + self.graph.predecessor_vertices(dst) } /// Iterator of node IDs `GraphNodeId`. @@ -316,8 +320,12 @@ impl HydroflowGraph { // Make corresponding operator instance (if `node` is an operator). let op_inst_opt = 'oc: { - let Node::Operator(operator) = &new_node else { break 'oc None; }; - let Some(op_constraints) = find_op_op_constraints(operator) else { break 'oc None; }; + let Node::Operator(operator) = &new_node else { + break 'oc None; + }; + let Some(op_constraints) = find_op_op_constraints(operator) else { + break 'oc None; + }; let (input_port, output_port) = self.ports.get(edge_id).cloned().unwrap(); let generics = get_operator_generics( &mut Vec::new(), // TODO(mingwei) diagnostics @@ -382,6 +390,100 @@ impl HydroflowGraph { let (_, dst_port) = self.ports.remove(succ_edge_id).unwrap(); self.ports.insert(new_edge_id, (src_port, dst_port)); } + + /// When modules are imported into a flat graph, they come with an input and output ModuleBoundary node. + /// The partitioner doesn't understand these nodes and will panic if it encounters them. + /// merge_modules removes them from the graph, stitching the input and ouput sides of the ModuleBondaries based on their ports + /// For example: + /// source_iter([]) -> \[myport\]ModuleBoundary(input)\[my_port\] -> map(|x| x) -> ModuleBoundary(output) -> null(); + /// in the above eaxmple, the \[myport\] port will be used to connect the source_iter with the map that is inside of the module. + /// The output module boundary has elided ports, this is also used to match up the input/output across the module boundary. + pub fn merge_modules(&mut self) -> Result<(), Diagnostic> { + let mut to_remove = Vec::new(); + + for (nid, node) in self.nodes() { + if matches!(node, Node::ModuleBoundary { .. }) { + to_remove.push(nid); + } + } + + for nid in to_remove { + self.remove_module_boundary(nid)?; + } + + Ok(()) + } + + /// see `merge_modules` + /// This function removes a singular module boundary from the graph and performs the necessary stitching to fix the graph aftward. + /// `merge_modules` calls this function for each module boundary in the graph. + fn remove_module_boundary(&mut self, nid: GraphNodeId) -> Result<(), Diagnostic> { + assert!( + self.node_subgraph.is_empty() && self.subgraph_nodes.is_empty(), + "Should not remove intermediate node after subgraph partitioning" + ); + + let mut predecessor_ports = BTreeMap::new(); + let mut successor_ports = BTreeMap::new(); + + for eid in self.node_predecessor_edges(nid) { + let (predecessor_port, successor_port) = self.edge_ports(eid); + predecessor_ports.insert(successor_port.clone(), (eid, predecessor_port.clone())); + } + + for eid in self.node_successor_edges(nid) { + let (predecessor_port, successor_port) = self.edge_ports(eid); + successor_ports.insert(predecessor_port.clone(), (eid, successor_port.clone())); + } + + if predecessor_ports.keys().collect::>() + != successor_ports.keys().collect::>() + { + // get module boundary node + match self.node(nid) { + Node::ModuleBoundary { input, import_expr } => { + if *input { + return Err(Diagnostic { + span: *import_expr, + level: Level::Error, + message: format!( + "The ports into the module did not match. input: {:?}, expected: {:?}", + predecessor_ports.keys().map(|x| x.to_string()).join(", "), + successor_ports.keys().map(|x| x.to_string()).join(", ") + ), + }); + } else { + return Err(Diagnostic { + span: *import_expr, + level: Level::Error, + message: format!("The ports out of the module did not match. output: {:?}, expected: {:?}", + successor_ports.keys().map(|x| x.to_string()).join(", "), + predecessor_ports.keys().map(|x| x.to_string()).join(", "), + )}); + } + } + _ => panic!(), + } + } + + for (port, (predecessor_edge, predecessor_port)) in predecessor_ports { + let (successor_edge, successor_port) = successor_ports.remove(&port).unwrap(); + + let (src, _) = self.graph.remove_edge(predecessor_edge).unwrap(); + let (_, dst) = self.graph.remove_edge(successor_edge).unwrap(); + + self.ports.remove(predecessor_edge); + self.ports.remove(successor_edge); + + let eid = self.graph.insert_edge(src, dst); + self.ports.insert(eid, (predecessor_port, successor_port)); + } + + self.graph.remove_vertex(nid); + self.nodes.remove(nid); + + Ok(()) + } } // Edge methods. impl HydroflowGraph { @@ -406,8 +508,7 @@ impl HydroflowGraph { pub fn edges( &self, ) -> impl '_ - + Iterator - + ExactSizeIterator + + ExactSizeIterator + FusedIterator + Clone + Debug { @@ -494,6 +595,24 @@ impl HydroflowGraph { self.subgraph_stratum.values().copied().max() } } +// Flow properties +impl HydroflowGraph { + /// Gets the flow properties associated with the edge, if set. + pub fn edge_flow_props(&self, edge_id: GraphEdgeId) -> Option { + self.flow_props.get(edge_id).copied() + } + + /// Sets the flow properties associated with the given edge. + /// + /// Returns the old flow properties, if set. + pub fn set_edge_flow_props( + &mut self, + edge_id: GraphEdgeId, + flow_props: FlowProps, + ) -> Option { + self.flow_props.insert(edge_id, flow_props) + } +} // Display/output stuff. impl HydroflowGraph { /// Helper to generate a deterministic `Ident` for the given node. @@ -505,11 +624,13 @@ impl HydroflowGraph { node_id.data(), if is_pred { "recv" } else { "send" } ), + Node::ModuleBoundary { .. } => panic!(), }; let span = match (is_pred, &self.nodes[node_id]) { (_, Node::Operator(operator)) => operator.span(), (true, &Node::Handoff { src_span, .. }) => src_span, (false, &Node::Handoff { dst_span, .. }) => dst_span, + (_, Node::ModuleBoundary { .. }) => panic!(), }; Ident::new(&*name, span) } @@ -554,6 +675,7 @@ impl HydroflowGraph { &self, root: &TokenStream, include_type_guards: bool, + prefix: TokenStream, diagnostics: &mut Vec, ) -> TokenStream { let hf = Ident::new(HYDROFLOW, Span::call_site()); @@ -565,6 +687,7 @@ impl HydroflowGraph { .filter_map(|(node_id, node)| match node { Node::Operator(_) => None, &Node::Handoff { src_span, dst_span } => Some((node_id, (src_span, dst_span))), + Node::ModuleBoundary { .. } => panic!(), }) .map(|(node_id, (src_span, dst_span))| { let ident_send = Ident::new(&*format!("hoff_{:?}_send", node_id.data()), dst_span); @@ -680,6 +803,11 @@ impl HydroflowGraph { .map(|&(_port, succ)| self.node_as_ident(succ, false)) .collect(); + // Corresponds 1:1 to inputs. + let flow_props_in = self.graph.predecessor_edges(node_id) + .map(|edge_id| self.flow_props.get(edge_id).copied()) + .collect::>(); + let is_pull = idx < pull_to_push_idx; let context_args = WriteContextArgs { @@ -703,6 +831,7 @@ impl HydroflowGraph { outputs: &*outputs, op_name, op_inst, + flow_props_in: &*flow_props_in, }; let write_result = (op_constraints.write_fn)(&context_args, diagnostics); @@ -805,6 +934,8 @@ impl HydroflowGraph { { // Determine pull and push halves of the `Pivot`. + #[allow(unknown_lints)] + #[allow(clippy::redundant_locals)] // https://github.com/rust-lang/rust-clippy/issues/11290 let pull_to_push_idx = pull_to_push_idx; let pull_ident = self.node_as_ident(subgraph_nodes[pull_to_push_idx - 1], false); @@ -816,10 +947,9 @@ impl HydroflowGraph { self.node_as_ident(node_id, false) } else { // Entire subgraph is pull (except for a single send/push handoff output). - assert_eq!( - 1, - send_ports.len(), - "If entire subgraph is pull, should have only one handoff output. Do you have a loose `null()` or other degenerate pipeline somewhere?" + assert!( + 1 == send_ports.len(), + "Degenerate subgraph detected, is there a disconnected `null()` or other degenerate pipeline somewhere?" ); send_ports[0].clone() }; @@ -881,6 +1011,8 @@ impl HydroflowGraph { { #[allow(unused_qualifications)] { + #prefix + use #root::{var_expr, var_args}; let mut #hf = #root::scheduled::graph::Hydroflow::new(); @@ -998,7 +1130,9 @@ impl HydroflowGraph { let mut sg_varname_nodes = SparseSecondaryMap::>>::new(); for (node_id, varname) in self.node_varnames.iter() { - let Some(sg_id) = self.node_subgraph(node_id) else { continue; }; + let Some(sg_id) = self.node_subgraph(node_id) else { + continue; + }; let varname_map = sg_varname_nodes.entry(sg_id).unwrap().or_default(); varname_map .entry(varname.clone()) @@ -1052,11 +1186,13 @@ impl HydroflowGraph { let delay_type = self .node_op_inst(dst_id) .and_then(|op_inst| (op_inst.op_constraints.input_delaytype_fn)(dst_port)); + let flow_props = self.edge_flow_props(edge_id); let label = helper_edge_label(src_port, dst_port); graph_write.write_edge( hoff_id, dst_id, delay_type, + flow_props, label.as_deref(), Some(subgraph_id), )?; @@ -1070,11 +1206,13 @@ impl HydroflowGraph { let delay_type = self.node_op_inst(dst_id).and_then(|op_inst| { (op_inst.op_constraints.input_delaytype_fn)(dst_port) }); + let flow_props = self.edge_flow_props(edge_id); let label = helper_edge_label(src_port, dst_port); graph_write.write_edge( src_id, dst_id, delay_type, + flow_props, label.as_deref(), Some(subgraph_id), )?; @@ -1110,23 +1248,25 @@ impl HydroflowGraph { Color::Hoff, None, )?; - - // write out edge - let (src_port, dst_port) = self.edge_ports(edge_id); - let delay_type = self - .node_op_inst(dst_id) - .and_then(|op_inst| (op_inst.op_constraints.input_delaytype_fn)(dst_port)); - let label = helper_edge_label(src_port, dst_port); - graph_write.write_edge(src_id, dst_id, delay_type, label.as_deref(), None)?; - } else if barrier_handoffs.contains(&dst_id) { - // write out edge - let (src_port, dst_port) = self.edge_ports(edge_id); - let delay_type = self - .node_op_inst(dst_id) - .and_then(|op_inst| (op_inst.op_constraints.input_delaytype_fn)(dst_port)); - let label = helper_edge_label(src_port, dst_port); - graph_write.write_edge(src_id, dst_id, delay_type, label.as_deref(), None)?; + } else if !barrier_handoffs.contains(&dst_id) { + continue; } + + // write out edge + let (src_port, dst_port) = self.edge_ports(edge_id); + let delay_type = self + .node_op_inst(dst_id) + .and_then(|op_inst| (op_inst.op_constraints.input_delaytype_fn)(dst_port)); + let flow_props = self.edge_flow_props(edge_id); + let label = helper_edge_label(src_port, dst_port); + graph_write.write_edge( + src_id, + dst_id, + delay_type, + flow_props, + label.as_deref(), + None, + )?; } } @@ -1150,6 +1290,7 @@ impl HydroflowGraph { writeln!(write, "{:?} = {};", key.data(), op.to_token_stream())?; } Node::Handoff { .. } => unimplemented!("HANDOFF IN FLAT GRAPH."), + Node::ModuleBoundary { .. } => panic!(), } } writeln!(write)?; @@ -1189,6 +1330,14 @@ impl HydroflowGraph { Node::Handoff { .. } => { writeln!(write, r#" {:?}{{"{}"}}"#, key.data(), HANDOFF_NODE_STR) } + Node::ModuleBoundary { .. } => { + writeln!( + write, + r#" {:?}{{"{}"}}"#, + key.data(), + MODULE_BOUNDARY_NODE_STR + ) + } }?; } writeln!(write)?; diff --git a/hydroflow_lang/src/graph/mod.rs b/hydroflow_lang/src/graph/mod.rs index 324ec37e2f7..d5b6869c190 100644 --- a/hydroflow_lang/src/graph/mod.rs +++ b/hydroflow_lang/src/graph/mod.rs @@ -20,17 +20,23 @@ mod di_mul_graph; mod eliminate_extra_unions_tees; mod flat_graph_builder; mod flat_to_partitioned; +mod flow_props; mod graph_write; mod hydroflow_graph; +use std::fmt::Display; +use std::path::PathBuf; + pub use di_mul_graph::DiMulGraph; pub use eliminate_extra_unions_tees::eliminate_extra_unions_tees; pub use flat_graph_builder::FlatGraphBuilder; pub use flat_to_partitioned::partition_graph; +pub use flow_props::*; pub use hydroflow_graph::HydroflowGraph; pub mod graph_algorithms; pub mod ops; +pub mod propegate_flow_props; new_key_type! { /// ID to identify a node (operator or handoff) in [`HydroflowGraph`]. @@ -49,6 +55,7 @@ const CONTEXT: &str = "context"; const HYDROFLOW: &str = "df"; const HANDOFF_NODE_STR: &str = "handoff"; +const MODULE_BOUNDARY_NODE_STR: &str = "module_boundary"; mod serde_syn { use serde::{Deserialize, Deserializer, Serializer}; @@ -74,28 +81,49 @@ mod serde_syn { #[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, Ord, PartialEq, Eq)] struct Varname(#[serde(with = "serde_syn")] pub Ident); +/// A node, corresponding to an operator or a handoff. #[derive(Clone, Serialize, Deserialize)] pub enum Node { + /// An operator. Operator(#[serde(with = "serde_syn")] Operator), + /// A handoff point, used between subgraphs (or within a subgraph to break a cycle). Handoff { + /// The span of the input into the handoff. #[serde(skip, default = "Span::call_site")] src_span: Span, + /// The span of the output out of the handoff. #[serde(skip, default = "Span::call_site")] dst_span: Span, }, + + /// Module Boundary, used for importing modules. Only exists prior to partitioning. + ModuleBoundary { + /// If this module is an input or output boundary. + input: bool, + + /// The span of the import!() expression that imported this module. + /// The value of this span when the ModuleBoundary node is still inside the module is Span::call_site() + /// TODO: This could one day reference into the module file itself? + #[serde(skip, default = "Span::call_site")] + import_expr: Span, + }, } impl Node { + /// Return the node as a human-readable string. pub fn to_pretty_string(&self) -> Cow<'static, str> { match self { Node::Operator(op) => op.to_pretty_string().into(), Node::Handoff { .. } => HANDOFF_NODE_STR.into(), + Node::ModuleBoundary { .. } => MODULE_BOUNDARY_NODE_STR.into(), } } + /// Return the source code span of the node (for operators) or input/otput spans for handoffs. pub fn span(&self) -> Span { match self { Self::Operator(op) => op.span(), &Self::Handoff { src_span, dst_span } => src_span.join(dst_span).unwrap_or(src_span), + Self::ModuleBoundary { import_expr, .. } => *import_expr, } } } @@ -106,10 +134,21 @@ impl std::fmt::Debug for Node { write!(f, "Node::Operator({} span)", PrettySpan(operator.span())) } Self::Handoff { .. } => write!(f, "Node::Handoff"), + Self::ModuleBoundary { input, .. } => { + write!(f, "Node::ModuleBoundary{{input: {}}}", input) + } } } } +/// Meta-data relating to operators which may be useful throughout the compilation process. +/// +/// This data can be generated from the graph, but it is useful to have it readily available +/// pre-computed as many algorithms use the same info. Stuff like port names, arguments, and the +/// [`OperatorConstraints`] for the operator. +/// +/// Because it is derived from the graph itself, there can be "cache invalidation"-esque issues +/// if this data is not kept in sync with the graph. #[derive(Clone, Debug)] pub struct OperatorInstance { /// Name of the operator (will match [`OperatorConstraints::name`]). @@ -128,6 +167,7 @@ pub struct OperatorInstance { pub arguments: Punctuated, } +/// Operator generic arguments, split into specific categories. #[derive(Clone, Debug)] pub struct OpInstGenerics { /// Operator generic (type or lifetime) arguments. @@ -138,6 +178,9 @@ pub struct OpInstGenerics { pub type_args: Vec, } +/// Gets the generic arguments for the operator. This helper method is here due to the special +/// handling of persistence lifetimes (`'static`, `'tick`, `'mutable`) which must come before +/// other generic parameters. pub fn get_operator_generics( diagnostics: &mut Vec, operator: &Operator, @@ -181,6 +224,7 @@ pub fn get_operator_generics( } } +/// Push, Pull, Comp, or Hoff polarity. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub enum Color { /// Pull (green) @@ -217,11 +261,17 @@ pub fn node_color(is_handoff: bool, inn_degree: usize, out_degree: usize) -> Opt /// Helper struct for [`PortIndex`] which keeps span information for elided ports. #[derive(Clone, Debug, Serialize, Deserialize)] pub enum PortIndexValue { + /// An integer value: `[0]`, `[1]`, etc. Can be negative although we don't use that (2023-08-16). Int(#[serde(with = "serde_syn")] IndexInt), + /// A name or path. `[pos]`, `[neg]`, etc. Can use `::` separators but we don't use that (2023-08-16). Path(#[serde(with = "serde_syn")] ExprPath), + /// Elided, unspecified port. We have this variant, rather than wrapping in `Option`, in order + /// to preserve the `Span` information. Elided(#[serde(skip)] Option), } impl PortIndexValue { + /// For a [`Ported`] value like `[port_in]name[port_out]`, get the `port_in` and `port_out` as + /// [`PortIndexValue`]s. pub fn from_ported(ported: Ported) -> (Self, Inner, Self) where Inner: Spanned, @@ -239,6 +289,7 @@ impl PortIndexValue { (port_inn, inner, port_out) } + /// Returns `true` if `self` is not [`PortIndexValue::Elided`]. pub fn is_specified(&self) -> bool { !matches!(self, Self::Elided(_)) } @@ -256,6 +307,7 @@ impl PortIndexValue { } } + /// Formats self as a human-readable string for error messages. pub fn as_error_message_string(&self) -> String { match self { PortIndexValue::Int(n) => format!("`{}`", n.value), @@ -264,6 +316,7 @@ impl PortIndexValue { } } + /// Returns the span of this port value. pub fn span(&self) -> Span { match self { PortIndexValue::Int(x) => x.span(), @@ -293,18 +346,7 @@ impl PartialEq for PortIndexValue { impl Eq for PortIndexValue {} impl PartialOrd for PortIndexValue { fn partial_cmp(&self, other: &Self) -> Option { - match (self, other) { - (Self::Int(s), Self::Int(o)) => s.partial_cmp(o), - (Self::Path(s), Self::Path(o)) => s - .to_token_stream() - .to_string() - .partial_cmp(&o.to_token_stream().to_string()), - (Self::Elided(_), Self::Elided(_)) => Some(std::cmp::Ordering::Equal), - (Self::Int(_), Self::Path(_)) => Some(std::cmp::Ordering::Less), - (Self::Path(_), Self::Int(_)) => Some(std::cmp::Ordering::Greater), - (_, Self::Elided(_)) => Some(std::cmp::Ordering::Less), - (Self::Elided(_), _) => Some(std::cmp::Ordering::Greater), - } + Some(self.cmp(other)) } } impl Ord for PortIndexValue { @@ -324,20 +366,50 @@ impl Ord for PortIndexValue { } } +impl Display for PortIndexValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PortIndexValue::Int(x) => write!(f, "{}", x.to_token_stream().to_string()), + PortIndexValue::Path(x) => write!(f, "{}", x.to_token_stream().to_string()), + PortIndexValue::Elided(_) => write!(f, "[]"), + } + } +} + +/// The main function of this module. Compiles a [`HfCode`] AST into a [`HydroflowGraph`] and +/// source code, or [`Diagnostic`] errors. pub fn build_hfcode( hf_code: HfCode, root: &TokenStream, + macro_invocation_path: PathBuf, ) -> (Option<(HydroflowGraph, TokenStream)>, Vec) { - let flat_graph_builder = FlatGraphBuilder::from_hfcode(hf_code); - let (mut flat_graph, mut diagnostics) = flat_graph_builder.build(); - eliminate_extra_unions_tees(&mut flat_graph); + let flat_graph_builder = FlatGraphBuilder::from_hfcode(hf_code, macro_invocation_path); + let (mut flat_graph, uses, mut diagnostics) = flat_graph_builder.build(); if !diagnostics.iter().any(Diagnostic::is_error) { + if let Err(diagnostic) = flat_graph.merge_modules() { + diagnostics.push(diagnostic); + return (None, diagnostics); + } + + eliminate_extra_unions_tees(&mut flat_graph); match partition_graph(flat_graph) { - Ok(part_graph) => { - let code = part_graph.as_code(root, true, &mut diagnostics); - if !diagnostics.iter().any(Diagnostic::is_error) { - // Success. - return (Some((part_graph, code)), diagnostics); + Ok(mut partitioned_graph) => { + // Propgeate flow properties throughout the graph. + // TODO(mingwei): Should this be done at a flat graph stage instead? + if let Ok(()) = propegate_flow_props::propegate_flow_props( + &mut partitioned_graph, + &mut diagnostics, + ) { + let code = partitioned_graph.as_code( + root, + true, + quote::quote! { #( #uses )* }, + &mut diagnostics, + ); + if !diagnostics.iter().any(Diagnostic::is_error) { + // Success. + return (Some((partitioned_graph, code)), diagnostics); + } } } Err(diagnostic) => diagnostics.push(diagnostic), diff --git a/hydroflow_lang/src/graph/ops/_lattice_fold_batch.rs b/hydroflow_lang/src/graph/ops/_lattice_fold_batch.rs new file mode 100644 index 00000000000..4d0ceefe5c9 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/_lattice_fold_batch.rs @@ -0,0 +1,112 @@ +use quote::{quote_spanned, ToTokens}; +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, PortListSpec, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::graph::{OpInstGenerics, OperatorInstance}; + +/// > 2 input streams, 1 output stream, no arguments. +/// +/// Batches streaming input and releases it downstream when a signal is delivered. This allows for buffering data and delivering it later while also folding it into a single lattice data structure. +/// This operator is similar to `defer_signal` in that it batches input and releases it when a signal is given. It is also similar to `lattice_fold` in that it folds the input into a single lattice. +/// So, `_lattice_fold_batch` is a combination of both `defer_signal` and `lattice_fold`. This operator is useful when trying to combine a sequence of `defer_signal` and `lattice_fold` operators without unnecessary memory consumption. +/// +/// There are two inputs to `_lattice_fold_batch`, they are `input` and `signal`. +/// `input` is the input data flow. Data that is delivered on this input is collected in order inside of the `_lattice_fold_batch` operator. +/// When anything is sent to `signal` the collected data is released downstream. The entire `signal` input is consumed each tick, so sending 5 things on `signal` will not release inputs on the next 5 consecutive ticks. +/// +/// ```hydroflow +/// use lattices::set_union::SetUnionHashSet; +/// use lattices::set_union::SetUnionSingletonSet; +/// +/// source_iter([1, 2, 3]) +/// -> map(SetUnionSingletonSet::new_from) +/// -> [input]batcher; +/// +/// source_iter([()]) +/// -> [signal]batcher; +/// +/// batcher = _lattice_fold_batch::>() +/// -> assert_eq([SetUnionHashSet::new_from([1, 2, 3])]); +/// ``` +pub const _LATTICE_FOLD_BATCH: OperatorConstraints = OperatorConstraints { + name: "_lattice_fold_batch", + categories: &[OperatorCategory::CompilerFusionOperator], + persistence_args: RANGE_0, + type_args: &(0..=1), + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + is_external_input: false, + ports_inn: Some(|| PortListSpec::Fixed(parse_quote! { input, signal })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::No, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + context, + hydroflow, + ident, + op_span, + root, + inputs, + is_pull, + op_inst: + OperatorInstance { + generics: OpInstGenerics { type_args, .. }, + .. + }, + .. + }, + _| { + assert!(is_pull); + + let lattice_type = type_args + .get(0) + .map(ToTokens::to_token_stream) + .unwrap_or(quote_spanned!(op_span=> _)); + + let lattice_ident = wc.make_ident("lattice"); + + let write_prologue = quote_spanned! {op_span=> + let #lattice_ident = #hydroflow.add_state(::std::cell::RefCell::new(<#lattice_type as ::std::default::Default>::default())); + }; + + let input = &inputs[0]; + let signal = &inputs[1]; + + let write_iterator = { + quote_spanned! {op_span=> + + { + let mut __lattice = #context.state_ref(#lattice_ident).borrow_mut(); + + for __item in #input { + #root::lattices::Merge::merge(&mut *__lattice, __item); + } + } + + let #ident = if #signal.count() > 0 { + ::std::option::Option::Some(#context.state_ref(#lattice_ident).take()) + } else { + ::std::option::Option::None + }.into_iter(); + } + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after: Default::default(), + ..Default::default() + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/_lattice_join_fused_join.rs b/hydroflow_lang/src/graph/ops/_lattice_join_fused_join.rs new file mode 100644 index 00000000000..b35caf78875 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/_lattice_join_fused_join.rs @@ -0,0 +1,112 @@ +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + WriteContextArgs, RANGE_1, +}; +use crate::graph::{OpInstGenerics, OperatorInstance}; + +/// > 2 input streams of type `(K, V1)` and `(K, V2)`, 1 output stream of type `(K, (V1', V2'))` where `V1`, `V2`, `V1'`, `V2'` are lattice types +/// +/// Performs a [`fold_keyed`](#fold_keyed) with lattice-merge aggregate function on each input and then forms the +/// equijoin of the resulting key/value pairs in the input streams by their first (key) attribute. Like [`join`](#join), the result nests the 2nd input field (values) into a tuple in the 2nd output field. +/// +/// You must specify the the accumulating lattice types, they cannot be inferred. The first type argument corresponds to the `[0]` input of the join, and the second to the `[1]` input. +/// Type arguments are specified in hydroflow using the rust turbofish syntax `::<>`, for example `_lattice_join_fused_join::, Max<_>>()` +/// The accumulating lattice type is not necessarily the same type as the input, see the below example involving SetUnion for such a case. +/// +/// Like [`join`](#join), `_lattice_join_fused_join` can also be provided with one or two generic lifetime persistence arguments, either +/// `'tick` or `'static`, to specify how join data persists. With `'tick`, pairs will only be +/// joined with corresponding pairs within the same tick. With `'static`, pairs will be remembered +/// across ticks and will be joined with pairs arriving in later ticks. When not explicitly +/// specified persistence defaults to `tick. +/// +/// Like [`join`](#join), when two persistence arguments are supplied the first maps to port `0` and the second maps to +/// port `1`. +/// When a single persistence argument is supplied, it is applied to both input ports. +/// When no persistence arguments are applied it defaults to `'tick` for both. +/// It is important to specify all persistence arguments before any type arguments, otherwise the persistence arguments will be ignored. +/// +/// The syntax is as follows: +/// ```hydroflow,ignore +/// _lattice_join_fused_join::, MaxRepr>(); // Or +/// _lattice_join_fused_join::<'static, MaxRepr, MaxRepr>(); +/// +/// _lattice_join_fused_join::<'tick, MaxRepr, MaxRepr>(); +/// +/// _lattice_join_fused_join::<'static, 'tick, MaxRepr, MaxRepr>(); +/// +/// _lattice_join_fused_join::<'tick, 'static, MaxRepr, MaxRepr>(); +/// // etc. +/// ``` +/// +/// ### Examples +/// +/// ```hydroflow +/// use hydroflow::lattices::Min; +/// use hydroflow::lattices::Max; +/// +/// source_iter([("key", Min::new(1)), ("key", Min::new(2))]) -> [0]my_join; +/// source_iter([("key", Max::new(1)), ("key", Max::new(2))]) -> [1]my_join; +/// +/// my_join = _lattice_join_fused_join::<'tick, Min, Max>() +/// -> assert_eq([("key", (Min::new(1), Max::new(2)))]); +/// ``` +/// +/// ```hydroflow +/// use hydroflow::lattices::set_union::SetUnionSingletonSet; +/// use hydroflow::lattices::set_union::SetUnionHashSet; +/// +/// source_iter([("key", SetUnionSingletonSet::new_from(0)), ("key", SetUnionSingletonSet::new_from(1))]) -> [0]my_join; +/// source_iter([("key", SetUnionHashSet::new_from([0])), ("key", SetUnionHashSet::new_from([1]))]) -> [1]my_join; +/// +/// my_join = _lattice_join_fused_join::<'tick, SetUnionHashSet, SetUnionHashSet>() +/// -> assert_eq([("key", (SetUnionHashSet::new_from([0, 1]), SetUnionHashSet::new_from([0, 1])))]); +/// ``` +pub const _LATTICE_JOIN_FUSED_JOIN: OperatorConstraints = OperatorConstraints { + name: "_lattice_join_fused_join", + categories: &[OperatorCategory::CompilerFusionOperator], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=2), + type_args: &(2..=2), + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + root, + op_inst: + op_inst @ OperatorInstance { + generics: OpInstGenerics { type_args, .. }, + .. + }, + .. + }, + diagnostics| { + let lhs_type = &type_args[0]; + let rhs_type = &type_args[1]; + + let wc = WriteContextArgs { + op_inst: &OperatorInstance { + arguments: parse_quote! { + FoldFrom(<#lhs_type as #root::lattices::LatticeFrom::<_>>::lattice_from, #root::lattices::Merge::merge), + FoldFrom(<#rhs_type as #root::lattices::LatticeFrom::<_>>::lattice_from, #root::lattices::Merge::merge) + }, + ..op_inst.clone() + }, + ..wc.clone() + }; + + (super::join_fused::JOIN_FUSED.write_fn)(&wc, diagnostics) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/anti_join.rs b/hydroflow_lang/src/graph/ops/anti_join.rs index 26bda91035f..78cc32dff15 100644 --- a/hydroflow_lang/src/graph/ops/anti_join.rs +++ b/hydroflow_lang/src/graph/ops/anti_join.rs @@ -3,21 +3,23 @@ use syn::parse_quote; use super::{ DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, - OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, + OperatorWriteOutput, Persistence, WriteContextArgs, RANGE_0, RANGE_1, }; -use crate::graph::PortIndexValue; +use crate::diagnostic::{Diagnostic, Level}; +use crate::graph::{OpInstGenerics, OperatorInstance, PortIndexValue}; /// > 2 input streams the first of type (K, T), the second of type K, /// > with output type (K, T) /// /// For a given tick, computes the anti-join of the items in the input -/// streams, returning items in the `pos` input that do not have matching keys -/// in the `neg` input. +/// streams, returning unique items in the `pos` input that do not have matching keys +/// in the `neg` input. Note this is set semantics, so duplicate items in the `pos` input +/// are output 0 or 1 times (if they do/do-not have a match in `neg` respectively.) /// /// ```hydroflow /// source_iter(vec![("dog", 1), ("cat", 2), ("elephant", 3)]) -> [pos]diff; /// source_iter(vec!["dog", "cat", "gorilla"]) -> [neg]diff; -/// diff = anti_join() -> assert([("elephant", 3)]); +/// diff = anti_join() -> assert_eq([("elephant", 3)]); /// ``` pub const ANTI_JOIN: OperatorConstraints = OperatorConstraints { name: "anti_join", @@ -27,7 +29,7 @@ pub const ANTI_JOIN: OperatorConstraints = OperatorConstraints { hard_range_out: RANGE_1, soft_range_out: RANGE_1, num_args: 0, - persistence_args: RANGE_0, + persistence_args: &(0..=2), type_args: RANGE_0, is_external_input: false, ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { pos, neg })), @@ -43,6 +45,7 @@ pub const ANTI_JOIN: OperatorConstraints = OperatorConstraints { } _else => None, }, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, @@ -50,48 +53,125 @@ pub const ANTI_JOIN: OperatorConstraints = OperatorConstraints { op_span, ident, inputs, + op_inst: + OperatorInstance { + generics: + OpInstGenerics { + persistence_args, .. + }, + .. + }, .. }, - _| { - let handle_ident = wc.make_ident("diffdata_handle"); + diagnostics| { + let persistences = match persistence_args[..] { + [] => [Persistence::Tick, Persistence::Tick], + [a] => [a, a], + [a, b] => [a, b], + _ => unreachable!(), + }; + + let mut make_antijoindata = |persistence, side| { + let antijoindata_ident = wc.make_ident(format!("antijoindata_{}", side)); + let borrow_ident = wc.make_ident(format!("antijoindata_{}_borrow", side)); + let (init, borrow) = match persistence { + Persistence::Tick => ( + quote_spanned! {op_span=> + #root::util::monotonic_map::MonotonicMap::<_, #root::rustc_hash::FxHashSet<_>>::default() + }, + quote_spanned! {op_span=> + &mut *#borrow_ident.get_mut_clear(#context.current_tick()) + }, + ), + Persistence::Static => ( + quote_spanned! {op_span=> + #root::rustc_hash::FxHashSet::default() + }, + quote_spanned! {op_span=> + &mut *#borrow_ident + }, + ), + Persistence::Mutable => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + } + }; + Ok((antijoindata_ident, borrow_ident, init, borrow)) + }; + + let (pos_antijoindata_ident, pos_borrow_ident, pos_init, pos_borrow) = + make_antijoindata(persistences[0], "pos")?; + let (neg_antijoindata_ident, neg_borrow_ident, neg_init, neg_borrow) = + make_antijoindata(persistences[1], "neg")?; + let tick_ident = wc.make_ident("persisttick"); + let write_prologue = quote_spanned! {op_span=> - let #handle_ident = #hydroflow.add_state(::std::cell::RefCell::new( - #root::util::monotonic_map::MonotonicMap::<_, #root::rustc_hash::FxHashSet<_>>::default(), + let #neg_antijoindata_ident = #hydroflow.add_state(std::cell::RefCell::new( + #neg_init + )); + let #pos_antijoindata_ident = #hydroflow.add_state(std::cell::RefCell::new( + #pos_init + )); + let #tick_ident = #hydroflow.add_state(std::cell::RefCell::new( + 0_usize )); }; - let write_iterator = { - let borrow_ident = wc.make_ident("borrow"); - let input_neg = &inputs[0]; // N before P - let input_pos = &inputs[1]; + let input_neg = &inputs[0]; // N before P + let input_pos = &inputs[1]; + let write_iterator = { quote_spanned! {op_span=> - let mut #borrow_ident = #context.state_ref(#handle_ident).borrow_mut(); + let mut #neg_borrow_ident = #context.state_ref(#neg_antijoindata_ident).borrow_mut(); + let mut #pos_borrow_ident = #context.state_ref(#pos_antijoindata_ident).borrow_mut(); let #ident = { /// Limit error propagation by bounding locally, erasing output iterator type. #[inline(always)] fn check_inputs<'a, K, I1, V, I2>( - input_pos: I1, - input_neg: I2, - borrow_state: &'a mut #root::rustc_hash::FxHashSet, + input_neg: I1, + input_pos: I2, + neg_state: &'a mut #root::rustc_hash::FxHashSet, + pos_state: &'a mut #root::rustc_hash::FxHashSet<(K, V)>, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + ::std::hash::Hash + Clone, - V: Eq + Clone, - I1: 'a + Iterator, - I2: 'a + Iterator, + V: Eq + ::std::hash::Hash + Clone, + I1: 'a + Iterator, + I2: 'a + Iterator, { - borrow_state.extend(input_neg); - input_pos.filter(move |x| !borrow_state.contains(&x.0)) + neg_state.extend(input_neg); + + #root::compiled::pull::anti_join_into_iter(input_pos, neg_state, pos_state, is_new_tick) } + let __is_new_tick = { + let mut __borrow_ident = #context.state_ref(#tick_ident).borrow_mut(); + + if *__borrow_ident <= #context.current_tick() { + *__borrow_ident = #context.current_tick() + 1; + // new tick + true + } else { + // same tick. + false + } + }; + check_inputs( - #input_pos, #input_neg, - #borrow_ident.get_mut_clear((#context.current_tick(), #context.current_stratum())) + #input_pos, + #neg_borrow, + #pos_borrow, + __is_new_tick, ) }; } }; + Ok(OperatorWriteOutput { write_prologue, write_iterator, diff --git a/hydroflow_lang/src/graph/ops/anti_join_multiset.rs b/hydroflow_lang/src/graph/ops/anti_join_multiset.rs new file mode 100644 index 00000000000..17d43426c79 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/anti_join_multiset.rs @@ -0,0 +1,201 @@ +use quote::{quote_spanned, ToTokens}; +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, Persistence, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::diagnostic::{Diagnostic, Level}; +use crate::graph::{OpInstGenerics, OperatorInstance, PortIndexValue}; + +/// > 2 input streams the first of type (K, T), the second of type K, +/// > with output type (K, T) +/// +/// For a given tick, computes the anti-join of the items in the input +/// streams, returning items in the `pos` input --that do not have matching keys +/// in the `neg` input. NOTE this uses multiset semantics on the positive side, +/// so duplicated positive inputs will appear in the output either 0 times (if matched in `neg`) +/// or as many times as they appear in the input (if not matched in `neg`) +/// +/// ```hydroflow +/// source_iter(vec![("cat", 2), ("cat", 2), ("elephant", 3), ("elephant", 3)]) -> [pos]diff; +/// source_iter(vec!["dog", "cat", "gorilla"]) -> [neg]diff; +/// diff = anti_join_multiset() -> assert_eq([("elephant", 3), ("elephant", 3)]); +/// ``` + +// This implementation is largely redundant to ANTI_JOIN and should be DRY'ed +pub const ANTI_JOIN_MULTISET: OperatorConstraints = OperatorConstraints { + name: "anti_join_multiset", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { pos, neg })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::No, + inconsistency_tainted: false, + }, + input_delaytype_fn: |idx| match idx { + PortIndexValue::Path(path) if "neg" == path.to_token_stream().to_string() => { + Some(DelayType::Stratum) + } + _else => None, + }, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + root, + context, + hydroflow, + op_span, + ident, + inputs, + op_inst: + OperatorInstance { + generics: + OpInstGenerics { + persistence_args, .. + }, + .. + }, + .. + }, + diagnostics| { + let persistences = match persistence_args[..] { + [] => [Persistence::Tick, Persistence::Tick], + [a] => [a, a], + [a, b] => [a, b], + _ => unreachable!(), + }; + + let mut make_antijoindata = |persistence, side| { + let antijoindata_ident = wc.make_ident(format!("antijoindata_{}", side)); + let borrow_ident = wc.make_ident(format!("antijoindata_{}_borrow", side)); + let (init, borrow) = match persistence { + Persistence::Tick => ( + quote_spanned! {op_span=> + #root::util::monotonic_map::MonotonicMap::<_, #root::rustc_hash::FxHashSet<_>>::default() + }, + quote_spanned! {op_span=> + (&mut *#borrow_ident).get_mut_clear(#context.current_tick()) + }, + ), + Persistence::Static => ( + quote_spanned! {op_span=> + #root::rustc_hash::FxHashSet::default() + }, + quote_spanned! {op_span=> + (&mut *#borrow_ident) + }, + ), + Persistence::Mutable => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + } + }; + Ok((antijoindata_ident, borrow_ident, init, borrow)) + }; + + let (neg_antijoindata_ident, neg_borrow_ident, neg_init, neg_borrow) = + make_antijoindata(persistences[1], "neg")?; + + // let vec_ident = wc.make_ident("persistvec"); + let pos_antijoindata_ident = wc.make_ident("antijoindata_pos_ident"); + let pos_borrow_ident = wc.make_ident("antijoindata_pos_borrow_ident"); + + let write_prologue_pos = match persistences[0] { + Persistence::Tick => quote_spanned! {op_span=>}, + Persistence::Static => quote_spanned! {op_span=> + let #pos_antijoindata_ident = #hydroflow.add_state(std::cell::RefCell::new(( + 0_usize, + ::std::vec::Vec::new() + ))); + }, + Persistence::Mutable => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + } + }; + + let write_prologue = quote_spanned! {op_span=> + let #neg_antijoindata_ident = #hydroflow.add_state(std::cell::RefCell::new( + #neg_init + )); + + #write_prologue_pos + }; + + let input_neg = &inputs[0]; // N before P + let input_pos = &inputs[1]; + let write_iterator = match persistences[0] { + Persistence::Tick => quote_spanned! {op_span => + let mut #neg_borrow_ident = #context.state_ref(#neg_antijoindata_ident).borrow_mut(); + + #[allow(clippy::needless_borrow)] + #neg_borrow.extend(#input_neg); + + let #ident = #input_pos.filter(|x| { + #[allow(clippy::needless_borrow)] + #[allow(clippy::unnecessary_mut_passed)] + !#neg_borrow.contains(&x.0) + }); + }, + Persistence::Static => quote_spanned! {op_span => + let mut #neg_borrow_ident = #context.state_ref(#neg_antijoindata_ident).borrow_mut(); + let mut #pos_borrow_ident = #context.state_ref(#pos_antijoindata_ident).borrow_mut(); + + #[allow(clippy::needless_borrow)] + let #ident = { + #[allow(clippy::clone_on_copy)] + #[allow(suspicious_double_ref_op)] + if #pos_borrow_ident.0 <= #context.current_tick() { + // Start of new tick + #neg_borrow.extend(#input_neg); + + #pos_borrow_ident.0 = 1 + #context.current_tick(); + #pos_borrow_ident.1.extend(#input_pos); + #pos_borrow_ident.1.iter() + } else { + // Called second or later times on the same tick. + let len = #pos_borrow_ident.1.len(); + #pos_borrow_ident.1.extend(#input_pos); + #pos_borrow_ident.1[len..].iter() + } + .filter(|x| { + #[allow(clippy::unnecessary_mut_passed)] + !#neg_borrow.contains(&x.0) + }) + .map(|(k, v)| (k.clone(), v.clone())) + }; + }, + Persistence::Mutable => quote_spanned! {op_span => + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + }, + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + ..Default::default() + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/assert.rs b/hydroflow_lang/src/graph/ops/assert.rs index c85ef9f307c..c3c889d0494 100644 --- a/hydroflow_lang/src/graph/ops/assert.rs +++ b/hydroflow_lang/src/graph/ops/assert.rs @@ -1,26 +1,20 @@ -use quote::quote_spanned; +use syn::parse_quote_spanned; use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - WriteContextArgs, RANGE_0, RANGE_1, + FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, WriteContextArgs, + RANGE_0, RANGE_1, }; use crate::graph::OperatorInstance; /// > 1 input stream, 1 optional output stream -/// > Arguments: A Vector, Slice, or Array containing objects that will be compared to the input stream. +/// > Arguments: a predicate function that will be applied to each item in the stream /// -/// The input stream will be collected into a vector and then compared with the input argument. If there is a difference then the program will panic. -/// -/// assert() is mainly useful for testing and documenting the behavior of hydroflow code inline. -/// -/// assert() will pass through the items to its output channel unchanged and in-order. -/// -/// assert() will perform the assertion on each tick, so it cannot be used to easily assert the contents of a stream across ticks. +/// If the predicate returns false for any input item then the operator will panic at runtime. /// /// ```hydroflow /// source_iter([1, 2, 3]) -/// -> assert([1, 2, 3]) -/// -> for_each(|x| println!("{x}")); +/// -> assert(|x| *x > 0) +/// -> assert_eq([1, 2, 3]); /// ``` pub const ASSERT: OperatorConstraints = OperatorConstraints { name: "assert", @@ -41,82 +35,31 @@ pub const ASSERT: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { - context, - hydroflow, - root, - op_span, - ident, - inputs, - outputs, - is_pull, - op_inst: OperatorInstance { arguments, .. }, - .. + op_span, op_inst, .. }, - _| { - let vec = &arguments[0]; - - let assert_data_ident = wc.make_ident("assert_data"); - - let (write_prologue, write_iterator, write_iterator_after) = if is_pull { - let input = &inputs[0]; + diagnostics| { + let args = &op_inst.arguments[0]; - ( - Default::default(), - quote_spanned! {op_span=> - let #assert_data_ident = #input.collect::<::std::vec::Vec<_>>(); - - assert_eq!(#assert_data_ident, #vec, "lhs = dataflow input, rhs = expected value (provided as argument to assert())"); - - let #ident = #assert_data_ident.into_iter(); - }, - Default::default(), - ) - } else { - ( - quote_spanned! {op_span=> - let #assert_data_ident = #hydroflow.add_state( - ::std::cell::Cell::new(::std::vec::Vec::default()) - ); - }, - { - match outputs { - [] => { - quote_spanned! {op_span=> - let #ident = #root::pusherator::for_each::ForEach::new(|v| { - let mut accum = #context.state_ref(#assert_data_ident).take(); - accum.push(v); - #context.state_ref(#assert_data_ident).set(accum); - }); - } - } - [output] => { - quote_spanned! {op_span=> - let #ident = #root::pusherator::inspect::Inspect::new(|v| { - let mut accum = #context.state_ref(#assert_data_ident).take(); - accum.push(v.clone()); - #context.state_ref(#assert_data_ident).set(accum); - }, #output); - } - } - // This should be a diagnostic but since output streams is limited by the declaration above to 0..=1 then this can never happen, so an assert is appropriate. - _ => panic!(), - } - }, - quote_spanned! {op_span=> - let accum = #context.state_ref(#assert_data_ident).take(); - - assert_eq!(accum, #vec, "lhs = dataflow input, rhs = expected value (provided as argument to assert())"); + let arguments = parse_quote_spanned! {op_span=> + |x| { + // This is to help constrain the types so that type inference works nicely. + fn __constrain_types(f: impl Fn(&T) -> bool, x: &T) -> bool { + (f)(x) + } + assert!(__constrain_types(#args, x)); + } + }; - #context.state_ref(#assert_data_ident).set(accum); - }, - ) + let wc = WriteContextArgs { + op_inst: &OperatorInstance { + arguments, + ..op_inst.clone() + }, + ..wc.clone() }; - Ok(OperatorWriteOutput { - write_prologue, - write_iterator, - write_iterator_after, - }) + (super::inspect::INSPECT.write_fn)(&wc, diagnostics) }, }; diff --git a/hydroflow_lang/src/graph/ops/assert_eq.rs b/hydroflow_lang/src/graph/ops/assert_eq.rs new file mode 100644 index 00000000000..d8125840dc5 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/assert_eq.rs @@ -0,0 +1,96 @@ +use quote::quote_spanned; +use syn::parse_quote_spanned; + +use super::{ + FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, WriteContextArgs, + RANGE_0, RANGE_1, +}; +use crate::graph::OperatorInstance; + +/// > 1 input stream, 1 optional output stream +/// > Arguments: A Vector, Slice, or Array containing objects that will be compared to the input stream. +/// +/// The input stream will be compared with the provided argument, element by element. If any elements do not match, `assert_eq` will panic. +/// If the input stream produces more elements than are in the provided argument, `assert_eq` will panic. +/// +/// The input stream is passed through `assert_eq` unchanged to the output stream. +/// +/// `assert_eq` is mainly useful for testing and documenting the behavior of hydroflow code inline. +/// +/// `assert_eq` will remember the stream position across ticks, see example. +/// +/// ```hydroflow +/// unioned = union(); +/// +/// source_iter([1]) -> assert_eq([1]) -> unioned; +/// source_iter([2]) -> defer_tick() -> assert_eq([2]) -> unioned; +/// +/// unioned -> assert_eq([1, 2]); +/// ``` +pub const ASSERT_EQ: OperatorConstraints = OperatorConstraints { + name: "assert_eq", + categories: &[OperatorCategory::Control], + hard_range_inn: RANGE_1, + soft_range_inn: RANGE_1, + hard_range_out: &(0..=1), + soft_range_out: &(0..=1), + num_args: 1, + persistence_args: RANGE_0, + type_args: RANGE_0, + is_external_input: false, + ports_inn: None, + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::DependsOnArgs, + monotonic: FlowPropertyVal::DependsOnArgs, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| None, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + context, + hydroflow, + op_span, + op_inst, + .. + }, + diagnostics| { + let assert_index_ident = wc.make_ident("assert_index"); + + let args = &op_inst.arguments[0]; + + let inspect_fn = parse_quote_spanned! {op_span=> + |__item| { + // This is to help constrain the types so that type inference works nicely. + fn __constrain_types(array: &impl ::std::ops::Index, index: usize) -> &T { + &array[index] + } + + let __index = #context.state_ref(#assert_index_ident).get(); + ::std::assert_eq!(__constrain_types(&#args, __index), __item, "Item (right) at index {} does not equal expected (left).", __index); + #context.state_ref(#assert_index_ident).set(__index + 1); + } + }; + + let wc = WriteContextArgs { + op_inst: &OperatorInstance { + arguments: inspect_fn, + ..op_inst.clone() + }, + ..wc.clone() + }; + + let mut owo = (super::inspect::INSPECT.write_fn)(&wc, diagnostics)?; + + let write_prologue = owo.write_prologue; + owo.write_prologue = quote_spanned! {op_span=> + let #assert_index_ident = #hydroflow.add_state( + ::std::cell::Cell::new(0usize) + ); + + #write_prologue + }; + + Ok(owo) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/batch.rs b/hydroflow_lang/src/graph/ops/batch.rs deleted file mode 100644 index e5d7db1df08..00000000000 --- a/hydroflow_lang/src/graph/ops/batch.rs +++ /dev/null @@ -1,149 +0,0 @@ -use quote::quote_spanned; - -use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - WriteContextArgs, RANGE_0, RANGE_1, -}; -use crate::graph::OperatorInstance; - -/// > 1 input stream, 1 output stream -/// -/// > Arguments: First argument is the maximum batch size that batch() will buffer up before completely releasing the batch. -/// The second argument is the receive end of a tokio channel that signals when to release the batch downstream. -/// -/// Given a [`Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html) -/// created in Rust code, `batch` -/// is passed the receive end of the channel and when receiving any element -/// will pass through all received inputs to the output unchanged. -/// -/// ```rustbook -/// let (tx, rx) = hydroflow::util::unbounded_channel::<()>(); -/// -/// let mut df = hydroflow::hydroflow_syntax! { -/// source_iter(0..5) -> persist() -> batch(10, rx) -> assert([0, 1, 2, 3, 4]); -/// }; -/// -/// tx.send(()).unwrap(); -/// -/// df.run_available(); -/// ``` -pub const BATCH: OperatorConstraints = OperatorConstraints { - name: "batch", - categories: &[OperatorCategory::Persistence], - persistence_args: RANGE_0, - type_args: RANGE_0, - hard_range_inn: RANGE_1, - soft_range_inn: RANGE_1, - hard_range_out: RANGE_1, - soft_range_out: RANGE_1, - num_args: 2, - is_external_input: false, - ports_inn: None, - ports_out: None, - properties: FlowProperties { - deterministic: FlowPropertyVal::Preserve, - monotonic: FlowPropertyVal::No, - inconsistency_tainted: false, - }, - input_delaytype_fn: |_| None, - write_fn: |wc @ &WriteContextArgs { - context, - hydroflow, - ident, - op_span, - root, - inputs, - outputs, - is_pull, - op_inst: OperatorInstance { arguments, .. }, - .. - }, - _| { - let internal_buffer = wc.make_ident("internal_buffer"); - - let max_queue_len = &arguments[0]; - let receiver = &arguments[1]; - let stream_ident = wc.make_ident("stream"); - - let write_prologue = quote_spanned! {op_span=> - let mut #stream_ident = ::std::boxed::Box::pin(#receiver); - let #internal_buffer = #hydroflow.add_state(::std::cell::RefCell::new(::std::vec::Vec::new())); - }; - - let (write_iterator, write_iterator_after) = if is_pull { - let input = &inputs[0]; - - ( - quote_spanned! {op_span=> - - { - let mut vec = #context.state_ref(#internal_buffer).borrow_mut(); - vec.extend(#input); - } - - let max_queue_len: usize = #max_queue_len; - - let mut vec = #context.state_ref(#internal_buffer).borrow_mut(); - let mut dummy = Vec::new(); - let #ident = match #root::futures::stream::Stream::poll_next(#stream_ident.as_mut(), &mut ::std::task::Context::from_waker(&context.waker())) { - ::std::task::Poll::Ready(Some(_)) => { - vec.drain(..) - } - ::std::task::Poll::Ready(None) | ::std::task::Poll::Pending => { - if vec.len() > max_queue_len { - vec.drain(..) - } else { - dummy.drain(..) - } - } - }; - }, - quote_spanned! {op_span=>}, - ) - } else { - let output = &outputs[0]; - - ( - quote_spanned! {op_span=> - - let mut out = #output; - - let max_queue_len: usize = #max_queue_len; - - let #ident = #root::pusherator::for_each::ForEach::new(|x| { - let mut vec = #context.state_ref(#internal_buffer).borrow_mut(); - - vec.push(x); - }); - }, - quote_spanned! {op_span=> - { - let mut vec = #context.state_ref(#internal_buffer).borrow_mut(); - - match #root::futures::stream::Stream::poll_next(#stream_ident.as_mut(), &mut ::std::task::Context::from_waker(&context.waker())) { - ::std::task::Poll::Ready(Some(_)) => { - for x in vec.drain(..) { - #root::pusherator::Pusherator::give(&mut out, x); - } - }, - ::std::task::Poll::Ready(None) | ::std::task::Poll::Pending => { - if vec.len() > max_queue_len { - for x in vec.drain(..) { - #root::pusherator::Pusherator::give(&mut out, x); - } - } - }, - } - } - }, - ) - }; - - Ok(OperatorWriteOutput { - write_prologue, - write_iterator, - write_iterator_after, - ..Default::default() - }) - }, -}; diff --git a/hydroflow_lang/src/graph/ops/cast.rs b/hydroflow_lang/src/graph/ops/cast.rs new file mode 100644 index 00000000000..a3509164e58 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/cast.rs @@ -0,0 +1,68 @@ +use quote::ToTokens; + +use super::{FlowPropArgs, OperatorCategory, OperatorConstraints}; +use crate::diagnostic::{Diagnostic, Level}; +use crate::graph::{FlowProps, LatticeFlowType}; + +/// TODO(MINGWEI) +pub const CAST: OperatorConstraints = OperatorConstraints { + name: "cast", + categories: &[OperatorCategory::Map], // TODO(mingwei):? + num_args: 1, + flow_prop_fn: Some( + |FlowPropArgs { + op_span, + op_name, + op_inst, + flow_props_in, + .. + }, + diagnostics| { + assert_eq!(1, op_inst.input_ports.len()); + assert_eq!(1, op_inst.output_ports.len()); + + let out_flow_type = match &*op_inst.arguments[0].to_token_stream().to_string() { + "Some(LatticeFlowType::Delta)" | "Some(Delta)" => Some(LatticeFlowType::Delta), + "Some(LatticeFlowType::Cumul)" | "Some(Cumul)" => Some(LatticeFlowType::Cumul), + "None" => None, + unexpected => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + format!( + "Unknown value `{}`, expected one of: `{:?}`, `{:?}`, or `None`.", + unexpected, + Some(LatticeFlowType::Delta), + Some(LatticeFlowType::Cumul), + ), + )); + return Err(()); + } + }; + let Some(in_props) = flow_props_in[0] else { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + format!("Failed to determine flow properties for `{}` input, this may be a Hydroflow bug.", op_name), + )); + return Err(()); + }; + if !LatticeFlowType::can_downcast(in_props.lattice_flow_type, out_flow_type) { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + format!( + "Cannot ilegally up-cast from `{:?}` to `{:?}`.", + in_props.lattice_flow_type, out_flow_type + ), + )); + return Err(()); + } + Ok(vec![Some(FlowProps { + star_ord: in_props.star_ord, + lattice_flow_type: out_flow_type, + })]) + }, + ), + ..super::identity::IDENTITY +}; diff --git a/hydroflow_lang/src/graph/ops/cross_join.rs b/hydroflow_lang/src/graph/ops/cross_join.rs index 6bcedf26531..1a80752b9d0 100644 --- a/hydroflow_lang/src/graph/ops/cross_join.rs +++ b/hydroflow_lang/src/graph/ops/cross_join.rs @@ -14,18 +14,11 @@ use super::{ /// ```hydroflow /// source_iter(vec!["happy", "sad"]) -> [0]my_join; /// source_iter(vec!["dog", "cat"]) -> [1]my_join; -/// my_join = cross_join() -> assert([("happy", "dog"), ("sad", "dog"), ("happy", "cat"), ("sad", "cat")]); +/// my_join = cross_join() -> assert_eq([("happy", "dog"), ("sad", "dog"), ("happy", "cat"), ("sad", "cat")]); /// ``` /// -/// `cross_join` can also be provided with one or two generic lifetime persistence arguments -/// in the same was as [`join`](#join), see [`join`'s documentation](#join) for more info. -/// -/// `cross_join` also accepts one type argument that controls how the join state is built up. This (currently) allows switching between a SetUnion and NonSetUnion implementation. -/// For example: -/// ```hydroflow,ignore -/// join::(); -/// join::(); -/// ``` +/// `cross_join` can be provided with one or two generic lifetime persistence arguments +/// in the same way as [`join`](#join), see [`join`'s documentation](#join) for more info. /// /// ```rustbook /// let (input_send, input_recv) = hydroflow::util::unbounded_channel::<&str>(); @@ -41,7 +34,8 @@ use super::{ /// flow.run_tick(); /// ``` /// Prints only `"(hello, oakland)"` and `"(bye, oakland)"`. The `source_iter` is only included in -/// the first tick, then forgotten. +/// the first tick, then forgotten, so when `"san francisco"` arrives on input `[1]` in the second tick, +/// there is nothing for it to match with from input `[0]`, and therefore it does appear in the output. pub const CROSS_JOIN: OperatorConstraints = OperatorConstraints { name: "cross_join", categories: &[OperatorCategory::MultiIn], @@ -61,6 +55,7 @@ pub const CROSS_JOIN: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { op_span, ident, diff --git a/hydroflow_lang/src/graph/ops/cross_join_multiset.rs b/hydroflow_lang/src/graph/ops/cross_join_multiset.rs new file mode 100644 index 00000000000..29568199543 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/cross_join_multiset.rs @@ -0,0 +1,69 @@ +use quote::quote_spanned; +use syn::parse_quote; + +use super::{ + FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, WriteContextArgs, + RANGE_1, +}; + +/// > 2 input streams of type S and T, 1 output stream of type (S, T) +/// +/// Forms the multiset cross-join (Cartesian product) of the (possibly duplicated) items in the input streams, returning all +/// tupled pairs regardless of duplicates. +/// +/// ```hydroflow +/// source_iter(vec!["happy", "happy", "sad"]) -> [0]my_join; +/// source_iter(vec!["dog", "cat", "cat"]) -> [1]my_join; +/// my_join = cross_join_multiset() -> sort() -> assert_eq([ +/// ("happy", "cat"), +/// ("happy", "cat"), +/// ("happy", "cat"), +/// ("happy", "cat"), +/// ("happy", "dog"), +/// ("happy", "dog"), +/// ("sad", "cat"), +/// ("sad", "cat"), +/// ("sad", "dog"), ]); +/// ``` +pub const CROSS_JOIN_MULTISET: OperatorConstraints = OperatorConstraints { + name: "cross_join_multiset", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=2), + type_args: &(0..=1), + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| None, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + op_span, + ident, + inputs, + .. + }, + diagnostics| { + let mut output = (super::join_multiset::JOIN_MULTISET.write_fn)(wc, diagnostics)?; + + let lhs = &inputs[0]; + let rhs = &inputs[1]; + let write_iterator = output.write_iterator; + output.write_iterator = quote_spanned!(op_span=> + let #lhs = #lhs.map(|a| ((), a)); + let #rhs = #rhs.map(|b| ((), b)); + #write_iterator + let #ident = #ident.map(|((), (a, b))| (a, b)); + ); + + Ok(output) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/defer_signal.rs b/hydroflow_lang/src/graph/ops/defer_signal.rs new file mode 100644 index 00000000000..a2cb5322889 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/defer_signal.rs @@ -0,0 +1,89 @@ +use quote::quote_spanned; +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, +}; + +/// > 2 input streams, 1 output stream, no arguments. +/// +/// Defers streaming input and releases it downstream when a signal is delivered. The order of input is preserved. This allows for buffering data and delivering it at a later, chosen, tick. +/// +/// There are two inputs to `defer_signal`, they are `input` and `signal`. +/// `input` is the input data flow. Data that is delivered on this input is collected in order inside of the `defer_signal` operator. +/// When anything is sent to `signal` the collected data is released downstream. The entire `signal` input is consumed each tick, so sending 5 things on `signal` will not release inputs on the next 5 consecutive ticks. +/// +/// ```hydroflow +/// gate = defer_signal(); +/// +/// source_iter([1, 2, 3]) -> [input]gate; +/// source_iter([()]) -> [signal]gate; +/// +/// gate -> assert_eq([1, 2, 3]); +/// ``` +pub const DEFER_SIGNAL: OperatorConstraints = OperatorConstraints { + name: "defer_signal", + categories: &[OperatorCategory::Persistence], + persistence_args: RANGE_0, + type_args: RANGE_0, + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { input, signal })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::No, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + context, + hydroflow, + ident, + op_span, + inputs, + is_pull, + .. + }, + _| { + assert!(is_pull); + + let internal_buffer = wc.make_ident("internal_buffer"); + let borrow_ident = wc.make_ident("borrow_ident"); + + let write_prologue = quote_spanned! {op_span=> + let #internal_buffer = #hydroflow.add_state(::std::cell::RefCell::new(::std::vec::Vec::new())); + }; + + let input = &inputs[0]; + let signal = &inputs[1]; + + let write_iterator = { + quote_spanned! {op_span=> + + let mut #borrow_ident = #context.state_ref(#internal_buffer).borrow_mut(); + + #borrow_ident.extend(#input); + + let #ident = if #signal.count() > 0 { + ::std::option::Option::Some(#borrow_ident.drain(..)) + } else { + ::std::option::Option::None + }.into_iter().flatten(); + } + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after: Default::default(), + ..Default::default() + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/defer_tick.rs b/hydroflow_lang/src/graph/ops/defer_tick.rs new file mode 100644 index 00000000000..8603a366619 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/defer_tick.rs @@ -0,0 +1,81 @@ +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + IDENTITY_WRITE_FN, RANGE_0, RANGE_1, +}; + +/// Buffers all input items and releases them in the next tick. +/// the state of the current tick. For example, +/// See the [book discussion of Hydroflow time](../concepts/life_and_times) for details on ticks. +/// A tick may be divided into multiple [strata](../concepts/stratification); see the [`next_stratum()`](#next_stratum) +/// operator. +/// +/// `defer_tick` is sometimes needed to separate conflicting data across time, +/// in order to preserve invariants. Consider the following example, which implements +/// a flip-flop -- the invariant is that it emit one of true or false in a given tick +/// (but never both!) +/// +/// ```rustbook +/// pub fn main() { +/// let mut df = hydroflow::hydroflow_syntax! { +/// source_iter(vec!(true)) +/// -> state; +/// state = union() +/// -> assert(|x| if context.current_tick() % 2 == 0 { *x == true } else { *x == false }) +/// -> map(|x| !x) +/// -> defer_tick() +/// -> state; +/// }; +/// for i in 1..100 { +/// println!("tick {}", i); +/// df.run_tick(); +/// } +/// } +/// ``` +/// +/// `defer_tick` can also be handy for comparing stream content across ticks. +/// In the example below `defer_tick()` is used alongside `difference()` to +/// filter out any items that arrive from `inp` in the current tick which match +/// an item from `inp` in the previous +/// tick. +/// ```rustbook +/// // Outputs 1 2 3 4 5 6 (on separate lines). +/// let (input_send, input_recv) = hydroflow::util::unbounded_channel::(); +/// let mut flow = hydroflow::hydroflow_syntax! { +/// inp = source_stream(input_recv) -> tee(); +/// inp -> [pos]diff; +/// inp -> defer_tick() -> [neg]diff; +/// diff = difference() -> for_each(|x| println!("{}", x)); +/// }; +/// +/// for x in [1, 2, 3, 4] { +/// input_send.send(x).unwrap(); +/// } +/// flow.run_tick(); +/// +/// for x in [3, 4, 5, 6] { +/// input_send.send(x).unwrap(); +/// } +/// flow.run_tick(); +/// ``` +pub const DEFER_TICK: OperatorConstraints = OperatorConstraints { + name: "defer_tick", + categories: &[OperatorCategory::Control], + hard_range_inn: RANGE_1, + soft_range_inn: RANGE_1, + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: RANGE_0, + type_args: RANGE_0, + is_external_input: false, + ports_inn: None, + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Tick), + flow_prop_fn: None, + write_fn: IDENTITY_WRITE_FN, +}; diff --git a/hydroflow_lang/src/graph/ops/demux.rs b/hydroflow_lang/src/graph/ops/demux.rs index efffe5cf216..f81658519cf 100644 --- a/hydroflow_lang/src/graph/ops/demux.rs +++ b/hydroflow_lang/src/graph/ops/demux.rs @@ -18,7 +18,7 @@ use crate::pretty_span::PrettySpan; /// > second argument is a variadic [`var_args!` tuple list](https://hydro-project.github.io/hydroflow/doc/hydroflow/macro.var_args.html) /// > where each item name is an output port. /// -/// Takes the input stream and allows the user to determine what elemnt(s) to +/// Takes the input stream and allows the user to determine which items to /// deliver to any number of output streams. /// /// > Note: Downstream operators may need explicit type annotations. @@ -61,6 +61,7 @@ pub const DEMUX: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, @@ -201,7 +202,7 @@ fn extract_closure_idents(arg2: &Pat) -> HashMap { match tt { TokenTree::Group(group) => { let a = stack.len(); - stack.extend(group.stream().into_iter()); + stack.extend(group.stream()); let b = stack.len(); stack[a..b].reverse(); } diff --git a/hydroflow_lang/src/graph/ops/dest_file.rs b/hydroflow_lang/src/graph/ops/dest_file.rs index 428ca939bff..4511601ac0a 100644 --- a/hydroflow_lang/src/graph/ops/dest_file.rs +++ b/hydroflow_lang/src/graph/ops/dest_file.rs @@ -39,6 +39,7 @@ pub const DEST_FILE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/dest_sink.rs b/hydroflow_lang/src/graph/ops/dest_sink.rs index 45b9e4b4e8b..255dfbf285e 100644 --- a/hydroflow_lang/src/graph/ops/dest_sink.rs +++ b/hydroflow_lang/src/graph/ops/dest_sink.rs @@ -100,6 +100,7 @@ pub const DEST_SINK: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, hydroflow, diff --git a/hydroflow_lang/src/graph/ops/dest_sink_serde.rs b/hydroflow_lang/src/graph/ops/dest_sink_serde.rs index acec52e4d6a..56cd3576dbb 100644 --- a/hydroflow_lang/src/graph/ops/dest_sink_serde.rs +++ b/hydroflow_lang/src/graph/ops/dest_sink_serde.rs @@ -7,7 +7,8 @@ use super::{ /// > Arguments: A [serializing async `Sink`](https://docs.rs/futures/latest/futures/sink/trait.Sink.html). /// -/// Consumes (payload, addr) pairs by serializing the payload and sending the resulting pair to an [async `Sink`](https://docs.rs/futures/latest/futures/sink/trait.Sink.html). +/// Consumes (payload, addr) pairs by serializing the payload and sending the resulting pair to an [async `Sink`](https://docs.rs/futures/latest/futures/sink/trait.Sink.html) +/// that delivers them to the `SocketAddr` specified by `addr`. /// /// Note this operator must be used within a Tokio runtime. /// ```rustbook @@ -41,6 +42,7 @@ pub const DEST_SINK_SERDE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/difference.rs b/hydroflow_lang/src/graph/ops/difference.rs index 061c2df3c18..77c238601cb 100644 --- a/hydroflow_lang/src/graph/ops/difference.rs +++ b/hydroflow_lang/src/graph/ops/difference.rs @@ -3,21 +3,26 @@ use syn::parse_quote; use super::{ DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, - OperatorWriteOutput, Persistence, WriteContextArgs, RANGE_0, RANGE_1, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, }; -use crate::diagnostic::{Diagnostic, Level}; -use crate::graph::{OpInstGenerics, OperatorInstance, PortIndexValue}; +use crate::graph::{OperatorInstance, PortIndexValue}; /// > 2 input streams of the same type T, 1 output stream of type T /// -/// For a given tick, forms the set difference of the items in the input +/// Forms the set difference of the items in the input /// streams, returning items in the `pos` input that are not found in the /// `neg` input. /// +/// `difference` can be provided with one or two generic lifetime persistence arguments +/// in the same way as [`join`](#join), see [`join`'s documentation](#join) for more info. +/// +/// Note set semantics here: duplicate items in the `pos` input +/// are output 0 or 1 times (if they do/do-not have a match in `neg` respectively.) +/// /// ```hydroflow /// source_iter(vec!["dog", "cat", "elephant"]) -> [pos]diff; /// source_iter(vec!["dog", "cat", "gorilla"]) -> [neg]diff; -/// diff = difference() -> assert(["elephant"]); +/// diff = difference() -> assert_eq(["elephant"]); /// ``` pub const DIFFERENCE: OperatorConstraints = OperatorConstraints { name: "difference", @@ -43,99 +48,32 @@ pub const DIFFERENCE: OperatorConstraints = OperatorConstraints { } _else => None, }, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { - root, - context, - hydroflow, op_span, ident, inputs, - op_inst: - OperatorInstance { - generics: - OpInstGenerics { - persistence_args, .. - }, - .. - }, + op_inst: OperatorInstance { .. }, .. }, diagnostics| { - let handle_ident = wc.make_ident("diffdata_handle"); - - let persistence = match &persistence_args[..] { - [] => Persistence::Tick, - [Persistence::Tick, Persistence::Tick] => Persistence::Tick, - [Persistence::Tick, Persistence::Static] => Persistence::Static, - other => { - diagnostics.push(Diagnostic::spanned( - op_span, - Level::Error, - &*format!( - "Unexpected persistence arguments for difference, expected two arguments with the first as `'tick`, got {:?}", // or whatever - other - ), - )); + let OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + } = (super::anti_join::ANTI_JOIN.write_fn)(wc, diagnostics)?; - Persistence::Tick - } + let pos = &inputs[1]; + let write_iterator = quote_spanned! {op_span=> + let #pos = #pos.map(|k| (k, ())); + #write_iterator + let #ident = #ident.map(|(k, ())| k); }; - let (write_prologue, write_iterator) = { - let borrow_ident = wc.make_ident("borrow"); - let negset_ident = wc.make_ident("negset"); - - let input_neg = &inputs[0]; // N before P - let input_pos = &inputs[1]; - - let (write_prologue, get_set) = match persistence { - Persistence::Tick => ( - quote_spanned! {op_span=> - let #handle_ident = #hydroflow.add_state(std::cell::RefCell::new( - #root::util::monotonic_map::MonotonicMap::<_, #root::rustc_hash::FxHashSet<_>>::default(), - )); - }, - quote_spanned! {op_span=> - let mut #borrow_ident = #context.state_ref(#handle_ident).borrow_mut(); - let #negset_ident = #borrow_ident - .get_mut_with((#context.current_tick(), #context.current_stratum()), || { - #input_neg.collect() - }); - }, - ), - - Persistence::Static => ( - quote_spanned! {op_span=> - let #handle_ident = #hydroflow.add_state(::std::cell::RefCell::new(#root::rustc_hash::FxHashSet::default())); - }, - quote_spanned! {op_span=> - let mut #negset_ident = #context.state_ref(#handle_ident).borrow_mut(); - #negset_ident.extend(#input_neg); - }, - ), - - Persistence::Mutable => { - diagnostics.push(Diagnostic::spanned( - op_span, - Level::Error, - "An implementation of 'mutable does not exist", - )); - return Err(()); - } - }; - - ( - write_prologue, - quote_spanned! {op_span=> - #get_set - let #ident = #input_pos.filter(move |x| !#negset_ident.contains(x)); - }, - ) - }; Ok(OperatorWriteOutput { write_prologue, write_iterator, - ..Default::default() + write_iterator_after, }) }, }; diff --git a/hydroflow_lang/src/graph/ops/difference_multiset.rs b/hydroflow_lang/src/graph/ops/difference_multiset.rs new file mode 100644 index 00000000000..e2582cae4ad --- /dev/null +++ b/hydroflow_lang/src/graph/ops/difference_multiset.rs @@ -0,0 +1,79 @@ +use quote::{quote_spanned, ToTokens}; +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::graph::{OperatorInstance, PortIndexValue}; + +/// > 2 input streams of the same type T, 1 output stream of type T +/// +/// Forms the set difference of the items in the input +/// streams, returning items in the `pos` input that are not found in the +/// `neg` input. +/// +/// `difference` can be provided with one or two generic lifetime persistence arguments +/// in the same way as [`join`](#join), see [`join`'s documentation](#join) for more info. +/// +/// Note multiset semantics here: each (possibly duplicated) item in the `pos` input +/// that has no match in `neg` is sent to the output. +/// +/// ```hydroflow +/// source_iter(vec!["cat", "cat", "elephant", "elephant"]) -> [pos]diff; +/// source_iter(vec!["cat", "gorilla"]) -> [neg]diff; +/// diff = difference_multiset() -> assert_eq(["elephant", "elephant"]); +/// ``` +pub const DIFFERENCE_MULTISET: OperatorConstraints = OperatorConstraints { + name: "difference_multiset", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { pos, neg })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::No, + inconsistency_tainted: false, + }, + input_delaytype_fn: |idx| match idx { + PortIndexValue::Path(path) if "neg" == path.to_token_stream().to_string() => { + Some(DelayType::Stratum) + } + _else => None, + }, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + op_span, + ident, + inputs, + op_inst: OperatorInstance { .. }, + .. + }, + diagnostics| { + let OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + } = (super::anti_join_multiset::ANTI_JOIN_MULTISET.write_fn)(wc, diagnostics)?; + + let pos = &inputs[1]; + let write_iterator = quote_spanned! {op_span=> + let #pos = #pos.map(|k| (k, ())); + #write_iterator + let #ident = #ident.map(|(k, ())| k); + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/enumerate.rs b/hydroflow_lang/src/graph/ops/enumerate.rs index ebdc0567620..7caeb5cff78 100644 --- a/hydroflow_lang/src/graph/ops/enumerate.rs +++ b/hydroflow_lang/src/graph/ops/enumerate.rs @@ -12,13 +12,14 @@ use crate::graph::OpInstGenerics; /// For each item passed in, enumerate it with its index: `(0, x_0)`, `(1, x_1)`, etc. /// /// `enumerate` can also be provided with one generic lifetime persistence argument, either -/// `'tick` or `'static`, to specify if indexing resets. If `'tick` is specified, indexing will -/// restart at zero at the start of each tick. Otherwise `'static` (the default) will never reset +/// `'tick` or `'static`, to specify if indexing resets. If `'tick` (the default) is specified, indexing will +/// restart at zero at the start of each tick. Otherwise `'static` will never reset /// and count monotonically upwards. /// /// ```hydroflow -/// source_iter(vec!["hello", "world"]) -> enumerate() -/// -> assert([(0, "hello"), (1, "world")]); +/// source_iter(vec!["hello", "world"]) +/// -> enumerate() +/// -> assert_eq([(0, "hello"), (1, "world")]); /// ``` pub const ENUMERATE: OperatorConstraints = OperatorConstraints { name: "enumerate", @@ -40,6 +41,7 @@ pub const ENUMERATE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: true, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, @@ -61,7 +63,7 @@ pub const ENUMERATE: OperatorConstraints = OperatorConstraints { }, diagnostics| { let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; diff --git a/hydroflow_lang/src/graph/ops/filter.rs b/hydroflow_lang/src/graph/ops/filter.rs index 9d14561b1a2..8913b250546 100644 --- a/hydroflow_lang/src/graph/ops/filter.rs +++ b/hydroflow_lang/src/graph/ops/filter.rs @@ -16,7 +16,7 @@ use super::{ /// /// ```hydroflow /// source_iter(vec!["hello", "world"]) -> filter(|x| x.starts_with('w')) -/// -> assert(["world"]); +/// -> assert_eq(["world"]); /// ``` pub const FILTER: OperatorConstraints = OperatorConstraints { name: "filter", @@ -37,6 +37,7 @@ pub const FILTER: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/filter_map.rs b/hydroflow_lang/src/graph/ops/filter_map.rs index b0ac12bc006..24c42be6b0d 100644 --- a/hydroflow_lang/src/graph/ops/filter_map.rs +++ b/hydroflow_lang/src/graph/ops/filter_map.rs @@ -14,7 +14,7 @@ use super::{ /// ```hydroflow /// source_iter(vec!["1", "hello", "world", "2"]) /// -> filter_map(|s| s.parse::().ok()) -/// -> assert([1, 2]); +/// -> assert_eq([1, 2]); /// ``` pub const FILTER_MAP: OperatorConstraints = OperatorConstraints { name: "filter_map", @@ -35,6 +35,7 @@ pub const FILTER_MAP: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/flat_map.rs b/hydroflow_lang/src/graph/ops/flat_map.rs index 4c42ad0b8ce..4ef3da3ff6f 100644 --- a/hydroflow_lang/src/graph/ops/flat_map.rs +++ b/hydroflow_lang/src/graph/ops/flat_map.rs @@ -18,7 +18,7 @@ use super::{ /// // should print out each character of each word on a separate line /// source_iter(vec!["hello", "world"]) /// -> flat_map(|x| x.chars()) -/// -> assert(['h', 'e', 'l', 'l', 'o', 'w', 'o', 'r', 'l', 'd']); +/// -> assert_eq(['h', 'e', 'l', 'l', 'o', 'w', 'o', 'r', 'l', 'd']); /// ``` pub const FLAT_MAP: OperatorConstraints = OperatorConstraints { name: "flat_map", @@ -39,6 +39,7 @@ pub const FLAT_MAP: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/flatten.rs b/hydroflow_lang/src/graph/ops/flatten.rs index 9cfb288a980..7a4a2df4db4 100644 --- a/hydroflow_lang/src/graph/ops/flatten.rs +++ b/hydroflow_lang/src/graph/ops/flatten.rs @@ -13,7 +13,7 @@ use super::{ /// ```hydroflow /// source_iter(vec![[1, 2], [3, 4], [5, 6]]) /// -> flatten() -/// -> assert([1, 2, 3, 4, 5, 6]); +/// -> assert_eq([1, 2, 3, 4, 5, 6]); /// ``` pub const FLATTEN: OperatorConstraints = OperatorConstraints { name: "flatten", @@ -34,6 +34,7 @@ pub const FLATTEN: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/fold.rs b/hydroflow_lang/src/graph/ops/fold.rs index 4c0f9e29662..d867c95d7e7 100644 --- a/hydroflow_lang/src/graph/ops/fold.rs +++ b/hydroflow_lang/src/graph/ops/fold.rs @@ -10,9 +10,9 @@ use crate::diagnostic::{Diagnostic, Level}; /// > 1 input stream, 1 output stream /// /// > Arguments: an initial value, and a closure which itself takes two arguments: -/// an 'accumulator', and an element. The closure returns the value that the accumulator should have for the next iteration. +/// an 'accumulator', and an element. /// -/// Akin to Rust's built-in fold operator. Folds every element into an accumulator by applying a closure, +/// Akin to Rust's built-in fold operator, except that it takes the accumulator by `&mut` instead of by value. Folds every element into an accumulator by applying a closure, /// returning the final result. /// /// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object). @@ -21,16 +21,15 @@ use crate::diagnostic::{Diagnostic, Level}; /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected /// within the same tick. With `'static`, values will be remembered across ticks and will be /// aggregated with pairs arriving in later ticks. When not explicitly specified persistence -/// defaults to `'static`. +/// defaults to `'tick`. /// /// ```hydroflow /// // should print `Reassembled vector [1,2,3,4,5]` /// source_iter([1,2,3,4,5]) -/// -> fold::<'tick>(Vec::new(), |mut accum, elem| { +/// -> fold::<'tick>(Vec::new(), |accum: &mut Vec<_>, elem| { /// accum.push(elem); -/// accum /// }) -/// -> assert([vec![1, 2, 3, 4, 5]]); +/// -> assert_eq([vec![1, 2, 3, 4, 5]]); /// ``` pub const FOLD: OperatorConstraints = OperatorConstraints { name: "fold", @@ -51,6 +50,7 @@ pub const FOLD: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { context, hydroflow, @@ -73,7 +73,7 @@ pub const FOLD: OperatorConstraints = OperatorConstraints { assert!(is_pull); let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; @@ -82,6 +82,8 @@ pub const FOLD: OperatorConstraints = OperatorConstraints { let init = &arguments[0]; let func = &arguments[1]; let folddata_ident = wc.make_ident("folddata"); + let accumulator_ident = wc.make_ident("accumulator"); + let iterator_item_ident = wc.make_ident("iterator_item"); let (write_prologue, write_iterator, write_iterator_after) = match persistence { // TODO(mingwei): Issues if initial value is not copy. @@ -89,8 +91,16 @@ pub const FOLD: OperatorConstraints = OperatorConstraints { Persistence::Tick => ( Default::default(), quote_spanned! {op_span=> - #[allow(clippy::unnecessary_fold)] - let #ident = ::std::iter::once(#input.fold(#init, #func)); + let #ident = { + let mut #accumulator_ident = #init; + + for #iterator_item_ident in #input { + #[allow(clippy::redundant_closure_call)] + (#func)(&mut #accumulator_ident, #iterator_item_ident); + } + + ::std::iter::once(#accumulator_ident) + }; }, Default::default(), ), @@ -102,13 +112,18 @@ pub const FOLD: OperatorConstraints = OperatorConstraints { }, quote_spanned! {op_span=> let #ident = { - let accum = #context.state_ref(#folddata_ident).take().expect("FOLD DATA MISSING"); - #[allow(clippy::unnecessary_fold)] - let accum = #input.fold(accum, #func); + let mut #accumulator_ident = #context.state_ref(#folddata_ident).take().expect("FOLD DATA MISSING"); + + for #iterator_item_ident in #input { + #[allow(clippy::redundant_closure_call)] + (#func)(&mut #accumulator_ident, #iterator_item_ident); + } + #context.state_ref(#folddata_ident).set( - ::std::option::Option::Some(::std::clone::Clone::clone(&accum)) + ::std::option::Option::Some(::std::clone::Clone::clone(&#accumulator_ident)) ); - ::std::iter::once(accum) + + ::std::iter::once(#accumulator_ident) }; }, quote_spanned! {op_span=> diff --git a/hydroflow_lang/src/graph/ops/fold_keyed.rs b/hydroflow_lang/src/graph/ops/fold_keyed.rs index bcd4c6a3114..447079fb045 100644 --- a/hydroflow_lang/src/graph/ops/fold_keyed.rs +++ b/hydroflow_lang/src/graph/ops/fold_keyed.rs @@ -17,16 +17,22 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// value that the accumulator should have for the next iteration. /// /// A special case of `fold`, in the spirit of SQL's GROUP BY and aggregation constructs. The input -/// is partitioned into groups by the first field, and for each group the values in the second +/// is partitioned into groups by the first field ("keys"), and for each group the values in the second /// field are accumulated via the closures in the arguments. /// /// > Note: The closures have access to the [`context` object](surface_flows.md#the-context-object). /// -/// `fold_keyed` can also be provided with one generic lifetime persistence argument, either +/// ```hydroflow +/// source_iter([("toy", 1), ("toy", 2), ("shoe", 11), ("shoe", 35), ("haberdashery", 7)]) +/// -> fold_keyed(|| 0, |old: &mut u32, val: u32| *old += val) +/// -> assert_eq([("toy", 3), ("shoe", 46), ("haberdashery", 7)]); +/// ``` +/// +/// `fold_keyed` can be provided with one generic lifetime persistence argument, either /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected /// within the same tick. With `'static`, values will be remembered across ticks and will be /// aggregated with pairs arriving in later ticks. When not explicitly specified persistence -/// defaults to `'static`. +/// defaults to `'tick`. /// /// `fold_keyed` can also be provided with two type arguments, the key type `K` and aggregated /// output value type `V2`. This is required when using `'static` persistence if the compiler @@ -35,7 +41,7 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// ```hydroflow /// source_iter([("toy", 1), ("toy", 2), ("shoe", 11), ("shoe", 35), ("haberdashery", 7)]) /// -> fold_keyed(|| 0, |old: &mut u32, val: u32| *old += val) -/// -> assert([("toy", 3), ("shoe", 46), ("haberdashery", 7)]); +/// -> assert_eq([("toy", 3), ("shoe", 46), ("haberdashery", 7)]); /// ``` /// /// Example using `'tick` persistence: @@ -79,6 +85,7 @@ pub const FOLD_KEYED: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { hydroflow, context, @@ -104,7 +111,7 @@ pub const FOLD_KEYED: OperatorConstraints = OperatorConstraints { assert!(is_pull); let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; @@ -142,6 +149,8 @@ pub const FOLD_KEYED: OperatorConstraints = OperatorConstraints { -> impl ::std::iter::Iterator { iter } for kv in check_input(#input) { + // TODO(mingwei): remove `unknown_lints` when `clippy::unwrap_or_default` is stabilized. + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = #hashtable_ident.entry(kv.0).or_insert_with(#initfn); #[allow(clippy::redundant_closure_call)] (#aggfn)(entry, kv.1); } @@ -169,6 +178,8 @@ pub const FOLD_KEYED: OperatorConstraints = OperatorConstraints { -> impl ::std::iter::Iterator { iter } for kv in check_input(#input) { + // TODO(mingwei): remove `unknown_lints` when `clippy::unwrap_or_default` is stabilized. + #[allow(unknown_lints, clippy::unwrap_or_default)] let entry = #hashtable_ident.entry(kv.0).or_insert_with(#initfn); #[allow(clippy::redundant_closure_call)] (#aggfn)(entry, kv.1); } diff --git a/hydroflow_lang/src/graph/ops/for_each.rs b/hydroflow_lang/src/graph/ops/for_each.rs index e4f7b63f7c4..d12ebf39b48 100644 --- a/hydroflow_lang/src/graph/ops/for_each.rs +++ b/hydroflow_lang/src/graph/ops/for_each.rs @@ -37,6 +37,7 @@ pub const FOR_EACH: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/identity.rs b/hydroflow_lang/src/graph/ops/identity.rs index 99e181cc543..c63ae7a4a00 100644 --- a/hydroflow_lang/src/graph/ops/identity.rs +++ b/hydroflow_lang/src/graph/ops/identity.rs @@ -10,7 +10,7 @@ use super::{ /// ```hydroflow /// source_iter(vec!["hello", "world"]) /// -> identity() -/// -> assert(["hello", "world"]); +/// -> assert_eq(["hello", "world"]); /// ``` /// /// You can also supply a type parameter `identity::()` to specify what items flow through the @@ -20,7 +20,7 @@ use super::{ /// // Use type parameter to ensure items are `i32`s. /// source_iter(0..10) /// -> identity::() -/// -> assert([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); +/// -> assert_eq([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); /// ``` pub const IDENTITY: OperatorConstraints = OperatorConstraints { name: "identity", @@ -41,5 +41,6 @@ pub const IDENTITY: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: IDENTITY_WRITE_FN, }; diff --git a/hydroflow_lang/src/graph/ops/initialize.rs b/hydroflow_lang/src/graph/ops/initialize.rs index 21ad265369c..3e42dc3fe99 100644 --- a/hydroflow_lang/src/graph/ops/initialize.rs +++ b/hydroflow_lang/src/graph/ops/initialize.rs @@ -14,7 +14,7 @@ use crate::graph::OperatorInstance; /// /// ```hydroflow /// initialize() -/// -> assert([()]); +/// -> assert_eq([()]); /// ``` pub const INITIALIZE: OperatorConstraints = OperatorConstraints { name: "initialize", @@ -35,6 +35,7 @@ pub const INITIALIZE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { op_span, .. }, diagnostics| { let wc = WriteContextArgs { op_inst: &OperatorInstance { diff --git a/hydroflow_lang/src/graph/ops/inspect.rs b/hydroflow_lang/src/graph/ops/inspect.rs index 224d46b0304..bb4b883cf28 100644 --- a/hydroflow_lang/src/graph/ops/inspect.rs +++ b/hydroflow_lang/src/graph/ops/inspect.rs @@ -16,8 +16,8 @@ use super::{ /// /// ```hydroflow /// source_iter([1, 2, 3, 4]) -/// -> inspect(|&x| println!("{}", x)) -/// -> assert([1, 2, 3, 4]); +/// -> inspect(|x| println!("{}", x)) +/// -> assert_eq([1, 2, 3, 4]); /// ``` pub const INSPECT: OperatorConstraints = OperatorConstraints { name: "inspect", @@ -38,6 +38,7 @@ pub const INSPECT: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, @@ -54,6 +55,10 @@ pub const INSPECT: OperatorConstraints = OperatorConstraints { quote_spanned! {op_span=> let #ident = #input.inspect(#arguments); } + } else if outputs.is_empty() { + quote_spanned! {op_span=> + let #ident = #root::pusherator::inspect::Inspect::new(#arguments, #root::pusherator::null::Null::new()); + } } else { let output = &outputs[0]; quote_spanned! {op_span=> diff --git a/hydroflow_lang/src/graph/ops/join.rs b/hydroflow_lang/src/graph/ops/join.rs index 1d6e0a03b95..c5118fb95c8 100644 --- a/hydroflow_lang/src/graph/ops/join.rs +++ b/hydroflow_lang/src/graph/ops/join.rs @@ -13,23 +13,22 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// Forms the equijoin of the tuples in the input streams by their first (key) attribute. Note that the result nests the 2nd input field (values) into a tuple in the 2nd output field. /// /// ```hydroflow -/// // should print `(hello, (world, cleveland))` -/// source_iter(vec![("hello", "world"), ("stay", "gold")]) -> [0]my_join; +/// source_iter(vec![("hello", "world"), ("stay", "gold"), ("hello", "world")]) -> [0]my_join; /// source_iter(vec![("hello", "cleveland")]) -> [1]my_join; /// my_join = join() -/// -> assert([("hello", ("world", "cleveland"))]); +/// -> assert_eq([("hello", ("world", "cleveland"))]); /// ``` /// /// `join` can also be provided with one or two generic lifetime persistence arguments, either /// `'tick` or `'static`, to specify how join data persists. With `'tick`, pairs will only be /// joined with corresponding pairs within the same tick. With `'static`, pairs will be remembered /// across ticks and will be joined with pairs arriving in later ticks. When not explicitly -/// specified persistence defaults to `static. +/// specified persistence defaults to `tick. /// /// When two persistence arguments are supplied the first maps to port `0` and the second maps to /// port `1`. /// When a single persistence argument is supplied, it is applied to both input ports. -/// When no persistence arguments are applied it defaults to `'static` for both. +/// When no persistence arguments are applied it defaults to `'tick` for both. /// /// The syntax is as follows: /// ```hydroflow,ignore @@ -44,12 +43,9 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// // etc. /// ``` /// -/// Join also accepts one type argument that controls how the join state is built up. This (currently) allows switching between a SetUnion and NonSetUnion implementation. -/// For example: -/// ```hydroflow,ignore -/// join::(); -/// join::(); -/// ``` +/// `join` is defined to treat its inputs as *sets*, meaning that it +/// eliminates duplicated values in its inputs. If you do not want +/// duplicates eliminated, use the [`join_multiset`](#join_multiset) operator. /// /// ### Examples /// @@ -103,6 +99,7 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, @@ -134,9 +131,6 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { // TODO: This is really bad. // This will break if the user aliases HalfSetJoinState to something else. Temporary hacky solution. // Note that cross_join() depends on the implementation here as well. - // Need to decide on what to do about multisetjoin. - // Should it be a separate operator (multisetjoin() and multisetcrossjoin())? - // Should the default be multiset join? And setjoin requires the use of lattice_join() with SetUnion lattice? let additional_trait_bounds = if join_type.to_string().contains("HalfSetJoinState") { quote_spanned!(op_span=> + ::std::cmp::Eq @@ -180,7 +174,7 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { }; let persistences = match persistence_args[..] { - [] => [Persistence::Static, Persistence::Static], + [] => [Persistence::Tick, Persistence::Tick], [a] => [a, a], [a, b] => [a, b], _ => unreachable!(), @@ -191,6 +185,9 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { let (rhs_joindata_ident, rhs_borrow_ident, rhs_init, rhs_borrow) = make_joindata(persistences[1], "rhs")?; + let tick_ident = wc.make_ident("persisttick"); + let tick_borrow_ident = wc.make_ident("persisttick_borrow"); + let write_prologue = quote_spanned! {op_span=> let #lhs_joindata_ident = #hydroflow.add_state(std::cell::RefCell::new( #lhs_init @@ -198,6 +195,9 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { let #rhs_joindata_ident = #hydroflow.add_state(std::cell::RefCell::new( #rhs_init )); + let #tick_ident = #hydroflow.add_state(std::cell::RefCell::new( + 0usize + )); }; let lhs = &inputs[0]; @@ -205,6 +205,7 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { let write_iterator = quote_spanned! {op_span=> let mut #lhs_borrow_ident = #context.state_ref(#lhs_joindata_ident).borrow_mut(); let mut #rhs_borrow_ident = #context.state_ref(#rhs_joindata_ident).borrow_mut(); + let mut #tick_borrow_ident = #context.state_ref(#tick_ident).borrow_mut(); let #ident = { // Limit error propagation by bounding locally, erasing output iterator type. #[inline(always)] @@ -213,6 +214,7 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { rhs: I2, lhs_state: &'a mut #join_type, rhs_state: &'a mut #join_type, + is_new_tick: bool, ) -> impl 'a + Iterator where K: Eq + std::hash::Hash + Clone, @@ -221,16 +223,36 @@ pub const JOIN: OperatorConstraints = OperatorConstraints { I1: 'a + Iterator, I2: 'a + Iterator, { - #root::compiled::pull::SymmetricHashJoin::new_from_mut(lhs, rhs, lhs_state, rhs_state) + #root::compiled::pull::symmetric_hash_join_into_iter(lhs, rhs, lhs_state, rhs_state, is_new_tick) + } + + { + let __is_new_tick = if *#tick_borrow_ident <= #context.current_tick() { + *#tick_borrow_ident = #context.current_tick() + 1; + true + } else { + false + }; + + check_inputs(#lhs, #rhs, #lhs_borrow, #rhs_borrow, __is_new_tick) } - check_inputs(#lhs, #rhs, #lhs_borrow, #rhs_borrow) }; }; + let write_iterator_after = + if persistences[0] == Persistence::Static || persistences[1] == Persistence::Static { + quote_spanned! {op_span=> + // TODO: Probably only need to schedule if #*_borrow.len() > 0? + #context.schedule_subgraph(#context.current_subgraph(), false); + } + } else { + quote_spanned! {op_span=>} + }; + Ok(OperatorWriteOutput { write_prologue, write_iterator, - ..Default::default() + write_iterator_after, }) }, }; diff --git a/hydroflow_lang/src/graph/ops/join_fused.rs b/hydroflow_lang/src/graph/ops/join_fused.rs new file mode 100644 index 00000000000..97fd5230e5d --- /dev/null +++ b/hydroflow_lang/src/graph/ops/join_fused.rs @@ -0,0 +1,346 @@ +use proc_macro2::{Ident, TokenStream}; +use quote::{quote_spanned, ToTokens}; +use syn::spanned::Spanned; +use syn::{parse_quote, Expr, ExprCall}; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, Persistence, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::diagnostic::{Diagnostic, Level}; +use crate::graph::{OpInstGenerics, OperatorInstance}; + +/// > 2 input streams of type <(K, V1)> and <(K, V2)>, 1 output stream of type <(K, (V1, V2))> +/// +/// `join_fused` takes two arguments, they are the configuration options for the left hand side and right hand side inputs respectively. +/// There are three available configuration options, they are `Reduce`: if the input type is the same as the accumulator type, +/// `Fold`: if the input type is different from the accumulator type, and the accumulator type has a sensible default value, and +/// `FoldFrom`: if the input type is different from the accumulator type, and the accumulator needs to be derived from the first input value. +/// Examples of all three configuration options are below: +/// ```hydroflow,ignore +/// // Left hand side input will use fold, right hand side input will use reduce, +/// join_fused(Fold(|| "default value", |x, y| *x += y), Reduce(|x, y| *x -= y)) +/// +/// // Left hand side input will use FoldFrom, and the right hand side input will use Reduce again +/// join_fused(FoldFrom(|x| "conversion function", |x, y| *x += y), Reduce(|x, y| *x *= y)) +/// ``` +/// The three currently supported fused operator types are `Fold(Fn() -> A, Fn(A, T) -> A)`, `Reduce(Fn(A, A) -> A)`, and `FoldFrom(Fn(T) -> A, Fn(A, T) -> A)` +/// +/// `join_fused` first performs a fold_keyed/reduce_keyed operation on each input stream before performing joining. See `join()`. There is currently no equivalent for `FoldFrom` in hydroflow operators. +/// +/// For example, the following two hydroflow programs are equivalent, the former would optimize into the latter: +/// +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) +/// -> reduce_keyed(|x: &mut _, y| *x += y) +/// -> [0]my_join; +/// source_iter(vec![("key", 2), ("key", 3)]) +/// -> fold_keyed(|| 1, |x: &mut _, y| *x *= y) +/// -> [1]my_join; +/// my_join = join_multiset() +/// -> assert_eq([("key", (3, 6))]); +/// ``` +/// +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) +/// -> [0]my_join; +/// source_iter(vec![("key", 2), ("key", 3)]) +/// -> [1]my_join; +/// my_join = join_fused(Reduce(|x, y| *x += y), Fold(|| 1, |x, y| *x *= y)) +/// -> assert_eq([("key", (3, 6))]); +/// ``` +/// +/// Here is an example of using FoldFrom to derive the accumulator from the first value: +/// +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) +/// -> [0]my_join; +/// source_iter(vec![("key", 2), ("key", 3)]) +/// -> [1]my_join; +/// my_join = join_fused(FoldFrom(|x| x + 3, |x, y| *x += y), Fold(|| 1, |x, y| *x *= y)) +/// -> assert_eq([("key", (6, 6))]); +/// ``` +/// +/// The benefit of this is that the state between the reducing/folding operator and the join is merged together. +/// +/// `join_fused` follows the same persistence rules as `join` and all other operators. By default, both the left hand side and right hand side are `'tick` persistence. They can be set to `'static` persistence +/// by specifying `'static` in the type arguments of the operator. +/// +/// for `join_fused::<'static>`, the operator will replay all _keys_ that the join has ever seen each tick, and not only the new matches from that specific tick. +/// This means that it behaves identically to if `persist()` were placed before the inputs and the persistence of +/// for example, the two following examples have identical behavior: +/// +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) -> persist() -> [0]my_join; +/// source_iter(vec![("key", 2)]) -> my_union; +/// source_iter(vec![("key", 3)]) -> defer_tick() -> my_union; +/// my_union = union() -> persist() -> [1]my_join; +/// +/// my_join = join_fused(Reduce(|x, y| *x += y), Fold(|| 1, |x, y| *x *= y)) +/// -> assert_eq([("key", (3, 2)), ("key", (3, 6))]); +/// ``` +/// +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) -> [0]my_join; +/// source_iter(vec![("key", 2)]) -> my_union; +/// source_iter(vec![("key", 3)]) -> defer_tick() -> my_union; +/// my_union = union() -> [1]my_join; +/// +/// my_join = join_fused::<'static>(Reduce(|x, y| *x += y), Fold(|| 1, |x, y| *x *= y)) +/// -> assert_eq([("key", (3, 2)), ("key", (3, 6))]); +/// ``` +pub const JOIN_FUSED: OperatorConstraints = OperatorConstraints { + name: "join_fused", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 2, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + context, + op_span, + ident, + inputs, + is_pull, + op_inst: + OperatorInstance { + arguments, + generics: + OpInstGenerics { + persistence_args, .. + }, + .. + }, + .. + }, + diagnostics| { + assert!(is_pull); + + let persistences = parse_persistences(persistence_args); + + let lhs_join_options = + parse_argument(&arguments[0]).map_err(|err| diagnostics.push(err))?; + let rhs_join_options = + parse_argument(&arguments[1]).map_err(|err| diagnostics.push(err))?; + + let (lhs_joindata_ident, lhs_borrow_ident, lhs_prologue, lhs_borrow) = + make_joindata(wc, persistences[0], &lhs_join_options, "lhs") + .map_err(|err| diagnostics.push(err))?; + + let (rhs_joindata_ident, rhs_borrow_ident, rhs_prologue, rhs_borrow) = + make_joindata(wc, persistences[1], &rhs_join_options, "rhs") + .map_err(|err| diagnostics.push(err))?; + + let write_prologue = quote_spanned! {op_span=> + #lhs_prologue + #rhs_prologue + }; + + let lhs = &inputs[0]; + let rhs = &inputs[1]; + + let arg0_span = arguments[0].span(); + let arg1_span = arguments[1].span(); + + let lhs_tokens = match lhs_join_options { + JoinOptions::FoldFrom(lhs_from, lhs_fold) => quote_spanned! {arg0_span=> + #lhs_borrow.fold_into(#lhs, #lhs_fold, #lhs_from); + }, + JoinOptions::Fold(lhs_default, lhs_fold) => quote_spanned! {arg0_span=> + #lhs_borrow.fold_into(#lhs, #lhs_fold, #lhs_default); + }, + JoinOptions::Reduce(lhs_reduce) => quote_spanned! {arg0_span=> + #lhs_borrow.reduce_into(#lhs, #lhs_reduce); + }, + }; + + let rhs_tokens = match rhs_join_options { + JoinOptions::FoldFrom(rhs_from, rhs_fold) => quote_spanned! {arg0_span=> + #rhs_borrow.fold_into(#rhs, #rhs_fold, #rhs_from); + }, + JoinOptions::Fold(rhs_default, rhs_fold) => quote_spanned! {arg1_span=> + #rhs_borrow.fold_into(#rhs, #rhs_fold, #rhs_default); + }, + JoinOptions::Reduce(rhs_reduce) => quote_spanned! {arg1_span=> + #rhs_borrow.reduce_into(#rhs, #rhs_reduce); + }, + }; + + // Since both input arguments are stratum blocking then we don't need to keep track of ticks to avoid emitting the same thing twice in the same tick. + let write_iterator = quote_spanned! {op_span=> + let mut #lhs_borrow_ident = #context.state_ref(#lhs_joindata_ident).borrow_mut(); + let mut #rhs_borrow_ident = #context.state_ref(#rhs_joindata_ident).borrow_mut(); + + let #ident = { + #lhs_tokens + #rhs_tokens + + // TODO: start the iterator with the smallest len() table rather than always picking rhs. + #[allow(clippy::clone_on_copy)] + #[allow(suspicious_double_ref_op)] + #rhs_borrow + .table + .iter() + .filter_map(|(k, v2)| #lhs_borrow.table.get(k).map(|v1| (k.clone(), (v1.clone(), v2.clone())))) + }; + }; + + let write_iterator_after = + if persistences[0] == Persistence::Static || persistences[1] == Persistence::Static { + quote_spanned! {op_span=> + // TODO: Probably only need to schedule if #*_borrow.len() > 0? + #context.schedule_subgraph(#context.current_subgraph(), false); + } + } else { + quote_spanned! {op_span=>} + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + }) + }, +}; + +pub(crate) enum JoinOptions<'a> { + FoldFrom(&'a Expr, &'a Expr), + Fold(&'a Expr, &'a Expr), + Reduce(&'a Expr), +} + +pub(crate) fn parse_argument(arg: &Expr) -> Result { + let Expr::Call(ExprCall { + attrs: _, + func, + paren_token: _, + args, + }) = arg else { + return Err(Diagnostic::spanned( + arg.span(), + Level::Error, + format!("Argument must be a function call: {arg:?}"), + )); + }; + + let mut elems = args.iter(); + let func_name = func.to_token_stream().to_string(); + + match func_name.as_str() { + "Fold" => match (elems.next(), elems.next()) { + (Some(default), Some(fold)) => Ok(JoinOptions::Fold(default, fold)), + _ => { + Err(Diagnostic::spanned( + args.span(), + Level::Error, + format!("Fold requires two arguments, first is the default function, second is the folding function: {func:?}"), + )) + } + }, + "FoldFrom" => match (elems.next(), elems.next()) { + (Some(from), Some(fold)) => Ok(JoinOptions::FoldFrom(from, fold)), + _ => { + Err(Diagnostic::spanned( + args.span(), + Level::Error, + format!("FoldFrom requires two arguments, first is the From function, second is the folding function: {func:?}"), + )) + } + }, + "Reduce" => match elems.next() { + Some(reduce) => Ok(JoinOptions::Reduce(reduce)), + _ => Err(Diagnostic::spanned( + args.span(), + Level::Error, + format!("Reduce requires one argument, the reducing function: {func:?}"), + )), + }, + _ => Err(Diagnostic::spanned( + func.span(), + Level::Error, + format!("Unknown summarizing function: {func:?}"), + )), + } +} + +pub(crate) fn make_joindata( + wc: &WriteContextArgs, + persistence: Persistence, + join_options: &JoinOptions<'_>, + side: &str, +) -> Result<(Ident, Ident, TokenStream, TokenStream), Diagnostic> { + let joindata_ident = wc.make_ident(format!("joindata_{}", side)); + let borrow_ident = wc.make_ident(format!("joindata_{}_borrow", side)); + + let context = wc.context; + let hydroflow = wc.hydroflow; + let root = wc.root; + let op_span = wc.op_span; + + let join_type = match *join_options { + JoinOptions::FoldFrom(_, _) => { + quote_spanned!(op_span=> #root::compiled::pull::HalfJoinStateFoldFrom) + } + JoinOptions::Fold(_, _) => { + quote_spanned!(op_span=> #root::compiled::pull::HalfJoinStateFold) + } + JoinOptions::Reduce(_) => { + quote_spanned!(op_span=> #root::compiled::pull::HalfJoinStateReduce) + } + }; + + let (prologue, borrow) = match persistence { + Persistence::Tick => ( + quote_spanned! {op_span=> + let #joindata_ident = #hydroflow.add_state(std::cell::RefCell::new( + #root::util::monotonic_map::MonotonicMap::new_init( + #join_type::default() + ) + )); + }, + quote_spanned! {op_span=> + #borrow_ident.get_mut_clear(#context.current_tick()) + }, + ), + Persistence::Static => ( + quote_spanned! {op_span=> + let #joindata_ident = #hydroflow.add_state(std::cell::RefCell::new( + #join_type::default() + )); + }, + quote_spanned! {op_span=> + #borrow_ident + }, + ), + Persistence::Mutable => { + return Err(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + } + }; + Ok((joindata_ident, borrow_ident, prologue, borrow)) +} + +pub(crate) fn parse_persistences(persistences: &[Persistence]) -> [Persistence; 2] { + match persistences { + [] => [Persistence::Tick, Persistence::Tick], + [a] => [*a, *a], + [a, b] => [*a, *b], + _ => panic!("Too many persistences: {persistences:?}"), + } +} diff --git a/hydroflow_lang/src/graph/ops/join_fused_lhs.rs b/hydroflow_lang/src/graph/ops/join_fused_lhs.rs new file mode 100644 index 00000000000..68642cc1ffb --- /dev/null +++ b/hydroflow_lang/src/graph/ops/join_fused_lhs.rs @@ -0,0 +1,184 @@ +use quote::{quote_spanned, ToTokens}; +use syn::parse_quote; +use syn::spanned::Spanned; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, Persistence, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::diagnostic::{Diagnostic, Level}; +use crate::graph::ops::join_fused::{ + make_joindata, parse_argument, parse_persistences, JoinOptions, +}; +use crate::graph::{OpInstGenerics, OperatorInstance, PortIndexValue}; + +/// See `join_fused` +/// +/// This operator is identical to `join_fused` except that the right hand side input `1` is a regular `join_multiset` input. +/// +/// This means that `join_fused_lhs` only takes one argument input, which is the reducing/folding operation for the left hand side only. +/// +/// For example: +/// ```hydroflow +/// source_iter(vec![("key", 0), ("key", 1), ("key", 2)]) -> [0]my_join; +/// source_iter(vec![("key", 2), ("key", 3)]) -> [1]my_join; +/// my_join = join_fused_lhs(Reduce(|x, y| *x += y)) +/// -> assert_eq([("key", (3, 2)), ("key", (3, 3))]); +/// ``` +pub const JOIN_FUSED_LHS: OperatorConstraints = OperatorConstraints { + name: "join_fused_lhs", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 1, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |idx| match idx { + PortIndexValue::Int(path) if "0" == path.to_token_stream().to_string() => { + Some(DelayType::Stratum) + } + _ => None, + }, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + context, + hydroflow, + op_span, + ident, + inputs, + is_pull, + op_inst: + OperatorInstance { + arguments, + generics: + OpInstGenerics { + persistence_args, .. + }, + .. + }, + .. + }, + diagnostics| { + assert!(is_pull); + + let arg0_span = arguments[0].span(); + + let persistences = parse_persistences(persistence_args); + + let lhs_join_options = + parse_argument(&arguments[0]).map_err(|err| diagnostics.push(err))?; + + let (lhs_joindata_ident, lhs_borrow_ident, lhs_prologue, lhs_borrow) = + make_joindata(wc, persistences[0], &lhs_join_options, "lhs") + .map_err(|err| diagnostics.push(err))?; + + let rhs_joindata_ident = wc.make_ident("rhs_joindata"); + let rhs_borrow_ident = wc.make_ident("rhs_joindata_borrow_ident"); + + let write_prologue_rhs = match persistences[1] { + Persistence::Tick => quote_spanned! {op_span=>}, + Persistence::Static => quote_spanned! {op_span=> + let #rhs_joindata_ident = #hydroflow.add_state(std::cell::RefCell::new(( + 0_usize, + ::std::vec::Vec::new() + ))); + }, + Persistence::Mutable => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + } + }; + + let write_prologue = quote_spanned! {op_span=> + #lhs_prologue + #write_prologue_rhs + }; + + let lhs = &inputs[0]; + let rhs = &inputs[1]; + + let lhs_fold_or_reduce_into_from = match lhs_join_options { + JoinOptions::FoldFrom(lhs_from, lhs_fold) => quote_spanned! {arg0_span=> + #lhs_borrow.fold_into(#lhs, #lhs_fold, #lhs_from); + }, + JoinOptions::Fold(lhs_default, lhs_fold) => quote_spanned! {arg0_span=> + #lhs_borrow.fold_into(#lhs, #lhs_fold, #lhs_default); + }, + JoinOptions::Reduce(lhs_reduce) => quote_spanned! {arg0_span=> + #lhs_borrow.reduce_into(#lhs, #lhs_reduce); + }, + }; + + let write_iterator = match persistences[1] { + Persistence::Tick => quote_spanned! {op_span=> + let mut #lhs_borrow_ident = #context.state_ref(#lhs_joindata_ident).borrow_mut(); + + let #ident = { + #lhs_fold_or_reduce_into_from + + #[allow(clippy::clone_on_copy)] + #rhs.filter_map(|(k, v2)| #lhs_borrow.table.get(&k).map(|v1| (k, (v1.clone(), v2.clone())))) + }; + }, + Persistence::Static => quote_spanned! {op_span=> + let mut #lhs_borrow_ident = #context.state_ref(#lhs_joindata_ident).borrow_mut(); + let mut #rhs_borrow_ident = #context.state_ref(#rhs_joindata_ident).borrow_mut(); + + let #ident = { + #lhs_fold_or_reduce_into_from + + #[allow(clippy::clone_on_copy)] + #[allow(suspicious_double_ref_op)] + if #rhs_borrow_ident.0 <= #context.current_tick() { + #rhs_borrow_ident.0 = 1 + #context.current_tick(); + #rhs_borrow_ident.1.extend(#rhs); + #rhs_borrow_ident.1.iter() + } else { + let len = #rhs_borrow_ident.1.len(); + #rhs_borrow_ident.1.extend(#rhs); + #rhs_borrow_ident.1[len..].iter() + } + .filter_map(|(k, v2)| #lhs_borrow.table.get(k).map(|v1| (k.clone(), (v1.clone(), v2.clone())))) + }; + }, + Persistence::Mutable => { + diagnostics.push(Diagnostic::spanned( + op_span, + Level::Error, + "An implementation of 'mutable does not exist", + )); + return Err(()); + } + }; + + let write_iterator_after = + if persistences[0] == Persistence::Static || persistences[1] == Persistence::Static { + quote_spanned! {op_span=> + // TODO: Probably only need to schedule if #*_borrow.len() > 0? + #context.schedule_subgraph(#context.current_subgraph(), false); + } + } else { + quote_spanned! {op_span=>} + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/join_fused_rhs.rs b/hydroflow_lang/src/graph/ops/join_fused_rhs.rs new file mode 100644 index 00000000000..3c178c78b3a --- /dev/null +++ b/hydroflow_lang/src/graph/ops/join_fused_rhs.rs @@ -0,0 +1,69 @@ +use quote::{quote_spanned, ToTokens}; +use syn::parse_quote; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, +}; +use crate::graph::PortIndexValue; + +/// See `join_fused_lhs` +/// +/// This operator is identical to `join_fused_lhs` except that it is the right hand side that is fused instead of the left hand side. +pub const JOIN_FUSED_RHS: OperatorConstraints = OperatorConstraints { + name: "join_fused_rhs", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 1, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |idx| match idx { + PortIndexValue::Int(path) if "1" == path.to_token_stream().to_string() => { + Some(DelayType::Stratum) + } + _ => None, + }, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + op_span, + ident, + inputs, + .. + }, + diagnostics| { + let inputs = inputs.iter().cloned().rev().collect::>(); + + let wc = WriteContextArgs { + inputs: &inputs[..], + ..wc.clone() + }; + + let OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + } = (super::join_fused_lhs::JOIN_FUSED_LHS.write_fn)(&wc, diagnostics)?; + + let write_iterator = quote_spanned! {op_span=> + #write_iterator + let #ident = #ident.map(|(k, (v1, v2))| (k, (v2, v1))); + }; + + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + write_iterator_after, + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/join_multiset.rs b/hydroflow_lang/src/graph/ops/join_multiset.rs new file mode 100644 index 00000000000..b93c4085a6d --- /dev/null +++ b/hydroflow_lang/src/graph/ops/join_multiset.rs @@ -0,0 +1,73 @@ +use syn::{parse_quote, parse_quote_spanned}; + +use super::{ + FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, WriteContextArgs, + RANGE_0, RANGE_1, +}; +use crate::graph::{OpInstGenerics, OperatorInstance}; + +/// > 2 input streams of type <(K, V1)> and <(K, V2)>, 1 output stream of type <(K, (V1, V2))> +/// +/// This operator is equivalent to `join` except that the LHS and RHS are collected into multisets rather than sets before joining. +/// +/// If you want +/// duplicates eliminated from the inputs, use the [`join`](#join) operator. +/// +/// For example: +/// ```hydroflow +/// lhs = source_iter([("a", 0), ("a", 0)]) -> tee(); +/// rhs = source_iter([("a", "hydro")]) -> tee(); +/// +/// lhs -> [0]multiset_join; +/// rhs -> [1]multiset_join; +/// multiset_join = join_multiset() -> assert_eq([("a", (0, "hydro")), ("a", (0, "hydro"))]); +/// +/// lhs -> [0]set_join; +/// rhs -> [1]set_join; +/// set_join = join() -> assert_eq([("a", (0, "hydro"))]); +/// ``` +pub const JOIN_MULTISET: OperatorConstraints = OperatorConstraints { + name: "join_multiset", + categories: &[OperatorCategory::MultiIn], + hard_range_inn: &(2..=2), + soft_range_inn: &(2..=2), + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=2), + type_args: RANGE_0, + is_external_input: false, + ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Preserve, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| None, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + root, + op_span, + op_inst: op_inst @ OperatorInstance { .. }, + .. + }, + diagnostics| { + let join_type = parse_quote_spanned! {op_span=> // Uses `lat_type.span()`! + #root::compiled::pull::HalfMultisetJoinState + }; + + let wc = WriteContextArgs { + op_inst: &OperatorInstance { + generics: OpInstGenerics { + type_args: vec![join_type], + ..wc.op_inst.generics.clone() + }, + ..op_inst.clone() + }, + ..wc.clone() + }; + + (super::join::JOIN.write_fn)(&wc, diagnostics) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/lattice_batch.rs b/hydroflow_lang/src/graph/ops/lattice_batch.rs deleted file mode 100644 index 6a4f409310c..00000000000 --- a/hydroflow_lang/src/graph/ops/lattice_batch.rs +++ /dev/null @@ -1,162 +0,0 @@ -use quote::{quote_spanned, ToTokens}; - -use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - WriteContextArgs, RANGE_0, RANGE_1, -}; -use crate::graph::{OpInstGenerics, OperatorInstance}; - -/// > 1 input stream, 1 output stream -/// -/// > Arguments: The one and only argument is the receive end of a tokio channel that signals when to release the batch downstream. -/// -/// Given a [`Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html) -/// created in Rust code, `lattice_batch` -/// is passed the receive end of the channel and when receiving any element -/// will pass through all received inputs to the output unchanged. -/// -/// ```rustbook -/// use hydroflow::lattices::Max; -/// -/// let (tx, rx) = hydroflow::util::unbounded_channel::<()>(); -/// -/// let mut df = hydroflow::hydroflow_syntax! { -/// source_iter(0..5) -> persist() -/// -> map(|x| Max::new(x)) -/// -> lattice_batch::>(rx) -/// -> assert([Max::new(4)]); -/// }; -/// -/// tx.send(()).unwrap(); -/// -/// df.run_available(); -/// ``` -pub const LATTICE_BATCH: OperatorConstraints = OperatorConstraints { - name: "lattice_batch", - categories: &[OperatorCategory::LatticeFold], - persistence_args: RANGE_0, - type_args: &(0..=1), - hard_range_inn: RANGE_1, - soft_range_inn: RANGE_1, - hard_range_out: RANGE_1, - soft_range_out: RANGE_1, - num_args: 1, - is_external_input: false, - ports_inn: None, - ports_out: None, - properties: FlowProperties { - deterministic: FlowPropertyVal::Preserve, - monotonic: FlowPropertyVal::No, - inconsistency_tainted: false, - }, - input_delaytype_fn: |_| None, - write_fn: |wc @ &WriteContextArgs { - context, - hydroflow, - ident, - op_span, - root, - inputs, - outputs, - is_pull, - op_inst: - OperatorInstance { - arguments, - generics: OpInstGenerics { type_args, .. }, - .. - }, - .. - }, - _| { - let internal_buffer = wc.make_ident("internal_buffer"); - - let lattice_type = type_args - .get(0) - .map(ToTokens::to_token_stream) - .unwrap_or(quote_spanned!(op_span=> _)); - - let receiver = &arguments[0]; - let stream_ident = wc.make_ident("stream"); - - let write_prologue = quote_spanned! {op_span=> - let mut #stream_ident = ::std::boxed::Box::pin(#receiver); - let #internal_buffer = #hydroflow.add_state(::std::cell::RefCell::>::new(None)); - }; - - let (write_iterator, write_iterator_after) = if is_pull { - let input = &inputs[0]; - - ( - quote_spanned! {op_span=> - - { - let mut lattice = #context.state_ref(#internal_buffer).borrow_mut(); - for i in #input { - if let Some(lattice) = &mut *lattice { - #root::lattices::Merge::merge(lattice, i); - } else { - *lattice = Some(#root::lattices::LatticeFrom::lattice_from(i)); - }; - } - } - - let mut lattice = #context.state_ref(#internal_buffer).borrow_mut(); - let #ident = match #root::futures::stream::Stream::poll_next(#stream_ident.as_mut(), &mut ::std::task::Context::from_waker(&context.waker())) { - ::std::task::Poll::Ready(Some(_)) => { - if let Some(lattice) = lattice.take() { - Some(lattice).into_iter() - } else { - None.into_iter() - } - } - ::std::task::Poll::Ready(None) | ::std::task::Poll::Pending => { - None.into_iter() - } - }; - }, - quote_spanned! {op_span=>}, - ) - } else { - let output = &outputs[0]; - - ( - quote_spanned! {op_span=> - - let mut out = #output; - - let #ident = #root::pusherator::for_each::ForEach::new(|x| { - let mut lattice = #context.state_ref(#internal_buffer).borrow_mut(); - - if let Some(lattice) = &mut *lattice { - #root::lattices::Merge::merge(&mut *lattice, x); - } else { - *lattice = Some(#root::lattices::LatticeFrom::lattice_from(x)); - }; - }); - }, - quote_spanned! {op_span=> - { - let mut lattice = #context.state_ref(#internal_buffer).borrow_mut(); - - match #root::futures::stream::Stream::poll_next(#stream_ident.as_mut(), &mut ::std::task::Context::from_waker(&context.waker())) { - ::std::task::Poll::Ready(Some(_)) => { - if let Some(lattice) = lattice.take() { - #root::pusherator::Pusherator::give(&mut out, lattice); - } - }, - ::std::task::Poll::Ready(None) | ::std::task::Poll::Pending => { - }, - } - } - }, - ) - }; - - Ok(OperatorWriteOutput { - write_prologue, - write_iterator, - write_iterator_after, - ..Default::default() - }) - }, -}; diff --git a/hydroflow_lang/src/graph/ops/lattice_fold.rs b/hydroflow_lang/src/graph/ops/lattice_fold.rs new file mode 100644 index 00000000000..551202f2248 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/lattice_fold.rs @@ -0,0 +1,81 @@ +use syn::parse_quote_spanned; +use syn::spanned::Spanned; + +use super::{ + DelayType, FlowProperties, FlowPropertyVal, OpInstGenerics, OperatorCategory, + OperatorConstraints, OperatorInstance, WriteContextArgs, LATTICE_FOLD_REDUCE_FLOW_PROP_FN, + RANGE_1, +}; + +/// > 1 input stream, 1 output stream +/// +/// > Generic parameters: A `Lattice` type, must implement [`Merge`](https://hydro-project.github.io/hydroflow/doc/lattices/trait.Merge.html) +/// type. +/// +/// A specialized operator for merging lattices together into a accumulated value. Like [`fold()`](#fold) +/// but specialized for lattice types. `lattice_fold::()` is equivalent to `fold(MyLattice::default(), hydroflow::lattices::Merge::merge_owned)`. +/// +/// `lattice_fold` can also be provided with one generic lifetime persistence argument, either +/// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected +/// within the same tick. With `'static`, values will be remembered across ticks and will be +/// aggregated with pairs arriving in later ticks. When not explicitly specified persistence +/// defaults to `'tick`. +/// +/// `lattice_fold` is differentiated from `lattice_reduce` in that `lattice_fold` can accumulate into a different type from its input. +/// But it also means that the accumulating type must implement `Default` +/// +/// ```hydroflow +/// source_iter([hydroflow::lattices::set_union::SetUnionSingletonSet::new_from(7)]) +/// -> lattice_fold::<'static, hydroflow::lattices::set_union::SetUnionHashSet>() +/// -> assert_eq([hydroflow::lattices::set_union::SetUnionHashSet::new_from([7])]); +/// ``` +pub const LATTICE_FOLD: OperatorConstraints = OperatorConstraints { + name: "lattice_fold", + categories: &[OperatorCategory::LatticeFold], + hard_range_inn: RANGE_1, + soft_range_inn: RANGE_1, + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 0, + persistence_args: &(0..=1), + type_args: RANGE_1, + is_external_input: false, + ports_inn: None, + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::Preserve, + monotonic: FlowPropertyVal::Yes, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: Some(LATTICE_FOLD_REDUCE_FLOW_PROP_FN), + write_fn: |wc @ &WriteContextArgs { + root, + is_pull, + op_inst: + op_inst @ OperatorInstance { + generics: OpInstGenerics { type_args, .. }, + .. + }, + .. + }, + diagnostics| { + assert!(is_pull); + + let lat_type = &type_args[0]; + + let arguments = parse_quote_spanned! {lat_type.span()=> // Uses `lat_type.span()`! + <#lat_type as ::std::default::Default>::default(), #root::lattices::Merge::merge + }; + + let wc = WriteContextArgs { + op_inst: &OperatorInstance { + arguments, + ..op_inst.clone() + }, + ..wc.clone() + }; + + (super::fold::FOLD.write_fn)(&wc, diagnostics) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/lattice_join.rs b/hydroflow_lang/src/graph/ops/lattice_join.rs deleted file mode 100644 index 8294e96b349..00000000000 --- a/hydroflow_lang/src/graph/ops/lattice_join.rs +++ /dev/null @@ -1,209 +0,0 @@ -use quote::{quote_spanned, ToTokens}; -use syn::parse_quote; - -use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - Persistence, WriteContextArgs, RANGE_1, -}; -use crate::diagnostic::{Diagnostic, Level}; -use crate::graph::{OpInstGenerics, OperatorInstance}; - -/// > 2 input streams of type <(K, V1)> and <(K, V2)>, 1 output stream of type <(K, (V1, V2))> -/// -/// Performs a group-by with lattice-merge aggregate function on LHS and RHS inputs and then forms the -/// equijoin of the tuples in the input streams by their first (key) attribute. Note that the result nests the 2nd input field (values) into a tuple in the 2nd output field. -/// -/// You must specify the LHS and RHS lattice types, they cannot be inferred. -/// -/// ```hydroflow -/// // should print `(key, (2, 1))` -/// my_join = lattice_join::, hydroflow::lattices::Max>(); -/// source_iter(vec![("key", hydroflow::lattices::Max::new(0)), ("key", hydroflow::lattices::Max::new(2))]) -> [0]my_join; -/// source_iter(vec![("key", hydroflow::lattices::Max::new(1))]) -> [1]my_join; -/// my_join -> for_each(|(k, (v1, v2))| println!("({}, ({:?}, {:?}))", k, v1, v2)); -/// ``` -/// -/// `lattice_join` can also be provided with one or two generic lifetime persistence arguments, either -/// `'tick` or `'static`, to specify how join data persists. With `'tick`, pairs will only be -/// joined with corresponding pairs within the same tick. With `'static`, pairs will be remembered -/// across ticks and will be joined with pairs arriving in later ticks. When not explicitly -/// specified persistence defaults to `static. -/// -/// When two persistence arguments are supplied the first maps to port `0` and the second maps to -/// port `1`. -/// When a single persistence argument is supplied, it is applied to both input ports. -/// When no persistence arguments are applied it defaults to `'static` for both. -/// -/// The syntax is as follows: -/// ```hydroflow,ignore -/// lattice_join::, MaxRepr>(); // Or -/// lattice_join::<'static, MaxRepr, MaxRepr>(); -/// -/// lattice_join::<'tick, MaxRepr, MaxRepr>(); -/// -/// lattice_join::<'static, 'tick, MaxRepr, MaxRepr>(); -/// -/// lattice_join::<'tick, 'static, MaxRepr, MaxRepr>(); -/// // etc. -/// ``` -/// -/// ### Examples -/// -/// ```rustbook -/// use hydroflow::lattices::Min; -/// use hydroflow::lattices::Max; -/// -/// let mut df = hydroflow::hydroflow_syntax! { -/// my_join = lattice_join::<'tick, Min, Max>(); -/// source_iter([(7, Min::new(1)), (7, Min::new(2))]) -> [0]my_join; -/// source_iter([(7, Max::new(1)), (7, Max::new(2))]) -> [1]my_join; -/// my_join -> assert([(7, (Min::new(1), Max::new(2)))]); -/// }; -/// df.run_available(); -/// ``` -pub const LATTICE_JOIN: OperatorConstraints = OperatorConstraints { - name: "lattice_join", - categories: &[OperatorCategory::MultiIn], - hard_range_inn: &(2..=2), - soft_range_inn: &(2..=2), - hard_range_out: RANGE_1, - soft_range_out: RANGE_1, - num_args: 0, - persistence_args: &(0..=2), - type_args: &(2..=2), - is_external_input: false, - ports_inn: Some(|| super::PortListSpec::Fixed(parse_quote! { 0, 1 })), - ports_out: None, - properties: FlowProperties { - deterministic: FlowPropertyVal::Preserve, - monotonic: FlowPropertyVal::Preserve, - inconsistency_tainted: false, - }, - input_delaytype_fn: |_| None, - write_fn: |wc @ &WriteContextArgs { - root, - context, - op_span, - ident, - inputs, - op_inst: - OperatorInstance { - generics: - OpInstGenerics { - persistence_args, - type_args, - .. - }, - .. - }, - .. - }, - diagnostics| { - let lhs_type = type_args - .get(0) - .map(ToTokens::to_token_stream) - .unwrap_or(quote_spanned!(op_span=> _)); - - let rhs_type = type_args - .get(1) - .map(ToTokens::to_token_stream) - .unwrap_or(quote_spanned!(op_span=> _)); - - let mut make_joindata = |persistence, side| { - let joindata_ident = wc.make_ident(format!("joindata_{}", side)); - let borrow_ident = wc.make_ident(format!("joindata_{}_borrow", side)); - let (init, borrow) = match persistence { - Persistence::Tick => ( - quote_spanned! {op_span=> - #root::util::monotonic_map::MonotonicMap::new_init( - #root::compiled::pull::HalfJoinStateLattice::default() - ) - }, - quote_spanned! {op_span=> - &mut *#borrow_ident.get_mut_clear(#context.current_tick()) - }, - ), - Persistence::Static => ( - quote_spanned! {op_span=> - #root::compiled::pull::HalfJoinStateLattice::default() - }, - quote_spanned! {op_span=> - &mut #borrow_ident - }, - ), - Persistence::Mutable => { - diagnostics.push(Diagnostic::spanned( - op_span, - Level::Error, - "An implementation of 'mutable does not exist", - )); - return Err(()); - } - }; - Ok((joindata_ident, borrow_ident, init, borrow)) - }; - - let persistences = match persistence_args[..] { - [] => [Persistence::Static, Persistence::Static], - [a] => [a, a], - [a, b] => [a, b], - _ => unreachable!(), - }; - - let (lhs_joindata_ident, lhs_borrow_ident, lhs_init, lhs_borrow) = - make_joindata(persistences[0], "lhs")?; - let (rhs_joindata_ident, rhs_borrow_ident, rhs_init, rhs_borrow) = - make_joindata(persistences[1], "rhs")?; - - let join_keys_ident = wc.make_ident("joinkeys"); - let join_keys_borrow_ident = wc.make_ident("joinkeys_borrow"); - - let write_prologue = quote_spanned! {op_span=> - let #lhs_joindata_ident = df.add_state(::std::cell::RefCell::new( - #lhs_init - )); - let #rhs_joindata_ident = df.add_state(::std::cell::RefCell::new( - #rhs_init - )); - let #join_keys_ident = df.add_state(::std::cell::RefCell::new( - #root::rustc_hash::FxHashSet::default() - )); - }; - - let lhs = &inputs[0]; - let rhs = &inputs[1]; - let write_iterator = quote_spanned! {op_span=> - let mut #lhs_borrow_ident = #context.state_ref(#lhs_joindata_ident).borrow_mut(); - let mut #rhs_borrow_ident = #context.state_ref(#rhs_joindata_ident).borrow_mut(); - let mut #join_keys_borrow_ident = #context.state_ref(#join_keys_ident).borrow_mut(); - - let #ident = { - /// Limit error propagation by bounding locally, erasing output iterator type. - #[inline(always)] - fn check_inputs<'a, Key, I1, Lhs, LhsDelta, I2, Rhs, RhsDelta>( - lhs: I1, - rhs: I2, - updated_keys: &'a mut #root::rustc_hash::FxHashSet, - lhs_state: &'a mut #root::compiled::pull::HalfJoinStateLattice, - rhs_state: &'a mut #root::compiled::pull::HalfJoinStateLattice, - ) -> impl 'a + Iterator - where - Key: Eq + std::hash::Hash + Clone, - Lhs: #root::lattices::Merge + Clone + #root::lattices::LatticeFrom, - Rhs: #root::lattices::Merge + Clone + #root::lattices::LatticeFrom, - I1: Iterator, - I2: Iterator, - { - #root::compiled::pull::SymmetricHashJoinLattice::new_from_mut(lhs, rhs, updated_keys, lhs_state, rhs_state) - } - check_inputs::<_, _, #lhs_type, _, _, #rhs_type, _>(#lhs, #rhs, &mut *#join_keys_borrow_ident, #lhs_borrow, #rhs_borrow) - }; - }; - - Ok(OperatorWriteOutput { - write_prologue, - write_iterator, - ..Default::default() - }) - }, -}; diff --git a/hydroflow_lang/src/graph/ops/lattice_merge.rs b/hydroflow_lang/src/graph/ops/lattice_reduce.rs similarity index 74% rename from hydroflow_lang/src/graph/ops/lattice_merge.rs rename to hydroflow_lang/src/graph/ops/lattice_reduce.rs index 1fda39d6706..ff4a46fcac2 100644 --- a/hydroflow_lang/src/graph/ops/lattice_merge.rs +++ b/hydroflow_lang/src/graph/ops/lattice_reduce.rs @@ -4,7 +4,8 @@ use syn::spanned::Spanned; use super::{ DelayType, FlowProperties, FlowPropertyVal, OpInstGenerics, OperatorCategory, - OperatorConstraints, OperatorInstance, WriteContextArgs, RANGE_1, + OperatorConstraints, OperatorInstance, WriteContextArgs, LATTICE_FOLD_REDUCE_FLOW_PROP_FN, + RANGE_1, }; use crate::graph::ops::OperatorWriteOutput; @@ -13,23 +14,26 @@ use crate::graph::ops::OperatorWriteOutput; /// > Generic parameters: A `Lattice` type, must implement [`Merge`](https://hydro-project.github.io/hydroflow/doc/lattices/trait.Merge.html) /// type. /// -/// A specialized operator for merging lattices together into a accumulated value. Like [`reduce()`](#reduce) -/// but specialized for lattice types. `lattice_merge::()` is equivalent to `reduce(hydroflow::lattices::Merge::merge_owned)`. +/// A specialized operator for merging lattices together into an accumulated value. Like [`reduce()`](#reduce) +/// but specialized for lattice types. `lattice_reduce::()` is equivalent to `reduce(hydroflow::lattices::Merge::merge_owned)`. /// -/// `lattice_merge` can also be provided with one generic lifetime persistence argument, either +/// `lattice_reduce` can also be provided with one generic lifetime persistence argument, either /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected /// within the same tick. With `'static`, values will be remembered across ticks and will be /// aggregated with pairs arriving in later ticks. When not explicitly specified persistence -/// defaults to `'static`. +/// defaults to `'tick`. +/// +/// `lattice_reduce` is differentiated from `lattice_fold` in that `lattice_reduce` does not require the accumulating type to implement `Default`. +/// But it also means that the accumulating function inputs and the accumulating type must be the same. /// /// ```hydroflow /// source_iter([1,2,3,4,5]) /// -> map(hydroflow::lattices::Max::new) -/// -> lattice_merge::<'static, hydroflow::lattices::Max>() -/// -> assert([hydroflow::lattices::Max(5)]); +/// -> lattice_reduce::<'static, hydroflow::lattices::Max>() +/// -> assert_eq([hydroflow::lattices::Max::new(5)]); /// ``` -pub const LATTICE_MERGE: OperatorConstraints = OperatorConstraints { - name: "lattice_merge", +pub const LATTICE_REDUCE: OperatorConstraints = OperatorConstraints { + name: "lattice_reduce", categories: &[OperatorCategory::LatticeFold], hard_range_inn: RANGE_1, soft_range_inn: RANGE_1, @@ -47,6 +51,7 @@ pub const LATTICE_MERGE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: Some(LATTICE_FOLD_REDUCE_FLOW_PROP_FN), write_fn: |wc @ &WriteContextArgs { root, inputs, @@ -68,7 +73,7 @@ pub const LATTICE_MERGE: OperatorConstraints = OperatorConstraints { let lat_type = &type_args[0]; let arguments = parse_quote_spanned! {lat_type.span()=> // Uses `lat_type.span()`! - #root::lattices::Merge::<#lat_type>::merge_owned + #root::lattices::Merge::<#lat_type>::merge }; let wc = WriteContextArgs { op_inst: &OperatorInstance { diff --git a/hydroflow_lang/src/graph/ops/map.rs b/hydroflow_lang/src/graph/ops/map.rs index c07a7ae2126..aa2db14c687 100644 --- a/hydroflow_lang/src/graph/ops/map.rs +++ b/hydroflow_lang/src/graph/ops/map.rs @@ -1,8 +1,8 @@ use quote::quote_spanned; use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorInstance, - OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, + FlowPropArgs, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorInstance, OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, }; /// > 1 input stream, 1 output stream @@ -17,7 +17,7 @@ use super::{ /// /// ```hydroflow /// source_iter(vec!["hello", "world"]) -> map(|x| x.to_uppercase()) -/// -> assert(["HELLO", "WORLD"]); +/// -> assert_eq(["HELLO", "WORLD"]); /// ``` pub const MAP: OperatorConstraints = OperatorConstraints { name: "map", @@ -38,7 +38,11 @@ pub const MAP: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, - write_fn: |&WriteContextArgs { + flow_prop_fn: Some(|FlowPropArgs { flow_props_in, .. }, _diagnostics| { + // Preserve input flow properties. + Ok(vec![flow_props_in[0]]) + }), + write_fn: |wc @ &WriteContextArgs { root, op_span, ident, @@ -49,15 +53,16 @@ pub const MAP: OperatorConstraints = OperatorConstraints { .. }, _| { + let func = wc.wrap_check_func_arg(&arguments[0]); let write_iterator = if is_pull { let input = &inputs[0]; quote_spanned! {op_span=> - let #ident = #input.map(#arguments); + let #ident = #input.map(#func); } } else { let output = &outputs[0]; quote_spanned! {op_span=> - let #ident = #root::pusherator::map::Map::new(#arguments, #output); + let #ident = #root::pusherator::map::Map::new(#func, #output); } }; Ok(OperatorWriteOutput { diff --git a/hydroflow_lang/src/graph/ops/mod.rs b/hydroflow_lang/src/graph/ops/mod.rs index 1574251cfc0..546c3f1b355 100644 --- a/hydroflow_lang/src/graph/ops/mod.rs +++ b/hydroflow_lang/src/graph/ops/mod.rs @@ -1,3 +1,5 @@ +//! Hydroflow's operators + use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::ops::{Bound, RangeBounds}; @@ -8,31 +10,49 @@ use quote::quote_spanned; use serde::{Deserialize, Serialize}; use slotmap::Key; use syn::punctuated::Punctuated; -use syn::{parse_quote_spanned, Token}; +use syn::{parse_quote_spanned, Expr, Token}; -use super::{GraphNodeId, GraphSubgraphId, Node, OpInstGenerics, OperatorInstance, PortIndexValue}; -use crate::diagnostic::Diagnostic; +use super::{ + FlowProps, GraphNodeId, GraphSubgraphId, LatticeFlowType, Node, OpInstGenerics, + OperatorInstance, PortIndexValue, +}; +use crate::diagnostic::{Diagnostic, Level}; use crate::parse::{Operator, PortIndex}; +/// The delay (soft barrier) type, for each input to an operator if needed. #[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Debug)] pub enum DelayType { + /// Input should be collected over the preceeding stratum. Stratum, + /// Input should be collected over the previous tick. Tick, } +/// Specification of the named (or unnamed) ports for an operator's inputs or outputs. pub enum PortListSpec { + /// Any number of unnamed (or optionally named) ports. Variadic, + /// A specific number of named ports. Fixed(Punctuated), } +/// Flow property preservation values. +/// TODO(mingwei): deprecated? #[derive(Default, Debug, Eq, PartialEq, Clone, Copy)] pub enum FlowPropertyVal { + /// Property is always false. #[default] No, + /// Property is always true. Yes, + /// Property is preserved from input. Preserve, + /// Property preservation depends on the arguments supplied to the operator. DependsOnArgs, } + +/// Flow properties of each edge. +/// TODO(mingwei): deprecated? #[derive(Default, Debug, Eq, PartialEq, Clone, Copy)] pub struct FlowProperties { /// Is the flow deterministic. @@ -82,9 +102,57 @@ pub struct OperatorConstraints { /// Determines if this input must be preceeded by a stratum barrier. pub input_delaytype_fn: fn(&PortIndexValue) -> Option, - /// Emit code in multiple locations. See [`OperatorWriteOutput`]. + /// Return the output flow types for the given input flow types. + /// + /// The first [`FlowPropArgs`] argument provides the input flow types (if set) and other + /// arguments such as the operator span, operator name, etc. + /// + /// The second argument is a vec to push [`Diagnostic`]s into to emit them. + /// + /// If only one flow type is returned for an operator with multiple outputs, that flow type + /// will be used for all outputs. Besides that case, it is an error to return a number of flow + /// props which does not match the number of outputs. + pub flow_prop_fn: Option, + + /// The operator's codegen. Returns code that is emited is several different locations. See [`OperatorWriteOutput`]. pub write_fn: WriteFn, } + +/// Type alias for [`OperatorConstraints::flow_prop_fn`]'s type. +pub type FlowPropFn = + fn(FlowPropArgs<'_>, &mut Vec) -> Result>, ()>; + +/// Type alias for [`OperatorConstraints::write_fn`]'s type. +pub type WriteFn = + fn(&WriteContextArgs<'_>, &mut Vec) -> Result; + +/// Arguments provided to [`OperatorConstraints::flow_prop_fn`]. +pub struct FlowPropArgs<'a> { + /// The source span of this operator. + pub op_span: Span, + + /// Operator name. + pub op_name: &'static str, + /// Operator instance arguments object. + pub op_inst: &'a OperatorInstance, + + /// Flow properties corresponding to each input. + /// + /// Sometimes (due to cycles, and for now usually due to no-yet-updated operators) the input + /// flow props might not be set yet (`None`). + pub flow_props_in: &'a [Option], + + /// Internal: TODO(mingwei): new token value for output flows. + pub(crate) new_star_ord: usize, +} +impl FlowPropArgs<'_> { + /// Returns a new `star_ord` token, representing a new order/provenance. + /// TODO(mingwei): This shouldn't return the same value for multiple calls. + pub fn new_star_ord(&self) -> usize { + self.new_star_ord + } +} + impl Debug for OperatorConstraints { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OperatorConstraints") @@ -100,14 +168,13 @@ impl Debug for OperatorConstraints { .field("ports_inn", &self.ports_inn) .field("ports_out", &self.ports_out) // .field("input_delaytype_fn", &self.input_delaytype_fn) + // .field("flow_prop_fn", &self.flow_prop_fn) // .field("write_fn", &self.write_fn) .finish() } } -pub type WriteFn = - fn(&WriteContextArgs<'_>, &mut Vec) -> Result; - +/// The code generated and returned by a [`OperatorConstraints::write_fn`]. #[derive(Default)] #[non_exhaustive] pub struct OperatorWriteOutput { @@ -125,10 +192,15 @@ pub struct OperatorWriteOutput { pub write_iterator_after: TokenStream, } +/// Convenience range: zero or more (any number). pub const RANGE_ANY: &'static dyn RangeTrait = &(0..); +/// Convenience range: exactly zero. pub const RANGE_0: &'static dyn RangeTrait = &(0..=0); +/// Convenience range: exactly one. pub const RANGE_1: &'static dyn RangeTrait = &(1..=1); +/// Helper to write the `write_iterator` portion of [`OperatorConstraints::write_fn`] output for +/// unary identity operators. pub fn identity_write_iterator_fn( &WriteContextArgs { root, @@ -164,11 +236,12 @@ pub fn identity_write_iterator_fn( let #ident = { fn check_output, Item>(push: Push) -> impl #root::pusherator::Pusherator { push } check_output::<_, #generic_type>(#output) - } + }; } } } +/// [`OperatorConstraints::write_fn`] for unary identity operators. pub const IDENTITY_WRITE_FN: WriteFn = |write_context_args, _| { let write_iterator = identity_write_iterator_fn(write_context_args); Ok(OperatorWriteOutput { @@ -177,6 +250,34 @@ pub const IDENTITY_WRITE_FN: WriteFn = |write_context_args, _| { }) }; +/// [`OperatorConstraints::flow_prop_fn`] for `lattice_fold` and `lattice_reduce`. +pub const LATTICE_FOLD_REDUCE_FLOW_PROP_FN: FlowPropFn = + |fp @ FlowPropArgs { + op_span, op_name, .. + }, + diagnostics| { + let input_flow_type = fp.flow_props_in[0].and_then(|fp| fp.lattice_flow_type); + match input_flow_type { + Some(LatticeFlowType::Delta) => (), + Some(LatticeFlowType::Cumul) => diagnostics.push(Diagnostic::spanned( + op_span, + Level::Warning, + format!("`{}` input is already cumulative lattice flow, this operator is redundant.", op_name), + )), + None => diagnostics.push(Diagnostic::spanned( + op_span, + Level::Warning, + format!("`{}` expects lattice flow input, has sequential input. This may be an error in the future.", op_name), + )), + } + Ok(vec![Some(FlowProps { + star_ord: fp.new_star_ord(), + lattice_flow_type: Some(LatticeFlowType::Cumul), + })]) + }; + +/// Helper to write the `write_iterator` portion of [`OperatorConstraints::write_fn`] output for +/// the null operator - an operator that ignores all inputs and produces no output. pub fn null_write_iterator_fn( &WriteContextArgs { root, @@ -206,11 +307,13 @@ pub fn null_write_iterator_fn( quote_spanned! {op_span=> #[allow(clippy::let_unit_value)] let _ = (#(#outputs),*); - let #ident = #root::pusherator::for_each::ForEach::<_, #iter_type>::new(std::mem::drop); + let #ident = #root::pusherator::null::Null::<#iter_type>::new(); } } } +/// [`OperatorConstraints::write_fn`] for the null operator - an operator that ignores all inputs +/// and produces no output. pub const NULL_WRITE_FN: WriteFn = |write_context_args, _| { let write_iterator = null_write_iterator_fn(write_context_args); Ok(OperatorWriteOutput { @@ -222,6 +325,7 @@ pub const NULL_WRITE_FN: WriteFn = |write_context_args, _| { macro_rules! declare_ops { ( $( $mod:ident :: $op:ident, )* ) => { $( pub(crate) mod $mod; )* + /// All Hydroflow operators. pub const OPERATORS: &[OperatorConstraints] = &[ $( $mod :: $op, )* ]; @@ -229,14 +333,18 @@ macro_rules! declare_ops { } declare_ops![ anti_join::ANTI_JOIN, + anti_join_multiset::ANTI_JOIN_MULTISET, assert::ASSERT, - batch::BATCH, + assert_eq::ASSERT_EQ, + cast::CAST, cross_join::CROSS_JOIN, + cross_join_multiset::CROSS_JOIN_MULTISET, demux::DEMUX, dest_file::DEST_FILE, dest_sink::DEST_SINK, dest_sink_serde::DEST_SINK_SERDE, difference::DIFFERENCE, + difference_multiset::DIFFERENCE_MULTISET, enumerate::ENUMERATE, filter::FILTER, filter_map::FILTER_MAP, @@ -248,20 +356,27 @@ declare_ops![ initialize::INITIALIZE, inspect::INSPECT, join::JOIN, + join_fused::JOIN_FUSED, + join_fused_lhs::JOIN_FUSED_LHS, + join_fused_rhs::JOIN_FUSED_RHS, + join_multiset::JOIN_MULTISET, fold_keyed::FOLD_KEYED, reduce_keyed::REDUCE_KEYED, - lattice_batch::LATTICE_BATCH, - lattice_join::LATTICE_JOIN, - lattice_merge::LATTICE_MERGE, + _lattice_fold_batch::_LATTICE_FOLD_BATCH, + lattice_fold::LATTICE_FOLD, + _lattice_join_fused_join::_LATTICE_JOIN_FUSED_JOIN, + lattice_reduce::LATTICE_REDUCE, map::MAP, union::UNION, multiset_delta::MULTISET_DELTA, next_stratum::NEXT_STRATUM, - next_tick::NEXT_TICK, + defer_signal::DEFER_SIGNAL, + defer_tick::DEFER_TICK, null::NULL, persist::PERSIST, persist_mut::PERSIST_MUT, persist_mut_keyed::PERSIST_MUT_KEYED, + py_udf::PY_UDF, reduce::REDUCE, spin::SPIN, sort::SORT, @@ -269,6 +384,7 @@ declare_ops![ source_file::SOURCE_FILE, source_interval::SOURCE_INTERVAL, source_iter::SOURCE_ITER, + source_iter_delta::SOURCE_ITER_DELTA, source_json::SOURCE_JSON, source_stdin::SOURCE_STDIN, source_stream::SOURCE_STREAM, @@ -280,11 +396,13 @@ declare_ops![ zip_longest::ZIP_LONGEST, ]; +/// Get the operator lookup table, generating it if needed. pub fn operator_lookup() -> &'static HashMap<&'static str, &'static OperatorConstraints> { pub static OPERATOR_LOOKUP: OnceLock> = OnceLock::new(); OPERATOR_LOOKUP.get_or_init(|| OPERATORS.iter().map(|op| (op.name, op)).collect()) } +/// Find an operator by [`Node`]. pub fn find_node_op_constraints(node: &Node) -> Option<&'static OperatorConstraints> { if let Node::Operator(operator) = node { find_op_op_constraints(operator) @@ -292,11 +410,13 @@ pub fn find_node_op_constraints(node: &Node) -> Option<&'static OperatorConstrai None } } +/// Find an operator by an AST [`Operator`]. pub fn find_op_op_constraints(operator: &Operator) -> Option<&'static OperatorConstraints> { let name = &*operator.name_string(); operator_lookup().get(name).copied() } +/// Context arguments provided to [`OperatorConstraints::write_fn`]. #[derive(Clone)] pub struct WriteContextArgs<'a> { /// `hydroflow` crate name for `use #root::something`. @@ -328,8 +448,19 @@ pub struct WriteContextArgs<'a> { pub op_name: &'static str, /// Operator instance arguments object. pub op_inst: &'a OperatorInstance, + + /// Flow properties corresponding to each input. + /// + /// Sometimes (due to cycles, and for now usually due to no-yet-updated operators) the input + /// flow props might not be set yet (`None`). + pub flow_props_in: &'a [Option], } impl WriteContextArgs<'_> { + /// Generate a (almost certainly) unique identifier with the given suffix. + /// + /// Includes the subgraph and node IDs in the generated identifier. + /// + /// This will always return the same identifier for a given `suffix`. pub fn make_ident(&self, suffix: impl AsRef) -> Ident { Ident::new( &*format!( @@ -341,18 +472,48 @@ impl WriteContextArgs<'_> { self.op_span, ) } + + /// Wraps the `func_arg` closure with a type checker macro corresponding to the first `flow_props` flow type. + /// + /// * `None` => No checking. + /// * `Some(Cumul) => Monotonic function. + /// * `Some(Delta) => Morphism. + pub fn wrap_check_func_arg(&self, func_arg: &Expr) -> TokenStream { + let root = self.root; + let span = self.op_span; + match self + .flow_props_in + .get(0) + .copied() + .flatten() + .and_then(|flow_props| flow_props.lattice_flow_type) + { + None => quote_spanned!(span=> #func_arg), + Some(LatticeFlowType::Cumul) => quote_spanned! {span=> + #root::monotonic_fn!(#func_arg) + }, + Some(LatticeFlowType::Delta) => quote_spanned! {span=> + #root::morphism!(#func_arg) + }, + } + } } +/// An object-safe version of [`RangeBounds`]. pub trait RangeTrait: Send + Sync + Debug where T: ?Sized, { + /// Start (lower) bound. fn start_bound(&self) -> Bound<&T>; + /// End (upper) bound. fn end_bound(&self) -> Bound<&T>; + /// Returns if `item` is contained in this range. fn contains(&self, item: &T) -> bool where T: PartialOrd; + /// Turn this range into a human-readable string. fn human_string(&self) -> String where T: Display + PartialEq, @@ -403,13 +564,18 @@ where } } +/// Persistence lifetimes: `'tick`, `'static`, or `'mutable`. #[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum Persistence { + /// Persistence for one tick at-a-time only. Tick, + /// Persistene across all ticks. Static, + /// Mutability. Mutable, } +/// Helper which creates a error message string literal for when the Tokio runtime is not found. fn make_missing_runtime_msg(op_name: &str) -> Literal { Literal::string(&*format!("`{}()` must be used within a Tokio runtime. For example, use `#[hydroflow::main]` on your main method.", op_name)) } @@ -417,6 +583,7 @@ fn make_missing_runtime_msg(op_name: &str) -> Literal { /// Operator categories, for docs. /// /// See source of [`Self::description`] for description of variants. +#[allow(missing_docs)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum OperatorCategory { Map, @@ -431,6 +598,7 @@ pub enum OperatorCategory { Source, Sink, Control, + CompilerFusionOperator, } impl OperatorCategory { /// Human-readible heading name, for docs. @@ -448,6 +616,7 @@ impl OperatorCategory { OperatorCategory::Source => "Sources", OperatorCategory::Sink => "Sinks", OperatorCategory::Control => "Control Flow Operators", + OperatorCategory::CompilerFusionOperator => "Compiler Fusion Operators", } } /// Human description, for docs. @@ -469,6 +638,9 @@ impl OperatorCategory { "Operators which consume input elements (and produce no outputs)." } OperatorCategory::Control => "Operators which affect control flow/scheduling.", + OperatorCategory::CompilerFusionOperator => { + "Operators which are necessary to implement certain optimizations and rewrite rules" + } } } } diff --git a/hydroflow_lang/src/graph/ops/multiset_delta.rs b/hydroflow_lang/src/graph/ops/multiset_delta.rs index eb352ebece9..36c9cf501c7 100644 --- a/hydroflow_lang/src/graph/ops/multiset_delta.rs +++ b/hydroflow_lang/src/graph/ops/multiset_delta.rs @@ -20,7 +20,7 @@ use super::{ /// input_send.send(4).unwrap(); /// input_send.send(3).unwrap(); /// flow.run_tick(); -/// // 3, 4, 3 +/// // 3, 4, /// /// input_send.send(3).unwrap(); /// input_send.send(5).unwrap(); @@ -49,6 +49,7 @@ pub const MULTISET_DELTA: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/next_stratum.rs b/hydroflow_lang/src/graph/ops/next_stratum.rs index 6b907f8005e..23cb0adcf8f 100644 --- a/hydroflow_lang/src/graph/ops/next_stratum.rs +++ b/hydroflow_lang/src/graph/ops/next_stratum.rs @@ -24,5 +24,6 @@ pub const NEXT_STRATUM: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: IDENTITY_WRITE_FN, }; diff --git a/hydroflow_lang/src/graph/ops/next_tick.rs b/hydroflow_lang/src/graph/ops/next_tick.rs deleted file mode 100644 index 054e87f7d49..00000000000 --- a/hydroflow_lang/src/graph/ops/next_tick.rs +++ /dev/null @@ -1,58 +0,0 @@ -use super::{ - DelayType, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, - IDENTITY_WRITE_FN, RANGE_0, RANGE_1, -}; - -/// Delays all elements which pass through to the next tick. In short, -/// execution of a hydroflow graph runs as a sequence of distinct "ticks". -/// Non-monotonic operators compute their output in terms of each tick so -/// execution doesn't have to block, and it is up to the user to coordinate -/// data between tick executions to achieve the desired result. -/// -/// An tick may be divided into multiple _strata_, see the [`next_stratum()`](#next_stratum) -/// operator. -/// -/// In the example below `next_tick()` is used alongside `difference()` to -/// ignore any items in the current tick that already appeared in the previous -/// tick. -/// ```rustbook -/// // Outputs 1 2 3 4 5 6 (on separate lines). -/// let (input_send, input_recv) = hydroflow::util::unbounded_channel::(); -/// let mut flow = hydroflow::hydroflow_syntax! { -/// inp = source_stream(input_recv) -> tee(); -/// diff = difference() -> for_each(|x| println!("{}", x)); -/// inp -> [pos]diff; -/// inp -> next_tick() -> [neg]diff; -/// }; -/// -/// for x in [1, 2, 3, 4] { -/// input_send.send(x).unwrap(); -/// } -/// flow.run_tick(); -/// -/// for x in [3, 4, 5, 6] { -/// input_send.send(x).unwrap(); -/// } -/// flow.run_tick(); -/// ``` -pub const NEXT_TICK: OperatorConstraints = OperatorConstraints { - name: "next_tick", - categories: &[OperatorCategory::Control], - hard_range_inn: RANGE_1, - soft_range_inn: RANGE_1, - hard_range_out: RANGE_1, - soft_range_out: RANGE_1, - num_args: 0, - persistence_args: RANGE_0, - type_args: RANGE_0, - is_external_input: false, - ports_inn: None, - ports_out: None, - properties: FlowProperties { - deterministic: FlowPropertyVal::Preserve, - monotonic: FlowPropertyVal::Preserve, - inconsistency_tainted: false, - }, - input_delaytype_fn: |_| Some(DelayType::Tick), - write_fn: IDENTITY_WRITE_FN, -}; diff --git a/hydroflow_lang/src/graph/ops/null.rs b/hydroflow_lang/src/graph/ops/null.rs index f0a0c6f0a20..775871c01ab 100644 --- a/hydroflow_lang/src/graph/ops/null.rs +++ b/hydroflow_lang/src/graph/ops/null.rs @@ -35,5 +35,6 @@ pub const NULL: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: NULL_WRITE_FN, }; diff --git a/hydroflow_lang/src/graph/ops/persist.rs b/hydroflow_lang/src/graph/ops/persist.rs index 7a29f00446c..a969eb075fb 100644 --- a/hydroflow_lang/src/graph/ops/persist.rs +++ b/hydroflow_lang/src/graph/ops/persist.rs @@ -12,13 +12,13 @@ use super::{ /// // on every tick. /// source_iter(["hello"]) /// -> persist() -/// -> assert(["hello"]); +/// -> assert_eq(["hello"]); /// ``` /// -/// `persist()` can be used to introduce statefulness into stateless pipelines. For example this -/// join only stores data for single `'tick`. The `persist()` operator introduces statefulness +/// `persist()` can be used to introduce statefulness into stateless pipelines. In the example below, the +/// join only stores data for single tick. The `persist()` operator introduces statefulness /// across ticks. This can be useful for optimization transformations within the hydroflow -/// compiler. +/// compiler. Equivalently, we could specify that the join has `static` persistence (`my_join = join::<'static>()`). /// ```rustbook /// let (input_send, input_recv) = hydroflow::util::unbounded_channel::<(&str, &str)>(); /// let mut flow = hydroflow::hydroflow_syntax! { @@ -53,6 +53,7 @@ pub const PERSIST: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/persist_mut.rs b/hydroflow_lang/src/graph/ops/persist_mut.rs index 57d19df1026..6d9088b010c 100644 --- a/hydroflow_lang/src/graph/ops/persist_mut.rs +++ b/hydroflow_lang/src/graph/ops/persist_mut.rs @@ -12,7 +12,7 @@ use super::{ /// ```hydroflow /// source_iter([hydroflow::util::Persistence::Persist(1), hydroflow::util::Persistence::Persist(2), hydroflow::util::Persistence::Delete(1)]) /// -> persist_mut() -/// -> assert([2]); +/// -> assert_eq([2]); /// ``` pub const PERSIST_MUT: OperatorConstraints = OperatorConstraints { name: "persist_mut", @@ -33,6 +33,7 @@ pub const PERSIST_MUT: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/persist_mut_keyed.rs b/hydroflow_lang/src/graph/ops/persist_mut_keyed.rs index 2c40b01fec7..f3efc19b23c 100644 --- a/hydroflow_lang/src/graph/ops/persist_mut_keyed.rs +++ b/hydroflow_lang/src/graph/ops/persist_mut_keyed.rs @@ -12,7 +12,7 @@ use super::{ /// ```hydroflow /// source_iter([hydroflow::util::PersistenceKeyed::Persist(0, 1), hydroflow::util::PersistenceKeyed::Persist(1, 1), hydroflow::util::PersistenceKeyed::Delete(1)]) /// -> persist_mut_keyed() -/// -> assert([(0, 1)]); +/// -> assert_eq([(0, 1)]); /// ``` pub const PERSIST_MUT_KEYED: OperatorConstraints = OperatorConstraints { name: "persist_mut_keyed", @@ -33,6 +33,7 @@ pub const PERSIST_MUT_KEYED: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/py_udf.rs b/hydroflow_lang/src/graph/ops/py_udf.rs new file mode 100644 index 00000000000..51480370b86 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/py_udf.rs @@ -0,0 +1,151 @@ +use proc_macro2::Literal; +use quote::quote_spanned; + +use super::{ + FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorInstance, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, +}; + +/// > Arguments: First, the source code for a python module, second, the name of a unary function +/// > defined within the module source code. +/// +/// **Requires the "python" feature to be enabled.** +/// +/// An operator which allows you to run a python udf. Input arguments must be a stream of tuples +/// whose items implement [`IntoPy`](https://docs.rs/pyo3/latest/pyo3/conversion/trait.IntoPy.html). +/// See the [relevant pyo3 docs here](https://pyo3.rs/latest/conversions/tables#mapping-of-rust-types-to-python-types). +/// +/// Output items are of type `PyResult>`. Rust native types can be extracted using +/// `.extract()`, see the [relevant pyo3 docs here](https://pyo3.rs/latest/conversions/traits#extract-and-the-frompyobject-trait) +/// or the examples below. +/// +/// ```hydroflow +/// use pyo3::prelude::*; +/// +/// source_iter(0..10) +/// -> map(|x| (x,)) +/// -> py_udf(r#" +/// def fib(n): +/// if n < 2: +/// return n +/// else: +/// return fib(n - 2) + fib(n - 1) +/// "#, "fib") +/// -> map(|x: PyResult>| Python::with_gil(|py| { +/// usize::extract(x.unwrap().as_ref(py)).unwrap() +/// })) +/// -> assert_eq([0, 1, 1, 2, 3, 5, 8, 13, 21, 34]); +/// ``` +/// +/// ```hydroflow +/// use pyo3::prelude::*; +/// +/// source_iter([(5,1)]) +/// -> py_udf(r#" +/// def add(a, b): +/// return a + b +/// "#, "add") +/// -> map(|x: PyResult>| Python::with_gil(|py| { +/// usize::extract(x.unwrap().as_ref(py)).unwrap() +/// })) +/// -> assert_eq([6]); +/// ``` +pub const PY_UDF: OperatorConstraints = OperatorConstraints { + name: "py_udf", + categories: &[OperatorCategory::Map], + hard_range_inn: RANGE_1, + soft_range_inn: RANGE_1, + hard_range_out: RANGE_1, + soft_range_out: RANGE_1, + num_args: 2, + persistence_args: RANGE_0, + type_args: RANGE_0, + is_external_input: false, + ports_inn: None, + ports_out: None, + properties: FlowProperties { + deterministic: FlowPropertyVal::DependsOnArgs, + monotonic: FlowPropertyVal::DependsOnArgs, + inconsistency_tainted: false, + }, + input_delaytype_fn: |_| None, + flow_prop_fn: None, + write_fn: |wc @ &WriteContextArgs { + root, + op_span, + context, + hydroflow, + ident, + inputs, + outputs, + is_pull, + op_name, + op_inst: OperatorInstance { arguments, .. }, + .. + }, + _| { + let py_src = &arguments[0]; + let py_func_name = &arguments[1]; + + let py_func_ident = wc.make_ident("py_func"); + + let err_lit = Literal::string(&*format!( + "Hydroflow 'python' feature must be enabled to use `{}`", + op_name + )); + + let write_prologue = quote_spanned! {op_span=> + #root::__python_feature_gate! { + { + let #py_func_ident = { + #root::pyo3::prepare_freethreaded_python(); + let func = #root::pyo3::Python::with_gil::<_, #root::pyo3::PyResult<#root::pyo3::Py<#root::pyo3::PyAny>>>(|py| { + Ok(#root::pyo3::types::PyModule::from_code( + py, + #py_src, + "_filename", + "_modulename", + )? + .getattr(#py_func_name)? + .into()) + }).expect("Failed to compile python."); + #hydroflow.add_state(func) + }; + }, + { + ::std::compile_error!(#err_lit); + } + } + }; + let closure = quote_spanned! {op_span=> + |x| { + #root::__python_feature_gate! { + { + // TODO(mingwei): maybe this can be outside the closure? + let py_func = #context.state_ref(#py_func_ident); + #root::pyo3::Python::with_gil(|py| py_func.call1(py, x)) + }, + { + panic!() + } + } + } + }; + let write_iterator = if is_pull { + let input = &inputs[0]; + quote_spanned! {op_span=> + let #ident = #input.map(#closure); + } + } else { + let output = &outputs[0]; + quote_spanned! {op_span=> + let #ident = #root::pusherator::map::Map::new(#closure, #output); + } + }; + Ok(OperatorWriteOutput { + write_prologue, + write_iterator, + ..Default::default() + }) + }, +}; diff --git a/hydroflow_lang/src/graph/ops/reduce.rs b/hydroflow_lang/src/graph/ops/reduce.rs index 23d51a7c257..2badee043b9 100644 --- a/hydroflow_lang/src/graph/ops/reduce.rs +++ b/hydroflow_lang/src/graph/ops/reduce.rs @@ -17,13 +17,18 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// /// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object). /// +/// `reduce` can also be provided with one generic lifetime persistence argument, either +/// `'tick` or `'static`, to specify how data persists. With `'tick`, elements will only be collected +/// within the same tick. With `'static`, the accumulator will be remembered across ticks and elements +/// are aggregated with elements arriving in later ticks. When not explicitly specified persistence +/// defaults to `'tick`. +/// /// ```hydroflow /// source_iter([1,2,3,4,5]) -/// -> reduce(|mut accum, elem| { -/// accum *= elem; -/// accum +/// -> reduce::<'tick>(|accum: &mut _, elem| { +/// *accum *= elem; /// }) -/// -> assert([120]); +/// -> assert_eq([120]); /// ``` pub const REDUCE: OperatorConstraints = OperatorConstraints { name: "reduce", @@ -44,6 +49,7 @@ pub const REDUCE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { context, hydroflow, @@ -66,7 +72,7 @@ pub const REDUCE: OperatorConstraints = OperatorConstraints { assert!(is_pull); let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; @@ -74,12 +80,28 @@ pub const REDUCE: OperatorConstraints = OperatorConstraints { let input = &inputs[0]; let func = &arguments[0]; let reducedata_ident = wc.make_ident("reducedata_ident"); + let accumulator_ident = wc.make_ident("accumulator"); + let ret_ident = wc.make_ident("ret"); + let iterator_item_ident = wc.make_ident("iterator_item"); let (write_prologue, write_iterator, write_iterator_after) = match persistence { Persistence::Tick => ( Default::default(), quote_spanned! {op_span=> - let #ident = #input.reduce(#func).into_iter(); + let #ident = { + let mut #input = #input; + let #accumulator_ident = #input.next(); + if let ::std::option::Option::Some(mut #accumulator_ident) = #accumulator_ident { + for #iterator_item_ident in #input { + #[allow(clippy::redundant_closure_call)] + (#func)(&mut #accumulator_ident, #iterator_item_ident); + } + + ::std::option::Option::Some(#accumulator_ident) + } else { + ::std::option::Option::None + }.into_iter() + }; }, Default::default(), ), @@ -91,13 +113,27 @@ pub const REDUCE: OperatorConstraints = OperatorConstraints { }, quote_spanned! {op_span=> let #ident = { - let opt = #context.state_ref(#reducedata_ident).take(); - let opt = match opt { - Some(accum) => Some(#input.fold(accum, #func)), - None => #input.reduce(#func), + let mut #input = #input; + let #accumulator_ident = if let ::std::option::Option::Some(#accumulator_ident) = #context.state_ref(#reducedata_ident).take() { + Some(#accumulator_ident) + } else { + #input.next() }; - #context.state_ref(#reducedata_ident).set(::std::clone::Clone::clone(&opt)); - opt.into_iter() + + let #ret_ident = if let ::std::option::Option::Some(mut #accumulator_ident) = #accumulator_ident { + for #iterator_item_ident in #input { + #[allow(clippy::redundant_closure_call)] + (#func)(&mut #accumulator_ident, #iterator_item_ident); + } + + ::std::option::Option::Some(#accumulator_ident) + } else { + ::std::option::Option::None + }; + + #context.state_ref(#reducedata_ident).set(::std::clone::Clone::clone(&#ret_ident)); + + #ret_ident.into_iter() }; }, quote_spanned! {op_span=> diff --git a/hydroflow_lang/src/graph/ops/reduce_keyed.rs b/hydroflow_lang/src/graph/ops/reduce_keyed.rs index 75da38d87c6..030b65622d8 100644 --- a/hydroflow_lang/src/graph/ops/reduce_keyed.rs +++ b/hydroflow_lang/src/graph/ops/reduce_keyed.rs @@ -11,12 +11,12 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// The output will have one tuple for each distinct `K`, with an accumulated (reduced) value of /// type `V`. /// -/// If you need the accumulated value to have a different type, use [`fold_keyed`](#keyed_fold). +/// If you need the accumulated value to have a different type than the input, use [`fold_keyed`](#keyed_fold). /// /// > Arguments: one Rust closures. The closure takes two arguments: an `&mut` 'accumulator', and /// an element. Accumulator should be updated based on the element. /// -/// A special case of `fold`, in the spirit of SQL's GROUP BY and aggregation constructs. The input +/// A special case of `reduce`, in the spirit of SQL's GROUP BY and aggregation constructs. The input /// is partitioned into groups by the first field, and for each group the values in the second /// field are accumulated via the closures in the arguments. /// @@ -26,7 +26,7 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// `'tick` or `'static`, to specify how data persists. With `'tick`, values will only be collected /// within the same tick. With `'static`, values will be remembered across ticks and will be /// aggregated with pairs arriving in later ticks. When not explicitly specified persistence -/// defaults to `'static`. +/// defaults to `'tick`. /// /// `reduce_keyed` can also be provided with two type arguments, the key and value type. This is /// required when using `'static` persistence if the compiler cannot infer the types. @@ -34,10 +34,10 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// ```hydroflow /// source_iter([("toy", 1), ("toy", 2), ("shoe", 11), ("shoe", 35), ("haberdashery", 7)]) /// -> reduce_keyed(|old: &mut u32, val: u32| *old += val) -/// -> assert([("toy", 3), ("shoe", 46), ("haberdashery", 7)]); +/// -> assert_eq([("toy", 3), ("shoe", 46), ("haberdashery", 7)]); /// ``` /// -/// Example using `'tick` persistence: +/// Example using `'tick` persistence and type arguments: /// ```rustbook /// let (input_send, input_recv) = hydroflow::util::unbounded_channel::<(&str, &str)>(); /// let mut flow = hydroflow::hydroflow_syntax! { @@ -75,6 +75,7 @@ pub const REDUCE_KEYED: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { hydroflow, context, @@ -100,7 +101,7 @@ pub const REDUCE_KEYED: OperatorConstraints = OperatorConstraints { assert!(is_pull); let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; diff --git a/hydroflow_lang/src/graph/ops/sort.rs b/hydroflow_lang/src/graph/ops/sort.rs index a77606618f8..64bd049c399 100644 --- a/hydroflow_lang/src/graph/ops/sort.rs +++ b/hydroflow_lang/src/graph/ops/sort.rs @@ -10,10 +10,10 @@ use super::{ /// ```hydroflow /// source_iter(vec![2, 3, 1]) /// -> sort() -/// -> assert([1, 2, 3]); +/// -> assert_eq([1, 2, 3]); /// ``` /// -/// `sort` is partially blocking. Only the values collected within a single tick will be sorted and +/// `sort` is blocking. Only the values collected within a single tick will be sorted and /// emitted. pub const SORT: OperatorConstraints = OperatorConstraints { name: "sort", @@ -34,6 +34,7 @@ pub const SORT: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |&WriteContextArgs { op_span, ident, diff --git a/hydroflow_lang/src/graph/ops/sort_by_key.rs b/hydroflow_lang/src/graph/ops/sort_by_key.rs index 72a56df3b6e..2835109fc6e 100644 --- a/hydroflow_lang/src/graph/ops/sort_by_key.rs +++ b/hydroflow_lang/src/graph/ops/sort_by_key.rs @@ -6,15 +6,15 @@ use super::{ }; use crate::graph::OperatorInstance; -/// Takes a stream as input and produces a version of the stream as output -/// sorted according to the key extracted by the closure. +/// Like sort, takes a stream as input and produces a version of the stream as output. +/// This operator sorts according to the key extracted by the closure. /// /// > Note: The closure has access to the [`context` object](surface_flows.md#the-context-object). /// /// ```hydroflow /// source_iter(vec![(2, 'y'), (3, 'x'), (1, 'z')]) /// -> sort_by_key(|(k, _v)| k) -/// -> assert([(1, 'z'), (2, 'y'), (3, 'x')]); +/// -> assert_eq([(1, 'z'), (2, 'y'), (3, 'x')]); /// ``` pub const SORT_BY_KEY: OperatorConstraints = OperatorConstraints { name: "sort_by_key", @@ -35,6 +35,7 @@ pub const SORT_BY_KEY: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/source_file.rs b/hydroflow_lang/src/graph/ops/source_file.rs index ca74cd730b3..61602cebf86 100644 --- a/hydroflow_lang/src/graph/ops/source_file.rs +++ b/hydroflow_lang/src/graph/ops/source_file.rs @@ -10,7 +10,7 @@ use crate::graph::OperatorInstance; /// > 0 input streams, 1 output stream /// /// > Arguments: An [`AsRef`](https://doc.rust-lang.org/std/convert/trait.AsRef.html)`<`[`Path`](https://doc.rust-lang.org/nightly/std/path/struct.Path.html)`>` -/// for a file to read as json. +/// for a file to read. /// /// Reads the referenced file one line at a time. The line will NOT include the line ending. /// @@ -38,6 +38,7 @@ pub const SOURCE_FILE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/source_interval.rs b/hydroflow_lang/src/graph/ops/source_interval.rs index 37d1de1182b..d008fa79abb 100644 --- a/hydroflow_lang/src/graph/ops/source_interval.rs +++ b/hydroflow_lang/src/graph/ops/source_interval.rs @@ -17,7 +17,7 @@ use crate::graph::OperatorInstance; /// /// Note that this requires the hydroflow instance be run within a [Tokio `Runtime`](https://docs.rs/tokio/1/tokio/runtime/struct.Runtime.html). /// The easiest way to do this is with a [`#[hydroflow::main]`](https://hydro-project.github.io/hydroflow/doc/hydroflow/macro.hydroflow_main.html) -/// annotation on `async fn main() { ... }`. +/// annotation on `async fn main() { ... }` as in the example below. /// /// ```rustbook /// use std::time::Duration; @@ -62,6 +62,7 @@ pub const SOURCE_INTERVAL: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/source_iter.rs b/hydroflow_lang/src/graph/ops/source_iter.rs index c83aff08da8..8779ba5b7f2 100644 --- a/hydroflow_lang/src/graph/ops/source_iter.rs +++ b/hydroflow_lang/src/graph/ops/source_iter.rs @@ -37,6 +37,7 @@ pub const SOURCE_ITER: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { op_span, ident, diff --git a/hydroflow_lang/src/graph/ops/source_iter_delta.rs b/hydroflow_lang/src/graph/ops/source_iter_delta.rs new file mode 100644 index 00000000000..91b41583ac7 --- /dev/null +++ b/hydroflow_lang/src/graph/ops/source_iter_delta.rs @@ -0,0 +1,25 @@ +use super::OperatorConstraints; +use crate::graph::{FlowProps, LatticeFlowType}; + +/// > 0 input streams, 1 output stream +/// +/// > Arguments: An iterable Rust object. +/// Takes the iterable object and delivers its elements downstream +/// one by one. +/// +/// Note that all elements are emitted during the first tick. +/// +/// ```hydroflow +/// source_iter(vec!["Hello", "World"]) +/// -> for_each(|x| println!("{}", x)); +/// ``` +pub const SOURCE_ITER_DELTA: OperatorConstraints = OperatorConstraints { + name: "source_iter_delta", + flow_prop_fn: Some(|fp, _diagnostics| { + Ok(vec![Some(FlowProps { + star_ord: fp.new_star_ord(), + lattice_flow_type: Some(LatticeFlowType::Delta), + })]) + }), + ..super::source_iter::SOURCE_ITER +}; diff --git a/hydroflow_lang/src/graph/ops/source_json.rs b/hydroflow_lang/src/graph/ops/source_json.rs index 484574e6941..08f8789e97b 100644 --- a/hydroflow_lang/src/graph/ops/source_json.rs +++ b/hydroflow_lang/src/graph/ops/source_json.rs @@ -35,6 +35,7 @@ pub const SOURCE_JSON: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/source_stdin.rs b/hydroflow_lang/src/graph/ops/source_stdin.rs index 19189c3b2fc..3dd2f82e26a 100644 --- a/hydroflow_lang/src/graph/ops/source_stdin.rs +++ b/hydroflow_lang/src/graph/ops/source_stdin.rs @@ -12,12 +12,10 @@ use super::{ /// `source_stdin` receives a Stream of lines from stdin /// and emits each of the elements it receives downstream. /// -/// ```rustbook -/// let mut flow = hydroflow::hydroflow_syntax! { -/// source_stdin() -> map(|x| x.unwrap().to_uppercase()) -/// -> for_each(|x| println!("{}", x)); -/// }; -/// flow.run_async(); +/// ```hydroflow +/// source_stdin() +/// -> map(|x| x.unwrap().to_uppercase()) +/// -> for_each(|x| println!("{}", x)); /// ``` pub const SOURCE_STDIN: OperatorConstraints = OperatorConstraints { name: "source_stdin", @@ -38,6 +36,7 @@ pub const SOURCE_STDIN: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/source_stream.rs b/hydroflow_lang/src/graph/ops/source_stream.rs index 95f305e69f3..6ccd52284a2 100644 --- a/hydroflow_lang/src/graph/ops/source_stream.rs +++ b/hydroflow_lang/src/graph/ops/source_stream.rs @@ -44,6 +44,7 @@ pub const SOURCE_STREAM: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/source_stream_serde.rs b/hydroflow_lang/src/graph/ops/source_stream_serde.rs index dd305acdcca..7b74cf16dc5 100644 --- a/hydroflow_lang/src/graph/ops/source_stream_serde.rs +++ b/hydroflow_lang/src/graph/ops/source_stream_serde.rs @@ -10,7 +10,7 @@ use super::{ /// > Arguments: [`Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html) /// /// Given a [`Stream`](https://docs.rs/futures/latest/futures/stream/trait.Stream.html) -/// of (serialized payload, addr) pairs, deserializes the payload and emits each of the +/// of `(serialized payload, addr)` pairs, deserializes the payload and emits each of the /// elements it receives downstream. /// /// ```rustbook @@ -43,6 +43,7 @@ pub const SOURCE_STREAM_SERDE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/spin.rs b/hydroflow_lang/src/graph/ops/spin.rs index 9ebeef494dd..84f7bd4bd05 100644 --- a/hydroflow_lang/src/graph/ops/spin.rs +++ b/hydroflow_lang/src/graph/ops/spin.rs @@ -5,7 +5,18 @@ use super::{ WriteContextArgs, RANGE_0, RANGE_1, }; -/// This operator will trigger the start of new ticks in order to repeat, which will cause spinning-like behavior. +/// This operator emits Unit, and triggers the start of a new tick at the end of each tick, +/// which will cause spinning-like behavior. Note that `run_available` will run forever, +/// so in the example below we illustrate running manually for 100 ticks. +/// +/// ```rustbook +/// let mut flow = hydroflow::hydroflow_syntax! { +/// spin() -> for_each(|x| println!("tick {}: {:?}", context.current_tick(), x)); +/// }; +/// for _ in 1..100 { +/// flow.run_tick(); +/// } +/// ``` pub const SPIN: OperatorConstraints = OperatorConstraints { name: "spin", categories: &[OperatorCategory::Source], @@ -25,6 +36,7 @@ pub const SPIN: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { context, op_span, diff --git a/hydroflow_lang/src/graph/ops/tee.rs b/hydroflow_lang/src/graph/ops/tee.rs index 3d24ce97a2f..861afd6adb2 100644 --- a/hydroflow_lang/src/graph/ops/tee.rs +++ b/hydroflow_lang/src/graph/ops/tee.rs @@ -1,8 +1,8 @@ use quote::{quote_spanned, ToTokens}; use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - WriteContextArgs, RANGE_0, RANGE_1, RANGE_ANY, + FlowPropArgs, FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, + OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, RANGE_ANY, }; /// > 1 input stream, *n* output streams @@ -12,9 +12,9 @@ use super::{ /// /// ```hydroflow /// my_tee = source_iter(vec!["Hello", "World"]) -> tee(); -/// my_tee -> map(|x: &str| x.to_uppercase()) -> assert(["HELLO", "WORLD"]); -/// my_tee -> map(|x: &str| x.to_lowercase()) -> assert(["hello", "world"]); -/// my_tee -> assert(["Hello", "World"]); +/// my_tee -> map(|x: &str| x.to_uppercase()) -> assert_eq(["HELLO", "WORLD"]); +/// my_tee -> map(|x: &str| x.to_lowercase()) -> assert_eq(["hello", "world"]); +/// my_tee -> assert_eq(["Hello", "World"]); /// ``` pub const TEE: OperatorConstraints = OperatorConstraints { name: "tee", @@ -35,6 +35,9 @@ pub const TEE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: Some(|FlowPropArgs { flow_props_in, .. }, _diagnostics| { + Ok(vec![flow_props_in[0]]) + }), write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/union.rs b/hydroflow_lang/src/graph/ops/union.rs index 462e7efbca8..860113a329f 100644 --- a/hydroflow_lang/src/graph/ops/union.rs +++ b/hydroflow_lang/src/graph/ops/union.rs @@ -1,9 +1,10 @@ use quote::{quote_spanned, ToTokens}; use super::{ - FlowProperties, FlowPropertyVal, OperatorCategory, OperatorConstraints, OperatorWriteOutput, - WriteContextArgs, RANGE_0, RANGE_1, RANGE_ANY, + FlowPropArgs, FlowProperties, FlowPropertyVal, FlowProps, OperatorCategory, + OperatorConstraints, OperatorWriteOutput, WriteContextArgs, RANGE_0, RANGE_1, RANGE_ANY, }; +use crate::diagnostic::{Diagnostic, Level}; /// > *n* input streams of the same type, 1 output stream of the same type /// @@ -18,7 +19,7 @@ use super::{ /// source_iter(vec!["don't", "give", "up"]) -> my_union; /// my_union = union() /// -> map(|x| x.to_uppercase()) -/// -> assert(["HELLO", "WORLD", "STAY", "GOLD", "DON'T", "GIVE", "UP"]); +/// -> assert_eq(["HELLO", "WORLD", "STAY", "GOLD", "DON'T", "GIVE", "UP"]); /// ``` pub const UNION: OperatorConstraints = OperatorConstraints { name: "union", @@ -39,6 +40,38 @@ pub const UNION: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: Some( + |fp @ FlowPropArgs { + op_name, + op_inst, + flow_props_in, + .. + }, + diagnostics| { + let lattice_flow_type = flow_props_in + .iter() + .map(|flow_props| flow_props.and_then(|fp| fp.lattice_flow_type)) + .reduce(std::cmp::min) + .flatten(); + // Warn if we are doing any implied downcasting. + for (fp_in, port_in) in flow_props_in.iter().zip(op_inst.input_ports.iter()) { + if lattice_flow_type != fp_in.and_then(|fp| fp.lattice_flow_type) { + diagnostics.push(Diagnostic::spanned( + port_in.span(), + Level::Warning, + format!( + "Input to `{}()` will be downcast to `{:?}` to match other inputs.", + op_name, lattice_flow_type, + ), + )); + } + } + Ok(vec![Some(FlowProps { + star_ord: fp.new_star_ord(), + lattice_flow_type, + })]) + }, + ), write_fn: |&WriteContextArgs { op_span, ident, diff --git a/hydroflow_lang/src/graph/ops/unique.rs b/hydroflow_lang/src/graph/ops/unique.rs index 7c479210a4c..e4ed1d07aab 100644 --- a/hydroflow_lang/src/graph/ops/unique.rs +++ b/hydroflow_lang/src/graph/ops/unique.rs @@ -13,11 +13,11 @@ use crate::graph::{OpInstGenerics, OperatorInstance}; /// ```hydroflow /// source_iter(vec![1, 1, 2, 3, 2, 1, 3]) /// -> unique() -/// -> assert([1, 2, 3]); +/// -> assert_eq([1, 2, 3]); /// ``` /// /// `unique` can also be provided with one generic lifetime persistence argument, either -/// `'tick` or `'static`, to specify how data persists. The default is `'static`. +/// `'tick` or `'static`, to specify how data persists. The default is `'tick`. /// With `'tick`, uniqueness is only considered within the current tick, so across multiple ticks /// duplicate values may be emitted. /// With `'static`, values will be remembered across ticks and no duplicates will ever be emitted. @@ -62,6 +62,7 @@ pub const UNIQUE: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, op_span, @@ -83,7 +84,7 @@ pub const UNIQUE: OperatorConstraints = OperatorConstraints { }, diagnostics| { let persistence = match persistence_args[..] { - [] => Persistence::Static, + [] => Persistence::Tick, [a] => a, _ => unreachable!(), }; diff --git a/hydroflow_lang/src/graph/ops/unzip.rs b/hydroflow_lang/src/graph/ops/unzip.rs index 92d2d6a6776..095d63d2a37 100644 --- a/hydroflow_lang/src/graph/ops/unzip.rs +++ b/hydroflow_lang/src/graph/ops/unzip.rs @@ -13,8 +13,8 @@ use super::{ /// /// ```hydroflow /// my_unzip = source_iter(vec![("Hello", "Foo"), ("World", "Bar")]) -> unzip(); -/// my_unzip[0] -> assert(["Hello", "World"]); -/// my_unzip[1] -> assert(["Foo", "Bar"]); +/// my_unzip[0] -> assert_eq(["Hello", "World"]); +/// my_unzip[1] -> assert_eq(["Foo", "Bar"]); /// ``` pub const UNZIP: OperatorConstraints = OperatorConstraints { name: "unzip", @@ -35,6 +35,7 @@ pub const UNZIP: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/ops/zip.rs b/hydroflow_lang/src/graph/ops/zip.rs index cf535a0c9f8..825a1259a53 100644 --- a/hydroflow_lang/src/graph/ops/zip.rs +++ b/hydroflow_lang/src/graph/ops/zip.rs @@ -16,7 +16,7 @@ use crate::diagnostic::{Diagnostic, Level}; /// ```hydroflow /// source_iter(0..3) -> [0]my_zip; /// source_iter(0..5) -> [1]my_zip; -/// my_zip = zip() -> assert([(0, 0), (1, 1), (2, 2)]); +/// my_zip = zip() -> assert_eq([(0, 0), (1, 1), (2, 2)]); /// ``` pub const ZIP: OperatorConstraints = OperatorConstraints { name: "zip", @@ -38,6 +38,7 @@ pub const ZIP: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| None, + flow_prop_fn: None, write_fn: |wc @ &WriteContextArgs { root, context, diff --git a/hydroflow_lang/src/graph/ops/zip_longest.rs b/hydroflow_lang/src/graph/ops/zip_longest.rs index 1d7e6da16e4..1fb7e28b3cc 100644 --- a/hydroflow_lang/src/graph/ops/zip_longest.rs +++ b/hydroflow_lang/src/graph/ops/zip_longest.rs @@ -18,7 +18,7 @@ use crate::diagnostic::{Diagnostic, Level}; /// source_iter(0..2) -> [0]my_zip_longest; /// source_iter(0..3) -> [1]my_zip_longest; /// my_zip_longest = zip_longest() -/// -> assert([ +/// -> assert_eq([ /// itertools::EitherOrBoth::Both(0, 0), /// itertools::EitherOrBoth::Both(1, 1), /// itertools::EitherOrBoth::Right(2)]); @@ -43,6 +43,7 @@ pub const ZIP_LONGEST: OperatorConstraints = OperatorConstraints { inconsistency_tainted: false, }, input_delaytype_fn: |_| Some(DelayType::Stratum), + flow_prop_fn: None, write_fn: |&WriteContextArgs { root, op_span, diff --git a/hydroflow_lang/src/graph/propegate_flow_props.rs b/hydroflow_lang/src/graph/propegate_flow_props.rs new file mode 100644 index 00000000000..048b4e1caea --- /dev/null +++ b/hydroflow_lang/src/graph/propegate_flow_props.rs @@ -0,0 +1,83 @@ +//! Module for determining flow properties. See [`propegate_flow_props`]. + +use super::{GraphNodeId, HydroflowGraph, Node}; +use crate::diagnostic::Diagnostic; +use crate::graph::graph_algorithms; +use crate::graph::ops::FlowPropArgs; + +/// Traverses the graph, propegating the flow properties from sources to sinks. +pub fn propegate_flow_props( + graph: &mut HydroflowGraph, + diagnostics: &mut Vec, +) -> Result<(), GraphNodeId> { + // Topological sort on strongly connected components. + // TODO(mingwei): order is broken within SCCs/cycles: https://github.com/hydro-project/hydroflow/issues/895 + let node_order = graph_algorithms::topo_sort_scc( + || graph.node_ids(), + |dst| graph.node_predecessor_nodes(dst), + |src| graph.node_successor_nodes(src), + ); + // Propegate flow props in order. + for (idx_star_ord, node_id) in node_order.into_iter().enumerate() { + match graph.node(node_id) { + Node::Operator(_) => { + let op_inst = graph + .node_op_inst(node_id) + .expect("Operator instance info must be set when calling `propegate_flow_props`. (This is a Hydroflow bug)."); + + if let Some(flow_prop_fn) = op_inst.op_constraints.flow_prop_fn { + // Collect the flow props on input edges. Input operators will naturally have no inputs. + let flow_props_in = graph + .node_predecessor_edges(node_id) + .map(|edge_id| graph.edge_flow_props(edge_id)) + .collect::>(); + + // Build the `FlowPropArgs` argument + let flow_prop_args = FlowPropArgs { + op_span: graph.node(node_id).span(), + op_name: op_inst.op_constraints.name, + op_inst, + flow_props_in: &flow_props_in, + new_star_ord: idx_star_ord, + }; + + // Call the `flow_prop_fn`. + // TODO(mingwei): don't exit early here? + let flow_props_out = + (flow_prop_fn)(flow_prop_args, diagnostics).map_err(|()| node_id)?; + assert!( + 1 == flow_props_out.len() + || graph.node_degree_out(node_id) == flow_props_out.len() + ); + + // Assign output flow props. + let out_edges = graph.node_successor_edges(node_id).collect::>(); + // In/out edges are in the same order as the in/out port names (in `op_inst`). + for (i, edge_id) in out_edges.into_iter().enumerate() { + if let Some(flow_prop_out) = *flow_props_out + .get(i) + .unwrap_or_else(|| flow_props_out.get(0).unwrap()) + { + graph.set_edge_flow_props(edge_id, flow_prop_out); + } + } + } + } + Node::Handoff { .. } => { + // Handoffs just copy over their one input [`FlowProps`] to their one output. + assert_eq!(1, graph.node_degree_in(node_id)); + assert_eq!(1, graph.node_degree_out(node_id)); + let in_edge = graph.node_predecessor_edges(node_id).next().unwrap(); + let out_edge = graph.node_successor_edges(node_id).next().unwrap(); + if let Some(flow_props) = graph.edge_flow_props(in_edge) { + graph.set_edge_flow_props(out_edge, flow_props); + } + } + _ => { + // If a module boundary is encountered then something has gone wrong. + panic!(); + } + } + } + Ok(()) +} diff --git a/hydroflow_lang/src/lib.rs b/hydroflow_lang/src/lib.rs index 2ab935bf966..14b392f802c 100644 --- a/hydroflow_lang/src/lib.rs +++ b/hydroflow_lang/src/lib.rs @@ -1,3 +1,6 @@ +//! Hydroflow surface syntax + +#![warn(missing_docs)] #![cfg_attr( feature = "diagnostics", feature(proc_macro_diagnostic, proc_macro_span) diff --git a/hydroflow_lang/src/parse.rs b/hydroflow_lang/src/parse.rs index 48ad5afc032..d75c21c0f45 100644 --- a/hydroflow_lang/src/parse.rs +++ b/hydroflow_lang/src/parse.rs @@ -1,4 +1,5 @@ //! AST for surface syntax, modelled on [`syn`]'s ASTs. +#![allow(missing_docs)] use std::hash::Hash; @@ -8,44 +9,50 @@ use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; use syn::token::{Bracket, Paren}; use syn::{ - bracketed, parenthesized, AngleBracketedGenericArguments, Expr, ExprPath, GenericArgument, - Ident, LitInt, Path, PathArguments, PathSegment, Token, + bracketed, parenthesized, AngleBracketedGenericArguments, Error, Expr, ExprPath, + GenericArgument, Ident, ItemUse, LitInt, LitStr, Path, PathArguments, PathSegment, Token, }; pub struct HfCode { - pub statements: Punctuated, + pub statements: Vec, } impl Parse for HfCode { fn parse(input: ParseStream) -> syn::Result { - let statements = Punctuated::parse_terminated(input)?; - if !statements.empty_or_trailing() { - return Err(input.parse::().unwrap_err()); + let mut statements = Vec::new(); + while !input.is_empty() { + statements.push(input.parse()?); } Ok(HfCode { statements }) } } impl ToTokens for HfCode { fn to_tokens(&self, tokens: &mut TokenStream) { - self.statements.to_tokens(tokens) + for statement in self.statements.iter() { + statement.to_tokens(tokens); + } } } pub enum HfStatement { + Use(ItemUse), Named(NamedHfStatement), - Pipeline(Pipeline), + Pipeline(PipelineStatement), } impl Parse for HfStatement { fn parse(input: ParseStream) -> syn::Result { - if input.peek2(Token![=]) { + if input.peek(Token![use]) { + Ok(Self::Use(ItemUse::parse(input)?)) + } else if input.peek2(Token![=]) { Ok(Self::Named(NamedHfStatement::parse(input)?)) } else { - Ok(Self::Pipeline(Pipeline::parse(input)?)) + Ok(Self::Pipeline(PipelineStatement::parse(input)?)) } } } impl ToTokens for HfStatement { fn to_tokens(&self, tokens: &mut TokenStream) { match self { + HfStatement::Use(x) => x.to_tokens(tokens), HfStatement::Named(x) => x.to_tokens(tokens), HfStatement::Pipeline(x) => x.to_tokens(tokens), } @@ -56,16 +63,19 @@ pub struct NamedHfStatement { pub name: Ident, pub equals: Token![=], pub pipeline: Pipeline, + pub semi_token: Token![;], } impl Parse for NamedHfStatement { fn parse(input: ParseStream) -> syn::Result { let name = input.parse()?; let equals = input.parse()?; let pipeline = input.parse()?; + let semi_token = input.parse()?; Ok(Self { name, equals, pipeline, + semi_token, }) } } @@ -74,14 +84,39 @@ impl ToTokens for NamedHfStatement { self.name.to_tokens(tokens); self.equals.to_tokens(tokens); self.pipeline.to_tokens(tokens); + self.semi_token.to_tokens(tokens); } } +pub struct PipelineStatement { + pub pipeline: Pipeline, + pub semi_token: Token![;], +} +impl Parse for PipelineStatement { + fn parse(input: ParseStream) -> syn::Result { + let pipeline = input.parse()?; + let semi_token = input.parse()?; + Ok(Self { + pipeline, + semi_token, + }) + } +} +impl ToTokens for PipelineStatement { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pipeline.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } +} + +#[derive(Debug)] pub enum Pipeline { Paren(Ported), Name(Ported), Link(PipelineLink), Operator(Operator), + ModuleBoundary(Ported), + Import(Import), } impl Pipeline { fn parse_one(input: ParseStream) -> syn::Result { @@ -99,19 +134,37 @@ impl Pipeline { else if lookahead2.peek(Ident) { Ok(Self::Name(Ported::parse_rest(Some(inn_idx), input)?)) } + // Indexed module boundary + else if lookahead2.peek(Token![mod]) { + Ok(Self::ModuleBoundary(Ported::parse_rest( + Some(inn_idx), + input, + )?)) + } // Emit lookahead expected tokens errors. else { Err(lookahead2.error()) } - } - // Ident - else if lookahead1.peek(Ident) { + // module input/output + } else if lookahead1.peek(Token![mod]) { + Ok(Self::ModuleBoundary(input.parse()?)) + // Ident or macro-style expression + } else if lookahead1.peek(Ident) { + let speculative = input.fork(); + let ident: Ident = speculative.parse()?; + let lookahead2 = speculative.lookahead1(); + // If has paren or generic next, it's an operator - if input.peek2(Paren) || input.peek2(Token![<]) || input.peek2(Token![::]) { + if lookahead2.peek(Paren) || lookahead2.peek(Token![<]) || lookahead2.peek(Token![::]) { Ok(Self::Operator(input.parse()?)) - } + // macro-style expression "x!.." + } else if lookahead2.peek(Token![!]) { + match ident.to_string().as_str() { + "import" => Ok(Self::Import(input.parse()?)), + _ => Err(Error::new(ident.span(), r#"Expected "import""#)), + } // Otherwise it's a name - else { + } else { Ok(Self::Name(input.parse()?)) } } @@ -145,10 +198,45 @@ impl ToTokens for Pipeline { Pipeline::Link(x) => x.to_tokens(tokens), Pipeline::Name(x) => x.to_tokens(tokens), Pipeline::Operator(x) => x.to_tokens(tokens), + Pipeline::ModuleBoundary(x) => x.to_tokens(tokens), + Pipeline::Import(x) => x.to_tokens(tokens), } } } +#[derive(Debug)] +pub struct Import { + pub import: Ident, + pub bang: Token![!], + pub paren_token: Paren, + pub filename: LitStr, +} +impl Parse for Import { + fn parse(input: ParseStream) -> syn::Result { + let import = input.parse()?; + let bang = input.parse()?; + let content; + let paren_token = parenthesized!(content in input); + let filename: LitStr = content.parse()?; + + Ok(Self { + import, + bang, + paren_token, + filename, + }) + } +} +impl ToTokens for Import { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.import.to_tokens(tokens); + self.bang.to_tokens(tokens); + self.paren_token + .surround(tokens, |tokens| self.filename.to_tokens(tokens)); + } +} + +#[derive(Debug)] pub struct Ported { pub inn: Option, pub inner: Inner, @@ -186,6 +274,7 @@ where } } +#[derive(Debug)] pub struct PipelineParen { pub paren_token: Paren, pub pipeline: Box, @@ -209,6 +298,7 @@ impl ToTokens for PipelineParen { } } +#[derive(Debug)] pub struct PipelineLink { pub lhs: Box, pub arrow: Token![->], @@ -231,6 +321,7 @@ impl ToTokens for PipelineLink { } } +#[derive(Debug)] pub struct Indexing { pub bracket_token: Bracket, pub index: PortIndex, @@ -284,7 +375,7 @@ impl ToTokens for PortIndex { } } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Operator { pub path: Path, pub paren_token: Paren, @@ -406,17 +497,17 @@ impl Hash for IndexInt { self.value.hash(state); } } -impl PartialOrd for IndexInt { - fn partial_cmp(&self, other: &Self) -> Option { - self.value.partial_cmp(&other.value) - } -} impl PartialEq for IndexInt { fn eq(&self, other: &Self) -> bool { self.value == other.value } } impl Eq for IndexInt {} +impl PartialOrd for IndexInt { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} impl Ord for IndexInt { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.value.cmp(&other.value) diff --git a/hydroflow_lang/src/pretty_span.rs b/hydroflow_lang/src/pretty_span.rs index de3d4fe0057..610eec3068d 100644 --- a/hydroflow_lang/src/pretty_span.rs +++ b/hydroflow_lang/src/pretty_span.rs @@ -1,25 +1,23 @@ +//! Pretty, human-readable printing of [`proc_macro2::Span`]s. + /// Helper struct which displays the span as `path:row:col` for human reading/IDE linking. /// Example: `hydroflow\tests\surface_syntax.rs:42:18`. pub struct PrettySpan(pub proc_macro2::Span); impl std::fmt::Display for PrettySpan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let span = self.0; - #[cfg(feature = "diagnostics")] - let path = span.unwrap().source_file().path(); + let (path, line, column) = ( + self.0.unwrap().source_file().path(), + self.0.unwrap().start().line(), + self.0.unwrap().start().column(), + ); #[cfg(feature = "diagnostics")] let location = path.display(); #[cfg(not(feature = "diagnostics"))] - let location = "unknown"; + let (location, line, column) = ("unknown", 0, 0); - write!( - f, - "{}:{}:{}", - location, - span.start().line, - span.start().column - ) + write!(f, "{}:{}:{}", location, line, column) } } diff --git a/hydroflow_lang/src/union_find.rs b/hydroflow_lang/src/union_find.rs index 01414779057..a91c7822da0 100644 --- a/hydroflow_lang/src/union_find.rs +++ b/hydroflow_lang/src/union_find.rs @@ -1,5 +1,12 @@ +//! Union-find data structure, see [`UnionFind`]. + use slotmap::{Key, SecondaryMap}; +/// Union-find data structure. +/// +/// Used to efficiently track sets of equivalent items. +/// +/// #[derive(Default, Clone)] pub struct UnionFind where @@ -11,16 +18,19 @@ impl UnionFind where K: Key, { + /// Creates a new `UnionFind`, same as [`Default::default()`]. #[allow(dead_code)] pub fn new() -> Self { Self::default() } + /// Creates a new `UnionFind` with the given key capacity pre-allocated. pub fn with_capacity(capacity: usize) -> Self { Self { links: SecondaryMap::with_capacity(capacity), } } + /// Combines two items `a` and `b` as equivalent, in the same set. pub fn union(&mut self, a: K, b: K) { let i = self.find(a); let j = self.find(b); @@ -29,6 +39,9 @@ where } self.links[i] = j; } + + /// Finds the "representative" item for `k`. Each set of equivalent items is represented by one + /// of its member items. pub fn find(&mut self, k: K) -> K { if let Some(next) = self.links.insert(k, k) { if k == next { @@ -38,6 +51,8 @@ where } self.links[k] } + + /// Returns if `a` and `b` are equivalent, i.e. in the same set. pub fn same_set(&mut self, a: K, b: K) -> bool { self.find(a) == self.find(b) } diff --git a/hydroflow_macro/CHANGELOG.md b/hydroflow_macro/CHANGELOG.md index 494705b4e07..d0dd108df6e 100644 --- a/hydroflow_macro/CHANGELOG.md +++ b/hydroflow_macro/CHANGELOG.md @@ -5,8 +5,81 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +### New Features + + - Add `use` statements to hydroflow syntax + And use in doc tests. + +### Commit Statistics + + + + - 1 commit contributed to the release over the course of 26 calendar days. + - 42 days passed between releases. + - 1 commit was understood as [conventional](https://www.conventionalcommits.org). + - 1 unique issue was worked on: [#845](https://github.com/hydro-project/hydroflow/issues/845) + +### Commit Details + + + +
view details + + * **[#845](https://github.com/hydro-project/hydroflow/issues/845)** + - Add `use` statements to hydroflow syntax ([`b4b9644`](https://github.com/hydro-project/hydroflow/commit/b4b9644a19e8e7e7725c9c5b88e3a6b8c2be7364)) +
+ +## 0.3.0 (2023-07-04) + +### Documentation + + - avoid double-extension for generated files which breaks navigation + +### New Features + + - emit `compile_error!` diagnostics for stable + - allow stable build, refactors behind `nightly` feature flag + +### Bug Fixes + + - update proc-macro2, use new span location API where possible + requires latest* rust nightly version + + *latest = 2023-06-28 or something + +### Commit Statistics + + + + - 5 commits contributed to the release over the course of 25 calendar days. + - 33 days passed between releases. + - 4 commits were understood as [conventional](https://www.conventionalcommits.org). + - 3 unique issues were worked on: [#758](https://github.com/hydro-project/hydroflow/issues/758), [#780](https://github.com/hydro-project/hydroflow/issues/780), [#801](https://github.com/hydro-project/hydroflow/issues/801) + +### Commit Details + + + +
view details + + * **[#758](https://github.com/hydro-project/hydroflow/issues/758)** + - Avoid double-extension for generated files which breaks navigation ([`c6b39e7`](https://github.com/hydro-project/hydroflow/commit/c6b39e72590a01590690b0e42237c9853b105edc)) + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Emit `compile_error!` diagnostics for stable ([`ea65349`](https://github.com/hydro-project/hydroflow/commit/ea65349d241873f8460d7a8b024d64c63180246f)) + - Allow stable build, refactors behind `nightly` feature flag ([`22abcaf`](https://github.com/hydro-project/hydroflow/commit/22abcaff806c7de6e4a7725656bbcf201e7d9259)) + * **[#801](https://github.com/hydro-project/hydroflow/issues/801)** + - Update proc-macro2, use new span location API where possible ([`8d3494b`](https://github.com/hydro-project/hydroflow/commit/8d3494b5afee858114a602a3e23077bb6d24dd77)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) +
+ ## 0.2.0 (2023-05-31) + + + ### Chore - manually bump versions for v0.2.0 release @@ -27,8 +100,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 4 commits contributed to the release. - - 2 days passed between releases. + - 5 commits contributed to the release. + - 1 day passed between releases. - 4 commits were understood as [conventional](https://www.conventionalcommits.org). - 2 unique issues were worked on: [#728](https://github.com/hydro-project/hydroflow/issues/728), [#730](https://github.com/hydro-project/hydroflow/issues/730) @@ -43,6 +116,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#730](https://github.com/hydro-project/hydroflow/issues/730)** - Categorize operators, organize op docs, fix #727 ([`989adcb`](https://github.com/hydro-project/hydroflow/commit/989adcbcd304ad0890d71351d56a22977bdcf73f)) * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Make `build.rs`s infallible, log to stderr, to fix release ([`554d563`](https://github.com/hydro-project/hydroflow/commit/554d563fe53a1303c5a5c9352197365235c607e3)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a)) diff --git a/hydroflow_macro/Cargo.toml b/hydroflow_macro/Cargo.toml index d725f62c461..75fe6b9fccb 100644 --- a/hydroflow_macro/Cargo.toml +++ b/hydroflow_macro/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hydroflow_macro" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/hydroflow_macro/" @@ -17,13 +17,13 @@ diagnostics = [ "hydroflow_lang/diagnostics" ] # Note: If we ever compile this proc macro crate to WASM (e.g., if we are # building on a WASM host), we may need to turn diagnostics off for WASM if # proc_macro2 still does not support WASM. -hydroflow_lang = { path = "../hydroflow_lang", version = "^0.2.0" } -proc-macro2 = "1.0.0" +hydroflow_lang = { path = "../hydroflow_lang", version = "^0.4.0" } +proc-macro2 = "1.0.57" proc-macro-crate = "1.1.0" quote = "1.0.0" syn = { version = "2.0.0", features = [ "parsing", "extra-traits" ] } [build-dependencies] -hydroflow_lang = { path = "../hydroflow_lang", version = "^0.2.0" } +hydroflow_lang = { path = "../hydroflow_lang", version = "^0.4.0" } itertools = "0.10" quote = "1.0.0" diff --git a/hydroflow_macro/build.rs b/hydroflow_macro/build.rs index 07dec19a0a6..d630504d1a8 100644 --- a/hydroflow_macro/build.rs +++ b/hydroflow_macro/build.rs @@ -1,6 +1,7 @@ //! Build script to generate operator book docs. use std::env::VarError; +use std::fmt::Write as _FmtWrite; use std::fs::File; use std::io::{BufReader, BufWriter, Result, Write}; use std::path::{Path, PathBuf}; @@ -38,7 +39,14 @@ fn write_operator_docgen(op_name: &str, mut write: &mut impl Write) -> Result<() fn update_book() -> Result<()> { let mut ops: Vec<_> = OPERATORS.iter().collect(); - ops.sort_by_key(|op| op.name); + // operators that have their name start with "_" are internal compiler operators, we should sort those after all the user-facing ops. + // but underscore by default sorts before all [A-z] + ops.sort_by(|a, b| { + a.name + .starts_with('_') + .cmp(&b.name.starts_with('_')) + .then_with(|| a.name.cmp(b.name)) + }); let mut write = book_file_writer(FILENAME)?; writeln!(write, "{}", PREFIX)?; @@ -104,8 +112,10 @@ fn update_book() -> Result<()> { op.name, ('A'..) .take(op.num_args) - .map(|c| format!("{}, ", c)) - .collect::() + .fold(String::new(), |mut s, c| { + write!(&mut s, "{}, ", c).unwrap(); + s + }) .strip_suffix(", ") .unwrap_or(""), if op.soft_range_out.contains(&0) { @@ -130,7 +140,7 @@ fn update_book() -> Result<()> { "> Input port names: {} ", port_names .into_iter() - .map(|idx| { + .fold(String::new(), |mut s, idx| { let port_ix = idx.clone().into(); let flow_str = if (op.input_delaytype_fn)(&port_ix).is_some() { blocking = true; @@ -138,9 +148,9 @@ fn update_book() -> Result<()> { } else { "streaming" }; - format!("`{}` ({}), ", idx.into_token_stream(), flow_str) + write!(&mut s, "`{}` ({}), ", idx.into_token_stream(), flow_str).unwrap(); + s }) - .collect::() .strip_suffix(", ") .unwrap_or("<EMPTY>") )), @@ -155,8 +165,10 @@ fn update_book() -> Result<()> { "> Output port names: {} ", port_names .into_iter() - .map(|idx| format!("`{}`, ", idx.into_token_stream())) - .collect::() + .fold(String::new(), |mut s, idx| { + write!(&mut s, "`{}`, ", idx.into_token_stream()).unwrap(); + s + }) .strip_suffix(", ") .unwrap_or("<EMPTY>") ), diff --git a/hydroflow_macro/src/lib.rs b/hydroflow_macro/src/lib.rs index 198944ac966..ed17fda89e9 100644 --- a/hydroflow_macro/src/lib.rs +++ b/hydroflow_macro/src/lib.rs @@ -59,9 +59,11 @@ fn hydroflow_syntax_internal( input: proc_macro::TokenStream, min_diagnostic_level: Option, ) -> proc_macro::TokenStream { + let macro_invocation_path = proc_macro::Span::call_site().source_file().path(); + let input = parse_macro_input!(input as HfCode); let root = root(); - let (graph_code_opt, diagnostics) = build_hfcode(input, &root); + let (graph_code_opt, diagnostics) = build_hfcode(input, &root, macro_invocation_path); let tokens = graph_code_opt .map(|(_graph, code)| code) .unwrap_or_else(|| quote! { #root::scheduled::graph::Hydroflow::new() }); @@ -96,20 +98,30 @@ fn hydroflow_syntax_internal( /// Used for testing, users will want to use [`hydroflow_syntax!`] instead. #[proc_macro] pub fn hydroflow_parser(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let macro_invocation_path = proc_macro::Span::call_site().source_file().path(); + let input = parse_macro_input!(input as HfCode); - let flat_graph_builder = FlatGraphBuilder::from_hfcode(input); - let (flat_graph, diagnostics) = flat_graph_builder.build(); - diagnostics.iter().for_each(Diagnostic::emit); - let flat_mermaid = flat_graph.mermaid_string_flat(); + let flat_graph_builder = FlatGraphBuilder::from_hfcode(input, macro_invocation_path); + let (mut flat_graph, _uses, mut diagnostics) = flat_graph_builder.build(); + if !diagnostics.iter().any(Diagnostic::is_error) { + if let Err(diagnostic) = flat_graph.merge_modules() { + diagnostics.push(diagnostic); + } else { + let flat_mermaid = flat_graph.mermaid_string_flat(); - let part_graph = partition_graph(flat_graph).unwrap(); - let part_mermaid = part_graph.to_mermaid(); + let part_graph = partition_graph(flat_graph).unwrap(); + let part_mermaid = part_graph.to_mermaid(); - let lit0 = Literal::string(&*flat_mermaid); - let lit1 = Literal::string(&*part_mermaid); + let lit0 = Literal::string(&*flat_mermaid); + let lit1 = Literal::string(&*part_mermaid); - quote! { println!("{}\n\n{}\n", #lit0, #lit1); }.into() + return quote! { println!("{}\n\n{}\n", #lit0, #lit1); }.into(); + } + } + + diagnostics.iter().for_each(Diagnostic::emit); + quote! {}.into() } #[doc(hidden)] @@ -154,6 +166,20 @@ fn hydroflow_wrap(item: proc_macro::TokenStream, attribute: Attribute) -> proc_m input.into_token_stream().into() } +/// Checks that the given closure is a morphism. For now does nothing. +#[proc_macro] +pub fn morphism(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + // TODO(mingwei): some sort of code analysis? + item +} + +/// Checks that the given closure is a monotonic function. For now does nothing. +#[proc_macro] +pub fn monotonic_fn(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + // TODO(mingwei): some sort of code analysis? + item +} + #[proc_macro_attribute] pub fn hydroflow_test( _: proc_macro::TokenStream, diff --git a/lattices/CHANGELOG.md b/lattices/CHANGELOG.md index a73630d5483..e27a621c7d1 100644 --- a/lattices/CHANGELOG.md +++ b/lattices/CHANGELOG.md @@ -5,8 +5,188 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.4.0 (2023-08-15) + +### Chore + + - fix lint, format errors for latest nightly version (without updated pinned) + For nightly version (d9c13cd45 2023-07-05) + +### Documentation + + - Improve `Atomize` docs + +### New Features + + - formalize `Default::default()` as returning bottom for lattice types + Not a breaking change since changed names were introduced only since last release + - Implement `SimpleKeyedRef` for map types + - Add atomize trait, impls, tests + +### Refactor + + - fix new clippy lints on latest nightly 1.73.0-nightly (db7ff98a7 2023-07-31) + - Change `Atomize` to require returning empty iff lattice is bottom + Previously was the opposite, `Atomize` always had to return non-empty. + + Not breaking since `Atomize` has not yet been published. + +### New Features (BREAKING) + + - Add bottom (+top) collapsing, implement `IsBot`/`IsTop` for all lattice types + * `WithBot(Some(BOTTOM))` and `WithBot(None)` are now considered to both be bottom, equal. Also, `MapUnion({})` and `MapUnion({key: BOTTOM})` are considered to both be bottom, equal. + * `WithTop(Some(TOP))` and `WithTop(None)` are now considered to both be top, equal. + * `check_lattice_bot/top` now check that `is_bot` and `is_top` must be consistent among all equal elements + +### Refactor (BREAKING) + + - Rename `Seq` -> `VecUnion` + +### Commit Statistics + + + + - 9 commits contributed to the release over the course of 39 calendar days. + - 42 days passed between releases. + - 9 commits were understood as [conventional](https://www.conventionalcommits.org). + - 8 unique issues were worked on: [#822](https://github.com/hydro-project/hydroflow/issues/822), [#849](https://github.com/hydro-project/hydroflow/issues/849), [#854](https://github.com/hydro-project/hydroflow/issues/854), [#860](https://github.com/hydro-project/hydroflow/issues/860), [#865](https://github.com/hydro-project/hydroflow/issues/865), [#866](https://github.com/hydro-project/hydroflow/issues/866), [#867](https://github.com/hydro-project/hydroflow/issues/867), [#879](https://github.com/hydro-project/hydroflow/issues/879) + +### Commit Details + + + +
view details + + * **[#822](https://github.com/hydro-project/hydroflow/issues/822)** + - Fix lint, format errors for latest nightly version (without updated pinned) ([`f60053f`](https://github.com/hydro-project/hydroflow/commit/f60053f70da3071c54de4a0eabb059a143aa2ccc)) + * **[#849](https://github.com/hydro-project/hydroflow/issues/849)** + - Rename `Seq` -> `VecUnion` ([`7b0485b`](https://github.com/hydro-project/hydroflow/commit/7b0485b20939ec86ed8e74ecc9c75ac1b5d01072)) + * **[#854](https://github.com/hydro-project/hydroflow/issues/854)** + - Add atomize trait, impls, tests ([`8ec75c6`](https://github.com/hydro-project/hydroflow/commit/8ec75c6d8998b7d7e5a0ae24ee53b0cdb6932683)) + * **[#860](https://github.com/hydro-project/hydroflow/issues/860)** + - Improve `Atomize` docs ([`a8b0d2d`](https://github.com/hydro-project/hydroflow/commit/a8b0d2d10eef3e45669f77a1f2460cd31a95d15b)) + * **[#865](https://github.com/hydro-project/hydroflow/issues/865)** + - Add bottom (+top) collapsing, implement `IsBot`/`IsTop` for all lattice types ([`7b752f7`](https://github.com/hydro-project/hydroflow/commit/7b752f743cbedc632b127dddf3f9a84e839eb47a)) + * **[#866](https://github.com/hydro-project/hydroflow/issues/866)** + - Implement `SimpleKeyedRef` for map types ([`b240699`](https://github.com/hydro-project/hydroflow/commit/b2406994a703f028724cc30065fec60f7f8a7247)) + * **[#867](https://github.com/hydro-project/hydroflow/issues/867)** + - Change `Atomize` to require returning empty iff lattice is bottom ([`262166e`](https://github.com/hydro-project/hydroflow/commit/262166e7cecf8ffb5a2c7bc989e8cf66c4524a68)) + * **[#879](https://github.com/hydro-project/hydroflow/issues/879)** + - Formalize `Default::default()` as returning bottom for lattice types ([`7282457`](https://github.com/hydro-project/hydroflow/commit/7282457e383407eabbeb1f931c130edb095c33ca)) + * **Uncategorized** + - Fix new clippy lints on latest nightly 1.73.0-nightly (db7ff98a7 2023-07-31) ([`6a2ad6b`](https://github.com/hydro-project/hydroflow/commit/6a2ad6b770c2ccf470548320d8753025b3a66c0a)) +
+ +## 0.3.0 (2023-07-04) + + + + + + + +### Documentation + + - List `WithTop` in README 4/4 + +### New Features + + - make unit `()` a point lattice + - impl `IsTop`, `IsBot` for `Min`, `Max` over numeric types + - Add `Conflict` lattice + - add top lattice, opposite of bottom + - Add `Seq` lattice. + +### Bug Fixes + + - removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` + - fix ConvertFrom for bottom to actually convert the type + * fix: fix type inference with doubly-nested bottom types +* fix: address comments + +### Refactor + + - Rename `bottom.rs` -> `with_bot.rs`, `top.rs` -> `with_top.rs` 1/4 + +### Style + + - `warn` missing docs (instead of `deny`) to allow code before docs + +### New Features (BREAKING) + + + + - Add `reveal` methods, make fields private + - Add `Provenance` generic param token to `Point`. + - Use `()` provenance for `kvs_bench` example. + +### Bug Fixes (BREAKING) + + - Remove `Default` impl for `WithTop` 3/4 + Is confusing, probably not what users want. + +### Refactor (BREAKING) + + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` + - Rename `Bottom` -> `WithBot`, `Top` -> `WithTop`, constructors now take `Option`s 2/4 + - Rename `Immut` -> `Point` lattice. + +### Commit Statistics + + + + - 18 commits contributed to the release over the course of 31 calendar days. + - 33 days passed between releases. + - 17 commits were understood as [conventional](https://www.conventionalcommits.org). + - 12 unique issues were worked on: [#742](https://github.com/hydro-project/hydroflow/issues/742), [#744](https://github.com/hydro-project/hydroflow/issues/744), [#761](https://github.com/hydro-project/hydroflow/issues/761), [#763](https://github.com/hydro-project/hydroflow/issues/763), [#765](https://github.com/hydro-project/hydroflow/issues/765), [#766](https://github.com/hydro-project/hydroflow/issues/766), [#767](https://github.com/hydro-project/hydroflow/issues/767), [#772](https://github.com/hydro-project/hydroflow/issues/772), [#773](https://github.com/hydro-project/hydroflow/issues/773), [#780](https://github.com/hydro-project/hydroflow/issues/780), [#789](https://github.com/hydro-project/hydroflow/issues/789), [#793](https://github.com/hydro-project/hydroflow/issues/793) + +### Commit Details + + + +
view details + + * **[#742](https://github.com/hydro-project/hydroflow/issues/742)** + - Fix ConvertFrom for bottom to actually convert the type ([`3c4eb16`](https://github.com/hydro-project/hydroflow/commit/3c4eb16833160f8813b812487a1297c023400138)) + * **[#744](https://github.com/hydro-project/hydroflow/issues/744)** + - Add top lattice, opposite of bottom ([`fc4dcbd`](https://github.com/hydro-project/hydroflow/commit/fc4dcbdfa703d79a0c183a2eb3f5dbb42260b67a)) + * **[#761](https://github.com/hydro-project/hydroflow/issues/761)** + - Rename `Immut` -> `Point` lattice. ([`1bdadb8`](https://github.com/hydro-project/hydroflow/commit/1bdadb82b25941d11f3fa24eaac35109927c852f)) + * **[#763](https://github.com/hydro-project/hydroflow/issues/763)** + - List `WithTop` in README 4/4 ([`ac4fd82`](https://github.com/hydro-project/hydroflow/commit/ac4fd827ccede0ad53dfc59079cdb7df5928e491)) + - Remove `Default` impl for `WithTop` 3/4 ([`5cfd2a0`](https://github.com/hydro-project/hydroflow/commit/5cfd2a0f48f11f6185070cab932f50b630e1f800)) + - Rename `Bottom` -> `WithBot`, `Top` -> `WithTop`, constructors now take `Option`s 2/4 ([`5c7e4d3`](https://github.com/hydro-project/hydroflow/commit/5c7e4d3aea1dfb61d51bcb0291740281824e3090)) + - Rename `bottom.rs` -> `with_bot.rs`, `top.rs` -> `with_top.rs` 1/4 ([`0cbbaea`](https://github.com/hydro-project/hydroflow/commit/0cbbaeaec5e192e2539771bb247926271c2dc4a3)) + * **[#765](https://github.com/hydro-project/hydroflow/issues/765)** + - Rename `ConvertFrom::from` -> `LatticeFrom::lattice_from` ([`4a727ec`](https://github.com/hydro-project/hydroflow/commit/4a727ecf1232e0f03f5300547282bfbe73342cfa)) + * **[#766](https://github.com/hydro-project/hydroflow/issues/766)** + - Add `IsBot::is_bot` and `IsTop::is_top` traits ([`deb26af`](https://github.com/hydro-project/hydroflow/commit/deb26af6bcd547f91bf339367387d36e5e59565a)) + * **[#767](https://github.com/hydro-project/hydroflow/issues/767)** + - Add `Conflict` lattice ([`f5e0d19`](https://github.com/hydro-project/hydroflow/commit/f5e0d19e8531c250bc4492b61b9731c947916daf)) + * **[#772](https://github.com/hydro-project/hydroflow/issues/772)** + - Add `Provenance` generic param token to `Point`. ([`7aec1ac`](https://github.com/hydro-project/hydroflow/commit/7aec1ac884e01a560770dfab7e0ba64d520415f6)) + * **[#773](https://github.com/hydro-project/hydroflow/issues/773)** + - `warn` missing docs (instead of `deny`) to allow code before docs ([`70c88a5`](https://github.com/hydro-project/hydroflow/commit/70c88a51c4c83a4dc2fc67a0cd344786a4ff26f7)) + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` ([`9bb5528`](https://github.com/hydro-project/hydroflow/commit/9bb5528d99e83fdae5aeca9456802379131c2f90)) + * **[#789](https://github.com/hydro-project/hydroflow/issues/789)** + - Add `reveal` methods, make fields private ([`931d938`](https://github.com/hydro-project/hydroflow/commit/931d93887c238025596cb22226e16d43e16a7425)) + * **[#793](https://github.com/hydro-project/hydroflow/issues/793)** + - Make unit `()` a point lattice ([`016abee`](https://github.com/hydro-project/hydroflow/commit/016abeea3ecd390a976dd8dbec371b08fe744655)) + - Impl `IsTop`, `IsBot` for `Min`, `Max` over numeric types ([`dc99c02`](https://github.com/hydro-project/hydroflow/commit/dc99c021640a47b704905d087eadcbc477f033f0)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) + - Add `Seq` lattice. ([`153cbab`](https://github.com/hydro-project/hydroflow/commit/153cbabd462d776eae395e371470abb4662642cd)) +
+ + + Add IsBot::is_bot and IsTop::is_top traitsAlso adds test::check_lattice_bot (inlcluded in test::check_all) and test::check_lattice_top (NOT in check_all) + ## 0.2.0 (2023-05-31) + + + ### Chore - manually bump versions for v0.2.0 release @@ -19,8 +199,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 2 commits contributed to the release. - - 2 days passed between releases. + - 3 commits contributed to the release. + - 1 day passed between releases. - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - 0 issues like '(#ID)' were seen in commit messages @@ -31,6 +211,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
view details * **Uncategorized** + - Release hydroflow_lang v0.2.0, hydroflow_datalog_core v0.2.0, hydroflow_datalog v0.2.0, hydroflow_macro v0.2.0, lattices v0.2.0, hydroflow v0.2.0, hydro_cli v0.2.0 ([`ca464c3`](https://github.com/hydro-project/hydroflow/commit/ca464c32322a7ad39eb53e1794777c849aa548a0)) - Manually bump versions for v0.2.0 release ([`fd896fb`](https://github.com/hydro-project/hydroflow/commit/fd896fbe925fbd8ef1d16be7206ac20ba585081a)) - Rename `Fake` -> `Immut` ([`10b3085`](https://github.com/hydro-project/hydroflow/commit/10b308532245db8f4480ce53b67aea050ae1918d))
diff --git a/lattices/Cargo.toml b/lattices/Cargo.toml index 2b123277afd..b7a681efb03 100644 --- a/lattices/Cargo.toml +++ b/lattices/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lattices" publish = true -version = "0.2.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/lattices/" diff --git a/lattices/README.md b/lattices/README.md index 2e63d4d0a29..edbb8b4ef6f 100644 --- a/lattices/README.md +++ b/lattices/README.md @@ -27,15 +27,16 @@ Take a look at the [`lattice` rustdocs](https://hydro-project.github.io/hydroflo `lattices` provides implementations of common lattice types: * [`Min`] and [`Max`] - totally-orderd lattices. -* [`set_union::SetUnion`] - set-union lattice of scalar values. -* [`map_union::MapUnion`] - scalar keys with nested lattice values. +* [`set_union::SetUnion`] - set-union lattice of scalar values. +* [`map_union::MapUnion`] - scalar keys with nested lattice values. +* [`VecUnion`] - growing `Vec` of nested lattices, like `MapUnion<>` but without missing entries. * [`WithBot`] - wraps a lattice in `Option` with `None` as the new bottom value. * [`WithTop`] - wraps a lattice in `Option` with `None` as the new _top_ value. * [`Pair`] - product of two nested lattices. -* [`Seq`] - growing `Vec` of nested lattices, like `MapUnion<>` but without missing entries. * [`DomPair`]* - a versioned pair where the `LatKey` dominates the `LatVal`. * [`Conflict`]* - adds a "conflict" top to domain `T`. Merging inequal `T`s results in top. -* [`Point`]* - a single "point lattice" value which cannot be merged with any inequal value. +* [`Point`]* - a single "point lattice" value which cannot be merged with any inequal value. +* [`()`](https://doc.rust-lang.org/std/primitive.unit.html) - the "unit" lattice, a "point lattice" with unit `()` as the only value in the domain. *Special implementations which do not obey all lattice properties but are still useful under certain circumstances. @@ -82,8 +83,17 @@ type, e.g. between [`set_union::SetUnionBTreeSet`] and [`set_union::SetUnionHash lattice (lattices with nested lattice types), the `LatticeFrom` implementation should be recursive for those nested lattices. -### `IsBot` and `IsTop` +### `IsBot`, `IsTop`, and `Default` A bottom (⊥) is strictly less than all other values. A top (⊤) is strictly greater than all other -values. `IsBot::is_bot` and `IsTop::is_top` determine if a lattice instance is top or -bottom respectively. +values. `IsBot::is_bot` and `IsTop::is_top` determine if a lattice instance is top or bottom +respectively. + +For lattice types, `Default::default()` must create a bottom value. `IsBot::is_bot(&Default::default())` +should always return true for all lattice types. + +### `Atomize` + +[`Atomize::atomize`] converts a lattice point into a bunch of smaller lattice points. When these +"atoms" are merged together they will form the original lattice point. See the docs for more +precise semantics. diff --git a/lattices/src/collections.rs b/lattices/src/collections.rs index 0a3e5ecaab2..f39aa11104e 100644 --- a/lattices/src/collections.rs +++ b/lattices/src/collections.rs @@ -3,10 +3,10 @@ use std::borrow::Borrow; use std::hash::Hash; -use crate::cc_traits::{ - covariant_item_mut, covariant_item_ref, covariant_key_ref, Collection, CollectionMut, - CollectionRef, Get, GetKeyValue, GetKeyValueMut, GetMut, Iter, IterMut, Keyed, KeyedRef, Len, - MapIter, MapIterMut, +use cc_traits::{ + covariant_item_mut, covariant_item_ref, covariant_key_ref, simple_keyed_ref, Collection, + CollectionMut, CollectionRef, Get, GetKeyValue, GetKeyValueMut, GetMut, Iter, IterMut, Keyed, + KeyedRef, Len, MapIter, MapIterMut, SimpleKeyedRef, }; /// A [`Vec`]-wrapper representing a naively-implemented set. @@ -198,6 +198,9 @@ where .find(|(k, _v)| key == K::borrow(k)) } } +impl SimpleKeyedRef for VecMap { + simple_keyed_ref!(); +} impl MapIter for VecMap { type Iter<'a> = std::iter::Zip, std::slice::Iter<'a, V>> where @@ -386,9 +389,9 @@ impl Iter for SingletonMap { std::iter::once(&self.1) } } -// impl SimpleKeyedRef for SingletonMap { -// simple_keyed_ref!(); -// } +impl SimpleKeyedRef for SingletonMap { + simple_keyed_ref!(); +} impl MapIter for SingletonMap { type Iter<'a> = std::iter::Once<(&'a K, &'a V)> where @@ -408,6 +411,225 @@ impl MapIterMut for SingletonMap { } } +/// A wrapper around `Option`, representing either a singleton or empty set. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct OptionSet(pub Option); +impl Default for OptionSet { + fn default() -> Self { + Self(None) + } +} +impl IntoIterator for OptionSet { + type Item = T; + type IntoIter = std::option::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} +impl From for OptionSet +where + U: Into>, +{ + fn from(value: U) -> Self { + Self(value.into()) + } +} +impl Collection for OptionSet { + type Item = T; +} +impl Len for OptionSet { + fn len(&self) -> usize { + self.0.is_some() as usize + } +} +impl CollectionRef for OptionSet { + type ItemRef<'a> = &'a Self::Item + where + Self: 'a; + + covariant_item_ref!(); +} +impl<'a, Q, T> Get<&'a Q> for OptionSet +where + T: Borrow, + Q: Eq + ?Sized, +{ + fn get(&self, key: &'a Q) -> Option> { + self.0.as_ref().filter(|inner| key == (**inner).borrow()) + } +} +impl CollectionMut for OptionSet { + type ItemMut<'a> = &'a mut T + where + Self: 'a; + + covariant_item_mut!(); +} +impl<'a, Q, T> GetMut<&'a Q> for OptionSet +where + T: Borrow, + Q: Eq + ?Sized, +{ + fn get_mut(&mut self, key: &'a Q) -> Option> { + self.0.as_mut().filter(|inner| key == (**inner).borrow()) + } +} +impl Iter for OptionSet { + type Iter<'a> = std::option::Iter<'a, T> + where + Self: 'a; + + fn iter(&self) -> Self::Iter<'_> { + self.0.iter() + } +} +impl IterMut for OptionSet { + type IterMut<'a> = std::option::IterMut<'a, T> + where + Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + self.0.iter_mut() + } +} + +/// A key-value entry wrapper around `Option<(K, V)>` representing a singleton or empty map. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct OptionMap(pub Option<(K, V)>); +impl Default for OptionMap { + fn default() -> Self { + Self(None) + } +} +impl IntoIterator for OptionMap { + type Item = (K, V); + type IntoIter = std::option::IntoIter<(K, V)>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} +impl From for OptionMap +where + U: Into>, +{ + fn from(kv: U) -> Self { + Self(kv.into()) + } +} +impl Collection for OptionMap { + type Item = V; +} +impl Len for OptionMap { + fn len(&self) -> usize { + self.0.is_some() as usize + } +} +impl CollectionRef for OptionMap { + type ItemRef<'a> = &'a Self::Item + where + Self: 'a; + + covariant_item_ref!(); +} +impl<'a, Q, K, V> Get<&'a Q> for OptionMap +where + K: Borrow, + Q: Eq + ?Sized, +{ + fn get(&self, key: &'a Q) -> Option> { + self.0 + .as_ref() + .filter(|(k, _v)| key == k.borrow()) + .map(|(_k, v)| v) + } +} +impl CollectionMut for OptionMap { + type ItemMut<'a> = &'a mut Self::Item + where + Self: 'a; + + covariant_item_mut!(); +} +impl<'a, Q, K, V> GetMut<&'a Q> for OptionMap +where + K: Borrow, + Q: Eq + ?Sized, +{ + fn get_mut(&mut self, key: &'a Q) -> Option> { + self.0 + .as_mut() + .filter(|(k, _v)| key == k.borrow()) + .map(|(_k, v)| v) + } +} +impl Keyed for OptionMap { + type Key = K; +} +impl KeyedRef for OptionMap { + type KeyRef<'a> = &'a Self::Key + where + Self: 'a; + + covariant_key_ref!(); +} +impl<'a, Q, K, V> GetKeyValue<&'a Q> for OptionMap +where + K: Borrow, + Q: Eq + ?Sized, +{ + fn get_key_value(&self, key: &'a Q) -> Option<(Self::KeyRef<'_>, Self::ItemRef<'_>)> { + self.0 + .as_ref() + .filter(|(k, _v)| key == k.borrow()) + .map(|(k, v)| (k, v)) + } +} +impl<'a, Q, K, V> GetKeyValueMut<&'a Q> for OptionMap +where + K: Borrow, + Q: Eq + ?Sized, +{ + fn get_key_value_mut(&mut self, key: &'a Q) -> Option<(Self::KeyRef<'_>, Self::ItemMut<'_>)> { + self.0 + .as_mut() + .filter(|(k, _v)| key == k.borrow()) + .map(|(k, v)| (&*k, v)) + } +} +impl Iter for OptionMap { + type Iter<'a> = std::option::IntoIter<&'a V> + where + Self: 'a; + + fn iter(&self) -> Self::Iter<'_> { + self.0.as_ref().map(|(_k, v)| v).into_iter() + } +} +impl SimpleKeyedRef for OptionMap { + simple_keyed_ref!(); +} +impl MapIter for OptionMap { + type Iter<'a> = std::option::IntoIter<(&'a K, &'a V)> + where + Self: 'a; + + fn iter(&self) -> Self::Iter<'_> { + self.0.as_ref().map(|(k, v)| (k, v)).into_iter() + } +} +impl MapIterMut for OptionMap { + type IterMut<'a> = std::option::IntoIter<(&'a K, &'a mut V)> + where + Self: 'a; + + fn iter_mut(&mut self) -> Self::IterMut<'_> { + self.0.as_mut().map(|(k, v)| (&*k, v)).into_iter() + } +} + /// An array wrapper representing a fixed-size set (modulo duplicate items). #[repr(transparent)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -600,6 +822,9 @@ impl Iter for ArrayMap { self.vals.iter() } } +impl SimpleKeyedRef for ArrayMap { + simple_keyed_ref!(); +} impl MapIter for ArrayMap { type Iter<'a> = std::iter::Zip, std::slice::Iter<'a, V>> where diff --git a/lattices/src/conflict.rs b/lattices/src/conflict.rs index 5ff2f680490..1e7586f9c6f 100644 --- a/lattices/src/conflict.rs +++ b/lattices/src/conflict.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering::{self, *}; -use crate::{IsTop, LatticeFrom, LatticeOrd, Merge}; +use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// A `Conflict` lattice, stores a single instance of `T` and goes to a "conflict" state (`None`) /// if inequal `T` instances are merged together. @@ -12,9 +12,9 @@ use crate::{IsTop, LatticeFrom, LatticeOrd, Merge}; /// /// This can be used to wrap non-lattice (scalar) data into a lattice type. #[repr(transparent)] -#[derive(Copy, Clone, Debug, Default, Eq)] +#[derive(Copy, Clone, Debug, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Conflict(pub Option); +pub struct Conflict(Option); impl Conflict { /// Create a new `Conflict` lattice instance from a value. pub fn new(val: Option) -> Self { @@ -25,6 +25,21 @@ impl Conflict { pub fn new_from(val: impl Into>) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> Option<&T> { + self.0.as_ref() + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> Option<&mut T> { + self.0.as_mut() + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Option { + self.0 + } } impl Merge> for Conflict @@ -76,6 +91,12 @@ where } } +impl IsBot for Conflict { + fn is_bot(&self) -> bool { + false + } +} + impl IsTop for Conflict { fn is_top(&self) -> bool { self.0.is_none() @@ -86,8 +107,8 @@ impl IsTop for Conflict { mod test { use super::*; use crate::test::{ - check_all, check_lattice_ord, check_lattice_properties, check_lattice_top, - check_partial_ord_properties, + check_all, check_lattice_is_bot, check_lattice_is_top, check_lattice_ord, + check_lattice_properties, check_partial_ord_properties, }; use crate::WithBot; @@ -101,8 +122,8 @@ mod test { check_lattice_ord(items); check_partial_ord_properties(items); check_lattice_properties(items); - // check_lattice_bot(items); - check_lattice_top(items); + check_lattice_is_bot(items); + check_lattice_is_top(items); } #[test] @@ -114,6 +135,5 @@ mod test { WithBot::new(None), ]; check_all(items); - check_lattice_top(items); } } diff --git a/lattices/src/dom_pair.rs b/lattices/src/dom_pair.rs index 2d9a14f853b..4090a612fe3 100644 --- a/lattices/src/dom_pair.rs +++ b/lattices/src/dom_pair.rs @@ -17,10 +17,12 @@ use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; #[derive(Copy, Clone, Debug, Default, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct DomPair { - /// The `Key` of the dominating pair lattice, usually a timestamp + /// The `Key` of the dominating pair lattice, usually a timestamp. + /// + /// This field is public as it is always monotonically increasing in its lattice. pub key: Key, /// The `Val` of the dominating pair lattice. - pub val: Val, + val: Val, } impl DomPair { @@ -33,6 +35,21 @@ impl DomPair { pub fn new_from(key: impl Into, val: impl Into) -> Self { Self::new(key.into(), val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> (&Key, &Val) { + (&self.key, &self.val) + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> (&mut Key, &mut Val) { + (&mut self.key, &mut self.val) + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> (Key, Val) { + (self.key, self.val) + } } impl Merge> @@ -139,7 +156,7 @@ mod test { use crate::ord::Max; use crate::set_union::SetUnionHashSet; use crate::test::{ - check_lattice_bot, check_lattice_ord, check_lattice_properties, check_lattice_top, + check_lattice_is_bot, check_lattice_is_top, check_lattice_ord, check_lattice_properties, check_partial_ord_properties, }; use crate::WithTop; @@ -159,7 +176,7 @@ mod test { check_lattice_ord(&test_vec); check_partial_ord_properties(&test_vec); - check_lattice_bot(&test_vec); + check_lattice_is_bot(&test_vec); // DomPair is not actually a lattice. assert!(std::panic::catch_unwind(|| check_lattice_properties(&test_vec)).is_err()); } @@ -191,8 +208,8 @@ mod test { check_lattice_ord(&test_vec); check_partial_ord_properties(&test_vec); - check_lattice_bot(&test_vec); - check_lattice_top(&test_vec); + check_lattice_is_bot(&test_vec); + check_lattice_is_top(&test_vec); // DomPair is not actually a lattice. assert!(std::panic::catch_unwind(|| check_lattice_properties(&test_vec)).is_err()); } diff --git a/lattices/src/lib.rs b/lattices/src/lib.rs index 263087814d3..7f4b973c590 100644 --- a/lattices/src/lib.rs +++ b/lattices/src/lib.rs @@ -13,9 +13,10 @@ pub mod map_union; mod ord; mod pair; mod point; -mod seq; pub mod set_union; pub mod test; +mod unit; +mod vec_union; mod with_bot; mod with_top; @@ -24,10 +25,16 @@ pub use dom_pair::DomPair; pub use ord::{Max, Min}; pub use pair::Pair; pub use point::Point; -pub use seq::Seq; +pub use vec_union::VecUnion; pub use with_bot::WithBot; pub use with_top::WithTop; +/// Alias trait for lattice types. +#[sealed] +pub trait Lattice: Sized + Merge + LatticeOrd + NaiveLatticeOrd + IsBot + IsTop {} +#[sealed] +impl Lattice for T where T: Sized + Merge + LatticeOrd + NaiveLatticeOrd + IsBot + IsTop {} + /// Trait for lattice merge (AKA "join" or "least upper bound"). pub trait Merge { /// Merge `other` into the `self` lattice. @@ -98,11 +105,37 @@ pub trait LatticeFrom { /// Trait to check if a lattice instance is bottom (⊥). pub trait IsBot { /// Returns if `self` is lattice bottom (⊥). + /// + /// Must be consistent with equality, any element equal to bottom is also considered to be bottom. fn is_bot(&self) -> bool; } /// Trait to check if a lattice instance is top (⊤) and therefore cannot change any futher. pub trait IsTop { /// Returns if `self` is lattice top (⊤). + /// + /// Must be consistent with equality, any element equal to top is also considered to be top. fn is_top(&self) -> bool; } + +/// Trait to atomize a lattice into individual elements. For example, a [`set_union::SetUnion`] +/// will be broken up into individual singleton elements. +/// +/// Formally, breaks up `Self` into an set of lattice points forming a (strong) [antichain](https://en.wikipedia.org/wiki/Antichain). +/// "Strong" in the sense that any pair of lattice points in the antichain should have a greatest +/// lower bound (GLB or "meet") of bottom. +pub trait Atomize: Merge { + /// The type of atoms for this lattice. + type Atom: 'static + IsBot; + + /// The iter type iterating the antichain atoms. + type AtomIter: 'static + Iterator; + + /// Atomize self: convert into an iter of atoms. + /// + /// The returned iterator should be empty if and only if `self.is_bot()` is true. + /// All atoms in the returned iterator should have `self.is_bot()` be false. + /// + /// Returned values must merge to reform a value equal to the original `self`. + fn atomize(self) -> Self::AtomIter; +} diff --git a/lattices/src/map_union.rs b/lattices/src/map_union.rs index 5523278940f..a873159d36b 100644 --- a/lattices/src/map_union.rs +++ b/lattices/src/map_union.rs @@ -4,11 +4,11 @@ use std::cmp::Ordering::{self, *}; use std::collections::{BTreeMap, HashMap}; use std::fmt::Debug; -use cc_traits::Len; +use cc_traits::{Iter, Len}; use crate::cc_traits::{GetMut, Keyed, Map, MapIter, SimpleKeyedRef}; -use crate::collections::{ArrayMap, SingletonMap, VecMap}; -use crate::{IsBot, LatticeFrom, LatticeOrd, Merge}; +use crate::collections::{ArrayMap, OptionMap, SingletonMap, VecMap}; +use crate::{Atomize, IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// Map-union compound lattice. /// @@ -17,7 +17,7 @@ use crate::{IsBot, LatticeFrom, LatticeOrd, Merge}; #[repr(transparent)] #[derive(Copy, Clone, Debug, Default)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct MapUnion(pub Map); +pub struct MapUnion(Map); impl MapUnion { /// Create a new `MapUnion` from a `Map`. pub fn new(val: Map) -> Self { @@ -28,6 +28,21 @@ impl MapUnion { pub fn new_from(val: impl Into) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> &Map { + &self.0 + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> &mut Map { + &mut self.0 + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Map { + self.0 + } } impl Merge> for MapUnion @@ -37,6 +52,7 @@ where + for<'a> GetMut<&'a K, Item = ValSelf>, MapOther: IntoIterator, ValSelf: Merge + LatticeFrom, + ValOther: IsBot, { fn merge(&mut self, other: MapUnion) -> bool { let mut changed = false; @@ -47,6 +63,7 @@ where let iter: Vec<_> = other .0 .into_iter() + .filter(|(_k_other, val_other)| !val_other.is_bot()) .filter_map(|(k_other, val_other)| { match self.0.get_mut(&k_other) { // Key collision, merge into `self`. @@ -88,22 +105,23 @@ impl PartialOrd> for where MapSelf: Map + MapIter + SimpleKeyedRef, MapOther: Map + MapIter + SimpleKeyedRef, - ValSelf: PartialOrd, + ValSelf: PartialOrd + IsBot, + ValOther: IsBot, { fn partial_cmp(&self, other: &MapUnion) -> Option { let mut self_any_greater = false; let mut other_any_greater = false; - for k in self + let self_keys = self + .0 + .iter() + .filter(|(_k, v)| !v.is_bot()) + .map(|(k, _v)| ::into_ref(k)); + let other_keys = other .0 .iter() - .map(|(k, _v)| ::into_ref(k)) - .chain( - other - .0 - .iter() - .map(|(k, _v)| ::into_ref(k)), - ) - { + .filter(|(_k, v)| !v.is_bot()) + .map(|(k, _v)| ::into_ref(k)); + for k in self_keys.chain(other_keys) { match (self.0.get(k), other.0.get(k)) { (Some(self_value), Some(other_value)) => { match self_value.partial_cmp(&*other_value) { @@ -149,24 +167,21 @@ impl PartialEq> for where MapSelf: Map + MapIter + SimpleKeyedRef, MapOther: Map + MapIter + SimpleKeyedRef, - ValSelf: PartialEq, + ValSelf: PartialEq + IsBot, + ValOther: IsBot, { fn eq(&self, other: &MapUnion) -> bool { - if self.0.len() != other.0.len() { - return false; - } - - for k in self + let self_keys = self + .0 + .iter() + .filter(|(_k, v)| !v.is_bot()) + .map(|(k, _v)| ::into_ref(k)); + let other_keys = other .0 .iter() - .map(|(k, _v)| ::into_ref(k)) - .chain( - other - .0 - .iter() - .map(|(k, _v)| ::into_ref(k)), - ) - { + .filter(|(_k, v)| !v.is_bot()) + .map(|(k, _v)| ::into_ref(k)); + for k in self_keys.chain(other_keys) { match (self.0.get(k), other.0.get(k)) { (Some(self_value), Some(other_value)) => { if *self_value != *other_value { @@ -183,19 +198,45 @@ where true } } -impl Eq for MapUnion +impl Eq for MapUnion where Self: PartialEq {} + +impl IsBot for MapUnion where - Self: PartialEq, - MapSelf: Eq, + Map: Iter, + Map::Item: IsBot, { + fn is_bot(&self) -> bool { + self.0.iter().all(|v| v.is_bot()) + } } -impl IsBot for MapUnion +impl IsTop for MapUnion { + fn is_top(&self) -> bool { + false + } +} + +impl Atomize for MapUnion where - Map: Len, + Map: 'static + + Len + + IntoIterator + + Keyed + + Extend<(K, Val)> + + for<'a> GetMut<&'a K, Item = Val>, + K: 'static + Clone, + Val: 'static + Atomize + LatticeFrom<::Atom>, { - fn is_bot(&self) -> bool { - self.0.is_empty() + type Atom = MapUnionOptionMap; + + // TODO: use impl trait. + type AtomIter = Box>; + + fn atomize(self) -> Self::AtomIter { + Box::new(self.0.into_iter().flat_map(|(k, val)| { + val.atomize() + .map(move |v| MapUnionOptionMap::new_from((k.clone(), v))) + })) } } @@ -215,7 +256,7 @@ pub type MapUnionArrayMap = MapUnion pub type MapUnionSingletonMap = MapUnion>; /// [`Option`]-backed [`MapUnion`] lattice. -pub type MapUnionOption = MapUnion>; +pub type MapUnionOptionMap = MapUnion>; #[cfg(test)] mod test { @@ -224,7 +265,7 @@ mod test { use super::*; use crate::collections::{SingletonMap, SingletonSet}; use crate::set_union::{SetUnionHashSet, SetUnionSingletonSet}; - use crate::test::{cartesian_power, check_all}; + use crate::test::{cartesian_power, check_all, check_atomize_each}; #[test] fn test_map_union() { @@ -240,7 +281,7 @@ mod test { } #[test] - fn consistency() { + fn consistency_atomize() { let mut test_vec = Vec::new(); // Size 0. @@ -263,5 +304,24 @@ mod test { } check_all(&test_vec); + check_atomize_each(&test_vec); + } + + /// Check that a key with a value of bottom is the same as an empty map, etc. + #[test] + fn test_collapes_bot() { + let map_empty = >>::default(); + let map_a_bot = >>::new(SingletonMap( + "a", + Default::default(), + )); + let map_b_bot = >>::new(SingletonMap( + "b", + Default::default(), + )); + + assert_eq!(map_empty, map_a_bot); + assert_eq!(map_empty, map_b_bot); + assert_eq!(map_a_bot, map_b_bot); } } diff --git a/lattices/src/ord.rs b/lattices/src/ord.rs index 4e19044c2b7..0a76f381615 100644 --- a/lattices/src/ord.rs +++ b/lattices/src/ord.rs @@ -1,12 +1,14 @@ use std::cmp::Ordering; -use crate::{LatticeFrom, LatticeOrd, Merge}; +use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// A totally ordered max lattice. Merging returns the larger value. +/// +/// Note that the [`Default::default()`] value for numeric type is `MIN`, not zero. #[repr(transparent)] -#[derive(Copy, Clone, Debug, Default, PartialOrd, Ord, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Max(pub T); +pub struct Max(T); impl Max { /// Create a new `Max` lattice instance from a `T`. pub fn new(val: T) -> Self { @@ -17,6 +19,21 @@ impl Max { pub fn from(val: impl Into) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> &T { + &self.0 + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> &mut T { + &mut self.0 + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> T { + self.0 + } } impl Merge> for Max @@ -45,10 +62,12 @@ impl LatticeOrd for Max where Self: PartialOrd {} /// /// This means the lattice order is the reverse of what you might naturally expect: 0 is greater /// than 1. +/// +/// Note that the [`Default::default()`] value for numeric type is `MAX`, not zero. #[repr(transparent)] -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Min(pub T); +pub struct Min(T); impl Min { /// Create a new `Min` lattice instance from a `T`. pub fn new(val: T) -> Self { @@ -59,6 +78,21 @@ impl Min { pub fn new_from(val: impl Into) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> &T { + &self.0 + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> &mut T { + &mut self.0 + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> T { + self.0 + } } impl Merge> for Min @@ -100,12 +134,140 @@ where } } +// IsTop, IsBot, Default impls +impl IsTop for Max<()> { + fn is_top(&self) -> bool { + true + } +} +impl IsBot for Max<()> { + fn is_bot(&self) -> bool { + true + } +} +impl IsTop for Min<()> { + fn is_top(&self) -> bool { + true + } +} +impl IsBot for Min<()> { + fn is_bot(&self) -> bool { + true + } +} + +impl IsTop for Max { + fn is_top(&self) -> bool { + self.0 + } +} +impl IsBot for Max { + fn is_bot(&self) -> bool { + !self.0 + } +} +impl Default for Max { + fn default() -> Self { + Self(false) + } +} +impl IsTop for Min { + fn is_top(&self) -> bool { + !self.0 + } +} +impl IsBot for Min { + fn is_bot(&self) -> bool { + self.0 + } +} +impl Default for Min { + fn default() -> Self { + Self(true) + } +} + +impl IsTop for Max { + fn is_top(&self) -> bool { + char::MAX == self.0 + } +} +impl IsBot for Max { + fn is_bot(&self) -> bool { + '\x00' == self.0 + } +} +impl Default for Max { + fn default() -> Self { + Self('\x00') + } +} +impl IsTop for Min { + fn is_top(&self) -> bool { + '\x00' == self.0 + } +} +impl IsBot for Min { + fn is_bot(&self) -> bool { + char::MAX == self.0 + } +} +impl Default for Min { + fn default() -> Self { + Self(char::MAX) + } +} + +macro_rules! impls_numeric { + ( + $( $x:ty ),* + ) => { + $( + impl IsTop for Max<$x> { + fn is_top(&self) -> bool { + <$x>::MAX == self.0 + } + } + impl IsBot for Max<$x> { + fn is_bot(&self) -> bool { + <$x>::MIN == self.0 + } + } + + impl Default for Max<$x> { + fn default() -> Self { + Self(<$x>::MIN) + } + } + + impl IsTop for Min<$x> { + fn is_top(&self) -> bool { + <$x>::MIN == self.0 + } + } + impl IsBot for Min<$x> { + fn is_bot(&self) -> bool { + <$x>::MAX == self.0 + } + } + impl Default for Min<$x> { + fn default() -> Self { + Self(<$x>::MAX) + } + } + )* + }; +} +impls_numeric! { + isize, i8, i16, i32, i64, i128, usize, u8, u16, u32, u64, u128 +} + #[cfg(test)] mod test { use std::cmp::Ordering::*; use super::*; - use crate::test::{check_lattice_ord, check_lattice_properties, check_partial_ord_properties}; + use crate::test::check_all; #[test] fn ordering() { @@ -130,15 +292,48 @@ mod test { } #[test] - fn consistency() { - let items_max = &[Max::new(0), Max::new(1)]; - check_lattice_ord(items_max); - check_partial_ord_properties(items_max); - check_lattice_properties(items_max); - - let items_min = &[Min::new(0), Min::new(1)]; - check_lattice_ord(items_min); - check_partial_ord_properties(items_min); - check_lattice_properties(items_min); + fn consistency_max_bool() { + let items = &[Max::new(false), Max::new(true)]; + check_all(items); + } + + #[test] + fn consistency_min_bool() { + let items = &[Min::new(false), Min::new(true)]; + check_all(items); + } + + #[test] + fn consistency_max_char() { + let items: Vec<_> = "\x00\u{10FFFF}✨🤦‍♀️踊るx".chars().map(Max::new).collect(); + check_all(&items); + } + + #[test] + fn consistency_min_char() { + let items: Vec<_> = "\x00\u{10FFFF}✨🤦‍♀️踊るx".chars().map(Min::new).collect(); + check_all(&items); + } + + #[test] + fn consistency_max_i32() { + let items = &[ + Max::new(0), + Max::new(1), + Max::new(i32::MIN), + Max::new(i32::MAX), + ]; + check_all(items); + } + + #[test] + fn consistency_min_i32() { + let items = &[ + Min::new(0), + Min::new(1), + Min::new(i32::MIN), + Min::new(i32::MAX), + ]; + check_all(items); } } diff --git a/lattices/src/pair.rs b/lattices/src/pair.rs index 78f612fb61f..e397b8c3e7a 100644 --- a/lattices/src/pair.rs +++ b/lattices/src/pair.rs @@ -24,6 +24,21 @@ impl Pair { pub fn new_from(a: impl Into, b: impl Into) -> Self { Self::new(a.into(), b.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> (&LatA, &LatB) { + (&self.a, &self.b) + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> (&mut LatA, &mut LatB) { + (&mut self.a, &mut self.b) + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> (LatA, LatB) { + (self.a, self.b) + } } impl Merge> @@ -116,7 +131,7 @@ mod test { use super::*; use crate::set_union::SetUnionHashSet; - use crate::test::{check_all, check_lattice_top}; + use crate::test::check_all; use crate::WithTop; #[test] @@ -161,6 +176,5 @@ mod test { } check_all(&test_vec); - check_lattice_top(&test_vec); } } diff --git a/lattices/src/seq.rs b/lattices/src/seq.rs deleted file mode 100644 index 4675eb78e4f..00000000000 --- a/lattices/src/seq.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::cmp::Ordering::{self, *}; - -use cc_traits::Iter; - -use crate::{IsBot, LatticeFrom, LatticeOrd, Merge}; - -/// Sequence compound lattice. -/// -/// Contains any number of `Lat` sub-lattices. Sub-lattices are indexed starting at zero, merging -/// combines corresponding sub-lattices and keeps any excess. -/// -/// Similar to [`MapUnion<>`](super::map_union::MapUnion) but requires the key indices -/// start at `0` and have no gaps. -#[derive(Clone, Debug, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Seq { - seq: Vec, -} - -impl Seq { - /// Create a new `Seq` from a `Vec` of `Lat` instances. - pub fn new(seq: Vec) -> Self { - Self { seq } - } - - /// Create a new `Seq` from an `Into>`. - pub fn new_from(seq: impl Into>) -> Self { - Self::new(seq.into()) - } -} - -impl Default for Seq { - fn default() -> Self { - Self { - seq: Default::default(), - } - } -} - -impl Merge> for Seq -where - LatSelf: Merge + LatticeFrom, -{ - fn merge(&mut self, mut other: Seq) -> bool { - let mut changed = false; - // Extend `self` if `other` is longer. - if self.seq.len() < other.seq.len() { - self.seq - .extend(other.seq.drain(self.seq.len()..).map(LatSelf::lattice_from)); - changed = true; - } - // Merge intersecting indices. - for (self_val, other_val) in self.seq.iter_mut().zip(other.seq.into_iter()) { - changed |= self_val.merge(other_val); - } - changed - } -} - -impl LatticeFrom> for Seq -where - LatSelf: LatticeFrom, -{ - fn lattice_from(other: Seq) -> Self { - Self::new(other.seq.into_iter().map(LatSelf::lattice_from).collect()) - } -} - -impl PartialEq> for Seq -where - LatSelf: PartialEq, -{ - fn eq(&self, other: &Seq) -> bool { - if self.seq.len() != other.seq.len() { - return false; - } - return self - .seq - .iter() - .zip(other.seq.iter()) - .all(|(val_self, val_other)| val_self == val_other); - } -} - -impl PartialOrd> for Seq -where - LatSelf: PartialOrd, -{ - fn partial_cmp(&self, other: &Seq) -> Option { - let (self_len, other_len) = (self.seq.len(), other.seq.len()); - let mut self_any_greater = other_len < self_len; - let mut other_any_greater = self_len < other_len; - for (self_val, other_val) in self.seq.iter().zip(other.seq.iter()) { - match self_val.partial_cmp(other_val) { - None => { - return None; - } - Some(Less) => { - other_any_greater = true; - } - Some(Greater) => { - self_any_greater = true; - } - Some(Equal) => {} - } - if self_any_greater && other_any_greater { - return None; - } - } - match (self_any_greater, other_any_greater) { - (true, false) => Some(Greater), - (false, true) => Some(Less), - (false, false) => Some(Equal), - // We check this one after each loop iteration above. - (true, true) => unreachable!(), - } - } -} -impl LatticeOrd> for Seq where - Self: PartialOrd> -{ -} - -impl IsBot for Seq { - fn is_bot(&self) -> bool { - self.seq.is_empty() - } -} - -#[cfg(test)] -mod test { - use std::collections::HashSet; - - use super::*; - use crate::set_union::SetUnionHashSet; - use crate::test::{cartesian_power, check_all}; - use crate::Max; - - #[test] - fn basic() { - let mut my_seq_a = Seq::>::default(); - let my_seq_b = Seq::new(vec![Max::new(9), Max::new(4), Max::new(5)]); - let my_seq_c = Seq::new(vec![Max::new(2), Max::new(5)]); - - assert!(my_seq_a.merge(my_seq_b.clone())); - assert!(!my_seq_a.merge(my_seq_b)); - assert!(my_seq_a.merge(my_seq_c.clone())); - assert!(!my_seq_a.merge(my_seq_c)); - } - - #[test] - fn consistency() { - let mut test_vec = vec![Seq::new(vec![] as Vec>)]; - - let vals = [vec![], vec![0], vec![1], vec![0, 1]] - .map(HashSet::from_iter) - .map(SetUnionHashSet::new); - - test_vec.extend( - cartesian_power::<_, 1>(&vals).map(|row| Seq::new(row.into_iter().cloned().collect())), - ); - test_vec.extend( - cartesian_power::<_, 2>(&vals).map(|row| Seq::new(row.into_iter().cloned().collect())), - ); - - check_all(&test_vec); - } -} diff --git a/lattices/src/set_union.rs b/lattices/src/set_union.rs index 205184babed..249901f4c3d 100644 --- a/lattices/src/set_union.rs +++ b/lattices/src/set_union.rs @@ -4,8 +4,8 @@ use std::cmp::Ordering::{self, *}; use std::collections::{BTreeSet, HashSet}; use crate::cc_traits::{Iter, Len, Set}; -use crate::collections::{ArraySet, SingletonSet}; -use crate::{IsBot, LatticeFrom, LatticeOrd, Merge}; +use crate::collections::{ArraySet, OptionSet, SingletonSet}; +use crate::{Atomize, IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// Set-union lattice. /// @@ -24,6 +24,21 @@ impl SetUnion { pub fn new_from(val: impl Into) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> &Set { + &self.0 + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> &mut Set { + &mut self.0 + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Set { + self.0 + } } impl Merge> for SetUnion @@ -108,6 +123,28 @@ where } } +impl IsTop for SetUnion { + fn is_top(&self) -> bool { + false + } +} + +impl Atomize for SetUnion +where + Set: Len + IntoIterator + Extend, + Set::IntoIter: 'static, + Item: 'static, +{ + type Atom = SetUnionOptionSet; + + // TODO: use impl trait. + type AtomIter = Box>; + + fn atomize(self) -> Self::AtomIter { + Box::new(self.0.into_iter().map(SetUnionOptionSet::new_from)) + } +} + /// [`std::collections::HashSet`]-backed [`SetUnion`] lattice. pub type SetUnionHashSet = SetUnion>; @@ -124,13 +161,13 @@ pub type SetUnionArray = SetUnion>; pub type SetUnionSingletonSet = SetUnion>; /// [`Option`]-backed [`SetUnion`] lattice. -pub type SetUnionOption = SetUnion>; +pub type SetUnionOptionSet = SetUnion>; #[cfg(test)] mod test { use super::*; use crate::collections::SingletonSet; - use crate::test::check_all; + use crate::test::{check_all, check_atomize_each}; #[test] fn test_set_union() { @@ -176,4 +213,15 @@ mod test { SetUnionHashSet::new_from([0, 1]), ]); } + + #[test] + fn atomize() { + check_atomize_each(&[ + SetUnionHashSet::new_from([]), + SetUnionHashSet::new_from([0]), + SetUnionHashSet::new_from([1]), + SetUnionHashSet::new_from([0, 1]), + SetUnionHashSet::new((0..10).collect()), + ]); + } } diff --git a/lattices/src/test.rs b/lattices/src/test.rs index 50d45ecfe67..7aa75130ae4 100644 --- a/lattices/src/test.rs +++ b/lattices/src/test.rs @@ -2,17 +2,17 @@ use std::fmt::Debug; -use crate::{IsBot, IsTop, LatticeOrd, Merge, NaiveLatticeOrd}; +use crate::{Atomize, IsBot, IsTop, Lattice, LatticeOrd, Merge, NaiveLatticeOrd}; -/// Helper which calls [`check_lattice_ord`], [`check_partial_ord_properties`], -/// [`check_lattice_properties`], and [`check_lattice_bot`]. -pub fn check_all + IsBot + Clone + Eq + Debug>( - items: &[T], -) { +/// Helper which calls many other `check_*` functions in this module. See source code for which +/// functions are called. +pub fn check_all(items: &[T]) { check_lattice_ord(items); check_partial_ord_properties(items); check_lattice_properties(items); - check_lattice_bot(items); + check_lattice_is_bot(items); + check_lattice_is_top(items); + check_lattice_default_is_bot::(); } /// Check that the lattice's `PartialOrd` implementation agrees with the `NaiveLatticeOrd` partial @@ -140,24 +140,61 @@ pub fn check_lattice_properties + Clone + Eq + Debug>(items: &[T]) { } /// Checks that the item which is bot is less than (or equal to) all other items. -pub fn check_lattice_bot(items: &[T]) { - let bot = items - .iter() - .find(|&x| IsBot::is_bot(x)) - .expect("Expected `items` to contain bottom."); +pub fn check_lattice_is_bot(items: &[T]) { + let Some(bot) = items.iter().find(|&x| IsBot::is_bot(x)) else { + return; + }; for x in items { assert!(bot <= x); + assert_eq!(bot == x, x.is_bot(), "{:?}", x); } } /// Checks that the item which is top is greater than (or equal to) all other items. -pub fn check_lattice_top(items: &[T]) { - let top = items - .iter() - .find(|&x| IsTop::is_top(x)) - .expect("Expected `items` to contain top."); +pub fn check_lattice_is_top(items: &[T]) { + let Some(top) = items.iter().find(|&x| IsTop::is_top(x)) else { + return; + }; for x in items { assert!(x <= top); + assert_eq!(top == x, x.is_top(), "{:?}", x); + } +} + +/// Asserts that [`IsBot`] is true for [`Default::default()`]. +pub fn check_lattice_default_is_bot() { + assert!(T::is_bot(&T::default())); +} + +/// Check that the atomized lattice points re-merge to form the same original lattice point, for each item in `items`. +pub fn check_atomize_each< + T: Atomize + Merge + LatticeOrd + IsBot + Default + Clone + Debug, +>( + items: &[T], +) where + T::Atom: Debug, +{ + for item in items { + let mut reformed = T::default(); + let mut atoms = item.clone().atomize().peekable(); + assert_eq!( + atoms.peek().is_none(), + item.is_bot(), + "`{:?}` atomize should return empty iterator ({}) if and only if item is bot ({}).", + item, + atoms.peek().is_none(), + item.is_bot() + ); + for atom in atoms { + assert!( + !atom.is_bot(), + "`{:?}` atomize illegally returned a bottom atom `{:?}`.", + item, + atom, + ); + reformed.merge(atom); + } + assert_eq!(item, &reformed, "`{:?}` atomize failed to reform", item); } } @@ -166,7 +203,7 @@ pub fn check_lattice_top(items: &[T]) { /// product of `items` with itself `N` times. pub fn cartesian_power( items: &[T], -) -> impl Iterator + ExactSizeIterator + Clone { +) -> impl ExactSizeIterator + Clone { struct CartesianPower<'a, T, const N: usize> { items: &'a [T], iters: [std::iter::Peekable>; N], @@ -219,7 +256,7 @@ pub fn cartesian_power( impl<'a, T, const N: usize> Clone for CartesianPower<'a, T, N> { fn clone(&self) -> Self { Self { - items: self.items.clone(), + items: self.items, iters: self.iters.clone(), } } diff --git a/lattices/src/unit.rs b/lattices/src/unit.rs new file mode 100644 index 00000000000..ab2d12e92cd --- /dev/null +++ b/lattices/src/unit.rs @@ -0,0 +1,37 @@ +use crate::{Atomize, IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; + +impl Merge for () { + fn merge(&mut self, _other: Self) -> bool { + false + } +} + +impl LatticeOrd for () {} + +impl LatticeFrom for () { + fn lattice_from(other: Self) -> Self { + other + } +} + +impl IsBot for () { + fn is_bot(&self) -> bool { + true + } +} + +impl IsTop for () { + fn is_top(&self) -> bool { + true + } +} + +impl Atomize for () { + type Atom = Self; + + type AtomIter = std::iter::Once; + + fn atomize(self) -> Self::AtomIter { + std::iter::once(self) + } +} diff --git a/lattices/src/vec_union.rs b/lattices/src/vec_union.rs new file mode 100644 index 00000000000..1682a28eeb0 --- /dev/null +++ b/lattices/src/vec_union.rs @@ -0,0 +1,191 @@ +use std::cmp::Ordering::{self, *}; + +use cc_traits::Iter; + +use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; + +/// Vec-union compound lattice. +/// +/// Contains any number of `Lat` sub-lattices. Sub-lattices are indexed starting at zero, merging +/// combines corresponding sub-lattices and keeps any excess. +/// +/// Similar to [`MapUnion<>`](super::map_union::MapUnion) but requires the key indices +/// start with `0`, `1`, `2`, etc: i.e. integers starting at zero with no gaps. +#[derive(Clone, Debug, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct VecUnion { + vec: Vec, +} + +impl VecUnion { + /// Create a new `VecUnion` from a `Vec` of `Lat` instances. + pub fn new(vec: Vec) -> Self { + Self { vec } + } + + /// Create a new `VecUnion` from an `Into>`. + pub fn new_from(vec: impl Into>) -> Self { + Self::new(vec.into()) + } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> &Vec { + &self.vec + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> &mut Vec { + &mut self.vec + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Vec { + self.vec + } +} + +impl Default for VecUnion { + fn default() -> Self { + Self { + vec: Default::default(), + } + } +} + +impl Merge> for VecUnion +where + LatSelf: Merge + LatticeFrom, +{ + fn merge(&mut self, mut other: VecUnion) -> bool { + let mut changed = false; + // Extend `self` if `other` is longer. + if self.vec.len() < other.vec.len() { + self.vec + .extend(other.vec.drain(self.vec.len()..).map(LatSelf::lattice_from)); + changed = true; + } + // Merge intersecting indices. + for (self_val, other_val) in self.vec.iter_mut().zip(other.vec) { + changed |= self_val.merge(other_val); + } + changed + } +} + +impl LatticeFrom> for VecUnion +where + LatSelf: LatticeFrom, +{ + fn lattice_from(other: VecUnion) -> Self { + Self::new(other.vec.into_iter().map(LatSelf::lattice_from).collect()) + } +} + +impl PartialEq> for VecUnion +where + LatSelf: PartialEq, +{ + fn eq(&self, other: &VecUnion) -> bool { + if self.vec.len() != other.vec.len() { + return false; + } + return self + .vec + .iter() + .zip(other.vec.iter()) + .all(|(val_self, val_other)| val_self == val_other); + } +} + +impl PartialOrd> for VecUnion +where + LatSelf: PartialOrd, +{ + fn partial_cmp(&self, other: &VecUnion) -> Option { + let (self_len, other_len) = (self.vec.len(), other.vec.len()); + let mut self_any_greater = other_len < self_len; + let mut other_any_greater = self_len < other_len; + for (self_val, other_val) in self.vec.iter().zip(other.vec.iter()) { + match self_val.partial_cmp(other_val) { + None => { + return None; + } + Some(Less) => { + other_any_greater = true; + } + Some(Greater) => { + self_any_greater = true; + } + Some(Equal) => {} + } + if self_any_greater && other_any_greater { + return None; + } + } + match (self_any_greater, other_any_greater) { + (true, false) => Some(Greater), + (false, true) => Some(Less), + (false, false) => Some(Equal), + // We check this one after each loop iteration above. + (true, true) => unreachable!(), + } + } +} +impl LatticeOrd> for VecUnion where + Self: PartialOrd> +{ +} + +impl IsBot for VecUnion { + fn is_bot(&self) -> bool { + self.vec.is_empty() + } +} + +impl IsTop for VecUnion { + fn is_top(&self) -> bool { + false + } +} + +#[cfg(test)] +mod test { + use std::collections::HashSet; + + use super::*; + use crate::set_union::SetUnionHashSet; + use crate::test::{cartesian_power, check_all}; + use crate::Max; + + #[test] + fn basic() { + let mut my_vec_a = VecUnion::>::default(); + let my_vec_b = VecUnion::new(vec![Max::new(9), Max::new(4), Max::new(5)]); + let my_vec_c = VecUnion::new(vec![Max::new(2), Max::new(5)]); + + assert!(my_vec_a.merge(my_vec_b.clone())); + assert!(!my_vec_a.merge(my_vec_b)); + assert!(my_vec_a.merge(my_vec_c.clone())); + assert!(!my_vec_a.merge(my_vec_c)); + } + + #[test] + fn consistency() { + let mut test_vec = vec![VecUnion::new(vec![] as Vec>)]; + + let vals = [vec![], vec![0], vec![1], vec![0, 1]] + .map(HashSet::from_iter) + .map(SetUnionHashSet::new); + + test_vec.extend( + cartesian_power::<_, 1>(&vals) + .map(|row| VecUnion::new(row.into_iter().cloned().collect())), + ); + test_vec.extend( + cartesian_power::<_, 2>(&vals) + .map(|row| VecUnion::new(row.into_iter().cloned().collect())), + ); + + check_all(&test_vec); + } +} diff --git a/lattices/src/with_bot.rs b/lattices/src/with_bot.rs index 1fa97328826..b53bf943476 100644 --- a/lattices/src/with_bot.rs +++ b/lattices/src/with_bot.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering::{self, *}; -use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; +use crate::{Atomize, IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// Wraps a lattice in [`Option`], treating [`None`] as a new bottom element which compares as less /// than to all other values. @@ -8,7 +8,7 @@ use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// This can be used for giving a sensible default/bottom element to lattices that don't /// necessarily have one. #[repr(transparent)] -#[derive(Copy, Clone, Debug, Eq)] +#[derive(Copy, Clone, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct WithBot(pub Option); impl WithBot { @@ -21,6 +21,21 @@ impl WithBot { pub fn new_from(val: impl Into>) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> Option<&Inner> { + self.0.as_ref() + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> Option<&mut Inner> { + self.0.as_mut() + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Option { + self.0 + } } // Cannot auto derive because the generated implementation has the wrong trait bounds. @@ -34,16 +49,16 @@ impl Default for WithBot { impl Merge> for WithBot where Inner: Merge + LatticeFrom, + Other: IsBot, { fn merge(&mut self, other: WithBot) -> bool { match (&mut self.0, other.0) { - (None, None) => false, - (Some(_), None) => false, - (this @ None, Some(other_inner)) => { + (this @ None, Some(other_inner)) if !other_inner.is_bot() => { *this = Some(LatticeFrom::lattice_from(other_inner)); true } (Some(self_inner), Some(other_inner)) => self_inner.merge(other_inner), + (_self, _none_or_bot) => false, } } } @@ -59,11 +74,14 @@ where impl PartialOrd> for WithBot where - Inner: PartialOrd, + Inner: PartialOrd + IsBot, + Other: IsBot, { fn partial_cmp(&self, other: &WithBot) -> Option { match (&self.0, &other.0) { (None, None) => Some(Equal), + (None, Some(bot)) if bot.is_bot() => Some(Equal), + (Some(bot), None) if bot.is_bot() => Some(Equal), (None, Some(_)) => Some(Less), (Some(_), None) => Some(Greater), (Some(this_inner), Some(other_inner)) => this_inner.partial_cmp(other_inner), @@ -77,21 +95,28 @@ impl LatticeOrd> for WithBot where impl PartialEq> for WithBot where - Inner: PartialEq, + Inner: PartialEq + IsBot, + Other: IsBot, { fn eq(&self, other: &WithBot) -> bool { match (&self.0, &other.0) { (None, None) => true, + (None, Some(bot)) if bot.is_bot() => true, + (Some(bot), None) if bot.is_bot() => true, (None, Some(_)) => false, (Some(_), None) => false, (Some(this_inner), Some(other_inner)) => this_inner == other_inner, } } } +impl Eq for WithBot where Self: PartialEq {} -impl IsBot for WithBot { +impl IsBot for WithBot +where + Inner: IsBot, +{ fn is_bot(&self) -> bool { - self.0.is_none() + self.0.as_ref().map_or(true, IsBot::is_bot) } } @@ -104,11 +129,30 @@ where } } +impl Atomize for WithBot +where + Inner: 'static + Atomize + LatticeFrom<::Atom>, +{ + type Atom = WithBot; + + // TODO: use impl trait. + type AtomIter = Box>; + + fn atomize(self) -> Self::AtomIter { + Box::new( + self.0 + .into_iter() + .flat_map(Atomize::atomize) + .map(WithBot::new_from), + ) + } +} + #[cfg(test)] mod test { use super::*; use crate::set_union::{SetUnionHashSet, SetUnionSingletonSet}; - use crate::test::check_all; + use crate::test::{check_all, check_atomize_each}; #[test] fn test_singly_nested_singleton_example() { @@ -137,16 +181,21 @@ mod test { type B = WithBot>; assert_eq!(B::default().partial_cmp(&B::default()), Some(Equal)); - assert_eq!(B::new_from(SetUnionHashSet::new_from([])).partial_cmp(&B::default()), Some(Greater)); - assert_eq!(B::default().partial_cmp(&B::new_from(SetUnionHashSet::new_from([]))), Some(Less)); + + // Test bot collapsing - `WithBot(Some(Bot))` equals `WithBot(None)`. + assert_eq!(B::new_from(SetUnionHashSet::new_from([])).partial_cmp(&B::default()), Some(Equal)); + assert_eq!(B::default().partial_cmp(&B::new_from(SetUnionHashSet::new_from([]))), Some(Equal)); + assert!(B::new_from(SetUnionHashSet::new_from([])).eq(&B::default())); + assert!(B::default().eq(&B::new_from(SetUnionHashSet::new_from([])))); + + // PartialOrd assert_eq!(B::new_from(SetUnionHashSet::new_from([])).partial_cmp(&B::new_from(SetUnionHashSet::new_from([]))), Some(Equal)); assert_eq!(B::new_from(SetUnionHashSet::new_from([0])).partial_cmp(&B::new_from(SetUnionHashSet::new_from([]))), Some(Greater)); assert_eq!(B::new_from(SetUnionHashSet::new_from([])).partial_cmp(&B::new_from(SetUnionHashSet::new_from([0]))), Some(Less)); assert_eq!(B::new_from(SetUnionHashSet::new_from([0])).partial_cmp(&B::new_from(SetUnionHashSet::new_from([1]))), None); + // PartialEq assert!(B::default().eq(&B::default())); - assert!(!B::new_from(SetUnionHashSet::new_from([])).eq(&B::default())); - assert!(!B::default().eq(&B::new_from(SetUnionHashSet::new_from([])))); assert!(B::new_from(SetUnionHashSet::new_from([])).eq(&B::new_from(SetUnionHashSet::new_from([])))); assert!(!B::new_from(SetUnionHashSet::new_from([0])).eq(&B::new_from(SetUnionHashSet::new_from([])))); assert!(!B::new_from(SetUnionHashSet::new_from([])).eq(&B::new_from(SetUnionHashSet::new_from([0])))); @@ -163,4 +212,16 @@ mod test { WithBot::new_from(SetUnionHashSet::new_from([0, 1])), ]) } + + #[test] + fn atomize() { + check_atomize_each(&[ + WithBot::default(), + WithBot::new_from(SetUnionHashSet::new_from([])), + WithBot::new_from(SetUnionHashSet::new_from([0])), + WithBot::new_from(SetUnionHashSet::new_from([1])), + WithBot::new_from(SetUnionHashSet::new_from([0, 1])), + WithBot::new_from(SetUnionHashSet::new((0..10).collect())), + ]); + } } diff --git a/lattices/src/with_top.rs b/lattices/src/with_top.rs index a0939742568..14ae8579cda 100644 --- a/lattices/src/with_top.rs +++ b/lattices/src/with_top.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering::{self, *}; -use crate::{IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; +use crate::{Atomize, IsBot, IsTop, LatticeFrom, LatticeOrd, Merge}; /// Wraps a lattice in [`Option`], treating [`None`] as a new top element which compares as greater /// than to all other values. @@ -21,6 +21,31 @@ impl WithTop { pub fn new_from(val: impl Into>) -> Self { Self::new(val.into()) } + + /// Reveal the inner value as a shared reference. + pub fn as_reveal_ref(&self) -> Option<&Inner> { + self.0.as_ref() + } + + /// Reveal the inner value as an exclusive reference. + pub fn as_reveal_mut(&mut self) -> Option<&mut Inner> { + self.0.as_mut() + } + + /// Gets the inner by value, consuming self. + pub fn into_reveal(self) -> Option { + self.0 + } +} + +// Use inner's default rather than `None` (which is top, not bot). +impl Default for WithTop +where + Inner: Default, +{ + fn default() -> Self { + Self(Some(Inner::default())) + } } impl Merge> for WithTop @@ -90,9 +115,29 @@ where } } -impl IsTop for WithTop { +impl IsTop for WithTop +where + Inner: IsTop, +{ fn is_top(&self) -> bool { - self.0.is_none() + self.0.as_ref().map_or(true, IsTop::is_top) + } +} + +impl Atomize for WithTop +where + Inner: Atomize + LatticeFrom<::Atom>, +{ + type Atom = WithTop; + + // TODO: use impl trait. + type AtomIter = Box>; + + fn atomize(self) -> Self::AtomIter { + match self.0 { + Some(inner) => Box::new(inner.atomize().map(WithTop::new_from)), + None => Box::new(std::iter::once(WithTop::new(None))), + } } } @@ -100,7 +145,7 @@ impl IsTop for WithTop { mod test { use super::*; use crate::set_union::{SetUnionHashSet, SetUnionSingletonSet}; - use crate::test::{check_all, check_lattice_top}; + use crate::test::{check_all, check_atomize_each, check_lattice_is_top}; #[test] fn test_singly_nested_singleton_example() { @@ -155,6 +200,18 @@ mod test { WithTop::new_from(SetUnionHashSet::new_from([0, 1])), ]; check_all(items); - check_lattice_top(items); + check_lattice_is_top(items); + } + + #[test] + fn atomize() { + check_atomize_each(&[ + WithTop::new(None), + WithTop::new_from(SetUnionHashSet::new_from([])), + WithTop::new_from(SetUnionHashSet::new_from([0])), + WithTop::new_from(SetUnionHashSet::new_from([1])), + WithTop::new_from(SetUnionHashSet::new_from([0, 1])), + WithTop::new_from(SetUnionHashSet::new((0..10).collect())), + ]); } } diff --git a/multiplatform_test/Cargo.toml b/multiplatform_test/Cargo.toml index 105f6726107..e18024bb75c 100644 --- a/multiplatform_test/Cargo.toml +++ b/multiplatform_test/Cargo.toml @@ -11,7 +11,7 @@ description = "A simple attribute macro to combine `#[test]` and `#[wasm_bindgen proc-macro = true [dependencies] -proc-macro2 = "1.0" +proc-macro2 = "1.0.57" quote = "1.0" [dev-dependencies] diff --git a/multiplatform_test/src/lib.rs b/multiplatform_test/src/lib.rs index fd94c1cbce8..3ec20b5ac63 100644 --- a/multiplatform_test/src/lib.rs +++ b/multiplatform_test/src/lib.rs @@ -141,7 +141,9 @@ fn multiplatform_test_impl( output.extend(body); } else { let mut body_head = body.into_iter().collect::>(); - let Some(proc_macro2::TokenTree::Group(body_code)) = body_head.pop() else { panic!(); }; + let Some(proc_macro2::TokenTree::Group(body_code)) = body_head.pop() else { + panic!(); + }; output.extend(body_head); output.extend(quote! { diff --git a/pusherator/CHANGELOG.md b/pusherator/CHANGELOG.md index 9a6d6f22b92..9ec3c68be43 100644 --- a/pusherator/CHANGELOG.md +++ b/pusherator/CHANGELOG.md @@ -5,8 +5,72 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.0.3 (2023-08-15) + +### Chore + + - fix lint, format errors for latest nightly version (without updated pinned) + For nightly version (d9c13cd45 2023-07-05) + +### New Features + + - rename assert => assert_eq, add assert, change underlying implementation to work across ticks + +### Commit Statistics + + + + - 2 commits contributed to the release over the course of 39 calendar days. + - 42 days passed between releases. + - 2 commits were understood as [conventional](https://www.conventionalcommits.org). + - 2 unique issues were worked on: [#822](https://github.com/hydro-project/hydroflow/issues/822), [#835](https://github.com/hydro-project/hydroflow/issues/835) + +### Commit Details + + + +
view details + + * **[#822](https://github.com/hydro-project/hydroflow/issues/822)** + - Fix lint, format errors for latest nightly version (without updated pinned) ([`f60053f`](https://github.com/hydro-project/hydroflow/commit/f60053f70da3071c54de4a0eabb059a143aa2ccc)) + * **[#835](https://github.com/hydro-project/hydroflow/issues/835)** + - Rename assert => assert_eq, add assert, change underlying implementation to work across ticks ([`8f306e2`](https://github.com/hydro-project/hydroflow/commit/8f306e2a36582e168417808099eedf8a9de3b419)) +
+ +## 0.0.2 (2023-07-04) + +### Bug Fixes + + - remove nightly feature `never_type` where unused + - removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` + +### Commit Statistics + + + + - 3 commits contributed to the release over the course of 12 calendar days. + - 44 days passed between releases. + - 2 commits were understood as [conventional](https://www.conventionalcommits.org). + - 1 unique issue was worked on: [#780](https://github.com/hydro-project/hydroflow/issues/780) + +### Commit Details + + + +
view details + + * **[#780](https://github.com/hydro-project/hydroflow/issues/780)** + - Remove nightly feature `never_type` where unused ([`a3c1fbb`](https://github.com/hydro-project/hydroflow/commit/a3c1fbbd1e3fa7a7299878f61b4bfd12dce0052c)) + - Removed unused nightly features `impl_trait_in_assoc_type`, `type_alias_impl_trait` ([`9bb5528`](https://github.com/hydro-project/hydroflow/commit/9bb5528d99e83fdae5aeca9456802379131c2f90)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.3.0, hydroflow_lang v0.3.0, hydroflow_datalog_core v0.3.0, hydroflow_datalog v0.3.0, hydroflow_macro v0.3.0, lattices v0.3.0, pusherator v0.0.2, hydroflow v0.3.0, hydro_cli v0.3.0, safety bump 5 crates ([`ec9633e`](https://github.com/hydro-project/hydroflow/commit/ec9633e2e393c2bf106223abeb0b680200fbdf84)) +
+ ## 0.0.1 (2023-05-21) + + + ### Style - rustfmt group imports @@ -16,8 +80,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - - 2 commits contributed to the release. - - 25 days passed between releases. + - 3 commits contributed to the release. + - 24 days passed between releases. - 2 commits were understood as [conventional](https://www.conventionalcommits.org). - 1 unique issue was worked on: [#660](https://github.com/hydro-project/hydroflow/issues/660) @@ -30,6 +94,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **[#660](https://github.com/hydro-project/hydroflow/issues/660)** - Rustfmt group imports ([`20a1b2c`](https://github.com/hydro-project/hydroflow/commit/20a1b2c0cd04a8b495a02ce345db3d48a99ea0e9)) - Rustfmt prescribe flat-module `use` format ([`1eda91a`](https://github.com/hydro-project/hydroflow/commit/1eda91a2ef8794711ef037240f15284e8085d863)) + * **Uncategorized** + - Release hydroflow_cli_integration v0.0.1, hydroflow_lang v0.0.1, hydroflow_datalog_core v0.0.1, hydroflow_datalog v0.0.1, hydroflow_macro v0.0.1, lattices v0.1.0, variadics v0.0.2, pusherator v0.0.1, hydroflow v0.0.2 ([`809395a`](https://github.com/hydro-project/hydroflow/commit/809395acddb78949d7a2bf036e1a94972f23b1ad)) ## 0.0.0 (2023-04-26) diff --git a/pusherator/Cargo.toml b/pusherator/Cargo.toml index 5f7bf1e9880..09eb7cf6bb6 100644 --- a/pusherator/Cargo.toml +++ b/pusherator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pusherator" publish = true -version = "0.0.1" +version = "0.0.3" edition = "2021" license = "Apache-2.0" documentation = "https://docs.rs/pusherator/" diff --git a/pusherator/src/lib.rs b/pusherator/src/lib.rs index ffe2fba1371..7837d7214f1 100644 --- a/pusherator/src/lib.rs +++ b/pusherator/src/lib.rs @@ -14,6 +14,7 @@ pub mod flatten; pub mod for_each; pub mod inspect; pub mod map; +pub mod null; pub mod partition; pub mod pivot; pub mod switch; @@ -235,7 +236,7 @@ mod tests { let mut right = Vec::new(); let pivot = Pivot::new( - a.into_iter().chain(b.into_iter()), + a.into_iter().chain(b), Partition::new( |x| x % 2 == 0, ForEach::new(|x| left.push(x)), diff --git a/pusherator/src/null.rs b/pusherator/src/null.rs new file mode 100644 index 00000000000..26ac6868621 --- /dev/null +++ b/pusherator/src/null.rs @@ -0,0 +1,27 @@ +use std::marker::PhantomData; + +use super::Pusherator; + +#[derive(Clone, Copy)] +pub struct Null { + phantom: PhantomData, +} + +impl Null { + pub fn new() -> Self { + Self { + phantom: Default::default(), + } + } +} + +impl Default for Null { + fn default() -> Self { + Self::new() + } +} + +impl Pusherator for Null { + type Item = In; + fn give(&mut self, _: Self::Item) {} +} diff --git a/relalg/Cargo.toml b/relalg/Cargo.toml index b6701cc3ecf..8216e4909d6 100644 --- a/relalg/Cargo.toml +++ b/relalg/Cargo.toml @@ -9,7 +9,7 @@ license = "Apache-2.0" anyhow = "1.0" datadriven = "0.6.0" hydroflow = { path = "../hydroflow" } -proc-macro2 = "1.0" +proc-macro2 = "1.0.57" quote = "1.0" syn = { version = "2.0.0", features = [ "parsing", "extra-traits" ] } prettyplease = "0.2.0" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b4ea1078295..a4a10b68f73 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2023-06-01" +channel = "nightly-2023-06-30" components = ["rustfmt", "clippy"] -targets = ["wasm32-unknown-unknown"] +targets = ["wasm32-unknown-unknown", "x86_64-unknown-linux-musl"] diff --git a/topolotree/.gitignore b/topolotree/.gitignore new file mode 100644 index 00000000000..afed0735dc9 --- /dev/null +++ b/topolotree/.gitignore @@ -0,0 +1 @@ +*.csv diff --git a/topolotree/Cargo.toml b/topolotree/Cargo.toml new file mode 100644 index 00000000000..6fde3c72c24 --- /dev/null +++ b/topolotree/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "topolotree" +publish = false +version = "0.0.0" +edition = "2021" + +[[bin]] +name = "topolotree" +path = "src/main.rs" + +[[bin]] +name = "pn" +path = "src/pn.rs" + +[[bin]] +name = "pn_delta" +path = "src/pn_delta.rs" + +[[bin]] +name = "latency_measure" +path = "src/latency_measure.rs" + +[dependencies] +hydroflow = { path = "../hydroflow", features = [ "cli_integration" ] } +hydroflow_datalog = { path = "../hydroflow_datalog" } + +tokio = { version = "1.16", features = [ "full" ] } +serde = { version = "1", features = ["rc"] } +serde_json = "1" +rand = "0.8.5" +dashmap = "5.4.0" + +futures = "0.3.28" + +tokio-tungstenite = "0.19.0" + +[target.'cfg(target_os = "linux")'.dependencies] +procinfo = "0.4.2" diff --git a/hydro_cli_examples/examples/topolotree_latency_measure/main.rs b/topolotree/src/latency_measure.rs similarity index 77% rename from hydro_cli_examples/examples/topolotree_latency_measure/main.rs rename to topolotree/src/latency_measure.rs index 56beda56453..5d8e8cc8f19 100644 --- a/hydro_cli_examples/examples/topolotree_latency_measure/main.rs +++ b/topolotree/src/latency_measure.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::atomic::AtomicU64; use std::sync::{mpsc, Arc}; use std::thread; @@ -5,18 +6,14 @@ use std::time::Instant; use futures::{SinkExt, StreamExt}; use hydroflow::bytes::Bytes; -use hydroflow::serde::{Deserialize, Serialize}; use hydroflow::tokio; use hydroflow::util::cli::{ConnectedDirect, ConnectedSink, ConnectedSource}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; -#[derive(Serialize, Deserialize, Clone, Debug)] -struct IncrementRequest { - tweet_id: u64, - likes: i32, -} +mod protocol; +use protocol::*; -#[hydroflow::main] +#[tokio::main] async fn main() { let mut ports = hydroflow::util::cli::init().await; let mut start_node = ports @@ -64,27 +61,37 @@ async fn main() { let mut queues = vec![]; for i in 0..num_clients { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::(); + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::(); queues.push(sender); let inc_sender = inc_sender.clone(); let latency_sender = latency_sender.clone(); let atomic_counter = atomic_counter.clone(); tokio::spawn(async move { + #[cfg(debug_assertions)] + let mut count_tracker = HashMap::new(); + loop { let id = ((rand::random::() % 1024) / (num_clients as u64)) * (num_clients as u64) + (i as u64); let increment = rand::random::(); + let change = if increment { 1 } else { -1 }; let start = Instant::now(); inc_sender - .send(serialize_to_bytes(IncrementRequest { - tweet_id: id, - likes: if increment { 1 } else { -1 }, + .send(serialize_to_bytes(OperationPayload { + key: id, + change, })) .unwrap(); - receiver.recv().await.unwrap(); + let received = receiver.recv().await.unwrap(); + #[cfg(debug_assertions)] + { + let count = count_tracker.entry(id).or_insert(0); + *count += change; + assert!(*count == received); + } latency_sender.send(start.elapsed().as_micros()).unwrap(); @@ -96,10 +103,10 @@ async fn main() { tokio::spawn(async move { loop { let updated = - deserialize_from_bytes::<(u64, i32)>(end_node.next().await.unwrap().unwrap()) + deserialize_from_bytes::(end_node.next().await.unwrap().unwrap()) .unwrap(); - if queues[(updated.0 % (num_clients as u64)) as usize] - .send(updated.1) + if queues[(updated.key % (num_clients as u64)) as usize] + .send(updated.value) .is_err() { break; diff --git a/topolotree/src/main.rs b/topolotree/src/main.rs new file mode 100644 index 00000000000..0a4353ff554 --- /dev/null +++ b/topolotree/src/main.rs @@ -0,0 +1,212 @@ +#[cfg(test)] +mod tests; + +use std::cell::RefCell; +use std::collections::HashMap; +use std::fmt::{Display, Debug}; +use std::io; +use std::rc::Rc; + +use futures::{SinkExt, Stream}; +use hydroflow::bytes::{Bytes, BytesMut}; +use hydroflow::hydroflow_syntax; +use hydroflow::scheduled::graph::Hydroflow; +use hydroflow::util::cli::{ + ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged, +}; + +mod protocol; +use hydroflow::util::{serialize_to_bytes, deserialize_from_bytes}; +use protocol::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct NodeID(pub u32); + +impl Display for NodeID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.0, f) + } +} + +fn run_topolotree( + neighbors: Vec, + input_recv: impl Stream> + Unpin + 'static, + increment_requests: impl Stream> + Unpin + 'static, + output_send: tokio::sync::mpsc::UnboundedSender<(u32, Bytes)>, + query_send: tokio::sync::mpsc::UnboundedSender, +) -> Hydroflow { + fn merge(x: &mut i64, y: i64) { + *x += y; + } + + // Timestamp stuff is a bit complicated, there is a proper data-flowy way to do it + // but it would require at least one more join and one more cross join just specifically for the local timestamps + // Until we need it to be proper then we can take a shortcut and use rc refcell + let self_timestamp = Rc::new(RefCell::new(HashMap::::new())); + + let self_timestamp1 = Rc::clone(&self_timestamp); + let self_timestamp2 = Rc::clone(&self_timestamp); + let self_timestamp3 = Rc::clone(&self_timestamp); + + // we use current tick to keep track of which *keys* have been modified + + hydroflow_syntax! { + from_neighbors = source_stream(input_recv) + -> map(Result::unwrap) + -> map(|(src, x)| (NodeID(src), deserialize_from_bytes::>(&x).unwrap())); + + from_neighbors + -> map(|(src, payload)| ((payload.key, src), (payload.key, payload.contents))) + -> fold_keyed::<'static>(|| (Timestamped { timestamp: -1, data: Default::default() }, 0), |acc: &mut (Timestamped, usize), (key, val): (u64, Timestamped)| { + if val.timestamp > acc.0.timestamp { + acc.0 = val; + *self_timestamp1.borrow_mut().entry(key).or_insert(0) += 1; + acc.1 = context.current_tick(); + } + }) + -> map(|((key, src), (payload, change_tick))| ((key, Some(src)), (payload.data, change_tick))) + -> from_neighbors_or_local; + + local_value = source_stream(increment_requests) + -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) + -> inspect(|change| { + *self_timestamp2.borrow_mut().entry(change.key).or_insert(0) += 1; + }) + -> map(|change_payload: OperationPayload| (change_payload.key, (change_payload.change, context.current_tick()))) + -> reduce_keyed::<'static>(|agg: &mut (i64, usize), change: (i64, usize)| { + agg.0 += change.0; + agg.1 = std::cmp::max(agg.1, change.1); + }); + + local_value -> map(|(key, data)| ((key, None), data)) -> from_neighbors_or_local; + + from_neighbors_or_local = union() -> tee(); + from_neighbors_or_local -> [0]all_neighbor_data; + + neighbors = source_iter(neighbors) + -> map(NodeID) + -> persist(); + + neighbors -> [1]all_neighbor_data; + + query_result = from_neighbors_or_local + -> map(|((key, _), payload): ((u64, _), (i64, usize))| { + (key, payload) + }) + -> reduce_keyed(|acc: &mut (i64, usize), (data, change_tick): (i64, usize)| { + merge(&mut acc.0, data); + acc.1 = std::cmp::max(acc.1, change_tick); + }) + -> filter(|(_, (_, change_tick))| *change_tick == context.current_tick()) + -> for_each(|(key, (data, _))| { + let serialized = serialize_to_bytes(QueryResponse { + key, + value: data + }); + query_send.send(serialized).unwrap(); + }); + + all_neighbor_data = cross_join_multiset() + -> filter(|(((_, aggregate_from_this_guy), _), target_neighbor): &(((u64, Option), (i64, usize)), NodeID)| { + aggregate_from_this_guy.iter().all(|source| source != target_neighbor) + }) + -> map(|(((key, _), payload), target_neighbor)| { + ((key, target_neighbor), payload) + }) + -> reduce_keyed(|acc: &mut (i64, usize), (data, change_tick): (i64, usize)| { + merge(&mut acc.0, data); + acc.1 = std::cmp::max(acc.1, change_tick); + }) + -> filter(|(_, (_, change_tick))| *change_tick == context.current_tick()) + -> map(|((key, target_neighbor), (data, _))| (target_neighbor, Payload { + key, + contents: Timestamped { + timestamp: self_timestamp3.borrow().get(&key).copied().unwrap_or(0), + data, + } + })) + -> for_each(|(target_neighbor, output): (NodeID, Payload)| { + let serialized = serialize_to_bytes(output); + output_send.send((target_neighbor.0, serialized)).unwrap(); + }); + } +} + +#[hydroflow::main] +async fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let neighbors: Vec = args.into_iter().map(|x| x.parse().unwrap()).collect(); + + let mut ports = hydroflow::util::cli::init().await; + + let input_recv = ports + .port("from_peer") + // connect to the port with a single recipient + .connect::>() + .await + .into_source(); + + let mut output_send = ports + .port("to_peer") + .connect::>() + .await + .into_sink(); + + let operations_send = ports + .port("increment_requests") + // connect to the port with a single recipient + .connect::() + .await + .into_source(); + + let mut query_responses = ports + .port("query_responses") + .connect::() + .await + .into_sink(); + + let (chan_tx, mut chan_rx) = tokio::sync::mpsc::unbounded_channel(); + + tokio::task::spawn_local(async move { + while let Some(msg) = chan_rx.recv().await { + output_send.send(msg).await.unwrap(); + } + }); + + let (query_tx, mut query_rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::task::spawn_local(async move { + while let Some(msg) = query_rx.recv().await { + query_responses.send(msg).await.unwrap(); + } + }); + + let flow = run_topolotree( + neighbors, + input_recv, + operations_send, + chan_tx, + query_tx + ); + + let f1 = async move { + #[cfg(target_os = "linux")] + loop { + let x = procinfo::pid::stat_self().unwrap(); + let bytes = x.rss * 1024 * 4; + println!("memory,{}", bytes); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + }; + + // initial memory + #[cfg(target_os = "linux")] + { + let x = procinfo::pid::stat_self().unwrap(); + let bytes = x.rss * 1024 * 4; + println!("memory,{}", bytes); + } + + let f1_handle = tokio::spawn(f1); + hydroflow::util::cli::launch_flow(flow).await; + f1_handle.abort(); +} diff --git a/hydro_cli_examples/examples/pn_counter/main.rs b/topolotree/src/pn.rs similarity index 83% rename from hydro_cli_examples/examples/pn_counter/main.rs rename to topolotree/src/pn.rs index e209fe2dcac..12c61d91073 100644 --- a/hydro_cli_examples/examples/pn_counter/main.rs +++ b/topolotree/src/pn.rs @@ -4,22 +4,19 @@ use std::ops::Deref; use std::rc::Rc; use hydroflow::serde::{Deserialize, Serialize}; -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow::{hydroflow_syntax, tokio}; -#[derive(Serialize, Deserialize, Clone, Debug)] -struct IncrementRequest { - tweet_id: u64, - likes: i32, -} +mod protocol; +use protocol::*; -type NextStateType = (u64, Rc, Vec)>>); +type NextStateType = (u64, Rc, Vec)>>); #[derive(Serialize, Deserialize, Clone, Debug)] enum GossipOrIncrement { Gossip(Vec), - Increment(u64, i32), + Increment(u64, i64), } #[hydroflow::main] @@ -51,7 +48,7 @@ async fn main() { let from_peer = ports .port("from_peer") - .connect::() + .connect::>() .await .into_source(); @@ -67,8 +64,8 @@ async fn main() { let df = hydroflow_syntax! { next_state = union() - -> fold::<'static>((HashMap::, Vec)>>>::new(), HashSet::new(), 0), |(mut cur_state, mut modified_tweets, last_tick), goi| { - if context.current_tick() != last_tick { + -> fold::<'static>((HashMap::, Vec)>>>::new(), HashSet::new(), 0), |(cur_state, modified_tweets, last_tick): &mut (HashMap<_, _>, HashSet<_>, _), goi| { + if context.current_tick() != *last_tick { modified_tweets.clear(); } @@ -102,16 +99,16 @@ async fn main() { let mut cur_value = cur_value.as_ref().borrow_mut(); if delta > 0 { - cur_value.0[my_id] += delta as u32; + cur_value.0[my_id] += delta as u64; } else { - cur_value.1[my_id] += (-delta) as u32; + cur_value.1[my_id] += (-delta) as u64; } modified_tweets.insert(counter_id); } } - (cur_state, modified_tweets, context.current_tick()) + *last_tick = context.current_tick(); }) -> filter(|(_, _, tick)| *tick == context.current_tick()) -> filter(|(_, modified_tweets, _)| !modified_tweets.is_empty()) @@ -119,12 +116,12 @@ async fn main() { -> tee(); source_stream(from_peer) - -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) + -> map(|x| deserialize_from_bytes::(&x.unwrap().1).unwrap()) -> next_state; source_stream(increment_requests) - -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) - -> map(|t| GossipOrIncrement::Increment(t.tweet_id, t.likes)) + -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) + -> map(|t| GossipOrIncrement::Increment(t.key, t.change)) -> next_state; all_peers = source_iter(0..num_replicas) @@ -143,10 +140,13 @@ async fn main() { a.into_iter().map(|(k, rc_array)| { let rc_borrowed = rc_array.as_ref().borrow(); let (pos, neg) = rc_borrowed.deref(); - (k, pos.iter().sum::() as i32 - neg.iter().sum::() as i32) + QueryResponse { + key: k, + value: pos.iter().sum::() as i64 - neg.iter().sum::() as i64 + } }).collect::>() }) - -> map(serialize_to_bytes::<(u64, i32)>) + -> map(serialize_to_bytes::) -> dest_sink(query_responses); }; diff --git a/hydro_cli_examples/examples/pn_counter_delta/main.rs b/topolotree/src/pn_delta.rs similarity index 83% rename from hydro_cli_examples/examples/pn_counter_delta/main.rs rename to topolotree/src/pn_delta.rs index 249f795d70e..3b3e6a7b1db 100644 --- a/hydro_cli_examples/examples/pn_counter_delta/main.rs +++ b/topolotree/src/pn_delta.rs @@ -4,23 +4,20 @@ use std::ops::Deref; use std::rc::Rc; use hydroflow::serde::{Deserialize, Serialize}; -use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource}; +use hydroflow::util::cli::{ConnectedDemux, ConnectedDirect, ConnectedSink, ConnectedSource, ConnectedTagged}; use hydroflow::util::{deserialize_from_bytes, serialize_to_bytes}; use hydroflow::{hydroflow_syntax, tokio}; -#[derive(Serialize, Deserialize, Clone, Debug)] -struct IncrementRequest { - tweet_id: u64, - likes: i32, -} +mod protocol; +use protocol::*; #[derive(Serialize, Deserialize, Clone, Debug)] enum GossipOrIncrement { - Gossip(Vec<(u64, (usize, u32, u32))>), - Increment(u64, i32), + Gossip(Vec<(u64, (usize, u64, u64))>), + Increment(u64, i64), } -type NextStateType = (u64, bool, Rc, Vec)>>); +type NextStateType = (u64, bool, Rc, Vec)>>); #[hydroflow::main] async fn main() { @@ -51,7 +48,7 @@ async fn main() { let from_peer = ports .port("from_peer") - .connect::() + .connect::>() .await .into_source(); @@ -67,8 +64,8 @@ async fn main() { let df = hydroflow_syntax! { next_state = union() - -> fold::<'static>((HashMap::, Vec)>>>::new(), HashMap::new(), 0), |(mut cur_state, mut modified_tweets, last_tick), goi| { - if context.current_tick() != last_tick { + -> fold::<'static>((HashMap::, Vec)>>>::new(), HashMap::new(), 0), |(cur_state, modified_tweets, last_tick): &mut (HashMap<_, _>, HashMap<_, _>, _), goi| { + if context.current_tick() != *last_tick { modified_tweets.clear(); } @@ -99,16 +96,16 @@ async fn main() { let mut cur_value = cur_value.as_ref().borrow_mut(); if delta > 0 { - cur_value.0[my_id] += delta as u32; + cur_value.0[my_id] += delta as u64; } else { - cur_value.1[my_id] += (-delta) as u32; + cur_value.1[my_id] += (-delta) as u64; } *modified_tweets.entry(counter_id).or_insert(false) |= true; } } - (cur_state, modified_tweets, context.current_tick()) + *last_tick = context.current_tick(); }) -> filter(|(_, _, tick)| *tick == context.current_tick()) -> filter(|(_, modified_tweets, _)| !modified_tweets.is_empty()) @@ -116,12 +113,12 @@ async fn main() { -> tee(); source_stream(from_peer) - -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) + -> map(|x| deserialize_from_bytes::(&x.unwrap().1).unwrap()) -> next_state; source_stream(increment_requests) - -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) - -> map(|t| GossipOrIncrement::Increment(t.tweet_id, t.likes)) + -> map(|x| deserialize_from_bytes::(&x.unwrap()).unwrap()) + -> map(|t| GossipOrIncrement::Increment(t.key, t.change)) -> next_state; all_peers = source_iter(0..num_replicas) @@ -144,10 +141,13 @@ async fn main() { a.into_iter().map(|(k, _, rc_array)| { let rc_borrowed = rc_array.as_ref().borrow(); let (pos, neg) = rc_borrowed.deref(); - (k, pos.iter().sum::() as i32 - neg.iter().sum::() as i32) + QueryResponse { + key: k, + value: pos.iter().sum::() as i64 - neg.iter().sum::() as i64 + } }).collect::>() }) - -> map(serialize_to_bytes::<(u64, i32)>) + -> map(serialize_to_bytes::) -> dest_sink(query_responses); }; diff --git a/topolotree/src/protocol.rs b/topolotree/src/protocol.rs new file mode 100644 index 00000000000..69199c080e9 --- /dev/null +++ b/topolotree/src/protocol.rs @@ -0,0 +1,27 @@ +use std::fmt::Debug; + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] +pub struct Timestamped { + pub timestamp: isize, + pub data: T, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] +pub struct Payload { + pub key: u64, + pub contents: Timestamped +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct OperationPayload { + pub key: u64, + pub change: i64, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] +pub struct QueryResponse { + pub key: u64, + pub value: i64 +} diff --git a/topolotree/src/tests.rs b/topolotree/src/tests.rs new file mode 100644 index 00000000000..93f05c23810 --- /dev/null +++ b/topolotree/src/tests.rs @@ -0,0 +1,381 @@ +use std::io; + +use hydroflow::bytes::{Bytes, BytesMut}; +use hydroflow::tokio_stream::wrappers::UnboundedReceiverStream; +use hydroflow::util::multiset::HashMultiSet; +use hydroflow::util::{collect_ready_async, unbounded_channel, deserialize_from_bytes, serialize_to_bytes}; +use tokio::sync::mpsc::error::SendError; +use tokio::sync::mpsc::UnboundedSender; + +use crate::protocol::{Timestamped, QueryResponse}; +use crate::{run_topolotree, OperationPayload, Payload}; + +pub fn simulate_input( + input_send: &mut UnboundedSender>, + (id, payload): (u32, Payload), +) -> Result<(), SendError>> { + input_send.send(Ok(( + id, + BytesMut::from(&serialize_to_bytes(&payload)[..]), + ))) +} + +pub fn simulate_operation( + input_send: &mut UnboundedSender>, + payload: OperationPayload, +) -> Result<(), SendError>> { + input_send.send(Ok(BytesMut::from(&serialize_to_bytes(&payload)[..]))) +} + +pub async fn read_all( + mut output_recv: &mut UnboundedReceiverStream<(u32, Bytes)>, +) -> HashMultiSet<(u32, Payload)> { + let collected = collect_ready_async::, _>(&mut output_recv).await; + collected + .iter() + .map(|(id, bytes)| { + ( + *id, + deserialize_from_bytes::>(&bytes[..]).unwrap(), + ) + }) + .collect::>() +} + +pub async fn read_all_query( + mut output_recv: &mut UnboundedReceiverStream, +) -> HashMultiSet { + let collected = collect_ready_async::, _>(&mut output_recv).await; + collected + .iter() + .map(|bytes| { + deserialize_from_bytes::(&bytes[..]).unwrap() + }) + .collect::>() +} + +#[hydroflow::test] +async fn simple_payload_test() { + let neighbors: Vec = vec![1, 2, 3]; + + let (_operations_tx, operations_rx) = unbounded_channel::>(); + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + #[rustfmt::skip] + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } })).unwrap(); + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (2, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 2 } + ])); +} + +#[hydroflow::test] +async fn idempotence_test() { + let neighbors: Vec = vec![1, 2, 3]; + let (_operations_tx, operations_rx) = unbounded_channel::>(); + + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 2 } })).unwrap(); + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 2 } })).unwrap(); + }; + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (2, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 2 } + ])); +} + +#[hydroflow::test] +async fn backwards_in_time_test() { + let neighbors: Vec = vec![1, 2, 3]; + + let (_operations_tx, operations_rx) = unbounded_channel::>(); + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 5, data: 7 } })).unwrap(); + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 2 } })).unwrap(); + }; + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (2, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 7 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 7 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 7 } + ])); +} + +#[hydroflow::test] +async fn multiple_input_sources_test() { + let neighbors: Vec = vec![1, 2, 3]; + let (_operations_tx, operations_rx) = unbounded_channel::>(); + + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 5, data: 7 } })).unwrap(); + simulate_input(&mut input_send, (2, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 2 } })).unwrap(); + }; + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 2 } }), + (2, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 7 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 9 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 9 } + ])); +} + +#[hydroflow::test] +async fn operations_across_ticks() { + let neighbors: Vec = vec![1, 2, 3]; + + let (mut operations_tx, operations_rx) = unbounded_channel::>(); + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 2 } })).unwrap(); + simulate_operation(&mut operations_tx, OperationPayload { key: 123, change: 5 }).unwrap(); + simulate_operation(&mut operations_tx, OperationPayload { key: 123, change: 7 }).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 3, data: 12 } }), + (2, Payload { key: 123, contents: Timestamped { timestamp: 3, data: 14 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 3, data: 14 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 14 } + ])); + + #[rustfmt::skip] + { + simulate_operation(&mut operations_tx, OperationPayload { key: 123, change: 1 }).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 13 } }), + (2, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 15 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 4, data: 15 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 15 } + ])); +} + +#[hydroflow::test] +async fn operations_multiple_keys() { + let neighbors: Vec = vec![1, 2, 3]; + + let (mut operations_tx, operations_rx) = unbounded_channel::>(); + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + #[rustfmt::skip] + { + simulate_operation(&mut operations_tx, OperationPayload { key: 123, change: 5 }).unwrap(); + simulate_operation(&mut operations_tx, OperationPayload { key: 456, change: 7 }).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 5 } }), + (2, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 5 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 5 } }), + + (1, Payload { key: 456, contents: Timestamped { timestamp: 1, data: 7 } }), + (2, Payload { key: 456, contents: Timestamped { timestamp: 1, data: 7 } }), + (3, Payload { key: 456, contents: Timestamped { timestamp: 1, data: 7 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 5 }, + QueryResponse { key: 456, value: 7 } + ])); + + #[rustfmt::skip] + { + simulate_operation(&mut operations_tx, OperationPayload { key: 123, change: 1 }).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 6 } }), + (2, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 6 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 6 } }) + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 6 } + ])); + + #[rustfmt::skip] + { + simulate_operation(&mut operations_tx, OperationPayload { key: 456, change: 2 }).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 456, contents: Timestamped { timestamp: 2, data: 9 } }), + (2, Payload { key: 456, contents: Timestamped { timestamp: 2, data: 9 } }), + (3, Payload { key: 456, contents: Timestamped { timestamp: 2, data: 9 } }) + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 456, value: 9 } + ])); +} + +#[hydroflow::test] +async fn gossip_multiple_keys() { + let neighbors: Vec = vec![1, 2, 3]; + + let (mut operations_tx, operations_rx) = unbounded_channel::>(); + let (mut input_send, input_recv) = unbounded_channel::>(); + let (output_send, mut output_recv) = unbounded_channel::<(u32, Bytes)>(); + let (query_send, mut query_recv) = unbounded_channel::(); + + let mut flow = run_topolotree(neighbors, input_recv, operations_rx, output_send, query_send); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (1, Payload { key: 123, contents: Timestamped { timestamp: 0, data: 5 } })).unwrap(); + simulate_input(&mut input_send, (2, Payload { key: 456, contents: Timestamped { timestamp: 0, data: 7 } })).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (2, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 5 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 1, data: 5 } }), + + (1, Payload { key: 456, contents: Timestamped { timestamp: 1, data: 7 } }), + (3, Payload { key: 456, contents: Timestamped { timestamp: 1, data: 7 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 5 }, + QueryResponse { key: 456, value: 7 }, + ])); + + #[rustfmt::skip] + { + simulate_input(&mut input_send, (2, Payload { key: 123, contents: Timestamped { timestamp: 0, data: 5 } })).unwrap(); + simulate_input(&mut input_send, (3, Payload { key: 456, contents: Timestamped { timestamp: 0, data: 7 } })).unwrap(); + }; + + flow.run_tick(); + + #[rustfmt::skip] + assert_eq!(read_all(&mut output_recv).await, HashMultiSet::from_iter([ + (1, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 5 } }), + (3, Payload { key: 123, contents: Timestamped { timestamp: 2, data: 10 } }), + + (1, Payload { key: 456, contents: Timestamped { timestamp: 2, data: 14 } }), + (2, Payload { key: 456, contents: Timestamped { timestamp: 2, data: 7 } }), + ])); + + #[rustfmt::skip] + assert_eq!(read_all_query(&mut query_recv).await, HashMultiSet::from_iter([ + QueryResponse { key: 123, value: 10 }, + QueryResponse { key: 456, value: 14 }, + ])); +} + +// idempotence test (issue two requests with the same timestamp and see that they don't change anything.) +// let input1 = (1, Payload {timestamp:4, data:2}); +// let input2 = (1, Payload {timestamp:4, data:2}); +// let output1: (u32, Payload) = (2, Payload {timestamp:1, data:2}); +// let output2: (u32, Payload) = (3, Payload {timestamp:1, data:2}); +// +// backward in time test (issue two requests, the second one with an earlier timestamp than the first. ) +// let input1 = (1, Payload {timestamp:5, data:7}); +// let input2 = (1, Payload {timestamp:4, data:2}); +// let output1: (u32, Payload) = (2, Payload {timestamp:1, data:7}); +// let output2: (u32, Payload) = (3, Payload {timestamp:1, data:7}); +// +// updates from multiple sources test +// let input1 = (1, Payload {timestamp:5, data:7}); +// let input2 = (2, Payload {timestamp:4, data:2}); +// let output1: (u32, Payload) = (1, Payload {timestamp:2, data:2}); +// let output2: (u32, Payload) = (2, Payload {timestamp:2, data:7}); +// let output3: (u32, Payload) = (3, Payload {timestamp:2, data:9}); diff --git a/topolotree/topolotree_latency.hydro.py b/topolotree/topolotree_latency.hydro.py new file mode 100644 index 00000000000..cda984202fd --- /dev/null +++ b/topolotree/topolotree_latency.hydro.py @@ -0,0 +1,352 @@ +import asyncio +from codecs import decode +from typing import List, Optional +import hydro +import json +from pathlib import Path +from aiostream import stream + +import pandas as pd +import numpy as np +import uuid + + +# given a list of IDs for each node in a binary tree, +# organize the nodes into a binary tree and compute the neighbors +# of each node +def get_neighbors_in_binary_tree(flat: List[int]) -> List[List[int]]: + tree = [] + for i in range(len(flat)): + tree.append([]) + if i > 0: + # add the parent + tree[i].append((i - 1) // 2) + if 2 * i + 1 < len(flat): + # add the left child + tree[i].append(2 * i + 1) + if 2 * i + 2 < len(flat): + # add the right child + tree[i].append(2 * i + 2) + return tree + +def get_leaves_in_binary_tree(flat: List[int]) -> List[int]: + tree = get_neighbors_in_binary_tree(flat) + leaves = [] + for i in range(len(tree)): + if len(tree[i]) == 1: + leaves.append(i) + return leaves + + +async def run_experiment( + deployment: hydro.Deployment, + localhost_machine: hydro.LocalhostHost, + profile, + machine_pool, + experiment_id, + summaries_file, + tree_arg, + depth_arg, + clients_arg, + is_gcp, + gcp_vpc, +): + tree_depth = int(depth_arg) + is_tree = tree_arg == "topolo" # or "pn" + + num_replicas = 2**tree_depth - 1 + + num_clients = int(clients_arg) + + print(f"Launching benchmark with protocol {tree_arg}, {num_replicas} replicas, and {num_clients} clients") + + currently_deployed = [] + + def create_machine(): + if len(machine_pool) > 0: + ret = machine_pool.pop() + currently_deployed.append(ret) + return ret + else: + if is_gcp: + out = deployment.GCPComputeEngineHost( + project="hydro-chrisdouglas", + machine_type="n2-standard-2", + image="debian-cloud/debian-11", + region="us-west1-a", + network=gcp_vpc, + ) + else: + out = localhost_machine + currently_deployed.append(out) + return out + + all_nodes = [] + neighbors = get_neighbors_in_binary_tree(list(range(num_replicas))) + cluster = [ + deployment.HydroflowCrate( + src=str( + Path(__file__).parent.absolute() + ), + profile=profile, + bin="topolotree", + args=[str(neighbor) for neighbor in neighbors[i]], + on=create_machine(), + ) if is_tree else deployment.HydroflowCrate( + src=str( + Path(__file__).parent.absolute() + ), + profile=profile, + bin="pn" if tree_arg == "pn" else "pn_delta", + args=[json.dumps([i]), json.dumps([num_replicas])], + on=create_machine(), + ) + for i in range(num_replicas) + ] + + for i in range(num_replicas): + cluster[i].ports.to_peer.tagged(i).send_to( + hydro.demux( + { + j: cluster[j].ports.from_peer.merge() + for j in range(num_replicas) + } + ) + ) + + all_nodes = cluster + + if is_tree: + leaves = get_leaves_in_binary_tree(list(range(num_replicas))) + source = cluster[leaves[0]] + dest = cluster[leaves[-1]] + else: + source = cluster[0] + dest = cluster[-1] + + for node in all_nodes: + if node is not dest: + node.ports.query_responses.send_to(hydro.null()) + if node is not source: + hydro.null().send_to(node.ports.increment_requests) + + latency_measurer = deployment.HydroflowCrate( + src=str(Path(__file__).parent.absolute()), + profile=profile, + bin="latency_measure", + args=[json.dumps([num_clients])], + on=create_machine(), + ) + + latency_measurer.ports.increment_start_node.send_to( + source.ports.increment_requests.merge() + ) + dest.ports.query_responses.send_to(latency_measurer.ports.end_node_query) + + await deployment.deploy() + + print("Deployed!") + + latency = [] + memory_per_node = [[] for _ in range(num_replicas)] + throughput_raw = [] + + throughput = [] + + latency_stdout = await latency_measurer.stdout() + + memories_streams_with_index = [ + stream.map(await node.stdout(), lambda x, i=i: (i, x)) + for i, node in enumerate(all_nodes) + ] + + async def memory_plotter(): + try: + async with stream.merge(*memories_streams_with_index).stream() as merged: + async for node_idx, line in merged: + line_split = line.split(",") + if line_split[0] == "memory": + memory_per_node[node_idx].append(int(line_split[1])) + except asyncio.CancelledError: + return + + memory_plotter_task = asyncio.create_task(memory_plotter()) + + async def latency_plotter(): + try: + async for line in latency_stdout: + line_split = line.split(",") + if line_split[0] == "throughput": + count = int(line_split[1]) + period = float(line_split[2]) + throughput_raw.append([count, period]) + throughput.append(count / period) + elif line_split[0] == "latency": + number = int(line_split[1]) # microseconds + latency.append(number) + except asyncio.CancelledError: + return + + latency_plotter_task = asyncio.create_task(latency_plotter()) + + await deployment.start() + print("Started! Please wait 30 seconds to collect data.") + + await asyncio.sleep(30) + + await latency_measurer.stop() + await asyncio.gather(*[node.stop() for node in all_nodes]) + + memory_plotter_task.cancel() + await memory_plotter_task + + latency_plotter_task.cancel() + await latency_plotter_task + + def summarize(v, kind): + print("mean = ", np.mean(v)) + print("std = ", np.std(v)) + print("min = ", np.min(v)) + print("max = ", np.max(v)) + print("percentile 99 = ", np.percentile(v, 99)) + print("percentile 75 = ", np.percentile(v, 75)) + print("percentile 50 = ", np.percentile(v, 50)) + print("percentile 25 = ", np.percentile(v, 25)) + print("percentile 1 = ", np.percentile(v, 1)) + + summaries_file.write("\n") + summaries_file.write(tree_arg + ",") + summaries_file.write(str(tree_depth) + ",") + summaries_file.write(str(num_clients) + ",") + summaries_file.write(kind + ",") + summaries_file.write(str(np.mean(v)) + ",") + summaries_file.write(str(np.std(v)) + ",") + summaries_file.write(str(np.min(v)) + ",") + summaries_file.write(str(np.max(v)) + ",") + summaries_file.write(str(np.percentile(v, 99)) + ",") + summaries_file.write(str(np.percentile(v, 75)) + ",") + summaries_file.write(str(np.percentile(v, 50)) + ",") + summaries_file.write(str(np.percentile(v, 25)) + ",") + summaries_file.write(str(np.percentile(v, 1))) + summaries_file.flush() + + print("latency:") + summarize(latency, "latency") + + print("throughput:") + summarize(throughput, "throughput") + + init_memory = [memory[0] for memory in memory_per_node] + print("init memory:") + summarize(init_memory, "init_memory") + + final_memory = [memory[-1] for memory in memory_per_node] + print("final memory:") + summarize(final_memory, "final_memory") + + pd.DataFrame(latency).to_csv( + "latency_" + + tree_arg + + "_tree_depth_" + + str(tree_depth) + + "_num_clients_" + + str(num_clients) + + "_" + + experiment_id + + ".csv", + index=False, + header=["latency"], + ) + pd.DataFrame(throughput_raw).to_csv( + "throughput_" + + tree_arg + + "_tree_depth_" + + str(tree_depth) + + "_num_clients_" + + str(num_clients) + + "_" + + experiment_id + + ".csv", + index=False, + header=["count", "period"], + ) + pd.DataFrame(init_memory).to_csv( + "init_memory_" + + tree_arg + + "_tree_depth_" + + str(tree_depth) + + "_num_clients_" + + str(num_clients) + + "_" + + experiment_id + + ".csv", + index=False, + header=["memory"], + ) + pd.DataFrame(final_memory).to_csv( + "final_memory_" + + tree_arg + + "_tree_depth_" + + str(tree_depth) + + "_num_clients_" + + str(num_clients) + + "_" + + experiment_id + + ".csv", + index=False, + header=["memory"], + ) + + for machine in currently_deployed: + machine_pool.append(machine) + + +# hydro deploy toplotree_latency.hydro.py -- local/gcp once/pn/pn_counter_delta DEPTH_OF_TREE NUM_CLIENTS +async def main(args): + # the current timestamp + import datetime + + experiment_id = str(datetime.datetime.now()) + + summaries_file = open(f"summaries_{experiment_id}.csv", "w") + summaries_file.write( + "protocol,tree_depth,num_clients,kind,mean,std,min,max,percentile_99,percentile_75,percentile_50,percentile_25,percentile_1" + ) + + deployment = hydro.Deployment() + localhost = deployment.Localhost() + pool = [] + + network = ( + hydro.GCPNetwork( + project="hydro-chrisdouglas", + ) + if args[0] == "gcp" + else None + ) + + for depth_arg in args[2].split(","): + for tree_arg in args[1].split(","): + for num_clients_arg in args[3].split(","): + await run_experiment( + deployment, + localhost, + "dev" if args[0] == "local" else None, + pool, + experiment_id, + summaries_file, + tree_arg, + depth_arg, + num_clients_arg, + args[0] == "gcp", + network, + ) + + summaries_file.close() + + +if __name__ == "__main__": + import sys + import hydro.async_wrapper + + hydro.async_wrapper.run(main, sys.argv[1:]) diff --git a/website_playground/Cargo.toml b/website_playground/Cargo.toml index f9f8af2023a..c7f5fc537f9 100644 --- a/website_playground/Cargo.toml +++ b/website_playground/Cargo.toml @@ -16,7 +16,7 @@ hydroflow_datalog_core = { path = "../hydroflow_datalog_core" } hydroflow_lang = { path = "../hydroflow_lang" } hydroflow = { path = "../hydroflow" } prettyplease = "0.2.0" -proc-macro2 = "1.0" +proc-macro2 = "1.0.57" quote = "1.0" serde = { version = "1.0", features = ["derive"] } serde-wasm-bindgen = "0.4" diff --git a/website_playground/src/lib.rs b/website_playground/src/lib.rs index 3e96ab7e06c..bbf61f43e9d 100644 --- a/website_playground/src/lib.rs +++ b/website_playground/src/lib.rs @@ -1,6 +1,7 @@ mod utils; use std::cell::RefCell; use std::collections::HashMap; +use std::path::PathBuf; use std::task::{Context, Poll}; use std::thread_local; @@ -107,7 +108,8 @@ pub struct HydroflowOutput { pub fn compile_hydroflow(program: String) -> JsValue { let out = match syn::parse_str(&program) { Ok(input) => { - let (graph_code_opt, diagnostics) = build_hfcode(input, "e!(hydroflow)); + let (graph_code_opt, diagnostics) = + build_hfcode(input, "e!(hydroflow), PathBuf::default()); let output = graph_code_opt.map(|(graph, code)| { let mermaid = graph.to_mermaid(); let file = syn::parse_quote! { @@ -149,7 +151,12 @@ pub fn compile_datalog(program: String) -> JsValue { let mut diagnostics = Vec::new(); let output = match partition_graph(flat_graph) { Ok(part_graph) => { - let out = part_graph.as_code("e!(hydroflow), true, &mut diagnostics); + let out = part_graph.as_code( + "e!(hydroflow), + true, + quote!(), + &mut diagnostics, + ); let file: syn::File = syn::parse_quote! { fn main() { #out