From bf1555d25e14c814bb1a97b0444debde31db6f7e Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Sat, 9 Nov 2024 16:20:50 -0800 Subject: [PATCH] internal: Add graph utils and rework project/task graphs. (#1711) * Add graph utils. * Polish. * Add utils to tg. * Clean up builders. * Add task data. * Add deps resolving. * Add to session. * Polish. * Fix ID resolving. * Add changelog. * Fix lints. * Fix tests. * Start on tests. * Add final tests. --- CHANGELOG.md | 14 + Cargo.lock | 21 + .../action-graph/src/action_graph_builder.rs | 12 +- .../tasks-ci-mismatch/ci/moon.yml | 12 + .../tests/__fixtures__/tasks/ci/moon.yml | 27 +- .../action-graph/tests/action_graph_test.rs | 75 +-- ...__runs_dependents_if_both_are_ci_true.snap | 2 +- crates/affected/src/affected.rs | 2 +- crates/affected/src/affected_tracker.rs | 14 +- crates/app/Cargo.toml | 1 + crates/app/src/commands/ci.rs | 11 - crates/app/src/commands/docker/scaffold.rs | 8 +- crates/app/src/commands/graph/project.rs | 3 +- crates/app/src/commands/graph/utils.rs | 4 +- crates/app/src/commands/project.rs | 2 +- crates/app/src/components.rs | 1 - crates/app/src/session.rs | 12 + crates/config/src/project/dep_config.rs | 11 + crates/config/src/project/task_config.rs | 10 + crates/graph-utils/Cargo.toml | 15 + crates/graph-utils/src/graph_formats.rs | 56 ++ crates/graph-utils/src/graph_traits.rs | 78 +++ crates/graph-utils/src/lib.rs | 5 + crates/notifier/Cargo.toml | 1 - crates/project-builder/src/project_builder.rs | 55 +- .../project-expander/src/expander_context.rs | 3 - .../project-expander/src/project_expander.rs | 2 +- crates/project-expander/src/tasks_expander.rs | 5 +- .../src/tasks_expander_new.rs | 214 ++++++ crates/project-expander/tests/utils.rs | 1 - crates/project-graph/Cargo.toml | 1 + crates/project-graph/src/lib.rs | 1 + crates/project-graph/src/project_graph.rs | 278 +++----- .../project-graph/tests/project_graph_test.rs | 88 ++- crates/project/src/project.rs | 10 + crates/task-builder/src/lib.rs | 2 + crates/task-builder/src/task_deps_builder.rs | 161 +++++ crates/task-builder/src/tasks_builder.rs | 30 +- .../task-builder/src/tasks_builder_error.rs | 63 ++ .../tests/task_deps_builder_test.rs | 634 ++++++++++++++++++ .../task-builder/tests/tasks_builder_test.rs | 7 +- crates/task-graph/Cargo.toml | 5 + crates/task-graph/src/lib.rs | 2 + crates/task-graph/src/task_graph.rs | 110 ++- crates/task-graph/src/task_graph_error.rs | 11 + crates/task/src/task.rs | 7 + crates/test-utils/src/workspace_mocker.rs | 3 +- crates/workspace/Cargo.toml | 3 + .../{project_build_data.rs => build_data.rs} | 52 ++ crates/workspace/src/lib.rs | 15 +- crates/workspace/src/tasks_querent.rs | 49 ++ crates/workspace/src/workspace_builder.rs | 242 +++++-- crates/workspace/src/workspace_cache.rs | 2 +- 53 files changed, 1954 insertions(+), 499 deletions(-) create mode 100644 crates/action-graph/tests/__fixtures__/tasks-ci-mismatch/ci/moon.yml create mode 100644 crates/graph-utils/Cargo.toml create mode 100644 crates/graph-utils/src/graph_formats.rs create mode 100644 crates/graph-utils/src/graph_traits.rs create mode 100644 crates/graph-utils/src/lib.rs create mode 100644 crates/project-expander/src/tasks_expander_new.rs create mode 100644 crates/task-builder/src/task_deps_builder.rs create mode 100644 crates/task-builder/tests/task_deps_builder_test.rs create mode 100644 crates/task-graph/src/task_graph_error.rs rename crates/workspace/src/{project_build_data.rs => build_data.rs} (51%) create mode 100644 crates/workspace/src/tasks_querent.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c3552c17c..92f8f07888b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased + +#### 💥 Breaking + +- If you renamed a project using the `id` setting in `moon.yml`, you can no longer reference that + project in dependencies and targets using its original ID. + +#### 🚀 Updates + +- Resolved the `strictProjectIds` experiment and you can no longer reference the original ID. +- Resolved the `disallowRunInCiMismatch` experiment and you can no longer have a CI based task + depend on a non-CI based task. +- Added a new task graph, that enables new granular based functionality for task related features. + ## 1.29.4 #### 🚀 Updates diff --git a/Cargo.lock b/Cargo.lock index 4e77999a743..845c04c9ee1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3022,6 +3022,7 @@ dependencies = [ "moon_rust_tool", "moon_system_platform", "moon_task", + "moon_task_graph", "moon_tool", "moon_toolchain", "moon_toolchain_plugin", @@ -3428,6 +3429,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "moon_graph_utils" +version = "0.0.1" +dependencies = [ + "miette", + "petgraph", + "rustc-hash 2.0.0", + "serde", + "starbase_utils", +] + [[package]] name = "moon_hash" version = "0.0.1" @@ -3735,6 +3747,7 @@ dependencies = [ "moon_cache", "moon_common", "moon_config", + "moon_graph_utils", "moon_project", "moon_project_expander", "moon_query", @@ -3914,11 +3927,16 @@ name = "moon_task_graph" version = "0.0.1" dependencies = [ "miette", + "moon_common", + "moon_config", + "moon_graph_utils", "moon_target", + "moon_task", "petgraph", "rustc-hash 2.0.0", "serde", "starbase_utils", + "thiserror", "tracing", ] @@ -4174,6 +4192,9 @@ dependencies = [ "moon_project_builder", "moon_project_constraints", "moon_project_graph", + "moon_task", + "moon_task_builder", + "moon_task_graph", "moon_vcs", "petgraph", "rustc-hash 2.0.0", diff --git a/crates/action-graph/src/action_graph_builder.rs b/crates/action-graph/src/action_graph_builder.rs index 30f9442c058..b21e42b65db 100644 --- a/crates/action-graph/src/action_graph_builder.rs +++ b/crates/action-graph/src/action_graph_builder.rs @@ -10,7 +10,7 @@ use moon_common::{color, Id}; use moon_config::{PlatformType, TaskDependencyConfig}; use moon_platform::{PlatformManager, Runtime}; use moon_project::{Project, ProjectError}; -use moon_project_graph::ProjectGraph; +use moon_project_graph::{GraphConnections, ProjectGraph}; use moon_query::{build_query, Criteria}; use moon_task::{Target, TargetError, TargetLocator, TargetScope, Task}; use moon_task_args::parse_task_args; @@ -458,8 +458,8 @@ impl<'app> ActionGraphBuilder<'app> { projects_to_build.push(self_project.clone()); // From other projects - for dependent_id in self.project_graph.dependents_of(&self_project)? { - projects_to_build.push(self.project_graph.get(dependent_id)?); + for dependent_id in self.project_graph.dependents_of(&self_project) { + projects_to_build.push(self.project_graph.get(&dependent_id)?); } for project in projects_to_build { @@ -672,12 +672,12 @@ impl<'app> ActionGraphBuilder<'app> { let mut edges = vec![setup_tool_index]; // And we should also depend on other projects - for dep_project_id in self.project_graph.dependencies_of(project)? { - if cycle.contains(dep_project_id) { + for dep_project_id in self.project_graph.dependencies_of(project) { + if cycle.contains(&dep_project_id) { continue; } - let dep_project = self.project_graph.get(dep_project_id)?; + let dep_project = self.project_graph.get(&dep_project_id)?; let dep_project_index = self.internal_sync_project(&dep_project, cycle)?; if index != dep_project_index { diff --git a/crates/action-graph/tests/__fixtures__/tasks-ci-mismatch/ci/moon.yml b/crates/action-graph/tests/__fixtures__/tasks-ci-mismatch/ci/moon.yml new file mode 100644 index 00000000000..7740cea1504 --- /dev/null +++ b/crates/action-graph/tests/__fixtures__/tasks-ci-mismatch/ci/moon.yml @@ -0,0 +1,12 @@ +tasks: + # Note: No longer allowed! + ci1-dependency: + inputs: + - 'input.txt' + options: + runInCI: false + ci1-dependent: + deps: + - ci1-dependency + options: + runInCI: true diff --git a/crates/action-graph/tests/__fixtures__/tasks/ci/moon.yml b/crates/action-graph/tests/__fixtures__/tasks/ci/moon.yml index 8bd8b544c71..bbea134a14a 100644 --- a/crates/action-graph/tests/__fixtures__/tasks/ci/moon.yml +++ b/crates/action-graph/tests/__fixtures__/tasks/ci/moon.yml @@ -1,21 +1,22 @@ tasks: - ci1-dependency: - inputs: - - 'input.txt' - options: - runInCI: false - ci1-dependant: - deps: - - ci1-dependency - options: - runInCI: true + # Note: No longer allowed! + # ci1-dependency: + # inputs: + # - 'input.txt' + # options: + # runInCI: false + # ci1-dependent: + # deps: + # - ci1-dependency + # options: + # runInCI: true ci2-dependency: inputs: - 'input.txt' options: runInCI: false - ci2-dependant: + ci2-dependent: deps: - ci2-dependency options: @@ -26,7 +27,7 @@ tasks: - 'input.txt' options: runInCI: true - ci3-dependant: + ci3-dependent: deps: - ci2-dependency options: @@ -37,7 +38,7 @@ tasks: - 'input.txt' options: runInCI: true - ci4-dependant: + ci4-dependent: deps: - ci4-dependency options: diff --git a/crates/action-graph/tests/action_graph_test.rs b/crates/action-graph/tests/action_graph_test.rs index 4e5b60f5e4e..76e72fa379d 100644 --- a/crates/action-graph/tests/action_graph_test.rs +++ b/crates/action-graph/tests/action_graph_test.rs @@ -1117,38 +1117,39 @@ mod action_graph { assert!(!topo(graph).is_empty()); } - #[tokio::test] - async fn runs_dependents_if_dependency_is_ci_false_but_affected() { - let sandbox = create_sandbox("tasks"); - let container = ActionGraphContainer::new(sandbox.path()).await; - let mut builder = container.create_builder(); - - let project = container.project_graph.get("ci").unwrap(); - let task = project.get_task("ci1-dependency").unwrap(); - - // Must be affected to run the dependent - let touched_files = - FxHashSet::from_iter([WorkspaceRelativePathBuf::from("ci/input.txt")]); - - builder.set_touched_files(&touched_files).unwrap(); - - builder - .run_task( - &project, - task, - &RunRequirements { - ci: true, - ci_check: true, - dependents: true, - ..RunRequirements::default() - }, - ) - .unwrap(); - - let graph = builder.build(); - - assert_snapshot!(graph.to_dot()); - } + // TODO: Enable after new task graph! + // #[tokio::test] + // async fn runs_dependents_if_dependency_is_ci_false_but_affected() { + // let sandbox = create_sandbox("tasks"); + // let container = ActionGraphContainer::new(sandbox.path()).await; + // let mut builder = container.create_builder(); + + // let project = container.project_graph.get("ci").unwrap(); + // let task = project.get_task("ci2-dependency").unwrap(); + + // // Must be affected to run the dependent + // let touched_files = + // FxHashSet::from_iter([WorkspaceRelativePathBuf::from("ci/input.txt")]); + + // builder.set_touched_files(&touched_files).unwrap(); + + // builder + // .run_task( + // &project, + // task, + // &RunRequirements { + // ci: true, + // ci_check: true, + // dependents: true, + // ..RunRequirements::default() + // }, + // ) + // .unwrap(); + + // let graph = builder.build(); + + // assert_snapshot!(graph.to_dot()); + // } #[tokio::test] async fn doesnt_run_dependents_if_dependency_is_ci_false_and_not_affected() { @@ -1157,7 +1158,7 @@ mod action_graph { let mut builder = container.create_builder(); let project = container.project_graph.get("ci").unwrap(); - let task = project.get_task("ci1-dependency").unwrap(); + let task = project.get_task("ci2-dependency").unwrap(); builder .run_task( @@ -1206,15 +1207,11 @@ mod action_graph { #[tokio::test] #[should_panic( - expected = "Task ci:ci1-dependant cannot depend on task ci:ci1-dependency" + expected = "Task ci:ci1-dependent cannot depend on task ci:ci1-dependency" )] async fn errors_if_dependency_is_ci_false_and_constraint_enabled() { - env::set_var("MOON_INTERNAL_CONSTRAINT_RUNINCI", "true"); - - let sandbox = create_sandbox("tasks"); + let sandbox = create_sandbox("tasks-ci-mismatch"); ActionGraphContainer::new(sandbox.path()).await; - - env::remove_var("MOON_INTERNAL_CONSTRAINT_RUNINCI"); } } } diff --git a/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__run_in_ci__runs_dependents_if_both_are_ci_true.snap b/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__run_in_ci__runs_dependents_if_both_are_ci_true.snap index 04163b48e8a..6b50d15c3ef 100644 --- a/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__run_in_ci__runs_dependents_if_both_are_ci_true.snap +++ b/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__run_in_ci__runs_dependents_if_both_are_ci_true.snap @@ -7,7 +7,7 @@ digraph { 1 [ label="SetupToolchain(system)" ] 2 [ label="SyncProject(system, ci)" ] 3 [ label="RunTask(ci:ci4-dependency)" ] - 4 [ label="RunTask(ci:ci4-dependant)" ] + 4 [ label="RunTask(ci:ci4-dependent)" ] 1 -> 0 [ ] 2 -> 1 [ ] 3 -> 2 [ ] diff --git a/crates/affected/src/affected.rs b/crates/affected/src/affected.rs index af1dce2565d..8442e483a71 100644 --- a/crates/affected/src/affected.rs +++ b/crates/affected/src/affected.rs @@ -6,7 +6,7 @@ use rustc_hash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; use std::fmt; -#[derive(PartialEq)] +#[derive(Debug, PartialEq)] pub enum AffectedBy { AlreadyMarked, AlwaysAffected, diff --git a/crates/affected/src/affected_tracker.rs b/crates/affected/src/affected_tracker.rs index ff4e093fca4..3d358af1a60 100644 --- a/crates/affected/src/affected_tracker.rs +++ b/crates/affected/src/affected_tracker.rs @@ -2,7 +2,7 @@ use crate::affected::*; use moon_common::path::WorkspaceRelativePathBuf; use moon_common::{color, Id}; use moon_project::Project; -use moon_project_graph::ProjectGraph; +use moon_project_graph::{GraphConnections, ProjectGraph}; use moon_task::{Target, TargetScope, Task}; use rustc_hash::{FxHashMap, FxHashSet}; use std::env; @@ -198,9 +198,9 @@ impl<'app> AffectedTracker<'app> { } } - for dep_id in self.project_graph.dependencies_of(project)? { + for dep_id in self.project_graph.dependencies_of(project) { self.projects - .entry(dep_id.to_owned()) + .entry(dep_id.clone()) .or_default() .push(AffectedBy::DownstreamProject(project.id.clone())); @@ -208,7 +208,7 @@ impl<'app> AffectedTracker<'app> { continue; } - let dep_project = self.project_graph.get(dep_id)?; + let dep_project = self.project_graph.get(&dep_id)?; self.track_project_dependencies(&dep_project, depth + 1)?; } @@ -240,9 +240,9 @@ impl<'app> AffectedTracker<'app> { } } - for dep_id in self.project_graph.dependents_of(project)? { + for dep_id in self.project_graph.dependents_of(project) { self.projects - .entry(dep_id.to_owned()) + .entry(dep_id.clone()) .or_default() .push(AffectedBy::UpstreamProject(project.id.clone())); @@ -250,7 +250,7 @@ impl<'app> AffectedTracker<'app> { continue; } - let dep_project = self.project_graph.get(dep_id)?; + let dep_project = self.project_graph.get(&dep_id)?; self.track_project_dependents(&dep_project, depth + 1)?; } diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index e4095646c8e..f6328d78c71 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -27,6 +27,7 @@ moon_project = { path = "../project" } moon_project_graph = { path = "../project-graph" } moon_query = { path = "../query" } moon_task = { path = "../task" } +moon_task_graph = { path = "../task-graph" } moon_toolchain = { path = "../toolchain" } moon_toolchain_plugin = { path = "../toolchain-plugin" } moon_vcs = { path = "../vcs" } diff --git a/crates/app/src/commands/ci.rs b/crates/app/src/commands/ci.rs index b309d795a42..1440e509e6d 100644 --- a/crates/app/src/commands/ci.rs +++ b/crates/app/src/commands/ci.rs @@ -14,7 +14,6 @@ use moon_task::{Target, TargetLocator}; use rustc_hash::FxHashSet; use starbase::AppResult; use starbase_styles::color; -use std::env; use std::sync::Arc; use tracing::instrument; @@ -85,16 +84,6 @@ impl CiConsole { } async fn generate_project_graph(session: &CliSession) -> AppResult> { - // We have no easy way of passing this experiment boolean into the - // project graph, so use an environment variable for now... - if session - .workspace_config - .experiments - .disallow_run_in_ci_mismatch - { - env::set_var("MOON_INTERNAL_CONSTRAINT_RUNINCI", "true"); - } - session.get_project_graph().await } diff --git a/crates/app/src/commands/docker/scaffold.rs b/crates/app/src/commands/docker/scaffold.rs index f174319704a..e0e2a01557b 100644 --- a/crates/app/src/commands/docker/scaffold.rs +++ b/crates/app/src/commands/docker/scaffold.rs @@ -5,7 +5,7 @@ use clap::Args; use moon_common::consts::*; use moon_common::{path, Id}; use moon_config::LanguageType; -use moon_project_graph::ProjectGraph; +use moon_project_graph::{GraphConnections, ProjectGraph}; use moon_rust_lang::cargo_toml::{CargoTomlCache, CargoTomlExt}; use moon_toolchain::detect::detect_language_files; use rustc_hash::FxHashSet; @@ -371,9 +371,9 @@ async fn scaffold_sources( } // Include non-focused projects in the manifest - for project_id in project_graph.ids() { - if !manifest.focused_projects.contains(project_id) { - manifest.unfocused_projects.insert(project_id.to_owned()); + for project_id in project_graph.get_node_keys() { + if !manifest.focused_projects.contains(&project_id) { + manifest.unfocused_projects.insert(project_id); } } diff --git a/crates/app/src/commands/graph/project.rs b/crates/app/src/commands/graph/project.rs index faed0da6f89..baa1b01b6d0 100644 --- a/crates/app/src/commands/graph/project.rs +++ b/crates/app/src/commands/graph/project.rs @@ -2,6 +2,7 @@ use super::utils::{project_graph_repr, respond_to_request, setup_server}; use crate::session::CliSession; use clap::Args; use moon_common::Id; +use moon_project_graph::{GraphToDot, GraphToJson}; use starbase::AppResult; use starbase_styles::color; use std::sync::Arc; @@ -27,7 +28,7 @@ pub async fn project_graph(session: CliSession, args: ProjectGraphArgs) -> AppRe let mut project_graph = session.get_project_graph().await?; if let Some(id) = &args.id { - project_graph = Arc::new(project_graph.into_focused(id, args.dependents)?); + project_graph = Arc::new(project_graph.focus_for(id, args.dependents)?); } // Force expand all projects diff --git a/crates/app/src/commands/graph/utils.rs b/crates/app/src/commands/graph/utils.rs index 93bdb4a2a66..25ddece3e02 100644 --- a/crates/app/src/commands/graph/utils.rs +++ b/crates/app/src/commands/graph/utils.rs @@ -1,7 +1,7 @@ use super::dto::{GraphEdgeDto, GraphInfoDto, GraphNodeDto}; use miette::IntoDiagnostic; use moon_action_graph::ActionGraph; -use moon_project_graph::ProjectGraph; +use moon_project_graph::{GraphConversions, ProjectGraph}; use petgraph::{graph::NodeIndex, Graph}; use rustc_hash::FxHashMap; use serde::Serialize; @@ -80,7 +80,7 @@ pub fn extract_nodes_and_edges_from_graph( /// Get a serialized representation of the project graph. pub async fn project_graph_repr(project_graph: &ProjectGraph) -> GraphInfoDto { - let labeled_graph = project_graph.labeled_graph(); + let labeled_graph = project_graph.to_labeled_graph(); extract_nodes_and_edges_from_graph(&labeled_graph, true) } diff --git a/crates/app/src/commands/project.rs b/crates/app/src/commands/project.rs index 6f49b42de54..bb8e2aa3d1a 100644 --- a/crates/app/src/commands/project.rs +++ b/crates/app/src/commands/project.rs @@ -20,7 +20,7 @@ pub async fn project(session: CliSession, args: ProjectArgs) -> AppResult { let project_graph = session .get_project_graph() .await? - .into_focused(&args.id, false)?; + .focus_for(&args.id, false)?; let project = project_graph.get(&args.id)?; let config = &project.config; diff --git a/crates/app/src/components.rs b/crates/app/src/components.rs index 3b73ed34c31..dc1bfb94800 100644 --- a/crates/app/src/components.rs +++ b/crates/app/src/components.rs @@ -61,7 +61,6 @@ pub async fn create_workspace_graph_context( extend_project: Emitter::::new(), extend_project_graph: Emitter::::new(), inherited_tasks: &session.tasks_config, - strict_project_ids: session.workspace_config.experiments.strict_project_ids, toolchain_config: &session.toolchain_config, vcs: Some(session.get_vcs_adapter()?), working_dir: &session.working_dir, diff --git a/crates/app/src/session.rs b/crates/app/src/session.rs index 7e91869ab24..9d294cb69d5 100644 --- a/crates/app/src/session.rs +++ b/crates/app/src/session.rs @@ -15,6 +15,7 @@ use moon_env::MoonEnvironment; use moon_extension_plugin::*; use moon_plugin::{PluginHostData, PluginId}; use moon_project_graph::ProjectGraph; +use moon_task_graph::TaskGraph; use moon_toolchain_plugin::*; use moon_vcs::{BoxedVcs, Git}; use moon_workspace::WorkspaceBuilder; @@ -45,6 +46,7 @@ pub struct CliSession { cache_engine: OnceCell>, extension_registry: OnceCell>, project_graph: OnceCell>, + task_graph: OnceCell>, toolchain_registry: OnceCell>, vcs_adapter: OnceCell>, @@ -72,6 +74,7 @@ impl CliSession { moon_env: Arc::new(MoonEnvironment::default()), project_graph: OnceCell::new(), proto_env: Arc::new(ProtoEnvironment::default()), + task_graph: OnceCell::new(), tasks_config: Arc::new(InheritedTasksManager::default()), toolchain_config: Arc::new(ToolchainConfig::default()), toolchain_registry: OnceCell::new(), @@ -144,6 +147,14 @@ impl CliSession { Ok(self.project_graph.get().map(Arc::clone).unwrap()) } + pub async fn get_task_graph(&self) -> AppResult> { + if self.task_graph.get().is_none() { + self.load_workspace_graph().await?; + } + + Ok(self.task_graph.get().map(Arc::clone).unwrap()) + } + pub async fn get_toolchain_registry(&self) -> AppResult> { let project_graph = self.get_project_graph().await?; @@ -205,6 +216,7 @@ impl CliSession { let result = builder.build().await?; let _ = self.project_graph.set(Arc::new(result.project_graph)); + let _ = self.task_graph.set(Arc::new(result.task_graph)); Ok(()) } diff --git a/crates/config/src/project/dep_config.rs b/crates/config/src/project/dep_config.rs index f4dd56b48ac..bb429f5f642 100644 --- a/crates/config/src/project/dep_config.rs +++ b/crates/config/src/project/dep_config.rs @@ -1,6 +1,17 @@ use moon_common::{cacheable, Id}; use schematic::{derive_enum, Config, ConfigEnum}; +derive_enum!( + /// The task-to-task relationship of the dependency. + #[derive(ConfigEnum, Copy, Default)] + pub enum DependencyType { + Cleanup, + #[default] + Required, + Optional, + } +); + derive_enum!( /// The scope and or relationship of the dependency. #[derive(ConfigEnum, Copy, Default)] diff --git a/crates/config/src/project/task_config.rs b/crates/config/src/project/task_config.rs index 8b66fee743e..8ecf15a39ac 100644 --- a/crates/config/src/project/task_config.rs +++ b/crates/config/src/project/task_config.rs @@ -128,6 +128,16 @@ impl TaskDependencyConfig { ..Default::default() } } + + pub fn optional(mut self) -> Self { + self.optional = Some(true); + self + } + + pub fn required(mut self) -> Self { + self.optional = Some(false); + self + } } cacheable!( diff --git a/crates/graph-utils/Cargo.toml b/crates/graph-utils/Cargo.toml new file mode 100644 index 00000000000..4665be28087 --- /dev/null +++ b/crates/graph-utils/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "moon_graph_utils" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +miette = { workspace = true } +petgraph = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_utils = { workspace = true } + +[lints] +workspace = true diff --git a/crates/graph-utils/src/graph_formats.rs b/crates/graph-utils/src/graph_formats.rs new file mode 100644 index 00000000000..b26725d0109 --- /dev/null +++ b/crates/graph-utils/src/graph_formats.rs @@ -0,0 +1,56 @@ +use crate::graph_traits::*; +use petgraph::dot::{Config, Dot}; +use petgraph::graph::DiGraph; +use petgraph::visit::{EdgeRef, NodeRef}; +use serde::Serialize; +use starbase_utils::json; +use std::fmt::{Debug, Display}; + +#[derive(Serialize)] +pub struct GraphCache<'graph, N, E> { + graph: &'graph DiGraph, + // data: &'graph FxHashMap, +} + +pub trait GraphToDot: + GraphData +{ + /// Format graph as a DOT string. + fn to_dot(&self) -> String { + let dot = Dot::with_attr_getters( + self.get_graph(), + &[Config::EdgeNoLabel, Config::NodeNoLabel], + &|_, e| { + let label = e.weight().to_string(); + + if e.source().index() == 0 { + format!("label=\"{label}\" arrowhead=none") + } else { + format!("label=\"{label}\" arrowhead=box, arrowtail=box") + } + }, + &|_, n| { + let label = n.weight().to_string(); + + format!( + "label=\"{label}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black" + ) + }, + ); + + format!("{dot:?}") + } +} + +pub trait GraphToJson: GraphData { + /// Format graph as a JSON string. + fn to_json(&self) -> miette::Result { + Ok(json::format( + &GraphCache { + graph: self.get_graph(), + // data: self.get_nodes(), + }, + true, + )?) + } +} diff --git a/crates/graph-utils/src/graph_traits.rs b/crates/graph-utils/src/graph_traits.rs new file mode 100644 index 00000000000..ce087a239fa --- /dev/null +++ b/crates/graph-utils/src/graph_traits.rs @@ -0,0 +1,78 @@ +use petgraph::graph::{DiGraph, NodeIndex}; +use petgraph::Direction; +use std::fmt::Display; + +pub trait GraphData { + fn get_graph(&self) -> &DiGraph; + fn get_node_index(&self, node: &N) -> NodeIndex; + fn get_node_key(&self, node: &N) -> K; +} + +pub trait GraphConnections: GraphData { + /// Return a list of node keys that the provided node depends on. + fn dependencies_of(&self, node: &N) -> Vec { + let graph = self.get_graph(); + + graph + .neighbors_directed(self.get_node_index(node), Direction::Outgoing) + .map(|idx| self.get_node_key(graph.node_weight(idx).unwrap())) + .collect() + } + + /// Return a list of node keys that require the provided node. + fn dependents_of(&self, node: &N) -> Vec { + let graph = self.get_graph(); + + graph + .neighbors_directed(self.get_node_index(node), Direction::Incoming) + .map(|idx| self.get_node_key(graph.node_weight(idx).unwrap())) + .collect() + } + + /// Return a list of keys for all nodes currently within the graph. + fn get_node_keys(&self) -> Vec { + self.get_graph() + .raw_nodes() + .iter() + .map(|n| self.get_node_key(&n.weight)) + .collect() + } +} + +pub trait GraphConversions: + GraphConnections +{ + /// Return the graph with display labels. + fn to_labeled_graph(&self) -> DiGraph { + self.get_graph() + .map(|_, node| node.to_string(), |_, edge| edge.to_string()) + } + + /// Return the graph focused for the provided node, and only include direct + /// dependents or dependencies. + fn to_focused_graph(&self, focus_node: &N, with_dependents: bool) -> DiGraph { + let upstream = self.dependencies_of(focus_node); + let downstream = self.dependents_of(focus_node); + let focus_key = self.get_node_key(focus_node); + + self.get_graph().filter_map( + |_, node| { + let node_key = self.get_node_key(node); + + if + // Self + node_key == focus_key || + // Dependencies + upstream.contains(&node_key) || + // Dependents + with_dependents && downstream.contains(&node_key) + { + Some(node.to_owned()) + } else { + None + } + }, + |_, edge| Some(edge.to_owned()), + ) + } +} diff --git a/crates/graph-utils/src/lib.rs b/crates/graph-utils/src/lib.rs new file mode 100644 index 00000000000..36af8ae1fe4 --- /dev/null +++ b/crates/graph-utils/src/lib.rs @@ -0,0 +1,5 @@ +mod graph_formats; +mod graph_traits; + +pub use graph_formats::*; +pub use graph_traits::*; diff --git a/crates/notifier/Cargo.toml b/crates/notifier/Cargo.toml index c3e98c310d4..a77d48064af 100644 --- a/crates/notifier/Cargo.toml +++ b/crates/notifier/Cargo.toml @@ -16,6 +16,5 @@ tokio = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } - [lints] workspace = true diff --git a/crates/project-builder/src/project_builder.rs b/crates/project-builder/src/project_builder.rs index 1e701df095b..32baa73b7cd 100644 --- a/crates/project-builder/src/project_builder.rs +++ b/crates/project-builder/src/project_builder.rs @@ -11,10 +11,9 @@ use moon_task::{TargetScope, Task}; use moon_task_builder::{TasksBuilder, TasksBuilderContext}; use moon_toolchain::detect::{detect_project_language, detect_project_platform}; use rustc_hash::FxHashMap; -use std::borrow::Cow; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use tracing::{debug, instrument, trace}; +use tracing::{instrument, trace}; pub struct ProjectBuilderContext<'app> { pub config_loader: &'app ConfigLoader, @@ -32,10 +31,10 @@ pub struct ProjectBuilder<'app> { local_config: Option, // Values to be continually built - id: Cow<'app, Id>, + id: &'app Id, source: &'app WorkspaceRelativePath, alias: Option<&'app str>, - project_root: PathBuf, + root: PathBuf, pub language: LanguageType, pub platform: PlatformType, @@ -55,9 +54,9 @@ impl<'app> ProjectBuilder<'app> { ); Ok(ProjectBuilder { - project_root: source.to_logical_path(context.workspace_root), + root: source.to_logical_path(context.workspace_root), context, - id: Cow::Borrowed(id), + id, source, alias: None, global_config: None, @@ -102,7 +101,7 @@ impl<'app> ProjectBuilder<'app> { pub async fn inherit_local_config(&mut self, config: &ProjectConfig) -> miette::Result<()> { // Use configured language or detect from environment self.language = if config.language == LanguageType::Unknown { - let mut language = detect_project_language(&self.project_root); + let mut language = detect_project_language(&self.root); if language == LanguageType::Unknown { language = config.language.clone(); @@ -122,7 +121,7 @@ impl<'app> ProjectBuilder<'app> { // Use configured platform or infer from language self.platform = config.platform.unwrap_or_else(|| { let platform = detect_project_platform( - &self.project_root, + &self.root, &self.language, &self.context.toolchain_config.get_enabled_platforms(), ); @@ -137,21 +136,6 @@ impl<'app> ProjectBuilder<'app> { platform }); - // Inherit the custom ID - if let Some(new_id) = &config.id { - if new_id != self.id.as_ref() { - debug!( - old_id = self.id.as_str(), - new_id = new_id.as_str(), - "Project has been configured with an explicit identifier of {}, renaming from {}", - color::id(new_id), - color::id(self.id.as_str()), - ); - - self.id = Cow::Owned(new_id.to_owned()); - } - } - self.local_config = Some(config.to_owned()); Ok(()) @@ -160,19 +144,7 @@ impl<'app> ProjectBuilder<'app> { /// Load a `moon.*` config file from the root of the project (derived from source). #[instrument(skip_all)] pub async fn load_local_config(&mut self) -> miette::Result<()> { - debug!( - project_id = self.id.as_str(), - "Attempting to load {} (optional)", - color::file( - self.source - .join(self.context.config_loader.get_debug_label("moon", false)) - ) - ); - - let config = self - .context - .config_loader - .load_project_config(&self.project_root)?; + let config = self.context.config_loader.load_project_config(&self.root)?; self.inherit_local_config(&config).await?; @@ -222,11 +194,12 @@ impl<'app> ProjectBuilder<'app> { alias: self.alias.map(|a| a.to_owned()), dependencies: self.build_dependencies(&tasks)?, file_groups: self.build_file_groups()?, + task_ids: tasks.keys().cloned().collect(), tasks, - id: self.id.into_owned(), + id: self.id.to_owned(), language: self.language, platform: self.platform, - root: self.project_root, + root: self.root, source: self.source.to_owned(), ..Project::default() }; @@ -271,7 +244,7 @@ impl<'app> ProjectBuilder<'app> { if let TargetScope::Project(dep_id) = &task_dep.target.scope { // Already a dependency, or references self if deps.contains_key(dep_id) - || self.id.as_ref() == dep_id + || self.id == dep_id || self.alias.as_ref().is_some_and(|a| *a == dep_id.as_str()) { continue; @@ -364,8 +337,8 @@ impl<'app> ProjectBuilder<'app> { trace!(id = self.id.as_str(), "Building tasks"); let mut tasks_builder = TasksBuilder::new( - self.id.as_ref(), - self.source.as_str(), + self.id, + self.source, &self.platform, TasksBuilderContext { monorepo: self.context.monorepo, diff --git a/crates/project-expander/src/expander_context.rs b/crates/project-expander/src/expander_context.rs index 0c857bc9d64..52f10886f2f 100644 --- a/crates/project-expander/src/expander_context.rs +++ b/crates/project-expander/src/expander_context.rs @@ -10,9 +10,6 @@ pub struct ExpanderContext<'graph, 'query> { /// Mapping of aliases to their project IDs. pub aliases: FxHashMap<&'graph str, &'graph Id>, - /// Check `runInCI` relationships are legitimate. - pub check_ci_relationships: bool, - /// The base unexpanded project. pub project: &'graph Project, diff --git a/crates/project-expander/src/project_expander.rs b/crates/project-expander/src/project_expander.rs index 4e7e17ce10d..8c16bce47db 100644 --- a/crates/project-expander/src/project_expander.rs +++ b/crates/project-expander/src/project_expander.rs @@ -18,7 +18,7 @@ impl<'graph, 'query> ProjectExpander<'graph, 'query> { } #[instrument(name = "expand_project", skip_all)] - pub fn expand(&mut self) -> miette::Result { + pub fn expand(mut self) -> miette::Result { // Clone before expanding! let mut project = self.context.project.to_owned(); diff --git a/crates/project-expander/src/tasks_expander.rs b/crates/project-expander/src/tasks_expander.rs index fd0ae13940e..8b89649aae5 100644 --- a/crates/project-expander/src/tasks_expander.rs +++ b/crates/project-expander/src/tasks_expander.rs @@ -108,10 +108,7 @@ impl<'graph, 'query> TasksExpander<'graph, 'query> { } // Do not depend on tasks that can't run in CI - if self.context.check_ci_relationships - && !dep_task.options.run_in_ci - && task.options.run_in_ci - { + if !dep_task.options.run_in_ci && task.options.run_in_ci { return Err(TasksExpanderError::RunInCiDepRequirement { dep: dep_task.target.to_owned(), task: task.target.to_owned(), diff --git a/crates/project-expander/src/tasks_expander_new.rs b/crates/project-expander/src/tasks_expander_new.rs new file mode 100644 index 00000000000..de6f5b276ec --- /dev/null +++ b/crates/project-expander/src/tasks_expander_new.rs @@ -0,0 +1,214 @@ +use crate::expander_context::*; +use crate::tasks_expander_error::TasksExpanderError; +use crate::token_expander::TokenExpander; +use moon_config::TaskArgs; +use moon_task::Task; +use moon_task_args::parse_task_args; +use rustc_hash::FxHashMap; +use std::mem; +use tracing::{instrument, trace, warn}; + +pub struct TasksExpander<'graph, 'query> { + pub context: &'graph ExpanderContext<'graph, 'query>, + pub token: TokenExpander<'graph, 'query>, +} + +impl<'graph, 'query> TasksExpander<'graph, 'query> { + pub fn new(context: &'graph ExpanderContext<'graph, 'query>) -> Self { + Self { + token: TokenExpander::new(context), + context, + } + } + + #[instrument(skip_all)] + pub fn expand_command(&mut self, task: &mut Task) -> miette::Result<()> { + trace!( + target = task.target.as_str(), + command = &task.command, + "Expanding tokens and variables in command" + ); + + task.command = self.token.expand_command(task)?; + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_script(&mut self, task: &mut Task) -> miette::Result<()> { + trace!( + target = task.target.as_str(), + script = task.script.as_ref(), + "Expanding tokens and variables in script" + ); + + task.script = Some(self.token.expand_script(task)?); + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_args(&mut self, task: &mut Task) -> miette::Result<()> { + if task.args.is_empty() { + return Ok(()); + } + + trace!( + target = task.target.as_str(), + args = ?task.args, + "Expanding tokens and variables in args", + ); + + task.args = self.token.expand_args(task)?; + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_deps(&mut self, task: &mut Task) -> miette::Result<()> { + if task.deps.is_empty() { + return Ok(()); + } + + trace!( + target = task.target.as_str(), + deps = ?task.deps.iter().map(|d| d.target.as_str()).collect::>(), + "Expanding tokens and variables in deps args and env", + ); + + let mut deps = mem::take(&mut task.deps); + + for dep in deps.iter_mut() { + let dep_args = self + .token + .expand_args_with_task(task, &parse_task_args(&dep.args)?)?; + let dep_env = self.token.expand_env_with_task(task, &dep.env)?; + + dep.args = if dep_args.is_empty() { + TaskArgs::None + } else { + TaskArgs::List(dep_args) + }; + dep.env = substitute_env_vars(dep_env); + } + + task.deps = deps; + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_env(&mut self, task: &mut Task) -> miette::Result<()> { + trace!( + target = task.target.as_str(), + env = ?task.env, + "Expanding environment variables" + ); + + let mut env = self.token.expand_env(task)?; + + // Load variables from an .env file + if let Some(env_files) = &task.options.env_files { + let env_paths = env_files + .iter() + .map(|file| { + file.to_workspace_relative(self.context.project.source.as_str()) + .to_path(self.context.workspace_root) + }) + .collect::>(); + + trace!( + target = task.target.as_str(), + env_files = ?env_paths, + "Loading environment variables from .env files", + ); + + let mut missing_paths = vec![]; + let mut merged_env_vars = FxHashMap::default(); + + // The file may not have been committed, so avoid crashing + for env_path in env_paths { + if env_path.exists() { + let handle_error = |error: dotenvy::Error| TasksExpanderError::InvalidEnvFile { + path: env_path.to_path_buf(), + error: Box::new(error), + }; + + for line in dotenvy::from_path_iter(&env_path).map_err(handle_error)? { + let (key, val) = line.map_err(handle_error)?; + + // Overwrite previous values + merged_env_vars.insert(key, val); + } + } else { + missing_paths.push(env_path); + } + } + + // Don't override task-level variables + for (key, val) in merged_env_vars { + env.entry(key).or_insert(val); + } + } + + task.env = substitute_env_vars(env); + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_inputs(&mut self, task: &mut Task) -> miette::Result<()> { + if task.inputs.is_empty() { + return Ok(()); + } + + trace!( + target = task.target.as_str(), + inputs = ?task.inputs.iter().map(|d| d.as_str()).collect::>(), + "Expanding inputs into file system paths" + ); + + // Expand inputs to file system paths and environment variables + let result = self.token.expand_inputs(task)?; + + task.input_env.extend(result.env); + task.input_files.extend(result.files); + task.input_globs.extend(result.globs); + + Ok(()) + } + + #[instrument(skip_all)] + pub fn expand_outputs(&mut self, task: &mut Task) -> miette::Result<()> { + if task.outputs.is_empty() { + return Ok(()); + } + + trace!( + target = task.target.as_str(), + outputs = ?task.outputs.iter().map(|d| d.as_str()).collect::>(), + "Expanding outputs into file system paths" + ); + + // Expand outputs to file system paths + let result = self.token.expand_outputs(task)?; + + // Aggregate paths first before globbing, as they are literal + for file in result.files { + // Outputs must *not* be considered an input, + // so if there's an input that matches an output, + // remove it! Is there a better way to do this? + task.input_files.remove(&file); + task.output_files.insert(file); + } + + // Aggregate globs second so we can match against the paths + for glob in result.globs { + // Same treatment here! + task.input_globs.remove(&glob); + task.output_globs.insert(glob); + } + + Ok(()) + } +} diff --git a/crates/project-expander/tests/utils.rs b/crates/project-expander/tests/utils.rs index 095372b1f67..c0ca6af07f2 100644 --- a/crates/project-expander/tests/utils.rs +++ b/crates/project-expander/tests/utils.rs @@ -16,7 +16,6 @@ pub fn create_context<'g, 'q>( ) -> ExpanderContext<'g, 'q> { ExpanderContext { aliases: FxHashMap::default(), - check_ci_relationships: true, project, query: Box::new(|_| Ok(vec![])), workspace_root, diff --git a/crates/project-graph/Cargo.toml b/crates/project-graph/Cargo.toml index 4bad9b7b3ed..d34f3902476 100644 --- a/crates/project-graph/Cargo.toml +++ b/crates/project-graph/Cargo.toml @@ -11,6 +11,7 @@ publish = false [dependencies] moon_common = { path = "../common" } moon_config = { path = "../config" } +moon_graph_utils = { path = "../graph-utils" } moon_project = { path = "../project" } moon_project_expander = { path = "../project-expander" } moon_query = { path = "../query" } diff --git a/crates/project-graph/src/lib.rs b/crates/project-graph/src/lib.rs index 62ca884fba2..126ca3c772b 100644 --- a/crates/project-graph/src/lib.rs +++ b/crates/project-graph/src/lib.rs @@ -2,6 +2,7 @@ mod project_graph; mod project_graph_error; mod project_matcher; +pub use moon_graph_utils::*; pub use project_graph::*; pub use project_graph_error::*; pub use project_matcher::*; diff --git a/crates/project-graph/src/project_graph.rs b/crates/project-graph/src/project_graph.rs index a00515106cd..0271db69ec7 100644 --- a/crates/project-graph/src/project_graph.rs +++ b/crates/project-graph/src/project_graph.rs @@ -3,18 +3,14 @@ use crate::project_matcher::matches_criteria; use moon_common::path::{PathExt, WorkspaceRelativePathBuf}; use moon_common::{color, Id}; use moon_config::DependencyScope; +use moon_graph_utils::*; use moon_project::Project; use moon_project_expander::{ExpanderContext, ProjectExpander}; use moon_query::{build_query, Criteria}; -use petgraph::dot::{Config, Dot}; use petgraph::graph::{DiGraph, NodeIndex}; -use petgraph::visit::EdgeRef; -use petgraph::Direction; use rustc_hash::FxHashMap; use scc::HashMap; -use serde::Serialize; -use starbase_utils::env::bool_var; -use starbase_utils::json; +use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tracing::{debug, instrument}; @@ -22,25 +18,19 @@ use tracing::{debug, instrument}; pub type ProjectGraphType = DiGraph; pub type ProjectsCache = FxHashMap>; -#[derive(Serialize)] -pub struct ProjectGraphCache<'graph> { - graph: &'graph ProjectGraphType, - projects: &'graph ProjectsCache, -} - #[derive(Clone, Debug, Default)] -pub struct ProjectNode { +pub struct ProjectMetadata { pub alias: Option, pub index: NodeIndex, pub original_id: Option, pub source: WorkspaceRelativePathBuf, } -impl ProjectNode { +impl ProjectMetadata { pub fn new(index: usize) -> Self { - ProjectNode { + ProjectMetadata { index: NodeIndex::new(index), - ..ProjectNode::default() + ..ProjectMetadata::default() } } } @@ -53,8 +43,8 @@ pub struct ProjectGraph { /// Directed-acyclic graph (DAG) of non-expanded projects and their dependencies. graph: ProjectGraphType, - /// Graph node information, mapped by project ID. - nodes: FxHashMap, + /// Project metadata, mapped by project ID. + metadata: FxHashMap, /// Expanded projects, mapped by project ID. projects: Arc>, @@ -72,14 +62,14 @@ pub struct ProjectGraph { impl ProjectGraph { pub fn new( graph: ProjectGraphType, - nodes: FxHashMap, + metadata: FxHashMap, workspace_root: &Path, ) -> Self { debug!("Creating project graph"); Self { graph, - nodes, + metadata, projects: Arc::new(RwLock::new(FxHashMap::default())), working_dir: workspace_root.to_owned(), workspace_root: workspace_root.to_owned(), @@ -90,57 +80,29 @@ impl ProjectGraph { /// Return a map of aliases to their project IDs. Projects without aliases are omitted. pub fn aliases(&self) -> FxHashMap<&str, &Id> { - self.nodes + self.metadata .iter() - .filter_map(|(id, node)| node.alias.as_ref().map(|alias| (alias.as_str(), id))) + .filter_map(|(id, metadata)| metadata.alias.as_ref().map(|alias| (alias.as_str(), id))) .collect() } - /// Return a list of project IDs that the provide project depends on. - pub fn dependencies_of(&self, project: &Project) -> miette::Result> { - let deps = self - .graph - .neighbors_directed( - self.nodes.get(&project.id).unwrap().index, - Direction::Outgoing, - ) - .map(|idx| &self.graph.node_weight(idx).unwrap().id) - .collect(); - - Ok(deps) - } - - /// Return a list of project IDs that require the provided project. - pub fn dependents_of(&self, project: &Project) -> miette::Result> { - let deps = self - .graph - .neighbors_directed( - self.nodes.get(&project.id).unwrap().index, - Direction::Incoming, - ) - .map(|idx| &self.graph.node_weight(idx).unwrap().id) - .collect(); - - Ok(deps) - } - - /// Return a project with the provided name or alias from the graph. + /// Return a project with the provided ID or alias from the graph. /// If the project does not exist or has been misconfigured, return an error. #[instrument(name = "get_project", skip(self))] pub fn get(&self, id_or_alias: &str) -> miette::Result> { self.internal_get(id_or_alias) } - /// Return an unexpanded project with the provided name or alias from the graph. + /// Return an unexpanded project with the provided ID or alias from the graph. pub fn get_unexpanded(&self, id_or_alias: &str) -> miette::Result<&Project> { let id = self.resolve_id(id_or_alias); - let node = self - .nodes + let metadata = self + .metadata .get(&id) .ok_or(ProjectGraphError::UnconfiguredID(id))?; - Ok(self.graph.node_weight(node.index).unwrap()) + Ok(self.graph.node_weight(metadata.index).unwrap()) } /// Return all projects from the graph. @@ -148,7 +110,7 @@ impl ProjectGraph { pub fn get_all(&self) -> miette::Result>> { let mut all = vec![]; - for id in self.nodes.keys() { + for id in self.metadata.keys() { all.push(self.internal_get(id)?); } @@ -183,23 +145,9 @@ impl ProjectGraph { self.get(&id) } - /// Return a list of IDs for all projects currently within the graph. - pub fn ids(&self) -> Vec<&Id> { - self.graph - .raw_nodes() - .iter() - .map(|n| &n.weight.id) - .collect() - } - - /// Get a labelled representation of the graph (which can be serialized easily). - pub fn labeled_graph(&self) -> DiGraph { - self.graph.map(|_, n| n.id.to_string(), |_, e| *e) - } - /// Return all expanded projects that match the query criteria. - #[instrument(name = "query_project", skip_all)] - pub fn query<'input, Q: AsRef>>( + #[instrument(name = "query_project", skip(self))] + pub fn query<'input, Q: AsRef> + Debug>( &self, query: Q, ) -> miette::Result>> { @@ -214,55 +162,35 @@ impl ProjectGraph { /// Return a map of project IDs to their file source paths. pub fn sources(&self) -> FxHashMap<&Id, &WorkspaceRelativePathBuf> { - self.nodes + self.metadata .iter() - .map(|(id, node)| (id, &node.source)) + .map(|(id, metadata)| (id, &metadata.source)) .collect() } - pub fn into_focused(&self, id_or_alias: &Id, with_dependents: bool) -> miette::Result { + /// Focus the graph for a specific project by ID. + pub fn focus_for(&self, id_or_alias: &Id, with_dependents: bool) -> miette::Result { let project = self.get(id_or_alias)?; - let upstream = self.dependencies_of(&project)?; - let downstream = self.dependents_of(&project)?; - let mut nodes = FxHashMap::default(); - - // Create a new graph - let graph = self.graph.filter_map( - |_, node| { - let node_id = &node.id; - - if - // Self - node_id == &project.id || - // Dependencies - upstream.contains(&node_id) || - // Dependents - with_dependents && downstream.contains(&node_id) - { - Some(node.clone()) - } else { - None - } - }, - |_, edge| Some(*edge), - ); + let graph = self.to_focused_graph(&project, with_dependents); + + // Copy over metadata + let mut metadata = FxHashMap::default(); - // Copy over nodes for new_index in graph.node_indices() { let project_id = &graph[new_index].id; - if let Some(old_node) = self.nodes.get(project_id) { + if let Some(old_node) = self.metadata.get(project_id) { let mut new_node = old_node.to_owned(); new_node.index = new_index; - nodes.insert(project_id.to_owned(), new_node); + metadata.insert(project_id.to_owned(), new_node); } } Ok(Self { fs_cache: HashMap::new(), graph, - nodes, + metadata, projects: self.projects.clone(), query_cache: HashMap::new(), working_dir: self.working_dir.clone(), @@ -270,83 +198,40 @@ impl ProjectGraph { }) } - /// Format graph as a DOT string. - pub fn to_dot(&self) -> String { - let dot = Dot::with_attr_getters( - &self.graph, - &[Config::EdgeNoLabel, Config::NodeNoLabel], - &|_, e| { - let label = e.weight().to_string(); - - if e.source().index() == 0 { - format!("label=\"{label}\" arrowhead=none") - } else { - format!("label=\"{label}\" arrowhead=box, arrowtail=box") - } - }, - &|_, n| { - let label = &n.1.id; - - format!( - "label=\"{label}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black" - ) - }, - ); - - format!("{dot:?}") - } - - /// Format graph as a JSON string. - pub fn to_json(&self) -> miette::Result { - let projects = self.read_cache(); - - Ok(json::format( - &ProjectGraphCache { - graph: &self.graph, - projects: &projects, - }, - true, - )?) - } - - fn internal_get(&self, alias_or_id: &str) -> miette::Result> { - let id = self.resolve_id(alias_or_id); + fn internal_get(&self, id_or_alias: &str) -> miette::Result> { + let id = self.resolve_id(id_or_alias); // Check if the expanded project has been created, if so return it - { - if let Some(project) = self.read_cache().get(&id) { - return Ok(Arc::clone(project)); - } + if let Some(project) = self.read_cache().get(&id) { + return Ok(Arc::clone(project)); } // Otherwise expand the project and cache it with an Arc - { - let query = |input: String| { - let mut results = vec![]; - - // Don't use get() for expanded projects, since it'll overflow the - // stack trying to recursively expand projects! Using unexpanded - // dependent projects works just fine for the this entire process. - for result_id in self.internal_query(build_query(&input)?)?.iter() { - results.push(self.get_unexpanded(result_id)?); - } + let query = |input: String| { + let mut results = vec![]; + + // Don't use get() for expanded projects, since it'll overflow the + // stack trying to recursively expand projects! Using unexpanded + // dependent projects works just fine for the this entire process. + for result_id in self.internal_query(build_query(&input)?)?.iter() { + results.push(self.get_unexpanded(result_id)?); + } - Ok(results) - }; + Ok(results) + }; - let mut expander = ProjectExpander::new(ExpanderContext { - aliases: self.aliases(), - check_ci_relationships: bool_var("MOON_INTERNAL_CONSTRAINT_RUNINCI"), - project: self.get_unexpanded(&id)?, - query: Box::new(query), - workspace_root: &self.workspace_root, - }); + let expander = ProjectExpander::new(ExpanderContext { + aliases: self.aliases(), + project: self.get_unexpanded(&id)?, + query: Box::new(query), + workspace_root: &self.workspace_root, + }); - self.write_cache() - .insert(id.clone(), Arc::new(expander.expand()?)); - } + let project = Arc::new(expander.expand()?); + + self.write_cache().insert(id.clone(), Arc::clone(&project)); - Ok(Arc::clone(self.read_cache().get(&id).unwrap())) + Ok(project) } fn internal_query<'input, Q: AsRef>>( @@ -370,9 +255,7 @@ impl ProjectGraph { // Don't use `get_all` as it recursively calls `query`, // which runs into a deadlock! This should be faster also... - for node in self.graph.raw_nodes() { - let project = &node.weight; - + for project in self.get_all_unexpanded() { if matches_criteria(project, query)? { project_ids.push(project.id.clone()); } @@ -407,12 +290,12 @@ impl ProjectGraph { let mut remaining_length = 1000; // Start with a really fake number let mut possible_id = String::new(); - for (id, node) in &self.nodes { - if !search.starts_with(node.source.as_str()) { + for (id, metadata) in &self.metadata { + if !search.starts_with(metadata.source.as_str()) { continue; } - if let Ok(diff) = search.relative_to(node.source.as_str()) { + if let Ok(diff) = search.relative_to(metadata.source.as_str()) { let diff_comps = diff.components().count(); // Exact match, abort @@ -439,21 +322,22 @@ impl ProjectGraph { } fn resolve_id(&self, id_or_alias: &str) -> Id { - Id::raw(if self.nodes.contains_key(id_or_alias) { + Id::raw(if self.metadata.contains_key(id_or_alias) { id_or_alias } else { - self.nodes + self.metadata .iter() - .find(|(_, node)| { - node.alias + .find_map(|(id, metadata)| { + if metadata + .alias .as_ref() .is_some_and(|alias| alias == id_or_alias) - || node - .original_id - .as_ref() - .is_some_and(|id| id == id_or_alias) + { + Some(id.as_str()) + } else { + None + } }) - .map(|(id, _)| id.as_str()) .unwrap_or(id_or_alias) }) } @@ -470,3 +354,25 @@ impl ProjectGraph { .expect("Failed to acquire write access to project graph!") } } + +impl GraphData for ProjectGraph { + fn get_graph(&self) -> &DiGraph { + &self.graph + } + + fn get_node_index(&self, node: &Project) -> NodeIndex { + self.metadata.get(&node.id).unwrap().index + } + + fn get_node_key(&self, node: &Project) -> Id { + node.id.clone() + } +} + +impl GraphConnections for ProjectGraph {} + +impl GraphConversions for ProjectGraph {} + +impl GraphToDot for ProjectGraph {} + +impl GraphToJson for ProjectGraph {} diff --git a/crates/project-graph/tests/project_graph_test.rs b/crates/project-graph/tests/project_graph_test.rs index 5c0a0ee16ae..7afad9b360a 100644 --- a/crates/project-graph/tests/project_graph_test.rs +++ b/crates/project-graph/tests/project_graph_test.rs @@ -4,7 +4,7 @@ use moon_config::{ WorkspaceProjectsConfig, }; use moon_project::{FileGroup, Project}; -use moon_project_graph::ProjectGraph; +use moon_project_graph::*; use moon_query::build_query; use moon_task::Target; use moon_test_utils2::*; @@ -29,8 +29,8 @@ pub fn append_file>(path: P, data: &str) { writeln!(file, "\n\n{data}").unwrap(); } -fn map_ids(ids: Vec<&Id>) -> Vec { - ids.iter().map(|id| id.to_string()).collect() +fn map_ids(ids: Vec) -> Vec { + ids.into_iter().map(|id| id.to_string()).collect() } fn get_ids_from_projects(projects: Vec>) -> Vec { @@ -294,7 +294,7 @@ mod project_graph { .await; let cached_graph = do_generate(sandbox.path()).await; - assert_eq!(graph.ids(), cached_graph.ids()); + assert_eq!(graph.get_node_keys(), cached_graph.get_node_keys()); } #[tokio::test] @@ -441,17 +441,17 @@ mod project_graph { ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("a").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("a").unwrap())), ["b"] ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("b").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("b").unwrap())), ["c"] ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("c").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("c").unwrap())), string_vec![] ); } @@ -477,7 +477,15 @@ mod project_graph { let graph = generate_inheritance_project_graph("inheritance/scoped").await; assert_eq!( - map_ids(graph.get("node").unwrap().tasks.keys().collect::>()), + map_ids( + graph + .get("node") + .unwrap() + .tasks + .keys() + .cloned() + .collect::>() + ), ["global", "global-node", "node"] ); @@ -488,6 +496,7 @@ mod project_graph { .unwrap() .tasks .keys() + .cloned() .collect::>() ), [ @@ -505,6 +514,7 @@ mod project_graph { .unwrap() .tasks .keys() + .cloned() .collect::>() ), ["global", "system-library"] @@ -516,7 +526,15 @@ mod project_graph { let graph = generate_inheritance_project_graph("inheritance/tagged").await; assert_eq!( - map_ids(graph.get("mage").unwrap().tasks.keys().collect::>()), + map_ids( + graph + .get("mage") + .unwrap() + .tasks + .keys() + .cloned() + .collect::>() + ), ["mage", "magic"] ); @@ -527,6 +545,7 @@ mod project_graph { .unwrap() .tasks .keys() + .cloned() .collect::>() ), ["warrior", "weapons"] @@ -539,6 +558,7 @@ mod project_graph { .unwrap() .tasks .keys() + .cloned() .collect::>() ), ["magic", "priest", "weapons"] @@ -687,19 +707,19 @@ mod project_graph { let graph = generate_project_graph("dependencies").await; assert_eq!( - map_ids(graph.dependencies_of(&graph.get("a").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("a").unwrap())), ["b"] ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("b").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("b").unwrap())), ["c"] ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("c").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("c").unwrap())), string_vec![] ); assert_eq!( - map_ids(graph.dependencies_of(&graph.get("d").unwrap()).unwrap()), + map_ids(graph.dependencies_of(&graph.get("d").unwrap())), ["c", "b", "a"] ); } @@ -709,19 +729,19 @@ mod project_graph { let graph = generate_project_graph("dependencies").await; assert_eq!( - map_ids(graph.dependents_of(&graph.get("a").unwrap()).unwrap()), + map_ids(graph.dependents_of(&graph.get("a").unwrap())), ["d"] ); assert_eq!( - map_ids(graph.dependents_of(&graph.get("b").unwrap()).unwrap()), + map_ids(graph.dependents_of(&graph.get("b").unwrap())), ["d", "a"] ); assert_eq!( - map_ids(graph.dependents_of(&graph.get("c").unwrap()).unwrap()), + map_ids(graph.dependents_of(&graph.get("c").unwrap())), ["d", "b"] ); assert_eq!( - map_ids(graph.dependents_of(&graph.get("d").unwrap()).unwrap()), + map_ids(graph.dependents_of(&graph.get("d").unwrap())), string_vec![] ); } @@ -736,7 +756,7 @@ mod project_graph { let graph = mock.build_project_graph_for(&["no-depends-on"]).await; - assert_eq!(map_ids(graph.ids()), ["no-depends-on"]); + assert_eq!(map_ids(graph.get_node_keys()), ["no-depends-on"]); } #[tokio::test] @@ -746,7 +766,10 @@ mod project_graph { let graph = mock.build_project_graph_for(&["some-depends-on"]).await; - assert_eq!(map_ids(graph.ids()), ["a", "c", "some-depends-on"]); + assert_eq!( + map_ids(graph.get_node_keys()), + ["a", "c", "some-depends-on"] + ); } #[tokio::test] @@ -756,7 +779,7 @@ mod project_graph { let graph = mock.build_project_graph_for(&["from-task-deps"]).await; - assert_eq!(map_ids(graph.ids()), ["b", "c", "from-task-deps"]); + assert_eq!(map_ids(graph.get_node_keys()), ["b", "c", "from-task-deps"]); let deps = &graph.get("from-task-deps").unwrap().dependencies; @@ -771,7 +794,10 @@ mod project_graph { let graph = mock.build_project_graph_for(&["from-root-task-deps"]).await; - assert_eq!(map_ids(graph.ids()), ["root", "from-root-task-deps"]); + assert_eq!( + map_ids(graph.get_node_keys()), + ["root", "from-root-task-deps"] + ); let deps = &graph.get("from-root-task-deps").unwrap().dependencies; @@ -785,7 +811,7 @@ mod project_graph { let graph = mock.build_project_graph_for(&["self-task-deps"]).await; - assert_eq!(map_ids(graph.ids()), ["self-task-deps"]); + assert_eq!(map_ids(graph.get_node_keys()), ["self-task-deps"]); } } } @@ -914,29 +940,17 @@ mod project_graph { let graph = generate_aliases_project_graph().await; assert_eq!( - map_ids( - graph - .dependencies_of(&graph.get("explicit").unwrap()) - .unwrap() - ), + map_ids(graph.dependencies_of(&graph.get("explicit").unwrap())), ["alias-two", "alias-one"] ); assert_eq!( - map_ids( - graph - .dependencies_of(&graph.get("explicit-and-implicit").unwrap()) - .unwrap() - ), + map_ids(graph.dependencies_of(&graph.get("explicit-and-implicit").unwrap())), ["alias-three", "alias-two"] ); assert_eq!( - map_ids( - graph - .dependencies_of(&graph.get("implicit").unwrap()) - .unwrap() - ), + map_ids(graph.dependencies_of(&graph.get("implicit").unwrap())), ["alias-three", "alias-one"] ); } diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index 98b6aed5e73..4ad8d4787fa 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -12,6 +12,7 @@ use moon_config::{ use moon_file_group::FileGroup; use moon_task::Task; use std::collections::BTreeMap; +use std::fmt; use std::path::PathBuf; cacheable!( @@ -59,6 +60,9 @@ cacheable!( /// Tasks specific to the project. Inherits all tasks from the global config. pub tasks: BTreeMap, + /// List of IDs of all tasks configured or inherited for the project. + pub task_ids: Vec, + /// The type of project. #[serde(rename = "type")] pub type_of: ProjectType, @@ -145,3 +149,9 @@ impl PartialEq for Project { && self.type_of == other.type_of } } + +impl fmt::Display for Project { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.id) + } +} diff --git a/crates/task-builder/src/lib.rs b/crates/task-builder/src/lib.rs index 1ca4da4d144..2c6ff74fc6d 100644 --- a/crates/task-builder/src/lib.rs +++ b/crates/task-builder/src/lib.rs @@ -1,5 +1,7 @@ +mod task_deps_builder; mod tasks_builder; mod tasks_builder_error; +pub use task_deps_builder::*; pub use tasks_builder::*; pub use tasks_builder_error::*; diff --git a/crates/task-builder/src/task_deps_builder.rs b/crates/task-builder/src/task_deps_builder.rs new file mode 100644 index 00000000000..00e1e1783e0 --- /dev/null +++ b/crates/task-builder/src/task_deps_builder.rs @@ -0,0 +1,161 @@ +use crate::tasks_builder_error::TasksBuilderError; +use moon_common::Id; +use moon_config::{DependencyConfig, TaskDependencyConfig}; +use moon_task::{Target, TargetScope, Task, TaskOptions}; +use std::mem; + +pub trait TasksQuerent { + fn query_projects_by_tag(&self, tag: &str) -> miette::Result>; + fn query_tasks( + &self, + project_ids: Vec<&Id>, + task_id: &Id, + ) -> miette::Result>; +} + +pub struct TaskDepsBuilder<'proj> { + pub querent: Box, + pub project_id: &'proj Id, + pub project_dependencies: &'proj [DependencyConfig], + pub task: &'proj mut Task, +} + +impl<'proj> TaskDepsBuilder<'proj> { + pub fn build(self) -> miette::Result<()> { + let mut deps = vec![]; + + for dep_config in mem::take(&mut self.task.deps) { + let (project_ids, skip_if_missing) = match &dep_config.target.scope { + // :task + TargetScope::All => { + return Err(TasksBuilderError::UnsupportedTargetScopeInDeps { + dep: dep_config.target.to_owned(), + task: self.task.target.to_owned(), + } + .into()); + } + // ^:task + TargetScope::Deps => ( + self.project_dependencies + .iter() + .map(|dep| &dep.id) + .collect::>(), + dep_config.optional.unwrap_or(true), + ), + // ~:task + TargetScope::OwnSelf => { + (vec![self.project_id], dep_config.optional.unwrap_or(false)) + } + // id:task + TargetScope::Project(project_id) => { + (vec![project_id], dep_config.optional.unwrap_or(false)) + } + // #tag:task + TargetScope::Tag(tag) => ( + self.querent + .query_projects_by_tag(tag)? + .into_iter() + .filter(|id| *id != self.project_id) + .collect(), + dep_config.optional.unwrap_or(true), + ), + }; + + let results = self + .querent + .query_tasks(project_ids, &dep_config.target.task_id)?; + + if results.is_empty() && !skip_if_missing { + return Err(match &dep_config.target.scope { + TargetScope::Deps => TasksBuilderError::UnknownDepTargetParentScope { + dep: dep_config.target.to_owned(), + task: self.task.target.to_owned(), + } + .into(), + TargetScope::Tag(_) => TasksBuilderError::UnknownDepTargetTagScope { + dep: dep_config.target.to_owned(), + task: self.task.target.to_owned(), + } + .into(), + _ => TasksBuilderError::UnknownDepTarget { + dep: dep_config.target.to_owned(), + task: self.task.target.to_owned(), + } + .into(), + }); + } + + for (dep_task_target, dep_task_options) in results { + // Avoid circular references + if dep_task_target + .get_project_id() + .is_some_and(|id| id == self.project_id) + && dep_task_target.task_id == self.task.target.task_id + { + continue; + } + + self.check_and_push_dep( + dep_task_target, + dep_task_options, + &dep_config, + &mut deps, + skip_if_missing, + )?; + } + } + + self.task.deps = deps; + + Ok(()) + } + + fn check_and_push_dep( + &self, + dep_task_target: &Target, + dep_task_options: &TaskOptions, + dep_config: &TaskDependencyConfig, + deps_list: &mut Vec, + _skip_if_missing: bool, + ) -> miette::Result<()> { + // Do not depend on tasks that can fail + if dep_task_options.allow_failure { + return Err(TasksBuilderError::AllowFailureDepRequirement { + dep: dep_task_target.to_owned(), + task: self.task.target.to_owned(), + } + .into()); + } + + // Do not depend on tasks that can't run in CI + if !dep_task_options.run_in_ci && self.task.options.run_in_ci { + return Err(TasksBuilderError::RunInCiDepRequirement { + dep: dep_task_target.to_owned(), + task: self.task.target.to_owned(), + } + .into()); + } + + // Enforce persistent constraints + if dep_task_options.persistent && !self.task.options.persistent { + return Err(TasksBuilderError::PersistentDepRequirement { + dep: dep_task_target.to_owned(), + task: self.task.target.to_owned(), + } + .into()); + } + + // Add the dep if it has not already been + let dep = TaskDependencyConfig { + target: dep_task_target.to_owned(), + // optional: Some(skip_if_missing), + ..dep_config.clone() + }; + + if !deps_list.contains(&dep) { + deps_list.push(dep); + } + + Ok(()) + } +} diff --git a/crates/task-builder/src/tasks_builder.rs b/crates/task-builder/src/tasks_builder.rs index 77a5a88e612..0773240d500 100644 --- a/crates/task-builder/src/tasks_builder.rs +++ b/crates/task-builder/src/tasks_builder.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use crate::tasks_builder_error::TasksBuilderError; -use moon_common::path::is_root_level_source; +use moon_common::path::{is_root_level_source, WorkspaceRelativePath}; use moon_common::{color, supports_pkl_configs, Id}; use moon_config::{ is_glob_like, InheritedTasksConfig, InputPath, PlatformType, ProjectConfig, @@ -73,10 +73,10 @@ pub struct TasksBuilderContext<'proj> { pub struct TasksBuilder<'proj> { context: TasksBuilderContext<'proj>, - project_id: &'proj str, + project_id: &'proj Id, project_env: FxHashMap<&'proj str, &'proj str>, project_platform: &'proj PlatformType, - project_source: &'proj str, + project_source: &'proj WorkspaceRelativePath, // Global settings for tasks to inherit implicit_deps: Vec<&'proj TaskDependency>, @@ -92,8 +92,8 @@ pub struct TasksBuilder<'proj> { impl<'proj> TasksBuilder<'proj> { pub fn new( - project_id: &'proj str, - project_source: &'proj str, + project_id: &'proj Id, + project_source: &'proj WorkspaceRelativePath, project_platform: &'proj PlatformType, context: TasksBuilderContext<'proj>, ) -> Self { @@ -135,7 +135,7 @@ impl<'proj> TasksBuilder<'proj> { } trace!( - project_id = self.project_id, + project_id = self.project_id.as_str(), tasks = ?global_config.tasks.keys().map(|k| k.as_str()).collect::>(), "Filtering global tasks", ); @@ -214,7 +214,7 @@ impl<'proj> TasksBuilder<'proj> { } trace!( - project_id = self.project_id, + project_id = self.project_id.as_str(), tasks = ?local_config.tasks.keys().map(|k| k.as_str()).collect::>(), "Loading local tasks", ); @@ -302,8 +302,8 @@ impl<'proj> TasksBuilder<'proj> { // Aggregate all values that are inherited from the global task configs, // and should always be included in the task, regardless of merge strategy. - let global_deps = self.build_global_deps(&target)?; - let mut global_inputs = self.build_global_inputs(&target, &task.options)?; + let global_deps = self.inherit_global_deps(&target)?; + let mut global_inputs = self.inherit_global_inputs(&target, &task.options)?; // Aggregate all values that that are inherited from the project, // and should be set on the task first, so that merge strategies can be applied. @@ -311,7 +311,7 @@ impl<'proj> TasksBuilder<'proj> { task.args = self.merge_vec(task.args, args, task.options.merge_args, index, false); } - task.env = self.build_env(&target)?; + task.env = self.inherit_project_env(&target)?; // Finally build the task itself, while applying our complex merge logic! let mut configured_inputs = 0; @@ -666,11 +666,11 @@ impl<'proj> TasksBuilder<'proj> { Ok(options) } - fn build_global_deps(&self, target: &Target) -> miette::Result> { + fn inherit_global_deps(&self, target: &Target) -> miette::Result> { let global_deps = self .implicit_deps .iter() - .map(|d| (*d).to_owned().into_config()) + .map(|dep| (*dep).to_owned().into_config()) .collect::>(); if !global_deps.is_empty() { @@ -684,7 +684,7 @@ impl<'proj> TasksBuilder<'proj> { Ok(global_deps) } - fn build_global_inputs( + fn inherit_global_inputs( &self, target: &Target, options: &TaskOptions, @@ -692,7 +692,7 @@ impl<'proj> TasksBuilder<'proj> { let mut global_inputs = self .implicit_inputs .iter() - .map(|d| (*d).to_owned()) + .map(|dep| (*dep).to_owned()) .collect::>(); global_inputs.push(InputPath::WorkspaceGlob(".moon/*.yml".into())); @@ -716,7 +716,7 @@ impl<'proj> TasksBuilder<'proj> { Ok(global_inputs) } - fn build_env(&self, target: &Target) -> miette::Result> { + fn inherit_project_env(&self, target: &Target) -> miette::Result> { let env = self .project_env .iter() diff --git a/crates/task-builder/src/tasks_builder_error.rs b/crates/task-builder/src/tasks_builder_error.rs index 5902bc356a0..f60ce28eaad 100644 --- a/crates/task-builder/src/tasks_builder_error.rs +++ b/crates/task-builder/src/tasks_builder_error.rs @@ -1,9 +1,38 @@ use miette::Diagnostic; use moon_common::{Id, Style, Stylize}; +use moon_task::Target; use thiserror::Error; #[derive(Error, Debug, Diagnostic)] pub enum TasksBuilderError { + #[diagnostic(code(task_builder::dependency::no_allowed_failures))] + #[error( + "Task {} cannot depend on task {}, as it is allowed to fail, which may cause unwanted side-effects.\nA task is marked to allow failure with the {} setting.", + .task.style(Style::Label), + .dep.style(Style::Label), + "options.allowFailure".style(Style::Property), + )] + AllowFailureDepRequirement { dep: Target, task: Target }, + + #[diagnostic(code(task_builder::dependency::run_in_ci_mismatch))] + #[error( + "Task {} cannot depend on task {}, as the dependency cannot run in CI because {} is disabled. Because of this, the pipeline will not run tasks correctly.", + .task.style(Style::Label), + .dep.style(Style::Label), + "options.runInCI".style(Style::Property), + )] + RunInCiDepRequirement { dep: Target, task: Target }, + + #[diagnostic(code(task_builder::dependency::persistent_requirement))] + #[error( + "Non-persistent task {} cannot depend on persistent task {}.\nA task is marked persistent with the {} setting.\n\nIf you're looking to avoid the cache, disable {} instead.", + .task.style(Style::Label), + .dep.style(Style::Label), + "options.persistent".style(Style::Property), + "options.cache".style(Style::Property), + )] + PersistentDepRequirement { dep: Target, task: Target }, + #[diagnostic( code(task_builder::unknown_extends), help = "Has the task been renamed or excluded?" @@ -14,4 +43,38 @@ pub enum TasksBuilderError { .target_id.style(Style::Id), )] UnknownExtendsSource { source_id: Id, target_id: Id }, + + #[diagnostic(code(task_builder::unknown_target))] + #[error( + "Invalid dependency {} for {}, target does not exist.", + .dep.style(Style::Label), + .task.style(Style::Label), + )] + UnknownDepTarget { dep: Target, task: Target }, + + #[diagnostic(code(task_builder::unknown_target_in_project_deps))] + #[error( + "Invalid dependency {} for {}, no matching targets in project dependencies. Mark the dependency as {} to allow no results.", + .dep.style(Style::Label), + .task.style(Style::Label), + "optional".style(Style::Property), + )] + UnknownDepTargetParentScope { dep: Target, task: Target }, + + #[diagnostic(code(task_builder::unknown_target_in_tag))] + #[error( + "Invalid dependency {} for {}, no matching targets within this tag. Mark the dependency as {} to allow no results.", + .dep.style(Style::Label), + .task.style(Style::Label), + "optional".style(Style::Property), + )] + UnknownDepTargetTagScope { dep: Target, task: Target }, + + #[diagnostic(code(task_builder::unsupported_target_scope))] + #[error( + "Invalid dependency {} for {}. All (:) scope is not supported.", + .dep.style(Style::Label), + .task.style(Style::Label), + )] + UnsupportedTargetScopeInDeps { dep: Target, task: Target }, } diff --git a/crates/task-builder/tests/task_deps_builder_test.rs b/crates/task-builder/tests/task_deps_builder_test.rs new file mode 100644 index 00000000000..dd5fa7a974c --- /dev/null +++ b/crates/task-builder/tests/task_deps_builder_test.rs @@ -0,0 +1,634 @@ +use moon_common::Id; +use moon_config::*; +use moon_task::{Target, Task, TaskOptions}; +use moon_task_builder::{TaskDepsBuilder, TasksQuerent}; +use rustc_hash::FxHashMap; + +#[derive(Default)] +struct TestQuerent { + pub data: FxHashMap, + pub tag_ids: Vec, +} + +impl TasksQuerent for TestQuerent { + fn query_projects_by_tag(&self, _tag: &str) -> miette::Result> { + Ok(self.tag_ids.iter().collect()) + } + + fn query_tasks( + &self, + project_ids: Vec<&Id>, + task_id: &Id, + ) -> miette::Result> { + Ok(self + .data + .iter() + .filter_map(|(target, options)| { + let project_id = target.get_project_id()?; + + if &target.task_id == task_id && project_ids.contains(&project_id) { + Some((target, options)) + } else { + None + } + }) + .collect::>()) + } +} + +fn create_task() -> Task { + Task { + id: Id::raw("task"), + target: Target::new("project", "task").unwrap(), + ..Task::default() + } +} + +fn build_task_deps(task: &mut Task) { + build_task_deps_with_data(task, FxHashMap::default()); +} + +fn build_task_deps_with_data(task: &mut Task, data: FxHashMap) { + let project_id = Id::raw("project"); + let project_dependencies = vec![]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data, + tag_ids: vec![], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task, + } + .build() + .unwrap() +} + +mod task_deps_builder { + use super::*; + + #[test] + #[should_panic(expected = "Task project:task cannot depend on task project:allow-failure")] + fn errors_if_dep_on_allow_failure() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("allow-failure").unwrap(), + )); + + build_task_deps_with_data( + &mut task, + FxHashMap::from_iter([( + Target::parse("project:allow-failure").unwrap(), + TaskOptions { + allow_failure: true, + ..Default::default() + }, + )]), + ); + } + + #[test] + #[should_panic(expected = "Task project:task cannot depend on task project:no-ci")] + fn errors_if_dep_not_run_in_ci() { + let mut task = create_task(); + task.options.run_in_ci = true; + task.deps + .push(TaskDependencyConfig::new(Target::parse("no-ci").unwrap())); + + build_task_deps_with_data( + &mut task, + FxHashMap::from_iter([( + Target::parse("project:no-ci").unwrap(), + TaskOptions { + run_in_ci: false, + ..Default::default() + }, + )]), + ); + } + + #[test] + fn doesnt_errors_if_dep_run_in_ci() { + let mut task = create_task(); + task.options.run_in_ci = false; + task.deps + .push(TaskDependencyConfig::new(Target::parse("ci").unwrap())); + + build_task_deps_with_data( + &mut task, + FxHashMap::from_iter([( + Target::parse("project:ci").unwrap(), + TaskOptions { + run_in_ci: true, + ..Default::default() + }, + )]), + ); + } + + #[test] + #[should_panic(expected = "Non-persistent task project:task cannot depend on persistent task")] + fn errors_for_invalid_persistent_chain() { + let mut task = create_task(); + task.options.persistent = false; + task.deps.push(TaskDependencyConfig::new( + Target::parse("persistent").unwrap(), + )); + + build_task_deps_with_data( + &mut task, + FxHashMap::from_iter([( + Target::parse("project:persistent").unwrap(), + TaskOptions { + persistent: true, + ..Default::default() + }, + )]), + ); + } + + #[test] + fn doesnt_errors_for_valid_persistent_chain() { + let mut task = create_task(); + task.options.persistent = true; + task.deps.push(TaskDependencyConfig::new( + Target::parse("not-persistent").unwrap(), + )); + + build_task_deps_with_data( + &mut task, + FxHashMap::from_iter([( + Target::parse("project:not-persistent").unwrap(), + TaskOptions { + persistent: false, + ..Default::default() + }, + )]), + ); + } + + mod all_scope { + use super::*; + + #[test] + #[should_panic( + expected = "Invalid dependency :build for project:task. All (:) scope is not" + )] + fn errors_for_all_scope() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse(":build").unwrap())); + + build_task_deps(&mut task); + } + } + + mod parent_deps_scope { + use super::*; + + #[test] + fn no_depends_on() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("^:build").unwrap())); + + build_task_deps(&mut task); + + assert!(task.deps.is_empty()); + } + + #[test] + fn returns_each_parent_task() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("^:build").unwrap())); + + let project_id = Id::raw("project"); + let project_dependencies = vec![ + DependencyConfig::new(Id::raw("foo")), + DependencyConfig::new(Id::raw("bar")), + DependencyConfig::new(Id::raw("baz")), + DependencyConfig::new(Id::raw("qux")), + ]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([ + (Target::parse("foo:build").unwrap(), TaskOptions::default()), + (Target::parse("bar:build").unwrap(), TaskOptions::default()), + (Target::parse("baz:build").unwrap(), TaskOptions::default()), + ]), + tag_ids: vec![], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + + assert_eq!( + task.deps, + vec![ + TaskDependencyConfig::new(Target::parse("baz:build").unwrap()), + TaskDependencyConfig::new(Target::parse("foo:build").unwrap()), + TaskDependencyConfig::new(Target::parse("bar:build").unwrap()), + ] + ); + } + + #[test] + fn returns_each_parent_task_only_if_id_matches() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("^:build").unwrap())); + + let project_id = Id::raw("project"); + let project_dependencies = vec![ + DependencyConfig::new(Id::raw("foo")), + DependencyConfig::new(Id::raw("bar")), + DependencyConfig::new(Id::raw("baz")), + ]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([ + (Target::parse("foo:build").unwrap(), TaskOptions::default()), + (Target::parse("bar:test").unwrap(), TaskOptions::default()), + (Target::parse("baz:lint").unwrap(), TaskOptions::default()), + // Ignored + (Target::parse("qux:build").unwrap(), TaskOptions::default()), + ]), + tag_ids: vec![], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + + assert_eq!( + task.deps, + vec![TaskDependencyConfig::new( + Target::parse("foo:build").unwrap() + )] + ); + } + + #[test] + #[should_panic( + expected = "Invalid dependency ^:build for project:task, no matching targets" + )] + fn can_error_if_non_optional_and_no_results() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("^:build").unwrap()).required()); + + let project_id = Id::raw("project"); + let project_dependencies = vec![ + DependencyConfig::new(Id::raw("foo")), + DependencyConfig::new(Id::raw("bar")), + DependencyConfig::new(Id::raw("baz")), + ]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::default(), + tag_ids: vec![], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + } + } + + mod self_scope { + use super::*; + + fn create_project_task_data() -> FxHashMap { + FxHashMap::from_iter([ + ( + Target::parse("project:build").unwrap(), + TaskOptions::default(), + ), + ( + Target::parse("project:lint").unwrap(), + TaskOptions::default(), + ), + ( + Target::parse("project:test").unwrap(), + TaskOptions::default(), + ), + // Self + ( + Target::parse("project:task").unwrap(), + TaskOptions::default(), + ), + ]) + } + + #[test] + fn returns_sibling_task() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("~:build").unwrap())); + // Without scope + task.deps + .push(TaskDependencyConfig::new(Target::parse("lint").unwrap())); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!( + task.deps, + vec![ + TaskDependencyConfig::new(Target::parse("project:build").unwrap()), + TaskDependencyConfig::new(Target::parse("project:lint").unwrap()), + ] + ); + } + + #[test] + fn ignores_self_cycle() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("~:task").unwrap())); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!(task.deps, vec![]); + } + + #[test] + fn ignores_dupe_ids() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("~:build").unwrap())); + task.deps + .push(TaskDependencyConfig::new(Target::parse("build").unwrap())); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!( + task.deps, + vec![TaskDependencyConfig::new( + Target::parse("project:build").unwrap() + )] + ); + } + + #[test] + #[should_panic( + expected = "Invalid dependency ~:unknown for project:task, target does not exist" + )] + fn errors_if_unknown() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("~:unknown").unwrap(), + )); + + build_task_deps_with_data(&mut task, create_project_task_data()); + } + + #[test] + fn doesnt_error_if_unknown_but_optional() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("~:unknown").unwrap()).optional()); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert!(task.deps.is_empty()); + } + } + + mod id_scope { + use super::*; + + fn create_project_task_data() -> FxHashMap { + FxHashMap::from_iter([ + (Target::parse("a:build").unwrap(), TaskOptions::default()), + (Target::parse("b:lint").unwrap(), TaskOptions::default()), + (Target::parse("c:test").unwrap(), TaskOptions::default()), + // Self + ( + Target::parse("project:task").unwrap(), + TaskOptions::default(), + ), + ]) + } + + #[test] + fn returns_task() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("a:build").unwrap())); + task.deps + .push(TaskDependencyConfig::new(Target::parse("c:test").unwrap())); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!( + task.deps, + vec![ + TaskDependencyConfig::new(Target::parse("a:build").unwrap()), + TaskDependencyConfig::new(Target::parse("c:test").unwrap()), + ] + ); + } + + #[test] + fn ignores_self_cycle() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("project:task").unwrap(), + )); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!(task.deps, vec![]); + } + + #[test] + fn ignores_dupe_ids() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("a:build").unwrap())); + task.deps + .push(TaskDependencyConfig::new(Target::parse("a:build").unwrap())); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert_eq!( + task.deps, + vec![TaskDependencyConfig::new(Target::parse("a:build").unwrap())] + ); + } + + #[test] + #[should_panic( + expected = "Invalid dependency d:unknown for project:task, target does not exist" + )] + fn errors_if_unknown() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("d:unknown").unwrap(), + )); + + build_task_deps_with_data(&mut task, create_project_task_data()); + } + + #[test] + fn doesnt_error_if_unknown_but_optional() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("d:unknown").unwrap()).optional()); + + build_task_deps_with_data(&mut task, create_project_task_data()); + + assert!(task.deps.is_empty()); + } + } + + mod tag_scope { + use super::*; + + #[test] + fn no_tags() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("#pkg:build").unwrap(), + )); + + build_task_deps(&mut task); + + assert!(task.deps.is_empty()); + } + + #[test] + fn returns_each_tag_task() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("#pkg:build").unwrap(), + )); + + let project_id = Id::raw("project"); + let project_dependencies = vec![]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([ + (Target::parse("foo:build").unwrap(), TaskOptions::default()), + (Target::parse("bar:build").unwrap(), TaskOptions::default()), + (Target::parse("baz:build").unwrap(), TaskOptions::default()), + ]), + tag_ids: vec![Id::raw("foo"), Id::raw("baz")], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + + assert_eq!( + task.deps, + vec![ + TaskDependencyConfig::new(Target::parse("baz:build").unwrap()), + TaskDependencyConfig::new(Target::parse("foo:build").unwrap()), + ] + ); + } + + #[test] + fn returns_each_tag_task_only_if_id_matches() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("#pkg:build").unwrap(), + )); + + let project_id = Id::raw("project"); + let project_dependencies = vec![]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([ + (Target::parse("foo:build").unwrap(), TaskOptions::default()), + (Target::parse("bar:lint").unwrap(), TaskOptions::default()), + (Target::parse("baz:test").unwrap(), TaskOptions::default()), + ]), + tag_ids: vec![Id::raw("foo"), Id::raw("baz")], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + + assert_eq!( + task.deps, + vec![TaskDependencyConfig::new( + Target::parse("foo:build").unwrap() + ),] + ); + } + + #[test] + #[should_panic( + expected = "Invalid dependency #pkg:build for project:task, no matching targets" + )] + fn can_error_if_non_optional_and_no_results() { + let mut task = create_task(); + task.deps + .push(TaskDependencyConfig::new(Target::parse("#pkg:build").unwrap()).required()); + + let project_id = Id::raw("project"); + let project_dependencies = vec![]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([]), + tag_ids: vec![Id::raw("foo"), Id::raw("baz")], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + } + + #[test] + fn ignores_self_cycle() { + let mut task = create_task(); + task.deps.push(TaskDependencyConfig::new( + Target::parse("#pkg:task").unwrap(), + )); + + let project_id = Id::raw("project"); + let project_dependencies = vec![]; + + TaskDepsBuilder { + querent: Box::new(TestQuerent { + data: FxHashMap::from_iter([( + Target::parse("project:task").unwrap(), + TaskOptions::default(), + )]), + tag_ids: vec![Id::raw("project")], + }), + project_id: &project_id, + project_dependencies: &project_dependencies, + task: &mut task, + } + .build() + .unwrap(); + + assert!(task.deps.is_empty()); + } + } +} diff --git a/crates/task-builder/tests/tasks_builder_test.rs b/crates/task-builder/tests/tasks_builder_test.rs index d155603dd1c..ae34ae3d382 100644 --- a/crates/task-builder/tests/tasks_builder_test.rs +++ b/crates/task-builder/tests/tasks_builder_test.rs @@ -1,3 +1,4 @@ +use moon_common::path::WorkspaceRelativePathBuf; use moon_common::Id; use moon_config::*; use moon_target::Target; @@ -17,10 +18,12 @@ async fn build_tasks_with_config( monorepo: bool, ) -> BTreeMap { let platform = local_config.platform.unwrap_or_default(); + let id = Id::raw("project"); + let source = WorkspaceRelativePathBuf::from(source); let mut builder = TasksBuilder::new( - "project", - source, + &id, + &source, &platform, TasksBuilderContext { monorepo, diff --git a/crates/task-graph/Cargo.toml b/crates/task-graph/Cargo.toml index 38ceb8707c5..222b0dbf8fa 100644 --- a/crates/task-graph/Cargo.toml +++ b/crates/task-graph/Cargo.toml @@ -9,12 +9,17 @@ repository = "https://github.com/moonrepo/moon" publish = false [dependencies] +moon_common = { path = "../common" } +moon_config = { path = "../config" } +moon_graph_utils = { path = "../graph-utils" } moon_target = { path = "../target" } +moon_task = { path = "../task" } miette = { workspace = true } petgraph = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } starbase_utils = { workspace = true, features = ["json"] } +thiserror = { workspace = true } tracing = { workspace = true } [lints] diff --git a/crates/task-graph/src/lib.rs b/crates/task-graph/src/lib.rs index cc499ba1d03..08dab862663 100644 --- a/crates/task-graph/src/lib.rs +++ b/crates/task-graph/src/lib.rs @@ -1,3 +1,5 @@ mod task_graph; +mod task_graph_error; pub use task_graph::*; +pub use task_graph_error::*; diff --git a/crates/task-graph/src/task_graph.rs b/crates/task-graph/src/task_graph.rs index 811a7d84a05..f60394ab84c 100644 --- a/crates/task-graph/src/task_graph.rs +++ b/crates/task-graph/src/task_graph.rs @@ -1,92 +1,74 @@ +use moon_config::DependencyType; +use moon_graph_utils::*; use moon_target::Target; -use petgraph::dot::{Config, Dot}; +use moon_task::Task; use petgraph::graph::{DiGraph, NodeIndex}; -use petgraph::visit::EdgeRef; -use petgraph::Direction; use rustc_hash::FxHashMap; -use serde::Serialize; -use starbase_utils::json; +use std::sync::Arc; +// use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tracing::debug; -pub type GraphType = DiGraph; +pub type TaskGraphType = DiGraph; +pub type TasksCache = FxHashMap>; -#[derive(Serialize)] -pub struct TaskGraphCache<'graph> { - graph: &'graph GraphType, +#[derive(Clone, Debug, Default)] +pub struct TaskMetadata { + pub index: NodeIndex, } #[derive(Default)] pub struct TaskGraph { - /// Directed-acyclic graph (DAG) of targets and their relationships. - graph: GraphType, + /// Directed-acyclic graph (DAG) of non-expanded tasks and their relationships. + graph: TaskGraphType, - /// Mapping of task targets to graph node indices. - nodes: FxHashMap, + /// Task metadata, mapped by target. + metadata: FxHashMap, + // /// Expanded tasks, mapped by target. + // tasks: Arc>, } impl TaskGraph { - pub fn new(graph: GraphType, nodes: FxHashMap) -> Self { + pub fn new(graph: TaskGraphType, metadata: FxHashMap) -> Self { debug!("Creating task graph"); - Self { graph, nodes } + Self { + graph, + metadata, + // tasks: Arc::new(RwLock::new(FxHashMap::default())), + } } - pub fn dependencies_of(&self, target: &Target) -> miette::Result> { - let deps = self - .graph - .neighbors_directed(*self.nodes.get(target).unwrap(), Direction::Outgoing) - .map(|idx| self.graph.node_weight(idx).unwrap()) - .collect(); - - Ok(deps) - } - - pub fn dependents_of(&self, target: &Target) -> miette::Result> { - let deps = self - .graph - .neighbors_directed(*self.nodes.get(target).unwrap(), Direction::Incoming) - .map(|idx| self.graph.node_weight(idx).unwrap()) - .collect(); + // fn read_cache(&self) -> RwLockReadGuard { + // self.tasks + // .read() + // .expect("Failed to acquire read access to task graph!") + // } + + // fn write_cache(&self) -> RwLockWriteGuard { + // self.tasks + // .write() + // .expect("Failed to acquire write access to task graph!") + // } +} - Ok(deps) +impl GraphData for TaskGraph { + fn get_graph(&self) -> &DiGraph { + &self.graph } - /// Return a list of targets for all tasks currently within the graph. - pub fn targets(&self) -> Vec<&Target> { - self.graph.raw_nodes().iter().map(|n| &n.weight).collect() + fn get_node_index(&self, node: &Task) -> NodeIndex { + self.metadata.get(&node.target).unwrap().index } - /// Get a labelled representation of the graph (which can be serialized easily). - pub fn labeled_graph(&self) -> DiGraph { - self.graph.map(|_, n| n.to_string(), |_, e| *e) + fn get_node_key(&self, node: &Task) -> Target { + node.target.clone() } +} - /// Format graph as a DOT string. - pub fn to_dot(&self) -> String { - let dot = Dot::with_attr_getters( - &self.graph, - &[Config::EdgeNoLabel, Config::NodeNoLabel], - &|_, e| { - if e.source().index() == 0 { - "arrowhead=none".into() - } else { - "arrowhead=box, arrowtail=box".into() - } - }, - &|_, n| { - let label = &n.1.id; +impl GraphConnections for TaskGraph {} - format!( - "label=\"{label}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black" - ) - }, - ); +impl GraphConversions for TaskGraph {} - format!("{dot:?}") - } +impl GraphToDot for TaskGraph {} - /// Format graph as a JSON string. - pub fn to_json(&self) -> miette::Result { - Ok(json::format(&TaskGraphCache { graph: &self.graph }, true)?) - } -} +impl GraphToJson for TaskGraph {} diff --git a/crates/task-graph/src/task_graph_error.rs b/crates/task-graph/src/task_graph_error.rs new file mode 100644 index 00000000000..784c9f83c61 --- /dev/null +++ b/crates/task-graph/src/task_graph_error.rs @@ -0,0 +1,11 @@ +use miette::Diagnostic; +use moon_common::{Style, Stylize}; +use moon_task::Target; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum TaskGraphError { + #[diagnostic(code(task_graph::unknown_target))] + #[error("No task has been configured with the target {}.", .0.style(Style::Id))] + UnconfiguredTarget(Target), +} diff --git a/crates/task/src/task.rs b/crates/task/src/task.rs index 2b847edd326..2dedba531e7 100644 --- a/crates/task/src/task.rs +++ b/crates/task/src/task.rs @@ -11,6 +11,7 @@ use moon_target::Target; use once_cell::sync::OnceCell; use rustc_hash::{FxHashMap, FxHashSet}; use starbase_utils::glob; +use std::fmt; use std::path::{Path, PathBuf}; cacheable!( @@ -215,3 +216,9 @@ impl Task { self.is_build_type() || self.is_test_type() } } + +impl fmt::Display for Task { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.target) + } +} diff --git a/crates/test-utils/src/workspace_mocker.rs b/crates/test-utils/src/workspace_mocker.rs index a8e488bfdc3..9fb2bad6ddf 100644 --- a/crates/test-utils/src/workspace_mocker.rs +++ b/crates/test-utils/src/workspace_mocker.rs @@ -108,7 +108,6 @@ impl WorkspaceMocker { extend_project: Emitter::::new(), extend_project_graph: Emitter::::new(), inherited_tasks: &self.inherited_tasks, - strict_project_ids: self.workspace_config.experiments.strict_project_ids, toolchain_config: &self.toolchain_config, vcs: self.vcs.clone(), working_dir: &self.workspace_root, @@ -154,6 +153,8 @@ impl WorkspaceMocker { } } + builder.load_tasks().await.unwrap(); + let project_graph = builder.build().await.unwrap().project_graph; if options.ids.is_empty() { diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml index 4ea75dadb98..7a1a9d740cd 100644 --- a/crates/workspace/Cargo.toml +++ b/crates/workspace/Cargo.toml @@ -17,6 +17,9 @@ moon_project = { path = "../project" } moon_project_builder = { path = "../project-builder" } moon_project_constraints = { path = "../project-constraints" } moon_project_graph = { path = "../project-graph" } +moon_task = { path = "../task" } +moon_task_builder = { path = "../task-builder" } +moon_task_graph = { path = "../task-graph" } moon_vcs = { path = "../vcs" } miette = { workspace = true } petgraph = { workspace = true } diff --git a/crates/workspace/src/project_build_data.rs b/crates/workspace/src/build_data.rs similarity index 51% rename from crates/workspace/src/project_build_data.rs rename to crates/workspace/src/build_data.rs index acb37b64337..237dd51c8ad 100644 --- a/crates/workspace/src/project_build_data.rs +++ b/crates/workspace/src/build_data.rs @@ -3,6 +3,7 @@ use moon_common::Id; use moon_config::{ DependencyConfig, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, TaskConfig, }; +use moon_task::{Target, TaskOptions}; use petgraph::graph::NodeIndex; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; @@ -27,6 +28,57 @@ pub struct ProjectBuildData { pub source: WorkspaceRelativePathBuf, } +impl ProjectBuildData { + pub fn resolve_id(id_or_alias: &str, project_data: &FxHashMap) -> Id { + if project_data.contains_key(id_or_alias) { + Id::raw(id_or_alias) + } else { + match project_data.iter().find_map(|(id, build_data)| { + if build_data + .alias + .as_ref() + .is_some_and(|alias| alias == id_or_alias) + { + Some(id) + } else { + None + } + }) { + Some(project_id) => project_id.to_owned(), + None => Id::raw(id_or_alias), + } + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub struct TaskBuildData { + #[serde(skip_serializing_if = "Option::is_none")] + pub node_index: Option, + + #[serde(skip)] + pub options: TaskOptions, +} + +impl TaskBuildData { + pub fn resolve_target( + target: &Target, + project_data: &FxHashMap, + ) -> Target { + // Target may be using an alias! + let project_id = ProjectBuildData::resolve_id( + target + .get_project_id() + .expect("Target requires a fully-qualified project scope!"), + project_data, + ); + + // IDs should be valid here, so ignore the result + Target::new(&project_id, &target.task_id).expect("Failed to format target!") + } +} + // Extend the project graph with additional information. #[derive(Debug)] diff --git a/crates/workspace/src/lib.rs b/crates/workspace/src/lib.rs index 84af662dc70..ecc0f2c5f29 100644 --- a/crates/workspace/src/lib.rs +++ b/crates/workspace/src/lib.rs @@ -1,12 +1,23 @@ -mod project_build_data; +mod build_data; mod projects_locator; mod repo_type; +mod tasks_querent; mod workspace_builder; mod workspace_builder_error; mod workspace_cache; -pub use project_build_data::*; +pub use build_data::*; pub use repo_type::*; +pub use tasks_querent::*; pub use workspace_builder::*; pub use workspace_builder_error::*; pub use workspace_cache::*; + +use moon_project_graph::ProjectGraph; +use moon_task_graph::TaskGraph; +use std::sync::Arc; + +pub struct WorkspaceGraph { + pub projects: Arc, + pub tasks: Arc, +} diff --git a/crates/workspace/src/tasks_querent.rs b/crates/workspace/src/tasks_querent.rs new file mode 100644 index 00000000000..e2fefe874f9 --- /dev/null +++ b/crates/workspace/src/tasks_querent.rs @@ -0,0 +1,49 @@ +use crate::build_data::{ProjectBuildData, TaskBuildData}; +use moon_common::Id; +use moon_task::{Target, TaskOptions}; +use moon_task_builder::TasksQuerent; +use rustc_hash::FxHashMap; + +pub struct WorkspaceBuilderTasksQuerent<'builder> { + pub project_data: &'builder FxHashMap, + pub projects_by_tag: &'builder FxHashMap>, + pub task_data: &'builder FxHashMap, +} + +impl<'builder> TasksQuerent for WorkspaceBuilderTasksQuerent<'builder> { + fn query_projects_by_tag(&self, tag: &str) -> miette::Result> { + Ok(self + .projects_by_tag + .get(tag) + .map(|list| list.iter().collect()) + .unwrap_or_default()) + } + + fn query_tasks( + &self, + project_ids: Vec<&Id>, + task_id: &Id, + ) -> miette::Result> { + // May be an alias! + let project_ids = project_ids + .iter() + .map(|id| ProjectBuildData::resolve_id(id, self.project_data)) + .collect::>(); + + let results = self + .task_data + .iter() + .filter_map(|(target, data)| { + let project_id = target.get_project_id()?; + + if &target.task_id == task_id && project_ids.contains(project_id) { + Some((target, &data.options)) + } else { + None + } + }) + .collect::>(); + + Ok(results) + } +} diff --git a/crates/workspace/src/workspace_builder.rs b/crates/workspace/src/workspace_builder.rs index f6a5294a478..67b2e0117ba 100644 --- a/crates/workspace/src/workspace_builder.rs +++ b/crates/workspace/src/workspace_builder.rs @@ -1,6 +1,7 @@ -use crate::project_build_data::*; +use crate::build_data::*; use crate::projects_locator::locate_projects_with_globs; use crate::repo_type::RepoType; +use crate::tasks_querent::*; use crate::workspace_builder_error::WorkspaceBuilderError; use crate::workspace_cache::*; use moon_cache::CacheEngine; @@ -10,13 +11,16 @@ use moon_common::{ Id, }; use moon_config::{ - ConfigLoader, DependencyScope, InheritedTasksManager, ProjectsSourcesList, ToolchainConfig, - WorkspaceConfig, WorkspaceProjects, + ConfigLoader, DependencyScope, DependencyType, InheritedTasksManager, ProjectsSourcesList, + ToolchainConfig, WorkspaceConfig, WorkspaceProjects, }; use moon_project::Project; use moon_project_builder::{ProjectBuilder, ProjectBuilderContext}; use moon_project_constraints::{enforce_project_type_relationships, enforce_tag_relationships}; -use moon_project_graph::{ProjectGraph, ProjectGraphError, ProjectGraphType, ProjectNode}; +use moon_project_graph::{ProjectGraph, ProjectGraphError, ProjectGraphType, ProjectMetadata}; +use moon_task::Target; +use moon_task_builder::TaskDepsBuilder; +use moon_task_graph::{TaskGraph, TaskGraphError, TaskGraphType, TaskMetadata}; use moon_vcs::BoxedVcs; use petgraph::prelude::*; use petgraph::visit::IntoNodeReferences; @@ -33,7 +37,6 @@ pub struct WorkspaceBuilderContext<'app> { pub extend_project: Emitter, pub extend_project_graph: Emitter, pub inherited_tasks: &'app InheritedTasksManager, - pub strict_project_ids: bool, pub toolchain_config: &'app ToolchainConfig, pub vcs: Option>, pub working_dir: &'app Path, @@ -43,6 +46,7 @@ pub struct WorkspaceBuilderContext<'app> { pub struct WorkspaceBuildResult { pub project_graph: ProjectGraph, + pub task_graph: TaskGraph, } #[derive(Deserialize, Serialize)] @@ -50,6 +54,9 @@ pub struct WorkspaceBuilder<'app> { #[serde(skip)] context: Option>>, + /// Projects grouped by tag, for use in task dependency resolution. + projects_by_tag: FxHashMap>, + /// Mapping of project IDs to associated data required for building /// the project itself. Currently we track the following: /// - The alias, derived from manifests (`package.json`). @@ -69,6 +76,14 @@ pub struct WorkspaceBuilder<'app> { /// The root project ID (only if a monorepo). root_project_id: Option, + + /// Mapping of task targets to associated data required for building + /// the project itself. Currently we track the following: + /// - Their task options, for resolving deps. + task_data: FxHashMap, + + /// The task DAG. + task_graph: TaskGraphType, } impl<'app> WorkspaceBuilder<'app> { @@ -80,11 +95,14 @@ impl<'app> WorkspaceBuilder<'app> { let mut graph = WorkspaceBuilder { context: Some(Arc::new(context)), + projects_by_tag: FxHashMap::default(), project_data: FxHashMap::default(), project_graph: ProjectGraphType::default(), renamed_project_ids: FxHashMap::default(), repo_type: RepoType::Unknown, root_project_id: None, + task_data: FxHashMap::default(), + task_graph: TaskGraphType::default(), }; graph.preload_build_data().await?; @@ -101,13 +119,14 @@ impl<'app> WorkspaceBuilder<'app> { let is_vcs_enabled = context .vcs .as_ref() - .expect("VCS is required for project graph caching!") + .expect("VCS is required for workspace graph caching!") .is_enabled(); let mut graph = Self::new(context).await?; // No VCS to hash with, so abort caching if !is_vcs_enabled { graph.load_projects().await?; + graph.load_tasks().await?; return Ok(graph); } @@ -150,6 +169,7 @@ impl<'app> WorkspaceBuilder<'app> { ); graph.load_projects().await?; + graph.load_tasks().await?; state.data.last_hash = hash; state.data.projects = graph.project_data.clone(); @@ -167,13 +187,13 @@ impl<'app> WorkspaceBuilder<'app> { let context = self.context.take().unwrap(); - let project_nodes = self + let project_metadata = self .project_data .into_iter() .map(|(id, data)| { ( id, - ProjectNode { + ProjectMetadata { alias: data.alias, index: data.node_index.unwrap_or_default(), original_id: data.original_id, @@ -184,11 +204,29 @@ impl<'app> WorkspaceBuilder<'app> { .collect::>(); let mut project_graph = - ProjectGraph::new(self.project_graph, project_nodes, context.workspace_root); + ProjectGraph::new(self.project_graph, project_metadata, context.workspace_root); project_graph.working_dir = context.working_dir.to_owned(); - Ok(WorkspaceBuildResult { project_graph }) + let task_metadata = self + .task_data + .into_iter() + .map(|(id, data)| { + ( + id, + TaskMetadata { + index: data.node_index.unwrap_or_default(), + }, + ) + }) + .collect::>(); + + let task_graph = TaskGraph::new(self.task_graph, task_metadata); + + Ok(WorkspaceBuildResult { + project_graph, + task_graph, + }) } /// Load a single project by ID or alias into the graph. @@ -204,20 +242,19 @@ impl<'app> WorkspaceBuilder<'app> { let ids = self.project_data.keys().cloned().collect::>(); for id in ids { - self.internal_load_project(&id, &mut FxHashSet::default()) - .await?; + self.load_project(&id).await?; } Ok(()) } - #[instrument(name = "load_project", skip(self, cycle))] + #[instrument(name = "load_project", skip(self))] async fn internal_load_project( &mut self, id_or_alias: &str, cycle: &mut FxHashSet, ) -> miette::Result<(Id, NodeIndex)> { - let id = self.resolve_project_id(id_or_alias); + let id = ProjectBuildData::resolve_id(id_or_alias, &self.project_data); { let Some(build_data) = self.project_data.get(&id) else { @@ -226,11 +263,6 @@ impl<'app> WorkspaceBuilder<'app> { // Already loaded, exit early with existing index if let Some(index) = &build_data.node_index { - trace!( - project_id = id.as_str(), - "Project already exists in the project graph, skipping load", - ); - return Ok((id, *index)); } } @@ -245,6 +277,14 @@ impl<'app> WorkspaceBuilder<'app> { cycle.insert(id.clone()); + // Then group projects by relevant data + for tag in &project.config.tags { + self.projects_by_tag + .entry(tag.to_owned()) + .or_default() + .push(id.clone()); + } + // Then build dependency projects let mut edges = vec![]; @@ -273,6 +313,17 @@ impl<'app> WorkspaceBuilder<'app> { } } + // Create task build data + for task in project.tasks.values() { + self.task_data.insert( + task.target.clone(), + TaskBuildData { + options: task.options.clone(), + ..Default::default() + }, + ); + } + // And finally add to the graph let index = self.project_graph.add_node(project); @@ -356,6 +407,117 @@ impl<'app> WorkspaceBuilder<'app> { Ok(project) } + /// Load a single task by target into the graph. + pub async fn load_task(&mut self, target: &Target) -> miette::Result<()> { + self.internal_load_task(target, &mut FxHashSet::default()) + .await?; + + Ok(()) + } + + /// Load all tasks into the graph, derived from the loaded projects. + pub async fn load_tasks(&mut self) -> miette::Result<()> { + let mut targets = vec![]; + + for project in self.project_graph.node_weights() { + for task in project.tasks.values() { + targets.push(task.target.clone()); + } + } + + for target in targets { + self.load_task(&target).await?; + } + + Ok(()) + } + + #[instrument(name = "load_task", skip(self))] + async fn internal_load_task( + &mut self, + target: &Target, + cycle: &mut FxHashSet, + ) -> miette::Result { + let target = TaskBuildData::resolve_target(target, &self.project_data); + + { + let Some(build_data) = self.task_data.get(&target) else { + return Err(TaskGraphError::UnconfiguredTarget(target).into()); + }; + + // Already loaded, exit early with existing index + if let Some(index) = &build_data.node_index { + return Ok(*index); + } + } + + // Not loaded, resolve the task + trace!( + target = target.as_str(), + "Task does not exist in the task graph, attempting to load", + ); + + let (_, project_index) = self + .internal_load_project(target.get_project_id().unwrap(), &mut FxHashSet::default()) + .await?; + + // TODO change to a remove in a follow-up PR + let project = self.project_graph.node_weight(project_index).unwrap(); + let mut task = project.tasks.get(&target.task_id).unwrap().clone(); + + cycle.insert(target.clone()); + + // Resolve the task dependencies so we can link edges correctly + TaskDepsBuilder { + querent: Box::new(WorkspaceBuilderTasksQuerent { + project_data: &self.project_data, + projects_by_tag: &self.projects_by_tag, + task_data: &self.task_data, + }), + project_id: &project.id, + project_dependencies: &project.dependencies, + task: &mut task, + } + .build()?; + + // Then resolve dependency tasks + let mut edges = vec![]; + + for dep_config in &mut task.deps { + if cycle.contains(&dep_config.target) { + debug!( + target = target.as_str(), + dependency_target = dep_config.target.as_str(), + "Encountered a dependency cycle (from task); will disconnect nodes to avoid recursion", + ); + + continue; + } + + edges.push(( + Box::pin(self.internal_load_task(&dep_config.target, cycle)).await?, + if dep_config.optional.is_some_and(|v| v) { + DependencyType::Optional + } else { + DependencyType::Required + }, + )); + } + + // And finally add to the graph + let index = self.task_graph.add_node(task); + + self.task_data.get_mut(&target).unwrap().node_index = Some(index); + + for edge in edges { + self.task_graph.add_edge(index, edge.0, edge.1); + } + + cycle.clear(); + + Ok(index) + } + /// Determine the repository type/structure based on the number of project /// sources, and where the point to. fn determine_repo_type(&mut self) -> miette::Result<()> { @@ -616,6 +778,14 @@ impl<'app> WorkspaceBuilder<'app> { // Track ID renames if let Some(new_id) = &config.id { if new_id != &id { + debug!( + old_id = id.as_str(), + new_id = new_id.as_str(), + "Project has been configured with an explicit identifier of {}, renaming from {}", + color::id(new_id), + color::id(id.as_str()), + ); + build_data.original_id = Some(id.clone()); if renamed_ids.contains_key(&id) { @@ -664,40 +834,6 @@ impl<'app> WorkspaceBuilder<'app> { Ok(()) } - fn resolve_project_id(&self, id_or_alias: &str) -> Id { - let id = if self.project_data.contains_key(id_or_alias) { - Id::raw(id_or_alias) - } else { - match self.project_data.iter().find_map(|(id, build_data)| { - if build_data - .alias - .as_ref() - .is_some_and(|alias| alias == id_or_alias) - { - Some(id) - } else { - None - } - }) { - Some(project_id) => project_id.to_owned(), - None => Id::raw(id_or_alias), - } - }; - - if self - .context - .as_ref() - .is_some_and(|ctx| ctx.strict_project_ids) - { - return id; - } - - match self.renamed_project_ids.get(&id) { - Some(new_id) => new_id.to_owned(), - None => id, - } - } - fn context(&self) -> Arc> { Arc::clone( self.context diff --git a/crates/workspace/src/workspace_cache.rs b/crates/workspace/src/workspace_cache.rs index 2b34e81a4c1..75409eba106 100644 --- a/crates/workspace/src/workspace_cache.rs +++ b/crates/workspace/src/workspace_cache.rs @@ -1,4 +1,4 @@ -use crate::project_build_data::ProjectBuildData; +use crate::build_data::ProjectBuildData; use moon_cache::cache_item; use moon_common::path::WorkspaceRelativePathBuf; use moon_common::{is_docker, Id};