Merge branch 'main' into mk/corepack-npm-tests

.github/.kodiak.toml

@@ -17,7 +17,7 @@ notify_on_conflict = false
 # Without this, only branches that are passing CI will get updated.
 # The benefit of this config is that PRs that we want to automerge, but
 # have failed because the test suite is flaky will get stuck, unless someone
-# presses the Update Branch button manually, or re-triggers the Github workflow
+# presses the Update Branch button manually, or re-triggers the GitHub workflow
 # for tests again. This causes delays in getting easy PRs in.
 always = true
 
@@ -28,7 +28,7 @@ always = true
 # (i.e. PRs with the "pr: automerge" label, which is configured above).
 # This label allows PR authors to keep their PR branch up-to-date without
 # opting into Kodiak's automerge feature. This is useful if you want to use
-# Github's AutoMerge feature instead.
+# GitHub's AutoMerge feature instead.
 autoupdate_label = "pr: autoupdate"
 
 [merge.message]

CONTRIBUTING.md

@@ -22,6 +22,10 @@ Thanks for your interest in contributing to Turbo!
 
 - [Rust](https://www.rust-lang.org/tools/install)
 - [cargo-groups](https://github.com/nicholaslyang/cargo-groups)
+- NodeJS v18
+- npm v10.5.0 (note: this is determined by the GitHub Actions CI, when in doubt, look at what the runner is using)
+- capnproto
+- protoc
 
 ### Linux Dependencies
 
@@ -48,7 +52,8 @@ Turborepo uses `reqwest`, a Rust HTTP client, to make requests to the Turbo API.
 implementations: `rustls` and `native-tls`. `rustls` is a pure Rust implementation of TLS, while `native-tls`
 is a wrapper around OpenSSL. Turborepo allows users to select which implementation they want with the `native-tls`
 and `rustls-tls` features. By default, the `rustls-tls` feature is selected---this is done so that `cargo build` works
-out of the box. If you wish to select `native-tls`, you may do so by passing `--no-default-features --features native-tls`
+out of the box. If you wish to select `native-tls`, you may do so by passing
+`--no-default-features --features native-tls`
 to the build command.
 
 ### Running Turborepo Tests
@@ -71,10 +76,6 @@ Then from the root directory, you can run:
   ```bash
   pnpm test -- --filter=cli
   ```
-- A single Go unit test (see more [in the Go docs](https://pkg.go.dev/cmd/go#hdr-Test_packages))
-  ```bash
-  cd cli && go test ./[path/to/package/]
-  ```
 - Rust unit tests ([install `nextest` first](https://nexte.st/book/pre-built-binaries.html))
   ```bash
   cargo nextest run -p turborepo-lib --features rustls-tls
@@ -96,6 +97,19 @@ Then from the root directory, you can run:
 
   Note: this is not through turbo, so you'll have to build turbo yourself first.
 
+- Updating Integration Tests
+
+  ```
+  turbo run build --filter=cli
+  pnpm --filter turborepo-tests-integration test:interactive
+  ```
+
+  You can pass a test name to run a single test, or a directory to run all tests in that directory.
+
+  ```
+  pnpm --filter turborepo-tests-integration test:interactive tests/turbo-help.t
+  ```
+
 - Example tests
   ```bash
   pnpm test -- --filter=turborepo-tests-examples -- <example-name> <packagemanager>
@@ -104,15 +118,17 @@ Then from the root directory, you can run:
 ## Debugging Turborepo
 
 1. Install `go install github.com/go-delve/delve/cmd/dlv@latest`
-1. In VS Code's "Run and Debug" tab, select `Build Basic` to start debugging the initial launch of `turbo` against the `build` target of the Basic Example. This task is configured in [launch.json](./.vscode/launch.json).
+1. In VS Code's "Run and Debug" tab, select `Build Basic` to start debugging the initial launch of `turbo` against the
+   `build` target of the Basic Example. This task is configured in [launch.json](./.vscode/launch.json).
 
 ## Benchmarking Turborepo
 
 Follow the instructions in the [`benchmark/README.md`](./benchmark/README.md).
 
 ## Updating `turbo`
 
-You might need to update `packages/turbo` in order to support a new platform. When you do that you will need to link the module in order to be able to continue working. As an example, with `npm link`:
+You might need to update `packages/turbo` in order to support a new platform. When you do that you will need to link the
+module in order to be able to continue working. As an example, with `npm link`:
 
 ```sh
 cd ~/repos/vercel/turbo/packages/turbo
@@ -146,8 +162,10 @@ Here's a checklist of testing strategies to cover:
 There are also multiple installation scenarios worth testing:
 
 - Global-only. `turbo` is installed as global binary, no local `turbo` in repository.
-- Local-only. `turbo` is installed as local binary, no global `turbo` in PATH. turbo` is invoked via a root package script.
-- Global + local. `turbo` is installed as global binary, and local `turbo` in repository. Global `turbo` delegates to local `turbo`
+- Local-only. `turbo` is installed as local binary, no global `turbo` in PATH. turbo` is invoked via a root package
+  script.
+- Global + local. `turbo` is installed as global binary, and local `turbo` in repository. Global `turbo` delegates to
+  local `turbo`
 
 Here are a few repositories that you can test on:
 
@@ -164,15 +182,18 @@ See [the publishing guide](./release.md#release-turborepo).
 
 ## Creating a new release blog post
 
-Creating a new release post can be done via a turborepo generator. Run the following command from anywhere within the repo:
+Creating a new release post can be done via a turborepo generator. Run the following command from anywhere within the
+repo:
 
 ```bash
 turbo generate run "blog - release post"
 ```
 
 This will walk you through creating a new blog post from start to finish.
 
-NOTE: If you would like to update the stats (github stars / npm downloads / time saved) for an existing blog post that has yet to be published (useful if time has passed since the blog post was created, and up to date stats are required before publishing) - run:
+NOTE: If you would like to update the stats (GitHub stars / npm downloads / time saved) for an existing blog post that
+has yet to be published (useful if time has passed since the blog post was created, and up to date stats are required
+before publishing) - run:
 
 ```bash
 turbo generate run "blog - "blog - update release post stats"

buildcontainer/README.md

@@ -20,7 +20,7 @@ This project is rather cookbook combing various projects into one. Special to [o
 
 ## Docker
 
-Docker images are available on both [Github](https://ghcr.io/goreleaser/goreleaser-cross) and [Docker hub](https://hub.docker.com/r/goreleaser/goreleaser-cross).
+Docker images are available on both [GitHub](https://ghcr.io/goreleaser/goreleaser-cross) and [Docker hub](https://hub.docker.com/r/goreleaser/goreleaser-cross).
 
 Images from version v1.17.4 are multi-arch. Supported host are listed in the table below
 

crates/turbo-trace/src/lib.rs

@@ -2,4 +2,4 @@
 mod import_finder;
 mod tracer;
 
-pub use tracer::{TraceError, Tracer};
+pub use tracer::{TraceError, TraceResult, Tracer};

crates/turbo-trace/src/tracer.rs

@@ -48,18 +48,18 @@ impl Tracer {
         cwd: AbsoluteSystemPathBuf,
         files: Vec<AbsoluteSystemPathBuf>,
         ts_config: Option<Utf8PathBuf>,
-    ) -> Result<Self, PathError> {
+    ) -> Self {
         let ts_config =
             ts_config.map(|ts_config| AbsoluteSystemPathBuf::from_unknown(&cwd, ts_config));
 
         let seen = HashSet::new();
 
-        Ok(Self {
+        Self {
             files,
             seen,
             ts_config,
             source_map: Rc::new(SourceMap::default()),
-        })
+        }
     }
 
     pub fn trace(mut self) -> TraceResult {

crates/turborepo-env/Cargo.toml

@@ -15,6 +15,8 @@ regex = { workspace = true }
 serde = { workspace = true }
 sha2 = { workspace = true }
 thiserror = { workspace = true }
+turborepo-ci = { workspace = true }
+turborepo-ui = { workspace = true }
 
 [dev-dependencies]
 test-case = { workspace = true }

crates/turborepo-env/src/lib.rs

@@ -11,6 +11,8 @@ use serde::Serialize;
 use sha2::{Digest, Sha256};
 use thiserror::Error;
 
+pub mod platform;
+
 const DEFAULT_ENV_VARS: [&str; 1] = ["VERCEL_ANALYTICS_ID"];
 
 #[derive(Clone, Debug, Error)]

crates/turborepo-env/src/platform.rs

@@ -0,0 +1,115 @@
+use turborepo_ci::Vendor;
+use turborepo_ui::{ceprint, ceprintln, color, ColorConfig, BOLD, GREY, UNDERLINE, YELLOW};
+
+use crate::EnvironmentVariableMap;
+
+pub struct PlatformEnv {
+    env_keys: Vec<String>,
+}
+
+impl Default for PlatformEnv {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+const TURBO_PLATFORM_ENV_KEY: &str = "TURBO_PLATFORM_ENV";
+const TURBO_PLATFORM_ENV_DISABLED_KEY: &str = "TURBO_PLATFORM_ENV_DISABLED";
+
+impl PlatformEnv {
+    pub fn new() -> Self {
+        let env_keys = std::env::var(TURBO_PLATFORM_ENV_KEY)
+            .unwrap_or_default()
+            .split(',')
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string())
+            .collect();
+
+        Self { env_keys }
+    }
+
+    pub fn disabled() -> bool {
+        let turbo_platform_env_disabled =
+            std::env::var(TURBO_PLATFORM_ENV_DISABLED_KEY).unwrap_or_default();
+        turbo_platform_env_disabled == "1" || turbo_platform_env_disabled == "true"
+    }
+
+    pub fn validate(&self, execution_env: &EnvironmentVariableMap) -> Vec<String> {
+        if Self::disabled() {
+            return vec![];
+        }
+
+        self.diff(execution_env)
+    }
+
+    pub fn diff(&self, execution_env: &EnvironmentVariableMap) -> Vec<String> {
+        self.env_keys
+            .iter()
+            .filter(|key| !execution_env.contains_key(*key))
+            .map(|s| s.to_string())
+            .collect()
+    }
+
+    pub fn output_header(is_strict: bool, color_config: ColorConfig) {
+        let ci = Vendor::get_constant().unwrap_or("unknown");
+
+        let strict_message = if is_strict {
+            "These variables WILL NOT be available to your application and may cause your build to \
+             fail."
+        } else {
+            "These variables WILL NOT be considered in your cache key and could cause inadvertent \
+             cache hits."
+        };
+
+        match ci {
+            "VERCEL" => {
+                ceprintln!(
+                    color_config,
+                    BOLD,
+                    "Warning - the following environment variables are set on your Vercel \
+                     project, but missing from \"turbo.json\". {}",
+                    strict_message
+                );
+            }
+            _ => {
+                ceprintln!(
+                    color_config,
+                    BOLD,
+                    "Warning - the following environment variables are missing from \
+                     \"turbo.json\". {}",
+                    strict_message
+                );
+            }
+        }
+
+        let docs = color!(
+            color_config,
+            UNDERLINE,
+            "https://turbo.build/repo/docs/platform-environment-variables"
+        );
+        ceprintln!(color_config, GREY, "Learn more at {docs}\n");
+    }
+
+    pub fn output_for_task(
+        missing: Vec<String>,
+        task_id_for_display: &str,
+        color_config: ColorConfig,
+    ) {
+        let ci = Vendor::get_constant().unwrap_or("unknown");
+        let log_prefix = match ci {
+            "VERCEL" => "[warn]",
+            _ => "",
+        };
+        ceprintln!(
+            color_config,
+            YELLOW,
+            "{} {}",
+            log_prefix,
+            task_id_for_display
+        );
+        for key in missing {
+            ceprint!(color_config, GREY, "{}   - ", log_prefix);
+            ceprint!(color_config, GREY, "{} \n", key);
+        }
+    }
+}

crates/turborepo-graph-utils/src/lib.rs

@@ -1,9 +1,12 @@
 mod walker;
 
-use std::fmt::Display;
+use std::{collections::HashSet, fmt::Display, hash::Hash};
 
 use itertools::Itertools;
-use petgraph::prelude::*;
+use petgraph::{
+    prelude::*,
+    visit::{depth_first_search, Reversed},
+};
 use thiserror::Error;
 
 #[derive(Debug, Error)]
@@ -14,6 +17,31 @@ pub enum Error {
     SelfDependency(String),
 }
 
+pub fn transitive_closure<N: Hash + Eq + PartialEq, I: IntoIterator<Item = NodeIndex>>(
+    graph: &Graph<N, ()>,
+    indices: I,
+    direction: petgraph::Direction,
+) -> HashSet<&N> {
+    let mut visited = HashSet::new();
+
+    let visitor = |event| {
+        if let petgraph::visit::DfsEvent::Discover(n, _) = event {
+            visited.insert(
+                graph
+                    .node_weight(n)
+                    .expect("node index found during dfs doesn't exist"),
+            );
+        }
+    };
+
+    match direction {
+        petgraph::Direction::Outgoing => depth_first_search(&graph, indices, visitor),
+        petgraph::Direction::Incoming => depth_first_search(Reversed(&graph), indices, visitor),
+    };
+
+    visited
+}
+
 pub fn validate_graph<G: Display>(graph: &Graph<G, ()>) -> Result<(), Error> {
     // This is equivalent to AcyclicGraph.Cycles from Go's dag library
     let cycles_lines = petgraph::algo::tarjan_scc(&graph)

crates/turborepo-lib/src/cli/mod.rs

@@ -417,20 +417,6 @@ impl Args {
         clap_args
     }
 
-    pub fn get_tasks(&self) -> &[String] {
-        match &self.command {
-            Some(Command::Run {
-                run_args: _,
-                execution_args: box ExecutionArgs { tasks, .. },
-            }) => tasks,
-            _ => self
-                .execution_args
-                .as_ref()
-                .map(|execution_args| execution_args.tasks.as_slice())
-                .unwrap_or(&[]),
-        }
-    }
-
     pub fn track(&self, tel: &GenericEventBuilder) {
         // track usage only
         track_usage!(tel, self.skip_infer, |val| val);

crates/turborepo-lib/src/commands/query.rs

@@ -68,7 +68,7 @@ pub async fn run(
         execution_args: Box::default(),
     });
 
-    let run_builder = RunBuilder::new(base)?;
+    let run_builder = RunBuilder::new(base)?.add_all_tasks();
     let run = run_builder.build(&handler, telemetry).await?;
 
     if let Some(query) = query {