Make CAS and AC server not to use the same store

When CAS and AC uses the same store, it might produce unexpected side-effects and wrong behaviors. To prevent system from such situation, it currently raises an error when CAS and AC uses the same store. An integration test is added for this using nix test file and github's nix workflow.
TraceMachina · Aug 16, 2024 · 57b63e8 · 57b63e8
1 parent 042f4a5
commit 57b63e8
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 6 deletions.
diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml
@@ -134,4 +134,5 @@ jobs:
 
       - name: Test nix run
         run: |
+          nix run -L .#nativelink-prevent-store-conflict-test
           nix run -L .#nativelink-is-executable-test
diff --git a/flake.nix b/flake.nix
@@ -215,6 +215,8 @@
 
         local-image-test = pkgs.callPackage ./tools/local-image-test.nix {};
 
+        nativelink-prevent-store-conflict-test = pkgs.callPackage ./tools/nativelink-prevent-store-conflict-test.nix {inherit nativelink;};
+
         nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;};
 
         rbe-configs-gen = pkgs.callPackage ./local-remote-execution/rbe-configs-gen.nix {};
@@ -357,6 +359,7 @@
               nativelink-debug
               nativelink-image
               nativelink-is-executable-test
+              nativelink-prevent-store-conflict-test
               nativelink-worker-init
               nativelink-x86_64-linux
               publish-ghcr

diff --git a/nativelink-scheduler/tests/utils/mock_scheduler.rs b/nativelink-scheduler/tests/utils/mock_scheduler.rs
@@ -17,12 +17,10 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use nativelink_error::{make_input_err, Error};
 use nativelink_metric::{MetricsComponent, RootMetricsComponent};
-use nativelink_util::{
-    action_messages::{ActionInfo, OperationId},
-    known_platform_property_provider::KnownPlatformPropertyProvider,
-    operation_state_manager::{
-        ActionStateResult, ActionStateResultStream, ClientStateManager, OperationFilter,
-    },
+use nativelink_util::action_messages::{ActionInfo, OperationId};
+use nativelink_util::known_platform_property_provider::KnownPlatformPropertyProvider;
+use nativelink_util::operation_state_manager::{
+    ActionStateResult, ActionStateResultStream, ClientStateManager, OperationFilter,
 };
 use tokio::sync::{mpsc, Mutex};
 

diff --git a/src/bin/nativelink.rs b/src/bin/nativelink.rs
@@ -243,6 +243,23 @@ async fn inner_main(
         // Currently we only support http as our socket type.
         let ListenerConfig::http(http_config) = server_cfg.listener;
 
+        // Check if CAS and AC use the same store.
+        if let Some(cas_config) = &services.cas {
+            if let Some(ac_config) = &services.ac {
+                for (instance_name, cas_store_config) in cas_config {
+                    if let Some(ac_store_config) = ac_config.get(instance_name) {
+                        if cas_store_config.cas_store == ac_store_config.ac_store {
+                            return Err(make_err!(
+                                Code::InvalidArgument,
+                                "CAS and AC cannot use the same store '{}' in the config",
+                                cas_store_config.cas_store
+                            ))?;
+                        }
+                    }
+                }
+            }
+        }
+
         let tonic_services = TonicServer::builder()
             .add_optional_service(
                 services

diff --git a/tools/nativelink-prevent-store-conflict-test.nix b/tools/nativelink-prevent-store-conflict-test.nix
@@ -0,0 +1,72 @@
+{
+  nativelink,
+  writeShellScriptBin,
+}:
+writeShellScriptBin "prevent-store-conflict-test" ''
+  set -xuo pipefail
+
+  cat > store_conflict_test.json <<EOF
+  {
+    "stores": {
+      "FILESYSTEM_STORE": {
+        "compression": {
+          "compression_algorithm": {
+            "lz4": {}
+          },
+          "backend": {
+            "filesystem": {
+              "content_path": "~/.cache/nativelink/content_path-cas",
+              "temp_path": "~/.cache/nativelink/tmp_path-cas",
+              "eviction_policy": {
+                // 10gb.
+                "max_bytes": 10000000000,
+              }
+            }
+          }
+        }
+      }
+    },
+    "servers": [{
+      "listener": {
+        "http": {
+          "socket_address": "0.0.0.0:50053"
+        }
+      },
+      "services": {
+        "cas": {
+          "main": {
+            "cas_store": "FILESYSTEM_STORE"
+          }
+        },
+        "ac": {
+          "main": {
+            "ac_store": "FILESYSTEM_STORE"
+          }
+        },
+        "capabilities": {},
+        "bytestream": {
+          "cas_stores": {
+            "main": "FILESYSTEM_STORE",
+          }
+        }
+      }
+    }]
+  }
+  EOF
+
+  nativelink_output="$(${nativelink}/bin/nativelink ./store_conflict_test.json 2>&1)"
+
+  rm ./store_conflict_test.json
+
+  print_error_output=$(cat <<EOF
+  Error: Error { code: InvalidArgument, messages: ["CAS and AC cannot use the same store 'FILESYSTEM_STORE' in the config"] }
+  EOF)
+
+  if [ "$nativelink_output" = "$print_error_output" ]; then
+    echo "The output of nativelink matches the print_error output."
+  else
+    echo 'The output of nativelink does not match the print_error output:'
+    diff <(echo "$nativelink_output") <(echo "$print_error_output") >&2
+    exit 1
+  fi
+''