diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f2fc29a71..e1b7c0bfd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,12 +21,12 @@ jobs: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Docker Buildx - uses: >- # v3.1.0 - docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c + uses: >- # v3.2.0 + docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 - name: Build Nativelink image - uses: >- # v5.1.0 - docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 + uses: >- # v5.3.0 + docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 with: context: . file: ./deployment-examples/docker-compose/Dockerfile @@ -38,8 +38,8 @@ jobs: tags: trace_machina/nativelink:latest - name: Build builder image - uses: >- # v5.1.0 - docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 + uses: >- # v5.3.0 + docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 with: context: . file: ./deployment-examples/docker-compose/Dockerfile @@ -85,12 +85,12 @@ jobs: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Docker Buildx - uses: >- # v3.1.0 - docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c + uses: >- # v3.2.0 + docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 - name: Build image - uses: >- # v5.1.0 - docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 + uses: >- # v5.3.0 + docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 with: context: . file: ./deployment-examples/docker-compose/Dockerfile diff --git a/deployment-examples/docker-compose/README.md b/deployment-examples/docker-compose/README.md index 9f9cd9e95..1789cef44 100644 --- a/deployment-examples/docker-compose/README.md +++ b/deployment-examples/docker-compose/README.md @@ -7,10 +7,11 @@ deployment of NativeLink's cache and remote execution system. 1. Install [Docker](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) on your system. -2. Open a terminal and run `docker-compose up -d` in this directory to start the +2. Open a terminal and run `docker compose up -d` in this directory to start the services. -It will take some time to apply, when it's finished everything should be running. The endpoints are: +It will take some time to apply, when it's finished everything should be +running. The endpoints are: ```sh CAS/AC: 0.0.0.0:50051 # Configures CAS & AC for SSL connections @@ -18,7 +19,8 @@ CAS/AC: 0.0.0.0:50071 # Configures CAS & AC for TLS connections Scheduler: 0.0.0.0:50052 ``` -As a reference you should be able to compile this project using Bazel with something like: +As a reference you should be able to compile this project using Bazel with +something like: ```sh bazel test //... \ @@ -35,7 +37,9 @@ built from the `Dockerfile` located at `./deployment-examples/docker-compose/Doc ### CAS -The CAS (Content Addressable Storage) service is used as a local cache for the NativeLink system. It's configured in the docker-compose.yml file under the nativelink_local_cas service. +The CAS (Content Addressable Storage) service is used as a local cache for the +NativeLink system. It's configured in the `docker-compose.yml` file under the +`nativelink_local_cas` service. ```yml nativelink_local_cas: @@ -65,8 +69,10 @@ The CAS (Content Addressable Storage) service is used as a local cache for the N ### Scheduler -The scheduler is currently the only single point of failure in the system. We currently only support one scheduler at a time. -The scheduler service is responsible for scheduling tasks in the NativeLink system. It's configured in the docker-compose.yml file under the nativelink_scheduler service. +The scheduler is currently the only single point of failure in the system. We +currently only support one scheduler at a time. The scheduler service is +responsible for scheduling tasks in the NativeLink system. It's configured in +the `docker-compose.yml` file under the nativelink_scheduler service. ```yml nativelink_scheduler: @@ -91,7 +97,8 @@ The scheduler service is responsible for scheduling tasks in the NativeLink syst ### Workers -Worker instances are responsible for executing tasks. They're configured in the docker-compose.yml file under the nativelink_executor service. +Worker instances are responsible for executing tasks. They're configured in the +`docker-compose.yml` file under the nativelink_executor service. ```yml nativelink_executor: @@ -124,7 +131,7 @@ in the system. The Docker Compose setup doesn't automatically delete old data. This could lead to storage issues over time if not managed properly. -If you decide to stop using this setup, you can use `docker-compose down` to +If you decide to stop using this setup, you can use `docker compose down` to stop and remove all the containers. However, in a non-local setup, additional steps may be required to ensure that all data is securely deleted. diff --git a/flake.nix b/flake.nix index 141972b66..6f8af5f1f 100644 --- a/flake.nix +++ b/flake.nix @@ -229,6 +229,7 @@ pkgs.yarn pkgs.vale pkgs.trivy + pkgs.docker-client # Additional tools from within our development environment. local-image-test diff --git a/run_integration_tests.sh b/run_integration_tests.sh index c2372b2b7..4e459ad35 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -73,7 +73,7 @@ sudo rm -rf ~/.cache/nativelink mkdir -p ~/.cache/nativelink # Ensure our docker compose is not running. -sudo docker-compose rm --stop -f +sudo docker compose rm --stop -f export TMPDIR=$HOME/.cache/nativelink/ mkdir -p "$TMPDIR" @@ -86,7 +86,7 @@ else fi export BAZEL_CACHE_DIR="$CACHE_DIR/bazel" -trap "sudo rm -rf $CACHE_DIR; sudo docker-compose rm --stop -f" EXIT +trap "sudo rm -rf $CACHE_DIR; sudo docker compose rm --stop -f" EXIT echo "" # New line. @@ -109,8 +109,8 @@ for pattern in "${TEST_PATTERNS[@]}"; do bazel --output_base="$BAZEL_CACHE_DIR" clean FILENAME=$(basename $fullpath) echo "Running test $FILENAME" - sudo docker-compose up -d - if perl -e 'alarm shift; exec @ARGV' 30 bash -c 'until sudo docker-compose logs | grep -q "Ready, listening on"; do sleep 1; done' + sudo docker compose up -d + if perl -e 'alarm shift; exec @ARGV' 30 bash -c 'until sudo docker compose logs | grep -q "Ready, listening on"; do sleep 1; done' then echo "String 'Ready, listening on' found in the logs." else @@ -124,10 +124,10 @@ for pattern in "${TEST_PATTERNS[@]}"; do echo "$FILENAME passed" else echo "$FILENAME failed with exit code $EXIT_CODE" - sudo docker-compose logs + sudo docker compose logs exit $EXIT_CODE fi - sudo docker-compose rm --stop -f + sudo docker compose rm --stop -f echo "" # New line. done done