diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a9dd633e1..146d4b8446 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,28 @@ jobs: run: | rustup default 1.62 rustup component add clippy rustfmt + - name: Get Rust version & build version + shell: bash + run: | + echo "::set-output name=RUST_VERSION::$(rustc --version)" + VERSION=$(src/ci/get-version.sh) + # it's a release build if version doesn't have a hyphen in it + IS_RELEASE_BUILD=$(if [[ "$VERSION" =~ '-' ]]; then echo 'false'; else echo 'true'; fi) + echo "::set-output name=RELEASE_BUILD::$IS_RELEASE_BUILD" + id: rust-version + - name: Rust artifact cache + id: cache-agent-artifacts + # don't cache the rust agent for relase builds as the version number will be incorrect + # don't cache on builds on the main branch for deployment to canary/nightly + if: steps.rust-version.outputs.RELEASE_BUILD == 'false' && github.ref_name != 'main' + uses: actions/cache@v3 + with: + # if nothing has changed inside src/agent, we can reuse the artifacts directory + path: artifacts + key: agent-artifacts-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}-${{ hashFiles('src/agent/**/*') }} + # note: also including the ACTIONS_CACHE_KEY_DATE to rebuild if the Prereq Cache is invalidated - name: Rust Prereq Cache + if: steps.cache-agent-artifacts.outputs.cache-hit != 'true' uses: actions/cache@v2 id: cache-rust-prereqs with: @@ -32,27 +53,28 @@ jobs: ~/.cargo/registry ~/.cargo/git ~/.cargo/bin - key: rust-${{ runner.os }}-${{ env.ACTIONS_CACHE_KEY_DATE }} + key: rust-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }} - name: Install Rust Prereqs - if: steps.cache-rust-prereqs.outputs.cache-hit != 'true' + if: steps.cache-rust-prereqs.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true' shell: bash run: src/ci/rust-prereqs.sh - name: Rust Compile Cache + if: steps.cache-agent-artifacts.outputs.cache-hit != 'true' uses: actions/cache@v2 with: path: | sccache src/agent/target - key: agent-${{ runner.os }}-${{ hashFiles('src/agent/Cargo.lock') }}-${{ env.ACTIONS_CACHE_KEY_DATE }} + key: agent-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}-${{ hashFiles('src/agent/Cargo.lock') }} restore-keys: | - agent-${{ runner.os }}-${{ hashFiles('src/agent/Cargo.lock') }}- - agent-${{ runner.os }}- + agent-${{ runner.os }}-${{steps.rust-version.outputs.RUST_VERSION}}-${{ env.ACTIONS_CACHE_KEY_DATE }}- - name: Linux Prereqs run: | sudo apt-get -y update sudo apt-get -y install libssl1.0-dev libunwind-dev - if: "${{ runner.os == 'Linux' }}" + if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true' - run: src/ci/agent.sh + if: steps.cache-agent-artifacts.outputs.cache-hit != 'true' shell: bash - uses: actions/upload-artifact@v2.2.2 with: diff --git a/src/api-service/__app__/onefuzzlib/workers/nodes.py b/src/api-service/__app__/onefuzzlib/workers/nodes.py index ed8e1a5a0b..fcc232f887 100644 --- a/src/api-service/__app__/onefuzzlib/workers/nodes.py +++ b/src/api-service/__app__/onefuzzlib/workers/nodes.py @@ -5,6 +5,7 @@ import datetime import logging +import os from typing import List, Optional, Tuple from uuid import UUID @@ -131,6 +132,10 @@ def search_outdated( @classmethod def mark_outdated_nodes(cls) -> None: + # if outdated agents are allowed, do not attempt to update + if os.environ["ONEFUZZ_ALLOW_OUTDATED_AGENT"] == "true": + return + # ony update 500 nodes at a time to mitigate timeout issues outdated = cls.search_outdated(exclude_update_scheduled=True, num_results=500) for node in outdated: @@ -263,7 +268,7 @@ def can_process_new_work(self) -> bool: from .pools import Pool from .scalesets import Scaleset - if self.is_outdated(): + if self.is_outdated() and os.environ["ONEFUZZ_ALLOW_OUTDATED_AGENT"] != "true": logging.info( "can_process_new_work agent and service versions differ, " "stopping node. "