diff --git a/.github/actions/tests/local-environment-tests/action.yml b/.github/actions/tests/local-environment-tests/action.yml index b4ab50ecbf..c2d70e11c1 100644 --- a/.github/actions/tests/local-environment-tests/action.yml +++ b/.github/actions/tests/local-environment-tests/action.yml @@ -11,7 +11,7 @@ inputs: description: "PC Artifact Tag" required: true markers: - description: 'Run tests by markers (-m). Available markers: ci, smoke, rpc, reserve, governed_map, delegator_rewards, ariadne, wizards or full' + description: 'Run tests by markers (-m). Available markers: ci, smoke, rpc, reserve, governed_map, delegator_rewards, ariadne, wizards, dolos or full' required: true outputs: {} @@ -42,6 +42,8 @@ runs: cd dev/local-environment if [ "${{ inputs.markers }}" = "wizards" ]; then bash setup.sh --non-interactive --postgres-password azMpOp4mTqhlKDmgCVQr --node-image ${{ inputs.image }} --tests -d 5 + elif [ "${{ inputs.markers }}" = "dolos" ]; then + bash setup.sh --non-interactive --postgres-password azMpOp4mTqhlKDmgCVQr --node-image ${{ inputs.image }} --tests -d 6 else bash setup.sh --non-interactive --postgres-password azMpOp4mTqhlKDmgCVQr --node-image ${{ inputs.image }} --tests fi diff --git a/.github/actions/tests/run-e2e-tests/action.yml b/.github/actions/tests/run-e2e-tests/action.yml index 4f029ebb5e..6f23987b4d 100644 --- a/.github/actions/tests/run-e2e-tests/action.yml +++ b/.github/actions/tests/run-e2e-tests/action.yml @@ -56,6 +56,8 @@ runs: markers_switch="" if [ "${{ inputs.markers }}" = "wizards" ]; then markers_switch="-m smoke" + elif [ "${{ inputs.markers }}" = "dolos" ]; then + markers_switch="-m smoke" elif [ -n "${{ inputs.markers }}" ]; then markers_switch="-m '${{ inputs.markers }}'" fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eac24f9f02..fc34e70354 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,11 +11,6 @@ on: description: 'Commit SHA to build' required: true type: string - wipe_cache: - description: 'Wipe /tmp/rust-cache* before build' - required: false - type: boolean - default: false run_post_merge_tests: description: 'Run post-merge tests' required: false @@ -36,105 +31,60 @@ jobs: ### Pre merge workflow ############################################################################################################### build-linux-x86_64-pre-merge: - runs-on: nixos + runs-on: ubuntu-latest if: github.event_name == 'pull_request' && github.event.pull_request.merged == false steps: - - name: Clean runner environment - run: rm -rf ./* - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: fetch-depth: 0 - clean: true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - name: Restore Rust cache from host (keyed by toolchain+lock) - run: | - set -e - echo "--- Restoring Rust cache from host ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - if [ -d "$CACHE" ] && [ -f "$CACHE/.ready" ]; then - echo "Ready cache found: $CACHE" - mkdir -p ~/.cargo/git ~/.cargo/registry - nix shell nixpkgs#rsync -c bash -c " - rsync -a $CACHE/cargo-git/ ~/.cargo/git/ - rsync -a $CACHE/cargo-registry/ ~/.cargo/registry/ - " - else - echo "No matching cache ($CACHE); skipping restore." - fi - - - name: Validate Rust cache - run: | - set -euo pipefail - CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" - - echo "--- Cache preflight ---" - # Compute cache key again (same as restore/save) - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - echo "Cargo.lock missing; skipping manifest check." - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - [[ -d "$CACHE" ]] || { echo "No shared cache present for key: $CACHE"; exit 0; } + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v11 + with: + extra-conf: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org/ + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - MAN="$CACHE/.manifest"; READY="$CACHE/.ready" - WANT="$(printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA")" - REASONS=() - echo "[1/2] Checking manifest/ready..." - if [[ ! -f "$READY" || ! -f "$MAN" ]]; then - echo "manifest/ready missing" - REASONS+=("manifest/ready missing") - else - HAVE="$(head -n 2 "$MAN" || true)" - if [[ "$WANT" != "$HAVE" ]]; then - echo "manifest mismatch" - REASONS+=("manifest mismatch") - fi - fi + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 - echo "[2/2] cargo fetch --locked..." - if ! nix develop -c bash -lc "cargo fetch --locked -q" 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2); then - echo "cargo fetch failed → clearing registry/src + git/checkouts, then re-fetching" - rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true - nix develop -c cargo fetch --locked 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2) - REASONS+=("re-fetched dependencies") - fi + - name: Enable sccache + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=5G" >> $GITHUB_ENV - echo "--- Preflight summary: ${REASONS[*]:-OK} ---" + - name: Cache sccache + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + sccache-${{ runner.os }}- - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-registry-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-registry-${{ runner.os }}- + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: v1 + shared-key: global + cache-all-crates: true + cache-targets: true + cache-bin: true + cache-on-failure: true + workspaces: | + . - name: Acquire AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -147,76 +97,19 @@ jobs: with: registry: ${{ secrets.ECR_REGISTRY_SECRET }} - - name: Enable sccache (RUSTC_WRAPPER) - run: | - set -e - if nix develop -c bash -lc 'command -v sccache' >/dev/null 2>&1; then - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "SCCACHE_DIR=/tmp/sccache" >> "$GITHUB_ENV" - echo "SCCACHE_CACHE_SIZE=30G" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - mkdir -p /tmp/sccache - nix develop -c sccache --stop-server || true - nix develop -c sccache --start-server || true - nix develop -c sccache --version || true - else - echo "sccache not in devshell; skipping wrapper." - fi - - name: Formatting - run: nix develop -c bash -c "cargo fmt --check" + run: nix develop --accept-flake-config -c bash -c "cargo fmt --check" - name: Build (Stable) - run: nix develop -c bash -c "cargo build --locked --release" + run: nix develop --accept-flake-config -c bash -c "cargo build --locked --release" - name: sccache stats if: always() - run: nix develop -c sccache -s || true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Save Rust cache to host (keyed by toolchain+lock) - run: | - set -e - echo "--- Saving Rust cache to host cache ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - mkdir -p "$CACHE" - nix shell nixpkgs#rsync -c bash -c " - rsync -a --delete ~/.cargo/git/ $CACHE/cargo-git/ - rsync -a --delete ~/.cargo/registry/ $CACHE/cargo-registry/ - " - echo "--- Verifying cache contents ($CACHE) ---" - ls -la "$CACHE" || true - ls -la "$CACHE"/cargo-git "$CACHE"/cargo-registry || true - - printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA" > "$CACHE/.manifest" - touch "$CACHE/.ready" - - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + run: sccache -s || true - name: Build chain specs run: | - nix develop -c bash -c ' + nix develop --accept-flake-config -c bash -c ' source ./dev/envs/devnet/.envrc target/release/partner-chains-demo-node build-spec --chain local --disable-default-bootnode > devnet_chain_spec.json @@ -230,7 +123,7 @@ jobs: - name: Build and push docker image run: | cp target/release/partner-chains-demo-node . - nix develop -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" + nix develop --accept-flake-config -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" docker build -f dev/ci/Dockerfile -t ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ github.sha }} . docker push ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ github.sha }} @@ -250,175 +143,78 @@ jobs: ./staging_preview_chain_spec.json build-nightly-pre-merge: - runs-on: nixos + runs-on: ubuntu-latest if: github.event_name == 'pull_request' && github.event.pull_request.merged == false steps: - - name: Clean runner environment - run: rm -rf ./* - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: fetch-depth: 0 - clean: true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Restore Rust cache from host (keyed by toolchain+lock) - run: | - set -e - echo "--- Restoring Rust cache from host ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - if [ -d "$CACHE" ] && [ -f "$CACHE/.ready" ]; then - echo "Ready cache found: $CACHE" - mkdir -p ~/.cargo/git ~/.cargo/registry - nix shell nixpkgs#rsync -c bash -c " - rsync -a $CACHE/cargo-git/ ~/.cargo/git/ - rsync -a $CACHE/cargo-registry/ ~/.cargo/registry/ - " - else - echo "No matching cache ($CACHE); skipping restore." - fi - - name: Validate Rust cache - run: | - set -euo pipefail - CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" - - echo "--- Cache preflight ---" - # Compute cache key again (same as restore/save) - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - echo "Cargo.lock missing; skipping manifest check." - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - [[ -d "$CACHE" ]] || { echo "No shared cache present for key: $CACHE"; exit 0; } - - MAN="$CACHE/.manifest"; READY="$CACHE/.ready" - WANT="$(printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA")" - REASONS=() + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v11 + with: + extra-conf: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org/ + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - echo "[1/2] Checking manifest/ready..." - if [[ ! -f "$READY" || ! -f "$MAN" ]]; then - echo "manifest/ready missing" - REASONS+=("manifest/ready missing") - else - HAVE="$(head -n 2 "$MAN" || true)" - if [[ "$WANT" != "$HAVE" ]]; then - echo "manifest mismatch" - REASONS+=("manifest mismatch") - fi - fi - echo "[2/2] cargo fetch --locked..." - if ! nix develop -c bash -lc "cargo fetch --locked -q" 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2); then - echo "cargo fetch failed → clearing registry/src + git/checkouts, then re-fetching" - rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true - nix develop -c cargo fetch --locked 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2) - REASONS+=("re-fetched dependencies") - fi + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 - echo "--- Preflight summary: ${REASONS[*]:-OK} ---" + - name: Enable sccache + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=2G" >> $GITHUB_ENV + echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + - name: Cache sccache + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + sccache-${{ runner.os }}- - - name: Enable sccache (RUSTC_WRAPPER) - run: | - set -e - if nix develop -c bash -lc 'command -v sccache' >/dev/null 2>&1; then - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "SCCACHE_DIR=/tmp/sccache" >> "$GITHUB_ENV" - echo "SCCACHE_CACHE_SIZE=30G" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - mkdir -p /tmp/sccache - nix develop -c sccache --stop-server || true - nix develop -c sccache --start-server || true - nix develop -c sccache --version || true - else - echo "sccache not in devshell; skipping wrapper." - fi + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-registry-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-registry-${{ runner.os }}- + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: v1-nightly + shared-key: global-nightly + cache-all-crates: true + cache-targets: true + cache-bin: true + cache-on-failure: true + workspaces: | + . - name: Build (Nightly) - run: nix develop -c bash -c "RUSTUP_TOOLCHAIN=nightly cargo build --locked" + env: + WASM_BUILD_WORKSPACE_HINT: ${{ github.workspace }} + run: nix develop --accept-flake-config -c bash -c "RUSTUP_TOOLCHAIN=nightly cargo build --locked --release" - name: Test (Nightly) - run: nix develop -c bash -c "RUSTUP_TOOLCHAIN=nightly cargo test --locked --all-features" + env: + WASM_BUILD_WORKSPACE_HINT: ${{ github.workspace }} + run: nix develop --accept-flake-config -c bash -c "RUSTUP_TOOLCHAIN=nightly cargo test --locked --release --all-features" - name: Lint (Nightly) - run: nix develop -c bash -c "RUSTUP_TOOLCHAIN=nightly RUSTFLAGS=-Dwarnings cargo clippy --locked --all-features" + run: nix develop --accept-flake-config -c bash -c "RUSTUP_TOOLCHAIN=nightly RUSTFLAGS=-Dwarnings cargo clippy --locked --all-features" - name: sccache stats if: always() - run: nix develop -c sccache -s || true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Save Rust cache to host (keyed by toolchain+lock) - run: | - set -e - echo "--- Saving Rust cache to host cache ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - mkdir -p "$CACHE" - nix shell nixpkgs#rsync -c bash -c " - rsync -a --delete ~/.cargo/git/ $CACHE/cargo-git/ - rsync -a --delete ~/.cargo/registry/ $CACHE/cargo-registry/ - " - echo "--- Verifying cache contents ($CACHE) ---" - ls -la "$CACHE" || true - ls -la "$CACHE"/cargo-git "$CACHE"/cargo-registry || true - - printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA" > "$CACHE/.manifest" - touch "$CACHE/.ready" - - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + run: sccache -s || true local-env: if: github.event_name == 'pull_request' && github.event.pull_request.merged == false @@ -426,7 +222,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - markers: [ci, smoke, rpc, reserve, governed_map, delegator_rewards, ariadne, wizards] + markers: [ci, smoke, rpc, reserve, governed_map, delegator_rewards, ariadne, wizards, dolos] steps: - name: Checkout uses: actions/checkout@v4 @@ -507,112 +303,67 @@ jobs: ### Post merge workflow ############################################################################################################### build-linux-x86_64-post-merge: - runs-on: nixos + runs-on: ubuntu-latest if: github.event_name == 'pull_request' && github.event.pull_request.merged == true outputs: sha: ${{ steps.get_sha.outputs.sha }} steps: - - name: Clean runner environment - run: rm -rf ./* - - name: Checkout master branch code - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: master - clean: true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Restore Rust cache from host (keyed by toolchain+lock) - run: | - set -e - echo "--- Restoring Rust cache from host ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - if [ -d "$CACHE" ] && [ -f "$CACHE/.ready" ]; then - echo "Ready cache found: $CACHE" - mkdir -p ~/.cargo/git ~/.cargo/registry - nix shell nixpkgs#rsync -c bash -c " - rsync -a $CACHE/cargo-git/ ~/.cargo/git/ - rsync -a $CACHE/cargo-registry/ ~/.cargo/registry/ - " - else - echo "No matching cache ($CACHE); skipping restore." - fi - - name: Validate Rust cache - run: | - set -euo pipefail - CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" - - echo "--- Cache preflight ---" - # Compute cache key again - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - echo "Cargo.lock missing; skipping manifest check." - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - [[ -d "$CACHE" ]] || { echo "No shared cache present for key: $CACHE"; exit 0; } + - name: Get current commit SHA + id: get_sha + run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - MAN="$CACHE/.manifest"; READY="$CACHE/.ready" - WANT="$(printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA")" - REASONS=() + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v11 + with: + extra-conf: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org/ + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - echo "[1/2] Checking manifest/ready..." - if [[ ! -f "$READY" || ! -f "$MAN" ]]; then - echo "manifest/ready missing" - REASONS+=("manifest/ready missing") - else - HAVE="$(head -n 2 "$MAN" || true)" - if [[ "$WANT" != "$HAVE" ]]; then - echo "manifest mismatch" - REASONS+=("manifest mismatch") - fi - fi - echo "[2/2] cargo fetch --locked..." - if ! nix develop -c bash -lc "cargo fetch --locked -q" 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2); then - echo "cargo fetch failed → clearing registry/src + git/checkouts, then re-fetching" - rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true - nix develop -c cargo fetch --locked 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2) - REASONS+=("re-fetched dependencies") - fi + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 - echo "--- Preflight summary: ${REASONS[*]:-OK} ---" + - name: Enable sccache + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=5G" >> $GITHUB_ENV - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + - name: Cache sccache + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + sccache-${{ runner.os }}- - - name: Get current commit SHA - id: get_sha - run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-registry-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-registry-${{ runner.os }}- + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: v1 + shared-key: global + cache-all-crates: true + cache-targets: true + cache-bin: true + cache-on-failure: true + workspaces: | + . - name: Acquire AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -632,32 +383,16 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Enable sccache (RUSTC_WRAPPER) - run: | - set -e - if nix develop -c bash -lc 'command -v sccache' >/dev/null 2>&1; then - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "SCCACHE_DIR=/tmp/sccache" >> "$GITHUB_ENV" - echo "SCCACHE_CACHE_SIZE=30G" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - mkdir -p /tmp/sccache - nix develop -c sccache --stop-server || true - nix develop -c sccache --start-server || true - nix develop -c sccache --version || true - else - echo "sccache not in devshell; skipping wrapper." - fi - - name: Build - run: nix develop -c bash -c "cargo build --locked --profile=release" + run: nix develop --accept-flake-config -c bash -c "cargo build --locked --profile=release" - name: sccache stats if: always() - run: nix develop -c sccache -s || true + run: sccache -s || true - name: Build chain specs run: | - nix develop -c bash -c ' + nix develop --accept-flake-config -c bash -c ' source ./dev/envs/devnet/.envrc target/release/partner-chains-demo-node build-spec --chain local --disable-default-bootnode > devnet_chain_spec.json @@ -671,7 +406,7 @@ jobs: - name: Build and push docker image run: | cp target/release/partner-chains-demo-node . - nix develop -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" + nix develop --accept-flake-config -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" docker build -f dev/ci/Dockerfile -t ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ steps.get_sha.outputs.sha }} . docker tag ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ steps.get_sha.outputs.sha }} ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:latest docker tag ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ steps.get_sha.outputs.sha }} ghcr.io/${{ github.repository }}/partner-chains-node-unstable:latest @@ -679,47 +414,6 @@ jobs: docker push ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:latest docker push ghcr.io/${{ github.repository }}/partner-chains-node-unstable:latest - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Save Rust cache to host (keyed by toolchain+lock) - run: | - set -e - echo "--- Saving Rust cache to host cache ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - mkdir -p "$CACHE" - nix shell nixpkgs#rsync -c bash -c " - rsync -a --delete ~/.cargo/git/ $CACHE/cargo-git/ - rsync -a --delete ~/.cargo/registry/ $CACHE/cargo-registry/ - " - echo "--- Verifying cache contents ($CACHE) ---" - ls -la "$CACHE" || true - ls -la "$CACHE"/cargo-git "$CACHE"/cargo-registry || true - - printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA" > "$CACHE/.manifest" - touch "$CACHE/.ready" - - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true - - name: Rename artifact run: | cp ./partner-chains-demo-node partner-chains-node-${{ steps.get_sha.outputs.sha }}-x86_64-linux @@ -895,113 +589,61 @@ jobs: ### Workflow dispatch flow ############################################################################################################### build-linux-x86_64-workflow-dispatch: - runs-on: nixos + runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' steps: - - name: Clean runner environment - run: rm -rf ./* - - name: Checkout target code to build - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.sha }} - clean: true - - name: (Optional) Wipe shared Rust caches - if: github.event_name == 'workflow_dispatch' && inputs.wipe_cache == true - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do sleep 2; done - rm -rf /tmp/rust-cache* # wipe all keyed caches - rmdir /tmp/rust-cache.lock || true || true + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v11 + with: + extra-conf: | + experimental-features = nix-command flakes + substituters = https://cache.nixos.org/ + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - name: Restore Rust cache from host (keyed by toolchain+lock) - run: | - set -e - echo "--- Restoring Rust cache from host ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - if [ -d "$CACHE" ] && [ -f "$CACHE/.ready" ]; then - echo "Ready cache found: $CACHE" - mkdir -p ~/.cargo/git ~/.cargo/registry - nix shell nixpkgs#rsync -c bash -c " - rsync -a $CACHE/cargo-git/ ~/.cargo/git/ - rsync -a $CACHE/cargo-registry/ ~/.cargo/registry/ - " - else - echo "No matching cache ($CACHE); skipping restore." - fi + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 - - name: Validate Rust cache + - name: Enable sccache run: | - set -euo pipefail - CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}" - - echo "--- Cache preflight ---" - # Compute cache key again - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - echo "Cargo.lock missing; skipping manifest check." - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - [[ -d "$CACHE" ]] || { echo "No shared cache present for key: $CACHE"; exit 0; } - - MAN="$CACHE/.manifest"; READY="$CACHE/.ready" - WANT="$(printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA")" - REASONS=() - - echo "[1/2] Checking manifest/ready..." - if [[ ! -f "$READY" || ! -f "$MAN" ]]; then - echo "manifest/ready missing" - REASONS+=("manifest/ready missing") - else - HAVE="$(head -n 2 "$MAN" || true)" - if [[ "$WANT" != "$HAVE" ]]; then - echo "manifest mismatch" - REASONS+=("manifest mismatch") - fi - fi + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=5G" >> $GITHUB_ENV - echo "[2/2] cargo fetch --locked..." - if ! nix develop -c bash -lc "cargo fetch --locked -q" 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2); then - echo "cargo fetch failed → clearing registry/src + git/checkouts, then re-fetching" - rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true - nix develop -c cargo fetch --locked 2> >(grep -v -E 'untrusted substituter|trusted-public-keys' >&2) - REASONS+=("re-fetched dependencies") - fi - - echo "--- Preflight summary: ${REASONS[*]:-OK} ---" + - name: Cache sccache + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + sccache-${{ runner.os }}- - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-registry-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-registry-${{ runner.os }}- + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: v1 + shared-key: global + cache-all-crates: true + cache-targets: true + cache-bin: true + cache-on-failure: true + workspaces: | + . - name: Acquire AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -1014,78 +656,21 @@ jobs: with: registry: ${{ secrets.ECR_REGISTRY_SECRET }} - - name: Enable sccache (RUSTC_WRAPPER) - run: | - set -e - if nix develop -c bash -lc 'command -v sccache' >/dev/null 2>&1; then - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "SCCACHE_DIR=/tmp/sccache" >> "$GITHUB_ENV" - echo "SCCACHE_CACHE_SIZE=30G" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - mkdir -p /tmp/sccache - nix develop -c sccache --stop-server || true - nix develop -c sccache --start-server || true - nix develop -c sccache --version || true - else - echo "sccache not in devshell; skipping wrapper." - fi - - name: Formatting - run: nix develop -c bash -c "cargo fmt --check" + run: nix develop --accept-flake-config -c bash -c "cargo fmt --check" - name: Build - run: nix develop -c bash -c "cargo build --locked --release" + run: nix develop --accept-flake-config -c bash -c "cargo build --locked --release" - name: Test - run: nix develop -c bash -c "cargo test --locked --release --all-features" + run: nix develop --accept-flake-config -c bash -c "cargo test --locked --release --all-features" - name: Lint - run: nix develop -c bash -c "RUSTFLAGS=-Dwarnings cargo clippy --locked --release --all-features" + run: nix develop --accept-flake-config -c bash -c "RUSTFLAGS=-Dwarnings cargo clippy --locked --release --all-features" - name: sccache stats if: always() - run: nix develop -c sccache -s || true - - - name: Acquire cache lock - run: | - until mkdir /tmp/rust-cache.lock 2>/dev/null; do - sleep 5 - done - - - name: Save Rust cache to host (keyed by toolchain+lock) - run: | - set -e - echo "--- Saving Rust cache to host cache ---" - # Compute cache key - RUSTC_FULL="$({ nix develop -c rustc -V; } 2>/dev/null || { rustc -V; } 2>/dev/null || true)" - if [[ -n "${RUSTC_FULL:-}" ]]; then - RUSTC_VER="${RUSTC_FULL#rustc }"; RUSTC_VER="${RUSTC_VER%% *}" - else - RUSTC_VER="unknown" - fi - if [[ -f Cargo.lock ]]; then - LOCK_LINE="$(sha256sum Cargo.lock)" - LOCK_SHA="${LOCK_LINE%% *}" - else - LOCK_SHA="nolock" - fi - CACHE="/tmp/rust-cache-${RUSTC_VER}-${LOCK_SHA}" - - mkdir -p "$CACHE" - nix shell nixpkgs#rsync -c bash -c " - rsync -a --delete ~/.cargo/git/ $CACHE/cargo-git/ - rsync -a --delete ~/.cargo/registry/ $CACHE/cargo-registry/ - " - echo "--- Verifying cache contents ($CACHE) ---" - ls -la "$CACHE" || true - ls -la "$CACHE"/cargo-git "$CACHE"/cargo-registry || true - - printf '%s\n%s\n' "$RUSTC_VER" "$LOCK_SHA" > "$CACHE/.manifest" - touch "$CACHE/.ready" - - - name: Release cache lock - if: always() - run: rmdir /tmp/rust-cache.lock || true + run: sccache -s || true #- name: Run cargo-deny to check licenses # uses: EmbarkStudios/cargo-deny-action@v1 @@ -1094,7 +679,7 @@ jobs: - name: Build chain specs run: | - nix develop -c bash -c ' + nix develop --accept-flake-config -c bash -c ' source ./dev/envs/devnet/.envrc target/release/partner-chains-demo-node build-spec --chain local --disable-default-bootnode > devnet_chain_spec.json @@ -1108,7 +693,7 @@ jobs: - name: Build and push docker image run: | cp target/release/partner-chains-demo-node . - nix develop -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" + nix develop --accept-flake-config -c bash -c "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 partner-chains-demo-node" docker build -f dev/ci/Dockerfile -t ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ inputs.sha }} . docker push ${{ secrets.ECR_REGISTRY_SECRET }}/partner-chains-node:${{ inputs.sha }} diff --git a/demo/node/src/data_sources.rs b/demo/node/src/data_sources.rs index 67fa66025d..faec588e96 100644 --- a/demo/node/src/data_sources.rs +++ b/demo/node/src/data_sources.rs @@ -81,7 +81,7 @@ pub(crate) async fn create_cached_data_sources( ServiceError::Application(format!("Failed to create mock data sources: {err}").into()) }), - DataSourceType::Dolos => create_dolos_data_sources().map_err(|err| { + DataSourceType::Dolos => create_dolos_data_sources(metrics_opt).await.map_err(|err| { ServiceError::Application(format!("Failed to create dolos data sources: {err}").into()) }), } @@ -101,16 +101,54 @@ pub fn create_mock_data_sources() }) } -pub fn create_dolos_data_sources() --> std::result::Result> { - use partner_chains_dolos_data_sources::*; +// TODO Currently uses db-sync for unimplemented Dolos data sources +pub async fn create_dolos_data_sources( + metrics_opt: Option, +) -> std::result::Result> { + let dolos_client = partner_chains_dolos_data_sources::get_connection_from_env()?; + let pool = partner_chains_db_sync_data_sources::get_connection_from_env().await?; + let block = Arc::new( + partner_chains_db_sync_data_sources::BlockDataSourceImpl::new_from_env(pool.clone()) + .await?, + ); Ok(DataSources { - sidechain_rpc: Arc::new(SidechainRpcDataSourceImpl::new()), - mc_hash: Arc::new(McHashDataSourceImpl::new()), - authority_selection: Arc::new(AuthoritySelectionDataSourceImpl::new()), - block_participation: Arc::new(StakeDistributionDataSourceImpl::new()), - governed_map: Arc::new(GovernedMapDataSourceImpl::default()), - bridge: Arc::new(TokenBridgeDataSourceImpl::new()), + sidechain_rpc: Arc::new( + partner_chains_dolos_data_sources::SidechainRpcDataSourceImpl::new( + dolos_client.clone(), + ), + ), + mc_hash: Arc::new(partner_chains_dolos_data_sources::McHashDataSourceImpl::new( + dolos_client.clone(), + )), + authority_selection: Arc::new( + partner_chains_dolos_data_sources::AuthoritySelectionDataSourceImpl::new( + dolos_client.clone(), + ), + ), + block_participation: Arc::new( + partner_chains_db_sync_data_sources::StakeDistributionDataSourceImpl::new( + pool.clone(), + metrics_opt.clone(), + STAKE_CACHE_SIZE, + ), + ), + governed_map: Arc::new( + partner_chains_db_sync_data_sources::GovernedMapDataSourceCachedImpl::new( + pool.clone(), + metrics_opt.clone(), + GOVERNED_MAP_CACHE_SIZE, + block.clone(), + ) + .await?, + ), + bridge: Arc::new( + partner_chains_db_sync_data_sources::CachedTokenBridgeDataSourceImpl::new( + pool, + metrics_opt, + block, + BRIDGE_TRANSFER_CACHE_LOOKAHEAD, + ), + ), }) } diff --git a/dev/local-environment/configurations/dolos/dolos.toml b/dev/local-environment/configurations/dolos/dolos.toml index dc8caba11b..60a26ef812 100644 --- a/dev/local-environment/configurations/dolos/dolos.toml +++ b/dev/local-environment/configurations/dolos/dolos.toml @@ -14,6 +14,7 @@ byron_path = "/shared/byron/genesis.json" shelley_path = "/shared/shelley/genesis.json" alonzo_path = "/shared/shelley/genesis.alonzo.json" conway_path = "/shared/conway/genesis.conway.json" +force_protocol = 9 [chain] type = "cardano" diff --git a/dev/local-environment/modules/partner-chains-nodes-dolos.txt b/dev/local-environment/modules/partner-chains-nodes-dolos.txt new file mode 100644 index 0000000000..c60aee0c7e --- /dev/null +++ b/dev/local-environment/modules/partner-chains-nodes-dolos.txt @@ -0,0 +1,156 @@ + partner-chains-node-1: + container_name: partner-chains-node-1 + image: ${PARTNER_CHAINS_NODE_IMAGE} + platform: linux/amd64 + volumes: + - shared-volume:/shared + - partner-chains-node-1-data:/data + - ./configurations/partner-chains-nodes/partner-chains-node-1/entrypoint.sh:/entrypoint.sh + - ./configurations/partner-chains-nodes/partner-chains-node-1/keystore:/keystore + environment: + DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" + DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" + CARDANO_DATA_SOURCE: "dolos" + CARDANO_SECURITY_PARAMETER: "5" + CARDANO_ACTIVE_SLOTS_COEFF: "0.4" + MC__FIRST_EPOCH_NUMBER: "0" + MC__FIRST_SLOT_NUMBER: "0" + MC__EPOCH_DURATION_MILLIS: "120000" + BLOCK_STABILITY_MARGIN: "0" + entrypoint: ["/bin/bash", "/entrypoint.sh"] + ports: + - "30333:30333" + - "9933:9933" + - "9615:9615" + restart: always + deploy: + resources: + limits: + cpus: ${CPU_PARTNER_CHAINS_NODE:-} + memory: ${MEM_PARTNER_CHAINS_NODE:-} + + partner-chains-node-2: + container_name: partner-chains-node-2 + image: ${PARTNER_CHAINS_NODE_IMAGE} + platform: linux/amd64 + volumes: + - partner-chains-node-2-data:/data + - shared-volume:/shared + - ./configurations/partner-chains-nodes/partner-chains-node-2/entrypoint.sh:/entrypoint.sh + - ./configurations/partner-chains-nodes/partner-chains-node-2/keystore:/keystore + environment: + DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" + DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" + CARDANO_DATA_SOURCE: "dolos" + CARDANO_SECURITY_PARAMETER: "5" + CARDANO_ACTIVE_SLOTS_COEFF: "0.4" + MC__FIRST_EPOCH_NUMBER: "0" + MC__FIRST_SLOT_NUMBER: "0" + MC__EPOCH_DURATION_MILLIS: "120000" + BLOCK_STABILITY_MARGIN: "0" + entrypoint: ["/bin/bash", "/entrypoint.sh"] + ports: + - "30334:30334" + - "9934:9934" + - "9616:9616" + restart: always + deploy: + resources: + limits: + cpus: ${CPU_PARTNER_CHAINS_NODE:-} + memory: ${MEM_PARTNER_CHAINS_NODE:-} + + partner-chains-node-3: + container_name: partner-chains-node-3 + image: ${PARTNER_CHAINS_NODE_IMAGE} + platform: linux/amd64 + volumes: + - partner-chains-node-3-data:/data + - shared-volume:/shared + - ./configurations/partner-chains-nodes/partner-chains-node-3/entrypoint.sh:/entrypoint.sh + - ./configurations/partner-chains-nodes/partner-chains-node-3/keystore:/keystore + environment: + DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" + DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" + CARDANO_DATA_SOURCE: "dolos" + CARDANO_SECURITY_PARAMETER: "5" + CARDANO_ACTIVE_SLOTS_COEFF: "0.4" + MC__FIRST_EPOCH_NUMBER: "0" + MC__FIRST_SLOT_NUMBER: "0" + MC__EPOCH_DURATION_MILLIS: "120000" + BLOCK_STABILITY_MARGIN: "0" + entrypoint: ["/bin/bash", "/entrypoint.sh"] + ports: + - "30335:30335" + - "9935:9935" + - "9617:9617" + restart: always + deploy: + resources: + limits: + cpus: ${CPU_PARTNER_CHAINS_NODE:-} + memory: ${MEM_PARTNER_CHAINS_NODE:-} + + partner-chains-node-4: + container_name: partner-chains-node-4 + image: ${PARTNER_CHAINS_NODE_IMAGE} + platform: linux/amd64 + volumes: + - partner-chains-node-4-data:/data + - shared-volume:/shared + - ./configurations/partner-chains-nodes/partner-chains-node-4/entrypoint.sh:/entrypoint.sh + - ./configurations/partner-chains-nodes/partner-chains-node-4/keystore:/keystore + - ./configurations/partner-chains-nodes/partner-chains-node-4/network:/network + environment: + DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" + DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" + CARDANO_DATA_SOURCE: "dolos" + CARDANO_SECURITY_PARAMETER: "5" + CARDANO_ACTIVE_SLOTS_COEFF: "0.4" + MC__FIRST_EPOCH_NUMBER: "0" + MC__FIRST_SLOT_NUMBER: "0" + MC__EPOCH_DURATION_MILLIS: "120000" + BLOCK_STABILITY_MARGIN: "0" + entrypoint: ["/bin/bash", "/entrypoint.sh"] + ports: + - "30336:30336" + - "9936:9936" + - "9618:9618" + restart: always + deploy: + resources: + limits: + cpus: ${CPU_PARTNER_CHAINS_NODE:-} + memory: ${MEM_PARTNER_CHAINS_NODE:-} + + partner-chains-node-5: + container_name: partner-chains-node-5 + image: ${PARTNER_CHAINS_NODE_IMAGE} + platform: linux/amd64 + volumes: + - partner-chains-node-5-data:/data + - shared-volume:/shared + - ./configurations/partner-chains-nodes/partner-chains-node-5/entrypoint.sh:/entrypoint.sh + - ./configurations/partner-chains-nodes/partner-chains-node-5/keystore:/keystore + - ./configurations/partner-chains-nodes/partner-chains-node-5/network:/network + environment: + DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" + DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" + CARDANO_DATA_SOURCE: "dolos" + CARDANO_SECURITY_PARAMETER: "5" + CARDANO_ACTIVE_SLOTS_COEFF: "0.4" + MC__FIRST_EPOCH_NUMBER: "0" + MC__FIRST_SLOT_NUMBER: "0" + MC__EPOCH_DURATION_MILLIS: "120000" + BLOCK_STABILITY_MARGIN: "0" + entrypoint: ["/bin/bash", "/entrypoint.sh"] + ports: + - "30337:30337" + - "9937:9937" + - "9619:9619" + restart: always + deploy: + resources: + limits: + cpus: ${CPU_PARTNER_CHAINS_NODE:-} + memory: ${MEM_PARTNER_CHAINS_NODE:-} diff --git a/dev/local-environment/modules/partner-chains-nodes.txt b/dev/local-environment/modules/partner-chains-nodes.txt index a795322f92..6765e07f43 100644 --- a/dev/local-environment/modules/partner-chains-nodes.txt +++ b/dev/local-environment/modules/partner-chains-nodes.txt @@ -10,7 +10,6 @@ - ./configurations/partner-chains-nodes/partner-chains-node-1/keystore:/keystore environment: DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" - DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" CARDANO_DATA_SOURCE: "db-sync" CARDANO_SECURITY_PARAMETER: "5" CARDANO_ACTIVE_SLOTS_COEFF: "0.4" @@ -41,7 +40,6 @@ - ./configurations/partner-chains-nodes/partner-chains-node-2/keystore:/keystore environment: DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" - DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" CARDANO_DATA_SOURCE: "db-sync" CARDANO_SECURITY_PARAMETER: "5" CARDANO_ACTIVE_SLOTS_COEFF: "0.4" @@ -72,7 +70,6 @@ - ./configurations/partner-chains-nodes/partner-chains-node-3/keystore:/keystore environment: DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" - DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" CARDANO_DATA_SOURCE: "db-sync" CARDANO_SECURITY_PARAMETER: "5" CARDANO_ACTIVE_SLOTS_COEFF: "0.4" @@ -104,7 +101,6 @@ - ./configurations/partner-chains-nodes/partner-chains-node-4/network:/network environment: DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" - DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" CARDANO_DATA_SOURCE: "db-sync" CARDANO_SECURITY_PARAMETER: "5" CARDANO_ACTIVE_SLOTS_COEFF: "0.4" @@ -136,7 +132,6 @@ - ./configurations/partner-chains-nodes/partner-chains-node-5/network:/network environment: DB_SYNC_POSTGRES_CONNECTION_STRING: "postgres://postgres:${POSTGRES_PASSWORD}@postgres:${POSTGRES_PORT}/cexplorer" - DOLOS_MINIBF_URL: "http://dolos:${DOLOS_MINIBF_PORT}" CARDANO_DATA_SOURCE: "db-sync" CARDANO_SECURITY_PARAMETER: "5" CARDANO_ACTIVE_SLOTS_COEFF: "0.4" diff --git a/dev/local-environment/setup.sh b/dev/local-environment/setup.sh index d530a3be56..c931018500 100755 --- a/dev/local-environment/setup.sh +++ b/dev/local-environment/setup.sh @@ -3,7 +3,7 @@ PARTNER_CHAINS_NODE_IMAGE="ghcr.io/input-output-hk/partner-chains/partner-chains-node-unstable:latest" CARDANO_IMAGE="ghcr.io/intersectmbo/cardano-node:10.5.1" DBSYNC_IMAGE="ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4" -DOLOS_IMAGE="ghcr.io/txpipe/dolos:1.0.0-beta.7" +DOLOS_IMAGE="ghcr.io/txpipe/dolos:1.0.0-beta.8" OGMIOS_IMAGE="cardanosolutions/ogmios:v6.13.0" POSTGRES_IMAGE="postgres:17.2" TESTS_IMAGE="python:3.12-slim" @@ -382,10 +382,14 @@ create_docker_compose() { cat ./modules/partner-chains-wizard.txt >> docker-compose.yml ;; 6) - echo -e "Including Cardano testnet, Ogmios, and Dolos services.\n" + echo -e "Including all services with Dolos data source.\n" cat ./modules/cardano.txt >> docker-compose.yml cat ./modules/ogmios.txt >> docker-compose.yml + cat ./modules/db-sync.txt >> docker-compose.yml + cat ./modules/postgres.txt >> docker-compose.yml cat ./modules/dolos.txt >> docker-compose.yml + cat ./modules/partner-chains-nodes-dolos.txt >> docker-compose.yml + cat ./modules/partner-chains-setup.txt >> docker-compose.yml ;; 0) echo -e "Including all services.\n" diff --git a/e2e-tests/docs/run-tests-on-local-env.md b/e2e-tests/docs/run-tests-on-local-env.md index c6047d1143..f7a8ee0938 100644 --- a/e2e-tests/docs/run-tests-on-local-env.md +++ b/e2e-tests/docs/run-tests-on-local-env.md @@ -26,7 +26,7 @@ 7. Run partner-chains tests on partner-chains local environment ```bash -pytest -rP -v --blockchain substrate --env local --init-timestamp=$(docker exec cardano-node-1 cat /shared/cardano.start) --log-cli-level debug -vv -s -m "not probability" +pytest -rP -v --blockchain substrate --env local --init-timestamp=$(docker exec cardano-node-1 cat /shared/cardano.start) --log-cli-level debug -vv -s -m "not probability" --node-host=127.0.0.1 ``` ## Substrate Portal diff --git a/e2e-tests/pytest.ini b/e2e-tests/pytest.ini index 85072e8cf4..9919ed2301 100644 --- a/e2e-tests/pytest.ini +++ b/e2e-tests/pytest.ini @@ -10,6 +10,7 @@ markers = reserve: reserve management system governed_map: arbitrary data stored on the main chain delegator_rewards: ADA delegator rewards + dolos: tests that require dolos data source # helper tags skip_on_new_chain: skip test on new chain (less than 2 MC epochs have passed) diff --git a/toolkit/committee-selection/pallet/src/migrations/v1.rs b/toolkit/committee-selection/pallet/src/migrations/v1.rs index 70847d7acb..a60781b441 100644 --- a/toolkit/committee-selection/pallet/src/migrations/v1.rs +++ b/toolkit/committee-selection/pallet/src/migrations/v1.rs @@ -2,7 +2,6 @@ #[cfg(feature = "try-runtime")] extern crate alloc; use frame_support::traits::UncheckedOnRuntimeUpgrade; -use sidechain_domain::ScEpochNumber; #[cfg(feature = "try-runtime")] use { alloc::vec::Vec, parity_scale_codec::Encode, sp_session_validator_management::CommitteeMember, @@ -68,6 +67,7 @@ where fn post_upgrade(state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { use frame_support::ensure; use parity_scale_codec::Decode; + use sidechain_domain::ScEpochNumber; use v0::LegacyCommitteeInfo; let (current_committee_v0, next_committee_v0): ( diff --git a/toolkit/data-sources/dolos/src/candidate.rs b/toolkit/data-sources/dolos/src/candidate.rs index 3212100221..02e6254d02 100644 --- a/toolkit/data-sources/dolos/src/candidate.rs +++ b/toolkit/data-sources/dolos/src/candidate.rs @@ -1,40 +1,390 @@ -use crate::Result; +use std::collections::HashMap; + +use crate::{client::*, *}; use async_trait::async_trait; use authority_selection_inherents::*; +use blockfrost_openapi::models::{ + address_utxo_content_inner::AddressUtxoContentInner, + asset_transactions_inner::AssetTransactionsInner, block_content::BlockContent, + pool_list_extended_inner::PoolListExtendedInner, +}; +use cardano_serialization_lib::PlutusData; +use futures::StreamExt; +use itertools::Itertools; +use partner_chains_plutus_data::{ + d_param::DParamDatum, permissioned_candidates::PermissionedCandidateDatums, + registered_candidates::RegisterValidatorDatum, +}; use sidechain_domain::*; -pub struct AuthoritySelectionDataSourceImpl {} +pub struct AuthoritySelectionDataSourceImpl { + client: MiniBFClient, +} impl AuthoritySelectionDataSourceImpl { - pub fn new() -> Self { - Self {} + pub fn new(client: MiniBFClient) -> Self { + Self { client } } } +async fn get_token_utxo_datum_for_epoch( + client: &impl MiniBFApi, + policy_id: PolicyId, + epoch: McEpochNumber, +) -> Result> { + let asset_utxos = client + .assets_transactions(AssetId { + policy_id: policy_id.clone(), + asset_name: AssetName::empty(), + }) + .await?; + let pred = |x: AssetTransactionsInner| async move { + let tx_hash = McTxHash::from_hex_unsafe(&x.tx_hash); + let tx = client.transaction_by_hash(tx_hash).await?; + let block = client.blocks_by_id(McBlockNumber(tx.block_height as u32)).await?; + Result::Ok(if block.epoch <= Some(epoch.0 as i32) { Some(tx_hash) } else { None }) + }; + let futures = asset_utxos.into_iter().map(|item| async move { pred(item.clone()).await }); + let tx_hash = futures::future::try_join_all(futures) + .await? + .into_iter() + .flatten() + .collect::>() + .first() + .ok_or("No policy utxo found after epoch")? + .to_owned(); + + let datum = client.transactions_utxos(tx_hash).await?.outputs.iter().find_map(|o| { + // TODO compare on the level of PolicyId instead of String + if o.amount.iter().any(|a| a.unit == &policy_id.to_hex_string()[2..]) { + o.inline_datum.clone() + } else { + None + } + }); + Ok(match datum { + Some(datum) => Some(PlutusData::from_hex(&datum)?), + None => None, + }) +} + #[async_trait] impl AuthoritySelectionDataSource for AuthoritySelectionDataSourceImpl { async fn get_ariadne_parameters( &self, - _epoch_number: McEpochNumber, - _d_parameter_validator: PolicyId, - _permissioned_candidates_validator: PolicyId, + epoch_number: McEpochNumber, + d_parameter_policy: PolicyId, + permissioned_candidate_policy: PolicyId, ) -> Result { - Err("not implemented".into()) + let epoch = self.get_epoch_of_data_storage(epoch_number)?; + + let (candidates_output_opt, d_output_opt) = tokio::try_join!( + get_token_utxo_datum_for_epoch(&self.client, permissioned_candidate_policy, epoch), + get_token_utxo_datum_for_epoch(&self.client, d_parameter_policy, epoch) + )?; + + let d_datum = d_output_opt + .ok_or(DataSourceError::ExpectedDataNotFound("DParameter Datum".to_string()))?; + + let d_parameter = DParamDatum::try_from(d_datum)?.into(); + + let permissioned_candidates = match candidates_output_opt { + None => None, + Some(candidates_datum) => { + Some(PermissionedCandidateDatums::try_from(candidates_datum)?.into()) + }, + }; + + Ok(AriadneParameters { d_parameter, permissioned_candidates }) } async fn get_candidates( &self, - _epoch: McEpochNumber, - _committee_candidate_address: MainchainAddress, + epoch_number: McEpochNumber, + committee_candidate_address: MainchainAddress, ) -> Result> { - Err("not implemented".into()) + let epoch = self.get_epoch_of_data_storage(epoch_number)?; + let candidates = self.get_registered_candidates(epoch, committee_candidate_address).await?; + let pools = self.client.pools_extended().await?; + let pred = |pool: PoolListExtendedInner| async move { + let history = self.client.pools_history(&pool.pool_id).await?; + Result::Ok(match history.into_iter().find(|h| h.epoch == epoch.0 as i32) { + Some(e) => Some(( + MainchainKeyHash(pool.pool_id.as_bytes().try_into()?), // TODO is pool_id a pool hash? + StakeDelegation(e.active_stake.parse::()?), + )), + None => None, + }) + }; + + let futures = pools.into_iter().map(|item| async move { pred(item.clone()).await }); + let stake_map: HashMap = + futures::future::try_join_all(futures).await?.into_iter().flatten().collect(); + + Ok(candidates + .into_iter() + .into_group_map_by(|c| c.stake_pool_pub_key.clone()) + .into_iter() + .map(|(mainchain_pub_key, candidate_registrations)| CandidateRegistrations { + stake_pool_public_key: mainchain_pub_key.clone(), + registrations: candidate_registrations + .into_iter() + .map(Self::make_registration_data) + .collect(), + stake_delegation: Self::get_stake_delegation(&stake_map, &mainchain_pub_key), + }) + .collect()) + } + + async fn get_epoch_nonce(&self, epoch_number: McEpochNumber) -> Result> { + let epoch = self.get_epoch_of_data_storage(epoch_number)?; + let nonce: String = self.client.epochs_parameters(epoch).await?.nonce; + Ok(Some(EpochNonce(nonce.into()))) + } + + async fn data_epoch(&self, for_epoch: McEpochNumber) -> Result { + self.get_epoch_of_data_storage(for_epoch) + } +} + +#[derive(Debug)] +struct RegisteredCandidate { + stake_pool_pub_key: StakePoolPublicKey, + registration_utxo: UtxoId, + tx_inputs: Vec, + sidechain_signature: SidechainSignature, + mainchain_signature: MainchainSignature, + cross_chain_signature: CrossChainSignature, + sidechain_pub_key: SidechainPublicKey, + cross_chain_pub_key: CrossChainPublicKey, + keys: CandidateKeys, + utxo_info: UtxoInfo, +} + +#[derive(Clone, Debug)] +struct ParsedCandidate { + utxo_info: UtxoInfo, + datum: RegisterValidatorDatum, + tx_inputs: Vec, +} + +impl AuthoritySelectionDataSourceImpl { + fn make_registration_data(c: RegisteredCandidate) -> RegistrationData { + RegistrationData { + registration_utxo: c.registration_utxo, + sidechain_signature: c.sidechain_signature, + mainchain_signature: c.mainchain_signature, + cross_chain_signature: c.cross_chain_signature, + sidechain_pub_key: c.sidechain_pub_key, + cross_chain_pub_key: c.cross_chain_pub_key, + keys: c.keys, + utxo_info: c.utxo_info, + tx_inputs: c.tx_inputs, + } + } + + fn get_stake_delegation( + stake_map: &HashMap, + stake_pool_pub_key: &StakePoolPublicKey, + ) -> Option { + if stake_map.is_empty() { + None + } else { + Some( + stake_map + .get(&MainchainKeyHash::from_vkey(&stake_pool_pub_key.0)) + .cloned() + .unwrap_or(StakeDelegation(0)), + ) + } + } + + // Converters + async fn convert_utxos_to_candidates( + &self, + outputs: &[AddressUtxoContentInner], + ) -> Result> { + Self::parse_candidates(&self.client, outputs) + .await + .into_iter() + .map(|c| { + match c.datum { + RegisterValidatorDatum::V0 { + stake_ownership, + sidechain_pub_key, + sidechain_signature, + registration_utxo, + own_pkh: _own_pkh, + aura_pub_key, + grandpa_pub_key, + } => Ok(RegisteredCandidate { + stake_pool_pub_key: stake_ownership.pub_key, + mainchain_signature: stake_ownership.signature, + // For now we use the same key for both cross chain and sidechain actions + cross_chain_pub_key: CrossChainPublicKey(sidechain_pub_key.0.clone()), + cross_chain_signature: CrossChainSignature(sidechain_signature.0.clone()), + sidechain_signature, + sidechain_pub_key, + keys: CandidateKeys(vec![aura_pub_key.into(), grandpa_pub_key.into()]), + registration_utxo, + tx_inputs: c.tx_inputs, + utxo_info: c.utxo_info, + }), + RegisterValidatorDatum::V1 { + stake_ownership, + sidechain_pub_key, + sidechain_signature, + registration_utxo, + own_pkh: _own_pkh, + keys, + } => Ok(RegisteredCandidate { + stake_pool_pub_key: stake_ownership.pub_key, + mainchain_signature: stake_ownership.signature, + // For now we use the same key for both cross chain and sidechain actions + cross_chain_pub_key: CrossChainPublicKey(sidechain_pub_key.0.clone()), + cross_chain_signature: CrossChainSignature(sidechain_signature.0.clone()), + sidechain_signature, + sidechain_pub_key, + keys, + registration_utxo, + tx_inputs: c.tx_inputs, + utxo_info: c.utxo_info, + }), + } + }) + .collect() } - async fn get_epoch_nonce(&self, _epoch_number: McEpochNumber) -> Result> { - Err("not implemented".into()) + async fn parse_candidate( + client: &impl MiniBFApi, + output: &AddressUtxoContentInner, + ) -> Result { + let datum_str = output.inline_datum.clone().ok_or(format!( + "Missing registration datum for {:?}:{:?}", + output.tx_hash, + output.clone().output_index + ))?; + let datum = cardano_serialization_lib::PlutusData::from_hex(&datum_str) + .map_err(|e| e.to_string())?; + let utxo_id = UtxoId { + tx_hash: output.tx_hash.as_bytes().try_into()?, + index: UtxoIndex(output.tx_index.try_into()?), + }; + let register_validator_datum = RegisterValidatorDatum::try_from(datum) + .map_err(|_| format!("Invalid registration datum for {:?}", utxo_id))?; + let block = client.blocks_by_id(output.block.clone()).await?; + let block_txs = client.blocks_txs(output.block.clone()).await?; + let tx_index_within_block = block_txs + .into_iter() + .position(|tx_hash| tx_hash == output.tx_hash) + .map(|pos| McTxIndexInBlock(pos as u32)) + .ok_or("output tx hash not found in blocks/txs response")?; + let utxos = client.transactions_utxos(utxo_id.tx_hash).await?; + let tx_inputs = utxos + .inputs + .into_iter() + .map(|input| { + Ok::>(UtxoId { + tx_hash: input.tx_hash.as_bytes().try_into()?, + index: UtxoIndex(input.output_index.try_into()?), + }) + }) + .collect::>>()?; + Ok(ParsedCandidate { + utxo_info: UtxoInfo { + utxo_id, + epoch_number: McEpochNumber(block.epoch.ok_or("block epoch missing")? as u32), + block_number: McBlockNumber(block.height.ok_or("block number missing")? as u32), + slot_number: McSlotNumber(block.slot.ok_or("block slot missing")? as u64), + tx_index_within_block, + }, + datum: register_validator_datum, + tx_inputs, + }) } - async fn data_epoch(&self, _for_epoch: McEpochNumber) -> Result { - Err("not implemented".into()) + async fn parse_candidates( + client: &impl MiniBFApi, + outputs: &[AddressUtxoContentInner], + ) -> Vec { + let results = futures::stream::iter(outputs) + .then(|output| async { Self::parse_candidate(client, output).await }) + .collect::>() + .await; + results + .into_iter() + .filter_map(|r| match r { + Ok(candidate) => Some(candidate.clone()), + Err(msg) => { + log::error!("{msg}"); + None + }, + }) + .collect() + } + + fn get_epoch_of_data_storage( + &self, + epoch_of_data_usage: McEpochNumber, + ) -> Result { + offset_data_epoch(&epoch_of_data_usage).map_err(|offset| { + DataSourceError::BadRequest(format!( + "Minimum supported epoch of data usage is {offset}, but {} was provided", + epoch_of_data_usage.0 + )) + .into() + }) + } + + /// Registrations state up to this block are considered as "active", after it - as "pending". + async fn get_last_block_for_epoch( + &self, + epoch_number: McEpochNumber, + ) -> Result> { + let block_option = self.client.epochs_blocks(epoch_number).await?.last().cloned(); + let block = match block_option { + Some(block) => Some(self.client.blocks_by_id(block).await?), + None => None, + }; + Ok(block) + } + + async fn get_registered_candidates( + &self, + epoch: McEpochNumber, + committee_candidate_address: MainchainAddress, + ) -> Result> { + let registrations_block_for_epoch_opt = self.get_last_block_for_epoch(epoch).await?; + let utxos = self.client.addresses_utxos(committee_candidate_address).await?; + let active_utxos = match registrations_block_for_epoch_opt { + Some(registrations_block_for_epoch) => { + let pred = |utxo: AddressUtxoContentInner| async move { + Ok::( + self.client + .blocks_by_id(utxo.block.clone()) + .await? + .height + .ok_or("committee candidate block height missing")? as u32 + >= registrations_block_for_epoch + .height + .ok_or("last_block_for_epoch block height missing")? as u32, + ) + }; + let futures = utxos.into_iter().map(|item| async move { + match pred(item.clone()).await { + Ok(true) => Ok(Some(item)), + Ok(false) => Ok(None), + Err(e) => Err(e), + } + }); + futures::future::try_join_all(futures) + .await? + .into_iter() + .flatten() + .collect::>() + }, + None => vec![], + }; + self.convert_utxos_to_candidates(&active_utxos).await } } diff --git a/toolkit/data-sources/dolos/src/client/minibf.rs b/toolkit/data-sources/dolos/src/client/minibf.rs index 33206d77c9..8979c8acca 100644 --- a/toolkit/data-sources/dolos/src/client/minibf.rs +++ b/toolkit/data-sources/dolos/src/client/minibf.rs @@ -49,23 +49,25 @@ impl MiniBFClient { method: &str, pagination: Pagination, ) -> Result, String> { - let mut req = url::form_urlencoded::Serializer::new(format!("{}/{}", self.addr, method)); - req.extend_pairs([ + let mut query_pairs = url::form_urlencoded::Serializer::new(String::new()); + query_pairs.extend_pairs([ ("count", &pagination.count.to_string()), ("page", &pagination.page.to_string()), ("order", &pagination.order.to_string()), ]); if let Some(from) = pagination.from { - req.append_pair("from", &from); + query_pairs.append_pair("from", &from); } if let Some(to) = pagination.to { - req.append_pair("to", &to); + query_pairs.append_pair("to", &to); } - let req_url = req.finish(); + let mut req_url = + url::Url::parse(&format!("{}/{}", self.addr, method)).map_err(|e| e.to_string())?; + req_url.set_query(Some(&query_pairs.finish())); log::trace!("Dolos request: {req_url:?}"); let resp = self .agent - .get(req_url) + .get(req_url.as_str()) .call() .map_err(|e| e.to_string()) .and_then(|mut r| r.body_mut().read_json().map_err(|e| e.to_string())); @@ -82,7 +84,7 @@ impl MiniBFClient { let mut res = Vec::new(); while !have_all_pages { let mut resp: Vec = self.paginated_request(method, pagination.clone()).await?; - if resp.len() < 100 { + if (resp.len() as i32) < pagination.count { have_all_pages = true } res.append(&mut resp); @@ -112,18 +114,16 @@ impl MiniBFApi for MiniBFClient { &self, asset_id: AssetId, ) -> Result, String> { - let AssetId { policy_id, asset_name } = asset_id; - self.paginated_request_all(&format!("assets/{policy_id}{asset_name}/transactions")) - .await + let asset_id_str = format_asset_id(&asset_id); + self.paginated_request_all(&format!("assets/{asset_id_str}/transactions")).await } async fn assets_addresses( &self, asset_id: AssetId, ) -> Result, String> { - let AssetId { policy_id, asset_name } = asset_id; - self.paginated_request_all(&format!("assets/{policy_id}{asset_name}/addresses")) - .await + let asset_id_str = format_asset_id(&asset_id); + self.paginated_request_all(&format!("assets/{asset_id_str}/addresses")).await } async fn blocks_latest(&self) -> Result { @@ -190,6 +190,11 @@ impl MiniBFApi for MiniBFClient { } } +fn format_asset_id(asset_id: &AssetId) -> String { + let AssetId { policy_id, asset_name } = asset_id; + format!("{}{}", &policy_id.to_hex_string()[2..], &asset_name.to_hex_string()[2..]) +} + #[derive(Clone)] #[allow(dead_code)] enum Order { diff --git a/toolkit/data-sources/dolos/src/lib.rs b/toolkit/data-sources/dolos/src/lib.rs index ccaa23955e..25598ea8b6 100644 --- a/toolkit/data-sources/dolos/src/lib.rs +++ b/toolkit/data-sources/dolos/src/lib.rs @@ -32,7 +32,8 @@ use crate::client::MiniBFClient; pub mod client; -type Result = std::result::Result>; +type ResultErr = Box; +type Result = std::result::Result; /// Error type returned by Dolos based data sources #[derive(Debug, PartialEq, thiserror::Error)] diff --git a/toolkit/data-sources/dolos/src/mc_hash.rs b/toolkit/data-sources/dolos/src/mc_hash.rs index 82547113d4..55bf1b01d7 100644 --- a/toolkit/data-sources/dolos/src/mc_hash.rs +++ b/toolkit/data-sources/dolos/src/mc_hash.rs @@ -1,12 +1,17 @@ -use crate::Result; +use crate::{ + Result, + client::{MiniBFClient, api::MiniBFApi, conversions::from_block_content}, +}; use async_trait::async_trait; use sidechain_domain::*; -pub struct McHashDataSourceImpl {} +pub struct McHashDataSourceImpl { + client: MiniBFClient, +} impl McHashDataSourceImpl { - pub fn new() -> Self { - Self {} + pub fn new(client: MiniBFClient) -> Self { + Self { client } } } @@ -16,18 +21,18 @@ impl sidechain_mc_hash::McHashDataSource for McHashDataSourceImpl { &self, _reference_timestamp: sp_timestamp::Timestamp, ) -> Result> { - Err("not implemented".into()) + Ok(Some(from_block_content(self.client.blocks_latest().await?)?)) } async fn get_stable_block_for( &self, - _hash: McBlockHash, + hash: McBlockHash, _reference_timestamp: sp_timestamp::Timestamp, ) -> Result> { - Err("not implemented".into()) + Ok(Some(from_block_content(self.client.blocks_by_id(hash).await?)?)) } - async fn get_block_by_hash(&self, _hash: McBlockHash) -> Result> { - Err("not implemented".into()) + async fn get_block_by_hash(&self, hash: McBlockHash) -> Result> { + Ok(Some(from_block_content(self.client.blocks_by_id(hash).await?)?)) } } diff --git a/toolkit/data-sources/dolos/src/sidechain_rpc.rs b/toolkit/data-sources/dolos/src/sidechain_rpc.rs index a5a22da44b..d2cc4daff5 100644 --- a/toolkit/data-sources/dolos/src/sidechain_rpc.rs +++ b/toolkit/data-sources/dolos/src/sidechain_rpc.rs @@ -1,18 +1,23 @@ -use crate::Result; +use crate::{ + Result, + client::{MiniBFClient, api::MiniBFApi, conversions::from_block_content}, +}; use pallet_sidechain_rpc::SidechainRpcDataSource; use sidechain_domain::MainchainBlock; -pub struct SidechainRpcDataSourceImpl {} +pub struct SidechainRpcDataSourceImpl { + client: MiniBFClient, +} impl SidechainRpcDataSourceImpl { - pub fn new() -> Self { - Self {} + pub fn new(client: MiniBFClient) -> Self { + Self { client } } } #[async_trait::async_trait] impl SidechainRpcDataSource for SidechainRpcDataSourceImpl { async fn get_latest_block_info(&self) -> Result { - Err("not implemented".into()) + Ok(from_block_content(self.client.blocks_latest().await?)?) } }