ci: run integration tests

.github/workflows/rust.yml

@@ -1,63 +1,94 @@
-name: Format, clippy, and test dlcdevkit 
-#
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  clippy_check:
+  clippy:
+    name: clippy
     runs-on: ubuntu-latest
     steps:
       - name: Install Protoc
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: nightly
-            components: clippy
-            override: true
+          toolchain: nightly
+          components: clippy
+          override: true
       - name: Check clippy
         run: cargo clippy -- -D warnings
-  fmt_check:
+  format:
+    name: format
     runs-on: ubuntu-latest
     steps:
       - name: Install Protoc
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: nightly
-            components: rustfmt
-            override: true
+          toolchain: nightly
+          components: rustfmt
+          override: true
       - name: Check format
         run: cargo fmt --check
-
-  # test:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     
-  #     - name: Install just
-  #       uses: extractions/setup-just@v1
-  #
-  #     - name: Install Protoc
-  #       run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-  #
-  #     - uses: actions-rs/toolchain@v1
-  #       with:
-  #         toolchain: stable
-  #       
-  #     - name: Start dependencies
-  #       run: just ci-deps
-  #       
-  #     - name: Setup Bitcoin Node
-  #       run: |
-  #         docker logs bitcoin
-  #         chmod +x ./testconfig/scripts/setup-bitcoind.sh
-  #         ./testconfig/scripts/setup-bitcoind.sh
-  #       
-  #     - name: Run tests
-  #       run: cargo test --all-features --verbose
-  #       
-  #     - name: Stop dependencies
-  #       run: just deps-down
+  integration_tests_prepare:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Install Protoc
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          components: rustfmt
+          override: true
+      - uses: actions/cache@v3
+        env:
+          cache-name: test-cache
+        with:
+          path: target/debug/deps
+          key: test-cache-${{ github.run_id }}-${{ github.run_number }}
+      - uses: actions/checkout@v4
+      - id: set-matrix
+        run: cargo test --no-run && echo "::set-output name=matrix::$(testconfig/scripts/get_test_list.sh manager_execution manager_tests contract_updater)"
+  integration_tests:
+    name: integration tests
+    needs: integration_tests_prepare
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        tests: ${{ fromJson(needs.integration_tests_prepare.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        env:
+          cache-name: test-cache
+        with:
+          path: target/debug/deps
+          key: test-cache-${{ github.run_id }}-${{ github.run_number }}
+      - name: Debug file location
+        run: |
+          ls -la ./testconfig/scripts/
+          pwd
+      - name: Start bitcoin node
+        run: docker compose up -d
+      - name: Wait for container to run
+        run: |
+          chmod +x ./testconfig/scripts/wait_for_container.sh
+          ./testconfig/scripts/wait_for_container.sh bitcoin
+      - name: Wait for electrs to be ready
+        run: |
+          chmod +x ./testconfig/scripts/wait_for_electrs.sh
+          ./testconfig/scripts/wait_for_electrs.sh
+      - name: Run test
+        run: RUST_BACKTRACE=1 ${{ matrix.tests }} --ignored
+      - name: Stop bitcoin node
+        run: docker compose down -v

ddk-manager/src/manager.rs

@@ -653,33 +653,40 @@ where
         if let Some((contract_info, adaptor_info, attestations)) = closable_contract_info {
             let offer = &contract.accepted_contract.offered_contract;
             let signer = self.signer_provider.derive_contract_signer(offer.keys_id)?;
-            let cet = crate::contract_updater::get_signed_cet(
+
+            //  === WARNING ===
+            // This code could potentiall be problematic. When running refund tests, it would look for a CET
+            // but the CET would be invalid and refund would not pass. By only updating with a valid CET,
+            // we then go to update. This way if it fails we can check for refund instead of bailing and getting locked
+            // funds.
+            if let Ok(cet) = crate::contract_updater::get_signed_cet(
                 &self.secp,
                 contract,
                 contract_info,
                 adaptor_info,
                 &attestations,
                 &signer,
-            )?;
-            match self
-                .close_contract(
-                    contract,
-                    cet,
-                    attestations.iter().map(|x| x.1.clone()).collect(),
-                )
-                .await
-            {
-                Ok(closed_contract) => {
-                    self.store.update_contract(&closed_contract)?;
-                    return Ok(());
-                }
-                Err(e) => {
-                    warn!(
-                        "Failed to close contract {}: {}",
-                        contract.accepted_contract.get_contract_id_string(),
-                        e
-                    );
-                    return Err(e);
+            ) {
+                match self
+                    .close_contract(
+                        contract,
+                        cet,
+                        attestations.iter().map(|x| x.1.clone()).collect(),
+                    )
+                    .await
+                {
+                    Ok(closed_contract) => {
+                        self.store.update_contract(&closed_contract)?;
+                        return Ok(());
+                    }
+                    Err(e) => {
+                        warn!(
+                            "Failed to close contract {}: {}",
+                            contract.accepted_contract.get_contract_id_string(),
+                            e
+                        );
+                        return Err(e);
+                    }
                 }
             }
         }

ddk-manager/tests/test_utils.rs

@@ -466,11 +466,10 @@ pub async fn get_digit_decomposition_oracles(
         };
 
         for oracle in &oracles {
-            oracle
+            let _sign_even_if_it_fails_spent_an_hour_tracking_ci_bug = oracle
                 .oracle
                 .sign_numeric_event(EVENT_ID.to_string(), cur_outcome as i64)
-                .await
-                .unwrap();
+                .await;
         }
     }
 

ddk/src/chain/esplora.rs

@@ -65,13 +65,28 @@ impl ddk_manager::Blockchain for EsploraClient {
             "Broadcasting transaction."
         );
 
+        if let Ok(status) = self
+            .async_client
+            .get_tx_status(&transaction.compute_txid())
+            .await
+        {
+            tracing::warn!(
+                txid = transaction.compute_txid().to_string(),
+                "Transaction already submitted",
+            );
+            println!("Transaction is already submitted: {:}", status.confirmed);
+            if status.confirmed {
+                return Ok(());
+            }
+        };
+
         if let Err(e) = self.async_client.broadcast(transaction).await {
             tracing::error!(
                 error =? e,
                 "Could not broadcast transaction {}",
                 transaction.compute_txid()
             );
-            // TODO handle already broadcasted.
+
             return Err(esplora_err_to_manager_err(e));
         }
 

testconfig/scripts/get_test_list.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+OS=$(uname -s)
+
+for TEST_PREFIX in "$@"; do
+    TEST_BIN=$(ls ./target/debug/deps/${TEST_PREFIX}* | grep -v '\.d\|\.o')
+
+    if [ "$OS" = "Darwin" ]; then  # macOS
+        LIST=$(${TEST_BIN} --list --format=terse | sed 's/: test$/,/' | sed "s|\([^,]*\)|\"${TEST_BIN} \1\"|g")
+    else  # Linux
+        LIST=$(${TEST_BIN} --list --format=terse | sed 's/\: test$/,/' | sed 's@[^[:space:],]\+@"'${TEST_BIN}' &"@g')
+    fi
+
+    RES+=(${LIST})
+done
+
+# Use BSD-compatible sed syntax for the final output
+echo $(echo [${RES[@]}] | sed 's/,]/]/')

testconfig/scripts/wait_for_container.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+while [ "`docker inspect -f {{.State.Status}} $1`" != "running" ]; do
+     sleep 2;
+done
+
+docker exec bitcoin bitcoin-cli --rpcport=18443 --rpcuser=ddk --rpcpassword=ddk createwallet ddk
+docker exec bitcoin bitcoin-cli --rpcport=18443 --rpcuser=ddk --rpcpassword=ddk --rpcwallet=ddk -generate 101

testconfig/scripts/wait_for_electrs.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+until $(curl --output /dev/null --silent --fail http://localhost:30000/blocks/tip/height); do
+    printf 'waiting for electrs to start'
+    docker exec bitcoin bitcoin-cli --rpcport=18443 --rpcuser=ddk --rpcpassword=ddk -rpcwallet=ddk getblockcount
+    docker exec bitcoin bitcoin-cli --rpcport=18443 --rpcuser=ddk --rpcpassword=ddk -rpcwallet=ddk -generate 1
+    sleep 5
+done