Fix challenge logic, implicit freezing, add integration test

.github/workflows/ci.yml

@@ -37,7 +37,14 @@ jobs:
         go build -o op ./cmd/operator
         go build -o cl ./cmd/client
 
-    - name: Test
+    - name: Unit Tests
       run: go test -timeout 60s ./...
+      env:
+        ERDSTALL_GANACHE_CMD: docker run --name=chain --rm --net=host trufflesuite/ganache-cli
+
+    - name: Integration Test
+      run: |
+        docker stop chain
+        go test -tags=integration -timeout 600s -v ./client -run TestWalkthroughs
       env:
         ERDSTALL_GANACHE_CMD: docker run --rm --net=host trufflesuite/ganache-cli

README.md

@@ -24,10 +24,10 @@ ganache-cli -e 100000000000000 -b 5 -m "pistol kiwi shrug future ozone ostrich m
 ./operator.bin
 # alice
 ./client.bin --contract 0x4fb8637afd28492a3209017556e95dc2f8086ddb
---account-index 7
+--account-index 2
 # bob
 ./client.bin --contract 0x4fb8637afd28492a3209017556e95dc2f8086ddb
---account-index 6
+--account-index 3
 ```
 
 ## Description

client/client.go

@@ -45,7 +45,8 @@ type Client struct {
 	params    *tee.Parameters
 	events    chan *Event
 	// balMtx protects balances.
-	balMtx sync.Mutex
+	balMtx            sync.RWMutex
+	stopFrozenWatcher context.CancelFunc
 }
 
 // EpochBalance describes the balance that a specific user has/has in a epoch.
@@ -105,11 +106,6 @@ const (
 	BENCH
 )
 
-const (
-	// How long do we wait for the deposit proof after the deposit phase is over.
-	depositProofGrace = time.Second * 30
-)
-
 // Trust describes how we perceive the operator.
 type Trust = string
 
@@ -206,11 +202,9 @@ func (c *Client) CmdSend(status chan *CmdStatus, args ...string) {
 }
 
 func (c *Client) createTransfer(receiver common.Address, amount *big.Int) (tee.Transaction, error) {
-	block := atomic.LoadUint64(&c.lastBlock) + 1
-
 	tx := tee.Transaction{
 		Nonce:     c.txNonce,
-		Epoch:     c.params.TxEpoch(block),
+		Epoch:     c.params.TxEpoch(c.ActiveBlock()),
 		Sender:    c.Address(),
 		Recipient: receiver,
 		Amount:    (*tee.Amount)(amount),
@@ -314,11 +308,11 @@ func (c *Client) CmdDeposit(status chan *CmdStatus, args ...string) {
 		}
 	}()
 	go func() {
-		depEndBlock := c.params.DepositDoneBlock(epoch)
+		depEndBlock := c.params.TxDoneBlock(epoch)
+		c.logOffChain("Waiting for deposit proof until block #%d", depEndBlock)
 		err := c.ethClient.WaitForBlock(ctx, depEndBlock) // Add PowDepth here if needed
 		if err == nil {
 			c.logOffChain("Deposit Phase for Epoch %d ended in block %d", epoch, depEndBlock)
-			time.Sleep(depositProofGrace)
 		}
 		waitErr <- err
 	}()
@@ -359,8 +353,8 @@ func (c *Client) CmdDeposit(status chan *CmdStatus, args ...string) {
 		status <- &CmdStatus{War: "Deposit proof: Operator timed out - resuming protocol"}
 		c.setOpTrust(UNKNOWN)
 	}
-	// TODO challenge
-	status <- &CmdStatus{Err: errors.New("TODO challenge")}
+
+	c.challengeDeposit(status)
 }
 
 // BalanceProofWatcher waits for the balance proof of an epoch and disputes
@@ -369,7 +363,7 @@ func (c *Client) CmdDeposit(status chan *CmdStatus, args ...string) {
 // Should be started in a go-routine.
 func (c *Client) BalanceProofWatcher() {
 	oldEpoch := uint64(0)
-	for {
+	for !c.IsClosed() {
 		proof, err := c.proofSub.BalanceProof(c.Ctx())
 		if err != nil {
 			c.logProof("Balance Proof error: %v", err)
@@ -395,11 +389,11 @@ func (c *Client) BalanceProofWatcher() {
 	}
 }
 
-// FrozenWatcher listens for Frozen events and calls WithdrawFrozen
+// frozenWatcher listens for Frozen events and calls WithdrawFrozen
 // if a balance proof is available.
-func (c *Client) FrozenWatcher() {
-	ctx, cancel := context.WithCancel(c.Ctx())
-	defer cancel()
+func (c *Client) frozenWatcher() {
+	var ctx context.Context
+	ctx, c.stopFrozenWatcher = context.WithCancel(c.Ctx())
 	sub, err := c.ethClient.SubscribeFrozen(ctx, c.contract, nil)
 	if err != nil {
 		c.logError("Frozen event subscription: %v", err)
@@ -408,6 +402,9 @@ func (c *Client) FrozenWatcher() {
 	defer sub.Unsubscribe()
 
 	select {
+	case <-ctx.Done():
+		c.logOnChain("Frozen watcher stopped")
+		return
 	case err := <-sub.Err():
 		c.logError("Frozen event subscription: %v", err)
 		return
@@ -419,15 +416,8 @@ func (c *Client) FrozenWatcher() {
 func (c *Client) handleFrozen(event *bindings.ErdstallFrozen) {
 	epoch := event.Epoch
 	c.setOpTrust(UNTRUSTED)
-	c.log("❄️ Contract Frozen in epoch #%d", epoch)
+	c.log("❄️Contract Frozen in epoch #%d", epoch)
 
-	c.balMtx.Lock()
-	defer c.balMtx.Unlock()
-	bal, ok := c.balances[epoch]
-	if !ok {
-		c.logOffChain("No balance-proof available for freeze")
-		return
-	}
 	status := make(chan *CmdStatus)
 	defer close(status)
 	go func() {
@@ -439,21 +429,33 @@ func (c *Client) handleFrozen(event *bindings.ErdstallFrozen) {
 			}
 		}
 	}()
+	c.withdrawFrozen(status, epoch)
+}
+
+func (c *Client) withdrawFrozen(status chan *CmdStatus, epoch uint64) {
+	c.balMtx.RLock()
+	bal, ok := c.balances[epoch]
+	c.balMtx.RUnlock()
+	if !ok {
+		c.logOffChain("No balance-proof available for freeze")
+		return
+	}
+
 	_, err := c.sendTx("WithdrawFrozen", func(auth *bind.TransactOpts) (*types.Transaction, error) {
-		return c.contract.WithdrawFrozen(auth, bindings.ErdstallBalance{Epoch: epoch, Account: bal.Account, Value: (*big.Int)(bal.Value)}, bal.Bal.Sig)
+		return c.contract.WithdrawFrozen(auth, bal.ToEthBal(), bal.Bal.Sig)
 	}, status)
 	if err != nil {
 		status <- &CmdStatus{Err: fmt.Errorf("WithdrawFrozen TX: %w", err)}
 		return
 	}
-	c.log("❄️ WithdrawFrozen: Complete")
+	c.logOnChain("❄️WithdrawFrozen: Complete")
 }
 
 func (c *Client) lastBal() *EpochBalance {
-	c.balMtx.Lock()
-	defer c.balMtx.Unlock()
-	epoch := c.params.ExitEpoch(atomic.LoadUint64(&c.lastBlock))
+	epoch := c.params.ExitEpoch(c.LastBlock())
+	c.balMtx.RLock()
 	bal, ok := c.balances[epoch]
+	c.balMtx.RUnlock()
 	if !ok || bal.Bal == nil || bal.Bal.Sig == nil {
 		return nil
 	}
@@ -473,7 +475,7 @@ func (c *Client) CmdLeave(status chan *CmdStatus, args ...string) {
 	}
 
 	rec, err := c.sendTx("Exit", func(opts *bind.TransactOpts) (*types.Transaction, error) {
-		return c.contract.Exit(opts, bindings.ErdstallBalance{Epoch: bal.Epoch, Account: bal.Account, Value: (*big.Int)(bal.Value)}, bal.Bal.Sig)
+		return c.contract.Exit(opts, bal.ToEthBal(), bal.Bal.Sig)
 	}, status)
 	if err != nil {
 		status <- &CmdStatus{Err: fmt.Errorf("Exit TX: %w", err)}
@@ -491,29 +493,60 @@ func (c *Client) CmdChallenge(status chan *CmdStatus, args ...string) {
 	}
 
 	// Wait for the next epoch, when we are currently in the response phase.
-	lastBlock := atomic.LoadUint64(&c.lastBlock)
-	if c.params.IsChallengeResponsePhase(lastBlock + 1) {
+	activeBlock := c.ActiveBlock()
+	if c.params.IsChallengeResponsePhase(activeBlock) {
 		status <- &CmdStatus{Msg: "Waiting for next epoch"}
-		next := c.params.DepositDoneBlock(c.params.ExitEpoch(lastBlock + 1))
+		next := c.params.DepositDoneBlock(c.params.DepositEpoch(activeBlock))
 		if err := c.ethClient.WaitForBlock(c.Ctx(), next); err != nil {
 			status <- &CmdStatus{Err: err}
 			return
 		}
 	}
 
+	// Get the BP of the sealed epoch.
+	activeBlock = c.ActiveBlock()
+	sealed := c.params.SealedEpoch(activeBlock)
+	c.balMtx.RLock()
+	bal, ok := c.balances[sealed]
+	c.balMtx.RUnlock()
+	if !ok || bal.Bal == nil {
+		status <- &CmdStatus{Err: fmt.Errorf("No Balance proof available")}
+		return
+	}
+
+	c.challenge(status, bal.Bal)
+}
+
+func (c *Client) challengeDeposit(status chan *CmdStatus) {
+	tx, err := c.sendTx("ChallengeDeposit", func(auth *bind.TransactOpts) (*types.Transaction, error) {
+		return c.contract.ChallengeDeposit(auth)
+	}, status)
+	if err != nil {
+		status <- &CmdStatus{Err: fmt.Errorf("ChallengeDeposit TX: %w", err)}
+		return
+	}
+	c.waitForResponseOrWithdraw(status, tx.BlockNumber.Uint64())
+}
+
+func (c *Client) challenge(status chan *CmdStatus, bp *tee.BalanceProof) {
 	tx, err := c.sendTx("Challenge", func(auth *bind.TransactOpts) (*types.Transaction, error) {
-		return c.contract.Challenge(auth)
+		return c.contract.Challenge(auth, bp.Balance.ToEthBal(), bp.Sig)
 	}, status)
 	if err != nil {
 		status <- &CmdStatus{Err: fmt.Errorf("Challenge TX: %w", err)}
 		return
 	}
+	c.waitForResponseOrWithdraw(status, tx.BlockNumber.Uint64())
+}
 
+// waitForResponseOrWithdraw waits for a challenge response. If it is not
+// received in time, it freezes the contract and withdraws.
+func (c *Client) waitForResponseOrWithdraw(status chan *CmdStatus, block uint64) {
 	// Wait for the operator til the end of the epoch and Freeze otherwise.
 	status <- &CmdStatus{Msg: "Exiting event: Waiting"}
 	subCtx, cancel := context.WithCancel(c.Ctx())
 	defer cancel()
-	exitEpoch := c.params.ExitEpoch(tx.BlockNumber.Uint64())
+	exitEpoch := c.params.ExitEpoch(block)
 	sub, err := c.ethClient.SubscribeExiting(subCtx, c.contract, []uint64{exitEpoch}, []common.Address{c.Address()})
 	if err != nil {
 		status <- &CmdStatus{Err: fmt.Errorf("Exiting subscription: %w", err)}
@@ -523,22 +556,22 @@ func (c *Client) CmdChallenge(status chan *CmdStatus, args ...string) {
 
 	done := make(chan struct{})
 	go func() {
-		next := c.params.DepositDoneBlock(c.params.DepositEpoch(tx.BlockNumber.Uint64()))
+		next := c.params.DepositDoneBlock(c.params.DepositEpoch(block))
 		c.ethClient.WaitForBlock(subCtx, next) // nolint: errcheck
 		close(done)
 	}()
 
 	select {
 	case <-done: // ChallengeReponse phase is over, freeze.
-		c.log("❄️ Freezing Contract")
-		_, err := c.sendTx("Freeze", func(auth *bind.TransactOpts) (*types.Transaction, error) {
-			return c.contract.Freeze(auth)
+		c.logOnChain("Freezing in epoch %d", exitEpoch)
+		c.stopFrozenWatcher()
+		_, err := c.sendTx("WithdrawChallenge", func(auth *bind.TransactOpts) (*types.Transaction, error) {
+			return c.contract.WithdrawChallenge(auth)
 		}, status)
 		if err != nil {
-			status <- &CmdStatus{Err: fmt.Errorf("Freezing TX: %w", err)}
+			status <- &CmdStatus{Err: fmt.Errorf("WithdrawChallenge TX: %w", err)}
 			return
 		}
-		// The FrozenWatcher will handle the withdrawal from hereon.
 	case err := <-sub.Err():
 		if err != nil {
 			c.logError("Exiting subscription: %v", err)
@@ -564,14 +597,8 @@ func (c *Client) sendTx(name string, f func(*bind.TransactOpts) (*types.Transact
 		return nil, err
 	}
 	status <- &CmdStatus{Msg: name + " TX: Mining"}
-	receipt, err := bind.WaitMined(txCtx(), c.ethClient, tx)
-	if err != nil {
-		return nil, err
-	}
-	if receipt.Status == types.ReceiptStatusFailed {
-		return nil, errors.New("Receipt failed")
-	}
-	return receipt, nil
+
+	return c.ethClient.ConfirmTransaction(txCtx(), tx, c.ethClient.Account())
 }
 
 func (c *Client) withdraw(exitEpoch uint64, status chan *CmdStatus) {
@@ -593,6 +620,7 @@ func (c *Client) withdraw(exitEpoch uint64, status chan *CmdStatus) {
 	c.logOnChain("Withdraw mined in block #%d", rec.BlockNumber.Uint64())
 	c.events <- &Event{Type: SET_BALANCE, Report: BalanceReport{Balance: big.NewInt(0)}}
 	c.setOpTrust(TRUSTED)
+	c.stopFrozenWatcher()
 }
 
 // writes to chainVvents
@@ -605,7 +633,7 @@ func (c *Client) listenOnChain() error {
 		subError <- c.ethClient.SubscribeEpochs(c.Ctx(), *c.params, epochs, blocks)
 	}()
 	go c.BalanceProofWatcher()
-	go c.FrozenWatcher()
+	go c.frozenWatcher()
 
 	for !c.IsClosed() {
 		select {
@@ -650,6 +678,14 @@ func (c *Client) Address() common.Address {
 	return c.ethClient.Account().Address
 }
 
+func (c *Client) ActiveBlock() uint64 {
+	return c.LastBlock() + 1
+}
+
+func (c *Client) LastBlock() uint64 {
+	return atomic.LoadUint64(&c.lastBlock)
+}
+
 func strToPerunAddress(str string) (pwallet.Address, error) {
 	if len(str) != 42 {
 		return nil, errors.New("Public keys must be chars 40 hex")