health.go: add a timeout duration

Add a duration parameter to the health command that will allow a timeout to fail the health check. The duration is specified in seconds.

Change-Id: I825e725413dea3334d15d8e61a0677cb1e918a61

cmd/health.go

@@ -15,25 +15,52 @@ import (
 )
 
 var table string
-var number int
+var number, timeout int
 
 func healthCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "health",
 		Short: "wait until cluster is healthy (10 storagenodes are registered in the db)",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return checkHealth(table, number)
+			return checkHealthWithTimeout(table, number, timeout)
 		},
 	}
 	cmd.PersistentFlags().StringVarP(&table, "table", "t", "nodes", "table to use for health check")
 	cmd.PersistentFlags().IntVarP(&number, "number", "n", 10, "number of entries to expect in the table")
+	cmd.PersistentFlags().IntVarP(&timeout, "duration", "d", 0, "time to wait (in seconds) for health check")
 	return cmd
 }
 
 func init() {
 	rootCmd.AddCommand(healthCmd())
 }
 
+// checkHealthWithTimeout polls the database until all storagenodes are checked in, or the timeout is exceeded. a timeout of 0 (default)
+// means no timeout.
+func checkHealthWithTimeout(table string, records, timeout int) error {
+	if timeout == 0 {
+		return checkHealth(table, records)
+	}
+	c1 := make(chan error, 1)
+	go func() {
+		err := checkHealth(table, records)
+		if err != nil {
+			c1 <- err
+		}
+		c1 <- nil
+	}()
+
+	select {
+	case err := <-c1:
+		if err != nil {
+			return err
+		}
+	case <-time.After(time.Duration(timeout) * time.Second):
+		return fmt.Errorf("health check failed. duration limit reached")
+	}
+	return nil
+}
+
 // checkHealth polls the database until all storagenodes are checked in.
 func checkHealth(table string, records int) error {
 	prevCount := -1

test/test-invoice.sh

@@ -49,7 +49,7 @@ ADDRESS=$(curl -X GET -s http://localhost:10000/api/v0/payments/wallet --header
 #ACCOUNT is defined with environment variables above
 for i in {1..15}; do cethacea token transfer 10 0x"$ADDRESS"; done
 
-storj-up health -t billing_transactions -n 3
+storj-up health -t billing_transactions -n 3 -d 12
 
 curl -X GET http://localhost:10000/api/v0/payments/wallet --header "$COOKIE"
 curl -X POST http://localhost:10000/api/v0/payments/account --header "$COOKIE"

test/test-reorg.sh

@@ -49,7 +49,7 @@ ADDRESS=$(curl -X GET -s http://localhost:10000/api/v0/payments/wallet --header
 
 #15 transactions means 3 are fully confirmed
 for i in {1..15}; do cethacea token transfer 1000 0x"$ADDRESS"; done
-storj-up health -t billing_transactions -n 3
+storj-up health -t billing_transactions -n 3 -d 12
 
 #save the last transaction of the base chain
 CONTENT=$(curl -s -X GET http://localhost:10000/api/v0/payments/wallet/payments --header "$COOKIE")
@@ -64,7 +64,7 @@ wait_for_geth
 
 #adding 5 transactions for a total of 20 means 8 should be fully confirmed
 for i in {1..5}; do cethacea token transfer 1 0x"$ADDRESS"; done
-storj-up health -t billing_transactions -n 8
+storj-up health -t billing_transactions -n 8 -d 12
 
 #save the last transaction (0) and compare to the base chain
 CONTENT=$(curl -s -X GET http://localhost:10000/api/v0/payments/wallet/payments --header "$COOKIE")
@@ -82,11 +82,11 @@ docker compose up -d
 
 wait_for_geth
 
-storj-up health -t storjscan_payments -n 15
+storj-up health -t storjscan_payments -n 15 -d 12
 
 #adding 5 different transactions to simulate reorg
 for i in {1..5}; do cethacea token transfer 1000 0x"$ADDRESS"; done
-storj-up health -t storjscan_payments -n 20
+storj-up health -t storjscan_payments -n 20 -d 12
 
 CONTENT=$(curl -s -X GET http://localhost:10000/api/v0/payments/wallet/payments --header "$COOKIE")
 POSTREORG4=$(jq -r '.payments[4].ID' <<< "${CONTENT}")

test/test-storjscan.sh

@@ -36,7 +36,7 @@ ADDRESS=$(curl -X GET -s http://localhost:10000/api/v0/payments/wallet --header
 #ACCOUNT is defined with environment variables above
 for i in {1..15}; do cethacea token transfer 1000 0x"$ADDRESS"; done
 
-storj-up health -t billing_transactions -n 3
+storj-up health -t billing_transactions -n 3 -d 12
 
 curl -X GET http://localhost:10000/api/v0/payments/wallet --header "$COOKIE"