build: add end to end integration test for recovery mechanism (#336)

4 files changed, 104 insertions, 3 deletions

diff --git a/.travis.yml b/.travis.yml
index fdefcc545..f59b817b9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -76,6 +76,11 @@ matrix:
         - ./run_test.sh --ignore-log
         - cd ..
         - ./build/fullnode-test.sh
+        - pkill -15 -f gdex
+        - cd test
+        - ./run_test.sh --continue --ignore-log
+        - cd ..
+        - ./build/recovery-test.sh
       addons:
         homebrew:
           packages:
diff --git a/build/recovery-test.sh b/build/recovery-test.sh
new file mode 100755
index 000000000..f305f743c
--- /dev/null
+++ b/build/recovery-test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+tarAndUpload()
+{
+  name=travis-fail-$(date +%s).tar.gz
+  tar -zcvf $name test
+  echo "Verify fail and upload $name"
+  PKG_CONFIG_PATH=/usr/local/opt/openssl/lib/pkgconfig go run build/testtool/testtool.go upload $name dexon-builds
+}
+
+endpoint=http://127.0.0.1:8545
+
+timeout=700
+
+echo "Wait for recovery"
+cmd="PKG_CONFIG_PATH=/usr/local/opt/openssl/lib/pkgconfig go run build/testtool/testtool.go waitForRecovery $endpoint $timeout"
+eval $cmd
+code=$?
+if [ $code == 1 ]; then
+  tarAndUpload
+  exit 1
+fi
diff --git a/build/testtool/testtool.go b/build/testtool/testtool.go
index 3f880b3bd..a801cfbb3 100644
--- a/build/testtool/testtool.go
+++ b/build/testtool/testtool.go
@@ -16,6 +16,7 @@ import (
 	"github.com/dexon-foundation/dexon"
 	"github.com/dexon-foundation/dexon/accounts/abi"
 	"github.com/dexon-foundation/dexon/cmd/zoo/monkey"
+	"github.com/dexon-foundation/dexon/core/types"
 	"github.com/dexon-foundation/dexon/core/vm"
 	"github.com/dexon-foundation/dexon/crypto"
 	"github.com/dexon-foundation/dexon/ethclient"
@@ -34,6 +35,8 @@ func main() {
 		doVerifyGovMPK(os.Args[2:])
 	case "monkeyTest":
 		doMonkeyTest(os.Args[2:])
+	case "waitForRecovery":
+		doWaitForRecovery(os.Args[2:])
 	case "upload":
 		doUpload(os.Args[2:])
 	}
@@ -206,3 +209,58 @@ func doMonkeyTest(args []string) {
 		}
 	}
 }
+
+func doWaitForRecovery(args []string) {
+	if len(args) < 2 {
+		log.Fatal("arg length is not enough")
+	}
+
+	client, err := ethclient.Dial(args[0])
+	if err != nil {
+		log.Fatalf("new ethclient fail: %v", err)
+	}
+
+	lastBlock, err := client.BlockByNumber(context.Background(), nil)
+	if err != nil {
+		log.Fatalf("get last block fail %v", err)
+	}
+	t, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		log.Fatalf("timeout args invalid, %s", args[1])
+	}
+
+	sleep := time.Duration(t) * time.Second
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	go func() {
+		for ctx.Err() == nil {
+			select {
+			case <-time.After(1 * time.Minute):
+				log.Printf("tick")
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	log.Printf("Sleep %s", sleep)
+	time.Sleep(sleep)
+	cancel()
+
+	// Check block height is increasing 5 time for safe.
+	var block *types.Block
+	for i := 0; i < 5; i++ {
+		block, err = client.BlockByNumber(context.Background(), nil)
+		if err != nil {
+			log.Fatalf("get current block fail, err=%v, i=%d", err, i)
+		}
+		if block.NumberU64() <= lastBlock.NumberU64() {
+			log.Fatalf("recovery fail, last=%d, current=%d",
+				lastBlock.NumberU64(), block.NumberU64())
+		}
+		log.Printf("last=%d, current=%d, ok, sleep a while", lastBlock.NumberU64(), block.NumberU64())
+		lastBlock = block
+		time.Sleep(2 * time.Second)
+	}
+}
diff --git a/test/run_test.sh b/test/run_test.sh
index b6db1ba09..1f19caa6a 100755
--- a/test/run_test.sh
+++ b/test/run_test.sh
@@ -6,6 +6,22 @@ GENESIS="genesis.json"
 GDEX="../build/bin/gdex"
 BOOTNODE="../build/bin/bootnode"
 
+
+CONTINUE=false
+IGNORELOG=false
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --continue)
+    CONTINUE=true
+    ;;
+    --ignore-log)
+    IGNORELOG=true
+    ;;
+  esac
+  shift
+done
+
+
 if [ ! -e "$BOOTNODE" ]; then
   echo "Building bootnode for the first time ..."
   go build -o $BOOTNODE ../cmd/bootnode
@@ -42,7 +58,7 @@ __FILE__
 
 # A standalone RPC server for accepting RPC requests.
 datadir=$PWD/Dexon.rpc
-if [ "$1" != "--continue" ]; then
+if ! $CONTINUE; then
   rm -rf $datadir
   $GDEX --datadir=$datadir init ${GENESIS}
 fi
@@ -64,7 +80,7 @@ NUM_NODES=$(cat ${GENESIS} | grep 'DEXON Test Node' | wc -l)
 for i in $(seq 0 $(($NUM_NODES - 1))); do
   datadir=$PWD/Dexon.$i
 
-  if [ "$1" != "--continue" ]; then
+  if ! $CONTINUE; then
     rm -rf $datadir
     $GDEX --datadir=$datadir init ${GENESIS}
   fi
@@ -85,6 +101,6 @@ for i in $(seq 0 $(($NUM_NODES - 1))); do
     > $logsdir/gdex.$i.log 2>&1 &
 done
 
-if [ "$1" != "--ignore-log" ]; then
+if ! $IGNORELOG; then
   tail -f $logsdir/gdex.*.log
 fi