core: do not panic in deliveryGuard but send a stop message instead

5 files changed, 19 insertions, 17 deletions

diff --git a/core/consensus.go b/core/consensus.go
index ed6c182..8024b54 100644
--- a/core/consensus.go
+++ b/core/consensus.go
@@ -1007,7 +1007,7 @@ func (con *Consensus) prepare(initBlock *types.Block) (err error) {
 }
 
 // Run starts running DEXON Consensus.
-func (con *Consensus) Run() {
+func (con *Consensus) Run(stopChan chan<- struct{}) {
 	// There may have emptys block in blockchain added by force sync.
 	blocksWithoutRandomness := con.bcModule.pendingBlocksWithoutRandomness()
 	// Launch BA routines.
@@ -1047,7 +1047,7 @@ func (con *Consensus) Run() {
 	// Take some time to bootstrap.
 	time.Sleep(3 * time.Second)
 	con.waitGroup.Add(1)
-	go con.deliveryGuard()
+	go con.deliveryGuard(stopChan)
 	// Block until done.
 	select {
 	case <-con.ctx.Done():
@@ -1442,7 +1442,7 @@ func (con *Consensus) processFinalizedBlock(b *types.Block) (err error) {
 	return
 }
 
-func (con *Consensus) deliveryGuard() {
+func (con *Consensus) deliveryGuard(stopChan chan<- struct{}) {
 	defer con.waitGroup.Done()
 	select {
 	case <-con.ctx.Done():
@@ -1465,7 +1465,9 @@ func (con *Consensus) deliveryGuard() {
 		case <-con.resetDeliveryGuardTicker:
 		case <-time.After(60 * time.Second):
 			con.logger.Error("No blocks delivered for too long", "ID", con.ID)
-			panic(fmt.Errorf("No blocks delivered for too long"))
+			stopChan <- struct{}{}
+			con.ctxCancel()
+			return
 		}
 	}
 }
diff --git a/core/consensus_test.go b/core/consensus_test.go
index aebb47b..eb5ab82 100644
--- a/core/consensus_test.go
+++ b/core/consensus_test.go
@@ -352,7 +352,7 @@ func (s *ConsensusTestSuite) TestSyncBA() {
 	s.Require().NoError(err)
 	prvKey := prvKeys[0]
 	_, con := s.prepareConsensus(time.Now().UTC(), gov, prvKey, conn)
-	go con.Run()
+	go con.Run(make(chan struct{}))
 	defer con.Stop()
 	hash := common.NewRandomHash()
 	signers := make([]*utils.Signer, 0, len(prvKeys))
diff --git a/integration_test/byzantine_test.go b/integration_test/byzantine_test.go
index 4d8e7ed..72639f7 100644
--- a/integration_test/byzantine_test.go
+++ b/integration_test/byzantine_test.go
@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/suite"
 	"gitlab.com/byzantine-lab/tangerine-consensus/common"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/crypto"
@@ -33,7 +34,6 @@ import (
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/test"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/types"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/utils"
-	"github.com/stretchr/testify/suite"
 )
 
 // There is no scheduler in these tests, we need to wait a long period to make
@@ -169,7 +169,7 @@ func (s *ByzantineTestSuite) TestOneSlowNodeOneDeadNode() {
 		if n.ID == deadNodeID {
 			continue
 		}
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 		defer n.con.Stop()
 	}
 	// Clean deadNode's network receive channel, or it might exceed the limit
@@ -235,7 +235,7 @@ func (s *ByzantineTestSuite) TestOneNodeWithoutVote() {
 	votelessNode := nodes[votelessNodeID]
 	votelessNode.network.SetCensor(&voteCensor{}, &voteCensor{})
 	for _, n := range nodes {
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 		defer n.con.Stop()
 	}
 Loop:
diff --git a/integration_test/consensus_test.go b/integration_test/consensus_test.go
index 57384f3..2fc60b9 100644
--- a/integration_test/consensus_test.go
+++ b/integration_test/consensus_test.go
@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/suite"
 	"gitlab.com/byzantine-lab/tangerine-consensus/common"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/crypto"
@@ -34,7 +35,6 @@ import (
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/test"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/types"
 	"gitlab.com/byzantine-lab/tangerine-consensus/core/utils"
-	"github.com/stretchr/testify/suite"
 )
 
 // There is no scheduler in these tests, we need to wait a long period to make
@@ -286,7 +286,7 @@ func (s *ConsensusTestSuite) TestSimple() {
 	// A short round interval.
 	nodes := s.setupNodes(dMoment, prvKeys, seedGov)
 	for _, n := range nodes {
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 		defer n.con.Stop()
 	}
 Loop:
@@ -366,7 +366,7 @@ func (s *ConsensusTestSuite) TestSetSizeChange() {
 		5, test.StateChangeNotarySetSize, uint32(4)))
 	// Run test.
 	for _, n := range nodes {
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 		defer n.con.Stop()
 	}
 Loop:
@@ -431,7 +431,7 @@ func (s *ConsensusTestSuite) TestSync() {
 		n.rEvt.Register(govHandlerGen(1, 0, n.gov, prohibitDKG))
 		n.rEvt.Register(govHandlerGen(1, 1, n.gov, unprohibitDKG))
 		if n.ID != syncNode.ID {
-			go n.con.Run()
+			go n.con.Run(make(chan struct{}))
 			if n.ID != stoppedNode.ID {
 				defer n.con.Stop()
 			}
@@ -497,7 +497,7 @@ ReachAlive:
 				stoppedNode, syncNode, syncerObj, syncedHeight)
 			if syncedCon != nil {
 				syncNode.con = syncedCon
-				go syncNode.con.Run()
+				go syncNode.con.Run(make(chan struct{}))
 				go func() {
 					<-runnerCtx.Done()
 					syncNode.con.Stop()
@@ -586,7 +586,7 @@ func (s *ConsensusTestSuite) TestForceSync() {
 	// A short round interval.
 	nodes := s.setupNodes(dMoment, prvKeys, seedGov)
 	for _, n := range nodes {
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 	}
 ReachStop:
 	for {
@@ -690,7 +690,7 @@ ReachStop:
 		nodes[nID].con = con
 	}
 	for _, node := range nodes {
-		go node.con.Run()
+		go node.con.Run(make(chan struct{}))
 		defer node.con.Stop()
 	}
 
@@ -744,7 +744,7 @@ func (s *ConsensusTestSuite) TestResetDKG() {
 		n.rEvt.Register(govHandlerGen(1, 2, n.gov, unprohibitDKG))
 		n.rEvt.Register(govHandlerGen(2, 0, n.gov, prohibitDKGExceptFinalize))
 		n.rEvt.Register(govHandlerGen(2, 1, n.gov, unprohibitDKG))
-		go n.con.Run()
+		go n.con.Run(make(chan struct{}))
 	}
 Loop:
 	for {
diff --git a/simulation/node.go b/simulation/node.go
index f7c242f..0c2dfa6 100644
--- a/simulation/node.go
+++ b/simulation/node.go
@@ -172,7 +172,7 @@ readyLoop:
 		n.netModule,
 		n.prvKey,
 		n.logger)
-	go n.consensus.Run()
+	go n.consensus.Run(make(chan struct{}))
 
 	// Blocks forever.
 MainLoop: