core: add cache for DKG MPK and complaint (#334)

Since deserializing DKG related items are extremely slow (takes about 3
seconds for 100 items), we cache it in the governance interface.

2 files changed, 90 insertions, 36 deletions

diff --git a/core/governance.go b/core/governance.go
index 855c85910..23e528a2f 100644
--- a/core/governance.go
+++ b/core/governance.go
@@ -4,6 +4,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
+	"sync"
 	"time"
 
 	coreCommon "github.com/dexon-foundation/dexon-consensus/common"
@@ -13,15 +14,17 @@ import (
 	coreTypes "github.com/dexon-foundation/dexon-consensus/core/types"
 	dkgTypes "github.com/dexon-foundation/dexon-consensus/core/types/dkg"
 	coreUtils "github.com/dexon-foundation/dexon-consensus/core/utils"
+	"github.com/hashicorp/golang-lru/simplelru"
 
 	"github.com/dexon-foundation/dexon/common"
 	"github.com/dexon-foundation/dexon/core/state"
 	"github.com/dexon-foundation/dexon/core/vm"
 	"github.com/dexon-foundation/dexon/crypto"
 	"github.com/dexon-foundation/dexon/log"
-	"github.com/dexon-foundation/dexon/rlp"
 )
 
+const dkgCacheSize = 5
+
 type GovernanceStateDB interface {
 	State() (*state.StateDB, error)
 	StateAt(height uint64) (*state.StateDB, error)
@@ -47,13 +50,32 @@ func (g *governanceStateDB) StateAt(height uint64) (*state.StateDB, error) {
 	return g.bc.StateAt(header.Root)
 }
 
+type dkgCacheItem struct {
+	Round               uint64
+	Reset               uint64
+	MasterPublicKeysLen uint64
+	MasterPublicKeys    []*dkgTypes.MasterPublicKey
+	ComplaintsLen       uint64
+	Complaints          []*dkgTypes.Complaint
+}
+
 type Governance struct {
 	db           GovernanceStateDB
 	nodeSetCache *dexCore.NodeSetCache
+	dkgCache     *simplelru.LRU
+	dkgCacheMu   sync.RWMutex
 }
 
 func NewGovernance(db GovernanceStateDB) *Governance {
-	g := &Governance{db: db}
+	cache, err := simplelru.NewLRU(dkgCacheSize, nil)
+	if err != nil {
+		log.Error("Failed to initialize DKG cache", "error", err)
+		return nil
+	}
+	g := &Governance{
+		db:       db,
+		dkgCache: cache,
+	}
 	g.nodeSetCache = dexCore.NewNodeSetCache(g)
 	return g
 }
@@ -210,29 +232,60 @@ func (g *Governance) DKGSetNodeKeyAddresses(round uint64) (map[common.Address]st
 	return r, nil
 }
 
-func (g *Governance) DKGComplaints(round uint64) []*dkgTypes.Complaint {
+func (g *Governance) getOrUpdateDKGCache(round uint64) *dkgCacheItem {
 	s := g.GetStateForDKGAtRound(round)
 	if s == nil {
+		log.Error("Failed to get DKG state", "round", round)
 		return nil
 	}
+	reset := s.DKGResetCount(new(big.Int).SetUint64(round))
+	mpksLen := s.LenDKGMasterPublicKeys().Uint64()
+	compsLen := s.LenDKGComplaints().Uint64()
 
-	var dkgComplaints []*dkgTypes.Complaint
-	for _, pk := range s.DKGComplaints() {
-		x := new(dkgTypes.Complaint)
-		if err := rlp.DecodeBytes(pk, x); err != nil {
-			panic(err)
-		}
-		dkgComplaints = append(dkgComplaints, x)
+	var cache *dkgCacheItem
+
+	g.dkgCacheMu.RLock()
+	if v, ok := g.dkgCache.Get(round); ok {
+		cache = v.(*dkgCacheItem)
+	}
+	g.dkgCacheMu.RUnlock()
+
+	if cache != nil && cache.Reset == reset.Uint64() &&
+		cache.MasterPublicKeysLen == mpksLen &&
+		cache.ComplaintsLen == compsLen {
+		return cache
+	}
+
+	g.dkgCacheMu.Lock()
+	defer g.dkgCacheMu.Unlock()
+
+	cache = &dkgCacheItem{
+		Round: round,
+		Reset: reset.Uint64(),
+	}
+
+	if cache == nil || cache.MasterPublicKeysLen != mpksLen {
+		cache.MasterPublicKeys = s.DKGMasterPublicKeyItems()
+		cache.MasterPublicKeysLen = uint64(len(cache.MasterPublicKeys))
 	}
-	return dkgComplaints
+
+	if cache == nil || cache.ComplaintsLen != compsLen {
+		cache.Complaints = s.DKGComplaintItems()
+		cache.ComplaintsLen = uint64(len(cache.Complaints))
+	}
+
+	g.dkgCache.Add(round, cache)
+	return cache
+}
+
+func (g *Governance) DKGComplaints(round uint64) []*dkgTypes.Complaint {
+	cache := g.getOrUpdateDKGCache(round)
+	return cache.Complaints
 }
 
 func (g *Governance) DKGMasterPublicKeys(round uint64) []*dkgTypes.MasterPublicKey {
-	s := g.GetStateForDKGAtRound(round)
-	if s == nil {
-		return nil
-	}
-	return s.UniqueDKGMasterPublicKeys()
+	cache := g.getOrUpdateDKGCache(round)
+	return cache.MasterPublicKeys
 }
 
 func (g *Governance) IsDKGMPKReady(round uint64) bool {
diff --git a/core/vm/oracle_contracts.go b/core/vm/oracle_contracts.go
index b794ac8e5..10459a870 100644
--- a/core/vm/oracle_contracts.go
+++ b/core/vm/oracle_contracts.go
@@ -599,27 +599,23 @@ func (s *GovernanceState) IncDKGResetCount(round *big.Int) {
 }
 
 // bytes[] public dkgMasterPublicKeys;
+func (s *GovernanceState) LenDKGMasterPublicKeys() *big.Int {
+	return s.getStateBigInt(big.NewInt(dkgMasterPublicKeysLoc))
+}
 func (s *GovernanceState) DKGMasterPublicKeys() [][]byte {
 	return s.read1DByteArray(big.NewInt(dkgMasterPublicKeysLoc))
 }
 func (s *GovernanceState) PushDKGMasterPublicKey(mpk []byte) {
 	s.appendTo1DByteArray(big.NewInt(dkgMasterPublicKeysLoc), mpk)
 }
-func (s *GovernanceState) UniqueDKGMasterPublicKeys() []*dkgTypes.MasterPublicKey {
+func (s *GovernanceState) DKGMasterPublicKeyItems() []*dkgTypes.MasterPublicKey {
 	// Prepare DKGMasterPublicKeys.
 	var dkgMasterPKs []*dkgTypes.MasterPublicKey
-	existence := make(map[coreTypes.NodeID]struct{})
 	for _, mpk := range s.DKGMasterPublicKeys() {
 		x := new(dkgTypes.MasterPublicKey)
 		if err := rlp.DecodeBytes(mpk, x); err != nil {
 			panic(err)
 		}
-
-		// Only the first DKG MPK submission is valid.
-		if _, exists := existence[x.ProposerID]; exists {
-			continue
-		}
-		existence[x.ProposerID] = struct{}{}
 		dkgMasterPKs = append(dkgMasterPKs, x)
 	}
 	return dkgMasterPKs
@@ -666,6 +662,9 @@ func (s *GovernanceState) ClearDKGMasterPublicKeyProposed() {
 }
 
 // bytes[] public dkgComplaints;
+func (s *GovernanceState) LenDKGComplaints() *big.Int {
+	return s.getStateBigInt(big.NewInt(dkgComplaintsLoc))
+}
 func (s *GovernanceState) DKGComplaints() [][]byte {
 	return s.read1DByteArray(big.NewInt(dkgComplaintsLoc))
 }
@@ -675,6 +674,17 @@ func (s *GovernanceState) PushDKGComplaint(complaint []byte) {
 func (s *GovernanceState) ClearDKGComplaints() {
 	s.erase1DByteArray(big.NewInt(dkgComplaintsLoc))
 }
+func (s *GovernanceState) DKGComplaintItems() []*dkgTypes.Complaint {
+	var dkgComplaints []*dkgTypes.Complaint
+	for _, pk := range s.DKGComplaints() {
+		x := new(dkgTypes.Complaint)
+		if err := rlp.DecodeBytes(pk, x); err != nil {
+			panic(err)
+		}
+		dkgComplaints = append(dkgComplaints, x)
+	}
+	return dkgComplaints
+}
 
 // mapping(bytes32 => bool) public dkgComplaintsProposed;
 func (s *GovernanceState) DKGComplaintProposed(id Bytes32) bool {
@@ -1244,19 +1254,10 @@ func (c *defaultCoreDKGUtils) NewGroupPublicKey(
 	state *GovernanceState, round *big.Int, threshold int) (tsigVerifierIntf, error) {
 
 	// Prepare DKGMasterPublicKeys.
-	mpks := state.UniqueDKGMasterPublicKeys()
-
-	// Prepare DKGComplaints.
-	var complaints []*dkgTypes.Complaint
-	for _, comp := range state.DKGComplaints() {
-		x := new(dkgTypes.Complaint)
-		if err := rlp.DecodeBytes(comp, x); err != nil {
-			panic(err)
-		}
-		complaints = append(complaints, x)
-	}
+	mpks := state.DKGMasterPublicKeyItems()
+	comps := state.DKGComplaintItems()
 
-	return dkgTypes.NewGroupPublicKey(round.Uint64(), mpks, complaints, threshold)
+	return dkgTypes.NewGroupPublicKey(round.Uint64(), mpks, comps, threshold)
 }
 
 func (g *GovernanceContract) Address() common.Address {