p2p: new dialer, peer management without locks

The most visible change is event-based dialing, which should be an
improvement over the timer-based system that we have at the moment.
The dialer gets a chance to compute new tasks whenever peers change or
dials complete. This is better than checking peers on a timer because
dials happen faster. The dialer can now make more precise decisions
about whom to dial based on the peer set and we can test those
decisions without actually opening any sockets.

Peer management is easier to test because the tests can inject
connections at checkpoints (after enc handshake, after protocol
handshake).

Most of the handshake stuff is now part of the RLPx code. It could be
exported or move to its own package because it is no longer entangled
with Server logic.

1 files changed, 440 insertions, 4 deletions

diff --git a/p2p/rlpx.go b/p2p/rlpx.go
index 6b533e275..e1cb13aae 100644
--- a/p2p/rlpx.go
+++ b/p2p/rlpx.go
@@ -4,23 +4,459 @@ import (
 	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/ecdsa"
+	"crypto/elliptic"
 	"crypto/hmac"
+	"crypto/rand"
 	"errors"
+	"fmt"
 	"hash"
 	"io"
+	"net"
+	"sync"
+	"time"
 
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/ecies"
+	"github.com/ethereum/go-ethereum/crypto/secp256k1"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
+const (
+	maxUint24 = ^uint32(0) >> 8
+
+	sskLen = 16 // ecies.MaxSharedKeyLength(pubKey) / 2
+	sigLen = 65 // elliptic S256
+	pubLen = 64 // 512 bit pubkey in uncompressed representation without format byte
+	shaLen = 32 // hash length (for nonce etc)
+
+	authMsgLen  = sigLen + shaLen + pubLen + shaLen + 1
+	authRespLen = pubLen + shaLen + 1
+
+	eciesBytes     = 65 + 16 + 32
+	encAuthMsgLen  = authMsgLen + eciesBytes  // size of the final ECIES payload sent as initiator's handshake
+	encAuthRespLen = authRespLen + eciesBytes // size of the final ECIES payload sent as receiver's handshake
+
+	// total timeout for encryption handshake and protocol
+	// handshake in both directions.
+	handshakeTimeout = 5 * time.Second
+
+	// This is the timeout for sending the disconnect reason.
+	// This is shorter than the usual timeout because we don't want
+	// to wait if the connection is known to be bad anyway.
+	discWriteTimeout = 1 * time.Second
+)
+
+// rlpx is the transport protocol used by actual (non-test) connections.
+// It wraps the frame encoder with locks and read/write deadlines.
+type rlpx struct {
+	fd net.Conn
+
+	rmu, wmu sync.Mutex
+	rw       *rlpxFrameRW
+}
+
+func newRLPX(fd net.Conn) transport {
+	fd.SetDeadline(time.Now().Add(handshakeTimeout))
+	return &rlpx{fd: fd}
+}
+
+func (t *rlpx) ReadMsg() (Msg, error) {
+	t.rmu.Lock()
+	defer t.rmu.Unlock()
+	t.fd.SetReadDeadline(time.Now().Add(frameReadTimeout))
+	return t.rw.ReadMsg()
+}
+
+func (t *rlpx) WriteMsg(msg Msg) error {
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	t.fd.SetWriteDeadline(time.Now().Add(frameWriteTimeout))
+	return t.rw.WriteMsg(msg)
+}
+
+func (t *rlpx) close(err error) {
+	t.wmu.Lock()
+	defer t.wmu.Unlock()
+	// Tell the remote end why we're disconnecting if possible.
+	if t.rw != nil {
+		if r, ok := err.(DiscReason); ok && r != DiscNetworkError {
+			t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout))
+			SendItems(t.rw, discMsg, r)
+		}
+	}
+	t.fd.Close()
+}
+
+// doEncHandshake runs the protocol handshake using authenticated
+// messages. the protocol handshake is the first authenticated message
+// and also verifies whether the encryption handshake 'worked' and the
+// remote side actually provided the right public key.
+func (t *rlpx) doProtoHandshake(our *protoHandshake) (their *protoHandshake, err error) {
+	// Writing our handshake happens concurrently, we prefer
+	// returning the handshake read error. If the remote side
+	// disconnects us early with a valid reason, we should return it
+	// as the error so it can be tracked elsewhere.
+	werr := make(chan error, 1)
+	go func() { werr <- Send(t.rw, handshakeMsg, our) }()
+	if their, err = readProtocolHandshake(t.rw, our); err != nil {
+		return nil, err
+	}
+	if err := <-werr; err != nil {
+		return nil, fmt.Errorf("write error: %v", err)
+	}
+	return their, nil
+}
+
+func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake, error) {
+	msg, err := rw.ReadMsg()
+	if err != nil {
+		return nil, err
+	}
+	if msg.Size > baseProtocolMaxMsgSize {
+		return nil, fmt.Errorf("message too big")
+	}
+	if msg.Code == discMsg {
+		// Disconnect before protocol handshake is valid according to the
+		// spec and we send it ourself if the posthanshake checks fail.
+		// We can't return the reason directly, though, because it is echoed
+		// back otherwise. Wrap it in a string instead.
+		var reason [1]DiscReason
+		rlp.Decode(msg.Payload, &reason)
+		return nil, reason[0]
+	}
+	if msg.Code != handshakeMsg {
+		return nil, fmt.Errorf("expected handshake, got %x", msg.Code)
+	}
+	var hs protoHandshake
+	if err := msg.Decode(&hs); err != nil {
+		return nil, err
+	}
+	// validate handshake info
+	if hs.Version != our.Version {
+		return nil, DiscIncompatibleVersion
+	}
+	if (hs.ID == discover.NodeID{}) {
+		return nil, DiscInvalidIdentity
+	}
+	return &hs, nil
+}
+
+func (t *rlpx) doEncHandshake(prv *ecdsa.PrivateKey, dial *discover.Node) (discover.NodeID, error) {
+	var (
+		sec secrets
+		err error
+	)
+	if dial == nil {
+		sec, err = receiverEncHandshake(t.fd, prv, nil)
+	} else {
+		sec, err = initiatorEncHandshake(t.fd, prv, dial.ID, nil)
+	}
+	if err != nil {
+		return discover.NodeID{}, err
+	}
+	t.wmu.Lock()
+	t.rw = newRLPXFrameRW(t.fd, sec)
+	t.wmu.Unlock()
+	return sec.RemoteID, nil
+}
+
+// encHandshake contains the state of the encryption handshake.
+type encHandshake struct {
+	initiator bool
+	remoteID  discover.NodeID
+
+	remotePub            *ecies.PublicKey  // remote-pubk
+	initNonce, respNonce []byte            // nonce
+	randomPrivKey        *ecies.PrivateKey // ecdhe-random
+	remoteRandomPub      *ecies.PublicKey  // ecdhe-random-pubk
+}
+
+// secrets represents the connection secrets
+// which are negotiated during the encryption handshake.
+type secrets struct {
+	RemoteID              discover.NodeID
+	AES, MAC              []byte
+	EgressMAC, IngressMAC hash.Hash
+	Token                 []byte
+}
+
+// secrets is called after the handshake is completed.
+// It extracts the connection secrets from the handshake values.
+func (h *encHandshake) secrets(auth, authResp []byte) (secrets, error) {
+	ecdheSecret, err := h.randomPrivKey.GenerateShared(h.remoteRandomPub, sskLen, sskLen)
+	if err != nil {
+		return secrets{}, err
+	}
+
+	// derive base secrets from ephemeral key agreement
+	sharedSecret := crypto.Sha3(ecdheSecret, crypto.Sha3(h.respNonce, h.initNonce))
+	aesSecret := crypto.Sha3(ecdheSecret, sharedSecret)
+	s := secrets{
+		RemoteID: h.remoteID,
+		AES:      aesSecret,
+		MAC:      crypto.Sha3(ecdheSecret, aesSecret),
+		Token:    crypto.Sha3(sharedSecret),
+	}
+
+	// setup sha3 instances for the MACs
+	mac1 := sha3.NewKeccak256()
+	mac1.Write(xor(s.MAC, h.respNonce))
+	mac1.Write(auth)
+	mac2 := sha3.NewKeccak256()
+	mac2.Write(xor(s.MAC, h.initNonce))
+	mac2.Write(authResp)
+	if h.initiator {
+		s.EgressMAC, s.IngressMAC = mac1, mac2
+	} else {
+		s.EgressMAC, s.IngressMAC = mac2, mac1
+	}
+
+	return s, nil
+}
+
+func (h *encHandshake) ecdhShared(prv *ecdsa.PrivateKey) ([]byte, error) {
+	return ecies.ImportECDSA(prv).GenerateShared(h.remotePub, sskLen, sskLen)
+}
+
+// initiatorEncHandshake negotiates a session token on conn.
+// it should be called on the dialing side of the connection.
+//
+// prv is the local client's private key.
+// token is the token from a previous session with this node.
+func initiatorEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey, remoteID discover.NodeID, token []byte) (s secrets, err error) {
+	h, err := newInitiatorHandshake(remoteID)
+	if err != nil {
+		return s, err
+	}
+	auth, err := h.authMsg(prv, token)
+	if err != nil {
+		return s, err
+	}
+	if _, err = conn.Write(auth); err != nil {
+		return s, err
+	}
+
+	response := make([]byte, encAuthRespLen)
+	if _, err = io.ReadFull(conn, response); err != nil {
+		return s, err
+	}
+	if err := h.decodeAuthResp(response, prv); err != nil {
+		return s, err
+	}
+	return h.secrets(auth, response)
+}
+
+func newInitiatorHandshake(remoteID discover.NodeID) (*encHandshake, error) {
+	// generate random initiator nonce
+	n := make([]byte, shaLen)
+	if _, err := rand.Read(n); err != nil {
+		return nil, err
+	}
+	// generate random keypair to use for signing
+	randpriv, err := ecies.GenerateKey(rand.Reader, crypto.S256(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rpub, err := remoteID.Pubkey()
+	if err != nil {
+		return nil, fmt.Errorf("bad remoteID: %v", err)
+	}
+	h := &encHandshake{
+		initiator:     true,
+		remoteID:      remoteID,
+		remotePub:     ecies.ImportECDSAPublic(rpub),
+		initNonce:     n,
+		randomPrivKey: randpriv,
+	}
+	return h, nil
+}
+
+// authMsg creates an encrypted initiator handshake message.
+func (h *encHandshake) authMsg(prv *ecdsa.PrivateKey, token []byte) ([]byte, error) {
+	var tokenFlag byte
+	if token == nil {
+		// no session token found means we need to generate shared secret.
+		// ecies shared secret is used as initial session token for new peers
+		// generate shared key from prv and remote pubkey
+		var err error
+		if token, err = h.ecdhShared(prv); err != nil {
+			return nil, err
+		}
+	} else {
+		// for known peers, we use stored token from the previous session
+		tokenFlag = 0x01
+	}
+
+	// sign known message:
+	//   ecdh-shared-secret^nonce for new peers
+	//   token^nonce for old peers
+	signed := xor(token, h.initNonce)
+	signature, err := crypto.Sign(signed, h.randomPrivKey.ExportECDSA())
+	if err != nil {
+		return nil, err
+	}
+
+	// encode auth message
+	// signature || sha3(ecdhe-random-pubk) || pubk || nonce || token-flag
+	msg := make([]byte, authMsgLen)
+	n := copy(msg, signature)
+	n += copy(msg[n:], crypto.Sha3(exportPubkey(&h.randomPrivKey.PublicKey)))
+	n += copy(msg[n:], crypto.FromECDSAPub(&prv.PublicKey)[1:])
+	n += copy(msg[n:], h.initNonce)
+	msg[n] = tokenFlag
+
+	// encrypt auth message using remote-pubk
+	return ecies.Encrypt(rand.Reader, h.remotePub, msg, nil, nil)
+}
+
+// decodeAuthResp decode an encrypted authentication response message.
+func (h *encHandshake) decodeAuthResp(auth []byte, prv *ecdsa.PrivateKey) error {
+	msg, err := crypto.Decrypt(prv, auth)
+	if err != nil {
+		return fmt.Errorf("could not decrypt auth response (%v)", err)
+	}
+	h.respNonce = msg[pubLen : pubLen+shaLen]
+	h.remoteRandomPub, err = importPublicKey(msg[:pubLen])
+	if err != nil {
+		return err
+	}
+	// ignore token flag for now
+	return nil
+}
+
+// receiverEncHandshake negotiates a session token on conn.
+// it should be called on the listening side of the connection.
+//
+// prv is the local client's private key.
+// token is the token from a previous session with this node.
+func receiverEncHandshake(conn io.ReadWriter, prv *ecdsa.PrivateKey, token []byte) (s secrets, err error) {
+	// read remote auth sent by initiator.
+	auth := make([]byte, encAuthMsgLen)
+	if _, err := io.ReadFull(conn, auth); err != nil {
+		return s, err
+	}
+	h, err := decodeAuthMsg(prv, token, auth)
+	if err != nil {
+		return s, err
+	}
+
+	// send auth response
+	resp, err := h.authResp(prv, token)
+	if err != nil {
+		return s, err
+	}
+	if _, err = conn.Write(resp); err != nil {
+		return s, err
+	}
+
+	return h.secrets(auth, resp)
+}
+
+func decodeAuthMsg(prv *ecdsa.PrivateKey, token []byte, auth []byte) (*encHandshake, error) {
+	var err error
+	h := new(encHandshake)
+	// generate random keypair for session
+	h.randomPrivKey, err = ecies.GenerateKey(rand.Reader, crypto.S256(), nil)
+	if err != nil {
+		return nil, err
+	}
+	// generate random nonce
+	h.respNonce = make([]byte, shaLen)
+	if _, err = rand.Read(h.respNonce); err != nil {
+		return nil, err
+	}
+
+	msg, err := crypto.Decrypt(prv, auth)
+	if err != nil {
+		return nil, fmt.Errorf("could not decrypt auth message (%v)", err)
+	}
+
+	// decode message parameters
+	// signature || sha3(ecdhe-random-pubk) || pubk || nonce || token-flag
+	h.initNonce = msg[authMsgLen-shaLen-1 : authMsgLen-1]
+	copy(h.remoteID[:], msg[sigLen+shaLen:sigLen+shaLen+pubLen])
+	rpub, err := h.remoteID.Pubkey()
+	if err != nil {
+		return nil, fmt.Errorf("bad remoteID: %#v", err)
+	}
+	h.remotePub = ecies.ImportECDSAPublic(rpub)
+
+	// recover remote random pubkey from signed message.
+	if token == nil {
+		// TODO: it is an error if the initiator has a token and we don't. check that.
+
+		// no session token means we need to generate shared secret.
+		// ecies shared secret is used as initial session token for new peers.
+		// generate shared key from prv and remote pubkey.
+		if token, err = h.ecdhShared(prv); err != nil {
+			return nil, err
+		}
+	}
+	signedMsg := xor(token, h.initNonce)
+	remoteRandomPub, err := secp256k1.RecoverPubkey(signedMsg, msg[:sigLen])
+	if err != nil {
+		return nil, err
+	}
+	h.remoteRandomPub, _ = importPublicKey(remoteRandomPub)
+	return h, nil
+}
+
+// authResp generates the encrypted authentication response message.
+func (h *encHandshake) authResp(prv *ecdsa.PrivateKey, token []byte) ([]byte, error) {
+	// responder auth message
+	// E(remote-pubk, ecdhe-random-pubk || nonce || 0x0)
+	resp := make([]byte, authRespLen)
+	n := copy(resp, exportPubkey(&h.randomPrivKey.PublicKey))
+	n += copy(resp[n:], h.respNonce)
+	if token == nil {
+		resp[n] = 0
+	} else {
+		resp[n] = 1
+	}
+	// encrypt using remote-pubk
+	return ecies.Encrypt(rand.Reader, h.remotePub, resp, nil, nil)
+}
+
+// importPublicKey unmarshals 512 bit public keys.
+func importPublicKey(pubKey []byte) (*ecies.PublicKey, error) {
+	var pubKey65 []byte
+	switch len(pubKey) {
+	case 64:
+		// add 'uncompressed key' flag
+		pubKey65 = append([]byte{0x04}, pubKey...)
+	case 65:
+		pubKey65 = pubKey
+	default:
+		return nil, fmt.Errorf("invalid public key length %v (expect 64/65)", len(pubKey))
+	}
+	// TODO: fewer pointless conversions
+	return ecies.ImportECDSAPublic(crypto.ToECDSAPub(pubKey65)), nil
+}
+
+func exportPubkey(pub *ecies.PublicKey) []byte {
+	if pub == nil {
+		panic("nil pubkey")
+	}
+	return elliptic.Marshal(pub.Curve, pub.X, pub.Y)[1:]
+}
+
+func xor(one, other []byte) (xor []byte) {
+	xor = make([]byte, len(one))
+	for i := 0; i < len(one); i++ {
+		xor[i] = one[i] ^ other[i]
+	}
+	return xor
+}
+
 var (
 	// this is used in place of actual frame header data.
 	// TODO: replace this when Msg contains the protocol type code.
 	zeroHeader = []byte{0xC2, 0x80, 0x80}
-
 	// sixteen zero bytes
 	zero16 = make([]byte, 16)
-
-	maxUint24 = ^uint32(0) >> 8
 )
 
 // rlpxFrameRW implements a simplified version of RLPx framing.
@@ -38,7 +474,7 @@ type rlpxFrameRW struct {
 	ingressMAC hash.Hash
 }
 
-func newRlpxFrameRW(conn io.ReadWriter, s secrets) *rlpxFrameRW {
+func newRLPXFrameRW(conn io.ReadWriter, s secrets) *rlpxFrameRW {
 	macc, err := aes.NewCipher(s.MAC)
 	if err != nil {
 		panic("invalid MAC secret: " + err.Error())