diff options
45 files changed, 2140 insertions, 2293 deletions
diff --git a/accounts/abi/bind/backends/remote.go b/accounts/abi/bind/backends/remote.go index 4793143e4..58edd791a 100644 --- a/accounts/abi/bind/backends/remote.go +++ b/accounts/abi/bind/backends/remote.go @@ -17,11 +17,7 @@ package backends import ( - "encoding/json" - "fmt" "math/big" - "sync" - "sync/atomic" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -37,119 +33,34 @@ var _ bind.ContractBackend = (*rpcBackend)(nil) // rpcBackend implements bind.ContractBackend, and acts as the data provider to // Ethereum contracts bound to Go structs. It uses an RPC connection to delegate // all its functionality. -// -// Note: The current implementation is a blocking one. This should be replaced -// by a proper async version when a real RPC client is created. type rpcBackend struct { - client rpc.Client // RPC client connection to interact with an API server - autoid uint32 // ID number to use for the next API request - lock sync.Mutex // Singleton access until we get to request multiplexing + client *rpc.Client // RPC client connection to interact with an API server } // NewRPCBackend creates a new binding backend to an RPC provider that can be // used to interact with remote contracts. -func NewRPCBackend(client rpc.Client) bind.ContractBackend { - return &rpcBackend{ - client: client, - } -} - -// request is a JSON RPC request package assembled internally from the client -// method calls. -type request struct { - JSONRPC string `json:"jsonrpc"` // Version of the JSON RPC protocol, always set to 2.0 - ID int `json:"id"` // Auto incrementing ID number for this request - Method string `json:"method"` // Remote procedure name to invoke on the server - Params []interface{} `json:"params"` // List of parameters to pass through (keep types simple) -} - -// response is a JSON RPC response package sent back from the API server. -type response struct { - JSONRPC string `json:"jsonrpc"` // Version of the JSON RPC protocol, always set to 2.0 - ID int `json:"id"` // Auto incrementing ID number for this request - Error *failure `json:"error"` // Any error returned by the remote side - Result json.RawMessage `json:"result"` // Whatever the remote side sends us in reply -} - -// failure is a JSON RPC response error field sent back from the API server. -type failure struct { - Code int `json:"code"` // JSON RPC error code associated with the failure - Message string `json:"message"` // Specific error message of the failure -} - -// request forwards an API request to the RPC server, and parses the response. -// -// This is currently painfully non-concurrent, but it will have to do until we -// find the time for niceties like this :P -func (b *rpcBackend) request(ctx context.Context, method string, params []interface{}) (json.RawMessage, error) { - b.lock.Lock() - defer b.lock.Unlock() - - if ctx == nil { - ctx = context.Background() - } - - // Ugly hack to serialize an empty list properly - if params == nil { - params = []interface{}{} - } - // Assemble the request object - reqID := int(atomic.AddUint32(&b.autoid, 1)) - req := &request{ - JSONRPC: "2.0", - ID: reqID, - Method: method, - Params: params, - } - if err := b.client.Send(req); err != nil { - return nil, err - } - res := new(response) - errc := make(chan error, 1) - go func() { - errc <- b.client.Recv(res) - }() - select { - case err := <-errc: - if err != nil { - return nil, err - } - case <-ctx.Done(): - return nil, ctx.Err() - } - if res.Error != nil { - if res.Error.Message == bind.ErrNoCode.Error() { - return nil, bind.ErrNoCode - } - return nil, fmt.Errorf("remote error: %s", res.Error.Message) - } - return res.Result, nil +func NewRPCBackend(client *rpc.Client) bind.ContractBackend { + return &rpcBackend{client: client} } // HasCode implements ContractVerifier.HasCode by retrieving any code associated // with the contract from the remote node, and checking its size. func (b *rpcBackend) HasCode(ctx context.Context, contract common.Address, pending bool) (bool, error) { - // Execute the RPC code retrieval block := "latest" if pending { block = "pending" } - res, err := b.request(ctx, "eth_getCode", []interface{}{contract.Hex(), block}) - if err != nil { - return false, err - } var hex string - if err := json.Unmarshal(res, &hex); err != nil { + err := b.client.CallContext(ctx, &hex, "eth_getCode", contract, block) + if err != nil { return false, err } - // Convert the response back to a Go byte slice and return return len(common.FromHex(hex)) > 0, nil } // ContractCall implements ContractCaller.ContractCall, delegating the execution of // a contract call to the remote node, returning the reply to for local processing. func (b *rpcBackend) ContractCall(ctx context.Context, contract common.Address, data []byte, pending bool) ([]byte, error) { - // Pack up the request into an RPC argument args := struct { To common.Address `json:"to"` Data string `json:"data"` @@ -157,63 +68,43 @@ func (b *rpcBackend) ContractCall(ctx context.Context, contract common.Address, To: contract, Data: common.ToHex(data), } - // Execute the RPC call and retrieve the response block := "latest" if pending { block = "pending" } - res, err := b.request(ctx, "eth_call", []interface{}{args, block}) - if err != nil { - return nil, err - } var hex string - if err := json.Unmarshal(res, &hex); err != nil { + err := b.client.CallContext(ctx, &hex, "eth_call", args, block) + if err != nil { return nil, err } - // Convert the response back to a Go byte slice and return return common.FromHex(hex), nil + } // PendingAccountNonce implements ContractTransactor.PendingAccountNonce, delegating // the current account nonce retrieval to the remote node. func (b *rpcBackend) PendingAccountNonce(ctx context.Context, account common.Address) (uint64, error) { - res, err := b.request(ctx, "eth_getTransactionCount", []interface{}{account.Hex(), "pending"}) + var hex rpc.HexNumber + err := b.client.CallContext(ctx, &hex, "eth_getTransactionCount", account.Hex(), "pending") if err != nil { return 0, err } - var hex string - if err := json.Unmarshal(res, &hex); err != nil { - return 0, err - } - nonce, ok := new(big.Int).SetString(hex, 0) - if !ok { - return 0, fmt.Errorf("invalid nonce hex: %s", hex) - } - return nonce.Uint64(), nil + return hex.Uint64(), nil } // SuggestGasPrice implements ContractTransactor.SuggestGasPrice, delegating the // gas price oracle request to the remote node. func (b *rpcBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) { - res, err := b.request(ctx, "eth_gasPrice", nil) - if err != nil { - return nil, err - } - var hex string - if err := json.Unmarshal(res, &hex); err != nil { + var hex rpc.HexNumber + if err := b.client.CallContext(ctx, &hex, "eth_gasPrice"); err != nil { return nil, err } - price, ok := new(big.Int).SetString(hex, 0) - if !ok { - return nil, fmt.Errorf("invalid price hex: %s", hex) - } - return price, nil + return (*big.Int)(&hex), nil } // EstimateGasLimit implements ContractTransactor.EstimateGasLimit, delegating // the gas estimation to the remote node. func (b *rpcBackend) EstimateGasLimit(ctx context.Context, sender common.Address, contract *common.Address, value *big.Int, data []byte) (*big.Int, error) { - // Pack up the request into an RPC argument args := struct { From common.Address `json:"from"` To *common.Address `json:"to"` @@ -226,19 +117,12 @@ func (b *rpcBackend) EstimateGasLimit(ctx context.Context, sender common.Address Value: rpc.NewHexNumber(value), } // Execute the RPC call and retrieve the response - res, err := b.request(ctx, "eth_estimateGas", []interface{}{args}) + var hex rpc.HexNumber + err := b.client.CallContext(ctx, &hex, "eth_estimateGas", args) if err != nil { return nil, err } - var hex string - if err := json.Unmarshal(res, &hex); err != nil { - return nil, err - } - estimate, ok := new(big.Int).SetString(hex, 0) - if !ok { - return nil, fmt.Errorf("invalid estimate hex: %s", hex) - } - return estimate, nil + return (*big.Int)(&hex), nil } // SendTransaction implements ContractTransactor.SendTransaction, delegating the @@ -248,13 +132,5 @@ func (b *rpcBackend) SendTransaction(ctx context.Context, tx *types.Transaction) if err != nil { return err } - res, err := b.request(ctx, "eth_sendRawTransaction", []interface{}{common.ToHex(data)}) - if err != nil { - return err - } - var hex string - if err := json.Unmarshal(res, &hex); err != nil { - return err - } - return nil + return b.client.CallContext(ctx, nil, "eth_sendRawTransaction", common.ToHex(data)) } diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 257050a62..8d53809ce 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -19,9 +19,12 @@ package main import ( "os" "os/signal" + "strings" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" "gopkg.in/urfave/cli.v1" ) @@ -99,7 +102,7 @@ func localConsole(ctx *cli.Context) error { // console to it. func remoteConsole(ctx *cli.Context) error { // Attach to a remotely running geth instance and start the JavaScript console - client, err := utils.NewRemoteRPCClient(ctx) + client, err := dialRPC(ctx.Args().First()) if err != nil { utils.Fatalf("Unable to attach to remote geth: %v", err) } @@ -127,6 +130,20 @@ func remoteConsole(ctx *cli.Context) error { return nil } +// dialRPC returns a RPC client which connects to the given endpoint. +// The check for empty endpoint implements the defaulting logic +// for "geth attach" and "geth monitor" with no argument. +func dialRPC(endpoint string) (*rpc.Client, error) { + if endpoint == "" { + endpoint = node.DefaultIPCEndpoint() + } else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { + // Backwards compatibility with geth < 1.5 which required + // these prefixes. + endpoint = endpoint[4:] + } + return rpc.Dial(endpoint) +} + // ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript // console to it, and each of the files specified as arguments and tears the // everything down. diff --git a/cmd/geth/monitorcmd.go b/cmd/geth/monitorcmd.go index 11fdca89c..d1490dce2 100644 --- a/cmd/geth/monitorcmd.go +++ b/cmd/geth/monitorcmd.go @@ -21,11 +21,10 @@ import ( "math" "reflect" "runtime" + "sort" "strings" "time" - "sort" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -36,7 +35,7 @@ import ( var ( monitorCommandAttachFlag = cli.StringFlag{ Name: "attach", - Value: "ipc:" + node.DefaultIPCEndpoint(), + Value: node.DefaultIPCEndpoint(), Usage: "API endpoint to attach to", } monitorCommandRowsFlag = cli.IntFlag{ @@ -69,12 +68,12 @@ to display multiple metrics simultaneously. // monitor starts a terminal UI based monitoring tool for the requested metrics. func monitor(ctx *cli.Context) error { var ( - client rpc.Client + client *rpc.Client err error ) // Attach to an Ethereum node over IPC or RPC endpoint := ctx.String(monitorCommandAttachFlag.Name) - if client, err = utils.NewRemoteRPCClientFromString(endpoint); err != nil { + if client, err = dialRPC(endpoint); err != nil { utils.Fatalf("Unable to attach to geth node: %v", err) } defer client.Close() @@ -159,30 +158,10 @@ func monitor(ctx *cli.Context) error { // retrieveMetrics contacts the attached geth node and retrieves the entire set // of collected system metrics. -func retrieveMetrics(client rpc.Client) (map[string]interface{}, error) { - req := map[string]interface{}{ - "id": new(int64), - "method": "debug_metrics", - "jsonrpc": "2.0", - "params": []interface{}{true}, - } - - if err := client.Send(req); err != nil { - return nil, err - } - - var res rpc.JSONSuccessResponse - if err := client.Recv(&res); err != nil { - return nil, err - } - - if res.Result != nil { - if mets, ok := res.Result.(map[string]interface{}); ok { - return mets, nil - } - } - - return nil, fmt.Errorf("unable to retrieve metrics") +func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) { + var metrics map[string]interface{} + err := client.Call(&metrics, "debug_metrics", true) + return metrics, err } // resolveMetrics takes a list of input metric patterns, and resolves each to one @@ -270,7 +249,7 @@ func fetchMetric(metrics map[string]interface{}, metric string) float64 { // refreshCharts retrieves a next batch of metrics, and inserts all the new // values into the active datasets and charts -func refreshCharts(client rpc.Client, metrics []string, data [][]float64, units []int, charts []*termui.LineChart, ctx *cli.Context, footer *termui.Par) (realign bool) { +func refreshCharts(client *rpc.Client, metrics []string, data [][]float64, units []int, charts []*termui.LineChart, ctx *cli.Context, footer *termui.Par) (realign bool) { values, err := retrieveMetrics(client) for i, metric := range metrics { if len(data) < 512 { diff --git a/cmd/utils/client.go b/cmd/utils/client.go deleted file mode 100644 index cc9647580..000000000 --- a/cmd/utils/client.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. - -package utils - -import ( - "fmt" - "strings" - - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" -) - -// NewRemoteRPCClient returns a RPC client which connects to a running geth instance. -// Depending on the given context this can either be a IPC or a HTTP client. -func NewRemoteRPCClient(ctx *cli.Context) (rpc.Client, error) { - if ctx.Args().Present() { - endpoint := ctx.Args().First() - return NewRemoteRPCClientFromString(endpoint) - } - // use IPC by default - return rpc.NewIPCClient(node.DefaultIPCEndpoint()) -} - -// NewRemoteRPCClientFromString returns a RPC client which connects to the given -// endpoint. It must start with either `ipc:` or `rpc:` (HTTP). -func NewRemoteRPCClientFromString(endpoint string) (rpc.Client, error) { - if strings.HasPrefix(endpoint, "ipc:") { - return rpc.NewIPCClient(endpoint[4:]) - } - if strings.HasPrefix(endpoint, "rpc:") { - return rpc.NewHTTPClient(endpoint[4:]) - } - if strings.HasPrefix(endpoint, "http://") { - return rpc.NewHTTPClient(endpoint) - } - if strings.HasPrefix(endpoint, "ws:") { - return rpc.NewWSClient(endpoint) - } - return nil, fmt.Errorf("invalid endpoint") -} diff --git a/console/bridge.go b/console/bridge.go index b23e06837..06cb41d80 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -31,13 +31,13 @@ import ( // bridge is a collection of JavaScript utility methods to bride the .js runtime // environment and the Go RPC connection backing the remote method calls. type bridge struct { - client rpc.Client // RPC client to execute Ethereum requests through + client *rpc.Client // RPC client to execute Ethereum requests through prompter UserPrompter // Input prompter to allow interactive user feedback printer io.Writer // Output writer to serialize any display strings to } // newBridge creates a new JavaScript wrapper around an RPC client. -func newBridge(client rpc.Client, prompter UserPrompter, printer io.Writer) *bridge { +func newBridge(client *rpc.Client, prompter UserPrompter, printer io.Writer) *bridge { return &bridge{ client: client, prompter: prompter, @@ -188,88 +188,86 @@ func (b *bridge) SleepBlocks(call otto.FunctionCall) (response otto.Value) { return otto.FalseValue() } -// Send will serialize the first argument, send it to the node and returns the response. +type jsonrpcCall struct { + Id int64 + Method string + Params []interface{} +} + +// Send implements the web3 provider "send" method. func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { - // Ensure that we've got a batch request (array) or a single request (object) - arg := call.Argument(0).Object() - if arg == nil || (arg.Class() != "Array" && arg.Class() != "Object") { - throwJSException("request must be an object or array") - } - // Convert the otto VM arguments to Go values - data, err := call.Otto.Call("JSON.stringify", nil, arg) + // Remarshal the request into a Go value. + JSON, _ := call.Otto.Object("JSON") + reqVal, err := JSON.Call("stringify", call.Argument(0)) if err != nil { throwJSException(err.Error()) } - reqjson, err := data.ToString() - if err != nil { - throwJSException(err.Error()) - } - var ( - reqs []rpc.JSONRequest - batch = true + rawReq = []byte(reqVal.String()) + reqs []jsonrpcCall + batch bool ) - if err = json.Unmarshal([]byte(reqjson), &reqs); err != nil { - // single request? - reqs = make([]rpc.JSONRequest, 1) - if err = json.Unmarshal([]byte(reqjson), &reqs[0]); err != nil { - throwJSException("invalid request") - } + if rawReq[0] == '[' { + batch = true + json.Unmarshal(rawReq, &reqs) + } else { batch = false + reqs = make([]jsonrpcCall, 1) + json.Unmarshal(rawReq, &reqs[0]) } - // Iteratively execute the requests - call.Otto.Set("response_len", len(reqs)) - call.Otto.Run("var ret_response = new Array(response_len);") - for i, req := range reqs { - // Execute the RPC request and parse the reply - if err = b.client.Send(&req); err != nil { - return newErrorResponse(call, -32603, err.Error(), req.Id) - } - result := make(map[string]interface{}) - if err = b.client.Recv(&result); err != nil { - return newErrorResponse(call, -32603, err.Error(), req.Id) + // Execute the requests. + resps, _ := call.Otto.Object("new Array()") + for _, req := range reqs { + resp, _ := call.Otto.Object(`({"jsonrpc":"2.0"})`) + resp.Set("id", req.Id) + var result json.RawMessage + err = b.client.Call(&result, req.Method, req.Params...) + switch err := err.(type) { + case nil: + if result == nil { + // Special case null because it is decoded as an empty + // raw message for some reason. + resp.Set("result", otto.NullValue()) + } else { + resultVal, err := JSON.Call("parse", string(result)) + if err != nil { + resp = newErrorResponse(call, -32603, err.Error(), &req.Id).Object() + } else { + resp.Set("result", resultVal) + } + } + case rpc.Error: + resp.Set("error", map[string]interface{}{ + "code": err.ErrorCode(), + "message": err.Error(), + }) + default: + resp = newErrorResponse(call, -32603, err.Error(), &req.Id).Object() } - // Feed the reply back into the JavaScript runtime environment - id, _ := result["id"] - jsonver, _ := result["jsonrpc"] - - call.Otto.Set("ret_id", id) - call.Otto.Set("ret_jsonrpc", jsonver) - call.Otto.Set("response_idx", i) + resps.Call("push", resp) + } - if res, ok := result["result"]; ok { - payload, _ := json.Marshal(res) - call.Otto.Set("ret_result", string(payload)) - response, err = call.Otto.Run(` - ret_response[response_idx] = { jsonrpc: ret_jsonrpc, id: ret_id, result: JSON.parse(ret_result) }; - `) - continue - } - if res, ok := result["error"]; ok { - payload, _ := json.Marshal(res) - call.Otto.Set("ret_result", string(payload)) - response, err = call.Otto.Run(` - ret_response[response_idx] = { jsonrpc: ret_jsonrpc, id: ret_id, error: JSON.parse(ret_result) }; - `) - continue - } - return newErrorResponse(call, -32603, fmt.Sprintf("Invalid response"), new(int64)) + // Return the responses either to the callback (if supplied) + // or directly as the return value. + if batch { + response = resps.Value() + } else { + response, _ = resps.Get("0") } - // Convert single requests back from batch ones - if !batch { - call.Otto.Run("ret_response = ret_response[0];") + if fn := call.Argument(1).Object(); fn != nil && fn.Class() == "function" { + fn.Call("apply", response) + return otto.UndefinedValue() } - // Execute any registered callbacks - if call.Argument(1).IsObject() { - call.Otto.Set("callback", call.Argument(1)) - call.Otto.Run(` - if (Object.prototype.toString.call(callback) == '[object Function]') { - callback(null, ret_response); - } - `) - } - return + return response +} + +func newErrorResponse(call otto.FunctionCall, code int, msg string, id interface{}) otto.Value { + // Bundle the error into a JSON RPC call response + m := map[string]interface{}{"version": "2.0", "id": id, "error": map[string]interface{}{"code": code, msg: msg}} + res, _ := json.Marshal(m) + val, _ := call.Otto.Run("(" + string(res) + ")") + return val } // throwJSException panics on an otto.Value. The Otto VM will recover from the @@ -281,37 +279,3 @@ func throwJSException(msg interface{}) otto.Value { } panic(val) } - -// newErrorResponse creates a JSON RPC error response for a specific request id, -// containing the specified error code and error message. Beside returning the -// error to the caller, it also sets the ret_error and ret_response JavaScript -// variables. -func newErrorResponse(call otto.FunctionCall, code int, msg string, id interface{}) (response otto.Value) { - // Bundle the error into a JSON RPC call response - res := rpc.JSONErrResponse{ - Version: rpc.JSONRPCVersion, - Id: id, - Error: rpc.JSONError{ - Code: code, - Message: msg, - }, - } - // Serialize the error response into JavaScript variables - errObj, err := json.Marshal(res.Error) - if err != nil { - glog.V(logger.Error).Infof("Failed to serialize JSON RPC error: %v", err) - } - resObj, err := json.Marshal(res) - if err != nil { - glog.V(logger.Error).Infof("Failed to serialize JSON RPC error response: %v", err) - } - - if _, err = call.Otto.Run("ret_error = " + string(errObj)); err != nil { - glog.V(logger.Error).Infof("Failed to set `ret_error` to the occurred error: %v", err) - } - resVal, err := call.Otto.Run("ret_response = " + string(resObj)) - if err != nil { - glog.V(logger.Error).Infof("Failed to set `ret_response` to the JSON RPC response: %v", err) - } - return resVal -} diff --git a/console/console.go b/console/console.go index 00d1fea1d..f224f0c2e 100644 --- a/console/console.go +++ b/console/console.go @@ -52,7 +52,7 @@ const DefaultPrompt = "> " type Config struct { DataDir string // Data directory to store the console history at DocRoot string // Filesystem path from where to load JavaScript files from - Client rpc.Client // RPC client to execute Ethereum requests through + Client *rpc.Client // RPC client to execute Ethereum requests through Prompt string // Input prompt prefix string (defaults to DefaultPrompt) Prompter UserPrompter // Input prompter to allow interactive user feedback (defaults to TerminalPrompter) Printer io.Writer // Output writer to serialize any display strings to (defaults to os.Stdout) @@ -63,7 +63,7 @@ type Config struct { // JavaScript console attached to a running node via an external or in-process RPC // client. type Console struct { - client rpc.Client // RPC client to execute Ethereum requests through + client *rpc.Client // RPC client to execute Ethereum requests through jsre *jsre.JSRE // JavaScript runtime environment running the interpreter prompt string // Input prompt prefix string prompter UserPrompter // Input prompter to allow interactive user feedback diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a10253b8e..aee21122a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -48,23 +48,17 @@ var ( MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request MaxStateFetch = 384 // Amount of node state values to allow fetching per request - MaxForkAncestry = 3 * params.EpochDuration.Uint64() // Maximum chain reorganisation - - hashTTL = 3 * time.Second // [eth/61] Time it takes for a hash request to time out - blockTargetRTT = 3 * time.Second / 2 // [eth/61] Target time for completing a block retrieval request - blockTTL = 3 * blockTargetRTT // [eth/61] Maximum time allowance before a block request is considered expired - - rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests - rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value - ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion - ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts + MaxForkAncestry = 3 * params.EpochDuration.Uint64() // Maximum chain reorganisation + rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests + rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests + rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value + ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion + ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts qosTuningPeers = 5 // Number of peers to tune based on (best peers) qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence qosTuningImpact = 0.25 // Impact that a new tuning target has on the previous value - maxQueuedHashes = 32 * 1024 // [eth/61] Maximum number of hashes to queue for import (DOS protection) maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) maxHeadersProcess = 2048 // Number of header download results to import at once into the chain maxResultsProcess = 2048 // Number of content download results to import at once into the chain @@ -84,16 +78,13 @@ var ( errStallingPeer = errors.New("peer is stalling") errNoPeers = errors.New("no peers to keep download active") errTimeout = errors.New("timeout") - errEmptyHashSet = errors.New("empty hash set by peer") errEmptyHeaderSet = errors.New("empty header set by peer") errPeersUnavailable = errors.New("no peers available or all tried for download") - errAlreadyInPool = errors.New("hash already in pool") errInvalidAncestor = errors.New("retrieved ancestor is invalid") errInvalidChain = errors.New("retrieved hash chain is invalid") errInvalidBlock = errors.New("retrieved block is invalid") errInvalidBody = errors.New("retrieved block body is invalid") errInvalidReceipt = errors.New("retrieved receipt is invalid") - errCancelHashFetch = errors.New("hash download canceled (requested)") errCancelBlockFetch = errors.New("block download canceled (requested)") errCancelHeaderFetch = errors.New("block header download canceled (requested)") errCancelBodyFetch = errors.New("block body download canceled (requested)") @@ -102,6 +93,7 @@ var ( errCancelHeaderProcessing = errors.New("header processing canceled (requested)") errCancelContentProcessing = errors.New("content processing canceled (requested)") errNoSyncActive = errors.New("no sync active") + errTooOld = errors.New("peer doesn't speak recent enough protocol version (need version >= 62)") ) type Downloader struct { @@ -146,13 +138,10 @@ type Downloader struct { // Channels newPeerCh chan *peer - hashCh chan dataPack // [eth/61] Channel receiving inbound hashes - blockCh chan dataPack // [eth/61] Channel receiving inbound blocks headerCh chan dataPack // [eth/62] Channel receiving inbound block headers bodyCh chan dataPack // [eth/62] Channel receiving inbound block bodies receiptCh chan dataPack // [eth/63] Channel receiving inbound receipts stateCh chan dataPack // [eth/63] Channel receiving inbound node state data - blockWakeCh chan bool // [eth/61] Channel to signal the block fetcher of new tasks bodyWakeCh chan bool // [eth/62] Channel to signal the block body fetcher of new tasks receiptWakeCh chan bool // [eth/63] Channel to signal the receipt fetcher of new tasks stateWakeCh chan bool // [eth/63] Channel to signal the state fetcher of new tasks @@ -199,13 +188,10 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha rollback: rollback, dropPeer: dropPeer, newPeerCh: make(chan *peer, 1), - hashCh: make(chan dataPack, 1), - blockCh: make(chan dataPack, 1), headerCh: make(chan dataPack, 1), bodyCh: make(chan dataPack, 1), receiptCh: make(chan dataPack, 1), stateCh: make(chan dataPack, 1), - blockWakeCh: make(chan bool, 1), bodyWakeCh: make(chan bool, 1), receiptWakeCh: make(chan bool, 1), stateWakeCh: make(chan bool, 1), @@ -251,12 +237,11 @@ func (d *Downloader) Synchronising() bool { // RegisterPeer injects a new download peer into the set of block source to be // used for fetching hashes and blocks from. func (d *Downloader) RegisterPeer(id string, version int, head common.Hash, - getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, getReceipts receiptFetcherFn, getNodeData stateFetcherFn) error { glog.V(logger.Detail).Infoln("Registering peer", id) - if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil { + if err := d.peers.Register(newPeer(id, version, head, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil { glog.V(logger.Error).Infoln("Register failed:", err) return err } @@ -291,7 +276,9 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode case errBusy: glog.V(logger.Detail).Infof("Synchronisation already in progress") - case errTimeout, errBadPeer, errStallingPeer, errEmptyHashSet, errEmptyHeaderSet, errPeersUnavailable, errInvalidAncestor, errInvalidChain: + case errTimeout, errBadPeer, errStallingPeer, + errEmptyHeaderSet, errPeersUnavailable, errTooOld, + errInvalidAncestor, errInvalidChain: glog.V(logger.Debug).Infof("Removing peer %v: %v", id, err) d.dropPeer(id) @@ -323,13 +310,13 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode d.queue.Reset() d.peers.Reset() - for _, ch := range []chan bool{d.blockWakeCh, d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} { + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} { select { case <-ch: default: } } - for _, ch := range []chan dataPack{d.hashCh, d.blockCh, d.headerCh, d.bodyCh, d.receiptCh, d.stateCh} { + for _, ch := range []chan dataPack{d.headerCh, d.bodyCh, d.receiptCh, d.stateCh} { for empty := false; !empty; { select { case <-ch: @@ -377,105 +364,73 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e d.mux.Post(DoneEvent{}) } }() + if p.version < 62 { + return errTooOld + } glog.V(logger.Debug).Infof("Synchronising with the network using: %s [eth/%d]", p.id, p.version) defer func(start time.Time) { glog.V(logger.Debug).Infof("Synchronisation terminated after %v", time.Since(start)) }(time.Now()) - switch { - case p.version == 61: - // Look up the sync boundaries: the common ancestor and the target block - latest, err := d.fetchHeight61(p) - if err != nil { - return err - } - origin, err := d.findAncestor61(p, latest) - if err != nil { - return err - } - d.syncStatsLock.Lock() - if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { - d.syncStatsChainOrigin = origin - } - d.syncStatsChainHeight = latest - d.syncStatsLock.Unlock() + // Look up the sync boundaries: the common ancestor and the target block + latest, err := d.fetchHeight(p) + if err != nil { + return err + } + height := latest.Number.Uint64() - // Initiate the sync using a concurrent hash and block retrieval algorithm - d.queue.Prepare(origin+1, d.mode, 0, nil) - if d.syncInitHook != nil { - d.syncInitHook(origin, latest) - } - return d.spawnSync(origin+1, - func() error { return d.fetchHashes61(p, td, origin+1) }, - func() error { return d.fetchBlocks61(origin + 1) }, - ) - - case p.version >= 62: - // Look up the sync boundaries: the common ancestor and the target block - latest, err := d.fetchHeight(p) - if err != nil { - return err - } - height := latest.Number.Uint64() + origin, err := d.findAncestor(p, height) + if err != nil { + return err + } + d.syncStatsLock.Lock() + if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { + d.syncStatsChainOrigin = origin + } + d.syncStatsChainHeight = height + d.syncStatsLock.Unlock() - origin, err := d.findAncestor(p, height) - if err != nil { - return err - } - d.syncStatsLock.Lock() - if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { - d.syncStatsChainOrigin = origin - } - d.syncStatsChainHeight = height - d.syncStatsLock.Unlock() - - // Initiate the sync using a concurrent header and content retrieval algorithm - pivot := uint64(0) - switch d.mode { - case LightSync: - pivot = height - case FastSync: - // Calculate the new fast/slow sync pivot point - if d.fsPivotLock == nil { - pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval))) - if err != nil { - panic(fmt.Sprintf("Failed to access crypto random source: %v", err)) - } - if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() { - pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64() - } - } else { - // Pivot point locked in, use this and do not pick a new one! - pivot = d.fsPivotLock.Number.Uint64() + // Initiate the sync using a concurrent header and content retrieval algorithm + pivot := uint64(0) + switch d.mode { + case LightSync: + pivot = height + case FastSync: + // Calculate the new fast/slow sync pivot point + if d.fsPivotLock == nil { + pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval))) + if err != nil { + panic(fmt.Sprintf("Failed to access crypto random source: %v", err)) } - // If the point is below the origin, move origin back to ensure state download - if pivot < origin { - if pivot > 0 { - origin = pivot - 1 - } else { - origin = 0 - } + if height > uint64(fsMinFullBlocks)+pivotOffset.Uint64() { + pivot = height - uint64(fsMinFullBlocks) - pivotOffset.Uint64() } - glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot) + } else { + // Pivot point locked in, use this and do not pick a new one! + pivot = d.fsPivotLock.Number.Uint64() } - d.queue.Prepare(origin+1, d.mode, pivot, latest) - if d.syncInitHook != nil { - d.syncInitHook(origin, height) + // If the point is below the origin, move origin back to ensure state download + if pivot < origin { + if pivot > 0 { + origin = pivot - 1 + } else { + origin = 0 + } } - return d.spawnSync(origin+1, - func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved - func() error { return d.processHeaders(origin+1, td) }, // Headers are always retrieved - func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync - func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync - func() error { return d.fetchNodeData() }, // Node state data is retrieved during fast sync - ) - - default: - // Something very wrong, stop right here - glog.V(logger.Error).Infof("Unsupported eth protocol: %d", p.version) - return errBadPeer + glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot) + } + d.queue.Prepare(origin+1, d.mode, pivot, latest) + if d.syncInitHook != nil { + d.syncInitHook(origin, height) } + return d.spawnSync(origin+1, + func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved + func() error { return d.processHeaders(origin+1, td) }, // Headers are always retrieved + func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync + func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync + func() error { return d.fetchNodeData() }, // Node state data is retrieved during fast sync + ) } // spawnSync runs d.process and all given fetcher functions to completion in @@ -540,452 +495,6 @@ func (d *Downloader) Terminate() { d.cancel() } -// fetchHeight61 retrieves the head block of the remote peer to aid in estimating -// the total time a pending synchronisation would take. -func (d *Downloader) fetchHeight61(p *peer) (uint64, error) { - glog.V(logger.Debug).Infof("%v: retrieving remote chain height", p) - - // Request the advertised remote head block and wait for the response - go p.getBlocks([]common.Hash{p.head}) - - timeout := time.After(hashTTL) - for { - select { - case <-d.cancelCh: - return 0, errCancelBlockFetch - - case packet := <-d.blockCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - blocks := packet.(*blockPack).blocks - if len(blocks) != 1 { - glog.V(logger.Debug).Infof("%v: invalid number of head blocks: %d != 1", p, len(blocks)) - return 0, errBadPeer - } - return blocks[0].NumberU64(), nil - - case <-timeout: - glog.V(logger.Debug).Infof("%v: head block timeout", p) - return 0, errTimeout - - case <-d.hashCh: - // Out of bounds hashes received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - -// findAncestor61 tries to locate the common ancestor block of the local chain and -// a remote peers blockchain. In the general case when our node was in sync and -// on the correct chain, checking the top N blocks should already get us a match. -// In the rare scenario when we ended up on a long reorganisation (i.e. none of -// the head blocks match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor61(p *peer, height uint64) (uint64, error) { - glog.V(logger.Debug).Infof("%v: looking for common ancestor", p) - - // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.headBlock().NumberU64() - if ceil >= MaxForkAncestry { - floor = int64(ceil - MaxForkAncestry) - } - // Request the topmost blocks to short circuit binary ancestor lookup - head := ceil - if head > height { - head = height - } - from := int64(head) - int64(MaxHashFetch) + 1 - if from < 0 { - from = 0 - } - go p.getAbsHashes(uint64(from), MaxHashFetch) - - // Wait for the remote response to the head fetch - number, hash := uint64(0), common.Hash{} - timeout := time.After(hashTTL) - - for finished := false; !finished; { - select { - case <-d.cancelCh: - return 0, errCancelHashFetch - - case packet := <-d.hashCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - hashes := packet.(*hashPack).hashes - if len(hashes) == 0 { - glog.V(logger.Debug).Infof("%v: empty head hash set", p) - return 0, errEmptyHashSet - } - // Check if a common ancestor was found - finished = true - for i := len(hashes) - 1; i >= 0; i-- { - // Skip any headers that underflow/overflow our requested set - header := d.getHeader(hashes[i]) - if header == nil || header.Number.Int64() < from || header.Number.Uint64() > head { - continue - } - // Otherwise check if we already know the header or not - if d.hasBlockAndState(hashes[i]) { - number, hash = header.Number.Uint64(), header.Hash() - break - } - } - - case <-timeout: - glog.V(logger.Debug).Infof("%v: head hash timeout", p) - return 0, errTimeout - - case <-d.blockCh: - // Out of bounds blocks received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } - // If the head fetch already found an ancestor, return - if !common.EmptyHash(hash) { - if int64(number) <= floor { - glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, number, hash[:4], floor) - return 0, errInvalidAncestor - } - glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash[:4]) - return number, nil - } - // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), head - if floor > 0 { - start = uint64(floor) - } - for start+1 < end { - // Split our chain interval in two, and request the hash to cross check - check := (start + end) / 2 - - timeout := time.After(hashTTL) - go p.getAbsHashes(uint64(check), 1) - - // Wait until a reply arrives to this request - for arrived := false; !arrived; { - select { - case <-d.cancelCh: - return 0, errCancelHashFetch - - case packet := <-d.hashCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - hashes := packet.(*hashPack).hashes - if len(hashes) != 1 { - glog.V(logger.Debug).Infof("%v: invalid search hash set (%d)", p, len(hashes)) - return 0, errBadPeer - } - arrived = true - - // Modify the search interval based on the response - if !d.hasBlockAndState(hashes[0]) { - end = check - break - } - block := d.getBlock(hashes[0]) // this doesn't check state, hence the above explicit check - if block.NumberU64() != check { - glog.V(logger.Debug).Infof("%v: non requested hash #%d [%x…], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check) - return 0, errBadPeer - } - start = check - - case <-timeout: - glog.V(logger.Debug).Infof("%v: search hash timeout", p) - return 0, errTimeout - - case <-d.blockCh: - // Out of bounds blocks received, ignore them - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } - } - // Ensure valid ancestry and return - if int64(start) <= floor { - glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, start, hash[:4], floor) - return 0, errInvalidAncestor - } - glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, start, hash[:4]) - return start, nil -} - -// fetchHashes61 keeps retrieving hashes from the requested number, until no more -// are returned, potentially throttling on the way. -func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error { - glog.V(logger.Debug).Infof("%v: downloading hashes from #%d", p, from) - - // Create a timeout timer, and the associated hash fetcher - request := time.Now() // time of the last fetch request - timeout := time.NewTimer(0) // timer to dump a non-responsive active peer - <-timeout.C // timeout channel should be initially empty - defer timeout.Stop() - - getHashes := func(from uint64) { - glog.V(logger.Detail).Infof("%v: fetching %d hashes from #%d", p, MaxHashFetch, from) - - request = time.Now() - timeout.Reset(hashTTL) - go p.getAbsHashes(from, MaxHashFetch) - } - // Start pulling hashes, until all are exhausted - getHashes(from) - gotHashes := false - - for { - select { - case <-d.cancelCh: - return errCancelHashFetch - - case packet := <-d.hashCh: - // Make sure the active peer is giving us the hashes - if packet.PeerId() != p.id { - glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId()) - break - } - hashReqTimer.UpdateSince(request) - timeout.Stop() - - // If no more hashes are inbound, notify the block fetcher and return - if packet.Items() == 0 { - glog.V(logger.Debug).Infof("%v: no available hashes", p) - - select { - case d.blockWakeCh <- false: - case <-d.cancelCh: - } - // If no hashes were retrieved at all, the peer violated it's TD promise that it had a - // better chain compared to ours. The only exception is if it's promised blocks were - // already imported by other means (e.g. fetcher): - // - // R <remote peer>, L <local node>: Both at block 10 - // R: Mine block 11, and propagate it to L - // L: Queue block 11 for import - // L: Notice that R's head and TD increased compared to ours, start sync - // L: Import of block 11 finishes - // L: Sync begins, and finds common ancestor at 11 - // L: Request new hashes up from 11 (R's TD was higher, it must have something) - // R: Nothing to give - if !gotHashes && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 { - return errStallingPeer - } - return nil - } - gotHashes = true - hashes := packet.(*hashPack).hashes - - // Otherwise insert all the new hashes, aborting in case of junk - glog.V(logger.Detail).Infof("%v: scheduling %d hashes from #%d", p, len(hashes), from) - - inserts := d.queue.Schedule61(hashes, true) - if len(inserts) != len(hashes) { - glog.V(logger.Debug).Infof("%v: stale hashes", p) - return errBadPeer - } - // Notify the block fetcher of new hashes, but stop if queue is full - if d.queue.PendingBlocks() < maxQueuedHashes { - // We still have hashes to fetch, send continuation wake signal (potential) - select { - case d.blockWakeCh <- true: - default: - } - } else { - // Hash limit reached, send a termination wake signal (enforced) - select { - case d.blockWakeCh <- false: - case <-d.cancelCh: - } - return nil - } - // Queue not yet full, fetch the next batch - from += uint64(len(hashes)) - getHashes(from) - - case <-timeout.C: - glog.V(logger.Debug).Infof("%v: hash request timed out", p) - hashTimeoutMeter.Mark(1) - return errTimeout - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - -// fetchBlocks61 iteratively downloads the scheduled hashes, taking any available -// peers, reserving a chunk of blocks for each, waiting for delivery and also -// periodically checking for timeouts. -func (d *Downloader) fetchBlocks61(from uint64) error { - glog.V(logger.Debug).Infof("Downloading blocks from #%d", from) - defer glog.V(logger.Debug).Infof("Block download terminated") - - // Create a timeout timer for scheduling expiration tasks - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - - update := make(chan struct{}, 1) - - // Fetch blocks until the hash fetcher's done - finished := false - for { - select { - case <-d.cancelCh: - return errCancelBlockFetch - - case packet := <-d.blockCh: - // If the peer was previously banned and failed to deliver it's pack - // in a reasonable time frame, ignore it's message. - if peer := d.peers.Peer(packet.PeerId()); peer != nil { - blocks := packet.(*blockPack).blocks - - // Deliver the received chunk of blocks and check chain validity - accepted, err := d.queue.DeliverBlocks(peer.id, blocks) - if err == errInvalidChain { - return err - } - // Unless a peer delivered something completely else than requested (usually - // caused by a timed out request which came through in the end), set it to - // idle. If the delivery's stale, the peer should have already been idled. - if err != errStaleDelivery { - peer.SetBlocksIdle(accepted) - } - // Issue a log to the user to see what's going on - switch { - case err == nil && len(blocks) == 0: - glog.V(logger.Detail).Infof("%s: no blocks delivered", peer) - case err == nil: - glog.V(logger.Detail).Infof("%s: delivered %d blocks", peer, len(blocks)) - default: - glog.V(logger.Detail).Infof("%s: delivery failed: %v", peer, err) - } - } - // Blocks arrived, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case cont := <-d.blockWakeCh: - // The hash fetcher sent a continuation flag, check if it's done - if !cont { - finished = true - } - // Hashes arrive, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case <-ticker.C: - // Sanity check update the progress - select { - case update <- struct{}{}: - default: - } - - case <-update: - // Short circuit if we lost all our peers - if d.peers.Len() == 0 { - return errNoPeers - } - // Check for block request timeouts and demote the responsible peers - for pid, fails := range d.queue.ExpireBlocks(blockTTL) { - if peer := d.peers.Peer(pid); peer != nil { - if fails > 1 { - glog.V(logger.Detail).Infof("%s: block delivery timeout", peer) - peer.SetBlocksIdle(0) - } else { - glog.V(logger.Debug).Infof("%s: stalling block delivery, dropping", peer) - d.dropPeer(pid) - } - } - } - // If there's nothing more to fetch, wait or terminate - if d.queue.PendingBlocks() == 0 { - if !d.queue.InFlightBlocks() && finished { - glog.V(logger.Debug).Infof("Block fetching completed") - return nil - } - break - } - // Send a download request to all idle peers, until throttled - throttled := false - idles, total := d.peers.BlockIdlePeers() - - for _, peer := range idles { - // Short circuit if throttling activated - if d.queue.ShouldThrottleBlocks() { - throttled = true - break - } - // Reserve a chunk of hashes for a peer. A nil can mean either that - // no more hashes are available, or that the peer is known not to - // have them. - request := d.queue.ReserveBlocks(peer, peer.BlockCapacity(blockTargetRTT)) - if request == nil { - continue - } - if glog.V(logger.Detail) { - glog.Infof("%s: requesting %d blocks", peer, len(request.Hashes)) - } - // Fetch the chunk and make sure any errors return the hashes to the queue - if err := peer.Fetch61(request); err != nil { - // Although we could try and make an attempt to fix this, this error really - // means that we've double allocated a fetch task to a peer. If that is the - // case, the internal state of the downloader and the queue is very wrong so - // better hard crash and note the error instead of silently accumulating into - // a much bigger issue. - panic(fmt.Sprintf("%v: fetch assignment failed", peer)) - } - } - // Make sure that we have peers available for fetching. If all peers have been tried - // and all failed throw an error - if !throttled && !d.queue.InFlightBlocks() && len(idles) == total { - return errPeersUnavailable - } - - case <-d.headerCh: - case <-d.bodyCh: - case <-d.stateCh: - case <-d.receiptCh: - // Ignore eth/{62,63} packets because this is eth/61. - // These can arrive as a late delivery from a previous sync. - } - } -} - // fetchHeight retrieves the head header of the remote peer to aid in estimating // the total time a pending synchronisation would take. func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { @@ -1022,11 +531,6 @@ func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1067,7 +571,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { for finished := false; !finished; { select { case <-d.cancelCh: - return 0, errCancelHashFetch + return 0, errCancelHeaderFetch case packet := <-d.headerCh: // Discard anything not from the origin peer @@ -1114,11 +618,6 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } // If the head fetch already found an ancestor, return @@ -1146,7 +645,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { for arrived := false; !arrived; { select { case <-d.cancelCh: - return 0, errCancelHashFetch + return 0, errCancelHeaderFetch case packer := <-d.headerCh: // Discard anything not from the origin peer @@ -1182,11 +681,6 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { case <-d.stateCh: case <-d.receiptCh: // Out of bounds delivery, ignore - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1305,11 +799,6 @@ func (d *Downloader) fetchHeaders(p *peer, from uint64) error { case <-d.cancelCh: } return errBadPeer - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1630,11 +1119,6 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv if !progressed && !throttled && !running && len(idles) == total && pending() > 0 { return errPeersUnavailable } - - case <-d.hashCh: - case <-d.blockCh: - // Ignore eth/61 packets because this is eth/62+. - // These can arrive as a late delivery from a previous sync. } } } @@ -1874,19 +1358,6 @@ func (d *Downloader) processContent() error { } } -// DeliverHashes injects a new batch of hashes received from a remote node into -// the download schedule. This is usually invoked through the BlockHashesMsg by -// the protocol handler. -func (d *Downloader) DeliverHashes(id string, hashes []common.Hash) (err error) { - return d.deliver(id, d.hashCh, &hashPack{id, hashes}, hashInMeter, hashDropMeter) -} - -// DeliverBlocks injects a new batch of blocks received from a remote node. -// This is usually invoked through the BlocksMsg by the protocol handler. -func (d *Downloader) DeliverBlocks(id string, blocks []*types.Block) (err error) { - return d.deliver(id, d.blockCh, &blockPack{id, blocks}, blockInMeter, blockDropMeter) -} - // DeliverHeaders injects a new batch of block headers received from a remote // node into the download schedule. func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) (err error) { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index fac6ef81c..4ca28091c 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -399,14 +399,12 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha var err error switch version { - case 61: - err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay), nil, nil, nil, nil, nil) case 62: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil) case 63: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) case 64: - err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) + err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) } if err == nil { // Assign the owned hashes, headers and blocks to the peer (deep copy) @@ -465,86 +463,6 @@ func (dl *downloadTester) dropPeer(id string) { dl.downloader.UnregisterPeer(id) } -// peerGetRelHashesFn constructs a GetHashes function associated with a specific -// peer in the download tester. The returned function can be used to retrieve -// batches of hashes from the particularly requested peer. -func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) func(head common.Hash) error { - return func(head common.Hash) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - // Gather the next batch of hashes - hashes := dl.peerHashes[id] - result := make([]common.Hash, 0, MaxHashFetch) - for i, hash := range hashes { - if hash == head { - i++ - for len(result) < cap(result) && i < len(hashes) { - result = append(result, hashes[i]) - i++ - } - break - } - } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dl.downloader.DeliverHashes(id, result) - }() - return nil - } -} - -// peerGetAbsHashesFn constructs a GetHashesFromNumber function associated with -// a particular peer in the download tester. The returned function can be used to -// retrieve batches of hashes from the particularly requested peer. -func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) func(uint64, int) error { - return func(head uint64, count int) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - // Gather the next batch of hashes - hashes := dl.peerHashes[id] - result := make([]common.Hash, 0, count) - for i := 0; i < count && len(hashes)-int(head)-1-i >= 0; i++ { - result = append(result, hashes[len(hashes)-int(head)-1-i]) - } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dl.downloader.DeliverHashes(id, result) - }() - return nil - } -} - -// peerGetBlocksFn constructs a getBlocks function associated with a particular -// peer in the download tester. The returned function can be used to retrieve -// batches of blocks from the particularly requested peer. -func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([]common.Hash) error { - return func(hashes []common.Hash) error { - time.Sleep(delay) - - dl.lock.RLock() - defer dl.lock.RUnlock() - - blocks := dl.peerBlocks[id] - result := make([]*types.Block, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - result = append(result, block) - } - } - go dl.downloader.DeliverBlocks(id, result) - - return nil - } -} - // peerGetRelHeadersFn constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. @@ -730,7 +648,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng // Tests that simple synchronization against a canonical chain works correctly. // In this test common ancestor lookup should be short circuited and not require // binary searching. -func TestCanonicalSynchronisation61(t *testing.T) { testCanonicalSynchronisation(t, 61, FullSync) } func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62, FullSync) } func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) } func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) } @@ -759,7 +676,6 @@ func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling61(t *testing.T) { testThrottling(t, 61, FullSync) } func TestThrottling62(t *testing.T) { testThrottling(t, 62, FullSync) } func TestThrottling63Full(t *testing.T) { testThrottling(t, 63, FullSync) } func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) } @@ -845,7 +761,6 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync61(t *testing.T) { testForkedSync(t, 61, FullSync) } func TestForkedSync62(t *testing.T) { testForkedSync(t, 62, FullSync) } func TestForkedSync63Full(t *testing.T) { testForkedSync(t, 63, FullSync) } func TestForkedSync63Fast(t *testing.T) { testForkedSync(t, 63, FastSync) } @@ -881,7 +796,6 @@ func testForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync61(t *testing.T) { testHeavyForkedSync(t, 61, FullSync) } func TestHeavyForkedSync62(t *testing.T) { testHeavyForkedSync(t, 62, FullSync) } func TestHeavyForkedSync63Full(t *testing.T) { testHeavyForkedSync(t, 63, FullSync) } func TestHeavyForkedSync63Fast(t *testing.T) { testHeavyForkedSync(t, 63, FastSync) } @@ -915,24 +829,9 @@ func testHeavyForkedSync(t *testing.T, protocol int, mode SyncMode) { assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork/2 + 1}) } -// Tests that an inactive downloader will not accept incoming hashes and blocks. -func TestInactiveDownloader61(t *testing.T) { - t.Parallel() - tester := newTester() - - // Check that neither hashes nor blocks are accepted - if err := tester.downloader.DeliverHashes("bad peer", []common.Hash{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } - if err := tester.downloader.DeliverBlocks("bad peer", []*types.Block{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } -} - // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync61(t *testing.T) { testBoundedForkedSync(t, 61, FullSync) } func TestBoundedForkedSync62(t *testing.T) { testBoundedForkedSync(t, 62, FullSync) } func TestBoundedForkedSync63Full(t *testing.T) { testBoundedForkedSync(t, 63, FullSync) } func TestBoundedForkedSync63Fast(t *testing.T) { testBoundedForkedSync(t, 63, FastSync) } @@ -968,7 +867,6 @@ func testBoundedForkedSync(t *testing.T, protocol int, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync61(t *testing.T) { testBoundedHeavyForkedSync(t, 61, FullSync) } func TestBoundedHeavyForkedSync62(t *testing.T) { testBoundedHeavyForkedSync(t, 62, FullSync) } func TestBoundedHeavyForkedSync63Full(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FullSync) } func TestBoundedHeavyForkedSync63Fast(t *testing.T) { testBoundedHeavyForkedSync(t, 63, FastSync) } @@ -1039,7 +937,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel61(t *testing.T) { testCancel(t, 61, FullSync) } func TestCancel62(t *testing.T) { testCancel(t, 62, FullSync) } func TestCancel63Full(t *testing.T) { testCancel(t, 63, FullSync) } func TestCancel63Fast(t *testing.T) { testCancel(t, 63, FastSync) } @@ -1081,7 +978,6 @@ func testCancel(t *testing.T, protocol int, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, 61, FullSync) } func TestMultiSynchronisation62(t *testing.T) { testMultiSynchronisation(t, 62, FullSync) } func TestMultiSynchronisation63Full(t *testing.T) { testMultiSynchronisation(t, 63, FullSync) } func TestMultiSynchronisation63Fast(t *testing.T) { testMultiSynchronisation(t, 63, FastSync) } @@ -1112,7 +1008,6 @@ func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation61(t *testing.T) { testMultiProtoSync(t, 61, FullSync) } func TestMultiProtoSynchronisation62(t *testing.T) { testMultiProtoSync(t, 62, FullSync) } func TestMultiProtoSynchronisation63Full(t *testing.T) { testMultiProtoSync(t, 63, FullSync) } func TestMultiProtoSynchronisation63Fast(t *testing.T) { testMultiProtoSync(t, 63, FastSync) } @@ -1131,7 +1026,6 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { tester := newTester() defer tester.terminate() - tester.newPeer("peer 61", 61, hashes, nil, blocks, nil) tester.newPeer("peer 62", 62, hashes, headers, blocks, nil) tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts) tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts) @@ -1143,7 +1037,7 @@ func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) { assertOwnChain(t, tester, targetBlocks+1) // Check that no peers have been dropped off - for _, version := range []int{61, 62, 63, 64} { + for _, version := range []int{62, 63, 64} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peerHashes[peer]; !ok { t.Errorf("%s dropped", peer) @@ -1368,7 +1262,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { // Tests that a peer advertising an high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack61(t *testing.T) { testHighTDStarvationAttack(t, 61, FullSync) } func TestHighTDStarvationAttack62(t *testing.T) { testHighTDStarvationAttack(t, 62, FullSync) } func TestHighTDStarvationAttack63Full(t *testing.T) { testHighTDStarvationAttack(t, 63, FullSync) } func TestHighTDStarvationAttack63Fast(t *testing.T) { testHighTDStarvationAttack(t, 63, FastSync) } @@ -1391,7 +1284,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping61(t *testing.T) { testBlockHeaderAttackerDropping(t, 61) } func TestBlockHeaderAttackerDropping62(t *testing.T) { testBlockHeaderAttackerDropping(t, 62) } func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) } func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) } @@ -1409,7 +1301,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { {errStallingPeer, true}, // Peer was detected to be stalling, drop it {errNoPeers, false}, // No peers to download from, soft race, no issue {errTimeout, true}, // No hashes received in due time, drop the peer - {errEmptyHashSet, true}, // No hashes were returned as a response, drop as it's a dead end {errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end {errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser {errInvalidAncestor, true}, // Agreed upon ancestor is not acceptable, drop the chain rewriter @@ -1417,7 +1308,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { {errInvalidBlock, false}, // A bad peer was detected, but not the sync origin {errInvalidBody, false}, // A bad peer was detected, but not the sync origin {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin - {errCancelHashFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelBlockFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelHeaderFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop {errCancelBodyFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop @@ -1450,7 +1340,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress61(t *testing.T) { testSyncProgress(t, 61, FullSync) } func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) } func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) } func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) } @@ -1524,7 +1413,6 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress61(t *testing.T) { testForkedSyncProgress(t, 61, FullSync) } func TestForkedSyncProgress62(t *testing.T) { testForkedSyncProgress(t, 62, FullSync) } func TestForkedSyncProgress63Full(t *testing.T) { testForkedSyncProgress(t, 63, FullSync) } func TestForkedSyncProgress63Fast(t *testing.T) { testForkedSyncProgress(t, 63, FastSync) } @@ -1601,7 +1489,6 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress61(t *testing.T) { testFailedSyncProgress(t, 61, FullSync) } func TestFailedSyncProgress62(t *testing.T) { testFailedSyncProgress(t, 62, FullSync) } func TestFailedSyncProgress63Full(t *testing.T) { testFailedSyncProgress(t, 63, FullSync) } func TestFailedSyncProgress63Fast(t *testing.T) { testFailedSyncProgress(t, 63, FastSync) } @@ -1679,7 +1566,6 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress61(t *testing.T) { testFakedSyncProgress(t, 61, FullSync) } func TestFakedSyncProgress62(t *testing.T) { testFakedSyncProgress(t, 62, FullSync) } func TestFakedSyncProgress63Full(t *testing.T) { testFakedSyncProgress(t, 63, FullSync) } func TestFakedSyncProgress63Fast(t *testing.T) { testFakedSyncProgress(t, 63, FastSync) } diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index d6fcfa25c..0d76c7dfd 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -23,16 +23,6 @@ import ( ) var ( - hashInMeter = metrics.NewMeter("eth/downloader/hashes/in") - hashReqTimer = metrics.NewTimer("eth/downloader/hashes/req") - hashDropMeter = metrics.NewMeter("eth/downloader/hashes/drop") - hashTimeoutMeter = metrics.NewMeter("eth/downloader/hashes/timeout") - - blockInMeter = metrics.NewMeter("eth/downloader/blocks/in") - blockReqTimer = metrics.NewTimer("eth/downloader/blocks/req") - blockDropMeter = metrics.NewMeter("eth/downloader/blocks/drop") - blockTimeoutMeter = metrics.NewMeter("eth/downloader/blocks/timeout") - headerInMeter = metrics.NewMeter("eth/downloader/headers/in") headerReqTimer = metrics.NewTimer("eth/downloader/headers/req") headerDropMeter = metrics.NewMeter("eth/downloader/headers/drop") diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 94d44fca4..c2b7a52d0 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -37,11 +37,6 @@ const ( measurementImpact = 0.1 // The impact a single measurement has on a peer's final throughput value. ) -// Hash and block fetchers belonging to eth/61 and below -type relativeHashFetcherFn func(common.Hash) error -type absoluteHashFetcherFn func(uint64, int) error -type blockFetcherFn func([]common.Hash) error - // Block header and body fetchers belonging to eth/62 and above type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error type absoluteHeaderFetcherFn func(uint64, int, int, bool) error @@ -79,10 +74,6 @@ type peer struct { lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) - getRelHashes relativeHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an origin hash - getAbsHashes absoluteHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an absolute position - getBlocks blockFetcherFn // [eth/61] Method to retrieve a batch of blocks - getRelHeaders relativeHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an origin hash getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies @@ -97,7 +88,6 @@ type peer struct { // newPeer create a new downloader peer, with specific hash and block retrieval // mechanisms. func newPeer(id string, version int, head common.Hash, - getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, getReceipts receiptFetcherFn, getNodeData stateFetcherFn) *peer { return &peer{ @@ -105,10 +95,6 @@ func newPeer(id string, version int, head common.Hash, head: head, lacking: make(map[common.Hash]struct{}), - getRelHashes: getRelHashes, - getAbsHashes: getAbsHashes, - getBlocks: getBlocks, - getRelHeaders: getRelHeaders, getAbsHeaders: getAbsHeaders, getBlockBodies: getBlockBodies, @@ -138,28 +124,6 @@ func (p *peer) Reset() { p.lacking = make(map[common.Hash]struct{}) } -// Fetch61 sends a block retrieval request to the remote peer. -func (p *peer) Fetch61(request *fetchRequest) error { - // Sanity check the protocol version - if p.version != 61 { - panic(fmt.Sprintf("block fetch [eth/61] requested on eth/%d", p.version)) - } - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) { - return errAlreadyFetching - } - p.blockStarted = time.Now() - - // Convert the hash set to a retrievable slice - hashes := make([]common.Hash, 0, len(request.Hashes)) - for hash, _ := range request.Hashes { - hashes = append(hashes, hash) - } - go p.getBlocks(hashes) - - return nil -} - // FetchHeaders sends a header retrieval request to the remote peer. func (p *peer) FetchHeaders(from uint64, count int) error { // Sanity check the protocol version @@ -481,20 +445,6 @@ func (ps *peerSet) AllPeers() []*peer { return list } -// BlockIdlePeers retrieves a flat list of all the currently idle peers within the -// active peer set, ordered by their reputation. -func (ps *peerSet) BlockIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { - return atomic.LoadInt32(&p.blockIdle) == 0 - } - throughput := func(p *peer) float64 { - p.lock.RLock() - defer p.lock.RUnlock() - return p.blockThroughput - } - return ps.idlePeers(61, 61, idle, throughput) -} - // HeaderIdlePeers retrieves a flat list of all the currently header-idle peers // within the active peer set, ordered by their reputation. func (ps *peerSet) HeaderIdlePeers() ([]*peer, int) { diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 01897af6d..fd239f7e4 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -45,7 +45,6 @@ var ( var ( errNoFetchesPending = errors.New("no fetches pending") - errStateSyncPending = errors.New("state trie sync already scheduled") errStaleDelivery = errors.New("stale delivery") ) @@ -74,10 +73,6 @@ type queue struct { mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching fastSyncPivot uint64 // Block number where the fast sync pivots into archive synchronisation mode - hashPool map[common.Hash]int // [eth/61] Pending hashes, mapping to their insertion index (priority) - hashQueue *prque.Prque // [eth/61] Priority queue of the block hashes to fetch - hashCounter int // [eth/61] Counter indexing the added hashes to ensure retrieval order - headerHead common.Hash // [eth/62] Hash of the last queued header to verify order // Headers are "special", they download in batches, supported by a skeleton chain @@ -85,7 +80,6 @@ type queue struct { headerTaskQueue *prque.Prque // [eth/62] Priority queue of the skeleton indexes to fetch the filling headers for headerPeerMiss map[string]map[uint64]struct{} // [eth/62] Set of per-peer header batches known to be unavailable headerPendPool map[string]*fetchRequest // [eth/62] Currently pending header retrieval operations - headerDonePool map[uint64]struct{} // [eth/62] Set of the completed header fetches headerResults []*types.Header // [eth/62] Result cache accumulating the completed headers headerProced int // [eth/62] Number of headers already processed from the results headerOffset uint64 // [eth/62] Number of the first header in the result cache @@ -124,8 +118,6 @@ type queue struct { func newQueue(stateDb ethdb.Database) *queue { lock := new(sync.Mutex) return &queue{ - hashPool: make(map[common.Hash]int), - hashQueue: prque.New(), headerPendPool: make(map[string]*fetchRequest), headerContCh: make(chan bool), blockTaskPool: make(map[common.Hash]*types.Header), @@ -158,10 +150,6 @@ func (q *queue) Reset() { q.mode = FullSync q.fastSyncPivot = 0 - q.hashPool = make(map[common.Hash]int) - q.hashQueue.Reset() - q.hashCounter = 0 - q.headerHead = common.Hash{} q.headerPendPool = make(map[string]*fetchRequest) @@ -208,7 +196,7 @@ func (q *queue) PendingBlocks() int { q.lock.Lock() defer q.lock.Unlock() - return q.hashQueue.Size() + q.blockTaskQueue.Size() + return q.blockTaskQueue.Size() } // PendingReceipts retrieves the number of block receipts pending for retrieval. @@ -272,7 +260,7 @@ func (q *queue) Idle() bool { q.lock.Lock() defer q.lock.Unlock() - queued := q.hashQueue.Size() + q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + q.stateTaskQueue.Size() + queued := q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + q.stateTaskQueue.Size() pending := len(q.blockPendPool) + len(q.receiptPendPool) + len(q.statePendPool) cached := len(q.blockDonePool) + len(q.receiptDonePool) @@ -323,34 +311,6 @@ func (q *queue) ShouldThrottleReceipts() bool { return pending >= len(q.resultCache)-len(q.receiptDonePool) } -// Schedule61 adds a set of hashes for the download queue for scheduling, returning -// the new hashes encountered. -func (q *queue) Schedule61(hashes []common.Hash, fifo bool) []common.Hash { - q.lock.Lock() - defer q.lock.Unlock() - - // Insert all the hashes prioritised in the arrival order - inserts := make([]common.Hash, 0, len(hashes)) - for _, hash := range hashes { - // Skip anything we already have - if old, ok := q.hashPool[hash]; ok { - glog.V(logger.Warn).Infof("Hash %x already scheduled at index %v", hash, old) - continue - } - // Update the counters and insert the hash - q.hashCounter = q.hashCounter + 1 - inserts = append(inserts, hash) - - q.hashPool[hash] = q.hashCounter - if fifo { - q.hashQueue.Push(hash, -float32(q.hashCounter)) // Lowest gets schedules first - } else { - q.hashQueue.Push(hash, float32(q.hashCounter)) // Highest gets schedules first - } - } - return inserts -} - // ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill // up an already retrieved header skeleton. func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { @@ -550,15 +510,6 @@ func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest { return request } -// ReserveBlocks reserves a set of block hashes for the given peer, skipping any -// previously failed download. -func (q *queue) ReserveBlocks(p *peer, count int) *fetchRequest { - q.lock.Lock() - defer q.lock.Unlock() - - return q.reserveHashes(p, count, q.hashQueue, nil, q.blockPendPool, len(q.resultCache)-len(q.blockDonePool)) -} - // ReserveNodeData reserves a set of node data hashes for the given peer, skipping // any previously failed download. func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest { @@ -753,11 +704,6 @@ func (q *queue) CancelHeaders(request *fetchRequest) { q.cancel(request, q.headerTaskQueue, q.headerPendPool) } -// CancelBlocks aborts a fetch request, returning all pending hashes to the queue. -func (q *queue) CancelBlocks(request *fetchRequest) { - q.cancel(request, q.hashQueue, q.blockPendPool) -} - // CancelBodies aborts a body fetch request, returning all pending headers to the // task queue. func (q *queue) CancelBodies(request *fetchRequest) { @@ -801,9 +747,6 @@ func (q *queue) Revoke(peerId string) { defer q.lock.Unlock() if request, ok := q.blockPendPool[peerId]; ok { - for hash, index := range request.Hashes { - q.hashQueue.Push(hash, float32(index)) - } for _, header := range request.Headers { q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) } @@ -832,15 +775,6 @@ func (q *queue) ExpireHeaders(timeout time.Duration) map[string]int { return q.expire(timeout, q.headerPendPool, q.headerTaskQueue, headerTimeoutMeter) } -// ExpireBlocks checks for in flight requests that exceeded a timeout allowance, -// canceling them and returning the responsible peers for penalisation. -func (q *queue) ExpireBlocks(timeout time.Duration) map[string]int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.expire(timeout, q.blockPendPool, q.hashQueue, blockTimeoutMeter) -} - // ExpireBodies checks for in flight block body requests that exceeded a timeout // allowance, canceling them and returning the responsible peers for penalisation. func (q *queue) ExpireBodies(timeout time.Duration) map[string]int { @@ -907,74 +841,6 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, return expiries } -// DeliverBlocks injects a block retrieval response into the download queue. The -// method returns the number of blocks accepted from the delivery and also wakes -// any threads waiting for data delivery. -func (q *queue) DeliverBlocks(id string, blocks []*types.Block) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - - // Short circuit if the blocks were never requested - request := q.blockPendPool[id] - if request == nil { - return 0, errNoFetchesPending - } - blockReqTimer.UpdateSince(request.Time) - delete(q.blockPendPool, id) - - // If no blocks were retrieved, mark them as unavailable for the origin peer - if len(blocks) == 0 { - for hash, _ := range request.Hashes { - request.Peer.MarkLacking(hash) - } - } - // Iterate over the downloaded blocks and add each of them - accepted, errs := 0, make([]error, 0) - for _, block := range blocks { - // Skip any blocks that were not requested - hash := block.Hash() - if _, ok := request.Hashes[hash]; !ok { - errs = append(errs, fmt.Errorf("non-requested block %x", hash)) - continue - } - // Reconstruct the next result if contents match up - index := int(block.Number().Int64() - int64(q.resultOffset)) - if index >= len(q.resultCache) || index < 0 { - errs = []error{errInvalidChain} - break - } - q.resultCache[index] = &fetchResult{ - Header: block.Header(), - Transactions: block.Transactions(), - Uncles: block.Uncles(), - } - q.blockDonePool[block.Hash()] = struct{}{} - - delete(request.Hashes, hash) - delete(q.hashPool, hash) - accepted++ - } - // Return all failed or missing fetches to the queue - for hash, index := range request.Hashes { - q.hashQueue.Push(hash, float32(index)) - } - // Wake up WaitResults - if accepted > 0 { - q.active.Signal() - } - // If none of the blocks were good, it's a stale delivery - switch { - case len(errs) == 0: - return accepted, nil - case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBlock): - return accepted, errs[0] - case len(errs) == len(blocks): - return accepted, errStaleDelivery - default: - return accepted, fmt.Errorf("multiple failures: %v", errs) - } -} - // DeliverHeaders injects a header retrieval response into the header results // cache. This method either accepts all headers it received, or none of them // if they do not map correctly to the skeleton. diff --git a/eth/downloader/types.go b/eth/downloader/types.go index b67fff1f8..e10510486 100644 --- a/eth/downloader/types.go +++ b/eth/downloader/types.go @@ -73,26 +73,6 @@ type dataPack interface { Stats() string } -// hashPack is a batch of block hashes returned by a peer (eth/61). -type hashPack struct { - peerId string - hashes []common.Hash -} - -func (p *hashPack) PeerId() string { return p.peerId } -func (p *hashPack) Items() int { return len(p.hashes) } -func (p *hashPack) Stats() string { return fmt.Sprintf("%d", len(p.hashes)) } - -// blockPack is a batch of blocks returned by a peer (eth/61). -type blockPack struct { - peerId string - blocks []*types.Block -} - -func (p *blockPack) PeerId() string { return p.peerId } -func (p *blockPack) Items() int { return len(p.blocks) } -func (p *blockPack) Stats() string { return fmt.Sprintf("%d", len(p.blocks)) } - // headerPack is a batch of block headers returned by a peer. type headerPack struct { peerId string diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 9300717c3..bd235bb9e 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -48,9 +48,6 @@ var ( // blockRetrievalFn is a callback type for retrieving a block from the local chain. type blockRetrievalFn func(common.Hash) *types.Block -// blockRequesterFn is a callback type for sending a block retrieval request. -type blockRequesterFn func([]common.Hash) error - // headerRequesterFn is a callback type for sending a header retrieval request. type headerRequesterFn func(common.Hash) error @@ -82,7 +79,6 @@ type announce struct { origin string // Identifier of the peer originating the notification - fetch61 blockRequesterFn // [eth/61] Fetcher function to retrieve an announced block fetchHeader headerRequesterFn // [eth/62] Fetcher function to retrieve the header of an announced block fetchBodies bodyRequesterFn // [eth/62] Fetcher function to retrieve the body of an announced block } @@ -191,14 +187,12 @@ func (f *Fetcher) Stop() { // Notify announces the fetcher of the potential availability of a new block in // the network. func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - blockFetcher blockRequesterFn, // eth/61 specific whole block fetcher headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { block := &announce{ hash: hash, number: number, time: time, origin: peer, - fetch61: blockFetcher, fetchHeader: headerFetcher, fetchBodies: bodyFetcher, } @@ -224,34 +218,6 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error { } } -// FilterBlocks extracts all the blocks that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *Fetcher) FilterBlocks(blocks types.Blocks) types.Blocks { - glog.V(logger.Detail).Infof("[eth/61] filtering %d blocks", len(blocks)) - - // Send the filter channel to the fetcher - filter := make(chan []*types.Block) - - select { - case f.blockFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the block list - select { - case filter <- blocks: - case <-f.quit: - return nil - } - // Retrieve the blocks remaining after filtering - select { - case blocks := <-filter: - return blocks - case <-f.quit: - return nil - } -} - // FilterHeaders extracts all the headers that were explicitly requested by the fetcher, // returning those that should be handled differently. func (f *Fetcher) FilterHeaders(headers []*types.Header, time time.Time) []*types.Header { @@ -413,7 +379,7 @@ func (f *Fetcher) loop() { } } } - // Send out all block (eth/61) or header (eth/62) requests + // Send out all block header requests for peer, hashes := range request { if glog.V(logger.Detail) && len(hashes) > 0 { list := "[" @@ -421,29 +387,17 @@ func (f *Fetcher) loop() { list += fmt.Sprintf("%x…, ", hash[:4]) } list = list[:len(list)-2] + "]" - - if f.fetching[hashes[0]].fetch61 != nil { - glog.V(logger.Detail).Infof("[eth/61] Peer %s: fetching blocks %s", peer, list) - } else { - glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching headers %s", peer, list) - } + glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching headers %s", peer, list) } // Create a closure of the fetch and schedule in on a new thread - fetchBlocks, fetchHeader, hashes := f.fetching[hashes[0]].fetch61, f.fetching[hashes[0]].fetchHeader, hashes + fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes go func() { if f.fetchingHook != nil { f.fetchingHook(hashes) } - if fetchBlocks != nil { - // Use old eth/61 protocol to retrieve whole blocks - blockFetchMeter.Mark(int64(len(hashes))) - fetchBlocks(hashes) - } else { - // Use new eth/62 protocol to retrieve headers first - for _, hash := range hashes { - headerFetchMeter.Mark(1) - fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals - } + for _, hash := range hashes { + headerFetchMeter.Mark(1) + fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals } }() } @@ -486,46 +440,6 @@ func (f *Fetcher) loop() { // Schedule the next fetch if blocks are still pending f.rescheduleComplete(completeTimer) - case filter := <-f.blockFilter: - // Blocks arrived, extract any explicit fetches, return all else - var blocks types.Blocks - select { - case blocks = <-filter: - case <-f.quit: - return - } - blockFilterInMeter.Mark(int64(len(blocks))) - - explicit, download := []*types.Block{}, []*types.Block{} - for _, block := range blocks { - hash := block.Hash() - - // Filter explicitly requested blocks from hash announcements - if f.fetching[hash] != nil && f.queued[hash] == nil { - // Discard if already imported by other means - if f.getBlock(hash) == nil { - explicit = append(explicit, block) - } else { - f.forgetHash(hash) - } - } else { - download = append(download, block) - } - } - - blockFilterOutMeter.Mark(int64(len(download))) - select { - case filter <- download: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range explicit { - if announce := f.fetching[block.Hash()]; announce != nil { - f.enqueue(announce.origin, block) - } - } - case filter := <-f.headerFilter: // Headers arrived from a remote peer. Extract those that were explicitly // requested by the fetcher, and return everything else so it's delivered diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 6a32be14c..ad955a577 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -151,28 +151,6 @@ func (f *fetcherTester) dropPeer(peer string) { f.drops[peer] = true } -// makeBlockFetcher retrieves a block fetcher associated with a simulated peer. -func (f *fetcherTester) makeBlockFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash) error { - // Gather the blocks to return - blocks := make([]*types.Block, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - blocks = append(blocks, block) - } - } - // Return on a new thread - go f.fetcher.FilterBlocks(blocks) - - return nil - } -} - // makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. func (f *fetcherTester) makeHeaderFetcher(blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { closure := make(map[common.Hash]*types.Block) @@ -293,7 +271,6 @@ func verifyImportDone(t *testing.T, imported chan *types.Block) { // Tests that a fetcher accepts block announcements and initiates retrievals for // them, successfully importing into the local chain. -func TestSequentialAnnouncements61(t *testing.T) { testSequentialAnnouncements(t, 61) } func TestSequentialAnnouncements62(t *testing.T) { testSequentialAnnouncements(t, 62) } func TestSequentialAnnouncements63(t *testing.T) { testSequentialAnnouncements(t, 63) } func TestSequentialAnnouncements64(t *testing.T) { testSequentialAnnouncements(t, 64) } @@ -304,7 +281,6 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { hashes, blocks := makeChain(targetBlocks, 0, genesis) tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -313,11 +289,7 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -325,7 +297,6 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { // Tests that if blocks are announced by multiple peers (or even the same buggy // peer), they will only get downloaded at most once. -func TestConcurrentAnnouncements61(t *testing.T) { testConcurrentAnnouncements(t, 61) } func TestConcurrentAnnouncements62(t *testing.T) { testConcurrentAnnouncements(t, 62) } func TestConcurrentAnnouncements63(t *testing.T) { testConcurrentAnnouncements(t, 63) } func TestConcurrentAnnouncements64(t *testing.T) { testConcurrentAnnouncements(t, 64) } @@ -337,15 +308,10 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { // Assemble a tester with a built in counter for the requests tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) counter := uint32(0) - blockWrapper := func(hashes []common.Hash) error { - atomic.AddUint32(&counter, uint32(len(hashes))) - return blockFetcher(hashes) - } headerWrapper := func(hash common.Hash) error { atomic.AddUint32(&counter, 1) return headerFetcher(hash) @@ -355,15 +321,9 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("first", hashes[i], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil) - tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout+time.Millisecond), blockWrapper, nil, nil) - tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout-time.Millisecond), blockWrapper, nil, nil) - } else { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), nil, headerWrapper, bodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), nil, headerWrapper, bodyFetcher) - } + tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), headerWrapper, bodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), headerWrapper, bodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) @@ -376,7 +336,6 @@ func testConcurrentAnnouncements(t *testing.T, protocol int) { // Tests that announcements arriving while a previous is being fetched still // results in a valid import. -func TestOverlappingAnnouncements61(t *testing.T) { testOverlappingAnnouncements(t, 61) } func TestOverlappingAnnouncements62(t *testing.T) { testOverlappingAnnouncements(t, 62) } func TestOverlappingAnnouncements63(t *testing.T) { testOverlappingAnnouncements(t, 63) } func TestOverlappingAnnouncements64(t *testing.T) { testOverlappingAnnouncements(t, 64) } @@ -387,7 +346,6 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { hashes, blocks := makeChain(targetBlocks, 0, genesis) tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -400,11 +358,7 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } for i := len(hashes) - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-imported: case <-time.After(time.Second): @@ -416,7 +370,6 @@ func testOverlappingAnnouncements(t *testing.T, protocol int) { } // Tests that announces already being retrieved will not be duplicated. -func TestPendingDeduplication61(t *testing.T) { testPendingDeduplication(t, 61) } func TestPendingDeduplication62(t *testing.T) { testPendingDeduplication(t, 62) } func TestPendingDeduplication63(t *testing.T) { testPendingDeduplication(t, 63) } func TestPendingDeduplication64(t *testing.T) { testPendingDeduplication(t, 64) } @@ -427,22 +380,11 @@ func testPendingDeduplication(t *testing.T, protocol int) { // Assemble a tester with a built in counter and delayed fetcher tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) delay := 50 * time.Millisecond counter := uint32(0) - blockWrapper := func(hashes []common.Hash) error { - atomic.AddUint32(&counter, uint32(len(hashes))) - - // Simulate a long running fetch - go func() { - time.Sleep(delay) - blockFetcher(hashes) - }() - return nil - } headerWrapper := func(hash common.Hash) error { atomic.AddUint32(&counter, 1) @@ -455,11 +397,7 @@ func testPendingDeduplication(t *testing.T, protocol int) { } // Announce the same block many times until it's fetched (wait for any pending ops) for tester.getBlock(hashes[0]) == nil { - if protocol < 62 { - tester.fetcher.Notify("repeater", hashes[0], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil) - } else { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher) - } + tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) time.Sleep(time.Millisecond) } time.Sleep(delay) @@ -475,7 +413,6 @@ func testPendingDeduplication(t *testing.T, protocol int) { // Tests that announcements retrieved in a random order are cached and eventually // imported when all the gaps are filled in. -func TestRandomArrivalImport61(t *testing.T) { testRandomArrivalImport(t, 61) } func TestRandomArrivalImport62(t *testing.T) { testRandomArrivalImport(t, 62) } func TestRandomArrivalImport63(t *testing.T) { testRandomArrivalImport(t, 63) } func TestRandomArrivalImport64(t *testing.T) { testRandomArrivalImport(t, 64) } @@ -487,7 +424,6 @@ func testRandomArrivalImport(t *testing.T, protocol int) { skip := targetBlocks / 2 tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -497,26 +433,17 @@ func testRandomArrivalImport(t *testing.T, protocol int) { for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) time.Sleep(time.Millisecond) } } // Finally announce the skipped entry and check full import - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[skip], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportCount(t, imported, len(hashes)-1) } // Tests that direct block enqueues (due to block propagation vs. hash announce) // are correctly schedule, filling and import queue gaps. -func TestQueueGapFill61(t *testing.T) { testQueueGapFill(t, 61) } func TestQueueGapFill62(t *testing.T) { testQueueGapFill(t, 62) } func TestQueueGapFill63(t *testing.T) { testQueueGapFill(t, 63) } func TestQueueGapFill64(t *testing.T) { testQueueGapFill(t, 64) } @@ -528,7 +455,6 @@ func testQueueGapFill(t *testing.T, protocol int) { skip := targetBlocks / 2 tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -538,11 +464,7 @@ func testQueueGapFill(t *testing.T, protocol int) { for i := len(hashes) - 1; i >= 0; i-- { if i != skip { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) time.Sleep(time.Millisecond) } } @@ -553,7 +475,6 @@ func testQueueGapFill(t *testing.T, protocol int) { // Tests that blocks arriving from various sources (multiple propagations, hash // announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication61(t *testing.T) { testImportDeduplication(t, 61) } func TestImportDeduplication62(t *testing.T) { testImportDeduplication(t, 62) } func TestImportDeduplication63(t *testing.T) { testImportDeduplication(t, 63) } func TestImportDeduplication64(t *testing.T) { testImportDeduplication(t, 64) } @@ -564,7 +485,6 @@ func testImportDeduplication(t *testing.T, protocol int) { // Create the tester and wrap the importer with a counter tester := newTester() - blockFetcher := tester.makeBlockFetcher(blocks) headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) bodyFetcher := tester.makeBodyFetcher(blocks, 0) @@ -580,11 +500,7 @@ func testImportDeduplication(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } // Announce the duplicating block, wait for retrieval, and also propagate directly - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[0], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) - } + tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) <-fetching tester.fetcher.Enqueue("valid", blocks[hashes[0]]) @@ -660,14 +576,14 @@ func testDistantAnnouncementDiscarding(t *testing.T, protocol int) { tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-time.After(50 * time.Millisecond): case <-fetching: t.Fatalf("fetcher requested stale header") } // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) select { case <-time.After(50 * time.Millisecond): case <-fetching: @@ -693,7 +609,7 @@ func testInvalidNumberAnnouncement(t *testing.T, protocol int) { tester.fetcher.importedHook = func(block *types.Block) { imported <- block } // Announce a block with a bad number, check for immediate drop - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, false) tester.lock.RLock() @@ -704,7 +620,7 @@ func testInvalidNumberAnnouncement(t *testing.T, protocol int) { t.Fatalf("peer with invalid numbered announcement not dropped") } // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) verifyImportEvent(t, imported, true) tester.lock.RLock() @@ -743,7 +659,7 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) { // Iteratively announce blocks until all are imported for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher) + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) // All announces should fetch the header verifyFetchingEvent(t, fetching, true) @@ -760,7 +676,6 @@ func testEmptyBlockShortCircuit(t *testing.T, protocol int) { // Tests that a peer is unable to use unbounded memory with sending infinite // block announcements to a node, but that even in the face of such an attack, // the fetcher remains operational. -func TestHashMemoryExhaustionAttack61(t *testing.T) { testHashMemoryExhaustionAttack(t, 61) } func TestHashMemoryExhaustionAttack62(t *testing.T) { testHashMemoryExhaustionAttack(t, 62) } func TestHashMemoryExhaustionAttack63(t *testing.T) { testHashMemoryExhaustionAttack(t, 63) } func TestHashMemoryExhaustionAttack64(t *testing.T) { testHashMemoryExhaustionAttack(t, 64) } @@ -781,29 +696,19 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) { // Create a valid chain and an infinite junk chain targetBlocks := hashLimit + 2*maxQueueDist hashes, blocks := makeChain(targetBlocks, 0, genesis) - validBlockFetcher := tester.makeBlockFetcher(blocks) validHeaderFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack) validBodyFetcher := tester.makeBodyFetcher(blocks, 0) attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerBlockFetcher := tester.makeBlockFetcher(nil) attackerHeaderFetcher := tester.makeHeaderFetcher(nil, -gatherSlack) attackerBodyFetcher := tester.makeBodyFetcher(nil, 0) // Feed the tester a huge hashset from the attacker, and a limited from the valid peer for i := 0; i < len(attack); i++ { if i < maxQueueDist { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], 0, time.Now(), validBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), nil, validHeaderFetcher, validBodyFetcher) - } - } - if protocol < 62 { - tester.fetcher.Notify("attacker", attack[i], 0, time.Now(), attackerBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), nil, attackerHeaderFetcher, attackerBodyFetcher) + tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) } + tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) } if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist { t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) @@ -813,11 +718,7 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) { // Feed the remaining valid hashes to ensure DOS protection state remains clean for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - if protocol < 62 { - tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), validBlockFetcher, nil, nil) - } else { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, validHeaderFetcher, validBodyFetcher) - } + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) verifyImportEvent(t, imported, true) } verifyImportDone(t, imported) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index b82d3ca01..1ed8075bf 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -33,12 +33,9 @@ var ( propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop") propBroadcastDOSMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos") - blockFetchMeter = metrics.NewMeter("eth/fetcher/fetch/blocks") headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers") bodyFetchMeter = metrics.NewMeter("eth/fetcher/fetch/bodies") - blockFilterInMeter = metrics.NewMeter("eth/fetcher/filter/blocks/in") - blockFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/blocks/out") headerFilterInMeter = metrics.NewMeter("eth/fetcher/filter/headers/in") headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out") bodyFilterInMeter = metrics.NewMeter("eth/fetcher/filter/bodies/in") diff --git a/eth/handler.go b/eth/handler.go index 1d5a04100..9ad430976 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -57,9 +57,6 @@ func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) } -type hashFetcherFn func(common.Hash) error -type blockFetcherFn func([]common.Hash) error - type ProtocolManager struct { networkId int @@ -275,9 +272,11 @@ func (pm *ProtocolManager) handle(p *peer) error { defer pm.removePeer(p.id) // Register the peer in the downloader. If the downloader considers it banned, we disconnect - if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(), - p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks, p.RequestHeadersByHash, - p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts, p.RequestNodeData); err != nil { + err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(), + p.RequestHeadersByHash, p.RequestHeadersByNumber, + p.RequestBodies, p.RequestReceipts, p.RequestNodeData, + ) + if err != nil { return err } // Propagate existing transactions. new transactions appearing @@ -331,108 +330,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Status messages should never arrive after the handshake return errResp(ErrExtraStatusMsg, "uncontrolled status message") - case p.version < eth62 && msg.Code == GetBlockHashesMsg: - // Retrieve the number of hashes to return and from which origin hash - var request getBlockHashesData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - if request.Amount > uint64(downloader.MaxHashFetch) { - request.Amount = uint64(downloader.MaxHashFetch) - } - // Retrieve the hashes from the block chain and return them - hashes := pm.blockchain.GetBlockHashesFromHash(request.Hash, request.Amount) - if len(hashes) == 0 { - glog.V(logger.Debug).Infof("invalid block hash %x", request.Hash.Bytes()[:4]) - } - return p.SendBlockHashes(hashes) - - case p.version < eth62 && msg.Code == GetBlockHashesFromNumberMsg: - // Retrieve and decode the number of hashes to return and from which origin number - var request getBlockHashesFromNumberData - if err := msg.Decode(&request); err != nil { - return errResp(ErrDecode, "%v: %v", msg, err) - } - if request.Amount > uint64(downloader.MaxHashFetch) { - request.Amount = uint64(downloader.MaxHashFetch) - } - // Calculate the last block that should be retrieved, and short circuit if unavailable - last := pm.blockchain.GetBlockByNumber(request.Number + request.Amount - 1) - if last == nil { - last = pm.blockchain.CurrentBlock() - request.Amount = last.NumberU64() - request.Number + 1 - } - if last.NumberU64() < request.Number { - return p.SendBlockHashes(nil) - } - // Retrieve the hashes from the last block backwards, reverse and return - hashes := []common.Hash{last.Hash()} - hashes = append(hashes, pm.blockchain.GetBlockHashesFromHash(last.Hash(), request.Amount-1)...) - - for i := 0; i < len(hashes)/2; i++ { - hashes[i], hashes[len(hashes)-1-i] = hashes[len(hashes)-1-i], hashes[i] - } - return p.SendBlockHashes(hashes) - - case p.version < eth62 && msg.Code == BlockHashesMsg: - // A batch of hashes arrived to one of our previous requests - var hashes []common.Hash - if err := msg.Decode(&hashes); err != nil { - break - } - // Deliver them all to the downloader for queuing - err := pm.downloader.DeliverHashes(p.id, hashes) - if err != nil { - glog.V(logger.Debug).Infoln(err) - } - - case p.version < eth62 && msg.Code == GetBlocksMsg: - // Decode the retrieval message - msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) - if _, err := msgStream.List(); err != nil { - return err - } - // Gather blocks until the fetch or network limits is reached - var ( - hash common.Hash - bytes common.StorageSize - blocks []*types.Block - ) - for len(blocks) < downloader.MaxBlockFetch && bytes < softResponseLimit { - //Retrieve the hash of the next block - err := msgStream.Decode(&hash) - if err == rlp.EOL { - break - } else if err != nil { - return errResp(ErrDecode, "msg %v: %v", msg, err) - } - // Retrieve the requested block, stopping if enough was found - if block := pm.blockchain.GetBlockByHash(hash); block != nil { - blocks = append(blocks, block) - bytes += block.Size() - } - } - return p.SendBlocks(blocks) - - case p.version < eth62 && msg.Code == BlocksMsg: - // Decode the arrived block message - var blocks []*types.Block - if err := msg.Decode(&blocks); err != nil { - glog.V(logger.Detail).Infoln("Decode error", err) - blocks = nil - } - // Update the receive timestamp of each block - for _, block := range blocks { - block.ReceivedAt = msg.ReceivedAt - block.ReceivedFrom = p - } - // Filter out any explicitly requested blocks, deliver the rest to the downloader - if blocks := pm.fetcher.FilterBlocks(blocks); len(blocks) > 0 { - pm.downloader.DeliverBlocks(p.id, blocks) - } - // Block header query, collect the requested headers and reply - case p.version >= eth62 && msg.Code == GetBlockHeadersMsg: + case msg.Code == GetBlockHeadersMsg: // Decode the complex header query var query getBlockHeadersData if err := msg.Decode(&query); err != nil { @@ -500,7 +399,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } return p.SendBlockHeaders(headers) - case p.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: // A batch of headers arrived to one of our previous requests var headers []*types.Header if err := msg.Decode(&headers); err != nil { @@ -552,7 +451,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } - case p.version >= eth62 && msg.Code == GetBlockBodiesMsg: + case msg.Code == GetBlockBodiesMsg: // Decode the retrieval message msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) if _, err := msgStream.List(); err != nil { @@ -579,7 +478,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } return p.SendBlockBodiesRLP(bodies) - case p.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: // A batch of block bodies arrived to one of our previous requests var request blockBodiesData if err := msg.Decode(&request); err != nil { @@ -730,11 +629,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } for _, block := range unknown { - if p.version < eth62 { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestBlocks, nil, nil) - } else { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), nil, p.RequestOneHeader, p.RequestBodies) - } + pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) } case msg.Code == NewBlockMsg: @@ -816,11 +711,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // Otherwise if the block is indeed in out own chain, announce it if pm.blockchain.HasBlock(hash) { for _, peer := range peers { - if peer.version < eth62 { - peer.SendNewBlockHashes61([]common.Hash{hash}) - } else { - peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) - } + peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) } glog.V(logger.Detail).Infof("announced block %x to %d peers in %v", hash[:4], len(peers), time.Since(block.ReceivedAt)) } diff --git a/eth/handler_test.go b/eth/handler_test.go index 66ff26809..91989cb8f 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -63,160 +63,6 @@ func TestProtocolCompatibility(t *testing.T) { } } -// Tests that hashes can be retrieved from a remote chain by hashes in reverse -// order. -func TestGetBlockHashes61(t *testing.T) { testGetBlockHashes(t, 61) } - -func testGetBlockHashes(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxHashFetch - tests := []struct { - origin common.Hash - number int - result int - }{ - {common.Hash{}, 1, 0}, // Make sure non existent hashes don't return results - {pm.blockchain.Genesis().Hash(), 1, 0}, // There are no hashes to retrieve up from the genesis - {pm.blockchain.GetBlockByNumber(5).Hash(), 5, 5}, // All the hashes including the genesis requested - {pm.blockchain.GetBlockByNumber(5).Hash(), 10, 5}, // More hashes than available till the genesis requested - {pm.blockchain.GetBlockByNumber(100).Hash(), 10, 10}, // All hashes available from the middle of the chain - {pm.blockchain.CurrentBlock().Hash(), 10, 10}, // All hashes available from the head of the chain - {pm.blockchain.CurrentBlock().Hash(), limit, limit}, // Request the maximum allowed hash count - {pm.blockchain.CurrentBlock().Hash(), limit + 1, limit}, // Request more than the maximum allowed hash count - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Assemble the hash response we would like to receive - resp := make([]common.Hash, tt.result) - if len(resp) > 0 { - from := pm.blockchain.GetBlockByHash(tt.origin).NumberU64() - 1 - for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from) - j)).Hash() - } - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x03, getBlockHashesData{tt.origin, uint64(tt.number)}) - if err := p2p.ExpectMsg(peer.app, 0x04, resp); err != nil { - t.Errorf("test %d: block hashes mismatch: %v", i, err) - } - } -} - -// Tests that hashes can be retrieved from a remote chain by numbers in forward -// order. -func TestGetBlockHashesFromNumber61(t *testing.T) { testGetBlockHashesFromNumber(t, 61) } - -func testGetBlockHashesFromNumber(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxHashFetch - tests := []struct { - origin uint64 - number int - result int - }{ - {pm.blockchain.CurrentBlock().NumberU64() + 1, 1, 0}, // Out of bounds requests should return empty - {pm.blockchain.CurrentBlock().NumberU64(), 1, 1}, // Make sure the head hash can be retrieved - {pm.blockchain.CurrentBlock().NumberU64() - 4, 5, 5}, // All hashes, including the head hash requested - {pm.blockchain.CurrentBlock().NumberU64() - 4, 10, 5}, // More hashes requested than available till the head - {pm.blockchain.CurrentBlock().NumberU64() - 100, 10, 10}, // All hashes available from the middle of the chain - {0, 10, 10}, // All hashes available from the root of the chain - {0, limit, limit}, // Request the maximum allowed hash count - {0, limit + 1, limit}, // Request more than the maximum allowed hash count - {0, 1, 1}, // Make sure the genesis hash can be retrieved - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Assemble the hash response we would like to receive - resp := make([]common.Hash, tt.result) - for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(tt.origin + uint64(j)).Hash() - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x08, getBlockHashesFromNumberData{tt.origin, uint64(tt.number)}) - if err := p2p.ExpectMsg(peer.app, 0x04, resp); err != nil { - t.Errorf("test %d: block hashes mismatch: %v", i, err) - } - } -} - -// Tests that blocks can be retrieved from a remote chain based on their hashes. -func TestGetBlocks61(t *testing.T) { testGetBlocks(t, 61) } - -func testGetBlocks(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil) - peer, _ := newTestPeer("peer", protocol, pm, true) - defer peer.close() - - // Create a batch of tests for various scenarios - limit := downloader.MaxBlockFetch - tests := []struct { - random int // Number of blocks to fetch randomly from the chain - explicit []common.Hash // Explicitly requested blocks - available []bool // Availability of explicitly requested blocks - expected int // Total number of existing blocks to expect - }{ - {1, nil, nil, 1}, // A single random block should be retrievable - {10, nil, nil, 10}, // Multiple random blocks should be retrievable - {limit, nil, nil, limit}, // The maximum possible blocks should be retrievable - {limit + 1, nil, nil, limit}, // No more than the possible block count should be returned - {0, []common.Hash{pm.blockchain.Genesis().Hash()}, []bool{true}, 1}, // The genesis block should be retrievable - {0, []common.Hash{pm.blockchain.CurrentBlock().Hash()}, []bool{true}, 1}, // The chains head block should be retrievable - {0, []common.Hash{common.Hash{}}, []bool{false}, 0}, // A non existent block should not be returned - - // Existing and non-existing blocks interleaved should not cause problems - {0, []common.Hash{ - common.Hash{}, - pm.blockchain.GetBlockByNumber(1).Hash(), - common.Hash{}, - pm.blockchain.GetBlockByNumber(10).Hash(), - common.Hash{}, - pm.blockchain.GetBlockByNumber(100).Hash(), - common.Hash{}, - }, []bool{false, true, false, true, false, true, false}, 3}, - } - // Run each of the tests and verify the results against the chain - for i, tt := range tests { - // Collect the hashes to request, and the response to expect - hashes, seen := []common.Hash{}, make(map[int64]bool) - blocks := []*types.Block{} - - for j := 0; j < tt.random; j++ { - for { - num := rand.Int63n(int64(pm.blockchain.CurrentBlock().NumberU64())) - if !seen[num] { - seen[num] = true - - block := pm.blockchain.GetBlockByNumber(uint64(num)) - hashes = append(hashes, block.Hash()) - if len(blocks) < tt.expected { - blocks = append(blocks, block) - } - break - } - } - } - for j, hash := range tt.explicit { - hashes = append(hashes, hash) - if tt.available[j] && len(blocks) < tt.expected { - blocks = append(blocks, pm.blockchain.GetBlockByHash(hash)) - } - } - // Send the hash request and verify the response - p2p.Send(peer.app, 0x05, hashes) - if err := p2p.ExpectMsg(peer.app, 0x06, blocks); err != nil { - t.Errorf("test %d: blocks mismatch: %v", i, err) - } - } -} - // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeaders62(t *testing.T) { testGetBlockHeaders(t, 62) } func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) } diff --git a/eth/metrics.go b/eth/metrics.go index e1a89d3a9..5fa2597d4 100644 --- a/eth/metrics.go +++ b/eth/metrics.go @@ -34,14 +34,6 @@ var ( propBlockInTrafficMeter = metrics.NewMeter("eth/prop/blocks/in/traffic") propBlockOutPacketsMeter = metrics.NewMeter("eth/prop/blocks/out/packets") propBlockOutTrafficMeter = metrics.NewMeter("eth/prop/blocks/out/traffic") - reqHashInPacketsMeter = metrics.NewMeter("eth/req/hashes/in/packets") - reqHashInTrafficMeter = metrics.NewMeter("eth/req/hashes/in/traffic") - reqHashOutPacketsMeter = metrics.NewMeter("eth/req/hashes/out/packets") - reqHashOutTrafficMeter = metrics.NewMeter("eth/req/hashes/out/traffic") - reqBlockInPacketsMeter = metrics.NewMeter("eth/req/blocks/in/packets") - reqBlockInTrafficMeter = metrics.NewMeter("eth/req/blocks/in/traffic") - reqBlockOutPacketsMeter = metrics.NewMeter("eth/req/blocks/out/packets") - reqBlockOutTrafficMeter = metrics.NewMeter("eth/req/blocks/out/traffic") reqHeaderInPacketsMeter = metrics.NewMeter("eth/req/headers/in/packets") reqHeaderInTrafficMeter = metrics.NewMeter("eth/req/headers/in/traffic") reqHeaderOutPacketsMeter = metrics.NewMeter("eth/req/headers/out/packets") @@ -95,14 +87,9 @@ func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) { // Account for the data traffic packets, traffic := miscInPacketsMeter, miscInTrafficMeter switch { - case rw.version < eth62 && msg.Code == BlockHashesMsg: - packets, traffic = reqHashInPacketsMeter, reqHashInTrafficMeter - case rw.version < eth62 && msg.Code == BlocksMsg: - packets, traffic = reqBlockInPacketsMeter, reqBlockInTrafficMeter - - case rw.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter - case rw.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter case rw.version >= eth63 && msg.Code == NodeDataMsg: @@ -127,14 +114,9 @@ func (rw *meteredMsgReadWriter) WriteMsg(msg p2p.Msg) error { // Account for the data traffic packets, traffic := miscOutPacketsMeter, miscOutTrafficMeter switch { - case rw.version < eth62 && msg.Code == BlockHashesMsg: - packets, traffic = reqHashOutPacketsMeter, reqHashOutTrafficMeter - case rw.version < eth62 && msg.Code == BlocksMsg: - packets, traffic = reqBlockOutPacketsMeter, reqBlockOutTrafficMeter - - case rw.version >= eth62 && msg.Code == BlockHeadersMsg: + case msg.Code == BlockHeadersMsg: packets, traffic = reqHeaderOutPacketsMeter, reqHeaderOutTrafficMeter - case rw.version >= eth62 && msg.Code == BlockBodiesMsg: + case msg.Code == BlockBodiesMsg: packets, traffic = reqBodyOutPacketsMeter, reqBodyOutTrafficMeter case rw.version >= eth63 && msg.Code == NodeDataMsg: diff --git a/eth/peer.go b/eth/peer.go index b97825c69..c8c207ecb 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/p2p" @@ -154,25 +153,6 @@ func (p *peer) SendTransactions(txs types.Transactions) error { return p2p.Send(p.rw, TxMsg, txs) } -// SendBlockHashes sends a batch of known hashes to the remote peer. -func (p *peer) SendBlockHashes(hashes []common.Hash) error { - return p2p.Send(p.rw, BlockHashesMsg, hashes) -} - -// SendBlocks sends a batch of blocks to the remote peer. -func (p *peer) SendBlocks(blocks []*types.Block) error { - return p2p.Send(p.rw, BlocksMsg, blocks) -} - -// SendNewBlockHashes61 announces the availability of a number of blocks through -// a hash notification. -func (p *peer) SendNewBlockHashes61(hashes []common.Hash) error { - for _, hash := range hashes { - p.knownBlocks.Add(hash) - } - return p2p.Send(p.rw, NewBlockHashesMsg, hashes) -} - // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -221,26 +201,6 @@ func (p *peer) SendReceiptsRLP(receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, receipts) } -// RequestHashes fetches a batch of hashes from a peer, starting at from, going -// towards the genesis block. -func (p *peer) RequestHashes(from common.Hash) error { - glog.V(logger.Debug).Infof("%v fetching hashes (%d) from %x...", p, downloader.MaxHashFetch, from[:4]) - return p2p.Send(p.rw, GetBlockHashesMsg, getBlockHashesData{from, uint64(downloader.MaxHashFetch)}) -} - -// RequestHashesFromNumber fetches a batch of hashes from a peer, starting at -// the requested block number, going upwards towards the genesis block. -func (p *peer) RequestHashesFromNumber(from uint64, count int) error { - glog.V(logger.Debug).Infof("%v fetching hashes (%d) from #%d...", p, count, from) - return p2p.Send(p.rw, GetBlockHashesFromNumberMsg, getBlockHashesFromNumberData{from, uint64(count)}) -} - -// RequestBlocks fetches a batch of blocks corresponding to the specified hashes. -func (p *peer) RequestBlocks(hashes []common.Hash) error { - glog.V(logger.Debug).Infof("%v fetching %v blocks", p, len(hashes)) - return p2p.Send(p.rw, GetBlocksMsg, hashes) -} - // RequestHeaders is a wrapper around the header query functions to fetch a // single header. It is used solely by the fetcher. func (p *peer) RequestOneHeader(hash common.Hash) error { diff --git a/eth/protocol.go b/eth/protocol.go index 7de0cb020..69b3be578 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -28,7 +28,6 @@ import ( // Constants to match up protocol versions and messages const ( - eth61 = 61 eth62 = 62 eth63 = 63 ) @@ -49,26 +48,15 @@ const ( // eth protocol message codes const ( - // Protocol messages belonging to eth/61 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TxMsg = 0x02 - GetBlockHashesMsg = 0x03 - BlockHashesMsg = 0x04 - GetBlocksMsg = 0x05 - BlocksMsg = 0x06 - NewBlockMsg = 0x07 - GetBlockHashesFromNumberMsg = 0x08 - - // Protocol messages belonging to eth/62 (new protocol from scratch) - // StatusMsg = 0x00 (uncomment after eth/61 deprecation) - // NewBlockHashesMsg = 0x01 (uncomment after eth/61 deprecation) - // TxMsg = 0x02 (uncomment after eth/61 deprecation) + // Protocol messages belonging to eth/62 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TxMsg = 0x02 GetBlockHeadersMsg = 0x03 BlockHeadersMsg = 0x04 GetBlockBodiesMsg = 0x05 BlockBodiesMsg = 0x06 - // NewBlockMsg = 0x07 (uncomment after eth/61 deprecation) + NewBlockMsg = 0x07 // Protocol messages belonging to eth/63 GetNodeDataMsg = 0x0d @@ -117,12 +105,6 @@ type txPool interface { GetTransactions() types.Transactions } -type chainManager interface { - GetBlockHashesFromHash(hash common.Hash, amount uint64) (hashes []common.Hash) - GetBlock(hash common.Hash) (block *types.Block) - Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) -} - // statusData is the network packet for the status message. type statusData struct { ProtocolVersion uint32 @@ -138,19 +120,6 @@ type newBlockHashesData []struct { Number uint64 // Number of one particular block being announced } -// getBlockHashesData is the network packet for the hash based hash retrieval. -type getBlockHashesData struct { - Hash common.Hash - Amount uint64 -} - -// getBlockHashesFromNumberData is the network packet for the number based hash -// retrieval. -type getBlockHashesFromNumberData struct { - Number uint64 - Amount uint64 -} - // getBlockHeadersData represents a block header query. type getBlockHeadersData struct { Origin hashOrNumber // Block from which to retrieve headers @@ -209,8 +178,3 @@ type blockBody struct { // blockBodiesData is the network packet for block content distribution. type blockBodiesData []*blockBody - -// nodeDataData is the network response packet for a node data retrieval. -type nodeDataData []struct { - Value []byte -} diff --git a/eth/protocol_test.go b/eth/protocol_test.go index f860d0a35..4633344da 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -37,7 +37,6 @@ func init() { var testAccount, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") // Tests that handshake failures are detected and reported correctly. -func TestStatusMsgErrors61(t *testing.T) { testStatusMsgErrors(t, 61) } func TestStatusMsgErrors62(t *testing.T) { testStatusMsgErrors(t, 62) } func TestStatusMsgErrors63(t *testing.T) { testStatusMsgErrors(t, 63) } @@ -90,7 +89,6 @@ func testStatusMsgErrors(t *testing.T, protocol int) { } // This test checks that received transactions are added to the local pool. -func TestRecvTransactions61(t *testing.T) { testRecvTransactions(t, 61) } func TestRecvTransactions62(t *testing.T) { testRecvTransactions(t, 62) } func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) } @@ -119,7 +117,6 @@ func testRecvTransactions(t *testing.T, protocol int) { } // This test checks that pending transactions are sent. -func TestSendTransactions61(t *testing.T) { testSendTransactions(t, 61) } func TestSendTransactions62(t *testing.T) { testSendTransactions(t, 62) } func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) } diff --git a/internal/jsre/pretty.go b/internal/jsre/pretty.go index 30d8660ff..f32e16243 100644 --- a/internal/jsre/pretty.go +++ b/internal/jsre/pretty.go @@ -116,7 +116,7 @@ func (ctx ppctx) printValue(v otto.Value, level int, inArray bool) { func (ctx ppctx) printObject(obj *otto.Object, level int, inArray bool) { switch obj.Class() { - case "Array": + case "Array", "GoArray": lv, _ := obj.Get("length") len, _ := lv.ToInteger() if len == 0 { diff --git a/node/node.go b/node/node.go index 1f517a027..ac8a7e8f0 100644 --- a/node/node.go +++ b/node/node.go @@ -505,16 +505,14 @@ func (n *Node) Restart() error { } // Attach creates an RPC client attached to an in-process API handler. -func (n *Node) Attach() (rpc.Client, error) { +func (n *Node) Attach() (*rpc.Client, error) { n.lock.RLock() defer n.lock.RUnlock() - // Short circuit if the node's not running if n.server == nil { return nil, ErrNodeStopped } - // Otherwise attach to the API and return - return rpc.NewInProcRPCClient(n.inprocHandler), nil + return rpc.DialInProc(n.inprocHandler), nil } // Server retrieves the currently running P2P network layer. This method is meant diff --git a/node/node_test.go b/node/node_test.go index 372fc6b10..d9b26453b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -507,21 +507,27 @@ func TestAPIGather(t *testing.T) { } // Register a batch of services with some configured APIs calls := make(chan string, 1) - + makeAPI := func(result string) *OneMethodApi { + return &OneMethodApi{fun: func() { calls <- result }} + } services := map[string]struct { APIs []rpc.API Maker InstrumentingWrapper }{ - "Zero APIs": {[]rpc.API{}, InstrumentedServiceMakerA}, - "Single API": {[]rpc.API{ - {"single", "1", &OneMethodApi{fun: func() { calls <- "single.v1" }}, true}, - }, InstrumentedServiceMakerB}, - "Many APIs": {[]rpc.API{ - {"multi", "1", &OneMethodApi{fun: func() { calls <- "multi.v1" }}, true}, - {"multi.v2", "2", &OneMethodApi{fun: func() { calls <- "multi.v2" }}, true}, - {"multi.v2.nested", "2", &OneMethodApi{fun: func() { calls <- "multi.v2.nested" }}, true}, - }, InstrumentedServiceMakerC}, + "Zero APIs": { + []rpc.API{}, InstrumentedServiceMakerA}, + "Single API": { + []rpc.API{ + {Namespace: "single", Version: "1", Service: makeAPI("single.v1"), Public: true}, + }, InstrumentedServiceMakerB}, + "Many APIs": { + []rpc.API{ + {Namespace: "multi", Version: "1", Service: makeAPI("multi.v1"), Public: true}, + {Namespace: "multi.v2", Version: "2", Service: makeAPI("multi.v2"), Public: true}, + {Namespace: "multi.v2.nested", Version: "2", Service: makeAPI("multi.v2.nested"), Public: true}, + }, InstrumentedServiceMakerC}, } + for id, config := range services { config := config constructor := func(*ServiceContext) (Service, error) { @@ -554,12 +560,8 @@ func TestAPIGather(t *testing.T) { {"multi.v2.nested_theOneMethod", "multi.v2.nested"}, } for i, test := range tests { - if err := client.Send(rpc.JSONRequest{Id: []byte("1"), Version: "2.0", Method: test.Method}); err != nil { - t.Fatalf("test %d: failed to send API request: %v", i, err) - } - reply := new(rpc.JSONSuccessResponse) - if err := client.Recv(reply); err != nil { - t.Fatalf("test %d: failed to read API reply: %v", i, err) + if err := client.Call(nil, test.Method); err != nil { + t.Errorf("test %d: API request failed: %v", i, err) } select { case result := <-calls: diff --git a/rpc/client.go b/rpc/client.go new file mode 100644 index 000000000..4ff9a8cb9 --- /dev/null +++ b/rpc/client.go @@ -0,0 +1,740 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rpc + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "reflect" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" +) + +var ( + ErrClientQuit = errors.New("client is closed") + ErrNoResult = errors.New("no result in JSON-RPC response") +) + +const ( + clientSubscriptionBuffer = 100 // if exceeded, the client stops reading + tcpKeepAliveInterval = 30 * time.Second + defaultDialTimeout = 10 * time.Second // used when dialing if the context has no deadline + defaultWriteTimeout = 10 * time.Second // used for calls if the context has no deadline + subscribeTimeout = 5 * time.Second // overall timeout eth_subscribe, rpc_modules calls +) + +// BatchElem is an element in a batch request. +type BatchElem struct { + Method string + Args []interface{} + // The result is unmarshaled into this field. Result must be set to a + // non-nil pointer value of the desired type, otherwise the response will be + // discarded. + Result interface{} + // Error is set if the server returns an error for this request, or if + // unmarshaling into Result fails. It is not set for I/O errors. + Error error +} + +// A value of this type can a JSON-RPC request, notification, successful response or +// error response. Which one it is depends on the fields. +type jsonrpcMessage struct { + Version string `json:"jsonrpc"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Error *jsonError `json:"error,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +func (msg *jsonrpcMessage) isNotification() bool { + return msg.ID == nil && msg.Method != "" +} + +func (msg *jsonrpcMessage) isResponse() bool { + return msg.hasValidID() && msg.Method == "" && len(msg.Params) == 0 +} + +func (msg *jsonrpcMessage) hasValidID() bool { + return len(msg.ID) > 0 && msg.ID[0] != '{' && msg.ID[0] != '[' +} + +func (msg *jsonrpcMessage) String() string { + b, _ := json.Marshal(msg) + return string(b) +} + +// Client represents a connection to an RPC server. +type Client struct { + idCounter uint32 + connectFunc func(ctx context.Context) (net.Conn, error) + isHTTP bool + + // writeConn is only safe to access outside dispatch, with the + // write lock held. The write lock is taken by sending on + // requestOp and released by sending on sendDone. + writeConn net.Conn + + // for dispatch + close chan struct{} + didQuit chan struct{} // closed when client quits + reconnected chan net.Conn // where write/reconnect sends the new connection + readErr chan error // errors from read + readResp chan []*jsonrpcMessage // valid messages from read + requestOp chan *requestOp // for registering response IDs + sendDone chan error // signals write completion, releases write lock + respWait map[string]*requestOp // active requests + subs map[string]*ClientSubscription // active subscriptions +} + +type requestOp struct { + ids []json.RawMessage + err error + resp chan *jsonrpcMessage // receives up to len(ids) responses + sub *ClientSubscription // only set for EthSubscribe requests +} + +func (op *requestOp) wait(ctx context.Context) (*jsonrpcMessage, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case resp := <-op.resp: + return resp, op.err + } +} + +// Dial creates a new client for the given URL. +// +// The currently supported URL schemes are "http", "https", "ws" and "wss". If rawurl is a +// file name with no URL scheme, a local socket connection is established using UNIX +// domain sockets on supported platforms and named pipes on Windows. If you want to +// configure transport options, use DialHTTP, DialWebsocket or DialIPC instead. +// +// For websocket connections, the origin is set to the local host name. +// +// The client reconnects automatically if the connection is lost. +func Dial(rawurl string) (*Client, error) { + return DialContext(context.Background(), rawurl) +} + +// DialContext creates a new RPC client, just like Dial. +// +// The context is used to cancel or time out the initial connection establishment. It does +// not affect subsequent interactions with the client. +func DialContext(ctx context.Context, rawurl string) (*Client, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + switch u.Scheme { + case "http", "https": + return DialHTTP(rawurl) + case "ws", "wss": + return DialWebsocket(ctx, rawurl, "") + case "": + return DialIPC(ctx, rawurl) + default: + return nil, fmt.Errorf("no known transport for URL scheme %q", u.Scheme) + } +} + +func newClient(initctx context.Context, connectFunc func(context.Context) (net.Conn, error)) (*Client, error) { + conn, err := connectFunc(initctx) + if err != nil { + return nil, err + } + _, isHTTP := conn.(*httpConn) + + c := &Client{ + writeConn: conn, + isHTTP: isHTTP, + connectFunc: connectFunc, + close: make(chan struct{}), + didQuit: make(chan struct{}), + reconnected: make(chan net.Conn), + readErr: make(chan error), + readResp: make(chan []*jsonrpcMessage), + requestOp: make(chan *requestOp), + sendDone: make(chan error, 1), + respWait: make(map[string]*requestOp), + subs: make(map[string]*ClientSubscription), + } + if !isHTTP { + go c.dispatch(conn) + } + return c, nil +} + +func (c *Client) nextID() json.RawMessage { + id := atomic.AddUint32(&c.idCounter, 1) + return []byte(strconv.FormatUint(uint64(id), 10)) +} + +// SupportedModules calls the rpc_modules method, retrieving the list of +// APIs that are available on the server. +func (c *Client) SupportedModules() (map[string]string, error) { + var result map[string]string + ctx, cancel := context.WithTimeout(context.Background(), subscribeTimeout) + defer cancel() + err := c.CallContext(ctx, &result, "rpc_modules") + return result, err +} + +// Close closes the client, aborting any in-flight requests. +func (c *Client) Close() { + if c.isHTTP { + return + } + select { + case c.close <- struct{}{}: + <-c.didQuit + case <-c.didQuit: + } +} + +// Call performs a JSON-RPC call with the given arguments and unmarshals into +// result if no error occurred. +// +// The result must be a pointer so that package json can unmarshal into it. You +// can also pass nil, in which case the result is ignored. +func (c *Client) Call(result interface{}, method string, args ...interface{}) error { + ctx := context.Background() + return c.CallContext(ctx, result, method, args...) +} + +// CallContext performs a JSON-RPC call with the given arguments. If the context is +// canceled before the call has successfully returned, CallContext returns immediately. +// +// The result must be a pointer so that package json can unmarshal into it. You +// can also pass nil, in which case the result is ignored. +func (c *Client) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + msg, err := c.newMessage(method, args...) + if err != nil { + return err + } + op := &requestOp{ids: []json.RawMessage{msg.ID}, resp: make(chan *jsonrpcMessage, 1)} + + if c.isHTTP { + err = c.sendHTTP(ctx, op, msg) + } else { + err = c.send(ctx, op, msg) + } + if err != nil { + return err + } + + // dispatch has accepted the request and will close the channel it when it quits. + switch resp, err := op.wait(ctx); { + case err != nil: + return err + case resp.Error != nil: + return resp.Error + case len(resp.Result) == 0: + return ErrNoResult + default: + return json.Unmarshal(resp.Result, &result) + } +} + +// BatchCall sends all given requests as a single batch and waits for the server +// to return a response for all of them. +// +// In contrast to Call, BatchCall only returns I/O errors. Any error specific to +// a request is reported through the Error field of the corresponding BatchElem. +// +// Note that batch calls may not be executed atomically on the server side. +func (c *Client) BatchCall(b []BatchElem) error { + ctx := context.Background() + return c.BatchCallContext(ctx, b) +} + +// BatchCall sends all given requests as a single batch and waits for the server +// to return a response for all of them. The wait duration is bounded by the +// context's deadline. +// +// In contrast to CallContext, BatchCallContext only returns I/O errors. Any +// error specific to a request is reported through the Error field of the +// corresponding BatchElem. +// +// Note that batch calls may not be executed atomically on the server side. +func (c *Client) BatchCallContext(ctx context.Context, b []BatchElem) error { + msgs := make([]*jsonrpcMessage, len(b)) + op := &requestOp{ + ids: make([]json.RawMessage, len(b)), + resp: make(chan *jsonrpcMessage, len(b)), + } + for i, elem := range b { + msg, err := c.newMessage(elem.Method, elem.Args...) + if err != nil { + return err + } + msgs[i] = msg + op.ids[i] = msg.ID + } + + var err error + if c.isHTTP { + err = c.sendBatchHTTP(ctx, op, msgs) + } else { + err = c.send(ctx, op, msgs) + } + + // Wait for all responses to come back. + for n := 0; n < len(b) && err == nil; n++ { + var resp *jsonrpcMessage + resp, err = op.wait(ctx) + if err != nil { + break + } + // Find the element corresponding to this response. + // The element is guaranteed to be present because dispatch + // only sends valid IDs to our channel. + var elem *BatchElem + for i := range msgs { + if bytes.Equal(msgs[i].ID, resp.ID) { + elem = &b[i] + break + } + } + if resp.Error != nil { + elem.Error = resp.Error + continue + } + if len(resp.Result) == 0 { + elem.Error = ErrNoResult + continue + } + elem.Error = json.Unmarshal(resp.Result, elem.Result) + } + return err +} + +// EthSubscribe calls the "eth_subscribe" method with the given arguments, +// registering a subscription. Server notifications for the subscription are +// sent to the given channel. The element type of the channel must match the +// expected type of content returned by the subscription. +// +// Callers should not use the same channel for multiple calls to EthSubscribe. +// The channel is closed when the notification is unsubscribed or an error +// occurs. The error can be retrieved via the Err method of the subscription. +// +// Slow subscribers will block the clients ingress path eventually. +func (c *Client) EthSubscribe(channel interface{}, args ...interface{}) (*ClientSubscription, error) { + // Check type of channel first. + chanVal := reflect.ValueOf(channel) + if chanVal.Kind() != reflect.Chan || chanVal.Type().ChanDir()&reflect.SendDir == 0 { + panic("first argument to EthSubscribe must be a writable channel") + } + if chanVal.IsNil() { + panic("channel given to EthSubscribe must not be nil") + } + if c.isHTTP { + return nil, ErrNotificationsUnsupported + } + + msg, err := c.newMessage(subscribeMethod, args...) + if err != nil { + return nil, err + } + op := &requestOp{ + ids: []json.RawMessage{msg.ID}, + resp: make(chan *jsonrpcMessage), + sub: newClientSubscription(c, chanVal), + } + ctx, cancel := context.WithTimeout(context.Background(), subscribeTimeout) + defer cancel() + + // Send the subscription request. + // The arrival and validity of the response is signaled on sub.quit. + if err := c.send(ctx, op, msg); err != nil { + return nil, err + } + if _, err := op.wait(ctx); err != nil { + return nil, err + } + return op.sub, nil +} + +func (c *Client) newMessage(method string, paramsIn ...interface{}) (*jsonrpcMessage, error) { + params, err := json.Marshal(paramsIn) + if err != nil { + return nil, err + } + return &jsonrpcMessage{Version: "2.0", ID: c.nextID(), Method: method, Params: params}, nil +} + +// send registers op with the dispatch loop, then sends msg on the connection. +// if sending fails, op is deregistered. +func (c *Client) send(ctx context.Context, op *requestOp, msg interface{}) error { + select { + case c.requestOp <- op: + if glog.V(logger.Detail) { + glog.Info("sending ", msg) + } + err := c.write(ctx, msg) + c.sendDone <- err + return err + case <-c.didQuit: + return ErrClientQuit + } +} + +func (c *Client) write(ctx context.Context, msg interface{}) error { + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(defaultWriteTimeout) + } + // The previous write failed. Try to establish a new connection. + if c.writeConn == nil { + if err := c.reconnect(ctx); err != nil { + return err + } + } + c.writeConn.SetWriteDeadline(deadline) + err := json.NewEncoder(c.writeConn).Encode(msg) + if err != nil { + c.writeConn = nil + } + return err +} + +func (c *Client) reconnect(ctx context.Context) error { + newconn, err := c.connectFunc(ctx) + if err != nil { + glog.V(logger.Detail).Infof("reconnect failed: %v", err) + return err + } + select { + case c.reconnected <- newconn: + c.writeConn = newconn + return nil + case <-c.didQuit: + newconn.Close() + return ErrClientQuit + } +} + +// dispatch is the main loop of the client. +// It sends read messages to waiting calls to Call and BatchCall +// and subscription notifications to registered subscriptions. +func (c *Client) dispatch(conn net.Conn) { + // Spawn the initial read loop. + go c.read(conn) + + var ( + lastOp *requestOp // tracks last send operation + requestOpLock = c.requestOp // nil while the send lock is held + reading = true // if true, a read loop is running + ) + defer close(c.didQuit) + defer func() { + c.closeRequestOps(ErrClientQuit) + conn.Close() + if reading { + // Empty read channels until read is dead. + for { + select { + case <-c.readResp: + case <-c.readErr: + return + } + } + } + }() + + for { + select { + case <-c.close: + return + + // Read path. + case batch := <-c.readResp: + for _, msg := range batch { + switch { + case msg.isNotification(): + if glog.V(logger.Detail) { + glog.Info("<-readResp: notification ", msg) + } + c.handleNotification(msg) + case msg.isResponse(): + if glog.V(logger.Detail) { + glog.Info("<-readResp: response ", msg) + } + c.handleResponse(msg) + default: + if glog.V(logger.Debug) { + glog.Error("<-readResp: dropping weird message", msg) + } + // TODO: maybe close + } + } + + case err := <-c.readErr: + glog.V(logger.Debug).Infof("<-readErr: %v", err) + c.closeRequestOps(err) + conn.Close() + reading = false + + case newconn := <-c.reconnected: + glog.V(logger.Debug).Infof("<-reconnected: (reading=%t) %v", reading, conn.RemoteAddr()) + if reading { + // Wait for the previous read loop to exit. This is a rare case. + conn.Close() + <-c.readErr + } + go c.read(newconn) + reading = true + conn = newconn + + // Send path. + case op := <-requestOpLock: + // Stop listening for further send ops until the current one is done. + requestOpLock = nil + lastOp = op + for _, id := range op.ids { + c.respWait[string(id)] = op + } + + case err := <-c.sendDone: + if err != nil { + // Remove response handlers for the last send. We remove those here + // because the error is already handled in Call or BatchCall. When the + // read loop goes down, it will signal all other current operations. + for _, id := range lastOp.ids { + delete(c.respWait, string(id)) + } + } + // Listen for send ops again. + requestOpLock = c.requestOp + lastOp = nil + } + } +} + +// closeRequestOps unblocks pending send ops and active subscriptions. +func (c *Client) closeRequestOps(err error) { + didClose := make(map[*requestOp]bool) + + for id, op := range c.respWait { + // Remove the op so that later calls will not close op.resp again. + delete(c.respWait, id) + + if !didClose[op] { + op.err = err + close(op.resp) + didClose[op] = true + } + } + for id, sub := range c.subs { + delete(c.subs, id) + sub.quitWithError(err, false) + } +} + +func (c *Client) handleNotification(msg *jsonrpcMessage) { + if msg.Method != notificationMethod { + glog.V(logger.Debug).Info("dropping non-subscription message: ", msg) + return + } + var subResult struct { + ID string `json:"subscription"` + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(msg.Params, &subResult); err != nil { + glog.V(logger.Debug).Info("dropping invalid subscription message: ", msg) + return + } + if c.subs[subResult.ID] != nil { + c.subs[subResult.ID].deliver(subResult.Result) + } +} + +func (c *Client) handleResponse(msg *jsonrpcMessage) { + op := c.respWait[string(msg.ID)] + if op == nil { + glog.V(logger.Debug).Infof("unsolicited response %v", msg) + return + } + delete(c.respWait, string(msg.ID)) + // For normal responses, just forward the reply to Call/BatchCall. + if op.sub == nil { + op.resp <- msg + return + } + // For subscription responses, start the subscription if the server + // indicates success. EthSubscribe gets unblocked in either case through + // the op.resp channel. + defer close(op.resp) + if msg.Error != nil { + op.err = msg.Error + return + } + if op.err = json.Unmarshal(msg.Result, &op.sub.subid); op.err == nil { + go op.sub.start() + c.subs[op.sub.subid] = op.sub + } +} + +// Reading happens on a dedicated goroutine. + +func (c *Client) read(conn net.Conn) error { + var ( + buf json.RawMessage + dec = json.NewDecoder(conn) + ) + readMessage := func() (rs []*jsonrpcMessage, err error) { + buf = buf[:0] + if err = dec.Decode(&buf); err != nil { + return nil, err + } + if isBatch(buf) { + err = json.Unmarshal(buf, &rs) + } else { + rs = make([]*jsonrpcMessage, 1) + err = json.Unmarshal(buf, &rs[0]) + } + return rs, err + } + + for { + resp, err := readMessage() + if err != nil { + c.readErr <- err + return err + } + c.readResp <- resp + } +} + +// Subscriptions. + +// A ClientSubscription represents a subscription established through EthSubscribe. +type ClientSubscription struct { + client *Client + etype reflect.Type + channel reflect.Value + subid string + in chan json.RawMessage + + quitOnce sync.Once // ensures quit is closed once + quit chan struct{} // quit is closed when the subscription exits + errOnce sync.Once // ensures err is closed once + err chan error +} + +func newClientSubscription(c *Client, channel reflect.Value) *ClientSubscription { + sub := &ClientSubscription{ + client: c, + etype: channel.Type().Elem(), + channel: channel, + quit: make(chan struct{}), + err: make(chan error, 1), + // in is buffered so dispatch can continue even if the subscriber is slow. + in: make(chan json.RawMessage, clientSubscriptionBuffer), + } + return sub +} + +// Err returns the subscription error channel. The intended use of Err is to schedule +// resubscription when the client connection is closed unexpectedly. +// +// The error channel receives a value when the subscription has ended due +// to an error. The received error is ErrClientQuit if Close has been called +// on the underlying client and no other error has occurred. +// +// The error channel is closed when Unsubscribe is called on the subscription. +func (sub *ClientSubscription) Err() <-chan error { + return sub.err +} + +// Unsubscribe unsubscribes the notification and closes the error channel. +// It can safely be called more than once. +func (sub *ClientSubscription) Unsubscribe() { + sub.quitWithError(nil, true) + sub.errOnce.Do(func() { close(sub.err) }) +} + +func (sub *ClientSubscription) quitWithError(err error, unsubscribeServer bool) { + sub.quitOnce.Do(func() { + if unsubscribeServer { + sub.requestUnsubscribe() + } + if err != nil { + sub.err <- err + } + close(sub.quit) + }) +} + +func (sub *ClientSubscription) deliver(result json.RawMessage) (ok bool) { + select { + case sub.in <- result: + return true + case <-sub.quit: + return false + } +} + +func (sub *ClientSubscription) start() { + sub.quitWithError(sub.forward()) +} + +func (sub *ClientSubscription) forward() (err error, unsubscribeServer bool) { + cases := []reflect.SelectCase{ + {Dir: reflect.SelectRecv, Chan: reflect.ValueOf(sub.quit)}, + {Dir: reflect.SelectSend, Chan: sub.channel}, + } + for { + select { + case result := <-sub.in: + val, err := sub.unmarshal(result) + if err != nil { + return err, true + } + cases[1].Send = val + switch chosen, _, _ := reflect.Select(cases); chosen { + case 0: // <-sub.quit + return nil, false + case 1: // sub.channel<- + continue + } + case <-sub.quit: + return nil, false + } + } +} + +func (sub *ClientSubscription) unmarshal(result json.RawMessage) (reflect.Value, error) { + val := reflect.New(sub.etype) + err := json.Unmarshal(result, val.Interface()) + return val.Elem(), err +} + +func (sub *ClientSubscription) requestUnsubscribe() error { + var result interface{} + return sub.client.Call(&result, unsubscribeMethod, sub.subid) +} diff --git a/rpc/client_context_go1.4.go b/rpc/client_context_go1.4.go new file mode 100644 index 000000000..ac956a17d --- /dev/null +++ b/rpc/client_context_go1.4.go @@ -0,0 +1,60 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build !go1.5 + +package rpc + +import ( + "net" + "net/http" + "time" + + "golang.org/x/net/context" +) + +// In older versions of Go (below 1.5), dials cannot be canceled +// via a channel or context. The context deadline can still applied. + +// contextDialer returns a dialer that applies the deadline value from the given context. +func contextDialer(ctx context.Context) *net.Dialer { + dialer := &net.Dialer{KeepAlive: tcpKeepAliveInterval} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline + } else { + dialer.Deadline = time.Now().Add(defaultDialTimeout) + } + return dialer +} + +// dialContext connects to the given address, aborting the dial if ctx is canceled. +func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return contextDialer(ctx).Dial(network, addr) +} + +// requestWithContext copies req, adding the cancelation channel and deadline from ctx. +func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { + // Set Timeout on the client if the context has a deadline. + // Note that there is no default timeout (unlike in contextDialer) because + // the timeout applies to the entire request, including reads from body. + if deadline, ok := ctx.Deadline(); ok { + c2 := *c + c2.Timeout = deadline.Sub(time.Now()) + c = &c2 + } + req2 := *req + return c, &req2 +} diff --git a/rpc/client_context_go1.5.go b/rpc/client_context_go1.5.go new file mode 100644 index 000000000..4a007d9f8 --- /dev/null +++ b/rpc/client_context_go1.5.go @@ -0,0 +1,61 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build go1.5,!go1.6 + +package rpc + +import ( + "net" + "net/http" + "time" + + "golang.org/x/net/context" +) + +// In Go 1.5, dials cannot be canceled via a channel or context. The context deadline can +// still be applied. Go 1.5 adds the ability to cancel HTTP requests via a channel. + +// contextDialer returns a dialer that applies the deadline value from the given context. +func contextDialer(ctx context.Context) *net.Dialer { + dialer := &net.Dialer{KeepAlive: tcpKeepAliveInterval} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline + } else { + dialer.Deadline = time.Now().Add(defaultDialTimeout) + } + return dialer +} + +// dialContext connects to the given address, aborting the dial if ctx is canceled. +func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return contextDialer(ctx).Dial(network, addr) +} + +// requestWithContext copies req, adding the cancelation channel and deadline from ctx. +func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { + // Set Timeout on the client if the context has a deadline. + // Note that there is no default timeout (unlike in contextDialer) because + // the timeout applies to the entire request, including reads from body. + if deadline, ok := ctx.Deadline(); ok { + c2 := *c + c2.Timeout = deadline.Sub(time.Now()) + c = &c2 + } + req2 := *req + req2.Cancel = ctx.Done() + return c, &req2 +} diff --git a/rpc/client_context_go1.6.go b/rpc/client_context_go1.6.go new file mode 100644 index 000000000..67777ddc6 --- /dev/null +++ b/rpc/client_context_go1.6.go @@ -0,0 +1,55 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build go1.6,!go1.7 + +package rpc + +import ( + "net" + "net/http" + "time" + + "golang.org/x/net/context" +) + +// In Go 1.6, net.Dialer gained the ability to cancel via a channel. + +// contextDialer returns a dialer that applies the deadline value from the given context. +func contextDialer(ctx context.Context) *net.Dialer { + dialer := &net.Dialer{Cancel: ctx.Done(), KeepAlive: tcpKeepAliveInterval} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline + } else { + dialer.Deadline = time.Now().Add(defaultDialTimeout) + } + return dialer +} + +// dialContext connects to the given address, aborting the dial if ctx is canceled. +func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return contextDialer(ctx).Dial(network, addr) +} + +// requestWithContext copies req, adding the cancelation channel and deadline from ctx. +func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { + // We set Timeout on the client for Go <= 1.5. There + // is no need to do that here because the dial will be canceled + // by package http. + req2 := *req + req2.Cancel = ctx.Done() + return c, &req2 +} diff --git a/rpc/client_context_go1.7.go b/rpc/client_context_go1.7.go new file mode 100644 index 000000000..56ce12ab8 --- /dev/null +++ b/rpc/client_context_go1.7.go @@ -0,0 +1,51 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build go1.7 + +package rpc + +import ( + "context" + "net" + "net/http" + "time" +) + +// In Go 1.7, context moved into the standard library and support +// for cancelation via context was added to net.Dialer and http.Request. + +// contextDialer returns a dialer that applies the deadline value from the given context. +func contextDialer(ctx context.Context) *net.Dialer { + dialer := &net.Dialer{Cancel: ctx.Done(), KeepAlive: tcpKeepAliveInterval} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline + } else { + dialer.Deadline = time.Now().Add(defaultDialTimeout) + } + return dialer +} + +// dialContext connects to the given address, aborting the dial if ctx is canceled. +func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + d := &net.Dialer{KeepAlive: tcpKeepAliveInterval} + return d.DialContext(ctx, network, addr) +} + +// requestWithContext copies req, adding the cancelation channel and deadline from ctx. +func requestWithContext(c *http.Client, req *http.Request, ctx context.Context) (*http.Client, *http.Request) { + return c, req.WithContext(ctx) +} diff --git a/rpc/client_example_test.go b/rpc/client_example_test.go new file mode 100644 index 000000000..84b4b67bb --- /dev/null +++ b/rpc/client_example_test.go @@ -0,0 +1,83 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rpc_test + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/rpc" +) + +// In this example, our client whishes to track the latest 'block number' +// known to the server. The server supports two methods: +// +// eth_getBlockByNumber("latest", {}) +// returns the latest block object. +// +// eth_subscribe("newBlocks") +// creates a subscription which fires block objects when new blocks arrive. + +type Block struct { + Number *big.Int +} + +func ExampleClientSubscription() { + // Connect the client. + client, _ := rpc.Dial("ws://127.0.0.1:8485") + subch := make(chan Block) + go subscribeBlocks(client, subch) + + // Print events from the subscription as they arrive. + for block := range subch { + fmt.Println("latest block:", block.Number) + } +} + +// subscribeBlocks runs in its own goroutine and maintains +// a subscription for new blocks. +func subscribeBlocks(client *rpc.Client, subch chan Block) { + for i := 0; ; i++ { + if i > 0 { + time.Sleep(2 * time.Second) + } + + // Subscribe to new blocks. + sub, err := client.EthSubscribe(subch, "newBlocks") + if err == rpc.ErrClientQuit { + return // Stop reconnecting if the client was closed. + } else if err != nil { + fmt.Println("subscribe error:", err) + continue + } + + // The connection is established now. + // Update the channel with the current block. + var lastBlock Block + if err := client.Call(&lastBlock, "eth_getBlockByNumber", "latest"); err != nil { + fmt.Println("can't get latest block:", err) + continue + } + subch <- lastBlock + + // The subscription will deliver events to the channel. Wait for the + // subscription to end for any reason, then loop around to re-establish + // the connection. + fmt.Println("connection lost: ", <-sub.Err()) + } +} diff --git a/rpc/client_test.go b/rpc/client_test.go new file mode 100644 index 000000000..58dceada0 --- /dev/null +++ b/rpc/client_test.go @@ -0,0 +1,489 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package rpc + +import ( + "fmt" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "os" + "reflect" + "runtime" + "sync" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" +) + +func TestClientRequest(t *testing.T) { + server := newTestServer("service", new(Service)) + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + var resp Result + if err := client.Call(&resp, "service_echo", "hello", 10, &Args{"world"}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(resp, Result{"hello", 10, &Args{"world"}}) { + t.Errorf("incorrect result %#v", resp) + } +} + +func TestClientBatchRequest(t *testing.T) { + server := newTestServer("service", new(Service)) + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + batch := []BatchElem{ + { + Method: "service_echo", + Args: []interface{}{"hello", 10, &Args{"world"}}, + Result: new(Result), + }, + { + Method: "service_echo", + Args: []interface{}{"hello2", 11, &Args{"world"}}, + Result: new(Result), + }, + { + Method: "no_such_method", + Args: []interface{}{1, 2, 3}, + Result: new(int), + }, + } + if err := client.BatchCall(batch); err != nil { + t.Fatal(err) + } + wantResult := []BatchElem{ + { + Method: "service_echo", + Args: []interface{}{"hello", 10, &Args{"world"}}, + Result: &Result{"hello", 10, &Args{"world"}}, + }, + { + Method: "service_echo", + Args: []interface{}{"hello2", 11, &Args{"world"}}, + Result: &Result{"hello2", 11, &Args{"world"}}, + }, + { + Method: "no_such_method", + Args: []interface{}{1, 2, 3}, + Result: new(int), + Error: &jsonError{Code: -32601, Message: "The method no_such_method_ does not exist/is not available"}, + }, + } + if !reflect.DeepEqual(batch, wantResult) { + t.Errorf("batch results mismatch:\ngot %swant %s", spew.Sdump(batch), spew.Sdump(wantResult)) + } +} + +// func TestClientCancelInproc(t *testing.T) { testClientCancel("inproc", t) } +func TestClientCancelWebsocket(t *testing.T) { testClientCancel("ws", t) } +func TestClientCancelHTTP(t *testing.T) { testClientCancel("http", t) } +func TestClientCancelIPC(t *testing.T) { testClientCancel("ipc", t) } + +// This test checks that requests made through CallContext can be canceled by canceling +// the context. +func testClientCancel(transport string, t *testing.T) { + server := newTestServer("service", new(Service)) + defer server.Stop() + + // What we want to achieve is that the context gets canceled + // at various stages of request processing. The interesting cases + // are: + // - cancel during dial + // - cancel while performing a HTTP request + // - cancel while waiting for a response + // + // To trigger those, the times are chosen such that connections + // are killed within the deadline for every other call (maxKillTimeout + // is 2x maxCancelTimeout). + // + // Once a connection is dead, there is a fair chance it won't connect + // successfully because the accept is delayed by 1s. + maxContextCancelTimeout := 300 * time.Millisecond + fl := &flakeyListener{ + maxAcceptDelay: 1 * time.Second, + maxKillTimeout: 600 * time.Millisecond, + } + + var client *Client + switch transport { + case "ws", "http": + c, hs := httpTestClient(server, transport, fl) + defer hs.Close() + client = c + case "ipc": + c, l := ipcTestClient(server, fl) + defer l.Close() + client = c + default: + panic("unknown transport: " + transport) + } + + // These tests take a lot of time, run them all at once. + // You probably want to run with -parallel 1 or comment out + // the call to t.Parallel if you enable the logging. + t.Parallel() + // glog.SetV(6) + // glog.SetToStderr(true) + // defer glog.SetToStderr(false) + // glog.Infoln("testing ", transport) + + // The actual test starts here. + var ( + wg sync.WaitGroup + nreqs = 10 + ncallers = 6 + ) + caller := func(index int) { + defer wg.Done() + for i := 0; i < nreqs; i++ { + var ( + ctx context.Context + cancel func() + timeout = time.Duration(rand.Int63n(int64(maxContextCancelTimeout))) + ) + if index < ncallers/2 { + // For half of the callers, create a context without deadline + // and cancel it later. + ctx, cancel = context.WithCancel(context.Background()) + time.AfterFunc(timeout, cancel) + } else { + // For the other half, create a context with a deadline instead. This is + // different because the context deadline is used to set the socket write + // deadline. + ctx, cancel = context.WithTimeout(context.Background(), timeout) + } + // Now perform a call with the context. + // The key thing here is that no call will ever complete successfully. + err := client.CallContext(ctx, nil, "service_sleep", 2*maxContextCancelTimeout) + if err != nil { + glog.V(logger.Debug).Infoln("got expected error:", err) + } else { + t.Errorf("no error for call with %v wait time", timeout) + } + cancel() + } + } + wg.Add(ncallers) + for i := 0; i < ncallers; i++ { + go caller(i) + } + wg.Wait() +} + +func TestClientSubscribeInvalidArg(t *testing.T) { + server := newTestServer("service", new(Service)) + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + check := func(shouldPanic bool, arg interface{}) { + defer func() { + err := recover() + if shouldPanic && err == nil { + t.Errorf("EthSubscribe should've panicked for %#v", arg) + } + if !shouldPanic && err != nil { + t.Errorf("EthSubscribe shouldn't have panicked for %#v", arg) + buf := make([]byte, 1024*1024) + buf = buf[:runtime.Stack(buf, false)] + t.Error(err) + t.Error(string(buf)) + } + }() + client.EthSubscribe(arg, "foo_bar") + } + check(true, nil) + check(true, 1) + check(true, (chan int)(nil)) + check(true, make(<-chan int)) + check(false, make(chan int)) + check(false, make(chan<- int)) +} + +func TestClientSubscribe(t *testing.T) { + server := newTestServer("eth", new(NotificationTestService)) + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + nc := make(chan int) + count := 10 + sub, err := client.EthSubscribe(nc, "someSubscription", count, 0) + if err != nil { + t.Fatal("can't subscribe:", err) + } + for i := 0; i < count; i++ { + if val := <-nc; val != i { + t.Fatalf("value mismatch: got %d, want %d", val, i) + } + } + + sub.Unsubscribe() + select { + case v := <-nc: + t.Fatal("received value after unsubscribe:", v) + case err := <-sub.Err(): + if err != nil { + t.Fatalf("Err returned a non-nil error after explicit unsubscribe: %q", err) + } + case <-time.After(1 * time.Second): + t.Fatalf("subscription not closed within 1s after unsubscribe") + } +} + +// In this test, the connection drops while EthSubscribe is +// waiting for a response. +func TestClientSubscribeClose(t *testing.T) { + service := &NotificationTestService{ + gotHangSubscriptionReq: make(chan struct{}), + unblockHangSubscription: make(chan struct{}), + } + server := newTestServer("eth", service) + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + var ( + nc = make(chan int) + errc = make(chan error) + sub *ClientSubscription + err error + ) + go func() { + sub, err = client.EthSubscribe(nc, "hangSubscription", 999) + errc <- err + }() + + <-service.gotHangSubscriptionReq + client.Close() + service.unblockHangSubscription <- struct{}{} + + select { + case err := <-errc: + if err == nil { + t.Errorf("EthSubscribe returned nil error after Close") + } + if sub != nil { + t.Error("EthSubscribe returned non-nil subscription after Close") + } + case <-time.After(1 * time.Second): + t.Fatalf("EthSubscribe did not return within 1s after Close") + } +} + +func TestClientHTTP(t *testing.T) { + server := newTestServer("service", new(Service)) + defer server.Stop() + + client, hs := httpTestClient(server, "http", nil) + defer hs.Close() + defer client.Close() + + // Launch concurrent requests. + var ( + results = make([]Result, 100) + errc = make(chan error) + wantResult = Result{"a", 1, new(Args)} + ) + defer client.Close() + for i := range results { + i := i + go func() { + errc <- client.Call(&results[i], "service_echo", + wantResult.String, wantResult.Int, wantResult.Args) + }() + } + + // Wait for all of them to complete. + timeout := time.NewTimer(5 * time.Second) + defer timeout.Stop() + for i := range results { + select { + case err := <-errc: + if err != nil { + t.Fatal(err) + } + case <-timeout.C: + t.Fatalf("timeout (got %d/%d) results)", i+1, len(results)) + } + } + + // Check results. + for i := range results { + if !reflect.DeepEqual(results[i], wantResult) { + t.Errorf("result %d mismatch: got %#v, want %#v", i, results[i], wantResult) + } + } +} + +func TestClientReconnect(t *testing.T) { + startServer := func(addr string) (*Server, net.Listener) { + srv := newTestServer("service", new(Service)) + l, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + go http.Serve(l, srv.WebsocketHandler("*")) + return srv, l + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Start a server and corresponding client. + s1, l1 := startServer("127.0.0.1:0") + client, err := DialContext(ctx, "ws://"+l1.Addr().String()) + if err != nil { + t.Fatal("can't dial", err) + } + + // Perform a call. This should work because the server is up. + var resp Result + if err := client.CallContext(ctx, &resp, "service_echo", "", 1, nil); err != nil { + t.Fatal(err) + } + + // Shut down the server and try calling again. It shouldn't work. + l1.Close() + s1.Stop() + if err := client.CallContext(ctx, &resp, "service_echo", "", 2, nil); err == nil { + t.Error("successful call while the server is down") + t.Logf("resp: %#v", resp) + } + + // Allow for some cool down time so we can listen on the same address again. + time.Sleep(2 * time.Second) + + // Start it up again and call again. The connection should be reestablished. + // We spawn multiple calls here to check whether this hangs somehow. + s2, l2 := startServer(l1.Addr().String()) + defer l2.Close() + defer s2.Stop() + + start := make(chan struct{}) + errors := make(chan error, 20) + for i := 0; i < cap(errors); i++ { + go func() { + <-start + var resp Result + errors <- client.CallContext(ctx, &resp, "service_echo", "", 3, nil) + }() + } + close(start) + errcount := 0 + for i := 0; i < cap(errors); i++ { + if err = <-errors; err != nil { + errcount++ + } + } + t.Log("err:", err) + if errcount > 1 { + t.Errorf("expected one error after disconnect, got %d", errcount) + } +} + +func newTestServer(serviceName string, service interface{}) *Server { + server := NewServer() + if err := server.RegisterName(serviceName, service); err != nil { + panic(err) + } + return server +} + +func httpTestClient(srv *Server, transport string, fl *flakeyListener) (*Client, *httptest.Server) { + // Create the HTTP server. + var hs *httptest.Server + switch transport { + case "ws": + hs = httptest.NewUnstartedServer(srv.WebsocketHandler("*")) + case "http": + hs = httptest.NewUnstartedServer(srv) + default: + panic("unknown HTTP transport: " + transport) + } + // Wrap the listener if required. + if fl != nil { + fl.Listener = hs.Listener + hs.Listener = fl + } + // Connect the client. + hs.Start() + client, err := Dial(transport + "://" + hs.Listener.Addr().String()) + if err != nil { + panic(err) + } + return client, hs +} + +func ipcTestClient(srv *Server, fl *flakeyListener) (*Client, net.Listener) { + // Listen on a random endpoint. + endpoint := fmt.Sprintf("go-ethereum-test-ipc-%d-%d", os.Getpid(), rand.Int63()) + if runtime.GOOS == "windows" { + endpoint = `\\.\pipe\` + endpoint + } else { + endpoint = os.TempDir() + "/" + endpoint + } + l, err := ipcListen(endpoint) + if err != nil { + panic(err) + } + // Connect the listener to the server. + if fl != nil { + fl.Listener = l + l = fl + } + go srv.ServeListener(l) + // Connect the client. + client, err := Dial(endpoint) + if err != nil { + panic(err) + } + return client, l +} + +// flakeyListener kills accepted connections after a random timeout. +type flakeyListener struct { + net.Listener + maxKillTimeout time.Duration + maxAcceptDelay time.Duration +} + +func (l *flakeyListener) Accept() (net.Conn, error) { + delay := time.Duration(rand.Int63n(int64(l.maxAcceptDelay))) + time.Sleep(delay) + + c, err := l.Listener.Accept() + if err == nil { + timeout := time.Duration(rand.Int63n(int64(l.maxKillTimeout))) + time.AfterFunc(timeout, func() { + glog.V(logger.Debug).Infof("killing conn %v after %v", c.LocalAddr(), timeout) + c.Close() + }) + } + return c, err +} diff --git a/rpc/errors.go b/rpc/errors.go index bc352fc45..9cf9dc60c 100644 --- a/rpc/errors.go +++ b/rpc/errors.go @@ -24,74 +24,43 @@ type methodNotFoundError struct { method string } -func (e *methodNotFoundError) Code() int { - return -32601 -} +func (e *methodNotFoundError) ErrorCode() int { return -32601 } func (e *methodNotFoundError) Error() string { return fmt.Sprintf("The method %s%s%s does not exist/is not available", e.service, serviceMethodSeparator, e.method) } // received message isn't a valid request -type invalidRequestError struct { - message string -} +type invalidRequestError struct{ message string } -func (e *invalidRequestError) Code() int { - return -32600 -} +func (e *invalidRequestError) ErrorCode() int { return -32600 } -func (e *invalidRequestError) Error() string { - return e.message -} +func (e *invalidRequestError) Error() string { return e.message } // received message is invalid -type invalidMessageError struct { - message string -} +type invalidMessageError struct{ message string } -func (e *invalidMessageError) Code() int { - return -32700 -} +func (e *invalidMessageError) ErrorCode() int { return -32700 } -func (e *invalidMessageError) Error() string { - return e.message -} +func (e *invalidMessageError) Error() string { return e.message } // unable to decode supplied params, or an invalid number of parameters -type invalidParamsError struct { - message string -} +type invalidParamsError struct{ message string } -func (e *invalidParamsError) Code() int { - return -32602 -} +func (e *invalidParamsError) ErrorCode() int { return -32602 } -func (e *invalidParamsError) Error() string { - return e.message -} +func (e *invalidParamsError) Error() string { return e.message } // logic error, callback returned an error -type callbackError struct { - message string -} +type callbackError struct{ message string } -func (e *callbackError) Code() int { - return -32000 -} +func (e *callbackError) ErrorCode() int { return -32000 } -func (e *callbackError) Error() string { - return e.message -} +func (e *callbackError) Error() string { return e.message } // issued when a request is received after the server is issued to stop. -type shutdownError struct { -} +type shutdownError struct{} -func (e *shutdownError) Code() int { - return -32000 -} +func (e *shutdownError) ErrorCode() int { return -32000 } -func (e *shutdownError) Error() string { - return "server is shutting down" -} +func (e *shutdownError) Error() string { return "server is shutting down" } diff --git a/rpc/http.go b/rpc/http.go index 9283ce0ec..afcdd4bd6 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -22,71 +22,108 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" - "net/url" "strings" + "sync" + "time" "github.com/rs/cors" + "golang.org/x/net/context" ) const ( maxHTTPRequestContentLength = 1024 * 128 ) -// httpClient connects to a geth RPC server over HTTP. -type httpClient struct { - endpoint *url.URL // HTTP-RPC server endpoint - httpClient http.Client // reuse connection - lastRes []byte // HTTP requests are synchronous, store last response +var nullAddr, _ = net.ResolveTCPAddr("tcp", "127.0.0.1:0") + +type httpConn struct { + client *http.Client + req *http.Request + closeOnce sync.Once + closed chan struct{} +} + +// httpConn is treated specially by Client. +func (hc *httpConn) LocalAddr() net.Addr { return nullAddr } +func (hc *httpConn) RemoteAddr() net.Addr { return nullAddr } +func (hc *httpConn) SetReadDeadline(time.Time) error { return nil } +func (hc *httpConn) SetWriteDeadline(time.Time) error { return nil } +func (hc *httpConn) SetDeadline(time.Time) error { return nil } +func (hc *httpConn) Write([]byte) (int, error) { panic("Write called") } + +func (hc *httpConn) Read(b []byte) (int, error) { + <-hc.closed + return 0, io.EOF +} + +func (hc *httpConn) Close() error { + hc.closeOnce.Do(func() { close(hc.closed) }) + return nil } -// NewHTTPClient create a new RPC clients that connection to a geth RPC server -// over HTTP. -func NewHTTPClient(endpoint string) (Client, error) { - url, err := url.Parse(endpoint) +// DialHTTP creates a new RPC clients that connection to an RPC server over HTTP. +func DialHTTP(endpoint string) (*Client, error) { + req, err := http.NewRequest("POST", endpoint, nil) if err != nil { return nil, err } - return &httpClient{endpoint: url}, nil -} + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") -// Send will serialize the given msg to JSON and sends it to the RPC server. -// Since HTTP is synchronous the response is stored until Recv is called. -func (client *httpClient) Send(msg interface{}) error { - var body []byte - var err error + initctx := context.Background() + return newClient(initctx, func(context.Context) (net.Conn, error) { + return &httpConn{client: new(http.Client), req: req, closed: make(chan struct{})}, nil + }) +} - client.lastRes = nil - if body, err = json.Marshal(msg); err != nil { +func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { + hc := c.writeConn.(*httpConn) + respBody, err := hc.doRequest(ctx, msg) + if err != nil { return err } + defer respBody.Close() + var respmsg jsonrpcMessage + if err := json.NewDecoder(respBody).Decode(&respmsg); err != nil { + return err + } + op.resp <- &respmsg + return nil +} - resp, err := client.httpClient.Post(client.endpoint.String(), "application/json", bytes.NewReader(body)) +func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonrpcMessage) error { + hc := c.writeConn.(*httpConn) + respBody, err := hc.doRequest(ctx, msgs) if err != nil { return err } - - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - client.lastRes, err = ioutil.ReadAll(resp.Body) + defer respBody.Close() + var respmsgs []jsonrpcMessage + if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } - - return fmt.Errorf("request failed: %s", resp.Status) -} - -// Recv will try to deserialize the last received response into the given msg. -func (client *httpClient) Recv(msg interface{}) error { - return json.Unmarshal(client.lastRes, &msg) + for _, respmsg := range respmsgs { + op.resp <- &respmsg + } + return nil } -// Close is not necessary for httpClient -func (client *httpClient) Close() { -} +func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadCloser, error) { + body, err := json.Marshal(msg) + if err != nil { + return nil, err + } + client, req := requestWithContext(hc.client, hc.req, ctx) + req.Body = ioutil.NopCloser(bytes.NewReader(body)) + req.ContentLength = int64(len(body)) -// SupportedModules will return the collection of offered RPC modules. -func (client *httpClient) SupportedModules() (map[string]string, error) { - return SupportedModules(client) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + return resp.Body, nil } // httpReadWriteNopCloser wraps a io.Reader and io.Writer with a NOP Close method. @@ -100,43 +137,39 @@ func (t *httpReadWriteNopCloser) Close() error { return nil } -// newJSONHTTPHandler creates a HTTP handler that will parse incoming JSON requests, -// send the request to the given API provider and sends the response back to the caller. -func newJSONHTTPHandler(srv *Server) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.ContentLength > maxHTTPRequestContentLength { - http.Error(w, - fmt.Sprintf("content length too large (%d>%d)", r.ContentLength, maxHTTPRequestContentLength), - http.StatusRequestEntityTooLarge) - return - } - - w.Header().Set("content-type", "application/json") - - // create a codec that reads direct from the request body until - // EOF and writes the response to w and order the server to process - // a single request. - codec := NewJSONCodec(&httpReadWriteNopCloser{r.Body, w}) - defer codec.Close() - srv.ServeSingleRequest(codec, OptionMethodInvocation) +// NewHTTPServer creates a new HTTP RPC server around an API provider. +// +// Deprecated: Server implements http.Handler +func NewHTTPServer(corsString string, srv *Server) *http.Server { + return &http.Server{Handler: newCorsHandler(srv, corsString)} +} + +// ServeHTTP serves JSON-RPC requests over HTTP. +func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.ContentLength > maxHTTPRequestContentLength { + http.Error(w, + fmt.Sprintf("content length too large (%d>%d)", r.ContentLength, maxHTTPRequestContentLength), + http.StatusRequestEntityTooLarge) + return } + w.Header().Set("content-type", "application/json") + + // create a codec that reads direct from the request body until + // EOF and writes the response to w and order the server to process + // a single request. + codec := NewJSONCodec(&httpReadWriteNopCloser{r.Body, w}) + defer codec.Close() + srv.ServeSingleRequest(codec, OptionMethodInvocation) } -// NewHTTPServer creates a new HTTP RPC server around an API provider. -func NewHTTPServer(corsString string, srv *Server) *http.Server { +func newCorsHandler(srv *Server, corsString string) http.Handler { var allowedOrigins []string for _, domain := range strings.Split(corsString, ",") { allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) } - c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: []string{"POST", "GET"}, }) - - handler := c.Handler(newJSONHTTPHandler(srv)) - - return &http.Server{ - Handler: handler, - } + return c.Handler(srv) } diff --git a/rpc/inproc.go b/rpc/inproc.go index 250f5c787..f72b97497 100644 --- a/rpc/inproc.go +++ b/rpc/inproc.go @@ -17,45 +17,18 @@ package rpc import ( - "encoding/json" - "io" "net" -) - -// inProcClient is an in-process buffer stream attached to an RPC server. -type inProcClient struct { - server *Server - cl io.Closer - enc *json.Encoder - dec *json.Decoder -} -// Close tears down the request channel of the in-proc client. -func (c *inProcClient) Close() { - c.cl.Close() -} - -// NewInProcRPCClient creates an in-process buffer stream attachment to a given -// RPC server. -func NewInProcRPCClient(handler *Server) Client { - p1, p2 := net.Pipe() - go handler.ServeCodec(NewJSONCodec(p1), OptionMethodInvocation|OptionSubscriptions) - return &inProcClient{handler, p2, json.NewEncoder(p2), json.NewDecoder(p2)} -} - -// Send marshals a message into a json format and injects in into the client -// request channel. -func (c *inProcClient) Send(msg interface{}) error { - return c.enc.Encode(msg) -} - -// Recv reads a message from the response channel and tries to parse it into the -// given msg interface. -func (c *inProcClient) Recv(msg interface{}) error { - return c.dec.Decode(msg) -} + "golang.org/x/net/context" +) -// Returns the collection of modules the RPC server offers. -func (c *inProcClient) SupportedModules() (map[string]string, error) { - return SupportedModules(c) +// NewInProcClient attaches an in-process connection to the given RPC server. +func DialInProc(handler *Server) *Client { + initctx := context.Background() + c, _ := newClient(initctx, func(context.Context) (net.Conn, error) { + p1, p2 := net.Pipe() + go handler.ServeCodec(NewJSONCodec(p1), OptionMethodInvocation|OptionSubscriptions) + return p2, nil + }) + return c } diff --git a/rpc/ipc.go b/rpc/ipc.go index 05d8909ca..c2b9e3871 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -17,68 +17,39 @@ package rpc import ( - "encoding/json" "net" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" ) -// CreateIPCListener creates an listener, on Unix platforms this is a unix socket, on Windows this is a named pipe +// CreateIPCListener creates an listener, on Unix platforms this is a unix socket, on +// Windows this is a named pipe func CreateIPCListener(endpoint string) (net.Listener, error) { return ipcListen(endpoint) } -// ipcClient represent an IPC RPC client. It will connect to a given endpoint and tries to communicate with a node using -// JSON serialization. -type ipcClient struct { - endpoint string - conn net.Conn - out *json.Encoder - in *json.Decoder -} - -// NewIPCClient create a new IPC client that will connect on the given endpoint. Messages are JSON encoded and encoded. -// On Unix it assumes the endpoint is the full path to a unix socket, and Windows the endpoint is an identifier for a -// named pipe. -func NewIPCClient(endpoint string) (Client, error) { - conn, err := newIPCConnection(endpoint) - if err != nil { - return nil, err - } - return &ipcClient{endpoint: endpoint, conn: conn, in: json.NewDecoder(conn), out: json.NewEncoder(conn)}, nil -} - -// Send will serialize the given message and send it to the server. -// When sending the message fails it will try to reconnect once and send the message again. -func (client *ipcClient) Send(msg interface{}) error { - if err := client.out.Encode(msg); err == nil { - return nil - } - - // retry once - client.conn.Close() - - conn, err := newIPCConnection(client.endpoint) - if err != nil { - return err +// ServeListener accepts connections on l, serving JSON-RPC on them. +func (srv *Server) ServeListener(l net.Listener) error { + for { + conn, err := l.Accept() + if err != nil { + return err + } + glog.V(logger.Detail).Infoln("accepted conn", conn.RemoteAddr()) + go srv.ServeCodec(NewJSONCodec(conn), OptionMethodInvocation|OptionSubscriptions) } - - client.conn = conn - client.in = json.NewDecoder(conn) - client.out = json.NewEncoder(conn) - - return client.out.Encode(msg) -} - -// Recv will read a message from the connection and tries to parse it. It assumes the received message is JSON encoded. -func (client *ipcClient) Recv(msg interface{}) error { - return client.in.Decode(&msg) -} - -// Close will close the underlying IPC connection -func (client *ipcClient) Close() { - client.conn.Close() } -// SupportedModules will return the collection of offered RPC modules. -func (client *ipcClient) SupportedModules() (map[string]string, error) { - return SupportedModules(client) +// DialIPC create a new IPC client that connects to the given endpoint. On Unix it assumes +// the endpoint is the full path to a unix socket, and Windows the endpoint is an +// identifier for a named pipe. +// +// The context is used for the initial connection establishment. It does not +// affect subsequent interactions with the client. +func DialIPC(ctx context.Context, endpoint string) (*Client, error) { + return newClient(ctx, func(ctx context.Context) (net.Conn, error) { + return newIPCConnection(ctx, endpoint) + }) } diff --git a/rpc/ipc_unix.go b/rpc/ipc_unix.go index 9ece01240..a25b21627 100644 --- a/rpc/ipc_unix.go +++ b/rpc/ipc_unix.go @@ -22,6 +22,8 @@ import ( "net" "os" "path/filepath" + + "golang.org/x/net/context" ) // ipcListen will create a Unix socket on the given endpoint. @@ -40,6 +42,6 @@ func ipcListen(endpoint string) (net.Listener, error) { } // newIPCConnection will connect to a Unix socket on the given endpoint. -func newIPCConnection(endpoint string) (net.Conn, error) { - return net.DialUnix("unix", nil, &net.UnixAddr{Name: endpoint, Net: "unix"}) +func newIPCConnection(ctx context.Context, endpoint string) (net.Conn, error) { + return dialContext(ctx, "unix", endpoint) } diff --git a/rpc/ipc_windows.go b/rpc/ipc_windows.go index 8342d04d5..68234d215 100644 --- a/rpc/ipc_windows.go +++ b/rpc/ipc_windows.go @@ -22,16 +22,27 @@ import ( "net" "time" + "golang.org/x/net/context" "gopkg.in/natefinch/npipe.v2" ) +// This is used if the dialing context has no deadline. It is much smaller than the +// defaultDialTimeout because named pipes are local and there is no need to wait so long. +const defaultPipeDialTimeout = 2 * time.Second + // ipcListen will create a named pipe on the given endpoint. func ipcListen(endpoint string) (net.Listener, error) { return npipe.Listen(endpoint) } // newIPCConnection will connect to a named pipe with the given endpoint as name. -func newIPCConnection(endpoint string) (net.Conn, error) { - timeout := 5 * time.Second +func newIPCConnection(ctx context.Context, endpoint string) (net.Conn, error) { + timeout := defaultPipeDialTimeout + if deadline, ok := ctx.Deadline(); ok { + timeout = deadline.Sub(time.Now()) + if timeout < 0 { + timeout = 0 + } + } return npipe.DialTimeout(endpoint, timeout) } diff --git a/rpc/json.go b/rpc/json.go index 151ed546e..a7053e3f5 100644 --- a/rpc/json.go +++ b/rpc/json.go @@ -30,49 +30,43 @@ import ( ) const ( - JSONRPCVersion = "2.0" + jsonrpcVersion = "2.0" serviceMethodSeparator = "_" subscribeMethod = "eth_subscribe" unsubscribeMethod = "eth_unsubscribe" notificationMethod = "eth_subscription" ) -// JSON-RPC request -type JSONRequest struct { +type jsonRequest struct { Method string `json:"method"` Version string `json:"jsonrpc"` Id json.RawMessage `json:"id,omitempty"` Payload json.RawMessage `json:"params,omitempty"` } -// JSON-RPC response -type JSONSuccessResponse struct { +type jsonSuccessResponse struct { Version string `json:"jsonrpc"` Id interface{} `json:"id,omitempty"` Result interface{} `json:"result"` } -// JSON-RPC error object -type JSONError struct { +type jsonError struct { Code int `json:"code"` Message string `json:"message"` Data interface{} `json:"data,omitempty"` } -// JSON-RPC error response -type JSONErrResponse struct { +type jsonErrResponse struct { Version string `json:"jsonrpc"` Id interface{} `json:"id,omitempty"` - Error JSONError `json:"error"` + Error jsonError `json:"error"` } -// JSON-RPC notification payload type jsonSubscription struct { Subscription string `json:"subscription"` Result interface{} `json:"result,omitempty"` } -// JSON-RPC notification type jsonNotification struct { Version string `json:"jsonrpc"` Method string `json:"method"` @@ -91,6 +85,17 @@ type jsonCodec struct { rw io.ReadWriteCloser // connection } +func (err *jsonError) Error() string { + if err.Message == "" { + return fmt.Sprintf("json-rpc error %d", err.Code) + } + return err.Message +} + +func (err *jsonError) ErrorCode() int { + return err.Code +} + // NewJSONCodec creates a new RPC server codec with support for JSON-RPC 2.0 func NewJSONCodec(rwc io.ReadWriteCloser) ServerCodec { d := json.NewDecoder(rwc) @@ -113,7 +118,7 @@ func isBatch(msg json.RawMessage) bool { // ReadRequestHeaders will read new requests without parsing the arguments. It will // return a collection of requests, an indication if these requests are in batch // form or an error when the incoming message could not be read/parsed. -func (c *jsonCodec) ReadRequestHeaders() ([]rpcRequest, bool, RPCError) { +func (c *jsonCodec) ReadRequestHeaders() ([]rpcRequest, bool, Error) { c.decMu.Lock() defer c.decMu.Unlock() @@ -148,8 +153,8 @@ func checkReqId(reqId json.RawMessage) error { // parseRequest will parse a single request from the given RawMessage. It will return // the parsed request, an indication if the request was a batch or an error when // the request could not be parsed. -func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCError) { - var in JSONRequest +func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) { + var in jsonRequest if err := json.Unmarshal(incomingMsg, &in); err != nil { return nil, false, &invalidMessageError{err.Error()} } @@ -182,12 +187,12 @@ func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCError) { method: unsubscribeMethod, params: in.Payload}}, false, nil } - // regular RPC call elems := strings.Split(in.Method, serviceMethodSeparator) if len(elems) != 2 { return nil, false, &methodNotFoundError{in.Method, ""} } + // regular RPC call if len(in.Payload) == 0 { return []rpcRequest{rpcRequest{service: elems[0], method: elems[1], id: &in.Id}}, false, nil } @@ -197,8 +202,8 @@ func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCError) { // parseBatchRequest will parse a batch request into a collection of requests from the given RawMessage, an indication // if the request was a batch or an error when the request could not be read. -func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCError) { - var in []JSONRequest +func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) { + var in []jsonRequest if err := json.Unmarshal(incomingMsg, &in); err != nil { return nil, false, &invalidMessageError{err.Error()} } @@ -236,15 +241,15 @@ func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCErro continue } - elems := strings.Split(r.Method, serviceMethodSeparator) - if len(elems) != 2 { - return nil, true, &methodNotFoundError{r.Method, ""} - } - if len(r.Payload) == 0 { - requests[i] = rpcRequest{service: elems[0], method: elems[1], id: id, params: nil} + requests[i] = rpcRequest{id: id, params: nil} + } else { + requests[i] = rpcRequest{id: id, params: r.Payload} + } + if elem := strings.Split(r.Method, serviceMethodSeparator); len(elem) == 2 { + requests[i].service, requests[i].method = elem[0], elem[1] } else { - requests[i] = rpcRequest{service: elems[0], method: elems[1], id: id, params: r.Payload} + requests[i].err = &methodNotFoundError{r.Method, ""} } } @@ -253,7 +258,7 @@ func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, RPCErro // ParseRequestArguments tries to parse the given params (json.RawMessage) with the given types. It returns the parsed // values or an error when the parsing failed. -func (c *jsonCodec) ParseRequestArguments(argTypes []reflect.Type, params interface{}) ([]reflect.Value, RPCError) { +func (c *jsonCodec) ParseRequestArguments(argTypes []reflect.Type, params interface{}) ([]reflect.Value, Error) { if args, ok := params.(json.RawMessage); !ok { return nil, &invalidParamsError{"Invalid params supplied"} } else { @@ -264,7 +269,7 @@ func (c *jsonCodec) ParseRequestArguments(argTypes []reflect.Type, params interf // parsePositionalArguments tries to parse the given args to an array of values with the given types. // It returns the parsed values or an error when the args could not be parsed. Missing optional arguments // are returned as reflect.Zero values. -func parsePositionalArguments(args json.RawMessage, callbackArgs []reflect.Type) ([]reflect.Value, RPCError) { +func parsePositionalArguments(args json.RawMessage, callbackArgs []reflect.Type) ([]reflect.Value, Error) { params := make([]interface{}, 0, len(callbackArgs)) for _, t := range callbackArgs { params = append(params, reflect.New(t).Interface()) @@ -302,31 +307,31 @@ func parsePositionalArguments(args json.RawMessage, callbackArgs []reflect.Type) // CreateResponse will create a JSON-RPC success response with the given id and reply as result. func (c *jsonCodec) CreateResponse(id interface{}, reply interface{}) interface{} { if isHexNum(reflect.TypeOf(reply)) { - return &JSONSuccessResponse{Version: JSONRPCVersion, Id: id, Result: fmt.Sprintf(`%#x`, reply)} + return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: fmt.Sprintf(`%#x`, reply)} } - return &JSONSuccessResponse{Version: JSONRPCVersion, Id: id, Result: reply} + return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: reply} } // CreateErrorResponse will create a JSON-RPC error response with the given id and error. -func (c *jsonCodec) CreateErrorResponse(id interface{}, err RPCError) interface{} { - return &JSONErrResponse{Version: JSONRPCVersion, Id: id, Error: JSONError{Code: err.Code(), Message: err.Error()}} +func (c *jsonCodec) CreateErrorResponse(id interface{}, err Error) interface{} { + return &jsonErrResponse{Version: jsonrpcVersion, Id: id, Error: jsonError{Code: err.ErrorCode(), Message: err.Error()}} } // CreateErrorResponseWithInfo will create a JSON-RPC error response with the given id and error. // info is optional and contains additional information about the error. When an empty string is passed it is ignored. -func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err RPCError, info interface{}) interface{} { - return &JSONErrResponse{Version: JSONRPCVersion, Id: id, - Error: JSONError{Code: err.Code(), Message: err.Error(), Data: info}} +func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err Error, info interface{}) interface{} { + return &jsonErrResponse{Version: jsonrpcVersion, Id: id, + Error: jsonError{Code: err.ErrorCode(), Message: err.Error(), Data: info}} } // CreateNotification will create a JSON-RPC notification with the given subscription id and event as params. func (c *jsonCodec) CreateNotification(subid string, event interface{}) interface{} { if isHexNum(reflect.TypeOf(event)) { - return &jsonNotification{Version: JSONRPCVersion, Method: notificationMethod, + return &jsonNotification{Version: jsonrpcVersion, Method: notificationMethod, Params: jsonSubscription{Subscription: subid, Result: fmt.Sprintf(`%#x`, event)}} } - return &jsonNotification{Version: JSONRPCVersion, Method: notificationMethod, + return &jsonNotification{Version: jsonrpcVersion, Method: notificationMethod, Params: jsonSubscription{Subscription: subid, Result: event}} } diff --git a/rpc/notification.go b/rpc/notification.go index e84e26a58..875433071 100644 --- a/rpc/notification.go +++ b/rpc/notification.go @@ -28,7 +28,7 @@ import ( var ( // ErrNotificationsUnsupported is returned when the connection doesn't support notifications - ErrNotificationsUnsupported = errors.New("notifications not supported") + ErrNotificationsUnsupported = errors.New("subscription notifications not supported by the current transport") // ErrNotificationNotFound is returned when the notification for the given id is not found ErrNotificationNotFound = errors.New("notification not found") diff --git a/rpc/notification_test.go b/rpc/notification_test.go index 1bcede177..280503222 100644 --- a/rpc/notification_test.go +++ b/rpc/notification_test.go @@ -19,20 +19,31 @@ package rpc import ( "encoding/json" "net" + "sync" "testing" "time" "golang.org/x/net/context" ) -type NotificationTestService struct{} +type NotificationTestService struct { + mu sync.Mutex + unsubscribed bool -var ( - unsubCallbackCalled = false -) + gotHangSubscriptionReq chan struct{} + unblockHangSubscription chan struct{} +} + +func (s *NotificationTestService) wasUnsubCallbackCalled() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.unsubscribed +} func (s *NotificationTestService) Unsubscribe(subid string) { - unsubCallbackCalled = true + s.mu.Lock() + s.unsubscribed = true + s.mu.Unlock() } func (s *NotificationTestService) SomeSubscription(ctx context.Context, n, val int) (Subscription, error) { @@ -60,6 +71,26 @@ func (s *NotificationTestService) SomeSubscription(ctx context.Context, n, val i return subscription, nil } +// HangSubscription blocks on s.unblockHangSubscription before +// sending anything. +func (s *NotificationTestService) HangSubscription(ctx context.Context, val int) (Subscription, error) { + notifier, supported := NotifierFromContext(ctx) + if !supported { + return nil, ErrNotificationsUnsupported + } + + s.gotHangSubscriptionReq <- struct{}{} + <-s.unblockHangSubscription + subscription, err := notifier.NewSubscription(s.Unsubscribe) + if err != nil { + return nil, err + } + go func() { + subscription.Notify(val) + }() + return subscription, nil +} + func TestNotifications(t *testing.T) { server := NewServer() service := &NotificationTestService{} @@ -90,7 +121,7 @@ func TestNotifications(t *testing.T) { } var subid string - response := JSONSuccessResponse{Result: subid} + response := jsonSuccessResponse{Result: subid} if err := in.Decode(&response); err != nil { t.Fatal(err) } @@ -114,7 +145,7 @@ func TestNotifications(t *testing.T) { clientConn.Close() // causes notification unsubscribe callback to be called time.Sleep(1 * time.Second) - if !unsubCallbackCalled { + if !service.wasUnsubCallbackCalled() { t.Error("unsubscribe callback not called after closing connection") } } diff --git a/rpc/server.go b/rpc/server.go index 7b7d22063..040805a5c 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -21,7 +21,6 @@ import ( "reflect" "runtime" "sync/atomic" - "time" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -30,8 +29,6 @@ import ( ) const ( - stopPendingRequestTimeout = 3 * time.Second // give pending requests stopPendingRequestTimeout the time to finish when the server is stopped - notificationBufferSize = 10000 // max buffered notifications before codec is closed MetadataApi = "rpc" @@ -183,7 +180,7 @@ func (s *Server) serveRequest(codec ServerCodec, singleShot bool, options CodecO for atomic.LoadInt32(&s.run) == 1 { reqs, batch, err := s.readRequest(codec) if err != nil { - glog.V(logger.Debug).Infof("%v\n", err) + glog.V(logger.Debug).Infof("read error %v\n", err) codec.Write(codec.CreateErrorResponse(nil, err)) return nil } @@ -240,13 +237,11 @@ func (s *Server) ServeSingleRequest(codec ServerCodec, options CodecOption) { func (s *Server) Stop() { if atomic.CompareAndSwapInt32(&s.run, 1, 0) { glog.V(logger.Debug).Infoln("RPC Server shutdown initiatied") - time.AfterFunc(stopPendingRequestTimeout, func() { - s.codecsMu.Lock() - defer s.codecsMu.Unlock() - s.codecs.Each(func(c interface{}) bool { - c.(ServerCodec).Close() - return true - }) + s.codecsMu.Lock() + defer s.codecsMu.Unlock() + s.codecs.Each(func(c interface{}) bool { + c.(ServerCodec).Close() + return true }) } } @@ -386,7 +381,7 @@ func (s *Server) execBatch(ctx context.Context, codec ServerCodec, requests []*s // readRequest requests the next (batch) request from the codec. It will return the collection // of requests, an indication if the request was a batch, the invalid request identifier and an // error when the request could not be read/parsed. -func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, RPCError) { +func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) { reqs, batch, err := codec.ReadRequestHeaders() if err != nil { return nil, batch, err @@ -399,6 +394,11 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, RPCErro var ok bool var svc *service + if r.err != nil { + requests[i] = &serverRequest{id: r.id, err: r.err} + continue + } + if r.isPubSub && r.method == unsubscribeMethod { requests[i] = &serverRequest{id: r.id, isUnsubscribe: true} argTypes := []reflect.Type{reflect.TypeOf("")} // expect subscription id as first arg diff --git a/rpc/server_test.go b/rpc/server_test.go index de47e1afd..e6840bde4 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -21,6 +21,7 @@ import ( "net" "reflect" "testing" + "time" "golang.org/x/net/context" ) @@ -48,6 +49,13 @@ func (s *Service) EchoWithCtx(ctx context.Context, str string, i int, args *Args return Result{str, i, args} } +func (s *Service) Sleep(ctx context.Context, duration time.Duration) { + select { + case <-time.After(duration): + case <-ctx.Done(): + } +} + func (s *Service) Rets() (string, error) { return "", nil } @@ -85,8 +93,8 @@ func TestServerRegisterName(t *testing.T) { t.Fatalf("Expected service calc to be registered") } - if len(svc.callbacks) != 4 { - t.Errorf("Expected 4 callbacks for service 'calc', got %d", len(svc.callbacks)) + if len(svc.callbacks) != 5 { + t.Errorf("Expected 5 callbacks for service 'calc', got %d", len(svc.callbacks)) } if len(svc.subscriptions) != 1 { @@ -126,7 +134,7 @@ func testServerMethodExecution(t *testing.T, method string) { t.Fatal(err) } - response := JSONSuccessResponse{Result: &Result{}} + response := jsonSuccessResponse{Result: &Result{}} if err := in.Decode(&response); err != nil { t.Fatal(err) } diff --git a/rpc/types.go b/rpc/types.go index a1f36fbd2..2a7268ad8 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -62,7 +62,7 @@ type serverRequest struct { callb *callback args []reflect.Value isUnsubscribe bool - err RPCError + err Error } type serviceRegistry map[string]*service // collection of services @@ -88,14 +88,13 @@ type rpcRequest struct { id interface{} isPubSub bool params interface{} + err Error // invalid batch element } -// RPCError implements RPC error, is add support for error codec over regular go errors -type RPCError interface { - // RPC error code - Code() int - // Error message - Error() string +// Error wraps RPC errors, which contain an error code in addition to the message. +type Error interface { + Error() string // returns the message + ErrorCode() int // returns the code } // ServerCodec implements reading, parsing and writing RPC messages for the server side of @@ -103,15 +102,15 @@ type RPCError interface { // multiple go-routines concurrently. type ServerCodec interface { // Read next request - ReadRequestHeaders() ([]rpcRequest, bool, RPCError) + ReadRequestHeaders() ([]rpcRequest, bool, Error) // Parse request argument to the given types - ParseRequestArguments([]reflect.Type, interface{}) ([]reflect.Value, RPCError) + ParseRequestArguments([]reflect.Type, interface{}) ([]reflect.Value, Error) // Assemble success response, expects response id and payload CreateResponse(interface{}, interface{}) interface{} // Assemble error response, expects response id and error - CreateErrorResponse(interface{}, RPCError) interface{} + CreateErrorResponse(interface{}, Error) interface{} // Assemble error response with extra information about the error through info - CreateErrorResponseWithInfo(id interface{}, err RPCError, info interface{}) interface{} + CreateErrorResponseWithInfo(id interface{}, err Error, info interface{}) interface{} // Create notification response CreateNotification(string, interface{}) interface{} // Write msg to client. @@ -273,14 +272,3 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { func (bn *BlockNumber) Int64() int64 { return (int64)(*bn) } - -// Client defines the interface for go client that wants to connect to a geth RPC endpoint -type Client interface { - // SupportedModules returns the collection of API's the server offers - SupportedModules() (map[string]string, error) - - Send(req interface{}) error - Recv(msg interface{}) error - - Close() -} diff --git a/rpc/utils.go b/rpc/utils.go index fe482e19d..1ac6698f5 100644 --- a/rpc/utils.go +++ b/rpc/utils.go @@ -20,7 +20,6 @@ import ( "crypto/rand" "encoding/hex" "errors" - "fmt" "math/big" "reflect" "unicode" @@ -227,31 +226,3 @@ func newSubscriptionID() (string, error) { } return "0x" + hex.EncodeToString(subid[:]), nil } - -// SupportedModules returns the collection of API's that the RPC server offers -// on which the given client connects. -func SupportedModules(client Client) (map[string]string, error) { - req := JSONRequest{ - Id: []byte("1"), - Version: "2.0", - Method: MetadataApi + "_modules", - } - if err := client.Send(req); err != nil { - return nil, err - } - - var response JSONSuccessResponse - if err := client.Recv(&response); err != nil { - return nil, err - } - if response.Result != nil { - mods := make(map[string]string) - if modules, ok := response.Result.(map[string]interface{}); ok { - for m, v := range modules { - mods[m] = fmt.Sprintf("%s", v) - } - return mods, nil - } - } - return nil, fmt.Errorf("unable to retrieve modules") -} diff --git a/rpc/websocket.go b/rpc/websocket.go index fe9354d94..fc3cd0709 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -17,36 +17,39 @@ package rpc import ( + "crypto/tls" "fmt" + "net" "net/http" + "net/url" "os" "strings" - "sync" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" "golang.org/x/net/websocket" "gopkg.in/fatih/set.v0" ) -// wsReaderWriterCloser reads and write payloads from and to a websocket connection. -type wsReaderWriterCloser struct { - c *websocket.Conn -} - -// Read will read incoming payload data into p. -func (rw *wsReaderWriterCloser) Read(p []byte) (int, error) { - return rw.c.Read(p) -} - -// Write writes p to the websocket. -func (rw *wsReaderWriterCloser) Write(p []byte) (int, error) { - return rw.c.Write(p) +// WebsocketHandler returns a handler that serves JSON-RPC to WebSocket connections. +// +// allowedOrigins should be a comma-separated list of allowed origin URLs. +// To allow connections with any origin, pass "*". +func (srv *Server) WebsocketHandler(allowedOrigins string) http.Handler { + return websocket.Server{ + Handshake: wsHandshakeValidator(strings.Split(allowedOrigins, ",")), + Handler: func(conn *websocket.Conn) { + srv.ServeCodec(NewJSONCodec(conn), OptionMethodInvocation|OptionSubscriptions) + }, + } } -// Close closes the websocket connection. -func (rw *wsReaderWriterCloser) Close() error { - return rw.c.Close() +// NewWSServer creates a new websocket RPC server around an API provider. +// +// Deprecated: use Server.WebsocketHandler +func NewWSServer(allowedOrigins string, srv *Server) *http.Server { + return &http.Server{Handler: srv.WebsocketHandler(allowedOrigins)} } // wsHandshakeValidator returns a handler that verifies the origin during the @@ -87,96 +90,63 @@ func wsHandshakeValidator(allowedOrigins []string) func(*websocket.Config, *http return f } -// NewWSServer creates a new websocket RPC server around an API provider. -func NewWSServer(allowedOrigins string, handler *Server) *http.Server { - return &http.Server{ - Handler: websocket.Server{ - Handshake: wsHandshakeValidator(strings.Split(allowedOrigins, ",")), - Handler: func(conn *websocket.Conn) { - handler.ServeCodec(NewJSONCodec(&wsReaderWriterCloser{conn}), - OptionMethodInvocation|OptionSubscriptions) - }, - }, +// DialWebsocket creates a new RPC client that communicates with a JSON-RPC server +// that is listening on the given endpoint. +// +// The context is used for the initial connection establishment. It does not +// affect subsequent interactions with the client. +func DialWebsocket(ctx context.Context, endpoint, origin string) (*Client, error) { + if origin == "" { + var err error + if origin, err = os.Hostname(); err != nil { + return nil, err + } + if strings.HasPrefix(endpoint, "wss") { + origin = "https://" + strings.ToLower(origin) + } else { + origin = "http://" + strings.ToLower(origin) + } + } + config, err := websocket.NewConfig(endpoint, origin) + if err != nil { + return nil, err } -} - -// wsClient represents a RPC client that communicates over websockets with a -// RPC server. -type wsClient struct { - endpoint string - connMu sync.Mutex - conn *websocket.Conn -} -// NewWSClientj creates a new RPC client that communicates with a RPC server -// that is listening on the given endpoint using JSON encoding. -func NewWSClient(endpoint string) (Client, error) { - return &wsClient{endpoint: endpoint}, nil + return newClient(ctx, func(ctx context.Context) (net.Conn, error) { + return wsDialContext(ctx, config) + }) } -// connection will return a websocket connection to the RPC server. It will -// (re)connect when necessary. -func (client *wsClient) connection() (*websocket.Conn, error) { - if client.conn != nil { - return client.conn, nil +func wsDialContext(ctx context.Context, config *websocket.Config) (*websocket.Conn, error) { + var conn net.Conn + var err error + switch config.Location.Scheme { + case "ws": + conn, err = dialContext(ctx, "tcp", wsDialAddress(config.Location)) + case "wss": + dialer := contextDialer(ctx) + conn, err = tls.DialWithDialer(dialer, "tcp", wsDialAddress(config.Location), config.TlsConfig) + default: + err = websocket.ErrBadScheme } - - origin, err := os.Hostname() if err != nil { return nil, err } - - origin = "http://" + origin - client.conn, err = websocket.Dial(client.endpoint, "", origin) - - return client.conn, err -} - -// SupportedModules is the collection of modules the RPC server offers. -func (client *wsClient) SupportedModules() (map[string]string, error) { - return SupportedModules(client) -} - -// Send writes the JSON serialized msg to the websocket. It will create a new -// websocket connection to the server if the client is currently not connected. -func (client *wsClient) Send(msg interface{}) (err error) { - client.connMu.Lock() - defer client.connMu.Unlock() - - var conn *websocket.Conn - if conn, err = client.connection(); err == nil { - if err = websocket.JSON.Send(conn, msg); err != nil { - client.conn.Close() - client.conn = nil - } + ws, err := websocket.NewClient(config, conn) + if err != nil { + conn.Close() + return nil, err } - - return err + return ws, err } -// Recv reads a JSON message from the websocket and unmarshals it into msg. -func (client *wsClient) Recv(msg interface{}) (err error) { - client.connMu.Lock() - defer client.connMu.Unlock() +var wsPortMap = map[string]string{"ws": "80", "wss": "443"} - var conn *websocket.Conn - if conn, err = client.connection(); err == nil { - if err = websocket.JSON.Receive(conn, msg); err != nil { - client.conn.Close() - client.conn = nil +func wsDialAddress(location *url.URL) string { + if _, ok := wsPortMap[location.Scheme]; ok { + if _, _, err := net.SplitHostPort(location.Host); err != nil { + return net.JoinHostPort(location.Host, wsPortMap[location.Scheme]) } } - return -} - -// Close closes the underlaying websocket connection. -func (client *wsClient) Close() { - client.connMu.Lock() - defer client.connMu.Unlock() - - if client.conn != nil { - client.conn.Close() - client.conn = nil - } - + return location.Host } |