cmd, core, eth/tracers: support fancier js tracing (#15516)

* cmd, core, eth/tracers: support fancier js tracing

* eth, internal/web3ext: rework trace API, concurrency, chain tracing

* eth/tracers: add three more JavaScript tracers

* eth/tracers, vendor: swap ottovm to duktape for tracing

* core, eth, internal: finalize call tracer and needed extras

* eth, tests: prestate tracer, call test suite, rewinding

* vendor: fix windows builds for tracer js engine

* vendor: temporary duktape fix

* eth/tracers: fix up 4byte and evmdis tracer

* vendor: pull in latest duktape with my upstream fixes

* eth: fix some review comments

* eth: rename rewind to reexec to make it more obvious

* core/vm: terminate tracing using defers

3 files changed, 24 insertions, 531 deletions

diff --git a/internal/ethapi/tracer.go b/internal/ethapi/tracer.go
deleted file mode 100644
index 71cafc6e9..000000000
--- a/internal/ethapi/tracer.go
+++ /dev/null
@@ -1,364 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package ethapi
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"math/big"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/robertkrimen/otto"
-)
-
-// fakeBig is used to provide an interface to Javascript for 'big.NewInt'
-type fakeBig struct{}
-
-// NewInt creates a new big.Int with the specified int64 value.
-func (fb *fakeBig) NewInt(x int64) *big.Int {
-	return big.NewInt(x)
-}
-
-// OpCodeWrapper provides a JavaScript-friendly wrapper around OpCode, to convince Otto to treat it
-// as an object, instead of a number.
-type opCodeWrapper struct {
-	op vm.OpCode
-}
-
-// toNumber returns the ID of this opcode as an integer
-func (ocw *opCodeWrapper) toNumber() int {
-	return int(ocw.op)
-}
-
-// toString returns the string representation of the opcode
-func (ocw *opCodeWrapper) toString() string {
-	return ocw.op.String()
-}
-
-// isPush returns true if the op is a Push
-func (ocw *opCodeWrapper) isPush() bool {
-	return ocw.op.IsPush()
-}
-
-// MarshalJSON serializes the opcode as JSON
-func (ocw *opCodeWrapper) MarshalJSON() ([]byte, error) {
-	return json.Marshal(ocw.op.String())
-}
-
-// toValue returns an otto.Value for the opCodeWrapper
-func (ocw *opCodeWrapper) toValue(vm *otto.Otto) otto.Value {
-	value, _ := vm.ToValue(ocw)
-	obj := value.Object()
-	obj.Set("toNumber", ocw.toNumber)
-	obj.Set("toString", ocw.toString)
-	obj.Set("isPush", ocw.isPush)
-	return value
-}
-
-// memoryWrapper provides a JS wrapper around vm.Memory
-type memoryWrapper struct {
-	memory *vm.Memory
-}
-
-// slice returns the requested range of memory as a byte slice
-func (mw *memoryWrapper) slice(begin, end int64) []byte {
-	return mw.memory.Get(begin, end-begin)
-}
-
-// getUint returns the 32 bytes at the specified address interpreted
-// as an unsigned integer
-func (mw *memoryWrapper) getUint(addr int64) *big.Int {
-	ret := big.NewInt(0)
-	ret.SetBytes(mw.memory.GetPtr(addr, 32))
-	return ret
-}
-
-// toValue returns an otto.Value for the memoryWrapper
-func (mw *memoryWrapper) toValue(vm *otto.Otto) otto.Value {
-	value, _ := vm.ToValue(mw)
-	obj := value.Object()
-	obj.Set("slice", mw.slice)
-	obj.Set("getUint", mw.getUint)
-	return value
-}
-
-// stackWrapper provides a JS wrapper around vm.Stack
-type stackWrapper struct {
-	stack *vm.Stack
-}
-
-// peek returns the nth-from-the-top element of the stack.
-func (sw *stackWrapper) peek(idx int) *big.Int {
-	return sw.stack.Data()[len(sw.stack.Data())-idx-1]
-}
-
-// length returns the length of the stack
-func (sw *stackWrapper) length() int {
-	return len(sw.stack.Data())
-}
-
-// toValue returns an otto.Value for the stackWrapper
-func (sw *stackWrapper) toValue(vm *otto.Otto) otto.Value {
-	value, _ := vm.ToValue(sw)
-	obj := value.Object()
-	obj.Set("peek", sw.peek)
-	obj.Set("length", sw.length)
-	return value
-}
-
-// dbWrapper provides a JS wrapper around vm.Database
-type dbWrapper struct {
-	db vm.StateDB
-}
-
-// getBalance retrieves an account's balance
-func (dw *dbWrapper) getBalance(addr []byte) *big.Int {
-	return dw.db.GetBalance(common.BytesToAddress(addr))
-}
-
-// getNonce retrieves an account's nonce
-func (dw *dbWrapper) getNonce(addr []byte) uint64 {
-	return dw.db.GetNonce(common.BytesToAddress(addr))
-}
-
-// getCode retrieves an account's code
-func (dw *dbWrapper) getCode(addr []byte) []byte {
-	return dw.db.GetCode(common.BytesToAddress(addr))
-}
-
-// getState retrieves an account's state data for the given hash
-func (dw *dbWrapper) getState(addr []byte, hash common.Hash) common.Hash {
-	return dw.db.GetState(common.BytesToAddress(addr), hash)
-}
-
-// exists returns true iff the account exists
-func (dw *dbWrapper) exists(addr []byte) bool {
-	return dw.db.Exist(common.BytesToAddress(addr))
-}
-
-// toValue returns an otto.Value for the dbWrapper
-func (dw *dbWrapper) toValue(vm *otto.Otto) otto.Value {
-	value, _ := vm.ToValue(dw)
-	obj := value.Object()
-	obj.Set("getBalance", dw.getBalance)
-	obj.Set("getNonce", dw.getNonce)
-	obj.Set("getCode", dw.getCode)
-	obj.Set("getState", dw.getState)
-	obj.Set("exists", dw.exists)
-	return value
-}
-
-// contractWrapper provides a JS wrapper around vm.Contract
-type contractWrapper struct {
-	contract *vm.Contract
-}
-
-func (c *contractWrapper) caller() common.Address {
-	return c.contract.Caller()
-}
-
-func (c *contractWrapper) address() common.Address {
-	return c.contract.Address()
-}
-
-func (c *contractWrapper) value() *big.Int {
-	return c.contract.Value()
-}
-
-func (c *contractWrapper) calldata() []byte {
-	return c.contract.Input
-}
-
-func (c *contractWrapper) toValue(vm *otto.Otto) otto.Value {
-	value, _ := vm.ToValue(c)
-	obj := value.Object()
-	obj.Set("caller", c.caller)
-	obj.Set("address", c.address)
-	obj.Set("value", c.value)
-	obj.Set("calldata", c.calldata)
-	return value
-}
-
-// JavascriptTracer provides an implementation of Tracer that evaluates a
-// Javascript function for each VM execution step.
-type JavascriptTracer struct {
-	vm       *otto.Otto             // Javascript VM instance
-	traceobj *otto.Object           // User-supplied object to call
-	op       *opCodeWrapper         // Wrapper around the VM opcode
-	log      map[string]interface{} // (Reusable) map for the `log` arg to `step`
-	logvalue otto.Value             // JS view of `log`
-	memory   *memoryWrapper         // Wrapper around the VM memory
-	stack    *stackWrapper          // Wrapper around the VM stack
-	db       *dbWrapper             // Wrapper around the VM environment
-	dbvalue  otto.Value             // JS view of `db`
-	contract *contractWrapper       // Wrapper around the contract object
-	err      error                  // Error, if one has occurred
-	result   interface{}            // Final result to return to the user
-}
-
-// NewJavascriptTracer instantiates a new JavascriptTracer instance.
-// code specifies a Javascript snippet, which must evaluate to an expression
-// returning an object with 'step' and 'result' functions.
-func NewJavascriptTracer(code string) (*JavascriptTracer, error) {
-	vm := otto.New()
-	vm.Interrupt = make(chan func(), 1)
-
-	// Set up builtins for this environment
-	vm.Set("big", &fakeBig{})
-	vm.Set("toHex", hexutil.Encode)
-
-	jstracer, err := vm.Object("(" + code + ")")
-	if err != nil {
-		return nil, err
-	}
-	// Check the required functions exist
-	step, err := jstracer.Get("step")
-	if err != nil {
-		return nil, err
-	}
-	if !step.IsFunction() {
-		return nil, fmt.Errorf("Trace object must expose a function step()")
-	}
-
-	result, err := jstracer.Get("result")
-	if err != nil {
-		return nil, err
-	}
-	if !result.IsFunction() {
-		return nil, fmt.Errorf("Trace object must expose a function result()")
-	}
-	// Create the persistent log object
-	var (
-		op       = new(opCodeWrapper)
-		mem      = new(memoryWrapper)
-		stack    = new(stackWrapper)
-		db       = new(dbWrapper)
-		contract = new(contractWrapper)
-	)
-	log := map[string]interface{}{
-		"op":       op.toValue(vm),
-		"memory":   mem.toValue(vm),
-		"stack":    stack.toValue(vm),
-		"contract": contract.toValue(vm),
-	}
-	logvalue, _ := vm.ToValue(log)
-
-	return &JavascriptTracer{
-		vm:       vm,
-		traceobj: jstracer,
-		op:       op,
-		log:      log,
-		logvalue: logvalue,
-		memory:   mem,
-		stack:    stack,
-		db:       db,
-		dbvalue:  db.toValue(vm),
-		contract: contract,
-		err:      nil,
-	}, nil
-}
-
-// Stop terminates execution of any JavaScript
-func (jst *JavascriptTracer) Stop(err error) {
-	jst.vm.Interrupt <- func() {
-		panic(err)
-	}
-}
-
-// callSafely executes a method on a JS object, catching any panics and
-// returning them as error objects.
-func (jst *JavascriptTracer) callSafely(method string, argumentList ...interface{}) (ret interface{}, err error) {
-	defer func() {
-		if caught := recover(); caught != nil {
-			switch caught := caught.(type) {
-			case error:
-				err = caught
-			case string:
-				err = errors.New(caught)
-			case fmt.Stringer:
-				err = errors.New(caught.String())
-			default:
-				panic(caught)
-			}
-		}
-	}()
-
-	value, err := jst.traceobj.Call(method, argumentList...)
-	ret, _ = value.Export()
-	return ret, err
-}
-
-func wrapError(context string, err error) error {
-	var message string
-	switch err := err.(type) {
-	case *otto.Error:
-		message = err.String()
-	default:
-		message = err.Error()
-	}
-	return fmt.Errorf("%v    in server-side tracer function '%v'", message, context)
-}
-
-// CaptureState implements the Tracer interface to trace a single step of VM execution
-func (jst *JavascriptTracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error {
-	if jst.err == nil {
-		jst.op.op = op
-		jst.memory.memory = memory
-		jst.stack.stack = stack
-		jst.db.db = env.StateDB
-		jst.contract.contract = contract
-
-		jst.log["pc"] = pc
-		jst.log["gas"] = gas
-		jst.log["cost"] = cost
-		jst.log["depth"] = depth
-		jst.log["account"] = contract.Address()
-
-		delete(jst.log, "error")
-		if err != nil {
-			jst.log["error"] = err
-		}
-		_, err := jst.callSafely("step", jst.logvalue, jst.dbvalue)
-		if err != nil {
-			jst.err = wrapError("step", err)
-		}
-	}
-	return nil
-}
-
-// CaptureEnd is called after the call finishes
-func (jst *JavascriptTracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error {
-	//TODO! @Arachnid please figure out of there's anything we can use this method for
-	return nil
-}
-
-// GetResult calls the Javascript 'result' function and returns its value, or any accumulated error
-func (jst *JavascriptTracer) GetResult() (result interface{}, err error) {
-	if jst.err != nil {
-		return nil, jst.err
-	}
-
-	result, err = jst.callSafely("result")
-	if err != nil {
-		err = wrapError("result", err)
-	}
-	return
-}
diff --git a/internal/ethapi/tracer_test.go b/internal/ethapi/tracer_test.go
deleted file mode 100644
index 0ef450ce3..000000000
--- a/internal/ethapi/tracer_test.go
+++ /dev/null
@@ -1,147 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package ethapi
-
-import (
-	"errors"
-	"math/big"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-type account struct{}
-
-func (account) SubBalance(amount *big.Int)                          {}
-func (account) AddBalance(amount *big.Int)                          {}
-func (account) SetAddress(common.Address)                           {}
-func (account) Value() *big.Int                                     { return nil }
-func (account) SetBalance(*big.Int)                                 {}
-func (account) SetNonce(uint64)                                     {}
-func (account) Balance() *big.Int                                   { return nil }
-func (account) Address() common.Address                             { return common.Address{} }
-func (account) ReturnGas(*big.Int)                                  {}
-func (account) SetCode(common.Hash, []byte)                         {}
-func (account) ForEachStorage(cb func(key, value common.Hash) bool) {}
-
-func runTrace(tracer *JavascriptTracer) (interface{}, error) {
-	env := vm.NewEVM(vm.Context{}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
-
-	contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000)
-	contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0}
-
-	_, err := env.Interpreter().Run(contract, []byte{})
-	if err != nil {
-		return nil, err
-	}
-
-	return tracer.GetResult()
-}
-
-func TestTracing(t *testing.T) {
-	tracer, err := NewJavascriptTracer("{count: 0, step: function() { this.count += 1; }, result: function() { return this.count; }}")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ret, err := runTrace(tracer)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	value, ok := ret.(float64)
-	if !ok {
-		t.Errorf("Expected return value to be float64, was %T", ret)
-	}
-	if value != 3 {
-		t.Errorf("Expected return value to be 3, got %v", value)
-	}
-}
-
-func TestStack(t *testing.T) {
-	tracer, err := NewJavascriptTracer("{depths: [], step: function(log) { this.depths.push(log.stack.length()); }, result: function() { return this.depths; }}")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ret, err := runTrace(tracer)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	expected := []int{0, 1, 2}
-	if !reflect.DeepEqual(ret, expected) {
-		t.Errorf("Expected return value to be %#v, got %#v", expected, ret)
-	}
-}
-
-func TestOpcodes(t *testing.T) {
-	tracer, err := NewJavascriptTracer("{opcodes: [], step: function(log) { this.opcodes.push(log.op.toString()); }, result: function() { return this.opcodes; }}")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ret, err := runTrace(tracer)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	expected := []string{"PUSH1", "PUSH1", "STOP"}
-	if !reflect.DeepEqual(ret, expected) {
-		t.Errorf("Expected return value to be %#v, got %#v", expected, ret)
-	}
-}
-
-func TestHalt(t *testing.T) {
-	timeout := errors.New("stahp")
-	tracer, err := NewJavascriptTracer("{step: function() { while(1); }, result: function() { return null; }}")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	go func() {
-		time.Sleep(1 * time.Second)
-		tracer.Stop(timeout)
-	}()
-
-	if _, err = runTrace(tracer); err.Error() != "stahp    in server-side tracer function 'step'" {
-		t.Errorf("Expected timeout error, got %v", err)
-	}
-}
-
-func TestHaltBetweenSteps(t *testing.T) {
-	tracer, err := NewJavascriptTracer("{step: function() {}, result: function() { return null; }}")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	env := vm.NewEVM(vm.Context{}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
-	contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0)
-
-	tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil)
-	timeout := errors.New("stahp")
-	tracer.Stop(timeout)
-	tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil)
-
-	if _, err := tracer.GetResult(); err.Error() != "stahp    in server-side tracer function 'step'" {
-		t.Errorf("Expected timeout error, got %v", err)
-	}
-}
diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go
index ef0d2b4e6..e11aa402f 100644
--- a/internal/web3ext/web3ext.go
+++ b/internal/web3ext/web3ext.go
@@ -197,26 +197,6 @@ web3._extend({
 			params: 1
 		}),
 		new web3._extend.Method({
-			name: 'traceBlock',
-			call: 'debug_traceBlock',
-			params: 1
-		}),
-		new web3._extend.Method({
-			name: 'traceBlockFromFile',
-			call: 'debug_traceBlockFromFile',
-			params: 1
-		}),
-		new web3._extend.Method({
-			name: 'traceBlockByNumber',
-			call: 'debug_traceBlockByNumber',
-			params: 1
-		}),
-		new web3._extend.Method({
-			name: 'traceBlockByHash',
-			call: 'debug_traceBlockByHash',
-			params: 1
-		}),
-		new web3._extend.Method({
 			name: 'seedHash',
 			call: 'debug_seedHash',
 			params: 1
@@ -333,6 +313,30 @@ web3._extend({
 			params: 1
 		}),
 		new web3._extend.Method({
+			name: 'traceBlock',
+			call: 'debug_traceBlock',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'traceBlockFromFile',
+			call: 'debug_traceBlockFromFile',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'traceBlockByNumber',
+			call: 'debug_traceBlockByNumber',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
+			name: 'traceBlockByHash',
+			call: 'debug_traceBlockByHash',
+			params: 2,
+			inputFormatter: [null, null]
+		}),
+		new web3._extend.Method({
 			name: 'traceTransaction',
 			call: 'debug_traceTransaction',
 			params: 2,