From 5da82077d1bd556a562568fe25c55996d0cdfb1c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Jul 2015 15:13:24 +0200 Subject: cmd/ethtest, tests: add support for RLP JSON tests --- cmd/ethtest/main.go | 4 +- tests/init.go | 1 + tests/rlp_test.go | 20 +++++++ tests/rlp_test_util.go | 156 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 tests/rlp_test.go create mode 100644 tests/rlp_test_util.go diff --git a/cmd/ethtest/main.go b/cmd/ethtest/main.go index 61276b177..0d6286407 100644 --- a/cmd/ethtest/main.go +++ b/cmd/ethtest/main.go @@ -35,7 +35,7 @@ var ( testExtension = ".json" defaultTest = "all" defaultDir = "." - allTests = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests"} + allTests = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"} skipTests = []string{} TestFlag = cli.StringFlag{ @@ -75,6 +75,8 @@ func runTestWithReader(test string, r io.Reader) error { err = tests.RunTransactionTestsWithReader(r, skipTests) case "vm", "vmtest", "vmtests": err = tests.RunVmTestWithReader(r, skipTests) + case "rlp", "rlptest", "rlptests": + err = tests.RunRLPTestWithReader(r, skipTests) default: err = fmt.Errorf("Invalid test type specified: %v", test) } diff --git a/tests/init.go b/tests/init.go index f149c11f1..30cff6f21 100644 --- a/tests/init.go +++ b/tests/init.go @@ -35,6 +35,7 @@ var ( stateTestDir = filepath.Join(baseDir, "StateTests") transactionTestDir = filepath.Join(baseDir, "TransactionTests") vmTestDir = filepath.Join(baseDir, "VMTests") + rlpTestDir = filepath.Join(baseDir, "RLPTests") BlockSkipTests = []string{ // These tests are not valid, as they are out of scope for RLP and diff --git a/tests/rlp_test.go b/tests/rlp_test.go new file mode 100644 index 000000000..70bd19627 --- /dev/null +++ b/tests/rlp_test.go @@ -0,0 +1,20 @@ +package tests + +import ( + "path/filepath" + "testing" +) + +func TestRLP(t *testing.T) { + err := RunRLPTest(filepath.Join(rlpTestDir, "rlptest.json"), nil) + if err != nil { + t.Fatal(err) + } +} + +func TestRLP_invalid(t *testing.T) { + err := RunRLPTest(filepath.Join(rlpTestDir, "invalidRLPTest.json"), nil) + if err != nil { + t.Fatal(err) + } +} diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go new file mode 100644 index 000000000..d7042eef7 --- /dev/null +++ b/tests/rlp_test_util.go @@ -0,0 +1,156 @@ +package tests + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/rlp" +) + +type RLPTest struct { + In interface{} + Out string +} + +func RunRLPTest(file string, skip []string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + return RunRLPTestWithReader(f, skip) +} + +func RunRLPTestWithReader(r io.Reader, skip []string) error { + var tests map[string]*RLPTest + if err := readJson(r, &tests); err != nil { + return err + } + for _, s := range skip { + delete(tests, s) + } + for name, test := range tests { + if err := test.Run(); err != nil { + return fmt.Errorf("test %q failed: %v", name, err) + } + } + return nil +} + +// Run executes the test. +func (t *RLPTest) Run() error { + outb, err := hex.DecodeString(t.Out) + if err != nil { + return fmt.Errorf("invalid hex in Out") + } + if t.In == "VALID" || t.In == "INVALID" { + return checkDecodeInterface(outb, t.In == "VALID") + } + + // Check whether encoding the value produces the same bytes. + in := translateJSON(t.In) + b, err := rlp.EncodeToBytes(in) + if err != nil { + return fmt.Errorf("encode failed: %v", err) + } + if !bytes.Equal(b, outb) { + return fmt.Errorf("encode produced %x, want %x", b, outb) + } + // Test decoding from a stream. + s := rlp.NewStream(bytes.NewReader(outb), 0) + return checkDecodeFromJSON(s, in) +} + +func checkDecodeInterface(b []byte, isValid bool) error { + err := rlp.DecodeBytes(b, new(interface{})) + switch { + case isValid && err != nil: + return fmt.Errorf("decoding failed: %v", err) + case !isValid && err == nil: + return fmt.Errorf("decoding of invalid value succeeded") + } + return nil +} + +// translateJSON makes test json values encodable with RLP. +func translateJSON(v interface{}) interface{} { + switch v := v.(type) { + case float64: + return uint64(v) + case string: + if len(v) > 0 && v[0] == '#' { // # starts a faux big int. + big, ok := new(big.Int).SetString(v[1:], 10) + if !ok { + panic(fmt.Errorf("bad test: bad big int: %q", v)) + } + return big + } + return []byte(v) + case []interface{}: + new := make([]interface{}, len(v)) + for i := range v { + new[i] = translateJSON(v[i]) + } + return new + default: + panic(fmt.Errorf("can't handle %T", v)) + } +} + +// checkDecodeFromJSON decodes from s guided by exp. For each JSON +// value, the value decoded from the RLP stream must match. +func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error { + switch exp := exp.(type) { + case uint64: + i, err := s.Uint() + if err != nil { + return addStack("Uint", exp, err) + } + if i != exp { + return addStack("Uint", exp, fmt.Errorf("result mismatch: got %d", i)) + } + case *big.Int: + big := new(big.Int) + if err := s.Decode(&big); err != nil { + return addStack("Big", exp, err) + } + if big.Cmp(exp) != 0 { + return addStack("Big", exp, fmt.Errorf("result mismatch: got %d", big)) + } + case []byte: + b, err := s.Bytes() + if err != nil { + return addStack("Bytes", exp, err) + } + if !bytes.Equal(b, exp) { + return addStack("Bytes", exp, fmt.Errorf("result mismatch: got %x", b)) + } + case []interface{}: + if _, err := s.List(); err != nil { + return addStack("List", exp, err) + } + for i, v := range exp { + if err := checkDecodeFromJSON(s, v); err != nil { + return addStack(fmt.Sprintf("[%d]", i), exp, err) + } + } + if err := s.ListEnd(); err != nil { + return addStack("ListEnd", exp, err) + } + default: + panic(fmt.Errorf("unhandled type: %T", exp)) + } + return nil +} + +func addStack(op string, val interface{}, err error) error { + lines := strings.Split(err.Error(), "\n") + lines = append(lines, fmt.Sprintf("\t%s: %v", op, val)) + return errors.New(strings.Join(lines, "\n")) +} -- cgit From 593b1b65e76bf1f92249078e45a97ee21b58778c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Jul 2015 15:42:23 +0200 Subject: tests: document RLP tests --- tests/rlp_test_util.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go index d7042eef7..c322b78c6 100644 --- a/tests/rlp_test_util.go +++ b/tests/rlp_test_util.go @@ -13,11 +13,22 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +// RLPTest is the JSON structure of a single RLP test. type RLPTest struct { - In interface{} + // If the value of In is "INVALID" or "VALID", the test + // checks whether Out can be decoded into a value of + // type interface{}. + // + // For other JSON values, In is treated as a driver for + // calls to rlp.Stream. The test also verifies that encoding + // In produces the bytes in Out. + In interface{} + + // Out is a hex-encoded RLP value. Out string } +// RunRLPTest runs the tests in the given file, skipping tests by name. func RunRLPTest(file string, skip []string) error { f, err := os.Open(file) if err != nil { @@ -27,6 +38,7 @@ func RunRLPTest(file string, skip []string) error { return RunRLPTestWithReader(f, skip) } +// RunRLPTest runs the tests encoded in r, skipping tests by name. func RunRLPTestWithReader(r io.Reader, skip []string) error { var tests map[string]*RLPTest if err := readJson(r, &tests); err != nil { @@ -49,6 +61,8 @@ func (t *RLPTest) Run() error { if err != nil { return fmt.Errorf("invalid hex in Out") } + + // Handle simple decoding tests with no actual In value. if t.In == "VALID" || t.In == "INVALID" { return checkDecodeInterface(outb, t.In == "VALID") } @@ -62,7 +76,7 @@ func (t *RLPTest) Run() error { if !bytes.Equal(b, outb) { return fmt.Errorf("encode produced %x, want %x", b, outb) } - // Test decoding from a stream. + // Test stream decoding. s := rlp.NewStream(bytes.NewReader(outb), 0) return checkDecodeFromJSON(s, in) } @@ -103,8 +117,10 @@ func translateJSON(v interface{}) interface{} { } } -// checkDecodeFromJSON decodes from s guided by exp. For each JSON -// value, the value decoded from the RLP stream must match. +// checkDecodeFromJSON decodes from s guided by exp. exp drives the +// Stream by invoking decoding operations (Uint, Big, List, ...) based +// on the type of each value. The value decoded from the RLP stream +// must match the JSON value. func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error { switch exp := exp.(type) { case uint64: -- cgit From cefd948267c6d16b317f7c15feb653a7725a3f1b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 18 Jul 2015 01:47:17 +0200 Subject: rlp: reject trailing data when using DecodeBytes --- rlp/decode.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/rlp/decode.go b/rlp/decode.go index 4462d4be4..f17690522 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -110,9 +110,17 @@ func Decode(r io.Reader, val interface{}) error { // DecodeBytes parses RLP data from b into val. // Please see the documentation of Decode for the decoding rules. +// The input must contain exactly one value and no trailing data. func DecodeBytes(b []byte, val interface{}) error { // TODO: this could use a Stream from a pool. - return NewStream(bytes.NewReader(b), uint64(len(b))).Decode(val) + r := bytes.NewReader(b) + if err := NewStream(r, uint64(len(b))).Decode(val); err != nil { + return err + } + if r.Len() > 0 { + return ErrMoreThanOneValue + } + return nil } type decodeError struct { @@ -517,6 +525,10 @@ var ( ErrElemTooLarge = errors.New("rlp: element is larger than containing list") ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + // This error is reported by DecodeBytes if the slice contains + // additional data after the first RLP value. + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + // internal errors errNotInList = errors.New("rlp: call of ListEnd outside of any list") errNotAtEOL = errors.New("rlp: call of ListEnd not positioned at EOL") -- cgit From 345590f27f81517fd7b560a3fafb0a2c8b498094 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 18 Jul 2015 16:13:20 +0200 Subject: rlp: fix check for canonical byte array size Decoding did not reject byte arrays of length one with a single element b where 55 < b < 128. Such byte arrays must be rejected because they must be encoded as the single byte b instead. --- rlp/decode.go | 6 +++--- rlp/decode_test.go | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index f17690522..cc402fc94 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -361,7 +361,7 @@ func decodeByteArray(s *Stream, val reflect.Value) error { return err } // Reject cases where single byte encoding should have been used. - if size == 1 && slice[0] < 56 { + if size == 1 && slice[0] < 128 { return wrapStreamError(ErrCanonSize, val.Type()) } case List: @@ -623,7 +623,7 @@ func (s *Stream) Bytes() ([]byte, error) { if err = s.readFull(b); err != nil { return nil, err } - if size == 1 && b[0] < 56 { + if size == 1 && b[0] < 128 { return nil, ErrCanonSize } return b, nil @@ -687,7 +687,7 @@ func (s *Stream) uint(maxbits int) (uint64, error) { return 0, ErrCanonInt case err != nil: return 0, err - case size > 0 && v < 56: + case size > 0 && v < 128: return 0, ErrCanonSize default: return v, nil diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 71dacaba4..6f90d6e1d 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -113,12 +113,16 @@ func TestStreamErrors(t *testing.T) { {"00", calls{"Uint"}, nil, ErrCanonInt}, {"820002", calls{"Uint"}, nil, ErrCanonInt}, {"8133", calls{"Uint"}, nil, ErrCanonSize}, - {"8156", calls{"Uint"}, nil, nil}, + {"817F", calls{"Uint"}, nil, ErrCanonSize}, + {"8180", calls{"Uint"}, nil, nil}, // Size tags must use the smallest possible encoding. // Leading zero bytes in the size tag are also rejected. {"8100", calls{"Uint"}, nil, ErrCanonSize}, {"8100", calls{"Bytes"}, nil, ErrCanonSize}, + {"8101", calls{"Bytes"}, nil, ErrCanonSize}, + {"817F", calls{"Bytes"}, nil, ErrCanonSize}, + {"8180", calls{"Bytes"}, nil, nil}, {"B800", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, {"B90000", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, {"B90055", calls{"Kind"}, withoutInputLimit, ErrCanonSize}, @@ -132,11 +136,11 @@ func TestStreamErrors(t *testing.T) { {"", calls{"Kind"}, nil, io.EOF}, {"", calls{"Uint"}, nil, io.EOF}, {"", calls{"List"}, nil, io.EOF}, - {"8158", calls{"Uint", "Uint"}, nil, io.EOF}, + {"8180", calls{"Uint", "Uint"}, nil, io.EOF}, {"C0", calls{"List", "ListEnd", "List"}, nil, io.EOF}, {"", calls{"List"}, withoutInputLimit, io.EOF}, - {"8158", calls{"Uint", "Uint"}, withoutInputLimit, io.EOF}, + {"8180", calls{"Uint", "Uint"}, withoutInputLimit, io.EOF}, {"C0", calls{"List", "ListEnd", "List"}, withoutInputLimit, io.EOF}, // Input limit errors. @@ -153,11 +157,11 @@ func TestStreamErrors(t *testing.T) { // down toward zero in Stream.remaining, reading too far can overflow // remaining to a large value, effectively disabling the limit. {"C40102030401", calls{"Raw", "Uint"}, withCustomInputLimit(5), io.EOF}, - {"C4010203048158", calls{"Raw", "Uint"}, withCustomInputLimit(6), ErrValueTooLarge}, + {"C4010203048180", calls{"Raw", "Uint"}, withCustomInputLimit(6), ErrValueTooLarge}, // Check that the same calls are fine without a limit. {"C40102030401", calls{"Raw", "Uint"}, withoutInputLimit, nil}, - {"C4010203048158", calls{"Raw", "Uint"}, withoutInputLimit, nil}, + {"C4010203048180", calls{"Raw", "Uint"}, withoutInputLimit, nil}, // Unexpected EOF. This only happens when there is // no input limit, so the reader needs to be 'dumbed down'. @@ -349,6 +353,7 @@ var decodeTests = []decodeTest{ // byte arrays {input: "02", ptr: new([1]byte), value: [1]byte{2}}, + {input: "8180", ptr: new([1]byte), value: [1]byte{128}}, {input: "850102030405", ptr: new([5]byte), value: [5]byte{1, 2, 3, 4, 5}}, // byte array errors @@ -359,6 +364,7 @@ var decodeTests = []decodeTest{ {input: "C3010203", ptr: new([5]byte), error: "rlp: expected input string or byte for [5]uint8"}, {input: "86010203040506", ptr: new([5]byte), error: "rlp: input string too long for [5]uint8"}, {input: "8105", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"}, + {input: "817F", ptr: new([1]byte), error: "rlp: non-canonical size information for [1]uint8"}, // zero sized byte arrays {input: "80", ptr: new([0]byte), value: [0]byte{}}, @@ -427,7 +433,8 @@ var decodeTests = []decodeTest{ {input: "80", ptr: new(*uint), value: uintp(0)}, {input: "C0", ptr: new(*uint), error: "rlp: expected input string or byte for uint"}, {input: "07", ptr: new(*uint), value: uintp(7)}, - {input: "8158", ptr: new(*uint), value: uintp(0x58)}, + {input: "817F", ptr: new(*uint), error: "rlp: non-canonical size information for uint"}, + {input: "8180", ptr: new(*uint), value: uintp(0x80)}, {input: "C109", ptr: new(*[]uint), value: &[]uint{9}}, {input: "C58403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}}, -- cgit