From 2714e8f091117b4f110198008348bfc19233ed60 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 26 Nov 2018 16:10:22 +0100 Subject: Remove multihash from Swarm bzz:// for Feeds (#18175) --- swarm/api/api.go | 25 +++++---- swarm/api/client/client_test.go | 112 ++++++++++++++++++++++++++++++---------- swarm/api/http/server_test.go | 104 ++++++++++++++++++++++++------------- 3 files changed, 165 insertions(+), 76 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index 099a46939..33a8e3539 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" @@ -417,7 +416,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return reader, mimeType, status, nil, err } // get the data of the update - _, rsrcData, err := a.feed.GetContent(entry.Feed) + _, contentAddr, err := a.feed.GetContent(entry.Feed) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -425,23 +424,23 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return reader, mimeType, status, nil, err } - // extract multihash - decodedMultihash, err := multihash.FromMultihash(rsrcData) - if err != nil { + // extract content hash + if len(contentAddr) != storage.AddressLength { apiGetInvalid.Inc(1) status = http.StatusUnprocessableEntity - log.Warn("invalid multihash in feed update", "err", err) - return reader, mimeType, status, nil, err + errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr)) + log.Warn(errorMessage) + return reader, mimeType, status, nil, errors.New(errorMessage) } - manifestAddr = storage.Address(decodedMultihash) - log.Trace("feed update contains multihash", "key", manifestAddr) + manifestAddr = storage.Address(contentAddr) + log.Trace("feed update contains swarm hash", "key", manifestAddr) - // get the manifest the multihash digest points to + // get the manifest the swarm hash points to trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err)) + log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err)) return reader, mimeType, status, nil, err } @@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage if entry == nil { status = http.StatusNotFound apiGetNotFound.Inc(1) - err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path) - log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path) + err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path) + log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path) return reader, mimeType, status, nil, err } } diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 76b349397..39f6e4797 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -25,13 +25,13 @@ import ( "sort" "testing" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage/feed" ) @@ -368,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) { return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash feed updates with bzz:// scheme +// Test the transparent resolving of feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a feed update -// retrieving the update with the multihash should return the manifest pointing directly to the data +// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update. +// This effectively uses a feed to store a pointer to content rather than the content itself +// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestClientCreateFeedMultihash(t *testing.T) { +func TestClientBzzWithFeed(t *testing.T) { signer, _ := newTestSigner() + // Initialize a Swarm test server srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - client := NewClient(srv.URL) + swarmClient := NewClient(srv.URL) defer srv.Close() - // add the data our multihash aliased manifest will point to - databytes := []byte("bar") - - swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false) + // put together some data for our test: + dataBytes := []byte(` + // + // Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update. + // So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it: + // + // MANIFEST HASH --> DATA + // + // Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this, + // we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash. + // + // FEED MANIFEST HASH --> MANIFEST HASH --> DATA + // + // Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash** + // stays constant, we have effectively created a fixed address to changing content. (Applause) + // + // FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) + // + `) + + // Create a virtual File out of memory containing the above data + f := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(len(dataBytes)), + }, + } + + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + manifestAddressHex, err := swarmClient.Upload(f, "", false) if err != nil { - t.Fatalf("Error uploading raw test data: %s", err) + t.Fatalf("Error creating manifest: %s", err) } - s := common.FromHex(swarmHash) - mh := multihash.ToMultihash(s) + // convert the hex-encoded manifest hash to a 32-byte slice + manifestAddress := common.FromHex(manifestAddressHex) + + if len(manifestAddress) != storage.AddressLength { + t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress)) + } - // our feed topic - topic, _ := feed.NewTopic("foo.eth", nil) + // Now create a **feed manifest**. For that, we need a topic: + topic, _ := feed.NewTopic("interesting topic indeed", nil) - createRequest := feed.NewFirstRequest(topic) + // Build a feed request to update data + request := feed.NewFirstRequest(topic) - createRequest.SetData(mh) - if err := createRequest.Sign(signer); err != nil { + // Put the 32-byte address of the manifest into the feed update + request.SetData(manifestAddress) + + // Sign the update + if err := request.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - feedManifestHash, err := client.CreateFeedWithManifest(createRequest) - + // Publish the update and at the same time request a **feed manifest** to be created + feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request) if err != nil { t.Fatalf("Error creating feed manifest: %s", err) } - correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" - if feedManifestHash != correctManifestAddrHex { - t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) + // Check we have received the exact **feed manifest** to be expected + // given the topic and user signing the updates: + correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2" + if feedManifestAddressHex != correctFeedManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex) } // Check we get a not found error when trying to get feed updates with a made-up manifest - _, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + _, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") if err != ErrNoFeedUpdatesFound { t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err) } - reader, err := client.QueryFeed(nil, correctManifestAddrHex) + // If we query the feed directly we should get **manifest hash** back: + reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex) if err != nil { t.Fatalf("Error retrieving feed updates: %s", err) } @@ -428,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(mh, gotData) { - t.Fatalf("Expected: %v, got %v", mh, gotData) + + //Check that indeed the **manifest hash** is retrieved + if !bytes.Equal(manifestAddress, gotData) { + t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) + } + + // Now the final test we were looking for: Use bzz:// and that should resolve all manifests + // and return the original data directly: + f, err = swarmClient.Download(feedManifestAddressHex, "") + if err != nil { + t.Fatal(err) + } + gotData, err = ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) } + // Check that we get back the original data: + if !bytes.Equal(dataBytes, gotData) { + t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) + } } // TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client. diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 1ef3deece..e82762ce0 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -45,7 +45,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" @@ -69,60 +68,91 @@ func newTestSigner() (*feed.GenericSigner, error) { return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash-containing feed updates with bzz:// scheme +// Test the transparent resolving of feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a feed update -// retrieving the update with the multihash should return the manifest pointing directly to the data +// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update. +// This effectively uses a feed to store a pointer to content rather than the content itself +// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestBzzFeedMultihash(t *testing.T) { +func TestBzzWithFeed(t *testing.T) { signer, _ := newTestSigner() + // Initialize Swarm test server srv := NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() - // add the data our multihash aliased manifest will point to - databytes := "bar" - testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL) - resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes))) + // put together some data for our test: + dataBytes := []byte(` + // + // Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update. + // So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it: + // + // MANIFEST HASH --> DATA + // + // Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this, + // we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash. + // + // FEED MANIFEST HASH --> MANIFEST HASH --> DATA + // + // Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash** + // stays constant, we have effectively created a fixed address to changing content. (Applause) + // + // FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) ... + // + `) + + // POST data to bzz and get back a content-addressed **manifest hash** pointing to it. + resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes))) if err != nil { t.Fatal(err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err := ioutil.ReadAll(resp.Body) + manifestAddressHex, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - s := common.FromHex(string(b)) - mh := multihash.ToMultihash(s) - log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) + manifestAddress := common.FromHex(string(manifestAddressHex)) - topic, _ := feed.NewTopic("foo.eth", nil) + log.Info("added data", "manifest", string(manifestAddressHex)) + + // At this point we have uploaded the data and have a manifest pointing to it + // Now store that manifest address in a feed update. + // We also want a feed manifest, so we can use it to refer to the feed. + + // First, create a topic for our feed: + topic, _ := feed.NewTopic("interesting topic indeed", nil) + + // Create a feed update request: updateRequest := feed.NewFirstRequest(topic) - updateRequest.SetData(mh) + // Store the **manifest address** as data into the feed update. + updateRequest.SetData(manifestAddress) + // Sign the update if err := updateRequest.Sign(signer); err != nil { t.Fatal(err) } - log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) + log.Info("added data", "data", common.ToHex(manifestAddress)) - testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) + // Build the feed update http request: + feedUpdateURL, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } - query := testUrl.Query() + query := feedUpdateURL.Query() body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted - query.Set("manifest", "1") // indicate we want a manifest back - testUrl.RawQuery = query.Encode() + query.Set("manifest", "1") // indicate we want a feed manifest back + feedUpdateURL.RawQuery = query.Encode() - // create the multihash update - resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body)) + // submit the feed update request to Swarm + resp, err = http.Post(feedUpdateURL.String(), "application/octet-stream", bytes.NewReader(body)) if err != nil { t.Fatal(err) } @@ -130,24 +160,25 @@ func TestBzzFeedMultihash(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err = ioutil.ReadAll(resp.Body) + + feedManifestAddressHex, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - rsrcResp := &storage.Address{} - err = json.Unmarshal(b, rsrcResp) + feedManifestAddress := &storage.Address{} + err = json.Unmarshal(feedManifestAddressHex, feedManifestAddress) if err != nil { - t.Fatalf("data %s could not be unmarshaled: %v", b, err) + t.Fatalf("data %s could not be unmarshaled: %v", feedManifestAddressHex, err) } - correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" - if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + correctManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2" + if feedManifestAddress.Hex() != correctManifestAddrHex { + t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestAddress.Hex()) } // get bzz manifest transparent feed update resolve - testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) - resp, err = http.Get(testBzzUrl) + getBzzURL := fmt.Sprintf("%s/bzz:/%s", srv.URL, feedManifestAddress) + resp, err = http.Get(getBzzURL) if err != nil { t.Fatal(err) } @@ -155,12 +186,12 @@ func TestBzzFeedMultihash(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("err %s", resp.Status) } - b, err = ioutil.ReadAll(resp.Body) + retrievedData, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - if !bytes.Equal(b, []byte(databytes)) { - t.Fatalf("retrieved data mismatch, expected %x, got %x", databytes, b) + if !bytes.Equal(retrievedData, []byte(dataBytes)) { + t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData) } } @@ -245,7 +276,8 @@ func TestBzzFeed(t *testing.T) { t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex()) } - // get bzz manifest transparent feed update resolve + // take the chance to have bzz: crash on resolving a feed update that does not contain + // a swarm hash: testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) resp, err = http.Get(testBzzUrl) if err != nil { @@ -253,7 +285,7 @@ func TestBzzFeed(t *testing.T) { } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { - t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") + t.Fatal("Expected error status since feed update does not contain a Swarm hash. Received 200 OK") } _, err = ioutil.ReadAll(resp.Body) if err != nil { -- cgit