Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate

2 files changed, 100 insertions, 43 deletions

diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index 3d06e9e1c..a6666144a 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -28,6 +28,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -595,13 +596,16 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error)
 	return string(data), nil
 }
 
+// ErrNoResourceUpdatesFound is returned when Swarm cannot find updates of the given resource
+var ErrNoResourceUpdatesFound = errors.New("No updates found for this resource")
+
 // CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided
 // data. Data is interpreted as multihash or not depending on the multihash parameter.
 // startTime=0 means "now"
 // Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent)
 // or reference future updates (Client.UpdateResource)
 func (c *Client) CreateResource(request *mru.Request) (string, error) {
-	responseStream, err := c.updateResource(request)
+	responseStream, err := c.updateResource(request, true)
 	if err != nil {
 		return "", err
 	}
@@ -621,17 +625,24 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) {
 
 // UpdateResource allows you to set a new version of your content
 func (c *Client) UpdateResource(request *mru.Request) error {
-	_, err := c.updateResource(request)
+	_, err := c.updateResource(request, false)
 	return err
 }
 
-func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
-	body, err := request.MarshalJSON()
+func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.ReadCloser, error) {
+	URL, err := url.Parse(c.Gateway)
 	if err != nil {
 		return nil, err
 	}
+	URL.Path = "/bzz-resource:/"
+	values := URL.Query()
+	body := request.AppendValues(values)
+	if createManifest {
+		values.Set("manifest", "1")
+	}
+	URL.RawQuery = values.Encode()
 
-	req, err := http.NewRequest("POST", c.Gateway+"/bzz-resource:/", bytes.NewBuffer(body))
+	req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body))
 	if err != nil {
 		return nil, err
 	}
@@ -642,28 +653,61 @@ func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
 	}
 
 	return res.Body, nil
-
 }
 
 // GetResource returns a byte stream with the raw content of the resource
 // manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
 // points to that address
-func (c *Client) GetResource(manifestAddressOrDomain string) (io.ReadCloser, error) {
+func (c *Client) GetResource(query *mru.Query, manifestAddressOrDomain string) (io.ReadCloser, error) {
+	return c.getResource(query, manifestAddressOrDomain, false)
+}
 
-	res, err := http.Get(c.Gateway + "/bzz-resource:/" + manifestAddressOrDomain)
+// getResource returns a byte stream with the raw content of the resource
+// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
+// points to that address
+// meta set to true will instruct the node return resource metainformation instead
+func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) {
+	URL, err := url.Parse(c.Gateway)
 	if err != nil {
 		return nil, err
 	}
-	return res.Body, nil
+	URL.Path = "/bzz-resource:/" + manifestAddressOrDomain
+	values := URL.Query()
+	if query != nil {
+		query.AppendValues(values) //adds query parameters
+	}
+	if meta {
+		values.Set("meta", "1")
+	}
+	URL.RawQuery = values.Encode()
+	res, err := http.Get(URL.String())
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		if res.StatusCode == http.StatusNotFound {
+			return nil, ErrNoResourceUpdatesFound
+		}
+		errorMessageBytes, err := ioutil.ReadAll(res.Body)
+		var errorMessage string
+		if err != nil {
+			errorMessage = "cannot retrieve error message: " + err.Error()
+		} else {
+			errorMessage = string(errorMessageBytes)
+		}
+		return nil, fmt.Errorf("Error retrieving resource: %s", errorMessage)
+	}
 
+	return res.Body, nil
 }
 
 // GetResourceMetadata returns a structure that describes the Mutable Resource
 // manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
 // points to that address
-func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Request, error) {
+func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain string) (*mru.Request, error) {
 
-	responseStream, err := c.GetResource(manifestAddressOrDomain + "/meta")
+	responseStream, err := c.getResource(query, manifestAddressOrDomain, true)
 	if err != nil {
 		return nil, err
 	}
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index f9312d48f..02980de1d 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -25,6 +25,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/swarm/api"
@@ -391,19 +393,12 @@ func TestClientCreateResourceMultihash(t *testing.T) {
 	s := common.FromHex(swarmHash)
 	mh := multihash.ToMultihash(s)
 
-	// our mutable resource "name"
-	resourceName := "foo.eth"
+	// our mutable resource topic
+	topic, _ := mru.NewTopic("foo.eth", nil)
 
-	createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
-		Name:      resourceName,
-		Frequency: 13,
-		StartTime: srv.GetCurrentTime(),
-		Owner:     signer.Address(),
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	createRequest.SetData(mh, true)
+	createRequest := mru.NewFirstRequest(topic)
+
+	createRequest.SetData(mh)
 	if err := createRequest.Sign(signer); err != nil {
 		t.Fatalf("Error signing update: %s", err)
 	}
@@ -414,12 +409,18 @@ func TestClientCreateResourceMultihash(t *testing.T) {
 		t.Fatalf("Error creating resource: %s", err)
 	}
 
-	correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
+	correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
 	if resourceManifestHash != correctManifestAddrHex {
-		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
+		t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
 	}
 
-	reader, err := client.GetResource(correctManifestAddrHex)
+	// Check we get a not found error when trying to get the resource with a made-up manifest
+	_, err = client.GetResource(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
+	if err != ErrNoResourceUpdatesFound {
+		t.Fatalf("Expected to receive ErrNoResourceUpdatesFound error. Got: %s", err)
+	}
+
+	reader, err := client.GetResource(nil, correctManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving resource: %s", err)
 	}
@@ -447,30 +448,22 @@ func TestClientCreateUpdateResource(t *testing.T) {
 	databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...")
 
 	// our mutable resource name
-	resourceName := "El Quijote"
+	topic, _ := mru.NewTopic("El Quijote", nil)
+	createRequest := mru.NewFirstRequest(topic)
 
-	createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
-		Name:      resourceName,
-		Frequency: 13,
-		StartTime: srv.GetCurrentTime(),
-		Owner:     signer.Address(),
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	createRequest.SetData(databytes, false)
+	createRequest.SetData(databytes)
 	if err := createRequest.Sign(signer); err != nil {
 		t.Fatalf("Error signing update: %s", err)
 	}
 
 	resourceManifestHash, err := client.CreateResource(createRequest)
 
-	correctManifestAddrHex := "cc7904c17b49f9679e2d8006fe25e87e3f5c2072c2b49cab50f15e544471b30a"
+	correctManifestAddrHex := "fcb8e75f53e480e197c083ad1976d265674d0ce776f2bf359c09c413fb5230b8"
 	if resourceManifestHash != correctManifestAddrHex {
-		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
+		t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
 	}
 
-	reader, err := client.GetResource(correctManifestAddrHex)
+	reader, err := client.GetResource(nil, correctManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving resource: %s", err)
 	}
@@ -486,12 +479,12 @@ func TestClientCreateUpdateResource(t *testing.T) {
 	// define different data
 	databytes = []byte("... no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero ...")
 
-	updateRequest, err := client.GetResourceMetadata(correctManifestAddrHex)
+	updateRequest, err := client.GetResourceMetadata(nil, correctManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving update request template: %s", err)
 	}
 
-	updateRequest.SetData(databytes, false)
+	updateRequest.SetData(databytes)
 	if err := updateRequest.Sign(signer); err != nil {
 		t.Fatalf("Error signing update: %s", err)
 	}
@@ -500,7 +493,7 @@ func TestClientCreateUpdateResource(t *testing.T) {
 		t.Fatalf("Error updating resource: %s", err)
 	}
 
-	reader, err = client.GetResource(correctManifestAddrHex)
+	reader, err = client.GetResource(nil, correctManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving resource: %s", err)
 	}
@@ -513,4 +506,24 @@ func TestClientCreateUpdateResource(t *testing.T) {
 		t.Fatalf("Expected: %v, got %v", databytes, gotData)
 	}
 
+	// now try retrieving resource without a manifest
+
+	view := &mru.View{
+		Topic: topic,
+		User:  signer.Address(),
+	}
+
+	lookupParams := mru.NewQueryLatest(view, lookup.NoClue)
+	reader, err = client.GetResource(lookupParams, "")
+	if err != nil {
+		t.Fatalf("Error retrieving resource: %s", err)
+	}
+	defer reader.Close()
+	gotData, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(databytes, gotData) {
+		t.Fatalf("Expected: %v, got %v", databytes, gotData)
+	}
 }