Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate

1 files changed, 96 insertions, 82 deletions

diff --git a/swarm/api/api.go b/swarm/api/api.go
index d7b6d8419..70c12a757 100644
--- a/swarm/api/api.go
+++ b/swarm/api/api.go
@@ -29,6 +29,8 @@ import (
 	"path"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
+
 	"bytes"
 	"mime"
 	"path/filepath"
@@ -401,77 +403,54 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 
 		// we need to do some extra work if this is a mutable resource manifest
 		if entry.ContentType == ResourceContentType {
-
-			// get the resource rootAddr
-			log.Trace("resource type", "menifestAddr", manifestAddr, "hash", entry.Hash)
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-			rootAddr := storage.Address(common.FromHex(entry.Hash))
-			rsrc, err := a.resource.Load(ctx, rootAddr)
+			if entry.ResourceView == nil {
+				return reader, mimeType, status, nil, fmt.Errorf("Cannot decode ResourceView in manifest")
+			}
+			_, err := a.resource.Lookup(ctx, mru.NewQueryLatest(entry.ResourceView, lookup.NoClue))
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
 				log.Debug(fmt.Sprintf("get resource content error: %v", err))
 				return reader, mimeType, status, nil, err
 			}
+			// get the data of the update
+			_, rsrcData, err := a.resource.GetContent(entry.ResourceView)
+			if err != nil {
+				apiGetNotFound.Inc(1)
+				status = http.StatusNotFound
+				log.Warn(fmt.Sprintf("get resource content error: %v", err))
+				return reader, mimeType, status, nil, err
+			}
+
+			// extract multihash
+			decodedMultihash, err := multihash.FromMultihash(rsrcData)
+			if err != nil {
+				apiGetInvalid.Inc(1)
+				status = http.StatusUnprocessableEntity
+				log.Warn("invalid resource multihash", "err", err)
+				return reader, mimeType, status, nil, err
+			}
+			manifestAddr = storage.Address(decodedMultihash)
+			log.Trace("resource is multihash", "key", manifestAddr)
 
-			// use this key to retrieve the latest update
-			params := mru.LookupLatest(rootAddr)
-			rsrc, err = a.resource.Lookup(ctx, params)
+			// get the manifest the multihash digest points to
+			trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
-				log.Debug(fmt.Sprintf("get resource content error: %v", err))
+				log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
 				return reader, mimeType, status, nil, err
 			}
 
-			// if it's multihash, we will transparently serve the content this multihash points to
-			// \TODO this resolve is rather expensive all in all, review to see if it can be achieved cheaper
-			if rsrc.Multihash() {
-
-				// get the data of the update
-				_, rsrcData, err := a.resource.GetContent(rootAddr)
-				if err != nil {
-					apiGetNotFound.Inc(1)
-					status = http.StatusNotFound
-					log.Warn(fmt.Sprintf("get resource content error: %v", err))
-					return reader, mimeType, status, nil, err
-				}
-
-				// validate that data as multihash
-				decodedMultihash, err := multihash.FromMultihash(rsrcData)
-				if err != nil {
-					apiGetInvalid.Inc(1)
-					status = http.StatusUnprocessableEntity
-					log.Warn("invalid resource multihash", "err", err)
-					return reader, mimeType, status, nil, err
-				}
-				manifestAddr = storage.Address(decodedMultihash)
-				log.Trace("resource is multihash", "key", manifestAddr)
-
-				// get the manifest the multihash digest points to
-				trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
-				if err != nil {
-					apiGetNotFound.Inc(1)
-					status = http.StatusNotFound
-					log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
-					return reader, mimeType, status, nil, err
-				}
-
-				// finally, get the manifest entry
-				// it will always be the entry on path ""
-				entry, _ = trie.getEntry(path)
-				if entry == nil {
-					status = http.StatusNotFound
-					apiGetNotFound.Inc(1)
-					err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
-					log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
-					return reader, mimeType, status, nil, err
-				}
-
-			} else {
-				// data is returned verbatim since it's not a multihash
-				return rsrc, "application/octet-stream", http.StatusOK, nil, nil
+			// finally, get the manifest entry
+			// it will always be the entry on path ""
+			entry, _ = trie.getEntry(path)
+			if entry == nil {
+				status = http.StatusNotFound
+				apiGetNotFound.Inc(1)
+				err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
+				log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
+				return reader, mimeType, status, nil, err
 			}
 		}
 
@@ -966,37 +945,27 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver
 }
 
 // ResourceLookup finds mutable resource updates at specific periods and versions
-func (a *API) ResourceLookup(ctx context.Context, params *mru.LookupParams) (string, []byte, error) {
-	var err error
-	rsrc, err := a.resource.Load(ctx, params.RootAddr())
+func (a *API) ResourceLookup(ctx context.Context, query *mru.Query) ([]byte, error) {
+	_, err := a.resource.Lookup(ctx, query)
 	if err != nil {
-		return "", nil, err
-	}
-	_, err = a.resource.Lookup(ctx, params)
-	if err != nil {
-		return "", nil, err
+		return nil, err
 	}
 	var data []byte
-	_, data, err = a.resource.GetContent(params.RootAddr())
+	_, data, err = a.resource.GetContent(&query.View)
 	if err != nil {
-		return "", nil, err
+		return nil, err
 	}
-	return rsrc.Name(), data, nil
-}
-
-// Create Mutable resource
-func (a *API) ResourceCreate(ctx context.Context, request *mru.Request) error {
-	return a.resource.New(ctx, request)
+	return data, nil
 }
 
 // ResourceNewRequest creates a Request object to update a specific mutable resource
-func (a *API) ResourceNewRequest(ctx context.Context, rootAddr storage.Address) (*mru.Request, error) {
-	return a.resource.NewUpdateRequest(ctx, rootAddr)
+func (a *API) ResourceNewRequest(ctx context.Context, view *mru.View) (*mru.Request, error) {
+	return a.resource.NewRequest(ctx, view)
 }
 
 // ResourceUpdate updates a Mutable Resource with arbitrary data.
 // Upon retrieval the update will be retrieved verbatim as bytes.
-func (a *API) ResourceUpdate(ctx context.Context, request *mru.SignedResourceUpdate) (storage.Address, error) {
+func (a *API) ResourceUpdate(ctx context.Context, request *mru.Request) (storage.Address, error) {
 	return a.resource.Update(ctx, request)
 }
 
@@ -1005,17 +974,62 @@ func (a *API) ResourceHashSize() int {
 	return a.resource.HashSize
 }
 
-// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
-func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
+// ErrCannotLoadResourceManifest is returned when looking up a resource manifest fails
+var ErrCannotLoadResourceManifest = errors.New("Cannot load resource manifest")
+
+// ErrNotAResourceManifest is returned when the address provided returned something other than a valid manifest
+var ErrNotAResourceManifest = errors.New("Not a resource manifest")
+
+// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the Resource's view ID.
+func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (*mru.View, error) {
 	trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt)
 	if err != nil {
-		return nil, fmt.Errorf("cannot load resource manifest: %v", err)
+		return nil, ErrCannotLoadResourceManifest
 	}
 
 	entry, _ := trie.getEntry("")
 	if entry.ContentType != ResourceContentType {
-		return nil, fmt.Errorf("not a resource manifest: %s", addr)
+		return nil, ErrNotAResourceManifest
 	}
 
-	return storage.Address(common.FromHex(entry.Hash)), nil
+	return entry.ResourceView, nil
+}
+
+// ErrCannotResolveResourceURI is returned when the ENS resolver is not able to translate a name to a resource
+var ErrCannotResolveResourceURI = errors.New("Cannot resolve Resource URI")
+
+// ErrCannotResolveResourceView is returned when values provided are not enough or invalid to recreate a
+// resource view out of them.
+var ErrCannotResolveResourceView = errors.New("Cannot resolve resource view")
+
+// ResolveResourceView attempts to extract View information out of the manifest, if provided
+// If not, it attempts to extract the View out of a set of key-value pairs
+func (a *API) ResolveResourceView(ctx context.Context, uri *URI, values mru.Values) (*mru.View, error) {
+	var view *mru.View
+	var err error
+	if uri.Addr != "" {
+		// resolve the content key.
+		manifestAddr := uri.Address()
+		if manifestAddr == nil {
+			manifestAddr, err = a.Resolve(ctx, uri.Addr)
+			if err != nil {
+				return nil, ErrCannotResolveResourceURI
+			}
+		}
+
+		// get the resource view from the manifest
+		view, err = a.ResolveResourceManifest(ctx, manifestAddr)
+		if err != nil {
+			return nil, err
+		}
+		log.Debug("handle.get.resource: resolved", "manifestkey", manifestAddr, "view", view.Hex())
+	} else {
+		var v mru.View
+		if err := v.FromValues(values); err != nil {
+			return nil, ErrCannotResolveResourceView
+
+		}
+		view = &v
+	}
+	return view, nil
 }