From 2c110c81ee92290d3e5ce6134a065c8d2abfbb60 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Fri, 28 Sep 2018 12:07:17 +0200 Subject: Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559) * swarm/storage/mru: Adaptive Frequency swarm/storage/mru/lookup: fixed getBaseTime Added NewEpoch constructor swarm/api/client: better error handling in GetResource() swarm/storage/mru: Renamed structures. Renamed ResourceMetadata to ResourceID. Renamed ResourceID.Name to ResourceID.Topic swarm/storage/mru: Added binarySerializer interface and test tools swarm/storage/mru/lookup: Changed base time to time and + marshallers swarm/storage/mru: Added ResourceID (former resourceMetadata) swarm/storage/mru: Added ResourceViewId and serialization tests swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals swarm/storage/mru: Fixes as per review comments cmd/swarm: reworded resource create/update help text regarding topic swarm/storage/mru: Added UpdateLookup and serializer tests swarm/storage/mru: Added UpdateHeader, serializers and tests swarm/storage/mru: changed UpdateAddr / epoch to Base() swarm/storage/mru: Added resourceUpdate serializer and tests swarm/storage/mru: Added SignedResourceUpdate tests and serializers swarm/storage/mru/lookup: fixed GetFirstEpoch bug swarm/storage/mru: refactor, comments, cleanup Also added tests for Topic swarm/storage/mru: handler tests pass swarm/storage/mru: all resource package tests pass swarm/storage/mru: resource test pass after adding timestamp checking support swarm/storage/mru: Added JSON serializers to ResourceIDView structures swarm/storage/mru: Sever, client, API test pass swarm/storage/mru: server test pass swarm/storage/mru: Added topic length check swarm/storage/mru: removed some literals, improved "previous lookup" test case swarm/storage/mru: some fixes and comments as per review swarm/storage/mru: first working version without metadata chunk swarm/storage/mru: Various fixes as per review swarm/storage/mru: client test pass swarm/storage/mru: resource query strings and manifest-less queries swarm/storage/mru: simplify naming swarm/storage/mru: first autofreq working version swarm/storage/mru: renamed ToValues to AppendValues swarm/resource/mru: Added ToValues / FromValues for URL query strings swarm/storage/mru: Changed POST resource to work with query strings. No more JSON. swarm/storage/mru: removed resourceid swarm/storage/mru: Opened up structures swarm/storage/mru: Merged Request and SignedResourceUpdate swarm/storage/mru: removed initial data from CLI resource create swarm/storage/mru: Refactor Topic as a direct fixed-length array swarm/storage/mru/lookup: Comprehensive GetNextLevel tests swarm/storage/mru: Added comments Added length checks in Topic swarm/storage/mru: fixes in tests and some code comments swarm/storage/mru/lookup: new optimized lookup algorithm swarm/api: moved getResourceView to api out of server swarm/storage/mru: Lookup algorithm working swarm/storage/mru: comments and renamed NewLookupParams Deleted commented code swarm/storage/mru/lookup: renamed Epoch.LaterThan to After swarm/storage/mru/lookup: Comments and tidying naming swarm/storage/mru: fix lookup algorithm swarm/storage/mru: exposed lookup hint removed updateheader swarm/storage/mru/lookup: changed GetNextEpoch for initial values swarm/storage/mru: resource tests pass swarm/storage/mru: valueSerializer interface and tests swarm/storage/mru/lookup: Comments, improvements, fixes, more tests swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query swarm/storage/mru: renamed query receiver var swarm/cmd: MRU CLI tests * cmd/swarm: remove rogue fmt * swarm/storage/mru: Add version / header for future use * swarm/storage/mru: Fixes/comments as per review cmd/swarm: remove rogue fmt swarm/storage/mru: Add version / header for future use- * swarm/storage/mru: fix linter errors * cmd/swarm: Speeded up TestCLIResourceUpdate --- swarm/api/client/client.go | 66 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 11 deletions(-) (limited to 'swarm/api/client/client.go') diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 3d06e9e1c..a6666144a 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -28,6 +28,7 @@ import ( "mime/multipart" "net/http" "net/textproto" + "net/url" "os" "path/filepath" "regexp" @@ -595,13 +596,16 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) return string(data), nil } +// ErrNoResourceUpdatesFound is returned when Swarm cannot find updates of the given resource +var ErrNoResourceUpdatesFound = errors.New("No updates found for this resource") + // CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided // data. Data is interpreted as multihash or not depending on the multihash parameter. // startTime=0 means "now" // Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent) // or reference future updates (Client.UpdateResource) func (c *Client) CreateResource(request *mru.Request) (string, error) { - responseStream, err := c.updateResource(request) + responseStream, err := c.updateResource(request, true) if err != nil { return "", err } @@ -621,17 +625,24 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) { // UpdateResource allows you to set a new version of your content func (c *Client) UpdateResource(request *mru.Request) error { - _, err := c.updateResource(request) + _, err := c.updateResource(request, false) return err } -func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) { - body, err := request.MarshalJSON() +func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.ReadCloser, error) { + URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } + URL.Path = "/bzz-resource:/" + values := URL.Query() + body := request.AppendValues(values) + if createManifest { + values.Set("manifest", "1") + } + URL.RawQuery = values.Encode() - req, err := http.NewRequest("POST", c.Gateway+"/bzz-resource:/", bytes.NewBuffer(body)) + req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body)) if err != nil { return nil, err } @@ -642,28 +653,61 @@ func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) { } return res.Body, nil - } // GetResource returns a byte stream with the raw content of the resource // manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver // points to that address -func (c *Client) GetResource(manifestAddressOrDomain string) (io.ReadCloser, error) { +func (c *Client) GetResource(query *mru.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { + return c.getResource(query, manifestAddressOrDomain, false) +} - res, err := http.Get(c.Gateway + "/bzz-resource:/" + manifestAddressOrDomain) +// getResource returns a byte stream with the raw content of the resource +// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// points to that address +// meta set to true will instruct the node return resource metainformation instead +func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { + URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } - return res.Body, nil + URL.Path = "/bzz-resource:/" + manifestAddressOrDomain + values := URL.Query() + if query != nil { + query.AppendValues(values) //adds query parameters + } + if meta { + values.Set("meta", "1") + } + URL.RawQuery = values.Encode() + res, err := http.Get(URL.String()) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + if res.StatusCode == http.StatusNotFound { + return nil, ErrNoResourceUpdatesFound + } + errorMessageBytes, err := ioutil.ReadAll(res.Body) + var errorMessage string + if err != nil { + errorMessage = "cannot retrieve error message: " + err.Error() + } else { + errorMessage = string(errorMessageBytes) + } + return nil, fmt.Errorf("Error retrieving resource: %s", errorMessage) + } + return res.Body, nil } // GetResourceMetadata returns a structure that describes the Mutable Resource // manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver // points to that address -func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Request, error) { +func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain string) (*mru.Request, error) { - responseStream, err := c.GetResource(manifestAddressOrDomain + "/meta") + responseStream, err := c.getResource(query, manifestAddressOrDomain, true) if err != nil { return nil, err } -- cgit