From 2c110c81ee92290d3e5ce6134a065c8d2abfbb60 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Fri, 28 Sep 2018 12:07:17 +0200 Subject: Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559) * swarm/storage/mru: Adaptive Frequency swarm/storage/mru/lookup: fixed getBaseTime Added NewEpoch constructor swarm/api/client: better error handling in GetResource() swarm/storage/mru: Renamed structures. Renamed ResourceMetadata to ResourceID. Renamed ResourceID.Name to ResourceID.Topic swarm/storage/mru: Added binarySerializer interface and test tools swarm/storage/mru/lookup: Changed base time to time and + marshallers swarm/storage/mru: Added ResourceID (former resourceMetadata) swarm/storage/mru: Added ResourceViewId and serialization tests swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals swarm/storage/mru: Fixes as per review comments cmd/swarm: reworded resource create/update help text regarding topic swarm/storage/mru: Added UpdateLookup and serializer tests swarm/storage/mru: Added UpdateHeader, serializers and tests swarm/storage/mru: changed UpdateAddr / epoch to Base() swarm/storage/mru: Added resourceUpdate serializer and tests swarm/storage/mru: Added SignedResourceUpdate tests and serializers swarm/storage/mru/lookup: fixed GetFirstEpoch bug swarm/storage/mru: refactor, comments, cleanup Also added tests for Topic swarm/storage/mru: handler tests pass swarm/storage/mru: all resource package tests pass swarm/storage/mru: resource test pass after adding timestamp checking support swarm/storage/mru: Added JSON serializers to ResourceIDView structures swarm/storage/mru: Sever, client, API test pass swarm/storage/mru: server test pass swarm/storage/mru: Added topic length check swarm/storage/mru: removed some literals, improved "previous lookup" test case swarm/storage/mru: some fixes and comments as per review swarm/storage/mru: first working version without metadata chunk swarm/storage/mru: Various fixes as per review swarm/storage/mru: client test pass swarm/storage/mru: resource query strings and manifest-less queries swarm/storage/mru: simplify naming swarm/storage/mru: first autofreq working version swarm/storage/mru: renamed ToValues to AppendValues swarm/resource/mru: Added ToValues / FromValues for URL query strings swarm/storage/mru: Changed POST resource to work with query strings. No more JSON. swarm/storage/mru: removed resourceid swarm/storage/mru: Opened up structures swarm/storage/mru: Merged Request and SignedResourceUpdate swarm/storage/mru: removed initial data from CLI resource create swarm/storage/mru: Refactor Topic as a direct fixed-length array swarm/storage/mru/lookup: Comprehensive GetNextLevel tests swarm/storage/mru: Added comments Added length checks in Topic swarm/storage/mru: fixes in tests and some code comments swarm/storage/mru/lookup: new optimized lookup algorithm swarm/api: moved getResourceView to api out of server swarm/storage/mru: Lookup algorithm working swarm/storage/mru: comments and renamed NewLookupParams Deleted commented code swarm/storage/mru/lookup: renamed Epoch.LaterThan to After swarm/storage/mru/lookup: Comments and tidying naming swarm/storage/mru: fix lookup algorithm swarm/storage/mru: exposed lookup hint removed updateheader swarm/storage/mru/lookup: changed GetNextEpoch for initial values swarm/storage/mru: resource tests pass swarm/storage/mru: valueSerializer interface and tests swarm/storage/mru/lookup: Comments, improvements, fixes, more tests swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query swarm/storage/mru: renamed query receiver var swarm/cmd: MRU CLI tests * cmd/swarm: remove rogue fmt * swarm/storage/mru: Add version / header for future use * swarm/storage/mru: Fixes/comments as per review cmd/swarm: remove rogue fmt swarm/storage/mru: Add version / header for future use- * swarm/storage/mru: fix linter errors * cmd/swarm: Speeded up TestCLIResourceUpdate --- swarm/api/http/server.go | 156 ++++++++++++++++++----------------------------- 1 file changed, 58 insertions(+), 98 deletions(-) (limited to 'swarm/api/http/server.go') diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index af1269b93..87ef05baa 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -487,6 +487,7 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { // The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) + uri := GetURI(r.Context()) log.Debug("handle.post.resource", "ruid", ruid) var err error @@ -496,9 +497,24 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { RespondError(w, r, err.Error(), http.StatusInternalServerError) return } + + view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query()) + if err != nil { // couldn't parse query string or retrieve manifest + getFail.Inc(1) + httpStatus := http.StatusBadRequest + if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI { + httpStatus = http.StatusNotFound + } + RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus) + return + } + var updateRequest mru.Request - if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON - RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error + updateRequest.View = *view + query := r.URL.Query() + + if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters + RespondError(w, r, err.Error(), http.StatusBadRequest) return } @@ -510,56 +526,40 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { RespondError(w, r, err.Error(), http.StatusForbidden) return } - } - - if updateRequest.IsNew() { - err = s.api.ResourceCreate(r.Context(), &updateRequest) - if err != nil { - code, err2 := s.translateResourceError(w, r, "resource creation fail", err) - RespondError(w, r, err2.Error(), code) - return - } - } - - if updateRequest.IsUpdate() { - _, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate) + _, err = s.api.ResourceUpdate(r.Context(), &updateRequest) if err != nil { RespondError(w, r, err.Error(), http.StatusInternalServerError) return } } - // at this point both possible operations (create, update or both) were successful - // so in case it was a new resource, then create a manifest and send it over. - - if updateRequest.IsNew() { + if query.Get("manifest") == "1" { // we create a manifest so we can retrieve the resource with bzz:// later - // this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource - // metadata chunk (rootAddr) - m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex()) + // this manifest has a special "resource type" manifest, and saves the + // resource view ID used to retrieve the resource later + m, err := s.api.NewResourceManifest(r.Context(), &updateRequest.View) if err != nil { RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) return } - // the key to the manifest will be passed back to the client - // the client can access the root chunk key directly through its Hash member - // the manifest key should be set as content in the resolver of the ENS name - // \TODO update manifest key automatically in ENS + // the client can access the view directly through its resourceView member + // the manifest key can be set as content in the resolver of the ENS name outdata, err := json.Marshal(m) if err != nil { RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) return } fmt.Fprint(w, string(outdata)) + + w.Header().Add("Content-type", "application/json") } - w.Header().Add("Content-type", "application/json") } // Retrieve mutable resource updates: // bzz-resource:// - get latest update -// bzz-resource:/// - get latest update on period n -// bzz-resource://// - get update version m of period n +// bzz-resource:///?period=n - get latest update on period n +// bzz-resource:///?period=n&version=m - get update version m of period n // bzz-resource:///meta - get metadata and next version information // = ens name or hash // TODO: Enable pass maxPeriod parameter @@ -569,84 +569,44 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { log.Debug("handle.get.resource", "ruid", ruid) var err error - // resolve the content key. - manifestAddr := uri.Address() - if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) - return - } - } else { - w.Header().Set("Cache-Control", "max-age=2147483648") - } - - // get the root chunk rootAddr from the manifest - rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr) - if err != nil { + view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query()) + if err != nil { // couldn't parse query string or retrieve manifest getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound) + httpStatus := http.StatusBadRequest + if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI { + httpStatus = http.StatusNotFound + } + RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus) return } - log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr) - // determine if the query specifies period and version or it is a metadata query - var params []string - if len(uri.Path) > 0 { - if uri.Path == "meta" { - unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound) - return - } - rawResponse, err := unsignedUpdateRequest.MarshalJSON() - if err != nil { - RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) - return - } - w.Header().Add("Content-type", "application/json") - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, string(rawResponse)) - return - - } - - params = strings.Split(uri.Path, "/") - - } - var name string - var data []byte - now := time.Now() - - switch len(params) { - case 0: // latest only - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatest(rootAddr)) - case 2: // specific period and version - var version uint64 - var period uint64 - version, err = strconv.ParseUint(params[1], 10, 32) + if r.URL.Query().Get("meta") == "1" { + unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), view) if err != nil { - break - } - period, err = strconv.ParseUint(params[0], 10, 32) - if err != nil { - break + getFail.Inc(1) + RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for view=%s: %s", view.Hex(), err), http.StatusNotFound) + return } - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupVersion(rootAddr, uint32(period), uint32(version))) - case 1: // last version of specific period - var period uint64 - period, err = strconv.ParseUint(params[0], 10, 32) + rawResponse, err := unsignedUpdateRequest.MarshalJSON() if err != nil { - break + RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) + return } - name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatestVersionInPeriod(rootAddr, uint32(period))) - default: // bogus - err = mru.NewError(storage.ErrInvalidValue, "invalid mutable resource request") + w.Header().Add("Content-type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, string(rawResponse)) + return } + lookupParams := &mru.Query{View: *view} + if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version + RespondError(w, r, fmt.Sprintf("invalid mutable resource request:%s", err), http.StatusBadRequest) + return + } + + data, err := s.api.ResourceLookup(r.Context(), lookupParams) + // any error from the switch statement will end up here if err != nil { code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err) @@ -655,9 +615,9 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { } // All ok, serve the retrieved update - log.Debug("Found update", "name", name, "ruid", ruid) + log.Debug("Found update", "view", view.Hex(), "ruid", ruid) w.Header().Set("Content-Type", "application/octet-stream") - http.ServeContent(w, r, "", now, bytes.NewReader(data)) + http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data)) } func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { -- cgit