From f7d3678c28c4b92e45a458e4785bd0f1cdc20e34 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 12 Jul 2018 15:42:45 +0200 Subject: swarm/api/http: http package refactoring 1/5 and 2/5 (#17164) --- swarm/api/api.go | 259 ++++++++++++++++-- swarm/api/client/client_test.go | 2 +- swarm/api/http/error_test.go | 1 - swarm/api/http/server.go | 565 +++++++++++++++------------------------- swarm/api/http/server_test.go | 126 ++++++++- 5 files changed, 576 insertions(+), 377 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index efc03d139..ff29877ff 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "archive/tar" "context" "fmt" "io" @@ -41,22 +42,32 @@ import ( ) var ( - apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) - apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) - apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) - apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) - apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) - apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) - apiGetHTTP300 = metrics.NewRegisteredCounter("api.get.http.300", nil) - apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) - apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) - apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) - apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) - apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) - apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) - apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) - apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) - apiGetInvalid = metrics.NewRegisteredCounter("api.get.invalid", nil) + apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) + apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) + apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) + apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) + apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) + apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) + apiGetHTTP300 = metrics.NewRegisteredCounter("api.get.http.300", nil) + apiManifestUpdateCount = metrics.NewRegisteredCounter("api.manifestupdate.count", nil) + apiManifestUpdateFail = metrics.NewRegisteredCounter("api.manifestupdate.fail", nil) + apiManifestListCount = metrics.NewRegisteredCounter("api.manifestlist.count", nil) + apiManifestListFail = metrics.NewRegisteredCounter("api.manifestlist.fail", nil) + apiDeleteCount = metrics.NewRegisteredCounter("api.delete.count", nil) + apiDeleteFail = metrics.NewRegisteredCounter("api.delete.fail", nil) + apiGetTarCount = metrics.NewRegisteredCounter("api.gettar.count", nil) + apiGetTarFail = metrics.NewRegisteredCounter("api.gettar.fail", nil) + apiUploadTarCount = metrics.NewRegisteredCounter("api.uploadtar.count", nil) + apiUploadTarFail = metrics.NewRegisteredCounter("api.uploadtar.fail", nil) + apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) + apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) + apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) + apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) + apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) + apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) + apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) + apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) + apiGetInvalid = metrics.NewRegisteredCounter("api.get.invalid", nil) ) // Resolver interface resolve a domain name to a hash using ENS @@ -424,6 +435,185 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string return } +func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Address, error) { + apiDeleteCount.Inc(1) + uri, err := Parse("bzz:/" + addr) + if err != nil { + apiDeleteFail.Inc(1) + return nil, err + } + key, err := a.Resolve(ctx, uri) + + if err != nil { + return nil, err + } + newKey, err := a.UpdateManifest(ctx, key, func(mw *ManifestWriter) error { + log.Debug(fmt.Sprintf("removing %s from manifest %s", path, key.Log())) + return mw.RemoveEntry(path) + }) + if err != nil { + apiDeleteFail.Inc(1) + return nil, err + } + + return newKey, nil +} + +// GetDirectoryTar fetches a requested directory as a tarstream +// it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser +func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) { + apiGetTarCount.Inc(1) + addr, err := a.Resolve(ctx, uri) + if err != nil { + return nil, err + } + walker, err := a.NewManifestWalker(ctx, addr, nil) + if err != nil { + apiGetTarFail.Inc(1) + return nil, err + } + + piper, pipew := io.Pipe() + + tw := tar.NewWriter(pipew) + + go func() { + err := walker.Walk(func(entry *ManifestEntry) error { + // ignore manifests (walk will recurse into them) + if entry.ContentType == ManifestType { + return nil + } + + // retrieve the entry's key and size + reader, _ := a.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash))) + size, err := reader.Size(nil) + if err != nil { + return err + } + + // write a tar header for the entry + hdr := &tar.Header{ + Name: entry.Path, + Mode: entry.Mode, + Size: size, + ModTime: entry.ModTime, + Xattrs: map[string]string{ + "user.swarm.content-type": entry.ContentType, + }, + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + // copy the file into the tar stream + n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) + if err != nil { + return err + } else if n != size { + return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) + } + + return nil + }) + if err != nil { + apiGetTarFail.Inc(1) + pipew.CloseWithError(err) + } else { + pipew.Close() + } + }() + + return piper, nil +} + +// GetManifestList lists the manifest entries for the specified address and prefix +// and returns it as a ManifestList +func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) { + apiManifestListCount.Inc(1) + walker, err := a.NewManifestWalker(ctx, addr, nil) + if err != nil { + apiManifestListFail.Inc(1) + return ManifestList{}, err + } + + err = walker.Walk(func(entry *ManifestEntry) error { + // handle non-manifest files + if entry.ContentType != ManifestType { + // ignore the file if it doesn't have the specified prefix + if !strings.HasPrefix(entry.Path, prefix) { + return nil + } + + // if the path after the prefix contains a slash, add a + // common prefix to the list, otherwise add the entry + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return nil + } + if entry.Path == "" { + entry.Path = "/" + } + list.Entries = append(list.Entries, entry) + return nil + } + + // if the manifest's path is a prefix of the specified prefix + // then just recurse into the manifest by returning nil and + // continuing the walk + if strings.HasPrefix(prefix, entry.Path) { + return nil + } + + // if the manifest's path has the specified prefix, then if the + // path after the prefix contains a slash, add a common prefix + // to the list and skip the manifest, otherwise recurse into + // the manifest by returning nil and continuing the walk + if strings.HasPrefix(entry.Path, prefix) { + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return ErrSkipManifest + } + return nil + } + + // the manifest neither has the prefix or needs recursing in to + // so just skip it + return ErrSkipManifest + }) + + if err != nil { + apiManifestListFail.Inc(1) + return ManifestList{}, err + } + + return list, nil +} + +func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update func(mw *ManifestWriter) error) (storage.Address, error) { + apiManifestUpdateCount.Inc(1) + mw, err := a.NewManifestWriter(ctx, addr, nil) + if err != nil { + apiManifestUpdateFail.Inc(1) + return nil, err + } + + if err := update(mw); err != nil { + apiManifestUpdateFail.Inc(1) + return nil, err + } + + addr, err = mw.Store() + if err != nil { + apiManifestUpdateFail.Inc(1) + return nil, err + } + log.Debug(fmt.Sprintf("generated manifest %s", addr)) + return addr, nil +} + // Modify loads manifest and checks the content hash before recalculating and storing the manifest. func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) { apiModifyCount.Inc(1) @@ -501,6 +691,43 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] return fkey, newMkey.String(), nil } +func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath string, mw *ManifestWriter) (storage.Address, error) { + apiUploadTarCount.Inc(1) + var contentKey storage.Address + tr := tar.NewReader(bodyReader) + defer bodyReader.Close() + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + apiUploadTarFail.Inc(1) + return nil, fmt.Errorf("error reading tar stream: %s", err) + } + + // only store regular files + if !hdr.FileInfo().Mode().IsRegular() { + continue + } + + // add the entry under the path from the request + manifestPath := path.Join(manifestPath, hdr.Name) + entry := &ManifestEntry{ + Path: manifestPath, + ContentType: hdr.Xattrs["user.swarm.content-type"], + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + contentKey, err = mw.AddEntry(ctx, tr, entry) + if err != nil { + apiUploadTarFail.Inc(1) + return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err) + } + } + return contentKey, nil +} + // RemoveFile removes a file entry in a manifest. func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname string, nameresolver bool) (string, error) { apiRmFileCount.Inc(1) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index a878bff17..e68147ab2 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -31,7 +31,7 @@ import ( ) func serverFunc(api *api.API) testutil.TestServer { - return swarmhttp.NewServer(api) + return swarmhttp.NewServer(api, "") } // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go index 86eff86b2..990961f60 100644 --- a/swarm/api/http/error_test.go +++ b/swarm/api/http/error_test.go @@ -29,7 +29,6 @@ import ( ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) defer srv.Close() diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 5897a1cb9..22c091026 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -20,10 +20,8 @@ A simple http server interface to Swarm package http import ( - "archive/tar" "bufio" "bytes" - "context" "encoding/json" "errors" "fmt" @@ -68,46 +66,185 @@ var ( getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil) getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil) getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil) - getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil) - getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil) getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil) getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) ) -// ServerConfig is the basic configuration needed for the HTTP server and also -// includes CORS settings. -type ServerConfig struct { - Addr string - CorsString string -} - -// browser API for registering bzz url scheme handlers: -// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers -// electron (chromium) api for registering bzz url scheme handlers: -// https://github.com/atom/electron/blob/master/docs/api/protocol.md - -// starts up http server -func StartHTTPServer(api *api.API, config *ServerConfig) { +func NewServer(api *api.API, corsString string) *Server { var allowedOrigins []string - for _, domain := range strings.Split(config.CorsString, ",") { + for _, domain := range strings.Split(corsString, ",") { allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) } c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, - AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"}, + AllowedMethods: []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodPatch, http.MethodPut}, MaxAge: 600, AllowedHeaders: []string{"*"}, }) - hdlr := c.Handler(NewServer(api)) - go http.ListenAndServe(config.Addr, hdlr) + mux := http.NewServeMux() + server := &Server{api: api} + mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz)) + mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw)) + mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable)) + mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash)) + mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList)) + mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource)) + + mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths)) + mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths)) + mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths)) + + server.Handler = c.Handler(mux) + return server +} +func (s *Server) ListenAndServe(addr string) error { + return http.ListenAndServe(addr, s) +} +func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + if r.RequestURI == "/" { + if strings.Contains(r.Header.Get("Accept"), "text/html") { + err := landingPageTemplate.Execute(w, nil) + if err != nil { + log.Error(fmt.Sprintf("error rendering landing page: %s", err)) + } + return + } + if strings.Contains(r.Header.Get("Accept"), "application/json") { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode("Welcome to Swarm!") + return + } + } + + if r.URL.Path == "/robots.txt" { + w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) + fmt.Fprintf(w, "User-agent: *\nDisallow: /") + return + } + Respond(w, r, "Bad Request", http.StatusBadRequest) + default: + Respond(w, r, "Not Found", http.StatusNotFound) + } +} +func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetBzz") + if r.Header.Get("Accept") == "application/x-tar" { + reader, err := s.api.GetDirectoryTar(r.Context(), r.uri) + if err != nil { + Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + } + defer reader.Close() + + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(http.StatusOK) + io.Copy(w, reader) + return + } + s.HandleGetFile(w, r) + case http.MethodPost: + log.Debug("handlePostFiles") + s.HandlePostFiles(w, r) + case http.MethodDelete: + log.Debug("handleBzzDelete") + s.HandleDelete(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } +} +func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetRaw") + s.HandleGet(w, r) + case http.MethodPost: + log.Debug("handlePostRaw") + s.HandlePostRaw(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } +} +func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetHash") + s.HandleGetList(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } +} +func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetHash") + s.HandleGet(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } } +func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetHash") + s.HandleGetList(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } +} +func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) { + switch r.Method { + case http.MethodGet: + log.Debug("handleGetResource") + s.HandleGetResource(w, r) + case http.MethodPost: + log.Debug("handlePostResource") + s.HandlePostResource(w, r) + default: + Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed) + } +} +func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now()) + req := &Request{Request: *r, ruid: uuid.New()[:8]} + metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1) + log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI) + + // wrapping the ResponseWriter, so that we get the response code set by http.ServeContent + w := newLoggingResponseWriter(rw) + if parseBzzUri { + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + if err != nil { + Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) + return + } + req.uri = uri -func NewServer(api *api.API) *Server { - return &Server{api} + log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme) + } + + h(w, req) // call original + log.Info("served response", "ruid", req.ruid, "code", w.statusCode) + }) } +// browser API for registering bzz url scheme handlers: +// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers +// electron (chromium) api for registering bzz url scheme handlers: +// https://github.com/atom/electron/blob/master/docs/api/protocol.md + +// browser API for registering bzz url scheme handlers: +// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers +// electron (chromium) api for registering bzz url scheme handlers: +// https://github.com/atom/electron/blob/master/docs/api/protocol.md + type Server struct { + http.Handler api *api.API } @@ -121,7 +258,7 @@ type Request struct { // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage address as a text/plain response -func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { log.Debug("handle.post.raw", "ruid", r.ruid) postRawCount.Inc(1) @@ -148,7 +285,8 @@ func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Re Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } - addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt) + + addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -167,7 +305,7 @@ func (s *Server) HandlePostRaw(ctx context.Context, w http.ResponseWriter, r *Re // (either a tar archive or multipart form), adds those files either to an // existing manifest or to a new manifest under and returns the // resulting manifest hash as a text/plain response -func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { log.Debug("handle.post.files", "ruid", r.ruid) postFilesCount.Inc(1) @@ -185,7 +323,7 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r * var addr storage.Address if r.uri.Addr != "" && r.uri.Addr != "encrypt" { - addr, err = s.api.Resolve(ctx, r.uri) + addr, err = s.api.Resolve(r.Context(), r.uri) if err != nil { postFilesFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) @@ -193,7 +331,7 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r * } log.Debug("resolved key", "ruid", r.ruid, "key", addr) } else { - addr, err = s.api.NewManifest(ctx, toEncrypt) + addr, err = s.api.NewManifest(r.Context(), toEncrypt) if err != nil { postFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -202,17 +340,21 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r * log.Debug("new manifest", "ruid", r.ruid, "key", addr) } - newAddr, err := s.updateManifest(ctx, addr, func(mw *api.ManifestWriter) error { + newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error { switch contentType { case "application/x-tar": - return s.handleTarUpload(ctx, r, mw) - + _, err := s.handleTarUpload(r, mw) + if err != nil { + Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError) + return err + } + return nil case "multipart/form-data": - return s.handleMultipartUpload(ctx, r, params["boundary"], mw) + return s.handleMultipartUpload(r, params["boundary"], mw) default: - return s.handleDirectUpload(ctx, r, mw) + return s.handleDirectUpload(r, mw) } }) if err != nil { @@ -228,41 +370,17 @@ func (s *Server) HandlePostFiles(ctx context.Context, w http.ResponseWriter, r * fmt.Fprint(w, newAddr) } -func (s *Server) handleTarUpload(ctx context.Context, req *Request, mw *api.ManifestWriter) error { - log.Debug("handle.tar.upload", "ruid", req.ruid) - tr := tar.NewReader(req.Body) - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil - } else if err != nil { - return fmt.Errorf("error reading tar stream: %s", err) - } - - // only store regular files - if !hdr.FileInfo().Mode().IsRegular() { - continue - } +func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) { + log.Debug("handle.tar.upload", "ruid", r.ruid) - // add the entry under the path from the request - path := path.Join(req.uri.Path, hdr.Name) - entry := &api.ManifestEntry{ - Path: path, - ContentType: hdr.Xattrs["user.swarm.content-type"], - Mode: hdr.Mode, - Size: hdr.Size, - ModTime: hdr.ModTime, - } - log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path) - contentKey, err := mw.AddEntry(ctx, tr, entry) - if err != nil { - return fmt.Errorf("error adding manifest entry from tar stream: %s", err) - } - log.Debug("stored content", "ruid", req.ruid, "key", contentKey) + key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw) + if err != nil { + return nil, err } + return key, nil } -func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, boundary string, mw *api.ManifestWriter) error { +func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error { log.Debug("handle.multipart.upload", "ruid", req.ruid) mr := multipart.NewReader(req.Body, boundary) for { @@ -312,7 +430,7 @@ func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, bounda ModTime: time.Now(), } log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path) - contentKey, err := mw.AddEntry(ctx, reader, entry) + contentKey, err := mw.AddEntry(req.Context(), reader, entry) if err != nil { return fmt.Errorf("error adding manifest entry from multipart form: %s", err) } @@ -320,9 +438,9 @@ func (s *Server) handleMultipartUpload(ctx context.Context, req *Request, bounda } } -func (s *Server) handleDirectUpload(ctx context.Context, req *Request, mw *api.ManifestWriter) error { +func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error { log.Debug("handle.direct.upload", "ruid", req.ruid) - key, err := mw.AddEntry(ctx, req.Body, &api.ManifestEntry{ + key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{ Path: req.uri.Path, ContentType: req.Header.Get("Content-Type"), Mode: 0644, @@ -339,24 +457,13 @@ func (s *Server) handleDirectUpload(ctx context.Context, req *Request, mw *api.M // HandleDelete handles a DELETE request to bzz://, removes // from and returns the resulting manifest hash as a // text/plain response -func (s *Server) HandleDelete(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { log.Debug("handle.delete", "ruid", r.ruid) - deleteCount.Inc(1) - key, err := s.api.Resolve(ctx, r.uri) - if err != nil { - deleteFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) - return - } - - newKey, err := s.updateManifest(ctx, key, func(mw *api.ManifestWriter) error { - log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log()), "ruid", r.ruid) - return mw.RemoveEntry(r.uri.Path) - }) + newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path) if err != nil { deleteFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot update manifest: %s", err), http.StatusInternalServerError) + Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError) return } @@ -400,7 +507,7 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { // The resource name will be verbatim what is passed as the address part of the url. // For example, if a POST is made to /bzz-resource:/foo.eth/raw/13 a new resource with frequency 13 // and name "foo.eth" will be created -func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { log.Debug("handle.post.resource", "ruid", r.ruid) var err error var addr storage.Address @@ -429,7 +536,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, // we create a manifest so we can retrieve the resource with bzz:// later // this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource // root chunk - m, err := s.api.NewResourceManifest(ctx, addr.Hex()) + m, err := s.api.NewResourceManifest(r.Context(), addr.Hex()) if err != nil { Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) return @@ -449,7 +556,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, // that means that we retrieve the manifest and inspect its Hash member. manifestAddr := r.uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(ctx, r.uri) + manifestAddr, err = s.api.Resolve(r.Context(), r.uri) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -460,7 +567,7 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, } // get the root chunk key from the manifest - addr, err = s.api.ResolveResourceManifest(ctx, manifestAddr) + addr, err = s.api.ResolveResourceManifest(r.Context(), manifestAddr) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -519,19 +626,15 @@ func (s *Server) HandlePostResource(ctx context.Context, w http.ResponseWriter, // bzz-resource:/// - get latest update on period n // bzz-resource://// - get update version m of period n // = ens name or hash -func (s *Server) HandleGetResource(ctx context.Context, w http.ResponseWriter, r *Request) { - s.handleGetResource(ctx, w, r) -} - // TODO: Enable pass maxPeriod parameter -func (s *Server) handleGetResource(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { log.Debug("handle.get.resource", "ruid", r.ruid) var err error // resolve the content key. manifestAddr := r.uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(ctx, r.uri) + manifestAddr, err = s.api.Resolve(r.Context(), r.uri) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -542,7 +645,7 @@ func (s *Server) handleGetResource(ctx context.Context, w http.ResponseWriter, r } // get the root chunk key from the manifest - key, err := s.api.ResolveResourceManifest(ctx, manifestAddr) + key, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -624,13 +727,13 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr // given storage key // - bzz-hash:// and responds with the hash of the content stored // at the given storage key as a text/plain response -func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri) getCount.Inc(1) var err error addr := r.uri.Address() if addr == nil { - addr, err = s.api.Resolve(ctx, r.uri) + addr, err = s.api.Resolve(r.Context(), r.uri) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -645,7 +748,7 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques // if path is set, interpret as a manifest and return the // raw entry at the given path if r.uri.Path != "" { - walker, err := s.api.NewManifestWalker(ctx, addr, nil) + walker, err := s.api.NewManifestWalker(r.Context(), addr, nil) if err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) @@ -693,7 +796,7 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques } // check the root chunk exists by retrieving the file's size - reader, isEncrypted := s.api.Retrieve(ctx, addr) + reader, isEncrypted := s.api.Retrieve(r.Context(), addr) if _, err := reader.Size(nil); err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) @@ -719,86 +822,10 @@ func (s *Server) HandleGet(ctx context.Context, w http.ResponseWriter, r *Reques } } -// HandleGetFiles handles a GET request to bzz:/ with an Accept -// header of "application/x-tar" and returns a tar stream of all files -// contained in the manifest -func (s *Server) HandleGetFiles(ctx context.Context, w http.ResponseWriter, r *Request) { - log.Debug("handle.get.files", "ruid", r.ruid, "uri", r.uri) - getFilesCount.Inc(1) - if r.uri.Path != "" { - getFilesFail.Inc(1) - Respond(w, r, "files request cannot contain a path", http.StatusBadRequest) - return - } - - addr, err := s.api.Resolve(ctx, r.uri) - if err != nil { - getFilesFail.Inc(1) - Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) - return - } - log.Debug("handle.get.files: resolved", "ruid", r.ruid, "key", addr) - - walker, err := s.api.NewManifestWalker(ctx, addr, nil) - if err != nil { - getFilesFail.Inc(1) - Respond(w, r, err.Error(), http.StatusInternalServerError) - return - } - - tw := tar.NewWriter(w) - defer tw.Close() - w.Header().Set("Content-Type", "application/x-tar") - w.WriteHeader(http.StatusOK) - - err = walker.Walk(func(entry *api.ManifestEntry) error { - // ignore manifests (walk will recurse into them) - if entry.ContentType == api.ManifestType { - return nil - } - - // retrieve the entry's key and size - reader, isEncrypted := s.api.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash))) - size, err := reader.Size(nil) - if err != nil { - return err - } - w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) - - // write a tar header for the entry - hdr := &tar.Header{ - Name: entry.Path, - Mode: entry.Mode, - Size: size, - ModTime: entry.ModTime, - Xattrs: map[string]string{ - "user.swarm.content-type": entry.ContentType, - }, - } - if err := tw.WriteHeader(hdr); err != nil { - return err - } - - // copy the file into the tar stream - n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) - if err != nil { - return err - } else if n != size { - return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) - } - - return nil - }) - if err != nil { - getFilesFail.Inc(1) - log.Error(fmt.Sprintf("error generating tar stream: %s", err)) - } -} - // HandleGetList handles a GET request to bzz-list:// and returns // a list of all files contained in under grouped into // common prefixes using "/" as a delimiter -func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri) getListCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work @@ -807,7 +834,7 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re return } - addr, err := s.api.Resolve(ctx, r.uri) + addr, err := s.api.Resolve(r.Context(), r.uri) if err != nil { getListFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -815,8 +842,7 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re } log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr) - list, err := s.getManifestList(ctx, addr, r.uri.Path) - + list, err := s.api.GetManifestList(r.Context(), addr, r.uri.Path) if err != nil { getListFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -846,65 +872,9 @@ func (s *Server) HandleGetList(ctx context.Context, w http.ResponseWriter, r *Re json.NewEncoder(w).Encode(&list) } -func (s *Server) getManifestList(ctx context.Context, addr storage.Address, prefix string) (list api.ManifestList, err error) { - walker, err := s.api.NewManifestWalker(ctx, addr, nil) - if err != nil { - return - } - - err = walker.Walk(func(entry *api.ManifestEntry) error { - // handle non-manifest files - if entry.ContentType != api.ManifestType { - // ignore the file if it doesn't have the specified prefix - if !strings.HasPrefix(entry.Path, prefix) { - return nil - } - - // if the path after the prefix contains a slash, add a - // common prefix to the list, otherwise add the entry - suffix := strings.TrimPrefix(entry.Path, prefix) - if index := strings.Index(suffix, "/"); index > -1 { - list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) - return nil - } - if entry.Path == "" { - entry.Path = "/" - } - list.Entries = append(list.Entries, entry) - return nil - } - - // if the manifest's path is a prefix of the specified prefix - // then just recurse into the manifest by returning nil and - // continuing the walk - if strings.HasPrefix(prefix, entry.Path) { - return nil - } - - // if the manifest's path has the specified prefix, then if the - // path after the prefix contains a slash, add a common prefix - // to the list and skip the manifest, otherwise recurse into - // the manifest by returning nil and continuing the walk - if strings.HasPrefix(entry.Path, prefix) { - suffix := strings.TrimPrefix(entry.Path, prefix) - if index := strings.Index(suffix, "/"); index > -1 { - list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) - return api.ErrSkipManifest - } - return nil - } - - // the manifest neither has the prefix or needs recursing in to - // so just skip it - return api.ErrSkipManifest - }) - - return list, nil -} - // HandleGetFile handles a GET request to bzz:/// and responds // with the content of the file at from the given -func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Request) { +func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { log.Debug("handle.get.file", "ruid", r.ruid) getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work @@ -916,7 +886,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re manifestAddr := r.uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(ctx, r.uri) + manifestAddr, err = s.api.Resolve(r.Context(), r.uri) if err != nil { getFileFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) @@ -927,8 +897,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re } log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr) - - reader, contentType, status, contentKey, err := s.api.Get(ctx, manifestAddr, r.uri.Path) + reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path) etag := common.Bytes2Hex(contentKey) noneMatchEtag := r.Header.Get("If-None-Match") @@ -955,8 +924,7 @@ func (s *Server) HandleGetFile(ctx context.Context, w http.ResponseWriter, r *Re //the request results in ambiguous files //e.g. /read with readme.md and readinglist.txt available in manifest if status == http.StatusMultipleChoices { - list, err := s.getManifestList(ctx, manifestAddr, r.uri.Path) - + list, err := s.api.GetManifestList(r.Context(), manifestAddr, r.uri.Path) if err != nil { getFileFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -1011,125 +979,6 @@ func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) { return b.s.Seek(offset, whence) } -func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - ctx := context.TODO() - - defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now()) - req := &Request{Request: *r, ruid: uuid.New()[:8]} - metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1) - log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI) - - // wrapping the ResponseWriter, so that we get the response code set by http.ServeContent - w := newLoggingResponseWriter(rw) - - if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") { - - err := landingPageTemplate.Execute(w, nil) - if err != nil { - log.Error(fmt.Sprintf("error rendering landing page: %s", err)) - } - return - } - - if r.URL.Path == "/robots.txt" { - w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) - fmt.Fprintf(w, "User-agent: *\nDisallow: /") - return - } - - if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "application/json") { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode("Welcome to Swarm!") - return - } - - uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) - if err != nil { - Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) - return - } - - req.uri = uri - - log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme) - - switch r.Method { - case "POST": - if uri.Raw() { - log.Debug("handlePostRaw") - s.HandlePostRaw(ctx, w, req) - } else if uri.Resource() { - log.Debug("handlePostResource") - s.HandlePostResource(ctx, w, req) - } else if uri.Immutable() || uri.List() || uri.Hash() { - log.Debug("POST not allowed on immutable, list or hash") - Respond(w, req, fmt.Sprintf("POST method on scheme %s not allowed", uri.Scheme), http.StatusMethodNotAllowed) - } else { - log.Debug("handlePostFiles") - s.HandlePostFiles(ctx, w, req) - } - - case "PUT": - Respond(w, req, fmt.Sprintf("PUT method to %s not allowed", uri), http.StatusBadRequest) - return - - case "DELETE": - if uri.Raw() { - Respond(w, req, fmt.Sprintf("DELETE method to %s not allowed", uri), http.StatusBadRequest) - return - } - s.HandleDelete(ctx, w, req) - - case "GET": - - if uri.Resource() { - s.HandleGetResource(ctx, w, req) - return - } - - if uri.Raw() || uri.Hash() { - s.HandleGet(ctx, w, req) - return - } - - if uri.List() { - s.HandleGetList(ctx, w, req) - return - } - - if r.Header.Get("Accept") == "application/x-tar" { - s.HandleGetFiles(ctx, w, req) - return - } - - s.HandleGetFile(ctx, w, req) - - default: - Respond(w, req, fmt.Sprintf("%s method is not supported", r.Method), http.StatusMethodNotAllowed) - } - - log.Info("served response", "ruid", req.ruid, "code", w.statusCode) -} - -func (s *Server) updateManifest(ctx context.Context, addr storage.Address, update func(mw *api.ManifestWriter) error) (storage.Address, error) { - mw, err := s.api.NewManifestWriter(ctx, addr, nil) - if err != nil { - return nil, err - } - - if err := update(mw); err != nil { - return nil, err - } - - addr, err = mw.Store() - if err != nil { - return nil, err - } - log.Debug(fmt.Sprintf("generated manifest %s", addr)) - return addr, nil -} - type loggingResponseWriter struct { http.ResponseWriter statusCode int diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index bfbc0a79d..8e1c26a33 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -17,6 +17,7 @@ package http import ( + "archive/tar" "bytes" "context" "crypto/rand" @@ -24,11 +25,13 @@ import ( "errors" "flag" "fmt" + "io" "io/ioutil" "net/http" "os" "strings" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -88,7 +91,7 @@ func TestResourcePostMode(t *testing.T) { } func serverFunc(api *api.API) testutil.TestServer { - return NewServer(api) + return NewServer(api, "") } // test the transparent resolving of multihash resource types with bzz:// scheme @@ -356,6 +359,11 @@ func TestBzzGetPath(t *testing.T) { testBzzGetPath(true, t) } +func TestBzzTar(t *testing.T) { + testBzzTar(false, t) + testBzzTar(true, t) +} + func testBzzGetPath(encrypted bool, t *testing.T) { var err error @@ -592,6 +600,122 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } } +func testBzzTar(encrypted bool, t *testing.T) { + srv := testutil.NewTestSwarmServer(t, serverFunc) + defer srv.Close() + fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} + fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} + + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + + for i, v := range fileNames { + size := int64(len(fileContents[i])) + hdr := &tar.Header{ + Name: v, + Mode: 0644, + Size: size, + ModTime: time.Now(), + Xattrs: map[string]string{ + "user.swarm.content-type": "text/plain", + }, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + // copy the file into the tar stream + n, err := io.Copy(tw, bytes.NewBufferString(fileContents[i])) + if err != nil { + t.Fatal(err) + } else if n != size { + t.Fatal("size mismatch") + } + } + + //post tar stream + url := srv.URL + "/bzz:/" + if encrypted { + url = url + "encrypt" + } + req, err := http.NewRequest("POST", url, buf) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/x-tar") + client := &http.Client{} + resp2, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + if resp2.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp2.Status) + } + swarmHash, err := ioutil.ReadAll(resp2.Body) + resp2.Body.Close() + t.Logf("uploaded tarball successfully and got manifest address at %s", string(swarmHash)) + if err != nil { + t.Fatal(err) + } + + // now do a GET to get a tarball back + req, err = http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s", string(swarmHash)), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Accept", "application/x-tar") + resp2, err = client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp2.Body.Close() + + file, err := ioutil.TempFile("", "swarm-downloaded-tarball") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + _, err = io.Copy(file, resp2.Body) + if err != nil { + t.Fatalf("error getting tarball: %v", err) + } + file.Sync() + file.Close() + + tarFileHandle, err := os.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(tarFileHandle) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + t.Fatalf("error reading tar stream: %s", err) + } + bb := make([]byte, hdr.Size) + _, err = tr.Read(bb) + if err != nil && err != io.EOF { + t.Fatal(err) + } + passed := false + for i, v := range fileNames { + if v == hdr.Name { + if string(bb) == fileContents[i] { + passed = true + break + } + } + } + if !passed { + t.Fatalf("file %s did not pass content assertion", hdr.Name) + } + } +} + // TestBzzRootRedirect tests that getting the root path of a manifest without // a trailing slash gets redirected to include the trailing slash so that // relative URLs work as expected. -- cgit