diff options
Diffstat (limited to 'swarm/api')
| -rw-r--r-- | swarm/api/api.go | 104 | ||||
| -rw-r--r-- | swarm/api/api_test.go | 10 | ||||
| -rw-r--r-- | swarm/api/client/client.go | 551 | ||||
| -rw-r--r-- | swarm/api/client/client_test.go | 260 | ||||
| -rw-r--r-- | swarm/api/filesystem.go | 27 | ||||
| -rw-r--r-- | swarm/api/filesystem_test.go | 32 | ||||
| -rw-r--r-- | swarm/api/http/roundtripper_test.go | 11 | ||||
| -rw-r--r-- | swarm/api/http/server.go | 757 | ||||
| -rw-r--r-- | swarm/api/http/server_test.go | 4 | ||||
| -rw-r--r-- | swarm/api/http/templates.go | 71 | ||||
| -rw-r--r-- | swarm/api/manifest.go | 170 | ||||
| -rw-r--r-- | swarm/api/storage.go | 40 | ||||
| -rw-r--r-- | swarm/api/storage_test.go | 2 | ||||
| -rw-r--r-- | swarm/api/swarmfs_unix.go | 9 | ||||
| -rw-r--r-- | swarm/api/uri.go | 96 | ||||
| -rw-r--r-- | swarm/api/uri_test.go | 120 |
16 files changed, 1686 insertions, 578 deletions
diff --git a/swarm/api/api.go b/swarm/api/api.go index 7af27208d..ba1156f7e 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "errors" "fmt" "io" "net/http" @@ -70,86 +71,50 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor type ErrResolve error // DNS Resolver -func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { - log.Trace(fmt.Sprintf("Resolving : %v", hostPort)) - if hashMatcher.MatchString(hostPort) || self.dns == nil { - log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort)) - return storage.Key(common.Hex2Bytes(hostPort)), nil +func (self *Api) Resolve(uri *URI) (storage.Key, error) { + log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) + if hashMatcher.MatchString(uri.Addr) { + log.Trace(fmt.Sprintf("addr is a hash: %q", uri.Addr)) + return storage.Key(common.Hex2Bytes(uri.Addr)), nil } - if !nameresolver { - return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort) + if uri.Immutable() { + return nil, errors.New("refusing to resolve immutable address") } - contentHash, err := self.dns.Resolve(hostPort) - if err != nil { - err = ErrResolve(err) - log.Warn(fmt.Sprintf("DNS error : %v", err)) - } - log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash)) - return contentHash[:], err -} -func Parse(uri string) (hostPort, path string) { - if uri == "" { - return - } - parts := slashes.Split(uri, 3) - var i int - if len(parts) == 0 { - return + if self.dns == nil { + return nil, fmt.Errorf("unable to resolve addr %q, resolver not configured", uri.Addr) } - // beginning with slash is now optional - for len(parts[i]) == 0 { - i++ - } - hostPort = parts[i] - for i < len(parts)-1 { - i++ - if len(path) > 0 { - path = path + "/" + parts[i] - } else { - path = parts[i] - } + hash, err := self.dns.Resolve(uri.Addr) + if err != nil { + log.Warn(fmt.Sprintf("DNS error resolving addr %q: %s", uri.Addr, err)) + return nil, ErrResolve(err) } - log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path)) - return -} - -func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) { - hostPort, path = Parse(uri) - //resolving host and port - contentHash, err := self.Resolve(hostPort, nameresolver) - log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)) - return contentHash[:], hostPort, path, err + log.Trace(fmt.Sprintf("addr lookup: %v -> %v", uri.Addr, hash)) + return hash[:], nil } // Put provides singleton manifest creation on top of dpa store -func (self *Api) Put(content, contentType string) (string, error) { +func (self *Api) Put(content, contentType string) (storage.Key, error) { r := strings.NewReader(content) wg := &sync.WaitGroup{} key, err := self.dpa.Store(r, int64(len(content)), wg, nil) if err != nil { - return "", err + return nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) if err != nil { - return "", err + return nil, err } wg.Wait() - return key.String(), nil + return key, nil } // Get uses iterative manifest retrieval and prefix matching // to resolve path to content using dpa retrieve // it returns a section reader, mimeType, status and an error -func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) { - key, _, path, err := self.parseAndResolve(uri, nameresolver) - if err != nil { - return nil, "", 500, fmt.Errorf("can't resolve: %v", err) - } - - quitC := make(chan bool) - trie, err := loadManifest(self.dpa, key, quitC) +func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + trie, err := loadManifest(self.dpa, key, nil) if err != nil { log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -173,32 +138,25 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR return } -func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { - root, _, path, err := self.parseAndResolve(uri, nameresolver) - if err != nil { - return "", fmt.Errorf("can't resolve: %v", err) - } - +func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { quitC := make(chan bool) - trie, err := loadManifest(self.dpa, root, quitC) + trie, err := loadManifest(self.dpa, key, quitC) if err != nil { - return + return nil, err } - if contentHash != "" { - entry := &manifestTrieEntry{ + entry := newManifestTrieEntry(&ManifestEntry{ Path: path, - Hash: contentHash, ContentType: contentType, - } + }, nil) + entry.Hash = contentHash trie.addEntry(entry, quitC) } else { trie.deleteEntry(path, quitC) } - err = trie.recalcAndStore() - if err != nil { - return + if err := trie.recalcAndStore(); err != nil { + return nil, err } - return trie.hash.String(), nil + return trie.hash, nil } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 16e90dd32..c2d78c2dc 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -23,6 +23,7 @@ import ( "os" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response { } // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { -func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { - reader, mimeType, status, err := api.Get(bzzhash, true) +func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse { + key := storage.Key(common.Hex2Bytes(bzzhash)) + reader, mimeType, status, err := api.Get(key, path) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - bzzhash, err := api.Put(content, exp.MimeType) + key, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, key.String(), "") checkResponse(t, resp, exp) }) } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index ef5335be3..f9c3e51e8 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -17,18 +17,23 @@ package client import ( + "archive/tar" "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" "mime" + "mime/multipart" "net/http" + "net/textproto" "os" "path/filepath" + "strconv" "strings" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" ) var ( @@ -36,18 +41,6 @@ var ( DefaultClient = NewClient(DefaultGateway) ) -// Manifest represents a swarm manifest. -type Manifest struct { - Entries []ManifestEntry `json:"entries,omitempty"` -} - -// ManifestEntry represents an entry in a swarm manifest. -type ManifestEntry struct { - Hash string `json:"hash,omitempty"` - ContentType string `json:"contentType,omitempty"` - Path string `json:"path,omitempty"` -} - func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, @@ -59,160 +52,207 @@ type Client struct { Gateway string } -func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) { - mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}")))) - if err != nil { - return "", fmt.Errorf("failed to upload empty manifest") +// UploadRaw uploads raw data to swarm and returns the resulting hash +func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { + if size <= 0 { + return "", errors.New("data size must be greater than zero") } - if len(defaultPath) > 0 { - fi, err := os.Stat(defaultPath) - if err != nil { - return "", err - } - mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi) - if err != nil { - return "", err - } + req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r) + if err != nil { + return "", err } - prefix := filepath.ToSlash(filepath.Clean(dir)) + "/" - err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil || fi.IsDir() { - return err - } - if !strings.HasPrefix(path, dir) { - return fmt.Errorf("path %s outside directory %s", path, dir) - } - uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix) - mhash, err = c.uploadToManifest(mhash, uripath, path, fi) - return err - }) - return mhash, err -} - -func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) { - var mimetype string - hash, err := c.uploadFileContent(file, fi) - if mimetype_hint != "" { - mimetype = mimetype_hint - log.Info("Mime type set by override", "mime", mimetype) - } else { - ext := filepath.Ext(file) - log.Info("Ext", "ext", ext, "file", file) - if ext != "" { - mimetype = mime.TypeByExtension(filepath.Ext(fi.Name())) - log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file)) - } else { - f, err := os.Open(file) - if err == nil { - first512 := make([]byte, 512) - fread, _ := f.ReadAt(first512, 0) - if fread > 0 { - mimetype = http.DetectContentType(first512[:fread]) - log.Info("Mime type set by autodetection", "mime", mimetype) - } - } - f.Close() - } - + req.ContentLength = size + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err } - m := ManifestEntry{ - Hash: hash, - ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())), + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) } - return m, err -} - -func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) { - fd, err := os.Open(file) + data, err := ioutil.ReadAll(res.Body) if err != nil { return "", err } - defer fd.Close() - log.Info("Uploading swarm content", "file", file, "bytes", fi.Size()) - return c.postRaw("application/octet-stream", fi.Size(), fd) + return string(data), nil } -func (c *Client) UploadManifest(m Manifest) (string, error) { - jsm, err := json.Marshal(m) +// DownloadRaw downloads raw data from swarm +func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) { + uri := c.Gateway + "/bzzr:/" + hash + res, err := http.DefaultClient.Get(uri) if err != nil { - panic(err) + return nil, err + } + if res.StatusCode != http.StatusOK { + res.Body.Close() + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) } - log.Info("Uploading swarm manifest") - return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm))) + return res.Body, nil } -func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) { - fd, err := os.Open(fpath) - if err != nil { - return "", err - } - defer fd.Close() - log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path) - req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd) +// File represents a file in a swarm manifest and is used for uploading and +// downloading content to and from swarm +type File struct { + io.ReadCloser + api.ManifestEntry +} + +// Open opens a local file which can then be passed to client.Upload to upload +// it to swarm +func Open(path string) (*File, error) { + f, err := os.Open(path) if err != nil { - return "", err + return nil, err } - req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name()))) - req.ContentLength = fi.Size() - resp, err := http.DefaultClient.Do(req) + stat, err := f.Stat() if err != nil { - return "", err + f.Close() + return nil, err } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return "", fmt.Errorf("bad status: %s", resp.Status) + return &File{ + ReadCloser: f, + ManifestEntry: api.ManifestEntry{ + ContentType: mime.TypeByExtension(filepath.Ext(path)), + Mode: int64(stat.Mode()), + Size: stat.Size(), + ModTime: stat.ModTime(), + }, + }, nil +} + +// Upload uploads a file to swarm and either adds it to an existing manifest +// (if the manifest argument is non-empty) or creates a new manifest containing +// the file, returning the resulting manifest hash (the file will then be +// available at bzz:/<hash>/<path>) +func (c *Client) Upload(file *File, manifest string) (string, error) { + if file.Size <= 0 { + return "", errors.New("file size must be greater than zero") } - content, err := ioutil.ReadAll(resp.Body) - return string(content), err + return c.TarUpload(manifest, &FileUploader{file}) } -func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) { - req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body) +// Download downloads a file with the given path from the swarm manifest with +// the given hash (i.e. it gets bzz:/<hash>/<path>) +func (c *Client) Download(hash, path string) (*File, error) { + uri := c.Gateway + "/bzz:/" + hash + "/" + path + res, err := http.DefaultClient.Get(uri) if err != nil { - return "", err + return nil, err } - req.Header.Set("content-type", mimetype) - req.ContentLength = size - resp, err := http.DefaultClient.Do(req) + if res.StatusCode != http.StatusOK { + res.Body.Close() + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + return &File{ + ReadCloser: res.Body, + ManifestEntry: api.ManifestEntry{ + ContentType: res.Header.Get("Content-Type"), + Size: res.ContentLength, + }, + }, nil +} + +// UploadDirectory uploads a directory tree to swarm and either adds the files +// to an existing manifest (if the manifest argument is non-empty) or creates a +// new manifest, returning the resulting manifest hash (files from the +// directory will then be available at bzz:/<hash>/path/to/file), with +// the file specified in defaultPath being uploaded to the root of the manifest +// (i.e. bzz:/<hash>/) +func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) { + stat, err := os.Stat(dir) if err != nil { return "", err + } else if !stat.IsDir() { + return "", fmt.Errorf("not a directory: %s", dir) } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return "", fmt.Errorf("bad status: %s", resp.Status) - } - content, err := ioutil.ReadAll(resp.Body) - return string(content), err + return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}) } -func (c *Client) DownloadManifest(mhash string) (Manifest, error) { +// DownloadDirectory downloads the files contained in a swarm manifest under +// the given path into a local directory (existing files will be overwritten) +func (c *Client) DownloadDirectory(hash, path, destDir string) error { + stat, err := os.Stat(destDir) + if err != nil { + return err + } else if !stat.IsDir() { + return fmt.Errorf("not a directory: %s", destDir) + } - mroot := Manifest{} - req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil) + uri := c.Gateway + "/bzz:/" + hash + "/" + path + req, err := http.NewRequest("GET", uri, nil) if err != nil { - return mroot, err + return err } - resp, err := http.DefaultClient.Do(req) + req.Header.Set("Accept", "application/x-tar") + res, err := http.DefaultClient.Do(req) if err != nil { - return mroot, err + return err } - defer resp.Body.Close() + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + tr := tar.NewReader(res.Body) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } else if err != nil { + return err + } + // ignore the default path file + if hdr.Name == "" { + continue + } - if resp.StatusCode >= 400 { - return mroot, fmt.Errorf("bad status: %s", resp.Status) + dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path))) + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + var mode os.FileMode = 0644 + if hdr.Mode > 0 { + mode = os.FileMode(hdr.Mode) + } + dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + n, err := io.Copy(dst, tr) + dst.Close() + if err != nil { + return err + } else if n != hdr.Size { + return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n) + } + } +} +// UploadManifest uploads the given manifest to swarm +func (c *Client) UploadManifest(m *api.Manifest) (string, error) { + data, err := json.Marshal(m) + if err != nil { + return "", err } - content, err := ioutil.ReadAll(resp.Body) + return c.UploadRaw(bytes.NewReader(data), int64(len(data))) +} - err = json.Unmarshal(content, &mroot) +// DownloadManifest downloads a swarm manifest +func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) { + res, err := c.DownloadRaw(hash) if err != nil { - return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) + return nil, err + } + defer res.Close() + var manifest api.Manifest + if err := json.NewDecoder(res).Decode(&manifest); err != nil { + return nil, err } - return mroot, err + return &manifest, nil } -// ManifestFileList downloads the manifest with the given hash and generates a -// list of files and directory prefixes which have the specified prefix. +// List list files in a swarm manifest which have the given prefix, grouping +// common prefixes using "/" as a delimiter. // // For example, if the manifest represents the following directory structure: // @@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) { // - a prefix of "" would return [dir1/, file1.txt, file2.txt] // - a prefix of "file" would return [file1.txt, file2.txt] // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] -func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) { - manifest, err := c.DownloadManifest(hash) +// +// where entries ending with "/" are common prefixes. +func (c *Client) List(hash, prefix string) (*api.ManifestList, error) { + res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true") if err != nil { return nil, err } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + var list api.ManifestList + if err := json.NewDecoder(res.Body).Decode(&list); err != nil { + return nil, err + } + return &list, nil +} + +// Uploader uploads files to swarm using a provided UploadFn +type Uploader interface { + Upload(UploadFn) error +} + +type UploaderFunc func(UploadFn) error + +func (u UploaderFunc) Upload(upload UploadFn) error { + return u(upload) +} + +// DirectoryUploader uploads all files in a directory, optionally uploading +// a file to the default path +type DirectoryUploader struct { + Dir string + DefaultPath string +} - // handleFile handles a manifest entry which is a direct reference to a - // file (i.e. it is not a swarm manifest) - handleFile := func(entry ManifestEntry) { - // ignore the file if it doesn't have the specified prefix - if !strings.HasPrefix(entry.Path, prefix) { - return +// Upload performs the upload of the directory and default path +func (d *DirectoryUploader) Upload(upload UploadFn) error { + if d.DefaultPath != "" { + file, err := Open(d.DefaultPath) + if err != nil { + return err } - // if the path after the prefix contains a directory separator, - // add a directory prefix to the entries, otherwise add the - // file - suffix := strings.TrimPrefix(entry.Path, prefix) - if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 { - entries = append(entries, ManifestEntry{ - Path: prefix + suffix[:sepIndex+1], - ContentType: "DIR", - }) - } else { - if entry.Path == "" { - entry.Path = "/" - } - entries = append(entries, entry) + if err := upload(file); err != nil { + return err } } - - // handleManifest handles a manifest entry which is a reference to - // another swarm manifest. - handleManifest := func(entry ManifestEntry) error { - // if the manifest's path is a prefix of the specified prefix - // then just recurse into the manifest by stripping its path - // from the prefix - if strings.HasPrefix(prefix, entry.Path) { - subPrefix := strings.TrimPrefix(prefix, entry.Path) - subEntries, err := c.ManifestFileList(entry.Hash, subPrefix) - if err != nil { - return err - } - // prefix the manifest's path to the sub entries and - // add them to the returned entries - for i, subEntry := range subEntries { - subEntry.Path = entry.Path + subEntry.Path - subEntries[i] = subEntry - } - entries = append(entries, subEntries...) + return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir() { return nil } + file, err := Open(path) + if err != nil { + return err + } + relPath, err := filepath.Rel(d.Dir, path) + if err != nil { + return err + } + file.Path = filepath.ToSlash(relPath) + return upload(file) + }) +} - // if the manifest's path has the specified prefix, then if the - // path after the prefix contains a directory separator, add a - // directory prefix to the entries, otherwise recurse into the - // manifest - if strings.HasPrefix(entry.Path, prefix) { - suffix := strings.TrimPrefix(entry.Path, prefix) - sepIndex := strings.Index(suffix, "/") - if sepIndex > -1 { - entries = append(entries, ManifestEntry{ - Path: prefix + suffix[:sepIndex+1], - ContentType: "DIR", - }) - return nil - } - subEntries, err := c.ManifestFileList(entry.Hash, "") - if err != nil { - return err - } - // prefix the manifest's path to the sub entries and - // add them to the returned entries - for i, subEntry := range subEntries { - subEntry.Path = entry.Path + subEntry.Path - subEntries[i] = subEntry - } - entries = append(entries, subEntries...) - return nil +// FileUploader uploads a single file +type FileUploader struct { + File *File +} + +// Upload performs the upload of the file +func (f *FileUploader) Upload(upload UploadFn) error { + return upload(f.File) +} + +// UploadFn is the type of function passed to an Uploader to perform the upload +// of a single file (for example, a directory uploader would call a provided +// UploadFn for each file in the directory tree) +type UploadFn func(file *File) error + +// TarUpload uses the given Uploader to upload files to swarm as a tar stream, +// returning the resulting manifest hash +func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) { + reqR, reqW := io.Pipe() + defer reqR.Close() + req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-tar") + + // use 'Expect: 100-continue' so we don't send the request body if + // the server refuses the request + req.Header.Set("Expect", "100-continue") + + tw := tar.NewWriter(reqW) + + // define an UploadFn which adds files to the tar stream + uploadFn := func(file *File) error { + hdr := &tar.Header{ + Name: file.Path, + Mode: file.Mode, + Size: file.Size, + ModTime: file.ModTime, + Xattrs: map[string]string{ + "user.swarm.content-type": file.ContentType, + }, + } + if err := tw.WriteHeader(hdr); err != nil { + return err } - return nil + _, err = io.Copy(tw, file) + return err } - for _, entry := range manifest.Entries { - if entry.ContentType == "application/bzz-manifest+json" { - if err := handleManifest(entry); err != nil { - return nil, err - } - } else { - handleFile(entry) + // run the upload in a goroutine so we can send the request headers and + // wait for a '100 Continue' response before sending the tar stream + go func() { + err := uploader.Upload(uploadFn) + if err == nil { + err = tw.Close() } + reqW.CloseWithError(err) + }() + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(data), nil +} + +// MultipartUpload uses the given Uploader to upload files to swarm as a +// multipart form, returning the resulting manifest hash +func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) { + reqR, reqW := io.Pipe() + defer reqR.Close() + req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) + if err != nil { + return "", err } - return + // use 'Expect: 100-continue' so we don't send the request body if + // the server refuses the request + req.Header.Set("Expect", "100-continue") + + mw := multipart.NewWriter(reqW) + req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) + + // define an UploadFn which adds files to the multipart form + uploadFn := func(file *File) error { + hdr := make(textproto.MIMEHeader) + hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path)) + hdr.Set("Content-Type", file.ContentType) + hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10)) + w, err := mw.CreatePart(hdr) + if err != nil { + return err + } + _, err = io.Copy(w, file) + return err + } + + // run the upload in a goroutine so we can send the request headers and + // wait for a '100 Continue' response before sending the multipart form + go func() { + err := uploader.Upload(uploadFn) + if err == nil { + err = mw.Close() + } + reqW.CloseWithError(err) + }() + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(data), nil } diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 135474475..4d02ceaf4 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -17,6 +17,7 @@ package client import ( + "bytes" "io/ioutil" "os" "path/filepath" @@ -24,52 +25,221 @@ import ( "sort" "testing" + "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/testutil" ) -func TestClientManifestFileList(t *testing.T) { +// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm +func TestClientUploadDownloadRaw(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() + client := NewClient(srv.URL) + + // upload some raw data + data := []byte("foo123") + hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + + // check we can download the same data + res, err := client.DownloadRaw(hash) + if err != nil { + t.Fatal(err) + } + defer res.Close() + gotData, err := ioutil.ReadAll(res) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(gotData, data) { + t.Fatalf("expected downloaded data to be %q, got %q", data, gotData) + } +} + +// TestClientUploadDownloadFiles test uploading and downloading files to swarm +// manifests +func TestClientUploadDownloadFiles(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + client := NewClient(srv.URL) + upload := func(manifest, path string, data []byte) string { + file := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + Path: path, + ContentType: "text/plain", + Size: int64(len(data)), + }, + } + hash, err := client.Upload(file, manifest) + if err != nil { + t.Fatal(err) + } + return hash + } + checkDownload := func(manifest, path string, expected []byte) { + file, err := client.Download(manifest, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if file.Size != int64(len(expected)) { + t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size) + } + if file.ContentType != file.ContentType { + t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType) + } + data, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, expected) { + t.Fatalf("expected downloaded data to be %q, got %q", expected, data) + } + } + + // upload a file to the root of a manifest + rootData := []byte("some-data") + rootHash := upload("", "", rootData) + + // check we can download the root file + checkDownload(rootHash, "", rootData) + + // upload another file to the same manifest + otherData := []byte("some-other-data") + newHash := upload(rootHash, "some/other/path", otherData) + + // check we can download both files from the new manifest + checkDownload(newHash, "", rootData) + checkDownload(newHash, "some/other/path", otherData) + + // replace the root file with different data + newHash = upload(newHash, "", otherData) + + // check both files have the other data + checkDownload(newHash, "", otherData) + checkDownload(newHash, "some/other/path", otherData) +} + +var testDirFiles = []string{ + "file1.txt", + "file2.txt", + "dir1/file3.txt", + "dir1/file4.txt", + "dir2/file5.txt", + "dir2/dir3/file6.txt", + "dir2/dir4/file7.txt", + "dir2/dir4/file8.txt", +} + +func newTestDirectory(t *testing.T) string { dir, err := ioutil.TempDir("", "swarm-client-test") if err != nil { t.Fatal(err) } - files := []string{ - "file1.txt", - "file2.txt", - "dir1/file3.txt", - "dir1/file4.txt", - "dir2/file5.txt", - "dir2/dir3/file6.txt", - "dir2/dir4/file7.txt", - "dir2/dir4/file8.txt", - } - for _, file := range files { + + for _, file := range testDirFiles { path := filepath.Join(dir, file) if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + os.RemoveAll(dir) t.Fatalf("error creating dir for %s: %s", path, err) } - if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil { + if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil { + os.RemoveAll(dir) t.Fatalf("error writing file %s: %s", path, err) } } + return dir +} + +// TestClientUploadDownloadDirectory tests uploading and downloading a +// directory of files to a swarm manifest +func TestClientUploadDownloadDirectory(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + dir := newTestDirectory(t) + defer os.RemoveAll(dir) + + // upload the directory client := NewClient(srv.URL) + defaultPath := filepath.Join(dir, testDirFiles[0]) + hash, err := client.UploadDirectory(dir, defaultPath, "") + if err != nil { + t.Fatalf("error uploading directory: %s", err) + } + + // check we can download the individual files + checkDownloadFile := func(path string, expected []byte) { + file, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + data, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, expected) { + t.Fatalf("expected data to be %q, got %q", expected, data) + } + } + for _, file := range testDirFiles { + checkDownloadFile(file, []byte(file)) + } + + // check we can download the default path + checkDownloadFile("", []byte(testDirFiles[0])) + + // check we can download the directory + tmp, err := ioutil.TempDir("", "swarm-client-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + if err := client.DownloadDirectory(hash, "", tmp); err != nil { + t.Fatal(err) + } + for _, file := range testDirFiles { + data, err := ioutil.ReadFile(filepath.Join(tmp, file)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, []byte(file)) { + t.Fatalf("expected data to be %q, got %q", file, data) + } + } +} + +// TestClientFileList tests listing files in a swarm manifest +func TestClientFileList(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() - hash, err := client.UploadDirectory(dir, "") + dir := newTestDirectory(t) + defer os.RemoveAll(dir) + + client := NewClient(srv.URL) + hash, err := client.UploadDirectory(dir, "", "") if err != nil { t.Fatalf("error uploading directory: %s", err) } ls := func(prefix string) []string { - entries, err := client.ManifestFileList(hash, prefix) + list, err := client.List(hash, prefix) if err != nil { t.Fatal(err) } - paths := make([]string, len(entries)) - for i, entry := range entries { - paths[i] = entry.Path + paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries)) + for _, prefix := range list.CommonPrefixes { + paths = append(paths, prefix) + } + for _, entry := range list.Entries { + paths = append(paths, entry.Path) } sort.Strings(paths) return paths @@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) { for prefix, expected := range tests { actual := ls(prefix) if !reflect.DeepEqual(actual, expected) { - t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual) + t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual) + } + } +} + +// TestClientMultipartUpload tests uploading files to swarm using a multipart +// upload +func TestClientMultipartUpload(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + // define an uploader which uploads testDirFiles with some data + data := []byte("some-data") + uploader := UploaderFunc(func(upload UploadFn) error { + for _, name := range testDirFiles { + file := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + Path: name, + ContentType: "text/plain", + Size: int64(len(data)), + }, + } + if err := upload(file); err != nil { + return err + } + } + return nil + }) + + // upload the files as a multipart upload + client := NewClient(srv.URL) + hash, err := client.MultipartUpload("", uploader) + if err != nil { + t.Fatal(err) + } + + // check we can download the individual files + checkDownloadFile := func(path string) { + file, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + gotData, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(gotData, data) { + t.Fatalf("expected data to be %q, got %q", data, gotData) } } + for _, file := range testDirFiles { + checkDownloadFile(file) + } } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index c2583e265..e7deaa32f 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "os" + "path" "path/filepath" "sync" @@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem { // Upload replicates a local directory as a manifest file and uploads it // using dpa store // TODO: localpath should point to a manifest +// +// DEPRECATED: Use the HTTP API instead func (self *FileSystem) Upload(lpath, index string) (string, error) { var list []*manifestTrieEntry localpath, err := filepath.Abs(filepath.Clean(lpath)) @@ -72,9 +75,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if path[:start] != localpath { return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath) } - entry := &manifestTrieEntry{ - Path: filepath.ToSlash(path), - } + entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil) list = append(list, entry) } return err @@ -91,9 +92,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if localpath[:start] != dir { return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir) } - entry := &manifestTrieEntry{ - Path: filepath.ToSlash(localpath), - } + entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil) list = append(list, entry) } @@ -153,11 +152,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { } entry.Path = RegularSlashes(entry.Path[start:]) if entry.Path == index { - ientry := &manifestTrieEntry{ - Path: "", - Hash: entry.Hash, + ientry := newManifestTrieEntry(&ManifestEntry{ ContentType: entry.ContentType, - } + }, nil) + ientry.Hash = entry.Hash trie.addEntry(ientry, quitC) } trie.addEntry(entry, quitC) @@ -174,6 +172,8 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { // Download replicates the manifest path structure on the local filesystem // under localpath +// +// DEPRECATED: Use the HTTP API instead func (self *FileSystem) Download(bzzpath, localpath string) error { lpath, err := filepath.Abs(filepath.Clean(localpath)) if err != nil { @@ -185,10 +185,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } //resolving host and port - key, _, path, err := self.api.parseAndResolve(bzzpath, true) + uri, err := Parse(path.Join("bzz:/", bzzpath)) + if err != nil { + return err + } + key, err := self.api.Resolve(uri) if err != nil { return err } + path := uri.Path if len(path) > 0 { path += "/" diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 4a27cb1da..8a15e735d 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -23,6 +23,9 @@ import ( "path/filepath" "sync" "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage" ) var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") @@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) { t.Fatalf("unexpected error: %v", err) } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index.html") + resp := testGet(t, api, bzzhash, "index.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) content = readPath(t, "testdata", "test0", "index.css") - resp = testGet(t, api, bzzhash+"/index.css") + resp = testGet(t, api, bzzhash, "index.css") exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, err = api.Get(bzzhash, true) + key := storage.Key(common.Hex2Bytes(bzzhash)) + _, _, _, err = api.Get(key, "") if err == nil { t.Fatalf("expected error: %v", err) } @@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) { return } - bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true) + key := storage.Key(common.Hex2Bytes(bzzhash)) + key, err = api.Modify(key, "index.html", "", "") if err != nil { t.Errorf("unexpected error: %v", err) return @@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true) + key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8") if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true) + key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8") if err != nil { t.Errorf("unexpected error: %v", err) return } + bzzhash = key.String() content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index2.html") + resp := testGet(t, api, bzzhash, "index2.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) - resp = testGet(t, api, bzzhash+"/img/logo.png") + resp = testGet(t, api, bzzhash, "img/logo.png") exp = expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) content = readPath(t, "testdata", "test0", "index.css") - resp = testGet(t, api, bzzhash+"/index.css") + resp = testGet(t, api, bzzhash, "index.css") exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, err = api.Get(bzzhash, true) + _, _, _, err = api.Get(key, "") if err == nil { t.Errorf("expected error: %v", err) } @@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, bzzhash, "") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) @@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index.html") + resp := testGet(t, api, bzzhash, "index.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) @@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, bzzhash, "") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go index fc74f5d3a..f99c4f35e 100644 --- a/swarm/api/http/roundtripper_test.go +++ b/swarm/api/http/roundtripper_test.go @@ -18,14 +18,14 @@ package http import ( "io/ioutil" + "net" "net/http" + "net/http/httptest" "strings" "testing" "time" ) -const port = "3222" - func TestRoundTripper(t *testing.T) { serveMux := http.NewServeMux() serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) { http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) } }) - go http.ListenAndServe(":"+port, serveMux) - rt := &RoundTripper{Port: port} + srv := httptest.NewServer(serveMux) + defer srv.Close() + + host, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + rt := &RoundTripper{Host: host, Port: port} trans := &http.Transport{} trans.RegisterProtocol("bzz", rt) client := &http.Client{Transport: trans} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 44e2c203a..849b9e10f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -20,13 +20,19 @@ A simple http server interface to Swarm package http import ( - "bytes" + "archive/tar" + "encoding/json" + "errors" "fmt" "io" + "io/ioutil" + "mime" + "mime/multipart" "net/http" - "regexp" + "os" + "path" + "strconv" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -36,26 +42,6 @@ import ( "github.com/rs/cors" ) -const ( - rawType = "application/octet-stream" -) - -var ( - // accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw) - bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+") - trailingSlashes = regexp.MustCompile("/+$") - rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$") - // forever = func() time.Time { return time.Unix(0, 0) } - forever = time.Now -) - -type sequentialReader struct { - reader io.Reader - pos int64 - ahead map[int64](chan bool) - lock sync.Mutex -} - // ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. type ServerConfig struct { @@ -94,242 +80,569 @@ type Server struct { api *api.Api } -func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - requestURL := r.URL - // This is wrong - // if requestURL.Host == "" { - // var err error - // requestURL, err = url.Parse(r.Referer() + requestURL.String()) - // if err != nil { - // http.Error(w, err.Error(), http.StatusBadRequest) - // return - // } - // } - log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))) - uri := requestURL.Path - var raw, nameresolver bool - var proto string - - // HTTP-based URL protocol handler - log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri)) - - path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string { - proto = p - return "" - }) +// Request wraps http.Request and also includes the parsed bzz URI +type Request struct { + http.Request + + uri *api.URI +} - // protocol identification (ugly) - if proto == "" { - log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri)) - http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest) +// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request +// body in swarm and returns the resulting storage key as a text/plain response +func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { + if r.uri.Path != "" { + s.BadRequest(w, r, "raw POST request cannot contain a path") return } - if len(proto) > 4 { - raw = proto[1:5] == "bzzr" - nameresolver = proto[1:5] != "bzzi" + + if r.Header.Get("Content-Length") == "" { + s.BadRequest(w, r, "missing Content-Length header in request") + return } - log.Debug("", "msg", log.Lazy{Fn: func() string { - return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path) - }}) + key, err := s.api.Store(r.Body, r.ContentLength, nil) + if err != nil { + s.Error(w, r, err) + return + } + s.logDebug("content for %s stored", key.Log()) + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, key) +} + +// HandlePostFiles handles a POST request (or deprecated PUT request) to +// bzz:/<hash>/<path> which contains either a single file or multiple files +// (either a tar archive or multipart form), adds those files either to an +// existing manifest or to a new manifest under <path> and returns the +// resulting manifest hash as a text/plain response +func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { + contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + s.BadRequest(w, r, err.Error()) + return + } - switch { - case r.Method == "POST" || r.Method == "PUT": - if r.Header.Get("content-length") == "" { - http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest) + var key storage.Key + if r.uri.Addr != "" { + key, err = s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } - key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil) - if err == nil { - log.Debug(fmt.Sprintf("Content for %v stored", key.Log())) - } else { - http.Error(w, err.Error(), http.StatusBadRequest) + } else { + key, err = s.api.NewManifest() + if err != nil { + s.Error(w, r, err) return } - if r.Method == "POST" { - if raw { - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key)))) - } else { - http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest) - return + } + + newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { + switch contentType { + + case "application/x-tar": + return s.handleTarUpload(r, mw) + + case "multipart/form-data": + return s.handleMultipartUpload(r, params["boundary"], mw) + + default: + return s.handleDirectUpload(r, mw) + } + }) + if err != nil { + s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, newKey) +} + +func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error { + tr := tar.NewReader(req.Body) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("error reading tar stream: %s", err) + } + + // only store regular files + if !hdr.FileInfo().Mode().IsRegular() { + continue + } + + // add the entry under the path from the request + path := path.Join(req.uri.Path, hdr.Name) + entry := &api.ManifestEntry{ + Path: path, + ContentType: hdr.Xattrs["user.swarm.content-type"], + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + contentKey, err := mw.AddEntry(tr, entry) + if err != nil { + return fmt.Errorf("error adding manifest entry from tar stream: %s", err) + } + s.logDebug("content for %s stored", contentKey.Log()) + } +} + +func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error { + mr := multipart.NewReader(req.Body, boundary) + for { + part, err := mr.NextPart() + if err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("error reading multipart form: %s", err) + } + + var size int64 + var reader io.Reader = part + if contentLength := part.Header.Get("Content-Length"); contentLength != "" { + size, err = strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return fmt.Errorf("error parsing multipart content length: %s", err) } + reader = part } else { - // PUT - if raw { - http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest) - return - } else { - path = api.RegularSlashes(path) - mime := r.Header.Get("Content-Type") - // TODO proper root hash separation - log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime)) - newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver) - if err == nil { - log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) - } else { - http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest) - return - } + // copy the part to a tmp file to get its size + tmp, err := ioutil.TempFile("", "swarm-multipart") + if err != nil { + return err } - } - case r.Method == "DELETE": - if raw { - http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest) - return - } else { - path = api.RegularSlashes(path) - log.Debug(fmt.Sprintf("Delete '%s'.", path)) - newKey, err := s.api.Modify(path, "", "", nameresolver) - if err == nil { - log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) - } else { - http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest) - return + defer os.Remove(tmp.Name()) + defer tmp.Close() + size, err = io.Copy(tmp, part) + if err != nil { + return fmt.Errorf("error copying multipart content: %s", err) + } + if _, err := tmp.Seek(0, os.SEEK_SET); err != nil { + return fmt.Errorf("error copying multipart content: %s", err) } + reader = tmp + } + + // add the entry under the path from the request + name := part.FileName() + if name == "" { + name = part.FormName() + } + path := path.Join(req.uri.Path, name) + entry := &api.ManifestEntry{ + Path: path, + ContentType: part.Header.Get("Content-Type"), + Size: size, + ModTime: time.Now(), } - case r.Method == "GET" || r.Method == "HEAD": - path = trailingSlashes.ReplaceAllString(path, "") - if path == "" { - http.Error(w, "Empty path not allowed", http.StatusBadRequest) + s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + contentKey, err := mw.AddEntry(reader, entry) + if err != nil { + return fmt.Errorf("error adding manifest entry from multipart form: %s", err) + } + s.logDebug("content for %s stored", contentKey.Log()) + } +} + +func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error { + key, err := mw.AddEntry(req.Body, &api.ManifestEntry{ + Path: req.uri.Path, + ContentType: req.Header.Get("Content-Type"), + Mode: 0644, + Size: req.ContentLength, + ModTime: time.Now(), + }) + if err != nil { + return err + } + s.logDebug("content for %s stored", key.Log()) + return nil +} + +// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes +// <path> from <manifest> and returns the resulting manifest hash as a +// text/plain response +func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { + s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log()) + return mw.RemoveEntry(r.uri.Path) + }) + if err != nil { + s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, newKey) +} + +// HandleGetRaw handles a GET request to bzzr://<key> and responds with +// the raw content stored at the given storage key +func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + // if path is set, interpret <key> as a manifest and return the + // raw entry at the given path + if r.uri.Path != "" { + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) return } - if raw { - var reader storage.LazySectionReader - parsedurl, _ := api.Parse(path) - - if parsedurl == path { - key, err := s.api.Resolve(parsedurl, nameresolver) - if err != nil { - log.Error(fmt.Sprintf("%v", err)) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - reader = s.api.Retrieve(key) - } else { - var status int - readertmp, _, status, err := s.api.Get(path, nameresolver) - if err != nil { - http.Error(w, err.Error(), status) - return - } - reader = readertmp + var entry *api.ManifestEntry + walker.Walk(func(e *api.ManifestEntry) error { + // if the entry matches the path, set entry and stop + // the walk + if e.Path == r.uri.Path { + entry = e + // return an error to cancel the walk + return errors.New("found") } - // retrieving content - - quitC := make(chan bool) - size, err := reader.Size(quitC) - if err != nil { - log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error())) - //An error on call to Size means we don't have the root chunk - http.Error(w, err.Error(), http.StatusNotFound) - return + // ignore non-manifest files + if e.ContentType != api.ManifestType { + return nil } - log.Debug(fmt.Sprintf("Reading %d bytes.", size)) - // setting mime type - qv := requestURL.Query() - mimeType := qv.Get("content_type") - if mimeType == "" { - mimeType = rawType + // if the manifest's path is a prefix of the + // requested path, recurse into it by returning + // nil and continuing the walk + if strings.HasPrefix(r.uri.Path, e.Path) { + return nil } - w.Header().Set("Content-Type", mimeType) - http.ServeContent(w, r, uri, forever(), reader) - log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType)) + return api.SkipManifest + }) + if entry == nil { + http.NotFound(w, &r.Request) + return + } + key = storage.Key(common.Hex2Bytes(entry.Hash)) + } - // retrieve path via manifest - } else { - log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri)) - // add trailing slash, if missing - if rootDocumentUri.MatchString(uri) { - http.Redirect(w, r, path+"/", http.StatusFound) - return + // check the root chunk exists by retrieving the file's size + reader := s.api.Retrieve(key) + if _, err := reader.Size(nil); err != nil { + s.logDebug("key not found %s: %s", key, err) + http.NotFound(w, &r.Request) + return + } + + // allow the request to overwrite the content type using a query + // parameter + contentType := "application/octet-stream" + if typ := r.URL.Query().Get("content_type"); typ != "" { + contentType = typ + } + w.Header().Set("Content-Type", contentType) + + http.ServeContent(w, &r.Request, "", time.Now(), reader) +} + +// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept +// header of "application/x-tar" and returns a tar stream of all files +// contained in the manifest +func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { + if r.uri.Path != "" { + s.BadRequest(w, r, "files request cannot contain a path") + return + } + + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.Error(w, r, err) + return + } + + tw := tar.NewWriter(w) + defer tw.Close() + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(http.StatusOK) + + err = walker.Walk(func(entry *api.ManifestEntry) error { + // ignore manifests (walk will recurse into them) + if entry.ContentType == api.ManifestType { + return nil + } + + // retrieve the entry's key and size + reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) + size, err := reader.Size(nil) + if err != nil { + return err + } + + // write a tar header for the entry + hdr := &tar.Header{ + Name: entry.Path, + Mode: entry.Mode, + Size: size, + ModTime: entry.ModTime, + Xattrs: map[string]string{ + "user.swarm.content-type": entry.ContentType, + }, + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + // copy the file into the tar stream + n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) + if err != nil { + return err + } else if n != size { + return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) + } + + return nil + }) + if err != nil { + s.logError("error generating tar stream: %s", err) + } +} + +// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has +// the "list" query parameter set to "true" and returns a list of all files +// contained in <manifest> under <path> grouped into common prefixes using +// "/" as a delimiter +func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { + // ensure the root path has a trailing slash so that relative URLs work + if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { + http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently) + return + } + + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.Error(w, r, err) + return + } + + var list api.ManifestList + prefix := r.uri.Path + err = walker.Walk(func(entry *api.ManifestEntry) error { + // handle non-manifest files + if entry.ContentType != api.ManifestType { + // ignore the file if it doesn't have the specified prefix + if !strings.HasPrefix(entry.Path, prefix) { + return nil } - reader, mimeType, status, err := s.api.Get(path, nameresolver) - if err != nil { - if _, ok := err.(api.ErrResolve); ok { - log.Debug(fmt.Sprintf("%v", err)) - status = http.StatusBadRequest - } else { - log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err)) - status = http.StatusNotFound - } - http.Error(w, err.Error(), status) - return + + // if the path after the prefix contains a slash, add a + // common prefix to the list, otherwise add the entry + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return nil } - // set mime type and status headers - w.Header().Set("Content-Type", mimeType) - if status > 0 { - w.WriteHeader(status) - } else { - status = 200 + if entry.Path == "" { + entry.Path = "/" } - quitC := make(chan bool) - size, err := reader.Size(quitC) - if err != nil { - log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error())) - //An error on call to Size means we don't have the root chunk - http.Error(w, err.Error(), http.StatusNotFound) - return + list.Entries = append(list.Entries, entry) + return nil + } + + // if the manifest's path is a prefix of the specified prefix + // then just recurse into the manifest by returning nil and + // continuing the walk + if strings.HasPrefix(prefix, entry.Path) { + return nil + } + + // if the manifest's path has the specified prefix, then if the + // path after the prefix contains a slash, add a common prefix + // to the list and skip the manifest, otherwise recurse into + // the manifest by returning nil and continuing the walk + if strings.HasPrefix(entry.Path, prefix) { + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return api.SkipManifest } - log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status)) + return nil + } - http.ServeContent(w, r, path, forever(), reader) + // the manifest neither has the prefix or needs recursing in to + // so just skip it + return api.SkipManifest + }) + if err != nil { + s.Error(w, r, err) + return + } + // if the client wants HTML (e.g. a browser) then render the list as a + // HTML index with relative URLs + if strings.Contains(r.Header.Get("Accept"), "text/html") { + w.Header().Set("Content-Type", "text/html") + err := htmlListTemplate.Execute(w, &htmlListData{ + URI: r.uri, + List: &list, + }) + if err != nil { + s.logError("error rendering list HTML: %s", err) } - default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + return } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(&list) } -func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) { - self.lock.Lock() - // assert self.pos <= off - if self.pos > off { - log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off)) - panic("Non-sequential read attempt") - } - if self.pos != off { - log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off)) - wait := make(chan bool) - self.ahead[off] = wait - self.lock.Unlock() - if <-wait { - // failed read behind - n = 0 - err = io.ErrUnexpectedEOF +// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds +// with the content of the file at <path> from the given <manifest> +func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + reader, contentType, _, err := s.api.Get(key, r.uri.Path) + if err != nil { + s.Error(w, r, err) + return + } + + // check the root chunk exists by retrieving the file's size + if _, err := reader.Size(nil); err != nil { + s.logDebug("file not found %s: %s", r.uri, err) + http.NotFound(w, &r.Request) + return + } + + w.Header().Set("Content-Type", contentType) + + http.ServeContent(w, &r.Request, "", time.Now(), reader) +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + if err != nil { + s.logError("Invalid URI %q: %s", r.URL.Path, err) + http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest) + return + } + s.logDebug("%s request received for %s", r.Method, uri) + + req := &Request{Request: *r, uri: uri} + switch r.Method { + case "POST": + if uri.Raw() { + s.HandlePostRaw(w, req) + } else { + s.HandlePostFiles(w, req) + } + + case "PUT": + // DEPRECATED: + // clients should send a POST request (the request creates a + // new manifest leaving the existing one intact, so it isn't + // strictly a traditional PUT request which replaces content + // at a URI, and POST is more ubiquitous) + if uri.Raw() { + http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return + } else { + s.HandlePostFiles(w, req) } - self.lock.Lock() - } - localPos := 0 - for localPos < len(target) { - n, err = self.reader.Read(target[localPos:]) - localPos += n - log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err)) - if err != nil { - log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err)) - for i := range self.ahead { - self.ahead[i] <- true - delete(self.ahead, i) - } - self.lock.Unlock() - return localPos, err + + case "DELETE": + if uri.Raw() { + http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + return + } + s.HandleDelete(w, req) + + case "GET": + if uri.Raw() { + s.HandleGetRaw(w, req) + return + } + + if r.Header.Get("Accept") == "application/x-tar" { + s.HandleGetFiles(w, req) + return + } + + if r.URL.Query().Get("list") == "true" { + s.HandleGetList(w, req) + return } - self.pos += int64(n) + + s.HandleGetFile(w, req) + + default: + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + + } +} + +func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) { + mw, err := s.api.NewManifestWriter(key, nil) + if err != nil { + return nil, err + } + + if err := update(mw); err != nil { + return nil, err } - wait := self.ahead[self.pos] - if wait != nil { - log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos)) - delete(self.ahead, self.pos) - close(wait) + + key, err = mw.Store() + if err != nil { + return nil, err } - self.lock.Unlock() - return localPos, err + s.logDebug("generated manifest %s", key) + return key, nil +} + +func (s *Server) logDebug(format string, v ...interface{}) { + log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) +} + +func (s *Server) logError(format string, v ...interface{}) { + log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) +} + +func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { + s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason) + http.Error(w, reason, http.StatusBadRequest) +} + +func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { + s.logError("error serving %s %s: %s", r.Method, r.uri, err) + http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 45a867f51..942f3ba0b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -40,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) { testrequests := make(map[string]int) testrequests["/"] = 0 - testrequests["/a"] = 1 - testrequests["/a/b"] = 2 + testrequests["/a/"] = 1 + testrequests["/a/b/"] = 2 testrequests["/x"] = 0 testrequests[""] = 0 diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go new file mode 100644 index 000000000..c3ef8c0f4 --- /dev/null +++ b/swarm/api/http/templates.go @@ -0,0 +1,71 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package http + +import ( + "html/template" + "path" + + "github.com/ethereum/go-ethereum/swarm/api" +) + +type htmlListData struct { + URI *api.URI + List *api.ManifestList +} + +var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(` +<!DOCTYPE html> +<html> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Swarm index of {{ .URI }}</title> +</head> + +<body> + <h1>Swarm index of {{ .URI }}</h1> + <hr> + <table> + <thead> + <tr> + <th>Path</th> + <th>Type</th> + <th>Size</th> + </tr> + </thead> + + <tbody> + {{ range .List.CommonPrefixes }} + <tr> + <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td> + <td>DIR</td> + <td>-</td> + </tr> + {{ end }} + + {{ range .List.Entries }} + <tr> + <td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td> + <td>{{ .ContentType }}</td> + <td>{{ .Size }}</td> + </tr> + {{ end }} + </table> + <hr> +</body> +`[1:])) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 199f259e1..6b3630fd0 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -19,8 +19,11 @@ package api import ( "bytes" "encoding/json" + "errors" "fmt" + "io" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -28,25 +31,152 @@ import ( ) const ( - manifestType = "application/bzz-manifest+json" + ManifestType = "application/bzz-manifest+json" ) +// Manifest represents a swarm manifest +type Manifest struct { + Entries []ManifestEntry `json:"entries,omitempty"` +} + +// ManifestEntry represents an entry in a swarm manifest +type ManifestEntry struct { + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + ContentType string `json:"contentType,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size,omitempty"` + ModTime time.Time `json:"mod_time,omitempty"` + Status int `json:"status,omitempty"` +} + +// ManifestList represents the result of listing files in a manifest +type ManifestList struct { + CommonPrefixes []string `json:"common_prefixes,omitempty"` + Entries []*ManifestEntry `json:"entries,omitempty"` +} + +// NewManifest creates and stores a new, empty manifest +func (a *Api) NewManifest() (storage.Key, error) { + var manifest Manifest + data, err := json.Marshal(&manifest) + if err != nil { + return nil, err + } + return a.Store(bytes.NewReader(data), int64(len(data)), nil) +} + +// ManifestWriter is used to add and remove entries from an underlying manifest +type ManifestWriter struct { + api *Api + trie *manifestTrie + quitC chan bool +} + +func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) { + trie, err := loadManifest(a.dpa, key, quitC) + if err != nil { + return nil, fmt.Errorf("error loading manifest %s: %s", key, err) + } + return &ManifestWriter{a, trie, quitC}, nil +} + +// AddEntry stores the given data and adds the resulting key to the manifest +func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { + key, err := m.api.Store(data, e.Size, nil) + if err != nil { + return nil, err + } + entry := newManifestTrieEntry(e, nil) + entry.Hash = key.String() + m.trie.addEntry(entry, m.quitC) + return key, nil +} + +// RemoveEntry removes the given path from the manifest +func (m *ManifestWriter) RemoveEntry(path string) error { + m.trie.deleteEntry(path, m.quitC) + return nil +} + +// Store stores the manifest, returning the resulting storage key +func (m *ManifestWriter) Store() (storage.Key, error) { + return m.trie.hash, m.trie.recalcAndStore() +} + +// ManifestWalker is used to recursively walk the entries in the manifest and +// all of its submanifests +type ManifestWalker struct { + api *Api + trie *manifestTrie + quitC chan bool +} + +func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) { + trie, err := loadManifest(a.dpa, key, quitC) + if err != nil { + return nil, fmt.Errorf("error loading manifest %s: %s", key, err) + } + return &ManifestWalker{a, trie, quitC}, nil +} + +// SkipManifest is used as a return value from WalkFn to indicate that the +// manifest should be skipped +var SkipManifest = errors.New("skip this manifest") + +// WalkFn is the type of function called for each entry visited by a recursive +// manifest walk +type WalkFn func(entry *ManifestEntry) error + +// Walk recursively walks the manifest calling walkFn for each entry in the +// manifest, including submanifests +func (m *ManifestWalker) Walk(walkFn WalkFn) error { + return m.walk(m.trie, "", walkFn) +} + +func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error { + for _, entry := range trie.entries { + if entry == nil { + continue + } + entry.Path = prefix + entry.Path + err := walkFn(&entry.ManifestEntry) + if err != nil { + if entry.ContentType == ManifestType && err == SkipManifest { + continue + } + return err + } + if entry.ContentType != ManifestType { + continue + } + if err := trie.loadSubTrie(entry, nil); err != nil { + return err + } + if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil { + return err + } + } + return nil +} + type manifestTrie struct { dpa *storage.DPA entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry hash storage.Key // if hash != nil, it is stored } -type manifestJSON struct { - Entries []*manifestTrieEntry `json:"entries"` +func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry { + return &manifestTrieEntry{ + ManifestEntry: *entry, + subtrie: subtrie, + } } type manifestTrieEntry struct { - Path string `json:"path"` - Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated - ContentType string `json:"contentType"` - Status int `json:"status"` - subtrie *manifestTrie + ManifestEntry + + subtrie *manifestTrie } func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand @@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp } log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log())) - man := manifestJSON{} + var man struct { + Entries []*manifestTrieEntry `json:"entries"` + } err = json.Unmarshal(manifestData, &man) if err != nil { err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) @@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { cpl++ } - if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { + if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) { if self.loadSubTrie(oldentry, quitC) != nil { return } @@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { subtrie.addEntry(entry, quitC) subtrie.addEntry(oldentry, quitC) - self.entries[b] = &manifestTrieEntry{ + self.entries[b] = newManifestTrieEntry(&ManifestEntry{ Path: commonPrefix, - Hash: "", - ContentType: manifestType, - subtrie: subtrie, - } + ContentType: ManifestType, + }, subtrie) } func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { @@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { } epl := len(entry.Path) - if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { + if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { if self.loadSubTrie(entry, quitC) != nil { return } @@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error { var buffer bytes.Buffer buffer.WriteString(`{"entries":[`) - list := &manifestJSON{} + list := &Manifest{} for _, entry := range self.entries { if entry != nil { if entry.Hash == "" { // TODO: paralellize @@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error { } entry.Hash = entry.subtrie.hash.String() } - list.Entries = append(list.Entries, entry) + list.Entries = append(list.Entries, entry.ManifestEntry) } } @@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, entry := self.entries[i] if entry != nil { epl := len(entry.Path) - if entry.ContentType == manifestType { + if entry.ContentType == ManifestType { l := plen if epl < l { l = epl @@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl)) if (len(path) >= epl) && (path[:epl] == entry.Path) { log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType)) - if entry.ContentType == manifestType { + if entry.ContentType == ManifestType { err := self.loadSubTrie(entry, quitC) if err != nil { return nil, 0 diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 31b484675..7e94a9653 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -16,6 +16,8 @@ package api +import "path" + type Response struct { MimeType string Status int @@ -25,6 +27,8 @@ type Response struct { } // implements a service +// +// DEPRECATED: Use the HTTP API instead type Storage struct { api *Api } @@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage { // Put uploads the content to the swarm with a simple manifest speficying // its content type +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Put(content, contentType string) (string, error) { - return self.api.Put(content, contentType) + key, err := self.api.Put(content, contentType) + if err != nil { + return "", err + } + return key.String(), err } // Get retrieves the content from bzzpath and reads the response in full @@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) { // NOTE: if error is non-nil, sResponse may still have partial content // the actual size of which is given in len(resp.Content), while the expected // size is resp.Size +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Get(bzzpath string) (*Response, error) { - reader, mimeType, status, err := self.api.Get(bzzpath, true) + uri, err := Parse(path.Join("bzz:/", bzzpath)) + if err != nil { + return nil, err + } + key, err := self.api.Resolve(uri) + if err != nil { + return nil, err + } + reader, mimeType, status, err := self.api.Get(key, uri.Path) if err != nil { return nil, err } @@ -65,6 +85,20 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { // Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash, // and merge on to it. creating an entry w conentType (mime) +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { - return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) + uri, err := Parse("bzz:/" + rootHash) + if err != nil { + return "", err + } + key, err := self.api.Resolve(uri) + if err != nil { + return "", err + } + key, err = self.api.Modify(key, path, contentHash, contentType) + if err != nil { + return "", err + } + return key.String(), nil } diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go index 72caf52df..d260dd61d 100644 --- a/swarm/api/storage_test.go +++ b/swarm/api/storage_test.go @@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) { t.Fatalf("unexpected error: %v", err) } // to check put against the Api#Get - resp0 := testGet(t, api.api, bzzhash) + resp0 := testGet(t, api.api, bzzhash, "") checkResponse(t, resp0, exp) // check storage#Get diff --git a/swarm/api/swarmfs_unix.go b/swarm/api/swarmfs_unix.go index e696c6b9a..a704c1ec2 100644 --- a/swarm/api/swarmfs_unix.go +++ b/swarm/api/swarmfs_unix.go @@ -91,11 +91,16 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { return nil, fmt.Errorf("%s is already mounted", cleanedMountPoint) } - key, _, path, err := self.swarmApi.parseAndResolve(mhash, true) + uri, err := Parse("bzz:/" + mhash) if err != nil { - return nil, fmt.Errorf("can't resolve %q: %v", mhash, err) + return nil, err + } + key, err := self.swarmApi.Resolve(uri) + if err != nil { + return nil, err } + path := uri.Path if len(path) > 0 { path += "/" } diff --git a/swarm/api/uri.go b/swarm/api/uri.go new file mode 100644 index 000000000..68ce04835 --- /dev/null +++ b/swarm/api/uri.go @@ -0,0 +1,96 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package api + +import ( + "fmt" + "net/url" + "strings" +) + +// URI is a reference to content stored in swarm. +type URI struct { + // Scheme has one of the following values: + // + // * bzz - an entry in a swarm manifest + // * bzzr - raw swarm content + // * bzzi - immutable URI of an entry in a swarm manifest + // (address is not resolved) + Scheme string + + // Addr is either a hexadecimal storage key or it an address which + // resolves to a storage key + Addr string + + // Path is the path to the content within a swarm manifest + Path string +} + +// Parse parses rawuri into a URI struct, where rawuri is expected to have one +// of the following formats: +// +// * <scheme>:/ +// * <scheme>:/<addr> +// * <scheme>:/<addr>/<path> +// * <scheme>:// +// * <scheme>://<addr> +// * <scheme>://<addr>/<path> +// +// with scheme one of bzz, bzzr or bzzi +func Parse(rawuri string) (*URI, error) { + u, err := url.Parse(rawuri) + if err != nil { + return nil, err + } + uri := &URI{Scheme: u.Scheme} + + // check the scheme is valid + switch uri.Scheme { + case "bzz", "bzzi", "bzzr": + default: + return nil, fmt.Errorf("unknown scheme %q", u.Scheme) + } + + // handle URIs like bzz://<addr>/<path> where the addr and path + // have already been split by url.Parse + if u.Host != "" { + uri.Addr = u.Host + uri.Path = strings.TrimLeft(u.Path, "/") + return uri, nil + } + + // URI is like bzz:/<addr>/<path> so split the addr and path from + // the raw path (which will be /<addr>/<path>) + parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2) + uri.Addr = parts[0] + if len(parts) == 2 { + uri.Path = parts[1] + } + return uri, nil +} + +func (u *URI) Raw() bool { + return u.Scheme == "bzzr" +} + +func (u *URI) Immutable() bool { + return u.Scheme == "bzzi" +} + +func (u *URI) String() string { + return u.Scheme + ":/" + u.Addr + "/" + u.Path +} diff --git a/swarm/api/uri_test.go b/swarm/api/uri_test.go new file mode 100644 index 000000000..dcb5fbbff --- /dev/null +++ b/swarm/api/uri_test.go @@ -0,0 +1,120 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package api + +import ( + "reflect" + "testing" +) + +func TestParseURI(t *testing.T) { + type test struct { + uri string + expectURI *URI + expectErr bool + expectRaw bool + expectImmutable bool + } + tests := []test{ + { + uri: "", + expectErr: true, + }, + { + uri: "foo", + expectErr: true, + }, + { + uri: "bzz", + expectErr: true, + }, + { + uri: "bzz:", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzzi:", + expectURI: &URI{Scheme: "bzzi"}, + expectImmutable: true, + }, + { + uri: "bzzr:", + expectURI: &URI{Scheme: "bzzr"}, + expectRaw: true, + }, + { + uri: "bzz:/", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzz:/abc123", + expectURI: &URI{Scheme: "bzz", Addr: "abc123"}, + }, + { + uri: "bzz:/abc123/path/to/entry", + expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, + }, + { + uri: "bzzr:/", + expectURI: &URI{Scheme: "bzzr"}, + expectRaw: true, + }, + { + uri: "bzzr:/abc123", + expectURI: &URI{Scheme: "bzzr", Addr: "abc123"}, + expectRaw: true, + }, + { + uri: "bzzr:/abc123/path/to/entry", + expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"}, + expectRaw: true, + }, + { + uri: "bzz://", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzz://abc123", + expectURI: &URI{Scheme: "bzz", Addr: "abc123"}, + }, + { + uri: "bzz://abc123/path/to/entry", + expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, + }, + } + for _, x := range tests { + actual, err := Parse(x.uri) + if x.expectErr { + if err == nil { + t.Fatalf("expected %s to error", x.uri) + } + continue + } + if err != nil { + t.Fatalf("error parsing %s: %s", x.uri, err) + } + if !reflect.DeepEqual(actual, x.expectURI) { + t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual) + } + if actual.Raw() != x.expectRaw { + t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw()) + } + if actual.Immutable() != x.expectImmutable { + t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable()) + } + } +} |