diff options
Diffstat (limited to 'swarm')
27 files changed, 3958 insertions, 430 deletions
diff --git a/swarm/api/api.go b/swarm/api/api.go index 7af27208d..26a9445d5 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -24,9 +24,13 @@ import ( "strings" "sync" + "bytes" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" + "mime" + "path/filepath" + "time" ) var ( @@ -58,6 +62,13 @@ func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { return } +// to be used only in TEST +func (self *Api) Upload(uploadDir, index string) (hash string, err error) { + fs := NewFileSystem(self) + hash, err = fs.Upload(uploadDir, index) + return hash, err +} + // DPA reader API func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) @@ -70,86 +81,53 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor type ErrResolve error // DNS Resolver -func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { - log.Trace(fmt.Sprintf("Resolving : %v", hostPort)) - if hashMatcher.MatchString(hostPort) || self.dns == nil { - log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort)) - return storage.Key(common.Hex2Bytes(hostPort)), nil +func (self *Api) Resolve(uri *URI) (storage.Key, error) { + log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) + + var err error + if !uri.Immutable() { + if self.dns != nil { + resolved, err := self.dns.Resolve(uri.Addr) + if err == nil { + return resolved[:], nil + } + } else { + err = fmt.Errorf("no DNS to resolve name") + } } - if !nameresolver { - return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort) + if hashMatcher.MatchString(uri.Addr) { + return storage.Key(common.Hex2Bytes(uri.Addr)), nil } - contentHash, err := self.dns.Resolve(hostPort) if err != nil { - err = ErrResolve(err) - log.Warn(fmt.Sprintf("DNS error : %v", err)) - } - log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash)) - return contentHash[:], err -} -func Parse(uri string) (hostPort, path string) { - if uri == "" { - return + return nil, fmt.Errorf("'%s' does not resolve: %v but is not a content hash", uri.Addr, err) } - parts := slashes.Split(uri, 3) - var i int - if len(parts) == 0 { - return - } - // beginning with slash is now optional - for len(parts[i]) == 0 { - i++ - } - hostPort = parts[i] - for i < len(parts)-1 { - i++ - if len(path) > 0 { - path = path + "/" + parts[i] - } else { - path = parts[i] - } - } - log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path)) - return + return nil, fmt.Errorf("'%s' is not a content hash", uri.Addr) } -func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) { - hostPort, path = Parse(uri) - //resolving host and port - contentHash, err := self.Resolve(hostPort, nameresolver) - log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)) - return contentHash[:], hostPort, path, err -} // Put provides singleton manifest creation on top of dpa store -func (self *Api) Put(content, contentType string) (string, error) { +func (self *Api) Put(content, contentType string) (storage.Key, error) { r := strings.NewReader(content) wg := &sync.WaitGroup{} key, err := self.dpa.Store(r, int64(len(content)), wg, nil) if err != nil { - return "", err + return nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) if err != nil { - return "", err + return nil, err } wg.Wait() - return key.String(), nil + return key, nil } // Get uses iterative manifest retrieval and prefix matching -// to resolve path to content using dpa retrieve +// to resolve basePath to content using dpa retrieve // it returns a section reader, mimeType, status and an error -func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) { - key, _, path, err := self.parseAndResolve(uri, nameresolver) - if err != nil { - return nil, "", 500, fmt.Errorf("can't resolve: %v", err) - } - - quitC := make(chan bool) - trie, err := loadManifest(self.dpa, key, quitC) +func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + trie, err := loadManifest(self.dpa, key, nil) if err != nil { log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -173,32 +151,203 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR return } -func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { - root, _, path, err := self.parseAndResolve(uri, nameresolver) - if err != nil { - return "", fmt.Errorf("can't resolve: %v", err) - } - +func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { quitC := make(chan bool) - trie, err := loadManifest(self.dpa, root, quitC) + trie, err := loadManifest(self.dpa, key, quitC) if err != nil { - return + return nil, err } - if contentHash != "" { - entry := &manifestTrieEntry{ + entry := newManifestTrieEntry(&ManifestEntry{ Path: path, - Hash: contentHash, ContentType: contentType, - } + }, nil) + entry.Hash = contentHash trie.addEntry(entry, quitC) } else { trie.deleteEntry(path, quitC) } - err = trie.recalcAndStore() + if err := trie.recalcAndStore(); err != nil { + return nil, err + } + return trie.hash, nil +} + +func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { + + uri, err := Parse("bzz:/" + mhash) if err != nil { - return + return nil, "", err + } + mkey, err := self.Resolve(uri) + if err != nil { + return nil, "", err + } + + // trim the root dir we added + if path[:1] == "/" { + path = path[1:] + } + + entry := &ManifestEntry{ + Path: filepath.Join(path, fname), + ContentType: mime.TypeByExtension(filepath.Ext(fname)), + Mode: 0700, + Size: int64(len(content)), + ModTime: time.Now(), + } + + mw, err := self.NewManifestWriter(mkey, nil) + if err != nil { + return nil, "", err + } + + fkey, err := mw.AddEntry(bytes.NewReader(content), entry) + if err != nil { + return nil, "", err + } + + newMkey, err := mw.Store() + if err != nil { + return nil, "", err + + } + + return fkey, newMkey.String(), nil + +} + +func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) { + + uri, err := Parse("bzz:/" + mhash) + if err != nil { + return "", err + } + mkey, err := self.Resolve(uri) + if err != nil { + return "", err + } + + // trim the root dir we added + if path[:1] == "/" { + path = path[1:] + } + + mw, err := self.NewManifestWriter(mkey, nil) + if err != nil { + return "", err + } + + err = mw.RemoveEntry(filepath.Join(path, fname)) + if err != nil { + return "", err + } + + newMkey, err := mw.Store() + if err != nil { + return "", err + + } + + return newMkey.String(), nil +} + +func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) { + + buffSize := offset + addSize + if buffSize < existingSize { + buffSize = existingSize + } + + buf := make([]byte, buffSize) + + oldReader := self.Retrieve(oldKey) + io.ReadAtLeast(oldReader, buf, int(offset)) + + newReader := bytes.NewReader(content) + io.ReadAtLeast(newReader, buf[offset:], int(addSize)) + + if buffSize < existingSize { + io.ReadAtLeast(oldReader, buf[addSize:], int(buffSize)) } - return trie.hash.String(), nil + + combinedReader := bytes.NewReader(buf) + totalSize := int64(len(buf)) + + // TODO(jmozah): to append using pyramid chunker when it is ready + //oldReader := self.Retrieve(oldKey) + //newReader := bytes.NewReader(content) + //combinedReader := io.MultiReader(oldReader, newReader) + + uri, err := Parse("bzz:/" + mhash) + if err != nil { + return nil, "", err + } + mkey, err := self.Resolve(uri) + if err != nil { + return nil, "", err + } + + // trim the root dir we added + if path[:1] == "/" { + path = path[1:] + } + + mw, err := self.NewManifestWriter(mkey, nil) + if err != nil { + return nil, "", err + } + + err = mw.RemoveEntry(filepath.Join(path, fname)) + if err != nil { + return nil, "", err + } + + entry := &ManifestEntry{ + Path: filepath.Join(path, fname), + ContentType: mime.TypeByExtension(filepath.Ext(fname)), + Mode: 0700, + Size: totalSize, + ModTime: time.Now(), + } + + fkey, err := mw.AddEntry(io.Reader(combinedReader), entry) + if err != nil { + return nil, "", err + } + + newMkey, err := mw.Store() + if err != nil { + return nil, "", err + + } + + return fkey, newMkey.String(), nil + +} + +func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { + + uri, err := Parse("bzz:/" + mhash) + if err != nil { + return nil, nil, err + } + key, err = self.Resolve(uri) + if err != nil { + return nil, nil, err + } + + quitC := make(chan bool) + rootTrie, err := loadManifest(self.dpa, key, quitC) + if err != nil { + return nil, nil, fmt.Errorf("can't load manifest %v: %v", key.String(), err) + } + + manifestEntryMap = map[string]*manifestTrieEntry{} + err = rootTrie.listWithPrefix(uri.Path, quitC, func(entry *manifestTrieEntry, suffix string) { + manifestEntryMap[suffix] = entry + }) + + return key, manifestEntryMap, nil } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 16e90dd32..c2d78c2dc 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -23,6 +23,7 @@ import ( "os" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response { } // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { -func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { - reader, mimeType, status, err := api.Get(bzzhash, true) +func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse { + key := storage.Key(common.Hex2Bytes(bzzhash)) + reader, mimeType, status, err := api.Get(key, path) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - bzzhash, err := api.Put(content, exp.MimeType) + key, err := api.Put(content, exp.MimeType) if err != nil { t.Fatalf("unexpected error: %v", err) } - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, key.String(), "") checkResponse(t, resp, exp) }) } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go new file mode 100644 index 000000000..f9c3e51e8 --- /dev/null +++ b/swarm/api/client/client.go @@ -0,0 +1,465 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package client + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "mime" + "mime/multipart" + "net/http" + "net/textproto" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/ethereum/go-ethereum/swarm/api" +) + +var ( + DefaultGateway = "http://localhost:8500" + DefaultClient = NewClient(DefaultGateway) +) + +func NewClient(gateway string) *Client { + return &Client{ + Gateway: gateway, + } +} + +// Client wraps interaction with a swarm HTTP gateway. +type Client struct { + Gateway string +} + +// UploadRaw uploads raw data to swarm and returns the resulting hash +func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { + if size <= 0 { + return "", errors.New("data size must be greater than zero") + } + req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r) + if err != nil { + return "", err + } + req.ContentLength = size + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(data), nil +} + +// DownloadRaw downloads raw data from swarm +func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) { + uri := c.Gateway + "/bzzr:/" + hash + res, err := http.DefaultClient.Get(uri) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + res.Body.Close() + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + return res.Body, nil +} + +// File represents a file in a swarm manifest and is used for uploading and +// downloading content to and from swarm +type File struct { + io.ReadCloser + api.ManifestEntry +} + +// Open opens a local file which can then be passed to client.Upload to upload +// it to swarm +func Open(path string) (*File, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + stat, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + return &File{ + ReadCloser: f, + ManifestEntry: api.ManifestEntry{ + ContentType: mime.TypeByExtension(filepath.Ext(path)), + Mode: int64(stat.Mode()), + Size: stat.Size(), + ModTime: stat.ModTime(), + }, + }, nil +} + +// Upload uploads a file to swarm and either adds it to an existing manifest +// (if the manifest argument is non-empty) or creates a new manifest containing +// the file, returning the resulting manifest hash (the file will then be +// available at bzz:/<hash>/<path>) +func (c *Client) Upload(file *File, manifest string) (string, error) { + if file.Size <= 0 { + return "", errors.New("file size must be greater than zero") + } + return c.TarUpload(manifest, &FileUploader{file}) +} + +// Download downloads a file with the given path from the swarm manifest with +// the given hash (i.e. it gets bzz:/<hash>/<path>) +func (c *Client) Download(hash, path string) (*File, error) { + uri := c.Gateway + "/bzz:/" + hash + "/" + path + res, err := http.DefaultClient.Get(uri) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + res.Body.Close() + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + return &File{ + ReadCloser: res.Body, + ManifestEntry: api.ManifestEntry{ + ContentType: res.Header.Get("Content-Type"), + Size: res.ContentLength, + }, + }, nil +} + +// UploadDirectory uploads a directory tree to swarm and either adds the files +// to an existing manifest (if the manifest argument is non-empty) or creates a +// new manifest, returning the resulting manifest hash (files from the +// directory will then be available at bzz:/<hash>/path/to/file), with +// the file specified in defaultPath being uploaded to the root of the manifest +// (i.e. bzz:/<hash>/) +func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) { + stat, err := os.Stat(dir) + if err != nil { + return "", err + } else if !stat.IsDir() { + return "", fmt.Errorf("not a directory: %s", dir) + } + return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}) +} + +// DownloadDirectory downloads the files contained in a swarm manifest under +// the given path into a local directory (existing files will be overwritten) +func (c *Client) DownloadDirectory(hash, path, destDir string) error { + stat, err := os.Stat(destDir) + if err != nil { + return err + } else if !stat.IsDir() { + return fmt.Errorf("not a directory: %s", destDir) + } + + uri := c.Gateway + "/bzz:/" + hash + "/" + path + req, err := http.NewRequest("GET", uri, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/x-tar") + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + tr := tar.NewReader(res.Body) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } else if err != nil { + return err + } + // ignore the default path file + if hdr.Name == "" { + continue + } + + dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path))) + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + var mode os.FileMode = 0644 + if hdr.Mode > 0 { + mode = os.FileMode(hdr.Mode) + } + dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + n, err := io.Copy(dst, tr) + dst.Close() + if err != nil { + return err + } else if n != hdr.Size { + return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n) + } + } +} + +// UploadManifest uploads the given manifest to swarm +func (c *Client) UploadManifest(m *api.Manifest) (string, error) { + data, err := json.Marshal(m) + if err != nil { + return "", err + } + return c.UploadRaw(bytes.NewReader(data), int64(len(data))) +} + +// DownloadManifest downloads a swarm manifest +func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) { + res, err := c.DownloadRaw(hash) + if err != nil { + return nil, err + } + defer res.Close() + var manifest api.Manifest + if err := json.NewDecoder(res).Decode(&manifest); err != nil { + return nil, err + } + return &manifest, nil +} + +// List list files in a swarm manifest which have the given prefix, grouping +// common prefixes using "/" as a delimiter. +// +// For example, if the manifest represents the following directory structure: +// +// file1.txt +// file2.txt +// dir1/file3.txt +// dir1/dir2/file4.txt +// +// Then: +// +// - a prefix of "" would return [dir1/, file1.txt, file2.txt] +// - a prefix of "file" would return [file1.txt, file2.txt] +// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] +// +// where entries ending with "/" are common prefixes. +func (c *Client) List(hash, prefix string) (*api.ManifestList, error) { + res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true") + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + var list api.ManifestList + if err := json.NewDecoder(res.Body).Decode(&list); err != nil { + return nil, err + } + return &list, nil +} + +// Uploader uploads files to swarm using a provided UploadFn +type Uploader interface { + Upload(UploadFn) error +} + +type UploaderFunc func(UploadFn) error + +func (u UploaderFunc) Upload(upload UploadFn) error { + return u(upload) +} + +// DirectoryUploader uploads all files in a directory, optionally uploading +// a file to the default path +type DirectoryUploader struct { + Dir string + DefaultPath string +} + +// Upload performs the upload of the directory and default path +func (d *DirectoryUploader) Upload(upload UploadFn) error { + if d.DefaultPath != "" { + file, err := Open(d.DefaultPath) + if err != nil { + return err + } + if err := upload(file); err != nil { + return err + } + } + return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir() { + return nil + } + file, err := Open(path) + if err != nil { + return err + } + relPath, err := filepath.Rel(d.Dir, path) + if err != nil { + return err + } + file.Path = filepath.ToSlash(relPath) + return upload(file) + }) +} + +// FileUploader uploads a single file +type FileUploader struct { + File *File +} + +// Upload performs the upload of the file +func (f *FileUploader) Upload(upload UploadFn) error { + return upload(f.File) +} + +// UploadFn is the type of function passed to an Uploader to perform the upload +// of a single file (for example, a directory uploader would call a provided +// UploadFn for each file in the directory tree) +type UploadFn func(file *File) error + +// TarUpload uses the given Uploader to upload files to swarm as a tar stream, +// returning the resulting manifest hash +func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) { + reqR, reqW := io.Pipe() + defer reqR.Close() + req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-tar") + + // use 'Expect: 100-continue' so we don't send the request body if + // the server refuses the request + req.Header.Set("Expect", "100-continue") + + tw := tar.NewWriter(reqW) + + // define an UploadFn which adds files to the tar stream + uploadFn := func(file *File) error { + hdr := &tar.Header{ + Name: file.Path, + Mode: file.Mode, + Size: file.Size, + ModTime: file.ModTime, + Xattrs: map[string]string{ + "user.swarm.content-type": file.ContentType, + }, + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + _, err = io.Copy(tw, file) + return err + } + + // run the upload in a goroutine so we can send the request headers and + // wait for a '100 Continue' response before sending the tar stream + go func() { + err := uploader.Upload(uploadFn) + if err == nil { + err = tw.Close() + } + reqW.CloseWithError(err) + }() + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(data), nil +} + +// MultipartUpload uses the given Uploader to upload files to swarm as a +// multipart form, returning the resulting manifest hash +func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) { + reqR, reqW := io.Pipe() + defer reqR.Close() + req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) + if err != nil { + return "", err + } + + // use 'Expect: 100-continue' so we don't send the request body if + // the server refuses the request + req.Header.Set("Expect", "100-continue") + + mw := multipart.NewWriter(reqW) + req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) + + // define an UploadFn which adds files to the multipart form + uploadFn := func(file *File) error { + hdr := make(textproto.MIMEHeader) + hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path)) + hdr.Set("Content-Type", file.ContentType) + hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10)) + w, err := mw.CreatePart(hdr) + if err != nil { + return err + } + _, err = io.Copy(w, file) + return err + } + + // run the upload in a goroutine so we can send the request headers and + // wait for a '100 Continue' response before sending the multipart form + go func() { + err := uploader.Upload(uploadFn) + if err == nil { + err = mw.Close() + } + reqW.CloseWithError(err) + }() + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) + } + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(data), nil +} diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go new file mode 100644 index 000000000..4d02ceaf4 --- /dev/null +++ b/swarm/api/client/client_test.go @@ -0,0 +1,327 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package client + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm +func TestClientUploadDownloadRaw(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + client := NewClient(srv.URL) + + // upload some raw data + data := []byte("foo123") + hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + + // check we can download the same data + res, err := client.DownloadRaw(hash) + if err != nil { + t.Fatal(err) + } + defer res.Close() + gotData, err := ioutil.ReadAll(res) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(gotData, data) { + t.Fatalf("expected downloaded data to be %q, got %q", data, gotData) + } +} + +// TestClientUploadDownloadFiles test uploading and downloading files to swarm +// manifests +func TestClientUploadDownloadFiles(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + client := NewClient(srv.URL) + upload := func(manifest, path string, data []byte) string { + file := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + Path: path, + ContentType: "text/plain", + Size: int64(len(data)), + }, + } + hash, err := client.Upload(file, manifest) + if err != nil { + t.Fatal(err) + } + return hash + } + checkDownload := func(manifest, path string, expected []byte) { + file, err := client.Download(manifest, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if file.Size != int64(len(expected)) { + t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size) + } + if file.ContentType != file.ContentType { + t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType) + } + data, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, expected) { + t.Fatalf("expected downloaded data to be %q, got %q", expected, data) + } + } + + // upload a file to the root of a manifest + rootData := []byte("some-data") + rootHash := upload("", "", rootData) + + // check we can download the root file + checkDownload(rootHash, "", rootData) + + // upload another file to the same manifest + otherData := []byte("some-other-data") + newHash := upload(rootHash, "some/other/path", otherData) + + // check we can download both files from the new manifest + checkDownload(newHash, "", rootData) + checkDownload(newHash, "some/other/path", otherData) + + // replace the root file with different data + newHash = upload(newHash, "", otherData) + + // check both files have the other data + checkDownload(newHash, "", otherData) + checkDownload(newHash, "some/other/path", otherData) +} + +var testDirFiles = []string{ + "file1.txt", + "file2.txt", + "dir1/file3.txt", + "dir1/file4.txt", + "dir2/file5.txt", + "dir2/dir3/file6.txt", + "dir2/dir4/file7.txt", + "dir2/dir4/file8.txt", +} + +func newTestDirectory(t *testing.T) string { + dir, err := ioutil.TempDir("", "swarm-client-test") + if err != nil { + t.Fatal(err) + } + + for _, file := range testDirFiles { + path := filepath.Join(dir, file) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + os.RemoveAll(dir) + t.Fatalf("error creating dir for %s: %s", path, err) + } + if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil { + os.RemoveAll(dir) + t.Fatalf("error writing file %s: %s", path, err) + } + } + + return dir +} + +// TestClientUploadDownloadDirectory tests uploading and downloading a +// directory of files to a swarm manifest +func TestClientUploadDownloadDirectory(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + dir := newTestDirectory(t) + defer os.RemoveAll(dir) + + // upload the directory + client := NewClient(srv.URL) + defaultPath := filepath.Join(dir, testDirFiles[0]) + hash, err := client.UploadDirectory(dir, defaultPath, "") + if err != nil { + t.Fatalf("error uploading directory: %s", err) + } + + // check we can download the individual files + checkDownloadFile := func(path string, expected []byte) { + file, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + data, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, expected) { + t.Fatalf("expected data to be %q, got %q", expected, data) + } + } + for _, file := range testDirFiles { + checkDownloadFile(file, []byte(file)) + } + + // check we can download the default path + checkDownloadFile("", []byte(testDirFiles[0])) + + // check we can download the directory + tmp, err := ioutil.TempDir("", "swarm-client-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + if err := client.DownloadDirectory(hash, "", tmp); err != nil { + t.Fatal(err) + } + for _, file := range testDirFiles { + data, err := ioutil.ReadFile(filepath.Join(tmp, file)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, []byte(file)) { + t.Fatalf("expected data to be %q, got %q", file, data) + } + } +} + +// TestClientFileList tests listing files in a swarm manifest +func TestClientFileList(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + dir := newTestDirectory(t) + defer os.RemoveAll(dir) + + client := NewClient(srv.URL) + hash, err := client.UploadDirectory(dir, "", "") + if err != nil { + t.Fatalf("error uploading directory: %s", err) + } + + ls := func(prefix string) []string { + list, err := client.List(hash, prefix) + if err != nil { + t.Fatal(err) + } + paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries)) + for _, prefix := range list.CommonPrefixes { + paths = append(paths, prefix) + } + for _, entry := range list.Entries { + paths = append(paths, entry.Path) + } + sort.Strings(paths) + return paths + } + + tests := map[string][]string{ + "": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"}, + "file": []string{"file1.txt", "file2.txt"}, + "file1": []string{"file1.txt"}, + "file2.txt": []string{"file2.txt"}, + "file12": []string{}, + "dir": []string{"dir1/", "dir2/"}, + "dir1": []string{"dir1/"}, + "dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file3.txt": []string{"dir1/file3.txt"}, + "dir1/file34": []string{}, + "dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, + "dir2/file": []string{"dir2/file5.txt"}, + "dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"}, + "dir2/dir3/": []string{"dir2/dir3/file6.txt"}, + "dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"}, + "dir2/dir4/file78": []string{}, + } + for prefix, expected := range tests { + actual := ls(prefix) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual) + } + } +} + +// TestClientMultipartUpload tests uploading files to swarm using a multipart +// upload +func TestClientMultipartUpload(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + // define an uploader which uploads testDirFiles with some data + data := []byte("some-data") + uploader := UploaderFunc(func(upload UploadFn) error { + for _, name := range testDirFiles { + file := &File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + Path: name, + ContentType: "text/plain", + Size: int64(len(data)), + }, + } + if err := upload(file); err != nil { + return err + } + } + return nil + }) + + // upload the files as a multipart upload + client := NewClient(srv.URL) + hash, err := client.MultipartUpload("", uploader) + if err != nil { + t.Fatal(err) + } + + // check we can download the individual files + checkDownloadFile := func(path string) { + file, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + gotData, err := ioutil.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(gotData, data) { + t.Fatalf("expected data to be %q, got %q", data, gotData) + } + } + for _, file := range testDirFiles { + checkDownloadFile(file) + } +} diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index c2583e265..f5dc90e2e 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "os" + "path" "path/filepath" "sync" @@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem { // Upload replicates a local directory as a manifest file and uploads it // using dpa store // TODO: localpath should point to a manifest +// +// DEPRECATED: Use the HTTP API instead func (self *FileSystem) Upload(lpath, index string) (string, error) { var list []*manifestTrieEntry localpath, err := filepath.Abs(filepath.Clean(lpath)) @@ -65,16 +68,13 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { log.Debug(fmt.Sprintf("uploading '%s'", localpath)) err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error { if (err == nil) && !info.IsDir() { - //fmt.Printf("lp %s path %s\n", localpath, path) if len(path) <= start { return fmt.Errorf("Path is too short") } if path[:start] != localpath { return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath) } - entry := &manifestTrieEntry{ - Path: filepath.ToSlash(path), - } + entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil) list = append(list, entry) } return err @@ -91,9 +91,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { if localpath[:start] != dir { return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir) } - entry := &manifestTrieEntry{ - Path: filepath.ToSlash(localpath), - } + entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil) list = append(list, entry) } @@ -153,11 +151,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { } entry.Path = RegularSlashes(entry.Path[start:]) if entry.Path == index { - ientry := &manifestTrieEntry{ - Path: "", - Hash: entry.Hash, + ientry := newManifestTrieEntry(&ManifestEntry{ ContentType: entry.ContentType, - } + }, nil) + ientry.Hash = entry.Hash trie.addEntry(ientry, quitC) } trie.addEntry(entry, quitC) @@ -172,8 +169,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { return hs, err2 } -// Download replicates the manifest path structure on the local filesystem +// Download replicates the manifest basePath structure on the local filesystem // under localpath +// +// DEPRECATED: Use the HTTP API instead func (self *FileSystem) Download(bzzpath, localpath string) error { lpath, err := filepath.Abs(filepath.Clean(localpath)) if err != nil { @@ -185,10 +184,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } //resolving host and port - key, _, path, err := self.api.parseAndResolve(bzzpath, true) + uri, err := Parse(path.Join("bzz:/", bzzpath)) + if err != nil { + return err + } + key, err := self.api.Resolve(uri) if err != nil { return err } + path := uri.Path if len(path) > 0 { path += "/" @@ -264,7 +268,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path string) error { - f, err := os.Create(path) // TODO: path separators + f, err := os.Create(path) // TODO: basePath separators if err != nil { return err } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 4a27cb1da..8a15e735d 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -23,6 +23,9 @@ import ( "path/filepath" "sync" "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage" ) var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") @@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) { t.Fatalf("unexpected error: %v", err) } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index.html") + resp := testGet(t, api, bzzhash, "index.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) content = readPath(t, "testdata", "test0", "index.css") - resp = testGet(t, api, bzzhash+"/index.css") + resp = testGet(t, api, bzzhash, "index.css") exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, err = api.Get(bzzhash, true) + key := storage.Key(common.Hex2Bytes(bzzhash)) + _, _, _, err = api.Get(key, "") if err == nil { t.Fatalf("expected error: %v", err) } @@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) { return } - bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true) + key := storage.Key(common.Hex2Bytes(bzzhash)) + key, err = api.Modify(key, "index.html", "", "") if err != nil { t.Errorf("unexpected error: %v", err) return @@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true) + key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8") if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true) + key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8") if err != nil { t.Errorf("unexpected error: %v", err) return } + bzzhash = key.String() content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index2.html") + resp := testGet(t, api, bzzhash, "index2.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) - resp = testGet(t, api, bzzhash+"/img/logo.png") + resp = testGet(t, api, bzzhash, "img/logo.png") exp = expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) content = readPath(t, "testdata", "test0", "index.css") - resp = testGet(t, api, bzzhash+"/index.css") + resp = testGet(t, api, bzzhash, "index.css") exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, err = api.Get(bzzhash, true) + _, _, _, err = api.Get(key, "") if err == nil { t.Errorf("expected error: %v", err) } @@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, bzzhash, "") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) @@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash+"/index.html") + resp := testGet(t, api, bzzhash, "index.html") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) @@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) { } content := readPath(t, "testdata", "test0", "index.html") - resp := testGet(t, api, bzzhash) + resp := testGet(t, api, bzzhash, "") exp := expResponse(content, "text/html; charset=utf-8", 0) checkResponse(t, resp, exp) }) diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go index fc74f5d3a..f99c4f35e 100644 --- a/swarm/api/http/roundtripper_test.go +++ b/swarm/api/http/roundtripper_test.go @@ -18,14 +18,14 @@ package http import ( "io/ioutil" + "net" "net/http" + "net/http/httptest" "strings" "testing" "time" ) -const port = "3222" - func TestRoundTripper(t *testing.T) { serveMux := http.NewServeMux() serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) { http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) } }) - go http.ListenAndServe(":"+port, serveMux) - rt := &RoundTripper{Port: port} + srv := httptest.NewServer(serveMux) + defer srv.Close() + + host, port, _ := net.SplitHostPort(srv.Listener.Addr().String()) + rt := &RoundTripper{Host: host, Port: port} trans := &http.Transport{} trans.RegisterProtocol("bzz", rt) client := &http.Client{Transport: trans} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index a61696678..849b9e10f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -20,13 +20,19 @@ A simple http server interface to Swarm package http import ( - "bytes" + "archive/tar" + "encoding/json" + "errors" "fmt" "io" + "io/ioutil" + "mime" + "mime/multipart" "net/http" - "regexp" + "os" + "path" + "strconv" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -36,29 +42,9 @@ import ( "github.com/rs/cors" ) -const ( - rawType = "application/octet-stream" -) - -var ( - // accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw) - bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+") - trailingSlashes = regexp.MustCompile("/+$") - rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$") - // forever = func() time.Time { return time.Unix(0, 0) } - forever = time.Now -) - -type sequentialReader struct { - reader io.Reader - pos int64 - ahead map[int64](chan bool) - lock sync.Mutex -} - -// Server is the basic configuration needs for the HTTP server and also +// ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. -type Server struct { +type ServerConfig struct { Addr string CorsString string } @@ -69,262 +55,594 @@ type Server struct { // https://github.com/atom/electron/blob/master/docs/api/protocol.md // starts up http server -func StartHttpServer(api *api.Api, server *Server) { - serveMux := http.NewServeMux() - serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - handler(w, r, api) - }) +func StartHttpServer(api *api.Api, config *ServerConfig) { var allowedOrigins []string - for _, domain := range strings.Split(server.CorsString, ",") { + for _, domain := range strings.Split(config.CorsString, ",") { allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) } c := cors.New(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"}, MaxAge: 600, + AllowedHeaders: []string{"*"}, }) - hdlr := c.Handler(serveMux) + hdlr := c.Handler(NewServer(api)) - go http.ListenAndServe(server.Addr, hdlr) - log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr)) + go http.ListenAndServe(config.Addr, hdlr) + log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr)) } -func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { - requestURL := r.URL - // This is wrong - // if requestURL.Host == "" { - // var err error - // requestURL, err = url.Parse(r.Referer() + requestURL.String()) - // if err != nil { - // http.Error(w, err.Error(), http.StatusBadRequest) - // return - // } - // } - log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))) - uri := requestURL.Path - var raw, nameresolver bool - var proto string - - // HTTP-based URL protocol handler - log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri)) - - path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string { - proto = p - return "" - }) +func NewServer(api *api.Api) *Server { + return &Server{api} +} + +type Server struct { + api *api.Api +} - // protocol identification (ugly) - if proto == "" { - log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri)) - http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest) +// Request wraps http.Request and also includes the parsed bzz URI +type Request struct { + http.Request + + uri *api.URI +} + +// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request +// body in swarm and returns the resulting storage key as a text/plain response +func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { + if r.uri.Path != "" { + s.BadRequest(w, r, "raw POST request cannot contain a path") + return + } + + if r.Header.Get("Content-Length") == "" { + s.BadRequest(w, r, "missing Content-Length header in request") return } - if len(proto) > 4 { - raw = proto[1:5] == "bzzr" - nameresolver = proto[1:5] != "bzzi" + + key, err := s.api.Store(r.Body, r.ContentLength, nil) + if err != nil { + s.Error(w, r, err) + return } + s.logDebug("content for %s stored", key.Log()) + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, key) +} - log.Debug("", "msg", log.Lazy{Fn: func() string { - return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path) - }}) +// HandlePostFiles handles a POST request (or deprecated PUT request) to +// bzz:/<hash>/<path> which contains either a single file or multiple files +// (either a tar archive or multipart form), adds those files either to an +// existing manifest or to a new manifest under <path> and returns the +// resulting manifest hash as a text/plain response +func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { + contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + s.BadRequest(w, r, err.Error()) + return + } - switch { - case r.Method == "POST" || r.Method == "PUT": - if r.Header.Get("content-length") == "" { - http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest) + var key storage.Key + if r.uri.Addr != "" { + key, err = s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } - key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil) - if err == nil { - log.Debug(fmt.Sprintf("Content for %v stored", key.Log())) - } else { - http.Error(w, err.Error(), http.StatusBadRequest) + } else { + key, err = s.api.NewManifest() + if err != nil { + s.Error(w, r, err) return } - if r.Method == "POST" { - if raw { - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key)))) - } else { - http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest) - return + } + + newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { + switch contentType { + + case "application/x-tar": + return s.handleTarUpload(r, mw) + + case "multipart/form-data": + return s.handleMultipartUpload(r, params["boundary"], mw) + + default: + return s.handleDirectUpload(r, mw) + } + }) + if err != nil { + s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, newKey) +} + +func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error { + tr := tar.NewReader(req.Body) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("error reading tar stream: %s", err) + } + + // only store regular files + if !hdr.FileInfo().Mode().IsRegular() { + continue + } + + // add the entry under the path from the request + path := path.Join(req.uri.Path, hdr.Name) + entry := &api.ManifestEntry{ + Path: path, + ContentType: hdr.Xattrs["user.swarm.content-type"], + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + contentKey, err := mw.AddEntry(tr, entry) + if err != nil { + return fmt.Errorf("error adding manifest entry from tar stream: %s", err) + } + s.logDebug("content for %s stored", contentKey.Log()) + } +} + +func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error { + mr := multipart.NewReader(req.Body, boundary) + for { + part, err := mr.NextPart() + if err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("error reading multipart form: %s", err) + } + + var size int64 + var reader io.Reader = part + if contentLength := part.Header.Get("Content-Length"); contentLength != "" { + size, err = strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return fmt.Errorf("error parsing multipart content length: %s", err) } + reader = part } else { - // PUT - if raw { - http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest) - return - } else { - path = api.RegularSlashes(path) - mime := r.Header.Get("Content-Type") - // TODO proper root hash separation - log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime)) - newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver) - if err == nil { - log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) - } else { - http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest) - return - } + // copy the part to a tmp file to get its size + tmp, err := ioutil.TempFile("", "swarm-multipart") + if err != nil { + return err } - } - case r.Method == "DELETE": - if raw { - http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest) - return - } else { - path = api.RegularSlashes(path) - log.Debug(fmt.Sprintf("Delete '%s'.", path)) - newKey, err := a.Modify(path, "", "", nameresolver) - if err == nil { - log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) - w.Header().Set("Content-Type", "text/plain") - http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) - } else { - http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest) - return + defer os.Remove(tmp.Name()) + defer tmp.Close() + size, err = io.Copy(tmp, part) + if err != nil { + return fmt.Errorf("error copying multipart content: %s", err) + } + if _, err := tmp.Seek(0, os.SEEK_SET); err != nil { + return fmt.Errorf("error copying multipart content: %s", err) } + reader = tmp + } + + // add the entry under the path from the request + name := part.FileName() + if name == "" { + name = part.FormName() + } + path := path.Join(req.uri.Path, name) + entry := &api.ManifestEntry{ + Path: path, + ContentType: part.Header.Get("Content-Type"), + Size: size, + ModTime: time.Now(), } - case r.Method == "GET" || r.Method == "HEAD": - path = trailingSlashes.ReplaceAllString(path, "") - if path == "" { - http.Error(w, "Empty path not allowed", http.StatusBadRequest) + s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size) + contentKey, err := mw.AddEntry(reader, entry) + if err != nil { + return fmt.Errorf("error adding manifest entry from multipart form: %s", err) + } + s.logDebug("content for %s stored", contentKey.Log()) + } +} + +func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error { + key, err := mw.AddEntry(req.Body, &api.ManifestEntry{ + Path: req.uri.Path, + ContentType: req.Header.Get("Content-Type"), + Mode: 0644, + Size: req.ContentLength, + ModTime: time.Now(), + }) + if err != nil { + return err + } + s.logDebug("content for %s stored", key.Log()) + return nil +} + +// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes +// <path> from <manifest> and returns the resulting manifest hash as a +// text/plain response +func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { + s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log()) + return mw.RemoveEntry(r.uri.Path) + }) + if err != nil { + s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, newKey) +} + +// HandleGetRaw handles a GET request to bzzr://<key> and responds with +// the raw content stored at the given storage key +func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + // if path is set, interpret <key> as a manifest and return the + // raw entry at the given path + if r.uri.Path != "" { + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) return } - if raw { - var reader storage.LazySectionReader - parsedurl, _ := api.Parse(path) - - if parsedurl == path { - key, err := a.Resolve(parsedurl, nameresolver) - if err != nil { - log.Error(fmt.Sprintf("%v", err)) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - reader = a.Retrieve(key) - } else { - var status int - readertmp, _, status, err := a.Get(path, nameresolver) - if err != nil { - http.Error(w, err.Error(), status) - return - } - reader = readertmp + var entry *api.ManifestEntry + walker.Walk(func(e *api.ManifestEntry) error { + // if the entry matches the path, set entry and stop + // the walk + if e.Path == r.uri.Path { + entry = e + // return an error to cancel the walk + return errors.New("found") } - // retrieving content - - quitC := make(chan bool) - size, err := reader.Size(quitC) - if err != nil { - log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error())) - //An error on call to Size means we don't have the root chunk - http.Error(w, err.Error(), http.StatusNotFound) - return + // ignore non-manifest files + if e.ContentType != api.ManifestType { + return nil } - log.Debug(fmt.Sprintf("Reading %d bytes.", size)) - // setting mime type - qv := requestURL.Query() - mimeType := qv.Get("content_type") - if mimeType == "" { - mimeType = rawType + // if the manifest's path is a prefix of the + // requested path, recurse into it by returning + // nil and continuing the walk + if strings.HasPrefix(r.uri.Path, e.Path) { + return nil } - w.Header().Set("Content-Type", mimeType) - http.ServeContent(w, r, uri, forever(), reader) - log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType)) + return api.SkipManifest + }) + if entry == nil { + http.NotFound(w, &r.Request) + return + } + key = storage.Key(common.Hex2Bytes(entry.Hash)) + } - // retrieve path via manifest - } else { - log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri)) - // add trailing slash, if missing - if rootDocumentUri.MatchString(uri) { - http.Redirect(w, r, path+"/", http.StatusFound) - return + // check the root chunk exists by retrieving the file's size + reader := s.api.Retrieve(key) + if _, err := reader.Size(nil); err != nil { + s.logDebug("key not found %s: %s", key, err) + http.NotFound(w, &r.Request) + return + } + + // allow the request to overwrite the content type using a query + // parameter + contentType := "application/octet-stream" + if typ := r.URL.Query().Get("content_type"); typ != "" { + contentType = typ + } + w.Header().Set("Content-Type", contentType) + + http.ServeContent(w, &r.Request, "", time.Now(), reader) +} + +// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept +// header of "application/x-tar" and returns a tar stream of all files +// contained in the manifest +func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { + if r.uri.Path != "" { + s.BadRequest(w, r, "files request cannot contain a path") + return + } + + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.Error(w, r, err) + return + } + + tw := tar.NewWriter(w) + defer tw.Close() + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(http.StatusOK) + + err = walker.Walk(func(entry *api.ManifestEntry) error { + // ignore manifests (walk will recurse into them) + if entry.ContentType == api.ManifestType { + return nil + } + + // retrieve the entry's key and size + reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) + size, err := reader.Size(nil) + if err != nil { + return err + } + + // write a tar header for the entry + hdr := &tar.Header{ + Name: entry.Path, + Mode: entry.Mode, + Size: size, + ModTime: entry.ModTime, + Xattrs: map[string]string{ + "user.swarm.content-type": entry.ContentType, + }, + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + // copy the file into the tar stream + n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) + if err != nil { + return err + } else if n != size { + return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) + } + + return nil + }) + if err != nil { + s.logError("error generating tar stream: %s", err) + } +} + +// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has +// the "list" query parameter set to "true" and returns a list of all files +// contained in <manifest> under <path> grouped into common prefixes using +// "/" as a delimiter +func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { + // ensure the root path has a trailing slash so that relative URLs work + if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { + http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently) + return + } + + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + walker, err := s.api.NewManifestWalker(key, nil) + if err != nil { + s.Error(w, r, err) + return + } + + var list api.ManifestList + prefix := r.uri.Path + err = walker.Walk(func(entry *api.ManifestEntry) error { + // handle non-manifest files + if entry.ContentType != api.ManifestType { + // ignore the file if it doesn't have the specified prefix + if !strings.HasPrefix(entry.Path, prefix) { + return nil } - reader, mimeType, status, err := a.Get(path, nameresolver) - if err != nil { - if _, ok := err.(api.ErrResolve); ok { - log.Debug(fmt.Sprintf("%v", err)) - status = http.StatusBadRequest - } else { - log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err)) - status = http.StatusNotFound - } - http.Error(w, err.Error(), status) - return + + // if the path after the prefix contains a slash, add a + // common prefix to the list, otherwise add the entry + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return nil } - // set mime type and status headers - w.Header().Set("Content-Type", mimeType) - if status > 0 { - w.WriteHeader(status) - } else { - status = 200 + if entry.Path == "" { + entry.Path = "/" } - quitC := make(chan bool) - size, err := reader.Size(quitC) - if err != nil { - log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error())) - //An error on call to Size means we don't have the root chunk - http.Error(w, err.Error(), http.StatusNotFound) - return + list.Entries = append(list.Entries, entry) + return nil + } + + // if the manifest's path is a prefix of the specified prefix + // then just recurse into the manifest by returning nil and + // continuing the walk + if strings.HasPrefix(prefix, entry.Path) { + return nil + } + + // if the manifest's path has the specified prefix, then if the + // path after the prefix contains a slash, add a common prefix + // to the list and skip the manifest, otherwise recurse into + // the manifest by returning nil and continuing the walk + if strings.HasPrefix(entry.Path, prefix) { + suffix := strings.TrimPrefix(entry.Path, prefix) + if index := strings.Index(suffix, "/"); index > -1 { + list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) + return api.SkipManifest } - log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status)) + return nil + } - http.ServeContent(w, r, path, forever(), reader) + // the manifest neither has the prefix or needs recursing in to + // so just skip it + return api.SkipManifest + }) + if err != nil { + s.Error(w, r, err) + return + } + // if the client wants HTML (e.g. a browser) then render the list as a + // HTML index with relative URLs + if strings.Contains(r.Header.Get("Accept"), "text/html") { + w.Header().Set("Content-Type", "text/html") + err := htmlListTemplate.Execute(w, &htmlListData{ + URI: r.uri, + List: &list, + }) + if err != nil { + s.logError("error rendering list HTML: %s", err) } - default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(&list) +} + +// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds +// with the content of the file at <path> from the given <manifest> +func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { + key, err := s.api.Resolve(r.uri) + if err != nil { + s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) + return + } + + reader, contentType, _, err := s.api.Get(key, r.uri.Path) + if err != nil { + s.Error(w, r, err) + return + } + + // check the root chunk exists by retrieving the file's size + if _, err := reader.Size(nil); err != nil { + s.logDebug("file not found %s: %s", r.uri, err) + http.NotFound(w, &r.Request) + return } + + w.Header().Set("Content-Type", contentType) + + http.ServeContent(w, &r.Request, "", time.Now(), reader) } -func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) { - self.lock.Lock() - // assert self.pos <= off - if self.pos > off { - log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off)) - panic("Non-sequential read attempt") - } - if self.pos != off { - log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off)) - wait := make(chan bool) - self.ahead[off] = wait - self.lock.Unlock() - if <-wait { - // failed read behind - n = 0 - err = io.ErrUnexpectedEOF +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + if err != nil { + s.logError("Invalid URI %q: %s", r.URL.Path, err) + http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest) + return + } + s.logDebug("%s request received for %s", r.Method, uri) + + req := &Request{Request: *r, uri: uri} + switch r.Method { + case "POST": + if uri.Raw() { + s.HandlePostRaw(w, req) + } else { + s.HandlePostFiles(w, req) + } + + case "PUT": + // DEPRECATED: + // clients should send a POST request (the request creates a + // new manifest leaving the existing one intact, so it isn't + // strictly a traditional PUT request which replaces content + // at a URI, and POST is more ubiquitous) + if uri.Raw() { + http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return + } else { + s.HandlePostFiles(w, req) } - self.lock.Lock() - } - localPos := 0 - for localPos < len(target) { - n, err = self.reader.Read(target[localPos:]) - localPos += n - log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err)) - if err != nil { - log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err)) - for i := range self.ahead { - self.ahead[i] <- true - delete(self.ahead, i) - } - self.lock.Unlock() - return localPos, err + + case "DELETE": + if uri.Raw() { + http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + return + } + s.HandleDelete(w, req) + + case "GET": + if uri.Raw() { + s.HandleGetRaw(w, req) + return } - self.pos += int64(n) + + if r.Header.Get("Accept") == "application/x-tar" { + s.HandleGetFiles(w, req) + return + } + + if r.URL.Query().Get("list") == "true" { + s.HandleGetList(w, req) + return + } + + s.HandleGetFile(w, req) + + default: + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + } - wait := self.ahead[self.pos] - if wait != nil { - log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos)) - delete(self.ahead, self.pos) - close(wait) +} + +func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) { + mw, err := s.api.NewManifestWriter(key, nil) + if err != nil { + return nil, err } - self.lock.Unlock() - return localPos, err + + if err := update(mw); err != nil { + return nil, err + } + + key, err = mw.Store() + if err != nil { + return nil, err + } + s.logDebug("generated manifest %s", key) + return key, nil +} + +func (s *Server) logDebug(format string, v ...interface{}) { + log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) +} + +func (s *Server) logError(format string, v ...interface{}) { + log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...)) +} + +func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { + s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason) + http.Error(w, reason, http.StatusBadRequest) +} + +func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { + s.logError("error serving %s %s: %s", r.Method, r.uri, err) + http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 88b49b9a5..ceb8db75b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package http +package http_test import ( "bytes" @@ -22,19 +22,16 @@ import ( "net/http" "sync" "testing" - "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/testutil" ) func TestBzzrGetPath(t *testing.T) { var err error - maxproxyattempts := 3 - testmanifest := []string{ `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, @@ -43,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) { testrequests := make(map[string]int) testrequests["/"] = 0 - testrequests["/a"] = 1 - testrequests["/a/b"] = 2 + testrequests["/a/"] = 1 + testrequests["/a/b/"] = 2 testrequests["/x"] = 0 testrequests[""] = 0 @@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) { key := [3]storage.Key{} - dir, _ := ioutil.TempDir("", "bzz-storage-test") - - storeparams := &storage.StoreParams{ - ChunkDbPath: dir, - DbCapacity: 5000000, - CacheCapacity: 5000, - Radius: 0, - } - - localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) - if err != nil { - t.Fatal(err) - } - chunker := storage.NewTreeChunker(storage.NewChunkerParams()) - dpa := &storage.DPA{ - Chunker: chunker, - ChunkStore: localStore, - } - dpa.Start() - defer dpa.Stop() + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() wg := &sync.WaitGroup{} for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) - key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) + key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil) if err != nil { t.Fatal(err) } wg.Wait() } - a := api.NewApi(dpa, nil) - - /// \todo iterate port numbers up if fail - StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""}) - // how to wait for ListenAndServe to have initialized? This is pretty cruuuude - // if we fix it we don't need maxproxyattempts anymore either - time.Sleep(1000 * time.Millisecond) - for i := 0; i <= maxproxyattempts; i++ { - _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") - if i == maxproxyattempts { - t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err) - } else if err != nil { - time.Sleep(100 * time.Millisecond) - continue - } - break + _, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a") + if err != nil { + t.Fatalf("Failed to connect to proxy: %v", err) } for k, v := range testrequests { var resp *http.Response var respbody []byte - url := "http://127.0.0.1:8504/bzzr:/" + url := srv.URL + "/bzzr:/" if k[:] != "" { url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" } @@ -133,4 +99,32 @@ func TestBzzrGetPath(t *testing.T) { } } + nonhashtests := []string{ + srv.URL + "/bzz:/name", + srv.URL + "/bzzi:/nonhash", + srv.URL + "/bzzr:/nonhash", + } + + nonhashresponses := []string{ + "error resolving name: 'name' does not resolve: no DNS to resolve name but is not a content hash\n", + "error resolving nonhash: 'nonhash' is not a content hash\n", + "error resolving nonhash: 'nonhash' does not resolve: no DNS to resolve name but is not a content hash\n", + } + + for i, url := range nonhashtests { + var resp *http.Response + var respbody []byte + + resp, err = http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + if string(respbody) != nonhashresponses[i] { + t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) + } + } + } diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go new file mode 100644 index 000000000..c3ef8c0f4 --- /dev/null +++ b/swarm/api/http/templates.go @@ -0,0 +1,71 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package http + +import ( + "html/template" + "path" + + "github.com/ethereum/go-ethereum/swarm/api" +) + +type htmlListData struct { + URI *api.URI + List *api.ManifestList +} + +var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(` +<!DOCTYPE html> +<html> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>Swarm index of {{ .URI }}</title> +</head> + +<body> + <h1>Swarm index of {{ .URI }}</h1> + <hr> + <table> + <thead> + <tr> + <th>Path</th> + <th>Type</th> + <th>Size</th> + </tr> + </thead> + + <tbody> + {{ range .List.CommonPrefixes }} + <tr> + <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td> + <td>DIR</td> + <td>-</td> + </tr> + {{ end }} + + {{ range .List.Entries }} + <tr> + <td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td> + <td>{{ .ContentType }}</td> + <td>{{ .Size }}</td> + </tr> + {{ end }} + </table> + <hr> +</body> +`[1:])) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 199f259e1..dbaaf4bff 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -19,8 +19,11 @@ package api import ( "bytes" "encoding/json" + "errors" "fmt" + "io" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" @@ -28,25 +31,152 @@ import ( ) const ( - manifestType = "application/bzz-manifest+json" + ManifestType = "application/bzz-manifest+json" ) +// Manifest represents a swarm manifest +type Manifest struct { + Entries []ManifestEntry `json:"entries,omitempty"` +} + +// ManifestEntry represents an entry in a swarm manifest +type ManifestEntry struct { + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + ContentType string `json:"contentType,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size,omitempty"` + ModTime time.Time `json:"mod_time,omitempty"` + Status int `json:"status,omitempty"` +} + +// ManifestList represents the result of listing files in a manifest +type ManifestList struct { + CommonPrefixes []string `json:"common_prefixes,omitempty"` + Entries []*ManifestEntry `json:"entries,omitempty"` +} + +// NewManifest creates and stores a new, empty manifest +func (a *Api) NewManifest() (storage.Key, error) { + var manifest Manifest + data, err := json.Marshal(&manifest) + if err != nil { + return nil, err + } + return a.Store(bytes.NewReader(data), int64(len(data)), nil) +} + +// ManifestWriter is used to add and remove entries from an underlying manifest +type ManifestWriter struct { + api *Api + trie *manifestTrie + quitC chan bool +} + +func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) { + trie, err := loadManifest(a.dpa, key, quitC) + if err != nil { + return nil, fmt.Errorf("error loading manifest %s: %s", key, err) + } + return &ManifestWriter{a, trie, quitC}, nil +} + +// AddEntry stores the given data and adds the resulting key to the manifest +func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { + key, err := m.api.Store(data, e.Size, nil) + if err != nil { + return nil, err + } + entry := newManifestTrieEntry(e, nil) + entry.Hash = key.String() + m.trie.addEntry(entry, m.quitC) + return key, nil +} + +// RemoveEntry removes the given path from the manifest +func (m *ManifestWriter) RemoveEntry(path string) error { + m.trie.deleteEntry(path, m.quitC) + return nil +} + +// Store stores the manifest, returning the resulting storage key +func (m *ManifestWriter) Store() (storage.Key, error) { + return m.trie.hash, m.trie.recalcAndStore() +} + +// ManifestWalker is used to recursively walk the entries in the manifest and +// all of its submanifests +type ManifestWalker struct { + api *Api + trie *manifestTrie + quitC chan bool +} + +func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) { + trie, err := loadManifest(a.dpa, key, quitC) + if err != nil { + return nil, fmt.Errorf("error loading manifest %s: %s", key, err) + } + return &ManifestWalker{a, trie, quitC}, nil +} + +// SkipManifest is used as a return value from WalkFn to indicate that the +// manifest should be skipped +var SkipManifest = errors.New("skip this manifest") + +// WalkFn is the type of function called for each entry visited by a recursive +// manifest walk +type WalkFn func(entry *ManifestEntry) error + +// Walk recursively walks the manifest calling walkFn for each entry in the +// manifest, including submanifests +func (m *ManifestWalker) Walk(walkFn WalkFn) error { + return m.walk(m.trie, "", walkFn) +} + +func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error { + for _, entry := range trie.entries { + if entry == nil { + continue + } + entry.Path = prefix + entry.Path + err := walkFn(&entry.ManifestEntry) + if err != nil { + if entry.ContentType == ManifestType && err == SkipManifest { + continue + } + return err + } + if entry.ContentType != ManifestType { + continue + } + if err := trie.loadSubTrie(entry, nil); err != nil { + return err + } + if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil { + return err + } + } + return nil +} + type manifestTrie struct { dpa *storage.DPA - entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry + entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry hash storage.Key // if hash != nil, it is stored } -type manifestJSON struct { - Entries []*manifestTrieEntry `json:"entries"` +func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry { + return &manifestTrieEntry{ + ManifestEntry: *entry, + subtrie: subtrie, + } } type manifestTrieEntry struct { - Path string `json:"path"` - Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated - ContentType string `json:"contentType"` - Status int `json:"status"` - subtrie *manifestTrie + ManifestEntry + + subtrie *manifestTrie } func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand @@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp } log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log())) - man := manifestJSON{} + var man struct { + Entries []*manifestTrieEntry `json:"entries"` + } err = json.Unmarshal(manifestData, &man) if err != nil { err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) @@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { cpl++ } - if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { + if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) { if self.loadSubTrie(oldentry, quitC) != nil { return } @@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { subtrie.addEntry(entry, quitC) subtrie.addEntry(oldentry, quitC) - self.entries[b] = &manifestTrieEntry{ + self.entries[b] = newManifestTrieEntry(&ManifestEntry{ Path: commonPrefix, - Hash: "", - ContentType: manifestType, - subtrie: subtrie, - } + ContentType: ManifestType, + }, subtrie) } func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { @@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { } epl := len(entry.Path) - if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { + if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { if self.loadSubTrie(entry, quitC) != nil { return } @@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error { var buffer bytes.Buffer buffer.WriteString(`{"entries":[`) - list := &manifestJSON{} + list := &Manifest{} for _, entry := range self.entries { if entry != nil { if entry.Hash == "" { // TODO: paralellize @@ -208,8 +338,9 @@ func (self *manifestTrie) recalcAndStore() error { } entry.Hash = entry.subtrie.hash.String() } - list.Entries = append(list.Entries, entry) + list.Entries = append(list.Entries, entry.ManifestEntry) } + } manifest, err := json.Marshal(list) @@ -254,7 +385,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, entry := self.entries[i] if entry != nil { epl := len(entry.Path) - if entry.ContentType == manifestType { + if entry.ContentType == ManifestType { l := plen if epl < l { l = epl @@ -300,7 +431,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl)) if (len(path) >= epl) && (path[:epl] == entry.Path) { log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType)) - if entry.ContentType == manifestType { + if entry.ContentType == ManifestType { err := self.loadSubTrie(entry, quitC) if err != nil { return nil, 0 diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 31b484675..0e3abecfe 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -16,6 +16,8 @@ package api +import "path" + type Response struct { MimeType string Status int @@ -25,6 +27,8 @@ type Response struct { } // implements a service +// +// DEPRECATED: Use the HTTP API instead type Storage struct { api *Api } @@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage { // Put uploads the content to the swarm with a simple manifest speficying // its content type +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Put(content, contentType string) (string, error) { - return self.api.Put(content, contentType) + key, err := self.api.Put(content, contentType) + if err != nil { + return "", err + } + return key.String(), err } // Get retrieves the content from bzzpath and reads the response in full @@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) { // NOTE: if error is non-nil, sResponse may still have partial content // the actual size of which is given in len(resp.Content), while the expected // size is resp.Size +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Get(bzzpath string) (*Response, error) { - reader, mimeType, status, err := self.api.Get(bzzpath, true) + uri, err := Parse(path.Join("bzz:/", bzzpath)) + if err != nil { + return nil, err + } + key, err := self.api.Resolve(uri) + if err != nil { + return nil, err + } + reader, mimeType, status, err := self.api.Get(key, uri.Path) if err != nil { return nil, err } @@ -63,8 +83,22 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { return &Response{mimeType, status, expsize, string(body[:size])}, err } -// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash, +// Modify(rootHash, basePath, contentHash, contentType) takes th e manifest trie rooted in rootHash, // and merge on to it. creating an entry w conentType (mime) +// +// DEPRECATED: Use the HTTP API instead func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { - return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) + uri, err := Parse("bzz:/" + rootHash) + if err != nil { + return "", err + } + key, err := self.api.Resolve(uri) + if err != nil { + return "", err + } + key, err = self.api.Modify(key, path, contentHash, contentType) + if err != nil { + return "", err + } + return key.String(), nil } diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go index 72caf52df..d260dd61d 100644 --- a/swarm/api/storage_test.go +++ b/swarm/api/storage_test.go @@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) { t.Fatalf("unexpected error: %v", err) } // to check put against the Api#Get - resp0 := testGet(t, api.api, bzzhash) + resp0 := testGet(t, api.api, bzzhash, "") checkResponse(t, resp0, exp) // check storage#Get diff --git a/swarm/api/uri.go b/swarm/api/uri.go new file mode 100644 index 000000000..68ce04835 --- /dev/null +++ b/swarm/api/uri.go @@ -0,0 +1,96 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package api + +import ( + "fmt" + "net/url" + "strings" +) + +// URI is a reference to content stored in swarm. +type URI struct { + // Scheme has one of the following values: + // + // * bzz - an entry in a swarm manifest + // * bzzr - raw swarm content + // * bzzi - immutable URI of an entry in a swarm manifest + // (address is not resolved) + Scheme string + + // Addr is either a hexadecimal storage key or it an address which + // resolves to a storage key + Addr string + + // Path is the path to the content within a swarm manifest + Path string +} + +// Parse parses rawuri into a URI struct, where rawuri is expected to have one +// of the following formats: +// +// * <scheme>:/ +// * <scheme>:/<addr> +// * <scheme>:/<addr>/<path> +// * <scheme>:// +// * <scheme>://<addr> +// * <scheme>://<addr>/<path> +// +// with scheme one of bzz, bzzr or bzzi +func Parse(rawuri string) (*URI, error) { + u, err := url.Parse(rawuri) + if err != nil { + return nil, err + } + uri := &URI{Scheme: u.Scheme} + + // check the scheme is valid + switch uri.Scheme { + case "bzz", "bzzi", "bzzr": + default: + return nil, fmt.Errorf("unknown scheme %q", u.Scheme) + } + + // handle URIs like bzz://<addr>/<path> where the addr and path + // have already been split by url.Parse + if u.Host != "" { + uri.Addr = u.Host + uri.Path = strings.TrimLeft(u.Path, "/") + return uri, nil + } + + // URI is like bzz:/<addr>/<path> so split the addr and path from + // the raw path (which will be /<addr>/<path>) + parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2) + uri.Addr = parts[0] + if len(parts) == 2 { + uri.Path = parts[1] + } + return uri, nil +} + +func (u *URI) Raw() bool { + return u.Scheme == "bzzr" +} + +func (u *URI) Immutable() bool { + return u.Scheme == "bzzi" +} + +func (u *URI) String() string { + return u.Scheme + ":/" + u.Addr + "/" + u.Path +} diff --git a/swarm/api/uri_test.go b/swarm/api/uri_test.go new file mode 100644 index 000000000..dcb5fbbff --- /dev/null +++ b/swarm/api/uri_test.go @@ -0,0 +1,120 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package api + +import ( + "reflect" + "testing" +) + +func TestParseURI(t *testing.T) { + type test struct { + uri string + expectURI *URI + expectErr bool + expectRaw bool + expectImmutable bool + } + tests := []test{ + { + uri: "", + expectErr: true, + }, + { + uri: "foo", + expectErr: true, + }, + { + uri: "bzz", + expectErr: true, + }, + { + uri: "bzz:", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzzi:", + expectURI: &URI{Scheme: "bzzi"}, + expectImmutable: true, + }, + { + uri: "bzzr:", + expectURI: &URI{Scheme: "bzzr"}, + expectRaw: true, + }, + { + uri: "bzz:/", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzz:/abc123", + expectURI: &URI{Scheme: "bzz", Addr: "abc123"}, + }, + { + uri: "bzz:/abc123/path/to/entry", + expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, + }, + { + uri: "bzzr:/", + expectURI: &URI{Scheme: "bzzr"}, + expectRaw: true, + }, + { + uri: "bzzr:/abc123", + expectURI: &URI{Scheme: "bzzr", Addr: "abc123"}, + expectRaw: true, + }, + { + uri: "bzzr:/abc123/path/to/entry", + expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"}, + expectRaw: true, + }, + { + uri: "bzz://", + expectURI: &URI{Scheme: "bzz"}, + }, + { + uri: "bzz://abc123", + expectURI: &URI{Scheme: "bzz", Addr: "abc123"}, + }, + { + uri: "bzz://abc123/path/to/entry", + expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"}, + }, + } + for _, x := range tests { + actual, err := Parse(x.uri) + if x.expectErr { + if err == nil { + t.Fatalf("expected %s to error", x.uri) + } + continue + } + if err != nil { + t.Fatalf("error parsing %s: %s", x.uri, err) + } + if !reflect.DeepEqual(actual, x.expectURI) { + t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual) + } + if actual.Raw() != x.expectRaw { + t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw()) + } + if actual.Immutable() != x.expectImmutable { + t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable()) + } + } +} diff --git a/swarm/fuse/fuse_dir.go b/swarm/fuse/fuse_dir.go new file mode 100644 index 000000000..91b236ae8 --- /dev/null +++ b/swarm/fuse/fuse_dir.go @@ -0,0 +1,155 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "golang.org/x/net/context" + "os" + "path/filepath" + "sync" +) + +var ( + _ fs.Node = (*SwarmDir)(nil) + _ fs.NodeRequestLookuper = (*SwarmDir)(nil) + _ fs.HandleReadDirAller = (*SwarmDir)(nil) + _ fs.NodeCreater = (*SwarmDir)(nil) + _ fs.NodeRemover = (*SwarmDir)(nil) + _ fs.NodeMkdirer = (*SwarmDir)(nil) +) + +type SwarmDir struct { + inode uint64 + name string + path string + directories []*SwarmDir + files []*SwarmFile + + mountInfo *MountInfo + lock *sync.RWMutex +} + +func NewSwarmDir(fullpath string, minfo *MountInfo) *SwarmDir { + newdir := &SwarmDir{ + inode: NewInode(), + name: filepath.Base(fullpath), + path: fullpath, + directories: []*SwarmDir{}, + files: []*SwarmFile{}, + mountInfo: minfo, + lock: &sync.RWMutex{}, + } + return newdir +} + +func (sd *SwarmDir) Attr(ctx context.Context, a *fuse.Attr) error { + a.Inode = sd.inode + a.Mode = os.ModeDir | 0700 + a.Uid = uint32(os.Getuid()) + a.Gid = uint32(os.Getegid()) + return nil +} + +func (sd *SwarmDir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { + + for _, n := range sd.files { + if n.name == req.Name { + return n, nil + } + } + for _, n := range sd.directories { + if n.name == req.Name { + return n, nil + } + } + return nil, fuse.ENOENT +} + +func (sd *SwarmDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + var children []fuse.Dirent + for _, file := range sd.files { + children = append(children, fuse.Dirent{Inode: file.inode, Type: fuse.DT_File, Name: file.name}) + } + for _, dir := range sd.directories { + children = append(children, fuse.Dirent{Inode: dir.inode, Type: fuse.DT_Dir, Name: dir.name}) + } + return children, nil +} + +func (sd *SwarmDir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + + newFile := NewSwarmFile(sd.path, req.Name, sd.mountInfo) + newFile.fileSize = 0 // 0 means, file is not in swarm yet and it is just created + + sd.lock.Lock() + defer sd.lock.Unlock() + sd.files = append(sd.files, newFile) + + return newFile, newFile, nil +} + +func (sd *SwarmDir) Remove(ctx context.Context, req *fuse.RemoveRequest) error { + + if req.Dir && sd.directories != nil { + newDirs := []*SwarmDir{} + for _, dir := range sd.directories { + if dir.name == req.Name { + removeDirectoryFromSwarm(dir) + } else { + newDirs = append(newDirs, dir) + } + } + if len(sd.directories) > len(newDirs) { + sd.lock.Lock() + defer sd.lock.Unlock() + sd.directories = newDirs + } + return nil + } else if !req.Dir && sd.files != nil { + newFiles := []*SwarmFile{} + for _, f := range sd.files { + if f.name == req.Name { + removeFileFromSwarm(f) + } else { + newFiles = append(newFiles, f) + } + } + if len(sd.files) > len(newFiles) { + sd.lock.Lock() + defer sd.lock.Unlock() + sd.files = newFiles + } + return nil + } + return fuse.ENOENT +} + +func (sd *SwarmDir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { + + newDir := NewSwarmDir(req.Name, sd.mountInfo) + + sd.lock.Lock() + defer sd.lock.Unlock() + sd.directories = append(sd.directories, newDir) + + return newDir, nil + +} diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go new file mode 100644 index 000000000..0cb59dfb3 --- /dev/null +++ b/swarm/fuse/fuse_file.go @@ -0,0 +1,144 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "errors" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage" + "golang.org/x/net/context" + "io" + "os" + "sync" +) + +const ( + MaxAppendFileSize = 10485760 // 10Mb +) + +var ( + errInvalidOffset = errors.New("Invalid offset during write") + errFileSizeMaxLimixReached = errors.New("File size exceeded max limit") +) + +var ( + _ fs.Node = (*SwarmFile)(nil) + _ fs.HandleReader = (*SwarmFile)(nil) + _ fs.HandleWriter = (*SwarmFile)(nil) +) + +type SwarmFile struct { + inode uint64 + name string + path string + key storage.Key + fileSize int64 + reader storage.LazySectionReader + + mountInfo *MountInfo + lock *sync.RWMutex +} + +func NewSwarmFile(path, fname string, minfo *MountInfo) *SwarmFile { + newFile := &SwarmFile{ + inode: NewInode(), + name: fname, + path: path, + key: nil, + fileSize: -1, // -1 means , file already exists in swarm and you need to just get the size from swarm + reader: nil, + + mountInfo: minfo, + lock: &sync.RWMutex{}, + } + return newFile +} + +func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { + + a.Inode = file.inode + //TODO: need to get permission as argument + a.Mode = 0700 + a.Uid = uint32(os.Getuid()) + a.Gid = uint32(os.Getegid()) + + if file.fileSize == -1 { + reader := file.mountInfo.swarmApi.Retrieve(file.key) + quitC := make(chan bool) + size, err := reader.Size(quitC) + if err != nil { + log.Warn("Couldnt get size of file %s : %v", file.path, err) + } + file.fileSize = int64(size) + } + a.Size = uint64(file.fileSize) + return nil +} + +func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { + + sf.lock.RLock() + defer sf.lock.RUnlock() + if sf.reader == nil { + sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key) + } + buf := make([]byte, req.Size) + n, err := sf.reader.ReadAt(buf, req.Offset) + if err == io.ErrUnexpectedEOF || err == io.EOF { + err = nil + } + resp.Data = buf[:n] + sf.reader = nil + return err + +} + +func (sf *SwarmFile) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + + if sf.fileSize == 0 && req.Offset == 0 { + + // A new file is created + err := addFileToSwarm(sf, req.Data, len(req.Data)) + if err != nil { + return err + } + resp.Size = len(req.Data) + + } else if req.Offset <= sf.fileSize { + + totalSize := sf.fileSize + int64(len(req.Data)) + if totalSize > MaxAppendFileSize { + log.Warn("Append file size reached (%v) : (%v)", sf.fileSize, len(req.Data)) + return errFileSizeMaxLimixReached + } + + err := appendToExistingFileInSwarm(sf, req.Data, req.Offset, int64(len(req.Data))) + if err != nil { + return err + } + resp.Size = int(sf.fileSize) + } else { + log.Warn("Invalid write request size(%v) : off(%v)", sf.fileSize, req.Offset) + return errInvalidOffset + } + + return nil +} diff --git a/swarm/fuse/fuse_root.go b/swarm/fuse/fuse_root.go new file mode 100644 index 000000000..b2262d1c5 --- /dev/null +++ b/swarm/fuse/fuse_root.go @@ -0,0 +1,35 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse/fs" +) + +var ( + _ fs.Node = (*SwarmDir)(nil) +) + +type SwarmRoot struct { + root *SwarmDir +} + +func (filesystem *SwarmRoot) Root() (fs.Node, error) { + return filesystem.root, nil +} diff --git a/swarm/fuse/swarmfs.go b/swarm/fuse/swarmfs.go new file mode 100644 index 000000000..2493bdab1 --- /dev/null +++ b/swarm/fuse/swarmfs.go @@ -0,0 +1,64 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +package fuse + +import ( + "github.com/ethereum/go-ethereum/swarm/api" + "sync" + "time" +) + +const ( + Swarmfs_Version = "0.1" + mountTimeout = time.Second * 5 + unmountTimeout = time.Second * 10 + maxFuseMounts = 5 +) + +var ( + swarmfs *SwarmFS // Swarm file system singleton + swarmfsLock sync.Once + + inode uint64 = 1 // global inode + inodeLock sync.RWMutex +) + +type SwarmFS struct { + swarmApi *api.Api + activeMounts map[string]*MountInfo + swarmFsLock *sync.RWMutex +} + +func NewSwarmFS(api *api.Api) *SwarmFS { + swarmfsLock.Do(func() { + swarmfs = &SwarmFS{ + swarmApi: api, + swarmFsLock: &sync.RWMutex{}, + activeMounts: map[string]*MountInfo{}, + } + }) + return swarmfs + +} + +// Inode numbers need to be unique, they are used for caching inside fuse +func NewInode() uint64 { + inodeLock.Lock() + defer inodeLock.Unlock() + inode += 1 + return inode +} diff --git a/swarm/fuse/swarmfs_fallback.go b/swarm/fuse/swarmfs_fallback.go new file mode 100644 index 000000000..4864c8689 --- /dev/null +++ b/swarm/fuse/swarmfs_fallback.go @@ -0,0 +1,51 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build !linux,!darwin,!freebsd + +package fuse + +import ( + "errors" +) + +var errNoFUSE = errors.New("FUSE is not supported on this platform") + +func isFUSEUnsupportedError(err error) bool { + return err == errNoFUSE +} + +type MountInfo struct { + MountPoint string + StartManifest string + LatestManifest string +} + +func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { + return nil, errNoFUSE +} + +func (self *SwarmFS) Unmount(mountpoint string) (bool, error) { + return false, errNoFUSE +} + +func (self *SwarmFS) Listmounts() ([]*MountInfo, error) { + return nil, errNoFUSE +} + +func (self *SwarmFS) Stop() error { + return nil +} diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go new file mode 100644 index 000000000..f307b38ea --- /dev/null +++ b/swarm/fuse/swarmfs_test.go @@ -0,0 +1,897 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "bytes" + "crypto/rand" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +type fileInfo struct { + perm uint64 + uid int + gid int + contents []byte +} + +func testFuseFileSystem(t *testing.T, f func(*api.Api)) { + + datadir, err := ioutil.TempDir("", "fuse") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + os.RemoveAll(datadir) + + dpa, err := storage.NewLocalDPA(datadir) + if err != nil { + return + } + api := api.NewApi(dpa, nil) + dpa.Start() + f(api) + dpa.Stop() +} + +func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string { + + os.RemoveAll(uploadDir) + + for fname, finfo := range files { + actualPath := filepath.Join(uploadDir, fname) + filePath := filepath.Dir(actualPath) + + err := os.MkdirAll(filePath, 0777) + if err != nil { + t.Fatalf("Error creating directory '%v' : %v", filePath, err) + } + + fd, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(finfo.perm)) + if err1 != nil { + t.Fatalf("Error creating file %v: %v", actualPath, err1) + } + + fd.Write(finfo.contents) + fd.Chown(finfo.uid, finfo.gid) + fd.Chmod(os.FileMode(finfo.perm)) + fd.Sync() + fd.Close() + } + + bzzhash, err := api.Upload(uploadDir, "") + if err != nil { + t.Fatalf("Error uploading directory %v: %v", uploadDir, err) + } + + return bzzhash +} + +func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash string, mountDir string) *SwarmFS { + + // Test Mount + os.RemoveAll(mountDir) + os.MkdirAll(mountDir, 0777) + swarmfs := NewSwarmFS(api) + _, err := swarmfs.Mount(bzzHash, mountDir) + if isFUSEUnsupportedError(err) { + t.Skip("FUSE not supported:", err) + } else if err != nil { + t.Fatalf("Error mounting hash %v: %v", bzzHash, err) + } + + found := false + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == mountDir { + if minfo.StartManifest != bzzHash || + minfo.LatestManifest != bzzHash || + minfo.fuseConnection == nil { + t.Fatalf("Error mounting: exp(%s): act(%s)", bzzHash, minfo.StartManifest) + } + found = true + } + } + + // Test listMounts + if found == false { + t.Fatalf("Error getting mounts information for %v: %v", mountDir, err) + } + + // Check if file and their attributes are as expected + compareGeneratedFileWithFileInMount(t, files, mountDir) + + return swarmfs + +} + +func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo, mountDir string) { + + err := filepath.Walk(mountDir, func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + return nil + } + fname := path[len(mountDir)+1:] + if _, ok := files[fname]; !ok { + t.Fatalf(" file %v present in mount dir and is not expected", fname) + } + return nil + }) + if err != nil { + t.Fatalf("Error walking dir %v", mountDir) + } + + for fname, finfo := range files { + + destinationFile := filepath.Join(mountDir, fname) + + dfinfo, err := os.Stat(destinationFile) + if err != nil { + t.Fatalf("Destination file %v missing in mount: %v", fname, err) + } + + if int64(len(finfo.contents)) != dfinfo.Size() { + t.Fatalf("file %v Size mismatch source (%v) vs destination(%v)", fname, int64(len(finfo.contents)), dfinfo.Size()) + } + + if dfinfo.Mode().Perm().String() != "-rwx------" { + t.Fatalf("file %v Permission mismatch source (-rwx------) vs destination(%v)", fname, dfinfo.Mode().Perm()) + } + + fileContents, err := ioutil.ReadFile(filepath.Join(mountDir, fname)) + if err != nil { + t.Fatalf("Could not readfile %v : %v", fname, err) + } + + if bytes.Compare(fileContents, finfo.contents) != 0 { + t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) + + } + + // TODO: check uid and gid + } +} + +func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { + + destinationFile := filepath.Join(testMountDir, fname) + dfinfo, err1 := os.Stat(destinationFile) + if err1 != nil { + t.Fatalf("Could not stat file %v", destinationFile) + } + if dfinfo.Size() != int64(len(contents)) { + t.Fatalf("Mismatch in size actual(%v) vs expected(%v)", dfinfo.Size(), int64(len(contents))) + } + + fd, err2 := os.OpenFile(destinationFile, os.O_RDONLY, os.FileMode(0665)) + if err2 != nil { + t.Fatalf("Could not open file %v", destinationFile) + } + newcontent := make([]byte, len(contents)) + fd.Read(newcontent) + fd.Close() + + if !bytes.Equal(contents, newcontent) { + t.Fatalf("File content mismatch expected (%v): received (%v) ", contents, newcontent) + } +} + +func getRandomBtes(size int) []byte { + contents := make([]byte, size) + rand.Read(contents) + return contents + +} + +func IsDirEmpty(name string) bool { + f, err := os.Open(name) + if err != nil { + return false + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true + } + return false +} + +func testMountListAndUnmount(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "fuse-source") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "fuse-dest") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)} + files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)} + files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)} + files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)} + files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)} + files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)} + files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)} + files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)} + files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)} + files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)} + files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + // Check unmount + _, err := swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + if !IsDirEmpty(testMountDir) { + t.Fatalf("unmount didnt work for %v", testMountDir) + } + +} + +func testMaxMounts(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir1, _ := ioutil.TempDir(os.TempDir(), "max-upload1") + bzzHash1 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir1) + mount1, _ := ioutil.TempDir(os.TempDir(), "max-mount1") + swarmfs1 := mountDir(t, api, files, bzzHash1, mount1) + defer swarmfs1.Stop() + + files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir2, _ := ioutil.TempDir(os.TempDir(), "max-upload2") + bzzHash2 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir2) + mount2, _ := ioutil.TempDir(os.TempDir(), "max-mount2") + swarmfs2 := mountDir(t, api, files, bzzHash2, mount2) + defer swarmfs2.Stop() + + files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir3, _ := ioutil.TempDir(os.TempDir(), "max-upload3") + bzzHash3 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir3) + mount3, _ := ioutil.TempDir(os.TempDir(), "max-mount3") + swarmfs3 := mountDir(t, api, files, bzzHash3, mount3) + defer swarmfs3.Stop() + + files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir4, _ := ioutil.TempDir(os.TempDir(), "max-upload4") + bzzHash4 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir4) + mount4, _ := ioutil.TempDir(os.TempDir(), "max-mount4") + swarmfs4 := mountDir(t, api, files, bzzHash4, mount4) + defer swarmfs4.Stop() + + files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir5, _ := ioutil.TempDir(os.TempDir(), "max-upload5") + bzzHash5 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir5) + mount5, _ := ioutil.TempDir(os.TempDir(), "max-mount5") + swarmfs5 := mountDir(t, api, files, bzzHash5, mount5) + defer swarmfs5.Stop() + + files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir6, _ := ioutil.TempDir(os.TempDir(), "max-upload6") + bzzHash6 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir6) + mount6, _ := ioutil.TempDir(os.TempDir(), "max-mount6") + + os.RemoveAll(mount6) + os.MkdirAll(mount6, 0777) + _, err := swarmfs.Mount(bzzHash6, mount6) + if err == nil { + t.Fatalf("Error: Going beyond max mounts %v", bzzHash6) + } + +} + +func testReMounts(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1") + bzzHash1 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir1) + testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1") + swarmfs := mountDir(t, api, files, bzzHash1, testMountDir1) + defer swarmfs.Stop() + + uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2") + bzzHash2 := createTestFilesAndUploadToSwarm(t, api, files, uploadDir2) + testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2") + + // try mounting the same hash second time + os.RemoveAll(testMountDir2) + os.MkdirAll(testMountDir2, 0777) + _, err := swarmfs.Mount(bzzHash1, testMountDir2) + if err != nil { + t.Fatalf("Error mounting hash %v", bzzHash1) + } + + // mount a different hash in already mounted point + _, err = swarmfs.Mount(bzzHash2, testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } + + // mount nonexistent hash + _, err = swarmfs.Mount("0xfea11223344", testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } + +} + +func testUnmount(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, uploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + swarmfs.Unmount(testMountDir) + + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } + } + +} + +func testUnmountWhenResourceBusy(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + actualPath := filepath.Join(testMountDir, "2.txt") + d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) + d.Write(getRandomBtes(10)) + + _, err = swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + d.Close() + + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } + } + +} +func testSeekInMultiChunkFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + // Create a new file seek the second chunk + actualPath := filepath.Join(testMountDir, "1.txt") + d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) + + d.Seek(5000, 0) + + contents := make([]byte, 1024) + d.Read(contents) + finfo := files["1.txt"] + + if bytes.Compare(finfo.contents[:6024][5000:], contents) != 0 { + t.Fatalf("File seek contents mismatch") + } + d.Close() + +} + +func testCreateNewFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", contents) + +} + +func testCreateNewFileInsideDirectory(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) + +} + +func testCreateNewFileInsideNewDirectory(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, 0777) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) + +} + +func testRemoveExistingFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "five.txt") + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "five.txt") + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveExistingFileInsideADir(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "one/five.txt") + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "one/five.txt") + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveNewlyAddedFile(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Adda a new file and remove it + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, os.FileMode(0665)) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + checkFile(t, testMountDir, "one/2.txt", contents) + + os.Remove(actualPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } + +} + +func testAddNewFileAndModifyContents(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + line1 := []byte("Line 1") + rand.Read(line1) + d.Write(line1) + d.Close() + + mi1, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, line1} + swarmfs2 := mountDir(t, api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", line1) + + mi2, err3 := swarmfs2.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + + // mount again and modify + swarmfs3 := mountDir(t, api, files, mi2.LatestManifest, testMountDir) + defer swarmfs3.Stop() + + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := []byte("Line 2") + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi3, err5 := swarmfs3.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["2.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs4 := mountDir(t, api, files, mi3.LatestManifest, testMountDir) + defer swarmfs4.Stop() + + checkFile(t, testMountDir, "2.txt", line1and2) + +} + +func testRemoveEmptyDir(api *api.Api, t *testing.T) { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777) + + mi, err3 := swarmfs1.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } + +} + +func testRemoveDirWhichHasFiles(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/five.txt") + delete(files, "two/six.txt") + + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testRemoveDirWhichHasSubDirs(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount") + + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/three/2.txt") + delete(files, "two/three/3.txt") + delete(files, "two/four/5.txt") + delete(files, "two/four/6.txt") + delete(files, "two/four/six/7.txt") + + swarmfs2 := mountDir(t, api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + +} + +func testAppendFileContentsToEnd(api *api.Api, t *testing.T) { + + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount") + + line1 := make([]byte, 10) + rand.Read(line1) + files["1.txt"] = fileInfo{0700, 333, 444, line1} + bzzHash := createTestFilesAndUploadToSwarm(t, api, files, testUploadDir) + + swarmfs1 := mountDir(t, api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() + + actualPath := filepath.Join(testMountDir, "1.txt") + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := make([]byte, 5) + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi1, err5 := swarmfs1.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v ", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["1.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs2 := mountDir(t, api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "1.txt", line1and2) + +} + +func TestSwarmFileSystem(t *testing.T) { + testFuseFileSystem(t, func(api *api.Api) { + + testMountListAndUnmount(api, t) + + testMaxMounts(api, t) + + testReMounts(api, t) + + testUnmount(api, t) + + testUnmountWhenResourceBusy(api, t) + + testSeekInMultiChunkFile(api, t) + + testCreateNewFile(api, t) + + testCreateNewFileInsideDirectory(api, t) + + testCreateNewFileInsideNewDirectory(api, t) + + testRemoveExistingFile(api, t) + + testRemoveExistingFileInsideADir(api, t) + + testRemoveNewlyAddedFile(api, t) + + testAddNewFileAndModifyContents(api, t) + + testRemoveEmptyDir(api, t) + + testRemoveDirWhichHasFiles(api, t) + + testRemoveDirWhichHasSubDirs(api, t) + + testAppendFileContentsToEnd(api, t) + + }) +} diff --git a/swarm/fuse/swarmfs_unix.go b/swarm/fuse/swarmfs_unix.go new file mode 100644 index 000000000..f4eecef24 --- /dev/null +++ b/swarm/fuse/swarmfs_unix.go @@ -0,0 +1,240 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "bazil.org/fuse" + "bazil.org/fuse/fs" + "errors" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + errEmptyMountPoint = errors.New("need non-empty mount point") + errMaxMountCount = errors.New("max FUSE mount count reached") + errMountTimeout = errors.New("mount timeout") + errAlreadyMounted = errors.New("mount point is already serving") +) + +func isFUSEUnsupportedError(err error) bool { + if perr, ok := err.(*os.PathError); ok { + return perr.Op == "open" && perr.Path == "/dev/fuse" + } + return err == fuse.ErrOSXFUSENotFound +} + +// information about every active mount +type MountInfo struct { + MountPoint string + StartManifest string + LatestManifest string + rootDir *SwarmDir + fuseConnection *fuse.Conn + swarmApi *api.Api + lock *sync.RWMutex +} + +// Inode numbers need to be unique, they are used for caching inside fuse +func newInode() uint64 { + inodeLock.Lock() + defer inodeLock.Unlock() + inode += 1 + return inode +} + +func NewMountInfo(mhash, mpoint string, sapi *api.Api) *MountInfo { + newMountInfo := &MountInfo{ + MountPoint: mpoint, + StartManifest: mhash, + LatestManifest: mhash, + rootDir: nil, + fuseConnection: nil, + swarmApi: sapi, + lock: &sync.RWMutex{}, + } + return newMountInfo +} + +func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { + + if mountpoint == "" { + return nil, errEmptyMountPoint + } + cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) + if err != nil { + return nil, err + } + + self.swarmFsLock.Lock() + defer self.swarmFsLock.Unlock() + + noOfActiveMounts := len(self.activeMounts) + if noOfActiveMounts >= maxFuseMounts { + return nil, errMaxMountCount + } + + if _, ok := self.activeMounts[cleanedMountPoint]; ok { + return nil, errAlreadyMounted + } + + log.Info(fmt.Sprintf("Attempting to mount %s ", cleanedMountPoint)) + key, manifestEntryMap, err := self.swarmApi.BuildDirectoryTree(mhash, true) + if err != nil { + return nil, err + } + + mi := NewMountInfo(mhash, cleanedMountPoint, self.swarmApi) + + dirTree := map[string]*SwarmDir{} + rootDir := NewSwarmDir("/", mi) + dirTree["/"] = rootDir + mi.rootDir = rootDir + + for suffix, entry := range manifestEntryMap { + + key = common.Hex2Bytes(entry.Hash) + fullpath := "/" + suffix + basepath := filepath.Dir(fullpath) + + parentDir := rootDir + dirUntilNow := "" + paths := strings.Split(basepath, "/") + for i := range paths { + if paths[i] != "" { + thisDir := paths[i] + dirUntilNow = dirUntilNow + "/" + thisDir + + if _, ok := dirTree[dirUntilNow]; !ok { + dirTree[dirUntilNow] = NewSwarmDir(dirUntilNow, mi) + parentDir.directories = append(parentDir.directories, dirTree[dirUntilNow]) + parentDir = dirTree[dirUntilNow] + + } else { + parentDir = dirTree[dirUntilNow] + } + + } + } + thisFile := NewSwarmFile(basepath, filepath.Base(fullpath), mi) + thisFile.key = key + + parentDir.files = append(parentDir.files, thisFile) + } + + fconn, err := fuse.Mount(cleanedMountPoint, fuse.FSName("swarmfs"), fuse.VolumeName(mhash)) + if isFUSEUnsupportedError(err) { + log.Warn("Fuse not installed", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + } else if err != nil { + fuse.Unmount(cleanedMountPoint) + log.Warn("Error mounting swarm manifest", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + } + mi.fuseConnection = fconn + + serverr := make(chan error, 1) + go func() { + log.Info(fmt.Sprintf("Serving %s at %s", mhash, cleanedMountPoint)) + filesys := &SwarmRoot{root: rootDir} + if err := fs.Serve(fconn, filesys); err != nil { + log.Warn(fmt.Sprintf("Could not Serve SwarmFileSystem error: %v", err)) + serverr <- err + } + + }() + + // Check if the mount process has an error to report. + select { + case <-time.After(mountTimeout): + fuse.Unmount(cleanedMountPoint) + return nil, errMountTimeout + + case err := <-serverr: + fuse.Unmount(cleanedMountPoint) + log.Warn("Error serving swarm FUSE FS", "mountpoint", cleanedMountPoint, "err", err) + return nil, err + + case <-fconn.Ready: + log.Info("Now serving swarm FUSE FS", "manifest", mhash, "mountpoint", cleanedMountPoint) + } + + self.activeMounts[cleanedMountPoint] = mi + return mi, nil +} + +func (self *SwarmFS) Unmount(mountpoint string) (*MountInfo, error) { + + self.swarmFsLock.Lock() + defer self.swarmFsLock.Unlock() + + cleanedMountPoint, err := filepath.Abs(filepath.Clean(mountpoint)) + if err != nil { + return nil, err + } + + mountInfo := self.activeMounts[cleanedMountPoint] + + if mountInfo == nil || mountInfo.MountPoint != cleanedMountPoint { + return nil, fmt.Errorf("%s is not mounted", cleanedMountPoint) + } + err = fuse.Unmount(cleanedMountPoint) + if err != nil { + err1 := externalUnMount(cleanedMountPoint) + if err1 != nil { + errStr := fmt.Sprintf("UnMount error: %v", err) + log.Warn(errStr) + return nil, err1 + } + } + + mountInfo.fuseConnection.Close() + delete(self.activeMounts, cleanedMountPoint) + + succString := fmt.Sprintf("UnMounting %v succeeded", cleanedMountPoint) + log.Info(succString) + + return mountInfo, nil +} + +func (self *SwarmFS) Listmounts() []*MountInfo { + self.swarmFsLock.RLock() + defer self.swarmFsLock.RUnlock() + + rows := make([]*MountInfo, 0, len(self.activeMounts)) + for _, mi := range self.activeMounts { + rows = append(rows, mi) + } + return rows +} + +func (self *SwarmFS) Stop() bool { + for mp := range self.activeMounts { + mountInfo := self.activeMounts[mp] + self.Unmount(mountInfo.MountPoint) + } + return true +} diff --git a/swarm/fuse/swarmfs_util.go b/swarm/fuse/swarmfs_util.go new file mode 100644 index 000000000..d20ab258e --- /dev/null +++ b/swarm/fuse/swarmfs_util.go @@ -0,0 +1,144 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. + +// +build linux darwin freebsd + +package fuse + +import ( + "fmt" + "github.com/ethereum/go-ethereum/log" + "os/exec" + "runtime" + "time" +) + +func externalUnMount(mountPoint string) error { + + var cmd *exec.Cmd + + switch runtime.GOOS { + + case "darwin": + cmd = exec.Command("/usr/bin/diskutil", "umount", "force", mountPoint) + + case "linux": + cmd = exec.Command("fusermount", "-u", mountPoint) + + default: + return fmt.Errorf("unmount: unimplemented") + } + + errc := make(chan error, 1) + go func() { + defer close(errc) + + if err := exec.Command("umount", mountPoint).Run(); err == nil { + return + } + errc <- cmd.Run() + }() + + select { + + case <-time.After(unmountTimeout): + return fmt.Errorf("umount timeout") + + case err := <-errc: + return err + } +} + +func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { + + fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true) + if err != nil { + return err + } + + sf.lock.Lock() + defer sf.lock.Unlock() + sf.key = fkey + sf.fileSize = int64(size) + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mhash + + log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash) + return nil + +} + +func removeFileFromSwarm(sf *SwarmFile) error { + + mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true) + if err != nil { + return err + } + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mkey + + log.Info("Removed file:", "fname", sf.name, "New Manifest hash", mkey) + return nil +} + +func removeDirectoryFromSwarm(sd *SwarmDir) error { + + if len(sd.directories) == 0 && len(sd.files) == 0 { + return nil + } + + for _, d := range sd.directories { + err := removeDirectoryFromSwarm(d) + if err != nil { + return err + } + } + + for _, f := range sd.files { + err := removeFileFromSwarm(f) + if err != nil { + return err + } + } + + return nil + +} + +func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error { + + fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true) + if err != nil { + return err + } + + sf.lock.Lock() + defer sf.lock.Unlock() + sf.key = fkey + sf.fileSize = sf.fileSize + int64(len(content)) + + sf.mountInfo.lock.Lock() + defer sf.mountInfo.lock.Unlock() + sf.mountInfo.LatestManifest = mhash + + log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash) + return nil + +} diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 8d731c038..bf976a3e1 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -116,7 +116,7 @@ func (self *Kademlia) DBCount() int { // On is the entry point called when a new nodes is added // unsafe in that node is not checked to be already active node (to be called once) func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error) { - log.Warn(fmt.Sprintf("%v", self)) + log.Debug(fmt.Sprintf("%v", self)) defer self.lock.Unlock() self.lock.Lock() diff --git a/swarm/services/swap/swap.go b/swarm/services/swap/swap.go index eb21a598d..093892e8d 100644 --- a/swarm/services/swap/swap.go +++ b/swarm/services/swap/swap.go @@ -17,6 +17,7 @@ package swap import ( + "context" "crypto/ecdsa" "fmt" "math/big" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/services/swap/swap" - "golang.org/x/net/context" ) // SwAP Swarm Accounting Protocol with diff --git a/swarm/swarm.go b/swarm/swarm.go index 44564a71d..442e68d51 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -18,6 +18,7 @@ package swarm import ( "bytes" + "context" "crypto/ecdsa" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/chequebook" "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" @@ -33,9 +35,9 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/fuse" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/storage" - "golang.org/x/net/context" ) // the swarm stack @@ -54,6 +56,7 @@ type Swarm struct { corsString string swapEnabled bool lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped + sfs *fuse.SwarmFS // need this to cleanup all the active mounts on node exit } type SwarmAPI struct { @@ -132,9 +135,13 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. // set up high level api transactOpts := bind.NewKeyedTransactor(self.privateKey) - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, self.backend) - if err != nil { - return nil, err + if backend == (*ethclient.Client)(nil) { + log.Warn("No ENS, please specify non-empty --ethapi to use domain name resolution") + } else { + self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, self.backend) + if err != nil { + return nil, err + } } log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) @@ -142,6 +149,9 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. // Manifests for Smart Hosting log.Debug(fmt.Sprintf("-> Web3 virtual server API")) + self.sfs = fuse.NewSwarmFS(self.api) + log.Debug("-> Initializing Fuse file system") + return self, nil } @@ -191,7 +201,10 @@ func (self *Swarm) Start(net *p2p.Server) error { // start swarm http proxy server if self.config.Port != "" { addr := ":" + self.config.Port - go httpapi.StartHttpServer(self.api, &httpapi.Server{Addr: addr, CorsString: self.corsString}) + go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{ + Addr: addr, + CorsString: self.corsString, + }) } log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) @@ -216,7 +229,7 @@ func (self *Swarm) Stop() error { if self.lstore != nil { self.lstore.DbStore.Close() } - + self.sfs.Stop() return self.config.Save() } @@ -237,12 +250,6 @@ func (self *Swarm) APIs() []rpc.API { { Namespace: "bzz", Version: "0.1", - Service: api.NewStorage(self.api), - Public: true, - }, - { - Namespace: "bzz", - Version: "0.1", Service: &Info{self.config, chequebook.ContractParams}, Public: true, }, @@ -250,11 +257,6 @@ func (self *Swarm) APIs() []rpc.API { { Namespace: "bzz", Version: "0.1", - Service: api.NewFileSystem(self.api), - Public: false}, - { - Namespace: "bzz", - Version: "0.1", Service: api.NewControl(self.api, self.hive), Public: false, }, @@ -264,6 +266,26 @@ func (self *Swarm) APIs() []rpc.API { Service: chequebook.NewApi(self.config.Swap.Chequebook), Public: false, }, + { + Namespace: "swarmfs", + Version: fuse.Swarmfs_Version, + Service: self.sfs, + Public: false, + }, + // storage APIs + // DEPRECATED: Use the HTTP API instead + { + Namespace: "bzz", + Version: "0.1", + Service: api.NewStorage(self.api), + Public: true, + }, + { + Namespace: "bzz", + Version: "0.1", + Service: api.NewFileSystem(self.api), + Public: false, + }, // {Namespace, Version, api.NewAdmin(self), false}, } } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go new file mode 100644 index 000000000..bf98d16eb --- /dev/null +++ b/swarm/testutil/http.go @@ -0,0 +1,56 @@ +package testutil + +import ( + "io/ioutil" + "net/http/httptest" + "os" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + httpapi "github.com/ethereum/go-ethereum/swarm/api/http" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func NewTestSwarmServer(t *testing.T) *TestSwarmServer { + dir, err := ioutil.TempDir("", "swarm-storage-test") + if err != nil { + t.Fatal(err) + } + storeparams := &storage.StoreParams{ + ChunkDbPath: dir, + DbCapacity: 5000000, + CacheCapacity: 5000, + Radius: 0, + } + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + if err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + chunker := storage.NewTreeChunker(storage.NewChunkerParams()) + dpa := &storage.DPA{ + Chunker: chunker, + ChunkStore: localStore, + } + dpa.Start() + a := api.NewApi(dpa, nil) + srv := httptest.NewServer(httpapi.NewServer(a)) + return &TestSwarmServer{ + Server: srv, + Dpa: dpa, + dir: dir, + } +} + +type TestSwarmServer struct { + *httptest.Server + + Dpa *storage.DPA + dir string +} + +func (t *TestSwarmServer) Close() { + t.Server.Close() + t.Dpa.Stop() + os.RemoveAll(t.dir) +} |