aboutsummaryrefslogtreecommitdiffstats
path: root/swarm/api/client/client.go
diff options
context:
space:
mode:
Diffstat (limited to 'swarm/api/client/client.go')
-rw-r--r--swarm/api/client/client.go551
1 files changed, 347 insertions, 204 deletions
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index ef5335be3..f9c3e51e8 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -17,18 +17,23 @@
package client
import (
+ "archive/tar"
"bytes"
"encoding/json"
+ "errors"
"fmt"
"io"
"io/ioutil"
"mime"
+ "mime/multipart"
"net/http"
+ "net/textproto"
"os"
"path/filepath"
+ "strconv"
"strings"
- "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/swarm/api"
)
var (
@@ -36,18 +41,6 @@ var (
DefaultClient = NewClient(DefaultGateway)
)
-// Manifest represents a swarm manifest.
-type Manifest struct {
- Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest.
-type ManifestEntry struct {
- Hash string `json:"hash,omitempty"`
- ContentType string `json:"contentType,omitempty"`
- Path string `json:"path,omitempty"`
-}
-
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
@@ -59,160 +52,207 @@ type Client struct {
Gateway string
}
-func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
- mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
- if err != nil {
- return "", fmt.Errorf("failed to upload empty manifest")
+// UploadRaw uploads raw data to swarm and returns the resulting hash
+func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+ if size <= 0 {
+ return "", errors.New("data size must be greater than zero")
}
- if len(defaultPath) > 0 {
- fi, err := os.Stat(defaultPath)
- if err != nil {
- return "", err
- }
- mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
- if err != nil {
- return "", err
- }
+ req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+ if err != nil {
+ return "", err
}
- prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
- err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
- if err != nil || fi.IsDir() {
- return err
- }
- if !strings.HasPrefix(path, dir) {
- return fmt.Errorf("path %s outside directory %s", path, dir)
- }
- uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
- mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
- return err
- })
- return mhash, err
-}
-
-func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
- var mimetype string
- hash, err := c.uploadFileContent(file, fi)
- if mimetype_hint != "" {
- mimetype = mimetype_hint
- log.Info("Mime type set by override", "mime", mimetype)
- } else {
- ext := filepath.Ext(file)
- log.Info("Ext", "ext", ext, "file", file)
- if ext != "" {
- mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
- log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
- } else {
- f, err := os.Open(file)
- if err == nil {
- first512 := make([]byte, 512)
- fread, _ := f.ReadAt(first512, 0)
- if fread > 0 {
- mimetype = http.DetectContentType(first512[:fread])
- log.Info("Mime type set by autodetection", "mime", mimetype)
- }
- }
- f.Close()
- }
-
+ req.ContentLength = size
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
}
- m := ManifestEntry{
- Hash: hash,
- ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
- return m, err
-}
-
-func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(file)
+ data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
- defer fd.Close()
- log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
- return c.postRaw("application/octet-stream", fi.Size(), fd)
+ return string(data), nil
}
-func (c *Client) UploadManifest(m Manifest) (string, error) {
- jsm, err := json.Marshal(m)
+// DownloadRaw downloads raw data from swarm
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+ uri := c.Gateway + "/bzzr:/" + hash
+ res, err := http.DefaultClient.Get(uri)
if err != nil {
- panic(err)
+ return nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ res.Body.Close()
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
- log.Info("Uploading swarm manifest")
- return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+ return res.Body, nil
}
-func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
- fd, err := os.Open(fpath)
- if err != nil {
- return "", err
- }
- defer fd.Close()
- log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
- req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+// File represents a file in a swarm manifest and is used for uploading and
+// downloading content to and from swarm
+type File struct {
+ io.ReadCloser
+ api.ManifestEntry
+}
+
+// Open opens a local file which can then be passed to client.Upload to upload
+// it to swarm
+func Open(path string) (*File, error) {
+ f, err := os.Open(path)
if err != nil {
- return "", err
+ return nil, err
}
- req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
- req.ContentLength = fi.Size()
- resp, err := http.DefaultClient.Do(req)
+ stat, err := f.Stat()
if err != nil {
- return "", err
+ f.Close()
+ return nil, err
}
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
+ return &File{
+ ReadCloser: f,
+ ManifestEntry: api.ManifestEntry{
+ ContentType: mime.TypeByExtension(filepath.Ext(path)),
+ Mode: int64(stat.Mode()),
+ Size: stat.Size(),
+ ModTime: stat.ModTime(),
+ },
+ }, nil
+}
+
+// Upload uploads a file to swarm and either adds it to an existing manifest
+// (if the manifest argument is non-empty) or creates a new manifest containing
+// the file, returning the resulting manifest hash (the file will then be
+// available at bzz:/<hash>/<path>)
+func (c *Client) Upload(file *File, manifest string) (string, error) {
+ if file.Size <= 0 {
+ return "", errors.New("file size must be greater than zero")
}
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
+ return c.TarUpload(manifest, &FileUploader{file})
}
-func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
- req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+// Download downloads a file with the given path from the swarm manifest with
+// the given hash (i.e. it gets bzz:/<hash>/<path>)
+func (c *Client) Download(hash, path string) (*File, error) {
+ uri := c.Gateway + "/bzz:/" + hash + "/" + path
+ res, err := http.DefaultClient.Get(uri)
if err != nil {
- return "", err
+ return nil, err
}
- req.Header.Set("content-type", mimetype)
- req.ContentLength = size
- resp, err := http.DefaultClient.Do(req)
+ if res.StatusCode != http.StatusOK {
+ res.Body.Close()
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ return &File{
+ ReadCloser: res.Body,
+ ManifestEntry: api.ManifestEntry{
+ ContentType: res.Header.Get("Content-Type"),
+ Size: res.ContentLength,
+ },
+ }, nil
+}
+
+// UploadDirectory uploads a directory tree to swarm and either adds the files
+// to an existing manifest (if the manifest argument is non-empty) or creates a
+// new manifest, returning the resulting manifest hash (files from the
+// directory will then be available at bzz:/<hash>/path/to/file), with
+// the file specified in defaultPath being uploaded to the root of the manifest
+// (i.e. bzz:/<hash>/)
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+ stat, err := os.Stat(dir)
if err != nil {
return "", err
+ } else if !stat.IsDir() {
+ return "", fmt.Errorf("not a directory: %s", dir)
}
- defer resp.Body.Close()
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("bad status: %s", resp.Status)
- }
- content, err := ioutil.ReadAll(resp.Body)
- return string(content), err
+ return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
}
-func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+// DownloadDirectory downloads the files contained in a swarm manifest under
+// the given path into a local directory (existing files will be overwritten)
+func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+ stat, err := os.Stat(destDir)
+ if err != nil {
+ return err
+ } else if !stat.IsDir() {
+ return fmt.Errorf("not a directory: %s", destDir)
+ }
- mroot := Manifest{}
- req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+ uri := c.Gateway + "/bzz:/" + hash + "/" + path
+ req, err := http.NewRequest("GET", uri, nil)
if err != nil {
- return mroot, err
+ return err
}
- resp, err := http.DefaultClient.Do(req)
+ req.Header.Set("Accept", "application/x-tar")
+ res, err := http.DefaultClient.Do(req)
if err != nil {
- return mroot, err
+ return err
}
- defer resp.Body.Close()
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ tr := tar.NewReader(res.Body)
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ return nil
+ } else if err != nil {
+ return err
+ }
+ // ignore the default path file
+ if hdr.Name == "" {
+ continue
+ }
- if resp.StatusCode >= 400 {
- return mroot, fmt.Errorf("bad status: %s", resp.Status)
+ dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
+ if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+ return err
+ }
+ var mode os.FileMode = 0644
+ if hdr.Mode > 0 {
+ mode = os.FileMode(hdr.Mode)
+ }
+ dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+ if err != nil {
+ return err
+ }
+ n, err := io.Copy(dst, tr)
+ dst.Close()
+ if err != nil {
+ return err
+ } else if n != hdr.Size {
+ return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
+ }
+ }
+}
+// UploadManifest uploads the given manifest to swarm
+func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+ data, err := json.Marshal(m)
+ if err != nil {
+ return "", err
}
- content, err := ioutil.ReadAll(resp.Body)
+ return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+}
- err = json.Unmarshal(content, &mroot)
+// DownloadManifest downloads a swarm manifest
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+ res, err := c.DownloadRaw(hash)
if err != nil {
- return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+ return nil, err
+ }
+ defer res.Close()
+ var manifest api.Manifest
+ if err := json.NewDecoder(res).Decode(&manifest); err != nil {
+ return nil, err
}
- return mroot, err
+ return &manifest, nil
}
-// ManifestFileList downloads the manifest with the given hash and generates a
-// list of files and directory prefixes which have the specified prefix.
+// List list files in a swarm manifest which have the given prefix, grouping
+// common prefixes using "/" as a delimiter.
//
// For example, if the manifest represents the following directory structure:
//
@@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
-func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
- manifest, err := c.DownloadManifest(hash)
+//
+// where entries ending with "/" are common prefixes.
+func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+ res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
if err != nil {
return nil, err
}
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ var list api.ManifestList
+ if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
+ return nil, err
+ }
+ return &list, nil
+}
+
+// Uploader uploads files to swarm using a provided UploadFn
+type Uploader interface {
+ Upload(UploadFn) error
+}
+
+type UploaderFunc func(UploadFn) error
+
+func (u UploaderFunc) Upload(upload UploadFn) error {
+ return u(upload)
+}
+
+// DirectoryUploader uploads all files in a directory, optionally uploading
+// a file to the default path
+type DirectoryUploader struct {
+ Dir string
+ DefaultPath string
+}
- // handleFile handles a manifest entry which is a direct reference to a
- // file (i.e. it is not a swarm manifest)
- handleFile := func(entry ManifestEntry) {
- // ignore the file if it doesn't have the specified prefix
- if !strings.HasPrefix(entry.Path, prefix) {
- return
+// Upload performs the upload of the directory and default path
+func (d *DirectoryUploader) Upload(upload UploadFn) error {
+ if d.DefaultPath != "" {
+ file, err := Open(d.DefaultPath)
+ if err != nil {
+ return err
}
- // if the path after the prefix contains a directory separator,
- // add a directory prefix to the entries, otherwise add the
- // file
- suffix := strings.TrimPrefix(entry.Path, prefix)
- if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
- entries = append(entries, ManifestEntry{
- Path: prefix + suffix[:sepIndex+1],
- ContentType: "DIR",
- })
- } else {
- if entry.Path == "" {
- entry.Path = "/"
- }
- entries = append(entries, entry)
+ if err := upload(file); err != nil {
+ return err
}
}
-
- // handleManifest handles a manifest entry which is a reference to
- // another swarm manifest.
- handleManifest := func(entry ManifestEntry) error {
- // if the manifest's path is a prefix of the specified prefix
- // then just recurse into the manifest by stripping its path
- // from the prefix
- if strings.HasPrefix(prefix, entry.Path) {
- subPrefix := strings.TrimPrefix(prefix, entry.Path)
- subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
- if err != nil {
- return err
- }
- // prefix the manifest's path to the sub entries and
- // add them to the returned entries
- for i, subEntry := range subEntries {
- subEntry.Path = entry.Path + subEntry.Path
- subEntries[i] = subEntry
- }
- entries = append(entries, subEntries...)
+ return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if f.IsDir() {
return nil
}
+ file, err := Open(path)
+ if err != nil {
+ return err
+ }
+ relPath, err := filepath.Rel(d.Dir, path)
+ if err != nil {
+ return err
+ }
+ file.Path = filepath.ToSlash(relPath)
+ return upload(file)
+ })
+}
- // if the manifest's path has the specified prefix, then if the
- // path after the prefix contains a directory separator, add a
- // directory prefix to the entries, otherwise recurse into the
- // manifest
- if strings.HasPrefix(entry.Path, prefix) {
- suffix := strings.TrimPrefix(entry.Path, prefix)
- sepIndex := strings.Index(suffix, "/")
- if sepIndex > -1 {
- entries = append(entries, ManifestEntry{
- Path: prefix + suffix[:sepIndex+1],
- ContentType: "DIR",
- })
- return nil
- }
- subEntries, err := c.ManifestFileList(entry.Hash, "")
- if err != nil {
- return err
- }
- // prefix the manifest's path to the sub entries and
- // add them to the returned entries
- for i, subEntry := range subEntries {
- subEntry.Path = entry.Path + subEntry.Path
- subEntries[i] = subEntry
- }
- entries = append(entries, subEntries...)
- return nil
+// FileUploader uploads a single file
+type FileUploader struct {
+ File *File
+}
+
+// Upload performs the upload of the file
+func (f *FileUploader) Upload(upload UploadFn) error {
+ return upload(f.File)
+}
+
+// UploadFn is the type of function passed to an Uploader to perform the upload
+// of a single file (for example, a directory uploader would call a provided
+// UploadFn for each file in the directory tree)
+type UploadFn func(file *File) error
+
+// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
+// returning the resulting manifest hash
+func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+ reqR, reqW := io.Pipe()
+ defer reqR.Close()
+ req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+ if err != nil {
+ return "", err
+ }
+ req.Header.Set("Content-Type", "application/x-tar")
+
+ // use 'Expect: 100-continue' so we don't send the request body if
+ // the server refuses the request
+ req.Header.Set("Expect", "100-continue")
+
+ tw := tar.NewWriter(reqW)
+
+ // define an UploadFn which adds files to the tar stream
+ uploadFn := func(file *File) error {
+ hdr := &tar.Header{
+ Name: file.Path,
+ Mode: file.Mode,
+ Size: file.Size,
+ ModTime: file.ModTime,
+ Xattrs: map[string]string{
+ "user.swarm.content-type": file.ContentType,
+ },
+ }
+ if err := tw.WriteHeader(hdr); err != nil {
+ return err
}
- return nil
+ _, err = io.Copy(tw, file)
+ return err
}
- for _, entry := range manifest.Entries {
- if entry.ContentType == "application/bzz-manifest+json" {
- if err := handleManifest(entry); err != nil {
- return nil, err
- }
- } else {
- handleFile(entry)
+ // run the upload in a goroutine so we can send the request headers and
+ // wait for a '100 Continue' response before sending the tar stream
+ go func() {
+ err := uploader.Upload(uploadFn)
+ if err == nil {
+ err = tw.Close()
}
+ reqW.CloseWithError(err)
+ }()
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ data, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return "", err
+ }
+ return string(data), nil
+}
+
+// MultipartUpload uses the given Uploader to upload files to swarm as a
+// multipart form, returning the resulting manifest hash
+func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+ reqR, reqW := io.Pipe()
+ defer reqR.Close()
+ req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+ if err != nil {
+ return "", err
}
- return
+ // use 'Expect: 100-continue' so we don't send the request body if
+ // the server refuses the request
+ req.Header.Set("Expect", "100-continue")
+
+ mw := multipart.NewWriter(reqW)
+ req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
+
+ // define an UploadFn which adds files to the multipart form
+ uploadFn := func(file *File) error {
+ hdr := make(textproto.MIMEHeader)
+ hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
+ hdr.Set("Content-Type", file.ContentType)
+ hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
+ w, err := mw.CreatePart(hdr)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(w, file)
+ return err
+ }
+
+ // run the upload in a goroutine so we can send the request headers and
+ // wait for a '100 Continue' response before sending the multipart form
+ go func() {
+ err := uploader.Upload(uploadFn)
+ if err == nil {
+ err = mw.Close()
+ }
+ reqW.CloseWithError(err)
+ }()
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+ }
+ data, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return "", err
+ }
+ return string(data), nil
}
generated by cgit (git 2.34.1) at 2024-10-11 08:28:39 +0800