cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.

4 files changed, 435 insertions, 62 deletions

diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
new file mode 100644
index 000000000..15e44f35d
--- /dev/null
+++ b/swarm/api/client/client.go
@@ -0,0 +1,298 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	DefaultGateway = "http://localhost:8500"
+	DefaultClient  = NewClient(DefaultGateway)
+)
+
+// Manifest represents a swarm manifest.
+type Manifest struct {
+	Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest.
+type ManifestEntry struct {
+	Hash        string `json:"hash,omitempty"`
+	ContentType string `json:"contentType,omitempty"`
+	Path        string `json:"path,omitempty"`
+}
+
+func NewClient(gateway string) *Client {
+	return &Client{
+		Gateway: gateway,
+	}
+}
+
+// Client wraps interaction with a swarm HTTP gateway.
+type Client struct {
+	Gateway string
+}
+
+func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
+	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
+	if err != nil {
+		return "", fmt.Errorf("failed to upload empty manifest")
+	}
+	if len(defaultPath) > 0 {
+		fi, err := os.Stat(defaultPath)
+		if err != nil {
+			return "", err
+		}
+		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
+		if err != nil {
+			return "", err
+		}
+	}
+	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
+	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil || fi.IsDir() {
+			return err
+		}
+		if !strings.HasPrefix(path, dir) {
+			return fmt.Errorf("path %s outside directory %s", path, dir)
+		}
+		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
+		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
+		return err
+	})
+	return mhash, err
+}
+
+func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) {
+	hash, err := c.uploadFileContent(file, fi)
+	m := ManifestEntry{
+		Hash:        hash,
+		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	}
+	return m, err
+}
+
+func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
+	return c.postRaw("application/octet-stream", fi.Size(), fd)
+}
+
+func (c *Client) UploadManifest(m Manifest) (string, error) {
+	jsm, err := json.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+	log.Info("Uploading swarm manifest")
+	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+}
+
+func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(fpath)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
+	req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
+	req.ContentLength = fi.Size()
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
+	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mimetype)
+	req.ContentLength = size
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+
+	mroot := Manifest{}
+	req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+	if err != nil {
+		return mroot, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return mroot, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+
+	err = json.Unmarshal(content, &mroot)
+	if err != nil {
+		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+	}
+	return mroot, err
+}
+
+// ManifestFileList downloads the manifest with the given hash and generates a
+// list of files and directory prefixes which have the specified prefix.
+//
+// For example, if the manifest represents the following directory structure:
+//
+// file1.txt
+// file2.txt
+// dir1/file3.txt
+// dir1/dir2/file4.txt
+//
+// Then:
+//
+// - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
+// - a prefix of "file"  would return [file1.txt, file2.txt]
+// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
+func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
+	manifest, err := c.DownloadManifest(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	// handleFile handles a manifest entry which is a direct reference to a
+	// file (i.e. it is not a swarm manifest)
+	handleFile := func(entry ManifestEntry) {
+		// ignore the file if it doesn't have the specified prefix
+		if !strings.HasPrefix(entry.Path, prefix) {
+			return
+		}
+		// if the path after the prefix contains a directory separator,
+		// add a directory prefix to the entries, otherwise add the
+		// file
+		suffix := strings.TrimPrefix(entry.Path, prefix)
+		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
+			entries = append(entries, ManifestEntry{
+				Path:        prefix + suffix[:sepIndex+1],
+				ContentType: "DIR",
+			})
+		} else {
+			if entry.Path == "" {
+				entry.Path = "/"
+			}
+			entries = append(entries, entry)
+		}
+	}
+
+	// handleManifest handles a manifest entry which is a reference to
+	// another swarm manifest.
+	handleManifest := func(entry ManifestEntry) error {
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by stripping its path
+		// from the prefix
+		if strings.HasPrefix(prefix, entry.Path) {
+			subPrefix := strings.TrimPrefix(prefix, entry.Path)
+			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a directory separator, add a
+		// directory prefix to the entries, otherwise recurse into the
+		// manifest
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			sepIndex := strings.Index(suffix, "/")
+			if sepIndex > -1 {
+				entries = append(entries, ManifestEntry{
+					Path:        prefix + suffix[:sepIndex+1],
+					ContentType: "DIR",
+				})
+				return nil
+			}
+			subEntries, err := c.ManifestFileList(entry.Hash, "")
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+		return nil
+	}
+
+	for _, entry := range manifest.Entries {
+		if entry.ContentType == "application/bzz-manifest+json" {
+			if err := handleManifest(entry); err != nil {
+				return nil, err
+			}
+		} else {
+			handleFile(entry)
+		}
+	}
+
+	return
+}
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
new file mode 100644
index 000000000..135474475
--- /dev/null
+++ b/swarm/api/client/client_test.go
@@ -0,0 +1,105 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+)
+
+func TestClientManifestFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := []string{
+		"file1.txt",
+		"file2.txt",
+		"dir1/file3.txt",
+		"dir1/file4.txt",
+		"dir2/file5.txt",
+		"dir2/dir3/file6.txt",
+		"dir2/dir4/file7.txt",
+		"dir2/dir4/file8.txt",
+	}
+	for _, file := range files {
+		path := filepath.Join(dir, file)
+		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			t.Fatalf("error creating dir for %s: %s", path, err)
+		}
+		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+			t.Fatalf("error writing file %s: %s", path, err)
+		}
+	}
+
+	client := NewClient(srv.URL)
+
+	hash, err := client.UploadDirectory(dir, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	ls := func(prefix string) []string {
+		entries, err := client.ManifestFileList(hash, prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		paths := make([]string, len(entries))
+		for i, entry := range entries {
+			paths[i] = entry.Path
+		}
+		sort.Strings(paths)
+		return paths
+	}
+
+	tests := map[string][]string{
+		"":                    []string{"dir1/", "dir2/", "file1.txt", "file2.txt"},
+		"file":                []string{"file1.txt", "file2.txt"},
+		"file1":               []string{"file1.txt"},
+		"file2.txt":           []string{"file2.txt"},
+		"file12":              []string{},
+		"dir":                 []string{"dir1/", "dir2/"},
+		"dir1":                []string{"dir1/"},
+		"dir1/":               []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file":           []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file3.txt":      []string{"dir1/file3.txt"},
+		"dir1/file34":         []string{},
+		"dir2/":               []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
+		"dir2/file":           []string{"dir2/file5.txt"},
+		"dir2/dir":            []string{"dir2/dir3/", "dir2/dir4/"},
+		"dir2/dir3/":          []string{"dir2/dir3/file6.txt"},
+		"dir2/dir4/":          []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file":      []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"},
+		"dir2/dir4/file78":    []string{},
+	}
+	for prefix, expected := range tests {
+		actual := ls(prefix)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+		}
+	}
+}
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index ae113750f..44e2c203a 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -56,9 +56,9 @@ type sequentialReader struct {
 	lock   sync.Mutex
 }
 
-// Server is the basic configuration needs for the HTTP server and also
+// ServerConfig is the basic configuration needed for the HTTP server and also
 // includes CORS settings.
-type Server struct {
+type ServerConfig struct {
 	Addr       string
 	CorsString string
 }
@@ -69,13 +69,9 @@ type Server struct {
 // https://github.com/atom/electron/blob/master/docs/api/protocol.md
 
 // starts up http server
-func StartHttpServer(api *api.Api, server *Server) {
-	serveMux := http.NewServeMux()
-	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		handler(w, r, api)
-	})
+func StartHttpServer(api *api.Api, config *ServerConfig) {
 	var allowedOrigins []string
-	for _, domain := range strings.Split(server.CorsString, ",") {
+	for _, domain := range strings.Split(config.CorsString, ",") {
 		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
 	}
 	c := cors.New(cors.Options{
@@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) {
 		MaxAge:         600,
 		AllowedHeaders: []string{"*"},
 	})
-	hdlr := c.Handler(serveMux)
+	hdlr := c.Handler(NewServer(api))
+
+	go http.ListenAndServe(config.Addr, hdlr)
+	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr))
+}
 
-	go http.ListenAndServe(server.Addr, hdlr)
-	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr))
+func NewServer(api *api.Api) *Server {
+	return &Server{api}
+}
+
+type Server struct {
+	api *api.Api
 }
 
-func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	requestURL := r.URL
 	// This is wrong
 	//	if requestURL.Host == "" {
@@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
 			return
 		}
-		key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
+		key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
 		if err == nil {
 			log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
 		} else {
@@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				mime := r.Header.Get("Content-Type")
 				// TODO proper root hash separation
 				log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
-				newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
+				newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
 				if err == nil {
 					log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 					w.Header().Set("Content-Type", "text/plain")
@@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		} else {
 			path = api.RegularSlashes(path)
 			log.Debug(fmt.Sprintf("Delete '%s'.", path))
-			newKey, err := a.Modify(path, "", "", nameresolver)
+			newKey, err := s.api.Modify(path, "", "", nameresolver)
 			if err == nil {
 				log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 				w.Header().Set("Content-Type", "text/plain")
@@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			parsedurl, _ := api.Parse(path)
 
 			if parsedurl == path {
-				key, err := a.Resolve(parsedurl, nameresolver)
+				key, err := s.api.Resolve(parsedurl, nameresolver)
 				if err != nil {
 					log.Error(fmt.Sprintf("%v", err))
 					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
-				reader = a.Retrieve(key)
+				reader = s.api.Retrieve(key)
 			} else {
 				var status int
-				readertmp, _, status, err := a.Get(path, nameresolver)
+				readertmp, _, status, err := s.api.Get(path, nameresolver)
 				if err != nil {
 					http.Error(w, err.Error(), status)
 					return
@@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				http.Redirect(w, r, path+"/", http.StatusFound)
 				return
 			}
-			reader, mimeType, status, err := a.Get(path, nameresolver)
+			reader, mimeType, status, err := s.api.Get(path, nameresolver)
 			if err != nil {
 				if _, ok := err.(api.ErrResolve); ok {
 					log.Debug(fmt.Sprintf("%v", err))
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 88b49b9a5..45a867f51 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package http
+package http_test
 
 import (
 	"bytes"
@@ -22,19 +22,16 @@ import (
 	"net/http"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestBzzrGetPath(t *testing.T) {
 
 	var err error
 
-	maxproxyattempts := 3
-
 	testmanifest := []string{
 		`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
 		`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
@@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) {
 
 	key := [3]storage.Key{}
 
-	dir, _ := ioutil.TempDir("", "bzz-storage-test")
-
-	storeparams := &storage.StoreParams{
-		ChunkDbPath:   dir,
-		DbCapacity:    5000000,
-		CacheCapacity: 5000,
-		Radius:        0,
-	}
-
-	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
-	if err != nil {
-		t.Fatal(err)
-	}
-	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
-	dpa := &storage.DPA{
-		Chunker:    chunker,
-		ChunkStore: localStore,
-	}
-	dpa.Start()
-	defer dpa.Stop()
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
 	wg := &sync.WaitGroup{}
 
 	for i, mf := range testmanifest {
 		reader[i] = bytes.NewReader([]byte(mf))
-		key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+		key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
 		wg.Wait()
 	}
 
-	a := api.NewApi(dpa, nil)
-
-	/// \todo iterate port numbers up if fail
-	StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
-	// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
-	// if we fix it we don't need maxproxyattempts anymore either
-	time.Sleep(1000 * time.Millisecond)
-	for i := 0; i <= maxproxyattempts; i++ {
-		_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
-		if i == maxproxyattempts {
-			t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
-		} else if err != nil {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		}
-		break
+	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	if err != nil {
+		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
 
 	for k, v := range testrequests {
 		var resp *http.Response
 		var respbody []byte
 
-		url := "http://127.0.0.1:8504/bzzr:/"
+		url := srv.URL + "/bzzr:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}