cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.

2 files changed, 32 insertions, 62 deletions

diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index ae113750f..44e2c203a 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -56,9 +56,9 @@ type sequentialReader struct {
 	lock   sync.Mutex
 }
 
-// Server is the basic configuration needs for the HTTP server and also
+// ServerConfig is the basic configuration needed for the HTTP server and also
 // includes CORS settings.
-type Server struct {
+type ServerConfig struct {
 	Addr       string
 	CorsString string
 }
@@ -69,13 +69,9 @@ type Server struct {
 // https://github.com/atom/electron/blob/master/docs/api/protocol.md
 
 // starts up http server
-func StartHttpServer(api *api.Api, server *Server) {
-	serveMux := http.NewServeMux()
-	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		handler(w, r, api)
-	})
+func StartHttpServer(api *api.Api, config *ServerConfig) {
 	var allowedOrigins []string
-	for _, domain := range strings.Split(server.CorsString, ",") {
+	for _, domain := range strings.Split(config.CorsString, ",") {
 		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
 	}
 	c := cors.New(cors.Options{
@@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) {
 		MaxAge:         600,
 		AllowedHeaders: []string{"*"},
 	})
-	hdlr := c.Handler(serveMux)
+	hdlr := c.Handler(NewServer(api))
+
+	go http.ListenAndServe(config.Addr, hdlr)
+	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr))
+}
 
-	go http.ListenAndServe(server.Addr, hdlr)
-	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr))
+func NewServer(api *api.Api) *Server {
+	return &Server{api}
+}
+
+type Server struct {
+	api *api.Api
 }
 
-func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	requestURL := r.URL
 	// This is wrong
 	//	if requestURL.Host == "" {
@@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
 			return
 		}
-		key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
+		key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
 		if err == nil {
 			log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
 		} else {
@@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				mime := r.Header.Get("Content-Type")
 				// TODO proper root hash separation
 				log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
-				newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
+				newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
 				if err == nil {
 					log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 					w.Header().Set("Content-Type", "text/plain")
@@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		} else {
 			path = api.RegularSlashes(path)
 			log.Debug(fmt.Sprintf("Delete '%s'.", path))
-			newKey, err := a.Modify(path, "", "", nameresolver)
+			newKey, err := s.api.Modify(path, "", "", nameresolver)
 			if err == nil {
 				log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 				w.Header().Set("Content-Type", "text/plain")
@@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			parsedurl, _ := api.Parse(path)
 
 			if parsedurl == path {
-				key, err := a.Resolve(parsedurl, nameresolver)
+				key, err := s.api.Resolve(parsedurl, nameresolver)
 				if err != nil {
 					log.Error(fmt.Sprintf("%v", err))
 					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
-				reader = a.Retrieve(key)
+				reader = s.api.Retrieve(key)
 			} else {
 				var status int
-				readertmp, _, status, err := a.Get(path, nameresolver)
+				readertmp, _, status, err := s.api.Get(path, nameresolver)
 				if err != nil {
 					http.Error(w, err.Error(), status)
 					return
@@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				http.Redirect(w, r, path+"/", http.StatusFound)
 				return
 			}
-			reader, mimeType, status, err := a.Get(path, nameresolver)
+			reader, mimeType, status, err := s.api.Get(path, nameresolver)
 			if err != nil {
 				if _, ok := err.(api.ErrResolve); ok {
 					log.Debug(fmt.Sprintf("%v", err))
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 88b49b9a5..45a867f51 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package http
+package http_test
 
 import (
 	"bytes"
@@ -22,19 +22,16 @@ import (
 	"net/http"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestBzzrGetPath(t *testing.T) {
 
 	var err error
 
-	maxproxyattempts := 3
-
 	testmanifest := []string{
 		`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
 		`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
@@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) {
 
 	key := [3]storage.Key{}
 
-	dir, _ := ioutil.TempDir("", "bzz-storage-test")
-
-	storeparams := &storage.StoreParams{
-		ChunkDbPath:   dir,
-		DbCapacity:    5000000,
-		CacheCapacity: 5000,
-		Radius:        0,
-	}
-
-	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
-	if err != nil {
-		t.Fatal(err)
-	}
-	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
-	dpa := &storage.DPA{
-		Chunker:    chunker,
-		ChunkStore: localStore,
-	}
-	dpa.Start()
-	defer dpa.Stop()
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
 	wg := &sync.WaitGroup{}
 
 	for i, mf := range testmanifest {
 		reader[i] = bytes.NewReader([]byte(mf))
-		key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+		key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
 		wg.Wait()
 	}
 
-	a := api.NewApi(dpa, nil)
-
-	/// \todo iterate port numbers up if fail
-	StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
-	// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
-	// if we fix it we don't need maxproxyattempts anymore either
-	time.Sleep(1000 * time.Millisecond)
-	for i := 0; i <= maxproxyattempts; i++ {
-		_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
-		if i == maxproxyattempts {
-			t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
-		} else if err != nil {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		}
-		break
+	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	if err != nil {
+		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
 
 	for k, v := range testrequests {
 		var resp *http.Response
 		var respbody []byte
 
-		url := "http://127.0.0.1:8504/bzzr:/"
+		url := srv.URL + "/bzzr:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}