swarm/api: refactor and improve HTTP API (#3773)

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).

1 files changed, 241 insertions, 19 deletions

diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index 135474475..4d02ceaf4 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -17,6 +17,7 @@
 package client
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,52 +25,221 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func TestClientManifestFileList(t *testing.T) {
+// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
+func TestClientUploadDownloadRaw(t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
+	client := NewClient(srv.URL)
+
+	// upload some raw data
+	data := []byte("foo123")
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the same data
+	res, err := client.DownloadRaw(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Close()
+	gotData, err := ioutil.ReadAll(res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(gotData, data) {
+		t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
+	}
+}
+
+// TestClientUploadDownloadFiles test uploading and downloading files to swarm
+// manifests
+func TestClientUploadDownloadFiles(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	client := NewClient(srv.URL)
+	upload := func(manifest, path string, data []byte) string {
+		file := &File{
+			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+			ManifestEntry: api.ManifestEntry{
+				Path:        path,
+				ContentType: "text/plain",
+				Size:        int64(len(data)),
+			},
+		}
+		hash, err := client.Upload(file, manifest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return hash
+	}
+	checkDownload := func(manifest, path string, expected []byte) {
+		file, err := client.Download(manifest, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		if file.Size != int64(len(expected)) {
+			t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
+		}
+		if file.ContentType != file.ContentType {
+			t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
+		}
+	}
+
+	// upload a file to the root of a manifest
+	rootData := []byte("some-data")
+	rootHash := upload("", "", rootData)
+
+	// check we can download the root file
+	checkDownload(rootHash, "", rootData)
+
+	// upload another file to the same manifest
+	otherData := []byte("some-other-data")
+	newHash := upload(rootHash, "some/other/path", otherData)
+
+	// check we can download both files from the new manifest
+	checkDownload(newHash, "", rootData)
+	checkDownload(newHash, "some/other/path", otherData)
+
+	// replace the root file with different data
+	newHash = upload(newHash, "", otherData)
+
+	// check both files have the other data
+	checkDownload(newHash, "", otherData)
+	checkDownload(newHash, "some/other/path", otherData)
+}
+
+var testDirFiles = []string{
+	"file1.txt",
+	"file2.txt",
+	"dir1/file3.txt",
+	"dir1/file4.txt",
+	"dir2/file5.txt",
+	"dir2/dir3/file6.txt",
+	"dir2/dir4/file7.txt",
+	"dir2/dir4/file8.txt",
+}
+
+func newTestDirectory(t *testing.T) string {
 	dir, err := ioutil.TempDir("", "swarm-client-test")
 	if err != nil {
 		t.Fatal(err)
 	}
-	files := []string{
-		"file1.txt",
-		"file2.txt",
-		"dir1/file3.txt",
-		"dir1/file4.txt",
-		"dir2/file5.txt",
-		"dir2/dir3/file6.txt",
-		"dir2/dir4/file7.txt",
-		"dir2/dir4/file8.txt",
-	}
-	for _, file := range files {
+
+	for _, file := range testDirFiles {
 		path := filepath.Join(dir, file)
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error creating dir for %s: %s", path, err)
 		}
-		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+		if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error writing file %s: %s", path, err)
 		}
 	}
 
+	return dir
+}
+
+// TestClientUploadDownloadDirectory tests uploading and downloading a
+// directory of files to a swarm manifest
+func TestClientUploadDownloadDirectory(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	// upload the directory
 	client := NewClient(srv.URL)
+	defaultPath := filepath.Join(dir, testDirFiles[0])
+	hash, err := client.UploadDirectory(dir, defaultPath, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string, expected []byte) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected data to be %q, got %q", expected, data)
+		}
+	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file, []byte(file))
+	}
+
+	// check we can download the default path
+	checkDownloadFile("", []byte(testDirFiles[0]))
+
+	// check we can download the directory
+	tmp, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range testDirFiles {
+		data, err := ioutil.ReadFile(filepath.Join(tmp, file))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, []byte(file)) {
+			t.Fatalf("expected data to be %q, got %q", file, data)
+		}
+	}
+}
+
+// TestClientFileList tests listing files in a swarm manifest
+func TestClientFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
-	hash, err := client.UploadDirectory(dir, "")
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	client := NewClient(srv.URL)
+	hash, err := client.UploadDirectory(dir, "", "")
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
 
 	ls := func(prefix string) []string {
-		entries, err := client.ManifestFileList(hash, prefix)
+		list, err := client.List(hash, prefix)
 		if err != nil {
 			t.Fatal(err)
 		}
-		paths := make([]string, len(entries))
-		for i, entry := range entries {
-			paths[i] = entry.Path
+		paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
+		for _, prefix := range list.CommonPrefixes {
+			paths = append(paths, prefix)
+		}
+		for _, entry := range list.Entries {
+			paths = append(paths, entry.Path)
 		}
 		sort.Strings(paths)
 		return paths
@@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
 	for prefix, expected := range tests {
 		actual := ls(prefix)
 		if !reflect.DeepEqual(actual, expected) {
-			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+			t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
+		}
+	}
+}
+
+// TestClientMultipartUpload tests uploading files to swarm using a multipart
+// upload
+func TestClientMultipartUpload(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	// define an uploader which uploads testDirFiles with some data
+	data := []byte("some-data")
+	uploader := UploaderFunc(func(upload UploadFn) error {
+		for _, name := range testDirFiles {
+			file := &File{
+				ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+				ManifestEntry: api.ManifestEntry{
+					Path:        name,
+					ContentType: "text/plain",
+					Size:        int64(len(data)),
+				},
+			}
+			if err := upload(file); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// upload the files as a multipart upload
+	client := NewClient(srv.URL)
+	hash, err := client.MultipartUpload("", uploader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		gotData, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(gotData, data) {
+			t.Fatalf("expected data to be %q, got %q", data, gotData)
 		}
 	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file)
+	}
 }