swarm/api: refactor and improve HTTP API (#3773)

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).

20 files changed, 1779 insertions, 663 deletions

diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go
index 3a68fef03..06d3883cf 100644
--- a/cmd/swarm/list.go
+++ b/cmd/swarm/list.go
@@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
 
 	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 	client := swarm.NewClient(bzzapi)
-	entries, err := client.ManifestFileList(manifest, prefix)
+	list, err := client.List(manifest, prefix)
 	if err != nil {
 		utils.Fatalf("Failed to generate file and directory list: %s", err)
 	}
@@ -52,7 +52,10 @@ func list(ctx *cli.Context) {
 	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
 	defer w.Flush()
 	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
-	for _, entry := range entries {
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
 		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
 	}
 }
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 698b8ddb8..9729022c0 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -42,7 +43,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	if len(args) > 3 {
@@ -76,7 +77,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -106,7 +107,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -125,11 +126,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		client           = swarm.NewClient(bzzapi)
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -163,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -173,9 +170,9 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := swarm.ManifestEntry{
-			Path:        path,
+		newEntry := api.ManifestEntry{
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
@@ -192,18 +189,10 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = swarm.NewClient(bzzapi)
-		newEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		newEntry         = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -237,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -250,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := swarm.ManifestEntry{
-					Path:        entry.Path,
+				myEntry := api.ManifestEntry{
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -276,18 +265,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = swarm.NewClient(bzzapi)
-		entryToRemove = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		entryToRemove    = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -319,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -331,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 46f10c4be..42673ae21 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -18,13 +18,15 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
+	"net/http"
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -42,12 +44,10 @@ func upload(ctx *cli.Context) {
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
 		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
 		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
 
-	var client = swarm.NewClient(bzzapi)
-	var entry swarm.ManifestEntry
-	var file string
-
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -66,41 +66,47 @@ func upload(ctx *cli.Context) {
 			utils.Fatalf("Need filename as the first and only argument")
 		}
 	} else {
-		file = args[0]
+		file = expandPath(args[0])
+	}
+
+	if !wantManifest {
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
+		return
 	}
 
-	fi, err := os.Stat(expandPath(file))
+	stat, err := os.Stat(file)
 	if err != nil {
-		utils.Fatalf("Failed to stat file: %v", err)
+		utils.Fatalf("Error opening file: %s", err)
 	}
-	if fi.IsDir() {
+	var hash string
+	if stat.IsDir() {
 		if !recursive {
 			utils.Fatalf("Argument is a directory and recursive upload is disabled")
 		}
-		if !wantManifest {
-			utils.Fatalf("Manifest is required for directory uploads")
+		hash, err = client.UploadDirectory(file, defaultPath, "")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
 		}
-		mhash, err := client.UploadDirectory(file, defaultPath)
+		f, err := swarm.Open(file)
 		if err != nil {
-			utils.Fatalf("Failed to upload directory: %v", err)
+			utils.Fatalf("Error opening file: %s", err)
 		}
-		fmt.Println(mhash)
-		return
+		defer f.Close()
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
 	}
-	entry, err = client.UploadFile(file, fi, mimeType)
 	if err != nil {
-		utils.Fatalf("Upload failed: %v", err)
-	}
-	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
-	hash, err := client.UploadManifest(mroot)
-	if err != nil {
-		utils.Fatalf("Manifest upload failed: %v", err)
+		utils.Fatalf("Upload failed: %s", err)
 	}
 	fmt.Println(hash)
 }
@@ -128,3 +134,19 @@ func homeDir() string {
 	}
 	return ""
 }
+
+func detectMimeType(file string) string {
+	if ext := filepath.Ext(file); ext != "" {
+		return mime.TypeByExtension(ext)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	buf := make([]byte, 512)
+	if n, _ := f.Read(buf); n > 0 {
+		return http.DetectContentType(buf)
+	}
+	return ""
+}
diff --git a/swarm/api/api.go b/swarm/api/api.go
index 7af27208d..ba1156f7e 100644
--- a/swarm/api/api.go
+++ b/swarm/api/api.go
@@ -17,6 +17,7 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -70,86 +71,50 @@ func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key stor
 type ErrResolve error
 
 // DNS Resolver
-func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
-	log.Trace(fmt.Sprintf("Resolving : %v", hostPort))
-	if hashMatcher.MatchString(hostPort) || self.dns == nil {
-		log.Trace(fmt.Sprintf("host is a contentHash: '%v'", hostPort))
-		return storage.Key(common.Hex2Bytes(hostPort)), nil
+func (self *Api) Resolve(uri *URI) (storage.Key, error) {
+	log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))
+	if hashMatcher.MatchString(uri.Addr) {
+		log.Trace(fmt.Sprintf("addr is a hash: %q", uri.Addr))
+		return storage.Key(common.Hex2Bytes(uri.Addr)), nil
 	}
-	if !nameresolver {
-		return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort)
+	if uri.Immutable() {
+		return nil, errors.New("refusing to resolve immutable address")
 	}
-	contentHash, err := self.dns.Resolve(hostPort)
-	if err != nil {
-		err = ErrResolve(err)
-		log.Warn(fmt.Sprintf("DNS error : %v", err))
-	}
-	log.Trace(fmt.Sprintf("host lookup: %v -> %v", hostPort, contentHash))
-	return contentHash[:], err
-}
-func Parse(uri string) (hostPort, path string) {
-	if uri == "" {
-		return
-	}
-	parts := slashes.Split(uri, 3)
-	var i int
-	if len(parts) == 0 {
-		return
+	if self.dns == nil {
+		return nil, fmt.Errorf("unable to resolve addr %q, resolver not configured", uri.Addr)
 	}
-	// beginning with slash is now optional
-	for len(parts[i]) == 0 {
-		i++
-	}
-	hostPort = parts[i]
-	for i < len(parts)-1 {
-		i++
-		if len(path) > 0 {
-			path = path + "/" + parts[i]
-		} else {
-			path = parts[i]
-		}
+	hash, err := self.dns.Resolve(uri.Addr)
+	if err != nil {
+		log.Warn(fmt.Sprintf("DNS error resolving addr %q: %s", uri.Addr, err))
+		return nil, ErrResolve(err)
 	}
-	log.Debug(fmt.Sprintf("host: '%s', path '%s' requested.", hostPort, path))
-	return
-}
-
-func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
-	hostPort, path = Parse(uri)
-	//resolving host and port
-	contentHash, err := self.Resolve(hostPort, nameresolver)
-	log.Debug(fmt.Sprintf("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path))
-	return contentHash[:], hostPort, path, err
+	log.Trace(fmt.Sprintf("addr lookup: %v -> %v", uri.Addr, hash))
+	return hash[:], nil
 }
 
 // Put provides singleton manifest creation on top of dpa store
-func (self *Api) Put(content, contentType string) (string, error) {
+func (self *Api) Put(content, contentType string) (storage.Key, error) {
 	r := strings.NewReader(content)
 	wg := &sync.WaitGroup{}
 	key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
 	r = strings.NewReader(manifest)
 	key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	wg.Wait()
-	return key.String(), nil
+	return key, nil
 }
 
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve path to content using dpa retrieve
 // it returns a section reader, mimeType, status and an error
-func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) {
-	key, _, path, err := self.parseAndResolve(uri, nameresolver)
-	if err != nil {
-		return nil, "", 500, fmt.Errorf("can't resolve: %v", err)
-	}
-
-	quitC := make(chan bool)
-	trie, err := loadManifest(self.dpa, key, quitC)
+func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
+	trie, err := loadManifest(self.dpa, key, nil)
 	if err != nil {
 		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
 		return
@@ -173,32 +138,25 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 	return
 }
 
-func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
-	root, _, path, err := self.parseAndResolve(uri, nameresolver)
-	if err != nil {
-		return "", fmt.Errorf("can't resolve: %v", err)
-	}
-
+func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
 	quitC := make(chan bool)
-	trie, err := loadManifest(self.dpa, root, quitC)
+	trie, err := loadManifest(self.dpa, key, quitC)
 	if err != nil {
-		return
+		return nil, err
 	}
-
 	if contentHash != "" {
-		entry := &manifestTrieEntry{
+		entry := newManifestTrieEntry(&ManifestEntry{
 			Path:        path,
-			Hash:        contentHash,
 			ContentType: contentType,
-		}
+		}, nil)
+		entry.Hash = contentHash
 		trie.addEntry(entry, quitC)
 	} else {
 		trie.deleteEntry(path, quitC)
 	}
 
-	err = trie.recalcAndStore()
-	if err != nil {
-		return
+	if err := trie.recalcAndStore(); err != nil {
+		return nil, err
 	}
-	return trie.hash.String(), nil
+	return trie.hash, nil
 }
diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go
index 16e90dd32..c2d78c2dc 100644
--- a/swarm/api/api_test.go
+++ b/swarm/api/api_test.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
@@ -81,8 +82,9 @@ func expResponse(content string, mimeType string, status int) *Response {
 }
 
 // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-	reader, mimeType, status, err := api.Get(bzzhash, true)
+func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
+	key := storage.Key(common.Hex2Bytes(bzzhash))
+	reader, mimeType, status, err := api.Get(key, path)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -107,11 +109,11 @@ func TestApiPut(t *testing.T) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
 		// exp := expResponse([]byte(content), "text/plain", 0)
-		bzzhash, err := api.Put(content, exp.MimeType)
+		key, err := api.Put(content, exp.MimeType)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, key.String(), "")
 		checkResponse(t, resp, exp)
 	})
 }
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index ef5335be3..f9c3e51e8 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -17,18 +17,23 @@
 package client
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime"
+	"mime/multipart"
 	"net/http"
+	"net/textproto"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 )
 
 var (
@@ -36,18 +41,6 @@ var (
 	DefaultClient  = NewClient(DefaultGateway)
 )
 
-// Manifest represents a swarm manifest.
-type Manifest struct {
-	Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest.
-type ManifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
 func NewClient(gateway string) *Client {
 	return &Client{
 		Gateway: gateway,
@@ -59,160 +52,207 @@ type Client struct {
 	Gateway string
 }
 
-func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
+// UploadRaw uploads raw data to swarm and returns the resulting hash
+func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+	if size <= 0 {
+		return "", errors.New("data size must be greater than zero")
 	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
+	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	if err != nil {
+		return "", err
 	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
-	var mimetype string
-	hash, err := c.uploadFileContent(file, fi)
-	if mimetype_hint != "" {
-		mimetype = mimetype_hint
-		log.Info("Mime type set by override", "mime", mimetype)
-	} else {
-		ext := filepath.Ext(file)
-		log.Info("Ext", "ext", ext, "file", file)
-		if ext != "" {
-			mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
-			log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
-		} else {
-			f, err := os.Open(file)
-			if err == nil {
-				first512 := make([]byte, 512)
-				fread, _ := f.ReadAt(first512, 0)
-				if fread > 0 {
-					mimetype = http.DetectContentType(first512[:fread])
-					log.Info("Mime type set by autodetection", "mime", mimetype)
-				}
-			}
-			f.Close()
-		}
-
+	req.ContentLength = size
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
 	}
-	m := ManifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	return m, err
-}
-
-func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
+	data, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", err
 	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
+	return string(data), nil
 }
 
-func (c *Client) UploadManifest(m Manifest) (string, error) {
-	jsm, err := json.Marshal(m)
+// DownloadRaw downloads raw data from swarm
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+	uri := c.Gateway + "/bzzr:/" + hash
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		panic(err)
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+	return res.Body, nil
 }
 
-func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+// File represents a file in a swarm manifest and is used for uploading and
+// downloading content to and from swarm
+type File struct {
+	io.ReadCloser
+	api.ManifestEntry
+}
+
+// Open opens a local file which can then be passed to client.Upload to upload
+// it to swarm
+func Open(path string) (*File, error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
+	stat, err := f.Stat()
 	if err != nil {
-		return "", err
+		f.Close()
+		return nil, err
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
+	return &File{
+		ReadCloser: f,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: mime.TypeByExtension(filepath.Ext(path)),
+			Mode:        int64(stat.Mode()),
+			Size:        stat.Size(),
+			ModTime:     stat.ModTime(),
+		},
+	}, nil
+}
+
+// Upload uploads a file to swarm and either adds it to an existing manifest
+// (if the manifest argument is non-empty) or creates a new manifest containing
+// the file, returning the resulting manifest hash (the file will then be
+// available at bzz:/<hash>/<path>)
+func (c *Client) Upload(file *File, manifest string) (string, error) {
+	if file.Size <= 0 {
+		return "", errors.New("file size must be greater than zero")
 	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &FileUploader{file})
 }
 
-func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+// Download downloads a file with the given path from the swarm manifest with
+// the given hash (i.e. it gets bzz:/<hash>/<path>)
+func (c *Client) Download(hash, path string) (*File, error) {
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	return &File{
+		ReadCloser: res.Body,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: res.Header.Get("Content-Type"),
+			Size:        res.ContentLength,
+		},
+	}, nil
+}
+
+// UploadDirectory uploads a directory tree to swarm and either adds the files
+// to an existing manifest (if the manifest argument is non-empty) or creates a
+// new manifest, returning the resulting manifest hash (files from the
+// directory will then be available at bzz:/<hash>/path/to/file), with
+// the file specified in defaultPath being uploaded to the root of the manifest
+// (i.e. bzz:/<hash>/)
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+	stat, err := os.Stat(dir)
 	if err != nil {
 		return "", err
+	} else if !stat.IsDir() {
+		return "", fmt.Errorf("not a directory: %s", dir)
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
 }
 
-func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+// DownloadDirectory downloads the files contained in a swarm manifest under
+// the given path into a local directory (existing files will be overwritten)
+func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+	stat, err := os.Stat(destDir)
+	if err != nil {
+		return err
+	} else if !stat.IsDir() {
+		return fmt.Errorf("not a directory: %s", destDir)
+	}
 
-	mroot := Manifest{}
-	req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	req.Header.Set("Accept", "application/x-tar")
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	defer resp.Body.Close()
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	tr := tar.NewReader(res.Body)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		// ignore the default path file
+		if hdr.Name == "" {
+			continue
+		}
 
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+		dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
+		if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+			return err
+		}
+		var mode os.FileMode = 0644
+		if hdr.Mode > 0 {
+			mode = os.FileMode(hdr.Mode)
+		}
+		dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+		if err != nil {
+			return err
+		}
+		n, err := io.Copy(dst, tr)
+		dst.Close()
+		if err != nil {
+			return err
+		} else if n != hdr.Size {
+			return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
+		}
+	}
+}
 
+// UploadManifest uploads the given manifest to swarm
+func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+	data, err := json.Marshal(m)
+	if err != nil {
+		return "", err
 	}
-	content, err := ioutil.ReadAll(resp.Body)
+	return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+}
 
-	err = json.Unmarshal(content, &mroot)
+// DownloadManifest downloads a swarm manifest
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+	res, err := c.DownloadRaw(hash)
 	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+		return nil, err
+	}
+	defer res.Close()
+	var manifest api.Manifest
+	if err := json.NewDecoder(res).Decode(&manifest); err != nil {
+		return nil, err
 	}
-	return mroot, err
+	return &manifest, nil
 }
 
-// ManifestFileList downloads the manifest with the given hash and generates a
-// list of files and directory prefixes which have the specified prefix.
+// List list files in a swarm manifest which have the given prefix, grouping
+// common prefixes using "/" as a delimiter.
 //
 // For example, if the manifest represents the following directory structure:
 //
@@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
 // - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
 // - a prefix of "file"  would return [file1.txt, file2.txt]
 // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
-func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
-	manifest, err := c.DownloadManifest(hash)
+//
+// where entries ending with "/" are common prefixes.
+func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
 	if err != nil {
 		return nil, err
 	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	var list api.ManifestList
+	if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
+		return nil, err
+	}
+	return &list, nil
+}
+
+// Uploader uploads files to swarm using a provided UploadFn
+type Uploader interface {
+	Upload(UploadFn) error
+}
+
+type UploaderFunc func(UploadFn) error
+
+func (u UploaderFunc) Upload(upload UploadFn) error {
+	return u(upload)
+}
+
+// DirectoryUploader uploads all files in a directory, optionally uploading
+// a file to the default path
+type DirectoryUploader struct {
+	Dir         string
+	DefaultPath string
+}
 
-	// handleFile handles a manifest entry which is a direct reference to a
-	// file (i.e. it is not a swarm manifest)
-	handleFile := func(entry ManifestEntry) {
-		// ignore the file if it doesn't have the specified prefix
-		if !strings.HasPrefix(entry.Path, prefix) {
-			return
+// Upload performs the upload of the directory and default path
+func (d *DirectoryUploader) Upload(upload UploadFn) error {
+	if d.DefaultPath != "" {
+		file, err := Open(d.DefaultPath)
+		if err != nil {
+			return err
 		}
-		// if the path after the prefix contains a directory separator,
-		// add a directory prefix to the entries, otherwise add the
-		// file
-		suffix := strings.TrimPrefix(entry.Path, prefix)
-		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
-			entries = append(entries, ManifestEntry{
-				Path:        prefix + suffix[:sepIndex+1],
-				ContentType: "DIR",
-			})
-		} else {
-			if entry.Path == "" {
-				entry.Path = "/"
-			}
-			entries = append(entries, entry)
+		if err := upload(file); err != nil {
+			return err
 		}
 	}
-
-	// handleManifest handles a manifest entry which is a reference to
-	// another swarm manifest.
-	handleManifest := func(entry ManifestEntry) error {
-		// if the manifest's path is a prefix of the specified prefix
-		// then just recurse into the manifest by stripping its path
-		// from the prefix
-		if strings.HasPrefix(prefix, entry.Path) {
-			subPrefix := strings.TrimPrefix(prefix, entry.Path)
-			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
+	return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if f.IsDir() {
 			return nil
 		}
+		file, err := Open(path)
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(d.Dir, path)
+		if err != nil {
+			return err
+		}
+		file.Path = filepath.ToSlash(relPath)
+		return upload(file)
+	})
+}
 
-		// if the manifest's path has the specified prefix, then if the
-		// path after the prefix contains a directory separator, add a
-		// directory prefix to the entries, otherwise recurse into the
-		// manifest
-		if strings.HasPrefix(entry.Path, prefix) {
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			sepIndex := strings.Index(suffix, "/")
-			if sepIndex > -1 {
-				entries = append(entries, ManifestEntry{
-					Path:        prefix + suffix[:sepIndex+1],
-					ContentType: "DIR",
-				})
-				return nil
-			}
-			subEntries, err := c.ManifestFileList(entry.Hash, "")
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
-			return nil
+// FileUploader uploads a single file
+type FileUploader struct {
+	File *File
+}
+
+// Upload performs the upload of the file
+func (f *FileUploader) Upload(upload UploadFn) error {
+	return upload(f.File)
+}
+
+// UploadFn is the type of function passed to an Uploader to perform the upload
+// of a single file (for example, a directory uploader would call a provided
+// UploadFn for each file in the directory tree)
+type UploadFn func(file *File) error
+
+// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
+// returning the resulting manifest hash
+func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-tar")
+
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	tw := tar.NewWriter(reqW)
+
+	// define an UploadFn which adds files to the tar stream
+	uploadFn := func(file *File) error {
+		hdr := &tar.Header{
+			Name:    file.Path,
+			Mode:    file.Mode,
+			Size:    file.Size,
+			ModTime: file.ModTime,
+			Xattrs: map[string]string{
+				"user.swarm.content-type": file.ContentType,
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
 		}
-		return nil
+		_, err = io.Copy(tw, file)
+		return err
 	}
 
-	for _, entry := range manifest.Entries {
-		if entry.ContentType == "application/bzz-manifest+json" {
-			if err := handleManifest(entry); err != nil {
-				return nil, err
-			}
-		} else {
-			handleFile(entry)
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the tar stream
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = tw.Close()
 		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+// MultipartUpload uses the given Uploader to upload files to swarm as a
+// multipart form, returning the resulting manifest hash
+func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
 	}
 
-	return
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	mw := multipart.NewWriter(reqW)
+	req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
+
+	// define an UploadFn which adds files to the multipart form
+	uploadFn := func(file *File) error {
+		hdr := make(textproto.MIMEHeader)
+		hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
+		hdr.Set("Content-Type", file.ContentType)
+		hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
+		w, err := mw.CreatePart(hdr)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(w, file)
+		return err
+	}
+
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the multipart form
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = mw.Close()
+		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
 }
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index 135474475..4d02ceaf4 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -17,6 +17,7 @@
 package client
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,52 +25,221 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func TestClientManifestFileList(t *testing.T) {
+// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
+func TestClientUploadDownloadRaw(t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
+	client := NewClient(srv.URL)
+
+	// upload some raw data
+	data := []byte("foo123")
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the same data
+	res, err := client.DownloadRaw(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Close()
+	gotData, err := ioutil.ReadAll(res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(gotData, data) {
+		t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
+	}
+}
+
+// TestClientUploadDownloadFiles test uploading and downloading files to swarm
+// manifests
+func TestClientUploadDownloadFiles(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	client := NewClient(srv.URL)
+	upload := func(manifest, path string, data []byte) string {
+		file := &File{
+			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+			ManifestEntry: api.ManifestEntry{
+				Path:        path,
+				ContentType: "text/plain",
+				Size:        int64(len(data)),
+			},
+		}
+		hash, err := client.Upload(file, manifest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return hash
+	}
+	checkDownload := func(manifest, path string, expected []byte) {
+		file, err := client.Download(manifest, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		if file.Size != int64(len(expected)) {
+			t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
+		}
+		if file.ContentType != file.ContentType {
+			t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
+		}
+	}
+
+	// upload a file to the root of a manifest
+	rootData := []byte("some-data")
+	rootHash := upload("", "", rootData)
+
+	// check we can download the root file
+	checkDownload(rootHash, "", rootData)
+
+	// upload another file to the same manifest
+	otherData := []byte("some-other-data")
+	newHash := upload(rootHash, "some/other/path", otherData)
+
+	// check we can download both files from the new manifest
+	checkDownload(newHash, "", rootData)
+	checkDownload(newHash, "some/other/path", otherData)
+
+	// replace the root file with different data
+	newHash = upload(newHash, "", otherData)
+
+	// check both files have the other data
+	checkDownload(newHash, "", otherData)
+	checkDownload(newHash, "some/other/path", otherData)
+}
+
+var testDirFiles = []string{
+	"file1.txt",
+	"file2.txt",
+	"dir1/file3.txt",
+	"dir1/file4.txt",
+	"dir2/file5.txt",
+	"dir2/dir3/file6.txt",
+	"dir2/dir4/file7.txt",
+	"dir2/dir4/file8.txt",
+}
+
+func newTestDirectory(t *testing.T) string {
 	dir, err := ioutil.TempDir("", "swarm-client-test")
 	if err != nil {
 		t.Fatal(err)
 	}
-	files := []string{
-		"file1.txt",
-		"file2.txt",
-		"dir1/file3.txt",
-		"dir1/file4.txt",
-		"dir2/file5.txt",
-		"dir2/dir3/file6.txt",
-		"dir2/dir4/file7.txt",
-		"dir2/dir4/file8.txt",
-	}
-	for _, file := range files {
+
+	for _, file := range testDirFiles {
 		path := filepath.Join(dir, file)
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error creating dir for %s: %s", path, err)
 		}
-		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+		if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error writing file %s: %s", path, err)
 		}
 	}
 
+	return dir
+}
+
+// TestClientUploadDownloadDirectory tests uploading and downloading a
+// directory of files to a swarm manifest
+func TestClientUploadDownloadDirectory(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	// upload the directory
 	client := NewClient(srv.URL)
+	defaultPath := filepath.Join(dir, testDirFiles[0])
+	hash, err := client.UploadDirectory(dir, defaultPath, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string, expected []byte) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected data to be %q, got %q", expected, data)
+		}
+	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file, []byte(file))
+	}
+
+	// check we can download the default path
+	checkDownloadFile("", []byte(testDirFiles[0]))
+
+	// check we can download the directory
+	tmp, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range testDirFiles {
+		data, err := ioutil.ReadFile(filepath.Join(tmp, file))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, []byte(file)) {
+			t.Fatalf("expected data to be %q, got %q", file, data)
+		}
+	}
+}
+
+// TestClientFileList tests listing files in a swarm manifest
+func TestClientFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
-	hash, err := client.UploadDirectory(dir, "")
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	client := NewClient(srv.URL)
+	hash, err := client.UploadDirectory(dir, "", "")
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
 
 	ls := func(prefix string) []string {
-		entries, err := client.ManifestFileList(hash, prefix)
+		list, err := client.List(hash, prefix)
 		if err != nil {
 			t.Fatal(err)
 		}
-		paths := make([]string, len(entries))
-		for i, entry := range entries {
-			paths[i] = entry.Path
+		paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
+		for _, prefix := range list.CommonPrefixes {
+			paths = append(paths, prefix)
+		}
+		for _, entry := range list.Entries {
+			paths = append(paths, entry.Path)
 		}
 		sort.Strings(paths)
 		return paths
@@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
 	for prefix, expected := range tests {
 		actual := ls(prefix)
 		if !reflect.DeepEqual(actual, expected) {
-			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+			t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
+		}
+	}
+}
+
+// TestClientMultipartUpload tests uploading files to swarm using a multipart
+// upload
+func TestClientMultipartUpload(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	// define an uploader which uploads testDirFiles with some data
+	data := []byte("some-data")
+	uploader := UploaderFunc(func(upload UploadFn) error {
+		for _, name := range testDirFiles {
+			file := &File{
+				ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+				ManifestEntry: api.ManifestEntry{
+					Path:        name,
+					ContentType: "text/plain",
+					Size:        int64(len(data)),
+				},
+			}
+			if err := upload(file); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// upload the files as a multipart upload
+	client := NewClient(srv.URL)
+	hash, err := client.MultipartUpload("", uploader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		gotData, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(gotData, data) {
+			t.Fatalf("expected data to be %q, got %q", data, gotData)
 		}
 	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file)
+	}
 }
diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go
index c2583e265..e7deaa32f 100644
--- a/swarm/api/filesystem.go
+++ b/swarm/api/filesystem.go
@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 
@@ -43,6 +44,8 @@ func NewFileSystem(api *Api) *FileSystem {
 // Upload replicates a local directory as a manifest file and uploads it
 // using dpa store
 // TODO: localpath should point to a manifest
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	var list []*manifestTrieEntry
 	localpath, err := filepath.Abs(filepath.Clean(lpath))
@@ -72,9 +75,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 				if path[:start] != localpath {
 					return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
 				}
-				entry := &manifestTrieEntry{
-					Path: filepath.ToSlash(path),
-				}
+				entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
 				list = append(list, entry)
 			}
 			return err
@@ -91,9 +92,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 		if localpath[:start] != dir {
 			return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
 		}
-		entry := &manifestTrieEntry{
-			Path: filepath.ToSlash(localpath),
-		}
+		entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
 		list = append(list, entry)
 	}
 
@@ -153,11 +152,10 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 		}
 		entry.Path = RegularSlashes(entry.Path[start:])
 		if entry.Path == index {
-			ientry := &manifestTrieEntry{
-				Path:        "",
-				Hash:        entry.Hash,
+			ientry := newManifestTrieEntry(&ManifestEntry{
 				ContentType: entry.ContentType,
-			}
+			}, nil)
+			ientry.Hash = entry.Hash
 			trie.addEntry(ientry, quitC)
 		}
 		trie.addEntry(entry, quitC)
@@ -174,6 +172,8 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 
 // Download replicates the manifest path structure on the local filesystem
 // under localpath
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *FileSystem) Download(bzzpath, localpath string) error {
 	lpath, err := filepath.Abs(filepath.Clean(localpath))
 	if err != nil {
@@ -185,10 +185,15 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	//resolving host and port
-	key, _, path, err := self.api.parseAndResolve(bzzpath, true)
+	uri, err := Parse(path.Join("bzz:/", bzzpath))
+	if err != nil {
+		return err
+	}
+	key, err := self.api.Resolve(uri)
 	if err != nil {
 		return err
 	}
+	path := uri.Path
 
 	if len(path) > 0 {
 		path += "/"
diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go
index 4a27cb1da..8a15e735d 100644
--- a/swarm/api/filesystem_test.go
+++ b/swarm/api/filesystem_test.go
@@ -23,6 +23,9 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
 var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
@@ -51,16 +54,17 @@ func TestApiDirUpload0(t *testing.T) {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index.html")
+		resp := testGet(t, api, bzzhash, "index.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
 		content = readPath(t, "testdata", "test0", "index.css")
-		resp = testGet(t, api, bzzhash+"/index.css")
+		resp = testGet(t, api, bzzhash, "index.css")
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, err = api.Get(bzzhash, true)
+		key := storage.Key(common.Hex2Bytes(bzzhash))
+		_, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -90,7 +94,8 @@ func TestApiDirUploadModify(t *testing.T) {
 			return
 		}
 
-		bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
+		key := storage.Key(common.Hex2Bytes(bzzhash))
+		key, err = api.Modify(key, "index.html", "", "")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -107,32 +112,33 @@ func TestApiDirUploadModify(t *testing.T) {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true)
+		key, err = api.Modify(key, "index2.html", hash.Hex(), "text/html; charset=utf-8")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true)
+		key, err = api.Modify(key, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
+		bzzhash = key.String()
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index2.html")
+		resp := testGet(t, api, bzzhash, "index2.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
-		resp = testGet(t, api, bzzhash+"/img/logo.png")
+		resp = testGet(t, api, bzzhash, "img/logo.png")
 		exp = expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 
 		content = readPath(t, "testdata", "test0", "index.css")
-		resp = testGet(t, api, bzzhash+"/index.css")
+		resp = testGet(t, api, bzzhash, "index.css")
 		exp = expResponse(content, "text/css", 0)
 		checkResponse(t, resp, exp)
 
-		_, _, _, err = api.Get(bzzhash, true)
+		_, _, _, err = api.Get(key, "")
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}
@@ -149,7 +155,7 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, bzzhash, "")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})
@@ -165,7 +171,7 @@ func TestApiFileUpload(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash+"/index.html")
+		resp := testGet(t, api, bzzhash, "index.html")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})
@@ -181,7 +187,7 @@ func TestApiFileUploadWithRootFile(t *testing.T) {
 		}
 
 		content := readPath(t, "testdata", "test0", "index.html")
-		resp := testGet(t, api, bzzhash)
+		resp := testGet(t, api, bzzhash, "")
 		exp := expResponse(content, "text/html; charset=utf-8", 0)
 		checkResponse(t, resp, exp)
 	})
diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go
index fc74f5d3a..f99c4f35e 100644
--- a/swarm/api/http/roundtripper_test.go
+++ b/swarm/api/http/roundtripper_test.go
@@ -18,14 +18,14 @@ package http
 
 import (
 	"io/ioutil"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 )
 
-const port = "3222"
-
 func TestRoundTripper(t *testing.T) {
 	serveMux := http.NewServeMux()
 	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -36,9 +36,12 @@ func TestRoundTripper(t *testing.T) {
 			http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
 		}
 	})
-	go http.ListenAndServe(":"+port, serveMux)
 
-	rt := &RoundTripper{Port: port}
+	srv := httptest.NewServer(serveMux)
+	defer srv.Close()
+
+	host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
+	rt := &RoundTripper{Host: host, Port: port}
 	trans := &http.Transport{}
 	trans.RegisterProtocol("bzz", rt)
 	client := &http.Client{Transport: trans}
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index 44e2c203a..849b9e10f 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -20,13 +20,19 @@ A simple http server interface to Swarm
 package http
 
 import (
-	"bytes"
+	"archive/tar"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
 	"net/http"
-	"regexp"
+	"os"
+	"path"
+	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -36,26 +42,6 @@ import (
 	"github.com/rs/cors"
 )
 
-const (
-	rawType = "application/octet-stream"
-)
-
-var (
-	// accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
-	bzzPrefix       = regexp.MustCompile("^/+bzz[ir]?:/+")
-	trailingSlashes = regexp.MustCompile("/+$")
-	rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$")
-	// forever         = func() time.Time { return time.Unix(0, 0) }
-	forever = time.Now
-)
-
-type sequentialReader struct {
-	reader io.Reader
-	pos    int64
-	ahead  map[int64](chan bool)
-	lock   sync.Mutex
-}
-
 // ServerConfig is the basic configuration needed for the HTTP server and also
 // includes CORS settings.
 type ServerConfig struct {
@@ -94,242 +80,569 @@ type Server struct {
 	api *api.Api
 }
 
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	requestURL := r.URL
-	// This is wrong
-	//	if requestURL.Host == "" {
-	//		var err error
-	//		requestURL, err = url.Parse(r.Referer() + requestURL.String())
-	//		if err != nil {
-	//			http.Error(w, err.Error(), http.StatusBadRequest)
-	//			return
-	//		}
-	//	}
-	log.Debug(fmt.Sprintf("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")))
-	uri := requestURL.Path
-	var raw, nameresolver bool
-	var proto string
-
-	// HTTP-based URL protocol handler
-	log.Debug(fmt.Sprintf("BZZ request URI: '%s'", uri))
-
-	path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
-		proto = p
-		return ""
-	})
+// Request wraps http.Request and also includes the parsed bzz URI
+type Request struct {
+	http.Request
+
+	uri *api.URI
+}
 
-	// protocol identification (ugly)
-	if proto == "" {
-		log.Error(fmt.Sprintf("[BZZ] Swarm: Protocol error in request `%s`.", uri))
-		http.Error(w, "Invalid request URL: need access protocol (bzz:/, bzzr:/, bzzi:/) as first element in path.", http.StatusBadRequest)
+// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// body in swarm and returns the resulting storage key as a text/plain response
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
+	if r.uri.Path != "" {
+		s.BadRequest(w, r, "raw POST request cannot contain a path")
 		return
 	}
-	if len(proto) > 4 {
-		raw = proto[1:5] == "bzzr"
-		nameresolver = proto[1:5] != "bzzi"
+
+	if r.Header.Get("Content-Length") == "" {
+		s.BadRequest(w, r, "missing Content-Length header in request")
+		return
 	}
 
-	log.Debug("", "msg", log.Lazy{Fn: func() string {
-		return fmt.Sprintf("[BZZ] Swarm: %s request over protocol %s '%s' received.", r.Method, proto, path)
-	}})
+	key, err := s.api.Store(r.Body, r.ContentLength, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+	s.logDebug("content for %s stored", key.Log())
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, key)
+}
+
+// HandlePostFiles handles a POST request (or deprecated PUT request) to
+// bzz:/<hash>/<path> which contains either a single file or multiple files
+// (either a tar archive or multipart form), adds those files either to an
+// existing manifest or to a new manifest under <path> and returns the
+// resulting manifest hash as a text/plain response
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
+	contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil {
+		s.BadRequest(w, r, err.Error())
+		return
+	}
 
-	switch {
-	case r.Method == "POST" || r.Method == "PUT":
-		if r.Header.Get("content-length") == "" {
-			http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
+	var key storage.Key
+	if r.uri.Addr != "" {
+		key, err = s.api.Resolve(r.uri)
+		if err != nil {
+			s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
 			return
 		}
-		key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
-		if err == nil {
-			log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
-		} else {
-			http.Error(w, err.Error(), http.StatusBadRequest)
+	} else {
+		key, err = s.api.NewManifest()
+		if err != nil {
+			s.Error(w, r, err)
 			return
 		}
-		if r.Method == "POST" {
-			if raw {
-				w.Header().Set("Content-Type", "text/plain")
-				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key))))
-			} else {
-				http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
-				return
+	}
+
+	newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+		switch contentType {
+
+		case "application/x-tar":
+			return s.handleTarUpload(r, mw)
+
+		case "multipart/form-data":
+			return s.handleMultipartUpload(r, params["boundary"], mw)
+
+		default:
+			return s.handleDirectUpload(r, mw)
+		}
+	})
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error creating manifest: %s", err))
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newKey)
+}
+
+func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error {
+	tr := tar.NewReader(req.Body)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("error reading tar stream: %s", err)
+		}
+
+		// only store regular files
+		if !hdr.FileInfo().Mode().IsRegular() {
+			continue
+		}
+
+		// add the entry under the path from the request
+		path := path.Join(req.uri.Path, hdr.Name)
+		entry := &api.ManifestEntry{
+			Path:        path,
+			ContentType: hdr.Xattrs["user.swarm.content-type"],
+			Mode:        hdr.Mode,
+			Size:        hdr.Size,
+			ModTime:     hdr.ModTime,
+		}
+		s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+		contentKey, err := mw.AddEntry(tr, entry)
+		if err != nil {
+			return fmt.Errorf("error adding manifest entry from tar stream: %s", err)
+		}
+		s.logDebug("content for %s stored", contentKey.Log())
+	}
+}
+
+func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
+	mr := multipart.NewReader(req.Body, boundary)
+	for {
+		part, err := mr.NextPart()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("error reading multipart form: %s", err)
+		}
+
+		var size int64
+		var reader io.Reader = part
+		if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
+			size, err = strconv.ParseInt(contentLength, 10, 64)
+			if err != nil {
+				return fmt.Errorf("error parsing multipart content length: %s", err)
 			}
+			reader = part
 		} else {
-			// PUT
-			if raw {
-				http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest)
-				return
-			} else {
-				path = api.RegularSlashes(path)
-				mime := r.Header.Get("Content-Type")
-				// TODO proper root hash separation
-				log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
-				newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
-				if err == nil {
-					log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
-					w.Header().Set("Content-Type", "text/plain")
-					http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
-				} else {
-					http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest)
-					return
-				}
+			// copy the part to a tmp file to get its size
+			tmp, err := ioutil.TempFile("", "swarm-multipart")
+			if err != nil {
+				return err
 			}
-		}
-	case r.Method == "DELETE":
-		if raw {
-			http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest)
-			return
-		} else {
-			path = api.RegularSlashes(path)
-			log.Debug(fmt.Sprintf("Delete '%s'.", path))
-			newKey, err := s.api.Modify(path, "", "", nameresolver)
-			if err == nil {
-				log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
-				w.Header().Set("Content-Type", "text/plain")
-				http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey)))
-			} else {
-				http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest)
-				return
+			defer os.Remove(tmp.Name())
+			defer tmp.Close()
+			size, err = io.Copy(tmp, part)
+			if err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
+			}
+			if _, err := tmp.Seek(0, os.SEEK_SET); err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
 			}
+			reader = tmp
+		}
+
+		// add the entry under the path from the request
+		name := part.FileName()
+		if name == "" {
+			name = part.FormName()
+		}
+		path := path.Join(req.uri.Path, name)
+		entry := &api.ManifestEntry{
+			Path:        path,
+			ContentType: part.Header.Get("Content-Type"),
+			Size:        size,
+			ModTime:     time.Now(),
 		}
-	case r.Method == "GET" || r.Method == "HEAD":
-		path = trailingSlashes.ReplaceAllString(path, "")
-		if path == "" {
-			http.Error(w, "Empty path not allowed", http.StatusBadRequest)
+		s.logDebug("adding %s (%d bytes) to new manifest", entry.Path, entry.Size)
+		contentKey, err := mw.AddEntry(reader, entry)
+		if err != nil {
+			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
+		}
+		s.logDebug("content for %s stored", contentKey.Log())
+	}
+}
+
+func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
+	key, err := mw.AddEntry(req.Body, &api.ManifestEntry{
+		Path:        req.uri.Path,
+		ContentType: req.Header.Get("Content-Type"),
+		Mode:        0644,
+		Size:        req.ContentLength,
+		ModTime:     time.Now(),
+	})
+	if err != nil {
+		return err
+	}
+	s.logDebug("content for %s stored", key.Log())
+	return nil
+}
+
+// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
+// <path> from <manifest> and returns the resulting manifest hash as a
+// text/plain response
+func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error {
+		s.logDebug("removing %s from manifest %s", r.uri.Path, key.Log())
+		return mw.RemoveEntry(r.uri.Path)
+	})
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error updating manifest: %s", err))
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newKey)
+}
+
+// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// the raw content stored at the given storage key
+func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	// if path is set, interpret <key> as a manifest and return the
+	// raw entry at the given path
+	if r.uri.Path != "" {
+		walker, err := s.api.NewManifestWalker(key, nil)
+		if err != nil {
+			s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key))
 			return
 		}
-		if raw {
-			var reader storage.LazySectionReader
-			parsedurl, _ := api.Parse(path)
-
-			if parsedurl == path {
-				key, err := s.api.Resolve(parsedurl, nameresolver)
-				if err != nil {
-					log.Error(fmt.Sprintf("%v", err))
-					http.Error(w, err.Error(), http.StatusBadRequest)
-					return
-				}
-				reader = s.api.Retrieve(key)
-			} else {
-				var status int
-				readertmp, _, status, err := s.api.Get(path, nameresolver)
-				if err != nil {
-					http.Error(w, err.Error(), status)
-					return
-				}
-				reader = readertmp
+		var entry *api.ManifestEntry
+		walker.Walk(func(e *api.ManifestEntry) error {
+			// if the entry matches the path, set entry and stop
+			// the walk
+			if e.Path == r.uri.Path {
+				entry = e
+				// return an error to cancel the walk
+				return errors.New("found")
 			}
 
-			// retrieving content
-
-			quitC := make(chan bool)
-			size, err := reader.Size(quitC)
-			if err != nil {
-				log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
-				//An error on call to Size means we don't have the root chunk
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
+			// ignore non-manifest files
+			if e.ContentType != api.ManifestType {
+				return nil
 			}
-			log.Debug(fmt.Sprintf("Reading %d bytes.", size))
 
-			// setting mime type
-			qv := requestURL.Query()
-			mimeType := qv.Get("content_type")
-			if mimeType == "" {
-				mimeType = rawType
+			// if the manifest's path is a prefix of the
+			// requested path, recurse into it by returning
+			// nil and continuing the walk
+			if strings.HasPrefix(r.uri.Path, e.Path) {
+				return nil
 			}
 
-			w.Header().Set("Content-Type", mimeType)
-			http.ServeContent(w, r, uri, forever(), reader)
-			log.Debug(fmt.Sprintf("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType))
+			return api.SkipManifest
+		})
+		if entry == nil {
+			http.NotFound(w, &r.Request)
+			return
+		}
+		key = storage.Key(common.Hex2Bytes(entry.Hash))
+	}
 
-			// retrieve path via manifest
-		} else {
-			log.Debug(fmt.Sprintf("Structured GET request '%s' received.", uri))
-			// add trailing slash, if missing
-			if rootDocumentUri.MatchString(uri) {
-				http.Redirect(w, r, path+"/", http.StatusFound)
-				return
+	// check the root chunk exists by retrieving the file's size
+	reader := s.api.Retrieve(key)
+	if _, err := reader.Size(nil); err != nil {
+		s.logDebug("key not found %s: %s", key, err)
+		http.NotFound(w, &r.Request)
+		return
+	}
+
+	// allow the request to overwrite the content type using a query
+	// parameter
+	contentType := "application/octet-stream"
+	if typ := r.URL.Query().Get("content_type"); typ != "" {
+		contentType = typ
+	}
+	w.Header().Set("Content-Type", contentType)
+
+	http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+// HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept
+// header of "application/x-tar" and returns a tar stream of all files
+// contained in the manifest
+func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
+	if r.uri.Path != "" {
+		s.BadRequest(w, r, "files request cannot contain a path")
+		return
+	}
+
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	walker, err := s.api.NewManifestWalker(key, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+	w.Header().Set("Content-Type", "application/x-tar")
+	w.WriteHeader(http.StatusOK)
+
+	err = walker.Walk(func(entry *api.ManifestEntry) error {
+		// ignore manifests (walk will recurse into them)
+		if entry.ContentType == api.ManifestType {
+			return nil
+		}
+
+		// retrieve the entry's key and size
+		reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
+		size, err := reader.Size(nil)
+		if err != nil {
+			return err
+		}
+
+		// write a tar header for the entry
+		hdr := &tar.Header{
+			Name:    entry.Path,
+			Mode:    entry.Mode,
+			Size:    size,
+			ModTime: entry.ModTime,
+			Xattrs: map[string]string{
+				"user.swarm.content-type": entry.ContentType,
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+
+		// copy the file into the tar stream
+		n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size))
+		if err != nil {
+			return err
+		} else if n != size {
+			return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n)
+		}
+
+		return nil
+	})
+	if err != nil {
+		s.logError("error generating tar stream: %s", err)
+	}
+}
+
+// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
+// the "list" query parameter set to "true" and returns a list of all files
+// contained in <manifest> under <path> grouped into common prefixes using
+// "/" as a delimiter
+func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
+	// ensure the root path has a trailing slash so that relative URLs work
+	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		return
+	}
+
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	walker, err := s.api.NewManifestWalker(key, nil)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	var list api.ManifestList
+	prefix := r.uri.Path
+	err = walker.Walk(func(entry *api.ManifestEntry) error {
+		// handle non-manifest files
+		if entry.ContentType != api.ManifestType {
+			// ignore the file if it doesn't have the specified prefix
+			if !strings.HasPrefix(entry.Path, prefix) {
+				return nil
 			}
-			reader, mimeType, status, err := s.api.Get(path, nameresolver)
-			if err != nil {
-				if _, ok := err.(api.ErrResolve); ok {
-					log.Debug(fmt.Sprintf("%v", err))
-					status = http.StatusBadRequest
-				} else {
-					log.Debug(fmt.Sprintf("error retrieving '%s': %v", uri, err))
-					status = http.StatusNotFound
-				}
-				http.Error(w, err.Error(), status)
-				return
+
+			// if the path after the prefix contains a slash, add a
+			// common prefix to the list, otherwise add the entry
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return nil
 			}
-			// set mime type and status headers
-			w.Header().Set("Content-Type", mimeType)
-			if status > 0 {
-				w.WriteHeader(status)
-			} else {
-				status = 200
+			if entry.Path == "" {
+				entry.Path = "/"
 			}
-			quitC := make(chan bool)
-			size, err := reader.Size(quitC)
-			if err != nil {
-				log.Debug(fmt.Sprintf("Could not determine size: %v", err.Error()))
-				//An error on call to Size means we don't have the root chunk
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
+			list.Entries = append(list.Entries, entry)
+			return nil
+		}
+
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by returning nil and
+		// continuing the walk
+		if strings.HasPrefix(prefix, entry.Path) {
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a slash, add a common prefix
+		// to the list and skip the manifest, otherwise recurse into
+		// the manifest by returning nil and continuing the walk
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			if index := strings.Index(suffix, "/"); index > -1 {
+				list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1])
+				return api.SkipManifest
 			}
-			log.Debug(fmt.Sprintf("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status))
+			return nil
+		}
 
-			http.ServeContent(w, r, path, forever(), reader)
+		// the manifest neither has the prefix or needs recursing in to
+		// so just skip it
+		return api.SkipManifest
+	})
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
 
+	// if the client wants HTML (e.g. a browser) then render the list as a
+	// HTML index with relative URLs
+	if strings.Contains(r.Header.Get("Accept"), "text/html") {
+		w.Header().Set("Content-Type", "text/html")
+		err := htmlListTemplate.Execute(w, &htmlListData{
+			URI:  r.uri,
+			List: &list,
+		})
+		if err != nil {
+			s.logError("error rendering list HTML: %s", err)
 		}
-	default:
-		http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+		return
 	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(&list)
 }
 
-func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) {
-	self.lock.Lock()
-	// assert self.pos <= off
-	if self.pos > off {
-		log.Error(fmt.Sprintf("non-sequential read attempted from sequentialReader; %d > %d", self.pos, off))
-		panic("Non-sequential read attempt")
-	}
-	if self.pos != off {
-		log.Debug(fmt.Sprintf("deferred read in POST at position %d, offset %d.", self.pos, off))
-		wait := make(chan bool)
-		self.ahead[off] = wait
-		self.lock.Unlock()
-		if <-wait {
-			// failed read behind
-			n = 0
-			err = io.ErrUnexpectedEOF
+// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
+// with the content of the file at <path> from the given <manifest>
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
+	key, err := s.api.Resolve(r.uri)
+	if err != nil {
+		s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err))
+		return
+	}
+
+	reader, contentType, _, err := s.api.Get(key, r.uri.Path)
+	if err != nil {
+		s.Error(w, r, err)
+		return
+	}
+
+	// check the root chunk exists by retrieving the file's size
+	if _, err := reader.Size(nil); err != nil {
+		s.logDebug("file not found %s: %s", r.uri, err)
+		http.NotFound(w, &r.Request)
+		return
+	}
+
+	w.Header().Set("Content-Type", contentType)
+
+	http.ServeContent(w, &r.Request, "", time.Now(), reader)
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept"))
+
+	uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+	if err != nil {
+		s.logError("Invalid URI %q: %s", r.URL.Path, err)
+		http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest)
+		return
+	}
+	s.logDebug("%s request received for %s", r.Method, uri)
+
+	req := &Request{Request: *r, uri: uri}
+	switch r.Method {
+	case "POST":
+		if uri.Raw() {
+			s.HandlePostRaw(w, req)
+		} else {
+			s.HandlePostFiles(w, req)
+		}
+
+	case "PUT":
+		// DEPRECATED:
+		//   clients should send a POST request (the request creates a
+		//   new manifest leaving the existing one intact, so it isn't
+		//   strictly a traditional PUT request which replaces content
+		//   at a URI, and POST is more ubiquitous)
+		if uri.Raw() {
+			http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
+		} else {
+			s.HandlePostFiles(w, req)
 		}
-		self.lock.Lock()
-	}
-	localPos := 0
-	for localPos < len(target) {
-		n, err = self.reader.Read(target[localPos:])
-		localPos += n
-		log.Debug(fmt.Sprintf("Read %d bytes into buffer size %d from POST, error %v.", n, len(target), err))
-		if err != nil {
-			log.Debug(fmt.Sprintf("POST stream's reading terminated with %v.", err))
-			for i := range self.ahead {
-				self.ahead[i] <- true
-				delete(self.ahead, i)
-			}
-			self.lock.Unlock()
-			return localPos, err
+
+	case "DELETE":
+		if uri.Raw() {
+			http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
+			return
+		}
+		s.HandleDelete(w, req)
+
+	case "GET":
+		if uri.Raw() {
+			s.HandleGetRaw(w, req)
+			return
+		}
+
+		if r.Header.Get("Accept") == "application/x-tar" {
+			s.HandleGetFiles(w, req)
+			return
+		}
+
+		if r.URL.Query().Get("list") == "true" {
+			s.HandleGetList(w, req)
+			return
 		}
-		self.pos += int64(n)
+
+		s.HandleGetFile(w, req)
+
+	default:
+		http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+
+	}
+}
+
+func (s *Server) updateManifest(key storage.Key, update func(mw *api.ManifestWriter) error) (storage.Key, error) {
+	mw, err := s.api.NewManifestWriter(key, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := update(mw); err != nil {
+		return nil, err
 	}
-	wait := self.ahead[self.pos]
-	if wait != nil {
-		log.Debug(fmt.Sprintf("deferred read in POST at position %d triggered.", self.pos))
-		delete(self.ahead, self.pos)
-		close(wait)
+
+	key, err = mw.Store()
+	if err != nil {
+		return nil, err
 	}
-	self.lock.Unlock()
-	return localPos, err
+	s.logDebug("generated manifest %s", key)
+	return key, nil
+}
+
+func (s *Server) logDebug(format string, v ...interface{}) {
+	log.Debug(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) logError(format string, v ...interface{}) {
+	log.Error(fmt.Sprintf("[BZZ] HTTP: "+format, v...))
+}
+
+func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) {
+	s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason)
+	http.Error(w, reason, http.StatusBadRequest)
+}
+
+func (s *Server) Error(w http.ResponseWriter, r *Request, err error) {
+	s.logError("error serving %s %s: %s", r.Method, r.uri, err)
+	http.Error(w, err.Error(), http.StatusInternalServerError)
 }
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 45a867f51..942f3ba0b 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -40,8 +40,8 @@ func TestBzzrGetPath(t *testing.T) {
 
 	testrequests := make(map[string]int)
 	testrequests["/"] = 0
-	testrequests["/a"] = 1
-	testrequests["/a/b"] = 2
+	testrequests["/a/"] = 1
+	testrequests["/a/b/"] = 2
 	testrequests["/x"] = 0
 	testrequests[""] = 0
 
diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go
new file mode 100644
index 000000000..c3ef8c0f4
--- /dev/null
+++ b/swarm/api/http/templates.go
@@ -0,0 +1,71 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"html/template"
+	"path"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+)
+
+type htmlListData struct {
+	URI  *api.URI
+	List *api.ManifestList
+}
+
+var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.FuncMap{"basename": path.Base}).Parse(`
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Swarm index of {{ .URI }}</title>
+</head>
+
+<body>
+  <h1>Swarm index of {{ .URI }}</h1>
+  <hr>
+  <table>
+    <thead>
+      <tr>
+	<th>Path</th>
+	<th>Type</th>
+	<th>Size</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      {{ range .List.CommonPrefixes }}
+	<tr>
+	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td>DIR</td>
+	  <td>-</td>
+	</tr>
+      {{ end }}
+
+      {{ range .List.Entries }}
+	<tr>
+	  <td><a href="{{ basename .Path }}">{{ basename .Path }}</a></td>
+	  <td>{{ .ContentType }}</td>
+	  <td>{{ .Size }}</td>
+	</tr>
+      {{ end }}
+  </table>
+  <hr>
+</body>
+`[1:]))
diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go
index 199f259e1..6b3630fd0 100644
--- a/swarm/api/manifest.go
+++ b/swarm/api/manifest.go
@@ -19,8 +19,11 @@ package api
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -28,25 +31,152 @@ import (
 )
 
 const (
-	manifestType = "application/bzz-manifest+json"
+	ManifestType = "application/bzz-manifest+json"
 )
 
+// Manifest represents a swarm manifest
+type Manifest struct {
+	Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest
+type ManifestEntry struct {
+	Hash        string    `json:"hash,omitempty"`
+	Path        string    `json:"path,omitempty"`
+	ContentType string    `json:"contentType,omitempty"`
+	Mode        int64     `json:"mode,omitempty"`
+	Size        int64     `json:"size,omitempty"`
+	ModTime     time.Time `json:"mod_time,omitempty"`
+	Status      int       `json:"status,omitempty"`
+}
+
+// ManifestList represents the result of listing files in a manifest
+type ManifestList struct {
+	CommonPrefixes []string         `json:"common_prefixes,omitempty"`
+	Entries        []*ManifestEntry `json:"entries,omitempty"`
+}
+
+// NewManifest creates and stores a new, empty manifest
+func (a *Api) NewManifest() (storage.Key, error) {
+	var manifest Manifest
+	data, err := json.Marshal(&manifest)
+	if err != nil {
+		return nil, err
+	}
+	return a.Store(bytes.NewReader(data), int64(len(data)), nil)
+}
+
+// ManifestWriter is used to add and remove entries from an underlying manifest
+type ManifestWriter struct {
+	api   *Api
+	trie  *manifestTrie
+	quitC chan bool
+}
+
+func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
+	trie, err := loadManifest(a.dpa, key, quitC)
+	if err != nil {
+		return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+	}
+	return &ManifestWriter{a, trie, quitC}, nil
+}
+
+// AddEntry stores the given data and adds the resulting key to the manifest
+func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
+	key, err := m.api.Store(data, e.Size, nil)
+	if err != nil {
+		return nil, err
+	}
+	entry := newManifestTrieEntry(e, nil)
+	entry.Hash = key.String()
+	m.trie.addEntry(entry, m.quitC)
+	return key, nil
+}
+
+// RemoveEntry removes the given path from the manifest
+func (m *ManifestWriter) RemoveEntry(path string) error {
+	m.trie.deleteEntry(path, m.quitC)
+	return nil
+}
+
+// Store stores the manifest, returning the resulting storage key
+func (m *ManifestWriter) Store() (storage.Key, error) {
+	return m.trie.hash, m.trie.recalcAndStore()
+}
+
+// ManifestWalker is used to recursively walk the entries in the manifest and
+// all of its submanifests
+type ManifestWalker struct {
+	api   *Api
+	trie  *manifestTrie
+	quitC chan bool
+}
+
+func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
+	trie, err := loadManifest(a.dpa, key, quitC)
+	if err != nil {
+		return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
+	}
+	return &ManifestWalker{a, trie, quitC}, nil
+}
+
+// SkipManifest is used as a return value from WalkFn to indicate that the
+// manifest should be skipped
+var SkipManifest = errors.New("skip this manifest")
+
+// WalkFn is the type of function called for each entry visited by a recursive
+// manifest walk
+type WalkFn func(entry *ManifestEntry) error
+
+// Walk recursively walks the manifest calling walkFn for each entry in the
+// manifest, including submanifests
+func (m *ManifestWalker) Walk(walkFn WalkFn) error {
+	return m.walk(m.trie, "", walkFn)
+}
+
+func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
+	for _, entry := range trie.entries {
+		if entry == nil {
+			continue
+		}
+		entry.Path = prefix + entry.Path
+		err := walkFn(&entry.ManifestEntry)
+		if err != nil {
+			if entry.ContentType == ManifestType && err == SkipManifest {
+				continue
+			}
+			return err
+		}
+		if entry.ContentType != ManifestType {
+			continue
+		}
+		if err := trie.loadSubTrie(entry, nil); err != nil {
+			return err
+		}
+		if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 type manifestTrie struct {
 	dpa     *storage.DPA
 	entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
 	hash    storage.Key             // if hash != nil, it is stored
 }
 
-type manifestJSON struct {
-	Entries []*manifestTrieEntry `json:"entries"`
+func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
+	return &manifestTrieEntry{
+		ManifestEntry: *entry,
+		subtrie:       subtrie,
+	}
 }
 
 type manifestTrieEntry struct {
-	Path        string `json:"path"`
-	Hash        string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
-	ContentType string `json:"contentType"`
-	Status      int    `json:"status"`
-	subtrie     *manifestTrie
+	ManifestEntry
+
+	subtrie *manifestTrie
 }
 
 func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
@@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
 	}
 
 	log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
-	man := manifestJSON{}
+	var man struct {
+		Entries []*manifestTrieEntry `json:"entries"`
+	}
 	err = json.Unmarshal(manifestData, &man)
 	if err != nil {
 		err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
@@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 		cpl++
 	}
 
-	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
+	if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
 		if self.loadSubTrie(oldentry, quitC) != nil {
 			return
 		}
@@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 	subtrie.addEntry(entry, quitC)
 	subtrie.addEntry(oldentry, quitC)
 
-	self.entries[b] = &manifestTrieEntry{
+	self.entries[b] = newManifestTrieEntry(&ManifestEntry{
 		Path:        commonPrefix,
-		Hash:        "",
-		ContentType: manifestType,
-		subtrie:     subtrie,
-	}
+		ContentType: ManifestType,
+	}, subtrie)
 }
 
 func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
 	}
 
 	epl := len(entry.Path)
-	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
+	if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
 		if self.loadSubTrie(entry, quitC) != nil {
 			return
 		}
@@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error {
 	var buffer bytes.Buffer
 	buffer.WriteString(`{"entries":[`)
 
-	list := &manifestJSON{}
+	list := &Manifest{}
 	for _, entry := range self.entries {
 		if entry != nil {
 			if entry.Hash == "" { // TODO: paralellize
@@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error {
 				}
 				entry.Hash = entry.subtrie.hash.String()
 			}
-			list.Entries = append(list.Entries, entry)
+			list.Entries = append(list.Entries, entry.ManifestEntry)
 		}
 	}
 
@@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
 		entry := self.entries[i]
 		if entry != nil {
 			epl := len(entry.Path)
-			if entry.ContentType == manifestType {
+			if entry.ContentType == ManifestType {
 				l := plen
 				if epl < l {
 					l = epl
@@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 	log.Trace(fmt.Sprintf("path = %v  entry.Path = %v  epl = %v", path, entry.Path, epl))
 	if (len(path) >= epl) && (path[:epl] == entry.Path) {
 		log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
-		if entry.ContentType == manifestType {
+		if entry.ContentType == ManifestType {
 			err := self.loadSubTrie(entry, quitC)
 			if err != nil {
 				return nil, 0
diff --git a/swarm/api/storage.go b/swarm/api/storage.go
index 31b484675..7e94a9653 100644
--- a/swarm/api/storage.go
+++ b/swarm/api/storage.go
@@ -16,6 +16,8 @@
 
 package api
 
+import "path"
+
 type Response struct {
 	MimeType string
 	Status   int
@@ -25,6 +27,8 @@ type Response struct {
 }
 
 // implements a service
+//
+// DEPRECATED: Use the HTTP API instead
 type Storage struct {
 	api *Api
 }
@@ -35,8 +39,14 @@ func NewStorage(api *Api) *Storage {
 
 // Put uploads the content to the swarm with a simple manifest speficying
 // its content type
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Put(content, contentType string) (string, error) {
-	return self.api.Put(content, contentType)
+	key, err := self.api.Put(content, contentType)
+	if err != nil {
+		return "", err
+	}
+	return key.String(), err
 }
 
 // Get retrieves the content from bzzpath and reads the response in full
@@ -45,8 +55,18 @@ func (self *Storage) Put(content, contentType string) (string, error) {
 // NOTE: if error is non-nil, sResponse may still have partial content
 // the actual size of which is given in len(resp.Content), while the expected
 // size is resp.Size
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Get(bzzpath string) (*Response, error) {
-	reader, mimeType, status, err := self.api.Get(bzzpath, true)
+	uri, err := Parse(path.Join("bzz:/", bzzpath))
+	if err != nil {
+		return nil, err
+	}
+	key, err := self.api.Resolve(uri)
+	if err != nil {
+		return nil, err
+	}
+	reader, mimeType, status, err := self.api.Get(key, uri.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -65,6 +85,20 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 
 // Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash,
 // and merge on  to it. creating an entry w conentType (mime)
+//
+// DEPRECATED: Use the HTTP API instead
 func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
-	return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
+	uri, err := Parse("bzz:/" + rootHash)
+	if err != nil {
+		return "", err
+	}
+	key, err := self.api.Resolve(uri)
+	if err != nil {
+		return "", err
+	}
+	key, err = self.api.Modify(key, path, contentHash, contentType)
+	if err != nil {
+		return "", err
+	}
+	return key.String(), nil
 }
diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go
index 72caf52df..d260dd61d 100644
--- a/swarm/api/storage_test.go
+++ b/swarm/api/storage_test.go
@@ -36,7 +36,7 @@ func TestStoragePutGet(t *testing.T) {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		// to check put against the Api#Get
-		resp0 := testGet(t, api.api, bzzhash)
+		resp0 := testGet(t, api.api, bzzhash, "")
 		checkResponse(t, resp0, exp)
 
 		// check storage#Get
diff --git a/swarm/api/swarmfs_unix.go b/swarm/api/swarmfs_unix.go
index e696c6b9a..a704c1ec2 100644
--- a/swarm/api/swarmfs_unix.go
+++ b/swarm/api/swarmfs_unix.go
@@ -91,11 +91,16 @@ func (self *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) {
 		return nil, fmt.Errorf("%s is already mounted", cleanedMountPoint)
 	}
 
-	key, _, path, err := self.swarmApi.parseAndResolve(mhash, true)
+	uri, err := Parse("bzz:/" + mhash)
 	if err != nil {
-		return nil, fmt.Errorf("can't resolve %q: %v", mhash, err)
+		return nil, err
+	}
+	key, err := self.swarmApi.Resolve(uri)
+	if err != nil {
+		return nil, err
 	}
 
+	path := uri.Path
 	if len(path) > 0 {
 		path += "/"
 	}
diff --git a/swarm/api/uri.go b/swarm/api/uri.go
new file mode 100644
index 000000000..68ce04835
--- /dev/null
+++ b/swarm/api/uri.go
@@ -0,0 +1,96 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// URI is a reference to content stored in swarm.
+type URI struct {
+	// Scheme has one of the following values:
+	//
+	// * bzz  - an entry in a swarm manifest
+	// * bzzr - raw swarm content
+	// * bzzi - immutable URI of an entry in a swarm manifest
+	//          (address is not resolved)
+	Scheme string
+
+	// Addr is either a hexadecimal storage key or it an address which
+	// resolves to a storage key
+	Addr string
+
+	// Path is the path to the content within a swarm manifest
+	Path string
+}
+
+// Parse parses rawuri into a URI struct, where rawuri is expected to have one
+// of the following formats:
+//
+// * <scheme>:/
+// * <scheme>:/<addr>
+// * <scheme>:/<addr>/<path>
+// * <scheme>://
+// * <scheme>://<addr>
+// * <scheme>://<addr>/<path>
+//
+// with scheme one of bzz, bzzr or bzzi
+func Parse(rawuri string) (*URI, error) {
+	u, err := url.Parse(rawuri)
+	if err != nil {
+		return nil, err
+	}
+	uri := &URI{Scheme: u.Scheme}
+
+	// check the scheme is valid
+	switch uri.Scheme {
+	case "bzz", "bzzi", "bzzr":
+	default:
+		return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
+	}
+
+	// handle URIs like bzz://<addr>/<path> where the addr and path
+	// have already been split by url.Parse
+	if u.Host != "" {
+		uri.Addr = u.Host
+		uri.Path = strings.TrimLeft(u.Path, "/")
+		return uri, nil
+	}
+
+	// URI is like bzz:/<addr>/<path> so split the addr and path from
+	// the raw path (which will be /<addr>/<path>)
+	parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
+	uri.Addr = parts[0]
+	if len(parts) == 2 {
+		uri.Path = parts[1]
+	}
+	return uri, nil
+}
+
+func (u *URI) Raw() bool {
+	return u.Scheme == "bzzr"
+}
+
+func (u *URI) Immutable() bool {
+	return u.Scheme == "bzzi"
+}
+
+func (u *URI) String() string {
+	return u.Scheme + ":/" + u.Addr + "/" + u.Path
+}
diff --git a/swarm/api/uri_test.go b/swarm/api/uri_test.go
new file mode 100644
index 000000000..dcb5fbbff
--- /dev/null
+++ b/swarm/api/uri_test.go
@@ -0,0 +1,120 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseURI(t *testing.T) {
+	type test struct {
+		uri             string
+		expectURI       *URI
+		expectErr       bool
+		expectRaw       bool
+		expectImmutable bool
+	}
+	tests := []test{
+		{
+			uri:       "",
+			expectErr: true,
+		},
+		{
+			uri:       "foo",
+			expectErr: true,
+		},
+		{
+			uri:       "bzz",
+			expectErr: true,
+		},
+		{
+			uri:       "bzz:",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:             "bzzi:",
+			expectURI:       &URI{Scheme: "bzzi"},
+			expectImmutable: true,
+		},
+		{
+			uri:       "bzzr:",
+			expectURI: &URI{Scheme: "bzzr"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzz:/",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:       "bzz:/abc123",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+		},
+		{
+			uri:       "bzz:/abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+		},
+		{
+			uri:       "bzzr:/",
+			expectURI: &URI{Scheme: "bzzr"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzzr:/abc123",
+			expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzzr:/abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+			expectRaw: true,
+		},
+		{
+			uri:       "bzz://",
+			expectURI: &URI{Scheme: "bzz"},
+		},
+		{
+			uri:       "bzz://abc123",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
+		},
+		{
+			uri:       "bzz://abc123/path/to/entry",
+			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
+		},
+	}
+	for _, x := range tests {
+		actual, err := Parse(x.uri)
+		if x.expectErr {
+			if err == nil {
+				t.Fatalf("expected %s to error", x.uri)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("error parsing %s: %s", x.uri, err)
+		}
+		if !reflect.DeepEqual(actual, x.expectURI) {
+			t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
+		}
+		if actual.Raw() != x.expectRaw {
+			t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
+		}
+		if actual.Immutable() != x.expectImmutable {
+			t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
+		}
+	}
+}
diff --git a/swarm/swarm.go b/swarm/swarm.go
index add28d205..5a7f43f8b 100644
--- a/swarm/swarm.go
+++ b/swarm/swarm.go
@@ -53,8 +53,8 @@ type Swarm struct {
 	privateKey  *ecdsa.PrivateKey
 	corsString  string
 	swapEnabled bool
-	lstore      *storage.LocalStore  // local store, needs to store for releasing resources after node stopped
-	sfs          *api.SwarmFS         // need this to cleanup all the active mounts on node exit
+	lstore      *storage.LocalStore // local store, needs to store for releasing resources after node stopped
+	sfs         *api.SwarmFS        // need this to cleanup all the active mounts on node exit
 }
 
 type SwarmAPI struct {
@@ -244,13 +244,6 @@ func (self *Swarm) APIs() []rpc.API {
 		{
 			Namespace: "bzz",
 			Version:   "0.1",
-			Service:   api.NewStorage(self.api),
-			Public:    true,
-		},
-
-		{
-			Namespace: "bzz",
-			Version:   "0.1",
 			Service:   &Info{self.config, chequebook.ContractParams},
 			Public:    true,
 		},
@@ -258,11 +251,6 @@ func (self *Swarm) APIs() []rpc.API {
 		{
 			Namespace: "bzz",
 			Version:   "0.1",
-			Service:   api.NewFileSystem(self.api),
-			Public:    false},
-		{
-			Namespace: "bzz",
-			Version:   "0.1",
 			Service:   api.NewControl(self.api, self.hive),
 			Public:    false,
 		},
@@ -278,6 +266,20 @@ func (self *Swarm) APIs() []rpc.API {
 			Service:   self.sfs,
 			Public:    false,
 		},
+		// storage APIs
+		// DEPRECATED: Use the HTTP API instead
+		{
+			Namespace: "bzz",
+			Version:   "0.1",
+			Service:   api.NewStorage(self.api),
+			Public:    true,
+		},
+		{
+			Namespace: "bzz",
+			Version:   "0.1",
+			Service:   api.NewFileSystem(self.api),
+			Public:    false,
+		},
 		// {Namespace, Version, api.NewAdmin(self), false},
 	}
 }