swarm: bzz-list, bzz-raw and bzz-immutable schemes (#15667)

* swarm/api: url scheme bzz-list for getting list of files from manifest

Replace query parameter list=true for listing all files contained
in a swarm manifest with a new URL scheme bzz-list.

* swarm: replaace bzzr and bzzi schemes with bzz-raw and bzz-immutable

New URI Shemes are added and old ones are deprecated, but not removed.
Old Schemes bzzr and bzzi are functional for backward compatibility.

* swarm/api: completely remove bzzr and bzzi schemes

Remove old schemes in favour of bzz-raw and
bzz-immutable.

* swarm/api: revert "completely remove bzzr and bzzi schemes"

Keep bzzr and bzzi schemes for backward compatibility. At least
until 0.3 swarm release.

4 files changed, 116 insertions, 23 deletions

diff --git a/swarm/api/http/roundtripper.go b/swarm/api/http/roundtripper.go
index 328177a21..019431771 100644
--- a/swarm/api/http/roundtripper.go
+++ b/swarm/api/http/roundtripper.go
@@ -35,8 +35,8 @@ import (
 client := httpclient.New()
 // for (private) swarm proxy running locally
 client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
 
 The port you give the Roundtripper is the port the swarm proxy is listening on.
 If Host is left empty, localhost is assumed.
diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go
index 65f6afab7..3872cbc4f 100644
--- a/swarm/api/http/server.go
+++ b/swarm/api/http/server.go
@@ -86,7 +86,7 @@ type Request struct {
 	uri *api.URI
 }
 
-// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage key as a text/plain response
 func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	if r.uri.Path != "" {
@@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	fmt.Fprint(w, newKey)
 }
 
-// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
 // the raw content stored at the given storage key
 func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
@@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 	}
 }
 
-// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
-// the "list" query parameter set to "true" and returns a list of all files
-// contained in <manifest> under <path> grouped into common prefixes using
-// "/" as a delimiter
+// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
+// a list of all files contained in <manifest> under <path> grouped into
+// common prefixes using "/" as a delimiter
 func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// ensure the root path has a trailing slash so that relative URLs work
 	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
@@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
 		err := htmlListTemplate.Execute(w, &htmlListData{
-			URI:  r.uri,
+			URI: &api.URI{
+				Scheme: "bzz",
+				Addr:   r.uri.Addr,
+				Path:   r.uri.Path,
+			},
 			List: &list,
 		})
 		if err != nil {
@@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case "POST":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandlePostRaw(w, req)
 		} else {
 			s.HandlePostFiles(w, req)
@@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		//   new manifest leaving the existing one intact, so it isn't
 		//   strictly a traditional PUT request which replaces content
 		//   at a URI, and POST is more ubiquitous)
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
 		} else {
@@ -609,25 +612,25 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 
 	case "DELETE":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
 			return
 		}
 		s.HandleDelete(w, req)
 
 	case "GET":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandleGetRaw(w, req)
 			return
 		}
 
-		if r.Header.Get("Accept") == "application/x-tar" {
-			s.HandleGetFiles(w, req)
+		if uri.List() {
+			s.HandleGetList(w, req)
 			return
 		}
 
-		if r.URL.Query().Get("list") == "true" {
-			s.HandleGetList(w, req)
+		if r.Header.Get("Accept") == "application/x-tar" {
+			s.HandleGetFiles(w, req)
 			return
 		}
 
diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index ffeaf6e0d..dbbfa3b07 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
 		wg.Wait()
 	}
 
-	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
 	if err != nil {
 		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
@@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
 		var resp *http.Response
 		var respbody []byte
 
-		url := srv.URL + "/bzzr:/"
+		url := srv.URL + "/bzz-raw:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}
@@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
 		}
 	}
 
+	for _, c := range []struct {
+		path string
+		json string
+		html string
+	}{
+		{
+			path: "/",
+			json: `{"common_prefixes":["a/"]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"a/\">a/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/",
+			json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"b/\">b/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"a\">a</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/b/",
+			json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"b\">b</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n\t<tr>\n\t  <td><a href=\"c\">c</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/x",
+		},
+		{
+			path: "",
+		},
+	} {
+		k := c.path
+		url := srv.URL + "/bzz-list:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:]
+		}
+		t.Run("json list "+c.path, func(t *testing.T) {
+			resp, err := http.Get(url)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			body := strings.TrimSpace(string(respbody))
+			if body != c.json {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
+				}
+			}
+		})
+		t.Run("html list "+c.path, func(t *testing.T) {
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				t.Fatalf("New request: %v", err)
+			}
+			req.Header.Set("Accept", "text/html")
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			if string(respbody) != c.html {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
+				}
+			}
+		})
+	}
+
 	nonhashtests := []string{
 		srv.URL + "/bzz:/name",
-		srv.URL + "/bzzi:/nonhash",
-		srv.URL + "/bzzr:/nonhash",
+		srv.URL + "/bzz-immutable:/nonhash",
+		srv.URL + "/bzz-raw:/nonhash",
+		srv.URL + "/bzz-list:/nonhash",
 	}
 
 	nonhashresponses := []string{
 		"error resolving name: no DNS to resolve name: &#34;name&#34;",
 		"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
 		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
+		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
 	}
 
 	for i, url := range nonhashtests {
diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go
index 9ae434a7e..53ce7b5a2 100644
--- a/swarm/api/http/templates.go
+++ b/swarm/api/http/templates.go
@@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
     <tbody>
       {{ range .List.CommonPrefixes }}
 	<tr>
-	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
 	  <td>DIR</td>
 	  <td>-</td>
 	</tr>