diff options
| -rw-r--r-- | .github/CODEOWNERS | 2 | ||||
| -rw-r--r-- | .travis.yml | 13 | ||||
| -rw-r--r-- | appveyor.yml | 4 | ||||
| -rw-r--r-- | cmd/evm/staterunner.go | 8 | ||||
| -rw-r--r-- | cmd/puppeth/module_node.go | 9 | ||||
| -rw-r--r-- | cmd/swarm/config.go | 43 | ||||
| -rw-r--r-- | cmd/swarm/db.go | 15 | ||||
| -rw-r--r-- | cmd/swarm/export_test.go | 4 | ||||
| -rw-r--r-- | cmd/swarm/feeds.go (renamed from cmd/swarm/mru.go) | 72 | ||||
| -rw-r--r-- | cmd/swarm/feeds_test.go (renamed from cmd/swarm/mru_test.go) | 44 | ||||
| -rw-r--r-- | cmd/swarm/main.go | 78 | ||||
| -rw-r--r-- | cmd/swarm/manifest_test.go | 17 | ||||
| -rw-r--r-- | cmd/swarm/run_test.go | 3 | ||||
| -rw-r--r-- | cmd/swarm/upload_test.go | 20 | ||||
| -rw-r--r-- | core/chain_indexer.go | 22 | ||||
| -rw-r--r-- | eth/protocol.go | 2 | ||||
| -rw-r--r-- | swarm/OWNERS | 2 | ||||
| -rw-r--r-- | swarm/api/api.go | 124 | ||||
| -rw-r--r-- | swarm/api/client/client.go | 61 | ||||
| -rw-r--r-- | swarm/api/client/client_test.go | 86 | ||||
| -rw-r--r-- | swarm/api/http/server.go | 132 | ||||
| -rw-r--r-- | swarm/api/http/server_test.go | 123 | ||||
| -rw-r--r-- | swarm/api/manifest.go | 32 | ||||
| -rw-r--r-- | swarm/api/uri.go | 6 | ||||
| -rw-r--r-- | swarm/network/README.md | 4 | ||||
| -rw-r--r-- | swarm/pss/notify/notify.go | 2 | ||||
| -rw-r--r-- | swarm/storage/database.go | 9 | ||||
| -rw-r--r-- | swarm/storage/feed/binaryserializer.go (renamed from swarm/storage/mru/binaryserializer.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/binaryserializer_test.go (renamed from swarm/storage/mru/binaryserializer_test.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/cacheentry.go (renamed from swarm/storage/mru/cacheentry.go) | 14 | ||||
| -rw-r--r-- | swarm/storage/feed/doc.go | 43 | ||||
| -rw-r--r-- | swarm/storage/feed/error.go (renamed from swarm/storage/mru/error.go) | 8 | ||||
| -rw-r--r-- | swarm/storage/feed/feed.go (renamed from swarm/storage/mru/view.go) | 66 | ||||
| -rw-r--r-- | swarm/storage/feed/feed_test.go (renamed from swarm/storage/mru/view_test.go) | 14 | ||||
| -rw-r--r-- | swarm/storage/feed/handler.go (renamed from swarm/storage/mru/handler.go) | 175 | ||||
| -rw-r--r-- | swarm/storage/feed/handler_test.go (renamed from swarm/storage/mru/handler_test.go) | 130 | ||||
| -rw-r--r-- | swarm/storage/feed/id.go (renamed from swarm/storage/mru/id.go) | 30 | ||||
| -rw-r--r-- | swarm/storage/feed/id_test.go (renamed from swarm/storage/mru/id_test.go) | 10 | ||||
| -rw-r--r-- | swarm/storage/feed/lookup/epoch.go (renamed from swarm/storage/mru/lookup/epoch.go) | 0 | ||||
| -rw-r--r-- | swarm/storage/feed/lookup/epoch_test.go (renamed from swarm/storage/mru/lookup/epoch_test.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/lookup/lookup.go (renamed from swarm/storage/mru/lookup/lookup.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/lookup/lookup_test.go (renamed from swarm/storage/mru/lookup/lookup_test.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/query.go (renamed from swarm/storage/mru/query.go) | 22 | ||||
| -rw-r--r-- | swarm/storage/feed/query_test.go (renamed from swarm/storage/mru/query_test.go) | 8 | ||||
| -rw-r--r-- | swarm/storage/feed/request.go (renamed from swarm/storage/mru/request.go) | 50 | ||||
| -rw-r--r-- | swarm/storage/feed/request_test.go (renamed from swarm/storage/mru/request_test.go) | 58 | ||||
| -rw-r--r-- | swarm/storage/feed/sign.go (renamed from swarm/storage/mru/resource_sign.go) | 6 | ||||
| -rw-r--r-- | swarm/storage/feed/testutil.go (renamed from swarm/storage/mru/testutil.go) | 14 | ||||
| -rw-r--r-- | swarm/storage/feed/timestampprovider.go (renamed from swarm/storage/mru/timestampprovider.go) | 4 | ||||
| -rw-r--r-- | swarm/storage/feed/topic.go (renamed from swarm/storage/mru/topic.go) | 6 | ||||
| -rw-r--r-- | swarm/storage/feed/topic_test.go (renamed from swarm/storage/mru/topic_test.go) | 2 | ||||
| -rw-r--r-- | swarm/storage/feed/update.go (renamed from swarm/storage/mru/update.go) | 26 | ||||
| -rw-r--r-- | swarm/storage/feed/update_test.go (renamed from swarm/storage/mru/update_test.go) | 20 | ||||
| -rw-r--r-- | swarm/storage/ldbstore.go | 33 | ||||
| -rw-r--r-- | swarm/storage/localstore.go | 39 | ||||
| -rw-r--r-- | swarm/storage/localstore_test.go | 4 | ||||
| -rw-r--r-- | swarm/storage/mru/doc.go | 44 | ||||
| -rw-r--r-- | swarm/storage/schema.go | 6 | ||||
| -rw-r--r-- | swarm/swarm.go | 19 | ||||
| -rw-r--r-- | swarm/testutil/http.go | 18 |
60 files changed, 918 insertions, 898 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a09baef7..5a717da00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,6 +27,6 @@ swarm/services @zelig swarm/state @justelad swarm/storage/encryption @gbalint @zelig @nagydani swarm/storage/mock @janos -swarm/storage/mru @nolash +swarm/storage/feed @nolash @jpeletier swarm/testutil @lmars whisper/ @gballet @gluk256 diff --git a/.travis.yml b/.travis.yml index 372f7a827..d6cd7d51c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,17 +6,6 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.9.x - script: - - sudo modprobe fuse - - sudo chmod 666 /dev/fuse - - sudo chown root:$USER /etc/fuse.conf - - go run build/ci.go install - - go run build/ci.go test -coverage $TEST_PACKAGES - - - os: linux - dist: trusty - sudo: required go: 1.10.x script: - sudo modprobe fuse @@ -155,7 +144,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/appveyor.yml b/appveyor.yml index b056cb3fd..11848ddb9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.windows-%GETH_ARCH%.zip - - 7z x go1.11.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip + - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 6d5ff069f..06c9be380 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} state, err := test.Run(st, cfg) + // print state root for evmlab tracing + if ctx.GlobalBool(MachineFlag.Name) && state != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() @@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error { result.State = &dump } } - // print state root for evmlab tracing (already committed above, so no need to delete objects again - if ctx.GlobalBool(MachineFlag.Name) && state != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) - } results = append(results, *result) diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 038152a3e..069adfe4f 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -99,6 +99,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{ "NetworkID": config.network, "Port": config.port, + "IP": client.address, "Peers": config.peersTotal, "LightFlag": lightFlag, "Bootnodes": strings.Join(bootnodes, ","), @@ -227,10 +228,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Container available, retrieve its node ID and its genesis json var out []byte - if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil { + if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil { return nil, ErrServiceUnreachable } - id := bytes.Trim(bytes.TrimSpace(out), "\"") + enode := bytes.Trim(bytes.TrimSpace(out), "\"") if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil { return nil, ErrServiceUnreachable @@ -265,7 +266,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) gasLimit: gasLimit, gasPrice: gasPrice, } - stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port) + stats.enode = string(enode) return stats, nil } diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 9d6fe41a7..16001010d 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -176,8 +176,12 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con } if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" { - if id, _ := strconv.Atoi(networkid); id != 0 { - currentConfig.NetworkID = uint64(id) + id, err := strconv.ParseUint(networkid, 10, 64) + if err != nil { + utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err) + } + if id != 0 { + currentConfig.NetworkID = id } } @@ -270,8 +274,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" { - if id, _ := strconv.Atoi(networkid); id != 0 { - currentConfig.NetworkID = uint64(id) + id, err := strconv.ParseUint(networkid, 10, 64) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err) + } + if id != 0 { + currentConfig.NetworkID = id } } @@ -289,27 +297,34 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" { - if swap, err := strconv.ParseBool(swapenable); err != nil { - currentConfig.SwapEnabled = swap + swap, err := strconv.ParseBool(swapenable) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err) } + currentConfig.SwapEnabled = swap } if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" { - if sync, err := strconv.ParseBool(syncdisable); err != nil { - currentConfig.SyncEnabled = !sync + sync, err := strconv.ParseBool(syncdisable) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err) } + currentConfig.SyncEnabled = !sync } if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" { - if skipCheck, err := strconv.ParseBool(v); err != nil { + skipCheck, err := strconv.ParseBool(v) + if err != nil { currentConfig.DeliverySkipCheck = skipCheck } } if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" { - if d, err := time.ParseDuration(v); err != nil { - currentConfig.SyncUpdateDelay = d + d, err := time.ParseDuration(v) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err) } + currentConfig.SyncUpdateDelay = d } if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" { @@ -321,9 +336,11 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" { - if lightnode, err := strconv.ParseBool(lne); err != nil { - currentConfig.LightNodeEnabled = lightnode + lightnode, err := strconv.ParseBool(lne) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err) } + currentConfig.LightNodeEnabled = lightnode } if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index fe03f2d16..107fbf100 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -93,21 +93,6 @@ func dbImport(ctx *cli.Context) { log.Info(fmt.Sprintf("successfully imported %d chunks", count)) } -func dbClean(ctx *cli.Context) { - args := ctx.Args() - if len(args) != 2 { - utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key") - } - - store, err := openLDBStore(args[0], common.Hex2Bytes(args[1])) - if err != nil { - utils.Fatalf("error opening local chunk database: %s", err) - } - defer store.Close() - - store.Cleanup() -} - func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return nil, fmt.Errorf("invalid chunkdb path: %s", err) diff --git a/cmd/swarm/export_test.go b/cmd/swarm/export_test.go index 525538ad7..c533511af 100644 --- a/cmd/swarm/export_test.go +++ b/cmd/swarm/export_test.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "net/http" "os" + "runtime" "strings" "testing" @@ -38,6 +39,9 @@ import ( // 5. imports the exported datastore // 6. fetches the uploaded random file from the second node func TestCLISwarmExportImport(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } cluster := newTestCluster(t, 1) // generate random 10mb file diff --git a/cmd/swarm/mru.go b/cmd/swarm/feeds.go index cc7f634cb..6806c6cf4 100644 --- a/cmd/swarm/mru.go +++ b/cmd/swarm/feeds.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. -// Command resource allows the user to create and update signed mutable resource updates +// Command feed allows the user to create and update signed Swarm feeds package main import ( @@ -27,17 +27,17 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" swarm "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "gopkg.in/urfave/cli.v1" ) -func NewGenericSigner(ctx *cli.Context) mru.Signer { - return mru.NewGenericSigner(getPrivKey(ctx)) +func NewGenericSigner(ctx *cli.Context) feed.Signer { + return feed.NewGenericSigner(getPrivKey(ctx)) } -func getTopic(ctx *cli.Context) (topic mru.Topic) { - var name = ctx.String(SwarmResourceNameFlag.Name) - var relatedTopic = ctx.String(SwarmResourceTopicFlag.Name) +func getTopic(ctx *cli.Context) (topic feed.Topic) { + var name = ctx.String(SwarmFeedNameFlag.Name) + var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name) var relatedTopicBytes []byte var err error @@ -48,42 +48,42 @@ func getTopic(ctx *cli.Context) (topic mru.Topic) { } } - topic, err = mru.NewTopic(name, relatedTopicBytes) + topic, err = feed.NewTopic(name, relatedTopicBytes) if err != nil { utils.Fatalf("Error parsing topic: %s", err) } return topic } -// swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]] -// swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false] -// swarm resource info <Manifest Address or ENS domain> +// swarm feed create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]] +// swarm feed update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false] +// swarm feed info <Manifest Address or ENS domain> -func resourceCreate(ctx *cli.Context) { +func feedCreateManifest(ctx *cli.Context) { var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") client = swarm.NewClient(bzzapi) ) - newResourceRequest := mru.NewFirstRequest(getTopic(ctx)) - newResourceRequest.View.User = resourceGetUser(ctx) + newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx)) + newFeedUpdateRequest.Feed.User = feedGetUser(ctx) - manifestAddress, err := client.CreateResource(newResourceRequest) + manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest) if err != nil { - utils.Fatalf("Error creating resource: %s", err.Error()) + utils.Fatalf("Error creating feed manifest: %s", err.Error()) return } fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up) } -func resourceUpdate(ctx *cli.Context) { +func feedUpdate(ctx *cli.Context) { args := ctx.Args() var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") client = swarm.NewClient(bzzapi) - manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name) + manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name) ) if len(args) < 1 { @@ -100,20 +100,20 @@ func resourceUpdate(ctx *cli.Context) { return } - var updateRequest *mru.Request - var query *mru.Query + var updateRequest *feed.Request + var query *feed.Query if manifestAddressOrDomain == "" { - query = new(mru.Query) + query = new(feed.Query) query.User = signer.Address() query.Topic = getTopic(ctx) } - // Retrieve resource status and metadata out of the manifest - updateRequest, err = client.GetResourceMetadata(query, manifestAddressOrDomain) + // Retrieve a feed update request + updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain) if err != nil { - utils.Fatalf("Error retrieving resource status: %s", err.Error()) + utils.Fatalf("Error retrieving feed status: %s", err.Error()) } // set the new data @@ -121,34 +121,34 @@ func resourceUpdate(ctx *cli.Context) { // sign update if err = updateRequest.Sign(signer); err != nil { - utils.Fatalf("Error signing resource update: %s", err.Error()) + utils.Fatalf("Error signing feed update: %s", err.Error()) } // post update - err = client.UpdateResource(updateRequest) + err = client.UpdateFeed(updateRequest) if err != nil { - utils.Fatalf("Error updating resource: %s", err.Error()) + utils.Fatalf("Error updating feed: %s", err.Error()) return } } -func resourceInfo(ctx *cli.Context) { +func feedInfo(ctx *cli.Context) { var ( bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") client = swarm.NewClient(bzzapi) - manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name) + manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name) ) - var query *mru.Query + var query *feed.Query if manifestAddressOrDomain == "" { - query = new(mru.Query) + query = new(feed.Query) query.Topic = getTopic(ctx) - query.User = resourceGetUser(ctx) + query.User = feedGetUser(ctx) } - metadata, err := client.GetResourceMetadata(query, manifestAddressOrDomain) + metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain) if err != nil { - utils.Fatalf("Error retrieving resource metadata: %s", err.Error()) + utils.Fatalf("Error retrieving feed metadata: %s", err.Error()) return } encodedMetadata, err := metadata.MarshalJSON() @@ -158,8 +158,8 @@ func resourceInfo(ctx *cli.Context) { fmt.Println(string(encodedMetadata)) } -func resourceGetUser(ctx *cli.Context) common.Address { - var user = ctx.String(SwarmResourceUserFlag.Name) +func feedGetUser(ctx *cli.Context) common.Address { + var user = ctx.String(SwarmFeedUserFlag.Name) if user != "" { return common.HexToAddress(user) } diff --git a/cmd/swarm/mru_test.go b/cmd/swarm/feeds_test.go index 142cf9cfd..46727c21d 100644 --- a/cmd/swarm/mru_test.go +++ b/cmd/swarm/feeds_test.go @@ -26,11 +26,11 @@ import ( "testing" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/swarm/testutil" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" @@ -38,12 +38,12 @@ import ( swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" ) -func TestCLIResourceUpdate(t *testing.T) { +func TestCLIFeedUpdate(t *testing.T) { srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer { return swarmhttp.NewServer(api, "") }, nil) - log.Info("starting 1 node cluster") + log.Info("starting a test swarm server") defer srv.Close() // create a private key file for signing @@ -65,7 +65,7 @@ func TestCLIResourceUpdate(t *testing.T) { } // compose a topic. We'll be doing quotes about Miguel de Cervantes - var topic mru.Topic + var topic feed.Topic subject := []byte("Miguel de Cervantes") copy(topic[:], subject[:]) name := "quotes" @@ -77,13 +77,13 @@ func TestCLIResourceUpdate(t *testing.T) { flags := []string{ "--bzzapi", srv.URL, "--bzzaccount", pkfile.Name(), - "resource", "update", + "feed", "update", "--topic", topic.Hex(), "--name", name, hexData} // create an update and expect an exit without errors - log.Info(fmt.Sprintf("updating a resource with 'swarm resource update'")) + log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'")) cmd := runSwarm(t, flags...) cmd.ExpectExit() @@ -95,22 +95,22 @@ func TestCLIResourceUpdate(t *testing.T) { // build the same topic as before, this time // we use NewTopic to create a topic automatically. - topic, err = mru.NewTopic(name, subject) + topic, err = feed.NewTopic(name, subject) if err != nil { t.Fatal(err) } - // View configures whose updates we will be looking up. - view := mru.View{ + // Feed configures whose updates we will be looking up. + fd := feed.Feed{ Topic: topic, User: address, } // Build a query to get the latest update - query := mru.NewQueryLatest(&view, lookup.NoClue) + query := feed.NewQueryLatest(&fd, lookup.NoClue) // retrieve content! - reader, err := client.GetResource(query, "") + reader, err := client.QueryFeed(query, "") if err != nil { t.Fatal(err) } @@ -128,45 +128,45 @@ func TestCLIResourceUpdate(t *testing.T) { // Now retrieve info for the next update flags = []string{ "--bzzapi", srv.URL, - "resource", "info", + "feed", "info", "--topic", topic.Hex(), "--user", address.Hex(), } - log.Info(fmt.Sprintf("getting resource info with 'swarm resource info'")) + log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'")) cmd = runSwarm(t, flags...) _, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout cmd.ExpectExit() // verify we can deserialize the result as a valid JSON - var request mru.Request + var request feed.Request err = json.Unmarshal([]byte(matches[0]), &request) if err != nil { t.Fatal(err) } - // make sure the retrieved view is the same - if request.View != view { - t.Fatalf("Expected view to be: %s, got %s", view, request.View) + // make sure the retrieved feed is the same + if request.Feed != fd { + t.Fatalf("Expected feed to be: %s, got %s", fd, request.Feed) } // test publishing a manifest flags = []string{ "--bzzapi", srv.URL, "--bzzaccount", pkfile.Name(), - "resource", "create", + "feed", "create", "--topic", topic.Hex(), } - log.Info(fmt.Sprintf("Publishing manifest with 'swarm resource create'")) + log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'")) cmd = runSwarm(t, flags...) _, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout cmd.ExpectExit() - manifestAddress := matches[0] // read the received resource manifest + manifestAddress := matches[0] // read the received feed manifest // now attempt to lookup the latest update using a manifest instead - reader, err = client.GetResource(nil, manifestAddress) + reader, err = client.QueryFeed(nil, manifestAddress) if err != nil { t.Fatal(err) } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index f1bec770e..88e6b0b0b 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -207,25 +207,25 @@ var ( Name: "compressed", Usage: "Prints encryption keys in compressed form", } - SwarmResourceNameFlag = cli.StringFlag{ + SwarmFeedNameFlag = cli.StringFlag{ Name: "name", - Usage: "User-defined name for the new resource, limited to 32 characters. If combined with topic, the resource will be a subtopic with this name", + Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", } - SwarmResourceTopicFlag = cli.StringFlag{ + SwarmFeedTopicFlag = cli.StringFlag{ Name: "topic", - Usage: "User-defined topic this resource is tracking, hex encoded. Limited to 64 hexadecimal characters", + Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters", } - SwarmResourceDataOnCreateFlag = cli.StringFlag{ + SwarmFeedDataOnCreateFlag = cli.StringFlag{ Name: "data", - Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x", + Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x", } - SwarmResourceManifestFlag = cli.StringFlag{ + SwarmFeedManifestFlag = cli.StringFlag{ Name: "manifest", - Usage: "Refers to the resource through a manifest", + Usage: "Refers to the feed through a manifest", } - SwarmResourceUserFlag = cli.StringFlag{ + SwarmFeedUserFlag = cli.StringFlag{ Name: "user", - Usage: "Indicates the user who updates the resource", + Usage: "Indicates the user who updates the feed", } ) @@ -256,12 +256,12 @@ func init() { utils.ListenPortFlag.Value = 30399 } -var app = utils.NewApp(gitCommit, "Ethereum Swarm") +var app = utils.NewApp("", "Ethereum Swarm") // This init function creates the cli.App. func init() { app.Action = bzzd - app.HideVersion = true // we have a command to print the version + app.Version = sv.ArchiveVersion(gitCommit) app.Copyright = "Copyright 2013-2016 The go-ethereum Authors" app.Commands = []cli.Command{ { @@ -346,62 +346,62 @@ func init() { }, { CustomHelpTemplate: helpTemplate, - Name: "resource", - Usage: "(Advanced) Create and update Mutable Resources", + Name: "feed", + Usage: "(Advanced) Create and update Swarm Feeds", ArgsUsage: "<create|update|info>", - Description: "Works with Mutable Resource Updates", + Description: "Works with Swarm Feeds", Subcommands: []cli.Command{ { - Action: resourceCreate, + Action: feedCreateManifest, CustomHelpTemplate: helpTemplate, Name: "create", - Usage: "creates and publishes a new Mutable Resource manifest", - Description: `creates and publishes a new Mutable Resource manifest pointing to a specified user's updates about a particular topic. - The resource topic can be built in the following ways: + Usage: "creates and publishes a new feed manifest", + Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic. + The feed topic can be built in the following ways: * use --topic to set the topic to an arbitrary binary hex string. * use --name to set the topic to a human-readable name. - For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. * use both --topic and --name to create named subtopics. For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning - the Mutable Resource tracks a discussion about that contract. + this feed tracks a discussion about that contract. The --user flag allows to have this manifest refer to a user other than yourself. If not specified, it will then default to your local account (--bzzaccount)`, - Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag}, + Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, }, { - Action: resourceUpdate, + Action: feedUpdate, CustomHelpTemplate: helpTemplate, Name: "update", - Usage: "updates the content of an existing Mutable Resource", + Usage: "updates the content of an existing Swarm Feed", ArgsUsage: "<0x Hex data>", Description: `publishes a new update on the specified topic - The resource topic can be built in the following ways: + The feed topic can be built in the following ways: * use --topic to set the topic to an arbitrary binary hex string. * use --name to set the topic to a human-readable name. - For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture. + For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture. * use both --topic and --name to create named subtopics. For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning - the Mutable Resource tracks a discussion about that contract. + this feed tracks a discussion about that contract. - If you have a manifest, you can specify it with --manifest to refer to the resource, + If you have a manifest, you can specify it with --manifest to refer to the feed, instead of using --topic / --name `, - Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag}, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag}, }, { - Action: resourceInfo, + Action: feedInfo, CustomHelpTemplate: helpTemplate, Name: "info", - Usage: "obtains information about an existing Mutable Resource", - Description: `obtains information about an existing Mutable Resource + Usage: "obtains information about an existing Swarm feed", + Description: `obtains information about an existing Swarm feed The topic can be specified directly with the --topic flag as an hex string If no topic is specified, the default topic (zero) will be used The --name flag can be used to specify subtopics with a specific name. The --user flag allows to refer to a user other than yourself. If not specified, it will then default to your local account (--bzzaccount) If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user - to refer to the resource`, - Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag}, + to refer to the feed`, + Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag}, }, }, }, @@ -537,14 +537,6 @@ pv(1) tool to get a progress bar: pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, }, - { - Action: dbClean, - CustomHelpTemplate: helpTemplate, - Name: "clean", - Usage: "remove corrupt entries from a local chunk database", - ArgsUsage: "<chunkdb>", - Description: "Remove corrupt entries from a local chunk database", - }, }, }, @@ -738,7 +730,7 @@ func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.Pr } // getPrivKey returns the private key of the specified bzzaccount -// Used only by client commands, such as `resource` +// Used only by client commands, such as `feed` func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey { // booting up the swarm node just as we do in bzzd action bzzconfig, err := buildConfig(ctx) diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go index 08fe0b2eb..9c7808dfa 100644 --- a/cmd/swarm/manifest_test.go +++ b/cmd/swarm/manifest_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "testing" "github.com/ethereum/go-ethereum/swarm/api" @@ -30,12 +31,20 @@ import ( // TestManifestChange tests manifest add, update and remove // cli commands without encryption. func TestManifestChange(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testManifestChange(t, false) } // TestManifestChange tests manifest add, update and remove // cli commands with encryption enabled. func TestManifestChangeEncrypted(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testManifestChange(t, true) } @@ -400,6 +409,10 @@ func testManifestChange(t *testing.T, encrypt bool) { // TestNestedDefaultEntryUpdate tests if the default entry is updated // if the file in nested manifest used for it is also updated. func TestNestedDefaultEntryUpdate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testNestedDefaultEntryUpdate(t, false) } @@ -407,6 +420,10 @@ func TestNestedDefaultEntryUpdate(t *testing.T) { // of encrypted upload is updated if the file in nested manifest // used for it is also updated. func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testNestedDefaultEntryUpdate(t, true) } diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 4be0495f8..6d2a86f4b 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -19,6 +19,7 @@ package main import ( "context" "crypto/ecdsa" + "flag" "fmt" "io/ioutil" "net" @@ -41,6 +42,8 @@ import ( "github.com/ethereum/go-ethereum/swarm" ) +var loglevel = flag.Int("loglevel", 3, "verbosity of logs") + func init() { // Run the app if we've been exec'd as "swarm-test" in runSwarm. reexec.Register("swarm-test", func() { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index aba4d0dda..e8161680f 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -18,7 +18,6 @@ package main import ( "bytes" - "flag" "fmt" "io" "io/ioutil" @@ -26,6 +25,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "testing" "time" @@ -35,8 +35,6 @@ import ( "github.com/mattn/go-colorable" ) -var loglevel = flag.Int("loglevel", 3, "verbosity of logs") - func init() { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) @@ -45,18 +43,31 @@ func init() { // TestCLISwarmUp tests that running 'swarm up' makes the resulting file // available from all nodes via the HTTP API func TestCLISwarmUp(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + testCLISwarmUp(false, t) } func TestCLISwarmUpRecursive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } testCLISwarmUpRecursive(false, t) } // TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file // available from all nodes via the HTTP API func TestCLISwarmUpEncrypted(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } testCLISwarmUp(true, t) } func TestCLISwarmUpEncryptedRecursive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } testCLISwarmUpRecursive(true, t) } @@ -277,6 +288,9 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { // TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute // default paths and with encryption. func TestCLISwarmUpDefaultPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } testCLISwarmUpDefaultPath(false, false, t) testCLISwarmUpDefaultPath(false, true, t) testCLISwarmUpDefaultPath(true, false, t) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 4bdd4ba1c..1adde1fcb 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -53,14 +53,14 @@ type ChainIndexerChain interface { // CurrentHeader retrieves the latest locally known header. CurrentHeader() *types.Header - // SubscribeChainEvent subscribes to new head header notifications. - SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription + // SubscribeChainHeadEvent subscribes to new head header notifications. + SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription } // ChainIndexer does a post-processing job for equally sized sections of the // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is // connected to the blockchain through the event system by starting a -// ChainEventLoop in a goroutine. +// ChainHeadEventLoop in a goroutine. // // Further child ChainIndexers can be added which use the output of the parent // section indexer. These child indexers receive new head notifications only @@ -142,8 +142,8 @@ func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { // cascading background processing. Children do not need to be started, they // are notified about new events by their parents. func (c *ChainIndexer) Start(chain ChainIndexerChain) { - events := make(chan ChainEvent, 10) - sub := chain.SubscribeChainEvent(events) + events := make(chan ChainHeadEvent, 10) + sub := chain.SubscribeChainHeadEvent(events) go c.eventLoop(chain.CurrentHeader(), events, sub) } @@ -190,7 +190,7 @@ func (c *ChainIndexer) Close() error { // eventLoop is a secondary - optional - event loop of the indexer which is only // started for the outermost indexer to push chain head events into a processing // queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) { +func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) { // Mark the chain indexer as active, requiring an additional teardown atomic.StoreUint32(&c.active, 1) @@ -219,13 +219,13 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE } header := ev.Block.Header() if header.ParentHash != prevHash { - // Reorg to the common ancestor (might not exist in light sync mode, skip reorg then) + // Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then) // TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly? - // TODO(karalabe): This operation is expensive and might block, causing the event system to - // potentially also lock up. We need to do with on a different thread somehow. - if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { - c.newHead(h.Number.Uint64(), true) + if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash { + if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { + c.newHead(h.Number.Uint64(), true) + } } } c.newHead(header.Number.Uint64(), false) diff --git a/eth/protocol.go b/eth/protocol.go index 0e90e6a2e..497ba4c59 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -37,7 +37,7 @@ const ( // ProtocolName is the official short name of the protocol used during capability negotiation. var ProtocolName = "eth" -// ProtocolVersions are the upported versions of the eth protocol (first is primary). +// ProtocolVersions are the supported versions of the eth protocol (first is primary). var ProtocolVersions = []uint{eth63, eth62} // ProtocolLengths are the number of implemented message corresponding to different protocol versions. diff --git a/swarm/OWNERS b/swarm/OWNERS index 774cd7db9..d4204e08c 100644 --- a/swarm/OWNERS +++ b/swarm/OWNERS @@ -22,5 +22,5 @@ swarm ├── storage ─────────────── ethersphere │ ├── encryption ──────── @gbalint, @zelig, @nagydani │ ├── mock ────────────── @janos -│ └── mru ─────────────── @nolash +│ └── feed ────────────── @nolash, @jpeletier └── testutil ────────────── @lmars
\ No newline at end of file diff --git a/swarm/api/api.go b/swarm/api/api.go index 7b8f04c13..7bb631967 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -45,9 +45,10 @@ import ( "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" - "github.com/opentracing/opentracing-go" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" + + opentracing "github.com/opentracing/opentracing-go" ) var ( @@ -235,18 +236,18 @@ on top of the FileStore it is the public interface of the FileStore which is included in the ethereum stack */ type API struct { - resource *mru.Handler + feed *feed.Handler fileStore *storage.FileStore dns Resolver Decryptor func(context.Context, string) DecryptFunc } // NewAPI the api constructor initialises a new API instance. -func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) { +func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handler, pk *ecdsa.PrivateKey) (self *API) { self = &API{ fileStore: fileStore, dns: dns, - resource: resourceHandler, + feed: feedHandler, Decryptor: func(ctx context.Context, credentials string) DecryptFunc { return self.doDecrypt(ctx, credentials, pk) }, @@ -403,24 +404,24 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage return a.Get(ctx, decrypt, adr, entry.Path) } - // we need to do some extra work if this is a mutable resource manifest - if entry.ContentType == ResourceContentType { - if entry.ResourceView == nil { - return reader, mimeType, status, nil, fmt.Errorf("Cannot decode ResourceView in manifest") + // we need to do some extra work if this is a Swarm feed manifest + if entry.ContentType == FeedContentType { + if entry.Feed == nil { + return reader, mimeType, status, nil, fmt.Errorf("Cannot decode Feed in manifest") } - _, err := a.resource.Lookup(ctx, mru.NewQueryLatest(entry.ResourceView, lookup.NoClue)) + _, err := a.feed.Lookup(ctx, feed.NewQueryLatest(entry.Feed, lookup.NoClue)) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Debug(fmt.Sprintf("get resource content error: %v", err)) + log.Debug(fmt.Sprintf("get feed update content error: %v", err)) return reader, mimeType, status, nil, err } // get the data of the update - _, rsrcData, err := a.resource.GetContent(entry.ResourceView) + _, rsrcData, err := a.feed.GetContent(entry.Feed) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Warn(fmt.Sprintf("get resource content error: %v", err)) + log.Warn(fmt.Sprintf("get feed update content error: %v", err)) return reader, mimeType, status, nil, err } @@ -429,18 +430,18 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage if err != nil { apiGetInvalid.Inc(1) status = http.StatusUnprocessableEntity - log.Warn("invalid resource multihash", "err", err) + log.Warn("invalid multihash in feed update", "err", err) return reader, mimeType, status, nil, err } manifestAddr = storage.Address(decodedMultihash) - log.Trace("resource is multihash", "key", manifestAddr) + log.Trace("feed update contains multihash", "key", manifestAddr) // get the manifest the multihash digest points to trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound - log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err)) + log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err)) return reader, mimeType, status, nil, err } @@ -450,13 +451,13 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage if entry == nil { status = http.StatusNotFound apiGetNotFound.Inc(1) - err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path) - log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path) + err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path) + log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path) return reader, mimeType, status, nil, err } } - // regardless of resource update manifests or normal manifests we will converge at this point + // regardless of feed update manifests or normal manifests we will converge at this point // get the key the manifest entry points to and serve it if it's unambiguous contentAddr = common.Hex2Bytes(entry.Hash) status = entry.Status @@ -956,68 +957,67 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver return addr, manifestEntryMap, nil } -// ResourceLookup finds mutable resource updates at specific periods and versions -func (a *API) ResourceLookup(ctx context.Context, query *mru.Query) ([]byte, error) { - _, err := a.resource.Lookup(ctx, query) +// FeedsLookup finds Swarm feeds updates at specific points in time, or the latest update +func (a *API) FeedsLookup(ctx context.Context, query *feed.Query) ([]byte, error) { + _, err := a.feed.Lookup(ctx, query) if err != nil { return nil, err } var data []byte - _, data, err = a.resource.GetContent(&query.View) + _, data, err = a.feed.GetContent(&query.Feed) if err != nil { return nil, err } return data, nil } -// ResourceNewRequest creates a Request object to update a specific mutable resource -func (a *API) ResourceNewRequest(ctx context.Context, view *mru.View) (*mru.Request, error) { - return a.resource.NewRequest(ctx, view) +// FeedsNewRequest creates a Request object to update a specific feed +func (a *API) FeedsNewRequest(ctx context.Context, feed *feed.Feed) (*feed.Request, error) { + return a.feed.NewRequest(ctx, feed) } -// ResourceUpdate updates a Mutable Resource with arbitrary data. -// Upon retrieval the update will be retrieved verbatim as bytes. -func (a *API) ResourceUpdate(ctx context.Context, request *mru.Request) (storage.Address, error) { - return a.resource.Update(ctx, request) +// FeedsUpdate publishes a new update on the given feed +func (a *API) FeedsUpdate(ctx context.Context, request *feed.Request) (storage.Address, error) { + return a.feed.Update(ctx, request) } -// ResourceHashSize returned the size of the digest produced by the Mutable Resource hashing function -func (a *API) ResourceHashSize() int { - return a.resource.HashSize +// FeedsHashSize returned the size of the digest produced by Swarm feeds' hashing function +func (a *API) FeedsHashSize() int { + return a.feed.HashSize } -// ErrCannotLoadResourceManifest is returned when looking up a resource manifest fails -var ErrCannotLoadResourceManifest = errors.New("Cannot load resource manifest") +// ErrCannotLoadFeedManifest is returned when looking up a feeds manifest fails +var ErrCannotLoadFeedManifest = errors.New("Cannot load feed manifest") -// ErrNotAResourceManifest is returned when the address provided returned something other than a valid manifest -var ErrNotAResourceManifest = errors.New("Not a resource manifest") +// ErrNotAFeedManifest is returned when the address provided returned something other than a valid manifest +var ErrNotAFeedManifest = errors.New("Not a feed manifest") -// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the Resource's view ID. -func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (*mru.View, error) { +// ResolveFeedManifest retrieves the Swarm feed manifest for the given address, and returns the referenced Feed. +func (a *API) ResolveFeedManifest(ctx context.Context, addr storage.Address) (*feed.Feed, error) { trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt) if err != nil { - return nil, ErrCannotLoadResourceManifest + return nil, ErrCannotLoadFeedManifest } entry, _ := trie.getEntry("") - if entry.ContentType != ResourceContentType { - return nil, ErrNotAResourceManifest + if entry.ContentType != FeedContentType { + return nil, ErrNotAFeedManifest } - return entry.ResourceView, nil + return entry.Feed, nil } -// ErrCannotResolveResourceURI is returned when the ENS resolver is not able to translate a name to a resource -var ErrCannotResolveResourceURI = errors.New("Cannot resolve Resource URI") +// ErrCannotResolveFeedURI is returned when the ENS resolver is not able to translate a name to a Swarm feed +var ErrCannotResolveFeedURI = errors.New("Cannot resolve Feed URI") -// ErrCannotResolveResourceView is returned when values provided are not enough or invalid to recreate a -// resource view out of them. -var ErrCannotResolveResourceView = errors.New("Cannot resolve resource view") +// ErrCannotResolveFeed is returned when values provided are not enough or invalid to recreate a +// feed out of them. +var ErrCannotResolveFeed = errors.New("Cannot resolve Feed") -// ResolveResourceView attempts to extract View information out of the manifest, if provided -// If not, it attempts to extract the View out of a set of key-value pairs -func (a *API) ResolveResourceView(ctx context.Context, uri *URI, values mru.Values) (*mru.View, error) { - var view *mru.View +// ResolveFeed attempts to extract feed information out of the manifest, if provided +// If not, it attempts to extract the feed out of a set of key-value pairs +func (a *API) ResolveFeed(ctx context.Context, uri *URI, values feed.Values) (*feed.Feed, error) { + var fd *feed.Feed var err error if uri.Addr != "" { // resolve the content key. @@ -1025,25 +1025,25 @@ func (a *API) ResolveResourceView(ctx context.Context, uri *URI, values mru.Valu if manifestAddr == nil { manifestAddr, err = a.Resolve(ctx, uri.Addr) if err != nil { - return nil, ErrCannotResolveResourceURI + return nil, ErrCannotResolveFeedURI } } - // get the resource view from the manifest - view, err = a.ResolveResourceManifest(ctx, manifestAddr) + // get the Swarm feed from the manifest + fd, err = a.ResolveFeedManifest(ctx, manifestAddr) if err != nil { return nil, err } - log.Debug("handle.get.resource: resolved", "manifestkey", manifestAddr, "view", view.Hex()) + log.Debug("handle.get.feed: resolved", "manifestkey", manifestAddr, "feed", fd.Hex()) } else { - var v mru.View - if err := v.FromValues(values); err != nil { - return nil, ErrCannotResolveResourceView + var f feed.Feed + if err := f.FromValues(values); err != nil { + return nil, ErrCannotResolveFeed } - view = &v + fd = &f } - return view, nil + return fd, nil } // MimeOctetStream default value of http Content-Type header diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 47a6980de..d9837ca73 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -35,7 +35,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" ) var ( @@ -601,16 +601,15 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) return string(data), nil } -// ErrNoResourceUpdatesFound is returned when Swarm cannot find updates of the given resource -var ErrNoResourceUpdatesFound = errors.New("No updates found for this resource") +// ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed +var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed") -// CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided -// data. Data is interpreted as multihash or not depending on the multihash parameter. -// startTime=0 means "now" -// Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent) -// or reference future updates (Client.UpdateResource) -func (c *Client) CreateResource(request *mru.Request) (string, error) { - responseStream, err := c.updateResource(request, true) +// CreateFeedWithManifest creates a feed manifest, initializing it with the provided +// data +// Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent) +// or reference future updates (Client.UpdateFeed) +func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) { + responseStream, err := c.updateFeed(request, true) if err != nil { return "", err } @@ -628,18 +627,18 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) { return manifestAddress, nil } -// UpdateResource allows you to set a new version of your content -func (c *Client) UpdateResource(request *mru.Request) error { - _, err := c.updateResource(request, false) +// UpdateFeed allows you to set a new version of your content +func (c *Client) UpdateFeed(request *feed.Request) error { + _, err := c.updateFeed(request, false) return err } -func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.ReadCloser, error) { +func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) { URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } - URL.Path = "/bzz-resource:/" + URL.Path = "/bzz-feed:/" values := URL.Query() body := request.AppendValues(values) if createManifest { @@ -660,23 +659,23 @@ func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.R return res.Body, nil } -// GetResource returns a byte stream with the raw content of the resource -// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// QueryFeed returns a byte stream with the raw content of the feed update +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address -func (c *Client) GetResource(query *mru.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { - return c.getResource(query, manifestAddressOrDomain, false) +func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { + return c.queryFeed(query, manifestAddressOrDomain, false) } -// getResource returns a byte stream with the raw content of the resource -// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// queryFeed returns a byte stream with the raw content of the feed update +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address -// meta set to true will instruct the node return resource metainformation instead -func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { +// meta set to true will instruct the node return feed metainformation instead +func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } - URL.Path = "/bzz-resource:/" + manifestAddressOrDomain + URL.Path = "/bzz-feed:/" + manifestAddressOrDomain values := URL.Query() if query != nil { query.AppendValues(values) //adds query parameters @@ -692,7 +691,7 @@ func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, m if res.StatusCode != http.StatusOK { if res.StatusCode == http.StatusNotFound { - return nil, ErrNoResourceUpdatesFound + return nil, ErrNoFeedUpdatesFound } errorMessageBytes, err := ioutil.ReadAll(res.Body) var errorMessage string @@ -701,18 +700,18 @@ func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, m } else { errorMessage = string(errorMessageBytes) } - return nil, fmt.Errorf("Error retrieving resource: %s", errorMessage) + return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage) } return res.Body, nil } -// GetResourceMetadata returns a structure that describes the Mutable Resource -// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver +// GetFeedRequest returns a structure that describes the referenced feed status +// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address -func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain string) (*mru.Request, error) { +func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) { - responseStream, err := c.getResource(query, manifestAddressOrDomain, true) + responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true) if err != nil { return nil, err } @@ -723,7 +722,7 @@ func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain s return nil, err } - var metadata mru.Request + var metadata feed.Request if err := metadata.UnmarshalJSON(body); err != nil { return nil, err } diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 02980de1d..03c6cbb28 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -25,14 +25,14 @@ import ( "sort" "testing" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/multihash" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -361,20 +361,20 @@ func TestClientMultipartUpload(t *testing.T) { } } -func newTestSigner() (*mru.GenericSigner, error) { +func newTestSigner() (*feed.GenericSigner, error) { privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") if err != nil { return nil, err } - return mru.NewGenericSigner(privKey), nil + return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash resource types with bzz:// scheme +// test the transparent resolving of multihash feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a resource update +// first upload data, and store the multihash to the resulting manifest in a feed update // retrieving the update with the multihash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestClientCreateResourceMultihash(t *testing.T) { +func TestClientCreateFeedMultihash(t *testing.T) { signer, _ := newTestSigner() @@ -393,36 +393,36 @@ func TestClientCreateResourceMultihash(t *testing.T) { s := common.FromHex(swarmHash) mh := multihash.ToMultihash(s) - // our mutable resource topic - topic, _ := mru.NewTopic("foo.eth", nil) + // our feed topic + topic, _ := feed.NewTopic("foo.eth", nil) - createRequest := mru.NewFirstRequest(topic) + createRequest := feed.NewFirstRequest(topic) createRequest.SetData(mh) if err := createRequest.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - resourceManifestHash, err := client.CreateResource(createRequest) + feedManifestHash, err := client.CreateFeedWithManifest(createRequest) if err != nil { - t.Fatalf("Error creating resource: %s", err) + t.Fatalf("Error creating feed manifest: %s", err) } - correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd" - if resourceManifestHash != correctManifestAddrHex { - t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash) + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" + if feedManifestHash != correctManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) } - // Check we get a not found error when trying to get the resource with a made-up manifest - _, err = client.GetResource(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") - if err != ErrNoResourceUpdatesFound { - t.Fatalf("Expected to receive ErrNoResourceUpdatesFound error. Got: %s", err) + // Check we get a not found error when trying to get feed updates with a made-up manifest + _, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + if err != ErrNoFeedUpdatesFound { + t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err) } - reader, err := client.GetResource(nil, correctManifestAddrHex) + reader, err := client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err := ioutil.ReadAll(reader) @@ -435,8 +435,8 @@ func TestClientCreateResourceMultihash(t *testing.T) { } -// TestClientCreateUpdateResource will check that mutable resources can be created and updated via the HTTP client. -func TestClientCreateUpdateResource(t *testing.T) { +// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client. +func TestClientCreateUpdateFeed(t *testing.T) { signer, _ := newTestSigner() @@ -444,28 +444,28 @@ func TestClientCreateUpdateResource(t *testing.T) { client := NewClient(srv.URL) defer srv.Close() - // set raw data for the resource + // set raw data for the feed update databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...") - // our mutable resource name - topic, _ := mru.NewTopic("El Quijote", nil) - createRequest := mru.NewFirstRequest(topic) + // our feed topic name + topic, _ := feed.NewTopic("El Quijote", nil) + createRequest := feed.NewFirstRequest(topic) createRequest.SetData(databytes) if err := createRequest.Sign(signer); err != nil { t.Fatalf("Error signing update: %s", err) } - resourceManifestHash, err := client.CreateResource(createRequest) + feedManifestHash, err := client.CreateFeedWithManifest(createRequest) - correctManifestAddrHex := "fcb8e75f53e480e197c083ad1976d265674d0ce776f2bf359c09c413fb5230b8" - if resourceManifestHash != correctManifestAddrHex { - t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash) + correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882" + if feedManifestHash != correctManifestAddrHex { + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) } - reader, err := client.GetResource(nil, correctManifestAddrHex) + reader, err := client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err := ioutil.ReadAll(reader) @@ -479,7 +479,7 @@ func TestClientCreateUpdateResource(t *testing.T) { // define different data databytes = []byte("... no ha mucho tiempo que vivÃa un hidalgo de los de lanza en astillero ...") - updateRequest, err := client.GetResourceMetadata(nil, correctManifestAddrHex) + updateRequest, err := client.GetFeedRequest(nil, correctManifestAddrHex) if err != nil { t.Fatalf("Error retrieving update request template: %s", err) } @@ -489,13 +489,13 @@ func TestClientCreateUpdateResource(t *testing.T) { t.Fatalf("Error signing update: %s", err) } - if err = client.UpdateResource(updateRequest); err != nil { - t.Fatalf("Error updating resource: %s", err) + if err = client.UpdateFeed(updateRequest); err != nil { + t.Fatalf("Error updating feed: %s", err) } - reader, err = client.GetResource(nil, correctManifestAddrHex) + reader, err = client.QueryFeed(nil, correctManifestAddrHex) if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err = ioutil.ReadAll(reader) @@ -506,17 +506,17 @@ func TestClientCreateUpdateResource(t *testing.T) { t.Fatalf("Expected: %v, got %v", databytes, gotData) } - // now try retrieving resource without a manifest + // now try retrieving feed updates without a manifest - view := &mru.View{ + fd := &feed.Feed{ Topic: topic, User: signer.Address(), } - lookupParams := mru.NewQueryLatest(view, lookup.NoClue) - reader, err = client.GetResource(lookupParams, "") + lookupParams := feed.NewQueryLatest(fd, lookup.NoClue) + reader, err = client.QueryFeed(lookupParams, "") if err != nil { - t.Fatalf("Error retrieving resource: %s", err) + t.Fatalf("Error retrieving feed updates: %s", err) } defer reader.Close() gotData, err = ioutil.ReadAll(reader) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 5ec69373d..370aca5a7 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -31,7 +31,6 @@ import ( "net/http" "os" "path" - "regexp" "strconv" "strings" "time" @@ -41,7 +40,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/rs/cors" ) @@ -145,13 +144,13 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares..., ), }) - mux.Handle("/bzz-resource:/", methodHandler{ + mux.Handle("/bzz-feed:/", methodHandler{ "GET": Adapt( - http.HandlerFunc(server.HandleGetResource), + http.HandlerFunc(server.HandleGetFeed), defaultMiddlewares..., ), "POST": Adapt( - http.HandlerFunc(server.HandlePostResource), + http.HandlerFunc(server.HandlePostFeed), defaultMiddlewares..., ), }) @@ -458,66 +457,35 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, newKey) } -// Parses a resource update post url to corresponding action -// possible combinations: -// / add multihash update to existing hash -// /raw add raw update to existing hash -// /# create new resource with first update as mulitihash -// /raw/# create new resource with first update raw -func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { - re, err := regexp.Compile("^(raw)?/?([0-9]+)?$") - if err != nil { - return isRaw, frequency, err - } - m := re.FindAllStringSubmatch(path, 2) - var freqstr = "0" - if len(m) > 0 { - if m[0][1] != "" { - isRaw = true - } - if m[0][2] != "" { - freqstr = m[0][2] - } - } else if len(path) > 0 { - return isRaw, frequency, fmt.Errorf("invalid path") - } - frequency, err = strconv.ParseUint(freqstr, 10, 64) - return isRaw, frequency, err -} - -// Handles creation of new mutable resources and adding updates to existing mutable resources -// There are two types of updates available, "raw" and "multihash." -// If the latter is used, a subsequent bzz:// GET call to the manifest of the resource will return -// the page that the multihash is pointing to, as if it held a normal swarm content manifest -// -// The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON` -// The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content -func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { +// Handles feed manifest creation and feed updates +// The POST request admits a JSON structure as defined in the feeds package: `feed.updateRequestJSON` +// The requests can be to a) create a feed manifest, b) update a feed or c) both a+b: create a feed manifest and publish a first update +func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) - log.Debug("handle.post.resource", "ruid", ruid) + log.Debug("handle.post.feed", "ruid", ruid) var err error - // Creation and update must send mru.updateRequestJSON JSON structure + // Creation and update must send feed.updateRequestJSON JSON structure body, err := ioutil.ReadAll(r.Body) if err != nil { RespondError(w, r, err.Error(), http.StatusInternalServerError) return } - view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query()) + fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query()) if err != nil { // couldn't parse query string or retrieve manifest getFail.Inc(1) httpStatus := http.StatusBadRequest - if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI { + if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus) + RespondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus) return } - var updateRequest mru.Request - updateRequest.View = *view + var updateRequest feed.Request + updateRequest.Feed = *fd query := r.URL.Query() if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters @@ -527,13 +495,13 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { if updateRequest.IsUpdate() { // Verify that the signature is intact and that the signer is authorized - // to update this resource - // Check this early, to avoid creating a resource and then not being able to set its first update. + // to update this feed + // Check this early, to avoid creating a feed and then not being able to set its first update. if err = updateRequest.Verify(); err != nil { RespondError(w, r, err.Error(), http.StatusForbidden) return } - _, err = s.api.ResourceUpdate(r.Context(), &updateRequest) + _, err = s.api.FeedsUpdate(r.Context(), &updateRequest) if err != nil { RespondError(w, r, err.Error(), http.StatusInternalServerError) return @@ -541,16 +509,16 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { } if query.Get("manifest") == "1" { - // we create a manifest so we can retrieve the resource with bzz:// later - // this manifest has a special "resource type" manifest, and saves the - // resource view ID used to retrieve the resource later - m, err := s.api.NewResourceManifest(r.Context(), &updateRequest.View) + // we create a manifest so we can retrieve feed updates with bzz:// later + // this manifest has a special "feed type" manifest, and saves the + // feed identification used to retrieve feed updates later + m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed) if err != nil { - RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError) return } // the key to the manifest will be passed back to the client - // the client can access the view directly through its resourceView member + // the client can access the feed directly through its Feed member // the manifest key can be set as content in the resolver of the ENS name outdata, err := json.Marshal(m) if err != nil { @@ -563,41 +531,49 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) { } } -// Retrieve mutable resource updates: -// bzz-resource://<id> - get latest update -// bzz-resource://<id>/?period=n - get latest update on period n -// bzz-resource://<id>/?period=n&version=m - get update version m of period n -// bzz-resource://<id>/meta - get metadata and next version information -// <id> = ens name or hash -// TODO: Enable pass maxPeriod parameter -func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { +// HandleGetFeed retrieves Swarm feeds updates: +// bzz-feed://<manifest address or ENS name> - get latest feed update, given a manifest address +// - or - +// specify user + topic (optional), subtopic name (optional) directly, without manifest: +// bzz-feed://?user=0x...&topic=0x...&name=subtopic name +// topic defaults to 0x000... if not specified. +// name defaults to empty string if not specified. +// thus, empty name and topic refers to the user's default feed. +// +// Optional parameters: +// time=xx - get the latest update before time (in epoch seconds) +// hint.time=xx - hint the lookup algorithm looking for updates at around that time +// hint.level=xx - hint the lookup algorithm looking for updates at around this frequency level +// meta=1 - get feed metadata and status information instead of performing a feed query +// NOTE: meta=1 will be deprecated in the near future +func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) - log.Debug("handle.get.resource", "ruid", ruid) + log.Debug("handle.get.feed", "ruid", ruid) var err error - view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query()) + fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query()) if err != nil { // couldn't parse query string or retrieve manifest getFail.Inc(1) httpStatus := http.StatusBadRequest - if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI { + if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI { httpStatus = http.StatusNotFound } - RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus) + RespondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus) return } // determine if the query specifies period and version or it is a metadata query if r.URL.Query().Get("meta") == "1" { - unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), view) + unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd) if err != nil { getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for view=%s: %s", view.Hex(), err), http.StatusNotFound) + RespondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound) return } rawResponse, err := unsignedUpdateRequest.MarshalJSON() if err != nil { - RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError) + RespondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError) return } w.Header().Add("Content-type", "application/json") @@ -606,31 +582,31 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { return } - lookupParams := &mru.Query{View: *view} + lookupParams := &feed.Query{Feed: *fd} if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version - RespondError(w, r, fmt.Sprintf("invalid mutable resource request:%s", err), http.StatusBadRequest) + RespondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest) return } - data, err := s.api.ResourceLookup(r.Context(), lookupParams) + data, err := s.api.FeedsLookup(r.Context(), lookupParams) // any error from the switch statement will end up here if err != nil { - code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err) + code, err2 := s.translateFeedError(w, r, "feed lookup fail", err) RespondError(w, r, err2.Error(), code) return } // All ok, serve the retrieved update - log.Debug("Found update", "view", view.Hex(), "ruid", ruid) + log.Debug("Found update", "feed", fd.Hex(), "ruid", ruid) w.Header().Set("Content-Type", api.MimeOctetStream) http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data)) } -func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { +func (s *Server) translateFeedError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) { code := 0 defaultErr := fmt.Errorf("%s: %v", supErr, err) - rsrcErr, ok := err.(*mru.Error) + rsrcErr, ok := err.(*feed.Error) if !ok && rsrcErr != nil { code = rsrcErr.Code() } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 817519a30..1cf7ff577 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -38,7 +38,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -48,7 +48,7 @@ import ( swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" ) @@ -58,65 +58,24 @@ func init() { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } -func TestResourcePostMode(t *testing.T) { - path := "" - errstr := "resourcePostMode for '%s' should be raw %v frequency %d, was raw %v, frequency %d" - r, f, err := resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if r || f != 0 { - t.Fatalf(errstr, path, false, 0, r, f) - } - - path = "raw" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if !r || f != 0 { - t.Fatalf(errstr, path, true, 0, r, f) - } - - path = "13" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if r || f == 0 { - t.Fatalf(errstr, path, false, 13, r, f) - } - - path = "raw/13" - r, f, err = resourcePostMode(path) - if err != nil { - t.Fatal(err) - } else if !r || f == 0 { - t.Fatalf(errstr, path, true, 13, r, f) - } - - path = "foo/13" - r, f, err = resourcePostMode(path) - if err == nil { - t.Fatal("resourcePostMode for 'foo/13' should fail, returned error nil") - } -} - func serverFunc(api *api.API) testutil.TestServer { return NewServer(api, "") } -func newTestSigner() (*mru.GenericSigner, error) { +func newTestSigner() (*feed.GenericSigner, error) { privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") if err != nil { return nil, err } - return mru.NewGenericSigner(privKey), nil + return feed.NewGenericSigner(privKey), nil } -// test the transparent resolving of multihash resource types with bzz:// scheme +// test the transparent resolving of multihash-containing feed updates with bzz:// scheme // -// first upload data, and store the multihash to the resulting manifest in a resource update +// first upload data, and store the multihash to the resulting manifest in a feed update // retrieving the update with the multihash should return the manifest pointing directly to the data // and raw retrieve of that hash should return the data -func TestBzzResourceMultihash(t *testing.T) { +func TestBzzFeedMultihash(t *testing.T) { signer, _ := newTestSigner() @@ -144,8 +103,8 @@ func TestBzzResourceMultihash(t *testing.T) { log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) - topic, _ := mru.NewTopic("foo.eth", nil) - updateRequest := mru.NewFirstRequest(topic) + topic, _ := feed.NewTopic("foo.eth", nil) + updateRequest := feed.NewFirstRequest(topic) updateRequest.SetData(mh) @@ -154,7 +113,7 @@ func TestBzzResourceMultihash(t *testing.T) { } log.Info("added data", "manifest", string(b), "data", common.ToHex(mh)) - testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL)) + testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } @@ -182,12 +141,12 @@ func TestBzzResourceMultihash(t *testing.T) { t.Fatalf("data %s could not be unmarshaled: %v", b, err) } - correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd" + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) } - // get bzz manifest transparent resource resolve + // get bzz manifest transparent feed update resolve testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) resp, err = http.Get(testBzzUrl) if err != nil { @@ -206,8 +165,8 @@ func TestBzzResourceMultihash(t *testing.T) { } } -// Test resource updates using the raw update methods -func TestBzzResource(t *testing.T) { +// Test Swarm feeds using the raw update methods +func TestBzzFeed(t *testing.T) { srv := testutil.NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() @@ -223,8 +182,8 @@ func TestBzzResource(t *testing.T) { //data for update 2 update2Data := []byte("foo") - topic, _ := mru.NewTopic("foo.eth", nil) - updateRequest := mru.NewFirstRequest(topic) + topic, _ := feed.NewTopic("foo.eth", nil) + updateRequest := feed.NewFirstRequest(topic) if err != nil { t.Fatal(err) } @@ -234,8 +193,8 @@ func TestBzzResource(t *testing.T) { t.Fatal(err) } - // creates resource and sets update 1 - testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL)) + // creates feed and sets update 1 + testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } @@ -262,9 +221,9 @@ func TestBzzResource(t *testing.T) { t.Fatalf("data %s could not be unmarshaled: %v", b, err) } - correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd" + correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b" if rsrcResp.Hex() != correctManifestAddrHex { - t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) + t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex()) } // get the manifest @@ -289,12 +248,12 @@ func TestBzzResource(t *testing.T) { if len(manifest.Entries) != 1 { t.Fatalf("Manifest has %d entries", len(manifest.Entries)) } - correctViewHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d" - if manifest.Entries[0].ResourceView.Hex() != correctViewHex { - t.Fatalf("Expected manifest Resource View '%s', got '%s'", correctViewHex, manifest.Entries[0].ResourceView.Hex()) + correctFeedHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d" + if manifest.Entries[0].Feed.Hex() != correctFeedHex { + t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex()) } - // get bzz manifest transparent resource resolve + // get bzz manifest transparent feed update resolve testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp) resp, err = http.Get(testBzzUrl) if err != nil { @@ -302,7 +261,7 @@ func TestBzzResource(t *testing.T) { } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { - t.Fatal("Expected error status since resource is not multihash. Received 200 OK") + t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK") } b, err = ioutil.ReadAll(resp.Body) if err != nil { @@ -310,21 +269,21 @@ func TestBzzResource(t *testing.T) { } // get non-existent name, should fail - testBzzResUrl := fmt.Sprintf("%s/bzz-resource:/bar", srv.URL) + testBzzResUrl := fmt.Sprintf("%s/bzz-feed:/bar", srv.URL) resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusNotFound { - t.Fatalf("Expected get non-existent resource to fail with StatusNotFound (404), got %d", resp.StatusCode) + t.Fatalf("Expected get non-existent feed manifest to fail with StatusNotFound (404), got %d", resp.StatusCode) } resp.Body.Close() - // get latest update (1.1) through resource directly + // get latest update through bzz-feed directly log.Info("get update latest = 1.1", "addr", correctManifestAddrHex) - testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex) + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s", srv.URL, correctManifestAddrHex) resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) @@ -346,29 +305,29 @@ func TestBzzResource(t *testing.T) { srv.CurrentTime++ log.Info("update 2") - // 1.- get metadata about this resource - testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex) + // 1.- get metadata about this feed + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s/", srv.URL, correctManifestAddrHex) resp, err = http.Get(testBzzResUrl + "?meta=1") if err != nil { t.Fatal(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - t.Fatalf("Get resource metadata returned %s", resp.Status) + t.Fatalf("Get feed metadata returned %s", resp.Status) } b, err = ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } - updateRequest = &mru.Request{} + updateRequest = &feed.Request{} if err = updateRequest.UnmarshalJSON(b); err != nil { - t.Fatalf("Error decoding resource metadata: %s", err) + t.Fatalf("Error decoding feed metadata: %s", err) } updateRequest.SetData(update2Data) if err = updateRequest.Sign(signer); err != nil { t.Fatal(err) } - testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL)) + testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } @@ -385,9 +344,9 @@ func TestBzzResource(t *testing.T) { t.Fatalf("Update returned %s", resp.Status) } - // get latest update (1.2) through resource directly + // get latest update through bzz-feed directly log.Info("get update 1.2") - testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex) + testBzzResUrl = fmt.Sprintf("%s/bzz-feed:/%s", srv.URL, correctManifestAddrHex) resp, err = http.Get(testBzzResUrl) if err != nil { t.Fatal(err) @@ -406,15 +365,15 @@ func TestBzzResource(t *testing.T) { // test manifest-less queries log.Info("get first update in update1Timestamp via direct query") - query := mru.NewQuery(&updateRequest.View, update1Timestamp, lookup.NoClue) + query := feed.NewQuery(&updateRequest.Feed, update1Timestamp, lookup.NoClue) - urlq, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL)) + urlq, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL)) if err != nil { t.Fatal(err) } values := urlq.Query() - query.AppendValues(values) // this adds view query parameters + query.AppendValues(values) // this adds feed query parameters urlq.RawQuery = values.Encode() resp, err = http.Get(urlq.String()) if err != nil { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 06be7323e..7c4cc88e4 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -27,7 +27,7 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/log" @@ -35,8 +35,8 @@ import ( ) const ( - ManifestType = "application/bzz-manifest+json" - ResourceContentType = "application/bzz-resource" + ManifestType = "application/bzz-manifest+json" + FeedContentType = "application/bzz-feed" manifestSizeLimit = 5 * 1024 * 1024 ) @@ -48,15 +48,15 @@ type Manifest struct { // ManifestEntry represents an entry in a swarm manifest type ManifestEntry struct { - Hash string `json:"hash,omitempty"` - Path string `json:"path,omitempty"` - ContentType string `json:"contentType,omitempty"` - Mode int64 `json:"mode,omitempty"` - Size int64 `json:"size,omitempty"` - ModTime time.Time `json:"mod_time,omitempty"` - Status int `json:"status,omitempty"` - Access *AccessEntry `json:"access,omitempty"` - ResourceView *mru.View `json:"resourceView,omitempty"` + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + ContentType string `json:"contentType,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size,omitempty"` + ModTime time.Time `json:"mod_time,omitempty"` + Status int `json:"status,omitempty"` + Access *AccessEntry `json:"access,omitempty"` + Feed *feed.Feed `json:"feed,omitempty"` } // ManifestList represents the result of listing files in a manifest @@ -80,13 +80,13 @@ func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, return addr, err } -// Manifest hack for supporting Mutable Resource Updates from the bzz: scheme +// Manifest hack for supporting Swarm feeds from the bzz: scheme // see swarm/api/api.go:API.Get() for more information -func (a *API) NewResourceManifest(ctx context.Context, view *mru.View) (storage.Address, error) { +func (a *API) NewFeedManifest(ctx context.Context, feed *feed.Feed) (storage.Address, error) { var manifest Manifest entry := ManifestEntry{ - ResourceView: view, - ContentType: ResourceContentType, + Feed: feed, + ContentType: FeedContentType, } manifest.Entries = append(manifest.Entries, entry) data, err := json.Marshal(&manifest) diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 808517088..09cfa4502 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -86,7 +86,7 @@ func Parse(rawuri string) (*URI, error) { // check the scheme is valid switch uri.Scheme { - case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-resource": + case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-feed": default: return nil, fmt.Errorf("unknown scheme %q", u.Scheme) } @@ -108,8 +108,8 @@ func Parse(rawuri string) (*URI, error) { } return uri, nil } -func (u *URI) Resource() bool { - return u.Scheme == "bzz-resource" +func (u *URI) Feed() bool { + return u.Scheme == "bzz-feed" } func (u *URI) Raw() bool { diff --git a/swarm/network/README.md b/swarm/network/README.md index e9c9d30d0..684ad0c8c 100644 --- a/swarm/network/README.md +++ b/swarm/network/README.md @@ -37,7 +37,7 @@ Using the streamer logic, various stream types are easy to implement: * live session syncing * historical syncing * simple retrieve requests and deliveries -* mutable resource updates streams +* swarm feeds streams * receipting for finger pointing ## Syncing @@ -57,7 +57,7 @@ receipts for a deleted chunk easily to refute their challenge. - syncing should be resilient to cut connections, metadata should be persisted that keep track of syncing state across sessions, historical syncing state should survive restart - extra data structures to support syncing should be kept at minimum -- syncing is organized separately for chunk types (resource update v content chunk) +- syncing is not organized separately for chunk types (Swarm feed updates v regular content chunk) - various types of streams should have common logic abstracted Syncing is now entirely mediated by the localstore, ie., no processes or memory leaks due to network contention. diff --git a/swarm/pss/notify/notify.go b/swarm/pss/notify/notify.go index 723092c32..3731fb9db 100644 --- a/swarm/pss/notify/notify.go +++ b/swarm/pss/notify/notify.go @@ -317,7 +317,7 @@ func (c *Controller) handleStartMsg(msg *Msg, keyid string) (err error) { return err } - // TODO this is set to zero-length byte pending decision on protocol for initial message, whether it should include message or not, and how to trigger the initial message so that current state of MRU is sent upon subscription + // TODO this is set to zero-length byte pending decision on protocol for initial message, whether it should include message or not, and how to trigger the initial message so that current state of Swarm feed is sent upon subscription notify := []byte{} replyMsg := NewMsg(MsgCodeNotifyWithKey, msg.namestring, make([]byte, len(notify)+symKeyLength)) copy(replyMsg.Payload, notify) diff --git a/swarm/storage/database.go b/swarm/storage/database.go index 3b5d003de..e25fce31f 100644 --- a/swarm/storage/database.go +++ b/swarm/storage/database.go @@ -20,8 +20,6 @@ package storage // no need for queueing/caching import ( - "fmt" - "github.com/ethereum/go-ethereum/metrics" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -46,13 +44,10 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { return database, nil } -func (db *LDBDatabase) Put(key []byte, value []byte) { +func (db *LDBDatabase) Put(key []byte, value []byte) error { metrics.GetOrRegisterCounter("ldbdatabase.put", nil).Inc(1) - err := db.db.Put(key, value, nil) - if err != nil { - fmt.Println("Error put", err) - } + return db.db.Put(key, value, nil) } func (db *LDBDatabase) Get(key []byte) ([]byte, error) { diff --git a/swarm/storage/mru/binaryserializer.go b/swarm/storage/feed/binaryserializer.go index 3123a82ee..4e4f67a09 100644 --- a/swarm/storage/mru/binaryserializer.go +++ b/swarm/storage/feed/binaryserializer.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import "github.com/ethereum/go-ethereum/common/hexutil" diff --git a/swarm/storage/mru/binaryserializer_test.go b/swarm/storage/feed/binaryserializer_test.go index f524157d6..37828d1c9 100644 --- a/swarm/storage/mru/binaryserializer_test.go +++ b/swarm/storage/feed/binaryserializer_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "encoding/json" diff --git a/swarm/storage/mru/cacheentry.go b/swarm/storage/feed/cacheentry.go index 280331f77..be42008e9 100644 --- a/swarm/storage/mru/cacheentry.go +++ b/swarm/storage/feed/cacheentry.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "bytes" @@ -26,23 +26,23 @@ import ( const ( hasherCount = 8 - resourceHashAlgorithm = storage.SHA3Hash + feedsHashAlgorithm = storage.SHA3Hash defaultRetrieveTimeout = 100 * time.Millisecond ) -// cacheEntry caches resource data and the metadata of its root chunk. +// cacheEntry caches the last known update of a specific Swarm feed. type cacheEntry struct { - ResourceUpdate + Update *bytes.Reader lastKey storage.Address } // implements storage.LazySectionReader func (r *cacheEntry) Size(ctx context.Context, _ chan bool) (int64, error) { - return int64(len(r.ResourceUpdate.data)), nil + return int64(len(r.Update.data)), nil } -//returns the resource's topic +//returns the feed's topic func (r *cacheEntry) Topic() Topic { - return r.View.Topic + return r.Feed.Topic } diff --git a/swarm/storage/feed/doc.go b/swarm/storage/feed/doc.go new file mode 100644 index 000000000..1f07948f2 --- /dev/null +++ b/swarm/storage/feed/doc.go @@ -0,0 +1,43 @@ +/* +Package feeds defines Swarm Feeds. + +Swarm Feeds allows a user to build an update feed about a particular topic +without resorting to ENS on each update. +The update scheme is built on swarm chunks with chunk keys following +a predictable, versionable pattern. + +A Feed is tied to a unique identifier that is deterministically generated out of +the chosen topic. + +A Feed is defined as the series of updates of a specific user about a particular topic + +Actual data updates are also made in the form of swarm chunks. The keys +of the updates are the hash of a concatenation of properties as follows: + +updateAddr = H(Feed, Epoch ID) +where H is the SHA3 hash function +Feed is the combination of Topic and the user address +Epoch ID is a time slot. See the lookup package for more information. + +A user looking up a the latest update in a Feed only needs to know the Topic +and the other user's address. + +The Feed Update data is: +updatedata = Feed|Epoch|data + +The full update data that goes in the chunk payload is: +updatedata|sign(updatedata) + +Structure Summary: + +Request: Feed Update with signature + Update: headers + data + Header: Protocol version and reserved for future use placeholders + ID: Information about how to locate a specific update + Feed: Represents a user's series of publications about a specific Topic + Topic: Item that the updates are about + User: User who updates the Feed + Epoch: time slot where the update is stored + +*/ +package feed diff --git a/swarm/storage/mru/error.go b/swarm/storage/feed/error.go index 18ab52558..206ba3316 100644 --- a/swarm/storage/mru/error.go +++ b/swarm/storage/feed/error.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "fmt" @@ -35,7 +35,7 @@ const ( ErrCnt ) -// Error is a the typed error object used for Mutable Resources +// Error is a the typed error object used for Swarm feeds type Error struct { code int err string @@ -47,12 +47,12 @@ func (e *Error) Error() string { } // Code returns the error code -// Error codes are enumerated in the error.go file within the mru package +// Error codes are enumerated in the error.go file within the feeds package func (e *Error) Code() int { return e.code } -// NewError creates a new Mutable Resource Error object with the specified code and custom error message +// NewError creates a new Swarm feeds Error object with the specified code and custom error message func NewError(code int, s string) error { if code < 0 || code >= ErrCnt { panic("no such error code!") diff --git a/swarm/storage/mru/view.go b/swarm/storage/feed/feed.go index 2e4ce4a0b..b6ea665a6 100644 --- a/swarm/storage/mru/view.go +++ b/swarm/storage/feed/feed.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "hash" @@ -25,21 +25,21 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -// View represents a particular user's view of a resource -type View struct { +// Feed represents a particular user's stream of updates on a topic +type Feed struct { Topic Topic `json:"topic"` User common.Address `json:"user"` } -// View layout: +// Feed layout: // TopicLength bytes // userAddr common.AddressLength bytes -const viewLength = TopicLength + common.AddressLength +const feedLength = TopicLength + common.AddressLength -// mapKey calculates a unique id for this view for the cache map in `Handler` -func (u *View) mapKey() uint64 { - serializedData := make([]byte, viewLength) - u.binaryPut(serializedData) +// mapKey calculates a unique id for this feed. Used by the cache map in `Handler` +func (f *Feed) mapKey() uint64 { + serializedData := make([]byte, feedLength) + f.binaryPut(serializedData) hasher := hashPool.Get().(hash.Hash) defer hashPool.Put(hasher) hasher.Reset() @@ -48,55 +48,55 @@ func (u *View) mapKey() uint64 { return *(*uint64)(unsafe.Pointer(&hash[0])) } -// binaryPut serializes this View instance into the provided slice -func (u *View) binaryPut(serializedData []byte) error { - if len(serializedData) != viewLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize View. Expected %d, got %d", viewLength, len(serializedData)) +// binaryPut serializes this feed instance into the provided slice +func (f *Feed) binaryPut(serializedData []byte) error { + if len(serializedData) != feedLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize feed. Expected %d, got %d", feedLength, len(serializedData)) } var cursor int - copy(serializedData[cursor:cursor+TopicLength], u.Topic[:TopicLength]) + copy(serializedData[cursor:cursor+TopicLength], f.Topic[:TopicLength]) cursor += TopicLength - copy(serializedData[cursor:cursor+common.AddressLength], u.User[:]) + copy(serializedData[cursor:cursor+common.AddressLength], f.User[:]) cursor += common.AddressLength return nil } // binaryLength returns the expected size of this structure when serialized -func (u *View) binaryLength() int { - return viewLength +func (f *Feed) binaryLength() int { + return feedLength } // binaryGet restores the current instance from the information contained in the passed slice -func (u *View) binaryGet(serializedData []byte) error { - if len(serializedData) != viewLength { - return NewErrorf(ErrInvalidValue, "Incorrect slice size to read View. Expected %d, got %d", viewLength, len(serializedData)) +func (f *Feed) binaryGet(serializedData []byte) error { + if len(serializedData) != feedLength { + return NewErrorf(ErrInvalidValue, "Incorrect slice size to read feed. Expected %d, got %d", feedLength, len(serializedData)) } var cursor int - copy(u.Topic[:], serializedData[cursor:cursor+TopicLength]) + copy(f.Topic[:], serializedData[cursor:cursor+TopicLength]) cursor += TopicLength - copy(u.User[:], serializedData[cursor:cursor+common.AddressLength]) + copy(f.User[:], serializedData[cursor:cursor+common.AddressLength]) cursor += common.AddressLength return nil } -// Hex serializes the View to a hex string -func (u *View) Hex() string { - serializedData := make([]byte, viewLength) - u.binaryPut(serializedData) +// Hex serializes the feed to a hex string +func (f *Feed) Hex() string { + serializedData := make([]byte, feedLength) + f.binaryPut(serializedData) return hexutil.Encode(serializedData) } // FromValues deserializes this instance from a string key-value store // useful to parse query strings -func (u *View) FromValues(values Values) (err error) { +func (f *Feed) FromValues(values Values) (err error) { topic := values.Get("topic") if topic != "" { - if err := u.Topic.FromHex(values.Get("topic")); err != nil { + if err := f.Topic.FromHex(values.Get("topic")); err != nil { return err } } else { // see if the user set name and relatedcontent @@ -108,18 +108,18 @@ func (u *View) FromValues(values Values) (err error) { } relatedContent = relatedContent[:storage.AddressLength] } - u.Topic, err = NewTopic(name, relatedContent) + f.Topic, err = NewTopic(name, relatedContent) if err != nil { return err } } - u.User = common.HexToAddress(values.Get("user")) + f.User = common.HexToAddress(values.Get("user")) return nil } // AppendValues serializes this structure into the provided string key-value store // useful to build query strings -func (u *View) AppendValues(values Values) { - values.Set("topic", u.Topic.Hex()) - values.Set("user", u.User.Hex()) +func (f *Feed) AppendValues(values Values) { + values.Set("topic", f.Topic.Hex()) + values.Set("user", f.User.Hex()) } diff --git a/swarm/storage/mru/view_test.go b/swarm/storage/feed/feed_test.go index 45720ba79..6a575594f 100644 --- a/swarm/storage/mru/view_test.go +++ b/swarm/storage/feed/feed_test.go @@ -13,24 +13,24 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "testing" ) -func getTestView() *View { +func getTestFeed() *Feed { topic, _ := NewTopic("world news report, every hour", nil) - return &View{ + return &Feed{ Topic: topic, User: newCharlieSigner().Address(), } } -func TestViewSerializerDeserializer(t *testing.T) { - testBinarySerializerRecovery(t, getTestView(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781c") +func TestFeedSerializerDeserializer(t *testing.T) { + testBinarySerializerRecovery(t, getTestFeed(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781c") } -func TestMetadataSerializerLengthCheck(t *testing.T) { - testBinarySerializerLengthCheck(t, getTestView()) +func TestFeedSerializerLengthCheck(t *testing.T) { + testBinarySerializerLengthCheck(t, getTestFeed()) } diff --git a/swarm/storage/mru/handler.go b/swarm/storage/feed/handler.go index 3e7654795..9e2640282 100644 --- a/swarm/storage/mru/handler.go +++ b/swarm/storage/feed/handler.go @@ -14,9 +14,9 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -// Handler is the API for Mutable Resources -// It enables creating, updating, syncing and retrieving resources and their update data -package mru +// Handler is the API for feeds +// It enables creating, updating, syncing and retrieving feed updates and their data +package feed import ( "bytes" @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" @@ -34,8 +34,8 @@ import ( type Handler struct { chunkStore *storage.NetStore HashSize int - resources map[uint64]*cacheEntry - resourceLock sync.RWMutex + cache map[uint64]*cacheEntry + cacheLock sync.RWMutex storeTimeout time.Duration queryMaxPeriods uint32 } @@ -52,35 +52,35 @@ var hashPool sync.Pool func init() { hashPool = sync.Pool{ New: func() interface{} { - return storage.MakeHashFunc(resourceHashAlgorithm)() + return storage.MakeHashFunc(feedsHashAlgorithm)() }, } } -// NewHandler creates a new Mutable Resource API +// NewHandler creates a new Swarm feeds API func NewHandler(params *HandlerParams) *Handler { - rh := &Handler{ - resources: make(map[uint64]*cacheEntry), + fh := &Handler{ + cache: make(map[uint64]*cacheEntry), } for i := 0; i < hasherCount; i++ { - hashfunc := storage.MakeHashFunc(resourceHashAlgorithm)() - if rh.HashSize == 0 { - rh.HashSize = hashfunc.Size() + hashfunc := storage.MakeHashFunc(feedsHashAlgorithm)() + if fh.HashSize == 0 { + fh.HashSize = hashfunc.Size() } hashPool.Put(hashfunc) } - return rh + return fh } -// SetStore sets the store backend for the Mutable Resource API +// SetStore sets the store backend for the Swarm feeds API func (h *Handler) SetStore(store *storage.NetStore) { h.chunkStore = store } // Validate is a chunk validation method -// If it looks like a resource update, the chunk address is checked against the userAddr of the update's signature +// If it looks like a feed update, the chunk address is checked against the userAddr of the update's signature // It implements the storage.ChunkValidator interface func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { dataLength := len(data) @@ -89,67 +89,67 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { } // check if it is a properly formatted update chunk with - // valid signature and proof of ownership of the resource it is trying + // valid signature and proof of ownership of the feed it is trying // to update // First, deserialize the chunk var r Request if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error()) + log.Debug("Invalid feed update chunk", "addr", chunkAddr.Hex(), "err", err.Error()) return false } - // Verify signatures and that the signer actually owns the resource + // Verify signatures and that the signer actually owns the feed // If it fails, it means either the signature is not valid, data is corrupted - // or someone is trying to update someone else's resource. + // or someone is trying to update someone else's feed. if err := r.Verify(); err != nil { - log.Debug("Invalid signature", "err", err) + log.Debug("Invalid feed update signature", "err", err) return false } return true } -// GetContent retrieves the data payload of the last synced update of the Mutable Resource -func (h *Handler) GetContent(view *View) (storage.Address, []byte, error) { - if view == nil { - return nil, nil, NewError(ErrInvalidValue, "view is nil") +// GetContent retrieves the data payload of the last synced update of the feed +func (h *Handler) GetContent(feed *Feed) (storage.Address, []byte, error) { + if feed == nil { + return nil, nil, NewError(ErrInvalidValue, "feed is nil") } - rsrc := h.get(view) - if rsrc == nil { - return nil, nil, NewError(ErrNotFound, "resource does not exist") + feedUpdate := h.get(feed) + if feedUpdate == nil { + return nil, nil, NewError(ErrNotFound, "feed update not cached") } - return rsrc.lastKey, rsrc.data, nil + return feedUpdate.lastKey, feedUpdate.data, nil } // NewRequest prepares a Request structure with all the necessary information to // just add the desired data and sign it. // The resulting structure can then be signed and passed to Handler.Update to be verified and sent -func (h *Handler) NewRequest(ctx context.Context, view *View) (request *Request, err error) { - if view == nil { - return nil, NewError(ErrInvalidValue, "view cannot be nil") +func (h *Handler) NewRequest(ctx context.Context, feed *Feed) (request *Request, err error) { + if feed == nil { + return nil, NewError(ErrInvalidValue, "feed cannot be nil") } now := TimestampProvider.Now().Time request = new(Request) request.Header.Version = ProtocolVersion - query := NewQueryLatest(view, lookup.NoClue) + query := NewQueryLatest(feed, lookup.NoClue) - rsrc, err := h.Lookup(ctx, query) + feedUpdate, err := h.Lookup(ctx, query) if err != nil { if err.(*Error).code != ErrNotFound { return nil, err } // not finding updates means that there is a network error - // or that the resource really does not have updates + // or that the feed really does not have updates } - request.View = *view + request.Feed = *feed // if we already have an update, then find next epoch - if rsrc != nil { - request.Epoch = lookup.GetNextEpoch(rsrc.Epoch, now) + if feedUpdate != nil { + request.Epoch = lookup.GetNextEpoch(feedUpdate.Epoch, now) } else { request.Epoch = lookup.GetFirstEpoch(now) } @@ -157,13 +157,10 @@ func (h *Handler) NewRequest(ctx context.Context, view *View) (request *Request, return request, nil } -// Lookup retrieves a specific or latest version of the resource -// Lookup works differently depending on the configuration of `ID` -// See the `ID` documentation and helper functions: -// `LookupLatest` and `LookupBefore` -// When looking for the latest update, it starts at the next period after the current time. -// upon failure tries the corresponding keys of each previous period until one is found -// (or startTime is reached, in which case there are no updates). +// Lookup retrieves a specific or latest feed update +// Lookup works differently depending on the configuration of `query` +// See the `query` documentation and helper functions: +// `NewQueryLatest` and `NewQuery` func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) { timeLimit := query.TimeLimit @@ -172,7 +169,7 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) } if query.Hint == lookup.NoClue { // try to use our cache - entry := h.get(&query.View) + entry := h.get(&query.Feed) if entry != nil && entry.Epoch.Time <= timeLimit { // avoid bad hints query.Hint = entry.Epoch } @@ -183,19 +180,19 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) return nil, NewError(ErrInit, "Call Handler.SetStore() before performing lookups") } - var ul ID - ul.View = query.View + var id ID + id.Feed = query.Feed var readCount int // Invoke the lookup engine. // The callback will be called every time the lookup algorithm needs to guess requestPtr, err := lookup.Lookup(timeLimit, query.Hint, func(epoch lookup.Epoch, now uint64) (interface{}, error) { readCount++ - ul.Epoch = epoch + id.Epoch = epoch ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout) defer cancel() - chunk, err := h.chunkStore.Get(ctx, ul.Addr()) + chunk, err := h.chunkStore.Get(ctx, id.Addr()) if err != nil { // TODO: check for catastrophic errors other than chunk not found return nil, nil } @@ -213,39 +210,39 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) return nil, err } - log.Info(fmt.Sprintf("Resource lookup finished in %d lookups", readCount)) + log.Info(fmt.Sprintf("Feed lookup finished in %d lookups", readCount)) request, _ := requestPtr.(*Request) if request == nil { - return nil, NewError(ErrNotFound, "no updates found") + return nil, NewError(ErrNotFound, "no feed updates found") } return h.updateCache(request) } -// update mutable resource cache map with specified content +// update feed updates cache with specified content func (h *Handler) updateCache(request *Request) (*cacheEntry, error) { updateAddr := request.Addr() - log.Trace("resource cache update", "topic", request.Topic.Hex(), "updatekey", updateAddr, "epoch time", request.Epoch.Time, "epoch level", request.Epoch.Level) + log.Trace("feed cache update", "topic", request.Topic.Hex(), "updateaddr", updateAddr, "epoch time", request.Epoch.Time, "epoch level", request.Epoch.Level) - rsrc := h.get(&request.View) - if rsrc == nil { - rsrc = &cacheEntry{} - h.set(&request.View, rsrc) + feedUpdate := h.get(&request.Feed) + if feedUpdate == nil { + feedUpdate = &cacheEntry{} + h.set(&request.Feed, feedUpdate) } // update our rsrcs entry map - rsrc.lastKey = updateAddr - rsrc.ResourceUpdate = request.ResourceUpdate - rsrc.Reader = bytes.NewReader(rsrc.data) - return rsrc, nil + feedUpdate.lastKey = updateAddr + feedUpdate.Update = request.Update + feedUpdate.Reader = bytes.NewReader(feedUpdate.data) + return feedUpdate, nil } -// Update adds an actual data update -// Uses the Mutable Resource metadata currently loaded in the resources map entry. -// It is the caller's responsibility to make sure that this data is not stale. -// Note that a Mutable Resource update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature. An error will be returned if the total length of the chunk payload will exceed this limit. +// Update publishes a feed update +// Note that a feed update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature. +// This results in a max payload of `maxUpdateDataLength` (check update.go for more details) +// An error will be returned if the total length of the chunk payload will exceed this limit. // Update can only check if the caller is trying to overwrite the very last known version, otherwise it just puts the update // on the network. func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Address, err error) { @@ -255,8 +252,8 @@ func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Ad return nil, NewError(ErrInit, "Call Handler.SetStore() before updating") } - rsrc := h.get(&r.View) - if rsrc != nil && rsrc.Epoch.Equals(r.Epoch) { // This is the only cheap check we can do for sure + feedUpdate := h.get(&r.Feed) + if feedUpdate != nil && feedUpdate.Epoch.Equals(r.Epoch) { // This is the only cheap check we can do for sure return nil, NewError(ErrInvalidValue, "A former update in this epoch is already known to exist") } @@ -267,32 +264,32 @@ func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Ad // send the chunk h.chunkStore.Put(ctx, chunk) - log.Trace("resource update", "updateAddr", r.idAddr, "epoch time", r.Epoch.Time, "epoch level", r.Epoch.Level, "data", chunk.Data()) - // update our resources map cache entry if the new update is older than the one we have, if we have it. - if rsrc != nil && r.Epoch.After(rsrc.Epoch) { - rsrc.Epoch = r.Epoch - rsrc.data = make([]byte, len(r.data)) - rsrc.lastKey = r.idAddr - copy(rsrc.data, r.data) - rsrc.Reader = bytes.NewReader(rsrc.data) + log.Trace("feed update", "updateAddr", r.idAddr, "epoch time", r.Epoch.Time, "epoch level", r.Epoch.Level, "data", chunk.Data()) + // update our feed updates map cache entry if the new update is older than the one we have, if we have it. + if feedUpdate != nil && r.Epoch.After(feedUpdate.Epoch) { + feedUpdate.Epoch = r.Epoch + feedUpdate.data = make([]byte, len(r.data)) + feedUpdate.lastKey = r.idAddr + copy(feedUpdate.data, r.data) + feedUpdate.Reader = bytes.NewReader(feedUpdate.data) } return r.idAddr, nil } -// Retrieves the resource cache value for the given nameHash -func (h *Handler) get(view *View) *cacheEntry { - mapKey := view.mapKey() - h.resourceLock.RLock() - defer h.resourceLock.RUnlock() - rsrc := h.resources[mapKey] - return rsrc +// Retrieves the feed update cache value for the given nameHash +func (h *Handler) get(feed *Feed) *cacheEntry { + mapKey := feed.mapKey() + h.cacheLock.RLock() + defer h.cacheLock.RUnlock() + feedUpdate := h.cache[mapKey] + return feedUpdate } -// Sets the resource cache value for the given View -func (h *Handler) set(view *View, rsrc *cacheEntry) { - mapKey := view.mapKey() - h.resourceLock.Lock() - defer h.resourceLock.Unlock() - h.resources[mapKey] = rsrc +// Sets the feed update cache value for the given feed +func (h *Handler) set(feed *Feed, feedUpdate *cacheEntry) { + mapKey := feed.mapKey() + h.cacheLock.Lock() + defer h.cacheLock.Unlock() + h.cache[mapKey] = feedUpdate } diff --git a/swarm/storage/mru/handler_test.go b/swarm/storage/feed/handler_test.go index 13eb9e51b..cf95bc1f5 100644 --- a/swarm/storage/mru/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "bytes" @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) var ( @@ -40,7 +40,7 @@ var ( Time: uint64(4200), } cleanF func() - resourceName = "føø.bar" + subtopicName = "føø.bar" hashfunc = storage.MakeHashFunc(storage.DefaultHash) ) @@ -73,7 +73,7 @@ func (f *fakeTimeProvider) Now() Timestamp { } // make updates and retrieve them based on periods and versions -func TestResourceHandler(t *testing.T) { +func TestFeedsHandler(t *testing.T) { // make fake timeProvider clock := &fakeTimeProvider{ @@ -83,18 +83,18 @@ func TestResourceHandler(t *testing.T) { // signer containing private key signer := newAliceSigner() - rh, datadir, teardownTest, err := setupTest(clock, signer) + feedsHandler, datadir, teardownTest, err := setupTest(clock, signer) if err != nil { t.Fatal(err) } defer teardownTest() - // create a new resource + // create a new feed ctx, cancel := context.WithCancel(context.Background()) defer cancel() - topic, _ := NewTopic("Mess with mru code and see what ghost catches you", nil) - view := View{ + topic, _ := NewTopic("Mess with Swarm feeds code and see what ghost catches you", nil) + fd := Feed{ Topic: topic, User: signer.Address(), } @@ -107,14 +107,14 @@ func TestResourceHandler(t *testing.T) { "clyde", // t=4285 } - request := NewFirstRequest(view.Topic) // this timestamps the update at t = 4200 (start time) - resourcekey := make(map[string]storage.Address) + request := NewFirstRequest(fd.Topic) // this timestamps the update at t = 4200 (start time) + chunkAddress := make(map[string]storage.Address) data := []byte(updates[0]) request.SetData(data) if err := request.Sign(signer); err != nil { t.Fatal(err) } - resourcekey[updates[0]], err = rh.Update(ctx, request) + chunkAddress[updates[0]], err = feedsHandler.Update(ctx, request) if err != nil { t.Fatal(err) } @@ -122,7 +122,7 @@ func TestResourceHandler(t *testing.T) { // move the clock ahead 21 seconds clock.FastForward(21) // t=4221 - request, err = rh.NewRequest(ctx, &request.View) // this timestamps the update at t = 4221 + request, err = feedsHandler.NewRequest(ctx, &request.Feed) // this timestamps the update at t = 4221 if err != nil { t.Fatal(err) } @@ -136,14 +136,14 @@ func TestResourceHandler(t *testing.T) { if err := request.Sign(signer); err != nil { t.Fatal(err) } - resourcekey[updates[1]], err = rh.Update(ctx, request) + chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request) if err == nil { t.Fatal("Expected update to fail since an update in this epoch already exists") } // move the clock ahead 21 seconds clock.FastForward(21) // t=4242 - request, err = rh.NewRequest(ctx, &request.View) + request, err = feedsHandler.NewRequest(ctx, &request.Feed) if err != nil { t.Fatal(err) } @@ -151,14 +151,14 @@ func TestResourceHandler(t *testing.T) { if err := request.Sign(signer); err != nil { t.Fatal(err) } - resourcekey[updates[1]], err = rh.Update(ctx, request) + chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request) if err != nil { t.Fatal(err) } // move the clock ahead 42 seconds clock.FastForward(42) // t=4284 - request, err = rh.NewRequest(ctx, &request.View) + request, err = feedsHandler.NewRequest(ctx, &request.Feed) if err != nil { t.Fatal(err) } @@ -167,14 +167,14 @@ func TestResourceHandler(t *testing.T) { if err := request.Sign(signer); err != nil { t.Fatal(err) } - resourcekey[updates[2]], err = rh.Update(ctx, request) + chunkAddress[updates[2]], err = feedsHandler.Update(ctx, request) if err != nil { t.Fatal(err) } // move the clock ahead 1 second clock.FastForward(1) // t=4285 - request, err = rh.NewRequest(ctx, &request.View) + request, err = feedsHandler.NewRequest(ctx, &request.Feed) if err != nil { t.Fatal(err) } @@ -187,56 +187,56 @@ func TestResourceHandler(t *testing.T) { if err := request.Sign(signer); err != nil { t.Fatal(err) } - resourcekey[updates[3]], err = rh.Update(ctx, request) + chunkAddress[updates[3]], err = feedsHandler.Update(ctx, request) if err != nil { t.Fatal(err) } time.Sleep(time.Second) - rh.Close() + feedsHandler.Close() // check we can retrieve the updates after close clock.FastForward(2000) // t=6285 - rhparams := &HandlerParams{} + feedParams := &HandlerParams{} - rh2, err := NewTestHandler(datadir, rhparams) + feedsHandler2, err := NewTestHandler(datadir, feedParams) if err != nil { t.Fatal(err) } - rsrc2, err := rh2.Lookup(ctx, NewQueryLatest(&request.View, lookup.NoClue)) + update2, err := feedsHandler2.Lookup(ctx, NewQueryLatest(&request.Feed, lookup.NoClue)) if err != nil { t.Fatal(err) } // last update should be "clyde" - if !bytes.Equal(rsrc2.data, []byte(updates[len(updates)-1])) { - t.Fatalf("resource data was %v, expected %v", string(rsrc2.data), updates[len(updates)-1]) + if !bytes.Equal(update2.data, []byte(updates[len(updates)-1])) { + t.Fatalf("feed update data was %v, expected %v", string(update2.data), updates[len(updates)-1]) } - if rsrc2.Level != 22 { - t.Fatalf("resource epoch level was %d, expected 22", rsrc2.Level) + if update2.Level != 22 { + t.Fatalf("feed update epoch level was %d, expected 22", update2.Level) } - if rsrc2.Base() != 0 { - t.Fatalf("resource epoch base time was %d, expected 0", rsrc2.Base()) + if update2.Base() != 0 { + t.Fatalf("feed update epoch base time was %d, expected 0", update2.Base()) } - log.Debug("Latest lookup", "epoch base time", rsrc2.Base(), "epoch level", rsrc2.Level, "data", rsrc2.data) + log.Debug("Latest lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data) // specific point in time - rsrc, err := rh2.Lookup(ctx, NewQuery(&request.View, 4284, lookup.NoClue)) + update, err := feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, 4284, lookup.NoClue)) if err != nil { t.Fatal(err) } // check data - if !bytes.Equal(rsrc.data, []byte(updates[2])) { - t.Fatalf("resource data (historical) was %v, expected %v", string(rsrc2.data), updates[2]) + if !bytes.Equal(update.data, []byte(updates[2])) { + t.Fatalf("feed update data (historical) was %v, expected %v", string(update2.data), updates[2]) } - log.Debug("Historical lookup", "epoch base time", rsrc2.Base(), "epoch level", rsrc2.Level, "data", rsrc2.data) + log.Debug("Historical lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data) // beyond the first should yield an error - rsrc, err = rh2.Lookup(ctx, NewQuery(&request.View, startTime.Time-1, lookup.NoClue)) + update, err = feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, startTime.Time-1, lookup.NoClue)) if err == nil { - t.Fatalf("expected previous to fail, returned epoch %s data %v", rsrc.Epoch.String(), rsrc.data) + t.Fatalf("expected previous to fail, returned epoch %s data %v", update.Epoch.String(), update.data) } } @@ -266,11 +266,11 @@ func TestSparseUpdates(t *testing.T) { defer teardownTest() defer os.RemoveAll(datadir) - // create a new resource + // create a new feed ctx, cancel := context.WithCancel(context.Background()) defer cancel() topic, _ := NewTopic("Very slow updates", nil) - view := View{ + fd := Feed{ Topic: topic, User: signer.Address(), } @@ -280,7 +280,7 @@ func TestSparseUpdates(t *testing.T) { var epoch lookup.Epoch var lastUpdateTime uint64 for T := uint64(0); T < today; T += 5 * Year { - request := NewFirstRequest(view.Topic) + request := NewFirstRequest(fd.Topic) request.Epoch = lookup.GetNextEpoch(epoch, T) request.data = generateData(T) // this generates some data that depends on T, so we can check later request.Sign(signer) @@ -295,14 +295,14 @@ func TestSparseUpdates(t *testing.T) { lastUpdateTime = T } - query := NewQuery(&view, today, lookup.NoClue) + query := NewQuery(&fd, today, lookup.NoClue) _, err = rh.Lookup(ctx, query) if err != nil { t.Fatal(err) } - _, content, err := rh.GetContent(&view) + _, content, err := rh.GetContent(&fd) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestSparseUpdates(t *testing.T) { t.Fatal(err) } - _, content, err = rh.GetContent(&view) + _, content, err = rh.GetContent(&fd) if err != nil { t.Fatal(err) } @@ -348,13 +348,13 @@ func TestValidator(t *testing.T) { } defer teardownTest() - // create new resource - topic, _ := NewTopic(resourceName, nil) - view := View{ + // create new feed + topic, _ := NewTopic(subtopicName, nil) + fd := Feed{ Topic: topic, User: signer.Address(), } - mr := NewFirstRequest(view.Topic) + mr := NewFirstRequest(fd.Topic) // chunk with address data := []byte("foo") @@ -382,7 +382,7 @@ func TestValidator(t *testing.T) { } // tests that the content address validator correctly checks the data -// tests that resource update chunks are passed through content address validator +// tests that feed update chunks are passed through content address validator // there is some redundancy in this test as it also tests content addressed chunks, // which should be evaluated as invalid chunks by this validator func TestValidatorInStore(t *testing.T) { @@ -396,7 +396,7 @@ func TestValidatorInStore(t *testing.T) { signer := newAliceSigner() // set up localstore - datadir, err := ioutil.TempDir("", "storage-testresourcevalidator") + datadir, err := ioutil.TempDir("", "storage-testfeedsvalidator") if err != nil { t.Fatal(err) } @@ -409,10 +409,10 @@ func TestValidatorInStore(t *testing.T) { t.Fatal(err) } - // set up resource handler and add is as a validator to the localstore - rhParams := &HandlerParams{} - rh := NewHandler(rhParams) - store.Validators = append(store.Validators, rh) + // set up Swarm feeds handler and add is as a validator to the localstore + fhParams := &HandlerParams{} + fh := NewHandler(fhParams) + store.Validators = append(store.Validators, fh) // create content addressed chunks, one good, one faulty chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) @@ -420,17 +420,17 @@ func TestValidatorInStore(t *testing.T) { badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data()) topic, _ := NewTopic("xyzzy", nil) - view := View{ + fd := Feed{ Topic: topic, User: signer.Address(), } - // create a resource update chunk with correct publickey + // create a feed update chunk with correct publickey id := ID{ Epoch: lookup.Epoch{Time: 42, Level: 1, }, - View: view, + Feed: fd, } updateAddr := id.Addr() @@ -438,7 +438,7 @@ func TestValidatorInStore(t *testing.T) { r := new(Request) r.idAddr = updateAddr - r.ResourceUpdate.ID = id + r.Update.ID = id r.data = data r.Sign(signer) @@ -451,20 +451,20 @@ func TestValidatorInStore(t *testing.T) { // put the chunks in the store and check their error status err = store.Put(context.Background(), goodChunk) if err == nil { - t.Fatal("expected error on good content address chunk with resource validator only, but got nil") + t.Fatal("expected error on good content address chunk with feed update validator only, but got nil") } err = store.Put(context.Background(), badChunk) if err == nil { - t.Fatal("expected error on bad content address chunk with resource validator only, but got nil") + t.Fatal("expected error on bad content address chunk with feed update validator only, but got nil") } err = store.Put(context.Background(), uglyChunk) if err != nil { - t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err) + t.Fatalf("expected no error on feed update chunk with feed update validator only, but got: %s", err) } } -// create rpc and resourcehandler -func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler, datadir string, teardown func(), err error) { +// create rpc and feeds Handler +func setupTest(timeProvider timestampProvider, signer Signer) (fh *TestHandler, datadir string, teardown func(), err error) { var fsClean func() var rpcClean func() @@ -478,7 +478,7 @@ func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler, } // temp datadir - datadir, err = ioutil.TempDir("", "rh") + datadir, err = ioutil.TempDir("", "fh") if err != nil { return nil, "", nil, err } @@ -487,9 +487,9 @@ func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler, } TimestampProvider = timeProvider - rhparams := &HandlerParams{} - rh, err = NewTestHandler(datadir, rhparams) - return rh, datadir, cleanF, err + fhParams := &HandlerParams{} + fh, err = NewTestHandler(datadir, fhParams) + return fh, datadir, cleanF, err } func newAliceSigner() *GenericSigner { diff --git a/swarm/storage/mru/id.go b/swarm/storage/feed/id.go index f008169ed..7e17743c1 100644 --- a/swarm/storage/mru/id.go +++ b/swarm/storage/feed/id.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "fmt" @@ -22,28 +22,28 @@ import ( "strconv" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" "github.com/ethereum/go-ethereum/swarm/storage" ) // ID uniquely identifies an update on the network. type ID struct { - View `json:"view"` + Feed `json:"feed"` lookup.Epoch `json:"epoch"` } // ID layout: -// View viewLength bytes +// Feed feedLength bytes // Epoch EpochLength -const idLength = viewLength + lookup.EpochLength +const idLength = feedLength + lookup.EpochLength -// Addr calculates the resource update chunk address corresponding to this ID +// Addr calculates the feed update chunk address corresponding to this ID func (u *ID) Addr() (updateAddr storage.Address) { serializedData := make([]byte, idLength) var cursor int - u.View.binaryPut(serializedData[cursor : cursor+viewLength]) - cursor += viewLength + u.Feed.binaryPut(serializedData[cursor : cursor+feedLength]) + cursor += feedLength eid := u.Epoch.ID() copy(serializedData[cursor:cursor+lookup.EpochLength], eid[:]) @@ -61,10 +61,10 @@ func (u *ID) binaryPut(serializedData []byte) error { return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize ID. Expected %d, got %d", idLength, len(serializedData)) } var cursor int - if err := u.View.binaryPut(serializedData[cursor : cursor+viewLength]); err != nil { + if err := u.Feed.binaryPut(serializedData[cursor : cursor+feedLength]); err != nil { return err } - cursor += viewLength + cursor += feedLength epochBytes, err := u.Epoch.MarshalBinary() if err != nil { @@ -88,10 +88,10 @@ func (u *ID) binaryGet(serializedData []byte) error { } var cursor int - if err := u.View.binaryGet(serializedData[cursor : cursor+viewLength]); err != nil { + if err := u.Feed.binaryGet(serializedData[cursor : cursor+feedLength]); err != nil { return err } - cursor += viewLength + cursor += feedLength if err := u.Epoch.UnmarshalBinary(serializedData[cursor : cursor+lookup.EpochLength]); err != nil { return err @@ -108,8 +108,8 @@ func (u *ID) FromValues(values Values) error { u.Epoch.Level = uint8(level) u.Epoch.Time, _ = strconv.ParseUint(values.Get("time"), 10, 64) - if u.View.User == (common.Address{}) { - return u.View.FromValues(values) + if u.Feed.User == (common.Address{}) { + return u.Feed.FromValues(values) } return nil } @@ -119,5 +119,5 @@ func (u *ID) FromValues(values Values) error { func (u *ID) AppendValues(values Values) { values.Set("level", fmt.Sprintf("%d", u.Epoch.Level)) values.Set("time", fmt.Sprintf("%d", u.Epoch.Time)) - u.View.AppendValues(values) + u.Feed.AppendValues(values) } diff --git a/swarm/storage/mru/id_test.go b/swarm/storage/feed/id_test.go index eba58fbf3..e561ff9b4 100644 --- a/swarm/storage/mru/id_test.go +++ b/swarm/storage/feed/id_test.go @@ -1,21 +1,21 @@ -package mru +package feed import ( "testing" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) func getTestID() *ID { return &ID{ - View: *getTestView(), + Feed: *getTestFeed(), Epoch: lookup.GetFirstEpoch(1000), } } func TestIDAddr(t *testing.T) { - ul := getTestID() - updateAddr := ul.Addr() + id := getTestID() + updateAddr := id.Addr() compareByteSliceToExpectedHex(t, "updateAddr", updateAddr, "0x8b24583ec293e085f4c78aaee66d1bc5abfb8b4233304d14a349afa57af2a783") } diff --git a/swarm/storage/mru/lookup/epoch.go b/swarm/storage/feed/lookup/epoch.go index bafe95477..bafe95477 100644 --- a/swarm/storage/mru/lookup/epoch.go +++ b/swarm/storage/feed/lookup/epoch.go diff --git a/swarm/storage/mru/lookup/epoch_test.go b/swarm/storage/feed/lookup/epoch_test.go index 62cf5523d..0629f3d1d 100644 --- a/swarm/storage/mru/lookup/epoch_test.go +++ b/swarm/storage/feed/lookup/epoch_test.go @@ -3,7 +3,7 @@ package lookup_test import ( "testing" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) func TestMarshallers(t *testing.T) { diff --git a/swarm/storage/mru/lookup/lookup.go b/swarm/storage/feed/lookup/lookup.go index c98248d70..2f862d81c 100644 --- a/swarm/storage/mru/lookup/lookup.go +++ b/swarm/storage/feed/lookup/lookup.go @@ -15,7 +15,7 @@ // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. /* -Package lookup defines resource lookup algorithms and provides tools to place updates +Package lookup defines feed lookup algorithms and provides tools to place updates so they can be found */ package lookup diff --git a/swarm/storage/mru/lookup/lookup_test.go b/swarm/storage/feed/lookup/lookup_test.go index 34bcb61f0..d71e81e95 100644 --- a/swarm/storage/mru/lookup/lookup_test.go +++ b/swarm/storage/feed/lookup/lookup_test.go @@ -22,7 +22,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/swarm/log" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) type Data struct { diff --git a/swarm/storage/mru/query.go b/swarm/storage/feed/query.go index 13a28eaab..8be78a952 100644 --- a/swarm/storage/mru/query.go +++ b/swarm/storage/feed/query.go @@ -14,20 +14,20 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "fmt" "strconv" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) // Query is used to specify constraints when performing an update lookup // TimeLimit indicates an upper bound for the search. Set to 0 for "now" type Query struct { - View + Feed Hint lookup.Epoch TimeLimit uint64 } @@ -41,8 +41,8 @@ func (q *Query) FromValues(values Values) error { level, _ := strconv.ParseUint(values.Get("hint.level"), 10, 32) q.Hint.Level = uint8(level) q.Hint.Time, _ = strconv.ParseUint(values.Get("hint.time"), 10, 64) - if q.View.User == (common.Address{}) { - return q.View.FromValues(values) + if q.Feed.User == (common.Address{}) { + return q.Feed.FromValues(values) } return nil } @@ -59,20 +59,20 @@ func (q *Query) AppendValues(values Values) { if q.Hint.Time != 0 { values.Set("hint.time", fmt.Sprintf("%d", q.Hint.Time)) } - q.View.AppendValues(values) + q.Feed.AppendValues(values) } // NewQuery constructs an Query structure to find updates on or before `time` // if time == 0, the latest update will be looked up -func NewQuery(view *View, time uint64, hint lookup.Epoch) *Query { +func NewQuery(feed *Feed, time uint64, hint lookup.Epoch) *Query { return &Query{ TimeLimit: time, - View: *view, + Feed: *feed, Hint: hint, } } -// NewQueryLatest generates lookup parameters that look for the latest version of a resource -func NewQueryLatest(view *View, hint lookup.Epoch) *Query { - return NewQuery(view, 0, hint) +// NewQueryLatest generates lookup parameters that look for the latest update to a feed +func NewQueryLatest(feed *Feed, hint lookup.Epoch) *Query { + return NewQuery(feed, 0, hint) } diff --git a/swarm/storage/mru/query_test.go b/swarm/storage/feed/query_test.go index 189a465d6..9fa5e2980 100644 --- a/swarm/storage/mru/query_test.go +++ b/swarm/storage/feed/query_test.go @@ -14,18 +14,18 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "testing" ) func getTestQuery() *Query { - ul := getTestID() + id := getTestID() return &Query{ TimeLimit: 5000, - View: ul.View, - Hint: ul.Epoch, + Feed: id.Feed, + Hint: id.Epoch, } } diff --git a/swarm/storage/mru/request.go b/swarm/storage/feed/request.go index f6d0f38ff..6968d8b9a 100644 --- a/swarm/storage/mru/request.go +++ b/swarm/storage/feed/request.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "bytes" @@ -24,15 +24,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) -// Request represents an update and/or resource create message +// Request represents a request to sign or signed feed update message type Request struct { - ResourceUpdate // actual content that will be put on the chunk, less signature - Signature *Signature - idAddr storage.Address // cached chunk address for the update (not serialized, for internal use) - binaryData []byte // cached serialized data (does not get serialized again!, for efficiency/internal use) + Update // actual content that will be put on the chunk, less signature + Signature *Signature + idAddr storage.Address // cached chunk address for the update (not serialized, for internal use) + binaryData []byte // cached serialized data (does not get serialized again!, for efficiency/internal use) } // updateRequestJSON represents a JSON-serialized UpdateRequest @@ -44,11 +44,11 @@ type updateRequestJSON struct { } // Request layout -// resourceUpdate bytes +// Update bytes // SignatureLength bytes const minimumSignedUpdateLength = minimumUpdateDataLength + signatureLength -// NewFirstRequest returns a ready to sign request to publish a first update +// NewFirstRequest returns a ready to sign request to publish a first feed update func NewFirstRequest(topic Topic) *Request { request := new(Request) @@ -56,13 +56,13 @@ func NewFirstRequest(topic Topic) *Request { // get the current time now := TimestampProvider.Now().Time request.Epoch = lookup.GetFirstEpoch(now) - request.View.Topic = topic + request.Feed.Topic = topic request.Header.Version = ProtocolVersion return request } -// SetData stores the payload data the resource will be updated with +// SetData stores the payload data the feed update will be updated with func (r *Request) SetData(data []byte) { r.data = data r.Signature = nil @@ -73,7 +73,7 @@ func (r *Request) IsUpdate() bool { return r.Signature != nil } -// Verify checks that signatures are valid and that the signer owns the resource to be updated +// Verify checks that signatures are valid func (r *Request) Verify() (err error) { if len(r.data) == 0 { return NewError(ErrInvalidValue, "Update does not contain data") @@ -88,7 +88,7 @@ func (r *Request) Verify() (err error) { } // get the address of the signer (which also checks that it's a valid signature) - r.View.User, err = getUserAddr(digest, *r.Signature) + r.Feed.User, err = getUserAddr(digest, *r.Signature) if err != nil { return err } @@ -103,9 +103,9 @@ func (r *Request) Verify() (err error) { return nil } -// Sign executes the signature to validate the resource +// Sign executes the signature to validate the update message func (r *Request) Sign(signer Signer) error { - r.View.User = signer.Address() + r.Feed.User = signer.Address() r.binaryData = nil //invalidate serialized data digest, err := r.GetDigest() // computes digest and serializes into .binaryData if err != nil { @@ -133,16 +133,16 @@ func (r *Request) Sign(signer Signer) error { return nil } -// GetDigest creates the resource update digest used in signatures +// GetDigest creates the feed update digest used in signatures // the serialized payload is cached in .binaryData func (r *Request) GetDigest() (result common.Hash, err error) { hasher := hashPool.Get().(hash.Hash) defer hashPool.Put(hasher) hasher.Reset() - dataLength := r.ResourceUpdate.binaryLength() + dataLength := r.Update.binaryLength() if r.binaryData == nil { r.binaryData = make([]byte, dataLength+signatureLength) - if err := r.ResourceUpdate.binaryPut(r.binaryData[:dataLength]); err != nil { + if err := r.Update.binaryPut(r.binaryData[:dataLength]); err != nil { return result, err } } @@ -161,10 +161,10 @@ func (r *Request) toChunk() (storage.Chunk, error) { return nil, NewError(ErrInvalidSignature, "toChunk called without a valid signature or payload data. Call .Sign() first.") } - resourceUpdateLength := r.ResourceUpdate.binaryLength() + updateLength := r.Update.binaryLength() // signature is the last item in the chunk data - copy(r.binaryData[resourceUpdateLength:], r.Signature[:]) + copy(r.binaryData[updateLength:], r.Signature[:]) chunk := storage.NewChunk(r.idAddr, r.binaryData) return chunk, nil @@ -174,14 +174,14 @@ func (r *Request) toChunk() (storage.Chunk, error) { func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error { // for update chunk layout see Request definition - //deserialize the resource update portion - if err := r.ResourceUpdate.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil { + //deserialize the feed update portion + if err := r.Update.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil { return err } // Extract the signature var signature *Signature - cursor := r.ResourceUpdate.binaryLength() + cursor := r.Update.binaryLength() sigdata := chunkdata[cursor : cursor+signatureLength] if len(sigdata) > 0 { signature = &Signature{} @@ -209,7 +209,7 @@ func (r *Request) FromValues(values Values, data []byte) error { r.Signature = new(Signature) copy(r.Signature[:], signatureBytes) } - err = r.ResourceUpdate.FromValues(values, data) + err = r.Update.FromValues(values, data) if err != nil { return err } @@ -223,7 +223,7 @@ func (r *Request) AppendValues(values Values) []byte { if r.Signature != nil { values.Set("signature", hexutil.Encode(r.Signature[:])) } - return r.ResourceUpdate.AppendValues(values) + return r.Update.AppendValues(values) } // fromJSON takes an update request JSON and populates an UpdateRequest diff --git a/swarm/storage/mru/request_test.go b/swarm/storage/feed/request_test.go index c32d5ec13..f5de32b74 100644 --- a/swarm/storage/mru/request_test.go +++ b/swarm/storage/feed/request_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "bytes" @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru/lookup" + "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" ) func areEqualJSON(s1, s2 string) (bool, error) { @@ -47,51 +47,49 @@ func areEqualJSON(s1, s2 string) (bool, error) { } // TestEncodingDecodingUpdateRequests ensures that requests are serialized properly -// while also checking cryptographically that only the owner of a resource can update it. +// while also checking cryptographically that only the owner of a feed can update it. func TestEncodingDecodingUpdateRequests(t *testing.T) { charlie := newCharlieSigner() //Charlie bob := newBobSigner() //Bob - // Create a resource to our good guy Charlie's name - topic, _ := NewTopic("a good resource name", nil) - createRequest := NewFirstRequest(topic) - createRequest.User = charlie.Address() + // Create a feed to our good guy Charlie's name + topic, _ := NewTopic("a good topic name", nil) + firstRequest := NewFirstRequest(topic) + firstRequest.User = charlie.Address() // We now encode the create message to simulate we send it over the wire - messageRawData, err := createRequest.MarshalJSON() + messageRawData, err := firstRequest.MarshalJSON() if err != nil { - t.Fatalf("Error encoding create resource request: %s", err) + t.Fatalf("Error encoding first feed update request: %s", err) } // ... the message arrives and is decoded... - var recoveredCreateRequest Request - if err := recoveredCreateRequest.UnmarshalJSON(messageRawData); err != nil { - t.Fatalf("Error decoding create resource request: %s", err) + var recoveredFirstRequest Request + if err := recoveredFirstRequest.UnmarshalJSON(messageRawData); err != nil { + t.Fatalf("Error decoding first feed update request: %s", err) } // ... but verification should fail because it is not signed! - if err := recoveredCreateRequest.Verify(); err == nil { + if err := recoveredFirstRequest.Verify(); err == nil { t.Fatal("Expected Verify to fail since the message is not signed") } - // We now assume that the resource was created and propagated. With rootAddr we can retrieve the resource metadata - // and recover the information above. To sign an update, we need the rootAddr and the metaHash to construct - // proof of ownership + // We now assume that the feed ypdate was created and propagated. - const expectedSignature = "0x32c2d2c7224e24e4d3ae6a10595fc6e945f1b3ecdf548a04d8247c240a50c9240076aa7730abad6c8a46dfea00cfb8f43b6211f02db5c4cc5ed8584cb0212a4d00" - const expectedJSON = `{"view":{"topic":"0x6120676f6f64207265736f75726365206e616d65000000000000000000000000","user":"0x876a8936a7cd0b79ef0735ad0896c1afe278781c"},"epoch":{"time":1000,"level":1},"protocolVersion":0,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421"}` + const expectedSignature = "0x7235b27a68372ddebcf78eba48543fa460864b0b0e99cb533fcd3664820e603312d29426dd00fb39628f5299480a69bf6e462838d78de49ce0704c754c9deb2601" + const expectedJSON = `{"feed":{"topic":"0x6120676f6f6420746f706963206e616d65000000000000000000000000000000","user":"0x876a8936a7cd0b79ef0735ad0896c1afe278781c"},"epoch":{"time":1000,"level":1},"protocolVersion":0,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421"}` //Put together an unsigned update request that we will serialize to send it to the signer. data := []byte("This hour's update: Swarm 99.0 has been released!") request := &Request{ - ResourceUpdate: ResourceUpdate{ + Update: Update{ ID: ID{ Epoch: lookup.Epoch{ Time: 1000, Level: 1, }, - View: createRequest.ResourceUpdate.View, + Feed: firstRequest.Update.Feed, }, data: data, }, @@ -138,7 +136,7 @@ func TestEncodingDecodingUpdateRequests(t *testing.T) { t.Fatal("Expected DecodeUpdateRequest to fail when trying to interpret a corrupt message with an invalid signature") } - // Now imagine Bob wants to create an update of his own about the same resource, + // Now imagine Bob wants to create an update of his own about the same feed, // signing a message with his private key if err := request.Sign(bob); err != nil { t.Fatalf("Error signing: %s", err) @@ -191,7 +189,7 @@ func TestEncodingDecodingUpdateRequests(t *testing.T) { func getTestRequest() *Request { return &Request{ - ResourceUpdate: *getTestResourceUpdate(), + Update: *getTestFeedUpdate(), } } @@ -230,7 +228,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { var recovered Request recovered.fromChunk(chunk.Address(), chunk.Data()) if !reflect.DeepEqual(recovered, r) { - t.Fatal("Expected recovered SignedResource update to equal the original one") + t.Fatal("Expected recovered feed update request to equal the original one") } } @@ -250,7 +248,7 @@ func TestReverse(t *testing.T) { // signer containing private key signer := newAliceSigner() - // set up rpc and create resourcehandler + // set up rpc and create feeds handler _, _, teardownTest, err := setupTest(timeProvider, signer) if err != nil { t.Fatal(err) @@ -258,7 +256,7 @@ func TestReverse(t *testing.T) { defer teardownTest() topic, _ := NewTopic("Cervantes quotes", nil) - view := View{ + fd := Feed{ Topic: topic, User: signer.Address(), } @@ -266,7 +264,7 @@ func TestReverse(t *testing.T) { data := []byte("Donde una puerta se cierra, otra se abre") request := new(Request) - request.View = view + request.Feed = fd request.Epoch = epoch request.data = data @@ -291,15 +289,15 @@ func TestReverse(t *testing.T) { if err != nil { t.Fatal(err) } - recoveredaddress, err := getUserAddr(checkdigest, *checkUpdate.Signature) + recoveredAddr, err := getUserAddr(checkdigest, *checkUpdate.Signature) if err != nil { t.Fatalf("Retrieve address from signature fail: %v", err) } - originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) + originalAddr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) // check that the metadata retrieved from the chunk matches what we gave it - if recoveredaddress != originaladdress { - t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress) + if recoveredAddr != originalAddr { + t.Fatalf("addresses dont match: %x != %x", originalAddr, recoveredAddr) } if !bytes.Equal(key[:], chunk.Address()[:]) { diff --git a/swarm/storage/mru/resource_sign.go b/swarm/storage/feed/sign.go index 58196f10e..5f0ea0b33 100644 --- a/swarm/storage/mru/resource_sign.go +++ b/swarm/storage/feed/sign.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "crypto/ecdsa" @@ -28,7 +28,7 @@ const signatureLength = 65 // Signature is an alias for a static byte array with the size of a signature type Signature [signatureLength]byte -// Signer signs Mutable Resource update payloads +// Signer signs feed update payloads type Signer interface { Sign(common.Hash) (Signature, error) Address() common.Address @@ -65,7 +65,7 @@ func (s *GenericSigner) Address() common.Address { return s.address } -// getUserAddr extracts the address of the resource update signer +// getUserAddr extracts the address of the feed update signer func getUserAddr(digest common.Hash, signature Signature) (common.Address, error) { pub, err := crypto.SigToPub(digest.Bytes(), signature[:]) if err != nil { diff --git a/swarm/storage/mru/testutil.go b/swarm/storage/feed/testutil.go index 7a5a9e4d9..b513fa1f2 100644 --- a/swarm/storage/mru/testutil.go +++ b/swarm/storage/feed/testutil.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "context" @@ -27,7 +27,7 @@ import ( ) const ( - testDbDirName = "mru" + testDbDirName = "feeds" ) type TestHandler struct { @@ -52,20 +52,20 @@ func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetF // NewTestHandler creates Handler object to be used for testing purposes. func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) { path := filepath.Join(datadir, testDbDirName) - rh := NewHandler(params) + fh := NewHandler(params) localstoreparams := storage.NewDefaultLocalStoreParams() localstoreparams.Init(path) localStore, err := storage.NewLocalStore(localstoreparams, nil) if err != nil { return nil, fmt.Errorf("localstore create fail, path %s: %v", path, err) } - localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(resourceHashAlgorithm))) - localStore.Validators = append(localStore.Validators, rh) + localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(feedsHashAlgorithm))) + localStore.Validators = append(localStore.Validators, fh) netStore, err := storage.NewNetStore(localStore, nil) if err != nil { return nil, err } netStore.NewNetFetcherFunc = newFakeNetFetcher - rh.SetStore(netStore) - return &TestHandler{rh}, nil + fh.SetStore(netStore) + return &TestHandler{fh}, nil } diff --git a/swarm/storage/mru/timestampprovider.go b/swarm/storage/feed/timestampprovider.go index 6ac153213..072dc3a48 100644 --- a/swarm/storage/mru/timestampprovider.go +++ b/swarm/storage/feed/timestampprovider.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "encoding/binary" @@ -22,7 +22,7 @@ import ( "time" ) -// TimestampProvider sets the time source of the mru package +// TimestampProvider sets the time source of the feeds package var TimestampProvider timestampProvider = NewDefaultTimestampProvider() // Timestamp encodes a point in time as a Unix epoch diff --git a/swarm/storage/mru/topic.go b/swarm/storage/feed/topic.go index f318a5593..43a7b4ba4 100644 --- a/swarm/storage/mru/topic.go +++ b/swarm/storage/feed/topic.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "bytes" @@ -29,7 +29,7 @@ import ( // TopicLength establishes the max length of a topic string const TopicLength = storage.AddressLength -// Topic represents what a resource talks about +// Topic represents what a feed is about type Topic [TopicLength]byte // ErrTopicTooLong is returned when creating a topic with a name/related content too long @@ -74,7 +74,7 @@ func (t *Topic) FromHex(hex string) error { return nil } -// Name will try to extract the resource name out of the topic +// Name will try to extract the topic name out of the Topic func (t *Topic) Name(relatedContent []byte) string { nameBytes := *t if relatedContent != nil { diff --git a/swarm/storage/mru/topic_test.go b/swarm/storage/feed/topic_test.go index dad7c7ddc..0403204f7 100644 --- a/swarm/storage/mru/topic_test.go +++ b/swarm/storage/feed/topic_test.go @@ -1,4 +1,4 @@ -package mru +package feed import ( "testing" diff --git a/swarm/storage/mru/update.go b/swarm/storage/feed/update.go index 6aa57fce1..627a537d1 100644 --- a/swarm/storage/mru/update.go +++ b/swarm/storage/feed/update.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "fmt" @@ -34,25 +34,25 @@ type Header struct { Padding [headerLength - 1]uint8 // reserved for future use } -// ResourceUpdate encapsulates the information sent as part of a resource update -type ResourceUpdate struct { +// Update encapsulates the information sent as part of a feed update +type Update struct { Header Header // - ID // Resource update identifying information + ID // Feed Update identifying information data []byte // actual data payload } const minimumUpdateDataLength = idLength + headerLength + 1 const maxUpdateDataLength = chunk.DefaultSize - signatureLength - idLength - headerLength -// binaryPut serializes the resource update information into the given slice -func (r *ResourceUpdate) binaryPut(serializedData []byte) error { +// binaryPut serializes the feed update information into the given slice +func (r *Update) binaryPut(serializedData []byte) error { datalength := len(r.data) if datalength == 0 { - return NewError(ErrInvalidValue, "cannot update a resource with no data") + return NewError(ErrInvalidValue, "a feed update must contain data") } if datalength > maxUpdateDataLength { - return NewErrorf(ErrInvalidValue, "data is too big (length=%d). Max length=%d", datalength, maxUpdateDataLength) + return NewErrorf(ErrInvalidValue, "feed update data is too big (length=%d). Max length=%d", datalength, maxUpdateDataLength) } if len(serializedData) != r.binaryLength() { @@ -79,14 +79,14 @@ func (r *ResourceUpdate) binaryPut(serializedData []byte) error { } // binaryLength returns the expected number of bytes this structure will take to encode -func (r *ResourceUpdate) binaryLength() int { +func (r *Update) binaryLength() int { return idLength + headerLength + len(r.data) } // binaryGet populates this instance from the information contained in the passed byte slice -func (r *ResourceUpdate) binaryGet(serializedData []byte) error { +func (r *Update) binaryGet(serializedData []byte) error { if len(serializedData) < minimumUpdateDataLength { - return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a resource update chunk", minimumUpdateDataLength) + return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a feed update chunk", minimumUpdateDataLength) } dataLength := len(serializedData) - idLength - headerLength // at this point we can be satisfied that we have the correct data length to read @@ -116,7 +116,7 @@ func (r *ResourceUpdate) binaryGet(serializedData []byte) error { // FromValues deserializes this instance from a string key-value store // useful to parse query strings -func (r *ResourceUpdate) FromValues(values Values, data []byte) error { +func (r *Update) FromValues(values Values, data []byte) error { r.data = data version, _ := strconv.ParseUint(values.Get("protocolVersion"), 10, 32) r.Header.Version = uint8(version) @@ -125,7 +125,7 @@ func (r *ResourceUpdate) FromValues(values Values, data []byte) error { // AppendValues serializes this structure into the provided string key-value store // useful to build query strings -func (r *ResourceUpdate) AppendValues(values Values) []byte { +func (r *Update) AppendValues(values Values) []byte { r.ID.AppendValues(values) values.Set("protocolVersion", fmt.Sprintf("%d", r.Header.Version)) return r.data diff --git a/swarm/storage/mru/update_test.go b/swarm/storage/feed/update_test.go index bd706d83a..4007223c6 100644 --- a/swarm/storage/mru/update_test.go +++ b/swarm/storage/feed/update_test.go @@ -14,37 +14,37 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. -package mru +package feed import ( "testing" ) -func getTestResourceUpdate() *ResourceUpdate { - return &ResourceUpdate{ +func getTestFeedUpdate() *Update { + return &Update{ ID: *getTestID(), data: []byte("El que lee mucho y anda mucho, ve mucho y sabe mucho"), } } -func TestResourceUpdateSerializer(t *testing.T) { - testBinarySerializerRecovery(t, getTestResourceUpdate(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f") +func TestUpdateSerializer(t *testing.T) { + testBinarySerializerRecovery(t, getTestFeedUpdate(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f") } -func TestResourceUpdateLengthCheck(t *testing.T) { - testBinarySerializerLengthCheck(t, getTestResourceUpdate()) +func TestUpdateLengthCheck(t *testing.T) { + testBinarySerializerLengthCheck(t, getTestFeedUpdate()) // Test fail if update is too big - update := getTestResourceUpdate() + update := getTestFeedUpdate() update.data = make([]byte, maxUpdateDataLength+100) serialized := make([]byte, update.binaryLength()) if err := update.binaryPut(serialized); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since update is too big") + t.Fatal("Expected update.binaryPut to fail since update is too big") } // test fail if data is empty or nil update.data = nil serialized = make([]byte, update.binaryLength()) if err := update.binaryPut(serialized); err == nil { - t.Fatal("Expected resourceUpdate.binaryPut to fail since data is empty") + t.Fatal("Expected update.binaryPut to fail since data is empty") } } diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index bde627394..2a7f51cb3 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" - ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" @@ -61,6 +60,7 @@ var ( keyDataIdx = []byte{4} keyData = byte(6) keyDistanceCnt = byte(7) + keySchema = []byte{8} ) var ( @@ -418,8 +418,8 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } } -func (s *LDBStore) Cleanup() { - //Iterates over the database and checks that there are no chunks bigger than 4kb +//Cleanup iterates over the database and deletes chunks if they pass the `f` condition +func (s *LDBStore) Cleanup(f func(*chunk) bool) { var errorsFound, removed, total int it := s.db.NewIterator() @@ -471,7 +471,8 @@ func (s *LDBStore) Cleanup() { cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) - if len(c.sdata) > ch.DefaultSize+8 { + // if chunk is to be removed + if f(c) { log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) s.delete(index.Idx, getIndexKey(key[1:]), po) removed++ @@ -730,6 +731,30 @@ func (s *LDBStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { return true } +// GetSchema is returning the current named schema of the datastore as read from LevelDB +func (s *LDBStore) GetSchema() (string, error) { + s.lock.Lock() + defer s.lock.Unlock() + + data, err := s.db.Get(keySchema) + if err != nil { + if err == leveldb.ErrNotFound { + return "", nil + } + return "", err + } + + return string(data), nil +} + +// PutSchema is saving a named schema to the LevelDB datastore +func (s *LDBStore) PutSchema(schema string) error { + s.lock.Lock() + defer s.lock.Unlock() + + return s.db.Put(keySchema, []byte(schema)) +} + func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) log.Trace("ldbstore.get", "key", addr) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 04701ee69..b28f62524 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -184,3 +184,42 @@ func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, func (ls *LocalStore) Close() { ls.DbStore.Close() } + +// Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match +func (ls *LocalStore) Migrate() error { + schema, err := ls.DbStore.GetSchema() + if err != nil { + log.Error(err.Error()) + return err + } + + log.Debug("found schema", "schema", schema, "runtime-schema", CurrentDbSchema) + if schema != CurrentDbSchema { + // run migrations + + if schema == "" { + log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema) + + cleanupFunc := func(c *chunk) bool { + // if one of the ls.Validators passes, it means a chunk is of particular type and it is valid + valid := false + for _, v := range ls.Validators { + if valid = v.Validate(c.Address(), c.Data()); valid { + break + } + } + return valid + } + + ls.DbStore.Cleanup(cleanupFunc) + + err := ls.DbStore.PutSchema(DbSchemaPurity) + if err != nil { + log.Error(err.Error()) + return err + } + } + } + + return nil +} diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 814d270d3..10f43f30f 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -30,8 +30,8 @@ var ( ) // tests that the content address validator correctly checks the data -// tests that resource update chunks are passed through content address validator -// the test checking the resouce update validator internal correctness is found in resource_test.go +// tests that feed update chunks are passed through content address validator +// the test checking the resouce update validator internal correctness is found in storage/feeds/handler_test.go func TestValidator(t *testing.T) { // set up localstore datadir, err := ioutil.TempDir("", "storage-testvalidator") diff --git a/swarm/storage/mru/doc.go b/swarm/storage/mru/doc.go deleted file mode 100644 index 19330e0c1..000000000 --- a/swarm/storage/mru/doc.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Package mru defines Mutable resource updates. - -A Mutable Resource is an entity which allows updates to a resource -without resorting to ENS on each update. -The update scheme is built on swarm chunks with chunk keys following -a predictable, versionable pattern. - -A Resource is tied to a unique identifier that is deterministically generated out of -the chosen topic. - -A Resource View is defined as a specific user's point of view about a particular resource. -Thus, a View is a Topic + the user's address (userAddr) - -Actual data updates are also made in the form of swarm chunks. The keys -of the updates are the hash of a concatenation of properties as follows: - -updateAddr = H(View, Epoch ID) -where H is the SHA3 hash function -View is the combination of Topic and the user address -Epoch ID is a time slot. See the lookup package for more information. - -A user looking up a resource would only need to know the View in order to -another user's updates - -The resource update data is: -resourcedata = View|Epoch|data - -the full update data that goes in the chunk payload is: -resourcedata|sign(resourcedata) - -Structure Summary: - -Request: Resource update with signature - ResourceUpdate: headers + data - Header: Protocol version and reserved for future use placeholders - ID: Information about how to locate a specific update - View: Author of the update and what is updating - Topic: Item that the updates are about - User: User who updates the resource - Epoch: time slot where the update is stored - -*/ -package mru diff --git a/swarm/storage/schema.go b/swarm/storage/schema.go new file mode 100644 index 000000000..fb8498a29 --- /dev/null +++ b/swarm/storage/schema.go @@ -0,0 +1,6 @@ +package storage + +// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5 +const DbSchemaPurity = "purity" + +const CurrentDbSchema = DbSchemaPurity diff --git a/swarm/swarm.go b/swarm/swarm.go index 0cd56d4eb..aea0989a1 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -49,8 +49,8 @@ import ( "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/storage/mock" - "github.com/ethereum/go-ethereum/swarm/storage/mru" "github.com/ethereum/go-ethereum/swarm/tracing" ) @@ -186,15 +186,20 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.fileStore = storage.NewFileStore(self.netStore, self.config.FileStoreParams) - var resourceHandler *mru.Handler - rhparams := &mru.HandlerParams{} + var feedsHandler *feed.Handler + fhParams := &feed.HandlerParams{} - resourceHandler = mru.NewHandler(rhparams) - resourceHandler.SetStore(self.netStore) + feedsHandler = feed.NewHandler(fhParams) + feedsHandler.SetStore(self.netStore) lstore.Validators = []storage.ChunkValidator{ storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)), - resourceHandler, + feedsHandler, + } + + err = lstore.Migrate() + if err != nil { + return nil, err } log.Debug("Setup local storage") @@ -210,7 +215,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } - self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) + self.api = api.NewAPI(self.fileStore, self.dns, feedsHandler, self.privateKey) self.sfs = fuse.NewSwarmFS(self.api) log.Debug("Initialized FUSE filesystem") diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 2309c39f0..cdf9239d0 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -25,7 +25,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/mru" + "github.com/ethereum/go-ethereum/swarm/storage/feed" ) type TestServer interface { @@ -48,14 +48,14 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso } fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams()) - // mutable resources test setup - resourceDir, err := ioutil.TempDir("", "swarm-resource-test") + // Swarm feeds test setup + feedsDir, err := ioutil.TempDir("", "swarm-feeds-test") if err != nil { t.Fatal(err) } - rhparams := &mru.HandlerParams{} - rh, err := mru.NewTestHandler(resourceDir, rhparams) + rhparams := &feed.HandlerParams{} + rh, err := feed.NewTestHandler(feedsDir, rhparams) if err != nil { t.Fatal(err) } @@ -71,11 +71,11 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, reso srv.Close() rh.Close() os.RemoveAll(dir) - os.RemoveAll(resourceDir) + os.RemoveAll(feedsDir) }, CurrentTime: 42, } - mru.TimestampProvider = tss + feed.TimestampProvider = tss return tss } @@ -92,6 +92,6 @@ func (t *TestSwarmServer) Close() { t.cleanup() } -func (t *TestSwarmServer) Now() mru.Timestamp { - return mru.Timestamp{Time: t.CurrentTime} +func (t *TestSwarmServer) Now() feed.Timestamp { + return feed.Timestamp{Time: t.CurrentTime} } |