diff options
| -rw-r--r-- | cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 323 | ||||
| -rw-r--r-- | cmd/swarm/swarm-smoke/main.go | 16 | ||||
| -rw-r--r-- | cmd/swarm/swarm-smoke/upload_and_sync.go | 18 |
3 files changed, 351 insertions, 6 deletions
diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go new file mode 100644 index 000000000..c7a1475d6 --- /dev/null +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -0,0 +1,323 @@ +package main + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/pborman/uuid" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/multihash" + "github.com/ethereum/go-ethereum/swarm/storage/feed" + + colorable "github.com/mattn/go-colorable" + + cli "gopkg.in/urfave/cli.v1" +) + +const ( + feedRandomDataLength = 8 +) + +// TODO: retrieve with manifest + extract repeating code +func cliFeedUploadAndSync(c *cli.Context) error { + + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) + + generateEndpoints(scheme, cluster, from, to) + + log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing") + + // create a random private key to sign updates with and derive the address + pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") + if err != nil { + return err + } + defer pkFile.Close() + defer os.Remove(pkFile.Name()) + + privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976" + privKey, err := crypto.HexToECDSA(privkeyHex) + if err != nil { + return err + } + user := crypto.PubkeyToAddress(privKey.PublicKey) + userHex := hexutil.Encode(user.Bytes()) + + // save the private key to a file + _, err = io.WriteString(pkFile, privkeyHex) + if err != nil { + return err + } + + // keep hex strings for topic and subtopic + var topicHex string + var subTopicHex string + + // and create combination hex topics for bzz-feed retrieval + // xor'ed with topic (zero-value topic if no topic) + var subTopicOnlyHex string + var mergedSubTopicHex string + + // generate random topic and subtopic and put a hex on them + topicBytes, err := generateRandomData(feed.TopicLength) + topicHex = hexutil.Encode(topicBytes) + subTopicBytes, err := generateRandomData(8) + subTopicHex = hexutil.Encode(subTopicBytes) + if err != nil { + return err + } + mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes) + if err != nil { + return err + } + mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:]) + subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil) + if err != nil { + return err + } + subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:]) + + // create feed manifest, topic only + var out bytes.Buffer + cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic) + } + log.Debug("create topic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, subtopic only + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithSubTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic) + } + log.Debug("create subtopic feed", "manifest", manifestWithTopic) + out.Reset() + + // create feed manifest, merged topic + cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) + cmd.Stdout = &out + log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + log.Error(err.Error()) + return err + } + manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a})) + if len(manifestWithMergedTopic) != 64 { + return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic) + } + log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic) + out.Reset() + + // create test data + data, err := generateRandomData(feedRandomDataLength) + if err != nil { + return err + } + h := md5.New() + h.Write(data) + dataHash := h.Sum(nil) + dataHex := hexutil.Encode(data) + + // update with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest subtopic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) + cmd.Stdout = &out + log.Debug("update feed manifest merged topic cmd", "cmd", cmd) + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + // retrieve the data + wg := sync.WaitGroup{} + for _, endpoint := range endpoints { + // raw retrieve, topic only + for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(endpoint string, ruid string) { + for { + err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(endpoint, ruid) + + } + } + wg.Wait() + log.Info("all endpoints synced random data successfully") + + // upload test file + log.Info("uploading to " + endpoints[0] + " and syncing") + + f, cleanup := generateRandomFile(filesize * 1000) + defer cleanup() + + hash, err := upload(f, endpoints[0]) + if err != nil { + return err + } + hashBytes, err := hexutil.Decode("0x" + hash) + if err != nil { + return err + } + multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes)) + + fileHash, err := digest(f) + if err != nil { + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) + + // update file with topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update topic", "out", out) + out.Reset() + + // update file with subtopic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update subtopic", "out", out) + out.Reset() + + // update file with merged topic + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) + cmd.Stdout = &out + err = cmd.Run() + if err != nil { + return err + } + log.Debug("feed update mergedtopic", "out", out) + out.Reset() + + time.Sleep(3 * time.Second) + + for _, endpoint := range endpoints { + + // manifest retrieve, topic only + for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { + wg.Add(1) + ruid := uuid.New()[:8] + go func(endpoint string, ruid string) { + for { + err := fetch(url, endpoint, fileHash, ruid) + if err != nil { + continue + } + + wg.Done() + return + } + }(endpoint, ruid) + } + + } + wg.Wait() + log.Info("all endpoints synced random file successfully") + + return nil +} + +func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + log.Trace("sleeping", "ruid", ruid) + time.Sleep(3 * time.Second) + + log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) + res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) + if err != nil { + return err + } + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 70aee1922..97054dd47 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -21,7 +21,6 @@ import ( "sort" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" cli "gopkg.in/urfave/cli.v1" ) @@ -34,11 +33,10 @@ var ( filesize int from int to int + verbosity int ) func main() { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) app := cli.NewApp() app.Name = "smoke-test" @@ -80,6 +78,12 @@ func main() { Usage: "file size for generated random file in KB", Destination: &filesize, }, + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity", + Destination: &verbosity, + }, } app.Commands = []cli.Command{ @@ -89,6 +93,12 @@ func main() { Usage: "upload and sync", Action: cliUploadAndSync, }, + { + Name: "feed_sync", + Aliases: []string{"f"}, + Usage: "feed update generate, upload and sync", + Action: cliFeedUploadAndSync, + }, } sort.Sort(cli.FlagsByName(app.Flags)) diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 5e0ff4b0f..358141c7c 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -19,7 +19,8 @@ package main import ( "bytes" "crypto/md5" - "crypto/rand" + crand "crypto/rand" + "errors" "fmt" "io" "io/ioutil" @@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error { wg := sync.WaitGroup{} for _, endpoint := range endpoints { - endpoint := endpoint ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { @@ -166,6 +166,18 @@ func digest(r io.Reader) ([]byte, error) { return h.Sum(nil), nil } +// generates random data in heap buffer +func generateRandomData(datasize int) ([]byte, error) { + b := make([]byte, datasize) + c, err := crand.Read(b) + if err != nil { + return nil, err + } else if c != datasize { + return nil, errors.New("short read") + } + return b, nil +} + // generateRandomFile is creating a temporary file with the requested byte size func generateRandomFile(size int) (f *os.File, teardown func()) { // create a tmp file @@ -181,7 +193,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) { } buf := make([]byte, size) - _, err = rand.Read(buf) + _, err = crand.Read(buf) if err != nil { panic(err) } |