diff options
author | Péter Szilágyi <peterke@gmail.com> | 2019-02-20 16:48:12 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-02-20 16:48:12 +0800 |
commit | c942700427557e3ff6de3aaf6b916e2f056c1ec2 (patch) | |
tree | cadf68e7206d6de42b1eefc6967214cf86e35ff2 /cmd | |
parent | 7fa3509e2eaf1a4ebc12344590e5699406690f15 (diff) | |
parent | cde35439e058b4f9579830fec9fb65ae0b998346 (diff) | |
download | go-tangerine-c942700427557e3ff6de3aaf6b916e2f056c1ec2.tar.gz go-tangerine-c942700427557e3ff6de3aaf6b916e2f056c1ec2.tar.zst go-tangerine-c942700427557e3ff6de3aaf6b916e2f056c1ec2.zip |
Merge pull request #19029 from holiman/update1.8v1.8.23
Update1.8
Diffstat (limited to 'cmd')
25 files changed, 1756 insertions, 403 deletions
diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ebaeba9f4..e60a27e43 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const ( @@ -121,6 +121,7 @@ var ( utils.DeveloperPeriodFlag, utils.TestnetFlag, utils.RinkebyFlag, + utils.GoerliFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.ConstantinopleOverrideFlag, @@ -164,7 +165,7 @@ var ( utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, } ) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 25a702dd7..6823aa36c 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) // AppHelpTemplate is the test template for the default, global app help topic. @@ -74,6 +74,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.NetworkIdFlag, utils.TestnetFlag, utils.RinkebyFlag, + utils.GoerliFlag, utils.SyncModeFlag, utils.GCModeFlag, utils.EthStatsURLFlag, @@ -229,7 +230,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, }, }, { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 8abfe7c41..6aed09f14 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -222,14 +222,18 @@ func (w *wizard) manageGenesis() { fmt.Println() fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) - + if w.conf.Genesis.Config.PetersburgBlock == nil { + w.conf.Genesis.Config.PetersburgBlock = w.conf.Genesis.Config.ConstantinopleBlock + } fmt.Println() - fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) - w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.PetersburgBlock) + w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.PetersburgBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) + w.conf.flush() + case "2": // Save whatever genesis configuration we currently have fmt.Println() diff --git a/cmd/swarm/bootnodes.go b/cmd/swarm/bootnodes.go index cbba9970d..ce3cd5288 100644 --- a/cmd/swarm/bootnodes.go +++ b/cmd/swarm/bootnodes.go @@ -17,61 +17,8 @@ package main var SwarmBootnodes = []string{ - // Foundation Swarm Gateway Cluster - "enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399", - "enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400", - "enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401", - "enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402", - "enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403", - "enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404", - "enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405", - "enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406", - "enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407", - "enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408", - "enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409", - "enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410", - "enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411", - "enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412", - "enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413", - "enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414", - "enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415", - "enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416", - "enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417", - "enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418", - "enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419", - "enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420", - "enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421", - "enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422", - "enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423", - "enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424", - "enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425", - "enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426", - "enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427", - "enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428", - "enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429", - "enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430", - "enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431", - "enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432", - "enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433", - "enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434", - "enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435", - "enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436", - "enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437", - "enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438", - "enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439", - "enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440", - "enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441", - "enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442", - "enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443", - "enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444", - "enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445", - "enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446", - - // Mainframe - "enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399", - "enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399", - "enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399", - "enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399", - "enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399", - "enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399", + // EF Swarm Bootnode - AWS - eu-central-1 + "enode://4c113504601930bf2000c29bcd98d1716b6167749f58bad703bae338332fe93cc9d9204f08afb44100dc7bea479205f5d162df579f9a8f76f8b402d339709023@3.122.203.99:30301", + // EF Swarm Bootnode - AWS - us-west-2 + "enode://89f2ede3371bff1ad9f2088f2012984e280287a4e2b68007c2a6ad994909c51886b4a8e9e2ecc97f9910aca538398e0a5804b0ee80a187fde1ba4f32626322ba@52.35.212.179:30301", } diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 3eea3057b..98d4dee7b 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -79,8 +79,10 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" + SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API" GETH_ENV_DATADIR = "GETH_DATADIR" ) @@ -164,10 +166,9 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config return config, err } -//override the current config with whatever is provided through the command line -//most values are not allowed a zero value (empty string), if not otherwise noted +// cmdLineOverride overrides the current config with whatever is provided through the command line +// most values are not allowed a zero value (empty string), if not otherwise noted func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config { - if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" { currentConfig.BzzAccount = keyid } @@ -258,14 +259,21 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity } + if ctx.GlobalIsSet(SwarmBootnodeModeFlag.Name) { + currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) + } + + if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) { + currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name) + } + return currentConfig } -//override the current config with whatver is provided in environment variables -//most values are not allowed a zero value (empty string), if not otherwise noted +// envVarsOverride overrides the current config with whatver is provided in environment variables +// most values are not allowed a zero value (empty string), if not otherwise noted func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { - if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" { currentConfig.BzzAccount = keyid } @@ -364,6 +372,18 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.Cors = cors } + if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" { + bootnodeMode, err := strconv.ParseBool(bm) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err) + } + currentConfig.BootnodeMode = bootnodeMode + } + + if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" { + currentConfig.GlobalStoreAPI = api + } + return currentConfig } diff --git a/cmd/swarm/explore.go b/cmd/swarm/explore.go new file mode 100644 index 000000000..5b5b8bf41 --- /dev/null +++ b/cmd/swarm/explore.go @@ -0,0 +1,59 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +// Command bzzhash computes a swarm tree hash. +package main + +import ( + "context" + "fmt" + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/swarm/storage" + "gopkg.in/urfave/cli.v1" +) + +var hashesCommand = cli.Command{ + Action: hashes, + CustomHelpTemplate: helpTemplate, + Name: "hashes", + Usage: "print all hashes of a file to STDOUT", + ArgsUsage: "<file>", + Description: "Prints all hashes of a file to STDOUT", +} + +func hashes(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 1 { + utils.Fatalf("Usage: swarm hashes <file name>") + } + f, err := os.Open(args[0]) + if err != nil { + utils.Fatalf("Error opening file " + args[1]) + } + defer f.Close() + + fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams()) + refs, err := fileStore.GetAllReferences(context.TODO(), f, false) + if err != nil { + utils.Fatalf("%v\n", err) + } else { + for _, r := range refs { + fmt.Println(r.String()) + } + } +} diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 12edc8cc9..b092a7747 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -156,6 +156,10 @@ var ( Name: "compressed", Usage: "Prints encryption keys in compressed form", } + SwarmBootnodeModeFlag = cli.BoolFlag{ + Name: "bootnode-mode", + Usage: "Run Swarm in Bootnode mode", + } SwarmFeedNameFlag = cli.StringFlag{ Name: "name", Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", @@ -172,4 +176,9 @@ var ( Name: "user", Usage: "Indicates the user who updates the feed", } + SwarmGlobalStoreAPIFlag = cli.StringFlag{ + Name: "globalstore-api", + Usage: "URL of the Global Store API provider (only for testing)", + EnvVar: SWARM_GLOBALSTORE_API, + } ) diff --git a/cmd/swarm/global-store/global_store.go b/cmd/swarm/global-store/global_store.go new file mode 100644 index 000000000..a55756e1c --- /dev/null +++ b/cmd/swarm/global-store/global_store.go @@ -0,0 +1,100 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "net" + "net/http" + "os" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" + cli "gopkg.in/urfave/cli.v1" +) + +// startHTTP starts a global store with HTTP RPC server. +// It is used for "http" cli command. +func startHTTP(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + log.Info("http", "address", listener.Addr().String()) + + return http.Serve(listener, server) +} + +// startWS starts a global store with WebSocket RPC server. +// It is used for "websocket" cli command. +func startWS(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + origins := ctx.StringSlice("origins") + log.Info("websocket", "address", listener.Addr().String(), "origins", origins) + + return http.Serve(listener, server.WebsocketHandler(origins)) +} + +// newServer creates a global store and returns its RPC server. +// Returned cleanup function should be called only if err is nil. +func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + + cleanup = func() {} + var globalStore mock.GlobalStorer + dir := ctx.String("dir") + if dir != "" { + dbStore, err := db.NewGlobalStore(dir) + if err != nil { + return nil, nil, err + } + cleanup = func() { + dbStore.Close() + } + globalStore = dbStore + log.Info("database global store", "dir", dir) + } else { + globalStore = mem.NewGlobalStore() + log.Info("in-memory global store") + } + + server = rpc.NewServer() + if err := server.RegisterName("mockStore", globalStore); err != nil { + cleanup() + return nil, nil, err + } + + return server, cleanup, nil +} diff --git a/cmd/swarm/global-store/global_store_test.go b/cmd/swarm/global-store/global_store_test.go new file mode 100644 index 000000000..85f361ed4 --- /dev/null +++ b/cmd/swarm/global-store/global_store_test.go @@ -0,0 +1,191 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" +) + +// TestHTTP_InMemory tests in-memory global store that exposes +// HTTP server. +func TestHTTP_InMemory(t *testing.T) { + testHTTP(t, true) +} + +// TestHTTP_Database tests global store with persisted database +// that exposes HTTP server. +func TestHTTP_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testHTTP(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testHTTP(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with HTTP server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testHTTP(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + client, err := rpc.DialHTTP("http://" + addr) + if err != nil { + t.Fatal(err) + } + + // wait until global store process is started as + // rpc.DialHTTP is actually not connecting + for i := 0; i < 1000; i++ { + _, err = http.DefaultClient.Get("http://" + addr) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// TestWebsocket_InMemory tests in-memory global store that exposes +// WebSocket server. +func TestWebsocket_InMemory(t *testing.T) { + testWebsocket(t, true) +} + +// TestWebsocket_Database tests global store with persisted database +// that exposes HTTP server. +func TestWebsocket_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testWebsocket(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testWebsocket(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with WebSocket server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testWebsocket(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + var client *rpc.Client + var err error + // wait until global store process is started + for i := 0; i < 1000; i++ { + client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// findFreeTCPAddress returns a local address (IP:Port) to which +// global store can listen on. +func findFreeTCPAddress(t *testing.T) (addr string) { + t.Helper() + + listener, err := net.Listen("tcp", "") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + return listener.Addr().String() +} diff --git a/cmd/swarm/global-store/main.go b/cmd/swarm/global-store/main.go new file mode 100644 index 000000000..51df0099a --- /dev/null +++ b/cmd/swarm/global-store/main.go @@ -0,0 +1,104 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Global Store. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Global Store") + + app.Name = "global-store" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 3, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "http", + Aliases: []string{"h"}, + Usage: "start swarm global store with http server", + Action: startHTTP, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for http connection", + }, + ), + }, + { + Name: "websocket", + Aliases: []string{"ws"}, + Usage: "start swarm global store with websocket server", + Action: startWS, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for websocket connection", + }, + cli.StringSliceFlag{ + Name: "origins", + Value: &cli.StringSlice{"*"}, + Usage: "websocket origins", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/global-store/run_test.go b/cmd/swarm/global-store/run_test.go new file mode 100644 index 000000000..d7ef626e5 --- /dev/null +++ b/cmd/swarm/global-store/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-global-store", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-global-store", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index ccbb24eec..3053ea1b3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,13 +39,16 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" "github.com/ethereum/go-ethereum/swarm/tracing" sv "github.com/ethereum/go-ethereum/swarm/version" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const clientIdentifier = "swarm" @@ -66,9 +69,10 @@ OPTIONS: {{end}}{{end}} ` -var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -) +// Git SHA1 commit hash of the release (set via linker flags) +// this variable will be assigned if corresponding parameter is passed with install, but not with test +// e.g.: go install -ldflags "-X main.gitCommit=ed1312d01b19e04ef578946226e5d8069d5dfd5a" ./cmd/swarm +var gitCommit string //declare a few constant error messages, useful for later error check comparisons in test var ( @@ -89,6 +93,7 @@ var defaultNodeConfig = node.DefaultConfig // This init function sets defaults so cmd/swarm can run alongside geth. func init() { + sv.GitCommit = gitCommit defaultNodeConfig.Name = clientIdentifier defaultNodeConfig.Version = sv.VersionWithCommit(gitCommit) defaultNodeConfig.P2P.ListenAddr = ":30399" @@ -140,6 +145,8 @@ func init() { dbCommand, // See config.go DumpConfigCommand, + // hashesCommand + hashesCommand, } // append a hidden help subcommand to all commands that have subcommands @@ -154,7 +161,6 @@ func init() { utils.BootnodesFlag, utils.KeyStoreDirFlag, utils.ListenPortFlag, - utils.NoDiscoverFlag, utils.DiscoveryV5Flag, utils.NetrestrictFlag, utils.NodeKeyFileFlag, @@ -187,10 +193,13 @@ func init() { SwarmUploadDefaultPath, SwarmUpFromStdinFlag, SwarmUploadMimeType, + // bootnode mode + SwarmBootnodeModeFlag, // storage flags SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, + SwarmGlobalStoreAPIFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, @@ -227,12 +236,17 @@ func main() { func keys(ctx *cli.Context) error { privateKey := getPrivKey(ctx) - pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey)) + pubkey := crypto.FromECDSAPub(&privateKey.PublicKey) + pubkeyhex := hex.EncodeToString(pubkey) pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + bzzkey := crypto.Keccak256Hash(pubkey).Hex() + if !ctx.Bool(SwarmCompressedFlag.Name) { - fmt.Println(fmt.Sprintf("publicKey=%s", pub)) + fmt.Println(fmt.Sprintf("bzzkey=%s", bzzkey[2:])) + fmt.Println(fmt.Sprintf("publicKey=%s", pubkeyhex)) } fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed)) + return nil } @@ -272,6 +286,10 @@ func bzzd(ctx *cli.Context) error { setSwarmBootstrapNodes(ctx, &cfg) //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) + + //always disable discovery from p2p package - swarm discovery is done with the `hive` protocol + cfg.P2P.NoDiscovery = true + stack, err := node.New(&cfg) if err != nil { utils.Fatalf("can't create node: %v", err) @@ -294,6 +312,15 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() + // add swarm bootnodes, because swarm doesn't use p2p package's discovery discv5 + go func() { + s := stack.Server() + + for _, n := range cfg.P2P.BootstrapNodes { + s.AddPeer(n) + } + }() + stack.Wait() return nil } @@ -301,8 +328,18 @@ func bzzd(ctx *cli.Context) error { func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) { //define the swarm service boot function boot := func(_ *node.ServiceContext) (node.Service, error) { - // In production, mockStore must be always nil. - return swarm.NewSwarm(bzzconfig, nil) + var nodeStore *mock.NodeStore + if bzzconfig.GlobalStoreAPI != "" { + // connect to global store + client, err := rpc.Dial(bzzconfig.GlobalStoreAPI) + if err != nil { + return nil, fmt.Errorf("global store: %v", err) + } + globalStore := mockrpc.NewGlobalStore(client) + // create a node store for this swarm key on global store + nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey)) + } + return swarm.NewSwarm(bzzconfig, nodeStore) } //register within the ethereum node if err := stack.Register(boot); err != nil { @@ -428,5 +465,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { } cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) } - log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) + } diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 680d238d0..4a6a56d9b 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -254,7 +254,6 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", @@ -330,7 +329,6 @@ func newTestNode(t *testing.T, dir string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 2c5e3fd23..6b3fed0c7 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -2,13 +2,10 @@ package main import ( "bytes" - "context" "crypto/md5" "fmt" "io" "io/ioutil" - "net/http" - "net/http/httptrace" "os" "os/exec" "strings" @@ -19,12 +16,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/ethereum/go-ethereum/swarm/testutil" - colorable "github.com/mattn/go-colorable" - opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -33,34 +26,28 @@ const ( feedRandomDataLength = 8 ) -func cliFeedUploadAndSync(c *cli.Context) error { - metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) - +func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error { errc := make(chan error) + go func() { - errc <- feedUploadAndSync(c) + errc <- feedUploadAndSync(ctx, tuid) }() select { case err := <-errc: if err != nil { - metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) } return err case <-time.After(time.Duration(timeout) * time.Second): - metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) } } -// TODO: retrieve with manifest + extract repeating code -func feedUploadAndSync(c *cli.Context) error { - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) - - generateEndpoints(scheme, cluster, appName, from, to) - - log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") +func feedUploadAndSync(c *cli.Context, tuid string) error { + log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing") // create a random private key to sign updates with and derive the address pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test") @@ -114,7 +101,7 @@ func feedUploadAndSync(c *cli.Context) error { // create feed manifest, topic only var out bytes.Buffer - cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex) + cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest topic cmd", "cmd", cmd) err = cmd.Run() @@ -129,7 +116,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // create feed manifest, subtopic only - cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex) + cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest subtopic cmd", "cmd", cmd) err = cmd.Run() @@ -144,7 +131,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // create feed manifest, merged topic - cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) + cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex) cmd.Stdout = &out log.Debug("create feed manifest mergetopic cmd", "cmd", cmd) err = cmd.Run() @@ -170,7 +157,7 @@ func feedUploadAndSync(c *cli.Context) error { dataHex := hexutil.Encode(data) // update with topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest topic cmd", "cmd", cmd) err = cmd.Run() @@ -181,7 +168,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update with subtopic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest subtopic cmd", "cmd", cmd) err = cmd.Run() @@ -192,7 +179,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update with merged topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex) cmd.Stdout = &out log.Debug("update feed manifest merged topic cmd", "cmd", cmd) err = cmd.Run() @@ -206,14 +193,14 @@ func feedUploadAndSync(c *cli.Context) error { // retrieve the data wg := sync.WaitGroup{} - for _, endpoint := range endpoints { + for _, host := range hosts { // raw retrieve, topic only for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} { wg.Add(1) ruid := uuid.New()[:8] go func(hex string, endpoint string, ruid string) { for { - err := fetchFeed(hex, userHex, endpoint, dataHash, ruid) + err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid) if err != nil { continue } @@ -221,20 +208,18 @@ func feedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(hex, endpoint, ruid) - + }(hex, httpEndpoint(host), ruid) } } wg.Wait() log.Info("all endpoints synced random data successfully") // upload test file - seed := int(time.Now().UnixNano() / 1e6) - log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) + log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed) randomBytes := testutil.RandomBytes(seed, filesize*1000) - hash, err := upload(&randomBytes, endpoints[0]) + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) if err != nil { return err } @@ -243,15 +228,12 @@ func feedUploadAndSync(c *cli.Context) error { return err } multihashHex := hexutil.Encode(hashBytes) - fileHash, err := digest(bytes.NewReader(randomBytes)) - if err != nil { - return err - } + fileHash := h.Sum(nil) log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash)) // update file with topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -261,7 +243,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update file with subtopic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -271,7 +253,7 @@ func feedUploadAndSync(c *cli.Context) error { out.Reset() // update file with merged topic - cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) + cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex) cmd.Stdout = &out err = cmd.Run() if err != nil { @@ -282,7 +264,7 @@ func feedUploadAndSync(c *cli.Context) error { time.Sleep(3 * time.Second) - for _, endpoint := range endpoints { + for _, host := range hosts { // manifest retrieve, topic only for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} { @@ -290,7 +272,7 @@ func feedUploadAndSync(c *cli.Context) error { ruid := uuid.New()[:8] go func(url string, endpoint string, ruid string) { for { - err := fetch(url, endpoint, fileHash, ruid) + err := fetch(url, endpoint, fileHash, ruid, "") if err != nil { continue } @@ -298,7 +280,7 @@ func feedUploadAndSync(c *cli.Context) error { wg.Done() return } - }(url, endpoint, ruid) + }(url, httpEndpoint(host), ruid) } } @@ -307,60 +289,3 @@ func feedUploadAndSync(c *cli.Context) error { return nil } - -func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { - ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") - defer sp.Finish() - - log.Trace("sleeping", "ruid", ruid) - time.Sleep(3 * time.Second) - - log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) - - var tn time.Time - reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user - req, _ := http.NewRequest("GET", reqUri, nil) - - opentracing.GlobalTracer().Inject( - sp.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) - - trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) - - req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) - transport := http.DefaultTransport - - //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - - tn = time.Now() - res, err := transport.RoundTrip(req) - if err != nil { - log.Error(err.Error(), "ruid", ruid) - return err - } - - log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) - - if res.StatusCode != 200 { - return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) - } - - defer res.Body.Close() - - rdigest, err := digest(res.Body) - if err != nil { - log.Warn(err.Error(), "ruid", ruid) - return err - } - - if !bytes.Equal(rdigest, original) { - err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) - - return nil -} diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 66cecdc5c..43d2c1ff5 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -37,18 +37,16 @@ var ( ) var ( - endpoints []string - includeLocalhost bool - cluster string - appName string - scheme string - filesize int - syncDelay int - from int - to int - verbosity int - timeout int - single bool + allhosts string + hosts []string + filesize int + syncDelay int + httpPort int + wsPort int + verbosity int + timeout int + single bool + trackTimeout int ) func main() { @@ -59,39 +57,22 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ - Name: "cluster-endpoint", - Value: "prod", - Usage: "cluster to point to (prod or a given namespace)", - Destination: &cluster, - }, - cli.StringFlag{ - Name: "app", - Value: "swarm", - Usage: "application to point to (swarm or swarm-private)", - Destination: &appName, + Name: "hosts", + Value: "", + Usage: "comma-separated list of swarm hosts", + Destination: &allhosts, }, cli.IntFlag{ - Name: "cluster-from", - Value: 8501, - Usage: "swarm node (from)", - Destination: &from, + Name: "http-port", + Value: 80, + Usage: "http port", + Destination: &httpPort, }, cli.IntFlag{ - Name: "cluster-to", - Value: 8512, - Usage: "swarm node (to)", - Destination: &to, - }, - cli.StringFlag{ - Name: "cluster-scheme", - Value: "http", - Usage: "http or https", - Destination: &scheme, - }, - cli.BoolFlag{ - Name: "include-localhost", - Usage: "whether to include localhost:8500 as an endpoint", - Destination: &includeLocalhost, + Name: "ws-port", + Value: 8546, + Usage: "ws port", + Destination: &wsPort, }, cli.IntFlag{ Name: "filesize", @@ -122,6 +103,12 @@ func main() { Usage: "whether to fetch content from a single node or from all nodes", Destination: &single, }, + cli.IntFlag{ + Name: "track-timeout", + Value: 5, + Usage: "timeout in seconds to wait for GetAllReferences to return", + Destination: &trackTimeout, + }, } app.Flags = append(app.Flags, []cli.Flag{ @@ -130,7 +117,7 @@ func main() { swarmmetrics.MetricsInfluxDBDatabaseFlag, swarmmetrics.MetricsInfluxDBUsernameFlag, swarmmetrics.MetricsInfluxDBPasswordFlag, - swarmmetrics.MetricsInfluxDBHostTagFlag, + swarmmetrics.MetricsInfluxDBTagsFlag, }...) app.Flags = append(app.Flags, tracing.Flags...) @@ -140,13 +127,25 @@ func main() { Name: "upload_and_sync", Aliases: []string{"c"}, Usage: "upload and sync", - Action: cliUploadAndSync, + Action: wrapCliCommand("upload-and-sync", uploadAndSyncCmd), }, { Name: "feed_sync", Aliases: []string{"f"}, Usage: "feed update generate, upload and sync", - Action: cliFeedUploadAndSync, + Action: wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd), + }, + { + Name: "upload_speed", + Aliases: []string{"u"}, + Usage: "measure upload speed", + Action: wrapCliCommand("upload-speed", uploadSpeedCmd), + }, + { + Name: "sliding_window", + Aliases: []string{"s"}, + Usage: "measure network aggregate capacity", + Action: wrapCliCommand("sliding-window", slidingWindowCmd), }, } @@ -177,13 +176,14 @@ func emitMetrics(ctx *cli.Context) error { database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + tags = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name) ) - return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ - "host": hosttag, - "version": gitCommit, - "filesize": fmt.Sprintf("%v", filesize), - }) + + tagsMap := utils.SplitTagsFlag(tags) + tagsMap["version"] = gitCommit + tagsMap["filesize"] = fmt.Sprintf("%v", filesize) + + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap) } return nil diff --git a/cmd/swarm/swarm-smoke/sliding_window.go b/cmd/swarm/swarm-smoke/sliding_window.go new file mode 100644 index 000000000..d313bbc37 --- /dev/null +++ b/cmd/swarm/swarm-smoke/sliding_window.go @@ -0,0 +1,131 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "bytes" + "fmt" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" + "github.com/pborman/uuid" + + cli "gopkg.in/urfave/cli.v1" +) + +type uploadResult struct { + hash string + digest []byte +} + +func slidingWindowCmd(ctx *cli.Context, tuid string) error { + errc := make(chan error) + + go func() { + errc <- slidingWindow(ctx, tuid) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func slidingWindow(ctx *cli.Context, tuid string) error { + hashes := []uploadResult{} //swarm hashes of the uploads + nodes := len(hosts) + const iterationTimeout = 30 * time.Second + log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout) + uploadedBytes := 0 + networkDepth := 0 + errored := false + +outer: + for { + log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed) + + t1 := time.Now() + + randomBytes := testutil.RandomBytes(seed, filesize*1000) + + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) + if err != nil { + log.Error(err.Error()) + return err + } + + metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1) + + fhash, err := digest(bytes.NewReader(randomBytes)) + if err != nil { + log.Error(err.Error()) + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay) + hashes = append(hashes, uploadResult{hash: hash, digest: fhash}) + time.Sleep(time.Duration(syncDelay) * time.Second) + uploadedBytes += filesize * 1000 + + for i, v := range hashes { + timeout := time.After(time.Duration(timeout) * time.Second) + errored = false + + inner: + for { + select { + case <-timeout: + errored = true + log.Error("error retrieving hash. timeout", "hash idx", i, "err", err) + metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1) + break inner + default: + idx := 1 + rand.Intn(len(hosts)-1) + ruid := uuid.New()[:8] + start := time.Now() + err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "") + if err != nil { + continue inner + } + metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start) + break inner + } + } + + if errored { + break outer + } + networkDepth = i + metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth)) + } + } + + log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize) + log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize) + + return nil +} diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index d605f79a3..90230df25 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -19,91 +19,122 @@ package main import ( "bytes" "context" - "crypto/md5" - crand "crypto/rand" - "errors" "fmt" - "io" "io/ioutil" "math/rand" - "net/http" - "net/http/httptrace" "os" "sync" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/api/client" - "github.com/ethereum/go-ethereum/swarm/spancontext" + "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" - opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -func generateEndpoints(scheme string, cluster string, app string, from int, to int) { - if cluster == "prod" { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) - } - } else { - for port := from; port < to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) - } - } - - if includeLocalhost { - endpoints = append(endpoints, "http://localhost:8500") - } -} - -func cliUploadAndSync(c *cli.Context) error { - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - - metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) +func uploadAndSyncCmd(ctx *cli.Context, tuid string) error { + randomBytes := testutil.RandomBytes(seed, filesize*1000) errc := make(chan error) + go func() { - errc <- uploadAndSync(c) + errc <- uplaodAndSync(ctx, randomBytes, tuid) }() select { case err := <-errc: if err != nil { - metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1) + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) } return err case <-time.After(time.Duration(timeout) * time.Second): - metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1) - return fmt.Errorf("timeout after %v sec", timeout) + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + e := fmt.Errorf("timeout after %v sec", timeout) + // trigger debug functionality on randomBytes + err := trackChunks(randomBytes[:]) + if err != nil { + e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err) + } + + return e } } -func uploadAndSync(c *cli.Context) error { - defer func(now time.Time) { - totalTime := time.Since(now) +func trackChunks(testData []byte) error { + log.Warn("Test timed out; running chunk debug sequence") - log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) - }(time.Now()) + addrs, err := getAllRefs(testData) + if err != nil { + return err + } + log.Trace("All references retrieved") - generateEndpoints(scheme, cluster, appName, from, to) - seed := int(time.Now().UnixNano() / 1e6) - log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) + // has-chunks + for _, host := range hosts { + httpHost := fmt.Sprintf("ws://%s:%d", host, 8546) + log.Trace("Calling `Has` on host", "httpHost", httpHost) + rpcClient, err := rpc.Dial(httpHost) + if err != nil { + log.Trace("Error dialing host", "err", err) + return err + } + log.Trace("rpc dial ok") + var hasInfo []api.HasInfo + err = rpcClient.Call(&hasInfo, "bzz_has", addrs) + if err != nil { + log.Trace("Error calling host", "err", err) + return err + } + log.Trace("rpc call ok") + count := 0 + for _, info := range hasInfo { + if !info.Has { + count++ + log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr) + } + } + if count == 0 { + log.Info("Host reported to have all chunks", "host", httpHost) + } + } + return nil +} - randomBytes := testutil.RandomBytes(seed, filesize*1000) +func getAllRefs(testData []byte) (storage.AddressCollection, error) { + log.Trace("Getting all references for given root hash") + datadir, err := ioutil.TempDir("", "chunk-debug") + if err != nil { + return nil, fmt.Errorf("unable to create temp dir: %v", err) + } + defer os.RemoveAll(datadir) + fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32)) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second) + defer cancel() + + reader := bytes.NewReader(testData) + return fileStore.GetAllReferences(ctx, reader, false) +} + +func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error { + log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed) t1 := time.Now() - hash, err := upload(&randomBytes, endpoints[0]) + hash, err := upload(randomBytes, httpEndpoint(hosts[0])) if err != nil { log.Error(err.Error()) return err } - metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) + t2 := time.Since(t1) + metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2) fhash, err := digest(bytes.NewReader(randomBytes)) if err != nil { @@ -111,147 +142,53 @@ func uploadAndSync(c *cli.Context) error { return err } - log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) + log.Info("uploaded successfully", "tuid", tuid, "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash)) time.Sleep(time.Duration(syncDelay) * time.Second) wg := sync.WaitGroup{} if single { - rand.Seed(time.Now().UTC().UnixNano()) - randIndex := 1 + rand.Intn(len(endpoints)-1) + randIndex := 1 + rand.Intn(len(hosts)-1) ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() - err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) + err := fetch(hash, endpoint, fhash, ruid, tuid) if err != nil { continue } + ended := time.Since(start) - metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended) + log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint) wg.Done() return } - }(endpoints[randIndex], ruid) + }(httpEndpoint(hosts[randIndex]), ruid) } else { - for _, endpoint := range endpoints { + for _, endpoint := range hosts[1:] { ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() - err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) + err := fetch(hash, endpoint, fhash, ruid, tuid) if err != nil { continue } + ended := time.Since(start) - metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).Update(ended) + log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint) wg.Done() return } - }(endpoint, ruid) + }(httpEndpoint(endpoint), ruid) } } wg.Wait() - log.Info("all endpoints synced random file successfully") + log.Info("all hosts synced random file successfully") return nil } - -// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file -func fetch(hash string, endpoint string, original []byte, ruid string) error { - ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") - defer sp.Finish() - - log.Trace("sleeping", "ruid", ruid) - time.Sleep(3 * time.Second) - log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - - var tn time.Time - reqUri := endpoint + "/bzz:/" + hash + "/" - req, _ := http.NewRequest("GET", reqUri, nil) - - opentracing.GlobalTracer().Inject( - sp.Context(), - opentracing.HTTPHeaders, - opentracing.HTTPHeadersCarrier(req.Header)) - - trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn) - - req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) - transport := http.DefaultTransport - - //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - - tn = time.Now() - res, err := transport.RoundTrip(req) - if err != nil { - log.Error(err.Error(), "ruid", ruid) - return err - } - log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) - - if res.StatusCode != 200 { - err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - defer res.Body.Close() - - rdigest, err := digest(res.Body) - if err != nil { - log.Warn(err.Error(), "ruid", ruid) - return err - } - - if !bytes.Equal(rdigest, original) { - err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) - log.Warn(err.Error(), "ruid", ruid) - return err - } - - log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) - - return nil -} - -// upload is uploading a file `f` to `endpoint` via the `swarm up` cmd -func upload(dataBytes *[]byte, endpoint string) (string, error) { - swarm := client.NewClient(endpoint) - f := &client.File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)), - ManifestEntry: api.ManifestEntry{ - ContentType: "text/plain", - Mode: 0660, - Size: int64(len(*dataBytes)), - }, - } - - // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. - return swarm.Upload(f, "", false) -} - -func digest(r io.Reader) ([]byte, error) { - h := md5.New() - _, err := io.Copy(h, r) - if err != nil { - return nil, err - } - return h.Sum(nil), nil -} - -// generates random data in heap buffer -func generateRandomData(datasize int) ([]byte, error) { - b := make([]byte, datasize) - c, err := crand.Read(b) - if err != nil { - return nil, err - } else if c != datasize { - return nil, errors.New("short read") - } - return b, nil -} diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go new file mode 100644 index 000000000..20bf7b86c --- /dev/null +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -0,0 +1,73 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "bytes" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" + + cli "gopkg.in/urfave/cli.v1" +) + +func uploadSpeedCmd(ctx *cli.Context, tuid string) error { + log.Info("uploading to "+hosts[0], "tuid", tuid, "seed", seed) + randomBytes := testutil.RandomBytes(seed, filesize*1000) + + errc := make(chan error) + + go func() { + errc <- uploadSpeed(ctx, tuid, randomBytes) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1) + + // trigger debug functionality on randomBytes + + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadSpeed(c *cli.Context, tuid string, data []byte) error { + t1 := time.Now() + hash, err := upload(data, hosts[0]) + if err != nil { + log.Error(err.Error()) + return err + } + metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1))) + + fhash, err := digest(bytes.NewReader(data)) + if err != nil { + log.Error(err.Error()) + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) + return nil +} diff --git a/cmd/swarm/swarm-smoke/util.go b/cmd/swarm/swarm-smoke/util.go new file mode 100644 index 000000000..87abb44b0 --- /dev/null +++ b/cmd/swarm/swarm-smoke/util.go @@ -0,0 +1,235 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "bytes" + "context" + "crypto/md5" + crand "crypto/rand" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptrace" + "os" + "strings" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" + opentracing "github.com/opentracing/opentracing-go" + "github.com/pborman/uuid" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + commandName = "" + seed = int(time.Now().UTC().UnixNano()) +) + +func init() { + rand.Seed(int64(seed)) +} + +func httpEndpoint(host string) string { + return fmt.Sprintf("http://%s:%d", host, httpPort) +} + +func wsEndpoint(host string) string { + return fmt.Sprintf("ws://%s:%d", host, wsPort) +} + +func wrapCliCommand(name string, command func(*cli.Context, string) error) func(*cli.Context) error { + return func(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + + // test uuid + tuid := uuid.New()[:8] + + commandName = name + + hosts = strings.Split(allhosts, ",") + + defer func(now time.Time) { + totalTime := time.Since(now) + log.Info("total time", "tuid", tuid, "time", totalTime, "kb", filesize) + metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime) + }(time.Now()) + + log.Info("smoke test starting", "tuid", tuid, "task", name, "timeout", timeout) + metrics.GetOrRegisterCounter(name, nil).Inc(1) + + return command(ctx, tuid) + } +} + +func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") + defer sp.Finish() + + log.Trace("sleeping", "ruid", ruid) + time.Sleep(3 * time.Second) + + log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) + + var tn time.Time + reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) + if err != nil { + log.Error(err.Error(), "ruid", ruid) + return err + } + + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid) + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} + +// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file +func fetch(hash string, endpoint string, original []byte, ruid string, tuid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") + defer sp.Finish() + + log.Info("http get request", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash) + + var tn time.Time + reqUri := endpoint + "/bzz:/" + hash + "/" + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) + if err != nil { + log.Error(err.Error(), "ruid", ruid) + return err + } + log.Info("http get response", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) + + if res.StatusCode != 200 { + err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + defer res.Body.Close() + + rdigest, err := digest(res.Body) + if err != nil { + log.Warn(err.Error(), "ruid", ruid) + return err + } + + if !bytes.Equal(rdigest, original) { + err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original) + log.Warn(err.Error(), "ruid", ruid) + return err + } + + log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength) + + return nil +} + +// upload an arbitrary byte as a plaintext file to `endpoint` using the api client +func upload(data []byte, endpoint string) (string, error) { + swarm := client.NewClient(endpoint) + f := &client.File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(len(data)), + }, + } + + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + return swarm.Upload(f, "", false) +} + +func digest(r io.Reader) ([]byte, error) { + h := md5.New() + _, err := io.Copy(h, r) + if err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +// generates random data in heap buffer +func generateRandomData(datasize int) ([]byte, error) { + b := make([]byte, datasize) + c, err := crand.Read(b) + if err != nil { + return nil, err + } else if c != datasize { + return nil, errors.New("short read") + } + return b, nil +} diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go new file mode 100644 index 000000000..127fde8ae --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create.go @@ -0,0 +1,157 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + cli "gopkg.in/urfave/cli.v1" +) + +// create is used as the entry function for "create" app command. +func create(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + if len(ctx.Args()) < 1 { + return errors.New("argument should be the filename to verify or write-to") + } + filename, err := touchPath(ctx.Args()[0]) + if err != nil { + return err + } + return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ",")) +} + +// createSnapshot creates a new snapshot on filesystem with provided filename, +// number of nodes and service names. +func createSnapshot(filename string, nodes int, services []string) (err error) { + log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services) + + sim := simulation.New(map[string]simulation.ServiceFunc{ + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + addr := network.NewAddr(ctx.Config.Node()) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + hp := network.NewHiveParams() + hp.KeepAliveInterval = time.Duration(200) * time.Millisecond + hp.Discovery = true // discovery must be enabled when creating a snapshot + + config := &network.BzzConfig{ + OverlayAddr: addr.Over(), + UnderlayAddr: addr.Under(), + HiveParams: hp, + } + return network.NewBzz(config, kad, nil, nil, nil), nil, nil + }, + }) + defer sim.Close() + + _, err = sim.AddNodes(nodes) + if err != nil { + return fmt.Errorf("add nodes: %v", err) + } + + err = sim.Net.ConnectNodesRing(nil) + if err != nil { + return fmt.Errorf("connect nodes: %v", err) + } + + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + if _, err := sim.WaitTillHealthy(ctx); err != nil { + return fmt.Errorf("wait for healthy kademlia: %v", err) + } + + var snap *simulations.Snapshot + if len(services) > 0 { + // If service names are provided, include them in the snapshot. + // But, check if "bzz" service is not among them to remove it + // form the snapshot as it exists on snapshot creation. + var removeServices []string + var wantBzz bool + for _, s := range services { + if s == "bzz" { + wantBzz = true + break + } + } + if !wantBzz { + removeServices = []string{"bzz"} + } + snap, err = sim.Net.SnapshotWithServices(services, removeServices) + } else { + snap, err = sim.Net.Snapshot() + } + if err != nil { + return fmt.Errorf("create snapshot: %v", err) + } + jsonsnapshot, err := json.Marshal(snap) + if err != nil { + return fmt.Errorf("json encode snapshot: %v", err) + } + return ioutil.WriteFile(filename, jsonsnapshot, 0666) +} + +// touchPath creates an empty file and all subdirectories +// that are missing. +func touchPath(filename string) (string, error) { + if path.IsAbs(filename) { + if _, err := os.Stat(filename); err == nil { + // path exists, overwrite + return filename, nil + } + } + + d, f := path.Split(filename) + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return "", err + } + + _, err = os.Stat(path.Join(dir, filename)) + if err == nil { + // path exists, overwrite + return filename, nil + } + + dirPath := path.Join(dir, d) + filePath := path.Join(dirPath, f) + if d != "" { + err = os.MkdirAll(dirPath, os.ModeDir) + if err != nil { + return "", err + } + } + + return filePath, nil +} diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go new file mode 100644 index 000000000..c9445168d --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -0,0 +1,143 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "runtime" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/p2p/simulations" +) + +// TestSnapshotCreate is a high level e2e test that tests for snapshot generation. +// It runs a few "create" commands with different flag values and loads generated +// snapshot files to validate their content. +func TestSnapshotCreate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + for _, v := range []struct { + name string + nodes int + services string + }{ + { + name: "defaults", + }, + { + name: "more nodes", + nodes: defaultNodes + 5, + }, + { + name: "services", + services: "stream,pss,zorglub", + }, + { + name: "services with bzz", + services: "bzz,pss", + }, + } { + t.Run(v.name, func(t *testing.T) { + t.Parallel() + + file, err := ioutil.TempFile("", "swarm-snapshot") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + if err = file.Close(); err != nil { + t.Error(err) + } + + args := []string{"create"} + if v.nodes > 0 { + args = append(args, "--nodes", strconv.Itoa(v.nodes)) + } + if v.services != "" { + args = append(args, "--services", v.services) + } + testCmd := runSnapshot(t, append(args, file.Name())...) + + testCmd.ExpectExit() + if code := testCmd.ExitStatus(); code != 0 { + t.Fatalf("command exit code %v, expected 0", code) + } + + f, err := os.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := f.Close() + if err != nil { + t.Error("closing snapshot file", "err", err) + } + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(b, &snap) + if err != nil { + t.Fatal(err) + } + + wantNodes := v.nodes + if wantNodes == 0 { + wantNodes = defaultNodes + } + gotNodes := len(snap.Nodes) + if gotNodes != wantNodes { + t.Errorf("got %v nodes, want %v", gotNodes, wantNodes) + } + + if len(snap.Conns) == 0 { + t.Error("no connections in a snapshot") + } + + var wantServices []string + if v.services != "" { + wantServices = strings.Split(v.services, ",") + } else { + wantServices = []string{"bzz"} + } + // sort service names so they can be comparable + // as strings to every node sorted services + sort.Strings(wantServices) + + for i, n := range snap.Nodes { + gotServices := n.Node.Config.Services + sort.Strings(gotServices) + if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) { + t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices) + } + } + + }) + } +} diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go new file mode 100644 index 000000000..184727e4d --- /dev/null +++ b/cmd/swarm/swarm-snapshot/main.go @@ -0,0 +1,82 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +// default value for "create" command --nodes flag +const defaultNodes = 10 + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Snapshot Utility. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Snapshot Utility") + + app.Name = "swarm-snapshot" + app.Usage = "" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "create", + Aliases: []string{"c"}, + Usage: "create a swarm snapshot", + Action: create, + // Flags only for "create" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.IntFlag{ + Name: "nodes", + Value: defaultNodes, + Usage: "number of nodes", + }, + cli.StringFlag{ + Name: "services", + Value: "bzz", + Usage: "comma separated list of services to boot the nodes with", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/swarm-snapshot/run_test.go b/cmd/swarm/swarm-snapshot/run_test.go new file mode 100644 index 000000000..d9a041597 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-snapshot", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-snapshot", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 60e45d095..55e84b876 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -57,7 +57,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -140,6 +140,10 @@ var ( Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", } + GoerliFlag = cli.BoolFlag{ + Name: "goerli", + Usage: "Görli network: pre-configured proof-of-authority test network", + } ConstantinopleOverrideFlag = cli.Uint64Flag{ Name: "override.constantinople", Usage: "Manually specify constantinople fork-block, overriding the bundled setting", @@ -614,14 +618,14 @@ var ( Usage: "Password to authorize access to the database", Value: "test", } - // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. - // It is used so that we can group all nodes and average a measurement across all of them, but also so - // that we can select a specific node and inspect its measurements. + // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // For example `host` tag could be used so that we can group all nodes and average a measurement + // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBHostTagFlag = cli.StringFlag{ - Name: "metrics.influxdb.host.tag", - Usage: "InfluxDB `host` tag attached to all measurements", - Value: "localhost", + MetricsInfluxDBTagsFlag = cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: "host=localhost", } EWASMInterpreterFlag = cli.StringFlag{ @@ -647,6 +651,9 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(RinkebyFlag.Name) { return filepath.Join(path, "rinkeby") } + if ctx.GlobalBool(GoerliFlag.Name) { + return filepath.Join(path, "goerli") + } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -701,6 +708,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.TestnetBootnodes case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes + case ctx.GlobalBool(GoerliFlag.Name): + urls = params.GoerliBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -728,6 +737,8 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { } case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes + case ctx.GlobalBool(GoerliFlag.Name): + urls = params.GoerliBootnodes case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -836,10 +847,11 @@ func makeDatabaseHandles() int { if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } - if err := fdlimit.Raise(uint64(limit)); err != nil { + raised, err := fdlimit.Raise(uint64(limit)) + if err != nil { Fatalf("Failed to raise file descriptor allowance: %v", err) } - return limit / 2 // Leave half for networking and other stuff + return int(raised / 2) // Leave half for networking and other stuff } // MakeAddress converts an account specified directly as a hex encoded string or @@ -980,7 +992,6 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setHTTP(ctx, cfg) setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) - setDataDir(ctx, cfg) if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { @@ -1004,6 +1015,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet") case ctx.GlobalBool(RinkebyFlag.Name): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") + case ctx.GlobalBool(GoerliFlag.Name): + cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") } } @@ -1160,7 +1173,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags - checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag) + checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag, GoerliFlag) checkExclusive(ctx, LightServFlag, SyncModeFlag, "light") ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -1256,6 +1269,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = 4 } cfg.Genesis = core.DefaultRinkebyGenesisBlock() + case ctx.GlobalBool(GoerliFlag.Name): + if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + cfg.NetworkId = 5 + } + cfg.Genesis = core.DefaultGoerliGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1360,18 +1378,35 @@ func SetupMetrics(ctx *cli.Context) { database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) if enableExport { + tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + log.Info("Enabling metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", map[string]string{ - "host": hosttag, - }) + + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) } } } +func SplitTagsFlag(tagsFlag string) map[string]string { + tags := strings.Split(tagsFlag, ",") + tagsMap := map[string]string{} + + for _, t := range tags { + if t != "" { + kv := strings.Split(t, "=") + + if len(kv) == 2 { + tagsMap[kv[0]] = kv[1] + } + } + } + + return tagsMap +} + // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { var ( @@ -1396,6 +1431,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultTestnetGenesisBlock() case ctx.GlobalBool(RinkebyFlag.Name): genesis = core.DefaultRinkebyGenesisBlock() + case ctx.GlobalBool(GoerliFlag.Name): + genesis = core.DefaultGoerliGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/cmd/utils/flags_test.go b/cmd/utils/flags_test.go new file mode 100644 index 000000000..adfdd0903 --- /dev/null +++ b/cmd/utils/flags_test.go @@ -0,0 +1,64 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. + +// Package utils contains internal helper functions for go-ethereum commands. +package utils + +import ( + "reflect" + "testing" +) + +func Test_SplitTagsFlag(t *testing.T) { + tests := []struct { + name string + args string + want map[string]string + }{ + { + "2 tags case", + "host=localhost,bzzkey=123", + map[string]string{ + "host": "localhost", + "bzzkey": "123", + }, + }, + { + "1 tag case", + "host=localhost123", + map[string]string{ + "host": "localhost123", + }, + }, + { + "empty case", + "", + map[string]string{}, + }, + { + "garbage", + "smth=smthelse=123", + map[string]string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SplitTagsFlag(tt.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitTagsFlag() = %v, want %v", got, tt.want) + } + }) + } +} |