diff options
26 files changed, 402 insertions, 116 deletions
diff --git a/Changelog.md b/Changelog.md index b5e48173..1058fdf5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,7 @@ ### 0.4.7 (unreleased) Features: + * Code generator: Inject the Swarm hash of a metadata file into the bytecode. * Optimizer: Some dead code elimination. Bugfixes: diff --git a/docs/miscellaneous.rst b/docs/miscellaneous.rst index 9f733979..1754752d 100644 --- a/docs/miscellaneous.rst +++ b/docs/miscellaneous.rst @@ -232,6 +232,142 @@ Either add ``--libraries "Math:0x12345678901234567890 Heap:0xabcdef0123456"`` to If ``solc`` is called with the option ``--link``, all input files are interpreted to be unlinked binaries (hex-encoded) in the ``__LibraryName____``-format given above and are linked in-place (if the input is read from stdin, it is written to stdout). All options except ``--libraries`` are ignored (including ``-o``) in this case. +***************** +Contract Metadata +***************** + +The Solidity compiler automatically generates a JSON file, the +contract metadata, that contains information about the current contract. +It can be used to query the compiler version, the sources used, the ABI +and NatSpec documentation in order to more safely interact with the contract +and to verify its source code. + +The compiler appends a Swarm hash of the metadata file to the end of the +bytecode (for details, see below) of each contract, so that you can retrieve +the file in an authenticated way without having to resort to a centralized +data provider. + +Of course, you have to publish the metadata file to Swarm (or some other service) +so that others can access it. The file can be output by using ``solc --metadata`` +and the file will be called ``ContractName_meta.json``. +It will contain Swarm references to the source code, so you have to upload +all source files and the metadata file. + +The metadata file has the following format. The example below is presented in a +human-readable way. Properly formatted metadata should use quotes correctly, +reduce whitespace to a minimum and sort the keys of all objects to arrive at a +unique formatting. +Comments are of course also not permitted and used here only for explanatory purposes. + +.. code-block:: none + + { + // Required: The version of the metadata format + version: "1", + // Required: Source code language, basically selects a "sub-version" + // of the specification + language: "Solidity", + // Required: Details about the compiler, contents are specific + // to the language. + compiler: { + // Required for Solidity: Version of the compiler + version: "0.4.6+commit.2dabbdf0.Emscripten.clang", + // Optional: Hash of the compiler binary which produced this output + keccak256: "0x123..." + }, + // Required: Compilation source files/source units, keys are file names + sources: + { + "myFile.sol": { + // Required: keccak256 hash of the source file + "keccak256": "0x123...", + // Required (unless "content" is used, see below): URL to the + // source file, protocol is more or less arbitrary, but a Swarm + // URL is recommended + "url": "bzzr://56ab..." + }, + "mortal": { + // Required: keccak256 hash of the source file + "keccak256": "0x234...", + // Required (unless "url" is used): literal contents of the source file + "content": "contract mortal is owned { function kill() { if (msg.sender == owner) selfdestruct(owner); } }" + } + }, + // Required: Compiler settings + settings: + { + // Required for Solidity: Sorted list of remappings + remappings: [ ":g/dir" ], + // Optional: Optimizer settings (enabled defaults to false) + optimizer: { + enabled: true, + runs: 500 + }, + // Required for Solidity: File and name of the contract or library this + // metadata is created for. + compilationTarget: { + "myFile.sol": "MyContract" + }, + // Required for Solidity: Addresses for libraries used + libraries: { + "MyLib": "0x123123..." + } + }, + // Required: Generated information about the contract. + output: + { + // Required: ABI definition of the contract + abi: [ ... ], + // Required: NatSpec user documentation of the contract + userdoc: [ ... ], + // Required: NatSpec developer documentation of the contract + devdoc: [ ... ], + } + } + + +Encoding of the Metadata Hash in the Bytecode +============================================= + +Because we might support other ways to retrieve the metadata file in the future, +the mapping ``{"bzzr0": <Swarm hash>}`` is stored +[CBOR](https://tools.ietf.org/html/rfc7049)-encoded. Since the beginning of that +encoding is not easy to find, its length is added in a two-byte big-endian +encoding. The current version of the Solidity compiler thus adds the following +to the end of the deployed bytecode:: + + 0xa1 0x65 'b' 'z' 'z' 'r' '0' 0x58 0x20 <32 bytes swarm hash> 0x00 0x29 + +So in order to retrieve the data, the end of the deployed bytecode can be checked +to match that pattern and use the Swarm hash to retrieve the file. + +Usage for Automatic Interface Generation and NatSpec +==================================================== + +The metadata is used in the following way: A component that wants to interact +with a contract (e.g. Mist) retrieves the code of the contract, from that +the Swarm hash of a file which is then retrieved. +That file is JSON-decoded into a structure like above. + +The component can then use the ABI to automatically generate a rudimentary +user interface for the contract. + +Furthermore, Mist can use the userdoc to display a confirmation message to the user +whenever they interact with the contract. + +Usage for Source Code Verification +================================== + +In order to verify the compilation, sources can be retrieved from Swarm +via the link in the metadata file. +The compiler of the correct version (which is checked to be part of the "official" compilers) +is invoked on that input with the specified settings. The resulting +bytecode is compared to the data of the creation transaction or CREATE opcode data. +This automatically verifies the metadata since its hash is part of the bytecode. +Excess data corresponds to the constructor input data, which should be decoded +according to the interface and presented to the user. + + *************** Tips and Tricks *************** diff --git a/libdevcore/JSON.h b/libdevcore/JSON.h index 7876dfb2..0d6e0d2e 100644 --- a/libdevcore/JSON.h +++ b/libdevcore/JSON.h @@ -28,13 +28,13 @@ namespace dev { /// Serialise the JSON object (@a _input) with identation -std::string jsonPrettyPrint(Json::Value const& _input) +inline std::string jsonPrettyPrint(Json::Value const& _input) { return Json::StyledWriter().write(_input); } /// Serialise theJ SON object (@a _input) without identation -std::string jsonCompactPrint(Json::Value const& _input) +inline std::string jsonCompactPrint(Json::Value const& _input) { Json::FastWriter writer; writer.omitEndingLineFeed(); diff --git a/libdevcore/SwarmHash.cpp b/libdevcore/SwarmHash.cpp index e7b844eb..aa98eafd 100644 --- a/libdevcore/SwarmHash.cpp +++ b/libdevcore/SwarmHash.cpp @@ -38,13 +38,14 @@ h256 swarmHashSimple(bytesConstRef _data, size_t _size) return keccak256(toLittleEndian(_size) + _data.toBytes()); } -h256 swarmHashIntermediate(bytes const& _input, size_t _offset, size_t _length) +h256 swarmHashIntermediate(string const& _input, size_t _offset, size_t _length) { + bytesConstRef ref; + bytes innerNodes; if (_length <= 0x1000) - return swarmHashSimple(bytesConstRef(_input.data() + _offset, _length), _length); + ref = bytesConstRef(_input).cropped(_offset, _length); else { - bytes innerNodes; size_t maxRepresentedSize = 0x1000; while (maxRepresentedSize * (0x1000 / 32) < _length) maxRepresentedSize *= (0x1000 / 32); @@ -53,11 +54,12 @@ h256 swarmHashIntermediate(bytes const& _input, size_t _offset, size_t _length) size_t size = std::min(maxRepresentedSize, _length - i); innerNodes += swarmHashIntermediate(_input, _offset + i, size).asBytes(); } - return swarmHashSimple(bytesConstRef(&innerNodes), _length); + ref = bytesConstRef(&innerNodes); } + return swarmHashSimple(ref, _length); } -h256 dev::swarmHash(bytes const& _input) +h256 dev::swarmHash(string const& _input) { return swarmHashIntermediate(_input, 0, _input.size()); } diff --git a/libdevcore/SwarmHash.h b/libdevcore/SwarmHash.h index 925509ab..f474ce11 100644 --- a/libdevcore/SwarmHash.h +++ b/libdevcore/SwarmHash.h @@ -20,12 +20,13 @@ #pragma once #include <libdevcore/FixedHash.h> -#include <libdevcore/Common.h> + +#include <string> namespace dev { /// Compute the "swarm hash" of @a _data -h256 swarmHash(bytes const& _data); +h256 swarmHash(std::string const& _data); } diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index b040917d..a9ca24dc 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -432,7 +432,7 @@ LinkerObject const& Assembly::assemble() const unsigned bytesPerTag = dev::bytesRequired(bytesRequiredForCode); byte tagPush = (byte)Instruction::PUSH1 - 1 + bytesPerTag; - unsigned bytesRequiredIncludingData = bytesRequiredForCode + 1; + unsigned bytesRequiredIncludingData = bytesRequiredForCode + 1 + m_auxiliaryData.size(); for (auto const& sub: m_subs) bytesRequiredIncludingData += sub->assemble().bytecode.size(); @@ -525,8 +525,10 @@ LinkerObject const& Assembly::assemble() const } } - if (!dataRef.empty() && !subRef.empty()) + if (!m_subs.empty() || !m_data.empty() || !m_auxiliaryData.empty()) + // Append a STOP just to be sure. ret.bytecode.push_back(0); + for (size_t i = 0; i < m_subs.size(); ++i) { auto references = subRef.equal_range(i); @@ -568,6 +570,9 @@ LinkerObject const& Assembly::assemble() const } ret.bytecode += dataItem.second; } + + ret.bytecode += m_auxiliaryData; + for (unsigned pos: sizeRef) { bytesRef r(ret.bytecode.data() + pos, bytesPerDataRef); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index a5fd4d51..9e7f9f7b 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -71,6 +71,9 @@ public: AssemblyItem appendJumpI(AssemblyItem const& _tag) { auto ret = append(_tag.pushTag()); append(solidity::Instruction::JUMPI); return ret; } AssemblyItem errorTag() { return AssemblyItem(PushTag, 0); } + /// Appends @a _data literally to the very end of the bytecode. + void appendAuxiliaryDataToEnd(bytes const& _data) { m_auxiliaryData += _data; } + template <class T> Assembly& operator<<(T const& _d) { append(_d); return *this; } AssemblyItems const& items() const { return m_items; } AssemblyItem const& back() const { return m_items.back(); } @@ -125,10 +128,12 @@ private: Json::Value createJsonValue(std::string _name, int _begin, int _end, std::string _value = std::string(), std::string _jumpType = std::string()) const; protected: - // 0 is reserved for exception + /// 0 is reserved for exception unsigned m_usedTags = 1; AssemblyItems m_items; std::map<h256, bytes> m_data; + /// Data that is appended to the very end of the contract. + bytes m_auxiliaryData; std::vector<std::shared_ptr<Assembly>> m_subs; std::map<h256, std::string> m_strings; std::map<h256, std::string> m_libraries; ///< Identifiers of libraries to be linked. diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index 9b478393..3cd1dfbe 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -63,6 +63,15 @@ SourceUnitAnnotation& SourceUnit::annotation() const return static_cast<SourceUnitAnnotation&>(*m_annotation); } +string Declaration::sourceUnitName() const +{ + solAssert(!!m_scope, ""); + ASTNode const* scope = m_scope; + while (dynamic_cast<Declaration const*>(scope) && dynamic_cast<Declaration const*>(scope)->m_scope) + scope = dynamic_cast<Declaration const*>(scope)->m_scope; + return dynamic_cast<SourceUnit const&>(*scope).annotation().path; +} + ImportAnnotation& ImportDirective::annotation() const { if (!m_annotation) diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index 89b96f09..ab4be1ea 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -158,6 +158,10 @@ public: ASTNode const* scope() const { return m_scope; } void setScope(ASTNode const* _scope) { m_scope = _scope; } + /// @returns the source name this declaration is present in. + /// Can be combined with annotation().canonicalName to form a globally unique name. + std::string sourceUnitName() const; + virtual bool isLValue() const { return false; } virtual bool isPartOfExternalInterface() const { return false; } diff --git a/libsolidity/codegen/Compiler.cpp b/libsolidity/codegen/Compiler.cpp index 22d25698..44264a07 100644 --- a/libsolidity/codegen/Compiler.cpp +++ b/libsolidity/codegen/Compiler.cpp @@ -30,11 +30,13 @@ using namespace dev::solidity; void Compiler::compileContract( ContractDefinition const& _contract, - std::map<const ContractDefinition*, eth::Assembly const*> const& _contracts + std::map<const ContractDefinition*, eth::Assembly const*> const& _contracts, + bytes const& _metadata ) { ContractCompiler runtimeCompiler(nullptr, m_runtimeContext, m_optimize); runtimeCompiler.compileContract(_contract, _contracts); + m_runtimeContext.appendAuxiliaryData(_metadata); // This might modify m_runtimeContext because it can access runtime functions at // creation time. @@ -42,12 +44,6 @@ void Compiler::compileContract( m_runtimeSub = creationCompiler.compileConstructor(_contract, _contracts); m_context.optimise(m_optimize, m_optimizeRuns); - - if (_contract.isLibrary()) - { - solAssert(m_runtimeSub != size_t(-1), ""); - m_context.injectVersionStampIntoSub(m_runtimeSub); - } } void Compiler::compileClone( diff --git a/libsolidity/codegen/Compiler.h b/libsolidity/codegen/Compiler.h index c8bf7861..eef078c1 100644 --- a/libsolidity/codegen/Compiler.h +++ b/libsolidity/codegen/Compiler.h @@ -42,7 +42,8 @@ public: void compileContract( ContractDefinition const& _contract, - std::map<ContractDefinition const*, eth::Assembly const*> const& _contracts + std::map<ContractDefinition const*, eth::Assembly const*> const& _contracts, + bytes const& _metadata ); /// Compiles a contract that uses DELEGATECALL to call into a pre-deployed version of the given /// contract at runtime, but contains the full creation-time code. diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp index b99b0548..2de5a3ec 100644 --- a/libsolidity/codegen/CompilerContext.cpp +++ b/libsolidity/codegen/CompilerContext.cpp @@ -220,13 +220,6 @@ void CompilerContext::appendInlineAssembly( solAssert(assembly::InlineAssemblyStack().parseAndAssemble(*assembly, *m_asm, identifierAccess), ""); } -void CompilerContext::injectVersionStampIntoSub(size_t _subIndex) -{ - eth::Assembly& sub = m_asm->sub(_subIndex); - sub.injectStart(Instruction::POP); - sub.injectStart(fromBigEndian<u256>(binaryVersion())); -} - FunctionDefinition const& CompilerContext::resolveVirtualFunction( FunctionDefinition const& _function, vector<ContractDefinition const*>::const_iterator _searchStart diff --git a/libsolidity/codegen/CompilerContext.h b/libsolidity/codegen/CompilerContext.h index 3c6d9d73..80671528 100644 --- a/libsolidity/codegen/CompilerContext.h +++ b/libsolidity/codegen/CompilerContext.h @@ -152,8 +152,8 @@ public: std::map<std::string, std::string> const& _replacements = std::map<std::string, std::string>{} ); - /// Prepends "PUSH <compiler version number> POP" - void injectVersionStampIntoSub(size_t _subIndex); + /// Appends arbitrary data to the end of the bytecode. + void appendAuxiliaryData(bytes const& _data) { m_asm->appendAuxiliaryDataToEnd(_data); } void optimise(bool _fullOptimsation, unsigned _runs = 200) { m_asm->optimise(_fullOptimsation, true, _runs); } diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 9305c5e3..b4fd6d87 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -38,7 +38,11 @@ #include <libsolidity/formal/Why3Translator.h> #include <libevmasm/Exceptions.h> -#include <libdevcore/SHA3.h> + +#include <libdevcore/SwarmHash.h> +#include <libdevcore/JSON.h> + +#include <json/json.h> #include <boost/algorithm/string.hpp> #include <boost/filesystem.hpp> @@ -79,6 +83,8 @@ void CompilerStack::reset(bool _keepSources) { m_sources.clear(); } + m_optimize = false; + m_optimizeRuns = 200; m_globalContext.reset(); m_sourceOrder.clear(); m_contracts.clear(); @@ -217,32 +223,37 @@ vector<string> CompilerStack::contractNames() const } -bool CompilerStack::compile(bool _optimize, unsigned _runs) +bool CompilerStack::compile(bool _optimize, unsigned _runs, map<string, h160> const& _libraries) { if (!m_parseSuccessful) if (!parse()) return false; + m_optimize = _optimize; + m_optimizeRuns = _runs; + m_libraries = _libraries; + map<ContractDefinition const*, eth::Assembly const*> compiledContracts; for (Source const* source: m_sourceOrder) for (ASTPointer<ASTNode> const& node: source->ast->nodes()) if (auto contract = dynamic_cast<ContractDefinition const*>(node.get())) - compileContract(_optimize, _runs, *contract, compiledContracts); + compileContract(*contract, compiledContracts); + this->link(); return true; } -bool CompilerStack::compile(string const& _sourceCode, bool _optimize) +bool CompilerStack::compile(string const& _sourceCode, bool _optimize, unsigned _runs) { - return parse(_sourceCode) && compile(_optimize); + return parse(_sourceCode) && compile(_optimize, _runs); } -void CompilerStack::link(const std::map<string, h160>& _libraries) +void CompilerStack::link() { for (auto& contract: m_contracts) { - contract.second.object.link(_libraries); - contract.second.runtimeObject.link(_libraries); - contract.second.cloneObject.link(_libraries); + contract.second.object.link(m_libraries); + contract.second.runtimeObject.link(m_libraries); + contract.second.cloneObject.link(m_libraries); } } @@ -356,20 +367,28 @@ Json::Value const& CompilerStack::metadata(string const& _contractName, Document if (!m_parseSuccessful) BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Parsing was not successful.")); + return metadata(contract(_contractName), _type); +} + +Json::Value const& CompilerStack::metadata(Contract const& _contract, DocumentationType _type) const +{ + if (!m_parseSuccessful) + BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Parsing was not successful.")); + + solAssert(_contract.contract, ""); std::unique_ptr<Json::Value const>* doc; - Contract const& currentContract = contract(_contractName); // checks wheather we already have the documentation switch (_type) { case DocumentationType::NatspecUser: - doc = ¤tContract.userDocumentation; + doc = &_contract.userDocumentation; break; case DocumentationType::NatspecDev: - doc = ¤tContract.devDocumentation; + doc = &_contract.devDocumentation; break; case DocumentationType::ABIInterface: - doc = ¤tContract.interface; + doc = &_contract.interface; break; default: BOOST_THROW_EXCEPTION(InternalCompilerError() << errinfo_comment("Illegal documentation type.")); @@ -377,11 +396,19 @@ Json::Value const& CompilerStack::metadata(string const& _contractName, Document // caches the result if (!*doc) - doc->reset(new Json::Value(InterfaceHandler::documentation(*currentContract.contract, _type))); + doc->reset(new Json::Value(InterfaceHandler::documentation(*_contract.contract, _type))); return *(*doc); } +string const& CompilerStack::onChainMetadata(string const& _contractName) const +{ + if (!m_parseSuccessful) + BOOST_THROW_EXCEPTION(CompilerError() << errinfo_comment("Parsing was not successful.")); + + return contract(_contractName).onChainMetadata; +} + Scanner const& CompilerStack::scanner(string const& _sourceName) const { return *source(_sourceName).scanner; @@ -572,8 +599,6 @@ string CompilerStack::absolutePath(string const& _path, string const& _reference } void CompilerStack::compileContract( - bool _optimize, - unsigned _runs, ContractDefinition const& _contract, map<ContractDefinition const*, eth::Assembly const*>& _compiledContracts ) @@ -581,19 +606,28 @@ void CompilerStack::compileContract( if (_compiledContracts.count(&_contract) || !_contract.annotation().isFullyImplemented) return; for (auto const* dependency: _contract.annotation().contractDependencies) - compileContract(_optimize, _runs, *dependency, _compiledContracts); + compileContract(*dependency, _compiledContracts); - shared_ptr<Compiler> compiler = make_shared<Compiler>(_optimize, _runs); - compiler->compileContract(_contract, _compiledContracts); + shared_ptr<Compiler> compiler = make_shared<Compiler>(m_optimize, m_optimizeRuns); Contract& compiledContract = m_contracts.at(_contract.name()); + string onChainMetadata = createOnChainMetadata(compiledContract); + bytes cborEncodedMetadata = + // CBOR-encoding of {"bzzr0": dev::swarmHash(onChainMetadata)} + bytes{0xa1, 0x65, 'b', 'z', 'z', 'r', '0', 0x58, 0x20} + + dev::swarmHash(onChainMetadata).asBytes(); + solAssert(cborEncodedMetadata.size() <= 0xffff, "Metadata too large"); + // 16-bit big endian length + cborEncodedMetadata += toCompactBigEndian(cborEncodedMetadata.size(), 2); + compiler->compileContract(_contract, _compiledContracts, cborEncodedMetadata); compiledContract.compiler = compiler; compiledContract.object = compiler->assembledObject(); compiledContract.runtimeObject = compiler->runtimeObject(); + compiledContract.onChainMetadata = onChainMetadata; _compiledContracts[compiledContract.contract] = &compiler->assembly(); try { - Compiler cloneCompiler(_optimize, _runs); + Compiler cloneCompiler(m_optimize, m_optimizeRuns); cloneCompiler.compileClone(_contract, _compiledContracts); compiledContract.cloneObject = cloneCompiler.assembledObject(); } @@ -637,6 +671,45 @@ CompilerStack::Source const& CompilerStack::source(string const& _sourceName) co return it->second; } +string CompilerStack::createOnChainMetadata(Contract const& _contract) const +{ + Json::Value meta; + meta["version"] = 1; + meta["language"] = "Solidity"; + meta["compiler"]["version"] = VersionString; + + meta["sources"] = Json::objectValue; + for (auto const& s: m_sources) + { + solAssert(s.second.scanner, "Scanner not available"); + meta["sources"][s.first]["keccak256"] = + "0x" + toHex(dev::keccak256(s.second.scanner->source()).asBytes()); + meta["sources"][s.first]["url"] = + "bzzr://" + toHex(dev::swarmHash(s.second.scanner->source()).asBytes()); + } + meta["settings"]["optimizer"]["enabled"] = m_optimize; + meta["settings"]["optimizer"]["runs"] = m_optimizeRuns; + meta["settings"]["compilationTarget"][_contract.contract->sourceUnitName()] = + _contract.contract->annotation().canonicalName; + + meta["settings"]["remappings"] = Json::arrayValue; + set<string> remappings; + for (auto const& r: m_remappings) + remappings.insert(r.context + ":" + r.prefix + "=" + r.target); + for (auto const& r: remappings) + meta["settings"]["remappings"].append(r); + + meta["settings"]["libraries"] = Json::objectValue; + for (auto const& library: m_libraries) + meta["settings"]["libraries"][library.first] = "0x" + toHex(library.second.asBytes()); + + meta["output"]["abi"] = metadata(_contract, DocumentationType::ABIInterface); + meta["output"]["userdoc"] = metadata(_contract, DocumentationType::NatspecUser); + meta["output"]["devdoc"] = metadata(_contract, DocumentationType::NatspecDev); + + return jsonCompactPrint(meta); +} + string CompilerStack::computeSourceMapping(eth::AssemblyItems const& _items) const { string ret; diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index ea706122..f98a457a 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -113,13 +113,14 @@ public: /// Compiles the source units that were previously added and parsed. /// @returns false on error. - bool compile(bool _optimize = false, unsigned _runs = 200); + bool compile( + bool _optimize = false, + unsigned _runs = 200, + std::map<std::string, h160> const& _libraries = std::map<std::string, h160>{} + ); /// Parses and compiles the given source code. /// @returns false on error. - bool compile(std::string const& _sourceCode, bool _optimize = false); - - /// Inserts the given addresses into the linker objects of all compiled contracts. - void link(std::map<std::string, h160> const& _libraries); + bool compile(std::string const& _sourceCode, bool _optimize = false, unsigned _runs = 200); /// Tries to translate all source files into a language suitable for formal analysis. /// @param _errors list to store errors - defaults to the internal error list. @@ -170,6 +171,7 @@ public: /// @param type The type of the documentation to get. /// Can be one of 4 types defined at @c DocumentationType Json::Value const& metadata(std::string const& _contractName, DocumentationType _type) const; + std::string const& onChainMetadata(std::string const& _contractName) const; /// @returns the previously used scanner, useful for counting lines during error reporting. Scanner const& scanner(std::string const& _sourceName = "") const; @@ -213,6 +215,7 @@ private: eth::LinkerObject object; eth::LinkerObject runtimeObject; eth::LinkerObject cloneObject; + std::string onChainMetadata; ///< The metadata json that will be hashed into the chain. mutable std::unique_ptr<Json::Value const> interface; mutable std::unique_ptr<Json::Value const> userDocumentation; mutable std::unique_ptr<Json::Value const> devDocumentation; @@ -233,16 +236,18 @@ private: std::string absolutePath(std::string const& _path, std::string const& _reference) const; /// Compile a single contract and put the result in @a _compiledContracts. void compileContract( - bool _optimize, - unsigned _runs, ContractDefinition const& _contract, std::map<ContractDefinition const*, eth::Assembly const*>& _compiledContracts ); + void link(); + Contract const& contract(std::string const& _contractName = "") const; Source const& source(std::string const& _sourceName = "") const; + std::string createOnChainMetadata(Contract const& _contract) const; std::string computeSourceMapping(eth::AssemblyItems const& _items) const; + Json::Value const& metadata(Contract const&, DocumentationType _type) const; struct Remapping { @@ -252,6 +257,9 @@ private: }; ReadFileCallback m_readFile; + bool m_optimize = false; + unsigned m_optimizeRuns = 200; + std::map<std::string, h160> m_libraries; /// list of path prefix remappings, e.g. mylibrary: github.com/ethereum = /usr/local/ethereum /// "context:prefix=target" std::vector<Remapping> m_remappings; diff --git a/libsolidity/parsing/Scanner.h b/libsolidity/parsing/Scanner.h index 65de8bd4..d6b48c6f 100644 --- a/libsolidity/parsing/Scanner.h +++ b/libsolidity/parsing/Scanner.h @@ -80,6 +80,8 @@ public: void reset() { m_position = 0; } + std::string const& source() const { return m_source; } + ///@{ ///@name Error printing helper functions /// Functions that help pretty-printing parse errors @@ -102,6 +104,8 @@ public: explicit Scanner(CharStream const& _source = CharStream(), std::string const& _sourceName = "") { reset(_source, _sourceName); } + std::string source() const { return m_source.source(); } + /// Resets the scanner as if newly constructed with _source and _sourceName as input. void reset(CharStream const& _source, std::string const& _sourceName); /// Resets scanner to the start of input. diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 49fac51c..6e59099a 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -78,6 +78,7 @@ static string const g_argCloneBinaryStr = "clone-bin"; static string const g_argOpcodesStr = "opcodes"; static string const g_argNatspecDevStr = "devdoc"; static string const g_argNatspecUserStr = "userdoc"; +static string const g_argMetadata = "metadata"; static string const g_argAddStandard = "add-std"; static string const g_stdinFileName = "<stdin>"; @@ -91,6 +92,7 @@ static set<string> const g_combinedJsonArgs{ "opcodes", "abi", "interface", + "metadata", "asm", "ast", "userdoc", @@ -117,6 +119,7 @@ static bool needsHumanTargetedStdout(po::variables_map const& _args) for (string const& arg: { g_argAbiStr, g_argSignatureHashes, + g_argMetadata, g_argNatspecUserStr, g_argAstJson, g_argNatspecDevStr, @@ -202,6 +205,18 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) cout << "Function signatures: " << endl << out; } +void CommandLineInterface::handleOnChainMetadata(string const& _contract) +{ + if (!m_args.count(g_argMetadata)) + return; + + string data = m_compiler->onChainMetadata(_contract); + if (m_args.count("output-dir")) + createFile(_contract + "_meta.json", data); + else + cout << "Metadata: " << endl << data << endl; +} + void CommandLineInterface::handleMeta(DocumentationType _type, string const& _contract) { std::string argName; @@ -467,6 +482,7 @@ Allowed options)", (g_argSignatureHashes.c_str(), "Function signature hashes of the contracts.") (g_argNatspecUserStr.c_str(), "Natspec user documentation of all contracts.") (g_argNatspecDevStr.c_str(), "Natspec developer documentation of all contracts.") + (g_argMetadata.c_str(), "Combined Metadata JSON whose Swarm hash is stored on-chain.") ("formal", "Translated source suitable for formal analysis."); desc.add(outputComponents); @@ -581,9 +597,7 @@ bool CommandLineInterface::processInput() // TODO: Perhaps we should not compile unless requested bool optimize = m_args.count("optimize") > 0; unsigned runs = m_args["optimize-runs"].as<unsigned>(); - bool successful = m_compiler->compile(optimize, runs); - if (successful) - m_compiler->link(m_libraries); + bool successful = m_compiler->compile(optimize, runs, m_libraries); if (successful && m_args.count("formal")) if (!m_compiler->prepareFormalAnalysis()) @@ -659,6 +673,8 @@ void CommandLineInterface::handleCombinedJSON() Json::Value contractData(Json::objectValue); if (requests.count("abi")) contractData["abi"] = dev::jsonCompactPrint(m_compiler->interface(contractName)); + if (requests.count("metadata")) + contractData["metadata"] = m_compiler->onChainMetadata(contractName); if (requests.count("bin")) contractData["bin"] = m_compiler->object(contractName).toHex(); if (requests.count("bin-runtime")) @@ -918,6 +934,7 @@ void CommandLineInterface::outputCompilationResults() handleBytecode(contract); handleSignatureHashes(contract); + handleOnChainMetadata(contract); handleMeta(DocumentationType::ABIInterface, contract); handleMeta(DocumentationType::NatspecDev, contract); handleMeta(DocumentationType::NatspecUser, contract); diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 33ae6ee6..b8fc1823 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -63,6 +63,7 @@ private: void handleOpcode(std::string const& _contract); void handleBytecode(std::string const& _contract); void handleSignatureHashes(std::string const& _contract); + void handleOnChainMetadata(std::string const& _contract); void handleMeta(DocumentationType _type, std::string const& _contract); void handleGasEstimation(std::string const& _contract); void handleFormal(); diff --git a/solc/jsonCompiler.cpp b/solc/jsonCompiler.cpp index 0f95ebe5..d761b541 100644 --- a/solc/jsonCompiler.cpp +++ b/solc/jsonCompiler.cpp @@ -218,6 +218,7 @@ string compile(StringMap const& _sources, bool _optimize, CStyleReadFileCallback contractData["bytecode"] = compiler.object(contractName).toHex(); contractData["runtimeBytecode"] = compiler.runtimeObject(contractName).toHex(); contractData["opcodes"] = solidity::disassemble(compiler.object(contractName).bytecode); + contractData["metadata"] = compiler.onChainMetadata(contractName); contractData["functionHashes"] = functionHashes(compiler.contractDefinition(contractName)); contractData["gasEstimates"] = estimateGas(compiler, contractName); auto sourceMap = compiler.sourceMapping(contractName); diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index 1274d400..f47f2743 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -279,7 +279,7 @@ protected: bytes data; }; - size_t m_optimizeRuns = 200; + unsigned m_optimizeRuns = 200; bool m_optimize = false; Address m_sender; Address m_contractAddress; diff --git a/test/libdevcore/SwarmHash.cpp b/test/libdevcore/SwarmHash.cpp index a23c36fd..7f3186ac 100644 --- a/test/libdevcore/SwarmHash.cpp +++ b/test/libdevcore/SwarmHash.cpp @@ -31,24 +31,24 @@ namespace test BOOST_AUTO_TEST_SUITE(SwarmHash) -string swarmHashHex(bytes const& _input) +string swarmHashHex(string const& _input) { return toHex(swarmHash(_input).asBytes()); } BOOST_AUTO_TEST_CASE(test_zeros) { - BOOST_CHECK_EQUAL(swarmHashHex(bytes()), string("011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x1000 - 1, 0)), string("32f0faabc4265ac238cd945087133ce3d7e9bb2e536053a812b5373c54043adb")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x1000, 0)), string("411dd45de7246e94589ff5888362c41e85bd3e582a92d0fda8f0e90b76439bec")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x1000 + 1, 0)), string("69754a0098432bbc2e84fe1205276870748a61a065ab6ef44d6a2e7b13ce044d")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x2000 - 1, 0)), string("69ad3c581043404f775ffa8d6f1b25ad4a9ee812971190e90209c0966116a321")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x2000, 0)), string("f00222373ff82d0a178dc6271c78953e9c88f74130a52d401f5ec51475f63c43")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x2000 + 1, 0)), string("86d6773e79e02fd8145ee1aedba89ace0c15f2566db1249654000039a9a134bf")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x80000, 0)), string("cc0854fe2c6b98e920d5c14b1a88e6d4223e55b8f78883f60939aa2485e361bf")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x80020, 0)), string("ee9ffca246e70d3704740ba4df450fa6988d14a1c2439c7e734c7a77a4eb6fd3")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(0x800020, 0)), string("78b90b20c90559fb904535181a7c28929ea2f30a2329dbc25232de579709f12f")); - BOOST_CHECK_EQUAL(swarmHashHex(bytes(2095104, 0)), string("a9958184589fc11b4027a4c233e777ebe2e99c66f96b74aef2a0638a94dd5439")); + BOOST_CHECK_EQUAL(swarmHashHex(string()), string("011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x1000 - 1, 0)), string("32f0faabc4265ac238cd945087133ce3d7e9bb2e536053a812b5373c54043adb")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x1000, 0)), string("411dd45de7246e94589ff5888362c41e85bd3e582a92d0fda8f0e90b76439bec")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x1000 + 1, 0)), string("69754a0098432bbc2e84fe1205276870748a61a065ab6ef44d6a2e7b13ce044d")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x2000 - 1, 0)), string("69ad3c581043404f775ffa8d6f1b25ad4a9ee812971190e90209c0966116a321")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x2000, 0)), string("f00222373ff82d0a178dc6271c78953e9c88f74130a52d401f5ec51475f63c43")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x2000 + 1, 0)), string("86d6773e79e02fd8145ee1aedba89ace0c15f2566db1249654000039a9a134bf")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x80000, 0)), string("cc0854fe2c6b98e920d5c14b1a88e6d4223e55b8f78883f60939aa2485e361bf")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x80020, 0)), string("ee9ffca246e70d3704740ba4df450fa6988d14a1c2439c7e734c7a77a4eb6fd3")); + BOOST_CHECK_EQUAL(swarmHashHex(string(0x800020, 0)), string("78b90b20c90559fb904535181a7c28929ea2f30a2329dbc25232de579709f12f")); + BOOST_CHECK_EQUAL(swarmHashHex(string(2095104, 0)), string("a9958184589fc11b4027a4c233e777ebe2e99c66f96b74aef2a0638a94dd5439")); } BOOST_AUTO_TEST_SUITE_END() diff --git a/test/libsolidity/Assembly.cpp b/test/libsolidity/Assembly.cpp index cc77bd4c..bdbe7dba 100644 --- a/test/libsolidity/Assembly.cpp +++ b/test/libsolidity/Assembly.cpp @@ -75,7 +75,7 @@ eth::AssemblyItems compileContract(const string& _sourceCode) if (ContractDefinition* contract = dynamic_cast<ContractDefinition*>(node.get())) { Compiler compiler; - compiler.compileContract(*contract, map<ContractDefinition const*, Assembly const*>{}); + compiler.compileContract(*contract, map<ContractDefinition const*, Assembly const*>{}, bytes()); return compiler.runtimeAssemblyItems(); } diff --git a/test/libsolidity/SolidityABIJSON.cpp b/test/libsolidity/SolidityABIJSON.cpp index 6fc2bcee..890db241 100644 --- a/test/libsolidity/SolidityABIJSON.cpp +++ b/test/libsolidity/SolidityABIJSON.cpp @@ -22,8 +22,11 @@ #include "../TestHelper.h" #include <libsolidity/interface/CompilerStack.h> -#include <json/json.h> + #include <libdevcore/Exceptions.h> +#include <libdevcore/SwarmHash.h> + +#include <json/json.h> namespace dev { @@ -51,7 +54,7 @@ public: ); } -private: +protected: CompilerStack m_compilerStack; Json::Reader m_reader; }; @@ -731,6 +734,26 @@ BOOST_AUTO_TEST_CASE(function_type) checkInterface(sourceCode, interface); } +BOOST_AUTO_TEST_CASE(metadata_stamp) +{ + // Check that the metadata stamp is at the end of the runtime bytecode. + char const* sourceCode = R"( + pragma solidity >=0.0; + contract test { + function g(function(uint) external returns (uint) x) {} + } + )"; + BOOST_REQUIRE(m_compilerStack.compile(std::string(sourceCode))); + bytes const& bytecode = m_compilerStack.runtimeObject("test").bytecode; + bytes hash = dev::swarmHash(m_compilerStack.onChainMetadata("test")).asBytes(); + BOOST_REQUIRE(hash.size() == 32); + BOOST_REQUIRE(bytecode.size() >= 2); + size_t metadataCBORSize = (size_t(bytecode.end()[-2]) << 8) + size_t(bytecode.end()[-1]); + BOOST_REQUIRE(metadataCBORSize < bytecode.size() - 2); + bytes expectation = bytes{0xa1, 0x65, 'b', 'z', 'z', 'r', '0', 0x58, 0x20} + hash; + BOOST_CHECK(std::equal(expectation.begin(), expectation.end(), bytecode.end() - metadataCBORSize - 2)); +} + BOOST_AUTO_TEST_SUITE_END() } diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 47b96dc2..aa1eb20a 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -6547,17 +6547,6 @@ BOOST_AUTO_TEST_CASE(calldata_offset) BOOST_CHECK(callContractFunction("last()", encodeArgs()) == encodeDyn(string("nd"))); } -BOOST_AUTO_TEST_CASE(version_stamp_for_libraries) -{ - char const* sourceCode = "library lib {}"; - m_optimize = true; - bytes runtimeCode = compileAndRun(sourceCode, 0, "lib"); - BOOST_CHECK(runtimeCode.size() >= 8); - BOOST_CHECK_EQUAL(runtimeCode[0], int(Instruction::PUSH6)); // might change once we switch to 1.x.x - BOOST_CHECK_EQUAL(runtimeCode[1], 4); // might change once we switch away from x.4.x - BOOST_CHECK_EQUAL(runtimeCode[7], int(Instruction::POP)); -} - BOOST_AUTO_TEST_CASE(contract_binary_dependencies) { char const* sourceCode = R"( @@ -7831,13 +7820,6 @@ BOOST_AUTO_TEST_CASE(mem_resize_is_not_paid_at_call) u160 cAddr = m_contractAddress; compileAndRun(sourceCode, 0, "D"); BOOST_CHECK(callContractFunction("f(address)", cAddr) == encodeArgs(u256(7))); - - m_optimize = true; - - compileAndRun(sourceCode, 0, "C"); - u160 cAddrOpt = m_contractAddress; - compileAndRun(sourceCode, 0, "D"); - BOOST_CHECK(callContractFunction("f(address)", cAddrOpt) == encodeArgs(u256(7))); } BOOST_AUTO_TEST_CASE(calling_uninitialized_function) diff --git a/test/libsolidity/SolidityExecutionFramework.h b/test/libsolidity/SolidityExecutionFramework.h index 0fab7aeb..03e3a881 100644 --- a/test/libsolidity/SolidityExecutionFramework.h +++ b/test/libsolidity/SolidityExecutionFramework.h @@ -56,7 +56,7 @@ public: std::string sourceCode = "pragma solidity >=0.0;\n" + _sourceCode; m_compiler.reset(false); m_compiler.addSource("", sourceCode); - if (!m_compiler.compile(m_optimize, m_optimizeRuns)) + if (!m_compiler.compile(m_optimize, m_optimizeRuns, _libraryAddresses)) { for (auto const& error: m_compiler.errors()) SourceReferenceFormatter::printExceptionInformation( @@ -68,7 +68,6 @@ public: BOOST_ERROR("Compiling contract failed"); } eth::LinkerObject obj = m_compiler.object(_contractName); - obj.link(_libraryAddresses); BOOST_REQUIRE(obj.linkReferences.empty()); sendMessage(obj.bytecode + _arguments, true, _value); return m_output; diff --git a/test/libsolidity/SolidityOptimizer.cpp b/test/libsolidity/SolidityOptimizer.cpp index 3c1f465a..2e2e0c6c 100644 --- a/test/libsolidity/SolidityOptimizer.cpp +++ b/test/libsolidity/SolidityOptimizer.cpp @@ -50,6 +50,25 @@ class OptimizerTestFramework: public SolidityExecutionFramework { public: OptimizerTestFramework() { } + + bytes const& compileAndRunWithOptimizer( + std::string const& _sourceCode, + u256 const& _value = 0, + std::string const& _contractName = "", + bool const _optimize = true, + unsigned const _optimizeRuns = 200 + ) + { + bool const c_optimize = m_optimize; + unsigned const c_optimizeRuns = m_optimizeRuns; + m_optimize = _optimize; + m_optimizeRuns = _optimizeRuns; + bytes const& ret = compileAndRun(_sourceCode, _value, _contractName); + m_optimize = c_optimize; + m_optimizeRuns = c_optimizeRuns; + return ret; + } + /// Compiles the source code with and without optimizing. void compileBothVersions( std::string const& _sourceCode, @@ -57,22 +76,16 @@ public: std::string const& _contractName = "" ) { - m_optimize = false; - bytes nonOptimizedBytecode = compileAndRun(_sourceCode, _value, _contractName); + bytes nonOptimizedBytecode = compileAndRunWithOptimizer(_sourceCode, _value, _contractName, false); m_nonOptimizedContract = m_contractAddress; - m_optimize = true; - bytes optimizedBytecode = compileAndRun(_sourceCode, _value, _contractName); - size_t nonOptimizedSize = 0; - solidity::eachInstruction(nonOptimizedBytecode, [&](Instruction, u256 const&) { - nonOptimizedSize++; - }); - size_t optimizedSize = 0; - solidity::eachInstruction(optimizedBytecode, [&](Instruction, u256 const&) { - optimizedSize++; - }); + bytes optimizedBytecode = compileAndRunWithOptimizer(_sourceCode, _value, _contractName, true); + size_t nonOptimizedSize = numInstructions(nonOptimizedBytecode); + size_t optimizedSize = numInstructions(optimizedBytecode); BOOST_CHECK_MESSAGE( - nonOptimizedSize > optimizedSize, - "Optimizer did not reduce bytecode size." + optimizedSize < nonOptimizedSize, + string("Optimizer did not reduce bytecode size. Non-optimized size: ") + + std::to_string(nonOptimizedSize) + " - optimized size: " + + std::to_string(optimizedSize) ); m_optimizedContract = m_contractAddress; } @@ -156,6 +169,22 @@ public: } protected: + /// @returns the number of intructions in the given bytecode, not taking the metadata hash + /// into account. + size_t numInstructions(bytes const& _bytecode) + { + BOOST_REQUIRE(_bytecode.size() > 5); + size_t metadataSize = (_bytecode[_bytecode.size() - 2] << 8) + _bytecode[_bytecode.size() - 1]; + BOOST_REQUIRE_MESSAGE(metadataSize == 0x29, "Invalid metadata size"); + BOOST_REQUIRE(_bytecode.size() >= metadataSize + 2); + bytes realCode = bytes(_bytecode.begin(), _bytecode.end() - metadataSize - 2); + size_t instructions = 0; + solidity::eachInstruction(realCode, [&](Instruction, u256 const&) { + instructions++; + }); + return instructions; + } + Address m_optimizedContract; Address m_nonOptimizedContract; }; @@ -315,8 +344,7 @@ BOOST_AUTO_TEST_CASE(retain_information_in_branches) compareVersions("f(uint256,bytes32)", 8, "def"); compareVersions("f(uint256,bytes32)", 10, "ghi"); - m_optimize = true; - bytes optimizedBytecode = compileAndRun(sourceCode, 0, "c"); + bytes optimizedBytecode = compileAndRunWithOptimizer(sourceCode, 0, "c", true); size_t numSHA3s = 0; eachInstruction(optimizedBytecode, [&](Instruction _instr, u256 const&) { if (_instr == Instruction::SHA3) @@ -359,8 +387,7 @@ BOOST_AUTO_TEST_CASE(store_tags_as_unions) compileBothVersions(sourceCode); compareVersions("f(uint256,bytes32)", 7, "abc"); - m_optimize = true; - bytes optimizedBytecode = compileAndRun(sourceCode, 0, "test"); + bytes optimizedBytecode = compileAndRunWithOptimizer(sourceCode, 0, "test", true); size_t numSHA3s = 0; eachInstruction(optimizedBytecode, [&](Instruction _instr, u256 const&) { if (_instr == Instruction::SHA3) @@ -1187,9 +1214,7 @@ BOOST_AUTO_TEST_CASE(computing_constants) compareVersions("set()"); compareVersions("get()"); - m_optimize = true; - m_optimizeRuns = 1; - bytes optimizedBytecode = compileAndRun(sourceCode, 0, "c"); + bytes optimizedBytecode = compileAndRunWithOptimizer(sourceCode, 0, "c", true, 1); bytes complicatedConstant = toBigEndian(u256("0x817416927846239487123469187231298734162934871263941234127518276")); unsigned occurrences = 0; for (auto iter = optimizedBytecode.cbegin(); iter < optimizedBytecode.cend(); ++occurrences) |