Merge pull request #3707 from ethereum/syntaxTestsTestRunner

Infrastructure for extracting syntax tests.

14 files changed, 348 insertions, 59 deletions

diff --git a/appveyor.yml b/appveyor.yml
index ef5f6907..5fd85482 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -71,7 +71,7 @@ build_script:
 
 test_script:
     - cd %APPVEYOR_BUILD_FOLDER%\build\test\%CONFIGURATION%
-    - soltest.exe --show-progress -- --no-ipc --no-smt
+    - soltest.exe --show-progress -- --testpath %APPVEYOR_BUILD_FOLDER%\test --no-ipc --no-smt
 # Skip bytecode compare if private key is not available
     - cd %APPVEYOR_BUILD_FOLDER%
     - ps: if ($env:priv_key) {
diff --git a/docs/contributing.rst b/docs/contributing.rst
index a5efba8b..8b4695e4 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -69,15 +69,22 @@ Solidity includes different types of tests. They are included in the application
 called ``soltest``. Some of them require the ``cpp-ethereum`` client in testing mode,
 some others require ``libz3`` to be installed.
 
-To disable the z3 tests, use ``./build/test/soltest -- --no-smt`` and
-to run a subset of the tests that do not require ``cpp-ethereum``, use ``./build/test/soltest -- --no-ipc``.
+``soltest`` reads test contracts that are annotated with expected results
+stored in ``./test/libsolidity/syntaxTests``. In order for soltest to find these
+tests the root test directory has to be specified using the ``--testpath`` command
+line option, e.g. ``./build/test/soltest -- --testpath ./test``.
+
+To disable the z3 tests, use ``./build/test/soltest -- --no-smt --testpath ./test`` and
+to run a subset of the tests that do not require ``cpp-ethereum``, use
+``./build/test/soltest -- --no-ipc --testpath ./test``.
 
 For all other tests, you need to install `cpp-ethereum <https://github.com/ethereum/cpp-ethereum/releases/download/solidityTester/eth>`_ and run it in testing mode: ``eth --test -d /tmp/testeth``.
 
-Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc``.
+Then you run the actual tests: ``./build/test/soltest -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``.
 
 To run a subset of tests, filters can be used:
-``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc``, where ``TestName`` can be a wildcard ``*``.
+``soltest -t TestSuite/TestName -- --ipcpath /tmp/testeth/geth.ipc --testpath ./test``,
+where ``TestName`` can be a wildcard ``*``.
 
 Alternatively, there is a testing script at ``scripts/test.sh`` which executes all tests and runs
 ``cpp-ethereum`` automatically if it is in the path (but does not download it).
diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py
index cfaef210..5bf577d3 100755
--- a/scripts/isolate_tests.py
+++ b/scripts/isolate_tests.py
@@ -86,10 +86,15 @@ if __name__ == '__main__':
     for root, subdirs, files in os.walk(path):
         if '_build' in subdirs:
           subdirs.remove('_build')
+        if 'compilationTests' in subdirs:
+          subdirs.remove('compilationTests')
         for f in files:
             path = join(root, f)
             if docs:
               cases = extract_docs_cases(path)
             else:
-              cases = extract_test_cases(path)
+              if f.endswith(".sol"):
+                cases = [open(path, "r").read()]
+              else:
+                cases = extract_test_cases(path)
             write_cases(cases)
diff --git a/scripts/tests.sh b/scripts/tests.sh
index bf4ee3d9..37ffafc2 100755
--- a/scripts/tests.sh
+++ b/scripts/tests.sh
@@ -116,7 +116,7 @@ do
         log=--logger=JUNIT,test_suite,$log_directory/noopt_$vm.xml $testargs_no_opt
       fi
     fi
-    "$REPO_ROOT"/build/test/soltest $progress $log -- "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc
+    "$REPO_ROOT"/build/test/soltest $progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc
   done
 done
 
diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp
index e0d4423d..77fa204f 100644
--- a/test/TestHelper.cpp
+++ b/test/TestHelper.cpp
@@ -43,6 +43,11 @@ Options::Options()
 			ipcPath = suite.argv[i + 1];
 			i++;
 		}
+		else if (string(suite.argv[i]) == "--testpath" && i + 1 < suite.argc)
+		{
+			testPath = suite.argv[i + 1];
+			i++;
+		}
 		else if (string(suite.argv[i]) == "--optimize")
 			optimize = true;
 		else if (string(suite.argv[i]) == "--evm-version")
@@ -60,6 +65,10 @@ Options::Options()
 	if (!disableIPC && ipcPath.empty())
 		if (auto path = getenv("ETH_TEST_IPC"))
 			ipcPath = path;
+
+	if (testPath.empty())
+		if (auto path = getenv("ETH_TEST_PATH"))
+			testPath = path;
 }
 
 dev::solidity::EVMVersion Options::evmVersion() const
diff --git a/test/TestHelper.h b/test/TestHelper.h
index 8c2eec36..f7b1d94c 100644
--- a/test/TestHelper.h
+++ b/test/TestHelper.h
@@ -35,6 +35,7 @@ namespace test
 struct Options: boost::noncopyable
 {
 	std::string ipcPath;
+	boost::filesystem::path testPath;
 	bool showMessages = false;
 	bool optimize = false;
 	bool disableIPC = false;
diff --git a/test/boostTest.cpp b/test/boostTest.cpp
index a3cc51c5..e557ff95 100644
--- a/test/boostTest.cpp
+++ b/test/boostTest.cpp
@@ -36,6 +36,7 @@
 #pragma GCC diagnostic pop
 
 #include <test/TestHelper.h>
+#include <test/libsolidity/SyntaxTest.h>
 
 using namespace boost::unit_test;
 
@@ -54,6 +55,11 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 {
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
+	solAssert(dev::solidity::test::SyntaxTest::registerTests(
+		master,
+		dev::test::Options::get().testPath / "libsolidity",
+		"syntaxTests"
+	) > 0, "no syntax tests found");
 	if (dev::test::Options::get().disableIPC)
 	{
 		for (auto suite: {
diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp
index 997b610e..1f76c01b 100644
--- a/test/libsolidity/SolidityNameAndTypeResolution.cpp
+++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp
@@ -43,27 +43,6 @@ namespace test
 
 BOOST_FIXTURE_TEST_SUITE(SolidityNameAndTypeResolution, AnalysisFramework)
 
-BOOST_AUTO_TEST_CASE(smoke_test)
-{
-	char const* text = R"(
-		contract test {
-			uint256 stateVariable1;
-			function fun(uint256 arg1) public { uint256 y; y = arg1; }
-		}
-	)";
-	CHECK_SUCCESS(text);
-}
-
-BOOST_AUTO_TEST_CASE(double_stateVariable_declaration)
-{
-	char const* text = R"(
-		contract test {
-			uint256 variable;
-			uint128 variable;
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared.");
-}
 
 BOOST_AUTO_TEST_CASE(double_function_declaration)
 {
@@ -76,37 +55,6 @@ BOOST_AUTO_TEST_CASE(double_function_declaration)
 	CHECK_ERROR(text, DeclarationError, "Function with same name and arguments defined twice.");
 }
 
-BOOST_AUTO_TEST_CASE(double_variable_declaration)
-{
-	string text = R"(
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared");
-}
-
-BOOST_AUTO_TEST_CASE(double_variable_declaration_050)
-{
-	string text = R"(
-		pragma experimental "v0.5.0";
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_WARNING_ALLOW_MULTI(text, (vector<string>{
-		"This declaration shadows an existing declaration.",
-		"Unused local variable",
-		"Unused local variable"
-	}));
-}
-
 BOOST_AUTO_TEST_CASE(double_variable_declaration_disjoint_scope)
 {
 	string text = R"(
diff --git a/test/libsolidity/SyntaxTest.cpp b/test/libsolidity/SyntaxTest.cpp
new file mode 100644
index 00000000..f1c60458
--- /dev/null
+++ b/test/libsolidity/SyntaxTest.cpp
@@ -0,0 +1,205 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTest.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <cctype>
+#include <fstream>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+namespace fs = boost::filesystem;
+using namespace boost::unit_test;
+
+template<typename IteratorType>
+void skipWhitespace(IteratorType& it, IteratorType end)
+{
+	while (it != end && isspace(*it))
+		++it;
+}
+
+template<typename IteratorType>
+void skipSlashes(IteratorType& it, IteratorType end)
+{
+	while (it != end && *it == '/')
+		++it;
+}
+
+SyntaxTest::SyntaxTest(string const& _filename)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
+	file.exceptions(ios::badbit);
+
+	m_source = parseSource(file);
+	m_expectations = parseExpectations(file);
+}
+
+bool SyntaxTest::run(ostream& _stream, string const& _indent)
+{
+	m_errorList = parseAnalyseAndReturnError(m_source, true, true, true).second;
+	if (!matchesExpectations(m_errorList))
+	{
+		std::string nextIndentLevel = _indent + "\t";
+		_stream << _indent << "Expected result:" << endl;
+		printExpected(_stream, nextIndentLevel);
+		_stream << _indent << "Obtained result:\n";
+		printErrorList(_stream, m_errorList, nextIndentLevel);
+		return false;
+	}
+	return true;
+}
+
+void SyntaxTest::printExpected(ostream& _stream, string const& _indent) const
+{
+	if (m_expectations.empty())
+		_stream << _indent << "Success" << endl;
+	else
+		for (auto const& expectation: m_expectations)
+			_stream << _indent << expectation.type << ": " << expectation.message << endl;
+}
+
+void SyntaxTest::printErrorList(
+	ostream& _stream,
+	ErrorList const& _errorList,
+	string const& _indent
+) const
+{
+	if (_errorList.empty())
+		_stream << _indent << "Success" << endl;
+	else
+		for (auto const& error: _errorList)
+			_stream << _indent << error->typeName() << ": " << errorMessage(*error) << endl;
+}
+
+bool SyntaxTest::matchesExpectations(ErrorList const& _errorList) const
+{
+	if (_errorList.size() != m_expectations.size())
+		return false;
+	else
+		for (size_t i = 0; i < _errorList.size(); i++)
+			if (
+				(_errorList[i]->typeName() != m_expectations[i].type) ||
+				(errorMessage(*_errorList[i]) != m_expectations[i].message)
+			)
+				return false;
+	return true;
+}
+
+string SyntaxTest::errorMessage(Error const& _e)
+{
+	if (_e.comment())
+		return boost::replace_all_copy(*_e.comment(), "\n", "\\n");
+	else
+		return "NONE";
+}
+
+string SyntaxTest::parseSource(istream& _stream)
+{
+	string source;
+	string line;
+	string const delimiter("// ----");
+	while (getline(_stream, line))
+		if (boost::algorithm::starts_with(line, delimiter))
+			break;
+		else
+			source += line + "\n";
+	return source;
+}
+
+vector<SyntaxTestExpectation> SyntaxTest::parseExpectations(istream& _stream)
+{
+	vector<SyntaxTestExpectation> expectations;
+	string line;
+	while (getline(_stream, line))
+	{
+		auto it = line.begin();
+
+		skipSlashes(it, line.end());
+		skipWhitespace(it, line.end());
+
+		if (it == line.end()) continue;
+
+		auto typeBegin = it;
+		while (it != line.end() && *it != ':')
+			++it;
+		string errorType(typeBegin, it);
+
+		// skip colon
+		if (it != line.end()) it++;
+
+		skipWhitespace(it, line.end());
+
+		string errorMessage(it, line.end());
+		expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)});
+	}
+	return expectations;
+}
+
+#if BOOST_VERSION < 105900
+test_case *make_test_case(
+	function<void()> const& _fn,
+	string const& _name,
+	string const& /* _filename */,
+	size_t /* _line */
+)
+{
+	return make_test_case(_fn, _name);
+}
+#endif
+
+int SyntaxTest::registerTests(
+	boost::unit_test::test_suite& _suite,
+	boost::filesystem::path const& _basepath,
+	boost::filesystem::path const& _path
+)
+{
+	int numTestsAdded = 0;
+	fs::path fullpath = _basepath / _path;
+	if (fs::is_directory(fullpath))
+	{
+		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
+		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+			fs::directory_iterator(fullpath),
+			fs::directory_iterator()
+		))
+			numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
+		_suite.add(sub_suite);
+	}
+	else
+	{
+		_suite.add(make_test_case(
+			[fullpath]
+			{
+				std::stringstream errorStream;
+				if (!SyntaxTest(fullpath.string()).run(errorStream, ""))
+					BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
+			},
+			_path.stem().string(),
+			_path.string(),
+			0
+		));
+		numTestsAdded = 1;
+	}
+	return numTestsAdded;
+}
diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h
new file mode 100644
index 00000000..4379c77b
--- /dev/null
+++ b/test/libsolidity/SyntaxTest.h
@@ -0,0 +1,77 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/AnalysisFramework.h>
+#include <libsolidity/interface/Exceptions.h>
+
+#include <boost/noncopyable.hpp>
+#include <boost/test/unit_test.hpp>
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+struct SyntaxTestExpectation
+{
+	std::string type;
+	std::string message;
+};
+
+
+class SyntaxTest: AnalysisFramework
+{
+public:
+	SyntaxTest(std::string const& _filename);
+
+	bool run(std::ostream& _stream, std::string const& _indent);
+
+	void printExpected(std::ostream& _stream, std::string const& _indent) const;
+	void printErrorList(
+		std::ostream& _stream,
+		ErrorList const& _errors,
+		std::string const& _indent
+	) const;
+
+	static int registerTests(
+		boost::unit_test::test_suite& _suite,
+		boost::filesystem::path const& _basepath,
+		boost::filesystem::path const& _path
+	);
+private:
+	bool matchesExpectations(ErrorList const& _errors) const;
+	static std::string errorMessage(Error const& _e);
+	static std::string parseSource(std::istream& _stream);
+	static std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
+
+	std::string m_source;
+	std::vector<SyntaxTestExpectation> m_expectations;
+	ErrorList m_errorList;
+};
+
+}
+}
+}
diff --git a/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
new file mode 100644
index 00000000..c5507b64
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 variable;
+	uint128 variable;
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration.sol b/test/libsolidity/syntaxTests/double_variable_declaration.sol
new file mode 100644
index 00000000..3349cfec
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration.sol
@@ -0,0 +1,8 @@
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration_050.sol b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
new file mode 100644
index 00000000..9c2d40d5
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
@@ -0,0 +1,11 @@
+pragma experimental "v0.5.0";
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// Warning: This declaration shadows an existing declaration.
+// Warning: Unused local variable.
+// Warning: Unused local variable.
diff --git a/test/libsolidity/syntaxTests/smoke_test.sol b/test/libsolidity/syntaxTests/smoke_test.sol
new file mode 100644
index 00000000..2d48098a
--- /dev/null
+++ b/test/libsolidity/syntaxTests/smoke_test.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 stateVariable1;
+	function fun(uint256 arg1) public { uint256 y; y = arg1; }
+}
+// ----
+// Warning: Function state mutability can be restricted to pure