Infrastructure for extracting syntax tests in separate test files.

15 files changed, 387 insertions, 55 deletions

diff --git a/appveyor.yml b/appveyor.yml
index ef5f6907..5fd85482 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -71,7 +71,7 @@ build_script:
 
 test_script:
     - cd %APPVEYOR_BUILD_FOLDER%\build\test\%CONFIGURATION%
-    - soltest.exe --show-progress -- --no-ipc --no-smt
+    - soltest.exe --show-progress -- --testpath %APPVEYOR_BUILD_FOLDER%\test --no-ipc --no-smt
 # Skip bytecode compare if private key is not available
     - cd %APPVEYOR_BUILD_FOLDER%
     - ps: if ($env:priv_key) {
diff --git a/scripts/isolate_tests.py b/scripts/isolate_tests.py
index cfaef210..5bf577d3 100755
--- a/scripts/isolate_tests.py
+++ b/scripts/isolate_tests.py
@@ -86,10 +86,15 @@ if __name__ == '__main__':
     for root, subdirs, files in os.walk(path):
         if '_build' in subdirs:
           subdirs.remove('_build')
+        if 'compilationTests' in subdirs:
+          subdirs.remove('compilationTests')
         for f in files:
             path = join(root, f)
             if docs:
               cases = extract_docs_cases(path)
             else:
-              cases = extract_test_cases(path)
+              if f.endswith(".sol"):
+                cases = [open(path, "r").read()]
+              else:
+                cases = extract_test_cases(path)
             write_cases(cases)
diff --git a/scripts/tests.sh b/scripts/tests.sh
index bf4ee3d9..37ffafc2 100755
--- a/scripts/tests.sh
+++ b/scripts/tests.sh
@@ -116,7 +116,7 @@ do
         log=--logger=JUNIT,test_suite,$log_directory/noopt_$vm.xml $testargs_no_opt
       fi
     fi
-    "$REPO_ROOT"/build/test/soltest $progress $log -- "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc
+    "$REPO_ROOT"/build/test/soltest $progress $log -- --testpath "$REPO_ROOT"/test "$optimize" --evm-version "$vm" --ipcpath /tmp/test/geth.ipc
   done
 done
 
diff --git a/test/TestHelper.cpp b/test/TestHelper.cpp
index e0d4423d..77fa204f 100644
--- a/test/TestHelper.cpp
+++ b/test/TestHelper.cpp
@@ -43,6 +43,11 @@ Options::Options()
 			ipcPath = suite.argv[i + 1];
 			i++;
 		}
+		else if (string(suite.argv[i]) == "--testpath" && i + 1 < suite.argc)
+		{
+			testPath = suite.argv[i + 1];
+			i++;
+		}
 		else if (string(suite.argv[i]) == "--optimize")
 			optimize = true;
 		else if (string(suite.argv[i]) == "--evm-version")
@@ -60,6 +65,10 @@ Options::Options()
 	if (!disableIPC && ipcPath.empty())
 		if (auto path = getenv("ETH_TEST_IPC"))
 			ipcPath = path;
+
+	if (testPath.empty())
+		if (auto path = getenv("ETH_TEST_PATH"))
+			testPath = path;
 }
 
 dev::solidity::EVMVersion Options::evmVersion() const
diff --git a/test/TestHelper.h b/test/TestHelper.h
index 8c2eec36..9c61be3b 100644
--- a/test/TestHelper.h
+++ b/test/TestHelper.h
@@ -35,6 +35,7 @@ namespace test
 struct Options: boost::noncopyable
 {
 	std::string ipcPath;
+	std::string testPath;
 	bool showMessages = false;
 	bool optimize = false;
 	bool disableIPC = false;
diff --git a/test/boostTest.cpp b/test/boostTest.cpp
index a3cc51c5..fa9df3fd 100644
--- a/test/boostTest.cpp
+++ b/test/boostTest.cpp
@@ -36,6 +36,7 @@
 #pragma GCC diagnostic pop
 
 #include <test/TestHelper.h>
+#include <test/libsolidity/SyntaxTester.h>
 
 using namespace boost::unit_test;
 
@@ -54,6 +55,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 {
 	master_test_suite_t& master = framework::master_test_suite();
 	master.p_name.value = "SolidityTests";
+	dev::solidity::test::SyntaxTester::registerTests();
 	if (dev::test::Options::get().disableIPC)
 	{
 		for (auto suite: {
diff --git a/test/libsolidity/SolidityNameAndTypeResolution.cpp b/test/libsolidity/SolidityNameAndTypeResolution.cpp
index 997b610e..1f76c01b 100644
--- a/test/libsolidity/SolidityNameAndTypeResolution.cpp
+++ b/test/libsolidity/SolidityNameAndTypeResolution.cpp
@@ -43,27 +43,6 @@ namespace test
 
 BOOST_FIXTURE_TEST_SUITE(SolidityNameAndTypeResolution, AnalysisFramework)
 
-BOOST_AUTO_TEST_CASE(smoke_test)
-{
-	char const* text = R"(
-		contract test {
-			uint256 stateVariable1;
-			function fun(uint256 arg1) public { uint256 y; y = arg1; }
-		}
-	)";
-	CHECK_SUCCESS(text);
-}
-
-BOOST_AUTO_TEST_CASE(double_stateVariable_declaration)
-{
-	char const* text = R"(
-		contract test {
-			uint256 variable;
-			uint128 variable;
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared.");
-}
 
 BOOST_AUTO_TEST_CASE(double_function_declaration)
 {
@@ -76,37 +55,6 @@ BOOST_AUTO_TEST_CASE(double_function_declaration)
 	CHECK_ERROR(text, DeclarationError, "Function with same name and arguments defined twice.");
 }
 
-BOOST_AUTO_TEST_CASE(double_variable_declaration)
-{
-	string text = R"(
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_ERROR(text, DeclarationError, "Identifier already declared");
-}
-
-BOOST_AUTO_TEST_CASE(double_variable_declaration_050)
-{
-	string text = R"(
-		pragma experimental "v0.5.0";
-		contract test {
-			function f() pure public {
-				uint256 x;
-				if (true) { uint256 x; }
-			}
-		}
-	)";
-	CHECK_WARNING_ALLOW_MULTI(text, (vector<string>{
-		"This declaration shadows an existing declaration.",
-		"Unused local variable",
-		"Unused local variable"
-	}));
-}
-
 BOOST_AUTO_TEST_CASE(double_variable_declaration_disjoint_scope)
 {
 	string text = R"(
diff --git a/test/libsolidity/SyntaxTestParser.cpp b/test/libsolidity/SyntaxTestParser.cpp
new file mode 100644
index 00000000..c37caca5
--- /dev/null
+++ b/test/libsolidity/SyntaxTestParser.cpp
@@ -0,0 +1,97 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTestParser.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <cctype>
+#include <fstream>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+
+template<typename IteratorType>
+void skipWhitespace(IteratorType& it, IteratorType end)
+{
+	while (it != end && isspace(*it))
+		++it;
+}
+
+template<typename IteratorType>
+void skipSlashes(IteratorType& it, IteratorType end)
+{
+	while (it != end && *it == '/')
+		++it;
+}
+
+std::string SyntaxTestParser::parseSource(std::istream& _stream)
+{
+	std::string source;
+	string line;
+	string const delimiter("// ----");
+	while (getline(_stream, line))
+		if (boost::algorithm::starts_with(line, delimiter))
+			break;
+		else
+			source += line + "\n";
+	return source;
+}
+
+std::vector<SyntaxTestExpectation> SyntaxTestParser::parseExpectations(std::istream& _stream)
+{
+	std::vector<SyntaxTestExpectation> expectations;
+	std::string line;
+	while (getline(_stream, line))
+	{
+		auto it = line.begin();
+
+		skipSlashes(it, line.end());
+		skipWhitespace(it, line.end());
+
+		if (it == line.end()) continue;
+
+		auto typeBegin = it;
+		while (it != line.end() && *it != ':')
+			++it;
+		string errorType(typeBegin, it);
+
+		// skip colon
+		if (it != line.end()) it++;
+
+		skipWhitespace(it, line.end());
+
+		string errorMessage(it, line.end());
+		expectations.emplace_back(SyntaxTestExpectation{move(errorType), move(errorMessage)});
+	}
+	return expectations;
+}
+
+SyntaxTest SyntaxTestParser::parse(string const& _filename)
+{
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("cannot open test contract: \"" + _filename + "\""));
+	file.exceptions(ios::badbit);
+
+	SyntaxTest result;
+	result.source = parseSource(file);
+	result.expectations = parseExpectations(file);
+	return result;
+}
diff --git a/test/libsolidity/SyntaxTestParser.h b/test/libsolidity/SyntaxTestParser.h
new file mode 100644
index 00000000..9e295a0b
--- /dev/null
+++ b/test/libsolidity/SyntaxTestParser.h
@@ -0,0 +1,57 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+struct SyntaxTestExpectation
+{
+	std::string type;
+	std::string message;
+};
+
+struct SyntaxTest
+{
+	std::string source;
+	std::vector<SyntaxTestExpectation> expectations;
+};
+
+class SyntaxTestParser
+{
+public:
+	SyntaxTestParser() = default;
+
+	SyntaxTest parse(std::string const& _filename);
+private:
+	std::string parseSource(std::istream& _stream);
+	std::vector<SyntaxTestExpectation> parseExpectations(std::istream& _stream);
+};
+
+}
+}
+}
diff --git a/test/libsolidity/SyntaxTester.cpp b/test/libsolidity/SyntaxTester.cpp
new file mode 100644
index 00000000..4e545760
--- /dev/null
+++ b/test/libsolidity/SyntaxTester.cpp
@@ -0,0 +1,134 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/SyntaxTester.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/TestHelper.h>
+#include <boost/algorithm/string/replace.hpp>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace std;
+using namespace boost::unit_test;
+namespace fs = boost::filesystem;
+
+void SyntaxTester::runTest(SyntaxTest const& _test)
+{
+	vector<string> unexpectedErrors;
+	auto expectations = _test.expectations;
+	auto errorList = parseAnalyseAndReturnError(_test.source, true, true, true).second;
+
+	bool errorsMatch = true;
+
+	if (errorList.size() != expectations.size())
+		errorsMatch = false;
+	else
+	{
+		for (size_t i = 0; i < errorList.size(); i++)
+		{
+			if (
+				!(errorList[i]->typeName() == expectations[i].type) ||
+				!(errorMessage(*errorList[i]) == expectations[i].message)
+			)
+			{
+				errorsMatch = false;
+				break;
+			}
+		}
+	}
+
+	if (!errorsMatch)
+	{
+		string msg = "Test expectation mismatch.\nExpected result:\n";
+		if (expectations.empty())
+			msg += "\tSuccess\n";
+		else
+			for (auto const& expectation: expectations)
+				msg += "\t" + expectation.type + ": " + expectation.message + "\n";
+		msg += "Obtained result:\n";
+		if (errorList.empty())
+			msg += "\tSuccess\n";
+		else
+			for (auto const& error: errorList)
+				msg += "\t" + error->typeName() + ": " + errorMessage(*error) + "\n";
+		BOOST_ERROR(msg);
+	}
+}
+
+std::string SyntaxTester::errorMessage(Error const& _e)
+{
+	if (_e.comment())
+		return boost::replace_all_copy(*_e.comment(), "\n", "\\n");
+	else
+		return "NONE";
+}
+
+int SyntaxTester::registerTests(
+	test_suite& _suite,
+	fs::path const& _basepath,
+	fs::path const& _path
+)
+{
+
+	int numTestsAdded = 0;
+	fs::path fullpath = _basepath / _path;
+	if (fs::is_directory(fullpath))
+	{
+		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
+		for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
+			fs::directory_iterator(fullpath),
+			fs::directory_iterator()
+		))
+			numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename());
+		_suite.add(sub_suite);
+	}
+	else
+	{
+		_suite.add(make_test_case(
+			[fullpath] { SyntaxTester().runTest(SyntaxTestParser().parse(fullpath.string())); },
+			_path.stem().string(),
+			_path.string(),
+			0
+		));
+		numTestsAdded = 1;
+	}
+	return numTestsAdded;
+}
+
+void SyntaxTester::registerTests()
+{
+	if(dev::test::Options::get().testPath.empty())
+		throw runtime_error(
+			"No path to the test files was specified. "
+			"Use the --testpath command line option or "
+			"the ETH_TEST_PATH environment variable."
+		);
+	auto testPath = fs::path(dev::test::Options::get().testPath);
+
+	if (fs::exists(testPath) && fs::is_directory(testPath))
+	{
+		int numTestsAdded = registerTests(
+			framework::master_test_suite(),
+			testPath / "libsolidity",
+			"syntaxTests"
+		);
+		solAssert(numTestsAdded > 0, "no syntax tests found in libsolidity/syntaxTests");
+	}
+	else
+		solAssert(false, "libsolidity/syntaxTests directory not found");
+}
diff --git a/test/libsolidity/SyntaxTester.h b/test/libsolidity/SyntaxTester.h
new file mode 100644
index 00000000..d61668aa
--- /dev/null
+++ b/test/libsolidity/SyntaxTester.h
@@ -0,0 +1,48 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/SyntaxTestParser.h>
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+class SyntaxTester: public AnalysisFramework
+{
+public:
+	static void registerTests();
+private:
+	static int registerTests(
+		boost::unit_test::test_suite& _suite,
+		boost::filesystem::path const& _basepath,
+		boost::filesystem::path const& _path
+	);
+	static std::string errorMessage(Error const& _e);
+	void runTest(SyntaxTest const& _test);
+};
+
+}
+}
+}
diff --git a/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
new file mode 100644
index 00000000..c5507b64
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_stateVariable_declaration.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 variable;
+	uint128 variable;
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration.sol b/test/libsolidity/syntaxTests/double_variable_declaration.sol
new file mode 100644
index 00000000..3349cfec
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration.sol
@@ -0,0 +1,8 @@
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// DeclarationError: Identifier already declared.
diff --git a/test/libsolidity/syntaxTests/double_variable_declaration_050.sol b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
new file mode 100644
index 00000000..9c2d40d5
--- /dev/null
+++ b/test/libsolidity/syntaxTests/double_variable_declaration_050.sol
@@ -0,0 +1,11 @@
+pragma experimental "v0.5.0";
+contract test {
+	function f() pure public {
+		uint256 x;
+		if (true) { uint256 x; }
+	}
+}
+// ----
+// Warning: This declaration shadows an existing declaration.
+// Warning: Unused local variable.
+// Warning: Unused local variable.
diff --git a/test/libsolidity/syntaxTests/smoke_test.sol b/test/libsolidity/syntaxTests/smoke_test.sol
new file mode 100644
index 00000000..2d48098a
--- /dev/null
+++ b/test/libsolidity/syntaxTests/smoke_test.sol
@@ -0,0 +1,6 @@
+contract test {
+	uint256 stateVariable1;
+	function fun(uint256 arg1) public { uint256 y; y = arg1; }
+}
+// ----
+// Warning: Function state mutability can be restricted to pure