Infrastructure for extracting JSON AST tests.

5 files changed, 302 insertions, 4 deletions

diff --git a/test/boostTest.cpp b/test/boostTest.cpp
index 6c68100c..cef3b06f 100644
--- a/test/boostTest.cpp
+++ b/test/boostTest.cpp
@@ -36,6 +36,7 @@
 #pragma GCC diagnostic pop
 
 #include <test/Options.h>
+#include <test/libsolidity/ASTJSONTest.h>
 #include <test/libsolidity/SyntaxTest.h>
 
 #include <boost/algorithm/string.hpp>
@@ -131,6 +132,12 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 		"syntaxTests",
 		SyntaxTest::create
 	) > 0, "no syntax tests found");
+	solAssert(registerTests(
+		master,
+		dev::test::Options::get().testPath / "libsolidity",
+		"ASTJSON",
+		ASTJSONTest::create
+	) > 0, "no JSON AST tests found");
 	if (dev::test::Options::get().disableIPC)
 	{
 		for (auto suite: {
diff --git a/test/libsolidity/ASTJSONTest.cpp b/test/libsolidity/ASTJSONTest.cpp
new file mode 100644
index 00000000..239ef9ff
--- /dev/null
+++ b/test/libsolidity/ASTJSONTest.cpp
@@ -0,0 +1,201 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <test/libsolidity/ASTJSONTest.h>
+#include <test/Options.h>
+#include <libsolidity/ast/ASTJsonConverter.h>
+#include <libsolidity/interface/CompilerStack.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/throw_exception.hpp>
+#include <cctype>
+#include <fstream>
+#include <memory>
+#include <stdexcept>
+
+using namespace dev;
+using namespace solidity;
+using namespace dev::solidity::test;
+using namespace dev::solidity::test::formatting;
+using namespace std;
+namespace fs = boost::filesystem;
+using namespace boost::unit_test;
+
+ASTJSONTest::ASTJSONTest(string const& _filename)
+{
+	if (!boost::algorithm::ends_with(_filename, ".sol"))
+		BOOST_THROW_EXCEPTION(runtime_error("Invalid test contract file name: \"" + _filename + "\"."));
+
+	m_astFilename = _filename.substr(0, _filename.size() - 4) + ".json";
+	m_legacyAstFilename = _filename.substr(0, _filename.size() - 4) + "_legacy.json";
+
+	ifstream file(_filename);
+	if (!file)
+		BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\"."));
+	file.exceptions(ios::badbit);
+
+	string sourceName;
+	string source;
+	string line;
+	string const sourceDelimiter("// ---- SOURCE: ");
+	string const delimiter("// ----");
+	while (getline(file, line))
+	{
+		if (boost::algorithm::starts_with(line, sourceDelimiter))
+		{
+			if (!sourceName.empty())
+				m_sources.emplace_back(sourceName, source);
+
+			sourceName = line.substr(sourceDelimiter.size(), string::npos);
+			source = string();
+		}
+		else if (!line.empty() && !boost::algorithm::starts_with(line, delimiter))
+			source += line + "\n";
+	}
+
+	m_sources.emplace_back(sourceName.empty() ? "a" : sourceName, source);
+
+	file.close();
+	file.open(m_astFilename);
+	if (file)
+	{
+		string line;
+		while (getline(file, line))
+			m_expectation += line + "\n";
+	}
+
+	file.close();
+	file.open(m_legacyAstFilename);
+	if (file)
+	{
+		string line;
+		while (getline(file, line))
+			m_expectationLegacy += line + "\n";
+	}
+}
+
+bool ASTJSONTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted)
+{
+	CompilerStack c;
+
+	map<string, unsigned> sourceIndices;
+	for (size_t i = 0; i < m_sources.size(); i++)
+	{
+		c.addSource(m_sources[i].first, m_sources[i].second);
+		sourceIndices[m_sources[i].first] = i;
+	}
+
+	c.setEVMVersion(dev::test::Options::get().evmVersion());
+	c.parseAndAnalyze();
+
+	for (size_t i = 0; i < m_sources.size(); i++)
+	{
+		ostringstream result;
+		ASTJsonConverter(false, sourceIndices).print(result, c.ast(m_sources[i].first));
+		m_result += result.str();
+		if (i != m_sources.size() - 1)
+			m_result += ",";
+		m_result += "\n";
+	}
+
+	bool resultsMatch = true;
+
+	if (m_expectation != m_result)
+	{
+		string nextIndentLevel = _linePrefix + "  ";
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
+		{
+			istringstream stream(m_expectation);
+			string line;
+			while (getline(stream, line))
+				_stream << nextIndentLevel << line << endl;
+		}
+		_stream << endl;
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
+		{
+			istringstream stream(m_result);
+			string line;
+			while (getline(stream, line))
+				_stream << nextIndentLevel << line << endl;
+		}
+		_stream << endl;
+		resultsMatch = false;
+	}
+
+	for (size_t i = 0; i < m_sources.size(); i++)
+	{
+		ostringstream result;
+		ASTJsonConverter(true, sourceIndices).print(result, c.ast(m_sources[i].first));
+		m_resultLegacy = result.str();
+		if (i != m_sources.size() - 1)
+			m_resultLegacy += ",";
+		m_resultLegacy += "\n";
+	}
+
+	if (m_expectationLegacy != m_resultLegacy)
+	{
+		string nextIndentLevel = _linePrefix + "  ";
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result (legacy):" << endl;
+		{
+			istringstream stream(m_expectationLegacy);
+			string line;
+			while (getline(stream, line))
+				_stream << nextIndentLevel << line << endl;
+		}
+		_stream << endl;
+		FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result (legacy):" << endl;
+		{
+			istringstream stream(m_resultLegacy);
+			string line;
+			while (getline(stream, line))
+				_stream << nextIndentLevel << line << endl;
+		}
+		_stream << endl;
+		resultsMatch = false;
+	}
+
+	return resultsMatch;
+}
+
+void ASTJSONTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const
+{
+	for (auto const& source: m_sources)
+	{
+		if (m_sources.size() > 1 || source.first != "a")
+			_stream << _linePrefix << "// ---- SOURCE: " << source.first << endl << endl;
+		stringstream stream(source.second);
+		string line;
+		while (getline(stream, line))
+			_stream << _linePrefix << line << endl;
+		_stream << endl;
+	}
+}
+
+void ASTJSONTest::printUpdatedExpectations(std::ostream&, std::string const&) const
+{
+	ofstream file(m_astFilename.c_str());
+	if (!file) BOOST_THROW_EXCEPTION(runtime_error("Cannot write AST expectation to \"" + m_astFilename + "\"."));
+	file.exceptions(ios::badbit);
+	file << m_result;
+	file.flush();
+	file.close();
+	file.open(m_legacyAstFilename.c_str());
+	if (!file) BOOST_THROW_EXCEPTION(runtime_error("Cannot write legacy AST expectation to \"" + m_legacyAstFilename + "\"."));
+	file << m_resultLegacy;
+	file.flush();
+	file.close();
+}
diff --git a/test/libsolidity/ASTJSONTest.h b/test/libsolidity/ASTJSONTest.h
new file mode 100644
index 00000000..6f24bb60
--- /dev/null
+++ b/test/libsolidity/ASTJSONTest.h
@@ -0,0 +1,58 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/FormattedScope.h>
+#include <test/libsolidity/TestCase.h>
+
+#include <iosfwd>
+#include <string>
+#include <vector>
+#include <utility>
+
+namespace dev
+{
+namespace solidity
+{
+namespace test
+{
+
+class ASTJSONTest: public TestCase
+{
+public:
+	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_filename)); }
+	ASTJSONTest(std::string const& _filename);
+
+	virtual bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;
+
+	virtual void printSource(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) const override;
+	virtual void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix) const override;
+private:
+	std::vector<std::pair<std::string, std::string>> m_sources;
+	std::string m_expectation;
+	std::string m_expectationLegacy;
+	std::string m_astFilename;
+	std::string m_legacyAstFilename;
+	std::string m_result;
+	std::string m_resultLegacy;
+};
+
+}
+}
+}
diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt
index 257b4f24..d6df0ac8 100644
--- a/test/tools/CMakeLists.txt
+++ b/test/tools/CMakeLists.txt
@@ -3,5 +3,5 @@ target_link_libraries(solfuzzer PRIVATE libsolc evmasm ${Boost_PROGRAM_OPTIONS_L
 
 add_executable(isoltest isoltest.cpp ../Options.cpp ../libsolidity/TestCase.cpp ../libsolidity/SyntaxTest.cpp
         ../libsolidity/AnalysisFramework.cpp ../libsolidity/SolidityExecutionFramework.cpp ../ExecutionFramework.cpp
-        ../RPCSession.cpp)
+        ../RPCSession.cpp ../libsolidity/ASTJSONTest.cpp)
 target_link_libraries(isoltest PRIVATE libsolc solidity evmasm ${Boost_PROGRAM_OPTIONS_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp
index bd4b0db9..7d15b07a 100644
--- a/test/tools/isoltest.cpp
+++ b/test/tools/isoltest.cpp
@@ -18,6 +18,7 @@
 #include <libdevcore/CommonIO.h>
 #include <test/libsolidity/AnalysisFramework.h>
 #include <test/libsolidity/SyntaxTest.h>
+#include <test/libsolidity/ASTJSONTest.h>
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/replace.hpp>
@@ -336,22 +337,53 @@ Allowed options)",
 		}
 	}
 
+	TestStats global_stats { 0, 0 };
+
 	fs::path syntaxTestPath = testPath / "libsolidity" / "syntaxTests";
 
 	if (fs::exists(syntaxTestPath) && fs::is_directory(syntaxTestPath))
 	{
 		auto stats = TestTool::processPath(SyntaxTest::create, testPath / "libsolidity", "syntaxTests", formatted);
 
-		cout << endl << "Summary: ";
+		cout << endl << "Syntax Test Summary: ";
 		FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
 			stats.successCount << "/" << stats.runCount;
-		cout << " tests successful." << endl;
+		cout << " tests successful." << endl << endl;
 
-		return stats ? 0 : 1;
+		global_stats.runCount += stats.runCount;
+		global_stats.successCount += stats.successCount;
 	}
 	else
 	{
 		cerr << "Syntax tests not found. Use the --testpath argument." << endl;
 		return 1;
 	}
+
+	fs::path astJsonTestPath = testPath / "libsolidity" / "ASTJSON";
+
+	if (fs::exists(astJsonTestPath) && fs::is_directory(astJsonTestPath))
+	{
+		auto stats = TestTool::processPath(ASTJSONTest::create, testPath / "libsolidity", "ASTJSON", formatted);
+
+		cout << endl << "JSON AST Test Summary: ";
+		FormattedScope(cout, formatted, {BOLD, stats ? GREEN : RED}) <<
+			stats.successCount << "/" << stats.runCount;
+		cout << " tests successful." << endl << endl;
+
+		global_stats.runCount += stats.runCount;
+		global_stats.successCount += stats.successCount;
+	}
+	else
+	{
+		cerr << "JSON AST tests not found." << endl;
+		return 1;
+	}
+
+	cout << endl << "Summary: ";
+	FormattedScope(cout, formatted, {BOLD, global_stats ? GREEN : RED}) <<
+		 global_stats.successCount << "/" << global_stats.runCount;
+	cout << " tests successful." << endl;
+
+
+	return global_stats ? 0 : 1;
 }