From ada31fa9b72434a24471116d0c7281d7499039b9 Mon Sep 17 00:00:00 2001
From: chriseth <c@ethdev.com>
Date: Fri, 2 Sep 2016 00:21:39 +0200
Subject: Do not emit non-utf8 strings for ast json.

---
 libsolidity/ast/ASTJsonConverter.cpp | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/libsolidity/ast/ASTJsonConverter.cpp b/libsolidity/ast/ASTJsonConverter.cpp
index 49ee6d34..0ea5e093 100644
--- a/libsolidity/ast/ASTJsonConverter.cpp
+++ b/libsolidity/ast/ASTJsonConverter.cpp
@@ -22,6 +22,7 @@
 
 #include <libsolidity/ast/ASTJsonConverter.h>
 #include <boost/algorithm/string/join.hpp>
+#include <libdevcore/UTF8.h>
 #include <libsolidity/ast/AST.h>
 
 using namespace std;
@@ -397,9 +398,21 @@ bool ASTJsonConverter::visit(ElementaryTypeNameExpression const& _node)
 bool ASTJsonConverter::visit(Literal const& _node)
 {
 	char const* tokenString = Token::toString(_node.token());
+	size_t invalidPos = 0;
+	Json::Value value{_node.value()};
+	if (!dev::validate(_node.value(), invalidPos))
+		value = Json::nullValue;
+	Token::Value subdenomination = Token::Value(_node.subDenomination());
 	addJsonNode(_node, "Literal", {
-		make_pair("string", tokenString ? tokenString : Json::Value()),
-		make_pair("value", _node.value()),
+		make_pair("token", tokenString ? tokenString : Json::Value()),
+		make_pair("value", value),
+		make_pair("hexvalue", toHex(_node.value())),
+		make_pair(
+			"subdenomination",
+			subdenomination == Token::Illegal ?
+			Json::nullValue :
+			Json::Value{Token::toString(subdenomination)}
+		),
 		make_pair("type", type(_node))
 	});
 	return true;
-- 
cgit 


From 43c28c1ede33a80b6c259988dde6059f493a41a9 Mon Sep 17 00:00:00 2001
From: chriseth <c@ethdev.com>
Date: Tue, 6 Sep 2016 11:07:56 +0200
Subject: Test case for non-utf8 characters in string literal.

---
 test/libsolidity/ASTJSON.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/libsolidity/ASTJSON.cpp b/test/libsolidity/ASTJSON.cpp
index ec60b668..c37f3542 100644
--- a/test/libsolidity/ASTJSON.cpp
+++ b/test/libsolidity/ASTJSON.cpp
@@ -195,6 +195,22 @@ BOOST_AUTO_TEST_CASE(placeholder_statement)
 	BOOST_CHECK_EQUAL(placeholder["src"], "26:1:1");
 }
 
+BOOST_AUTO_TEST_CASE(non_utf8)
+{
+	CompilerStack c;
+	c.addSource("a", "contract C { function f() { var x = hex\"ff\"; } }");
+	c.parse();
+	map<string, unsigned> sourceIndices;
+	sourceIndices["a"] = 1;
+	Json::Value astJson = ASTJsonConverter(c.ast("a"), sourceIndices).json();
+	Json::Value literal = astJson["children"][0]["children"][0]["children"][2]["children"][0]["children"][1];
+	BOOST_CHECK_EQUAL(literal["name"], "Literal");
+	BOOST_CHECK_EQUAL(literal["attributes"]["hexvalue"], "ff");
+	BOOST_CHECK_EQUAL(literal["attributes"]["token"], Json::nullValue);
+	BOOST_CHECK_EQUAL(literal["attributes"]["value"], Json::nullValue);
+	BOOST_CHECK(literal["attributes"]["type"].asString().find("invalid") != string::npos);
+}
+
 BOOST_AUTO_TEST_SUITE_END()
 
 }
-- 
cgit