added two functions in Token to handle long identifiers, redid fromIdentifierOrKeyword, and made complementary changes in scanner and parser

1 files changed, 8 insertions, 6 deletions

diff --git a/libsolidity/parsing/Scanner.cpp b/libsolidity/parsing/Scanner.cpp
index 9307b100..510d283e 100644
--- a/libsolidity/parsing/Scanner.cpp
+++ b/libsolidity/parsing/Scanner.cpp
@@ -381,12 +381,14 @@ Token::Value Scanner::scanSlash()
 void Scanner::scanToken()
 {
 	m_nextToken.literal.clear();
-	m_nextToken.extendedTokenInfo.clear();
+	m_nextToken.extendedTokenInfo = make_tuple(0, 0);
 	m_nextSkippedComment.literal.clear();
-	m_nextSkippedComment.extendedTokenInfo.clear();
+	m_nextSkippedComment.extendedTokenInfo = make_tuple(0, 0);
 
 	Token::Value token;
-	string tokenExtension;
+	// M and N are for the purposes of grabbing different type sizes
+	unsigned M;
+	unsigned N;
 	do
 	{
 		// Remember the position of the next token
@@ -554,7 +556,7 @@ void Scanner::scanToken()
 			break;
 		default:
 			if (isIdentifierStart(m_char))
-				tie(token, tokenExtension) = scanIdentifierOrKeyword();
+				tie(token, M, N) = scanIdentifierOrKeyword();
 			else if (isDecimalDigit(m_char))
 				token = scanNumber();
 			else if (skipWhitespace())
@@ -571,7 +573,7 @@ void Scanner::scanToken()
 	while (token == Token::Whitespace);
 	m_nextToken.location.end = sourcePos();
 	m_nextToken.token = token;
-	m_nextToken.extendedTokenInfo = tokenExtension;
+	m_nextToken.extendedTokenInfo = make_tuple(M,N);
 }
 
 bool Scanner::scanEscape()
@@ -709,7 +711,7 @@ Token::Value Scanner::scanNumber(char _charSeen)
 	return Token::Number;
 }
 
-tuple<Token::Value, string> Scanner::scanIdentifierOrKeyword()
+tuple<Token::Value, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
 {
 	solAssert(isIdentifierStart(m_char), "");
 	LiteralScope literal(this, LITERAL_TYPE_STRING);