changes to redefine the token list, the scanner, and the parser and how they pass around variable types of different sizes

not ready for change to FixedPoint just yet

made this more const correct and added a switch statement for easier reading

5 files changed, 114 insertions, 125 deletions

diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp
index d9ec1a49..9f944a0c 100644
--- a/libsolidity/parsing/Parser.cpp
+++ b/libsolidity/parsing/Parser.cpp
@@ -194,7 +194,8 @@ ASTPointer<ContractDefinition> Parser::parseContractDefinition(bool _isLibrary)
 	expectToken(Token::LBrace);
 	while (true)
 	{
-		Token::Value currentTokenValue= m_scanner->currentToken();
+		Token::Value currentTokenValue = m_scanner->currentToken();
+		string lit = m_scanner->currentLiteral();
 		if (currentTokenValue == Token::RBrace)
 			break;
 		else if (currentTokenValue == Token::Function)
@@ -590,7 +591,8 @@ ASTPointer<TypeName> Parser::parseTypeName(bool _allowVar)
 	Token::Value token = m_scanner->currentToken();
 	if (Token::isElementaryTypeName(token))
 	{
-		type = ASTNodeFactory(*this).createNode<ElementaryTypeName>(token);
+		ElementaryTypeNameToken elemTypeName(token, m_scanner->currentTokenInfo());
+		type = ASTNodeFactory(*this).createNode<ElementaryTypeName>(elemTypeName);
 		m_scanner->next();
 	}
 	else if (token == Token::Var)
@@ -626,10 +628,12 @@ ASTPointer<Mapping> Parser::parseMapping()
 	ASTNodeFactory nodeFactory(*this);
 	expectToken(Token::Mapping);
 	expectToken(Token::LParen);
-	if (!Token::isElementaryTypeName(m_scanner->currentToken()))
-		fatalParserError(string("Expected elementary type name for mapping key type"));
 	ASTPointer<ElementaryTypeName> keyType;
-	keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>(m_scanner->currentToken());
+	if (Token::isElementaryTypeName(m_scanner->currentToken()))
+		keyType = ASTNodeFactory(*this).createNode<ElementaryTypeName>
+			(ElementaryTypeNameToken(m_scanner->currentToken(), m_scanner->currentTokenInfo()));
+	else
+		fatalParserError(string("Expected elementary type name for mapping key type"));
 	m_scanner->next();
 	expectToken(Token::Arrow);
 	bool const allowVar = false;
@@ -815,12 +819,10 @@ ASTPointer<Statement> Parser::parseSimpleStatement(ASTPointer<ASTString> const&
 	default:
 		break;
 	}
-
 	// At this point, we have 'Identifier "["' or 'Identifier "." Identifier' or 'ElementoryTypeName "["'.
 	// We parse '(Identifier ("." Identifier)* |ElementaryTypeName) ( "[" Expression "]" )+'
 	// until we can decide whether to hand this over to ExpressionStatement or create a
 	// VariableDeclarationStatement out of it.
-
 	vector<ASTPointer<PrimaryExpression>> path;
 	bool startedWithElementary = false;
 	if (m_scanner->currentToken() == Token::Identifier)
@@ -828,7 +830,8 @@ ASTPointer<Statement> Parser::parseSimpleStatement(ASTPointer<ASTString> const&
 	else
 	{
 		startedWithElementary = true;
-		path.push_back(ASTNodeFactory(*this).createNode<ElementaryTypeNameExpression>(m_scanner->currentToken()));
+		ElementaryTypeNameToken elemToken(m_scanner->currentToken(), m_scanner->currentTokenInfo());
+		path.push_back(ASTNodeFactory(*this).createNode<ElementaryTypeNameExpression>(elemToken));
 		m_scanner->next();
 	}
 	while (!startedWithElementary && m_scanner->currentToken() == Token::Period)
@@ -1066,6 +1069,7 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
 	ASTNodeFactory nodeFactory(*this);
 	Token::Value token = m_scanner->currentToken();
 	ASTPointer<Expression> expression;
+
 	switch (token)
 	{
 	case Token::TrueLiteral:
@@ -1134,8 +1138,8 @@ ASTPointer<Expression> Parser::parsePrimaryExpression()
 	default:
 		if (Token::isElementaryTypeName(token))
 		{
-			// used for casts
-			expression = nodeFactory.createNode<ElementaryTypeNameExpression>(token);
+			ElementaryTypeNameToken elementaryExpression(m_scanner->currentToken(), m_scanner->currentTokenInfo());
+			expression = nodeFactory.createNode<ElementaryTypeNameExpression>(elementaryExpression);
 			m_scanner->next();
 		}
 		else
@@ -1226,7 +1230,7 @@ ASTPointer<TypeName> Parser::typeNameIndexAccessStructure(
 	if (auto typeName = dynamic_cast<ElementaryTypeNameExpression const*>(_path.front().get()))
 	{
 		solAssert(_path.size() == 1, "");
-		type = nodeFactory.createNode<ElementaryTypeName>(typeName->typeToken());
+		type = nodeFactory.createNode<ElementaryTypeName>(typeName->typeName());
 	}
 	else
 	{
diff --git a/libsolidity/parsing/Scanner.cpp b/libsolidity/parsing/Scanner.cpp
index fe0807d5..5d40e55b 100644
--- a/libsolidity/parsing/Scanner.cpp
+++ b/libsolidity/parsing/Scanner.cpp
@@ -82,13 +82,9 @@ bool isWhiteSpace(char c)
 {
 	return c == ' ' || c == '\n' || c == '\t' || c == '\r';
 }
-bool isIdentifierStart(char c)
-{
-	return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
-}
 bool isIdentifierPart(char c)
 {
-	return isIdentifierStart(c) || isDecimalDigit(c);
+	return c == '_' || c == '$' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
 }
 
 int hexValue(char c)
@@ -382,8 +378,12 @@ Token::Value Scanner::scanSlash()
 void Scanner::scanToken()
 {
 	m_nextToken.literal.clear();
+	m_nextToken.extendedTokenInfo.clear();
 	m_nextSkippedComment.literal.clear();
+	m_nextSkippedComment.extendedTokenInfo.clear();
+
 	Token::Value token;
+	string tokenExtension = "";
 	do
 	{
 		// Remember the position of the next token
@@ -550,8 +550,8 @@ void Scanner::scanToken()
 			token = selectToken(Token::BitNot);
 			break;
 		default:
-			if (isIdentifierStart(m_char))
-				token = scanIdentifierOrKeyword();
+			if (isIdentifierPart(m_char))
+				tie(token, tokenExtension) = scanIdentifierOrKeyword();
 			else if (isDecimalDigit(m_char))
 				token = scanNumber();
 			else if (skipWhitespace())
@@ -568,6 +568,7 @@ void Scanner::scanToken()
 	while (token == Token::Whitespace);
 	m_nextToken.location.end = sourcePos();
 	m_nextToken.token = token;
+	m_nextToken.extendedTokenInfo = tokenExtension;
 }
 
 bool Scanner::scanEscape()
@@ -699,22 +700,34 @@ Token::Value Scanner::scanNumber(char _charSeen)
 	// not be an identifier start or a decimal digit; see ECMA-262
 	// section 7.8.3, page 17 (note that we read only one decimal digit
 	// if the value is 0).
-	if (isDecimalDigit(m_char) || isIdentifierStart(m_char))
+	if (isDecimalDigit(m_char) || isIdentifierPart(m_char))
 		return Token::Illegal;
 	literal.complete();
 	return Token::Number;
 }
 
-Token::Value Scanner::scanIdentifierOrKeyword()
+tuple<Token::Value, string> Scanner::scanIdentifierOrKeyword()
 {
-	solAssert(isIdentifierStart(m_char), "");
+	solAssert(isIdentifierPart(m_char), "");
 	LiteralScope literal(this, LITERAL_TYPE_STRING);
 	addLiteralCharAndAdvance();
 	// Scan the rest of the identifier characters.
-	while (isIdentifierPart(m_char))
+	string keyword = "";
+	string description = "";
+	while (isIdentifierPart(m_char)) //get main keyword
+		addLiteralCharAndAdvance();
+	keyword = m_nextToken.literal;
+	while (isDecimalDigit(m_char) || isIdentifierPart(m_char)) //get the description
 		addLiteralCharAndAdvance();
 	literal.complete();
-	return Token::fromIdentifierOrKeyword(m_nextToken.literal);
+	if (m_nextToken.literal.find_first_of("0123456789") != string::npos)
+	{
+		description = m_nextToken.literal.substr(m_nextToken.literal.find_first_of("0123456789"));
+		keyword += "M";
+		if (description.find('x') != string::npos)
+			keyword += "xN";
+	}
+	return make_tuple(Token::fromIdentifierOrKeyword(keyword), description);
 }
 
 char CharStream::advanceAndGet(size_t _chars)
diff --git a/libsolidity/parsing/Scanner.h b/libsolidity/parsing/Scanner.h
index a1a5c9c1..fa1f118b 100644
--- a/libsolidity/parsing/Scanner.h
+++ b/libsolidity/parsing/Scanner.h
@@ -122,6 +122,7 @@ public:
 
 	SourceLocation currentLocation() const { return m_currentToken.location; }
 	std::string const& currentLiteral() const { return m_currentToken.literal; }
+	std::string const& currentTokenInfo() const { return m_currentToken.extendedTokenInfo; }
 	///@}
 
 	///@{
@@ -160,6 +161,7 @@ private:
 		Token::Value token;
 		SourceLocation location;
 		std::string literal;
+		std::string extendedTokenInfo;
 	};
 
 	///@{
@@ -190,7 +192,7 @@ private:
 
 	void scanDecimalDigits();
 	Token::Value scanNumber(char _charSeen = 0);
-	Token::Value scanIdentifierOrKeyword();
+	std::tuple<Token::Value, std::string> scanIdentifierOrKeyword();
 
 	Token::Value scanString();
 	Token::Value scanSingleLineDocComment();
diff --git a/libsolidity/parsing/Token.cpp b/libsolidity/parsing/Token.cpp
index cda639fb..7ea94112 100644
--- a/libsolidity/parsing/Token.cpp
+++ b/libsolidity/parsing/Token.cpp
@@ -50,6 +50,44 @@ namespace dev
 namespace solidity
 {
 
+bool ElementaryTypeNameToken::isElementaryTypeName(Token::Value _baseType, string const& _info)
+{
+	if (!Token::isElementaryTypeName(_baseType))
+		return false;
+	string baseType = Token::toString(_baseType);
+	if (baseType.find('M') == string::npos)
+		return true;
+	short m;
+	m = stoi(_info.substr(_info.find_first_of("0123456789")));
+
+	if (baseType == "bytesM")
+		return (0 < m && m <= 32) ? true : false;
+	else if (baseType == "uintM" || baseType == "intM")
+		return (0 < m && m <= 256 && m % 8 == 0) ? true : false;
+	else if (baseType == "ufixedMxN" || baseType == "fixedMxN")
+	{
+		short n;
+		m = stoi(_info.substr(_info.find_first_of("0123456789"), _info.find_last_of("x") - 1));
+		n = stoi(_info.substr(_info.find_last_of("x") + 1));
+		return (0 < n + m && n + m <= 256 && ((n % 8 == 0) && (m % 8 == 0)));
+	}
+	return false;
+}
+
+tuple<string, unsigned int, unsigned int> ElementaryTypeNameToken::setTypes(Token::Value _baseType, string const& _toSet)
+{
+	string baseType = Token::toString(_baseType);
+	if (_toSet.find_first_of("0123456789") == string::npos)
+		return make_tuple(baseType, 0, 0);
+	baseType = baseType.substr(0, baseType.find('M') - 1) + _toSet;
+	size_t index = _toSet.find('x') == string::npos ? string::npos : _toSet.find('x') - 1;
+	unsigned int m = stoi(_toSet.substr(0, index));
+	unsigned int n = 0;
+	if (baseType == "fixed" || baseType == "ufixed")
+		n = stoi(_toSet.substr(_toSet.find('x') + 1));
+	return make_tuple(baseType, m, n);
+}
+
 #define T(name, string, precedence) #name,
 char const* const Token::m_name[NUM_TOKENS] =
 {
@@ -80,7 +118,7 @@ char const Token::m_tokenType[] =
 {
 	TOKEN_LIST(KT, KK)
 };
-Token::Value Token::fromIdentifierOrKeyword(const std::string& _name)
+Token::Value Token::fromIdentifierOrKeyword(const string& _name)
 {
 	// The following macros are used inside TOKEN_LIST and cause non-keyword tokens to be ignored
 	// and keywords to be put inside the keywords variable.
diff --git a/libsolidity/parsing/Token.h b/libsolidity/parsing/Token.h
index 88e77345..5f8141ae 100644
--- a/libsolidity/parsing/Token.h
+++ b/libsolidity/parsing/Token.h
@@ -187,113 +187,21 @@ namespace solidity
 	K(SubWeek, "weeks", 0)                                             \
 	K(SubYear, "years", 0)                                             \
 	K(After, "after", 0)                                               \
-	/* type keywords, keep them in this order, keep int as first keyword
-	 * the implementation in Types.cpp has to be synced to this here */\
+	/* type keywords*/                                                 \
 	K(Int, "int", 0)                                                   \
-	K(Int8, "int8", 0)                                                 \
-	K(Int16, "int16", 0)                                               \
-	K(Int24, "int24", 0)                                               \
-	K(Int32, "int32", 0)                                               \
-	K(Int40, "int40", 0)                                               \
-	K(Int48, "int48", 0)                                               \
-	K(Int56, "int56", 0)                                               \
-	K(Int64, "int64", 0)                                               \
-	K(Int72, "int72", 0)                                               \
-	K(Int80, "int80", 0)                                               \
-	K(Int88, "int88", 0)                                               \
-	K(Int96, "int96", 0)                                               \
-	K(Int104, "int104", 0)                                             \
-	K(Int112, "int112", 0)                                             \
-	K(Int120, "int120", 0)                                             \
-	K(Int128, "int128", 0)                                             \
-	K(Int136, "int136", 0)                                             \
-	K(Int144, "int144", 0)                                             \
-	K(Int152, "int152", 0)                                             \
-	K(Int160, "int160", 0)                                             \
-	K(Int168, "int168", 0)                                             \
-	K(Int176, "int176", 0)                                             \
-	K(Int184, "int184", 0)                                             \
-	K(Int192, "int192", 0)                                             \
-	K(Int200, "int200", 0)                                             \
-	K(Int208, "int208", 0)                                             \
-	K(Int216, "int216", 0)                                             \
-	K(Int224, "int224", 0)                                             \
-	K(Int232, "int232", 0)                                             \
-	K(Int240, "int240", 0)                                             \
-	K(Int248, "int248", 0)                                             \
-	K(Int256, "int256", 0)                                             \
+	K(IntM, "intM", 0)                                                 \
 	K(UInt, "uint", 0)                                                 \
-	K(UInt8, "uint8", 0)                                               \
-	K(UInt16, "uint16", 0)                                             \
-	K(UInt24, "uint24", 0)                                             \
-	K(UInt32, "uint32", 0)                                             \
-	K(UInt40, "uint40", 0)                                             \
-	K(UInt48, "uint48", 0)                                             \
-	K(UInt56, "uint56", 0)                                             \
-	K(UInt64, "uint64", 0)                                             \
-	K(UInt72, "uint72", 0)                                             \
-	K(UInt80, "uint80", 0)                                             \
-	K(UInt88, "uint88", 0)                                             \
-	K(UInt96, "uint96", 0)                                             \
-	K(UInt104, "uint104", 0)                                           \
-	K(UInt112, "uint112", 0)                                           \
-	K(UInt120, "uint120", 0)                                           \
-	K(UInt128, "uint128", 0)                                           \
-	K(UInt136, "uint136", 0)                                           \
-	K(UInt144, "uint144", 0)                                           \
-	K(UInt152, "uint152", 0)                                           \
-	K(UInt160, "uint160", 0)                                           \
-	K(UInt168, "uint168", 0)                                           \
-	K(UInt176, "uint176", 0)                                           \
-	K(UInt184, "uint184", 0)                                           \
-	K(UInt192, "uint192", 0)                                           \
-	K(UInt200, "uint200", 0)                                           \
-	K(UInt208, "uint208", 0)                                           \
-	K(UInt216, "uint216", 0)                                           \
-	K(UInt224, "uint224", 0)                                           \
-	K(UInt232, "uint232", 0)                                           \
-	K(UInt240, "uint240", 0)                                           \
-	K(UInt248, "uint248", 0)                                           \
-	K(UInt256, "uint256", 0)                                           \
-	K(Bytes1, "bytes1", 0)                                             \
-	K(Bytes2, "bytes2", 0)                                             \
-	K(Bytes3, "bytes3", 0)                                             \
-	K(Bytes4, "bytes4", 0)                                             \
-	K(Bytes5, "bytes5", 0)                                             \
-	K(Bytes6, "bytes6", 0)                                             \
-	K(Bytes7, "bytes7", 0)                                             \
-	K(Bytes8, "bytes8", 0)                                             \
-	K(Bytes9, "bytes9", 0)                                             \
-	K(Bytes10, "bytes10", 0)                                           \
-	K(Bytes11, "bytes11", 0)                                           \
-	K(Bytes12, "bytes12", 0)                                           \
-	K(Bytes13, "bytes13", 0)                                           \
-	K(Bytes14, "bytes14", 0)                                           \
-	K(Bytes15, "bytes15", 0)                                           \
-	K(Bytes16, "bytes16", 0)                                           \
-	K(Bytes17, "bytes17", 0)                                           \
-	K(Bytes18, "bytes18", 0)                                           \
-	K(Bytes19, "bytes19", 0)                                           \
-	K(Bytes20, "bytes20", 0)                                           \
-	K(Bytes21, "bytes21", 0)                                           \
-	K(Bytes22, "bytes22", 0)                                           \
-	K(Bytes23, "bytes23", 0)                                           \
-	K(Bytes24, "bytes24", 0)                                           \
-	K(Bytes25, "bytes25", 0)                                           \
-	K(Bytes26, "bytes26", 0)                                           \
-	K(Bytes27, "bytes27", 0)                                           \
-	K(Bytes28, "bytes28", 0)                                           \
-	K(Bytes29, "bytes29", 0)                                           \
-	K(Bytes30, "bytes30", 0)                                           \
-	K(Bytes31, "bytes31", 0)                                           \
-	K(Bytes32, "bytes32", 0)                                           \
+	K(UIntM, "uintM", 0)                                               \
 	K(Bytes, "bytes", 0)                                               \
+	K(BytesM, "bytesM", 0)                                             \
 	K(Byte, "byte", 0)                                                 \
 	K(String, "string", 0)                                             \
 	K(Address, "address", 0)                                           \
 	K(Bool, "bool", 0)                                                 \
-	K(Real, "real", 0)                                                 \
-	K(UReal, "ureal", 0)                                               \
+	K(Fixed, "fixed", 0)                                               \
+	K(FixedMxN, "fixedMxN", 0)                                         \
+	K(UFixed, "ufixed", 0)                                             \
+	K(UFixedMxN, "ufixedMxN", 0)                                       \
 	T(TypesEnd, NULL, 0) /* used as type enum end marker */            \
 	\
 	/* Literals */                                                     \
@@ -403,5 +311,29 @@ private:
 	static char const m_tokenType[NUM_TOKENS];
 };
 
+class ElementaryTypeNameToken
+{
+public:
+	ElementaryTypeNameToken(Token::Value token, std::string const& description)
+	{
+		solAssert(isElementaryTypeName(token, description), "");
+		std::tie(m_name, M, N) = setTypes(token, description);
+		tok = token;
+	}
+	
+	std::string toString(bool const& tokValue = false) const& { return tokValue ? Token::toString(tok) : m_name; }
+	unsigned int const& returnM() const& { return M; }
+	unsigned int const& returnN() const& { return N; }
+	Token::Value const& returnTok() const& { return tok; }
+	static bool isElementaryTypeName(Token::Value _baseType, std::string const& _info);
+
+private:
+	Token::Value tok;
+	std::string m_name;
+	unsigned int M;
+	unsigned int N;
+	std::tuple<std::string, unsigned int, unsigned int> setTypes(Token::Value _baseType, std::string const& _toSet);
+};
+
 }
 }