diff options
11 files changed, 79 insertions, 36 deletions
diff --git a/liblangutil/Scanner.cpp b/liblangutil/Scanner.cpp index 3d7527d4..091e9b89 100644 --- a/liblangutil/Scanner.cpp +++ b/liblangutil/Scanner.cpp @@ -311,7 +311,7 @@ Token Scanner::skipMultiLineComment() } } // Unterminated multi-line comment. - return Token::Illegal; + return Token::IllegalCommentTerminator; } Token Scanner::scanMultiLineDocComment() @@ -362,7 +362,7 @@ Token Scanner::scanMultiLineDocComment() } literal.complete(); if (!endFound) - return Token::Illegal; + return Token::IllegalCommentTerminator; else return Token::CommentLiteral; } @@ -392,7 +392,7 @@ Token Scanner::scanSlash() { // doxygen style /** natspec comment if (!advance()) /* slash star comment before EOS */ - return Token::Illegal; + return Token::IllegalCommentTerminator; else if (m_char == '*') { advance(); //consume the last '*' at /** @@ -409,8 +409,9 @@ Token Scanner::scanSlash() comment = scanMultiLineDocComment(); m_nextSkippedComment.location.end = sourcePos(); m_nextSkippedComment.token = comment; - if (comment == Token::Illegal) - return Token::Illegal; + // @todo possibly: if (comment.isIllegal) return comment; to pass all errors + if (comment == Token::IllegalCommentTerminator) + return Token::IllegalCommentTerminator; else return Token::Whitespace; } @@ -620,6 +621,7 @@ void Scanner::scanToken() else if (isSourcePastEndOfInput()) token = Token::EOS; else + // @todo verfiy if this is actually an "IllegalUnknown" case token = selectToken(Token::Illegal); break; } @@ -713,13 +715,13 @@ Token Scanner::scanString() if (c == '\\') { if (isSourcePastEndOfInput() || !scanEscape()) - return Token::Illegal; + return Token::IllegalStringEscape; } else addLiteralChar(c); } if (m_char != quote) - return Token::Illegal; + return Token::IllegalStringEndQuote; literal.complete(); advance(); // consume quote return Token::StringLiteral; @@ -767,7 +769,8 @@ Token Scanner::scanNumber(char _charSeen) // we have already seen a decimal point of the float addLiteralChar('.'); if (m_char == '_') - return Token::Illegal; + // @todo add test-case (change of return value did not break test) + return Token::IllegalNumberSeparator; scanDecimalDigits(); // we know we have at least one digit } else @@ -784,14 +787,14 @@ Token Scanner::scanNumber(char _charSeen) kind = HEX; addLiteralCharAndAdvance(); if (!isHexDigit(m_char)) - return Token::Illegal; // we must have at least one hex digit after 'x' + return Token::IllegalHexDigit; // we must have at least one hex digit after 'x' while (isHexDigit(m_char) || m_char == '_') // We keep the underscores for later validation addLiteralCharAndAdvance(); } else if (isDecimalDigit(m_char)) // We do not allow octal numbers - return Token::Illegal; + return Token::IllegalOctalNotAllowed; } // Parse decimal digits and allow trailing fractional part. if (kind == DECIMAL) @@ -823,7 +826,8 @@ Token Scanner::scanNumber(char _charSeen) { solAssert(kind != HEX, "'e'/'E' must be scanned as part of the hex number"); if (kind != DECIMAL) - return Token::Illegal; + // @todo add test (change introduced no failing) + return Token::IllegalExponent; else if (!m_source.isPastEndOfInput(1) && m_source.get(1) == '_') { // Recover from wrongly placed underscore as delimiter in literal with scientific @@ -839,7 +843,7 @@ Token Scanner::scanNumber(char _charSeen) if (m_char == '+' || m_char == '-') addLiteralCharAndAdvance(); if (!isDecimalDigit(m_char)) - return Token::Illegal; // we must have at least one decimal digit after 'e'/'E' + return Token::IllegalExponent; // we must have at least one decimal digit after 'e'/'E' scanDecimalDigits(); } // The source character immediately following a numeric literal must @@ -847,7 +851,7 @@ Token Scanner::scanNumber(char _charSeen) // section 7.8.3, page 17 (note that we read only one decimal digit // if the value is 0). if (isDecimalDigit(m_char) || isIdentifierStart(m_char)) - return Token::Illegal; + return Token::IllegalNumberEnd; literal.complete(); return Token::Number; } diff --git a/liblangutil/Token.h b/liblangutil/Token.h index d997b138..55d474f5 100644 --- a/liblangutil/Token.h +++ b/liblangutil/Token.h @@ -265,6 +265,15 @@ namespace langutil T(Illegal, "ILLEGAL", 0) \ /* Illegal hex token */ \ T(IllegalHex, "ILLEGAL_HEX", 0) \ + T(IllegalCommentTerminator, "ILLEGAL_COMMENT_TERMINATOR", 0) \ + T(IllegalStringEscape, "ILLEGAL_STRING_ESCAPE", 0) \ + T(IllegalStringEndQuote, "ILLEGAL_STRING_END_QUOTE", 0) \ + T(IllegalNumberSeparator, "ILLEGAL_NUMER_SEPARATOR", 0) \ + T(IllegalHexDigit, "ILLEGAL_HEX_DIGIT", 0) \ + T(IllegalOctalNotAllowed, "ILLEGAL_OCTAL_NOT_ALLOWED", 0) \ + T(IllegalExponent, "ILLEGAL_EXPONENT", 0) \ + T(IllegalNumberEnd, "ILLEGAL_NUMBER_END", 0) \ + T(IllegalEnd, NULL, 0) /* used as type Illegal enum end marker */ \ \ /* Scanner-internal use only. */ \ T(Whitespace, nullptr, 0) @@ -312,6 +321,9 @@ namespace TokenTraits constexpr bool isTimeSubdenomination(Token op) { return op == Token::SubSecond || op == Token::SubMinute || op == Token::SubHour || op == Token::SubDay || op == Token::SubWeek || op == Token::SubYear; } constexpr bool isReservedKeyword(Token op) { return (Token::Abstract <= op && op <= Token::Unchecked); } + // @returns true if token is illegal + constexpr bool isIllegal(Token tok) { return Token::Illegal <= tok && tok < Token::IllegalEnd; }; + inline Token AssignmentToBinaryOp(Token op) { solAssert(isAssignmentOp(op) && op != Token::Assign, ""); diff --git a/libsolidity/parsing/Parser.cpp b/libsolidity/parsing/Parser.cpp index ffe2be83..b78c58c6 100644 --- a/libsolidity/parsing/Parser.cpp +++ b/libsolidity/parsing/Parser.cpp @@ -1558,6 +1558,30 @@ ASTPointer<Expression> Parser::parsePrimaryExpression() case Token::IllegalHex: fatalParserError("Expected even number of hex-nibbles within double-quotes."); break; + case Token::IllegalCommentTerminator: + fatalParserError("Expected multi-line comment-terminator."); + break; + case Token::IllegalStringEscape: + fatalParserError("Invalid String Escape."); + break; + case Token::IllegalStringEndQuote: + fatalParserError("Expected String end-quote."); + break; + case Token::IllegalNumberSeparator: + fatalParserError("Invalid use of Number Separator '_'."); + break; + case Token::IllegalHexDigit: + fatalParserError("Hex Digit missing or invalid."); + break; + case Token::IllegalOctalNotAllowed: + fatalParserError("Octal Numbers not allowed."); + break; + case Token::IllegalExponent: + fatalParserError("Invalid Exponent."); + break; + case Token::IllegalNumberEnd: + fatalParserError("Digit or Identifier-Start not allowed at end of Number."); + break; default: if (TokenTraits::isElementaryTypeName(token)) { diff --git a/test/libsolidity/SolidityScanner.cpp b/test/libsolidity/SolidityScanner.cpp index 2d164ae3..9d1383db 100644 --- a/test/libsolidity/SolidityScanner.cpp +++ b/test/libsolidity/SolidityScanner.cpp @@ -59,6 +59,8 @@ BOOST_AUTO_TEST_CASE(smoke_test) BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } +// STRING ESCAPES + BOOST_AUTO_TEST_CASE(string_escapes) { Scanner scanner(CharStream(" { \"a\\x61\"")); @@ -87,12 +89,12 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal) { Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)")); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEscape); BOOST_CHECK_EQUAL(scanner.currentLiteral(), ""); // TODO recovery from illegal tokens should be improved - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalNumberEnd); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } @@ -110,18 +112,19 @@ BOOST_AUTO_TEST_CASE(hex_numbers) BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234"); scanner.reset(CharStream("0X1234"), ""); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + // @todo update underlying code to return "Expected lower-case 'x' for Hex-Numbers." or similar. + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalNumberEnd); } BOOST_AUTO_TEST_CASE(octal_numbers) { Scanner scanner(CharStream("07")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalOctalNotAllowed); scanner.reset(CharStream("007"), ""); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalOctalNotAllowed); scanner.reset(CharStream("-07"), ""); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalOctalNotAllowed); scanner.reset(CharStream("-.07"), ""); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub); BOOST_CHECK_EQUAL(scanner.next(), Token::Number); @@ -204,7 +207,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal) BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal) { Scanner scanner(CharStream("0x_abc")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalHexDigit); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } @@ -469,7 +472,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape) { Scanner scanner(CharStream("{ \"\\uFFnicode\"")); BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEscape); } // HEX STRING LITERAL @@ -516,7 +519,7 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close) { // This used to parse as "comment", "identifier" Scanner scanner(CharStream("/** / x")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } @@ -524,14 +527,14 @@ BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos) { // This used to parse as "whitespace" Scanner scanner(CharStream("/**")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } BOOST_AUTO_TEST_CASE(multiline_comment_at_eos) { Scanner scanner(CharStream("/*")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalCommentTerminator); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } @@ -594,10 +597,10 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings) for (auto const& nl: {"\n", "\r"}) { Scanner scanner(CharStream("\"abc " + string(nl) + " def\"")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalStringEndQuote); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def"); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } } @@ -607,12 +610,12 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings) for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"}) { Scanner scanner(CharStream("\"abc " + string(nl) + " def\"")); - BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.currentToken(), Token::IllegalStringEndQuote); for (size_t i = 0; i < string(nl).size(); i++) BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier); BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def"); - BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal); + BOOST_CHECK_EQUAL(scanner.next(), Token::IllegalStringEndQuote); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); } } diff --git a/test/libsolidity/syntaxTests/inlineAssembly/invalid/invalid_number.sol b/test/libsolidity/syntaxTests/inlineAssembly/invalid/invalid_number.sol index 715913de..99a906de 100644 --- a/test/libsolidity/syntaxTests/inlineAssembly/invalid/invalid_number.sol +++ b/test/libsolidity/syntaxTests/inlineAssembly/invalid/invalid_number.sol @@ -7,4 +7,4 @@ contract C { } // ---- // ParserError: (72-73): Literal, identifier or instruction expected. -// ParserError: (72-73): Expected primary expression. +// ParserError: (72-73): Octal Numbers not allowed. diff --git a/test/libsolidity/syntaxTests/parsing/invalid_fixed_conversion_leading_zeroes_check.sol b/test/libsolidity/syntaxTests/parsing/invalid_fixed_conversion_leading_zeroes_check.sol index fb267ba3..4ac008a8 100644 --- a/test/libsolidity/syntaxTests/parsing/invalid_fixed_conversion_leading_zeroes_check.sol +++ b/test/libsolidity/syntaxTests/parsing/invalid_fixed_conversion_leading_zeroes_check.sol @@ -4,4 +4,4 @@ contract test { } } // ---- -// ParserError: (44-47): Expected primary expression. +// ParserError: (44-47): Digit or Identifier-Start not allowed at end of Number. diff --git a/test/libsolidity/syntaxTests/string/string_new_line.sol b/test/libsolidity/syntaxTests/string/string_new_line.sol index da2240f7..5a52534b 100644 --- a/test/libsolidity/syntaxTests/string/string_new_line.sol +++ b/test/libsolidity/syntaxTests/string/string_new_line.sol @@ -6,4 +6,4 @@ contract test { } } // ---- -// ParserError: (100-112): Expected primary expression. +// ParserError: (100-112): Expected String end-quote. diff --git a/test/libsolidity/syntaxTests/string/string_terminated_by_backslash.sol b/test/libsolidity/syntaxTests/string/string_terminated_by_backslash.sol index 3eaba6af..000d3b0e 100644 --- a/test/libsolidity/syntaxTests/string/string_terminated_by_backslash.sol +++ b/test/libsolidity/syntaxTests/string/string_terminated_by_backslash.sol @@ -5,4 +5,4 @@ contract test { } } // ---- -// ParserError: (100-109): Expected primary expression.
\ No newline at end of file +// ParserError: (100-109): Expected String end-quote.
\ No newline at end of file diff --git a/test/libsolidity/syntaxTests/string/string_unterminated.sol b/test/libsolidity/syntaxTests/string/string_unterminated.sol index 3291781e..6772a300 100644 --- a/test/libsolidity/syntaxTests/string/string_unterminated.sol +++ b/test/libsolidity/syntaxTests/string/string_unterminated.sol @@ -4,4 +4,4 @@ contract test { } } // ---- -// ParserError: (100-112): Expected primary expression.
\ No newline at end of file +// ParserError: (100-112): Expected String end-quote.
\ No newline at end of file diff --git a/test/libsolidity/syntaxTests/string/string_unterminated_no_new_line.sol b/test/libsolidity/syntaxTests/string/string_unterminated_no_new_line.sol index e7be50d2..734ce37f 100644 --- a/test/libsolidity/syntaxTests/string/string_unterminated_no_new_line.sol +++ b/test/libsolidity/syntaxTests/string/string_unterminated_no_new_line.sol @@ -1,4 +1,4 @@ contract test { function f() pure public { "abc\ // ---- -// ParserError: (47-53): Expected primary expression.
\ No newline at end of file +// ParserError: (47-53): Expected String end-quote.
\ No newline at end of file diff --git a/test/libsolidity/syntaxTests/unicode_escape_literals.sol b/test/libsolidity/syntaxTests/unicode_escape_literals.sol index a340487b..ac9aa08a 100644 --- a/test/libsolidity/syntaxTests/unicode_escape_literals.sol +++ b/test/libsolidity/syntaxTests/unicode_escape_literals.sol @@ -28,4 +28,4 @@ contract test { } // ---- -// ParserError: (678-681): Expected primary expression. +// ParserError: (678-681): Invalid String Escape. |