liblangutil: extends CharStream to know about the respective (file-)name (and adapt codebase to it)

13 files changed, 83 insertions, 80 deletions

diff --git a/liblangutil/CharStream.h b/liblangutil/CharStream.h
index 72aacacf..6f2da3bb 100644
--- a/liblangutil/CharStream.h
+++ b/liblangutil/CharStream.h
@@ -68,7 +68,8 @@ class CharStream
 {
 public:
 	CharStream(): m_position(0) {}
-	explicit CharStream(std::string const& _source): m_source(_source), m_position(0) {}
+	explicit CharStream(std::string const& _source, std::string const& name):
+		m_source(_source), m_name(name), m_position(0) {}
 
 	int position() const { return m_position; }
 	bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); }
@@ -80,6 +81,7 @@ public:
 	void reset() { m_position = 0; }
 
 	std::string const& source() const { return m_source; }
+	std::string const& name() const noexcept { return m_name; }
 
 	///@{
 	///@name Error printing helper functions
@@ -91,6 +93,7 @@ public:
 
 private:
 	std::string m_source;
+	std::string m_name;
 	size_t m_position;
 };
 
diff --git a/libsolidity/codegen/CompilerContext.cpp b/libsolidity/codegen/CompilerContext.cpp
index 2fd62de2..16bc4225 100644
--- a/libsolidity/codegen/CompilerContext.cpp
+++ b/libsolidity/codegen/CompilerContext.cpp
@@ -360,7 +360,7 @@ void CompilerContext::appendInlineAssembly(
 
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly), "--CODEGEN--");
+	auto scanner = make_shared<langutil::Scanner>(langutil::CharStream(_assembly, "--CODEGEN--"), "--CODEGEN--");
 	auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false);
 #ifdef SOL_OUTPUT_ASM
 	cout << yul::AsmPrinter()(*parserResult) << endl;
diff --git a/libsolidity/interface/AssemblyStack.cpp b/libsolidity/interface/AssemblyStack.cpp
index 24e190b3..5268c5aa 100644
--- a/libsolidity/interface/AssemblyStack.cpp
+++ b/libsolidity/interface/AssemblyStack.cpp
@@ -69,7 +69,7 @@ bool AssemblyStack::parseAndAnalyze(std::string const& _sourceName, std::string
 {
 	m_errors.clear();
 	m_analysisSuccessful = false;
-	m_scanner = make_shared<Scanner>(CharStream(_source), _sourceName);
+	m_scanner = make_shared<Scanner>(CharStream(_source, _sourceName), _sourceName);
 	m_parserResult = yul::ObjectParser(m_errorReporter, languageToAsmFlavour(m_language)).parse(m_scanner, false);
 	if (!m_errorReporter.errors().empty())
 		return false;
diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp
index de4a7ec2..cc5a2102 100644
--- a/libsolidity/interface/CompilerStack.cpp
+++ b/libsolidity/interface/CompilerStack.cpp
@@ -125,7 +125,7 @@ bool CompilerStack::addSource(string const& _name, string const& _content, bool
 {
 	bool existed = m_sources.count(_name) != 0;
 	reset(true);
-	m_sources[_name].scanner = make_shared<Scanner>(CharStream(_content), _name);
+	m_sources[_name].scanner = make_shared<Scanner>(CharStream(_content, _name), _name);
 	m_sources[_name].isLibrary = _isLibrary;
 	m_stackState = SourcesSet;
 	return existed;
@@ -160,7 +160,7 @@ bool CompilerStack::parse()
 			{
 				string const& newPath = newSource.first;
 				string const& newContents = newSource.second;
-				m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents), newPath);
+				m_sources[newPath].scanner = make_shared<Scanner>(CharStream(newContents, newPath), newPath);
 				sourcesToParse.push_back(newPath);
 			}
 		}
diff --git a/test/libsolidity/Assembly.cpp b/test/libsolidity/Assembly.cpp
index 926e29fe..620f9661 100644
--- a/test/libsolidity/Assembly.cpp
+++ b/test/libsolidity/Assembly.cpp
@@ -58,7 +58,7 @@ eth::AssemblyItems compileContract(string const& _sourceCode)
 	ErrorReporter errorReporter(errors);
 	Parser parser(errorReporter);
 	ASTPointer<SourceUnit> sourceUnit;
-	BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(CharStream(_sourceCode))));
+	BOOST_REQUIRE_NO_THROW(sourceUnit = parser.parse(make_shared<Scanner>(CharStream(_sourceCode, ""))));
 	BOOST_CHECK(!!sourceUnit);
 
 	map<ASTNode const*, shared_ptr<DeclarationContainer>> scopes;
diff --git a/test/libsolidity/SemVerMatcher.cpp b/test/libsolidity/SemVerMatcher.cpp
index 2e147847..2980acd1 100644
--- a/test/libsolidity/SemVerMatcher.cpp
+++ b/test/libsolidity/SemVerMatcher.cpp
@@ -41,7 +41,7 @@ BOOST_AUTO_TEST_SUITE(SemVerMatcher)
 
 SemVerMatchExpression parseExpression(string const& _input)
 {
-	Scanner scanner{CharStream(_input)};
+	Scanner scanner{CharStream(_input, "")};
 	vector<string> literals;
 	vector<Token> tokens;
 	while (scanner.currentToken() != Token::EOS)
diff --git a/test/libsolidity/SolidityExpressionCompiler.cpp b/test/libsolidity/SolidityExpressionCompiler.cpp
index a68eb3df..8bce26c1 100644
--- a/test/libsolidity/SolidityExpressionCompiler.cpp
+++ b/test/libsolidity/SolidityExpressionCompiler.cpp
@@ -101,7 +101,7 @@ bytes compileFirstExpression(
 	{
 		ErrorList errors;
 		ErrorReporter errorReporter(errors);
-		sourceUnit = Parser(errorReporter).parse(make_shared<Scanner>(CharStream(_sourceCode)));
+		sourceUnit = Parser(errorReporter).parse(make_shared<Scanner>(CharStream(_sourceCode, "")));
 		if (!sourceUnit)
 			return bytes();
 	}
diff --git a/test/libsolidity/SolidityParser.cpp b/test/libsolidity/SolidityParser.cpp
index 3534dd5b..d86d3d39 100644
--- a/test/libsolidity/SolidityParser.cpp
+++ b/test/libsolidity/SolidityParser.cpp
@@ -43,7 +43,7 @@ namespace
 ASTPointer<ContractDefinition> parseText(std::string const& _source, ErrorList& _errors)
 {
 	ErrorReporter errorReporter(_errors);
-	ASTPointer<SourceUnit> sourceUnit = Parser(errorReporter).parse(std::make_shared<Scanner>(CharStream(_source)));
+	ASTPointer<SourceUnit> sourceUnit = Parser(errorReporter).parse(std::make_shared<Scanner>(CharStream(_source, "")));
 	if (!sourceUnit)
 		return ASTPointer<ContractDefinition>();
 	for (ASTPointer<ASTNode> const& node: sourceUnit->nodes())
diff --git a/test/libsolidity/SolidityScanner.cpp b/test/libsolidity/SolidityScanner.cpp
index 02d91d32..95d56568 100644
--- a/test/libsolidity/SolidityScanner.cpp
+++ b/test/libsolidity/SolidityScanner.cpp
@@ -37,13 +37,13 @@ BOOST_AUTO_TEST_SUITE(SolidityScanner)
 
 BOOST_AUTO_TEST_CASE(test_empty)
 {
-	Scanner scanner(CharStream(""));
+	Scanner scanner(CharStream{});
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(smoke_test)
 {
-	Scanner scanner(CharStream("function break;765  \t  \"string1\",'string2'\nidentifier1"));
+	Scanner scanner(CharStream("function break;765  \t  \"string1\",'string2'\nidentifier1", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Function);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Break);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
@@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(smoke_test)
 
 BOOST_AUTO_TEST_CASE(string_escapes)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\""));
+	Scanner scanner(CharStream("  { \"a\\x61\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa");
@@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(string_escapes)
 
 BOOST_AUTO_TEST_CASE(string_escapes_all)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\\b\\f\\n\\r\\t\\v\""));
+	Scanner scanner(CharStream("  { \"a\\x61\\b\\f\\n\\r\\t\\v\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "aa\b\f\n\r\t\v");
@@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_all)
 
 BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
 {
-	Scanner scanner(CharStream("  { \"a\\x61\\x00abc\""));
+	Scanner scanner(CharStream("  { \"a\\x61\\x00abc\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("aa\0abc", 6));
@@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(string_escapes_with_zero)
 
 BOOST_AUTO_TEST_CASE(string_escape_illegal)
 {
-	Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)"));
+	Scanner scanner(CharStream(" bla \"\\x6rf\" (illegalescape)", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalEscapeSequence);
@@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE(string_escape_illegal)
 
 BOOST_AUTO_TEST_CASE(hex_numbers)
 {
-	Scanner scanner(CharStream("var x = 0x765432536763762734623472346;"));
+	Scanner scanner(CharStream("var x = 0x765432536763762734623472346;", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -107,34 +107,34 @@ BOOST_AUTO_TEST_CASE(hex_numbers)
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x765432536763762734623472346");
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Semicolon);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("0x1234"), "");
+	scanner.reset(CharStream("0x1234", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "0x1234");
-	scanner.reset(CharStream("0X1234"), "");
+	scanner.reset(CharStream("0X1234", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 }
 
 BOOST_AUTO_TEST_CASE(octal_numbers)
 {
-	Scanner scanner(CharStream("07"));
+	Scanner scanner(CharStream("07", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
-	scanner.reset(CharStream("007"), "");
+	scanner.reset(CharStream("007", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
-	scanner.reset(CharStream("-07"), "");
+	scanner.reset(CharStream("-07", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
-	scanner.reset(CharStream("-.07"), "");
+	scanner.reset(CharStream("-.07", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Sub);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
-	scanner.reset(CharStream("0"), "");
+	scanner.reset(CharStream("0", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
-	scanner.reset(CharStream("0.1"), "");
+	scanner.reset(CharStream("0.1", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 }
 
 BOOST_AUTO_TEST_CASE(scientific_notation)
 {
-	Scanner scanner(CharStream("var x = 2e10;"));
+	Scanner scanner(CharStream("var x = 2e10;", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -146,19 +146,19 @@ BOOST_AUTO_TEST_CASE(scientific_notation)
 
 BOOST_AUTO_TEST_CASE(trailing_dot)
 {
-	Scanner scanner(CharStream("2.5"));
+	Scanner scanner(CharStream("2.5", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("2.5e10"), "");
+	scanner.reset(CharStream("2.5e10", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(".5"), "");
+	scanner.reset(CharStream(".5", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream(".5e10"), "");
+	scanner.reset(CharStream(".5e10", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
-	scanner.reset(CharStream("2."), "");
+	scanner.reset(CharStream("2.", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Period);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -167,7 +167,7 @@ BOOST_AUTO_TEST_CASE(trailing_dot)
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("_1.2"));
+	Scanner scanner(CharStream("_1.2", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -176,11 +176,11 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_is_identifier)
 BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("1._2"));
+	Scanner scanner(CharStream("1._2", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("1._"), "");
+	scanner.reset(CharStream("1._", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -188,7 +188,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_decimal_after_dot_illegal)
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("_1e2"));
+	Scanner scanner(CharStream("_1e2", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -196,7 +196,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_are_identifier)
 BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("1e_2"));
+	Scanner scanner(CharStream("1e_2", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), "1e_2");
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_exp_after_e_illegal)
 
 BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 {
-	Scanner scanner(CharStream("0x_abc"));
+	Scanner scanner(CharStream("0x_abc", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
@@ -213,7 +213,7 @@ BOOST_AUTO_TEST_CASE(leading_underscore_hex_illegal)
 BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("12._1234_1234"));
+	Scanner scanner(CharStream("12._1234_1234", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -221,22 +221,22 @@ BOOST_AUTO_TEST_CASE(fixed_number_invalid_underscore_front)
 BOOST_AUTO_TEST_CASE(number_literals_with_trailing_underscore_at_eos)
 {
 	// Actual error is cought by SyntaxChecker.
-	Scanner scanner(CharStream("0x123_"));
+	Scanner scanner(CharStream("0x123_", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("123_"), "");
+	scanner.reset(CharStream("123_", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 
-	scanner.reset(CharStream("12.34_"), "");
+	scanner.reset(CharStream("12.34_", ""), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Number);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(negative_numbers)
 {
-	Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;"));
+	Scanner scanner(CharStream("var x = -.2 + -0x78 + -7.3 + 8.9 + 2e-2;", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Var);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Assign);
@@ -263,7 +263,7 @@ BOOST_AUTO_TEST_CASE(negative_numbers)
 
 BOOST_AUTO_TEST_CASE(locations)
 {
-	Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment"));
+	Scanner scanner(CharStream("function_identifier has ; -0x743/*comment*/\n ident //comment", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.currentLocation().start, 0);
 	BOOST_CHECK_EQUAL(scanner.currentLocation().end, 19);
@@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(locations)
 BOOST_AUTO_TEST_CASE(ambiguities)
 {
 	// test scanning of some operators which need look-ahead
-	Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<="));
+	Scanner scanner(CharStream("<=" "<" "+ +=a++ =>" "<<" ">>" " >>=" ">>>" ">>>=" " >>>>>=><<=", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LessThanOrEqual);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LessThan);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Add);
@@ -308,21 +308,21 @@ BOOST_AUTO_TEST_CASE(ambiguities)
 
 BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin)
 {
-	Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user"));
+	Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
 }
 
 BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed_begin)
 {
-	Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/"));
+	Scanner scanner(CharStream("/** Send $(value / 1000) chocolates to the user*/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "Send $(value / 1000) chocolates to the user");
 }
 
 BOOST_AUTO_TEST_CASE(documentation_comments_parsed)
 {
-	Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user"));
+	Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -334,7 +334,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_comments_parsed)
 {
 	Scanner scanner(CharStream("some other tokens /**\n"
 							   "* Send $(value / 1000) chocolates to the user\n"
-							   "*/"));
+							   "*/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -346,7 +346,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_no_stars)
 {
 	Scanner scanner(CharStream("some other tokens /**\n"
 							   " Send $(value / 1000) chocolates to the user\n"
-							   "*/"));
+							   "*/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -358,7 +358,7 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
 {
 	Scanner scanner(CharStream("some other tokens /** \t \r \n"
 							   "\t \r  * Send $(value / 1000) chocolates to the user\n"
-							   "*/"));
+							   "*/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
@@ -368,28 +368,28 @@ BOOST_AUTO_TEST_CASE(multiline_documentation_whitespace_hell)
 
 BOOST_AUTO_TEST_CASE(comment_before_eos)
 {
-	Scanner scanner(CharStream("//"));
+	Scanner scanner(CharStream("//", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(documentation_comment_before_eos)
 {
-	Scanner scanner(CharStream("///"));
+	Scanner scanner(CharStream("///", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(empty_multiline_comment)
 {
-	Scanner scanner(CharStream("/**/"));
+	Scanner scanner(CharStream("/**/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
 
 BOOST_AUTO_TEST_CASE(empty_multiline_documentation_comment_before_eos)
 {
-	Scanner scanner(CharStream("/***/"));
+	Scanner scanner(CharStream("/***/", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::EOS);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 }
@@ -398,7 +398,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
 {
 	Scanner scanner(CharStream("hello_world ///documentation comment \n"
 							   "//simple comment \n"
-							   "<<"));
+							   "<<", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SHL);
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "documentation comment ");
@@ -406,7 +406,7 @@ BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence)
 
 BOOST_AUTO_TEST_CASE(ether_subdenominations)
 {
-	Scanner scanner(CharStream("wei szabo finney ether"));
+	Scanner scanner(CharStream("wei szabo finney ether", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubWei);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubSzabo);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubFinney);
@@ -415,7 +415,7 @@ BOOST_AUTO_TEST_CASE(ether_subdenominations)
 
 BOOST_AUTO_TEST_CASE(time_subdenominations)
 {
-	Scanner scanner(CharStream("seconds minutes hours days weeks years"));
+	Scanner scanner(CharStream("seconds minutes hours days weeks years", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::SubSecond);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubMinute);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::SubHour);
@@ -426,7 +426,7 @@ BOOST_AUTO_TEST_CASE(time_subdenominations)
 
 BOOST_AUTO_TEST_CASE(empty_comment)
 {
-	Scanner scanner(CharStream("//\ncontract{}"));
+	Scanner scanner(CharStream("//\ncontract{}", ""));
 	BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Contract);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::LBrace);
@@ -436,7 +436,7 @@ BOOST_AUTO_TEST_CASE(empty_comment)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
 {
-	Scanner scanner(CharStream("{ \"\\u00DAnicode\""));
+	Scanner scanner(CharStream("{ \"\\u00DAnicode\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xC3\x9Anicode", 8));
@@ -444,7 +444,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
 {
-	Scanner scanner(CharStream("{ \"\\u007Fnicode\""));
+	Scanner scanner(CharStream("{ \"\\u007Fnicode\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x7Fnicode", 7));
@@ -452,7 +452,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7f)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
 {
-	Scanner scanner(CharStream("{ \"\\u07FFnicode\""));
+	Scanner scanner(CharStream("{ \"\\u07FFnicode\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xDF\xBFnicode", 8));
@@ -460,7 +460,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_7ff)
 
 BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
 {
-	Scanner scanner(CharStream("{ \"\\uFFFFnicode\""));
+	Scanner scanner(CharStream("{ \"\\uFFFFnicode\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\xEF\xBF\xBFnicode", 9));
@@ -468,7 +468,7 @@ BOOST_AUTO_TEST_CASE(valid_unicode_string_escape_ffff)
 
 BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
 {
-	Scanner scanner(CharStream("{ \"\\uFFnicode\""));
+	Scanner scanner(CharStream("{ \"\\uFFnicode\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 }
@@ -477,7 +477,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_unicode_string_escape)
 
 BOOST_AUTO_TEST_CASE(valid_hex_literal)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF\""));
+	Scanner scanner(CharStream("{ hex\"00112233FF\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::StringLiteral);
 	BOOST_CHECK_EQUAL(scanner.currentLiteral(), std::string("\x00\x11\x22\x33\xFF", 5));
@@ -485,7 +485,7 @@ BOOST_AUTO_TEST_CASE(valid_hex_literal)
 
 BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
 {
-	Scanner scanner(CharStream("{ hex\"00112233F\""));
+	Scanner scanner(CharStream("{ hex\"00112233F\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -493,7 +493,7 @@ BOOST_AUTO_TEST_CASE(invalid_short_hex_literal)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF \""));
+	Scanner scanner(CharStream("{ hex\"00112233FF \"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -501,7 +501,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_space)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
 {
-	Scanner scanner(CharStream("{ hex\"00112233FF'"));
+	Scanner scanner(CharStream("{ hex\"00112233FF'", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -509,7 +509,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_with_wrong_quotes)
 
 BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
 {
-	Scanner scanner(CharStream("{ hex\"hello\""));
+	Scanner scanner(CharStream("{ hex\"hello\"", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::LBrace);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.currentError(), ScannerError::IllegalHexString);
@@ -520,7 +520,7 @@ BOOST_AUTO_TEST_CASE(invalid_hex_literal_nonhex_string)
 BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
 {
 	// This used to parse as "comment", "identifier"
-	Scanner scanner(CharStream("/** / x"));
+	Scanner scanner(CharStream("/** / x", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -528,14 +528,14 @@ BOOST_AUTO_TEST_CASE(invalid_multiline_comment_close)
 BOOST_AUTO_TEST_CASE(multiline_doc_comment_at_eos)
 {
 	// This used to parse as "whitespace"
-	Scanner scanner(CharStream("/**"));
+	Scanner scanner(CharStream("/**", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
 
 BOOST_AUTO_TEST_CASE(multiline_comment_at_eos)
 {
-	Scanner scanner(CharStream("/*"));
+	Scanner scanner(CharStream("/*", ""));
 	BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 	BOOST_CHECK_EQUAL(scanner.next(), Token::EOS);
 }
@@ -544,7 +544,7 @@ BOOST_AUTO_TEST_CASE(regular_line_break_in_single_line_comment)
 {
 	for (auto const& nl: {"\r", "\n"})
 	{
-		Scanner scanner(CharStream("// abc " + string(nl) + " def "));
+		Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -556,7 +556,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_comment)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("// abc " + string(nl) + " def "));
+		Scanner scanner(CharStream("// abc " + string(nl) + " def ", ""));
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -571,7 +571,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_single_line_doc_comment)
 {
 	for (auto const& nl: {"\r", "\n"})
 	{
-		Scanner scanner(CharStream("/// abc " + string(nl) + " def "));
+		Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -583,7 +583,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_single_line_doc_comment)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("/// abc " + string(nl) + " def "));
+		Scanner scanner(CharStream("/// abc " + string(nl) + " def ", ""));
 		BOOST_CHECK_EQUAL(scanner.currentCommentLiteral(), "abc ");
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size() - 1; i++)
@@ -598,7 +598,7 @@ BOOST_AUTO_TEST_CASE(regular_line_breaks_in_strings)
 {
 	for (auto const& nl: {"\n", "\r"})
 	{
-		Scanner scanner(CharStream("\"abc " + string(nl) + " def\""));
+		Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		BOOST_CHECK_EQUAL(scanner.next(), Token::Identifier);
 		BOOST_CHECK_EQUAL(scanner.currentLiteral(), "def");
@@ -611,7 +611,7 @@ BOOST_AUTO_TEST_CASE(irregular_line_breaks_in_strings)
 {
 	for (auto const& nl: {"\v", "\f", "\xE2\x80\xA8", "\xE2\x80\xA9"})
 	{
-		Scanner scanner(CharStream("\"abc " + string(nl) + " def\""));
+		Scanner scanner(CharStream("\"abc " + string(nl) + " def\"", ""));
 		BOOST_CHECK_EQUAL(scanner.currentToken(), Token::Illegal);
 		for (size_t i = 0; i < string(nl).size(); i++)
 			BOOST_CHECK_EQUAL(scanner.next(), Token::Illegal);
diff --git a/test/libyul/Common.cpp b/test/libyul/Common.cpp
index 2e70d086..36065db7 100644
--- a/test/libyul/Common.cpp
+++ b/test/libyul/Common.cpp
@@ -57,7 +57,7 @@ pair<shared_ptr<Block>, shared_ptr<yul::AsmAnalysisInfo>> yul::test::parse(strin
 	auto flavour = _yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict;
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	auto scanner = make_shared<Scanner>(CharStream(_source), "");
+	auto scanner = make_shared<Scanner>(CharStream(_source, ""), "");
 	auto parserResult = yul::Parser(errorReporter, flavour).parse(scanner, false);
 	if (parserResult)
 	{
diff --git a/test/libyul/Parser.cpp b/test/libyul/Parser.cpp
index 6f946362..caaf2719 100644
--- a/test/libyul/Parser.cpp
+++ b/test/libyul/Parser.cpp
@@ -51,7 +51,7 @@ bool parse(string const& _source, ErrorReporter& errorReporter)
 {
 	try
 	{
-		auto scanner = make_shared<Scanner>(CharStream(_source));
+		auto scanner = make_shared<Scanner>(CharStream(_source, ""));
 		auto parserResult = yul::Parser(errorReporter, yul::AsmFlavour::Yul).parse(scanner, false);
 		if (parserResult)
 		{
diff --git a/test/libyul/YulOptimizerTest.cpp b/test/libyul/YulOptimizerTest.cpp
index c857d8a5..8f282aac 100644
--- a/test/libyul/YulOptimizerTest.cpp
+++ b/test/libyul/YulOptimizerTest.cpp
@@ -259,7 +259,7 @@ bool YulOptimizerTest::parse(ostream& _stream, string const& _linePrefix, bool c
 	yul::AsmFlavour flavour = m_yul ? yul::AsmFlavour::Yul : yul::AsmFlavour::Strict;
 	ErrorList errors;
 	ErrorReporter errorReporter(errors);
-	shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(m_source), "");
+	shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(m_source, ""), "");
 	m_ast = yul::Parser(errorReporter, flavour).parse(scanner, false);
 	if (!m_ast || !errorReporter.errors().empty())
 	{
diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp
index 5a8a5106..8ecafb29 100644
--- a/test/tools/yulopti.cpp
+++ b/test/tools/yulopti.cpp
@@ -81,7 +81,7 @@ public:
 	bool parse(string const& _input)
 	{
 		ErrorReporter errorReporter(m_errors);
-		shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_input), "");
+		shared_ptr<Scanner> scanner = make_shared<Scanner>(CharStream(_input, ""), "");
 		m_ast = yul::Parser(errorReporter, yul::AsmFlavour::Strict).parse(scanner, false);
 		if (!m_ast || !errorReporter.errors().empty())
 		{