Splitting out CharStream from Scanner.

4 files changed, 216 insertions, 81 deletions

diff --git a/liblangutil/CharStream.cpp b/liblangutil/CharStream.cpp
new file mode 100644
index 00000000..04490fa6
--- /dev/null
+++ b/liblangutil/CharStream.cpp
@@ -0,0 +1,115 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.cc", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#include <liblangutil/CharStream.h>
+#include <liblangutil/Exceptions.h>
+#include <algorithm>
+#include <tuple>
+
+using namespace std;
+
+namespace dev
+{
+namespace solidity
+{
+
+char CharStream::advanceAndGet(size_t _chars)
+{
+	if (isPastEndOfInput())
+		return 0;
+	m_position += _chars;
+	if (isPastEndOfInput())
+		return 0;
+	return m_source[m_position];
+}
+
+char CharStream::rollback(size_t _amount)
+{
+	solAssert(m_position >= _amount, "");
+	m_position -= _amount;
+	return get();
+}
+
+string CharStream::lineAtPosition(int _position) const
+{
+	// if _position points to \n, it returns the line before the \n
+	using size_type = string::size_type;
+	size_type searchStart = min<size_type>(m_source.size(), _position);
+	if (searchStart > 0)
+		searchStart--;
+	size_type lineStart = m_source.rfind('\n', searchStart);
+	if (lineStart == string::npos)
+		lineStart = 0;
+	else
+		lineStart++;
+	return m_source.substr(lineStart, min(m_source.find('\n', lineStart),
+										  m_source.size()) - lineStart);
+}
+
+tuple<int, int> CharStream::translatePositionToLineColumn(int _position) const
+{
+	using size_type = string::size_type;
+	size_type searchPosition = min<size_type>(m_source.size(), _position);
+	int lineNumber = count(m_source.begin(), m_source.begin() + searchPosition, '\n');
+	size_type lineStart;
+	if (searchPosition == 0)
+		lineStart = 0;
+	else
+	{
+		lineStart = m_source.rfind('\n', searchPosition - 1);
+		lineStart = lineStart == string::npos ? 0 : lineStart + 1;
+	}
+	return tuple<int, int>(lineNumber, searchPosition - lineStart);
+}
+
+}
+}
diff --git a/liblangutil/CharStream.h b/liblangutil/CharStream.h
new file mode 100644
index 00000000..7c5488a0
--- /dev/null
+++ b/liblangutil/CharStream.h
@@ -0,0 +1,100 @@
+/*
+	This file is part of solidity.
+
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+
+	This file is derived from the file "scanner.h", which was part of the
+	V8 project. The original copyright header follows:
+
+	Copyright 2006-2012, the V8 project authors. All rights reserved.
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are
+	met:
+
+	* Redistributions of source code must retain the above copyright
+	  notice, this list of conditions and the following disclaimer.
+	* Redistributions in binary form must reproduce the above
+	  copyright notice, this list of conditions and the following
+	  disclaimer in the documentation and/or other materials provided
+	  with the distribution.
+	* Neither the name of Google Inc. nor the names of its
+	  contributors may be used to endorse or promote products derived
+	  from this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/**
+ * @author Christian <c@ethdev.com>
+ * @date 2014
+ * Solidity scanner.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <tuple>
+
+namespace dev
+{
+namespace solidity
+{
+
+/**
+ * Bidirectional stream of characters.
+ *
+ * This CharStream is used by lexical analyzers as the source.
+ */
+class CharStream
+{
+public:
+	CharStream(): m_position(0) {}
+	explicit CharStream(std::string const& _source): m_source(_source), m_position(0) {}
+
+	int position() const { return m_position; }
+	bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); }
+
+	char get(size_t _charsForward = 0) const { return m_source[m_position + _charsForward]; }
+	char advanceAndGet(size_t _chars = 1);
+	char rollback(size_t _amount);
+
+	void reset() { m_position = 0; }
+
+	std::string const& source() const { return m_source; }
+
+	///@{
+	///@name Error printing helper functions
+	/// Functions that help pretty-printing parse errors
+	/// Do only use in error cases, they are quite expensive.
+	std::string lineAtPosition(int _position) const;
+	std::tuple<int, int> translatePositionToLineColumn(int _position) const;
+	///@}
+
+private:
+	std::string m_source;
+	size_t m_position;
+};
+
+}
+}
diff --git a/liblangutil/Scanner.cpp b/liblangutil/Scanner.cpp
index 246f5ea6..beb39a4f 100644
--- a/liblangutil/Scanner.cpp
+++ b/liblangutil/Scanner.cpp
@@ -866,55 +866,5 @@ tuple<Token, unsigned, unsigned> Scanner::scanIdentifierOrKeyword()
 	return TokenTraits::fromIdentifierOrKeyword(m_nextToken.literal);
 }
 
-char CharStream::advanceAndGet(size_t _chars)
-{
-	if (isPastEndOfInput())
-		return 0;
-	m_position += _chars;
-	if (isPastEndOfInput())
-		return 0;
-	return m_source[m_position];
-}
-
-char CharStream::rollback(size_t _amount)
-{
-	solAssert(m_position >= _amount, "");
-	m_position -= _amount;
-	return get();
-}
-
-string CharStream::lineAtPosition(int _position) const
-{
-	// if _position points to \n, it returns the line before the \n
-	using size_type = string::size_type;
-	size_type searchStart = min<size_type>(m_source.size(), _position);
-	if (searchStart > 0)
-		searchStart--;
-	size_type lineStart = m_source.rfind('\n', searchStart);
-	if (lineStart == string::npos)
-		lineStart = 0;
-	else
-		lineStart++;
-	return m_source.substr(lineStart, min(m_source.find('\n', lineStart),
-										  m_source.size()) - lineStart);
-}
-
-tuple<int, int> CharStream::translatePositionToLineColumn(int _position) const
-{
-	using size_type = string::size_type;
-	size_type searchPosition = min<size_type>(m_source.size(), _position);
-	int lineNumber = count(m_source.begin(), m_source.begin() + searchPosition, '\n');
-	size_type lineStart;
-	if (searchPosition == 0)
-		lineStart = 0;
-	else
-	{
-		lineStart = m_source.rfind('\n', searchPosition - 1);
-		lineStart = lineStart == string::npos ? 0 : lineStart + 1;
-	}
-	return tuple<int, int>(lineNumber, searchPosition - lineStart);
-}
-
-
 }
 }
diff --git a/liblangutil/Scanner.h b/liblangutil/Scanner.h
index d89f0c84..5e5ddfdd 100644
--- a/liblangutil/Scanner.h
+++ b/liblangutil/Scanner.h
@@ -54,6 +54,7 @@
 
 #include <libdevcore/Common.h>
 #include <libdevcore/CommonData.h>
+#include <liblangutil/CharStream.h>
 #include <liblangutil/SourceLocation.h>
 #include <liblangutil/Token.h>
 
@@ -62,41 +63,10 @@ namespace dev
 namespace solidity
 {
 
-
 class AstRawString;
 class AstValueFactory;
 class ParserRecorder;
 
-class CharStream
-{
-public:
-	CharStream(): m_position(0) {}
-	explicit CharStream(std::string const& _source): m_source(_source), m_position(0) {}
-	int position() const { return m_position; }
-	bool isPastEndOfInput(size_t _charsForward = 0) const { return (m_position + _charsForward) >= m_source.size(); }
-	char get(size_t _charsForward = 0) const { return m_source[m_position + _charsForward]; }
-	char advanceAndGet(size_t _chars = 1);
-	char rollback(size_t _amount);
-
-	void reset() { m_position = 0; }
-
-	std::string const& source() const { return m_source; }
-
-	///@{
-	///@name Error printing helper functions
-	/// Functions that help pretty-printing parse errors
-	/// Do only use in error cases, they are quite expensive.
-	std::string lineAtPosition(int _position) const;
-	std::tuple<int, int> translatePositionToLineColumn(int _position) const;
-	///@}
-
-private:
-	std::string m_source;
-	size_t m_position;
-};
-
-
-
 class Scanner
 {
 	friend class LiteralScope;