path: root/test/tools/isoltest.cpp

/*
    This file is part of solidity.

    solidity is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    solidity is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with solidity.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <libdevcore/CommonIO.h>

#include <test/Common.h>
#include <test/libsolidity/AnalysisFramework.h>
#include <test/libsolidity/SyntaxTest.h>
#include <test/libsolidity/ASTJSONTest.h>
#include <test/libsolidity/SMTCheckerJSONTest.h>
#include <test/libyul/YulOptimizerTest.h>

#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>

#include <cstdlib>
#include <iostream>
#include <fstream>
#include <queue>

#if defined(_WIN32)
#include <windows.h>
#endif

using namespace dev;
using namespace dev::solidity;
using namespace dev::solidity::test;
using namespace dev::solidity::test::formatting;
using namespace std;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

struct TestStats
{
    int successCount;
    int testCount;
    operator bool() const { return successCount == testCount; }
    TestStats& operator+=(TestStats const& _other) noexcept
    {
        successCount += _other.successCount;
        testCount += _other.testCount;
        return *this;
    }
};

class TestTool
{
public:
    TestTool(
        TestCase::TestCaseCreator _testCaseCreator,
        string const& _name,
        fs::path const& _path,
        bool _formatted
    ): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path)
    {}

    enum class Result
    {
        Success,
        Failure,
        Exception
    };

    Result process();

    static TestStats processPath(
        TestCase::TestCaseCreator _testCaseCreator,
        fs::path const& _basepath,
        fs::path const& _path,
        bool const _formatted
    );

    static string editor;
private:
    enum class Request
    {
        Skip,
        Rerun,
        Quit
    };

    Request handleResponse(bool const _exception);

    TestCase::TestCaseCreator m_testCaseCreator;
    bool const m_formatted = false;
    string const m_name;
    fs::path const m_path;
    unique_ptr<TestCase> m_test;
    static bool m_exitRequested;
};

string TestTool::editor;
bool TestTool::m_exitRequested = false;

TestTool::Result TestTool::process()
{
    bool success;
    std::stringstream outputMessages;

    (FormattedScope(cout, m_formatted, {BOLD}) << m_name << ": ").flush();

    try
    {
        m_test = m_testCaseCreator(m_path.string());
        success = m_test->run(outputMessages, "  ", m_formatted);
    }
    catch(boost::exception const& _e)
    {
        FormattedScope(cout, m_formatted, {BOLD, RED}) <<
            "Exception during syntax test: " << boost::diagnostic_information(_e) << endl;
        return Result::Exception;
    }
    catch (std::exception const& _e)
    {
        FormattedScope(cout, m_formatted, {BOLD, RED}) <<
            "Exception during syntax test: " << _e.what() << endl;
        return Result::Exception;
    }
    catch (...)
    {
        FormattedScope(cout, m_formatted, {BOLD, RED}) <<
            "Unknown exception during syntax test." << endl;
        return Result::Exception;
    }

    if (success)
    {
        FormattedScope(cout, m_formatted, {BOLD, GREEN}) << "OK" << endl;
        return Result::Success;
    }
    else
    {
        FormattedScope(cout, m_formatted, {BOLD, RED}) << "FAIL" << endl;

        FormattedScope(cout, m_formatted, {BOLD, CYAN}) << "  Contract:" << endl;
        m_test->printSource(cout, "    ", m_formatted);

        cout << endl << outputMessages.str() << endl;
        return Result::Failure;
    }
}

TestTool::Request TestTool::handleResponse(bool const _exception)
{
    if (_exception)
        cout << "(e)dit/(s)kip/(q)uit? ";
    else
        cout << "(e)dit/(u)pdate expectations/(s)kip/(q)uit? ";
    cout.flush();

    while (true)
    {
        switch(readStandardInputChar())
        {
        case 's':
            cout << endl;
            return Request::Skip;
        case 'u':
            if (_exception)
                break;
            else
            {
                cout << endl;
                ofstream file(m_path.string(), ios::trunc);
                m_test->printSource(file);
                file << "// ----" << endl;
                m_test->printUpdatedExpectations(file, "// ");
                return Request::Rerun;
            }
        case 'e':
            cout << endl << endl;
            if (system((TestTool::editor + " \"" + m_path.string() + "\"").c_str()))
                cerr << "Error running editor command." << endl << endl;
            return Request::Rerun;
        case 'q':
            cout << endl;
            return Request::Quit;
        default:
            break;
        }
    }
}

TestStats TestTool::processPath(
    TestCase::TestCaseCreator _testCaseCreator,
    fs::path const& _basepath,
    fs::path const& _path,
    bool const _formatted
)
{
    std::queue<fs::path> paths;
    paths.push(_path);
    int successCount = 0;
    int testCount = 0;

    while (!paths.empty())
    {
        auto currentPath = paths.front();

        fs::path fullpath = _basepath / currentPath;
        if (fs::is_directory(fullpath))
        {
            paths.pop();
            for (auto const& entry: boost::iterator_range<fs::directory_iterator>(
                fs::directory_iterator(fullpath),
                fs::directory_iterator()
            ))
                if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
                    paths.push(currentPath / entry.path().filename());
        }
        else if (m_exitRequested)
        {
            ++testCount;
            paths.pop();
        }
        else
        {
            ++testCount;
            TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted);
            auto result = testTool.process();

            switch(result)
            {
            case Result::Failure:
            case Result::Exception:
                switch(testTool.handleResponse(result == Result::Exception))
                {
                case Request::Quit:
                    paths.pop();
                    m_exitRequested = true;
                    break;
                case Request::Rerun:
                    cout << "Re-running test case..." << endl;
                    --testCount;
                    break;
                case Request::Skip:
                    paths.pop();
                    break;
                }
                break;
            case Result::Success:
                paths.pop();
                ++successCount;
                break;
            }
        }
    }

    return { successCount, testCount };

}

namespace
{

void setupTerminal()
{
#if defined(_WIN32) && defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
    // Set output mode to handle virtual terminal (ANSI escape sequences)
    // ignore any error, as this is just a "nice-to-have"
    // only windows needs to be taken care of, as other platforms (Linux/OSX) support them natively.
    HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
    if (hOut == INVALID_HANDLE_VALUE)
        return;

    DWORD dwMode = 0;
    if (!GetConsoleMode(hOut, &dwMode))
        return;

    dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
    if (!SetConsoleMode(hOut, dwMode))
        return;
#endif
}

boost::optional<TestStats> runTestSuite(
    string const& _name,
    fs::path const& _basePath,
    fs::path const& _subdirectory,
    TestCase::TestCaseCreator _testCaseCreator,
    bool _formatted
)
{
    fs::path testPath = _basePath / _subdirectory;

    if (!fs::exists(testPath) || !fs::is_directory(testPath))
    {
        cerr << _name << " tests not found. Use the --testpath argument." << endl;
        return {};
    }

    TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _formatted);

    cout << endl << _name << " Test Summary: ";
    FormattedScope(cout, _formatted, {BOLD, stats ? GREEN : RED}) <<
        stats.successCount <<
        "/" <<
        stats.testCount;
    cout << " tests successful." << endl << endl;

    return stats;
}

}

int main(int argc, char *argv[])
{
    setupTerminal();

    if (getenv("EDITOR"))
        TestTool::editor = getenv("EDITOR");
    else if (fs::exists("/usr/bin/editor"))
        TestTool::editor = "/usr/bin/editor";

    fs::path testPath;
    bool disableSMT = false;
    bool formatted = true;
    po::options_description options(
        R"(isoltest, tool for interactively managing test contracts.
Usage: isoltest [Options] --testpath path
Interactively validates test contracts.

Allowed options)",
        po::options_description::m_default_line_length,
        po::options_description::m_default_line_length - 23);
    options.add_options()
        ("help", "Show this help screen.")
        ("testpath", po::value<fs::path>(&testPath), "path to test files")
        ("no-smt", "disable SMT checker")
        ("no-color", "don't use colors")
        ("editor", po::value<string>(&TestTool::editor), "editor for opening contracts");

    po::variables_map arguments;
    try
    {
        po::command_line_parser cmdLineParser(argc, argv);
        cmdLineParser.options(options);
        po::store(cmdLineParser.run(), arguments);

        if (arguments.count("help"))
        {
            cout << options << endl;
            return 0;
        }

        if (arguments.count("no-color"))
            formatted = false;

        po::notify(arguments);

        if (arguments.count("no-smt"))
            disableSMT = true;
    }
    catch (std::exception const& _exception)
    {
        cerr << _exception.what() << endl;
        return 1;
    }

    if (testPath.empty())
        testPath = dev::test::discoverTestPath();

    TestStats global_stats{0, 0};

    // Actually run the tests.
    // If you add new tests here, you also have to add them in boostTest.cpp
    if (auto stats = runTestSuite("Syntax", testPath / "libsolidity", "syntaxTests", SyntaxTest::create, formatted))
        global_stats += *stats;
    else
        return 1;

    if (auto stats = runTestSuite("JSON AST", testPath / "libsolidity", "ASTJSON", ASTJSONTest::create, formatted))
        global_stats += *stats;
    else
        return 1;

    if (auto stats = runTestSuite(
        "Yul Optimizer",
        testPath / "libyul",
        "yulOptimizerTests",
        yul::test::YulOptimizerTest::create,
        formatted
    ))
        global_stats += *stats;
    else
        return 1;

    if (!disableSMT)
    {
        if (auto stats = runTestSuite(
            "SMT Checker",
            testPath / "libsolidity",
            "smtCheckerTests",
            SyntaxTest::create,
            formatted
        ))
            global_stats += *stats;
        else
            return 1;

        if (auto stats = runTestSuite(
            "SMT Checker JSON",
            testPath / "libsolidity",
            "smtCheckerTestsJSON",
            SMTCheckerTest::create,
            formatted
        ))
            global_stats += *stats;
        else
            return 1;
    }

    cout << endl << "Summary: ";
    FormattedScope(cout, formatted, {BOLD, global_stats ? GREEN : RED}) <<
         global_stats.successCount << "/" << global_stats.testCount;
    cout << " tests successful." << endl;

    return global_stats ? 0 : 1;
}