From 190634e1f9156519d0a78124332ec4f83c8db0b9 Mon Sep 17 00:00:00 2001 From: Erik Kundt Date: Wed, 6 Feb 2019 12:10:05 +0100 Subject: [PATCH 1/2] Implements infrastructure for semantic tests. --- test/ExecutionFramework.cpp | 9 +- test/ExecutionFramework.h | 1 + test/InteractiveTests.h | 2 + test/TestCase.h | 8 +- test/libsolidity/ASTJSONTest.h | 4 +- test/libsolidity/SMTCheckerJSONTest.h | 4 +- test/libsolidity/SemanticTest.cpp | 222 ++++++++++++++++++ test/libsolidity/SemanticTest.h | 99 ++++++++ .../SolidityExecutionFramework.cpp | 7 +- test/libsolidity/SolidityExecutionFramework.h | 1 + test/libsolidity/SyntaxTest.h | 4 +- test/libyul/ObjectCompilerTest.h | 4 +- test/libyul/YulOptimizerTest.h | 4 +- test/tools/CMakeLists.txt | 2 + 14 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 test/libsolidity/SemanticTest.cpp create mode 100644 test/libsolidity/SemanticTest.h diff --git a/test/ExecutionFramework.cpp b/test/ExecutionFramework.cpp index 3922d5e9da40..ebb4e6bca187 100644 --- a/test/ExecutionFramework.cpp +++ b/test/ExecutionFramework.cpp @@ -49,8 +49,13 @@ string getIPCSocketPath() } -ExecutionFramework::ExecutionFramework() : - m_rpc(RPCSession::instance(getIPCSocketPath())), +ExecutionFramework::ExecutionFramework(): + ExecutionFramework(getIPCSocketPath()) +{ +} + +ExecutionFramework::ExecutionFramework(string const& _ipcPath): + m_rpc(RPCSession::instance(_ipcPath)), m_evmVersion(dev::test::Options::get().evmVersion()), m_optimize(dev::test::Options::get().optimize), m_showMessages(dev::test::Options::get().showMessages), diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index 86c1bcca1241..782c1d9410e2 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -53,6 +53,7 @@ class ExecutionFramework public: ExecutionFramework(); + explicit ExecutionFramework(std::string const& _ipcPath); virtual ~ExecutionFramework() = default; virtual bytes const& compileAndRunWithoutCheck( diff --git a/test/InteractiveTests.h b/test/InteractiveTests.h index be0760590849..a085fc32380d 100644 --- a/test/InteractiveTests.h +++ b/test/InteractiveTests.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ Testsuite const g_interactiveTestsuites[] = { {"Yul Optimizer", "libyul", "yulOptimizerTests", false, false, &yul::test::YulOptimizerTest::create}, {"Yul Object Compiler", "libyul", "objectCompiler", false, false, &yul::test::ObjectCompilerTest::create}, {"Syntax", "libsolidity", "syntaxTests", false, false, &SyntaxTest::create}, + {"Semantic", "libsolidity", "semanticTests", false, true, &SemanticTest::create}, {"JSON AST", "libsolidity", "ASTJSON", false, false, &ASTJSONTest::create}, {"SMT Checker", "libsolidity", "smtCheckerTests", true, false, &SyntaxTest::create}, {"SMT Checker JSON", "libsolidity", "smtCheckerTestsJSON", true, false, &SMTCheckerTest::create} diff --git a/test/TestCase.h b/test/TestCase.h index 27320009ff4b..9e28e23aeb6b 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -34,7 +34,13 @@ namespace test class TestCase { public: - using TestCaseCreator = std::unique_ptr(*)(std::string const&); + struct Config + { + std::string filename; + std::string ipcPath; + }; + + using TestCaseCreator = std::unique_ptr(*)(Config const&); virtual ~TestCase() = default; diff --git a/test/libsolidity/ASTJSONTest.h b/test/libsolidity/ASTJSONTest.h index fb63d719808b..ed349710064e 100644 --- a/test/libsolidity/ASTJSONTest.h +++ b/test/libsolidity/ASTJSONTest.h @@ -35,8 +35,8 @@ namespace test class ASTJSONTest: public TestCase { public: - static std::unique_ptr create(std::string const& _filename) - { return std::unique_ptr(new ASTJSONTest(_filename)); } + static std::unique_ptr create(Config const& _config) + { return std::unique_ptr(new ASTJSONTest(_config.filename)); } ASTJSONTest(std::string const& _filename); bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; diff --git a/test/libsolidity/SMTCheckerJSONTest.h b/test/libsolidity/SMTCheckerJSONTest.h index cf41acac55fa..256056689009 100644 --- a/test/libsolidity/SMTCheckerJSONTest.h +++ b/test/libsolidity/SMTCheckerJSONTest.h @@ -33,9 +33,9 @@ namespace test class SMTCheckerTest: public SyntaxTest { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new SMTCheckerTest(_filename)); + return std::unique_ptr(new SMTCheckerTest(_config.filename)); } SMTCheckerTest(std::string const& _filename); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp new file mode 100644 index 000000000000..f4fb000a1d61 --- /dev/null +++ b/test/libsolidity/SemanticTest.cpp @@ -0,0 +1,222 @@ +/* + This file is part of solidity. + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace dev; +using namespace solidity; +using namespace dev::solidity::test; +using namespace dev::solidity::test::formatting; +using namespace std; +namespace fs = boost::filesystem; +using namespace boost; +using namespace boost::algorithm; +using namespace boost::unit_test; + +namespace +{ + using ParamList = dev::solidity::test::ParameterList; + using FunctionCallTest = dev::solidity::test::SemanticTest::FunctionCallTest; + using FunctionCall = dev::solidity::test::FunctionCall; + + string formatBytes(bytes const& _bytes, ParamList const& _params, bool const _formatInvalid = false) + { + stringstream resultStream; + if (_bytes.empty()) + resultStream.str(); + auto it = _bytes.begin(); + for (auto const& param: _params) + { + bytes byteRange{it, it + param.abiType.size}; + // FIXME Check range + // TODO Check range + switch (param.abiType.type) + { + case ABIType::SignedDec: + if (*byteRange.begin() & 0x80) + resultStream << u2s(fromBigEndian(byteRange)); + else + resultStream << fromBigEndian(byteRange); + break; + case ABIType::UnsignedDec: + // Check if the detected type was wrong and if this could + // be signed. If an unsigned was detected in the expectations, + // but the actual result returned a signed, it would be formatted + // incorrectly. + if (*byteRange.begin() & 0x80) + resultStream << u2s(fromBigEndian(byteRange)); + else + resultStream << fromBigEndian(byteRange); + break; + case ABIType::Failure: + // If expectations are empty, the encoding type is invalid. + // In order to still print the actual result even if + // empty expectations were detected, it must be forced. + if (_formatInvalid) + resultStream << fromBigEndian(byteRange); + break; + case ABIType::None: + // If expectations are empty, the encoding type is NONE. + if (_formatInvalid) + resultStream << fromBigEndian(byteRange); + break; + } + it += param.abiType.size; + if (it != _bytes.end() && !(param.abiType.type == ABIType::None)) + resultStream << ", "; + } + return resultStream.str(); + } + + string formatFunctionCallTest( + FunctionCallTest const& _test, + string const& _linePrefix = "", + bool const _renderResult = false, + bool const _higlight = false + ) + { + stringstream _stream; + FunctionCall call = _test.call; + bool hightlight = !_test.matchesExpectation() && _higlight; + + auto formatOutput = [&](bool const _singleLine) + { + _stream << _linePrefix << "// " << call.signature; + if (call.value > u256(0)) + _stream << TestFileParser::formatToken(SoltToken::Comma) + << call.value << " " + << TestFileParser::formatToken(SoltToken::Ether); + if (!call.arguments.rawBytes().empty()) + _stream << ": " + << formatBytes(call.arguments.rawBytes(), call.arguments.parameters); + if (!_singleLine) + _stream << endl << _linePrefix << "// "; + if (_singleLine) + _stream << " "; + _stream << "-> "; + if (!_singleLine) + _stream << endl << _linePrefix << "// "; + if (hightlight) + _stream << formatting::RED_BACKGROUND; + bytes output; + if (_renderResult) + output = call.expectations.rawBytes(); + else + output = _test.rawBytes; + if (!output.empty()) + _stream << formatBytes(output, call.expectations.result); + if (hightlight) + _stream << formatting::RESET; + }; + + if (call.displayMode == FunctionCall::DisplayMode::SingleLine) + formatOutput(true); + else + formatOutput(false); + _stream << endl; + + return _stream.str(); + } +} + +SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): + SolidityExecutionFramework(_ipcPath) +{ + ifstream file(_filename); + if (!file) + BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\".")); + file.exceptions(ios::badbit); + + m_source = parseSource(file); + parseExpectations(file); +} + +bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) +{ + if (!deploy("", 0, bytes())) + BOOST_THROW_EXCEPTION(runtime_error("Failed to deploy contract.")); + + bool success = true; + for (auto& test: m_tests) + test.reset(); + + for (auto& test: m_tests) + { + bytes output = callContractFunctionWithValueNoEncoding( + test.call.signature, + test.call.value, + test.call.arguments.rawBytes() + ); + + if ((m_transactionSuccessful == test.call.expectations.failure) || (output != test.call.expectations.rawBytes())) + success = false; + + test.failure = !m_transactionSuccessful; + test.rawBytes = std::move(output); + } + + if (!success) + { + FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, _linePrefix, false, true); + + FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, _linePrefix, true, true); + + FormattedScope(_stream, _formatted, {BOLD, RED}) << _linePrefix + << "Attention: Updates on the test will apply the detected format displayed." << endl; + return false; + } + return true; +} + +void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const +{ + stringstream stream(m_source); + string line; + while (getline(stream, line)) + _stream << _linePrefix << line << endl; +} + +void SemanticTest::printUpdatedExpectations(ostream& _stream, string const& _linePrefix) const +{ + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, _linePrefix, false, false); +} + +void SemanticTest::parseExpectations(istream& _stream) +{ + TestFileParser parser{_stream}; + for (auto const& call: parser.parseFunctionCalls()) + m_tests.emplace_back(FunctionCallTest{call, bytes{}, string{}}); +} + +bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments) +{ + auto output = compileAndRunWithoutCheck(m_source, _value, _contractName, _arguments); + return !output.empty() && m_transactionSuccessful; +} diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h new file mode 100644 index 000000000000..d3adff416d9b --- /dev/null +++ b/test/libsolidity/SemanticTest.h @@ -0,0 +1,99 @@ +/* + This file is part of solidity. + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace dev +{ +namespace solidity +{ +namespace test +{ + +/** + * Class that represents a semantic test (or end-to-end test) and allows running it as part of the + * boost unit test environment or isoltest. It reads the Solidity source and an additional comment + * section from the given file. This comment section should define a set of functions to be called + * and an expected result they return after being executed. + */ +class SemanticTest: public SolidityExecutionFramework, public TestCase +{ +public: + /** + * Represents a function call and the result it returned. It stores the call + * representation itself, the actual byte result (if any) and a string representation + * used for the interactive update routine provided by isoltest. It also provides + * functionality to compare the actual result with the expectations attached to the + * call object, as well as a way to reset the result if executed multiple times. + */ + struct FunctionCallTest + { + FunctionCall call; + bytes rawBytes; + std::string output; + bool failure = true; + /// Compares raw expectations (which are converted to a byte representation before), + /// and also the expected transaction status of the function call to the actual test results. + bool matchesExpectation() const + { + return failure == call.expectations.failure && rawBytes == call.expectations.rawBytes(); + } + /// Resets current results in case the function was called and the result + /// stored already (e.g. if test case was updated via isoltest). + void reset() + { + failure = true; + rawBytes = bytes{}; + output = std::string{}; + } + }; + + static std::unique_ptr create(Config const& _options) + { return std::make_unique(_options.filename, _options.ipcPath); } + + explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath); + + bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; + void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override; + void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override; + + /// Instantiates a test file parser that parses the additional comment section at the end of + /// the input stream \param _stream. Each function call is represented using a `FunctionCallTest` + /// and added to the list of call to be executed when `run()` is called. + /// Throws if parsing expectations failed. + void parseExpectations(std::istream& _stream); + + /// Compiles and deploys currently held source. + /// Returns true if deployment was successful, false otherwise. + bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments); + + std::string m_source; + std::vector m_tests; +}; + +} +} +} diff --git a/test/libsolidity/SolidityExecutionFramework.cpp b/test/libsolidity/SolidityExecutionFramework.cpp index bb9695d1ecf2..934c563f44ff 100644 --- a/test/libsolidity/SolidityExecutionFramework.cpp +++ b/test/libsolidity/SolidityExecutionFramework.cpp @@ -28,7 +28,12 @@ using namespace dev::test; using namespace dev::solidity; using namespace dev::solidity::test; -SolidityExecutionFramework::SolidityExecutionFramework() : +SolidityExecutionFramework::SolidityExecutionFramework(): ExecutionFramework() { } + +SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath): + ExecutionFramework(_ipcPath) +{ +} diff --git a/test/libsolidity/SolidityExecutionFramework.h b/test/libsolidity/SolidityExecutionFramework.h index 73377eb98cd1..bda1c2486c80 100644 --- a/test/libsolidity/SolidityExecutionFramework.h +++ b/test/libsolidity/SolidityExecutionFramework.h @@ -43,6 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework public: SolidityExecutionFramework(); + SolidityExecutionFramework(std::string const& _ipcPath); virtual bytes const& compileAndRunWithoutCheck( std::string const& _sourceCode, diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index 69394f4af861..c331636aba11 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -53,8 +53,8 @@ struct SyntaxTestError class SyntaxTest: AnalysisFramework, public TestCase { public: - static std::unique_ptr create(std::string const& _filename) - { return std::unique_ptr(new SyntaxTest(_filename)); } + static std::unique_ptr create(Config const& _config) + { return std::unique_ptr(new SyntaxTest(_config.filename)); } SyntaxTest(std::string const& _filename); bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; diff --git a/test/libyul/ObjectCompilerTest.h b/test/libyul/ObjectCompilerTest.h index a5f8d777bda2..d81372eec874 100644 --- a/test/libyul/ObjectCompilerTest.h +++ b/test/libyul/ObjectCompilerTest.h @@ -40,9 +40,9 @@ namespace test class ObjectCompilerTest: public dev::solidity::test::TestCase { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new ObjectCompilerTest(_filename)); + return std::unique_ptr(new ObjectCompilerTest(_config.filename)); } explicit ObjectCompilerTest(std::string const& _filename); diff --git a/test/libyul/YulOptimizerTest.h b/test/libyul/YulOptimizerTest.h index 5009b82c7b45..cb21090050e4 100644 --- a/test/libyul/YulOptimizerTest.h +++ b/test/libyul/YulOptimizerTest.h @@ -41,9 +41,9 @@ namespace test class YulOptimizerTest: public dev::solidity::test::TestCase { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new YulOptimizerTest(_filename)); + return std::unique_ptr(new YulOptimizerTest(_config.filename)); } explicit YulOptimizerTest(std::string const& _filename); diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt index 7e070ebb551b..8200806d7589 100644 --- a/test/tools/CMakeLists.txt +++ b/test/tools/CMakeLists.txt @@ -13,7 +13,9 @@ add_executable(isoltest ../Options.cpp ../Common.cpp ../TestCase.cpp + ../libsolidity/util/TestFileParser.cpp ../libsolidity/SyntaxTest.cpp + ../libsolidity/SemanticTest.cpp ../libsolidity/AnalysisFramework.cpp ../libsolidity/SolidityExecutionFramework.cpp ../ExecutionFramework.cpp From dacad629ef4dbfa58a5a335748cb4c2cc0deec43 Mon Sep 17 00:00:00 2001 From: Erik Kundt Date: Wed, 6 Feb 2019 12:10:48 +0100 Subject: [PATCH 2/2] Adds semantic tests to test framework and isoltest. --- test/RPCSession.cpp | 13 +- test/TestCase.h | 8 + test/boostTest.cpp | 9 +- test/libsolidity/SemanticTest.cpp | 159 ++++++++++++------ test/libsolidity/SemanticTest.h | 2 +- test/libsolidity/semanticTests/smoke_test.sol | 17 ++ .../semanticTests/smoke_test_multiline.sol | 11 ++ .../smoke_test_multiline_comments.sol | 17 ++ test/libsolidity/util/TestFileParser.cpp | 23 ++- test/libsolidity/util/TestFileParser.h | 13 +- test/libsolidity/util/TestFileParserTests.cpp | 57 ++++++- test/tools/isoltest.cpp | 47 ++++-- 12 files changed, 291 insertions(+), 85 deletions(-) create mode 100644 test/libsolidity/semanticTests/smoke_test.sol create mode 100644 test/libsolidity/semanticTests/smoke_test_multiline.sol create mode 100644 test/libsolidity/semanticTests/smoke_test_multiline_comments.sol diff --git a/test/RPCSession.cpp b/test/RPCSession.cpp index dd276417b0a4..19328477f938 100644 --- a/test/RPCSession.cpp +++ b/test/RPCSession.cpp @@ -139,9 +139,16 @@ string IPCSocket::sendRequest(string const& _req) RPCSession& RPCSession::instance(const string& _path) { - static RPCSession session(_path); - BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path); - return session; + try + { + static RPCSession session(_path); + BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path); + return session; + } + catch (std::exception const&) + { + BOOST_THROW_EXCEPTION(std::runtime_error("Error creating RPC session for socket: " + _path)); + } } string RPCSession::eth_getCode(string const& _address, string const& _blockNumber) diff --git a/test/TestCase.h b/test/TestCase.h index 9e28e23aeb6b..52bca52749cb 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -30,6 +30,14 @@ namespace solidity namespace test { +#define soltestAssert(CONDITION, DESCRIPTION) \ + do \ + { \ + if (!(CONDITION)) \ + BOOST_THROW_EXCEPTION(runtime_error(DESCRIPTION)); \ + } \ + while (false) + /** Common superclass of SyntaxTest and SemanticsTest. */ class TestCase { diff --git a/test/boostTest.cpp b/test/boostTest.cpp index d6e75cb92146..5cda2fae4bbf 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -74,11 +74,13 @@ int registerTests( boost::unit_test::test_suite& _suite, boost::filesystem::path const& _basepath, boost::filesystem::path const& _path, + std::string const& _ipcPath, TestCase::TestCaseCreator _testCaseCreator ) { int numTestsAdded = 0; fs::path fullpath = _basepath / _path; + TestCase::Config config{fullpath.string(), _ipcPath}; if (fs::is_directory(fullpath)) { test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); @@ -87,7 +89,7 @@ int registerTests( fs::directory_iterator() )) if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename())) - numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator); + numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _ipcPath, _testCaseCreator); _suite.add(sub_suite); } else @@ -96,13 +98,13 @@ int registerTests( filenames.emplace_back(new string(_path.string())); _suite.add(make_test_case( - [fullpath, _testCaseCreator] + [config, _testCaseCreator] { BOOST_REQUIRE_NO_THROW({ try { stringstream errorStream; - if (!_testCaseCreator(fullpath.string())->run(errorStream)) + if (!_testCaseCreator(config)->run(errorStream)) BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str()); } catch (boost::exception const& _e) @@ -142,6 +144,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) master, options.testPath / ts.path, ts.subpath, + options.ipcPath, ts.testCaseCreator ) > 0, std::string("no ") + ts.title + " tests found"); } diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index f4fb000a1d61..2f95fd2168d8 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -28,30 +28,33 @@ using namespace dev; using namespace solidity; using namespace dev::solidity::test; -using namespace dev::solidity::test::formatting; +using namespace dev::formatting; using namespace std; -namespace fs = boost::filesystem; using namespace boost; using namespace boost::algorithm; using namespace boost::unit_test; +namespace fs = boost::filesystem; namespace { - using ParamList = dev::solidity::test::ParameterList; - using FunctionCallTest = dev::solidity::test::SemanticTest::FunctionCallTest; + using FunctionCallTest = SemanticTest::FunctionCallTest; using FunctionCall = dev::solidity::test::FunctionCall; + using ParamList = dev::solidity::test::ParameterList; + - string formatBytes(bytes const& _bytes, ParamList const& _params, bool const _formatInvalid = false) + string formatBytes(bytes const& _bytes, ParamList const& _params) { stringstream resultStream; if (_bytes.empty()) - resultStream.str(); + return {}; auto it = _bytes.begin(); for (auto const& param: _params) { - bytes byteRange{it, it + param.abiType.size}; - // FIXME Check range - // TODO Check range + long offset = static_cast(param.abiType.size); + auto offsetIter = it + offset; + soltestAssert(offsetIter <= _bytes.end(), "Byte range can not be extended past the end of given bytes."); + + bytes byteRange{it, offsetIter}; switch (param.abiType.type) { case ABIType::SignedDec: @@ -71,22 +74,29 @@ namespace resultStream << fromBigEndian(byteRange); break; case ABIType::Failure: - // If expectations are empty, the encoding type is invalid. - // In order to still print the actual result even if - // empty expectations were detected, it must be forced. - if (_formatInvalid) - resultStream << fromBigEndian(byteRange); break; case ABIType::None: - // If expectations are empty, the encoding type is NONE. - if (_formatInvalid) - resultStream << fromBigEndian(byteRange); break; } - it += param.abiType.size; + it += offset; if (it != _bytes.end() && !(param.abiType.type == ABIType::None)) resultStream << ", "; } + soltestAssert(it == _bytes.end(), "Parameter encoding too short for the given byte range."); + return resultStream.str(); + } + + string formatRawArguments(ParamList const& _params, string const& _linePrefix = "") + { + stringstream resultStream; + for (auto const& param: _params) + { + if (param.format.newline) + resultStream << endl << _linePrefix << "//"; + resultStream << " " << param.rawString; + if (¶m != &_params.back()) + resultStream << ","; + } return resultStream.str(); } @@ -94,41 +104,86 @@ namespace FunctionCallTest const& _test, string const& _linePrefix = "", bool const _renderResult = false, - bool const _higlight = false + bool const _highlight = false ) { + using namespace soltest; + using Token = soltest::Token; + stringstream _stream; FunctionCall call = _test.call; - bool hightlight = !_test.matchesExpectation() && _higlight; + bool highlight = !_test.matchesExpectation() && _highlight; auto formatOutput = [&](bool const _singleLine) { - _stream << _linePrefix << "// " << call.signature; + string ws = " "; + string arrow = formatToken(Token::Arrow); + string colon = formatToken(Token::Colon); + string comma = formatToken(Token::Comma); + string comment = formatToken(Token::Comment); + string ether = formatToken(Token::Ether); + string newline = formatToken(Token::Newline); + string failure = formatToken(Token::Failure); + + /// Prints the function signature. This is the same independent from the display-mode. + _stream << _linePrefix << newline << ws << call.signature; if (call.value > u256(0)) - _stream << TestFileParser::formatToken(SoltToken::Comma) - << call.value << " " - << TestFileParser::formatToken(SoltToken::Ether); + _stream << comma << ws << call.value << ws << ether; if (!call.arguments.rawBytes().empty()) - _stream << ": " - << formatBytes(call.arguments.rawBytes(), call.arguments.parameters); - if (!_singleLine) - _stream << endl << _linePrefix << "// "; + { + string output = formatRawArguments(call.arguments.parameters, _linePrefix); + _stream << colon << output; + } + + /// Prints comments on the function parameters and the arrow taking + /// the display-mode into account. if (_singleLine) - _stream << " "; - _stream << "-> "; - if (!_singleLine) - _stream << endl << _linePrefix << "// "; - if (hightlight) - _stream << formatting::RED_BACKGROUND; - bytes output; - if (_renderResult) - output = call.expectations.rawBytes(); + { + if (!call.arguments.comment.empty()) + _stream << ws << comment << call.arguments.comment << comment; + _stream << ws << arrow << ws; + } + else + { + _stream << endl << _linePrefix << newline << ws; + if (!call.arguments.comment.empty()) + { + _stream << comment << call.arguments.comment << comment; + _stream << endl << _linePrefix << newline << ws; + } + _stream << arrow << ws; + } + + /// Print either the expected output or the actual result output + string result; + if (!_renderResult) + { + bytes output = call.expectations.rawBytes(); + bool const isFailure = call.expectations.failure; + result = isFailure ? failure : formatBytes(output, call.expectations.result); + } else - output = _test.rawBytes; - if (!output.empty()) - _stream << formatBytes(output, call.expectations.result); - if (hightlight) - _stream << formatting::RESET; + { + bytes output = _test.rawBytes; + bool const isFailure = _test.failure; + result = isFailure ? failure : formatBytes(output, call.expectations.result); + } + AnsiColorized(_stream, highlight, {RED_BACKGROUND}) << result; + + /// Print comments on expectations taking the display-mode into account. + if (_singleLine) + { + if (!call.expectations.comment.empty()) + _stream << ws << comment << call.expectations.comment << comment; + } + else + { + if (!call.expectations.comment.empty()) + { + _stream << endl << _linePrefix << newline << ws; + _stream << comment << call.expectations.comment << comment; + } + } }; if (call.displayMode == FunctionCall::DisplayMode::SingleLine) @@ -145,8 +200,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): SolidityExecutionFramework(_ipcPath) { ifstream file(_filename); - if (!file) - BOOST_THROW_EXCEPTION(runtime_error("Cannot open test contract: \"" + _filename + "\".")); + soltestAssert(file, "Cannot open test contract: \"" + _filename + "\"."); file.exceptions(ios::badbit); m_source = parseSource(file); @@ -155,8 +209,7 @@ SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) { - if (!deploy("", 0, bytes())) - BOOST_THROW_EXCEPTION(runtime_error("Failed to deploy contract.")); + soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract."); bool success = true; for (auto& test: m_tests) @@ -179,15 +232,15 @@ bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _ if (!success) { - FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; for (auto const& test: m_tests) - _stream << formatFunctionCallTest(test, _linePrefix, false, true); + _stream << formatFunctionCallTest(test, _linePrefix, false, true & _formatted); - FormattedScope(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; for (auto const& test: m_tests) - _stream << formatFunctionCallTest(test, _linePrefix, true, true); + _stream << formatFunctionCallTest(test, _linePrefix, true, true & _formatted); - FormattedScope(_stream, _formatted, {BOLD, RED}) << _linePrefix + AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix << "Attention: Updates on the test will apply the detected format displayed." << endl; return false; } @@ -202,10 +255,10 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool _stream << _linePrefix << line << endl; } -void SemanticTest::printUpdatedExpectations(ostream& _stream, string const& _linePrefix) const +void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const { for (auto const& test: m_tests) - _stream << formatFunctionCallTest(test, _linePrefix, false, false); + _stream << formatFunctionCallTest(test, "", true, false); } void SemanticTest::parseExpectations(istream& _stream) diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h index d3adff416d9b..a8dd4ee66d79 100644 --- a/test/libsolidity/SemanticTest.h +++ b/test/libsolidity/SemanticTest.h @@ -15,11 +15,11 @@ #pragma once #include -#include #include #include #include #include +#include #include #include diff --git a/test/libsolidity/semanticTests/smoke_test.sol b/test/libsolidity/semanticTests/smoke_test.sol new file mode 100644 index 000000000000..d4b5d33f8150 --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test.sol @@ -0,0 +1,17 @@ +contract C { + function f() public returns (uint) { + return 1; + } + function g(uint x, uint y) public returns (uint) { + return x - y; + } + function h() public payable returns (uint) { + return f(); + } +} +// ---- +// f() -> 1 +// g(uint256,uint256): 1, -2 -> 3 +// h(), 1 ether -> 1 +// j() -> FAILURE +// i() # Does not exist. # -> FAILURE # Reverts. # diff --git a/test/libsolidity/semanticTests/smoke_test_multiline.sol b/test/libsolidity/semanticTests/smoke_test_multiline.sol new file mode 100644 index 000000000000..7395b1c3b1a0 --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test_multiline.sol @@ -0,0 +1,11 @@ +contract C { + function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) { + return a + b + c + d + e; + } +} +// ---- +// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1 +// -> 5 +// g() +// # g() does not exist # +// -> FAILURE diff --git a/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol b/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol new file mode 100644 index 000000000000..17de40fc4dab --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) { + return a + b + c + d + e; + } +} +// ---- +// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1 +// # A comment on the function parameters. # +// -> 5 +// f(uint256,uint256,uint256,uint256,uint256): +// 1, +// 1, +// 1, +// 1, +// 1 +// -> 5 +// # Should return sum of all parameters. # diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp index 5f1a420e9a59..9f05e72ffbd9 100644 --- a/test/libsolidity/util/TestFileParser.cpp +++ b/test/libsolidity/util/TestFileParser.cpp @@ -85,6 +85,8 @@ vector TestFileParser::parseFunctionCalls() expect(Token::Arrow); call.expectations = parseFunctionCallExpectations(); + + accept(Token::Newline, true); call.expectations.comment = parseComment(); calls.emplace_back(std::move(call)); @@ -194,38 +196,45 @@ Parameter TestFileParser::parseParameter() if (accept(Token::Newline, true)) parameter.format.newline = true; auto literal = parseABITypeLiteral(); - parameter.rawBytes = literal.first; - parameter.abiType = literal.second; + parameter.rawBytes = get<0>(literal); + parameter.abiType = get<1>(literal); + parameter.rawString = get<2>(literal); return parameter; } -pair TestFileParser::parseABITypeLiteral() +tuple TestFileParser::parseABITypeLiteral() { try { u256 number{0}; ABIType abiType{ABIType::None, 0}; + string rawString; if (accept(Token::Sub)) { abiType = ABIType{ABIType::SignedDec, 32}; expect(Token::Sub); - number = convertNumber(parseNumber()) * -1; + rawString += formatToken(Token::Sub); + string parsed = parseNumber(); + rawString += parsed; + number = convertNumber(parsed) * -1; } else { if (accept(Token::Number)) { abiType = ABIType{ABIType::UnsignedDec, 32}; - number = convertNumber(parseNumber()); + string parsed = parseNumber(); + rawString += parsed; + number = convertNumber(parsed); } else if (accept(Token::Failure, true)) { abiType = ABIType{ABIType::Failure, 0}; - return make_pair(bytes{}, abiType); + return make_tuple(bytes{}, abiType, rawString); } } - return make_pair(toBigEndian(number), abiType); + return make_tuple(toBigEndian(number), abiType, rawString); } catch (std::exception const&) { diff --git a/test/libsolidity/util/TestFileParser.h b/test/libsolidity/util/TestFileParser.h index 6b79008833fd..3dbe90a3248b 100644 --- a/test/libsolidity/util/TestFileParser.h +++ b/test/libsolidity/util/TestFileParser.h @@ -114,7 +114,7 @@ struct ABIType */ struct FormatInfo { - bool newline; + bool newline = false; }; /** @@ -132,6 +132,9 @@ struct Parameter /// compared to the actual result of a function call /// and used for validating it. bytes rawBytes; + /// Stores the raw string representation of this parameter. + /// Used to print the unformatted arguments of a function call. + std::string rawString; /// Types that were used to encode `rawBytes`. Expectations /// are usually comma separated literals. Their type is auto- /// detected and retained in order to format them later on. @@ -327,13 +330,15 @@ class TestFileParser Parameter parseParameter(); /// Parses and converts the current literal to its byte representation and - /// preserves the chosen ABI type. Based on that type information, the driver of - /// this parser can format arguments, expectations and results. Supported types: + /// preserves the chosen ABI type, as well as a raw, unformatted string representation + /// of this literal. + /// Based on the type information retrieved, the driver of this parser may format arguments, + /// expectations and results. Supported types: /// - unsigned and signed decimal number literals. /// Returns invalid ABI type for empty literal. This is needed in order /// to detect empty expectations. Throws a ParserError if data is encoded incorrectly or /// if data type is not supported. - std::pair parseABITypeLiteral(); + std::tuple parseABITypeLiteral(); /// Recursively parses an identifier or a tuple definition that contains identifiers /// and / or parentheses like `((uint, uint), (uint, (uint, uint)), uint)`. diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index 5b89341da4e8..1898da5fefc4 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -56,7 +56,8 @@ void testFunctionCall( bytes _expectations = bytes{}, u256 _value = 0, string _argumentComment = "", - string _expectationComment = "" + string _expectationComment = "", + vector _rawArguments = vector{} ) { BOOST_REQUIRE_EQUAL(_call.expectations.failure, _failure); @@ -67,6 +68,17 @@ void testFunctionCall( BOOST_REQUIRE_EQUAL(_call.value, _value); BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment); BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment); + + if (!_rawArguments.empty()) + { + BOOST_REQUIRE_EQUAL(_call.arguments.parameters.size(), _rawArguments.size()); + size_t index = 0; + for (Parameter const& param: _call.arguments.parameters) + { + BOOST_REQUIRE_EQUAL(param.rawString, _rawArguments[index]); + ++index; + } + } } BOOST_AUTO_TEST_SUITE(TestFileParserTest) @@ -112,11 +124,16 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) { char const* source = R"( // f(uint256, uint256): 1, 1 + // # Comment on the parameters. # // -> // # This call should not return a value, but still succeed. # + // f() + // # Comment on no parameters. # + // -> 1 + // # This comment should be parsed. # )"; auto const calls = parse(source); - BOOST_REQUIRE_EQUAL(calls.size(), 1); + BOOST_REQUIRE_EQUAL(calls.size(), 2); testFunctionCall( calls.at(0), Mode::MultiLine, @@ -125,9 +142,20 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) fmt::encodeArgs(1, 1), fmt::encodeArgs(), 0, - "", + " Comment on the parameters. ", " This call should not return a value, but still succeed. " ); + testFunctionCall( + calls.at(1), + Mode::MultiLine, + "f()", + false, + fmt::encodeArgs(), + fmt::encodeArgs(1), + 0, + " Comment on no parameters. ", + " This comment should be parsed. " + ); } BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success) @@ -383,7 +411,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) ); } -BOOST_AUTO_TEST_CASE(call_signature) +BOOST_AUTO_TEST_CASE(call_signature_valid) { char const* source = R"( // f(uint256, uint8, string) -> FAILURE @@ -395,6 +423,27 @@ BOOST_AUTO_TEST_CASE(call_signature) testFunctionCall(calls.at(1), Mode::SingleLine, "f(invalid,xyz,foo)", true); } +BOOST_AUTO_TEST_CASE(call_raw_arguments) +{ + char const* source = R"( + // f(): 1, -2, -3 -> + )"; + auto const calls = parse(source); + BOOST_REQUIRE_EQUAL(calls.size(), 1); + testFunctionCall( + calls.at(0), + Mode::SingleLine, + "f()", + false, + fmt::encodeArgs(1, -2, -3), + fmt::encodeArgs(), + 0, + "", + "", + {"1", "-2", "-3"} + ); +} + BOOST_AUTO_TEST_CASE(call_newline_invalid) { char const* source = R"( diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index c15183f8f613..1d3b9eda90a7 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -64,8 +64,9 @@ class TestTool TestCase::TestCaseCreator _testCaseCreator, string const& _name, fs::path const& _path, + string const& _ipcPath, bool _formatted - ): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path) + ): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted) {} enum class Result @@ -81,6 +82,7 @@ class TestTool TestCase::TestCaseCreator _testCaseCreator, fs::path const& _basepath, fs::path const& _path, + string const& _ipcPath, bool const _formatted ); @@ -96,9 +98,10 @@ class TestTool Request handleResponse(bool const _exception); TestCase::TestCaseCreator m_testCaseCreator; - bool const m_formatted = false; string const m_name; fs::path const m_path; + string m_ipcPath; + bool const m_formatted = false; unique_ptr m_test; static bool m_exitRequested; }; @@ -115,25 +118,25 @@ TestTool::Result TestTool::process() try { - m_test = m_testCaseCreator(m_path.string()); + m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath}); success = m_test->run(outputMessages, " ", m_formatted); } catch(boost::exception const& _e) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Exception during syntax test: " << boost::diagnostic_information(_e) << endl; + "Exception during test: " << boost::diagnostic_information(_e) << endl; return Result::Exception; } catch (std::exception const& _e) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Exception during syntax test: " << _e.what() << endl; + "Exception during test: " << _e.what() << endl; return Result::Exception; } catch (...) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Unknown exception during syntax test." << endl; + "Unknown exception during test." << endl; return Result::Exception; } @@ -199,6 +202,7 @@ TestStats TestTool::processPath( TestCase::TestCaseCreator _testCaseCreator, fs::path const& _basepath, fs::path const& _path, + string const& _ipcPath, bool const _formatted ) { @@ -230,7 +234,7 @@ TestStats TestTool::processPath( else { ++testCount; - TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted); + TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted); auto result = testTool.process(); switch(result) @@ -291,6 +295,7 @@ boost::optional runTestSuite( string const& _name, fs::path const& _basePath, fs::path const& _subdirectory, + string const& _ipcPath, TestCase::TestCaseCreator _testCaseCreator, bool _formatted ) @@ -303,7 +308,7 @@ boost::optional runTestSuite( return {}; } - TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _formatted); + TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted); cout << endl << _name << " Test Summary: "; AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) << @@ -327,11 +332,13 @@ int main(int argc, char *argv[]) TestTool::editor = "/usr/bin/editor"; fs::path testPath; + string ipcPath; + bool disableIPC = false; bool disableSMT = false; bool formatted = true; po::options_description options( R"(isoltest, tool for interactively managing test contracts. -Usage: isoltest [Options] --testpath path +Usage: isoltest [Options] --ipcpath ipcpath Interactively validates test contracts. Allowed options)", @@ -340,6 +347,8 @@ Allowed options)", options.add_options() ("help", "Show this help screen.") ("testpath", po::value(&testPath), "path to test files") + ("ipcpath", po::value(&ipcPath), "path to ipc socket") + ("no-ipc", "disable semantic tests") ("no-smt", "disable SMT checker") ("no-color", "don't use colors") ("editor", po::value(&TestTool::editor), "editor for opening contracts"); @@ -362,8 +371,23 @@ Allowed options)", po::notify(arguments); + if (arguments.count("no-ipc")) + disableIPC = true; + else + { + solAssert( + !ipcPath.empty(), + "No ipc path specified. The --ipcpath argument is required, unless --no-ipc is used." + ); + solAssert( + fs::exists(ipcPath), + "Invalid ipc path specified." + ); + } + if (arguments.count("no-smt")) disableSMT = true; + } catch (std::exception const& _exception) { @@ -380,10 +404,13 @@ Allowed options)", // Interactive tests are added in InteractiveTests.h for (auto const& ts: g_interactiveTestsuites) { + if (ts.ipc && disableIPC) + continue; + if (ts.smt && disableSMT) continue; - if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ts.testCaseCreator, formatted)) + if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ipcPath, ts.testCaseCreator, formatted)) global_stats += *stats; else return 1;