diff --git a/test/ExecutionFramework.cpp b/test/ExecutionFramework.cpp index 3922d5e9da40..ebb4e6bca187 100644 --- a/test/ExecutionFramework.cpp +++ b/test/ExecutionFramework.cpp @@ -49,8 +49,13 @@ string getIPCSocketPath() } -ExecutionFramework::ExecutionFramework() : - m_rpc(RPCSession::instance(getIPCSocketPath())), +ExecutionFramework::ExecutionFramework(): + ExecutionFramework(getIPCSocketPath()) +{ +} + +ExecutionFramework::ExecutionFramework(string const& _ipcPath): + m_rpc(RPCSession::instance(_ipcPath)), m_evmVersion(dev::test::Options::get().evmVersion()), m_optimize(dev::test::Options::get().optimize), m_showMessages(dev::test::Options::get().showMessages), diff --git a/test/ExecutionFramework.h b/test/ExecutionFramework.h index 86c1bcca1241..782c1d9410e2 100644 --- a/test/ExecutionFramework.h +++ b/test/ExecutionFramework.h @@ -53,6 +53,7 @@ class ExecutionFramework public: ExecutionFramework(); + explicit ExecutionFramework(std::string const& _ipcPath); virtual ~ExecutionFramework() = default; virtual bytes const& compileAndRunWithoutCheck( diff --git a/test/InteractiveTests.h b/test/InteractiveTests.h index be0760590849..a085fc32380d 100644 --- a/test/InteractiveTests.h +++ b/test/InteractiveTests.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ Testsuite const g_interactiveTestsuites[] = { {"Yul Optimizer", "libyul", "yulOptimizerTests", false, false, &yul::test::YulOptimizerTest::create}, {"Yul Object Compiler", "libyul", "objectCompiler", false, false, &yul::test::ObjectCompilerTest::create}, {"Syntax", "libsolidity", "syntaxTests", false, false, &SyntaxTest::create}, + {"Semantic", "libsolidity", "semanticTests", false, true, &SemanticTest::create}, {"JSON AST", "libsolidity", "ASTJSON", false, false, &ASTJSONTest::create}, {"SMT Checker", "libsolidity", "smtCheckerTests", true, false, &SyntaxTest::create}, {"SMT Checker JSON", "libsolidity", "smtCheckerTestsJSON", true, false, &SMTCheckerTest::create} diff --git a/test/RPCSession.cpp b/test/RPCSession.cpp index dd276417b0a4..19328477f938 100644 --- a/test/RPCSession.cpp +++ b/test/RPCSession.cpp @@ -139,9 +139,16 @@ string IPCSocket::sendRequest(string const& _req) RPCSession& RPCSession::instance(const string& _path) { - static RPCSession session(_path); - BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path); - return session; + try + { + static RPCSession session(_path); + BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path); + return session; + } + catch (std::exception const&) + { + BOOST_THROW_EXCEPTION(std::runtime_error("Error creating RPC session for socket: " + _path)); + } } string RPCSession::eth_getCode(string const& _address, string const& _blockNumber) diff --git a/test/TestCase.h b/test/TestCase.h index 27320009ff4b..52bca52749cb 100644 --- a/test/TestCase.h +++ b/test/TestCase.h @@ -30,11 +30,25 @@ namespace solidity namespace test { +#define soltestAssert(CONDITION, DESCRIPTION) \ + do \ + { \ + if (!(CONDITION)) \ + BOOST_THROW_EXCEPTION(runtime_error(DESCRIPTION)); \ + } \ + while (false) + /** Common superclass of SyntaxTest and SemanticsTest. */ class TestCase { public: - using TestCaseCreator = std::unique_ptr(*)(std::string const&); + struct Config + { + std::string filename; + std::string ipcPath; + }; + + using TestCaseCreator = std::unique_ptr(*)(Config const&); virtual ~TestCase() = default; diff --git a/test/boostTest.cpp b/test/boostTest.cpp index d6e75cb92146..5cda2fae4bbf 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -74,11 +74,13 @@ int registerTests( boost::unit_test::test_suite& _suite, boost::filesystem::path const& _basepath, boost::filesystem::path const& _path, + std::string const& _ipcPath, TestCase::TestCaseCreator _testCaseCreator ) { int numTestsAdded = 0; fs::path fullpath = _basepath / _path; + TestCase::Config config{fullpath.string(), _ipcPath}; if (fs::is_directory(fullpath)) { test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string()); @@ -87,7 +89,7 @@ int registerTests( fs::directory_iterator() )) if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename())) - numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator); + numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _ipcPath, _testCaseCreator); _suite.add(sub_suite); } else @@ -96,13 +98,13 @@ int registerTests( filenames.emplace_back(new string(_path.string())); _suite.add(make_test_case( - [fullpath, _testCaseCreator] + [config, _testCaseCreator] { BOOST_REQUIRE_NO_THROW({ try { stringstream errorStream; - if (!_testCaseCreator(fullpath.string())->run(errorStream)) + if (!_testCaseCreator(config)->run(errorStream)) BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str()); } catch (boost::exception const& _e) @@ -142,6 +144,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) master, options.testPath / ts.path, ts.subpath, + options.ipcPath, ts.testCaseCreator ) > 0, std::string("no ") + ts.title + " tests found"); } diff --git a/test/libsolidity/ASTJSONTest.h b/test/libsolidity/ASTJSONTest.h index fb63d719808b..ed349710064e 100644 --- a/test/libsolidity/ASTJSONTest.h +++ b/test/libsolidity/ASTJSONTest.h @@ -35,8 +35,8 @@ namespace test class ASTJSONTest: public TestCase { public: - static std::unique_ptr create(std::string const& _filename) - { return std::unique_ptr(new ASTJSONTest(_filename)); } + static std::unique_ptr create(Config const& _config) + { return std::unique_ptr(new ASTJSONTest(_config.filename)); } ASTJSONTest(std::string const& _filename); bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; diff --git a/test/libsolidity/SMTCheckerJSONTest.h b/test/libsolidity/SMTCheckerJSONTest.h index cf41acac55fa..256056689009 100644 --- a/test/libsolidity/SMTCheckerJSONTest.h +++ b/test/libsolidity/SMTCheckerJSONTest.h @@ -33,9 +33,9 @@ namespace test class SMTCheckerTest: public SyntaxTest { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new SMTCheckerTest(_filename)); + return std::unique_ptr(new SMTCheckerTest(_config.filename)); } SMTCheckerTest(std::string const& _filename); diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp new file mode 100644 index 000000000000..2f95fd2168d8 --- /dev/null +++ b/test/libsolidity/SemanticTest.cpp @@ -0,0 +1,275 @@ +/* + This file is part of solidity. + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace dev; +using namespace solidity; +using namespace dev::solidity::test; +using namespace dev::formatting; +using namespace std; +using namespace boost; +using namespace boost::algorithm; +using namespace boost::unit_test; +namespace fs = boost::filesystem; + +namespace +{ + using FunctionCallTest = SemanticTest::FunctionCallTest; + using FunctionCall = dev::solidity::test::FunctionCall; + using ParamList = dev::solidity::test::ParameterList; + + + string formatBytes(bytes const& _bytes, ParamList const& _params) + { + stringstream resultStream; + if (_bytes.empty()) + return {}; + auto it = _bytes.begin(); + for (auto const& param: _params) + { + long offset = static_cast(param.abiType.size); + auto offsetIter = it + offset; + soltestAssert(offsetIter <= _bytes.end(), "Byte range can not be extended past the end of given bytes."); + + bytes byteRange{it, offsetIter}; + switch (param.abiType.type) + { + case ABIType::SignedDec: + if (*byteRange.begin() & 0x80) + resultStream << u2s(fromBigEndian(byteRange)); + else + resultStream << fromBigEndian(byteRange); + break; + case ABIType::UnsignedDec: + // Check if the detected type was wrong and if this could + // be signed. If an unsigned was detected in the expectations, + // but the actual result returned a signed, it would be formatted + // incorrectly. + if (*byteRange.begin() & 0x80) + resultStream << u2s(fromBigEndian(byteRange)); + else + resultStream << fromBigEndian(byteRange); + break; + case ABIType::Failure: + break; + case ABIType::None: + break; + } + it += offset; + if (it != _bytes.end() && !(param.abiType.type == ABIType::None)) + resultStream << ", "; + } + soltestAssert(it == _bytes.end(), "Parameter encoding too short for the given byte range."); + return resultStream.str(); + } + + string formatRawArguments(ParamList const& _params, string const& _linePrefix = "") + { + stringstream resultStream; + for (auto const& param: _params) + { + if (param.format.newline) + resultStream << endl << _linePrefix << "//"; + resultStream << " " << param.rawString; + if (¶m != &_params.back()) + resultStream << ","; + } + return resultStream.str(); + } + + string formatFunctionCallTest( + FunctionCallTest const& _test, + string const& _linePrefix = "", + bool const _renderResult = false, + bool const _highlight = false + ) + { + using namespace soltest; + using Token = soltest::Token; + + stringstream _stream; + FunctionCall call = _test.call; + bool highlight = !_test.matchesExpectation() && _highlight; + + auto formatOutput = [&](bool const _singleLine) + { + string ws = " "; + string arrow = formatToken(Token::Arrow); + string colon = formatToken(Token::Colon); + string comma = formatToken(Token::Comma); + string comment = formatToken(Token::Comment); + string ether = formatToken(Token::Ether); + string newline = formatToken(Token::Newline); + string failure = formatToken(Token::Failure); + + /// Prints the function signature. This is the same independent from the display-mode. + _stream << _linePrefix << newline << ws << call.signature; + if (call.value > u256(0)) + _stream << comma << ws << call.value << ws << ether; + if (!call.arguments.rawBytes().empty()) + { + string output = formatRawArguments(call.arguments.parameters, _linePrefix); + _stream << colon << output; + } + + /// Prints comments on the function parameters and the arrow taking + /// the display-mode into account. + if (_singleLine) + { + if (!call.arguments.comment.empty()) + _stream << ws << comment << call.arguments.comment << comment; + _stream << ws << arrow << ws; + } + else + { + _stream << endl << _linePrefix << newline << ws; + if (!call.arguments.comment.empty()) + { + _stream << comment << call.arguments.comment << comment; + _stream << endl << _linePrefix << newline << ws; + } + _stream << arrow << ws; + } + + /// Print either the expected output or the actual result output + string result; + if (!_renderResult) + { + bytes output = call.expectations.rawBytes(); + bool const isFailure = call.expectations.failure; + result = isFailure ? failure : formatBytes(output, call.expectations.result); + } + else + { + bytes output = _test.rawBytes; + bool const isFailure = _test.failure; + result = isFailure ? failure : formatBytes(output, call.expectations.result); + } + AnsiColorized(_stream, highlight, {RED_BACKGROUND}) << result; + + /// Print comments on expectations taking the display-mode into account. + if (_singleLine) + { + if (!call.expectations.comment.empty()) + _stream << ws << comment << call.expectations.comment << comment; + } + else + { + if (!call.expectations.comment.empty()) + { + _stream << endl << _linePrefix << newline << ws; + _stream << comment << call.expectations.comment << comment; + } + } + }; + + if (call.displayMode == FunctionCall::DisplayMode::SingleLine) + formatOutput(true); + else + formatOutput(false); + _stream << endl; + + return _stream.str(); + } +} + +SemanticTest::SemanticTest(string const& _filename, string const& _ipcPath): + SolidityExecutionFramework(_ipcPath) +{ + ifstream file(_filename); + soltestAssert(file, "Cannot open test contract: \"" + _filename + "\"."); + file.exceptions(ios::badbit); + + m_source = parseSource(file); + parseExpectations(file); +} + +bool SemanticTest::run(ostream& _stream, string const& _linePrefix, bool const _formatted) +{ + soltestAssert(deploy("", 0, bytes()), "Failed to deploy contract."); + + bool success = true; + for (auto& test: m_tests) + test.reset(); + + for (auto& test: m_tests) + { + bytes output = callContractFunctionWithValueNoEncoding( + test.call.signature, + test.call.value, + test.call.arguments.rawBytes() + ); + + if ((m_transactionSuccessful == test.call.expectations.failure) || (output != test.call.expectations.rawBytes())) + success = false; + + test.failure = !m_transactionSuccessful; + test.rawBytes = std::move(output); + } + + if (!success) + { + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl; + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, _linePrefix, false, true & _formatted); + + AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl; + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, _linePrefix, true, true & _formatted); + + AnsiColorized(_stream, _formatted, {BOLD, RED}) << _linePrefix + << "Attention: Updates on the test will apply the detected format displayed." << endl; + return false; + } + return true; +} + +void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool const) const +{ + stringstream stream(m_source); + string line; + while (getline(stream, line)) + _stream << _linePrefix << line << endl; +} + +void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const +{ + for (auto const& test: m_tests) + _stream << formatFunctionCallTest(test, "", true, false); +} + +void SemanticTest::parseExpectations(istream& _stream) +{ + TestFileParser parser{_stream}; + for (auto const& call: parser.parseFunctionCalls()) + m_tests.emplace_back(FunctionCallTest{call, bytes{}, string{}}); +} + +bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments) +{ + auto output = compileAndRunWithoutCheck(m_source, _value, _contractName, _arguments); + return !output.empty() && m_transactionSuccessful; +} diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h new file mode 100644 index 000000000000..a8dd4ee66d79 --- /dev/null +++ b/test/libsolidity/SemanticTest.h @@ -0,0 +1,99 @@ +/* + This file is part of solidity. + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace dev +{ +namespace solidity +{ +namespace test +{ + +/** + * Class that represents a semantic test (or end-to-end test) and allows running it as part of the + * boost unit test environment or isoltest. It reads the Solidity source and an additional comment + * section from the given file. This comment section should define a set of functions to be called + * and an expected result they return after being executed. + */ +class SemanticTest: public SolidityExecutionFramework, public TestCase +{ +public: + /** + * Represents a function call and the result it returned. It stores the call + * representation itself, the actual byte result (if any) and a string representation + * used for the interactive update routine provided by isoltest. It also provides + * functionality to compare the actual result with the expectations attached to the + * call object, as well as a way to reset the result if executed multiple times. + */ + struct FunctionCallTest + { + FunctionCall call; + bytes rawBytes; + std::string output; + bool failure = true; + /// Compares raw expectations (which are converted to a byte representation before), + /// and also the expected transaction status of the function call to the actual test results. + bool matchesExpectation() const + { + return failure == call.expectations.failure && rawBytes == call.expectations.rawBytes(); + } + /// Resets current results in case the function was called and the result + /// stored already (e.g. if test case was updated via isoltest). + void reset() + { + failure = true; + rawBytes = bytes{}; + output = std::string{}; + } + }; + + static std::unique_ptr create(Config const& _options) + { return std::make_unique(_options.filename, _options.ipcPath); } + + explicit SemanticTest(std::string const& _filename, std::string const& _ipcPath); + + bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; + void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool const _formatted = false) const override; + void printUpdatedExpectations(std::ostream& _stream, std::string const& _linePrefix = "") const override; + + /// Instantiates a test file parser that parses the additional comment section at the end of + /// the input stream \param _stream. Each function call is represented using a `FunctionCallTest` + /// and added to the list of call to be executed when `run()` is called. + /// Throws if parsing expectations failed. + void parseExpectations(std::istream& _stream); + + /// Compiles and deploys currently held source. + /// Returns true if deployment was successful, false otherwise. + bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments); + + std::string m_source; + std::vector m_tests; +}; + +} +} +} diff --git a/test/libsolidity/SolidityExecutionFramework.cpp b/test/libsolidity/SolidityExecutionFramework.cpp index bb9695d1ecf2..934c563f44ff 100644 --- a/test/libsolidity/SolidityExecutionFramework.cpp +++ b/test/libsolidity/SolidityExecutionFramework.cpp @@ -28,7 +28,12 @@ using namespace dev::test; using namespace dev::solidity; using namespace dev::solidity::test; -SolidityExecutionFramework::SolidityExecutionFramework() : +SolidityExecutionFramework::SolidityExecutionFramework(): ExecutionFramework() { } + +SolidityExecutionFramework::SolidityExecutionFramework(std::string const& _ipcPath): + ExecutionFramework(_ipcPath) +{ +} diff --git a/test/libsolidity/SolidityExecutionFramework.h b/test/libsolidity/SolidityExecutionFramework.h index 73377eb98cd1..bda1c2486c80 100644 --- a/test/libsolidity/SolidityExecutionFramework.h +++ b/test/libsolidity/SolidityExecutionFramework.h @@ -43,6 +43,7 @@ class SolidityExecutionFramework: public dev::test::ExecutionFramework public: SolidityExecutionFramework(); + SolidityExecutionFramework(std::string const& _ipcPath); virtual bytes const& compileAndRunWithoutCheck( std::string const& _sourceCode, diff --git a/test/libsolidity/SyntaxTest.h b/test/libsolidity/SyntaxTest.h index 69394f4af861..c331636aba11 100644 --- a/test/libsolidity/SyntaxTest.h +++ b/test/libsolidity/SyntaxTest.h @@ -53,8 +53,8 @@ struct SyntaxTestError class SyntaxTest: AnalysisFramework, public TestCase { public: - static std::unique_ptr create(std::string const& _filename) - { return std::unique_ptr(new SyntaxTest(_filename)); } + static std::unique_ptr create(Config const& _config) + { return std::unique_ptr(new SyntaxTest(_config.filename)); } SyntaxTest(std::string const& _filename); bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override; diff --git a/test/libsolidity/semanticTests/smoke_test.sol b/test/libsolidity/semanticTests/smoke_test.sol new file mode 100644 index 000000000000..d4b5d33f8150 --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test.sol @@ -0,0 +1,17 @@ +contract C { + function f() public returns (uint) { + return 1; + } + function g(uint x, uint y) public returns (uint) { + return x - y; + } + function h() public payable returns (uint) { + return f(); + } +} +// ---- +// f() -> 1 +// g(uint256,uint256): 1, -2 -> 3 +// h(), 1 ether -> 1 +// j() -> FAILURE +// i() # Does not exist. # -> FAILURE # Reverts. # diff --git a/test/libsolidity/semanticTests/smoke_test_multiline.sol b/test/libsolidity/semanticTests/smoke_test_multiline.sol new file mode 100644 index 000000000000..7395b1c3b1a0 --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test_multiline.sol @@ -0,0 +1,11 @@ +contract C { + function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) { + return a + b + c + d + e; + } +} +// ---- +// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1 +// -> 5 +// g() +// # g() does not exist # +// -> FAILURE diff --git a/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol b/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol new file mode 100644 index 000000000000..17de40fc4dab --- /dev/null +++ b/test/libsolidity/semanticTests/smoke_test_multiline_comments.sol @@ -0,0 +1,17 @@ +contract C { + function f(uint a, uint b, uint c, uint d, uint e) public returns (uint) { + return a + b + c + d + e; + } +} +// ---- +// f(uint256,uint256,uint256,uint256,uint256): 1, 1, 1, 1, 1 +// # A comment on the function parameters. # +// -> 5 +// f(uint256,uint256,uint256,uint256,uint256): +// 1, +// 1, +// 1, +// 1, +// 1 +// -> 5 +// # Should return sum of all parameters. # diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp index 5f1a420e9a59..9f05e72ffbd9 100644 --- a/test/libsolidity/util/TestFileParser.cpp +++ b/test/libsolidity/util/TestFileParser.cpp @@ -85,6 +85,8 @@ vector TestFileParser::parseFunctionCalls() expect(Token::Arrow); call.expectations = parseFunctionCallExpectations(); + + accept(Token::Newline, true); call.expectations.comment = parseComment(); calls.emplace_back(std::move(call)); @@ -194,38 +196,45 @@ Parameter TestFileParser::parseParameter() if (accept(Token::Newline, true)) parameter.format.newline = true; auto literal = parseABITypeLiteral(); - parameter.rawBytes = literal.first; - parameter.abiType = literal.second; + parameter.rawBytes = get<0>(literal); + parameter.abiType = get<1>(literal); + parameter.rawString = get<2>(literal); return parameter; } -pair TestFileParser::parseABITypeLiteral() +tuple TestFileParser::parseABITypeLiteral() { try { u256 number{0}; ABIType abiType{ABIType::None, 0}; + string rawString; if (accept(Token::Sub)) { abiType = ABIType{ABIType::SignedDec, 32}; expect(Token::Sub); - number = convertNumber(parseNumber()) * -1; + rawString += formatToken(Token::Sub); + string parsed = parseNumber(); + rawString += parsed; + number = convertNumber(parsed) * -1; } else { if (accept(Token::Number)) { abiType = ABIType{ABIType::UnsignedDec, 32}; - number = convertNumber(parseNumber()); + string parsed = parseNumber(); + rawString += parsed; + number = convertNumber(parsed); } else if (accept(Token::Failure, true)) { abiType = ABIType{ABIType::Failure, 0}; - return make_pair(bytes{}, abiType); + return make_tuple(bytes{}, abiType, rawString); } } - return make_pair(toBigEndian(number), abiType); + return make_tuple(toBigEndian(number), abiType, rawString); } catch (std::exception const&) { diff --git a/test/libsolidity/util/TestFileParser.h b/test/libsolidity/util/TestFileParser.h index 6b79008833fd..3dbe90a3248b 100644 --- a/test/libsolidity/util/TestFileParser.h +++ b/test/libsolidity/util/TestFileParser.h @@ -114,7 +114,7 @@ struct ABIType */ struct FormatInfo { - bool newline; + bool newline = false; }; /** @@ -132,6 +132,9 @@ struct Parameter /// compared to the actual result of a function call /// and used for validating it. bytes rawBytes; + /// Stores the raw string representation of this parameter. + /// Used to print the unformatted arguments of a function call. + std::string rawString; /// Types that were used to encode `rawBytes`. Expectations /// are usually comma separated literals. Their type is auto- /// detected and retained in order to format them later on. @@ -327,13 +330,15 @@ class TestFileParser Parameter parseParameter(); /// Parses and converts the current literal to its byte representation and - /// preserves the chosen ABI type. Based on that type information, the driver of - /// this parser can format arguments, expectations and results. Supported types: + /// preserves the chosen ABI type, as well as a raw, unformatted string representation + /// of this literal. + /// Based on the type information retrieved, the driver of this parser may format arguments, + /// expectations and results. Supported types: /// - unsigned and signed decimal number literals. /// Returns invalid ABI type for empty literal. This is needed in order /// to detect empty expectations. Throws a ParserError if data is encoded incorrectly or /// if data type is not supported. - std::pair parseABITypeLiteral(); + std::tuple parseABITypeLiteral(); /// Recursively parses an identifier or a tuple definition that contains identifiers /// and / or parentheses like `((uint, uint), (uint, (uint, uint)), uint)`. diff --git a/test/libsolidity/util/TestFileParserTests.cpp b/test/libsolidity/util/TestFileParserTests.cpp index 5b89341da4e8..1898da5fefc4 100644 --- a/test/libsolidity/util/TestFileParserTests.cpp +++ b/test/libsolidity/util/TestFileParserTests.cpp @@ -56,7 +56,8 @@ void testFunctionCall( bytes _expectations = bytes{}, u256 _value = 0, string _argumentComment = "", - string _expectationComment = "" + string _expectationComment = "", + vector _rawArguments = vector{} ) { BOOST_REQUIRE_EQUAL(_call.expectations.failure, _failure); @@ -67,6 +68,17 @@ void testFunctionCall( BOOST_REQUIRE_EQUAL(_call.value, _value); BOOST_REQUIRE_EQUAL(_call.arguments.comment, _argumentComment); BOOST_REQUIRE_EQUAL(_call.expectations.comment, _expectationComment); + + if (!_rawArguments.empty()) + { + BOOST_REQUIRE_EQUAL(_call.arguments.parameters.size(), _rawArguments.size()); + size_t index = 0; + for (Parameter const& param: _call.arguments.parameters) + { + BOOST_REQUIRE_EQUAL(param.rawString, _rawArguments[index]); + ++index; + } + } } BOOST_AUTO_TEST_SUITE(TestFileParserTest) @@ -112,11 +124,16 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) { char const* source = R"( // f(uint256, uint256): 1, 1 + // # Comment on the parameters. # // -> // # This call should not return a value, but still succeed. # + // f() + // # Comment on no parameters. # + // -> 1 + // # This comment should be parsed. # )"; auto const calls = parse(source); - BOOST_REQUIRE_EQUAL(calls.size(), 1); + BOOST_REQUIRE_EQUAL(calls.size(), 2); testFunctionCall( calls.at(0), Mode::MultiLine, @@ -125,9 +142,20 @@ BOOST_AUTO_TEST_CASE(call_arguments_comments_success) fmt::encodeArgs(1, 1), fmt::encodeArgs(), 0, - "", + " Comment on the parameters. ", " This call should not return a value, but still succeed. " ); + testFunctionCall( + calls.at(1), + Mode::MultiLine, + "f()", + false, + fmt::encodeArgs(), + fmt::encodeArgs(1), + 0, + " Comment on no parameters. ", + " This comment should be parsed. " + ); } BOOST_AUTO_TEST_CASE(simple_single_line_call_comment_success) @@ -383,7 +411,7 @@ BOOST_AUTO_TEST_CASE(call_multiple_arguments_mixed_format) ); } -BOOST_AUTO_TEST_CASE(call_signature) +BOOST_AUTO_TEST_CASE(call_signature_valid) { char const* source = R"( // f(uint256, uint8, string) -> FAILURE @@ -395,6 +423,27 @@ BOOST_AUTO_TEST_CASE(call_signature) testFunctionCall(calls.at(1), Mode::SingleLine, "f(invalid,xyz,foo)", true); } +BOOST_AUTO_TEST_CASE(call_raw_arguments) +{ + char const* source = R"( + // f(): 1, -2, -3 -> + )"; + auto const calls = parse(source); + BOOST_REQUIRE_EQUAL(calls.size(), 1); + testFunctionCall( + calls.at(0), + Mode::SingleLine, + "f()", + false, + fmt::encodeArgs(1, -2, -3), + fmt::encodeArgs(), + 0, + "", + "", + {"1", "-2", "-3"} + ); +} + BOOST_AUTO_TEST_CASE(call_newline_invalid) { char const* source = R"( diff --git a/test/libyul/ObjectCompilerTest.h b/test/libyul/ObjectCompilerTest.h index a5f8d777bda2..d81372eec874 100644 --- a/test/libyul/ObjectCompilerTest.h +++ b/test/libyul/ObjectCompilerTest.h @@ -40,9 +40,9 @@ namespace test class ObjectCompilerTest: public dev::solidity::test::TestCase { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new ObjectCompilerTest(_filename)); + return std::unique_ptr(new ObjectCompilerTest(_config.filename)); } explicit ObjectCompilerTest(std::string const& _filename); diff --git a/test/libyul/YulOptimizerTest.h b/test/libyul/YulOptimizerTest.h index 5009b82c7b45..cb21090050e4 100644 --- a/test/libyul/YulOptimizerTest.h +++ b/test/libyul/YulOptimizerTest.h @@ -41,9 +41,9 @@ namespace test class YulOptimizerTest: public dev::solidity::test::TestCase { public: - static std::unique_ptr create(std::string const& _filename) + static std::unique_ptr create(Config const& _config) { - return std::unique_ptr(new YulOptimizerTest(_filename)); + return std::unique_ptr(new YulOptimizerTest(_config.filename)); } explicit YulOptimizerTest(std::string const& _filename); diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt index 7e070ebb551b..8200806d7589 100644 --- a/test/tools/CMakeLists.txt +++ b/test/tools/CMakeLists.txt @@ -13,7 +13,9 @@ add_executable(isoltest ../Options.cpp ../Common.cpp ../TestCase.cpp + ../libsolidity/util/TestFileParser.cpp ../libsolidity/SyntaxTest.cpp + ../libsolidity/SemanticTest.cpp ../libsolidity/AnalysisFramework.cpp ../libsolidity/SolidityExecutionFramework.cpp ../ExecutionFramework.cpp diff --git a/test/tools/isoltest.cpp b/test/tools/isoltest.cpp index c15183f8f613..1d3b9eda90a7 100644 --- a/test/tools/isoltest.cpp +++ b/test/tools/isoltest.cpp @@ -64,8 +64,9 @@ class TestTool TestCase::TestCaseCreator _testCaseCreator, string const& _name, fs::path const& _path, + string const& _ipcPath, bool _formatted - ): m_testCaseCreator(_testCaseCreator), m_formatted(_formatted), m_name(_name), m_path(_path) + ): m_testCaseCreator(_testCaseCreator), m_name(_name), m_path(_path), m_ipcPath(_ipcPath), m_formatted(_formatted) {} enum class Result @@ -81,6 +82,7 @@ class TestTool TestCase::TestCaseCreator _testCaseCreator, fs::path const& _basepath, fs::path const& _path, + string const& _ipcPath, bool const _formatted ); @@ -96,9 +98,10 @@ class TestTool Request handleResponse(bool const _exception); TestCase::TestCaseCreator m_testCaseCreator; - bool const m_formatted = false; string const m_name; fs::path const m_path; + string m_ipcPath; + bool const m_formatted = false; unique_ptr m_test; static bool m_exitRequested; }; @@ -115,25 +118,25 @@ TestTool::Result TestTool::process() try { - m_test = m_testCaseCreator(m_path.string()); + m_test = m_testCaseCreator(TestCase::Config{m_path.string(), m_ipcPath}); success = m_test->run(outputMessages, " ", m_formatted); } catch(boost::exception const& _e) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Exception during syntax test: " << boost::diagnostic_information(_e) << endl; + "Exception during test: " << boost::diagnostic_information(_e) << endl; return Result::Exception; } catch (std::exception const& _e) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Exception during syntax test: " << _e.what() << endl; + "Exception during test: " << _e.what() << endl; return Result::Exception; } catch (...) { AnsiColorized(cout, m_formatted, {BOLD, RED}) << - "Unknown exception during syntax test." << endl; + "Unknown exception during test." << endl; return Result::Exception; } @@ -199,6 +202,7 @@ TestStats TestTool::processPath( TestCase::TestCaseCreator _testCaseCreator, fs::path const& _basepath, fs::path const& _path, + string const& _ipcPath, bool const _formatted ) { @@ -230,7 +234,7 @@ TestStats TestTool::processPath( else { ++testCount; - TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _formatted); + TestTool testTool(_testCaseCreator, currentPath.string(), fullpath, _ipcPath, _formatted); auto result = testTool.process(); switch(result) @@ -291,6 +295,7 @@ boost::optional runTestSuite( string const& _name, fs::path const& _basePath, fs::path const& _subdirectory, + string const& _ipcPath, TestCase::TestCaseCreator _testCaseCreator, bool _formatted ) @@ -303,7 +308,7 @@ boost::optional runTestSuite( return {}; } - TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _formatted); + TestStats stats = TestTool::processPath(_testCaseCreator, _basePath, _subdirectory, _ipcPath, _formatted); cout << endl << _name << " Test Summary: "; AnsiColorized(cout, _formatted, {BOLD, stats ? GREEN : RED}) << @@ -327,11 +332,13 @@ int main(int argc, char *argv[]) TestTool::editor = "/usr/bin/editor"; fs::path testPath; + string ipcPath; + bool disableIPC = false; bool disableSMT = false; bool formatted = true; po::options_description options( R"(isoltest, tool for interactively managing test contracts. -Usage: isoltest [Options] --testpath path +Usage: isoltest [Options] --ipcpath ipcpath Interactively validates test contracts. Allowed options)", @@ -340,6 +347,8 @@ Allowed options)", options.add_options() ("help", "Show this help screen.") ("testpath", po::value(&testPath), "path to test files") + ("ipcpath", po::value(&ipcPath), "path to ipc socket") + ("no-ipc", "disable semantic tests") ("no-smt", "disable SMT checker") ("no-color", "don't use colors") ("editor", po::value(&TestTool::editor), "editor for opening contracts"); @@ -362,8 +371,23 @@ Allowed options)", po::notify(arguments); + if (arguments.count("no-ipc")) + disableIPC = true; + else + { + solAssert( + !ipcPath.empty(), + "No ipc path specified. The --ipcpath argument is required, unless --no-ipc is used." + ); + solAssert( + fs::exists(ipcPath), + "Invalid ipc path specified." + ); + } + if (arguments.count("no-smt")) disableSMT = true; + } catch (std::exception const& _exception) { @@ -380,10 +404,13 @@ Allowed options)", // Interactive tests are added in InteractiveTests.h for (auto const& ts: g_interactiveTestsuites) { + if (ts.ipc && disableIPC) + continue; + if (ts.smt && disableSMT) continue; - if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ts.testCaseCreator, formatted)) + if (auto stats = runTestSuite(ts.title, testPath / ts.path, ts.subpath, ipcPath, ts.testCaseCreator, formatted)) global_stats += *stats; else return 1;