[isoltest] Add support for builtin functions and test hooks.

Author: aarlt

liblangutil/Common.h

@@ -45,6 +45,11 @@ inline bool isIdentifierPart(char c)
 	return isIdentifierStart(c) || isDecimalDigit(c);
 }
 
+inline bool isIdentifierPartWithDot(char c)
+{
+	return isIdentifierPart(c) || c == '.';
+}
+
 inline int hexValue(char c)
 {
 	if (c >= '0' && c <= '9')

test/CMakeLists.txt

@@ -58,6 +58,8 @@ set(liblangutil_sources
 detect_stray_source_files("${liblangutil_sources}" "liblangutil/")
 
 set(libsolidity_sources
+    libsolidity/TestHook.h
+    libsolidity/hooks/SmokeHook.h
     libsolidity/ABIDecoderTests.cpp
     libsolidity/ABIEncoderTests.cpp
     libsolidity/ABIJsonTest.cpp

test/libsolidity/SemanticTest.cpp

@@ -13,9 +13,14 @@
 */
 
 #include <test/libsolidity/SemanticTest.h>
+
 #include <libsolutil/Whiskers.h>
 #include <libyul/Exceptions.h>
 #include <test/Common.h>
+#include <test/libsolidity/util/BytesUtils.h>
+#include <test/libsolidity/hooks/SmokeHook.h>
+#include <test/libsolidity/TestHook.h>
+
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/trim.hpp>
@@ -25,6 +30,7 @@
 #include <cctype>
 #include <fstream>
 #include <memory>
+#include <optional>
 #include <stdexcept>
 
 using namespace std;
@@ -47,6 +53,17 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 	m_lineOffset(m_reader.lineNumber()),
 	m_enforceViaYul(enforceViaYul)
 {
+	using namespace placeholders;
+	auto simpleSmokeBuiltin = bind(&SemanticTest::builtinSmokeTest, this, _1);
+	m_builtins = {
+		{"smoke.test0", simpleSmokeBuiltin},
+		{"smoke.test1", simpleSmokeBuiltin},
+		{"smoke.test2", simpleSmokeBuiltin},
+	};
+	m_testHooks = {
+		make_shared<SmokeHook>()
+	};
+
 	string choice = m_reader.stringSetting("compileViaYul", "default");
 	if (choice == "also")
 	{
@@ -119,7 +136,18 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref
 	return result;
 }
 
-TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm)
+void SemanticTest::addBuiltin(string _name, Builtin _builtin)
+{
+	m_builtins[_name] = _builtin;
+}
+
+TestCase::TestResult SemanticTest::runTest(
+	ostream& _stream,
+	string const& _linePrefix,
+	bool _formatted,
+	bool _compileViaYul,
+	bool _compileToEwasm
+)
 {
 	bool success = true;
 
@@ -142,21 +170,40 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 	if (_compileViaYul)
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Running via Yul:" << endl;
 
-	for (auto& test: m_tests)
+	for (TestFunctionCall& test: m_tests)
 		test.reset();
 
 	map<string, solidity::test::Address> libraries;
 
 	bool constructed = false;
 
-	for (auto& test: m_tests)
+	// Iterate through the test calls and set the previous call.
+	TestFunctionCall* previousCall{nullptr};
+	for (TestFunctionCall& test: m_tests)
+	{
+		test.setPreviousCall(previousCall);
+		test.setTestHooks(&m_testHooks);
+		previousCall = &test;
+	}
+
+	for (std::shared_ptr<TestHook>& hook: m_testHooks)
+		hook->beginTestCase();
+
+	for (TestFunctionCall& test: m_tests)
 	{
+		for (std::shared_ptr<TestHook>& hook: m_testHooks)
+			hook->beforeFunctionCall(test);
+
 		if (constructed)
 		{
-			soltestAssert(test.call().kind != FunctionCall::Kind::Library, "Libraries have to be deployed before any other call.");
+			soltestAssert(
+				test.call().kind != FunctionCall::Kind::Library,
+				"Libraries have to be deployed before any other call."
+			);
 			soltestAssert(
 				test.call().kind != FunctionCall::Kind::Constructor,
-				"Constructor has to be the first function call expect for library deployments.");
+				"Constructor has to be the first function call expect for library deployments."
+			);
 		}
 		else if (test.call().kind == FunctionCall::Kind::Library)
 		{
@@ -197,6 +244,18 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 			bytes output;
 			if (test.call().kind == FunctionCall::Kind::LowLevel)
 				output = callLowLevel(test.call().arguments.rawBytes(), test.call().value.value);
+			else if (test.call().kind == FunctionCall::Kind::Builtin)
+			{
+				auto builtin = m_builtins[test.call().signature];
+				std::optional<bytes> builtinOutput{builtin(test.call())};
+				if (builtinOutput.has_value())
+				{
+					test.setFailure(false);
+					output = builtinOutput.value();
+				}
+				else
+					test.setFailure(true);
+			}
 			else
 			{
 				soltestAssert(
@@ -212,20 +271,60 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 				);
 			}
 
-			bool outputMismatch = (output != test.call().expectations.rawBytes());
-			// Pre byzantium, it was not possible to return failure data, so we disregard
-			// output mismatch for those EVM versions.
-			if (test.call().expectations.failure && !m_transactionSuccessful && !m_evmVersion.supportsReturndata())
-				outputMismatch = false;
-			if (m_transactionSuccessful != !test.call().expectations.failure || outputMismatch)
-				success = false;
+			bytes expectationOutput;
+			if (test.call().expectations.builtin)
+			{
+				auto builtin = m_builtins[test.call().expectations.builtin->signature];
+				if (std::optional<bytes> builtinResult = builtin(*test.call().expectations.builtin))
+					expectationOutput = builtinResult.value();
+				else
+					test.setFailure(true);
+			}
+			else
+				expectationOutput = test.call().expectations.rawBytes();
 
-			test.setFailure(!m_transactionSuccessful);
+			bool outputMismatch = (output != expectationOutput);
+			if (test.call().kind == FunctionCall::Kind::Builtin)
+			{
+				if (outputMismatch)
+					success = false;
+			}
+			else
+			{
+				// Pre byzantium, it was not possible to return failure data, so we disregard
+				// output mismatch for those EVM versions.
+				if (test.call().expectations.failure && !m_transactionSuccessful && !m_evmVersion.supportsReturndata())
+					outputMismatch = false;
+				if (m_transactionSuccessful != !test.call().expectations.failure || outputMismatch)
+					success = false;
+				test.setFailure(!m_transactionSuccessful);
+			}
 			test.setRawBytes(std::move(output));
 			test.setContractABI(m_compiler.contractABI(m_compiler.lastContractName()));
 		}
+
+		for (std::shared_ptr<TestHook>& hook: m_testHooks)
+			hook->afterFunctionCall(test);
+	}
+
+	// The artificialFunctionCall is an artificially created function call,
+	// where it's previous call is pointing to the last call of the test.
+	TestFunctionCall artificialFunctionCall(FunctionCall{});
+	artificialFunctionCall.setTestHooks(&m_testHooks);
+	artificialFunctionCall.setPreviousCall(previousCall);
+	for (std::shared_ptr<TestHook>& hook: m_testHooks)
+	{
+		hook->beforeFunctionCall(artificialFunctionCall);
+		hook->afterFunctionCall(artificialFunctionCall);
 	}
 
+	for (std::shared_ptr<TestHook>& hook: m_testHooks)
+		hook->endTestCase();
+
+	for (TestFunctionCall& test: m_tests)
+		for (std::shared_ptr<TestHook>& hook: m_testHooks)
+			success &= hook->verifyFunctionCall(test);
+
 	if (!m_runWithYul && _compileViaYul)
 	{
 		m_compileViaYulCanBeSet = success;
@@ -241,15 +340,15 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 	if (!success && (m_runWithYul || !_compileViaYul))
 	{
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
-		for (auto const& test: m_tests)
+		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
 			_stream << test.format(errorReporter, _linePrefix, false, _formatted) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		_stream << endl;
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
-		for (auto const& test: m_tests)
+		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
 			_stream << test.format(errorReporter, _linePrefix, true, _formatted) << endl;
@@ -320,7 +419,7 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool
 
 void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
 {
-	for (auto const& test: m_tests)
+	for (TestFunctionCall const& test: m_tests)
 		_stream << test.format("", true, false) << endl;
 }
 
@@ -340,13 +439,31 @@ void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePre
 
 void SemanticTest::parseExpectations(istream& _stream)
 {
-	TestFileParser parser{_stream};
+	TestFileParser parser{_stream, &this->m_builtins};
 	auto functionCalls = parser.parseFunctionCalls(m_lineOffset);
-	std::move(functionCalls.begin(), functionCalls.end(), back_inserter(m_tests));
+	move(functionCalls.begin(), functionCalls.end(), back_inserter(m_tests));
 }
 
-bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments, map<string, solidity::test::Address> const& _libraries)
+bool SemanticTest::deploy(
+	string const& _contractName,
+	u256 const& _value,
+	bytes const& _arguments,
+	map<string, solidity::test::Address> const& _libraries
+)
 {
 	auto output = compileAndRunWithoutCheck(m_sources.sources, _value, _contractName, _arguments, _libraries);
 	return !output.empty() && m_transactionSuccessful;
 }
+
+std::optional<bytes> SemanticTest::builtinSmokeTest(FunctionCall const& call)
+{
+	// This function is only used in test/libsolidity/semanticTests/builtins/smoke.sol.
+	std::optional<bytes> result;
+	if (call.arguments.parameters.size() < 3)
+	{
+		result = bytes();
+		for (const auto& parameter: call.arguments.parameters)
+			result.value() += util::toBigEndian(u256{util::fromHex(parameter.rawString)});
+	}
+	return result;
+}

test/libsolidity/SemanticTest.h

@@ -14,13 +14,14 @@
 
 #pragma once
 
-#include <test/libsolidity/util/TestFileParser.h>
-#include <test/libsolidity/util/TestFunctionCall.h>
-#include <test/libsolidity/SolidityExecutionFramework.h>
-#include <test/libsolidity/AnalysisFramework.h>
-#include <test/TestCase.h>
 #include <liblangutil/Exceptions.h>
 #include <libsolutil/AnsiColorized.h>
+#include <test/TestCase.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/SolidityExecutionFramework.h>
+#include <test/libsolidity/TestHook.h>
+#include <test/libsolidity/util/TestFileParser.h>
+#include <test/libsolidity/util/TestFunctionCall.h>
 
 #include <iosfwd>
 #include <string>
@@ -58,7 +59,13 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	/// Compiles and deploys currently held source.
 	/// Returns true if deployment was successful, false otherwise.
 	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments, std::map<std::string, solidity::test::Address> const& _libraries = {});
+
+	void addBuiltin(std::string _name, Builtin _builtin);
+
 private:
+	// builtin functions
+	std::optional<bytes> builtinSmokeTest(FunctionCall const& call);
+
 	TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm);
 	SourceMap m_sources;
 	std::size_t m_lineOffset;
@@ -70,6 +77,8 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	bool m_runWithABIEncoderV1Only = false;
 	bool m_allowNonExistingFunctions = false;
 	bool m_compileViaYulCanBeSet = false;
+	Builtins m_builtins;
+	TestHooks m_testHooks;
 };
 
 }

test/libsolidity/TestHook.h

@@ -0,0 +1,51 @@
+/*
+	This file is part of solidity.
+	solidity is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+	solidity is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+	You should have received a copy of the GNU General Public License
+	along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <test/libsolidity/util/SoltestErrors.h>
+
+#include <memory>
+#include <vector>
+
+namespace solidity::frontend::test
+{
+
+class TestFunctionCall;
+
+class TestHook
+{
+public:
+	TestHook() = default;
+	virtual ~TestHook() = default;
+
+	virtual void beginTestCase() = 0;
+	virtual void beforeFunctionCall(TestFunctionCall const& _call)  = 0;
+	virtual void afterFunctionCall(TestFunctionCall const& _call) = 0;
+	virtual void endTestCase() = 0;
+
+	virtual bool verifyFunctionCall(TestFunctionCall const& _call) = 0;
+
+	virtual std::string formatFunctionCall(
+		const TestFunctionCall& _call,
+		ErrorReporter& _errorReporter,
+		std::string const& _linePrefix,
+		bool const _renderResult,
+		bool const _highlight
+	) const = 0;
+};
+
+using TestHooks = std::vector<std::shared_ptr<TestHook>>;
+
+} // namespace solidity::frontend::test