[isoltest] Add support for builtin functions and test hooks.

Author: aarlt

liblangutil/Common.h

@@ -45,6 +45,11 @@ inline bool isIdentifierPart(char c)
 	return isIdentifierStart(c) || isDecimalDigit(c);
 }
 
+inline bool isIdentifierPartWithDot(char c)
+{
+	return isIdentifierPart(c) || c == '.';
+}
+
 inline int hexValue(char c)
 {
 	if (c >= '0' && c <= '9')

test/CMakeLists.txt

@@ -58,6 +58,8 @@ set(liblangutil_sources
 detect_stray_source_files("${liblangutil_sources}" "liblangutil/")
 
 set(libsolidity_sources
+    libsolidity/TestHook.h
+    libsolidity/hooks/SmokeHook.h
     libsolidity/ABIDecoderTests.cpp
     libsolidity/ABIEncoderTests.cpp
     libsolidity/ABIJsonTest.cpp

test/libsolidity/SemanticTest.cpp

@@ -13,9 +13,14 @@
 */
 
 #include <test/libsolidity/SemanticTest.h>
+
 #include <libsolutil/Whiskers.h>
 #include <libyul/Exceptions.h>
 #include <test/Common.h>
+#include <test/libsolidity/util/BytesUtils.h>
+#include <test/libsolidity/hooks/SmokeHook.h>
+#include <test/libsolidity/TestHook.h>
+
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/trim.hpp>
@@ -25,7 +30,9 @@
 #include <cctype>
 #include <fstream>
 #include <memory>
+#include <optional>
 #include <stdexcept>
+#include <utility>
 
 using namespace std;
 using namespace solidity;
@@ -47,6 +54,16 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 	m_lineOffset(m_reader.lineNumber()),
 	m_enforceViaYul(enforceViaYul)
 {
+	using namespace placeholders;
+
+	auto simpleSmokeBuiltin = bind(&SemanticTest::builtinSmokeTest, this, _1);
+	addBuiltin("smoke.test0", simpleSmokeBuiltin);
+	addBuiltin("smoke.test1", simpleSmokeBuiltin);
+	addBuiltin("smoke.test2", simpleSmokeBuiltin);
+	addBuiltin("smoke_test3", simpleSmokeBuiltin);
+
+	addTestHook(make_unique<SmokeHook>());
+
 	string choice = m_reader.stringSetting("compileViaYul", "default");
 	if (choice == "also")
 	{
@@ -119,7 +136,23 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref
 	return result;
 }
 
-TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm)
+void SemanticTest::addBuiltin(string _name, Builtin _builtin)
+{
+	m_builtins[std::move(_name)] = std::move(_builtin);
+}
+
+void SemanticTest::addTestHook(std::unique_ptr<TestHook> _testHook)
+{
+	m_testHooks.emplace_back(std::move(_testHook));
+}
+
+TestCase::TestResult SemanticTest::runTest(
+	ostream& _stream,
+	string const& _linePrefix,
+	bool _formatted,
+	bool _compileViaYul,
+	bool _compileToEwasm
+)
 {
 	bool success = true;
 
@@ -142,21 +175,31 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 	if (_compileViaYul)
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Running via Yul:" << endl;
 
-	for (auto& test: m_tests)
+	for (TestFunctionCall& test: m_tests)
 		test.reset();
 
 	map<string, solidity::test::Address> libraries;
 
 	bool constructed = false;
 
-	for (auto& test: m_tests)
+	for (std::unique_ptr<TestHook> const& hook: m_testHooks)
+		hook->beginTestCase();
+
+	for (TestFunctionCall& test: m_tests)
 	{
+		for (std::unique_ptr<TestHook> const& hook: m_testHooks)
+			hook->beforeFunctionCall(test);
+
 		if (constructed)
 		{
-			soltestAssert(test.call().kind != FunctionCall::Kind::Library, "Libraries have to be deployed before any other call.");
+			soltestAssert(
+				test.call().kind != FunctionCall::Kind::Library,
+				"Libraries have to be deployed before any other call."
+			);
 			soltestAssert(
 				test.call().kind != FunctionCall::Kind::Constructor,
-				"Constructor has to be the first function call expect for library deployments.");
+				"Constructor has to be the first function call expect for library deployments."
+			);
 		}
 		else if (test.call().kind == FunctionCall::Kind::Library)
 		{
@@ -197,6 +240,18 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 			bytes output;
 			if (test.call().kind == FunctionCall::Kind::LowLevel)
 				output = callLowLevel(test.call().arguments.rawBytes(), test.call().value.value);
+			else if (test.call().kind == FunctionCall::Kind::Builtin)
+			{
+				auto builtin = m_builtins[test.call().signature];
+				std::optional<bytes> builtinOutput{builtin(test.call())};
+				if (builtinOutput.has_value())
+				{
+					test.setFailure(false);
+					output = builtinOutput.value();
+				}
+				else
+					test.setFailure(true);
+			}
 			else
 			{
 				soltestAssert(
@@ -212,20 +267,49 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 				);
 			}
 
-			bool outputMismatch = (output != test.call().expectations.rawBytes());
-			// Pre byzantium, it was not possible to return failure data, so we disregard
-			// output mismatch for those EVM versions.
-			if (test.call().expectations.failure && !m_transactionSuccessful && !m_evmVersion.supportsReturndata())
-				outputMismatch = false;
-			if (m_transactionSuccessful != !test.call().expectations.failure || outputMismatch)
-				success = false;
+			bytes expectationOutput;
+			if (test.call().expectations.builtin)
+			{
+				auto builtin = m_builtins[test.call().expectations.builtin->signature];
+				if (std::optional<bytes> builtinResult = builtin(*test.call().expectations.builtin))
+					expectationOutput = builtinResult.value();
+				else
+					test.setFailure(true);
+			}
+			else
+				expectationOutput = test.call().expectations.rawBytes();
 
-			test.setFailure(!m_transactionSuccessful);
+			bool outputMismatch = (output != expectationOutput);
+			if (test.call().kind == FunctionCall::Kind::Builtin)
+			{
+				if (outputMismatch)
+					success = false;
+			}
+			else
+			{
+				// Pre byzantium, it was not possible to return failure data, so we disregard
+				// output mismatch for those EVM versions.
+				if (test.call().expectations.failure && !m_transactionSuccessful && !m_evmVersion.supportsReturndata())
+					outputMismatch = false;
+				if (m_transactionSuccessful != !test.call().expectations.failure || outputMismatch)
+					success = false;
+				test.setFailure(!m_transactionSuccessful);
+			}
 			test.setRawBytes(std::move(output));
 			test.setContractABI(m_compiler.contractABI(m_compiler.lastContractName()));
 		}
+
+		for (std::unique_ptr<TestHook> const& hook: m_testHooks)
+			hook->afterFunctionCall(test);
 	}
 
+	for (std::unique_ptr<TestHook> const& hook: m_testHooks)
+		hook->endTestCase();
+
+	for (TestFunctionCall& test: m_tests)
+		for (std::unique_ptr<TestHook> const& hook: m_testHooks)
+			success &= hook->verifyFunctionCall(test);
+
 	if (!m_runWithYul && _compileViaYul)
 	{
 		m_compileViaYulCanBeSet = success;
@@ -241,18 +325,18 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 	if (!success && (m_runWithYul || !_compileViaYul))
 	{
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
-		for (auto const& test: m_tests)
+		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, false, _formatted) << endl;
+			_stream << test.format(errorReporter, _linePrefix, false, _formatted, &m_testHooks) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		_stream << endl;
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
-		for (auto const& test: m_tests)
+		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, true, _formatted) << endl;
+			_stream << test.format(errorReporter, _linePrefix, true, _formatted, &m_testHooks) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		AnsiColorized(_stream, _formatted, {BOLD, RED})
@@ -320,8 +404,8 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool
 
 void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
 {
-	for (auto const& test: m_tests)
-		_stream << test.format("", true, false) << endl;
+	for (TestFunctionCall const& test: m_tests)
+		_stream << test.format("", true, false, &m_testHooks) << endl;
 }
 
 void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePrefix)
@@ -340,13 +424,31 @@ void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePre
 
 void SemanticTest::parseExpectations(istream& _stream)
 {
-	TestFileParser parser{_stream};
+	TestFileParser parser{_stream, &this->m_builtins};
 	auto functionCalls = parser.parseFunctionCalls(m_lineOffset);
-	std::move(functionCalls.begin(), functionCalls.end(), back_inserter(m_tests));
+	move(functionCalls.begin(), functionCalls.end(), back_inserter(m_tests));
 }
 
-bool SemanticTest::deploy(string const& _contractName, u256 const& _value, bytes const& _arguments, map<string, solidity::test::Address> const& _libraries)
+bool SemanticTest::deploy(
+	string const& _contractName,
+	u256 const& _value,
+	bytes const& _arguments,
+	map<string, solidity::test::Address> const& _libraries
+)
 {
 	auto output = compileAndRunWithoutCheck(m_sources.sources, _value, _contractName, _arguments, _libraries);
 	return !output.empty() && m_transactionSuccessful;
 }
+
+std::optional<bytes> SemanticTest::builtinSmokeTest(FunctionCall const& call)
+{
+	// This function is only used in test/libsolidity/semanticTests/builtins/smoke.sol.
+	std::optional<bytes> result;
+	if (call.arguments.parameters.size() < 3)
+	{
+		result = bytes();
+		for (const auto& parameter: call.arguments.parameters)
+			result.value() += util::toBigEndian(u256{util::fromHex(parameter.rawString)});
+	}
+	return result;
+}

test/libsolidity/SemanticTest.h

@@ -14,13 +14,14 @@
 
 #pragma once
 
-#include <test/libsolidity/util/TestFileParser.h>
-#include <test/libsolidity/util/TestFunctionCall.h>
-#include <test/libsolidity/SolidityExecutionFramework.h>
-#include <test/libsolidity/AnalysisFramework.h>
-#include <test/TestCase.h>
 #include <liblangutil/Exceptions.h>
 #include <libsolutil/AnsiColorized.h>
+#include <test/TestCase.h>
+#include <test/libsolidity/AnalysisFramework.h>
+#include <test/libsolidity/SolidityExecutionFramework.h>
+#include <test/libsolidity/TestHook.h>
+#include <test/libsolidity/util/TestFileParser.h>
+#include <test/libsolidity/util/TestFunctionCall.h>
 
 #include <iosfwd>
 #include <string>
@@ -58,7 +59,14 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	/// Compiles and deploys currently held source.
 	/// Returns true if deployment was successful, false otherwise.
 	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments, std::map<std::string, solidity::test::Address> const& _libraries = {});
+
+	void addBuiltin(std::string _name, Builtin _builtin);
+	void addTestHook(std::unique_ptr<TestHook> _testHook);
+
 private:
+	// builtin functions
+	std::optional<bytes> builtinSmokeTest(FunctionCall const& call);
+
 	TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm);
 	SourceMap m_sources;
 	std::size_t m_lineOffset;
@@ -70,6 +78,8 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	bool m_runWithABIEncoderV1Only = false;
 	bool m_allowNonExistingFunctions = false;
 	bool m_compileViaYulCanBeSet = false;
+	Builtins m_builtins{};
+	std::vector<std::unique_ptr<TestHook>> m_testHooks{};
 };
 
 }