[isoltest] Add support for effects.

aarlt · aarlt · commit a1c65502df2f · 2021-03-04T06:30:13.000-05:00
diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp
@@ -68,6 +68,12 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 	m_builtins["smoke_test1"] = simpleSmokeBuiltin;
 	m_builtins["smoke_test2"] = simpleSmokeBuiltin;
 
+	m_hooks.emplace_back([](FunctionCall const& _call) -> std::vector<std::string> {
+	  if (_call.signature.find("smoke_") != string::npos)
+			return {_call.signature};
+		return {};
+	});
+
 	string choice = m_reader.stringSetting("compileViaYul", "default");
 	if (choice == "also")
 	{
@@ -140,6 +146,15 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref
 	return result;
 }
 
+vector<string> SemanticTest::effectsOfCall(FunctionCall const& _call) const
+{
+	vector<string> effects;
+	for (auto const& hook: m_hooks)
+		for (auto const& effect: hook(_call))
+			effects.emplace_back(effect);
+	return effects;
+}
+
 TestCase::TestResult SemanticTest::runTest(
 	ostream& _stream,
 	string const& _linePrefix,
@@ -266,6 +281,8 @@ TestCase::TestResult SemanticTest::runTest(
 			test.setRawBytes(std::move(output));
 			test.setContractABI(m_compiler.contractABI(m_compiler.lastContractName()));
 		}
+
+		success &= test.call().effects == effectsOfCall(test.call());
 	}
 
 	if (!m_runWithYul && _compileViaYul)
@@ -286,15 +303,15 @@ TestCase::TestResult SemanticTest::runTest(
 		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, false, _formatted) << endl;
+			_stream << test.format(errorReporter, _linePrefix, false, _formatted, effectsOfCall(test.call())) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		_stream << endl;
 		AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Obtained result:" << endl;
 		for (TestFunctionCall const& test: m_tests)
 		{
 			ErrorReporter errorReporter;
-			_stream << test.format(errorReporter, _linePrefix, true, _formatted) << endl;
+			_stream << test.format(errorReporter, _linePrefix, true, _formatted, effectsOfCall(test.call())) << endl;
 			_stream << errorReporter.format(_linePrefix, _formatted);
 		}
 		AnsiColorized(_stream, _formatted, {BOLD, RED})
@@ -363,7 +380,7 @@ void SemanticTest::printSource(ostream& _stream, string const& _linePrefix, bool
 void SemanticTest::printUpdatedExpectations(ostream& _stream, string const&) const
 {
 	for (TestFunctionCall const& test: m_tests)
-		_stream << test.format("", true, false) << endl;
+		_stream << test.format("", true, false, effectsOfCall(test.call())) << endl;
 }
 
 void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePrefix)
diff --git a/test/libsolidity/SemanticTest.h b/test/libsolidity/SemanticTest.h
@@ -59,6 +59,8 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	/// Returns true if deployment was successful, false otherwise.
 	bool deploy(std::string const& _contractName, u256 const& _value, bytes const& _arguments, std::map<std::string, solidity::test::Address> const& _libraries = {});
 
+	std::vector<std::string> effectsOfCall(FunctionCall const& _call) const;
+
 private:
 	TestResult runTest(std::ostream& _stream, std::string const& _linePrefix, bool _formatted, bool _compileViaYul, bool _compileToEwasm);
 	SourceMap m_sources;
@@ -72,6 +74,7 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	bool m_allowNonExistingFunctions = false;
 	bool m_compileViaYulCanBeSet = false;
 	std::map<std::string, Builtin> m_builtins{};
+	std::vector<Hook> m_hooks{};
 };
 
 }
diff --git a/test/libsolidity/semanticTests/builtins/smoke.sol b/test/libsolidity/semanticTests/builtins/smoke.sol
@@ -4,7 +4,12 @@ contract ClientReceipt {
 // compileViaYul: also
 // ----
 // smoke_test0 ->
+// - smoke_test0
 // smoke_test1: 1 -> 1
+// - smoke_test1
 // smoke_test2: 2, 3 -> 2, 3
+// - smoke_test2
 // smoke_test2: 2, 3, 4 -> FAILURE
+// - smoke_test2
 // smoke_test0 ->
+// - smoke_test0
diff --git a/test/libsolidity/util/SoltestTypes.h b/test/libsolidity/util/SoltestTypes.h
@@ -42,6 +42,7 @@ namespace solidity::frontend::test
 	T(Comma,  ",", 0)              \
 	T(Period, ".", 0)              \
 	T(Arrow, "->", 0)              \
+	T(Effect, "- ", 0)             \
 	T(Newline, "//", 0)            \
 	/* Literals & identifier */    \
 	T(Comment, "#", 0)             \
@@ -59,7 +60,7 @@ namespace solidity::frontend::test
 	K(Library, "library", 0)       \
 	K(Right, "right", 0)           \
 	K(Failure, "FAILURE", 0)       \
-	K(Storage, "storage", 0) \
+	K(Storage, "storage", 0)       \
 
 namespace soltest
 {
@@ -299,8 +300,11 @@ struct FunctionCall
 	/// Marks this function call as "short-handed", meaning
 	/// no `->` declared.
 	bool omitsArrow = true;
+	/// A textual representation of the effect of the function call.
+	std::vector<std::string> effects{};
 };
 
 using Builtin = std::function<std::optional<bytes>(FunctionCall const&)>;
+using Hook = std::function<std::vector<std::string>(FunctionCall const&)>;
 
 }
diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp
@@ -149,6 +149,8 @@ vector<solidity::frontend::test::FunctionCall> TestFileParser::parseFunctionCall
 							call.kind = FunctionCall::Kind::Constructor;
 					}
 
+					call.effects = parseFunctionCallEffects();
+
 					calls.emplace_back(std::move(call));
 				}
 				catch (TestParserError const& _e)
@@ -284,6 +286,25 @@ FunctionCallExpectations TestFileParser::parseFunctionCallExpectations()
 	return expectations;
 }
 
+vector<string> TestFileParser::parseFunctionCallEffects()
+{
+	vector<string> result;
+	while (accept(Token::Effect, false))
+	{
+		string effect = m_scanner.currentLiteral();
+		result.emplace_back(effect);
+
+		if (m_scanner.currentToken() == Token::Effect)
+			m_scanner.scanNextToken();
+		if (m_scanner.currentToken() == Token::Newline)
+			m_scanner.scanNextToken();
+		if (m_scanner.currentToken() == Token::EOS)
+			break;
+	}
+
+	return result;
+}
+
 Parameter TestFileParser::parseParameter()
 {
 	Parameter parameter;
@@ -535,6 +556,11 @@ void TestFileParser::Scanner::scanNextToken()
 				advance();
 				selectToken(Token::Arrow);
 			}
+			else if (peek() == ' ')
+			{
+				advance();
+				selectToken(Token::Effect, scanEffectString());
+			}
 			else
 				selectToken(Token::Sub);
 			break;
@@ -590,8 +616,18 @@ void TestFileParser::Scanner::scanNextToken()
 				BOOST_THROW_EXCEPTION(TestParserError("Unexpected character: '" + string{current()} + "'"));
 			break;
 		}
+	} while (m_currentToken == Token::Whitespace);
+}
+
+string TestFileParser::Scanner::scanEffectString()
+{
+	string effect;
+	while (peek() != '/' && peek() != '\0')
+	{
+		advance();
+		effect += current();
 	}
-	while (m_currentToken == Token::Whitespace);
+	return effect;
 }
 
 string TestFileParser::Scanner::scanComment()
diff --git a/test/libsolidity/util/TestFileParser.h b/test/libsolidity/util/TestFileParser.h
@@ -83,6 +83,7 @@ class TestFileParser
 		std::string currentLiteral() { return m_currentLiteral; }
 
 		std::string scanComment();
+		std::string scanEffectString();
 		std::string scanIdentifierOrKeyword();
 		std::string scanDecimalNumber();
 		std::string scanHexNumber();
@@ -141,6 +142,9 @@ class TestFileParser
 	/// Parses the expected result of a function call execution.
 	FunctionCallExpectations parseFunctionCallExpectations();
 
+	/// Parses the expected effects of a function call execution.
+	std::vector<std::string> parseFunctionCallEffects();
+
 	/// Parses the next parameter in a comma separated list.
 	/// Takes a newly parsed, and type-annotated `bytes` argument,
 	/// appends it to the internal `bytes` buffer of the parameter. It can also
diff --git a/test/libsolidity/util/TestFunctionCall.cpp b/test/libsolidity/util/TestFunctionCall.cpp
@@ -36,8 +36,8 @@ string TestFunctionCall::format(
 	ErrorReporter& _errorReporter,
 	string const& _linePrefix,
 	bool const _renderResult,
-	bool const _highlight
-) const
+	bool const _highlight,
+	std::vector<std::string> _effects) const
 {
 	stringstream stream;
 
@@ -204,6 +204,17 @@ string TestFunctionCall::format(
 				stream << comment << m_call.expectations.comment << comment;
 			}
 		}
+
+		if (!_effects.empty())
+		{
+			stream << std::endl;
+			for (string const& effect: _effects)
+			{
+				stream << _linePrefix << "// - " << effect;
+				if (effect != *_effects.rbegin())
+					stream << std::endl;
+			}
+		}
 	};
 
 	formatOutput(m_call.displayMode == FunctionCall::DisplayMode::SingleLine);
diff --git a/test/libsolidity/util/TestFunctionCall.h b/test/libsolidity/util/TestFunctionCall.h
@@ -56,19 +56,21 @@ class TestFunctionCall
 		ErrorReporter& _errorReporter,
 		std::string const& _linePrefix = "",
 		bool const _renderResult = false,
-		bool const _highlight = false
+		bool const _highlight = false,
+		std::vector<std::string> _effects = {}
 	) const;
 
 	/// Overloaded version that passes an error reporter which is never used outside
 	/// of this function.
 	std::string format(
 		std::string const& _linePrefix = "",
 		bool const _renderResult = false,
-		bool const _highlight = false
+		bool const _highlight = false,
+		std::vector<std::string> _effects = {}
 	) const
 	{
 		ErrorReporter reporter;
-		return format(reporter, _linePrefix, _renderResult, _highlight);
+		return format(reporter, _linePrefix, _renderResult, _highlight, _effects);
 	}
 
 	/// Resets current results in case the function was called and the result
