[test] Add support for --enforce-compile-to-ewasm.

Author: aarlt

test/Common.cpp

@@ -104,6 +104,7 @@ CommonOptions::CommonOptions(std::string _caption):
 		("optimize", po::bool_switch(&optimize), "enables optimization")
 		("enforce-via-yul", po::bool_switch(&enforceViaYul), "Enforce compiling all tests via yul to see if additional tests can be activated.")
 		("abiencoderv1", po::bool_switch(&useABIEncoderV1), "enables abi encoder v1")
+		("enforce-compile-to-ewasm", po::bool_switch(&enforceCompileToEwasm), "Enforce compiling all tests to ewasm to see if additional tests can be activated.")
 		("show-messages", po::bool_switch(&showMessages), "enables message output")
 		("show-metadata", po::bool_switch(&showMetadata), "enables metadata output");
 }

test/Common.h

@@ -56,6 +56,7 @@ struct CommonOptions: boost::noncopyable
 	bool ewasm = false;
 	bool optimize = false;
 	bool enforceViaYul = false;
+	bool enforceCompileToEwasm = false;
 	bool disableSMT = false;
 	bool useABIEncoderV1 = false;
 	bool showMessages = false;

test/TestCase.h

@@ -41,6 +41,7 @@ class TestCase
 		langutil::EVMVersion evmVersion;
 		std::vector<boost::filesystem::path> vmPaths;
 		bool enforceCompileViaYul;
+		bool enforceCompileToEwasm;
 	};
 
 	enum class TestResult { Success, Failure, FatalError };

test/boostTest.cpp

@@ -65,13 +65,14 @@ int registerTests(
 	boost::filesystem::path const& _basepath,
 	boost::filesystem::path const& _path,
 	bool _enforceViaYul,
+	bool _enforceCompileToEwasm,
 	vector<string> const& _labels,
 	TestCase::TestCaseCreator _testCaseCreator
 )
 {
 	int numTestsAdded = 0;
 	fs::path fullpath = _basepath / _path;
-	TestCase::Config config{fullpath.string(), solidity::test::CommonOptions::get().evmVersion(), solidity::test::CommonOptions::get().vmPaths, _enforceViaYul};
+	TestCase::Config config{fullpath.string(), solidity::test::CommonOptions::get().evmVersion(), solidity::test::CommonOptions::get().vmPaths, _enforceViaYul, _enforceCompileToEwasm};
 	if (fs::is_directory(fullpath))
 	{
 		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
@@ -84,6 +85,7 @@ int registerTests(
 					*sub_suite,
 					_basepath, _path / entry.path().filename(),
 					_enforceViaYul,
+					_enforceCompileToEwasm,
 					_labels,
 					_testCaseCreator
 				);
@@ -188,6 +190,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 			options.testPath / ts.path,
 			ts.subpath,
 			options.enforceViaYul,
+			options.enforceCompileToEwasm,
 			ts.labels,
 			ts.testCaseCreator
 		) > 0, std::string("no ") + ts.title + " tests found");

test/libsolidity/SemanticTest.cpp

@@ -39,12 +39,13 @@ using namespace boost::unit_test;
 namespace fs = boost::filesystem;
 
 
-SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVersion, vector<boost::filesystem::path> const& _vmPaths, bool enforceViaYul):
+SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVersion, vector<boost::filesystem::path> const& _vmPaths, bool enforceViaYul, bool enforceCompileToEwasm):
 	SolidityExecutionFramework(_evmVersion, _vmPaths),
 	EVMVersionRestrictedTestCase(_filename),
 	m_sources(m_reader.sources()),
 	m_lineOffset(m_reader.lineNumber()),
-	m_enforceViaYul(enforceViaYul)
+	m_enforceViaYul(enforceViaYul),
+	m_enforceCompileToEwasm(enforceCompileToEwasm)
 {
 	string choice = m_reader.stringSetting("compileViaYul", "default");
 	if (choice == "also")
@@ -63,6 +64,7 @@ SemanticTest::SemanticTest(string const& _filename, langutil::EVMVersion _evmVer
 		m_runWithoutYul = true;
 		// Do not try to run via yul if explicitly denied.
 		m_enforceViaYul = false;
+		m_enforceCompileToEwasm = false;
 	}
 	else if (choice == "default")
 	{
@@ -105,16 +107,26 @@ TestCase::TestResult SemanticTest::run(ostream& _stream, string const& _linePref
 {
 	TestResult result = TestResult::Success;
 	bool compileViaYul = m_runWithYul || m_enforceViaYul;
+	bool compileToEwasm = m_runWithEwasm || m_enforceCompileToEwasm;
 
 	if (m_runWithoutYul)
 		result = runTest(_stream, _linePrefix, _formatted, false, false);
 
 	if (compileViaYul && result == TestResult::Success)
 		result = runTest(_stream, _linePrefix, _formatted, true, false);
 
-	if (m_runWithEwasm && result == TestResult::Success)
-		result = runTest(_stream, _linePrefix, _formatted, true, true);
-
+	if (compileToEwasm && result == TestResult::Success)
+	{
+		try
+		{
+			result = runTest(_stream, _linePrefix, _formatted, true, true);
+		}
+		catch (...)
+		{
+			if (!m_enforceCompileToEwasm)
+				throw;
+		}
+	}
 	return result;
 }
 
@@ -139,6 +151,7 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 		}
 
 		m_compileViaYulCanBeSet = false;
+		m_compileToEwasmCanBeSet = false;
 
 		if (_compileViaYul)
 			AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Running via Yul:" << endl;
@@ -236,8 +249,23 @@ TestCase::TestResult SemanticTest::runTest(ostream& _stream, string const& _line
 			return TestResult::Failure;
 		}
 
+		if (success && !m_runWithEwasm && _compileToEwasm)
+		{
+			if (m_revertStrings != RevertStrings::Default)
+				return TestResult::Success;
+
+			m_compileToEwasmCanBeSet = true;
+			AnsiColorized(_stream, _formatted, {BOLD, YELLOW}) <<
+				_linePrefix << endl <<
+				_linePrefix << "Test can pass via Yul (ewasm) and marked with compileToEwasm: false." << endl;
+			return TestResult::Failure;
+		}
+
 		if (!success && (m_runWithYul || !_compileViaYul))
 		{
+			if (!m_runWithEwasm && m_enforceCompileToEwasm)
+				return TestResult::Success;
+
 			AnsiColorized(_stream, _formatted, {BOLD, CYAN}) << _linePrefix << "Expected result:" << endl;
 			for (auto const& test: m_tests)
 			{
@@ -355,11 +383,28 @@ void SemanticTest::printUpdatedSettings(ostream& _stream, string const& _linePre
 		return;
 
 	_stream << _linePrefix << "// ====" << endl;
-	if (m_compileViaYulCanBeSet)
+	if (m_compileToEwasmCanBeSet)
+	{
+		// if test was already configured to run via yul,
+		// we need to preserve the original settings.
+		if (m_runWithYul)
+		{
+			// if test was also configured to run without yul.
+			if (m_runWithoutYul)
+				_stream << _linePrefix << "// compileViaYul: also\n";
+			// if test was configured only to run with yul.
+			else
+				_stream << _linePrefix << "// compileViaYul: true\n";
+		}
+
+		_stream << _linePrefix << "// compileToEwasm: also\n";
+	}
+	else if (m_compileViaYulCanBeSet)
 		_stream << _linePrefix << "// compileViaYul: also\n";
+
 	for (auto const& setting: settings)
-		if (!m_compileViaYulCanBeSet || setting.first != "compileViaYul")
-		_stream << _linePrefix << "// " << setting.first << ": " << setting.second << endl;
+		if (!(m_compileViaYulCanBeSet || m_compileToEwasmCanBeSet) || setting.first != "compileViaYul")
+			_stream << _linePrefix << "// " << setting.first << ": " << setting.second << endl;
 }
 
 void SemanticTest::parseExpectations(istream& _stream)

test/libsolidity/SemanticTest.h

@@ -40,9 +40,9 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 {
 public:
 	static std::unique_ptr<TestCase> create(Config const& _options)
-	{ return std::make_unique<SemanticTest>(_options.filename, _options.evmVersion, _options.vmPaths, _options.enforceCompileViaYul); }
+	{ return std::make_unique<SemanticTest>(_options.filename, _options.evmVersion, _options.vmPaths, _options.enforceCompileViaYul, _options.enforceCompileToEwasm); }
 
-	explicit SemanticTest(std::string const& _filename, langutil::EVMVersion _evmVersion, std::vector<boost::filesystem::path> const& _vmPaths, bool _enforceViaYul = false);
+	explicit SemanticTest(std::string const& _filename, langutil::EVMVersion _evmVersion, std::vector<boost::filesystem::path> const& _vmPaths, bool _enforceViaYul = false, bool _enforceCompileToEwasm = false);
 
 	TestResult run(std::ostream& _stream, std::string const& _linePrefix = "", bool _formatted = false) override;
 	void printSource(std::ostream &_stream, std::string const& _linePrefix = "", bool _formatted = false) const override;
@@ -67,9 +67,11 @@ class SemanticTest: public SolidityExecutionFramework, public EVMVersionRestrict
 	bool m_runWithEwasm = false;
 	bool m_runWithoutYul = true;
 	bool m_enforceViaYul = false;
+	bool m_enforceCompileToEwasm = false;
 	bool m_runWithABIEncoderV1Only = false;
 	bool m_allowNonExistingFunctions = false;
 	bool m_compileViaYulCanBeSet = false;
+	bool m_compileToEwasmCanBeSet = false;
 };
 
 }

test/tools/isoltest.cpp

@@ -162,7 +162,8 @@ TestTool::Result TestTool::process()
 				m_path.string(),
 				m_options.evmVersion(),
 				m_options.vmPaths,
-				m_options.enforceViaYul
+				m_options.enforceViaYul,
+				m_options.enforceCompileToEwasm,
 			});
 			if (m_test->shouldRun())
 				switch (TestCase::TestResult result = m_test->run(outputMessages, "  ", formatted))