Merge pull request #5872 from ethereum/semantic-tests-split

Semantic test infrastructure

Author: chriseth

test/ExecutionFramework.cpp

@@ -49,8 +49,13 @@ string getIPCSocketPath()
 
 }
 
-ExecutionFramework::ExecutionFramework() :
-	m_rpc(RPCSession::instance(getIPCSocketPath())),
+ExecutionFramework::ExecutionFramework():
+	ExecutionFramework(getIPCSocketPath())
+{
+}
+
+ExecutionFramework::ExecutionFramework(string const& _ipcPath):
+	m_rpc(RPCSession::instance(_ipcPath)),
 	m_evmVersion(dev::test::Options::get().evmVersion()),
 	m_optimize(dev::test::Options::get().optimize),
 	m_showMessages(dev::test::Options::get().showMessages),

test/ExecutionFramework.h

@@ -53,6 +53,7 @@ class ExecutionFramework
 
 public:
 	ExecutionFramework();
+	explicit ExecutionFramework(std::string const& _ipcPath);
 	virtual ~ExecutionFramework() = default;
 
 	virtual bytes const& compileAndRunWithoutCheck(

test/InteractiveTests.h

@@ -20,6 +20,7 @@
 #include <test/TestCase.h>
 #include <test/libsolidity/ASTJSONTest.h>
 #include <test/libsolidity/SyntaxTest.h>
+#include <test/libsolidity/SemanticTest.h>
 #include <test/libsolidity/SMTCheckerJSONTest.h>
 #include <test/libyul/YulOptimizerTest.h>
 #include <test/libyul/ObjectCompilerTest.h>
@@ -52,6 +53,7 @@ Testsuite const g_interactiveTestsuites[] = {
 	{"Yul Optimizer",       "libyul",      "yulOptimizerTests",   false, false, &yul::test::YulOptimizerTest::create},
 	{"Yul Object Compiler", "libyul",      "objectCompiler",      false, false, &yul::test::ObjectCompilerTest::create},
 	{"Syntax",              "libsolidity", "syntaxTests",         false, false, &SyntaxTest::create},
+	{"Semantic",            "libsolidity", "semanticTests",       false, true,  &SemanticTest::create},
 	{"JSON AST",            "libsolidity", "ASTJSON",             false, false, &ASTJSONTest::create},
 	{"SMT Checker",         "libsolidity", "smtCheckerTests",     true,  false, &SyntaxTest::create},
 	{"SMT Checker JSON",    "libsolidity", "smtCheckerTestsJSON", true,  false, &SMTCheckerTest::create}

test/RPCSession.cpp

@@ -139,9 +139,16 @@ string IPCSocket::sendRequest(string const& _req)
 
 RPCSession& RPCSession::instance(string const& _path)
 {
-	static RPCSession session(_path);
-	BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path);
-	return session;
+	try
+	{
+		static RPCSession session(_path);
+		BOOST_REQUIRE_EQUAL(session.m_ipcSocket.path(), _path);
+		return session;
+	}
+	catch (std::exception const&)
+	{
+		BOOST_THROW_EXCEPTION(std::runtime_error("Error creating RPC session for socket: " + _path));
+	}
 }
 
 string RPCSession::eth_getCode(string const& _address, string const& _blockNumber)

test/TestCase.h

@@ -30,11 +30,25 @@ namespace solidity
 namespace test
 {
 
+#define soltestAssert(CONDITION, DESCRIPTION) \
+	do \
+	{ \
+		if (!(CONDITION)) \
+			BOOST_THROW_EXCEPTION(runtime_error(DESCRIPTION)); \
+	} \
+	while (false)
+
 /** Common superclass of SyntaxTest and SemanticsTest. */
 class TestCase
 {
 public:
-	using TestCaseCreator = std::unique_ptr<TestCase>(*)(std::string const&);
+	struct Config
+	{
+		std::string filename;
+		std::string ipcPath;
+	};
+
+	using TestCaseCreator = std::unique_ptr<TestCase>(*)(Config const&);
 
 	virtual ~TestCase() = default;
 

test/boostTest.cpp

@@ -74,11 +74,13 @@ int registerTests(
 	boost::unit_test::test_suite& _suite,
 	boost::filesystem::path const& _basepath,
 	boost::filesystem::path const& _path,
+	std::string const& _ipcPath,
 	TestCase::TestCaseCreator _testCaseCreator
 )
 {
 	int numTestsAdded = 0;
 	fs::path fullpath = _basepath / _path;
+	TestCase::Config config{fullpath.string(), _ipcPath};
 	if (fs::is_directory(fullpath))
 	{
 		test_suite* sub_suite = BOOST_TEST_SUITE(_path.filename().string());
@@ -87,7 +89,7 @@ int registerTests(
 			fs::directory_iterator()
 		))
 			if (fs::is_directory(entry.path()) || TestCase::isTestFilename(entry.path().filename()))
-				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _testCaseCreator);
+				numTestsAdded += registerTests(*sub_suite, _basepath, _path / entry.path().filename(), _ipcPath, _testCaseCreator);
 		_suite.add(sub_suite);
 	}
 	else
@@ -96,13 +98,13 @@ int registerTests(
 
 		filenames.emplace_back(new string(_path.string()));
 		_suite.add(make_test_case(
-			[fullpath, _testCaseCreator]
+			[config, _testCaseCreator]
 			{
 				BOOST_REQUIRE_NO_THROW({
 					try
 					{
 						stringstream errorStream;
-						if (!_testCaseCreator(fullpath.string())->run(errorStream))
+						if (!_testCaseCreator(config)->run(errorStream))
 							BOOST_ERROR("Test expectation mismatch.\n" + errorStream.str());
 					}
 					catch (boost::exception const& _e)
@@ -142,6 +144,7 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] )
 			master,
 			options.testPath / ts.path,
 			ts.subpath,
+			options.ipcPath,
 			ts.testCaseCreator
 		) > 0, std::string("no ") + ts.title + " tests found");
 	}

test/libsolidity/ASTJSONTest.h

@@ -35,8 +35,8 @@ namespace test
 class ASTJSONTest: public TestCase
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
-	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_filename)); }
+	static std::unique_ptr<TestCase> create(Config const& _config)
+	{ return std::unique_ptr<TestCase>(new ASTJSONTest(_config.filename)); }
 	ASTJSONTest(std::string const& _filename);
 
 	bool run(std::ostream& _stream, std::string const& _linePrefix = "", bool const _formatted = false) override;

test/libsolidity/SMTCheckerJSONTest.h

@@ -33,9 +33,9 @@ namespace test
 class SMTCheckerTest: public SyntaxTest
 {
 public:
-	static std::unique_ptr<TestCase> create(std::string const& _filename)
+	static std::unique_ptr<TestCase> create(Config const& _config)
 	{
-		return std::unique_ptr<TestCase>(new SMTCheckerTest(_filename));
+		return std::unique_ptr<TestCase>(new SMTCheckerTest(_config.filename));
 	}
 	SMTCheckerTest(std::string const& _filename);