fix testeth -t <TestSuite> -- --singletest <path> <testname> running a test from file

CHANGELOG.md

@@ -5,6 +5,7 @@
 - Added: [#5699](https://github.com/ethereum/aleth/pull/5699) EIP 2046: Reduced gas cost for static calls made to precompiles.
 - Added: [#5741](https://github.com/ethereum/aleth/pull/5741) Support for individual EIP activation to facilitate EIP-centric network upgrade process.
 - Added: [#5752](https://github.com/ethereum/aleth/pull/5752) [#5753](https://github.com/ethereum/aleth/pull/5753) Implement EIP1380 (reduced gas costs for call-to-self).
+- Changed: [#5750](https://github.com/ethereum/aleth/pull/5750) Use `testeth -t <SUITE_NAME> -- --testfile <PATH>` to run the tests from file at any path. Use `testeth -t <SUITE_NAME> -- --testfile <PATH> --singletest <TEST_NAME>` to run only single test from any file.
 - Changed: [#5801](https://github.com/ethereum/aleth/pull/5801) `testeth -t BlockchainTests` command now doesn't run the tests for the forks before Istanbul. To run those tests use a separate LegacyTests suite with command `testeth -t LegacyTests/Constantinople/BlockchainTests`.
 - Changed: [#5807](https://github.com/ethereum/aleth/pull/5807) Optimize selfdestruct opcode in LegacyVM by reducing state accesses in certain out-of-gas scenarios.
 - Changed: [#5806](https://github.com/ethereum/aleth/pull/5806) Optimize selfdestruct opcode in aleth-interpreter by reducing state accesses in certain out-of-gas scenarios.

test/tools/jsontests/BlockChainTests.cpp

@@ -122,7 +122,8 @@ json_spirit::mValue doBCTest(
                 Options const& opt = test::Options::get();
 
                 // Select BC Test by singleTest
-                if (opt.singleTest && !boost::algorithm::starts_with(testname, opt.singleTestName))
+                if (!opt.singleTestName.empty() &&
+                    !boost::algorithm::starts_with(testname, opt.singleTestName))
                     continue;
 
                 // Select BC Test by singleNet

test/tools/libtesteth/Options.cpp

@@ -34,8 +34,11 @@ void printHelp()
     cout << setw(35) << "-g <index>" << setw(25) << "Set the transaction gas array index when running GeneralStateTests\n";
     cout << setw(35) << "-v <index>" << setw(25) << "Set the transaction value array index when running GeneralStateTests\n";
     cout << setw(35) << "--singletest <TestName>" << "Run on a single test\n";
-    cout << setw(35) << "--singletest <TestFile> <TestName>" << "Run on a single test from file\n";
     cout << setw(35) << "--singlenet <networkId>" << setw(25) << "Run tests for a specific network (Frontier|Homestead|EIP150|EIP158|Byzantium|Constantinople|ConstantinopleFix)\n";
+    cout << setw(35) << "--testfile <TestFile>"
+         << "Run tests from a file. Requires -t <TestSuite>\n";
+    cout << setw(35) << "--testfile <TestFile> --singletest <TestName>"
+         << "Run on a single test from a file. Requires -t <TestSuite>\n";
     cout << setw(35) << "--verbosity <level>" << setw(25) << "Set logs verbosity. 0 - silent, 1 - only errors, 2 - informative, >2 - detailed\n";
     cout << setw(35) << "--vm <name|path> (=legacy)" << setw(25) << "Set VM type for VMTests suite. Available options are: interpreter, legacy.\n";
     cout << setw(35) << "--evmc <option>=<value>" << "EVMC option\n";
@@ -198,31 +201,24 @@ Options::Options(int argc, const char** argv)
         else if (arg == "--singletest")
         {
             throwIfNoArgumentFollows();
-            singleTest = true;
-            auto name1 = std::string{argv[++i]};
-            if (i + 1 < argc) // two params
-            {
-                auto name2 = std::string{argv[++i]};
-                if (name2[0] == '-') // not param, another option
-                {
-                    singleTestName = std::move(name1);
-                    i--;
-                }
-                else
-                {
-                    singleTestFile = std::move(name1);
-                    singleTestName = std::move(name2);
-                }
-            }
-            else
-                singleTestName = std::move(name1);
+            singleTestName = std::string{argv[++i]};
         }
         else if (arg == "--singlenet")
         {
             throwIfNoArgumentFollows();
             singleTestNet = std::string{argv[++i]};
             ImportTest::checkAllowedNetwork(singleTestNet);
         }
+        else if (arg == "--testfile")
+        {
+            throwIfNoArgumentFollows();
+            singleTestFile = std::string{argv[++i]};
+            if (!boost::filesystem::exists(singleTestFile))
+            {
+                cerr << "Test file not found: " << singleTestFile << "\n";
+                exit(1);
+            }
+        }
         else if (arg == "--fullstate")
             fullstate = true;
         else if (arg == "--verbosity")
@@ -329,8 +325,8 @@ Options::Options(int argc, const char** argv)
     //check restricted options
     if (createRandomTest)
     {
-        if (trValueIndex >= 0 || trGasIndex >= 0 || trDataIndex >= 0 || singleTest || all ||
-            stats || filltests || fillchain)
+        if (trValueIndex >= 0 || trGasIndex >= 0 || trDataIndex >= 0 || !singleTestName.empty() ||
+            all || stats || filltests || fillchain)
         {
             cerr << "--createRandomTest cannot be used with any of the options: " <<
                     "trValueIndex, trGasIndex, trDataIndex, singleTest, all, " <<

test/tools/libtesteth/Options.h

@@ -40,8 +40,7 @@ class Options
 
     /// Test selection
     /// @{
-    bool singleTest = false;
-    boost::optional<std::string> singleTestFile;
+    std::string singleTestFile;
     std::string singleTestName;
     std::string singleTestNet;
     int trDataIndex;	///< GeneralState data

test/tools/libtesteth/TestOutputHelper.cpp

@@ -36,10 +36,11 @@ void TestOutputHelper::initTest(size_t _maxTests)
 
 bool TestOutputHelper::checkTest(std::string const& _testName)
 {
-	if (test::Options::get().singleTest && test::Options::get().singleTestName != _testName)
-		return false;
+    if (!test::Options::get().singleTestName.empty() &&
+        test::Options::get().singleTestName != _testName)
+        return false;
 
-	m_currentTestName = _testName;
+    m_currentTestName = _testName;
 	return true;
 }
 

test/tools/libtesteth/TestSuite.cpp

@@ -224,8 +224,8 @@ void TestSuite::executeTest(string const& _testFolder, fs::path const& _testFile
 		}
 		else
 		{
-			if (!Options::get().singleTest)
-				cnote << "Populating tests...";
+            if (Options::get().singleTestName.empty())
+                cnote << "Populating tests...";
 
             TestFileData fillerData = readTestFile(_testFileName);
             removeComments(fillerData.data);

test/tools/libtesteth/boostTest.cpp

@@ -44,9 +44,9 @@ void customTestSuite()
     }
 
     // if running a singletest
-    if (opt.singleTestFile.is_initialized())
+    if (!opt.singleTestFile.empty())
     {
-        boost::filesystem::path file(opt.singleTestFile.get());
+        boost::filesystem::path const file(opt.singleTestFile);
         if (opt.rCurrentTestSuite.find("GeneralStateTests") != std::string::npos)
         {
             dev::test::StateTestSuite suite;
@@ -93,7 +93,7 @@ int main(int argc, const char* argv[])
     }
 
     dev::test::Options const& opt = dev::test::Options::get();
-    if (opt.createRandomTest || opt.singleTestFile.is_initialized())
+    if (opt.createRandomTest || !opt.singleTestFile.empty())
     {
         bool testSuiteFound = false;
         for (int i = 0; i < argc; i++)
@@ -114,7 +114,7 @@ int main(int argc, const char* argv[])
             std::cerr << "createRandomTest requires a test suite to be set -t <TestSuite>\n";
             return -1;
         }
-        if (!testSuiteFound && opt.singleTestFile.is_initialized())
+        if (!testSuiteFound && !opt.singleTestFile.empty())
         {
             std::cerr
                 << "singletest <file> <testname>  requires a test suite to be set -t <TestSuite>\n";
@@ -146,7 +146,8 @@ int main(int argc, const char* argv[])
     }
 
     // Print suggestions of a test case if test suite not found
-    if (!sMinusTArg.empty() && !dev::test::inArray(c_allTestNames, sMinusTArg))
+    if (!sMinusTArg.empty() && !dev::test::inArray(c_allTestNames, sMinusTArg) &&
+        sMinusTArg != "customTestSuite")
     {
         std::cerr << "Error: '" + sMinusTArg + "' suite not found! \n";
         printTestSuiteSuggestions(sMinusTArg);