No specs found when running tests in parallel

[JonWallsten]

So, I realize that this is super hard to reproduce, but I don't know what else to do than reporting it as a bug. I've tried Gitter and read all relevant issues as well as Google searches for days.

We have just enabled shardTestFiles in our project to reduce the running time on our test.
Everything works fine when running locally on our Windows machines.
But after pushing the project into our CI (Jenkins) the problems started.
We get "No specs found" in a random number of test cases. Sometimes it's all of them. Sometimes it's just a few. I have added console statements inside the test files to make sure there's no issue with access/permission/path etc, and as you can see in the output it finds executes the describe, but ignores the it and outputs "No specs found".
I have checked and deducted everything. I can't figure out the source of the issue.
Has anyone any idea of why the "No specs found" message is thrown, when I clearly have an it in the file? And why it actually works half of the times? I'm truly desperate to post a ticket without a repo where it can be reproduced, but I have tried everything else.

Bug report

• Node Version: 10.9.0
• Protractor Version: 5.4.2
• Angular Version: 8.3.0
• Browser(s): chromium 71.0.3563.0
• Operating System and Version Red Hat Enterprise Linux Server 7.2 (Maipo)
• Your protractor configuration file

const JasmineConsoleReporter = require('jasmine-console-reporter');
const HtmlScreenshotReporter = require('protractor-jasmine2-screenshot-reporter');
const jasmineReporters = require('jasmine-reporters');
const chromeDriverPath = require('chromedriver').path;
const { logInfo, developerRoles, changeBusinessRoles, xmlImport, TESTOA34 } = require('./test/e2e/testdata/create.testdata');
const { Screencast, VideoReporter, directories } = require('@project/internal-lib-e2e');
const puppeteer = require('puppeteer');

const IS_DEV = false;
let USE_RECORDING = process.env.USE_RECORDING === 'true';

const videoReporter = new VideoReporter();

const consoleReporter = new JasmineConsoleReporter({
    colors: IS_DEV ? 1 : 0,
    cleanStack: 1,
    verbosity: 4,
    listStyle: 'indent',
    activity: false
});

const screenshotReporter = new HtmlScreenshotReporter({
    dest: directories.htmlReport,
    filename: 'test-report.html',
    reportOnlyFailedSpecs: false,
    captureOnlyFailedSpecs: true
});


const junitReporter = new jasmineReporters.JUnitXmlReporter({
    savePath: directories.junitReport,
    consolidateAll: false
});

exports.config = {
    framework: 'jasmine',
    chromeDriver: chromeDriverPath,
    multiCapabilities: [{
        browserName: 'chrome',
        chromeOptions: {
            args: puppeteer ? ['--headless', '--log-level=5', '--disable-gpu', '--no-sandbox', '--disable-extensions', '--disable-dev-shm-usage', '--window-size=1920,1200', '--disable-setuid-sandbox', '--auth-server-whitelist=*domain.com'] : [],
            binary: puppeteer.executablePath()
        },
        shardTestFiles: !IS_DEV,
        maxInstances: 5
    }],
    skipSourceMapSupport: true,
    getPageTimeout: 90000,   // How long to wait for a page to load
    allScriptsTimeout: 90000,  // The timeout in milliseconds for each script run on the browser. This should be longer than the maximum time your application needs to stabilize between tasks.

    plugins: [
        {
            package: 'jasmine2-protractor-utils',
            disableHTMLReport: true,
            disableScreenshot: false,
            screenshotOnExpectFailure: true,
            screenshotOnSpecFailure: false,
            screenshotPath: '.test_results/html_report',
            clearFoldersBeforeTest: false,
            htmlReportDir:  '.test_results/html_report',
            failTestOnErrorLog: {
                failTestOnErrorLogLevel: 2000
            }
        }
    ],

    suites: {
        full: [
            './src/app/**/*.e2e-spec.ts',
            './test/e2e/suite/**/*.e2e-spec.ts'
        ],
        auth: './test/e2e/authority/*.e2e-spec.ts'
    },
    // Default suite
    suite: 'full',

    directConnect: true,

    jasmineNodeOpts: {
        showColors: true,
        defaultTimeoutInterval: 220000   // Default time to wait in ms before a test fails.
    },
    beforeLaunch: () => {
        'use strict';

        const testDataFiles = [
            ['CO_structure.xml', 'CO_Test_Structure.txt'],
            ['Parts.xml', 'Parts.txt']
        ];

        const createTestData = async () => {
            await testDataFiles.reduce( async (previousPromise, paths) => {
                await previousPromise;
                return xmlImport(paths[0], paths[1]);
            }, Promise.resolve());
        };

        console.log('Creating test data. This will take a minute or two. Please be patient.'); //eslint-disable-line no-console

        return createTestData()
            .then(() => {
                console.log('\nCreating test data completed.\n'); //eslint-disable-line no-console
            })
            .catch(() => {
                console.log('\nCreating test data failed.\n'); //eslint-disable-line no-console
            });
    },
    onPrepare: () => {
        'use strict';
        let chromeRemoteDebuggingPort = 0;
        if (!process.env.HEADLESS) {
            browser.driver.manage().window().maximize();
        }
        // Get Remote debugging port for chrome
        browser.getCapabilities().then((capabilities) => {
            const chromeOptions = capabilities.get('goog:chromeOptions');
            if( chromeOptions && chromeOptions.debuggerAddress) {
                chromeRemoteDebuggingPort = chromeOptions.debuggerAddress.split(':')[1];
            }
        });

        if(!process.env.PROD_BUILD) {
            jasmine.getEnv().addReporter(consoleReporter);
        }

        jasmine.getEnv().addReporter(junitReporter);
        jasmine.getEnv().addReporter(screenshotReporter);

        if(USE_RECORDING) {
            jasmine.getEnv().addReporter(videoReporter);
        }

        // Log info about environment and running user
        logInfo();

        const prepareAsync = async () => {
            // Need to be able to change business roles when running tests locally
            if (IS_DEV) {
                await changeBusinessRoles(TESTOA34);
            }

            if (USE_RECORDING) {
                if(chromeRemoteDebuggingPort) {
                    await Screencast.start({port: chromeRemoteDebuggingPort});
                } else {
                    // No remote port found, so we can't record.
                    USE_RECORDING = false;
                }
            }
        };

        return prepareAsync();
    },
    onComplete: () => {
        'use strict';
        const completeAsync = async () => {
            // Allow the last images to be received
            await new Promise((resolve) => {
                setTimeout(() => {
                    resolve();
                }, 2000);
            });

            // Need to be able to restore business roles when running tests locally
            if (IS_DEV) {
                await changeBusinessRoles(developerRoles)
                    .catch((error) => {
                        console.error('Error:', error); //eslint-disable-line no-console
                        // Resolve so the tests don't fail just because we can't restore the roles
                        return Promise.resolve();
                    });
            }

            if (USE_RECORDING) {
                await Screencast.stop();
                await Screencast.renderVideos();
            }
            return true;
        };
        return completeAsync();
    },
    afterLaunch: (exitCode) => {
        'use strict';

        return new Promise((resolve) => {
            screenshotReporter.afterLaunch(resolve.bind(this, exitCode));
        });
    }
};

• A relevant example test

console.log('---Read file---');
describe('Simple test 10', () => {
    console.log('---Run describe---');
    it('Test 1', () => {
        console.log('---Run it---');
        expect('1').toEqual('1');
    });
});

• Output from running the test

08:07:03 [OASEdit]  | [08:07:02] I/testLogger - [chrome #01-6] PID: 10115
08:07:03 [OASEdit]  | [chrome #01-6] Specs: /packages/web-app-edit/opt/jenkins/slave/workspace/master/packages/web-app-edit/test/e2e/suite/components/test.e2e-spec.ts
08:07:03 [OASEdit]  | [chrome #01-6] 
08:07:03 [OASEdit]  | [chrome #01-6] [08:06:59] I/direct - Using ChromeDriver directly...
08:07:03 [OASEdit]  | [chrome #01-6] ****************************************
08:07:03 [OASEdit]  | [chrome #01-6] * Backend Server Id: 10
08:07:03 [OASEdit]  | [chrome #01-6] * Host name: api.ourdomain.com
08:07:03 [OASEdit]  | [chrome #01-6] * Using frontend server: http://test5.ourdomain.com
08:07:03 [OASEdit]  | [chrome #01-6] * Using backend server: https://api.ourdomain.com:18443
08:07:03 [OASEdit]  | [chrome #01-6] * Running as user: TESTOA34
08:07:03 [OASEdit]  | [chrome #01-6] ****************************************
08:07:03 [OASEdit]  | [chrome #01-6] ---Read file---
08:07:03 [OASEdit]  | [chrome #01-6] ---Run describe---
08:07:03 [OASEdit]  | [chrome #01-6] Started
08:07:03 [OASEdit]  | [chrome #01-6] 
08:07:03 [OASEdit]  | [chrome #01-6] 
08:07:03 [OASEdit]  | [chrome #01-6] No specs found
08:07:03 [OASEdit]  | [chrome #01-6] Finished in 0.714 seconds
08:07:03 [OASEdit]  | [chrome #01-6] 
08:07:03 [OASEdit]  | 
08:07:03 [OASEdit]  | [08:07:02] I/testLogger - 
08:07:03 [OASEdit]  | 
08:07:03 [OASEdit]  | [08:07:02] I/launcher - 5 instance(s) of WebDriver still running
08:07:03 [OASEdit]  | [08:07:02] I/testLogger - 

• Steps to reproduce the bug
  Run parallel test cases in Jenkins CI on Red Hat server

[tymfear]

Can you show the place where you apply register in ts-node? Because it tries to execute /packages/web-app-edit/opt/jenkins/slave/workspace/master/packages/web-app-edit/test/e2e/suite/components/test.e2e-spec.ts And which command do you use to execute your tests?

[JonWallsten]

@tymfear: We're using a monorepo. So it might look confusing.
The workspace is the monorepo root. And Edit is located in packages/web-app-edit.
I run a script in the monorepo root which triggers a script in packages/web-app-edit/package.json.
You can see the dir in the output below.

09:54:08 > oas-web-src@6.0.182 all /opt/jenkins/slave/workspace/master
09:54:08 > ts-node ./build-tools/run-all/index.ts "protractor -- --suite full" "--include" "web-app-edit"

09:54:09 > oas-web-src@6.0.182 all /opt/jenkins/slave/workspace/master/packages/web-app-edit
09:54:09 > ts-node ../../node_modules/protractor/bin/protractor

It seems like the issue only occurs when combining parallel executions in Jenkins and in Protractor.
So we're starting one protractor script for each app in the monorepo. And they are using shardTestFiles to create multiple instances themselves.

Edit: I changed so that each app now has it's own root as the cwd instead on the monorepo root. Now the tests are working in our branch-builds but not the master, even though they run the same code. Could it possible be a cache issue like in the question below?

https://stackoverflow.com/questions/49995731/jasmine-library-no-specs-found-when-running-same-test-twice

Would it be possible to get more detailer error information from Jasmine why it can't find any specs?

[JonWallsten]

This is so frustrating. Here's logs from the same test case in two different branches.
One works, the other one doesn't.

13:43:57 [Prime] [13:43:56] I/testLogger - [chrome #1-0] PID: 6352
13:43:57 [Prime] [chrome #1-0] Specs: /opt/jenkins/slave/workspace/B-Components_oas-web_master-PTCXA55GXMBRSIFNTKINTF4SAHS37GVGIU5K7FABZP76HFQY5GKQ_2/packages/web-app/test/e2e/PrimeTests.js
13:43:57 [Prime] [chrome #1-0]
13:43:57 [Prime] [chrome #1-0]*************************************
13:43:57 [Prime] [chrome #1-0] * Backend Server Id: 10
13:43:57 [Prime] [chrome #1-0] * Host name: server1.sss.se.domain.com
13:43:57 [Prime] [chrome #1-0] * Using frontend server: http://oastest1.domain.com
13:43:57 [Prime] [chrome #1-0] * Using backend server: https://server1.sss.se.domain.com:18443
13:43:57 [Prime] [chrome #1-0] ****************************************
13:43:57 [Prime] [chrome #1-0] [13:43:49] I/direct - Using ChromeDriver directly...
13:43:57 [Prime] [chrome #1-0] Started
13:43:57 [Prime] [chrome #1-0]
13:43:57 [Prime] [chrome #1-0]
13:43:57 [Prime] [chrome #1-0] No specs found
13:43:57 [Prime] [chrome #1-0] Finished in 4.378 seconds
13:43:57 [Prime] [chrome #1-0]
13:43:57 [Prime]
13:43:57 [Prime] [13:43:56] I/testLogger -
13:43:57 [Prime]
13:43:57 [Prime] [13:43:56] I/launcher - 5 instance(s) of WebDriver still running
13:43:57 [Prime] [13:43:56] I/testLogger -

10:28:04 [Prime] [10:28:03] I/testLogger - [chrome #1-0] PID: 14705
10:28:04 [Prime] [chrome #1-0] Specs: /opt/jenkins/slave/workspace/_fix-add-object-cell-always-IC2TWQHDTR2MALHGN4UX77XXAM6LSUNENVI7TPKQFFX6SVE42FIQ/packages/web-app/test/e2e/PrimeTests.js
10:28:04 [Prime] [chrome #1-0]
10:28:04 [Prime] [chrome #1-0] ****************************************
10:28:04 [Prime] [chrome #1-0] * Backend Server Id: 10
10:28:04 [Prime] [chrome #1-0] * Host name: server1.sss.se.domain.com
10:28:04 [Prime] [chrome #1-0] * Using frontend server: http://oastest6.domain.com
10:28:04 [Prime] [chrome #1-0] * Using backend server: https://server1.sss.se.domain.com:18443
10:28:04 [Prime] [chrome #1-0] ****************************************
10:28:04 [Prime] [chrome #1-0] [10:27:52] I/direct - Using ChromeDriver directly...
10:28:04 [Prime] [chrome #1-0] Started
10:28:04 [Prime] [chrome #1-0] ..
10:28:04 [Prime] [chrome #1-0]
10:28:04 [Prime] [chrome #1-0]
10:28:04 [Prime] [chrome #1-0] 2 specs, 0 failures
10:28:04 [Prime] [chrome #1-0] Finished in 8.736 seconds
10:28:04 [Prime] [chrome #1-0]
10:28:04 [Prime]
10:28:04 [Prime] [10:28:03] I/testLogger -
10:28:04 [Prime]
10:28:04 [Prime] [10:28:03] I/launcher - 5 instance(s) of WebDriver still running
10:28:09 [Prime] ........[10:28:09] I/testLogger -

[JonWallsten]

This was one of those edgy edge cases. One of my reporters is creating a folder for each spec for screenshots. And node doesn't continue passed the fs.mkdir. No error is thrown, and then the spec is considered not found.