[CI] Split build and verify into parallel jobs

Signed-off-by: Miki <miki@amazon.com>
AMoo-Miki · Jun 30, 2023 · 2f4f6ba · 2f4f6ba
1 parent 63b66f9
commit 2f4f6ba
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 13 deletions.
diff --git a/.github/workflows/build_and_test_workflow.yml b/.github/workflows/build_and_test_workflow.yml
@@ -29,11 +29,12 @@ env:
 
 jobs:
   build-lint-test:
-    name: Build and Verify on ${{ matrix.name }}
+    name: Build and Verify on ${{ matrix.name }} batch ${{ matrix.ci-batch }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
+        ci-batch: [1, 2, 3]
         include:
           - os: ubuntu-latest
             name: Linux
@@ -95,10 +96,12 @@ jobs:
         run: yarn osd bootstrap || yarn osd bootstrap
 
       - name: Run linter
+        if: matrix.ci-batch == 3
         id: linter
         run: yarn lint
 
       - name: Validate NOTICE file
+        if: matrix.ci-batch == 3
         id: notice-validate
         run: yarn notice:validate
 
@@ -107,6 +110,7 @@ jobs:
         run: yarn test:jest:ci:coverage
 
       - name: Run mocha tests with coverage
+        if: matrix.ci-batch == 3
         id: mocha-tests
         run: yarn test:mocha:coverage
 
@@ -115,9 +119,10 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           directory: ./target/opensearch-dashboards-coverage
-          flags: ${{ matrix.name }}
+          flags: ${{ matrix.name }} b${{ matrix.ci-batch }}
 
       - name: Run integration tests
+        if: matrix.ci-batch == 3
         id: integration-tests
         run: yarn test:jest_integration:ci
 

diff --git a/src/dev/jest/config.js b/src/dev/jest/config.js
@@ -30,25 +30,56 @@
 
 import { RESERVED_DIR_JEST_INTEGRATION_TESTS } from '../constants';
 
-export default {
-  rootDir: '../../..',
-  roots: [
-    '<rootDir>/src/plugins',
-    '<rootDir>/src/legacy/ui',
-    '<rootDir>/src/core',
-    '<rootDir>/src/legacy/server',
-    '<rootDir>/src/cli',
+const rootBatches = [
+  [
+    /* CI Batch 0 is left empty to make numbering natural */
+  ],
+  [
+    // CI Batch 1
+    '<rootDir>/src/plugins', //800
+  ],
+  [
+    // CI Batch 2
+    '<rootDir>/src/cli', //48
     '<rootDir>/src/cli_keystore',
     '<rootDir>/src/cli_plugin',
+    '<rootDir>/src/legacy/ui',
+    '<rootDir>/src/core', //340
+    '<rootDir>/src/legacy/server', //5
     '<rootDir>/packages/osd-test/target/functional_test_runner',
-    '<rootDir>/src/dev',
+    '<rootDir>/src/dev', //7
     '<rootDir>/src/optimize',
     '<rootDir>/src/legacy/utils',
-    '<rootDir>/src/setup_node_env',
-    '<rootDir>/packages',
+  ],
+  [
+    // CI Batch 3
+    '<rootDir>/src/setup_node_env', //150
+    '<rootDir>/packages', //242
     '<rootDir>/src/test_utils',
     '<rootDir>/test/functional/services/remote',
   ],
+];
+
+const roots = [];
+
+// Looks for --ci-batch=<number> and captures the number
+const ciBatchPattern = /^--ci-batch=(\d+)$/;
+const ciBatches = process.argv.reduce((acc, arg) => {
+  const match = arg.match(ciBatchPattern);
+  if (isFinite(match?.[1])) acc.push(parseInt(match[1], 10));
+  return acc;
+}, []);
+
+if (ciBatches.length > 0) {
+  console.log(`Requested batch${ciBatches.length === 1 ? '' : 'es'}: ${ciBatches.join(', ')}`);
+  ciBatches.forEach((id) => roots.push(...rootBatches[id]));
+} else {
+  roots.push(...rootBatches.flat());
+}
+
+export default {
+  rootDir: '../../..',
+  roots,
   moduleNameMapper: {
     '@elastic/eui$': '<rootDir>/node_modules/@elastic/eui/test-env',
     '@elastic/eui/lib/(.*)?': '<rootDir>/node_modules/@elastic/eui/test-env/$1',