ci: review and enhancements

- Removed `rules` from child pipelines and other jobs with dependencies - Separated value for `parallel` into a variable for easier editing - Uploading of the dist separated into its own job so multiple parallel jobs aren't uploading it - Removed `--runInBand` since we want to make use of multiple cores Relates to #65
MatrixAI · Jun 29, 2022 · 9fcd851 · 9fcd851
1 parent f450a32
commit 9fcd851
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 57 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -74,6 +74,7 @@ check:nix-dry:
 
 check:test-generate:
   stage: check
+  needs: []
   script:
     - mkdir -p ./tmp
     - >
@@ -102,12 +103,6 @@ check:test:
       - artifact: tmp/test-pipelines.yml
         job: check:test-generate
     strategy: depend
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Manually run on commits other than master and staging and ignore version commits
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-      when: manual
 
 build:merge:
   stage: build
@@ -137,7 +132,7 @@ build:merge:
     # Runs on tag pipeline where the tag is a prerelease or release version
     - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
-build:build-generate:
+build:platforms-generate:
   stage: build
   script:
     - mkdir -p ./tmp
@@ -172,10 +167,28 @@ build:platforms:
     # Runs on tag pipeline where the tag is a prerelease or release version
     - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 
+build:dist:
+  stage: build
+  needs: []
+  script:
+    - >
+        nix-shell --run '
+        npm run build --verbose;
+        '
+  artifacts:
+    when: always
+    paths:
+      - ./dist
+  rules:
+    # Runs on staging commits and ignores version commits
+    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+    # Runs on tag pipeline where the tag is a prerelease or release version
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
+
 build:prerelease:
   stage: build
   needs:
-    - build:platforms
+    - build:dist
   # Don't interrupt publishing job
   interruptible: false
   before_script:
@@ -198,7 +211,7 @@ build:prerelease:
 integration:builds:
   stage: integration
   needs:
-    - build:platforms
+    - build:dist
   script:
     - mkdir -p ./builds
     - >
@@ -395,7 +408,7 @@ integration:merge:
   stage: integration
   needs:
     - build:merge
-    - job: build:platforms
+    - job: build:dist
       optional: true
     - job: integration:nix
       optional: true
@@ -471,7 +484,7 @@ release:deployment:tag:
 release:distribution:
   stage: release
   needs:
-    - build:platforms
+    - build:dist
     - integration:builds
     - integration:merge
     - release:deployment:tag

diff --git a/scripts/build-pipelines.sh b/scripts/build-pipelines.sh
@@ -5,14 +5,9 @@ shopt -s nullglob
 
 # Quote the heredoc to prevent shell expansion
 cat << "EOF"
-workflow:
-  rules:
-    # Disable merge request pipelines
-    - if: $CI_MERGE_REQUEST_ID
-      when: never
-    - when: always
 default:
   interruptible: true
+
 variables:
   GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
   GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
@@ -27,6 +22,7 @@ variables:
   TS_CACHED_TRANSPILE_PORTABLE: "true"
   # Homebrew cache only used by macos runner
   HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew"
+
 # Cached directories shared between jobs & pipelines per-branch per-runner
 cache:
   key: $CI_COMMIT_REF_SLUG
@@ -37,8 +33,10 @@ cache:
     - ./tmp/Homebrew
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
+
 stages:
   - build       # Cross-platform library compilation, unit tests
+
 image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
 EOF
 
@@ -48,16 +46,20 @@ printf "\n"
 # In the future we can incorporate test durations rather than using
 # a static value for the parallel keyword
 
+# Number of parallel shards to split the test suite into
+CI_PARALLEL=2
+
+
 cat << "EOF"
 build:linux:
   stage: build
   needs: []
-  parallel: 2
+  parallel: $CI_PARALLEL
   script:
     - >
         nix-shell --run '
-        npm run build --verbose;
-        npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
+        npm run build --verbose
+        npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
         '
   artifacts:
     when: always
@@ -67,15 +69,7 @@ build:linux:
       coverage_report:
         coverage_format: cobertura
         path: ./tmp/coverage/cobertura-coverage.xml
-    paths:
-      # Only the build:linux preserves the dist
-      - ./dist
   coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Runs on tag pipeline where the tag is a prerelease or release version
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 EOF
 
 printf "\n"
@@ -84,7 +78,7 @@ cat << "EOF"
 build:windows:
   stage: build
   needs: []
-  parallel: 2
+  parallel: $CI_PARALLEL
   tags:
     - windows
   before_script:
@@ -95,17 +89,12 @@ build:windows:
     - npm install --ignore-scripts
     - $env:Path = "$(npm bin);" + $env:Path
     - npm run build --verbose
-    - npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
+    - npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
   artifacts:
     when: always
     reports:
       junit:
         - ./tmp/junit/junit.xml
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Runs on tag pipeline where the tag is a prerelease or release version
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 EOF
 
 printf "\n"
@@ -114,7 +103,7 @@ cat << "EOF"
 build:macos:
   stage: build
   needs: []
-  parallel: 2
+  parallel: $CI_PARALLEL
   tags:
     - shared-macos-amd64
   image: macos-11-xcode-12
@@ -130,17 +119,12 @@ build:macos:
     - npm install --ignore-scripts
     - export PATH="$(npm bin):$PATH"
     - npm run build --verbose
-    - npm test -- --ci --runInBand --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
+    - npm test -- --ci --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
   artifacts:
     when: always
     reports:
       junit:
         - ./tmp/junit/junit.xml
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH == 'staging' && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Runs on tag pipeline where the tag is a prerelease or release version
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
 EOF
 
 printf "\n"
diff --git a/scripts/test-pipelines.sh b/scripts/test-pipelines.sh
@@ -5,14 +5,9 @@ shopt -s nullglob
 
 # Quote the heredoc to prevent shell expansion
 cat << "EOF"
-workflow:
-  rules:
-    # Disable merge request pipelines
-    - if: $CI_MERGE_REQUEST_ID
-      when: never
-    - when: always
 default:
   interruptible: true
+
 variables:
   GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}"
   GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git"
@@ -25,6 +20,7 @@ variables:
   # It must use an absolute path, otherwise ts-node calls will CWD
   TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache"
   TS_CACHED_TRANSPILE_PORTABLE: "true"
+
 # Cached directories shared between jobs & pipelines per-branch per-runner
 cache:
   key: $CI_COMMIT_REF_SLUG
@@ -33,8 +29,10 @@ cache:
     - ./tmp/ts-node-cache/
     # `jest` cache is configured in jest.config.js
     - ./tmp/jest/
+
 stages:
   - check       # Linting, unit tests
+
 image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner
 EOF
 
@@ -43,16 +41,19 @@ printf "\n"
 # Using shards to optimise tests
 # In the future we can incorporate test durations rather than using
 # a static value for the parallel keyword
+
+# Number of parallel shards to split the test suite into
+CI_PARALLEL=2
+
 cat << "EOF"
 check:test:
   stage: check
   needs: []
-  parallel: 2
+  parallel: $CI_PARALLEL
   script:
     - >
         nix-shell --run '
-        npm run build --verbose;
-        npm test -- --ci --runInBand --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
+        npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL;
         '
   artifacts:
     when: always
@@ -63,12 +64,6 @@ check:test:
         coverage_format: cobertura
         path: ./tmp/coverage/cobertura-coverage.xml
   coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
-  rules:
-    # Runs on staging commits and ignores version commits
-    - if: $CI_COMMIT_BRANCH =~ /^feature.*$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-    # Manually run on commits other than master and staging and ignore version commits
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH !~ /^(?:master|staging)$/ && $CI_COMMIT_TITLE !~ /^[0-9]+\.[0-9]+\.[0-9]+(?:-.*[0-9]+)?$/
-      when: manual
 EOF
 
 printf "\n"