From 924e29a70308d960bca5f317ad9ebe706508f3f9 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana Date: Fri, 23 Feb 2024 21:43:43 +0000 Subject: [PATCH] Add cigroup matrix for parallel runs Signed-off-by: Manasvini B Suryanarayana --- .github/workflows/cypress_workflow.yml | 88 ++++++++++++++----- .../workflows/dashboards_cypress_workflow.yml | 45 +++++----- cypress/utils/commands.js | 15 ++++ package.json | 6 +- scripts/common/opensearch_service.sh | 1 + scripts/cypress_tests.sh | 6 +- 6 files changed, 109 insertions(+), 52 deletions(-) create mode 100644 cypress/utils/commands.js diff --git a/.github/workflows/cypress_workflow.yml b/.github/workflows/cypress_workflow.yml index 14975e23b17a..8dfd18193c2c 100644 --- a/.github/workflows/cypress_workflow.yml +++ b/.github/workflows/cypress_workflow.yml @@ -2,6 +2,8 @@ name: Run cypress tests # trigger on every PR for all branches on: + push: + branches: [ '**' ] pull_request: branches: [ '**' ] paths-ignore: @@ -47,7 +49,7 @@ jobs: strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5, 6, 7, 8, 9] + group: [1, 2, 10, 11] container: image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 options: --user 1001 @@ -81,8 +83,8 @@ jobs: - name: Checkout code uses: actions/checkout@v2 with: - repository: ${{ env.SOURCE_REPO }} - ref: '${{ env.SOURCE_BRANCH }}' + repository: ${{ env.SOURCE_REPO || github.repository}} + ref: '${{ env.SOURCE_BRANCH || github.base_ref}}' - name: Setup Node uses: actions/setup-node@v2 @@ -101,7 +103,7 @@ jobs: - name: Build plugins run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12 - - name: Checkout + - name: Checkout FT repo uses: actions/checkout@v2 with: path: ${{ env.FTR_PATH }} @@ -114,7 +116,7 @@ jobs: echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV - name: Setup spec files - if: ${{ inputs.specs == '' }} + if: ${{ inputs.specs == '' && matrix.group != 10 && matrix.group != 11 }} working-directory: ${{ env.FTR_PATH }} shell: bash run: | @@ -126,51 +128,91 @@ jobs: echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV - name: Get Cypress version + if: ${{ matrix.group != 10 && matrix.group != 11 }} id: cypress_version run: | echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT - - name: Cache Cypress - id: cache-cypress - uses: actions/cache@v1 - with: - path: ~/.cache/Cypress - key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }} - env: - CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }} - - run: npx cypress cache list - - run: npx cypress cache path - - - name: Run tests + - name: Setup spec files for Dashboards tests + if: ${{ inputs.specs == '' && matrix.group == 10 || matrix.group == 11 }} + shell: bash + run: | + IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }}) + DASHBOARDS_SPEC='' + for i in "${SPEC_ARRAY[@]}"; do + DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," + done + echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV + + - name: Run FT repo tests + if: ${{ matrix.group != 10 && matrix.group != 11 }} uses: cypress-io/github-action@v2 with: working-directory: ${{ env.FTR_PATH }} start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }} wait-on: 'http://localhost:9200, http://localhost:5601' command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }} + + - name: Run Dashboards Cypress tests + if: ${{ matrix.group == 10 || matrix.group == 11 }} + uses: cypress-io/github-action@v6 + with: + install-command: | + yarn cypress cache clear + yarn osd bootstrap + start: node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false, node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true + wait-on: 'http://localhost:9200, http://localhost:5601' + command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }} # Screenshots are only captured on failure, will change this once we do visual regression tests - - uses: actions/upload-artifact@v3 - if: failure() + - name: Upload FT repo screenshots + uses: actions/upload-artifact@v3 + if: failure() && (matrix.group != 10 && matrix.group != 11) with: name: ftr-cypress-screenshots path: ${{ env.FTR_PATH }}/cypress/screenshots retention-days: 1 + + - name: Upload Dashboards screenshots + if: failure() && (matrix.group == 10 || matrix.group == 11) + uses: actions/upload-artifact@v3 + with: + name: dashboards-cypress-screenshots + path: cypress/screenshots + retention-days: 1 - - uses: actions/upload-artifact@v3 - if: always() + - name: Upload FT repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group != 10 && matrix.group != 11) with: name: ftr-cypress-videos path: ${{ env.FTR_PATH }}/cypress/videos retention-days: 1 - - uses: actions/upload-artifact@v3 - if: always() + - name: Upload Dashboards repo videos + uses: actions/upload-artifact@v3 + if: always() && (matrix.group == 10 || matrix.group == 11) + with: + name: dashboards-cypress-videos + path: cypress/videos + retention-days: 1 + + - name: Upload FT repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group != 10 && matrix.group != 11) with: name: ftr-cypress-results path: ${{ env.FTR_PATH }}/cypress/results retention-days: 1 + - name: Upload Dashboards repo results + uses: actions/upload-artifact@v3 + if: always() && (matrix.group == 10 || matrix.group == 11) + with: + name: dashboards-cypress-results + path: cypress/results + retention-days: 1 + add-comment: needs: [cypress-tests] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }} diff --git a/.github/workflows/dashboards_cypress_workflow.yml b/.github/workflows/dashboards_cypress_workflow.yml index 7f0a6bb9c3d7..e961ee975382 100644 --- a/.github/workflows/dashboards_cypress_workflow.yml +++ b/.github/workflows/dashboards_cypress_workflow.yml @@ -41,25 +41,32 @@ env: TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }} TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}" OSD_PATH: 'osd' - CYPRESS_BROWSER: 'chromium' + CYPRESS_BROWSER: 'electron' JOB_ID: ${{ inputs.UNIQUE_ID}} OPENSEARCH: ${{ inputs.OS_URL != '' && inputs.OS_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }} DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }} OPENSEARCH_DIR: 'cypress/opensearch' DASHBOARDS_DIR: 'cypress/opensearch-dashboards' SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }} - SPEC: 'cypress/integration/core_opensearch_dashboards/*' jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch-dashboards + cypress-tests: + needs: Get-CI-Image-Tag runs-on: ubuntu-latest strategy: matrix: - spec_group: - [1, 2, 3, 4] # Add more spec files as needed + spec_group: [10, 11] # Add more spec files as needed container: - image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 - options: --user 1001 + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root env: # prevents extra Cypress installation progress messages CI: 1 @@ -88,32 +95,22 @@ jobs: - name: Setup spec files if: ${{ inputs.specs == '' }} - working-directory: ${{ env.OSD_PATH }} shell: bash run: | + cd ${{ env.OSD_PATH }} IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.spec_group }}) FORMATTED_SPEC='' for i in "${SPEC_ARRAY[@]}"; do - FORMATTED_SPEC+="cypress/integration/core-opensearch-dashboards/${i}," + FORMATTED_SPEC+="cypress/integration/core_opensearch_dashboards/${i}," done + SPEC=${FORMATTED_SPEC} # Set SPEC to FORMATTED_SPEC echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV - echo "SPec found: $SPEC" + echo "SPEC found: $SPEC" - name: Get Cypress version id: cypress_version run: | echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT - - - name: Cache Cypress - id: cache-cypress - uses: actions/cache@v1 - with: - path: ~/.cache/Cypress - key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }} - env: - CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }} - - run: npx cypress cache list - - run: npx cypress cache path - name: Get package version (Linux) run: | @@ -123,7 +120,9 @@ jobs: - name: Run bootstrap run: | cd ${{ env.OSD_PATH }} - yarn osd bootstrap + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v && + yarn osd bootstrap" - name: Download and extract Opensearch artifacts run: | @@ -141,8 +140,8 @@ jobs: - name: Run Cypress tests run: | - source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh - run_dashboards_cypress_tests + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh && run_dashboards_cypress_tests" # Screenshots are only captured on failures - uses: actions/upload-artifact@v3 diff --git a/cypress/utils/commands.js b/cypress/utils/commands.js new file mode 100644 index 000000000000..56a1fd0cff0e --- /dev/null +++ b/cypress/utils/commands.js @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +// --- Typed commands -- + +Cypress.Commands.add('getElementByTestId', (testId, options = {}) => { + return cy.get(`[data-test-subj="${testId}"]`, options); +}); + +Cypress.Commands.add('getElementsByTestIds', (testIds, options = {}) => { + const selectors = [testIds].flat(Infinity).map((testId) => `[data-test-subj="${testId}"]`); + return cy.get(selectors.join(','), options); +}); diff --git a/package.json b/package.json index 8523ef59885c..1bf6f640a145 100644 --- a/package.json +++ b/package.json @@ -80,10 +80,8 @@ "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"", "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false", "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500", - "osd:ciGroup1": "echo \"\"", - "osd:ciGroup2": "echo \"\"", - "osd:ciGroup3": "echo \"\"", - "osd:ciGroup4": "echo \"dashboard_sanity_test_spec.js\"" + "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"", + "osd:ciGroup11": "echo \"dashboard_sanity_test_spec.js\"" }, "repository": { "type": "git", diff --git a/scripts/common/opensearch_service.sh b/scripts/common/opensearch_service.sh index 09f3b5d50c8c..120caee12d26 100755 --- a/scripts/common/opensearch_service.sh +++ b/scripts/common/opensearch_service.sh @@ -9,6 +9,7 @@ function setup_opensearch() { cd "$OPENSEARCH_DIR" echo "network.host: 0.0.0.0" >> config/opensearch.yml echo "discovery.type: single-node" >> config/opensearch.yml + echo "cluster.routing.allocation.disk.threshold_enabled: false" >> config/opensearch.yml [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml # Required for IM [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml diff --git a/scripts/cypress_tests.sh b/scripts/cypress_tests.sh index e4c2cbfc9352..93e58664f438 100644 --- a/scripts/cypress_tests.sh +++ b/scripts/cypress_tests.sh @@ -67,10 +67,12 @@ function check_status() { # Starts OpenSearch Dashboards and run tests in the cypress folder function run_dashboards_cypress_tests() { + run_opensearch + cd $CWD echo "[ OpenSearch Dashboards setup before it starts... ]" setup_dashboards >> /dev/null 2>&1 & sleep 100 - cd "$DASHBOARDS_DIR" + cd $CWD/"$DASHBOARDS_DIR" if [ -x "./bin/opensearch-dashboards" ]; then spawn_process_and_save_PID "./bin/opensearch-dashboards &" else @@ -79,11 +81,11 @@ function run_dashboards_cypress_tests() { fi check_status $DASHBOARDS_URL $DASHBOARDS_MSG # Run cypress tests + cd "$CWD"/osd run_cypress } function run_cypress() { - cd "$CWD"/osd echo "SPEC found: $SPEC" if [ $SECURITY_ENABLED = "true" ]; then echo "run security enabled tests"