Support /test-whole-conformance and daily job (antrea-io#826)

* Support /test-whole-conformance to trigger a whole conformance test
* Support daily whole conformance test
* If whole conformance test fails, Jenkins will send an email to projectantrea-dev@googlegroups.com
Fixed issue antrea-io#819

ci/jenkins/README.md

@@ -21,13 +21,18 @@ should be deleted. This ensures that all tests are run on a clean testbed.
 * [conformance [gated check-in]](https://jenkins.antrea-ci.rocks/job/antrea-conformance-for-pull-request/):
   community tests using sonobuoy, focusing on "Conformance", and skipping "Slow",
   "Serial", "Disruptive", "Flaky", "Feature", "sig-cli",
-  "sig-storage", "sig-auth", "sig-api-machinery", "sig-apps" and "sig-node". [gated check-in]
+  "sig-storage", "sig-auth", "sig-api-machinery", "sig-apps" and "sig-node".
 * [network policy [gated check-in]](https://jenkins.antrea-ci.rocks/job/antrea-networkpolicy-for-pull-request/):
   community tests using sonobuoy, focusing on "Feature:NetworkPolicy".
+* [whole-conformance [daily]](https://jenkins.antrea-ci.rocks/job/antrea-whole-conformance-for-pull-request/):
+  community tests using sonobuoy, with certified-conformance mode.
+* [daily-whole-conformance](https://jenkins.antrea-ci.rocks/job/antrea-daily-whole-conformance-for-period/):
+  daily community tests using sonobuoy, with certified-conformance mode. If build fails, Jenkins will
+  send an email to projectantrea-dev@googlegroups.com for notification.
 * Microsoft Windows conformance: community tests related to Microsoft Windows.
   It focuses on: "[sig-network].+[Conformance]|[sig-windows]".
   It skips: "[LinuxOnly]|[Slow]|[Serial]|[Disruptive]|[Flaky]|[Feature:.+]|[sig-cli]|[sig-storage]|[sig-auth]|[sig-api-machinery]|[sig-apps]|[sig-node]|[Privileged]|should be able to change the type from|[sig-network] Services should be able to create a functioning NodePort service [Conformance]".
-* jenkins jobs validator [gated check-in]: this job only executes for PRs that include changes to
+* Jenkins jobs validator [gated check-in]: this job only executes for PRs that include changes to
   [ci/jenkins/jobs](/ci/jenkins/jobs). It validates the syntax of the jenkins jobs'
   configuration.
 * [EKS conformance/network policy [bi-daily]](http://jenkins.antrea-ci.rocks/view/cloud/job/cloud-antrea-eks-conformance-test-native/)

ci/jenkins/jobs/job-templates.yaml

@@ -127,6 +127,7 @@
         url: 'https://github.com/{org_repo}'
         wipe-workspace: true
         included-regions: '{included_regions}'
+    publishers: '{publishers}'
 
 - job-template:
     name: '{name}-{test_name}-for-gc'

ci/jenkins/jobs/macros.yaml

@@ -171,7 +171,7 @@
 
           rm -f COMMENT_EXIST
           rm -f body.json
-          echo "{{\"body\": \"Thanks for your PR.\\\nUnit tests and code linters are run automatically every time the PR is updated.\\\nE2e, conformance and network policy tests can only be triggered by a member of the vmware-tanzu organization. Regular contributors to the project should join the org.\\\n\\\nThe following commands are available:\\\n* \`/test-e2e\`: to trigger e2e tests.\\\n* \`/skip-e2e\`: to skip e2e tests.\\\n* \`/test-conformance\`: to trigger conformance tests.\\\n* \`/skip-conformance\`: to skip conformance tests.\\\n* \`/test-networkpolicy\`: to trigger networkpolicy tests.\\\n* \`/skip-networkpolicy\`: to skip networkpolicy tests.\\\n* \`/test-windows-conformance\`: to trigger windows conformance tests.\\\n* \`/skip-windows-conformance\`: to skip windows conformance tests.\\\n* \`/test-all\`: to trigger all tests.\\\n* \`/skip-all\`: to skip all tests.\\\n\\\nThese commands can only be run by members of the vmware-tanzu organization.\"}}" > body.json
+          echo "{{\"body\": \"Thanks for your PR.\\\nUnit tests and code linters are run automatically every time the PR is updated.\\\nE2e, conformance and network policy tests can only be triggered by a member of the vmware-tanzu organization. Regular contributors to the project should join the org.\\\n\\\nThe following commands are available:\\\n* \`/test-e2e\`: to trigger e2e tests.\\\n* \`/skip-e2e\`: to skip e2e tests.\\\n* \`/test-conformance\`: to trigger conformance tests.\\\n* \`/skip-conformance\`: to skip conformance tests.\\\n* \`/test-whole-conformance\`: to trigger all conformance tests on linux.\\\n* \`/test-networkpolicy\`: to trigger networkpolicy tests.\\\n* \`/skip-networkpolicy\`: to skip networkpolicy tests.\\\n* \`/test-windows-conformance\`: to trigger windows conformance tests.\\\n* \`/skip-windows-conformance\`: to skip windows conformance tests.\\\n* \`/test-all\`: to trigger all tests.\\\n* \`/skip-all\`: to skip all tests.\\\n\\\nThese commands can only be run by members of the vmware-tanzu organization.\"}}" > body.json
 
           B="$(cat body.json | jq .body)"
 
@@ -340,7 +340,11 @@
             echo  "=== Run sonobuoy with conformance image version ${{conformance_image_version}} ==="
 
             sonobuoy delete --wait --kubeconfig jenkins/out/kubeconfig
-            sonobuoy run --wait --timeout 1800 --e2e-focus '{focus_regex}' --e2e-skip '{skip_regex}' --e2e-parallel y --kube-conformance-image-version ${{conformance_image_version}} --kubeconfig jenkins/out/kubeconfig
+            if [ '{mode_regex}' == "" ]; then
+              sonobuoy run --wait --timeout 1800 --e2e-focus '{focus_regex}' --e2e-skip '{skip_regex}' --e2e-parallel y --kube-conformance-image-version ${{conformance_image_version}} --kubeconfig jenkins/out/kubeconfig
+            else
+              sonobuoy run --wait --timeout 7200 --mode='{mode_regex}' --e2e-parallel y --kube-conformance-image-version ${{conformance_image_version}} --kubeconfig jenkins/out/kubeconfig
+            fi
             sonobuoy retrieve --kubeconfig jenkins/out/kubeconfig
             echo '=== Print all results ==='
             sonobuoy results *sonobuoy*.tar.gz

ci/jenkins/jobs/projects.yaml

@@ -17,6 +17,27 @@
           - ci/jenkins/jobs/.*
           cron: 'H/15 * * * *'
           ignore_post_commit_hooks: false
+          publishers: []
+      - '{name}-{test_name}-for-period':
+          test_name: daily-whole-conformance
+          node: 'antrea-test-node'
+          description: 'This is for testing all conformance daily.'
+          builders:
+            - builder-workload-cluster-setup
+            - builder-prepare-antrea
+            - builder-conformance:
+                mode_regex: 'certified-conformance'
+                focus_regex: ''
+                skip_regex: ''
+            - builder-workload-cluster-cleanup
+          branches:
+          - '*/master'
+          included_regions: []
+          cron: 'H H * * *'
+          ignore_post_commit_hooks: false
+          publishers:
+          - email:
+              recipients: projectantrea-dev@googlegroups.com
       - '{name}-{test_name}-for-gc':
           test_name: workload-cluster
           node: 'antrea-test-node'
@@ -186,6 +207,7 @@
             - builder-workload-cluster-setup
             - builder-prepare-antrea
             - builder-conformance:
+                mode_regex: ''
                 focus_regex: '\[Conformance\]'
                 skip_regex: '\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[sig-cli\]|\[sig-storage\]|\[sig-auth\]|\[sig-api-machinery\]|\[sig-apps\]|\[sig-node\]'
             - builder-workload-cluster-cleanup
@@ -237,6 +259,88 @@
           started_status: null
           wrappers: []
           publishers: []
+      - '{name}-{test_name}-for-pull-request':
+          test_name: whole-conformance-pending-label
+          node: null
+          description: 'This is for marking PR as pending for whole conformance test.'
+          builders:
+            - builder-pending-label
+          trigger_phrase: null
+          allow_whitelist_orgs_as_admins: false
+          admin_list: []
+          org_list: []
+          white_list: []
+          only_trigger_phrase: false
+          trigger_permit_all: true
+          status_context: jenkins-whole-conformance
+          status_url: --none--
+          success_status: Pending test. Mark as failure. Not required for merging. Add comment /test-whole-conformance to trigger.
+          failure_status: Pending test. Mark as failure. Not required for merging. Add comment /test-whole-conformance to trigger.
+          error_status: Pending test. Mark as failure. Not required for merging. Add comment /test-whole-conformance to trigger.
+          triggered_status: null
+          started_status: null
+          wrappers: []
+          publishers: []
+      - '{name}-{test_name}-for-pull-request':
+          test_name: whole-conformance
+          node: 'antrea-test-node'
+          description: 'This is the {test_name} test for {name}.'
+          builders:
+            - builder-workload-cluster-setup
+            - builder-prepare-antrea
+            - builder-conformance:
+                mode_regex: 'certified-conformance'
+                focus_regex: ''
+                skip_regex: ''
+            - builder-workload-cluster-cleanup
+          trigger_phrase: .*/test-(whole-conformance|all).*
+          allow_whitelist_orgs_as_admins: true
+          admin_list: '{antrea_admin_list}'
+          org_list: '{antrea_org_list}'
+          white_list: '{antrea_white_list}'
+          only_trigger_phrase: true
+          trigger_permit_all: false
+          status_context: jenkins-whole-conformance
+          status_url: null
+          success_status: Build finished.
+          failure_status: Failed. Add comment /test-whole-conformance to re-trigger.
+          error_status: Failed. Add comment /test-whole-conformance to re-trigger.
+          triggered_status: null
+          started_status: null
+          wrappers:
+          - timeout:
+              fail: true
+              timeout: 120
+              type: absolute
+          publishers:
+          - archive:
+              allow-empty: true
+              artifacts: '*sonobuoy*.tar.gz'
+              case-sensitive: true
+              default-excludes: true
+              fingerprint: false
+              only-if-success: false
+      - '{name}-{test_name}-for-pull-request':
+          test_name: whole-conformance-skip
+          node: null
+          description: 'This is for marking PR as passed.'
+          builders: []
+          allow_whitelist_orgs_as_admins: true
+          admin_list: '{antrea_admin_list}'
+          org_list: '{antrea_org_list}'
+          white_list: '{antrea_white_list}'
+          only_trigger_phrase: true
+          trigger_permit_all: false
+          trigger_phrase: .*/skip-(whole-conformance|all).*
+          status_context: jenkins-whole-conformance
+          status_url: --none--
+          success_status: Skipped test. Mark as succeeded.
+          failure_status: Skipped test. Mark as succeeded.
+          error_status: Skipped test. Mark as succeeded.
+          triggered_status: null
+          started_status: null
+          wrappers: []
+          publishers: []
       - '{name}-{test_name}-for-pull-request':
           test_name: networkpolicy-pending-label
           node: null
@@ -267,6 +371,7 @@
             - builder-workload-cluster-setup
             - builder-prepare-antrea
             - builder-conformance:
+                mode_regex: ''
                 focus_regex: '\[Feature:NetworkPolicy\]'
                 skip_regex: 'SKIP_NO_TESTCASE'
             - builder-workload-cluster-cleanup