E2E Tests: Add Scheduled job for a Pre-Release Gutenberg block tests

.github/workflows/e2e-tests.yml

@@ -8,16 +8,37 @@ on:
     branches: [master]
     paths-ignore:
     - '**.md'
+  schedule:
+    - cron:  '0 */12 * * *'
 
 jobs:
+  create-matrix:
+    name: "Determine tests matrix"
+    runs-on: ubuntu-latest
+    timeout-minutes: 1  # 2021-02-03: Should only take a second.
+    outputs:
+      matrix: ${{ steps.create-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+      - id: create-matrix
+        run: |
+          MATRIX='[{"group":"pre-connection" },{"group":"connection" },{"group":"post-connection" }]'
+          if [ ${{ github.event_name }} == schedule ]; then
+            MATRIX=$(echo $MATRIX | jq '. + [{"group": "gutenberg"}]')
+          fi
+          echo $MATRIX
+          echo "::set-output name=matrix::$MATRIX"
+
+
   e2e-tests:
     name: "E2E ${{ matrix.group }} tests"
     runs-on: ubuntu-latest
+    needs: create-matrix
     timeout-minutes: 25  # 2021-04-16: Successful runs seem to take 11-17 minutes
     strategy:
       fail-fast: false
       matrix:
-        group: [ 'pre-connection', 'connection', 'post-connection', 'gutenberg' ]
+        include: ${{ fromJson( needs.create-matrix.outputs.matrix ) }}
     env:
       GUTENBERG: bundle # default value. Can be overridden later based on matrix.group value
     if: github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name
@@ -85,21 +106,25 @@ jobs:
         yarn install
         yarn jetpack build plugins/jetpack -v --production
 
-
-    - name: Set Gutenberg version
-      run: if [ ${{ matrix.group }} == gutenberg ]; then echo "GUTENBERG=latest" >> $GITHUB_ENV; fi
-
     - name: Environment set-up
       working-directory: projects/plugins/jetpack/tests/e2e
       env:
         CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }}
       run: |
         yarn install
         yarn test-decrypt-config
-        echo "Gutenberg: $GUTENBERG"
         yarn env-start
         yarn tunnel-on
 
+    - name: Set up Gutenberg
+      if: matrix.group == 'gutenberg'
+      working-directory: projects/plugins/jetpack/tests/e2e
+      run: |
+        export GUTENBERG=latest
+        echo "$GUTENBERG" >> $GITHUB_ENV;
+        ./bin/env.sh config-gb
+
+
     - name: Run ${{ matrix.group }} tests
       working-directory: projects/plugins/jetpack/tests/e2e
       run: yarn test-e2e --group=${{ matrix.group }}

projects/plugins/jetpack/changelog/add-e2e-scheduled-pre-release-gb-job

@@ -0,0 +1,5 @@
+Significance: patch
+Type: enhancement
+Comment: E2E releated changes
+
+

projects/plugins/jetpack/tests/e2e/bin/container-setup.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Exit if any command fails.
+set -eo pipefail
+
+#####
+# This script is designed to be running inside the WP container
+# it is basically a hacky way to insert arbitrary PHP code into wp-config without messing with bash escaping etc.
+# Also, it creates a debug log file
+
+function usage {
+	echo "usage: $0 command"
+	echo "  wp-setup                     Setup wp-config"
+	echo "  gb-setup                     Set up Gutenberg plugin"
+	echo "  -h | usage                   Output this message"
+	exit 1
+}
+
+function wp_config {
+	touch wp-content/debug.log
+	chown www-data:www-data wp-content/debug.log
+	chmod 755 wp-content/debug.log
+
+	# Remove default config entries
+	sed -i '/WP_SITEURL/d' wp-config.php
+	sed -i '/WP_HOME/d' wp-config.php
+	sed -i '/WP_TESTS_DOMAIN/d' wp-config.php
+	sed -i '/E2E_REQUEST_URL/d' wp-config.php
+
+	sed -i "/\/\* That's all, stop editing! Happy publishing. \*\//i\
+	define( 'E2E_REQUEST_URL', ( ! empty( \\\$_SERVER['HTTPS'] ) ? 'https://' : 'http://' ) . ( ! empty( \\\$_SERVER['HTTP_HOST'] ) ? \\\$_SERVER['HTTP_HOST'] : 'localhost' ) );\n\
+	define( 'WP_SITEURL', E2E_REQUEST_URL );\n\
+	define( 'WP_HOME', E2E_REQUEST_URL );\n\
+	" wp-config.php
+}
+
+function gb_setup {
+	ZIP_PATH=${1}
+	GB_URL=$(curl -s https://api.github.com/repos/Wordpress/gutenberg/releases/latest | grep browser_download_url | cut -d '"' -f 4)
+
+	rm -rf $ZIP_PATH wp-content/plugins/gutenberg
+	curl -L $GB_URL --output $ZIP_PATH
+	echo "Latest pre-release Gutenberg successfuly downloaded in $ZIP_PATH"
+}
+
+if [ "${1}" == "wp-config" ]; then
+	wp_config
+elif [ "${1}" == "gb-setup" ]; then
+	gb_setup ${2}
+elif [ "${1}" == "usage" ]; then
+	usage
+else
+	usage
+fi

projects/plugins/jetpack/tests/e2e/bin/env.sh

@@ -7,6 +7,7 @@ function usage {
 	echo "usage: $0 command"
 	echo "  start                        Setup the docker containers for E2E tests"
 	echo "  reset                        Reset the containers state"
+	echo "  gb-setup                     Setup Gutenberg plugin"
 	echo "  -h | usage                   Output this message"
 	exit 1
 }
@@ -21,14 +22,15 @@ reset_env() {
 	configure_wp_env
 }
 
-configure_wp_env() {
-	yarn wp-env run tests-wordpress sh wp-content/plugins/jetpack-dev/tests/e2e/bin/wp-setup.sh
-
-	if [ "$GUTENBERG" == "latest" ]; then
-		echo "Installing latest Gutenberg"
-		yarn wp-env run tests-cli wp plugin install gutenberg --activate
-	fi
+gb_setup() {
+	GB_ZIP="wp-content/gutenberg.zip"
+	yarn wp-env run tests-wordpress "./wp-content/plugins/jetpack-dev/tests/e2e/bin/container-setup.sh gb-setup $GB_ZIP"
+	yarn wp-env run tests-cli "wp plugin install $GB_ZIP"
+	yarn wp-env run tests-cli "wp plugin activate gutenberg"
+}
 
+configure_wp_env() {
+	yarn wp-env run tests-wordpress ./wp-content/plugins/jetpack-dev/tests/e2e/bin/container-setup.sh wp-config
 	yarn wp-env run tests-cli wp plugin activate jetpack-dev
 
 	echo
@@ -40,6 +42,8 @@ if [ "${1}" == "start" ]; then
 	start_env
 elif [ "${1}" == "reset" ]; then
 	reset_env
+elif [ "${1}" == "gb-setup" ]; then
+	gb_setup
 elif [ "${1}" == "usage" ]; then
 	usage
 else

projects/plugins/jetpack/tests/e2e/lib/reporters/slack.js

@@ -21,7 +21,7 @@ export default class SlackReporter {
 		} );
 		this.runURL = `${ repoUrl }/actions/runs/${ GITHUB_RUN_ID }`;
 		this.runType =
-			GUTENBERG === 'latest' ? 'with latest :gutenberg: plugin' : 'with no :gutenberg: plugin';
+			GUTENBERG === 'latest' ? 'with latest :gutenberg: plugin' : 'with bundled :gutenberg:';
 
 		this.mentions = mentions
 			.map( function ( userId ) {