refactor(tests): Introduce loading shims, use in playgrounds (#7380)

* refactor(build): Simplify implementation of posixPath

* fix(closure): Make safe to import in node.js

  Make the implementation declareModlueId safe to import and
  execute in node.js, which does not provide a window global.

  (N.B. because this is an ESM and therefore automatically
  strict, using window?.goog?.declareModuleId doesn't work
  because window being undefined is an early error.)

* feat(tests): Introduce chunk loading shims

  - Add a buildShims task to build_tasks.js that, for each chunk,
    creates a correspondingly-named build/<chunk>.mjs that will
    either (in uncompressed mode) import and reexport that chunk's
    entry point module (e.g. core/blockly.js) or (in compressed
    mode) load dist/<chunk>_compressed.js using a <script> tag
    and then export the corresponding properties on the chunk's
    exports object.

  - Provide helper methods used by these shims in
    tests/scripts/loading.mjs, including code to detect whether
    to load in compressed or uncompressed mode.

  - Add a quote() function to scripts/helpers.js, used by
    buildShims.  This is copied from tests/bootstrap_helper.js,
    which will be removed in a later commit.

* refactor(tests): Update playground.html to use new loading shims

* refactor(tests): Update advanced_playground.html to use new loading shims

* refactor(tests): Update multi_playground.html to use new loading shims

* chore(tests): Delete playgrounds/shared_procedures.html

  Shared procedure support was moved to a plugin and this should
  have been removed from core along with it.

* docs(tests): Typo corrections.

* chore(tests): Add ".loader" infix to shim filenames.

  Per suggestion on PR #7380, have buildShims name the shims
  ${chunk.name}.loader.mjs instead of just `${chunk.name}.mjs`.

closure/goog/goog.js

@@ -72,9 +72,9 @@ export const global = globalThis;
 // export const scope = goog.scope;
 // export const defineClass = goog.defineClass;
 export const declareModuleId = function(namespace) {
- if (window.goog && window.goog.declareModuleId) {
-  window.goog.declareModuleId.call(this, namespace);
- }
+ // Use globalThis instead of window to find goog, so this can be
+ // imported in node.js (e.g. when running buildShims gulp task).
+ globalThis?.goog?.declareModuleId.call(this, namespace);
 };
 
 

scripts/gulpfiles/build_tasks.js

@@ -15,6 +15,7 @@ gulp.sourcemaps = require('gulp-sourcemaps');
 
 const path = require('path');
 const fs = require('fs');
+const fsPromises = require('fs/promises');
 const {exec, execSync} = require('child_process');
 
 const closureCompiler = require('google-closure-compiler').gulp();
@@ -24,7 +25,7 @@ const {rimraf} = require('rimraf');
 const {BUILD_DIR, DEPS_FILE, RELEASE_DIR, TEST_DEPS_FILE, TSC_OUTPUT_DIR, TYPINGS_BUILD_DIR} = require('./config');
 const {getPackageJson} = require('./helper_tasks');
 
-const {posixPath} = require('../helpers');
+const {posixPath, quote} = require('../helpers');
 
 ////////////////////////////////////////////////////////////
 // Build //
@@ -106,6 +107,7 @@ const chunks = [
  {
  name: 'blockly',
  entry: path.join(TSC_OUTPUT_DIR, 'core', 'main.js'),
+ moduleEntry: path.join(TSC_OUTPUT_DIR, 'core', 'blockly.js'),
  exports: 'module$build$src$core$blockly',
  scriptExport: 'Blockly',
  },
@@ -672,6 +674,57 @@ function buildCompiled() {
  .pipe(gulp.dest(RELEASE_DIR));
 }
 
+/**
+ * This task builds the shims used by the playgrounds and tests to
+ * load Blockly in either compressed or uncompressed mode, creating
+ * build/blockly.loader.mjs, blocks.loader.mjs, javascript.loader.mjs,
+ * etc.
+ *
+ * Prerequisite: getChunkOptions (via buildCompiled, for chunks[].parent).
+ */
+async function buildShims() {
+ // Install a package.json file in BUILD_DIR to tell node.js that the
+ // .js files therein are ESM not CJS, so we can import the
+ // entrypoints to enumerate their exported names.
+ //
+ // N.B.: There is an exception: core/main.js is a goog.module not
+ // ESM, but fortunately we don't attempt to import or require this
+ // file from node.js - we only feed it to Closure Compiler, which
+ // uses the type information in deps.js rather than package.json.
+ await fsPromises.writeFile(
+ path.join(BUILD_DIR, 'package.json'),
+ '{"type": "module"}'
+ );
+
+ // Import each entrypoint module, enumerate its exports, and write
+ // a shim to load the chunk either by importing the entrypoint
+ // module or by loading the compiled script.
+ await Promise.all(chunks.map(async (chunk) => {
+ const modulePath = posixPath(chunk.moduleEntry ?? chunk.entry);
+ const scriptPath =
+ path.posix.join(RELEASE_DIR, `${chunk.name}${COMPILED_SUFFIX}.js`);
+ const shimPath = path.join(BUILD_DIR, `${chunk.name}.loader.mjs`);
+ const parentImport =
+ chunk.parent ? `import ${quote(`./${chunk.parent.name}.mjs`)};` : '';
+ const exports = await import(`../../${modulePath}`);
+
+ await fsPromises.writeFile(shimPath,
+ `import {loadChunk} from '../tests/scripts/load.mjs';
+${parentImport}
+
+export const {
+${Object.keys(exports).map((name) => ` ${name},`).join('\n')}
+} = await loadChunk(
+ ${quote(modulePath)},
+ ${quote(scriptPath)},
+ ${quote(chunk.scriptExport)},
+);
+`);
+ }));
+}
+
+
+
 /**
  * This task builds Blockly core, blocks and generators together and uses
  * Closure Compiler's ADVANCED_COMPILATION mode.
@@ -728,7 +781,7 @@ exports.cleanBuildDir = cleanBuildDir;
 exports.langfiles = buildLangfiles; // Build build/msg/*.js from msg/json/*.
 exports.tsc = buildJavaScript;
 exports.deps = gulp.series(exports.tsc, buildDeps);
-exports.minify = gulp.series(exports.deps, buildCompiled);
+exports.minify = gulp.series(exports.deps, buildCompiled, buildShims);
 exports.build = gulp.parallel(exports.minify, exports.langfiles);
 
 // Manually-invokable targets, with prerequisites where required.

scripts/helpers.js

@@ -11,25 +11,64 @@
 
 const path = require('path');
 
-/**
- * Escape regular expression pattern
- * @param {string} pattern regular expression pattern
- * @return {string} escaped regular expression pattern
- */
-function escapeRegex(pattern) {
- return pattern.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
-}
-
 /**
  * Replaces OS-specific path with POSIX style path.
+ * Simplified implementation based on
+ * https://stackoverflow.com/a/63251716/4969945
+ *
  * @param {string} target target path
  * @return {string} posix path
  */
 function posixPath(target) {
- const osSpecificSep = new RegExp(escapeRegex(path.sep), 'g');
- return target.replace(osSpecificSep, path.posix.sep);
+ return target.split(path.sep).join(path.posix.sep);
+}
+
+/**
+ * Convert a string into a string literal. Strictly speaking we
+ * only need to escape backslash, \r, \n, \u2028 (line separator),
+ * \u2029 (paragraph separator) and whichever quote character we're
+ * using, but for simplicity we escape all the control characters.
+ *
+ * Based on https://github.com/google/CodeCity/blob/master/server/code.js
+ *
+ * @param {string} str The string to convert.
+ * @return {string} The value s as a eval-able string literal.
+ */
+function quote(str) {
+ /* eslint-disable no-control-regex, no-multi-spaces */
+ /** Regexp for characters to be escaped in a single-quoted string. */
+ const singleRE = /[\x00-\x1f\\\u2028\u2029']/g;
+
+ /** Map of control character replacements. */
+ const replacements = {
+ '\x00': '\\0',
+ '\x01': '\\x01',
+ '\x02': '\\x02',
+ '\x03': '\\x03',
+ '\x04': '\\x04',
+ '\x05': '\\x05',
+ '\x06': '\\x06',
+ '\x07': '\\x07',
+ '\x08': '\\b',
+ '\x09': '\\t',
+ '\x0a': '\\n',
+ '\x0b': '\\v',
+ '\x0c': '\\f',
+ '\x0d': '\\r',
+ '\x0e': '\\x0e',
+ '\x0f': '\\x0f',
+ '"': '\\"',
+ "'": "\\'",
+ '\\': '\\\\',
+ '\u2028': '\\u2028',
+ '\u2029': '\\u2029',
+ };
+ /* eslint-enable no-control-regex, no-multi-spaces */
+
+ return "'" + str.replace(singleRE, (c) => replacements[c]) + "'";
 }
 
 module.exports = {
  posixPath,
+ quote,
 };

tests/multi_playground.html

@@ -4,14 +4,13 @@
  <meta charset="utf-8" />
  <title>Multi-toolbox Playground</title>
 
- <!-- This script loads uncompressed when on localhost and loads
- compressed when it is being hosted or on Internet Explorer. -->
- <script src="bootstrap.js"></script>
  <script type="module">
- // Wait for Blockly to finish loading.
- import './bootstrap_done.mjs';
+ import {COMPRESSED, loadScript} from './scripts/load.mjs';
 
- const IS_UNCOMPRESSED = !window.bootstrapInfo.compressed; // See bootstrap.js
+ import * as Blockly from '../build/blockly.loader.mjs';
+ import '../build/blocks.loader.mjs';
+
+ await loadScript('../build/msg/en.js');
 
  var options = {
  comments: true,
@@ -87,7 +86,7 @@
 
  function setBackgroundColour() {
  // Set background colour to differentiate between compressed and uncompressed mode.
- if (IS_UNCOMPRESSED) {
+ if (!COMPRESSED) {
  document.body.style.backgroundColor = '#d6d6ff'; // Familiar lilac.
  } else {
  document.body.style.backgroundColor = '#60fcfc'; // Unfamiliar blue.

tests/playground.html

@@ -4,23 +4,21 @@
  <meta charset="utf-8" />
  <title>Blockly Playground</title>
 
- <!-- This script loads uncompressed when on localhost and loads
- compressed when it is being hosted or on Internet Explorer. -->
- <script>
- var BLOCKLY_BOOTSTRAP_OPTIONS = {
- scripts: [
- 'build/msg/en.js',
- 'tests/playgrounds/screenshot.js',
- 'node_modules/@blockly/dev-tools/dist/index.js',
- ],
- };
- </script>
- <script src="bootstrap.js"></script>
  <script type="module">
- // Wait for Blockly to finish loading.
- import './bootstrap_done.mjs';
+ import {COMPRESSED, loadScript} from './scripts/load.mjs';
+
+ import * as Blockly from '../build/blockly.loader.mjs';
+ import '../build/blocks.loader.mjs';
+ import {dartGenerator} from '../build/dart.loader.mjs';
+ import {luaGenerator} from '../build/lua.loader.mjs';
+ import {javascriptGenerator} from '../build/javascript.loader.mjs';
+ import {phpGenerator} from '../build/php.loader.mjs';
+ import {pythonGenerator} from '../build/python.loader.mjs';
+
+ await loadScript('../build/msg/en.js');
+ await loadScript('playgrounds/screenshot.js');
+ await loadScript('../node_modules/@blockly/dev-tools/dist/index.js');
 
- const IS_UNCOMPRESSED = !window.bootstrapInfo.compressed; // See bootstrap.js
  var workspace = null;
 
  function start() {
@@ -96,9 +94,12 @@
  addEventHandlers();
  }
 
+ /**
+ * Set background colour to differentiate between compressed and
+ * uncompressed mode.
+ */
  function setBackgroundColour() {
- // Set background colour to differentiate between compressed and uncompressed mode.
- if (IS_UNCOMPRESSED) {
+ if (!COMPRESSED) {
  document.body.style.backgroundColor = '#d6d6ff'; // Familiar lilac.
  } else {
  document.body.style.backgroundColor = '#60fcfc'; // Unfamiliar blue.

tests/playgrounds/advanced_playground.html

@@ -4,22 +4,20 @@
  <meta charset="utf-8" />
  <title>Advanced Blockly Playground</title>
 
- <script>
- var BLOCKLY_BOOTSTRAP_OPTIONS = {
- scripts: [
- 'build/msg/en.js',
- 'tests/playgrounds/screenshot.js',
- 'tests/themes/test_themes.js',
- 'node_modules/@blockly/dev-tools/dist/index.js',
- 'node_modules/@blockly/theme-modern/dist/index.js',
- ],
- };
- </script>
- <script src="../bootstrap.js"></script>
-
  <script type="module">
- // Wait for Blockly to finish loading.
- import '../bootstrap_done.mjs';
+ import {loadScript} from '../scripts/load.mjs';
+
+ import * as Blockly from '../../build/blockly.loader.mjs';
+ import '../../build/blocks.loader.mjs';
+ // Generators not needed (but see also bug #6597).
+
+ await loadScript('../../build/msg/en.js');
+ await loadScript('screenshot.js');
+ await loadScript('../themes/test_themes.js');
+ await loadScript('../../node_modules/@blockly/dev-tools/dist/index.js');
+ await loadScript(
+ '../../node_modules/@blockly/theme-modern/dist/index.js',
+ );
 
  function start() {
  setBackgroundColour();