From baf7223c3ee0ea3257e33ed15217cdd48fc460a9 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 1 Nov 2023 14:17:54 +0200 Subject: [PATCH 01/30] Add depCache functionality to the builder --- lib/lbt/bundle/Builder.js | 51 ++++++++++++++++++++++++++++++ lib/lbt/bundle/BundleDefinition.js | 10 +++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index cd4e28c8e..3fbe8a15b 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -71,6 +71,14 @@ const EVOBundleFormat = { resolvedModule.executes(MODULE__UI5LOADER_AUTOCONFIG) || resolvedModule.executes(MODULE__JQUERY_SAP_GLOBAL) || resolvedModule.executes(MODULE__SAP_UI_CORE_CORE); + }, + + beforeDepCache(outW) { + outW.writeln(`sap.ui.loader.config({depCacheUI5:{`); + }, + + afterDepCache(outW) { + outW.writeln(`}});`); } }; @@ -220,6 +228,8 @@ class BundleBuilder { return this.writeBundleInfos([section]); case SectionType.Require: return this.writeRequires(section); + case SectionType.DepCache: + return this.writeDepCache(section); default: throw new Error("unknown section mode " + section.mode); } @@ -549,6 +559,47 @@ class BundleBuilder { }); } + async writeDepCache(section) { + const outW = this.outW; + + outW.ensureNewLine(); + + const sequence = section.modules.slice().sort(); + + if (sequence.length > 0) { + this.targetBundleFormat.beforeDepCache(outW, section); + let i = 0; + for (const module of sequence) { + const resource = await this.pool.findResourceWithInfo(module); + if (resource != null) { + const deps = resource.info.dependencies.filter( + (dep) => + !resource.info.isConditionalDependency(dep) && + !resource.info.isImplicitDependency(dep) + ); + if (deps.length > 0) { + if (i > 0) { + outW.writeln(","); + } + outW.write( + `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}]` + ); + i++; + } else { + log.verbose(" skipped %s, no dependencies", module); + } + } else { + log.error(" couldn't find %s", module); + } + } + + if (i > 0) { + outW.writeln(); + } + this.targetBundleFormat.afterDepCache(outW, section); + } + } + async getSourceMapForModule({moduleName, moduleContent, resourcePath}) { let moduleSourceMap = null; let newModuleContent = moduleContent; diff --git a/lib/lbt/bundle/BundleDefinition.js b/lib/lbt/bundle/BundleDefinition.js index d4716b8b7..42b3a3ff4 100644 --- a/lib/lbt/bundle/BundleDefinition.js +++ b/lib/lbt/bundle/BundleDefinition.js @@ -27,5 +27,13 @@ export const SectionType = { * Usually used as the last section in a merged module to enforce loading and * execution of some specific module or modules. */ - Require: "require" + Require: "require", + + /** + * Dependency cache information that lists modules and their dependencies + * of all types: JS, declarative views/fragments. + * Only the dependencies of the modules are stored as 'depCache' configuration. + * Requires UI5 Evolution runtime. + */ + DepCache: "depCache" }; From 7d4a7fa8e938f5798ed70ec88cd779a197c9f0c4 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 1 Nov 2023 14:19:16 +0200 Subject: [PATCH 02/30] depCache for ComponentPreload task --- lib/tasks/bundlers/generateComponentPreload.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tasks/bundlers/generateComponentPreload.js b/lib/tasks/bundlers/generateComponentPreload.js index 68af2c71c..c4d689109 100644 --- a/lib/tasks/bundlers/generateComponentPreload.js +++ b/lib/tasks/bundlers/generateComponentPreload.js @@ -131,6 +131,13 @@ export default async function({ resolve: false, resolveConditional: false, renderer: false + }, + { + mode: "depCache", + filters: filters, + resolve: false, + resolveConditional: false, + renderer: false } ] }; From b451870337536f1a6c8b7b1fad99e0619d7a5877 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 1 Nov 2023 14:19:27 +0200 Subject: [PATCH 03/30] depCache for LibraryPreload task --- lib/tasks/bundlers/generateLibraryPreload.js | 126 +++++++++++-------- 1 file changed, 73 insertions(+), 53 deletions(-) diff --git a/lib/tasks/bundlers/generateLibraryPreload.js b/lib/tasks/bundlers/generateLibraryPreload.js index 6275c5e47..dcc8476f6 100644 --- a/lib/tasks/bundlers/generateLibraryPreload.js +++ b/lib/tasks/bundlers/generateLibraryPreload.js @@ -31,11 +31,63 @@ function getDefaultLibraryPreloadFilters(namespace, excludes) { return filters; } -function getBundleDefinition(namespace, excludes) { +function getBundleDefinition(namespace, excludes, preloadMode = "preload") { + const preloadBundleSuffix = preloadMode === "preload" ? "preload" : "depcache-preload"; + let preloadDef; + let depCachePreloadDef; // TODO: move to config of actual core project if (namespace === "sap/ui/core") { + preloadDef = { + mode: "preload", + filters: [ + // Note: Don't pass configured preload excludes for sap.ui.core + // as they are already hardcoded below. + // In future the sap/ui/core/library-preload should be configured + // as a custom bundle in the ui5.yaml. + ...getDefaultLibraryPreloadFilters(namespace), + + `!${namespace}/cldr/`, + "*.js", + "sap/base/", + "sap/ui/base/", + "sap/ui/dom/", + "sap/ui/events/", + "sap/ui/model/", + "sap/ui/security/", + "sap/ui/util/", + "sap/ui/Global.js", + + // include only thirdparty that is very likely to be used + "sap/ui/thirdparty/crossroads.js", + "sap/ui/thirdparty/caja-html-sanitizer.js", + "sap/ui/thirdparty/hasher.js", + "sap/ui/thirdparty/signals.js", + "sap/ui/thirdparty/jquery-mobile-custom.js", + "sap/ui/thirdparty/jqueryui/jquery-ui-core.js", + "sap/ui/thirdparty/jqueryui/jquery-ui-position.js", + + // other excludes (not required for productive scenarios) + "!sap-ui-*.js", + "!sap/ui/core/support/", + "!sap/ui/core/plugin/DeclarativeSupport.js", + "!sap/ui/core/plugin/LessSupport.js" + + ], + resolve: false, + resolveConditional: false, + renderer: true + }; + + // TODO: Adjust filters + if (preloadMode === "depCache") { + depCachePreloadDef = { + ...preloadDef, + ...{mode: "depCache"} + }; + } + return { - name: `${namespace}/library-preload.js`, + name: `${namespace}/library-${preloadBundleSuffix}.js`, sections: [ { // exclude the content of sap-ui-core by declaring it as 'provided' @@ -46,60 +98,28 @@ function getBundleDefinition(namespace, excludes) { ], resolve: true }, - { - mode: "preload", - filters: [ - // Note: Don't pass configured preload excludes for sap.ui.core - // as they are already hardcoded below. - // In future the sap/ui/core/library-preload should be configured - // as a custom bundle in the ui5.yaml. - ...getDefaultLibraryPreloadFilters(namespace), - - `!${namespace}/cldr/`, - "*.js", - "sap/base/", - "sap/ui/base/", - "sap/ui/dom/", - "sap/ui/events/", - "sap/ui/model/", - "sap/ui/security/", - "sap/ui/util/", - "sap/ui/Global.js", - - // include only thirdparty that is very likely to be used - "sap/ui/thirdparty/crossroads.js", - "sap/ui/thirdparty/caja-html-sanitizer.js", - "sap/ui/thirdparty/hasher.js", - "sap/ui/thirdparty/signals.js", - "sap/ui/thirdparty/jquery-mobile-custom.js", - "sap/ui/thirdparty/jqueryui/jquery-ui-core.js", - "sap/ui/thirdparty/jqueryui/jquery-ui-position.js", - - // other excludes (not required for productive scenarios) - "!sap-ui-*.js", - "!sap/ui/core/support/", - "!sap/ui/core/plugin/DeclarativeSupport.js", - "!sap/ui/core/plugin/LessSupport.js" - - ], - resolve: false, - resolveConditional: false, - renderer: true - } + preloadDef, + ...[depCachePreloadDef] ] }; } + + preloadDef = { + mode: "preload", + filters: getDefaultLibraryPreloadFilters(namespace, excludes), + resolve: false, + resolveConditional: false, + renderer: true + }; + + // TODO: Adjust filters + if (preloadMode === "depCache") { + depCachePreloadDef = {...preloadDef, ...{mode: "depCache"}}; + } + return { - name: `${namespace}/library-preload.js`, - sections: [ - { - mode: "preload", - filters: getDefaultLibraryPreloadFilters(namespace, excludes), - resolve: false, - resolveConditional: false, - renderer: true - } - ] + name: `${namespace}/library-${preloadBundleSuffix}.js`, + sections: [preloadDef, ...[depCachePreloadDef]], }; } @@ -385,7 +405,7 @@ export default async function({workspace, taskUtil, options: {skipBundles = [], const results = await Promise.all([ execModuleBundlerIfNeeded({ options: { - bundleDefinition: getBundleDefinition(libraryNamespace, excludes), + bundleDefinition: getBundleDefinition(libraryNamespace, excludes, "depCache"), bundleOptions: { optimize: true, usePredefineCalls: true, From ba079779fbb0c90055173567dc935469989adf90 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 3 Nov 2023 09:10:04 +0200 Subject: [PATCH 04/30] Do not modify preloads for lib & comp default tasks --- .../bundlers/generateComponentPreload.js | 7 - lib/tasks/bundlers/generateLibraryPreload.js | 126 ++++++++---------- 2 files changed, 53 insertions(+), 80 deletions(-) diff --git a/lib/tasks/bundlers/generateComponentPreload.js b/lib/tasks/bundlers/generateComponentPreload.js index c4d689109..68af2c71c 100644 --- a/lib/tasks/bundlers/generateComponentPreload.js +++ b/lib/tasks/bundlers/generateComponentPreload.js @@ -131,13 +131,6 @@ export default async function({ resolve: false, resolveConditional: false, renderer: false - }, - { - mode: "depCache", - filters: filters, - resolve: false, - resolveConditional: false, - renderer: false } ] }; diff --git a/lib/tasks/bundlers/generateLibraryPreload.js b/lib/tasks/bundlers/generateLibraryPreload.js index dcc8476f6..6275c5e47 100644 --- a/lib/tasks/bundlers/generateLibraryPreload.js +++ b/lib/tasks/bundlers/generateLibraryPreload.js @@ -31,63 +31,11 @@ function getDefaultLibraryPreloadFilters(namespace, excludes) { return filters; } -function getBundleDefinition(namespace, excludes, preloadMode = "preload") { - const preloadBundleSuffix = preloadMode === "preload" ? "preload" : "depcache-preload"; - let preloadDef; - let depCachePreloadDef; +function getBundleDefinition(namespace, excludes) { // TODO: move to config of actual core project if (namespace === "sap/ui/core") { - preloadDef = { - mode: "preload", - filters: [ - // Note: Don't pass configured preload excludes for sap.ui.core - // as they are already hardcoded below. - // In future the sap/ui/core/library-preload should be configured - // as a custom bundle in the ui5.yaml. - ...getDefaultLibraryPreloadFilters(namespace), - - `!${namespace}/cldr/`, - "*.js", - "sap/base/", - "sap/ui/base/", - "sap/ui/dom/", - "sap/ui/events/", - "sap/ui/model/", - "sap/ui/security/", - "sap/ui/util/", - "sap/ui/Global.js", - - // include only thirdparty that is very likely to be used - "sap/ui/thirdparty/crossroads.js", - "sap/ui/thirdparty/caja-html-sanitizer.js", - "sap/ui/thirdparty/hasher.js", - "sap/ui/thirdparty/signals.js", - "sap/ui/thirdparty/jquery-mobile-custom.js", - "sap/ui/thirdparty/jqueryui/jquery-ui-core.js", - "sap/ui/thirdparty/jqueryui/jquery-ui-position.js", - - // other excludes (not required for productive scenarios) - "!sap-ui-*.js", - "!sap/ui/core/support/", - "!sap/ui/core/plugin/DeclarativeSupport.js", - "!sap/ui/core/plugin/LessSupport.js" - - ], - resolve: false, - resolveConditional: false, - renderer: true - }; - - // TODO: Adjust filters - if (preloadMode === "depCache") { - depCachePreloadDef = { - ...preloadDef, - ...{mode: "depCache"} - }; - } - return { - name: `${namespace}/library-${preloadBundleSuffix}.js`, + name: `${namespace}/library-preload.js`, sections: [ { // exclude the content of sap-ui-core by declaring it as 'provided' @@ -98,28 +46,60 @@ function getBundleDefinition(namespace, excludes, preloadMode = "preload") { ], resolve: true }, - preloadDef, - ...[depCachePreloadDef] + { + mode: "preload", + filters: [ + // Note: Don't pass configured preload excludes for sap.ui.core + // as they are already hardcoded below. + // In future the sap/ui/core/library-preload should be configured + // as a custom bundle in the ui5.yaml. + ...getDefaultLibraryPreloadFilters(namespace), + + `!${namespace}/cldr/`, + "*.js", + "sap/base/", + "sap/ui/base/", + "sap/ui/dom/", + "sap/ui/events/", + "sap/ui/model/", + "sap/ui/security/", + "sap/ui/util/", + "sap/ui/Global.js", + + // include only thirdparty that is very likely to be used + "sap/ui/thirdparty/crossroads.js", + "sap/ui/thirdparty/caja-html-sanitizer.js", + "sap/ui/thirdparty/hasher.js", + "sap/ui/thirdparty/signals.js", + "sap/ui/thirdparty/jquery-mobile-custom.js", + "sap/ui/thirdparty/jqueryui/jquery-ui-core.js", + "sap/ui/thirdparty/jqueryui/jquery-ui-position.js", + + // other excludes (not required for productive scenarios) + "!sap-ui-*.js", + "!sap/ui/core/support/", + "!sap/ui/core/plugin/DeclarativeSupport.js", + "!sap/ui/core/plugin/LessSupport.js" + + ], + resolve: false, + resolveConditional: false, + renderer: true + } ] }; } - - preloadDef = { - mode: "preload", - filters: getDefaultLibraryPreloadFilters(namespace, excludes), - resolve: false, - resolveConditional: false, - renderer: true - }; - - // TODO: Adjust filters - if (preloadMode === "depCache") { - depCachePreloadDef = {...preloadDef, ...{mode: "depCache"}}; - } - return { - name: `${namespace}/library-${preloadBundleSuffix}.js`, - sections: [preloadDef, ...[depCachePreloadDef]], + name: `${namespace}/library-preload.js`, + sections: [ + { + mode: "preload", + filters: getDefaultLibraryPreloadFilters(namespace, excludes), + resolve: false, + resolveConditional: false, + renderer: true + } + ] }; } @@ -405,7 +385,7 @@ export default async function({workspace, taskUtil, options: {skipBundles = [], const results = await Promise.all([ execModuleBundlerIfNeeded({ options: { - bundleDefinition: getBundleDefinition(libraryNamespace, excludes, "depCache"), + bundleDefinition: getBundleDefinition(libraryNamespace, excludes), bundleOptions: { optimize: true, usePredefineCalls: true, From cbb22b41bbdb53f6813dc6a5bbf4b93ae804e158 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 3 Nov 2023 11:43:16 +0200 Subject: [PATCH 05/30] (Re-)include filtered files for depCache mode --- lib/lbt/bundle/Resolver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lbt/bundle/Resolver.js b/lib/lbt/bundle/Resolver.js index e14c40c7e..45c3e7f32 100644 --- a/lib/lbt/bundle/Resolver.js +++ b/lib/lbt/bundle/Resolver.js @@ -198,7 +198,7 @@ class BundleResolver { let oldIgnoredResources; let oldSelectedResourcesSequence; - if ( section.mode == SectionType.Require ) { + if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) { oldSelectedResources = selectedResources; oldIgnoredResources = visitedResources; oldSelectedResourcesSequence = selectedResourcesSequence; @@ -254,7 +254,7 @@ class BundleResolver { }); return Promise.all(promises).then( function() { - if ( section.mode == SectionType.Require ) { + if ( [SectionType.Require, SectionType.DepCache].includes(section.mode) ) { newKeys = selectedResourcesSequence; selectedResources = oldSelectedResources; visitedResources = oldIgnoredResources; From 31a88aef4f46bfcbb5d6bbeec79dced4735fa131 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 3 Nov 2023 12:21:01 +0200 Subject: [PATCH 06/30] Do not duplicate bundleInfo's subModules --- lib/lbt/bundle/ResolvedBundleDefinition.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lbt/bundle/ResolvedBundleDefinition.js b/lib/lbt/bundle/ResolvedBundleDefinition.js index 013b433a7..e8cf0353a 100644 --- a/lib/lbt/bundle/ResolvedBundleDefinition.js +++ b/lib/lbt/bundle/ResolvedBundleDefinition.js @@ -64,7 +64,11 @@ class ResolvedBundleDefinition { return Promise.all( modules.map( (submodule) => { return pool.getModuleInfo(submodule).then( - (subinfo) => bundleInfo.addSubModule(subinfo) + (subinfo) => { + if (!bundleInfo.subModules.includes(subinfo.name)) { + bundleInfo.addSubModule(subinfo); + } + } ); }) ); From e3d461520774c7b738767800dd8155b24e81a474 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 3 Nov 2023 15:00:22 +0200 Subject: [PATCH 07/30] Add tests --- test/lib/lbt/bundle/Builder.js | 98 ++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index 8c24e4ceb..c6b01f05d 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -817,6 +817,104 @@ ${SOURCE_MAPPING_URL}=library-preload.js.map ]); }); +test.serial("integration: createBundle with depCache", async (t) => { + const pool = new ResourcePool(); + pool.addResource({ + name: "a.js", + getPath: () => "a.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([\"./b\", \"./c2\"],function(b, c){return {};});" + }); + pool.addResource({ + name: "b.js", + getPath: () => "b.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Two(){return 2;}" + }); + pool.addResource({ + name: "c2.js", + getPath: () => "c2.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([\"./c1\", \"./c3\"],function(c1, c3){return {};});" + }); + pool.addResource({ + name: "c1.js", + getPath: () => "c1.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.1;}" + }); + pool.addResource({ + name: "c3.js", + getPath: () => "c3.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.3;}" + }); + pool.addResource({ + name: "a.library", + getPath: () => "a.library", + string: function() { + return this.buffer(); + }, + buffer: async () => ` + + + + + + + + +` + }); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "preload", + name: "preload-section", + filters: ["a.js"] + }, { + mode: "depCache", + filters: ["*.js"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {}); + t.is(oResult.name, "library-depCache-preload.js"); + const expectedContent = `//@ui5-bundle library-depCache-preload.js +sap.ui.require.preload({ + "a.js":function(){ +sap.ui.define(["./b", "./c2"],function(b, c){return {};}); +} +},"preload-section"); +sap.ui.loader.config({depCacheUI5:{ +"a.js": ["b.js","c2.js"], +"c2.js": ["c1.js","c3.js"] +}}); +${SOURCE_MAPPING_URL}=library-depCache-preload.js.map +`; + t.deepEqual(oResult.content, expectedContent, "EVOBundleFormat " + + "should contain:" + + " preload part from a.js" + + " depCache part from a.js && c2.js"); + t.is(oResult.bundleInfo.name, "library-depCache-preload.js", "bundle info name is correct"); + t.deepEqual(oResult.bundleInfo.size, expectedContent.length, "bundle info size is correct"); + t.deepEqual(oResult.bundleInfo.subModules, ["a.js", "b.js", "c2.js", "c1.js", "c3.js"], + "bundle info subModules are correct"); +}); + test("integration: createBundle using predefine calls with source maps and a single, simple source", async (t) => { const pool = new ResourcePool(); From 08e61ff0359cc92f0dde4853fe06532f72f44703 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 09:31:50 +0200 Subject: [PATCH 08/30] Update lib/lbt/bundle/BundleDefinition.js MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Oßwald --- lib/lbt/bundle/BundleDefinition.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lbt/bundle/BundleDefinition.js b/lib/lbt/bundle/BundleDefinition.js index 42b3a3ff4..bcb617267 100644 --- a/lib/lbt/bundle/BundleDefinition.js +++ b/lib/lbt/bundle/BundleDefinition.js @@ -33,7 +33,6 @@ export const SectionType = { * Dependency cache information that lists modules and their dependencies * of all types: JS, declarative views/fragments. * Only the dependencies of the modules are stored as 'depCache' configuration. - * Requires UI5 Evolution runtime. */ DepCache: "depCache" }; From 16c065f59096109293a7064da5dd0e24b91c66e9 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 09:56:34 +0200 Subject: [PATCH 09/30] Replace placeholders with string literals & refactor resource fetch exception handling --- lib/lbt/bundle/Builder.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 3fbe8a15b..5213daa0d 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -562,15 +562,21 @@ class BundleBuilder { async writeDepCache(section) { const outW = this.outW; - outW.ensureNewLine(); - const sequence = section.modules.slice().sort(); if (sequence.length > 0) { + outW.ensureNewLine(); + this.targetBundleFormat.beforeDepCache(outW, section); let i = 0; for (const module of sequence) { - const resource = await this.pool.findResourceWithInfo(module); + let resource = null; + try { + resource = await this.pool.findResourceWithInfo(module); + } catch (e) { + log.error(` couldn't find ${module}`); + } + if (resource != null) { const deps = resource.info.dependencies.filter( (dep) => @@ -586,10 +592,8 @@ class BundleBuilder { ); i++; } else { - log.verbose(" skipped %s, no dependencies", module); + log.verbose(` skipped ${module}, no dependencies`); } - } else { - log.error(" couldn't find %s", module); } } From 44396b0e4a40e2da6dcdc2bcdec52313f7113c5f Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 09:56:50 +0200 Subject: [PATCH 10/30] Update JSDoc --- lib/lbt/bundle/BundleDefinition.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lbt/bundle/BundleDefinition.js b/lib/lbt/bundle/BundleDefinition.js index bcb617267..b71bb8476 100644 --- a/lib/lbt/bundle/BundleDefinition.js +++ b/lib/lbt/bundle/BundleDefinition.js @@ -32,7 +32,6 @@ export const SectionType = { /** * Dependency cache information that lists modules and their dependencies * of all types: JS, declarative views/fragments. - * Only the dependencies of the modules are stored as 'depCache' configuration. */ DepCache: "depCache" }; From 52d1c3ac599877097081e824a0da2cb7a7fd2545 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 14:15:06 +0200 Subject: [PATCH 11/30] Add depCache to the AutoSplitter --- lib/lbt/bundle/AutoSplitter.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 995d1a915..d85616fcd 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -74,6 +74,25 @@ class AutoSplitter { totalSize += "sap.ui.requireSync('');".length + toRequireJSName(module).length; }); break; + case SectionType.DepCache: + moduleSizes["__depCacheSize"] = "sap.ui.loader.config({depCacheUI5:{}});".length; + + section.modules.forEach( (module) => { + promises.push((async () => { + const resource = await this.pool.findResourceWithInfo(module); + const deps = resource.info.dependencies.filter( + (dep) => + !resource.info.isConditionalDependency(dep) && + !resource.info.isImplicitDependency(dep) + ); + if (deps.length > 0) { + const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length; + moduleSizes["__depCacheSize"] += depSize; + totalSize += depSize; + } + })()); + }); + break; default: break; } @@ -180,6 +199,15 @@ class AutoSplitter { totalSize += 21 + toRequireJSName(module).length; }); break; + case SectionType.DepCache: + currentSection = { + name: section.name, + mode: SectionType.DepCache, + filters: section.modules.slice() + }; + currentModule.sections.push( currentSection ); + totalSize += moduleSizes["__depCacheSize"]; + break; default: break; } From acdfe6a57c5049bac8bf93a14779e295bde314a7 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 14:35:05 +0200 Subject: [PATCH 12/30] Do not write depCache data if there are no dependencies at all --- lib/lbt/bundle/Builder.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 5213daa0d..5af5c4c47 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -561,14 +561,12 @@ class BundleBuilder { async writeDepCache(section) { const outW = this.outW; + let hasDepCache = false; const sequence = section.modules.slice().sort(); if (sequence.length > 0) { - outW.ensureNewLine(); - - this.targetBundleFormat.beforeDepCache(outW, section); - let i = 0; + // let i = 0; for (const module of sequence) { let resource = null; try { @@ -584,23 +582,24 @@ class BundleBuilder { !resource.info.isImplicitDependency(dep) ); if (deps.length > 0) { - if (i > 0) { - outW.writeln(","); + if (!hasDepCache) { + hasDepCache = true; + outW.ensureNewLine(); + this.targetBundleFormat.beforeDepCache(outW, section); } - outW.write( - `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}]` + + outW.writeln( + `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],` ); - i++; } else { log.verbose(` skipped ${module}, no dependencies`); } } } - if (i > 0) { - outW.writeln(); + if (hasDepCache) { + this.targetBundleFormat.afterDepCache(outW, section); } - this.targetBundleFormat.afterDepCache(outW, section); } } From a27f4d9caff7bde574ec266da9f1f129f34f4106 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 14:35:14 +0200 Subject: [PATCH 13/30] Add tests --- test/lib/lbt/bundle/Builder.js | 96 +++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index c6b01f05d..743f3733f 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -901,7 +901,7 @@ sap.ui.define(["./b", "./c2"],function(b, c){return {};}); },"preload-section"); sap.ui.loader.config({depCacheUI5:{ "a.js": ["b.js","c2.js"], -"c2.js": ["c1.js","c3.js"] +"c2.js": ["c1.js","c3.js"], }}); ${SOURCE_MAPPING_URL}=library-depCache-preload.js.map `; @@ -915,6 +915,100 @@ ${SOURCE_MAPPING_URL}=library-depCache-preload.js.map "bundle info subModules are correct"); }); +test.serial("integration: createBundle with depCache with NO dependencies", async (t) => { + const pool = new ResourcePool(); + pool.addResource({ + name: "a.js", + getPath: () => "a.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([],function(){return {};});" + }); + pool.addResource({ + name: "b.js", + getPath: () => "b.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Two(){return 2;}" + }); + pool.addResource({ + name: "c2.js", + getPath: () => "c2.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "sap.ui.define([],function(){return {};});" + }); + pool.addResource({ + name: "c1.js", + getPath: () => "c1.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.1;}" + }); + pool.addResource({ + name: "c3.js", + getPath: () => "c3.js", + string: function() { + return this.buffer(); + }, + buffer: async () => "function Three(){return 3.3;}" + }); + pool.addResource({ + name: "a.library", + getPath: () => "a.library", + string: function() { + return this.buffer(); + }, + buffer: async () => ` + + + + + + + + +` + }); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "preload", + name: "preload-section", + filters: ["a.js"] + }, { + mode: "depCache", + filters: ["*.js"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {}); + t.is(oResult.name, "library-depCache-preload.js"); + const expectedContent = `//@ui5-bundle library-depCache-preload.js +sap.ui.require.preload({ + "a.js":function(){ +sap.ui.define([],function(){return {};}); +} +},"preload-section"); +${SOURCE_MAPPING_URL}=library-depCache-preload.js.map +`; + t.deepEqual(oResult.content, expectedContent, "EVOBundleFormat " + + "should contain:" + + " preload part from a.js" + + " depCache part from a.js && c2.js"); + t.is(oResult.bundleInfo.name, "library-depCache-preload.js", "bundle info name is correct"); + t.deepEqual(oResult.bundleInfo.size, expectedContent.length, "bundle info size is correct"); + t.deepEqual(oResult.bundleInfo.subModules, ["a.js", "b.js", "c2.js", "c1.js", "c3.js"], + "bundle info subModules are correct"); +}); + test("integration: createBundle using predefine calls with source maps and a single, simple source", async (t) => { const pool = new ResourcePool(); From 798db531a87bda0e1ef37c779051a3fc934f9984 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 14:46:32 +0200 Subject: [PATCH 14/30] Revert JSDoc --- lib/lbt/bundle/BundleDefinition.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/lbt/bundle/BundleDefinition.js b/lib/lbt/bundle/BundleDefinition.js index b71bb8476..bcb617267 100644 --- a/lib/lbt/bundle/BundleDefinition.js +++ b/lib/lbt/bundle/BundleDefinition.js @@ -32,6 +32,7 @@ export const SectionType = { /** * Dependency cache information that lists modules and their dependencies * of all types: JS, declarative views/fragments. + * Only the dependencies of the modules are stored as 'depCache' configuration. */ DepCache: "depCache" }; From 6def2784425d38ae2b1cb6af62998ba0d8b527d7 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Wed, 15 Nov 2023 15:37:58 +0200 Subject: [PATCH 15/30] Remove obsolete code --- lib/lbt/bundle/Builder.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 5af5c4c47..d1db2eda8 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -566,7 +566,6 @@ class BundleBuilder { const sequence = section.modules.slice().sort(); if (sequence.length > 0) { - // let i = 0; for (const module of sequence) { let resource = null; try { From daad21d072f2116a3a2e38c4d103f73aaaafc6c3 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Thu, 16 Nov 2023 09:34:30 +0200 Subject: [PATCH 16/30] Use Symbols for moduleSizes map --- lib/lbt/bundle/AutoSplitter.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index d85616fcd..e68a7dbff 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -41,6 +41,7 @@ class AutoSplitter { const numberOfParts = options.numberOfParts; let totalSize = 0; const moduleSizes = Object.create(null); + const depCacheSymbol = Symbol("depCache"); this.optimize = !!options.optimize; // ---- resolve module definition @@ -75,7 +76,8 @@ class AutoSplitter { }); break; case SectionType.DepCache: - moduleSizes["__depCacheSize"] = "sap.ui.loader.config({depCacheUI5:{}});".length; + moduleSizes[depCacheSymbol] = "sap.ui.loader.config({depCacheUI5:{}});".length; + totalSize += moduleSizes[depCacheSymbol]; section.modules.forEach( (module) => { promises.push((async () => { @@ -87,7 +89,7 @@ class AutoSplitter { ); if (deps.length > 0) { const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length; - moduleSizes["__depCacheSize"] += depSize; + moduleSizes[depCacheSymbol] += depSize; totalSize += depSize; } })()); @@ -206,7 +208,7 @@ class AutoSplitter { filters: section.modules.slice() }; currentModule.sections.push( currentSection ); - totalSize += moduleSizes["__depCacheSize"]; + totalSize += moduleSizes[depCacheSymbol]; break; default: break; From 10d570e763703af387e3521c80c50feda570e8ed Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Thu, 16 Nov 2023 09:44:15 +0200 Subject: [PATCH 17/30] Add tests for the Splitter --- test/lib/lbt/bundle/AutoSplitter.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index c766b704b..ca9305d30 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -102,6 +102,10 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { mode: "require", filters: ["a.js", "c.js"], modules: ["a.js", "c.js"] + }, { + mode: "depCache", + filters: ["*.js"], + modules: ["a.js", "c.js", "b.json", "c.properties", "x.view.xml"] }], configuration: {} }; @@ -134,11 +138,15 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, { mode: "require", filters: ["a.js", "c.js"] + }, + { + filters: ["a.js", "c.js"], + mode: "depCache", + name: undefined, }] }, "second part should contain the other resources"); }); - test("_calcMinSize: compressedSize", async (t) => { const pool = { findResourceWithInfo: function() { From ba48390e1ea127afafdd84f2071b6f85a3468b64 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Thu, 16 Nov 2023 09:45:56 +0200 Subject: [PATCH 18/30] Enhance tests for AutoSplitter --- lib/lbt/bundle/AutoSplitter.js | 1 - test/lib/lbt/bundle/AutoSplitter.js | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index e68a7dbff..81752d2bb 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -203,7 +203,6 @@ class AutoSplitter { break; case SectionType.DepCache: currentSection = { - name: section.name, mode: SectionType.DepCache, filters: section.modules.slice() }; diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index ca9305d30..c78d654c4 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -141,8 +141,7 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, { filters: ["a.js", "c.js"], - mode: "depCache", - name: undefined, + mode: "depCache" }] }, "second part should contain the other resources"); }); From 3df4cef858c8ddc92ac5790ae368230dcb3980f8 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Thu, 16 Nov 2023 09:48:06 +0200 Subject: [PATCH 19/30] Re-arrange tests --- test/lib/lbt/bundle/AutoSplitter.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index c78d654c4..4891f8bd6 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -75,6 +75,10 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { name: `Component-preload.js`, defaultFileTypes: [".js", ".fragment.xml", ".view.xml", ".properties", ".json"], sections: [{ + mode: "depCache", + filters: ["*.js"], + modules: ["a.js", "c.js", "b.json", "c.properties", "x.view.xml"] + }, { mode: "preload", filters: ["a.js", "b.json", "x.view.xml"], resolve: false, @@ -102,10 +106,6 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { mode: "require", filters: ["a.js", "c.js"], modules: ["a.js", "c.js"] - }, { - mode: "depCache", - filters: ["*.js"], - modules: ["a.js", "c.js", "b.json", "c.properties", "x.view.xml"] }], configuration: {} }; @@ -114,6 +114,9 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { t.deepEqual(oResult[0], { name: `Component-preload-0.js`, sections: [{ + filters: ["a.js", "c.js"], + mode: "depCache" + }, { mode: "preload", filters: ["a.js"], name: undefined @@ -138,10 +141,6 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, { mode: "require", filters: ["a.js", "c.js"] - }, - { - filters: ["a.js", "c.js"], - mode: "depCache" }] }, "second part should contain the other resources"); }); From 0ecfd7dced28f8f0e749e506b3675904a4585a3e Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 17 Nov 2023 10:05:19 +0200 Subject: [PATCH 20/30] Enable auto split for depCache --- lib/lbt/bundle/AutoSplitter.js | 42 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 81752d2bb..7de0d9721 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -41,7 +41,9 @@ class AutoSplitter { const numberOfParts = options.numberOfParts; let totalSize = 0; const moduleSizes = Object.create(null); - const depCacheSymbol = Symbol("depCache"); + const depCacheSizes = []; + let depCacheLoaderSize = 0; + let depCacheSet = new Set(); this.optimize = !!options.optimize; // ---- resolve module definition @@ -76,8 +78,8 @@ class AutoSplitter { }); break; case SectionType.DepCache: - moduleSizes[depCacheSymbol] = "sap.ui.loader.config({depCacheUI5:{}});".length; - totalSize += moduleSizes[depCacheSymbol]; + depCacheLoaderSize = "sap.ui.loader.config({depCacheUI5:{}});".length; + totalSize += depCacheLoaderSize; section.modules.forEach( (module) => { promises.push((async () => { @@ -89,8 +91,12 @@ class AutoSplitter { ); if (deps.length > 0) { const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length; - moduleSizes[depCacheSymbol] += depSize; totalSize += depSize; + + depCacheSizes.push({ + size: depSize, + filters: [module, ...deps] + }); } })()); }); @@ -204,10 +210,34 @@ class AutoSplitter { case SectionType.DepCache: currentSection = { mode: SectionType.DepCache, - filters: section.modules.slice() + filters: [] }; currentModule.sections.push( currentSection ); - totalSize += moduleSizes[depCacheSymbol]; + totalSize += depCacheLoaderSize; + + depCacheSizes.forEach((depCache) => { + if ( part + 1 < numberOfParts && totalSize + depCache.size / 2 > partSize ) { + currentSection.filters = Array.from(depCacheSet); + depCacheSet = new Set(); + part++; + // start a new module + totalSize = depCacheLoaderSize; + currentSection = { + mode: SectionType.DepCache, + filters: [] + }; + currentModule = { + name: moduleNameWithPart.replace(/__part__/, part), + sections: [currentSection] + }; + splittedModules.push(currentModule); + } + + depCache.filters.forEach((moduleName) => depCacheSet.add(moduleName)); + totalSize += depCache.size; + }); + + currentSection.filters = Array.from(depCacheSet); break; default: break; From 7574c76e66a8e354a89da44b95ed50ce3981012f Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 17 Nov 2023 10:05:58 +0200 Subject: [PATCH 21/30] fix: Remove depCache duplicate definitions across bundles --- lib/lbt/bundle/Builder.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index d1db2eda8..71a839fb9 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -559,6 +559,11 @@ class BundleBuilder { }); } + // When AutoSplit is enabled for depCache, we need to ensure that modules + // are not duplicated across files. This might happen due to the filters provided. + // So, certain modules that are included in depCache could be dependencies of another + // module in the next file. This will also duplicate its dependency definition if we do not filter. + #depCacheSet = new Set(); async writeDepCache(section) { const outW = this.outW; let hasDepCache = false; @@ -567,6 +572,11 @@ class BundleBuilder { if (sequence.length > 0) { for (const module of sequence) { + if (this.#depCacheSet.has(module)) { + continue; + } + + this.#depCacheSet.add(module); let resource = null; try { resource = await this.pool.findResourceWithInfo(module); From 981c6fc8f399d834c451ff1e60575cea695709ab Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 17 Nov 2023 11:35:57 +0200 Subject: [PATCH 22/30] Enhance tests for depCache in AutoSplitter --- test/lib/lbt/bundle/AutoSplitter.js | 41 ++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 4891f8bd6..7d432f7cf 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -114,7 +114,7 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { t.deepEqual(oResult[0], { name: `Component-preload-0.js`, sections: [{ - filters: ["a.js", "c.js"], + filters: ["a.js", "b.json", "c.js"], mode: "depCache" }, { mode: "preload", @@ -145,6 +145,45 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, "second part should contain the other resources"); }); +test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { + const modules = new Array(50).fill(null).map((val, index) => `a${index}.js`); + const pool = { + findResourceWithInfo: async (name) => { + const info = new ModuleInfo(name); + modules + .filter((moduleName) => moduleName !== name) + .forEach((dependency) => { + info.addDependency(dependency); + }); + return {info}; + }, + resources: modules.map((res) => ({name: res})) + }; + const autoSplitter = new AutoSplitter(pool, new BundleResolver(pool)); + const bundleDefinition = { + name: `test-depCache-preload.js`, + sections: [{ + mode: "depCache", + filters: ["*.js"], + modules + }] + }; + const oResult = await autoSplitter.run(bundleDefinition, {numberOfParts: 50, optimize: false}); + t.is(oResult.length, 50, "50 parts expected"); + + // Sections are the same as all modules depend on each other, + // therefore, the filters is the same (just the order of the names is slightly different). + for (let i= 0; i < 50; i++) { + t.deepEqual(oResult[i], { + name: `test-depCache-preload-${i}.js`, + sections: [{ + filters: [`a${i}.js`, ...modules.filter((name) => `a${i}.js` !== name)], + mode: "depCache" + }] + }); + } +}); + test("_calcMinSize: compressedSize", async (t) => { const pool = { findResourceWithInfo: function() { From 169d52322396e6419f463f633c12dc9462df8b71 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 17 Nov 2023 13:52:57 +0200 Subject: [PATCH 23/30] Refactor depCache autosplitter --- lib/lbt/bundle/AutoSplitter.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 7de0d9721..41c681169 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -43,7 +43,6 @@ class AutoSplitter { const moduleSizes = Object.create(null); const depCacheSizes = []; let depCacheLoaderSize = 0; - let depCacheSet = new Set(); this.optimize = !!options.optimize; // ---- resolve module definition @@ -217,8 +216,6 @@ class AutoSplitter { depCacheSizes.forEach((depCache) => { if ( part + 1 < numberOfParts && totalSize + depCache.size / 2 > partSize ) { - currentSection.filters = Array.from(depCacheSet); - depCacheSet = new Set(); part++; // start a new module totalSize = depCacheLoaderSize; @@ -233,11 +230,11 @@ class AutoSplitter { splittedModules.push(currentModule); } - depCache.filters.forEach((moduleName) => depCacheSet.add(moduleName)); + depCache.filters + .filter((moduleName) => !currentSection.filters.includes(moduleName)) + .forEach((moduleName) => currentSection.filters.push(moduleName)); totalSize += depCache.size; }); - - currentSection.filters = Array.from(depCacheSet); break; default: break; From 8b5802467e31c2fd00dd652ee8ef88f9bb6ca97e Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Fri, 17 Nov 2023 13:53:25 +0200 Subject: [PATCH 24/30] DepCache test for overlapping modules --- test/lib/lbt/bundle/Builder.js | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index 743f3733f..2410d9e21 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -915,6 +915,47 @@ ${SOURCE_MAPPING_URL}=library-depCache-preload.js.map "bundle info subModules are correct"); }); +test.serial("integration: createBundle with depCache with splitted modules", async (t) => { + const pool = new ResourcePool(); + new Array(20).fill(null).forEach((val, index, arr) => { + const strDeps = []; + const deps = []; + for (let i = index + 1; i < arr.length; i++ ) { + strDeps.push(`"./a${i}.js"`); + deps.push(`a${i}`); + } + pool.addResource({ + name: `a${index}.js`, + getPath: () => `a${index}.js`, + string: function() { + return this.buffer(); + }, + buffer: async () => `sap.ui.define([${strDeps.join(", ")}],function(${deps.join(", ")}){return {};});` + }); + }); + + const bundleDefinition = { + name: `library-depCache-preload.js`, + sections: [{ + mode: "depCache", + filters: ["*.js"] + }] + }; + + const builder = new Builder(pool); + const oResult = await builder.createBundle(bundleDefinition, {numberOfParts: 2}); + t.is(oResult.length, 2, "The bundle got split into 2 parts"); + + t.falsy( + oResult[0].bundleInfo.subModules.find((module) => + oResult[1].bundleInfo.subModules.includes(module) + ), "Submodules do not overlap" + ); + + const allSubmodules = [...oResult[0].bundleInfo.subModules, ...oResult[1].bundleInfo.subModules]; + t.is(allSubmodules.length, 19, "All modules have dependencies, except for the last one"); +}); + test.serial("integration: createBundle with depCache with NO dependencies", async (t) => { const pool = new ResourcePool(); pool.addResource({ From c02ee043f5fb9293450a9abe356fcb5afe8da216 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Mon, 20 Nov 2023 09:33:05 +0200 Subject: [PATCH 25/30] Fix typo --- test/lib/lbt/bundle/AutoSplitter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 7d432f7cf..215a90fae 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -172,7 +172,7 @@ test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { t.is(oResult.length, 50, "50 parts expected"); // Sections are the same as all modules depend on each other, - // therefore, the filters is the same (just the order of the names is slightly different). + // therefore, the filters are the same (just the order of the names is slightly different). for (let i= 0; i < 50; i++) { t.deepEqual(oResult[i], { name: `test-depCache-preload-${i}.js`, From 822d6ab0135f96bb749ac175aa5873c93d6eb499 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Mon, 4 Dec 2023 09:21:07 +0200 Subject: [PATCH 26/30] Fix AutoSplitter dependencies resolution --- lib/lbt/bundle/AutoSplitter.js | 13 +++++-------- test/lib/lbt/bundle/AutoSplitter.js | 6 +++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/lbt/bundle/AutoSplitter.js b/lib/lbt/bundle/AutoSplitter.js index 41c681169..5e5f0d1f1 100644 --- a/lib/lbt/bundle/AutoSplitter.js +++ b/lib/lbt/bundle/AutoSplitter.js @@ -92,10 +92,7 @@ class AutoSplitter { const depSize = `"${module}": [${deps.map((dep) => `"${dep}"`).join(",")}],`.length; totalSize += depSize; - depCacheSizes.push({ - size: depSize, - filters: [module, ...deps] - }); + depCacheSizes.push({size: depSize, module}); } })()); }); @@ -230,10 +227,10 @@ class AutoSplitter { splittedModules.push(currentModule); } - depCache.filters - .filter((moduleName) => !currentSection.filters.includes(moduleName)) - .forEach((moduleName) => currentSection.filters.push(moduleName)); - totalSize += depCache.size; + if (!currentSection.filters.includes(depCache.module)) { + currentSection.filters.push(depCache.module); + totalSize += depCache.size; + } }); break; default: diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 215a90fae..b4ea95ef0 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -114,7 +114,7 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { t.deepEqual(oResult[0], { name: `Component-preload-0.js`, sections: [{ - filters: ["a.js", "b.json", "c.js"], + filters: ["a.js", "c.js"], mode: "depCache" }, { mode: "preload", @@ -122,7 +122,7 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { name: undefined }], configuration: {} - }, "first part should contain only a.js since its size is only 2048"); + }, "bundle properly and correct dependencies & sizes"); t.deepEqual(oResult[1], { name: `Component-preload-1.js`, sections: [{ @@ -177,7 +177,7 @@ test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { t.deepEqual(oResult[i], { name: `test-depCache-preload-${i}.js`, sections: [{ - filters: [`a${i}.js`, ...modules.filter((name) => `a${i}.js` !== name)], + filters: [`a${i}.js`], mode: "depCache" }] }); From 3f2dfc43c062bcd4c295d307a26d3295da67b322 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Mon, 4 Dec 2023 16:11:45 +0200 Subject: [PATCH 27/30] Enhance tests for builder to support splitting and properly test split + filtering --- test/lib/lbt/bundle/Builder.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index 2410d9e21..bfae3af08 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -920,13 +920,19 @@ test.serial("integration: createBundle with depCache with splitted modules", asy new Array(20).fill(null).forEach((val, index, arr) => { const strDeps = []; const deps = []; + // Defines dependencies with different namespace for (let i = index + 1; i < arr.length; i++ ) { - strDeps.push(`"./a${i}.js"`); - deps.push(`a${i}`); + strDeps.push(`"another/path/dep/b${i}.js"`); + deps.push(`b${i}`); } + + // Test filtering by registering different namespaces in the pool, + // but using only the foo/bar namespace. + const curResourceName = (index % 2) ? + `foo/bar/a${index}.js` : `bizz/buzz/a${index}.js`; pool.addResource({ - name: `a${index}.js`, - getPath: () => `a${index}.js`, + name: curResourceName, + getPath: () => curResourceName, string: function() { return this.buffer(); }, @@ -938,7 +944,7 @@ test.serial("integration: createBundle with depCache with splitted modules", asy name: `library-depCache-preload.js`, sections: [{ mode: "depCache", - filters: ["*.js"] + filters: ["foo/bar/**"] }] }; @@ -953,7 +959,8 @@ test.serial("integration: createBundle with depCache with splitted modules", asy ); const allSubmodules = [...oResult[0].bundleInfo.subModules, ...oResult[1].bundleInfo.subModules]; - t.is(allSubmodules.length, 19, "All modules have dependencies, except for the last one"); + t.is(allSubmodules.length, 9, "'Half' of the defined modules are actually cached. Due to the filters. The last one is always ignored as it doesn't have dependencies"); + t.true(allSubmodules.every((module) => module.startsWith("foo/bar")), "Every (included) submodule starts with foo/bar namespace. The rest are filtered."); }); test.serial("integration: createBundle with depCache with NO dependencies", async (t) => { From 31bb2c448ade5e8f0b46a838576ff966cae1e322 Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Tue, 5 Dec 2023 13:07:26 +0200 Subject: [PATCH 28/30] Provide reliable integration test for depCache with AutoSplit --- test/lib/lbt/bundle/Builder.js | 54 +++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/test/lib/lbt/bundle/Builder.js b/test/lib/lbt/bundle/Builder.js index bfae3af08..b28e4597b 100644 --- a/test/lib/lbt/bundle/Builder.js +++ b/test/lib/lbt/bundle/Builder.js @@ -916,29 +916,36 @@ ${SOURCE_MAPPING_URL}=library-depCache-preload.js.map }); test.serial("integration: createBundle with depCache with splitted modules", async (t) => { + const resolvedModulesCount = 10; const pool = new ResourcePool(); - new Array(20).fill(null).forEach((val, index, arr) => { - const strDeps = []; - const deps = []; - // Defines dependencies with different namespace - for (let i = index + 1; i < arr.length; i++ ) { - strDeps.push(`"another/path/dep/b${i}.js"`); - deps.push(`b${i}`); - } - // Test filtering by registering different namespaces in the pool, - // but using only the foo/bar namespace. - const curResourceName = (index % 2) ? - `foo/bar/a${index}.js` : `bizz/buzz/a${index}.js`; - pool.addResource({ - name: curResourceName, - getPath: () => curResourceName, - string: function() { - return this.buffer(); - }, - buffer: async () => `sap.ui.define([${strDeps.join(", ")}],function(${deps.join(", ")}){return {};});` + // Builds N resources by adding provided "dependencies" as resource dependencies. + // Also adds the remaining I resources into dependency list + const buildDependencies = function(count, namespace, dependencies = []) { + return new Array(count).fill(null).map((val, index, arr) => { + const strDeps = dependencies.map((dep) => "\"" + dep + "\""); + const deps = dependencies.map((val, i) => `b${i}`); + for (let i = index + 1; i < arr.length; i++ ) { + strDeps.push(`"${namespace}${i}"`); + deps.push(`a${i}`); + } + + const curResourceName = `${namespace}${index}`; + pool.addResource({ + name: `${curResourceName}.js`, + getPath: () => `${curResourceName}.js`, + string: function() { + return this.buffer(); + }, + buffer: async () => `sap.ui.define([${strDeps.join(", ")}],function(${deps.join(", ")}){return {};});` + }); + + return curResourceName; }); - }); + }; + + const nonCachedDependencies = buildDependencies(5, "fizz/buzz/b"); + const cachedDependencies = buildDependencies(resolvedModulesCount, "foo/bar/a", nonCachedDependencies); const bundleDefinition = { name: `library-depCache-preload.js`, @@ -959,7 +966,12 @@ test.serial("integration: createBundle with depCache with splitted modules", asy ); const allSubmodules = [...oResult[0].bundleInfo.subModules, ...oResult[1].bundleInfo.subModules]; - t.is(allSubmodules.length, 9, "'Half' of the defined modules are actually cached. Due to the filters. The last one is always ignored as it doesn't have dependencies"); + t.is(allSubmodules.length, resolvedModulesCount, `${resolvedModulesCount} of all defined modules in the pool are actually cached as the filter is only for foo/bar namespace`); + t.deepEqual( + allSubmodules.sort(), + cachedDependencies.sort().map((dep) => `${dep}.js`), + "Cached dependencies are the correct ones" + ); t.true(allSubmodules.every((module) => module.startsWith("foo/bar")), "Every (included) submodule starts with foo/bar namespace. The rest are filtered."); }); From 5a19d479a14c18f0a2da668ea634eecb3f90660c Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Tue, 5 Dec 2023 13:40:34 +0200 Subject: [PATCH 29/30] Add tests for AutoSplitter --- test/lib/lbt/bundle/AutoSplitter.js | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index b4ea95ef0..2eaefae3b 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -145,8 +145,16 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, "second part should contain the other resources"); }); -test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { - const modules = new Array(50).fill(null).map((val, index) => `a${index}.js`); +test.only("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { + const includedNamespace = "foo/bar/a"; + const excludedNamespace = "fizz/buzz/b"; + const modules = new Array(150) + .fill(null) + .map((val, index) => + index % 2 ? + `${includedNamespace}${index}.js` : + `${excludedNamespace}${index}.js` + ); const pool = { findResourceWithInfo: async (name) => { const info = new ModuleInfo(name); @@ -164,24 +172,27 @@ test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { name: `test-depCache-preload.js`, sections: [{ mode: "depCache", - filters: ["*.js"], + filters: ["foo/bar/**"], modules }] }; const oResult = await autoSplitter.run(bundleDefinition, {numberOfParts: 50, optimize: false}); t.is(oResult.length, 50, "50 parts expected"); - // Sections are the same as all modules depend on each other, - // therefore, the filters are the same (just the order of the names is slightly different). for (let i= 0; i < 50; i++) { - t.deepEqual(oResult[i], { - name: `test-depCache-preload-${i}.js`, - sections: [{ - filters: [`a${i}.js`], - mode: "depCache" - }] - }); + t.is(oResult[i].name, `test-depCache-preload-${i}.js`, "Correct preload bundles got created"); } + + // Merge filters from all bundles + const allFilters = oResult.flatMap((res) => + res.sections.flatMap((section) => section.filters) + ).sort(); + + t.deepEqual(Array.from(new Set(allFilters)).sort(), allFilters, "There are no duplicate filters"); + t.true( + allFilters.every((filter) => filter.startsWith("foo/bar")), + "Every (included) filter starts with foo/bar namespace. The rest are filtered." + ); }); test("_calcMinSize: compressedSize", async (t) => { From f66676f848607b4df201412d71ad93c1a00367fb Mon Sep 17 00:00:00 2001 From: Yavor Ivanov Date: Tue, 5 Dec 2023 13:42:13 +0200 Subject: [PATCH 30/30] Enable all tests for AutoSplitter --- test/lib/lbt/bundle/AutoSplitter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/lbt/bundle/AutoSplitter.js b/test/lib/lbt/bundle/AutoSplitter.js index 2eaefae3b..77a8e1e1b 100644 --- a/test/lib/lbt/bundle/AutoSplitter.js +++ b/test/lib/lbt/bundle/AutoSplitter.js @@ -145,7 +145,7 @@ test("integration: AutoSplitter with numberOfParts 2", async (t) => { }, "second part should contain the other resources"); }); -test.only("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { +test("integration: Extreme AutoSplitter with numberOfParts 50", async (t) => { const includedNamespace = "foo/bar/a"; const excludedNamespace = "fizz/buzz/b"; const modules = new Array(150)