From ecf3659eda005fa343c0cb18050815e4c8ec93fe Mon Sep 17 00:00:00 2001 From: Bharat Raju Date: Tue, 29 Oct 2024 15:10:26 -0400 Subject: [PATCH] Update the top level template json file based on ZCL extensions (#1475) - Add the zcl extension files to part of its crc check. - If the crc changes based on the inclusion of extensions then reload the top level template json file and turn its IN_SYNC to true and turn the old on to false - These changes make sure that when a zcl extension file (eg json files in zcl section such as cluster-to-component.json) is updated then everything from the top level json files are loaded again - Add tests to make sure top level template json file is reloaded when the zcl extension file is modified - JIRA: ZAPP-1092 --- docs/api.md | 46 ++++++++------ src-electron/db/db-mapping.js | 3 +- src-electron/db/query-package.js | 3 +- src-electron/generator/generation-engine.js | 55 +++++++++++++--- src-electron/util/env.js | 4 +- test/test-util.js | 4 +- test/zcl-loader.test.js | 69 +++++++++++++++++++++ 7 files changed, 151 insertions(+), 33 deletions(-) diff --git a/docs/api.md b/docs/api.md index c21557c392..72adca6605 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7713,10 +7713,10 @@ Get endpoint type events from the given endpoint type ID. ## JS API: generator logic * [JS API: generator logic](#module_JS API_ generator logic) - * [~loadGenTemplateFromFile(path)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ + * [~loadGenTemplateFromFile(templatePath)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ * [~recordPackageIfNonexistent(db, packagePath, parentId, packageType, version, category, description)](#module_JS API_ generator logic..recordPackageIfNonexistent) ⇒ * [~loadTemplateOptionsFromJsonFile(db, packageId, category, externalPath)](#module_JS API_ generator logic..loadTemplateOptionsFromJsonFile) ⇒ - * [~recordTemplatesPackage(context)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ + * [~recordTemplatesPackage(context, isTopLevelPackageInSync)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ * [~decodePackageExtensionEntity(entityType, entity)](#module_JS API_ generator logic..decodePackageExtensionEntity) ⇒ * [~loadZclExtensions(zclExt)](#module_JS API_ generator logic..loadZclExtensions) ⇒ * [~loadTemplates(db, genTemplatesJsonArray)](#module_JS API_ generator logic..loadTemplates) @@ -7762,7 +7762,7 @@ Get endpoint type events from the given endpoint type ID. -### JS API: generator logic~loadGenTemplateFromFile(path) ⇒ +### JS API: generator logic~loadGenTemplateFromFile(templatePath) ⇒ Given a path, it will read generation template object into memory. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) @@ -7770,7 +7770,7 @@ Given a path, it will read generation template object into memory. | Param | Type | | --- | --- | -| path | \* | +| templatePath | \* | @@ -7807,8 +7807,9 @@ Insert the template options from the json meta data file. -### JS API: generator logic~recordTemplatesPackage(context) ⇒ -Given a loading context, it records the package into the packages table and adds the packageId field into the resolved context. +### JS API: generator logic~recordTemplatesPackage(context, isTopLevelPackageInSync) ⇒ +Given a loading context and whether the package is in sync, it records the +package into the packages table and adds the packageId field into the resolved context. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) **Returns**: promise that resolves with the same context passed in, except packageId added to it @@ -7816,6 +7817,7 @@ Given a loading context, it records the package into the packages table and adds | Param | Type | | --- | --- | | context | \* | +| isTopLevelPackageInSync | \* | @@ -13019,10 +13021,10 @@ This module contains the API for templating. For more detailed instructions, rea ## JS API: generator logic * [JS API: generator logic](#module_JS API_ generator logic) - * [~loadGenTemplateFromFile(path)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ + * [~loadGenTemplateFromFile(templatePath)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ * [~recordPackageIfNonexistent(db, packagePath, parentId, packageType, version, category, description)](#module_JS API_ generator logic..recordPackageIfNonexistent) ⇒ * [~loadTemplateOptionsFromJsonFile(db, packageId, category, externalPath)](#module_JS API_ generator logic..loadTemplateOptionsFromJsonFile) ⇒ - * [~recordTemplatesPackage(context)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ + * [~recordTemplatesPackage(context, isTopLevelPackageInSync)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ * [~decodePackageExtensionEntity(entityType, entity)](#module_JS API_ generator logic..decodePackageExtensionEntity) ⇒ * [~loadZclExtensions(zclExt)](#module_JS API_ generator logic..loadZclExtensions) ⇒ * [~loadTemplates(db, genTemplatesJsonArray)](#module_JS API_ generator logic..loadTemplates) @@ -13068,7 +13070,7 @@ This module contains the API for templating. For more detailed instructions, rea -### JS API: generator logic~loadGenTemplateFromFile(path) ⇒ +### JS API: generator logic~loadGenTemplateFromFile(templatePath) ⇒ Given a path, it will read generation template object into memory. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) @@ -13076,7 +13078,7 @@ Given a path, it will read generation template object into memory. | Param | Type | | --- | --- | -| path | \* | +| templatePath | \* | @@ -13113,8 +13115,9 @@ Insert the template options from the json meta data file. -### JS API: generator logic~recordTemplatesPackage(context) ⇒ -Given a loading context, it records the package into the packages table and adds the packageId field into the resolved context. +### JS API: generator logic~recordTemplatesPackage(context, isTopLevelPackageInSync) ⇒ +Given a loading context and whether the package is in sync, it records the +package into the packages table and adds the packageId field into the resolved context. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) **Returns**: promise that resolves with the same context passed in, except packageId added to it @@ -13122,6 +13125,7 @@ Given a loading context, it records the package into the packages table and adds | Param | Type | | --- | --- | | context | \* | +| isTopLevelPackageInSync | \* | @@ -13675,10 +13679,10 @@ Function wrapper that can be used when a helper is deprecated. ## JS API: generator logic * [JS API: generator logic](#module_JS API_ generator logic) - * [~loadGenTemplateFromFile(path)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ + * [~loadGenTemplateFromFile(templatePath)](#module_JS API_ generator logic..loadGenTemplateFromFile) ⇒ * [~recordPackageIfNonexistent(db, packagePath, parentId, packageType, version, category, description)](#module_JS API_ generator logic..recordPackageIfNonexistent) ⇒ * [~loadTemplateOptionsFromJsonFile(db, packageId, category, externalPath)](#module_JS API_ generator logic..loadTemplateOptionsFromJsonFile) ⇒ - * [~recordTemplatesPackage(context)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ + * [~recordTemplatesPackage(context, isTopLevelPackageInSync)](#module_JS API_ generator logic..recordTemplatesPackage) ⇒ * [~decodePackageExtensionEntity(entityType, entity)](#module_JS API_ generator logic..decodePackageExtensionEntity) ⇒ * [~loadZclExtensions(zclExt)](#module_JS API_ generator logic..loadZclExtensions) ⇒ * [~loadTemplates(db, genTemplatesJsonArray)](#module_JS API_ generator logic..loadTemplates) @@ -13724,7 +13728,7 @@ Function wrapper that can be used when a helper is deprecated. -### JS API: generator logic~loadGenTemplateFromFile(path) ⇒ +### JS API: generator logic~loadGenTemplateFromFile(templatePath) ⇒ Given a path, it will read generation template object into memory. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) @@ -13732,7 +13736,7 @@ Given a path, it will read generation template object into memory. | Param | Type | | --- | --- | -| path | \* | +| templatePath | \* | @@ -13769,8 +13773,9 @@ Insert the template options from the json meta data file. -### JS API: generator logic~recordTemplatesPackage(context) ⇒ -Given a loading context, it records the package into the packages table and adds the packageId field into the resolved context. +### JS API: generator logic~recordTemplatesPackage(context, isTopLevelPackageInSync) ⇒ +Given a loading context and whether the package is in sync, it records the +package into the packages table and adds the packageId field into the resolved context. **Kind**: inner method of [JS API: generator logic](#module_JS API_ generator logic) **Returns**: promise that resolves with the same context passed in, except packageId added to it @@ -13778,6 +13783,7 @@ Given a loading context, it records the package into the packages table and adds | Param | Type | | --- | --- | | context | \* | +| isTopLevelPackageInSync | \* | @@ -17201,7 +17207,7 @@ Get save file format. ### JS API: Environment utilities.builtinSilabsZclSpecialMetafile() ⇒ -Used to retrive zcl-special.json by zcl reload test in zcl-loader.test.js +Used to retrieve zcl-special.json by zcl reload test in zcl-loader.test.js **Kind**: static method of [JS API: Environment utilities](#module_JS API_ Environment utilities) **Returns**: path to zcl-special.json file used by zcl-loader.test.js @@ -17213,7 +17219,7 @@ Used to retrive zcl-special.json by zcl reload test in zcl-loader.test.js ### JS API: Environment utilities.builtinSilabsSpecialZclGeneralSpecialXmlFile() ⇒ -Used to retrive general-special.xml by zcl reload test in zcl-loader.test.js +Used to retrieve general-special.xml by zcl reload test in zcl-loader.test.js **Kind**: static method of [JS API: Environment utilities](#module_JS API_ Environment utilities) **Returns**: path to general-special.xml file used by zcl-loader.test.js diff --git a/src-electron/db/db-mapping.js b/src-electron/db/db-mapping.js index 152d87f552..11309da1ad 100644 --- a/src-electron/db/db-mapping.js +++ b/src-electron/db/db-mapping.js @@ -36,7 +36,8 @@ exports.map = { category: x.CATEGORY, description: x.DESCRIPTION, version: x.VERSION, - parentId: x.PARENT_PACKAGE_REF + parentId: x.PARENT_PACKAGE_REF, + isInSync: x.IS_IN_SYNC } }, options: (x) => { diff --git a/src-electron/db/query-package.js b/src-electron/db/query-package.js index ae110c715e..be351533fa 100644 --- a/src-electron/db/query-package.js +++ b/src-electron/db/query-package.js @@ -34,7 +34,8 @@ SELECT CRC, VERSION, CATEGORY, - DESCRIPTION + DESCRIPTION, + IS_IN_SYNC FROM PACKAGE ` /** diff --git a/src-electron/generator/generation-engine.js b/src-electron/generator/generation-engine.js index 657297bd89..a8e6032f9d 100644 --- a/src-electron/generator/generation-engine.js +++ b/src-electron/generator/generation-engine.js @@ -36,20 +36,45 @@ const queryNotification = require('../db/query-package-notification.js') /** * Given a path, it will read generation template object into memory. * - * @param {*} path + * @param {*} templatePath * @returns Object that contains: data, crc, templateData */ -async function loadGenTemplateFromFile(path) { +async function loadGenTemplateFromFile(templatePath) { let ret = {} - ret.data = await fsPromise.readFile(path, 'utf8') + ret.data = await fsPromise.readFile(templatePath, 'utf8') ret.crc = util.checksum(ret.data) ret.templateData = JSON.parse(ret.data) + let zclExtension = ret.templateData.zcl + let zclExtensionFileContent = '' + // Adding zcl extension files to the template json crc + if (zclExtension && typeof zclExtension === 'object') { + for (const key of Object.keys(zclExtension)) { + let extension = zclExtension[key] + for (const key2 of Object.keys(extension)) { + let defaultExtensionValue = extension[key2].defaults + if ( + typeof defaultExtensionValue === 'string' || + defaultExtensionValue instanceof String + ) { + // Data is a string, so we will treat it as a relative path to the JSON file. + let externalPath = path.resolve( + path.join(path.dirname(templatePath), defaultExtensionValue) + ) + zclExtensionFileContent += await fsPromise.readFile( + externalPath, + 'utf8' + ) + } + } + } + ret.crc = util.checksum(ret.data + zclExtensionFileContent) + } let requiredFeatureLevel = 0 if ('requiredFeatureLevel' in ret.templateData) { requiredFeatureLevel = ret.templateData.requiredFeatureLevel } - let status = util.matchFeatureLevel(requiredFeatureLevel, path) + let status = util.matchFeatureLevel(requiredFeatureLevel, templatePath) if (status.match) { return ret } else { @@ -133,12 +158,14 @@ async function loadTemplateOptionsFromJsonFile( } /** - * Given a loading context, it records the package into the packages table and adds the packageId field into the resolved context. + * Given a loading context and whether the package is in sync, it records the + * package into the packages table and adds the packageId field into the resolved context. * * @param {*} context + * @param {*} isTopLevelPackageInSync * @returns promise that resolves with the same context passed in, except packageId added to it */ -async function recordTemplatesPackage(context) { +async function recordTemplatesPackage(context, isTopLevelPackageInSync) { let topLevel = await queryPackage.registerTopLevelPackage( context.db, context.path, @@ -147,7 +174,7 @@ async function recordTemplatesPackage(context) { context.templateData.version, context.templateData.category, context.templateData.description, - true + isTopLevelPackageInSync ) context.packageId = topLevel.id if (topLevel.existedPreviously) return context @@ -596,7 +623,19 @@ async function loadGenTemplatesJsonFile(db, genTemplatesJson) { if (!isTransactionAlreadyExisting) await dbApi.dbBeginTransaction(db) try { Object.assign(context, await loadGenTemplateFromFile(file)) - context = await recordTemplatesPackage(context) + let isTopLevelPackageInSync = true + // Check if that package already exist with the same crc + let existingPackage = await queryPackage.getPackageByPathAndType( + db, + file, + dbEnum.packageType.genTemplatesJson + ) + if (existingPackage && existingPackage.crc !== context.crc) { + // Package crc has changed so turning the old package out of sync(IN_SYNC=0) + await queryPackage.updatePackageIsInSync(db, existingPackage.id, 0) + isTopLevelPackageInSync = false + } + context = await recordTemplatesPackage(context, isTopLevelPackageInSync) return context } catch (err) { env.logInfo(`Can not read templates from: ${file}`) diff --git a/src-electron/util/env.js b/src-electron/util/env.js index 83a9de52ef..0e39ef5c64 100644 --- a/src-electron/util/env.js +++ b/src-electron/util/env.js @@ -55,7 +55,7 @@ export function builtinSilabsZclMetafile() { } /** - * Used to retrive zcl-special.json by zcl reload test in zcl-loader.test.js + * Used to retrieve zcl-special.json by zcl reload test in zcl-loader.test.js * * @returns path to zcl-special.json file used by zcl-loader.test.js */ @@ -73,7 +73,7 @@ export function builtinSilabsZclGeneralXmlFile() { } /** - * Used to retrive general-special.xml by zcl reload test in zcl-loader.test.js + * Used to retrieve general-special.xml by zcl reload test in zcl-loader.test.js * * @returns path to general-special.xml file used by zcl-loader.test.js */ diff --git a/test/test-util.js b/test/test-util.js index 7aff4c5e92..10e484bc18 100644 --- a/test/test-util.js +++ b/test/test-util.js @@ -95,7 +95,9 @@ exports.testTemplate = { dotdotCount: 5, unittest: './test/gen-template/test/gen-test.json', testCount: 3, - meta: './test/resource/meta/gen-test.json' + meta: './test/resource/meta/gen-test.json', + zclExtensionClusterToComponentFile: + './test/gen-template/zigbee2/cluster-to-component-dependencies.json' } exports.otherTestFile = { diff --git a/test/zcl-loader.test.js b/test/zcl-loader.test.js index 2c5740584c..e3f304b014 100644 --- a/test/zcl-loader.test.js +++ b/test/zcl-loader.test.js @@ -31,6 +31,7 @@ const types = require('../src-electron/util/types') const testUtil = require('./test-util') const testQuery = require('./test-query') const fs = require('fs') +const genEngine = require('../src-electron/generator/generation-engine') beforeAll(async () => { env.setDevelopmentEnv() @@ -282,6 +283,74 @@ test( testUtil.timeout.long() ) +test( + 'test changing the zcl extension file in a top level templates json file and make sure it is re-loaded again', + async () => { + let db = await dbApi.initRamDatabase() + try { + await dbApi.loadSchema(db, env.schemaFile(), env.zapVersion()) + let context = await genEngine.loadTemplates( + db, + testUtil.testTemplate.zigbee2 + ) + let existingPackageId = context.packageId + + // Reload package + context = await genEngine.loadTemplates(db, testUtil.testTemplate.zigbee2) + expect(existingPackageId).toEqual(context.packageId) + let existingPackageDetails = await queryPackage.getPackageByPackageId( + db, + existingPackageId + ) + expect(existingPackageDetails.isInSync).toEqual(1) + + // Update the cluster-to-component.json extension file + let extensionFile = + testUtil.testTemplate.zclExtensionClusterToComponentFile + let originalString = '"clusterCode": "zll commissioning-server"' + let editString = '"clusterCode": "zll commissioningEdit-server"' + let generalExtensionFileOriginalContent = fs.readFileSync( + extensionFile, + 'utf8' + ) + let generalExtensionFileUpdatedContent = + generalExtensionFileOriginalContent.replace(originalString, editString) + fs.writeFileSync( + extensionFile, + generalExtensionFileUpdatedContent, + 'utf8' + ) + + // Reload the templates json package after an extension file change above + context = await genEngine.loadTemplates(db, testUtil.testTemplate.zigbee2) + expect(existingPackageId).not.toEqual(context.packageId) + existingPackageDetails = await queryPackage.getPackageByPackageId( + db, + existingPackageId + ) + // The old package should no longer be in sync + expect(existingPackageDetails.isInSync).toEqual(0) + + let newPackageDetails = await queryPackage.getPackageByPackageId( + db, + context.packageId + ) + // The new package should now be in sync + expect(newPackageDetails.isInSync).toEqual(1) + + // Revert the zcl extension json file change which was done to run this test. + fs.writeFileSync( + extensionFile, + generalExtensionFileOriginalContent, + 'utf8' + ) + } finally { + await dbApi.closeDatabase(db) + } + }, + testUtil.timeout.long() +) + test( 'test Dotdot zcl data loading in memory', async () => {