Optimize deploy

package.json

@@ -16,6 +16,7 @@
     "@adobe/aio-lib-env": "^3.0.0",
     "archiver": "^6.0.1",
     "execa": "^4.0.3",
+    "folder-hash": "^4.0.4",
     "fs-extra": "^11.1.1",
     "globby": "^11.0.1",
     "js-yaml": "^4.1.0",

src/build-actions.js

@@ -11,13 +11,14 @@ governing permissions and limitations under the License.
 */
 
 const fs = require('fs-extra')
-const path = require('path')
+const path = require('node:path')
 const webpack = require('webpack')
 const globby = require('globby')
 const utils = require('./utils')
 const aioLogger = require('@adobe/aio-lib-core-logging')('@adobe/aio-lib-runtime:action-builder', { provider: 'debug' })
 const cloneDeep = require('lodash.clonedeep')
 const { getCliEnv } = require('@adobe/aio-lib-env')
+const { hashElement } = require('folder-hash')
 
 const uniqueArr = (items) => {
   return [...new Set(items)]
@@ -164,9 +165,9 @@ const loadWebpackConfig = async (configPath, actionPath, tempBuildDir, outBuildF
  * @returns {Promise<ActionBuild>} Relevant data for the zip process..
  */
 const prepareToBuildAction = async (action, root, dist) => {
+  // dist is something like ext-id/actions/ typically
   const { name: actionName, defaultPackage, packageName } = action
   const zipFileName = utils.getActionZipFileName(packageName, actionName, false)
-  let statsInfo // this is the object returned by bundler run, it has the hash
   // path.resolve supports both relative and absolute action.function
   const actionPath = path.resolve(root, action.function)
   const outPath = path.join(dist, `${zipFileName}.zip`)
@@ -187,17 +188,23 @@ const prepareToBuildAction = async (action, root, dist) => {
     }
   })
 
+  // quick helper
+  const filePathExists = (dir, file) => {
+    return fs.existsSync(path.join(dir, file))
+  }
+
+  const actionDir = path.dirname(actionPath)
+  const srcHash = await hashElement(actionDir, { folders: { exclude: ['node_modules'] } })
   if (isDirectory) {
-    // make sure package.json exists OR index.js
-    const packageJsonPath = path.join(actionPath, 'package.json')
-    if (!fs.existsSync(packageJsonPath)) {
-      if (!fs.existsSync(path.join(actionPath, 'index.js'))) {
-        throw new Error(`missing required ${utils._relApp(root, packageJsonPath)} or index.js for folder actions`)
+    // make sure package.json exists OR index.js exists
+    if (!filePathExists(actionPath, 'package.json')) {
+      if (!filePathExists(actionPath, 'index.js')) {
+        throw new Error('missing required package.json or index.js for folder actions')
       }
       aioLogger.debug('action directory has an index.js, allowing zip')
     } else {
       // make sure package.json exposes main or there is an index.js
-      const expectedActionName = utils.getActionEntryFile(packageJsonPath)
+      const expectedActionName = utils.getActionEntryFile(path.join(actionPath, 'package.json'))
       if (!fs.existsSync(path.join(actionPath, expectedActionName))) {
         throw new Error(`the directory ${action.function} must contain either a package.json with a 'main' flag or an index.js file at its root`)
       }
@@ -210,15 +217,17 @@ const prepareToBuildAction = async (action, root, dist) => {
     const webpackConfig = await loadWebpackConfig(webpackConfigPath, actionPath, tempBuildDir, 'index.js')
     const compiler = webpack(webpackConfig)
 
-    // run the compiler and wait for a result
-    statsInfo = await new Promise((resolve, reject) => {
+    // run the compiler and wait
+    await new Promise((resolve, reject) => {
       compiler.run((err, stats) => {
         if (err) {
           reject(err)
         }
         // stats must be defined at this point
         const info = stats.toJson()
         if (stats.hasWarnings()) {
+          // this might need to be evaluated, in most cases the user would not see this but
+          // probably should by default
           aioLogger.warn(`webpack compilation warnings:\n${info.warnings}`)
         }
         if (stats.hasErrors()) {
@@ -229,21 +238,11 @@ const prepareToBuildAction = async (action, root, dist) => {
     })
   }
 
-  let buildHash
-  let contentHash
-  if (isDirectory) {
-    contentHash = actionFileStats.mtime.valueOf()
-    buildHash = { [zipFileName]: contentHash }
-  } else {
-    contentHash = statsInfo.hash
-    buildHash = { [zipFileName]: contentHash }
-  }
-
   return {
     actionName,
-    buildHash,
     legacy: defaultPackage,
     outPath,
+    srcHash,
     tempBuildDir,
     tempActionName: 'index.js'
   }
@@ -257,10 +256,9 @@ const prepareToBuildAction = async (action, root, dist) => {
  * @param {Array<ActionBuild>} buildsList Array of data about actions available to be zipped.
  * @param {string} lastBuildsPath Path to the last built actions data.
  * @param {string} distFolder Path to the output root.
- * @param {boolean} skipCheck If true, zip all the actions from the buildsList
  * @returns {string[]} Array of zipped actions.
  */
-const zipActions = async (buildsList, lastBuildsPath, distFolder, skipCheck) => {
+const zipActions = async (buildsList, lastBuildsPath, distFolder) => {
   let dumpData = {}
   const builtList = []
   let lastBuiltData = ''
@@ -270,22 +268,17 @@ const zipActions = async (buildsList, lastBuildsPath, distFolder, skipCheck) =>
   for (const build of buildsList) {
     const { outPath, buildHash, tempBuildDir } = build
     aioLogger.debug(`action buildHash ${JSON.stringify(buildHash)}`)
-    const previouslyBuilt = utils.actionBuiltBefore(lastBuiltData, buildHash)
-    if (!previouslyBuilt || skipCheck) {
-      aioLogger.debug(`action ${build.actionName} has changed since last build, zipping`)
-      dumpData = { ...dumpData, ...buildHash }
-      await utils.zip(tempBuildDir, outPath)
-      builtList.push(outPath)
-    } else {
-      aioLogger.debug(`action ${build.actionName} was not modified since last build, skipping`)
-    }
+    aioLogger.debug(`action ${build.actionName} has changed since last build, zipping`)
+    dumpData = { ...dumpData, ...buildHash }
+    await utils.zip(tempBuildDir, outPath)
+    builtList.push(outPath)
   }
   const parsedLastBuiltData = utils.safeParse(lastBuiltData)
   await utils.dumpActionsBuiltInfo(lastBuildsPath, dumpData, parsedLastBuiltData)
   return builtList
 }
 
-const buildActions = async (config, filterActions, skipCheck = false, emptyDist = true) => {
+const buildActions = async (config, filterActions, skipCheck = false, emptyDist = false) => {
   if (!config.app.hasBackend) {
     throw new Error('cannot build actions, app has no backend')
   }
@@ -296,30 +289,56 @@ const buildActions = async (config, filterActions, skipCheck = false, emptyDist
     // If using old format of <actionname>, convert it to <package>/<actionname> using default/first package in the manifest
     sanitizedFilterActions = sanitizedFilterActions.map(actionName => actionName.indexOf('/') === -1 ? modifiedConfig.ow.package + '/' + actionName : actionName)
   }
+  // action specific, ext-id/actions/
   const distFolder = config.actions.dist
-
   // clear out dist dir
   if (emptyDist) {
     fs.emptyDirSync(distFolder)
   }
   const toBuildList = []
   const lastBuiltActionsPath = path.join(config.root, 'dist', 'last-built-actions.json')
+  let lastBuiltData = {}
+  if (fs.existsSync(lastBuiltActionsPath)) {
+    lastBuiltData = await fs.readJson(lastBuiltActionsPath)
+  }
+
   for (const [pkgName, pkg] of Object.entries(modifiedConfig.manifest.full.packages)) {
     const actionsToBuild = Object.entries(pkg.actions || {})
     // build all sequentially (todo make bundler execution parallel)
     for (const [actionName, action] of actionsToBuild) {
       const actionFullName = pkgName + '/' + actionName
+      // here we check if this action should be skipped
       if (Array.isArray(sanitizedFilterActions) && !sanitizedFilterActions.includes(actionFullName)) {
         continue
       }
       action.name = actionName
       action.packageName = pkgName
       action.defaultPackage = modifiedConfig.ow.package === pkgName
-      toBuildList.push(await prepareToBuildAction(action, config.root, distFolder))
+
+      // here we should check if there are changes since the last build
+      const actionPath = path.resolve(config.root, action.function)
+      const actionDir = path.dirname(actionPath)
+
+      // get a hash of the current action folder
+      const srcHash = await hashElement(actionDir, { folders: { exclude: ['node_modules'] } })
+      // lastBuiltData[actionName] === contentHash
+      // if the flag to skip is set, then we ALWAYS build
+      // if the hash is different, we build
+      // if the user has specified a filter, we build even if hash is the same, they are explicitly asking for it
+      // but we don't need to add a case, before we are called, skipCheck is set to true if there is a filter
+      if (skipCheck || lastBuiltData[actionFullName] !== srcHash.hash) {
+        // todo: inform the user that the action has changed and we are rebuilding
+        // console.log('action has changed since last build, zipping', actionFullName)
+        const buildResult = await prepareToBuildAction(action, config.root, distFolder)
+        buildResult.buildHash = { [actionFullName]: srcHash.hash }
+        toBuildList.push(buildResult)
+      } else {
+        // inform the user that the action has not changed ???
+        aioLogger.debug(`action ${actionFullName} has not changed`)
+      }
     }
   }
-
-  return zipActions(toBuildList, lastBuiltActionsPath, distFolder, skipCheck)
+  return zipActions(toBuildList, lastBuiltActionsPath, distFolder)
 }
 
 module.exports = buildActions

src/deploy-actions.js

@@ -18,13 +18,13 @@ const aioLogger = require('@adobe/aio-lib-core-logging')('@adobe/aio-lib-runtime
 const IOruntime = require('./RuntimeAPI')
 const PACKAGE_ITEMS = ['actions', 'sequences']
 const FILTERABLE_ITEMS = ['apis', 'triggers', 'rules', 'dependencies', ...PACKAGE_ITEMS]
-
+const { createHash } = require('node:crypto')
 /**
  * runs the command
  *
  * @param {object} config app config
  * @param {object} [deployConfig={}] deployment config
- * @param {boolean} [deployConfig.isLocalDev] local dev flag
+ * @param {boolean} [deployConfig.isLocalDev] local dev flag // todo: remove
  * @param {object} [deployConfig.filterEntities] add filters to deploy only specified OpenWhisk entities
  * @param {Array} [deployConfig.filterEntities.actions] filter list of actions to deploy by provided array, e.g. ['name1', ..]
  * @param {boolean} [deployConfig.filterEntities.byBuiltActions] if true, trim actions from the manifest based on the already built actions
@@ -33,13 +33,17 @@ const FILTERABLE_ITEMS = ['apis', 'triggers', 'rules', 'dependencies', ...PACKAG
  * @param {Array} [deployConfig.filterEntities.rules] filter list of rules to deploy, e.g. ['name1', ..]
  * @param {Array} [deployConfig.filterEntities.apis] filter list of apis to deploy, e.g. ['name1', ..]
  * @param {Array} [deployConfig.filterEntities.dependencies] filter list of package dependencies to deploy, e.g. ['name1', ..]
+ * @param {boolean} [deployConfig.useForce] force deploy of actions
  * @param {object} [logFunc] custom logger function
  * @returns {Promise<object>} deployedEntities
  */
 async function deployActions (config, deployConfig = {}, logFunc) {
-  if (!config.app.hasBackend) throw new Error('cannot deploy actions, app has no backend')
+  if (!config.app.hasBackend) {
+    throw new Error('cannot deploy actions, app has no backend')
+  }
 
-  const isLocalDev = deployConfig.isLocalDev
+  const isLocalDev = deployConfig.isLocalDev // todo: remove
+  const useForce = deployConfig.useForce
   const log = logFunc || console.log
   let filterEntities = deployConfig.filterEntities
 
@@ -64,6 +68,8 @@ async function deployActions (config, deployConfig = {}, logFunc) {
     aioLogger.debug('Trimming out the manifest\'s actions...')
     filterEntities = undefined
     const builtActions = []
+    // this is a little weird, we are getting the list of built actions from the dist folder
+    // instead of it being passed, or simply reading manifest/config
     const distFiles = fs.readdirSync(path.resolve(__dirname, dist))
     distFiles.forEach(distFile => {
       const packageFolder = path.resolve(__dirname, dist, distFile)
@@ -121,12 +127,14 @@ async function deployActions (config, deployConfig = {}, logFunc) {
       }
     })
   }
+
   // 2. deploy manifest
   const deployedEntities = await deployWsk(
     modifiedConfig,
     manifest,
     log,
-    filterEntities
+    filterEntities,
+    useForce
   )
   // enrich actions array with urls
   if (Array.isArray(deployedEntities.actions)) {
@@ -148,9 +156,11 @@ async function deployActions (config, deployConfig = {}, logFunc) {
  * @param {object} manifestContent manifest
  * @param {object} logFunc custom logger function
  * @param {object} filterEntities entities (actions, sequences, triggers, rules etc) to be filtered
+ * @param {boolean} useForce force deploy of actions
  * @returns {Promise<object>} deployedEntities
  */
-async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities) {
+async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities, useForce) {
+  // note, logFunc is always defined here because we can only ever be called by deployActions
   const packageName = scriptConfig.ow.package
   const manifestPath = scriptConfig.manifest.src
   const owOptions = {
@@ -160,6 +170,17 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
     namespace: scriptConfig.ow.namespace
   }
 
+  const lastDeployedActionsPath = path.join(scriptConfig.root, 'dist', 'last-deployed-actions.json')
+  let lastDeployData = {}
+  if (useForce) {
+    logFunc('Force deploy enabled, skipping last deployed actions')
+  } else if (fs.existsSync(lastDeployedActionsPath)) {
+    lastDeployData = await fs.readJson(lastDeployedActionsPath)
+  } else {
+    // we will create it later
+    logFunc('lastDeployedActions not found, it will be created after first deployment')
+  }
+
   const ow = await new IOruntime().init(owOptions)
 
   /**
@@ -204,6 +225,25 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
   // note we must filter before processPackage, as it expect all built actions to be there
   const entities = utils.processPackage(packages, {}, {}, {}, false, owOptions)
 
+  // entities.actions is an array of actions
+  entities.actions = entities.actions?.filter(action => {
+    // action name here includes the package name, ie manyactions/__secured_generic1
+    const hash = createHash('sha256')
+    hash.update(JSON.stringify(action))
+    const actionHash = hash.digest('hex')
+    if (lastDeployData[action.name] !== actionHash) {
+      lastDeployData[action.name] = actionHash
+      return true
+    }
+    lastDeployData[action.name] = actionHash
+    return false
+  })
+
+  fs.ensureFileSync(lastDeployedActionsPath)
+  fs.writeJSONSync(lastDeployedActionsPath,
+    lastDeployData,
+    { spaces: 2 })
+
   // Note1: utils.processPackage sets the headless-v2 validator for all
   //   require-adobe-auth annotated actions. Here, we have the context on whether
   //   an app has a frontend or not.
@@ -224,7 +264,6 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
     }
     const DEFAULT_VALIDATOR = DEFAULT_VALIDATORS[env]
     const APP_REGISTRY_VALIDATOR = APP_REGISTRY_VALIDATORS[env]
-
     const replaceValidator = { [DEFAULT_VALIDATOR]: APP_REGISTRY_VALIDATOR }
     entities.actions.forEach(a => {
       const needsReplacement = a.exec && a.exec.kind === 'sequence' && a.exec.components && a.exec.components.includes(DEFAULT_VALIDATOR)
@@ -236,6 +275,7 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
   }
 
   // do the deployment, manifestPath and manifestContent needed for creating a project hash
+  //
   await utils.syncProject(packageName, manifestPath, manifestContent, entities, ow, logFunc, scriptConfig.imsOrgId, deleteOldEntities)
   return entities
 }

src/utils.js

@@ -2039,25 +2039,6 @@ function activationLogBanner (logFunc, activation, activationLogs) {
   }
 }
 
-/**
- * Will tell if the action was built before based on it's contentHash.
- *
- * @param {string} lastBuildsData Data with the last builds
- * @param {object} buildData Object where key is the name of the action and value is its contentHash
- * @returns {boolean} true if the action was built before
- */
-function actionBuiltBefore (lastBuildsData, buildData) {
-  if (buildData && Object.keys(buildData).length > 0) {
-    const [actionName, contentHash] = Object.entries(buildData)[0]
-    const storedData = safeParse(lastBuildsData)
-    if (contentHash) {
-      return storedData[actionName] === contentHash
-    }
-  }
-  aioLogger.debug('actionBuiltBefore > Invalid actionBuiltData')
-  return false
-}
-
 /**
  * Will dump the previously actions built data information.
  *
@@ -2148,7 +2129,6 @@ module.exports = {
   getActionZipFileName,
   getActionNameFromZipFile,
   dumpActionsBuiltInfo,
-  actionBuiltBefore,
   safeParse,
   isSupportedActionKind,
   DEFAULT_PACKAGE_RESERVED_NAME