Add Jobs for Product Creation, Builds, and Publishing

scripts/DB/default_workflow.sql

@@ -99,8 +99,8 @@ DO UPDATE SET
 	"Type" = excluded."Type",
   "ProductType" = excluded."ProductType";
 
-INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES
-(9, 'pwa_cloud', '1', 'SIL Default Workflow for Publishing PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud', 'SIL_AppBuilders_Web_Flow', 3, 1, 3)
+INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES
+(9, 'pwa_cloud', '1', 'SIL Default Workflow for Publishing PWA to Cloud', 'SIL_Default_AppBuilders_Pwa_Cloud', 'SIL_AppBuilders_Web_Flow', 3, 1, '{ "build:targets" : "pwa" }', 3)
 ON CONFLICT ("Id")
 DO UPDATE SET
 	"Name" = excluded."Name",
@@ -112,8 +112,8 @@ DO UPDATE SET
 	"Type" = excluded."Type",
   "ProductType" = excluded."ProductType";
 
-INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES
-(10,	'pwa_cloud_rebuild',	'1',	'SIL Default Workflow for Rebuilding PWA to Cloud',	'SIL_Default_AppBuilders_Pwa_Cloud_Rebuild',	'SIL_AppBuilders_Web_Flow',	3,	2,  3)
+INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES
+(10,	'pwa_cloud_rebuild',	'1',	'SIL Default Workflow for Rebuilding PWA to Cloud',	'SIL_Default_AppBuilders_Pwa_Cloud_Rebuild',	'SIL_AppBuilders_Web_Flow',	3,	2, '{ "build:targets" : "pwa" }', 3)
 ON CONFLICT ("Id")
 DO UPDATE SET
 	"Name" = excluded."Name",
@@ -151,8 +151,8 @@ DO UPDATE SET
 	"Type" = excluded."Type",
   "ProductType" = excluded."ProductType";
 
-INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES
-(13,	'asset_package',	'1',	'SIL Default Workflow for Publishing Asset Packages',	'SIL_NoAdmin_AppBuilders_Android_S3',	'SIL_AppBuilders_AssetPackage_Flow',	2,	1, '{ "build:targets" : "asset-package" }', 2)
+INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES
+(13,	'asset_package',	'1',	'SIL Default Workflow for Publishing Asset Packages',	'SIL_NoAdmin_AppBuilders_Android_S3',	'SIL_AppBuilders_AssetPackage_Flow',	2,	1, 2)
 ON CONFLICT ("Id")
 DO UPDATE SET
 	"Name" = excluded."Name",
@@ -165,8 +165,8 @@ DO UPDATE SET
 	"Properties" = excluded."Properties",
   "ProductType" = excluded."ProductType";
 
-INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "Properties", "ProductType") VALUES
-(14,	'asset_package_rebuild',	'1',	'SIL Default Workflow for Rebuilding Asset Packages',	'SIL_Default_AppBuilders_Android_S3_Rebuild',	'SIL_AppBuilders_AssetPackage_Flow',	2,	2, '{ "build:targets" : "asset-package" }', 3)
+INSERT INTO "WorkflowDefinitions" ("Id", "Name", "Enabled", "Description", "WorkflowScheme", "WorkflowBusinessFlow", "StoreTypeId", "Type", "ProductType") VALUES
+(14,	'asset_package_rebuild',	'1',	'SIL Default Workflow for Rebuilding Asset Packages',	'SIL_Default_AppBuilders_Android_S3_Rebuild',	'SIL_AppBuilders_AssetPackage_Flow',	2,	2, 3)
 ON CONFLICT ("Id")
 DO UPDATE SET
 	"Name" = excluded."Name",

source/SIL.AppBuilder.Portal/common/bullmq/queues.ts

@@ -2,12 +2,36 @@ import { Queue } from 'bullmq';
 import type { Job } from './types.js';
 import { QueueName } from './types.js';
 
+/** Queue for Product Builds */
+export const Builds = new Queue<Job>(QueueName.Builds, {
+  connection: {
+    host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis'
+  }
+});
 /** Queue for default recurring jobs such as the BuildEngine status check */
 export const DefaultRecurring = new Queue<Job>(QueueName.DefaultRecurring, {
   connection: {
     host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis'
   }
 });
+/** Queue for miscellaneous jobs such as Product and Project Creation */
+export const Miscellaneous = new Queue<Job>(QueueName.Miscellaneous, {
+  connection: {
+    host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis'
+  }
+});
+/** Queue for Product Publishing  */
+export const Publishing = new Queue<Job>(QueueName.Publishing, {
+  connection: {
+    host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis'
+  }
+});
+/** Queue for jobs that poll BuildEngine, such as checking the status of a build */
+export const RemotePolling = new Queue<Job>(QueueName.RemotePolling, {
+  connection: {
+    host: process.env.NODE_ENV === 'development' ? 'localhost' : 'redis'
+  }
+});
 /** Queue for operations on UserTasks */
 export const UserTasks = new Queue<Job>(QueueName.UserTasks, {
   connection: {

source/SIL.AppBuilder.Portal/common/bullmq/types.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-namespace */
+import type { Channels } from '../build-engine-api/types.js';
 import { RoleId } from '../public/prisma.js';
 
 interface RetryOptions {
@@ -17,18 +18,97 @@ export const Retry5e5: RetryOptions = {
   }
 };
 
+interface RepeatOptions {
+  readonly repeat: {
+    readonly pattern: string;
+  };
+}
+/** Repeat a job every minute */
+export const RepeatEveryMinute: RepeatOptions = {
+  repeat: {
+    pattern: '*/1 * * * *' // every minute
+  }
+};
+
 export enum QueueName {
+  Builds = 'Builds',
   DefaultRecurring = 'Default Recurring',
+  Miscellaneous = 'Miscellaneous',
+  Publishing = 'Publishing',
+  RemotePolling = 'Remote Polling',
   UserTasks = 'User Tasks'
 }
 
 export enum JobType {
+  // Build Tasks
+  Build_Product = 'Build Product',
+  Build_Check = 'Check Product Build',
+  // Product Tasks
+  Product_Create = 'Create Product',
+  Product_Delete = 'Delete Product',
+  Product_GetVersionCode = 'Get VersionCode for Uploaded Product',
+  // Publishing Tasks
+  Publish_Product = 'Publish Product',
+  Publish_Check = 'Check Product Publish',
   // System Tasks
   System_CheckStatuses = 'Check System Statuses',
   // UserTasks
   UserTasks_Modify = 'Modify UserTasks'
 }
 
+export namespace Build {
+  export interface Product {
+    type: JobType.Build_Product;
+    productId: string;
+    targets: string;
+    environment: { [key: string]: string };
+  }
+  export interface Check {
+    type: JobType.Build_Check;
+    organizationId: number;
+    productId: string;
+    jobId: number;
+    buildId: number;
+    productBuildId: number;
+  }
+}
+
+export namespace Product {
+  export interface Create {
+    type: JobType.Product_Create;
+    productId: string;
+  }
+  export interface Delete {
+    type: JobType.Product_Delete;
+    organizationId: number;
+    workflowJobId: number;
+  }
+  export interface GetVersionCode {
+    type: JobType.Product_GetVersionCode;
+    productId: string;
+  }
+}
+
+export namespace Publish {
+  export interface Product {
+    type: JobType.Publish_Product;
+    productId: string;
+    channel: Channels;
+    targets: string;
+    environment: { [key: string]: string };
+  }
+
+  export interface Check {
+    type: JobType.Publish_Check;
+    organizationId: number;
+    productId: string;
+    jobId: number;
+    buildId: number;
+    releaseId: number;
+    publicationId: number;
+  }
+}
+
 export namespace System {
   export interface CheckStatuses {
     type: JobType.System_CheckStatuses;
@@ -75,6 +155,13 @@ export namespace UserTasks {
 export type Job = JobTypeMap[keyof JobTypeMap];
 
 export type JobTypeMap = {
+  [JobType.Build_Product]: Build.Product;
+  [JobType.Build_Check]: Build.Check;
+  [JobType.Product_Create]: Product.Create;
+  [JobType.Product_Delete]: Product.Delete;
+  [JobType.Product_GetVersionCode]: Product.GetVersionCode;
+  [JobType.Publish_Product]: Publish.Product;
+  [JobType.Publish_Check]: Publish.Check;
   [JobType.System_CheckStatuses]: System.CheckStatuses;
   [JobType.UserTasks_Modify]: UserTasks.Modify;
   // Add more mappings here as needed

source/SIL.AppBuilder.Portal/common/databaseProxy/Products.ts

@@ -1,5 +1,9 @@
 import type { Prisma } from '@prisma/client';
+import { Workflow } from 'sil.appbuilder.portal.common';
+import { BullMQ, Queues } from '../index.js';
 import prisma from '../prisma.js';
+import { WorkflowType } from '../public/prisma.js';
+import { update as projectUpdate } from './Projects.js';
 import type { RequirePrimitive } from './utility.js';
 
 export async function create(
@@ -10,7 +14,7 @@ export async function create(
       productData.ProjectId,
       productData.ProductDefinitionId,
       productData.StoreId,
-      productData.StoreLanguageId
+      productData.StoreLanguageId ?? undefined
     ))
   )
     return false;
@@ -21,6 +25,36 @@ export async function create(
     const res = await prisma.products.create({
       data: productData
     });
+
+    if (res) {
+      const flowDefinition = (
+        await prisma.productDefinitions.findUnique({
+          where: {
+            Id: productData.ProductDefinitionId
+          },
+          select: {
+            Workflow: {
+              select: {
+                Id: true,
+                Type: true,
+                ProductType: true,
+                WorkflowOptions: true
+              }
+            }
+          }
+        })
+      )?.Workflow;
+
+      if (flowDefinition?.Type === WorkflowType.Startup) {
+        await Workflow.create(res.Id, {
+          productType: flowDefinition.ProductType,
+          options: new Set(flowDefinition.WorkflowOptions)
+        });
+        updateProjectDateActive(productData.ProjectId);
+      }
+      await updateProjectDateActive(productData.ProjectId);
+    }
+
     return res.Id;
   } catch (e) {
     return false;
@@ -55,16 +89,40 @@ export async function update(
       },
       data: productData
     });
+    await updateProjectDateActive(projectId);
     // TODO: Are there any other updates that need to be done?
   } catch (e) {
     return false;
   }
   return true;
 }
 
-function deleteProduct(productId: string) {
+async function deleteProduct(productId: string) {
   // Delete all userTasks for this product, and delete the product
-  return prisma.$transaction([
+  const product = await prisma.products.findUnique({
+    where: {
+      Id: productId
+    },
+    select: {
+      Project: {
+        select: {
+          Id: true,
+          OrganizationId: true
+        }
+      },
+      WorkflowJobId: true
+    }
+  });
+  await Queues.Miscellaneous.add(
+    `Delete Product #${productId} from BuildEngine`,
+    {
+      type: BullMQ.JobType.Product_Delete,
+      organizationId: product!.Project.OrganizationId,
+      workflowJobId: product!.WorkflowJobId
+    },
+    BullMQ.Retry5e5
+  );
+  const res = prisma.$transaction([
     prisma.workflowInstances.deleteMany({
       where: {
         ProductId: productId
@@ -81,7 +139,8 @@ function deleteProduct(productId: string) {
       }
     })
   ]);
-  // TODO: delete from BuildEngine
+  await updateProjectDateActive(product!.Project.Id);
+  return res;
 }
 export { deleteProduct as delete };
 
@@ -140,7 +199,7 @@ async function validateProductBase(
                       Id: true,
                       // StoreLanguage must be allowed by Store, if the StoreLanguage is defined
                       StoreLanguages:
-                        storeLanguageId === undefined || storeLanguageId === null
+                        storeLanguageId === undefined
                           ? undefined
                           : {
                             where: {
@@ -166,7 +225,6 @@ async function validateProductBase(
       }
     }
   });
-
   // 3. The store is allowed by the organization
   return (
     (project?.Organization.OrganizationStores.length ?? 0) > 0 &&
@@ -175,11 +233,54 @@ async function validateProductBase(
       project?.Organization.OrganizationStores[0].Store.StoreType.Id &&
     // 2. The project has a WorkflowProjectUrl
     // handled by query
-    // 4. The language is allowed by the store
-    (storeLanguageId ??
+    // 4. The language, if specified, is allowed by the store
+    ((storeLanguageId &&
       (project?.Organization.OrganizationStores[0].Store.StoreType.StoreLanguages.length ?? 0) >
-        0) &&
+        0) ||
+      storeLanguageId === undefined) &&
     // 5. The product type is allowed by the organization
     (project?.Organization.OrganizationProductDefinitions.length ?? 0) > 0
   );
 }
+
+async function updateProjectDateActive(projectId: number) {
+  const project = await prisma.projects.findUnique({
+    where: {
+      Id: projectId
+    },
+    select: {
+      Products: {
+        select: {
+          WorkflowInstance: {
+            select: {
+              Id: true
+            }
+          },
+          DateUpdated: true
+        }
+      },
+      DateActive: true
+    }
+  });
+
+  const projectDateActive = project.DateActive;
+
+  let dateActive = new Date(0);
+  project.Products.forEach((product) => {
+    if (product.WorkflowInstance) {
+      if (product.DateUpdated > dateActive) {
+        dateActive = product.DateUpdated;
+      }
+    }
+  });
+
+  if (dateActive > new Date(0)) {
+    project.DateActive = dateActive;
+  } else {
+    project.DateActive = null;
+  }
+
+  if (project.DateActive != projectDateActive) {
+    await projectUpdate(projectId, { DateActive: project.DateActive });
+  }
+}

source/SIL.AppBuilder.Portal/common/public/prisma.ts

@@ -18,3 +18,5 @@ export enum WorkflowType {
   Rebuild,
   Republish
 }
+
+export const WorkflowTypeString = ['', 'Startup', 'Rebuild', 'Republish'];