Developemnt 313 mass deployment

.github/workflows/mass_deployments.yml

@@ -2,7 +2,7 @@ name: Mass Deployments
 
 on:
   schedule:
-    - cron: "0 1 * * *"
+    - cron: "0 */6 * * *"
   workflow_dispatch:
 
 jobs:

packages/grid_client/scripts/mass_deployments.ts

@@ -1,119 +1,211 @@
 import {
-  DiskModel,
   FarmFilterOptions,
   FilterOptions,
   generateString,
+  GridClient,
   MachineModel,
   MachinesModel,
   NetworkModel,
-  randomChoice,
+  NodeInfo,
+  TwinDeployment,
 } from "../src";
 import { config, getClient } from "./client_loader";
 import { log } from "./utils";
 
+async function pingNodes(
+  grid3: GridClient,
+  nodes: any[],
+): Promise<Promise<{ node: NodeInfo; error?: Error; res?: unknown }[]>> {
+  const pingPromises = nodes.map(async node => {
+    try {
+      const res = await grid3.zos.pingNode({ nodeId: node.nodeId });
+      return { node, res };
+    } catch (error) {
+      return { node, error };
+    }
+  });
+
+  const results = await Promise.all(pingPromises);
+  return results;
+}
+
 async function main() {
   const grid3 = await getClient();
 
+  // Timeout for deploying vm is 2 min
+  grid3.clientOptions.deploymentTimeoutMinutes = 2;
+  await grid3._connect();
+
   const errors: any = [];
+  const offlineNodes: number[] = [];
   let failedCount = 0;
   let successCount = 0;
+  const batchSize = 5;
+  const totalVMs = 10;
+  const batches = totalVMs / batchSize;
+
+  // resources
+  const cru = 1;
+  const mru = 256;
+  const rootFs = 1;
+  const publicIp = false;
+
+  console.time("Farms Time");
+  const farms = await grid3.capacity.filterFarms({
+    nodeMRU: mru / 1024,
+    nodeSRU: rootFs,
+    publicIp: publicIp,
+    availableFor: await grid3.twins.get_my_twin_id(),
+    randomize: true,
+  } as FarmFilterOptions);
+  console.timeEnd("Farms Time");
+
+  if (farms.length < 1) {
+    throw new Error("No farms found");
+  }
+
+  console.time("Total Deployment Time");
+
+  for (let batch = 0; batch < batches; batch++) {
+    console.time("Batch " + (batch + 1));
+
+    const nodesPromises = Array.from({ length: batchSize }, async () => {
+      const farmIds = farms.map(farm => farm.farmId);
+      return grid3.capacity.filterNodes({
+        cru: cru,
+        mru: mru / 1024,
+        sru: rootFs,
+        availableFor: await grid3.twins.get_my_twin_id(),
+        farmIds: farmIds,
+        randomize: true,
+      } as FilterOptions);
+    });
 
-  for (let i = 0; i < 100; i++) {
-    //Generating the resources
-    const cru = 1;
-    const mru = 256;
-    const diskSize = 5;
-    const rootFs = 1;
-    const vmName = "vm" + generateString(8);
-    const deploymentName = "dep" + generateString(8);
-    const publicIp = false;
-
-    // create network Object
-    const n = new NetworkModel();
-    n.name = "nw" + generateString(5);
-    n.ip_range = "10.238.0.0/16";
-
-    // create disk Object
-    const disk1 = new DiskModel();
-    disk1.name = "d" + generateString(5);
-    disk1.size = diskSize;
-    disk1.mountpoint = "/newDisk1";
-
-    //Farm Selection
-    const farms = await grid3.capacity.filterFarms({
-      nodeMRU: mru / 1024,
-      nodeSRU: diskSize + rootFs,
-      publicIp: publicIp,
-      availableFor: await grid3.twins.get_my_twin_id(),
-      randomize: true,
-    } as FarmFilterOptions);
-
-    if (farms.length < 1) {
-      throw new Error("No farms found");
+    const nodes = await Promise.all(nodesPromises);
+
+    let flattenedNodes = nodes.flat();
+
+    console.time("Ping Nodes");
+    try {
+      const results = await pingNodes(grid3, flattenedNodes);
+      results.forEach(({ node, res, error }) => {
+        if (res) {
+          console.log(`Node ${node.nodeId} is online`);
+        } else {
+          offlineNodes.push(node.nodeId);
+          console.log(`Node ${node.nodeId} is offline`);
+          if (error) {
+            console.error("Error:", error);
+          }
+        }
+      });
+    } catch (error) {
+      console.error("Error pinging nodes:", error);
+    } finally {
+      console.timeEnd("Ping Nodes");
+    }
+
+    flattenedNodes = flattenedNodes.filter(node => !offlineNodes.includes(node.nodeId));
+
+    // Batch Deployment
+    const batchVMs: MachinesModel[] = [];
+    for (let i = 0; i < batchSize; i++) {
+      const vmName = "vm" + generateString(8);
+
+      if (flattenedNodes.length < 0) {
+        errors.push("No online nodes available for deployment");
+        failedCount++;
+        continue;
+      }
+
+      const selectedNode = flattenedNodes.pop();
+
+      // create vm node Object
+      const vm = new MachineModel();
+      vm.name = vmName;
+      vm.node_id = selectedNode.nodeId;
+      vm.disks = [];
+      vm.public_ip = publicIp;
+      vm.planetary = true;
+      vm.cpu = cru;
+      vm.memory = mru;
+      vm.rootfs_size = rootFs;
+      vm.flist = "https://hub.grid.tf/tf-official-apps/base:latest.flist";
+      vm.entrypoint = "/sbin/zinit init";
+      vm.env = {
+        SSH_KEY: config.ssh_key,
+      };
+
+      // create network model for each vm
+      const n = new NetworkModel();
+      n.name = "nw" + generateString(5);
+      n.ip_range = "10.238.0.0/16";
+      n.addAccess = true;
+
+      // create VMs Object for each vm
+      const vms = new MachinesModel();
+      vms.name = "batch" + (batch + 1);
+      vms.network = n;
+      vms.machines = [vm];
+      vms.metadata = "";
+      vms.description = "Test deploying vm with name " + vm.name + " via ts grid3 client - Batch " + (batch + 1);
+
+      batchVMs.push(vms);
     }
 
-    //Node Selection
-    const nodes = await grid3.capacity.filterNodes({
-      cru: cru,
-      mru: mru / 1024,
-      sru: rootFs + diskSize,
-      availableFor: await grid3.twins.get_my_twin_id(),
-      farmId: +randomChoice(farms).farmId,
-      randomize: true,
-    } as FilterOptions);
-
-    if (nodes.length < 1) {
-      errors.push("Node not found");
-      failedCount++;
-      continue;
+    const allTwinDeployments: TwinDeployment[] = [];
+
+    const deploymentPromises = batchVMs.map(async (vms, index) => {
+      try {
+        const [twinDeployments, _, __] = await grid3.machines._createDeployment(vms);
+        return { twinDeployments, batchIndex: index };
+      } catch (error) {
+        log(`Error creating deployment for batch ${batch + 1}: ${error}`);
+        return { twinDeployments: null, batchIndex: index };
+      }
+    });
+    console.time("Preparing Batch " + (batch + 1));
+    const deploymentResults = await Promise.all(deploymentPromises);
+    console.timeEnd("Preparing Batch " + (batch + 1));
+
+    for (const { twinDeployments } of deploymentResults) {
+      if (twinDeployments) {
+        allTwinDeployments.push(...twinDeployments);
+      }
     }
 
-    // create vm node Object
-    const vm = new MachineModel();
-    vm.name = vmName;
-    vm.node_id = nodes[0].nodeId;
-    vm.disks = [disk1];
-    vm.public_ip = publicIp;
-    vm.planetary = true;
-    vm.cpu = cru;
-    vm.memory = mru;
-    vm.rootfs_size = rootFs;
-    vm.flist = "https://hub.grid.tf/tf-official-apps/base:latest.flist";
-    vm.entrypoint = "/sbin/zinit init";
-    vm.env = {
-      SSH_KEY: config.ssh_key,
-    };
-
-    // create VMs Object
-    const vms = new MachinesModel();
-    vms.name = deploymentName;
-    vms.network = n;
-    vms.machines = [vm];
-    vms.metadata = "";
-    vms.description = "test deploying VMs via ts grid3 client";
-
-    // deploy vm
     try {
-      await grid3.machines.deploy(vms);
-      successCount++;
+      await grid3.machines.twinDeploymentHandler.handle(allTwinDeployments);
+      successCount += batchSize;
+      log(`Successfully handled and saved contracts for all twin deployments`);
     } catch (error) {
-      log(error);
+      failedCount += batchSize;
       errors.push(error);
-      failedCount++;
-      continue;
+      log(`Error handling contracts for all twin deployments: ${error}`);
     }
+
+    console.timeEnd("Batch " + (batch + 1));
   }
 
+  console.timeEnd("Total Deployment Time");
+
   log("Successful Deployments: " + successCount);
   log("Failed Deployments: " + failedCount);
 
-  // List of failed deployments errors
+  // List of failed deployments' errors
   log("Failed deployments errors: ");
   for (let i = 0; i < errors.length; i++) {
     log(errors[i]);
     log("---------------------------------------------");
   }
 
+  // List of offline nodes
+  log("Failed Nodes: ");
+  for (let i = 0; i < offlineNodes.length; i++) {
+    log(offlineNodes[i]);
+    log("---------------------------------------------");
+  }
+
   await grid3.disconnect();
 }
 

packages/grid_client/src/modules/models.ts

@@ -580,6 +580,7 @@ class FilterOptions {
   @Expose() @IsOptional() @IsBoolean() gateway?: boolean;
   @Expose() @IsOptional() @IsBoolean() certified?: boolean;
   @Expose() @IsOptional() @IsInt({ each: true }) @Min(1, { each: true }) nodeExclude?: number[];
+  @Expose() @IsOptional() @IsInt({ each: true }) @Min(1, { each: true }) farmIds?: number[];
   @Expose() @IsOptional() @IsInt() @Min(1) farmId?: number;
   @Expose() @IsOptional() @IsString() farmName?: string;
   @Expose() @IsOptional() @IsString() country?: string;

packages/grid_client/src/modules/zos.ts

@@ -45,7 +45,7 @@ class Zos {
   @validateInput
   async pingNode(options: PingNodeOptionsModel): Promise<boolean> {
     const nodeTwinId = await this.capacity.getNodeTwinId(options.nodeId);
-    return await this.rmb.request([nodeTwinId], "zos.system.version", "", 20, 1);
+    return await this.rmb.request([nodeTwinId], "zos.system.version", "", 10, 1);
   }
 
   @expose

packages/grid_client/src/primitives/nodes.ts

@@ -47,6 +47,7 @@ interface NodeInfo {
   updatedAt: string;
   total_resources: NodeResources;
   used_resources: NodeResources;
+  farm_ids: number[];
   location: {
     country: string;
     city: string;
@@ -373,7 +374,7 @@ class Nodes {
       ipv6: options.accessNodeV6,
       gateway: options.gateway,
       certification_type: options.certified ? "Certified" : "",
-      farm_ids: [options.farmId],
+      farm_ids: options.farmIds ? options.farmIds : [options.farmId],
       farm_name: options.farmName,
       country: options.country,
       city: options.city,