Detect and Resolve nodes' resources overlapping k8s/caprover

packages/playground/src/components/caprover_worker.vue

@@ -32,6 +32,8 @@
     </input-tooltip>
 
     <TfSelectionDetails
+      :selected-machines="selectedMachines"
+      :nodes-lock="nodesLock"
       :filters="{
         ipv4: true,
         certified: $props.modelValue.certified,
@@ -47,8 +49,10 @@
 </template>
 
 <script lang="ts">
+import type AwaitLock from "await-lock";
 import { computed, type PropType } from "vue";
 
+import type { SelectedMachine } from "@/types/nodeSelector";
 import { manual } from "@/utils/manual";
 
 import Networks from "../components/networks.vue";
@@ -61,17 +65,50 @@ export function createWorker(name: string = generateName({ prefix: "wr" })): Cap
   return { name, mycelium: true };
 }
 
+function toMachine(rootFilesystemSize: number, worker?: CaproverWorker): SelectedMachine | undefined {
+  if (!worker || !worker.selectionDetails || !worker.selectionDetails.node) {
+    return undefined;
+  }
+
+  return {
+    nodeId: worker.selectionDetails.node.nodeId,
+    cpu: worker.solution?.cpu ?? 0,
+    memory: worker.solution?.memory ?? 0,
+    disk: (worker.solution?.disk ?? 0) + (rootFilesystemSize ?? 0),
+  };
+}
+
 export default {
   name: "CaproverWorker",
   components: { SelectSolutionFlavor, Networks },
-  props: { modelValue: { type: Object as PropType<CaproverWorker>, required: true } },
+  props: {
+    modelValue: {
+      type: Object as PropType<CaproverWorker>,
+      required: true,
+    },
+    otherWorkers: {
+      type: Array as PropType<CaproverWorker[]>,
+      default: () => [],
+    },
+    nodesLock: Object as PropType<AwaitLock>,
+  },
   setup(props) {
     const rootFilesystemSize = computed(() => {
       const { cpu = 0, memory = 0 } = props.modelValue.solution || {};
       return rootFs(cpu, memory);
     });
 
-    return { rootFilesystemSize, manual };
+    const selectedMachines = computed(() => {
+      return props.otherWorkers.reduce((res, worker) => {
+        const machine = toMachine(rootFilesystemSize.value, worker);
+        if (machine) {
+          res.push(machine);
+        }
+        return res;
+      }, [] as SelectedMachine[]);
+    });
+
+    return { rootFilesystemSize, manual, selectedMachines };
   },
 };
 </script>

packages/playground/src/components/k8s_worker.vue

@@ -84,6 +84,8 @@
     </input-tooltip>
 
     <TfSelectionDetails
+      :selected-machines="selectedMachines"
+      :nodes-lock="nodesLock"
       :filters-validators="{
         memory: { min: 1024 },
         rootFilesystemSize: { min: rootFs($props.modelValue.cpu ?? 0, $props.modelValue.memory ?? 0) },
@@ -103,8 +105,10 @@
 </template>
 
 <script lang="ts">
-import type { PropType } from "vue";
+import type AwaitLock from "await-lock";
+import { computed, type PropType } from "vue";
 
+import type { SelectedMachine } from "@/types/nodeSelector";
 import { manual } from "@/utils/manual";
 
 import Networks from "../components/networks.vue";
@@ -130,6 +134,19 @@ export function createWorker(name: string = generateName({ prefix: "wr" })): K8S
   };
 }
 
+function toMachine(worker?: K8SWorker): SelectedMachine | undefined {
+  if (!worker || !worker.selectionDetails || !worker.selectionDetails.node) {
+    return undefined;
+  }
+
+  return {
+    nodeId: worker.selectionDetails.node.nodeId,
+    cpu: worker.cpu,
+    memory: worker.memory,
+    disk: (worker.diskSize ?? 0) + (worker.rootFsSize ?? 0),
+  };
+}
+
 export default {
   name: "K8SWorker",
   components: { RootFsSize, Networks },
@@ -138,9 +155,24 @@ export default {
       type: Object as PropType<K8SWorker>,
       required: true,
     },
+    otherWorkers: {
+      type: Array as PropType<K8SWorker[]>,
+      default: () => [],
+    },
+    nodesLock: Object as PropType<AwaitLock>,
   },
-  setup() {
-    return { rootFs, manual };
+  setup(props) {
+    const selectedMachines = computed(() => {
+      return props.otherWorkers.reduce((res, worker) => {
+        const machine = toMachine(worker);
+        if (machine) {
+          res.push(machine);
+        }
+        return res;
+      }, [] as SelectedMachine[]);
+    });
+
+    return { rootFs, manual, selectedMachines };
   },
 };
 </script>

packages/playground/src/components/node_selector/TfAutoNodeSelector.vue

@@ -144,22 +144,25 @@
 <script lang="ts">
 import type { FarmInfo, FilterOptions, NodeInfo } from "@threefold/grid_client";
 import { RequestError } from "@threefold/types";
+import type AwaitLock from "await-lock";
 import equals from "lodash/fp/equals.js";
-import sample from "lodash/fp/sample.js";
 import { computed, nextTick, onMounted, onUnmounted, type PropType, ref } from "vue";
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 import type { VCard } from "vuetify/components/VCard";
 
 import { useAsync, usePagination, useWatchDeep } from "../../hooks";
 import { ValidatorStatus } from "../../hooks/form_validator";
 import { useGrid } from "../../stores";
-import type { SelectedLocation, SelectionDetailsFilters } from "../../types/nodeSelector";
+import type { SelectedLocation, SelectedMachine, SelectionDetailsFilters } from "../../types/nodeSelector";
 import {
   checkNodeCapacityPool,
   getNodePageCount,
+  isNodeValid,
   loadValidNodes,
   normalizeNodeFilters,
   normalizeNodeOptions,
+  release,
+  selectValidNode,
   validateRentContract,
 } from "../../utils/nodeSelector";
 import TfNodeDetailsCard from "./TfNodeDetailsCard.vue";
@@ -177,39 +180,57 @@ export default {
     location: Object as PropType<SelectedLocation>,
     farm: Object as PropType<FarmInfo>,
     status: String as PropType<ValidatorStatus>,
+    selectedMachines: {
+      type: Array as PropType<SelectedMachine[]>,
+      required: true,
+    },
+    nodesLock: Object as PropType<AwaitLock>,
   },
   emits: {
     "update:model-value": (node?: NodeInfo) => true || node,
     "update:status": (status: ValidatorStatus) => true || status,
   },
   setup(props, ctx) {
     const gridStore = useGrid();
-    const loadedNodes = ref<NodeInfo[]>([]);
+    const _loadedNodes = ref<NodeInfo[]>([]);
+    const loadedNodes = computed(() => {
+      return _loadedNodes.value.filter(
+        node => node.nodeId === props.modelValue?.nodeId || isNodeValid(node, props.selectedMachines, filters.value),
+      );
+    });
     const nodesTask = useAsync(loadValidNodes, {
       shouldRun: () => props.validFilters,
       onBeforeTask() {
         const oldNode = props.modelValue;
         bindModelValue();
         return oldNode?.nodeId;
       },
-      onAfterTask({ data }, oldNodeId: number) {
-        loadedNodes.value = loadedNodes.value.concat(data as NodeInfo[]);
+      async onAfterTask({ data }, oldNodeId: number) {
+        _loadedNodes.value = _loadedNodes.value.concat(data as NodeInfo[]);
 
-        const node = loadedNodes.value.find(n => n.nodeId === oldNodeId) || sample(loadedNodes.value);
-        node && bindModelValue(node);
-        node && nodeInputValidateTask.value.run(node);
+        await _setValidNode(oldNodeId);
         pagination.value.next();
       },
       default: [],
     });
 
+    async function _setValidNode(oldNodeId?: number) {
+      const node = await selectValidNode(_loadedNodes.value, props.selectedMachines, filters.value, oldNodeId);
+      if (node) {
+        bindModelValue(node);
+        nodeInputValidateTask.value.run(node);
+      } else {
+        release(props.nodesLock);
+      }
+    }
+
     const pageCountTask = useAsync(getNodePageCount, { default: 1, shouldRun: () => props.validFilters });
     const pagination = usePagination();
 
     const options = computed(() => normalizeNodeOptions(gridStore, props.location, pagination, props.farm));
     const filters = computed(() => normalizeNodeFilters(props.filters, options.value));
 
-    const reloadNodes = () => nodesTask.value.run(gridStore, props.filters, filters.value, pagination);
+    const reloadNodes = () => nodesTask.value.run(gridStore, props.filters, filters.value, pagination, props.nodesLock);
     const loadingError = computed(() => {
       if (!nodesTask.value.error) return "";
       if (nodesTask.value.error instanceof RequestError) return "Failed to fetch nodes due to a network error";
@@ -259,7 +280,7 @@ export default {
       await pageCountTask.value.run(gridStore, filters.value);
       pagination.value.reset(pageCountTask.value.data as number);
       await nextTick();
-      loadedNodes.value = [];
+      _loadedNodes.value = [];
       return reloadNodes();
     }
 
@@ -274,6 +295,7 @@ export default {
         shouldRun: () => props.validFilters,
         onBeforeTask: () => bindStatus(ValidatorStatus.Pending),
         onAfterTask({ data }) {
+          release(props.nodesLock);
           bindStatus(data ? ValidatorStatus.Valid : ValidatorStatus.Invalid);
           const container = nodesContainer.value as HTMLDivElement;
           if (container) {
@@ -314,6 +336,18 @@ export default {
 
     const nodesContainer = ref<HTMLDivElement>();
 
+    useWatchDeep(
+      () => props.selectedMachines.map(m => m.nodeId),
+      () => {
+        if (props.modelValue || nodesTask.value.loading) {
+          return;
+        }
+
+        _setValidNode();
+      },
+      { debounce: 1000 },
+    );
+
     return {
       pageCountTask,
       nodesTask,

packages/playground/src/components/node_selector/TfManualNodeSelector.vue

@@ -1,6 +1,7 @@
 <template>
   <div>
     <TfNodeDetailsCard
+      :selected-machines="selectedMachines.filter(m => m.nodeId === nodeId)"
       v-show="modelValue || placeholderNode"
       :key="modelValue?.rentedByTwinId"
       flat
@@ -52,16 +53,17 @@
 
 <script lang="ts">
 import type { NodeInfo } from "@threefold/grid_client";
+import type AwaitLock from "await-lock";
 import isInt from "validator/lib/isInt";
 import { onUnmounted, type PropType, ref, watch } from "vue";
 
 import { gridProxyClient } from "../../clients";
 import { useAsync, useWatchDeep } from "../../hooks";
 import { ValidatorStatus } from "../../hooks/form_validator";
 import { useGrid } from "../../stores";
-import type { SelectionDetailsFilters } from "../../types/nodeSelector";
+import type { SelectedMachine, SelectionDetailsFilters } from "../../types/nodeSelector";
 import { normalizeError } from "../../utils/helpers";
-import { checkNodeCapacityPool, resolveAsync, validateRentContract } from "../../utils/nodeSelector";
+import { checkNodeCapacityPool, release, resolveAsync, validateRentContract } from "../../utils/nodeSelector";
 import TfNodeDetailsCard from "./TfNodeDetailsCard.vue";
 
 const _defaultError =
@@ -78,6 +80,11 @@ export default {
       required: true,
     },
     status: String as PropType<ValidatorStatus>,
+    selectedMachines: {
+      type: Array as PropType<SelectedMachine[]>,
+      required: true,
+    },
+    nodesLock: Object as PropType<AwaitLock>,
   },
   emits: {
     "update:model-value": (node?: NodeInfo) => true || node,
@@ -159,7 +166,14 @@ export default {
         }
 
         const { cru, mru, sru } = resources;
-        const { cpu = 0, memory = 0, ssdDisks = [], solutionDisk = 0, rootFilesystemSize = 0 } = props.filters;
+        const { ssdDisks = [], rootFilesystemSize = 0 } = props.filters;
+
+        const machinesWithSameNode = props.selectedMachines.filter(machine => machine.nodeId === nodeId);
+        let { cpu = 0, memory = 0, solutionDisk = 0 } = props.filters;
+
+        cpu += machinesWithSameNode.reduce((res, machine) => res + machine.cpu, 0);
+        memory += machinesWithSameNode.reduce((res, machine) => res + machine.memory, 0);
+        solutionDisk += machinesWithSameNode.reduce((res, machine) => res + machine.disk, 0);
 
         const memorySize = memory / 1024;
         const requiredMru = Math.ceil(Math.round(memorySize) * 1024 ** 3);
@@ -190,16 +204,25 @@ export default {
         tries: 1,
         onReset: bindStatus,
         shouldRun: () => props.validFilters,
-        onBeforeTask: () => bindStatus(ValidatorStatus.Pending),
+        async onBeforeTask() {
+          await props.nodesLock?.acquireAsync();
+          bindStatus(ValidatorStatus.Pending);
+        },
         onAfterTask: ({ data }) => {
           bindStatus(data ? ValidatorStatus.Valid : ValidatorStatus.Invalid);
+          release(props.nodesLock);
         },
       },
     );
 
     // reset validation to prevent form from being valid
     useWatchDeep(() => props.filters, validationTask.value.reset);
     useWatchDeep(nodeId, validationTask.value.reset);
+    useWatchDeep(
+      () => props.selectedMachines.map(m => m.nodeId),
+      () => nodeId.value && validationTask.value.run(nodeId.value),
+      { debounce: 1000 },
+    );
 
     // revalidate if filters updated
     useWatchDeep(