feat!: new provider.yaml and new contracts v2 [JS-850, JS-851] (#1082)

* feat: new provider.yaml * feat: new provider.yaml * use 1.1.1.1 * Apply automatic changes * use 2, fix offerName * Apply automatic changes * improve validation, try some ideas * Apply automatic changes * use ones again * Apply automatic changes * group resource names by type, rename vcpu to cpu * Apply automatic changes * feat: add balance keeper * feat: use registerMarketOfferV2 * move over the rest of v2 I could find * improve resource definition in provider.yaml * Apply automatic changes * rename nox to computePeer * Apply automatic changes * product review fixes * fix integer * Apply automatic changes * add resource validation and autocomplete * use snapshot * prepare for getting resources from chain * Apply automatic changes * fix offer-create * more offer update functionality * use containers * use chain to get resources, temporary remove resource validation, temporary hardcode subgraph * use latest containers * add ip validation across compute peers * add 2x when creating offer, check enough ram per core * Apply automatic changes * fix storage description * Apply automatic changes * fix add, remove and display CU * remove type assertions * fix * fix * convert stuff * Apply automatic changes * use megabytes and megabits per second for network * up containers and snapshot * fix some update problems * fix provider test * fix test * fix change CU? * Add docker login to docs job * another fix for CU update * improve resource update? * use supply to 0 instead of removePeerResource * first update ram then CPU * first remove resources then add and check if details changed * fix update * use mebibytes, fix cpu price so it is per VCPU on chain but is still per physical core in config, add descriptions for prices in config * Apply automatic changes * fix offer update --------- Co-authored-by: shamsartem <shamsartem@users.noreply.github.com> Co-authored-by: Anatolios Laskaris <github_me@nahsi.dev>
fluencelabs · Jan 8, 2025 · 3894079 · 3894079
1 parent 7668819
commit 3894079
Show file tree

Hide file tree

Showing 30 changed files with 3,352 additions and 832 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -16,6 +16,9 @@ env:
 
 jobs:
   docs:
+    permissions:
+      contents: write
+      id-token: write
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
@@ -50,6 +53,28 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-turbo-
 
+      - name: Import secrets
+        uses: hashicorp/vault-action@v3.0.0
+        id: secrets
+        with:
+          url: https://vault.fluence.dev
+          path: jwt/github
+          role: ci
+          method: jwt
+          jwtGithubAudience: "https://github.com/fluencelabs"
+          jwtTtl: 300
+          exportToken: false
+          secrets: |
+            kv/docker-registry/basicauth/ci username | DOCKER_USERNAME ;
+            kv/docker-registry/basicauth/ci password | DOCKER_PASSWORD ;
+
+      - name: Login to private docker registry
+        uses: docker/login-action@v3
+        with:
+          registry: docker.fluence.dev
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+
       - name: Run on each commit
         run: yarn on-each-commit
 

diff --git a/packages/cli/package/docs/configs/README.md b/packages/cli/package/docs/configs/README.md
@@ -2,7 +2,7 @@
 
 ## [provider.yaml](./provider.md)
 
-Defines config used for provider set up
+Defines provider configuration
 
 ## [provider-secrets.yaml](./provider-secrets.md)
 

diff --git a/packages/cli/package/docs/configs/env.md b/packages/cli/package/docs/configs/env.md
@@ -22,9 +22,10 @@ Deployed contract address overrides
 
 ### Properties
 
-| Property     | Type   | Required | Description |
-|--------------|--------|----------|-------------|
-| `diamond`    | string | No       |             |
-| `multicall3` | string | No       |             |
-| `usdc`       | string | No       |             |
+| Property        | Type   | Required | Description |
+|-----------------|--------|----------|-------------|
+| `balanceKeeper` | string | No       |             |
+| `diamond`       | string | No       |             |
+| `multicall3`    | string | No       |             |
+| `usdc`          | string | No       |             |
 
diff --git a/packages/cli/package/docs/configs/provider-artifacts.md b/packages/cli/package/docs/configs/provider-artifacts.md
@@ -29,11 +29,11 @@ Created offers
 
 #### Properties
 
-| Property  | Type               | Required | Description |
-|-----------|--------------------|----------|-------------|
-| `noxName` | [object](#noxname) | No       |             |
+| Property    | Type                 | Required | Description |
+|-------------|----------------------|----------|-------------|
+| `offerName` | [object](#offername) | No       |             |
 
-#### noxName
+#### offerName
 
 ##### Properties
 
@@ -48,11 +48,11 @@ Created offers
 
 #### Properties
 
-| Property  | Type               | Required | Description |
-|-----------|--------------------|----------|-------------|
-| `noxName` | [object](#noxname) | No       |             |
+| Property    | Type                 | Required | Description |
+|-------------|----------------------|----------|-------------|
+| `offerName` | [object](#offername) | No       |             |
 
-#### noxName
+#### offerName
 
 ##### Properties
 
@@ -67,11 +67,11 @@ Created offers
 
 #### Properties
 
-| Property  | Type               | Required | Description |
-|-----------|--------------------|----------|-------------|
-| `noxName` | [object](#noxname) | No       |             |
+| Property    | Type                 | Required | Description |
+|-------------|----------------------|----------|-------------|
+| `offerName` | [object](#offername) | No       |             |
 
-#### noxName
+#### offerName
 
 ##### Properties
 
@@ -86,11 +86,11 @@ Created offers
 
 #### Properties
 
-| Property  | Type               | Required | Description |
-|-----------|--------------------|----------|-------------|
-| `noxName` | [object](#noxname) | No       |             |
+| Property    | Type                 | Required | Description |
+|-------------|----------------------|----------|-------------|
+| `offerName` | [object](#offername) | No       |             |
 
-#### noxName
+#### offerName
 
 ##### Properties
 
@@ -105,11 +105,11 @@ Created offers
 
 #### Properties
 
-| Property  | Type               | Required | Description |
-|-----------|--------------------|----------|-------------|
-| `noxName` | [object](#noxname) | No       |             |
+| Property    | Type                 | Required | Description |
+|-------------|----------------------|----------|-------------|
+| `offerName` | [object](#offername) | No       |             |
 
-#### noxName
+#### offerName
 
 ##### Properties
 

diff --git a/packages/cli/package/docs/configs/provider.md b/packages/cli/package/docs/configs/provider.md
diff --git a/packages/cli/package/package.json b/packages/cli/package/package.json
@@ -56,7 +56,7 @@
     "whatwg-url": "^14.0.0"
   },
   "dependencies": {
-    "@fluencelabs/deal-ts-clients": "0.22.1",
+    "@fluencelabs/deal-ts-clients": "0.23.2-feat-marketplace-v2-resources-03f7996-7429-1.0",
     "@kubernetes/client-node": "github:fluencelabs/kubernetes-client-javascript#e72ee00a52fec4eb4a8327632895d888ee504f4d",
     "@libp2p/crypto": "4.0.1",
     "@libp2p/peer-id-factory": "4.0.5",

diff --git a/packages/cli/package/src/commands/provider/deploy.ts b/packages/cli/package/src/commands/provider/deploy.ts
@@ -19,13 +19,10 @@ import { readFile } from "node:fs/promises";
 import { isAbsolute, resolve } from "node:path";
 
 import type k8s from "@kubernetes/client-node";
-import { color } from "@oclif/color";
 
 import { BaseCommand } from "../../baseCommand.js";
 import { commandObj } from "../../lib/commandObj.js";
-import { ensureReadonlyProviderConfig } from "../../lib/configs/project/provider/provider.js";
 import { CHAIN_FLAGS, PEER_AND_OFFER_NAMES_FLAGS } from "../../lib/const.js";
-import { splitErrorsAndResults } from "../../lib/helpers/utils.js";
 import { initCli } from "../../lib/lifeCycle.js";
 import { projectRootDir } from "../../lib/paths.js";
 import { resolveComputePeersByNames } from "../../lib/resolveComputePeersByNames.js";
@@ -41,35 +38,7 @@ export default class Deploy extends BaseCommand<typeof Deploy> {
     const { flags } = await initCli(this, await this.parse(Deploy));
     const computePeers = await resolveComputePeersByNames(flags);
 
-    const [invalidPeers, peers] = splitErrorsAndResults(
-      computePeers,
-      ({ kubeconfigPath, ipSupplies, name, ...restPeer }) => {
-        return kubeconfigPath === undefined || ipSupplies.length === 0
-          ? {
-              error: [
-                kubeconfigPath === undefined &&
-                  `computePeer ${color.yellow(name)} must have a ${color.yellow("kubeconfigPath")} property`,
-                ipSupplies.length === 0 &&
-                  `computePeer ${color.yellow(name)} must have a ${color.yellow("resources.ip.supply")} property, which must be a non-empty array`,
-              ]
-                .filter((err) => {
-                  return typeof err === "string";
-                })
-                .join("\n"),
-            }
-          : { result: { name, ipSupplies, kubeconfigPath, ...restPeer } };
-      },
-    );
-
-    if (invalidPeers.length > 0) {
-      const providerConfig = await ensureReadonlyProviderConfig();
-
-      commandObj.error(
-        `Invalid config at ${providerConfig.$getPath()}:\n${invalidPeers.join("\n")}`,
-      );
-    }
-
-    for (const { kubeconfigPath, name, manifestPath } of peers) {
+    for (const { kubeconfigPath, name, manifestPath } of computePeers) {
       await kubectlApply(kubeconfigPath, manifestPath, name);
     }
   }

diff --git a/packages/cli/package/src/lib/ajvInstance.ts b/packages/cli/package/src/lib/ajvInstance.ts
@@ -21,14 +21,16 @@ import addFormats from "ajv-formats";
 
 import { jsonStringify } from "../common.js";
 
+import { HEX_REGEX } from "./helpers/validations.js";
+
 export function getAjv() {
   const ajv = new Ajv.default({
     allowUnionTypes: true,
     code: { esm: true },
   });
 
   addFormats.default(ajv);
-  ajv.addFormat("hex", /^0x[0-9a-fA-F]*$/);
+  ajv.addFormat("hex", HEX_REGEX);
   return ajv;
 }
 

diff --git a/packages/cli/package/src/lib/chain/commitment.ts b/packages/cli/package/src/lib/chain/commitment.ts
@@ -495,6 +495,7 @@ export async function collateralWithdraw(
   })) {
     const { ccId, name: peerName } = commitment;
 
+    // TODO: improve how we get this info
     const [unitIds, isExitedStatuses] =
       await contracts.diamond.getUnitExitStatuses(ccId);
 
@@ -544,30 +545,20 @@ export async function collateralWithdraw(
       return unit.unitInfo.deal !== ZeroAddress;
     });
 
-    const unitIdsByOnChainWorkerId: Record<string, string[]> = {};
+    const onChainWorkerIds = new Set<string>();
 
-    for (const { unitId, unitInfo } of unitsWithDeals) {
-      let unitIds = unitIdsByOnChainWorkerId[unitInfo.onchainWorkerId];
-
-      if (unitIds === undefined) {
-        unitIds = [];
-        unitIdsByOnChainWorkerId[unitInfo.onchainWorkerId] = unitIds;
-      }
-
-      unitIds.push(unitId);
+    for (const { unitInfo } of unitsWithDeals) {
+      onChainWorkerIds.add(unitInfo.onchainWorkerId);
     }
 
-    const moveResourcesFromDealTxs = Object.entries(
-      unitIdsByOnChainWorkerId,
-    ).flatMap(([onchainWorkerId, unitIds]) => {
-      return unitIds.map((unit) => {
+    const moveResourcesFromDealTxs = Array.from(onChainWorkerIds).map(
+      (onchainWorkerId) => {
         return populateTx(
-          contracts.diamond.moveResourcesFromDeal,
-          [unit],
+          contracts.diamond.moveResourcesFromDealV2,
           onchainWorkerId,
         );
-      });
-    });
+      },
+    );
 
     const [firstMoveResourcesFromDealTx, ...restMoveResourcesFromDealTxs] =
       moveResourcesFromDealTxs;