fix(back): fluidattacks#1234 compute on aws batch

- adjust python env for the computeOnAwsBatch util
- support dry run to enable the util build check
- add a job that checks the build of the util
- update docs

Signed-off-by: Daniel F. Murcia Rivera <danmur97@outlook.com>

.github/workflows/dev.yml

@@ -628,6 +628,22 @@ jobs:
       - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac
       - name: /testTerraform/module
         run: nix-env -if . && m . /testTerraform/module
+
+  linux_computeOnAwsBatch_module:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222
+      - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845
+        name: /tests/computeOnAwsBatch
+        with:
+          args: sh -c "chown -R root:root /github/workspace && nix-env -if . && m . /tests/computeOnAwsBatch
+  macos_computeOnAwsBatch_module:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222
+      - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac
+      - name: /tests/computeOnAwsBatch
+        run: nix-env -if . && m . /tests/computeOnAwsBatch
 name: dev
 on:
   pull_request:

.github/workflows/prod.yml

@@ -821,6 +821,22 @@ jobs:
         run: nix-env -if . && m . /testTerraform/module
         env:
           CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+  linux_computeOnAwsBatch_module:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222
+      - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845
+        name: /tests/computeOnAwsBatch
+        with:
+          args: sh -c "chown -R root:root /github/workspace && nix-env -if . && m . /tests/computeOnAwsBatch
+  macos_computeOnAwsBatch_module:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222
+      - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac
+      - name: /tests/computeOnAwsBatch
+        run: nix-env -if . && m . /tests/computeOnAwsBatch
 name: prod
 on:
   push:

docs/src/api/builtins/deploy.md

@@ -8,11 +8,20 @@ When used as a Makes declaration (at makes.nix attrs):
     Job groups to submit.
     Defaults to `{ }`.
 
-Types:
+When used as a makes input:
 
 - computeOnAwsBatch: `JobType -> SourceAble`
     Source able batch file to send jobs to aws batch.
 
+???+ warning
+
+    When used as a makes input, all arguments are required
+    and defaults are not available.
+    However nested jobs (see `nextJob` argument)
+    do have defaults enabled.
+
+Types:
+
 - `JobType` = `attrs`
     - allowDuplicates: `bool` (Optional Attr)
         Set to `false` in order to prevent submitting the job
@@ -32,6 +41,8 @@ Types:
         It overrides the one specified
         in the Batch job definition.
         Additional arguments can be propagated when running this module output.
+    - dryRun: `bool`  (Optional Attr) (Not supported on nextJob)
+        Do not send any job. Only check the correctness of the pipeline definition.
     - definition: `str`
         Name of the Batch job definition
         that we will use as base for submitting the job.

makes/tests/computeOnAwsBatch/main.nix

@@ -0,0 +1,20 @@
+{computeOnAwsBatch, ...}:
+computeOnAwsBatch {
+  dryRun = true;
+  allowDuplicates = true;
+  attempts = 1;
+  attemptDurationSeconds = 60;
+  command = ["foo"];
+  definition = "foo";
+  environment = [];
+  includePositionalArgsInName = true;
+  name = "foo";
+  nextJob = {};
+  memory = 1;
+  parallel = 1;
+  propagateTags = true;
+  queue = "foo";
+  setup = [];
+  tags = {};
+  vcpus = 1;
+}

src/args/compute-on-aws-batch/batch-client/batch_client/_cli.py

@@ -8,6 +8,7 @@
 )
 from .core import (
     EnvVarPointer,
+    JobPipelineDraft,
     QueueName,
 )
 import click
@@ -37,9 +38,9 @@ def _action() -> JsonObj:
     return Cmd.from_cmd(_action)
 
 
-@click.command()  # type: ignore[misc]
-@click.option("--pipeline", type=click.Path(exists=True), required=True)  # type: ignore[misc]
-@click.argument("args", nargs=-1)  # type: ignore[misc]
+@click.command()
+@click.option("--pipeline", type=click.Path(exists=True), required=True)
+@click.argument("args", nargs=-1)
 def submit_job(
     pipeline: str,
     args: FrozenList[str],
@@ -74,32 +75,31 @@ def _sep(item: str) -> bool:
         .map(lambda x: tuple(x))
         .to_list()
     )
-    drafts = _queue_from_env.bind(
+    pipeline_draft = _queue_from_env.bind(
         lambda queue: _root.map(
             lambda root: decode.decode_all_drafts(root, arg_groups, queue)
         )
-    ).map(
-        lambda t: (
-            t[0].map(
-                lambda r: r.alt(
-                    lambda e: Exception(f"Invalid job draft i.e. {e}")
-                ).unwrap()
-            ),
-            t[1],
-        )
     )
-    cmd: Cmd[None] = drafts.bind(
-        lambda d: new_client().bind(
-            lambda c: utils.extract_single(d[0]).map(
-                lambda j: actions.send_single_job(c, j, d[1]),
+
+    def _execute(draft: JobPipelineDraft) -> Cmd[None]:
+        # Handle dry run logic
+        action = new_client().bind(
+            lambda c: utils.extract_single(draft.drafts).map(
+                lambda j: actions.send_single_job(
+                    c, j, draft.allow_duplicates
+                ),
                 lambda p: actions.send_pipeline(c, p),
             )
         )
-    )
+        if draft.dry_run:
+            return Cmd.from_cmd(lambda: None)
+        return action
+
+    cmd: Cmd[None] = pipeline_draft.bind(_execute)
     cmd.compute()
 
 
-@click.group()  # type: ignore[misc]
+@click.group()
 def main() -> None:
     pass
 

src/args/compute-on-aws-batch/batch-client/batch_client/actions.py

@@ -2,7 +2,6 @@
     ApiClient,
 )
 from batch_client.core import (
-    AllowDuplicates,
     DependentJobDraft,
     JobDependencies,
     JobDraft,
@@ -25,7 +24,7 @@
 def send_single_job(
     client: ApiClient,
     draft: JobDraft,
-    allow_duplicates: AllowDuplicates,
+    allow_duplicates: bool,
 ) -> Cmd[None]:
     dup_msg = Cmd.from_cmd(lambda: LOG.info("Detecting duplicates..."))
     skipped_msg = Cmd.from_cmd(

src/args/compute-on-aws-batch/batch-client/batch_client/core.py

@@ -20,6 +20,7 @@
     FrozenDict,
     FrozenList,
     Maybe,
+    PureIter,
     Result,
     ResultE,
 )
@@ -240,6 +241,7 @@ class RawJobDraft:
     allow_duplicates: bool
     args_in_name: bool
     propagate_tags: bool
+    dry_run: bool
     next_job: Maybe[RawJobDraft]
 
 
@@ -266,3 +268,10 @@ class DependentJobDraft:
 @dataclass(frozen=True)
 class AllowDuplicates:
     value: bool
+
+
+@dataclass(frozen=True)
+class JobPipelineDraft:
+    drafts: PureIter[JobDraft]
+    allow_duplicates: bool
+    dry_run: bool

src/args/compute-on-aws-batch/batch-client/batch_client/decode.py

@@ -6,13 +6,13 @@
     utils,
 )
 from batch_client.core import (
-    AllowDuplicates,
     Attempts,
     Command,
     EnvVarPointer,
     JobDefinition,
     JobDraft,
     JobName,
+    JobPipelineDraft,
     JobSize,
     Manifest,
     QueueName,
@@ -131,6 +131,7 @@ def _decode_raw_draft(raw: JsonObj, unwrapper: ResultUnwrapper) -> RawJobDraft:
     _allow_duplicates = _require(raw, "allowDuplicates", _to_bool)
     _args_in_name = _require(raw, "includePositionalArgsInName", _to_bool)
     _propagate_tags = _require(raw, "propagateTags", _to_bool)
+    _dry_run = _require(raw, "dryRun", _to_bool)
     _next = _require(raw, "nextJob", Unfolder.to_json).bind(decode_raw_draft)
     return RawJobDraft(
         unwrapper.unwrap(_name),
@@ -147,6 +148,7 @@ def _decode_raw_draft(raw: JsonObj, unwrapper: ResultUnwrapper) -> RawJobDraft:
         unwrapper.unwrap(_allow_duplicates),
         unwrapper.unwrap(_args_in_name),
         unwrapper.unwrap(_propagate_tags),
+        unwrapper.unwrap(_dry_run),
         unwrapper.unwrap(_next),
     )
 
@@ -222,7 +224,7 @@ def decode_all_drafts(
     root: RawJobDraft,
     args: FrozenList[FrozenList[str]],
     queue_from_env: ResultE[QueueName],
-) -> Tuple[PureIter[ResultE[JobDraft]], AllowDuplicates]:
+) -> JobPipelineDraft:
     items = (
         _raw_jobs(root)
         .enumerate(0)
@@ -234,4 +236,9 @@ def decode_all_drafts(
             )
         )
     )
-    return (items, AllowDuplicates(root.allow_duplicates))
+    drafts = items.map(
+        lambda r: r.alt(
+            lambda e: Exception(f"Invalid job draft i.e. {e}")
+        ).unwrap()
+    )
+    return JobPipelineDraft(drafts, root.allow_duplicates, root.dry_run)

src/args/compute-on-aws-batch/batch-client/build/default.nix

@@ -1,11 +1,11 @@
 {
-  makePythonPyprojectPackage,
+  makes_inputs,
   nixpkgs,
   python_version,
   src,
 }: let
   deps = import ./deps {
-    inherit nixpkgs python_version;
+    inherit makes_inputs nixpkgs python_version;
   };
   pkgDeps = {
     runtime_deps = with deps.python_pkgs; [
@@ -24,7 +24,7 @@
       pytest
     ];
   };
-  packages = makePythonPyprojectPackage {
+  packages = makes_inputs.makePythonPyprojectPackage {
     inherit (deps.lib) buildEnv buildPythonPackage;
     inherit pkgDeps src;
   };

src/args/compute-on-aws-batch/batch-client/build/deps/arch_lint.nix

@@ -1,13 +1,29 @@
 {
+  lib,
+  makes_inputs,
   nixpkgs,
+  python_pkgs,
   python_version,
 }: let
-  commit = "72a495bb933f052ad812292b468ca3e18fd9dde4";
-  src = builtins.fetchTarball {
-    sha256 = "sha256:0413zl4y92dbdfmck070x7dhp5cxx66xd2pxpxg3gbhaw0yqzhqd";
+  commit = "fd64a300bda15c2389f5bfb314f48fb5b2a0e47a"; # 2.4.0+2
+  raw_src = builtins.fetchTarball {
+    sha256 = "sha256:0g1md5fiyzqi9xfh1qxf0mh32k8nb06w0yhc17rr5a0ijiskb8i4";
     url = "https://gitlab.com/dmurciaatfluid/arch_lint/-/archive/${commit}/arch_lint-${commit}.tar";
   };
-in
-  import "${src}/build" {
+  src = import "${raw_src}/build/filter.nix" nixpkgs.nix-filter raw_src;
+  bundle = import "${raw_src}/build" {
+    makesLib = makes_inputs;
     inherit nixpkgs python_version src;
-  }
+  };
+in
+  bundle.build_bundle (
+    default: required_deps: builder:
+      builder lib (
+        required_deps (
+          python_pkgs
+          // {
+            inherit (default.python_pkgs) grimp;
+          }
+        )
+      )
+  )

src/args/compute-on-aws-batch/batch-client/build/deps/default.nix

@@ -1,4 +1,5 @@
 {
+  makes_inputs,
   nixpkgs,
   python_version,
 }: let
@@ -8,32 +9,32 @@
     inherit (nixpkgs.python3Packages) fetchPypi;
   };
 
-  utils = import ./override_utils.nix;
-  pkgs_overrides = override: python_pkgs: builtins.mapAttrs (_: override python_pkgs) python_pkgs;
-
-  arch-lint = import ./arch_lint.nix {inherit nixpkgs python_version;};
-  fa-purity = let
-    core = import ./fa_purity.nix {inherit nixpkgs python_version;};
-  in {
-    "${python_version}" = core;
-  };
+  utils = makes_inputs.pythonOverrideUtils;
 
   layer_1 = python_pkgs:
     python_pkgs
     // {
-      arch-lint = arch-lint.pkg;
-      fa-purity = fa-purity."${python_version}".pkg;
+      arch-lint = let
+        result = import ./arch_lint.nix {
+          inherit lib makes_inputs nixpkgs python_pkgs python_version;
+        };
+      in
+        result.pkg;
       mypy-boto3-batch = import ./boto3/batch-stubs.nix {inherit lib python_pkgs;};
       types-boto3 = import ./boto3/stubs.nix {inherit lib python_pkgs;};
     };
+  layer_2 = python_pkgs:
+    python_pkgs
+    // {
+      fa-purity = let
+        result = import ./fa_purity.nix {
+          inherit lib makes_inputs nixpkgs python_pkgs python_version;
+        };
+      in
+        result.pkg;
+    };
 
-  fa_purity_override = python_pkgs: utils.replace_pkg ["fa_purity"] python_pkgs.fa-purity;
-  overrides = map pkgs_overrides [
-    fa_purity_override
-    (_: utils.no_check_override)
-  ];
-
-  python_pkgs = utils.compose ([layer_1] ++ overrides) nixpkgs."${python_version}Packages";
+  python_pkgs = utils.compose [layer_2 layer_1] nixpkgs."${python_version}Packages";
 in {
   inherit lib python_pkgs;
 }