From 85ce7a998e0cbaf370e329f33e7392d76fac7d4d Mon Sep 17 00:00:00 2001 From: "Daniel F. Murcia Rivera" Date: Thu, 4 Jan 2024 15:34:41 -0500 Subject: [PATCH] fix(back): #1234 computeOnAwsBatch - adjust python env for the computeOnAwsBatch util - support dry run to enable the util build check - add a job that checks the build of the util - update docs Signed-off-by: Daniel F. Murcia Rivera --- .github/workflows/dev.yml | 16 +++++++ .github/workflows/prod.yml | 16 +++++++ docs/src/api/builtins/deploy.md | 13 +++++- makes/tests/computeOnAwsBatch/main.nix | 20 ++++++++ .../batch-client/batch_client/_cli.py | 38 +++++++-------- .../batch-client/batch_client/actions.py | 3 +- .../batch-client/batch_client/core.py | 9 ++++ .../batch-client/batch_client/decode.py | 13 ++++-- .../batch-client/build/default.nix | 6 +-- .../batch-client/build/deps/arch_lint.nix | 28 ++++++++--- .../batch-client/build/deps/default.nix | 37 +++++++-------- .../batch-client/build/deps/fa_purity.nix | 28 ++++++++--- .../build/deps/override_utils.nix | 46 ------------------- .../batch-client/entrypoint.nix | 9 +++- src/args/compute-on-aws-batch/default.nix | 10 ++-- .../modules/compute-on-aws-batch/default.nix | 4 ++ 16 files changed, 186 insertions(+), 110 deletions(-) create mode 100644 makes/tests/computeOnAwsBatch/main.nix delete mode 100644 src/args/compute-on-aws-batch/batch-client/build/deps/override_utils.nix diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 48e3e3eb..5b6fa5fa 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -628,6 +628,22 @@ jobs: - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac - name: /testTerraform/module run: nix-env -if . && m . /testTerraform/module + + linux_computeOnAwsBatch_module: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 + - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845 + name: /tests/computeOnAwsBatch + with: + args: nix-env -if . && m . /tests/computeOnAwsBatch + macos_computeOnAwsBatch_module: + runs-on: macos-latest + steps: + - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 + - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac + - name: /tests/computeOnAwsBatch + run: nix-env -if . && m . /tests/computeOnAwsBatch name: dev on: pull_request: diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index ab11fd67..cb791c1f 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -821,6 +821,22 @@ jobs: run: nix-env -if . && m . /testTerraform/module env: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + + linux_computeOnAwsBatch_module: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 + - uses: docker://docker.io/nixos/nix@sha256:c3db4c484f6b1ee6c9bb8ca90307cfbeca8ef88156840911356a677eeaff4845 + name: /tests/computeOnAwsBatch + with: + args: nix-env -if . && m . /tests/computeOnAwsBatch + macos_computeOnAwsBatch_module: + runs-on: macos-latest + steps: + - uses: actions/checkout@f095bcc56b7c2baf48f3ac70d6d6782f4f553222 + - uses: cachix/install-nix-action@6ed004b9ccb68dbc28e7c85bee15fa93dbd214ac + - name: /tests/computeOnAwsBatch + run: nix-env -if . && m . /tests/computeOnAwsBatch name: prod on: push: diff --git a/docs/src/api/builtins/deploy.md b/docs/src/api/builtins/deploy.md index 08af910b..946b09c4 100644 --- a/docs/src/api/builtins/deploy.md +++ b/docs/src/api/builtins/deploy.md @@ -8,11 +8,20 @@ When used as a Makes declaration (at makes.nix attrs): Job groups to submit. Defaults to `{ }`. -Types: +When used as a makes input: - computeOnAwsBatch: `JobType -> SourceAble` Source able batch file to send jobs to aws batch. +???+ warning + + When used as a makes input, all arguments are required + and defaults are not available. + However nested jobs (see `nextJob` argument) + do have defaults enabled. + +Types: + - `JobType` = `attrs` - allowDuplicates: `bool` (Optional Attr) Set to `false` in order to prevent submitting the job @@ -32,6 +41,8 @@ Types: It overrides the one specified in the Batch job definition. Additional arguments can be propagated when running this module output. + - dryRun: `bool` (Optional Attr) (Not supported on nextJob) + Do not send any job. Only check the correctness of the pipeline definition. - definition: `str` Name of the Batch job definition that we will use as base for submitting the job. diff --git a/makes/tests/computeOnAwsBatch/main.nix b/makes/tests/computeOnAwsBatch/main.nix new file mode 100644 index 00000000..a011b30f --- /dev/null +++ b/makes/tests/computeOnAwsBatch/main.nix @@ -0,0 +1,20 @@ +{computeOnAwsBatch, ...}: +computeOnAwsBatch { + dryRun = true; + allowDuplicates = true; + attempts = 1; + attemptDurationSeconds = 60; + command = ["foo"]; + definition = "foo"; + environment = []; + includePositionalArgsInName = true; + name = "foo"; + nextJob = {}; + memory = 1; + parallel = 1; + propagateTags = true; + queue = "foo"; + setup = []; + tags = {}; + vcpus = 1; +} diff --git a/src/args/compute-on-aws-batch/batch-client/batch_client/_cli.py b/src/args/compute-on-aws-batch/batch-client/batch_client/_cli.py index e5310b98..6c82b038 100644 --- a/src/args/compute-on-aws-batch/batch-client/batch_client/_cli.py +++ b/src/args/compute-on-aws-batch/batch-client/batch_client/_cli.py @@ -8,6 +8,7 @@ ) from .core import ( EnvVarPointer, + JobPipelineDraft, QueueName, ) import click @@ -37,9 +38,9 @@ def _action() -> JsonObj: return Cmd.from_cmd(_action) -@click.command() # type: ignore[misc] -@click.option("--pipeline", type=click.Path(exists=True), required=True) # type: ignore[misc] -@click.argument("args", nargs=-1) # type: ignore[misc] +@click.command() +@click.option("--pipeline", type=click.Path(exists=True), required=True) +@click.argument("args", nargs=-1) def submit_job( pipeline: str, args: FrozenList[str], @@ -74,32 +75,31 @@ def _sep(item: str) -> bool: .map(lambda x: tuple(x)) .to_list() ) - drafts = _queue_from_env.bind( + pipeline_draft = _queue_from_env.bind( lambda queue: _root.map( lambda root: decode.decode_all_drafts(root, arg_groups, queue) ) - ).map( - lambda t: ( - t[0].map( - lambda r: r.alt( - lambda e: Exception(f"Invalid job draft i.e. {e}") - ).unwrap() - ), - t[1], - ) ) - cmd: Cmd[None] = drafts.bind( - lambda d: new_client().bind( - lambda c: utils.extract_single(d[0]).map( - lambda j: actions.send_single_job(c, j, d[1]), + + def _execute(draft: JobPipelineDraft) -> Cmd[None]: + # Handle dry run logic + action = new_client().bind( + lambda c: utils.extract_single(draft.drafts).map( + lambda j: actions.send_single_job( + c, j, draft.allow_duplicates + ), lambda p: actions.send_pipeline(c, p), ) ) - ) + if draft.dry_run: + return Cmd.from_cmd(lambda: None) + return action + + cmd: Cmd[None] = pipeline_draft.bind(_execute) cmd.compute() -@click.group() # type: ignore[misc] +@click.group() def main() -> None: pass diff --git a/src/args/compute-on-aws-batch/batch-client/batch_client/actions.py b/src/args/compute-on-aws-batch/batch-client/batch_client/actions.py index 9dbc36be..f8a2b656 100644 --- a/src/args/compute-on-aws-batch/batch-client/batch_client/actions.py +++ b/src/args/compute-on-aws-batch/batch-client/batch_client/actions.py @@ -2,7 +2,6 @@ ApiClient, ) from batch_client.core import ( - AllowDuplicates, DependentJobDraft, JobDependencies, JobDraft, @@ -25,7 +24,7 @@ def send_single_job( client: ApiClient, draft: JobDraft, - allow_duplicates: AllowDuplicates, + allow_duplicates: bool, ) -> Cmd[None]: dup_msg = Cmd.from_cmd(lambda: LOG.info("Detecting duplicates...")) skipped_msg = Cmd.from_cmd( diff --git a/src/args/compute-on-aws-batch/batch-client/batch_client/core.py b/src/args/compute-on-aws-batch/batch-client/batch_client/core.py index 7ce0578d..76180375 100644 --- a/src/args/compute-on-aws-batch/batch-client/batch_client/core.py +++ b/src/args/compute-on-aws-batch/batch-client/batch_client/core.py @@ -20,6 +20,7 @@ FrozenDict, FrozenList, Maybe, + PureIter, Result, ResultE, ) @@ -240,6 +241,7 @@ class RawJobDraft: allow_duplicates: bool args_in_name: bool propagate_tags: bool + dry_run: bool next_job: Maybe[RawJobDraft] @@ -266,3 +268,10 @@ class DependentJobDraft: @dataclass(frozen=True) class AllowDuplicates: value: bool + + +@dataclass(frozen=True) +class JobPipelineDraft: + drafts: PureIter[JobDraft] + allow_duplicates: bool + dry_run: bool diff --git a/src/args/compute-on-aws-batch/batch-client/batch_client/decode.py b/src/args/compute-on-aws-batch/batch-client/batch_client/decode.py index fa0377b0..2b531f49 100644 --- a/src/args/compute-on-aws-batch/batch-client/batch_client/decode.py +++ b/src/args/compute-on-aws-batch/batch-client/batch_client/decode.py @@ -6,13 +6,13 @@ utils, ) from batch_client.core import ( - AllowDuplicates, Attempts, Command, EnvVarPointer, JobDefinition, JobDraft, JobName, + JobPipelineDraft, JobSize, Manifest, QueueName, @@ -131,6 +131,7 @@ def _decode_raw_draft(raw: JsonObj, unwrapper: ResultUnwrapper) -> RawJobDraft: _allow_duplicates = _require(raw, "allowDuplicates", _to_bool) _args_in_name = _require(raw, "includePositionalArgsInName", _to_bool) _propagate_tags = _require(raw, "propagateTags", _to_bool) + _dry_run = _require(raw, "dryRun", _to_bool) _next = _require(raw, "nextJob", Unfolder.to_json).bind(decode_raw_draft) return RawJobDraft( unwrapper.unwrap(_name), @@ -147,6 +148,7 @@ def _decode_raw_draft(raw: JsonObj, unwrapper: ResultUnwrapper) -> RawJobDraft: unwrapper.unwrap(_allow_duplicates), unwrapper.unwrap(_args_in_name), unwrapper.unwrap(_propagate_tags), + unwrapper.unwrap(_dry_run), unwrapper.unwrap(_next), ) @@ -222,7 +224,7 @@ def decode_all_drafts( root: RawJobDraft, args: FrozenList[FrozenList[str]], queue_from_env: ResultE[QueueName], -) -> Tuple[PureIter[ResultE[JobDraft]], AllowDuplicates]: +) -> JobPipelineDraft: items = ( _raw_jobs(root) .enumerate(0) @@ -234,4 +236,9 @@ def decode_all_drafts( ) ) ) - return (items, AllowDuplicates(root.allow_duplicates)) + drafts = items.map( + lambda r: r.alt( + lambda e: Exception(f"Invalid job draft i.e. {e}") + ).unwrap() + ) + return JobPipelineDraft(drafts, root.allow_duplicates, root.dry_run) diff --git a/src/args/compute-on-aws-batch/batch-client/build/default.nix b/src/args/compute-on-aws-batch/batch-client/build/default.nix index 2895ecb4..91229dfc 100644 --- a/src/args/compute-on-aws-batch/batch-client/build/default.nix +++ b/src/args/compute-on-aws-batch/batch-client/build/default.nix @@ -1,11 +1,11 @@ { - makePythonPyprojectPackage, + makes_inputs, nixpkgs, python_version, src, }: let deps = import ./deps { - inherit nixpkgs python_version; + inherit makes_inputs nixpkgs python_version; }; pkgDeps = { runtime_deps = with deps.python_pkgs; [ @@ -24,7 +24,7 @@ pytest ]; }; - packages = makePythonPyprojectPackage { + packages = makes_inputs.makePythonPyprojectPackage { inherit (deps.lib) buildEnv buildPythonPackage; inherit pkgDeps src; }; diff --git a/src/args/compute-on-aws-batch/batch-client/build/deps/arch_lint.nix b/src/args/compute-on-aws-batch/batch-client/build/deps/arch_lint.nix index 99846ab5..7edacbe3 100644 --- a/src/args/compute-on-aws-batch/batch-client/build/deps/arch_lint.nix +++ b/src/args/compute-on-aws-batch/batch-client/build/deps/arch_lint.nix @@ -1,13 +1,29 @@ { + lib, + makes_inputs, nixpkgs, + python_pkgs, python_version, }: let - commit = "72a495bb933f052ad812292b468ca3e18fd9dde4"; - src = builtins.fetchTarball { - sha256 = "sha256:0413zl4y92dbdfmck070x7dhp5cxx66xd2pxpxg3gbhaw0yqzhqd"; + commit = "fd64a300bda15c2389f5bfb314f48fb5b2a0e47a"; # 2.4.0+2 + raw_src = builtins.fetchTarball { + sha256 = "sha256:0g1md5fiyzqi9xfh1qxf0mh32k8nb06w0yhc17rr5a0ijiskb8i4"; url = "https://gitlab.com/dmurciaatfluid/arch_lint/-/archive/${commit}/arch_lint-${commit}.tar"; }; -in - import "${src}/build" { + src = import "${raw_src}/build/filter.nix" nixpkgs.nix-filter raw_src; + bundle = import "${raw_src}/build" { + makesLib = makes_inputs; inherit nixpkgs python_version src; - } + }; +in + bundle.build_bundle ( + default: required_deps: builder: + builder lib ( + required_deps ( + python_pkgs + // { + inherit (default.python_pkgs) grimp; + } + ) + ) + ) diff --git a/src/args/compute-on-aws-batch/batch-client/build/deps/default.nix b/src/args/compute-on-aws-batch/batch-client/build/deps/default.nix index e5e0b0c1..bb033634 100644 --- a/src/args/compute-on-aws-batch/batch-client/build/deps/default.nix +++ b/src/args/compute-on-aws-batch/batch-client/build/deps/default.nix @@ -1,4 +1,5 @@ { + makes_inputs, nixpkgs, python_version, }: let @@ -8,32 +9,32 @@ inherit (nixpkgs.python3Packages) fetchPypi; }; - utils = import ./override_utils.nix; - pkgs_overrides = override: python_pkgs: builtins.mapAttrs (_: override python_pkgs) python_pkgs; - - arch-lint = import ./arch_lint.nix {inherit nixpkgs python_version;}; - fa-purity = let - core = import ./fa_purity.nix {inherit nixpkgs python_version;}; - in { - "${python_version}" = core; - }; + utils = makes_inputs.pythonOverrideUtils; layer_1 = python_pkgs: python_pkgs // { - arch-lint = arch-lint.pkg; - fa-purity = fa-purity."${python_version}".pkg; + arch-lint = let + result = import ./arch_lint.nix { + inherit lib makes_inputs nixpkgs python_pkgs python_version; + }; + in + result.pkg; mypy-boto3-batch = import ./boto3/batch-stubs.nix {inherit lib python_pkgs;}; types-boto3 = import ./boto3/stubs.nix {inherit lib python_pkgs;}; }; + layer_2 = python_pkgs: + python_pkgs + // { + fa-purity = let + result = import ./fa_purity.nix { + inherit lib makes_inputs nixpkgs python_pkgs python_version; + }; + in + result.pkg; + }; - fa_purity_override = python_pkgs: utils.replace_pkg ["fa_purity"] python_pkgs.fa-purity; - overrides = map pkgs_overrides [ - fa_purity_override - (_: utils.no_check_override) - ]; - - python_pkgs = utils.compose ([layer_1] ++ overrides) nixpkgs."${python_version}Packages"; + python_pkgs = utils.compose [layer_2 layer_1] nixpkgs."${python_version}Packages"; in { inherit lib python_pkgs; } diff --git a/src/args/compute-on-aws-batch/batch-client/build/deps/fa_purity.nix b/src/args/compute-on-aws-batch/batch-client/build/deps/fa_purity.nix index 8f3e417d..d5db0f16 100644 --- a/src/args/compute-on-aws-batch/batch-client/build/deps/fa_purity.nix +++ b/src/args/compute-on-aws-batch/batch-client/build/deps/fa_purity.nix @@ -1,13 +1,29 @@ { + lib, + makes_inputs, nixpkgs, + python_pkgs, python_version, }: let - commit = "e0b5cf459a16eb92d86ca6c024edbedd52d72589"; - src = builtins.fetchTarball { - sha256 = "sha256:08h1b94mn74lqz47cj8m5dmm5xddddfd1clrb6zqi898w3q1bylr"; + commit = "5ff1ddb203afcc0a636b0305398581845f68a153"; # v1.40.0 + raw_src = builtins.fetchTarball { + sha256 = "sha256:1i357nzd15a7c8av0ff7495gbcfd4pc7xxprlrbnz4savwnxd5x1"; url = "https://gitlab.com/dmurciaatfluid/purity/-/archive/${commit}/purity-${commit}.tar"; }; + src = import "${raw_src}/build/filter.nix" nixpkgs.nix-filter raw_src; + bundle = import "${raw_src}/build" { + makesLib = makes_inputs; + inherit nixpkgs python_version src; + }; in - import "${src}/build" { - inherit src nixpkgs python_version; - } + bundle.build_bundle ( + default: required_deps: builder: + builder lib ( + required_deps ( + python_pkgs + // { + inherit (default.python_pkgs) types-simplejson; + } + ) + ) + ) diff --git a/src/args/compute-on-aws-batch/batch-client/build/deps/override_utils.nix b/src/args/compute-on-aws-batch/batch-client/build/deps/override_utils.nix deleted file mode 100644 index 901a5827..00000000 --- a/src/args/compute-on-aws-batch/batch-client/build/deps/override_utils.nix +++ /dev/null @@ -1,46 +0,0 @@ -let - recursive_python_pkg_override = is_pkg: override: let - # is_pkg: Derivation -> Bool - # override: Derivation -> Derivation - self = recursive_python_pkg_override is_pkg override; - in - pkg: - if is_pkg pkg - then override pkg - else if pkg ? overridePythonAttrs && pkg ? pname - then - pkg.overridePythonAttrs ( - builtins.mapAttrs (_: value: - if builtins.isList value - then map self value - else self value) - ) - else pkg; - - # no_check_override: Derivation -> Derivation - no_check_override = recursive_python_pkg_override (pkg: pkg ? overridePythonAttrs && pkg ? pname) ( - pkg: - pkg.overridePythonAttrs ( - old: - ( - builtins.mapAttrs (_: value: - if builtins.isList value - then map no_check_override value - else no_check_override value) - old - ) - // { - doCheck = false; - } - ) - ); - - # replace_pkg: List[str] -> Derivation -> Derivation - replace_pkg = names: new_pkg: - recursive_python_pkg_override ( - x: x ? overridePythonAttrs && x ? pname && builtins.elem x.pname names - ) (_: new_pkg); -in { - inherit recursive_python_pkg_override no_check_override replace_pkg; - compose = functions: val: builtins.foldl' (x: f: f x) val functions; -} diff --git a/src/args/compute-on-aws-batch/batch-client/entrypoint.nix b/src/args/compute-on-aws-batch/batch-client/entrypoint.nix index f1bcfcca..3ea678cc 100644 --- a/src/args/compute-on-aws-batch/batch-client/entrypoint.nix +++ b/src/args/compute-on-aws-batch/batch-client/entrypoint.nix @@ -1,5 +1,5 @@ { - makePythonPyprojectPackage, + makes_inputs, nixpkgs, }: let nix-filter = let @@ -11,7 +11,12 @@ import src; python_version = "python311"; out = import ./build { - inherit makePythonPyprojectPackage nixpkgs python_version; + inherit makes_inputs python_version; + nixpkgs = + nixpkgs + // { + inherit nix-filter; + }; src = nix-filter { root = ./.; include = [ diff --git a/src/args/compute-on-aws-batch/default.nix b/src/args/compute-on-aws-batch/default.nix index 959be866..cb6d64d0 100644 --- a/src/args/compute-on-aws-batch/default.nix +++ b/src/args/compute-on-aws-batch/default.nix @@ -4,12 +4,13 @@ makeScript, toFileJson, ... -}: { +} @ makes_inputs: { allowDuplicates, attempts, attemptDurationSeconds, command, definition, + dryRun, environment, includePositionalArgsInName, name, @@ -23,7 +24,7 @@ vcpus, } @ self: let batch-client = import ./batch-client/entrypoint.nix { - inherit makePythonPyprojectPackage; + inherit makes_inputs; nixpkgs = __nixpkgs__; }; @@ -51,6 +52,7 @@ attemptDurationSeconds, command, definition, + dryRun ? false, environment ? [], includePositionalArgsInName ? true, name, @@ -62,14 +64,14 @@ tags ? {}, vcpus, } @ result: { - inherit allowDuplicates attempts attemptDurationSeconds command definition environment; + inherit allowDuplicates attempts attemptDurationSeconds command definition dryRun environment; inherit includePositionalArgsInName name nextJob memory parallel propagateTags queue tags vcpus; }; encode_draft = _draft: let draft = apply_defaults (removeAttrs _draft ["setup"]); in { inherit (draft) allowDuplicates attempts attemptDurationSeconds command; - inherit (draft) definition includePositionalArgsInName memory parallel; + inherit (draft) definition dryRun includePositionalArgsInName memory parallel; inherit (draft) propagateTags queue name tags vcpus; environment = encode_envs draft.environment; nextJob = diff --git a/src/evaluator/modules/compute-on-aws-batch/default.nix b/src/evaluator/modules/compute-on-aws-batch/default.nix index b928234c..5d38dfc9 100644 --- a/src/evaluator/modules/compute-on-aws-batch/default.nix +++ b/src/evaluator/modules/compute-on-aws-batch/default.nix @@ -48,6 +48,10 @@ in { command = lib.mkOption { type = lib.types.listOf lib.types.str; }; + dryRun = lib.mkOption { + default = false; + type = lib.types.bool; + }; definition = lib.mkOption { type = lib.types.str; };