From 6d5072d46558b1fe4a9e14bd73424625dc916ce3 Mon Sep 17 00:00:00 2001 From: Claire Carouge Date: Fri, 1 Nov 2024 15:05:51 +1100 Subject: [PATCH] Initial commit --- .github/workflows/cd.yml | 18 ++++ .github/workflows/ci.yml | 50 ++++++++++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++ README.md | 81 ++++++++++++++++ config/versions.json | 5 + spack.yaml | 61 ++++++++++++ 6 files changed, 416 insertions(+) create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/ci.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/versions.json create mode 100644 spack.yaml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..a2ccbfc --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,18 @@ +name: CD +on: + push: + branches: + - main + - backport/*.* + paths: + - config/** + - spack.yaml +jobs: + cd: + name: CD + uses: access-nri/build-cd/.github/workflows/cd.yml@main + with: + model: ${{ vars.NAME }} + permissions: + contents: write + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ff48aed --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +# Requires vars.NAME to be set as a variable. +name: CI +on: + pull_request: + types: + - opened + - reopened + - synchronize + - closed + branches: + - main + - backport/*.* + paths: + - config/** + - spack.yaml + issue_comment: + types: + - created + - edited +jobs: + pr-ci: + name: CI + if: github.event_name == 'pull_request' && github.event.action != 'closed' + uses: access-nri/build-cd/.github/workflows/ci.yml@main + with: + model: ${{ vars.NAME }} + # root-sbd: if different from vars.NAME + permissions: + pull-requests: write + contents: write + secrets: inherit + + pr-comment: + name: Comment + if: github.event_name == 'issue_comment' + uses: access-nri/build-cd/.github/workflows/ci-comment.yml@main + with: + model: ${{ vars.NAME }} + # root-sbd: if different from vars.NAME + permissions: + pull-requests: write + contents: write + + pr-closed: + name: Closed + if: github.event_name == 'pull_request' && github.event.action == 'closed' + uses: access-nri/build-cd/.github/workflows/ci-closed.yml@main + with: + model: ${{ vars.NAME }} + secrets: inherit diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..15752aa --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# model-deployment-template + +A template repository for the deployment of `spack`-based models. + +> [!NOTE] +> Feel free to replace this README with information on the model once the TODOs have been ticked off. + +## Things TODO to get your model deployed + +### Settings + +#### Repository Settings + +Branch protections should be set up on `main` and the special `backport/*.*` branches, which are used for backporting of fixes to major releases (the `YEAR.MONTH` portion of the `YEAR.MONTH.MINOR` version) of models. + +#### Repository Secrets/Variables + +There are a few secrets and variables that must be set at the repository level. + +##### Repository Secrets + +* `BUILD_DB_CONNECTION_STR`: A postgresql connection url to the release provenance database +* `GH_COMMIT_CHECK_TOKEN`: GitHub Token that allows workflows to run based on workflow-authored commits (in the case where a user uses `!bump` commands in PRs that bumps the version of the model) + +##### Repository Variables + +* `BUILD_DB_PACKAGES`: List of `spack` packages that are model components that will be uploaded to the release provenance database +* `NAME`: which corresponds to the model name - which is usually the repository name +* `CONFIG_VERSIONS_SCHEMA_VERSION`: Version of the [`config/versions.json` schema](https://github.com/ACCESS-NRI/schema/tree/main/au.org.access-nri/model/deployment/config/versions) used in this repository +* `SPACK_YAML_SCHEMA_VERSION`: Version of the [ACCESS-NRI-style `spack.yaml` schema](https://github.com/ACCESS-NRI/schema/tree/main/au.org.access-nri/model/spack/environment/deployment) used in this repository + +#### Environment Secrets/Variables + +GitHub Environments are sets of variables and secrets that are used specifically to deploy software, and hence have more security requirements for their use. + +Currently, we have two Environments per deployment target - one for `Release` and one for `Prerelease`. Our current list of deployment targets and Environments can be found in this [deployment configuration file in `build-cd`](https://github.com/ACCESS-NRI/build-cd/blob/main/config/deployment-environment.json). + +In order to deploy to a given deployment target: + +* Environments with the name of the deployment target must be created _in this repository_ and have the associated secrets/variables set ([see below](#environment-secrets)) +* There must be a `Prerelease` Environment associated with the `Release` Environment. For example, if we are deploying to `SUPERCOMPUTER`, we require Environments with the names `SUPERCOMPUTER`, `SUPERCOMPUTER Prerelease`. + +When setting the environment up, remember to require sign off by a member of ACCESS-NRI when deploying as a `Release`. + +Regarding the secrets and variables that must be created: + +##### Environment Secrets + +* `HOST`: The deployment location SSH Host +* `HOST_DATA`: The deployment location SSH Host for data transfer (may be the same as `HOST`) +* `SSH_KEY`: A SSH Key that allows access to the above `HOST`/`HOST_DATA` +* `USER`: A Username to login to the above `HOST`/`HOST_DATA` + +##### Environment Variables + +* `DEPLOYMENT_TARGET`: Name of the deployment target for logging purposes +* `SPACK_INSTALLS_ROOT_LOCATION`: Path to the directory that contains all versions of a deployment of `spack`. For example, if `/some/apps/spack` is the `SPACK_INSTALLS_ROOT_LOCATION`, that directory will contain directories like `0.20`, `0.21`, `0.22`, which in turn contain an install of `spack`, `spack-packages` and `spack-config` +* `SPACK_YAML_LOCATION`: Path to a directory that will contain the `spack.yaml` from this repository during deployment +* (Optional) `SPACK_INSTALL_PARALLEL_JOBS`: Explicit number of parallel jobs for the installation of the given model. Must be either of the form `--jobs N` or unset (for the default `--jobs 16`). + +### File Modifications + +#### In `.github/workflows` + +* Reminder that these workflows use `vars.NAME` (as well as inherit the above environment secrets) and hence these must be set. +* If the name of the root SBD for the model (in [`spack-packages`](https://github.com/ACCESS-NRI/spack-packages/tree/main/packages)) is different from the model name (for example, `ACCESS-ESM1.5`s root SBD is `access-esm1p5`), you must uncomment and set the `jobs.[pr-ci|pr-comment].with.root-sbd` line to the appropriate SBD name. + +#### In `config/versions.json` + +* `.spack` must be given a version. For example, it will clone the associated `releases/VERSION` branch of `ACCESS-NRI/spack` if you give it `VERSION`. +* `.spack-packages` should also have a CalVer-compliant tag as the version. See the [associated repo](https://github.com/ACCESS-NRI/spack-packages/tags) for a list of available tags. + +#### In `spack.yaml` + +There are a few TODOs for the `spack.yaml`: + +* `spack.specs`: Set the root SBD as the only element of `spack.specs`. This must also have an `@git.YEAR.MONTH.MINOR` version as it is the version of the entire deployment (and indeed will be a tag in this repository). +* `spack.packages.*`: In this section, you can specify the versions and variants of dependencies. Note that the first element of the `spack.packages.*.require` must be only a version. Variants and other configuration can be done on subsequent lines. +* `spack.packages.all`: Can set configuration for all packages. For example, the compiler used, or the target architecture. +* `spack.modules.default.tcl.include`: List of package names that will be explicitly included and available to `module load`. +* `spack.modules.default.tcl.projections`: For included modules, you must set the name of the module to be the same as the `spack.packages.*.require[0]` version, without the `@git.`. diff --git a/config/versions.json b/config/versions.json new file mode 100644 index 0000000..a6f4b3a --- /dev/null +++ b/config/versions.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://github.com/ACCESS-NRI/schema/blob/main/au.org.access-nri/model/deployment/config/versions/3-0-0.json", + "spack": "0.22", + "spack-packages": "SOME_SPECIFIC_TAG" +} diff --git a/spack.yaml b/spack.yaml new file mode 100644 index 0000000..2c5f4d3 --- /dev/null +++ b/spack.yaml @@ -0,0 +1,61 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. +spack: + specs: + # TODO: Replace the MODEL and VERSION. + # The root SBD for the model and overall version of the deployment: + # - MODEL@git.VERSION + packages: + # TODO: Specify versions and variants of dependencies where required + # Specification of dependency versions and variants goes here. + # CI/CD requires that the first element of the require is only a version: + # openmpi: + # require: + # - '@4.0.2' + + # Specifications that apply to all packages + all: + # TODO: Specify compiler/targets for all packages + # require: + # - '%intel@19.0.5.281' + # - 'target=x86_64' + view: true + concretizer: + unify: true + modules: + default: + enable: + - tcl + roots: + tcl: $spack/../release/modules + lmod: $spack/../release/lmod + tcl: + hash_length: 0 + include: + # Explicitly, which packages are accessible as modules + # TODO: Add packages that will be included as modules + # - MODEL + exclude_implicits: true + all: + autoload: direct + conflict: + - '{name}' + environment: + set: + 'SPACK_{name}_ROOT': '{prefix}' + projections: + # TODO: Add explicit projections for modules that will be found with `module load`. + # Naming scheme for the above included modules. + # These projection versions must be the same as the + # `spack.packages.*.require[0]` version but without the `@git.`. + # Ex. `require` version `@git.2024.04.21` -> projection `2024.04.21`. + all: '{name}/{version}' + # MODEL: '{name}/VERSION' + # config: + # overridden spack configurations, if needed + # mirrors: + # overridden spack package tarball directories, if needed + # repos: + # overridden repo sources, if needed