From c7706472542aa03f8d7b0b6a632c0ba780f66d73 Mon Sep 17 00:00:00 2001 From: alicechai <134899205+alicechaitea@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:46:57 +0700 Subject: [PATCH 1/3] fix: UDC deprecation (#148) * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * fix: token recognition error at: 'FUNCTION' * fix: bash earthfile * fix: update earthfile * fix: command to function * fix: token issue * Add files via upload * Update Earthfile * fix: change to correct earthfile * Update go.sum * Update go.mod * Update go.mod * Update go.mod * Update go.mod * Update go.sum * Update action.yml * Update action.yml * Update action.yml * Update run.yml * Update action.yml * Update ci.yml * Update action.yml * Update run.yml * Update pages.yml * Update release.yml * Update publish.yml * Update deploy.yml * Update go.mod * Update Earthfile * Update Earthfile * fix: update ast go version * Update project.dic * Update go.mod * Update go.sum * Update Earthfile * Update Earthfile * Update earthfile.go * Update earthfile.go * Update earthfile.go * Update earthfile.go * Update earthfile.go * Update earthly_parser.go * Update earthfile.go * Update go.mod * Update go.mod * Update go.mod * Update go.mod * Update earthfile.go * Update earthly_parser.go * Update Earthfile * Update go.sum * Revert "fix: update ast go version" This reverts commit 9a4c484ae17f25366e9c908d8cb235fe887131a7. * fix: update go dependency * fix: update earthfile struct * fix: branch name typo * fix: clean ci-lint go cache * fix: remove clean cache * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * fix: edit COMMAND to FUNCTION * update: deprecation of UDCs in Earthly 0.8 * fix: trailing spaces * fix: style * fix: styling * fix: edited DO to RUN for function calling * fix: update consolidate checking script * fix: added edit-docs back to the script * fix: added full stop to line 108 * fix: change RUN back to DO at line 87 * fix: deleted check and comment * fix: added warning for deprecation of UDCs at line 159 * fix: moved warning to line 77 and pasted Blue's version * fix: edited formatting * fix: added blank line * fix: formatting revision * fix: update branch name and replace udc in doc * fix: update udc and branch name * fix: reroll UDC change * fix: change UDC back to Function * fix: change branch name to fdcDeprecation * fix: linting error * fix: linting error * fix: linting error * fix: linting error * fix: trailing spaces at line 37 * fix: RUN to DO * fix: required value for a flag missing * fix: deleted --dot * fix: cannot execute command * fix: grammatical error * fix: edited Function target to Function * fix: changed file name from udc.md to function.md * fix: renamed branch to master * fix: true to false --------- Co-authored-by: vasapornchaiyakul <134899205+vasapornchaiyakul@users.noreply.github.com> Co-authored-by: bkioshn <35752733+bkioshn@users.noreply.github.com> Co-authored-by: bkioshn --- .github/workflows/ci.yml | 2 +- .github/workflows/deploy.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- Earthfile | 4 +-- cli/go.mod | 6 ++-- cli/go.sum | 10 +++---- cli/pkg/earthfile.go | 2 +- cli/pkg/parsers/earthly_parser.go | 2 +- docs/src/guides/docs.md | 6 ++-- docs/src/guides/languages/go.md | 16 +++++------ docs/src/guides/languages/postgresql.md | 8 +++--- docs/src/guides/languages/python.md | 2 +- docs/src/guides/languages/rust.md | 16 +++++------ docs/src/guides/markdown.md | 10 +++---- docs/src/guides/spellcheck.md | 2 +- docs/src/reference/function.md | 38 +++++++++++++++++++++++++ docs/src/reference/udc.md | 37 ------------------------ docs/src/style/index.md | 18 ++++++++---- earthly/bash/Earthfile | 5 ++-- earthly/cspell/Earthfile | 6 ++-- earthly/docs/Earthfile | 10 +++---- earthly/go/Earthfile | 12 ++++---- earthly/mdlint/Earthfile | 8 +++--- earthly/postgresql/Earthfile | 14 ++++----- earthly/python/Earthfile | 4 +-- earthly/rust/Earthfile | 12 ++++---- examples/go/Earthfile | 4 +-- examples/postgresql/Earthfile | 4 +-- utilities/scripts/Earthfile | 8 +++--- 30 files changed, 142 insertions(+), 130 deletions(-) create mode 100644 docs/src/reference/function.md delete mode 100644 docs/src/reference/udc.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 497ed948e..61d518d4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,4 +203,4 @@ jobs: deployment_repo: ${{ inputs.deployment_repo }} images: ${{ inputs.deployment_images }} secrets: - token: ${{ secrets.deployment_token }} \ No newline at end of file + token: ${{ secrets.deployment_token }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2f1402a48..414b179aa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,4 +50,4 @@ jobs: author_name: catalyst-cibot author_email: ci@projectcatalyst.io message: "chore: updates ${{ github.event.repository.name }} deployments" - push: true \ No newline at end of file + push: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a9a5fc91b..a631fe7bf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -172,4 +172,4 @@ jobs: ${{ inputs.aws_ecr_registry }} ghcr.io/${{ github.repository }} tags: | - ${{ steps.fetch_tag.outputs.tag }} \ No newline at end of file + ${{ steps.fetch_tag.outputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43433fd2f..4f9613777 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -170,4 +170,4 @@ jobs: - name: Create release uses: softprops/action-gh-release@v1 with: - files: ${{ steps.collect.outputs.artifacts }} \ No newline at end of file + files: ${{ steps.collect.outputs.artifacts }} diff --git a/Earthfile b/Earthfile index 070b9a50f..8ecf21e4b 100644 --- a/Earthfile +++ b/Earthfile @@ -1,5 +1,5 @@ # Set the Earthly version to 0.7 -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words livedocs sitedocs @@ -48,4 +48,4 @@ repo-config: edit-docs: LOCALLY - RUN ./earthly/docs/dev/local.py cat-ci-docs:latest \ No newline at end of file + RUN ./earthly/docs/dev/local.py cat-ci-docs:latest diff --git a/cli/go.mod b/cli/go.mod index fec7b7495..f3466188a 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -6,11 +6,10 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/alecthomas/kong v0.7.1 github.com/aws/aws-sdk-go v1.44.274 - github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b + github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb github.com/onsi/ginkgo/v2 v2.9.7 github.com/onsi/gomega v1.27.7 github.com/spf13/afero v1.9.5 - gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -24,8 +23,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.9.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/go.sum b/cli/go.sum index 33bb07292..687fd8a03 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -61,8 +61,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b h1:kwdLqircFHqYNLrBuSOybPpGPnqaqXEcRvJOWiSqRZw= -github.com/earthly/earthly/ast v0.0.2-0.20230524145153-17fad8c8f79b/go.mod h1:OoxU/3mRER8gdd4l1lxF6EgK+0vf7zQ8m9g269y6hWI= +github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb h1:dP0Wcrfbq5whzPCT/ZHGTvWVvx7S6BoF32nRHqhKt7E= +github.com/earthly/earthly/ast v0.0.2-0.20231213200555-95fef7ce99cb/go.mod h1:UX9246125xgKvMIKim7WTxR8nookovxWx8/Ze8kpYlQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -164,7 +164,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/poy/onpar v0.3.1 h1:iqwl2S3U08PY8Bm9Hfh9v3fxdig7hZy9bsZfTephng8= +github.com/poy/onpar v0.3.2 h1:yo8ZRqU3C4RlvkXPWUWfonQiTodAgpKQZ1g8VTNU9xU= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -325,8 +325,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/cli/pkg/earthfile.go b/cli/pkg/earthfile.go index 208a60dce..8182daa71 100644 --- a/cli/pkg/earthfile.go +++ b/cli/pkg/earthfile.go @@ -14,7 +14,7 @@ type Earthfile struct { Path string `json:"path,omitempty"` SourceLocation *spec.SourceLocation `json:"sourceLocation,omitempty"` Targets []spec.Target `json:"targets,omitempty"` - UserCommands []spec.UserCommand `json:"userCommands,omitempty"` + Functions []spec.Function `json:"functions,omitempty"` Version *spec.Version `json:"version,omitempty"` } diff --git a/cli/pkg/parsers/earthly_parser.go b/cli/pkg/parsers/earthly_parser.go index 56586188f..3befdfdf1 100644 --- a/cli/pkg/parsers/earthly_parser.go +++ b/cli/pkg/parsers/earthly_parser.go @@ -24,7 +24,7 @@ func (e EarthlyParser) Parse(path string) (pkg.Earthfile, error) { Path: path, SourceLocation: ef.SourceLocation, Targets: ef.Targets, - UserCommands: ef.UserCommands, + Functions: ef.Functions, Version: ef.Version, }, nil } diff --git a/docs/src/guides/docs.md b/docs/src/guides/docs.md index 1c5831f9a..890cc27e4 100644 --- a/docs/src/guides/docs.md +++ b/docs/src/guides/docs.md @@ -52,7 +52,7 @@ src: The first step of building process it preparing a source files. It is mandatory to have a `src` directory with all documentation md files in it and `mkdocs.yml` file. -This directory and file will be picked during the execution of `+SRC` UDC target. +This directory and file will be picked during the execution of `+SRC` Function. Also it is possible to replace defined `includes`, `macros` and `overrides` dirs to customize some docs appearance and configuration. @@ -78,7 +78,7 @@ docs: DO ../earthly/docs+BUILD ``` -To build a docs artifact which will be used later just invoke `+BUILD` UDC target +To build a docs artifact which will be used later just invoke `+BUILD` Function on the already prepared docs environment `+src` target which we have discussed before. ```Earthfile @@ -93,7 +93,7 @@ local: ``` To finally build a docker image which is pretty strait forward process, -you should firstly invoke `+PACKAGE` UDC target which will prepare an environment for future docs image, +you should firstly invoke `+PACKAGE` Function which will prepare an environment for future docs image, next step is to copy builded artifact from the previous step to `/usr/share/nginx/html` folder. And the last step is to save a docker image with the specified name, tag and registry if it is needed. diff --git a/docs/src/guides/languages/go.md b/docs/src/guides/languages/go.md index 595cdb279..1d35e90df 100644 --- a/docs/src/guides/languages/go.md +++ b/docs/src/guides/languages/go.md @@ -56,7 +56,7 @@ deps: # Any build dependencies should also be captured in this target. RUN apk add --no-cache gcc musl-dev - # This UDC automatically copies the go.mod and go.sum files and runs + # This Function automatically copies the go.mod and go.sum files and runs # `go mod download` to install the dependencies. DO ../../earthly/go+DEPS --ginkgo="false" ``` @@ -75,11 +75,11 @@ This target is also going to build responsible for installing external build dep These are dependencies that are not specific to a language and usually get installed system-wide. In our case, since we're building a static binary, we need `gcc` and `musl`. -Finally, the actual logic we will be using is encapsulate in a UDC. +Finally, the actual logic we will be using is encapsulate in a Function. This is a very common pattern, as an `Earthfile` can get repetitive across a repository. -In our case, we use the `go+DEPS` UDC that will automatically copy our `go.mod` and `go.sum` files and then execute +In our case, we use the `go+DEPS` Function that will automatically copy our `go.mod` and `go.sum` files and then execute `go mod download`. -The UDC will also establish a cache for the Go tooling. +The Function will also establish a cache for the Go tooling. This means that, even if our source code changes, we'll see a substantial speed boost when compiling because the cache is preserved across Earthly runs. @@ -96,10 +96,10 @@ check: # This target checks the overall health of the source code. FROM +src - # This UDC validates the code is formatted according to Go standards. + # This Function validates the code is formatted according to Go standards. DO ../../earthly/go+FMT --src="go.mod go.sum cmd" - # This UDC runs golangci-lint to check for common errors. + # This Function runs golangci-lint to check for common errors. DO ../../earthly/go+LINT --src="go.mod go.sum cmd" ``` @@ -111,8 +111,8 @@ Any future targets which need access to the source code will inherit from this t Now that the source code is available, we can begin performing static checks. These checks are intended to verify the code is healthy and conforms to a certain standard. -As we did in the previous section, here we rely on UDCs again to perform these checks. -These two UDCs will validate the code formatting is correct and also perform a series of lints to validate code quality. +As we did in the previous section, here we rely on Functions again to perform these checks. +These two Functions will validate the code formatting is correct and also perform a series of lints to validate code quality. Note that these checks are fast (compared to later steps) and perform quick feedback on code quality. Since this is the first target run in CI, we want to fail the CI as quickly as possible if we can easily find code quality issues. diff --git a/docs/src/guides/languages/postgresql.md b/docs/src/guides/languages/postgresql.md index b6d86ac13..700c913ba 100644 --- a/docs/src/guides/languages/postgresql.md +++ b/docs/src/guides/languages/postgresql.md @@ -62,7 +62,7 @@ builder: The first target we are going to consider will be responsible to prepare a PostgreSQL environment (Earthly `+postgres-base` target), migrations, migrations configuration and seed data (`COPY --dir ./migrations ./data ./refinery.toml .`), -doing some final build step (Earthly `+BUILDER` UDC target). +doing some final build step (Earthly `+BUILDER` Function). In the next steps we are going to inheriting from this `+builder` target which contains all necessary data, dependencies, environment to properly run PostgreSQL database. @@ -92,8 +92,8 @@ These checks are intended to verify the code is healthy and well formatted to a and done with the help of the `sqlfluff` tool which is already configured during the `+postgres-base` target. To apply and fix some formatting issues you can run `+format` target which will picks up directory -where your Earthly file lies in as a source dir for formatting and run `+FORMAT` UDC target. -Under the hood `+FORMAT` UDC target runs `sqlfluff-image` docker image, +where your Earthly file lies in as a source dir for formatting and run `+FORMAT` Function. +Under the hood `+FORMAT` Function runs `sqlfluff-image` docker image, which contains the same configuration and setup which is applied during the `+check`. @@ -118,7 +118,7 @@ With the `*.sql` files validation out of the way, we can finally build our Postg Since we need migration and seed data files, we'll inherit from the `builder` target. The actual image build process is pretty straight-forward -and fully defined under the `+BUILD` UDC target. +and fully defined under the `+BUILD` Function. The only thing it is needed to specify is a few arguments: * `tag` - the tag of the image, default value `latest`. diff --git a/docs/src/guides/languages/python.md b/docs/src/guides/languages/python.md index a36d2ee24..b6ae6e6a5 100644 --- a/docs/src/guides/languages/python.md +++ b/docs/src/guides/languages/python.md @@ -62,7 +62,7 @@ so it is mandatory to have `pyproject.toml` and `poetry.lock` files in the root The fist step of the `builder` target is prepare a Python environment with poetry via `+python-base` target. Next step is to copy source code of the project and finally finalize the build -with some poetry project setup which is done with `+BUILDER` UDC target. +with some poetry project setup which is done with `+BUILDER` Function. ### Running checks diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index c2f81932f..f43f85197 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -69,8 +69,8 @@ The fist step of the `builder` target is to prepare a Rust environment via `+rus Next step is to copy source code of the project. Note that you need to copy only needed files for Rust build process, any other irrelevant stuff should omitted. -And finally finalize the build with `+SETUP` UDC target. -The `+SETUP` UDC target requires `rust-toolchain.toml` file, +And finally finalize the build with `+SETUP` Function. +The `+SETUP` Function requires `rust-toolchain.toml` file, with the specified `channel` option in it. This `rust-toolchain.toml` file could be specified via the `toolchain` argument of the `+SETUP` target like this @@ -113,8 +113,8 @@ check: With prepared environment and all data, we're now ready to start operating with the source code and configuration files. The `hosted-check` target which actually performs all checks and validation -with the help of `+CHECK` UDC target. -The `+CHECK` UDC target performs static checks of the Rust project as +with the help of `+CHECK` Function. +The `+CHECK` Function performs static checks of the Rust project as `cargo fmt`, `cargo machete`, `cargo deny` which will validate formatting, find unused dependencies and any supply chain issues with dependencies. Here is the list of steps (look at `./earthly/rust/scripts/std_checks.sh`): @@ -181,8 +181,8 @@ Obviously it inherits `builder` target environment and than performs build of th Important to note that in this particular example we are dealing with the executable Rust project, so it produces binary as a final artifact. Another case of the building Rust library we will consider later. -Actual build process is done with `+BUILD` UDC target. -The `+BUILD` UDC have few arguments `libs` and `bins`, +Actual build process is done with `+BUILD` Function. +The `+BUILD` Function have few arguments `libs` and `bins`, they should be specified to properly generate `cargo-modules` docs (see description below). The `libs` argument takes a list of library crate's names in your Rust project, e.g. `--libs="crate1 crate2"`. @@ -209,11 +209,11 @@ Generated artifacts are `doc/workspace.dot`, `doc/full.dot`, `doc/all.dot` files Generated artifacts are `doc/$crate.$bin.bin.modules.tree`, `doc/$crate.$bin.bin.modules.dot` for the specified `--bins="crate1/bin1"` argument and `target/doc/$crate.lib.modules.tree`, `target/doc/$crate.lib.modules.dot` -for the specified `--libs="crate1"` argument of the `+BUILD` UDC. +for the specified `--libs="crate1"` argument of the `+BUILD` Function. Next steps is mandatory if you are going to produce a binary as an artifact, for Rust libraries the are not mandatory and could be omitted. -The `+SMOKE_TEST` UDC target checks that produced binary with the specified name (`--bin` argument) +The `+SMOKE_TEST` Function checks that produced binary with the specified name (`--bin` argument) is executable, isn't a busted mess. Final step is to provide desired artifacts: docs and binary. diff --git a/docs/src/guides/markdown.md b/docs/src/guides/markdown.md index 148f33b35..d9819a906 100644 --- a/docs/src/guides/markdown.md +++ b/docs/src/guides/markdown.md @@ -4,14 +4,14 @@ icon: simple/markdown # Markdown Check -This Earthly Target and UDC enables uniform linting of Markdown files to maintain consistency and quality. +This Earthly Target and Function enables uniform linting of Markdown files to maintain consistency and quality. -This UDC is **NOT** intended to be used inside container builds. +This Function is **NOT** intended to be used inside container builds. Its sole purpose is to enforce uniform style rules for all markdown files in a repository. It makes no assumptions about which files may or may not end up inside a container or are part of a build. This is *INTENTIONAL*. -IF this UDC is used inside a container build, it is **NOT** a bug if it does not do the correct thing. +IF this Function is used inside a container build, it is **NOT** a bug if it does not do the correct thing. ## Introduction @@ -83,7 +83,7 @@ markdown-check-fix: DO github.com/input-output-hk/catalyst-ci/earthly/mdlint:+MDLINT_LOCALLY --src=$(echo ${PWD}) --fix=--fix ``` -In this use case, the UDC is run Locally, so that the markdown in the repo can be directly checked. +In this use case, the Function is run Locally, so that the markdown in the repo can be directly checked. !!! Note @@ -105,4 +105,4 @@ For more example, please refer to this [doc](https://github.com/DavidAnson/markd mdlint-cli2 is integrated into VSCode and may be integrated into other Editors. The editor integration should pick up both the `.markdownlint.jsonc` and `.markdownlint-cli2.jsonc` configuration files. -It will then behave exactly the same as the Earthly UDC. +It will then behave exactly the same as the Earthly Function. diff --git a/docs/src/guides/spellcheck.md b/docs/src/guides/spellcheck.md index 053356e1a..02e30f501 100644 --- a/docs/src/guides/spellcheck.md +++ b/docs/src/guides/spellcheck.md @@ -142,4 +142,4 @@ For these files/paths, exclude them from the spell check by adding their filenam `cspell` is integrated into VSCode and may be integrated into other Editors. -The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly UDC. +The editor integration should pick up the `cspell.json` configuration file and behave exactly the same as the Earthly Function. diff --git a/docs/src/reference/function.md b/docs/src/reference/function.md new file mode 100644 index 000000000..cb954c567 --- /dev/null +++ b/docs/src/reference/function.md @@ -0,0 +1,38 @@ +# Functions + +## Overview + +The Catalyst CI repository provides a number of Earthly +[Functions](https://docs.earthly.dev/docs/guides/functions). +You can think of a Function as a reusable snippet of Earthly code that serves the same purpose as functions in a common programming +language. +Functions are helpful for several reasons: + +1. They keep Earthfiles DRY +2. They can encapsulate complex logic into a simple contractual interface +3. They enforce standardization and prevent solving the same problem in multiple different ways + +The third reason is particularly useful for Catalyst as we have multiple repositories with dozens of Earthfiles often solving +similar problems. + +## Usage + +You are highly encouraged to review the currently available Functions in the +[Catalyst CI repository](https://github.com/input-output-hk/catalyst-ci/tree/master/earthly). +The folder structure is broken out by language/technology and should be relatively easy to navigate. +You can incorporate these Functions into your Earthfiles by using something like below: + +```Earthfile +DO github.com/input-output-hk/catalyst-ci/earthly/+ --arg1=value1 +``` + +Replacing `folder` and `Function_NAME` respectively. +The passing of an argument is optional, as some Functions do not require any input arguments. + +## Contributing + +Please feel encouraged to contribute Functions to the repository. +If you're seeing the same logic being re-used across multiple Earthfiles in a +repository, this is a good candidate for refactoring into a Function. +Additionally, for SMEs who are aware of language best practices, encoding those +into a Function will help increase the overall health of the CI process. diff --git a/docs/src/reference/udc.md b/docs/src/reference/udc.md deleted file mode 100644 index 8dfc85249..000000000 --- a/docs/src/reference/udc.md +++ /dev/null @@ -1,37 +0,0 @@ -# UDCs - -## Overview - -The Catalyst CI repository provides a number of Earthly [User Defined Commands](https://docs.earthly.dev/docs/guides/udc) (UDCs). -You can think of a UDC as a reusable snippet of Earthly code that serves the same purpose as functions in a common programming -language. -UDCs are helpful for several reasons: - -1. They keep Earthfiles DRY -2. They can encapsulate complex logic into a simple contractual interface -3. They enforce standardization and prevent solving the same problem in multiple different ways - -The third reason is particularly useful for Catalyst as we have multiple repositories with dozens of Earthfiles often solving -similar problems. - -## Usage - -You are highly encouraged to review the currently available UDCs in the -[Catalyst CI repository](https://github.com/input-output-hk/catalyst-ci/tree/master/earthly). -The folder structure is broken out by language/technology and should be relatively easy to navigate. -You can incorporate these UDCs into your Earthfiles by using something like below: - -```Earthfile -DO github.com/input-output-hk/catalyst-ci/earthly/+ --arg1=value1 -``` - -Replacing `folder` and `UDC_NAME` respectively. -The passing of an argument is optional, as some UDCs do not require any input arguments. - -## Contributing - -Please feel encouraged to contribute UDCs to the repository. -If you're seeing the same logic being re-used across multiple Earthfiles in a repository, this is a good candidate for refactoring -into a UDC. -Additionally, for SMEs who are aware of language best practices, encoding those into a UDC will help increase the overall health of -the CI process. diff --git a/docs/src/style/index.md b/docs/src/style/index.md index 5d55acc97..59725b1e0 100644 --- a/docs/src/style/index.md +++ b/docs/src/style/index.md @@ -12,6 +12,13 @@ In most circumstances, the standards provided by this style guide should *not* b If an exception must me made, the rationale should be included in the respective PR. Any `Earthfile` which does not adhere to this style guide will be rejected if no further justification is made. +### Earthly User Defined Commands (Functions) deprecated starting from Earthly 0.8 + + +!!! Warning +Please be advised that Earthly's File Descriptor Cache (FDC) is deprecated, effective upon updating to Earthly 0.8. + + ## Organization ### Adhere to a consistent structure @@ -156,13 +163,14 @@ This can be through inheriting from it or by directly copying artifacts. The main point is that a subproject should not have multiple builds scattered in various places. Each subproject has an authoritative `build` target that *all* targets use when fetching build artifacts. -### Prefer User Defined Commands (UDCs) +### Prefer FUNCTION -The primary purpose of a UDC is to reduce boilerplate and promote reusing common workflows. +The primary purpose of a Function is to reduce boilerplate and promote reusing common workflows. Many build patterns tend to be repetitive. For example, copying a package lockfile and installing dependencies is very common. - -In these cases, a UDC should be preferred. -The `catalyst-ci` repository provides a number of UDCs in the `earthly` subdirectory. +In these cases, a FUNCTION should be preferred. +The catalyst-ci repository provides a number of FUNCTIONs in the earthly subdirectory. These should be used prior to writing a new one. If a common use case is not covered in this subdirectory, a PR should be opened to add it. +The use of functions in Earthly contributes to a more modular and organized build system, +enhancing code readability and maintainability. diff --git a/earthly/bash/Earthfile b/earthly/bash/Earthfile index b1d09a400..93617567f 100644 --- a/earthly/bash/Earthfile +++ b/earthly/bash/Earthfile @@ -1,5 +1,5 @@ # cspell UDCs and Containers. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # Internal: builder creates a container we can use to execute shellcheck builder: @@ -22,7 +22,7 @@ builder: # shellcheck - Check all shell files recursively in the src with shellcheck. SHELLCHECK: - COMMAND + FUNCTION ARG --required src FROM +builder @@ -30,3 +30,4 @@ SHELLCHECK: COPY --dir $src /src RUN ./check-all.sh /src + diff --git a/earthly/cspell/Earthfile b/earthly/cspell/Earthfile index 5b9932233..8ef1fb488 100644 --- a/earthly/cspell/Earthfile +++ b/earthly/cspell/Earthfile @@ -1,10 +1,10 @@ # cspell UDCs and Containers. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 CHECK: # Spell checking all docs and code is done with cspell # See: cspell.org - COMMAND + FUNCTION # Where we want to run the `lint` from. Use `.` to check the whole repo. ARG src=. @@ -17,4 +17,4 @@ CHECK: COPY $src . - RUN cspell-cli lint . --dot + RUN cspell-cli lint . --dot \ No newline at end of file diff --git a/earthly/docs/Earthfile b/earthly/docs/Earthfile index e26dd20f1..74b8a16e0 100644 --- a/earthly/docs/Earthfile +++ b/earthly/docs/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words libgcc freetype lcms openjpeg etag @@ -64,7 +64,7 @@ common: # Common src setup SRC: - COMMAND + FUNCTION FROM +deps @@ -81,7 +81,7 @@ SRC: # Build the docs - We always do this in a `docs` target. # The only target that needs customizing is the `src` target. BUILD: - COMMAND + FUNCTION RUN /scripts/build.sh @@ -95,7 +95,7 @@ BUILD: # We do however build a docs package which is used for locally serving the docs # which assists in documentation writing. PACKAGE: - COMMAND + FUNCTION # Use the official Nginx base image FROM nginx:alpine3.18-slim @@ -121,4 +121,4 @@ PACKAGE: # And then save the image # SAVE IMAGE image_name - \ No newline at end of file + diff --git a/earthly/go/Earthfile b/earthly/go/Earthfile index e459ae29e..df5c8e5e2 100644 --- a/earthly/go/Earthfile +++ b/earthly/go/Earthfile @@ -1,7 +1,7 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 DEPS: - COMMAND + FUNCTION ARG ginkgo="true" @@ -24,7 +24,7 @@ DEPS: SAVE ARTIFACT go.sum AS LOCAL go.sum FMT: - COMMAND + FUNCTION # latest linux/amd64 FROM cytopia/gofmt@sha256:ae76abdbc96780819ce175f7ba4bf1fec98e9f708eb680d0b949d1737a959983 @@ -39,7 +39,7 @@ FMT: RUN gofmt -l . | grep . && exit 1 || exit 0 LINT: - COMMAND + FUNCTION # v1.54-alpine FROM golangci/golangci-lint@sha256:a8423cab177c2863b863ee6d9f00c58f101cadd6120e195b2bb59fdfa189c1ae @@ -55,6 +55,8 @@ LINT: ENV GOLANGCI_LINT_CACHE=/cache/golangci-lint ENV GOCACHE=/cache/go-build + RUN go get ./... + RUN golangci-lint run \ --timeout 5m \ -E errchkjson \ @@ -64,4 +66,4 @@ LINT: -E gosec \ -E nilnil \ -E revive \ - -E testpackage \ No newline at end of file + -E testpackage diff --git a/earthly/mdlint/Earthfile b/earthly/mdlint/Earthfile index a41b5a877..edb0555f4 100644 --- a/earthly/mdlint/Earthfile +++ b/earthly/mdlint/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words markdownlint MDLINT_LOCALLY: @@ -14,7 +14,7 @@ MDLINT_LOCALLY: # in the `next` branch of `markdownlint`. # Container is built by: # https://github.com/DavidAnson/markdownlint-cli2/blob/next/docker/Dockerfile-rules - COMMAND + FUNCTION # Directory we need to run lint checks against. ARG --required src @@ -53,7 +53,7 @@ CHECK: # in the `next` branch of `markdownlint`. # Container is built by: # https://github.com/DavidAnson/markdownlint-cli2/blob/next/docker/Dockerfile-rules - COMMAND + FUNCTION ARG src = . @@ -84,4 +84,4 @@ mdlint-test: ARG src=$(echo ${PWD}/../../) - DO +MDLINT_LOCALLY --src=${src} + RUN +MDLINT_LOCALLY --src=${src} diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index 0a0e3647f..ea494a6e4 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,5 +1,5 @@ # Common PostgreSQL Earthly builders -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words colordiff nushell psycopg dbviz @@ -59,7 +59,7 @@ postgres-base: # seed - OPTIONAL - Location of Seed data directory - DEFAULT: ./seed # refinery_toml - OPTIONAL - Location of refinery,toml which configures migrations. DEFAULT: ./refinery.toml BUILDER: - COMMAND + FUNCTION ARG migrations=./migrations ARG seed=./seed @@ -88,7 +88,7 @@ BUILDER: # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+BUILD --image_name= # DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:+DOCS DOCS: - COMMAND + FUNCTION ARG diagrams=./diagrams.json ARG migrations=./migrations @@ -109,7 +109,7 @@ DOCS: # Linter checks for sql files CHECK: - COMMAND + FUNCTION RUN /scripts/std_checks.sh @@ -118,7 +118,7 @@ CHECK: # Can not do it LOCALLY as the command will change ownership of the fixed files # if run within docker. FORMAT: - COMMAND + FUNCTION # Can not run format LOCALLY as it changes permissions of files to root:root RUN --no-cache printf "%s\n%s\n%s\n" \ @@ -143,7 +143,7 @@ integration-test: INTEGRATION_TEST_SETUP: - COMMAND + FUNCTION FROM +integration-test @@ -156,7 +156,7 @@ INTEGRATION_TEST_SETUP: # Arguments: # * image_name: The name of the image (required). BUILD: - COMMAND + FUNCTION ARG --required image_name USER postgres:postgres diff --git a/earthly/python/Earthfile b/earthly/python/Earthfile index 2b078886c..dbe99a9a0 100644 --- a/earthly/python/Earthfile +++ b/earthly/python/Earthfile @@ -1,5 +1,5 @@ # Common Python UDCs and Builders. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words libgcc @@ -33,7 +33,7 @@ python-base: # Parameters: # opts : Options passed to the `poetry install` command. BUILDER: - COMMAND + FUNCTION ARG opts # Copy our dependencies. diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index ae702cb41..2164b8ea9 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,5 +1,5 @@ # Common Rust UDCs and Builders. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset readelf depgraph lcov # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT @@ -96,7 +96,7 @@ rust-base-all-hosts: # Parameters: # * toolchain : The `rust-toolchain` toml file. SETUP: - COMMAND + FUNCTION ARG toolchain=./rust-toolchain.toml # Copy our toolchain dependency. @@ -116,7 +116,7 @@ SETUP: # Steps we do during the `check` CI phase for all Rust programs CHECK: - COMMAND + FUNCTION # This is set up so that ALL checks are run and it will fail if any fail. # This improves visibility into all issues that need to be corrected for `check` @@ -128,7 +128,7 @@ CHECK: # * libs : The list of lib crates `cargo-modules` docs to build. # * bins : The list of binaries `cargo-modules` docs to build. BUILD: - COMMAND + FUNCTION ARG libs="" ARG bins="" @@ -136,7 +136,7 @@ BUILD: # Run unit tests and generates test and coverage report artifacts UNIT_TEST: - COMMAND + FUNCTION # remove artifacts that may affect the coverage results RUN cargo llvm-cov clean --workspace # run unit tests and display test result and test coverage @@ -146,7 +146,7 @@ UNIT_TEST: # Check if the build executable, isn't a busted mess. SMOKE_TEST: - COMMAND + FUNCTION ARG --required bin RUN ldd target/$TARGETARCH/release/$bin diff --git a/examples/go/Earthfile b/examples/go/Earthfile index 74a9062e3..8e102e6d9 100644 --- a/examples/go/Earthfile +++ b/examples/go/Earthfile @@ -1,6 +1,6 @@ # WARNING: If you modify this file, please update the guide that it is dependent # on in docs/guides/languages/go.md. -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # The structure of this Earthfile is derived from the style guide: # https://input-output-hk.github.io/catalyst-ci/style/#adhere-to-a-consistent-structure @@ -73,4 +73,4 @@ publish-example: COPY +build/hello . # Use the cached artifact from the build target. ENTRYPOINT ["/app/hello"] - SAVE IMAGE hello:${tag} \ No newline at end of file + SAVE IMAGE hello:${tag} diff --git a/examples/postgresql/Earthfile b/examples/postgresql/Earthfile index b3aac3253..de8830be3 100644 --- a/examples/postgresql/Earthfile +++ b/examples/postgresql/Earthfile @@ -1,4 +1,4 @@ -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # cspell: words psql @@ -43,7 +43,7 @@ all-tests: # Internal Function to run an specific integration test script. INTEGRATION_TEST_RUN: - COMMAND + FUNCTION ARG seed_data ARG test_script diff --git a/utilities/scripts/Earthfile b/utilities/scripts/Earthfile index b9b32b5db..a52567a91 100644 --- a/utilities/scripts/Earthfile +++ b/utilities/scripts/Earthfile @@ -1,5 +1,5 @@ # Earthfile containing common scripts for easy reference -VERSION --global-cache 0.7 +VERSION --global-cache --use-function-keyword 0.7 # Internal: bash-scripts : Common bash scripts. bash-scripts: @@ -19,12 +19,12 @@ python-scripts: # UDC to add our common bash scripts to a container image. ADD_BASH_SCRIPTS: - COMMAND + FUNCTION COPY --dir +bash-scripts/include /scripts/include # UDC to add our common bash scripts to a container image. ADD_PYTHON_SCRIPTS: - COMMAND + FUNCTION - COPY --dir +python-scripts/python /scripts/python \ No newline at end of file + COPY --dir +python-scripts/python /scripts/python \ No newline at end of file From 33c07b69836a9ba65cd39d39a51bf5229e66d50e Mon Sep 17 00:00:00 2001 From: bkioshn <35752733+bkioshn@users.noreply.github.com> Date: Thu, 18 Jan 2024 00:03:06 +0700 Subject: [PATCH 2/3] fix: change ci cli version (#162) --- .github/workflows/dogfood.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 9f89ee808..d01228b16 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -15,7 +15,7 @@ jobs: aws_ecr_registry: 332405224602.dkr.ecr.eu-central-1.amazonaws.com aws_role_arn: arn:aws:iam::332405224602:role/ci aws_region: eu-central-1 - ci_cli_version: 2.2.1 + ci_cli_version: 2.3.0 secrets: dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} From 9b5e1731c133c7d8a0c5a2dc413da0faf7f1bd33 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 18 Jan 2024 15:53:28 +0700 Subject: [PATCH 3/3] feat: Update Rust builder, add `target` flag (#140) * feat(rust): add support for wasm32 targets * feat(rust): Add verify-component-adapter from wasmtime as rust tooling * update * replace rust std_checks.sh with python version * update * fix std_checks.py * add std_build.py * fix spelling * fix dbviz build * remove uneeded TARGETARCH usage * remove std_build.sh * fix docs * add target flag to std_build.py * wip * fix * fix * add package argument * wip * cleanup * add std_docs.py script * fix spelling * wip * wip * disable cargo progress bar * fix * removed --locked flag * fix * remove SETUP udc * update docs * fix markdown * fix spell check * update config file * fix comments * wip * remove std_docs.py file * add TRY/FINALLY usage * fix spellcheck * fix * fix * add nightly channel section * add with_bench flag * Move SMOKE_TEST udc to the std_build.py script * remove nushell * fix markdown check --------- Co-authored-by: Mr-Leshiy --- docs/src/guides/languages/rust.md | 162 +++++++------- earthly/postgresql/Earthfile | 8 +- earthly/postgresql/scripts/python | 1 - earthly/rust/Earthfile | 53 +---- earthly/rust/scripts/std_build.py | 197 ++++++++++++++++++ earthly/rust/scripts/std_build.sh | 85 -------- earthly/rust/scripts/std_checks.py | 59 ++++++ earthly/rust/scripts/std_checks.sh | 47 ----- earthly/rust/stdcfgs/cargo_config.toml | 24 ++- earthly/rust/stdcfgs/deny.toml | 5 +- earthly/rust/stdcfgs/nextest.toml | 2 +- examples/rust/.cargo/config.toml | 24 ++- examples/rust/.config/nextest.toml | 2 +- examples/rust/Cargo.toml | 3 - examples/rust/Earthfile | 100 +++------ examples/rust/deny.toml | 5 +- examples/rust/rust-toolchain.toml | 4 +- utilities/dbviz/.cargo/config.toml | 24 ++- utilities/dbviz/.config/nextest.toml | 2 +- utilities/dbviz/Earthfile | 85 ++------ utilities/dbviz/deny.toml | 5 +- utilities/nushell/Earthfile | 65 ------ utilities/scripts/python/cli.py | 7 +- .../scripts/python/vendor_files_check.py | 11 + 24 files changed, 472 insertions(+), 508 deletions(-) delete mode 120000 earthly/postgresql/scripts/python create mode 100755 earthly/rust/scripts/std_build.py delete mode 100755 earthly/rust/scripts/std_build.sh create mode 100755 earthly/rust/scripts/std_checks.py delete mode 100755 earthly/rust/scripts/std_checks.sh delete mode 100644 utilities/nushell/Earthfile create mode 100644 utilities/scripts/python/vendor_files_check.py diff --git a/docs/src/guides/languages/rust.md b/docs/src/guides/languages/rust.md index f43f85197..e7650adda 100644 --- a/docs/src/guides/languages/rust.md +++ b/docs/src/guides/languages/rust.md @@ -9,7 +9,7 @@ tags: # :simple-rust: Rust - + ## Introduction @@ -80,47 +80,29 @@ By default `toolchain` setup to `rust-toolchain.toml`. ### Running checks ```Earthfile -# Test rust build container - Use best architecture host tools. -hosted-check: +# Run check using the most efficient host tooling +# CI Automated Entry point. +check: FROM +builder - DO ./../../earthly/rust+CHECK + RUN /scripts/std_checks.py # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. -all-hosts-check: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-check - -## Standard CI targets. -## -## These targets are discovered and executed automatically by CI. - -# Run check using the most efficient host tooling -# CI Automated Entry point. -check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-check - ELSE - BUILD --platform=linux/amd64 +hosted-check - END +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check ``` With prepared environment and all data, we're now ready to start operating with the source code and configuration files. -The `hosted-check` target which actually performs all checks and validation -with the help of `+CHECK` Function. -The `+CHECK` Function performs static checks of the Rust project as +The `check` target which actually performs all checks and validation +with the help of `std_checks.py` script. +This script performs static checks of the Rust project as `cargo fmt`, `cargo machete`, `cargo deny` which will validate formatting, find unused dependencies and any supply chain issues with dependencies. -Here is the list of steps (look at `./earthly/rust/scripts/std_checks.sh`): +Here is the list of steps (look at `./earthly/rust/scripts/std_checks.py`): 1. `cargo fmtchk` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), -look at `./earthly/rust/stdcfgs/config.toml`)Checking Rust Code Format. +look at `./earthly/rust/stdcfgs/cargo_config.toml`)Checking Rust Code Format. 2. Checking configuration files for consistency. 3. `cargo machete` - Checking for Unused Dependencies. 4. `cargo deny check` - Checking for Supply Chain Issues. @@ -131,8 +113,8 @@ to be the same as defined in `earthly/rust/stdcfgs` directory of the `catalyst-c So when you are going to setup a new Rust project copy these configuration files described above to the appropriate location of your Rust project. -Another targets as `all-hosts-check` and `check` (running on CI) just invoke `hosted-check` -with the specified `--platform`. +Another target as `all-hosts-check` just invokes `check` with the specified `--platform`. +It is needed for the local development to double check that everything is works for different platforms. It is important to define a `linux` target platform with a proper cpu architecture for the Rust project when you are building it inside Docker and check the build process with different scenarios. @@ -141,56 +123,84 @@ The same approach we will see for the another targets of this guide. ### Build ```Earthfile -# Build the service. -hosted-build: +# Run build using the most efficient host tooling +# CI Automated Entry point. +build: FROM +builder - DO ./../../earthly/rust+BUILD --libs="bar" --bins="foo/foo" + TRY + RUN /scripts/std_build.py --build_flags="" \ + --with_test \ + --with_bench \ + --cov_report="coverage-report.info" \ + --libs="bar" \ + --bins="foo/foo" + FINALLY + SAVE ARTIFACT target/nextest/ci/junit.xml example.junit-report.xml AS LOCAL + SAVE ARTIFACT coverage-report.info example.coverage-report.info AS LOCAL + END - DO ./../../earthly/rust+SMOKE_TEST --bin="foo" + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/foo foo - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/foo foo + DO ./../../earthly/rust+SMOKE_TEST --bin="foo" # Test which runs check with all supported host tooling. Needs qemu or rosetta to run. # Only used to validate tooling is working across host toolsets. -all-hosts-build: - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted - -# Run build using the most efficient host tooling -# CI Automated Entry point. -build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-build - ELSE - BUILD --platform=linux/amd64 +hosted-build - END +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build ``` -After successful performing checks of the Rust project we can finally build artifacts. -As it was discussed in the previous section, actual job is done with `hosted-build` target, -other targets needs to configure different platform running options. -So we will focus on `hosted-build` target. +After successful performing checks of the Rust project we can finally `build` artifacts. Obviously it inherits `builder` target environment and than performs build of the binary. Important to note that in this particular example we are dealing with the executable Rust project, so it produces binary as a final artifact. Another case of the building Rust library we will consider later. -Actual build process is done with `+BUILD` Function. -The `+BUILD` Function have few arguments `libs` and `bins`, -they should be specified to properly generate `cargo-modules` docs (see description below). -The `libs` argument takes a list of library crate's names in your Rust project, e.g. +Actual build process is done with the `std_build.py` script. +Here is the full list of configuration of this script: + +```bash + usage: std_build.py [-h] [--build_flags BUILD_FLAGS] + [--doctest_flags DOCTEST_FLAGS] [--test_flags TEST_FLAGS] + [--bench_flags BENCH_FLAGS] [--with_test] + [--cov_report COV_REPORT] [--with_bench] [--libs LIBS] + [--bins BINS] + + Rust build processing. + + options: + -h, --help show this help message and exit + --build_flags BUILD_FLAGS + Additional command-line flags that can be passed to + the `cargo build` command. + --doctest_flags DOCTEST_FLAGS + Additional command-line flags that can be passed to + the `cargo testdocs` command. + --test_flags TEST_FLAGS + Additional command-line flags that can be passed to + the `cargo testunit` command. + --bench_flags BENCH_FLAGS + Additional command-line flags that can be passed to + the `cargo bench` command. + --with_test Flag to indicate whether to run tests (including unit + tests and doc tests). + --cov_report COV_REPORT + The output coverage report file path. If omitted, + coverage will not be run. + --with_bench Flag to indicate whether to run benchmarks. + --libs LIBS The list of lib crates `cargo-modules` docs to build + separated by comma. + --bins BINS The list of binaries `cargo-modules` docs to build and + made a smoke tests on them. . +``` + +Note that the `libs` argument takes a list of library crate's names in your Rust project, e.g. `--libs="crate1 crate2"`. The `bins` argument takes a list of binary crate's names and binary names in your Rust project, e.g. `--bins="crate1/bin1 crate1/bin2 crate2/bin1"`, note that each binary name correspond to each crate and separated in this list with `/` symbol. Under this build process we perform different steps of compiling and validating of our Rust project, -here is the list of steps (look at `./earthly/rust/scripts/std_build.sh`): +here is the list of steps (look at `./earthly/rust/scripts/std_build.py` and `./earthly/rust/scripts/std_docs.py`): 1. `cargo build` - Building all code in the workspace. 2. `cargo lint` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), @@ -202,19 +212,17 @@ look at `./earthly/rust/stdcfgs/config.toml`)Checking Documentation can be gener look at `./earthly/rust/stdcfgs/config.toml`)Checking Self contained Unit tests all pass. 5. `cargo testdocs` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), look at `./earthly/rust/stdcfgs/config.toml`)Checking Documentation tests all pass. -6. `cargo bench` - Checking Benchmarks all run to completion. -7. `cargo depgraph` - Generating dependency graph based on the Rust code. +6. `cargo testcov` ([cargo alias](https://doc.rust-lang.org/cargo/reference/config.html#alias), +look at `./earthly/rust/stdcfgs/config.toml`)Checking Self contained Unit tests all pass and collect coverage. +7. `cargo bench` - Checking Benchmarks all run to completion. +8. `cargo depgraph` - Generating dependency graph based on the Rust code. Generated artifacts are `doc/workspace.dot`, `doc/full.dot`, `doc/all.dot` files. -8. `cargo modules` - Generating modules trees and graphs based on the Rust code. +9. `cargo modules` - Generating modules trees and graphs based on the Rust code. Generated artifacts are `doc/$crate.$bin.bin.modules.tree`, `doc/$crate.$bin.bin.modules.dot` for the specified `--bins="crate1/bin1"` argument and `target/doc/$crate.lib.modules.tree`, `target/doc/$crate.lib.modules.dot` -for the specified `--libs="crate1"` argument of the `+BUILD` Function. - -Next steps is mandatory if you are going to produce a binary as an artifact, -for Rust libraries the are not mandatory and could be omitted. -The `+SMOKE_TEST` Function checks that produced binary with the specified name (`--bin` argument) -is executable, isn't a busted mess. +for the specified `--libs="crate1"` argument. +10. Running smoke tests on provided binary names (`--bins` argument). Final step is to provide desired artifacts: docs and binary. @@ -231,6 +239,14 @@ please follow this [guide](./../../onboarding/index.md). It is pretty strait forward for this builder process, because as a part of `+build` target we already creating a docker image. +## Rust `nightly` channel + +Be aware that we are running some tools in the `nightly` channel, such as `cargo fmt` and `cargo docs`. +It is highly likely that the `nightly` toolchain version on the CI machines differs from what you have locally. +Unfortunately, Rust tooling does not have the capability to preserve and maintain consistency between +`stable` and `nightly` toolchains simultaneously. +In our builds, we only preserve the `stable` toolchain version (`rust-toolchain.toml` file). + ## Conclusion You can see the final `Earthfile` [here](https://github.com/input-output-hk/catalyst-ci/blob/master/examples/rust/Earthfile) diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index ea494a6e4..465a66fea 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -1,7 +1,7 @@ # Common PostgreSQL Earthly builders VERSION --global-cache --use-function-keyword 0.7 -# cspell: words colordiff nushell psycopg dbviz +# cspell: words colordiff psycopg dbviz postgres-base: FROM postgres:16.0-alpine3.18 @@ -38,9 +38,6 @@ postgres-base: COPY ../../utilities/dbviz+build/dbviz /bin RUN dbviz --help - # Get nushell - COPY ../../utilities/nushell+nushell-build/nu /bin - # Copy our set SQL files COPY --dir sql /sql @@ -134,9 +131,6 @@ integration-test: bash \ postgresql15-client # We use PostgreSQL 16, but v15 clients will work OK with it. - # Get nushell - COPY ../../utilities/nushell+nushell-build/nu /bin - COPY +postgres-base/scripts /scripts WORKDIR /test diff --git a/earthly/postgresql/scripts/python b/earthly/postgresql/scripts/python deleted file mode 120000 index 168bb203b..000000000 --- a/earthly/postgresql/scripts/python +++ /dev/null @@ -1 +0,0 @@ -../../../utilities/scripts/python \ No newline at end of file diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index 2164b8ea9..36a040256 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,7 +1,7 @@ # Common Rust UDCs and Builders. VERSION --global-cache --use-function-keyword 0.7 -# cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset readelf depgraph lcov +# cspell: words rustup miri nextest ripgrep colordiff rustfmt stdcfgs toolset depgraph lcov psycopg # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT # Base Rustup build container. @@ -34,10 +34,13 @@ rust-base: # Install necessary packages # Expand this list as needed, rather than adding more tools in later containers. - RUN apk add --no-cache \ + RUN apk add --no-cache \ musl-dev \ mold \ clang \ + py3-pip \ + py3-rich \ + py3-psycopg \ ripgrep \ bash \ colordiff \ @@ -70,7 +73,8 @@ rust-base: cargo install cargo-deny --version=0.14.3 && \ cargo install cargo-modules --version=0.10.2 && \ cargo install cargo-depgraph --version=1.5.0 && \ - cargo install cargo-llvm-cov --version=0.5.39 + cargo install cargo-llvm-cov --version=0.5.39 && \ + cargo install --git https://github.com/bytecodealliance/wasmtime --tag v16.0.0 verify-component-adapter SAVE ARTIFACT $CARGO_HOME/bin/refinery refinery @@ -78,6 +82,7 @@ rust-base: COPY --dir scripts /scripts # Copy our common scripts so we can use them inside the container. DO ../../utilities/scripts+ADD_BASH_SCRIPTS + DO ../../utilities/scripts+ADD_PYTHON_SCRIPTS # Standardized Rust configs. # Build will refuse to proceed if the projects rust configs do not match these. @@ -113,45 +118,3 @@ SETUP: rustup show && \ cargo --version && \ cargo +nightly --version - -# Steps we do during the `check` CI phase for all Rust programs -CHECK: - FUNCTION - - # This is set up so that ALL checks are run and it will fail if any fail. - # This improves visibility into all issues that need to be corrected for `check` - # to pass without needing to iterate excessively. - RUN /scripts/std_checks.sh - -# Step we do during the `build` CI phase for all Rust programs -# Parameters: -# * libs : The list of lib crates `cargo-modules` docs to build. -# * bins : The list of binaries `cargo-modules` docs to build. -BUILD: - FUNCTION - ARG libs="" - ARG bins="" - - RUN /scripts/std_build.sh "$libs" "$bins" - -# Run unit tests and generates test and coverage report artifacts -UNIT_TEST: - FUNCTION - # remove artifacts that may affect the coverage results - RUN cargo llvm-cov clean --workspace - # run unit tests and display test result and test coverage - RUN cargo llvm-cov nextest --release --bins --lib --workspace --locked -P ci - # generate lcov report - RUN cargo llvm-cov report --release --output-path coverage-report.info - -# Check if the build executable, isn't a busted mess. -SMOKE_TEST: - FUNCTION - ARG --required bin - - RUN ldd target/$TARGETARCH/release/$bin - RUN readelf -p .comment target/$TARGETARCH/release/$bin - RUN strip -v target/$TARGETARCH/release/$bin - - # ALL executables MUST have `--help` as an option. - RUN target/$TARGETARCH/release/$bin --help diff --git a/earthly/rust/scripts/std_build.py b/earthly/rust/scripts/std_build.py new file mode 100755 index 000000000..22eea24d4 --- /dev/null +++ b/earthly/rust/scripts/std_build.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +# cspell: words lcov testdocs nextest testunit depgraph testcov readelf + +import python.cli as cli +import argparse +import rich + +# This script is run inside the `build` stage. +# This is set up so that ALL build steps are run and it will fail if any fail. +# This improves visibility into all issues that need to be corrected for `build` +# to pass without needing to iterate excessively. + +def cargo_build(results: cli.Results, flags: str): + results.add(cli.run("cargo build " + + f"{flags} " + + "--release ", + name="Build all code in the workspace")) + +def cargo_clippy(results: cli.Results): + results.add(cli.run("cargo lint ", + name="Clippy Lints in the workspace check")) + +def cargo_doctest(results: cli.Results, flags: str): + results.add(cli.run("cargo testdocs " + + f"{flags} ", + name="Documentation tests all pass check")) + +def cargo_nextest(results: cli.Results, flags: str): + results.add(cli.run("cargo testunit " + + f"{flags} ", + name="Self contained Unit tests all pass check")) + +def cargo_llvm_cov(results: cli.Results, flags: str, cov_report: str): + # Remove artifacts that may affect the coverage results + res = cli.run("cargo llvm-cov clean", + name="Remove artifacts that may affect the coverage results") + results.add(res) + # Run unit tests and generates test and coverage report artifacts + if res.ok(): + res = cli.run("cargo testcov " + + f"{flags} ", + name="Self contained Unit tests and collect coverage") + results.add(res) + # Save coverage report to file if it is provided + if res.ok(): + res = cli.run("cargo llvm-cov report " + + f"{flags} " + + "--release " + + f"--output-path {cov_report} ", + name=f"Generate lcov report to {cov_report}") + results.add(res) + +def cargo_bench(results: cli.Results, flags: str): + results.add(cli.run("cargo bench " + + f"{flags} " + + "--all-targets ", + name="Benchmarks all run to completion check")) + +def cargo_doc(results: cli.Results): + results.add(cli.run("cargo +nightly docs ", + name="Documentation build")) + +def cargo_depgraph(results: cli.Results): + results.add(cli.run("cargo depgraph " + + "--workspace-only " + + "--dedup-transitive-deps " + + "> target/doc/workspace.dot ", + name="Workspace dependency graphs generation")) + results.add(cli.run("cargo depgraph " + + "--dedup-transitive-deps " + + "> target/doc/full.dot ", + name="Full dependency graphs generation")) + results.add(cli.run("cargo depgraph " + + "--all-deps " + + "--dedup-transitive-deps " + + "> target/doc/all.dot ", + name="All dependency graphs generation")) + +def cargo_modules_lib(results: cli.Results, lib: str): + # Generate tree + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate tree --orphans --types --traits --tests --all-features " + + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.tree' ", + name=f"Generate Module Trees for {lib}")) + # Generate graph + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate graph --all-features --modules " + + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.dot' ", + name=f"Generate Module Graphs for {lib}")) + +def cargo_modules_bin(results: cli.Results, package: str, bin: str): + # Generate tree + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate tree --orphans --types --traits --tests --all-features " + + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.tree' ", + name=f"Generate Module Trees for {package}/{bin}")) + # Generate graph + results.add(cli.run("NO_COLOR=1 " + + "cargo modules generate graph --all-features --modules " + + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.dot' ", + name=f"Generate Module Graphs for {package}/{bin}")) + +# ALL executables MUST have `--help` as an option. +def help_check(results: cli.Results, bin: str): + results.add(cli.run(f"target/release/{bin} --help", + name=f"Executable '{bin}' MUST have `--help` as an option.")) + +def ldd(results: cli.Results, bin: str): + results.add(cli.run(f"ldd target/release/{bin}", + name=f"ldd for '{bin}'", + verbose=True)) + +def readelf(results: cli.Results, bin: str): + results.add(cli.run(f"readelf -p .comment target/release/{bin}", + name=f"readelf for '{bin}'", + verbose=True)) + +def strip(results: cli.Results, bin: str): + results.add(cli.run(f"strip -v target/release/{bin}", + name=f"strip for '{bin}'", + verbose=True)) + +def main(): + # Force color output in CI + rich.reconfigure(color_system="256") + + parser = argparse.ArgumentParser( + description="Rust build processing." + ) + parser.add_argument("--build_flags", default="", help="Additional command-line flags that can be passed to the `cargo build` command.") + parser.add_argument("--doctest_flags", default="", help="Additional command-line flags that can be passed to the `cargo testdocs` command.") + parser.add_argument("--test_flags", default="", help="Additional command-line flags that can be passed to the `cargo testunit` command.") + parser.add_argument("--bench_flags", default="", help="Additional command-line flags that can be passed to the `cargo bench` command.") + parser.add_argument("--with_test", action='store_true', help="Flag to indicate whether to run tests (including unit tests and doc tests).") + parser.add_argument("--cov_report", default="", help="The output coverage report file path. If omitted, coverage will not be run.") + parser.add_argument("--with_bench", action='store_true', help="Flag to indicate whether to run benchmarks.") + parser.add_argument("--libs", default="", help="The list of lib crates `cargo-modules` docs to build separated by comma.") + parser.add_argument("--bins", default="", help="The list of binaries `cargo-modules` docs to build and made a smoke tests on them.") + args = parser.parse_args() + + libs = filter(lambda lib: lib != "", args.libs.split(", ")) + bins = list(filter(lambda bin: bin != "", args.bins.split(", "))) + + results = cli.Results("Rust build") + # Build the code. + cargo_build(results, args.build_flags) + # Check the code passes all clippy lint checks. + cargo_clippy(results) + # Check if all Self contained tests pass (Test that need no external resources). + if args.with_test: + # Check if all documentation tests pass. + cargo_doctest(results, args.doctest_flags) + if args.cov_report == "": + # Without coverage report + cargo_nextest(results, args.test_flags) + else: + # With coverage report + cargo_llvm_cov(results, args.test_flags, args.cov_report) + + # Check if any benchmarks defined run (We don't validate the results.) + if args.with_bench: + cargo_bench(results, args.bench_flags) + + # Generate all the documentation. + cargo_doc(results) + # Generate dependency graphs + cargo_depgraph(results) + + for lib in libs: + cargo_modules_lib(results, lib) + + for bin in bins: + package, bin_name = bin.split('/') + cargo_modules_bin(results, package, bin_name) + + results.print() + if not results.ok(): + exit(1) + + # Check if the build executable, isn't a busted mess. + results = cli.Results("Smoke test") + + for bin in bins: + _, bin_name = bin.split('/') + help_check(results, bin_name) + ldd(results, bin_name) + readelf(results, bin_name) + strip(results, bin_name) + + results.print() + if not results.ok(): + exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/earthly/rust/scripts/std_build.sh b/earthly/rust/scripts/std_build.sh deleted file mode 100755 index 0829dd432..000000000 --- a/earthly/rust/scripts/std_build.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -# cspell: words testunit testdocs RUSTDOCFLAGS Zunstable depgraph - -# shellcheck disable=SC2046,SC2312 - -# This script is run inside the `check` stage for rust projects to perform all -# high level non-compilation checks. -# These are the Standard checks which ALL rust targets must pass before they -# will be scheduled to be `build`. -# Individual targets can add extra `check` steps, but these checks must always -# pass. - -source "/scripts/include/colors.sh" - -# This is set up so that ALL build steps are run and it will fail if any fail. -# This improves visibility into all issues that need to be corrected for `build` -# to pass without needing to iterate excessively. - -rc=0 - -## Build the code -status "${rc}" "Building all code in the workspace" \ - cargo build --release --workspace --locked -rc=$? - -## Check the code passes all clippy lint checks. -status "${rc}" "Checking all Clippy Lints in the workspace" \ - cargo lint -rc=$? - -## Check we can generate all the documentation -status "${rc}" "Checking Documentation can be generated OK" \ - cargo docs -rc=$? - -## Check if all documentation tests pass. -status "${rc}" "Checking Documentation tests all pass" \ - cargo testdocs -rc=$? - -## Check if any benchmarks defined run (We don;t validate the results.) -status "${rc}" "Checking Benchmarks all run to completion" \ - cargo bench --all-targets -rc=$? - -## Generate dependency graphs -status "${rc}" "Generating workspace dependency graphs" \ - $(cargo depgraph --workspace-only --dedup-transitive-deps > target/doc/workspace.dot) -rc=$? -status "${rc}" "Generating full dependency graphs" \ - $(cargo depgraph --dedup-transitive-deps > target/doc/full.dot) -rc=$? -status "${rc}" "Generating all dependency graphs" \ - $(cargo depgraph --all-deps --dedup-transitive-deps > target/doc/all.dot) -rc=$? - -export NO_COLOR=1 -## Generate Module Trees for documentation purposes. -for lib in $1; do - status "${rc}" "Generate Module Trees for ${lib}" \ - $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package "${lib}" --lib > "target/doc/${lib}.lib.modules.tree") - rc=$? - - status "${rc}" "Generate Module Graphs for ${lib}" \ - $(cargo modules generate graph --all-features --modules \ - --package "${lib}" --lib > "target/doc/${lib}.lib.modules.dot") - rc=$? -done -for bin in $2; do - IFS="/" read -r package bin <<< "${bin}" - status "${rc}" "Generate Module Trees for ${package}/${bin}" \ - $(cargo modules generate tree --orphans --types --traits --tests --all-features \ - --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.tree") - rc=$? - - status "${rc}" "Generate Module Graphs for ${package}/${bin}" \ - $(cargo modules generate graph --all-features --modules \ - --package "${package}" --bin "${bin}" > "target/doc/${package}.${bin}.bin.modules.dot") - rc=$? -done - -# Return an error if any of this fails. -exit "${rc}" diff --git a/earthly/rust/scripts/std_checks.py b/earthly/rust/scripts/std_checks.py new file mode 100755 index 000000000..700acabaf --- /dev/null +++ b/earthly/rust/scripts/std_checks.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# cspell: words depgraph fmtchk fmtfix colordiff stdcfgs rustfmt stdcfgs nextest + +import python.cli as cli +import python.vendor_files_check as vendor_files_check +import argparse +import rich +from rich import print +import os + +# This script is run inside the `check` stage for rust projects to perform all +# high level non-compilation checks. +# These are the Standard checks which ALL rust targets must pass before they +# will be scheduled to be `build`. +# Individual targets can add extra `check` steps, but these checks must always +# pass. + +# This is set up so that ALL checks are run and it will fail if any fail. +# This improves visibility into all issues that need to be corrected for `check` +# to pass without needing to iterate excessively. + +def main(): + # Force color output in CI + rich.reconfigure(color_system="256") + + parser = argparse.ArgumentParser( + description="Rust high level non-compilation checks processing." + ) + + results = cli.Results("Rust checks") + + # Check if the rust src is properly formatted. + res = cli.run("cargo +nightly fmtchk ", + name="Rust Code Format Check") + results.add(res) + if not res.ok(): + print("[yellow]You can locally fix format errors by running: [/yellow] \n [red bold]cargo +nightly fmtfix [/red bold]") + + # Check config files. + results.add(vendor_files_check.colordiff_check(f"{os.environ.get('CARGO_HOME')}/config.toml", ".cargo/config.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/rustfmt.toml", "rustfmt.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/nextest.toml", ".config/nextest.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/clippy.toml", "clippy.toml")) + results.add(vendor_files_check.colordiff_check("/stdcfgs/deny.toml", "deny.toml")) + + # Check if we have unused dependencies declared in our Cargo.toml files. + results.add(cli.run("cargo machete", name="Unused Dependencies Check")) + # Check if we have any supply chain issues with dependencies. + results.add(cli.run("cargo deny check", name="Supply Chain Issues Check")) + + + results.print() + if not results.ok(): + exit(1) + + +if __name__ == "__main__": + main() diff --git a/earthly/rust/scripts/std_checks.sh b/earthly/rust/scripts/std_checks.sh deleted file mode 100755 index e3e18ad0f..000000000 --- a/earthly/rust/scripts/std_checks.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# cspell: words fmtchk fmtfix rustfmt stdcfgs nextest - -# This script is run inside the `check` stage for rust projects to perform all -# high level non-compilation checks. -# These are the Standard checks which ALL rust targets must pass before they -# will be scheduled to be `build`. -# Individual targets can add extra `check` steps, but these checks must always -# pass. - -source "/scripts/include/colors.sh" - -# This is set up so that ALL checks are run and it will fail if any fail. -# This improves visibility into all issues that need to be corrected for `check` -# to pass without needing to iterate excessively. - -# Check if the rust src is properly formatted. -# Note, we run this first so we can print help how to fix. -status 0 "Checking Rust Code Format" cargo +nightly fmtchk -rc=$? -if [[ "${rc}" -ne 0 ]]; then - echo -e " ${YELLOW}You can locally fix format errors by running: \`cargo +nightly fmtfix\`${NC}" -fi - -## Check if .cargo.config.toml has been modified. -check_vendored_files "${rc}" .cargo/config.toml "${CARGO_HOME:?}"/config.toml -rc=$? -check_vendored_files "${rc}" rustfmt.toml /stdcfgs/rustfmt.toml -rc=$? -check_vendored_files "${rc}" .config/nextest.toml /stdcfgs/nextest.toml -rc=$? -check_vendored_files "${rc}" clippy.toml /stdcfgs/clippy.toml -rc=$? -check_vendored_files "${rc}" deny.toml /stdcfgs/deny.toml -rc=$? - -# Check if we have unused dependencies declared in our Cargo.toml files. -status "${rc}" "Checking for Unused Dependencies" cargo machete -rc=$? - -# Check if we have any supply chain issues with dependencies. -status "${rc}" "Checking for Supply Chain Issues" cargo deny check -rc=$? - -# Return an error if any of this fails. -exit "${rc}" diff --git a/earthly/rust/stdcfgs/cargo_config.toml b/earthly/rust/stdcfgs/cargo_config.toml index 0eae0eb21..8f2abc5db 100644 --- a/earthly/rust/stdcfgs/cargo_config.toml +++ b/earthly/rust/stdcfgs/cargo_config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/earthly/rust/stdcfgs/deny.toml b/earthly/rust/stdcfgs/deny.toml index d28c1100d..0c80c3331 100644 --- a/earthly/rust/stdcfgs/deny.toml +++ b/earthly/rust/stdcfgs/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/earthly/rust/stdcfgs/nextest.toml b/earthly/rust/stdcfgs/nextest.toml index ebc7d7390..86c177122 100644 --- a/earthly/rust/stdcfgs/nextest.toml +++ b/earthly/rust/stdcfgs/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/examples/rust/.cargo/config.toml b/examples/rust/.cargo/config.toml index 0eae0eb21..8f2abc5db 100644 --- a/examples/rust/.cargo/config.toml +++ b/examples/rust/.cargo/config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/examples/rust/.config/nextest.toml b/examples/rust/.config/nextest.toml index ebc7d7390..86c177122 100644 --- a/examples/rust/.config/nextest.toml +++ b/examples/rust/.config/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml index 0dee97239..e6cfc794c 100644 --- a/examples/rust/Cargo.toml +++ b/examples/rust/Cargo.toml @@ -9,6 +9,3 @@ members = [ edition = "2021" version = "0.0.1" license = "MIT OR Apache-2.0" - - - diff --git a/examples/rust/Earthfile b/examples/rust/Earthfile index 804e57db7..5de5f553b 100644 --- a/examples/rust/Earthfile +++ b/examples/rust/Earthfile @@ -1,6 +1,6 @@ -VERSION --global-cache 0.7 +VERSION --try --global-cache 0.7 -# cspell: words TARGETARCH USERARCH toolsets rustfmt nextest +# cspell: words USERARCH toolsets rustfmt nextest # Set up our target toolchains, and copy our files. builder: @@ -12,42 +12,6 @@ builder: DO ./../../earthly/rust+SETUP -# Test rust build container - Use best architecture host tools. -hosted-check: - FROM +builder - - DO ./../../earthly/rust+CHECK - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-check: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-check - - -# Build the service. -hosted-build: - FROM +builder - - DO ./../../earthly/rust+BUILD --libs="bar" --bins="foo/foo" - - DO ./../../earthly/rust+SMOKE_TEST --bin="foo" - - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/foo foo - -# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets. -all-hosts-build: - BUILD --platform=linux/amd64 --platform=linux/arm64 +hosted-build - -test-hosted: - FROM +builder - - DO ./../../earthly/rust+UNIT_TEST - - SAVE ARTIFACT target/nextest/ci/junit.xml junit-report.xml - SAVE ARTIFACT coverage-report.info coverage-report.info - ## ----------------------------------------------------------------------------- ## ## Standard CI targets. @@ -57,48 +21,36 @@ test-hosted: # Run check using the most efficient host tooling # CI Automated Entry point. check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-check - ELSE - BUILD --platform=linux/amd64 +hosted-check - END + RUN /scripts/std_checks.py + +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check # Run build using the most efficient host tooling # CI Automated Entry point. build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +hosted-build - ELSE - BUILD --platform=linux/amd64 +hosted-build + TRY + RUN /scripts/std_build.py --build_flags="" \ + --with_test \ + --with_bench \ + --cov_report="coverage-report.info" \ + --libs="bar" \ + --bins="foo/foo" + FINALLY + SAVE ARTIFACT target/nextest/ci/junit.xml AS LOCAL example.junit-report.xml + SAVE ARTIFACT coverage-report.info AS LOCAL example.coverage-report.info END -test: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/foo foo - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +test-hosted - ELSE - BUILD --platform=linux/amd64 +test-hosted - END - -# This step simulates the full CI run for local purposes only. -local-ci-run: - BUILD +check - BUILD +build - BUILD +test \ No newline at end of file +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/examples/rust/deny.toml b/examples/rust/deny.toml index d28c1100d..0c80c3331 100644 --- a/examples/rust/deny.toml +++ b/examples/rust/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/examples/rust/rust-toolchain.toml b/examples/rust/rust-toolchain.toml index dca770e1b..f970caf8a 100644 --- a/examples/rust/rust-toolchain.toml +++ b/examples/rust/rust-toolchain.toml @@ -1,5 +1,3 @@ [toolchain] channel = "1.73.0" -profile = "default" -components = [ ] -targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", "aarch64-apple-darwin"] \ No newline at end of file +profile = "default" \ No newline at end of file diff --git a/utilities/dbviz/.cargo/config.toml b/utilities/dbviz/.cargo/config.toml index 0eae0eb21..8f2abc5db 100644 --- a/utilities/dbviz/.cargo/config.toml +++ b/utilities/dbviz/.cargo/config.toml @@ -1,7 +1,7 @@ # Use MOLD linker where possible, but ONLY in CI applicable targets. # cspell: words rustflags armv gnueabihf msvc nextest idents rustdocflags # cspell: words rustdoc lintfix lintrestrict testfast testdocs codegen testci testunit -# cspell: words fmtchk fmtfix +# cspell: words fmtchk fmtfix testcov # Configure how Docker container targets build. @@ -28,6 +28,8 @@ rustflags = [ "-C", "target-feature=-crt-static" ] +[target.wasm32-unknown-unknown] +rustflags = ["--cap-lints", "warn"] [build] @@ -110,21 +112,21 @@ codegen-units = 16 [alias] lint = "clippy --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" lintfix = "clippy --all-targets --fix --allow-dirty -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -lintrestrict = "clippy -- -D warnings -D clippy::pedantic -D clippy::restriction -D clippy::missing_docs_in_private_items" -lint-vscode = "clippy --workspace --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" +lint-vscode = "clippy --message-format=json-diagnostic-rendered-ansi --all-targets -- -D warnings -D clippy::pedantic -D clippy::unwrap_used -D clippy::expect_used -D clippy::exit -D clippy::get_unwrap -D clippy::index_refutable_slice -D clippy::indexing_slicing -D clippy::match_on_vec_items -D clippy::match_wild_err_arm -D clippy::missing_panics_doc -D clippy::panic -D clippy::string_slice -D clippy::unchecked_duration_subtraction -D clippy::unreachable -D clippy::missing_docs_in_private_items" -docs = "doc --workspace -r --all-features --no-deps --bins --document-private-items --examples --locked" +docs = "doc --release --no-deps --document-private-items --bins --lib --examples" # nightly docs build broken... when they are'nt we can enable these docs... --unit-graph --timings=html,json -Z unstable-options" -testfast = "nextest --release --workspace --locked" -testdocs = "test --doc --release --workspace --locked" +testunit = "nextest run --release --bins --lib -P ci" +testcov = "llvm-cov nextest --release --bins --lib -P ci" +testdocs = "test --doc --release" # Rust formatting, MUST be run with +nightly fmtchk = "fmt -- --check -v --color=always" fmtfix = "fmt -- -v" [term] -quiet = false # whether cargo output is quiet -verbose = true # whether cargo provides verbose output -color = 'always' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. -progress.when = 'auto' # whether cargo shows progress bar -progress.width = 80 # width of progress bar +quiet = false # whether cargo output is quiet +verbose = false # whether cargo provides verbose output +color = 'auto' # whether cargo colorizes output use `CARGO_TERM_COLOR="off"` to disable. +progress.when = 'never' # whether cargo shows progress bar +progress.width = 80 # width of progress bar diff --git a/utilities/dbviz/.config/nextest.toml b/utilities/dbviz/.config/nextest.toml index ebc7d7390..86c177122 100644 --- a/utilities/dbviz/.config/nextest.toml +++ b/utilities/dbviz/.config/nextest.toml @@ -35,7 +35,7 @@ path = "junit.xml" # The name of the top-level "report" element in JUnit report. If aggregating # reports across different test runs, it may be useful to provide separate names # for each report. -report-name = "cat-gateway" +report-name = "nextest" # Whether standard output and standard error for passing tests should be stored in the JUnit report. # Output is stored in the and elements of the element. diff --git a/utilities/dbviz/Earthfile b/utilities/dbviz/Earthfile index 8b0ab7c4e..0e78c1e6a 100644 --- a/utilities/dbviz/Earthfile +++ b/utilities/dbviz/Earthfile @@ -2,49 +2,19 @@ VERSION --global-cache 0.7 # cspell: words rustup rustc automake autotools xutils miri nextest kani -# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT +# cspell: words TARGETPLATFORM TARGETOS TARGETVARIANT # cspell: words USERPLATFORM USEROS USERARCH USERVARIANT -# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets readelf +# cspell: words ripgrep colordiff rustfmt stdcfgs toolset toolsets # Internal: Set up our target toolchains, and copy our files. builder: FROM ../../earthly/rust+rust-base - DO ../../earthly/rust+SETUP --toolchain=rust-toolchain.toml - - COPY --dir .cargo .config Cargo.* clippy.toml deny.toml rustfmt.toml src . - -# Internal: Run Project Checks - Use best architecture host tools. -check-hosted: - FROM +builder - - DO ../../earthly/rust+CHECK - -# check-all-hosts - A developers test which runs check with all supported host tooling. -# Needs qemu or rosetta to run. -# Only used to validate tooling is working across host toolsets by developers. -check-all-hosts: - BUILD --platform=linux/amd64 --platform=linux/arm64 +check-hosted - -build-hosted: - ARG TARGETPLATFORM - - # Build the service - FROM +builder - - RUN /scripts/std_build.sh - - DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.0.3+SMOKE_TEST --bin=dbviz - - SAVE ARTIFACT target/$TARGETARCH/doc doc - SAVE ARTIFACT target/$TARGETARCH/release/dbviz dbviz - -# build-all-hosts - A developers test which runs build with all supported host tooling. -# Needs qemu or rosetta to run. -# Only used to validate build tooling is working across host toolsets. -build-all-hosts: - BUILD --platform=linux/amd64 --platform=linux/arm64 +build-hosted + COPY --dir .cargo .config src . + COPY Cargo.lock Cargo.toml . + COPY clippy.toml deny.toml rustfmt.toml . + DO ./../../earthly/rust+SETUP ## ----------------------------------------------------------------------------- ## @@ -55,37 +25,28 @@ build-all-hosts: # check - Run check using the most efficient host tooling # CI Automated Entry point. check: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH + FROM +builder - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +check-hosted - ELSE - BUILD --platform=linux/amd64 +check-hosted - END + RUN /scripts/std_checks.py + +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-check: + BUILD --platform=linux/amd64 --platform=linux/arm64 +check # build - Run build using the most efficient host tooling # CI Automated Entry point. build: - FROM busybox - # This is necessary to pick the correct architecture build to suit the native machine. - # It primarily ensures that Darwin/Arm builds work as expected without needing x86 emulation. - # All target implementation of this should follow this pattern. - ARG USERARCH - - IF [ "$USERARCH" == "arm64" ] - BUILD --platform=linux/arm64 +build-hosted - COPY +build-hosted/doc /doc - ELSE - BUILD --platform=linux/amd64 +build-hosted - END + FROM +builder + + RUN /scripts/std_build.py --with_test \ + --bins="dbviz/dbviz" - COPY --dir +build-hosted/doc . - COPY +build-hosted/dbviz . + SAVE ARTIFACT target/doc doc + SAVE ARTIFACT target/release/dbviz dbviz - SAVE ARTIFACT doc doc - SAVE ARTIFACT dbviz dbviz +# Test which runs check with all supported host tooling. Needs qemu or rosetta to run. +# Only used to validate tooling is working across host toolsets. +all-hosts-build: + BUILD --platform=linux/amd64 --platform=linux/arm64 +build diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml index d28c1100d..0c80c3331 100644 --- a/utilities/dbviz/deny.toml +++ b/utilities/dbviz/deny.toml @@ -98,6 +98,8 @@ ignore = [ [licenses] # The lint level for crates which do not have a detectable license unlicensed = "deny" +# Don't warn if a listed license isn't found +unused-allowed-license="allow" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses # [possible values: any SPDX 3.11 short identifier (+ optional exception)]. @@ -106,7 +108,8 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", - "BlueOak-1.0.0" + "BlueOak-1.0.0", + "Apache-2.0 WITH LLVM-exception" ] # List of explicitly disallowed licenses # See https://spdx.org/licenses/ for list of possible licenses diff --git a/utilities/nushell/Earthfile b/utilities/nushell/Earthfile deleted file mode 100644 index e047c3367..000000000 --- a/utilities/nushell/Earthfile +++ /dev/null @@ -1,65 +0,0 @@ -# Common Rust UDCs and Builders. -VERSION --global-cache 0.7 - -# cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT -# cspell: words nushell dataframe nufmt rustup toolset - -# Base Rustup build container. -nushell-build: - ARG TARGETPLATFORM - ARG TARGETOS - ARG TARGETARCH - ARG TARGETVARIANT - ARG USERPLATFORM - ARG USEROS - ARG USERARCH - ARG USERVARIANT - - # This is our base Host toolset, and rustup. - # The ACTUAL version of rust that will be used, and available targets - # is controlled by a `rust-toolchain.toml` file when the `SETUP` UDC is run. - # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version. - FROM rust:1.73-alpine3.18 - - RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \ - echo "TARGETOS = $TARGETOS"; \ - echo "TARGETARCH = $TARGETARCH"; \ - echo "TARGETVARIANT = $TARGETVARIANT"; \ - echo "USERPLATFORM = $USERPLATFORM"; \ - echo "USEROS = $USEROS"; \ - echo "USERARCH = $USERARCH"; \ - echo "USERVARIANT = $USERVARIANT"; - - WORKDIR /root - - # Install necessary packages - # Expand this list as needed, rather than adding more tools in later containers. - RUN apk add --no-cache \ - musl-dev \ - mold \ - clang \ - alpine-sdk \ - openssl-dev \ - perl \ - openssl-libs-static - - RUN OPENSSL_NO_VENDOR=Y cargo install nu --version=0.87.1 --features static-link-openssl,extra,dataframe - - # Reduce the size of `nu` and make sure it can execute. - RUN strip $CARGO_HOME/bin/nu - RUN $CARGO_HOME/bin/nu --help - - # Build nufmt from latest git. - RUN git clone https://github.com/nushell/nufmt - - WORKDIR /root/nufmt - - RUN cargo build --release - RUN cargo install --path . - - # Reduce the size of `nufmt` and make sure it can execute. - RUN strip $CARGO_HOME/bin/nufmt - RUN $CARGO_HOME/bin/nufmt --help - - SAVE ARTIFACT $CARGO_HOME/bin/nu nu - SAVE ARTIFACT $CARGO_HOME/bin/nufmt nufmt \ No newline at end of file diff --git a/utilities/scripts/python/cli.py b/utilities/scripts/python/cli.py index ddfd734bd..58d16622b 100755 --- a/utilities/scripts/python/cli.py +++ b/utilities/scripts/python/cli.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python3 - # cspell: words rtype import subprocess from typing import Optional from rich import print +from rich.text import Text from rich.table import Table from dataclasses import dataclass import textwrap @@ -128,8 +127,8 @@ def print( f"[bold cyan]{self.get_name():<{name_width}}[/bold cyan] : {self.duration()} : {self.status()}" ) if verbose or (self.rc != 0 and verbose_errors): - print(f"[italic]{indent(self.get_command(), ' $ ',' . ' )}[/italic]") - print(f"{indent(self.out, ' > ', ' . ')}") + print(f"[italic bright_red]{indent(self.get_command(), ' $ ', ' . ')}[/italic bright_red]") + print(Text(indent(self.out, ' > ', ' . '))) def run( diff --git a/utilities/scripts/python/vendor_files_check.py b/utilities/scripts/python/vendor_files_check.py new file mode 100644 index 000000000..24b086210 --- /dev/null +++ b/utilities/scripts/python/vendor_files_check.py @@ -0,0 +1,11 @@ + +# cspell: words colordiff + +# Checks if two files that should exist DO, and are equal. +# used to enforce consistency between local config files and the expected config locked in CI. + +import python.cli as cli + +def colordiff_check(vendor_file_path: str, provided_file_path: str) -> cli.Result: + return cli.run(f"colordiff -Nau {provided_file_path} {vendor_file_path}", + name=f"Checking if Provided File {provided_file_path} == Vendored File {vendor_file_path}")