From a767fcb60d4416aa035b529d452ec521d43697d3 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 6 Feb 2024 09:33:03 -0500 Subject: [PATCH 1/4] refactor: docker updates with info command and rpc host update Includes: - macos-14 for checks on ci - some doc changes - action-validator in flake for validation when changing github action workflows - added a couple todos --- .github/workflows/builds.yml | 11 +++++++ .github/workflows/docker.yml | 10 +++++- .github/workflows/tests_and_checks.yml | 6 ++-- DEVELOPMENT.md | 4 +-- docker/Dockerfile | 13 ++++++-- flake.lock | 42 ++++++++++++------------- flake.nix | 5 ++- homestar-runtime/build.rs | 3 +- homestar-runtime/config/settings.toml | 12 +++++++ homestar-runtime/src/event_handler.rs | 6 ++-- homestar-runtime/src/runner/response.rs | 9 +++++- 11 files changed, 87 insertions(+), 34 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 01b21473..4ecc60ab 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -328,6 +328,11 @@ jobs: id: crate-version run: echo version=$(cargo get workspace.package.version)-rc.$(date +%s) >> $GITHUB_OUTPUT + - name: Set Env + run: | + echo "GIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "GIT_TIMESTAMP=$(git log -1 --pretty=format:'%cI')" >> $GITHUB_ENV + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -373,6 +378,9 @@ jobs: file: docker/Dockerfile context: . platforms: linux/amd64,linux/arm64 + build-args: | + git_sha=${{ env.GIT_SHA }} + git_timestamp=${{ env.GIT_TIMESTAMP }} push: true tags: ${{ steps.meta-release.outputs.tags }} labels: ${{ steps.meta-release.outputs.labels }} @@ -386,6 +394,9 @@ jobs: file: docker/Dockerfile context: . platforms: linux/amd64,linux/arm64 + build-args: | + git_sha=${{ env.GIT_SHA }} + git_timestamp=${{ env.GIT_TIMESTAMP }} push: true tags: ${{ steps.meta-dispatch.outputs.tags }} labels: ${{ steps.meta-dispatch.outputs.labels }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b6a3175b..ad4cc8cf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,7 @@ name: 🐳 Docker on: schedule: - - cron: '0 */4 * * *' + - cron: "0 */4 * * *" jobs: build-docker: @@ -15,6 +15,11 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 + - name: Set Env + run: | + echo "GIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "GIT_TIMESTAMP=$(git log -1 --pretty=format:'%cI')" >> $GITHUB_ENV + # https://github.com/docker/setup-qemu-action - name: Setup QEMU uses: docker/setup-qemu-action@v3 @@ -33,6 +38,9 @@ jobs: file: docker/Dockerfile context: . platforms: linux/amd64 + build-args: | + git_sha=${{ env.GIT_SHA }} + git_timestamp=${{ env.GIT_TIMESTAMP }} push: false tags: | ${{ github.repository_owner }}/homestar:latest diff --git a/.github/workflows/tests_and_checks.yml b/.github/workflows/tests_and_checks.yml index e0d78fcb..5cf11869 100644 --- a/.github/workflows/tests_and_checks.yml +++ b/.github/workflows/tests_and_checks.yml @@ -53,7 +53,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-14, windows-latest] rust-toolchain: [stable, nightly, 1.73.0] runs-on: ${{ matrix.os }} steps: @@ -89,7 +89,7 @@ jobs: - name: Run Linter run: cargo clippy --all -- -D warnings - continue-on-error: ${{ matrix.rust-toolchain == 'nightly' && matrix.os == 'macos-latest' }} + continue-on-error: ${{ matrix.rust-toolchain == 'nightly' && matrix.os == 'macos-14' }} - name: Install cargo-hakari if: ${{ matrix.rust-toolchain == 'stable' }} @@ -205,6 +205,7 @@ jobs: - name: Run Doc Tests if: ${{ matrix.default-features == 'all' }} run: cargo test --doc --workspace + continue-on-error: ${{ matrix.rust-toolchain == 'nightly' }} run-tests-windows: needs: changes @@ -258,6 +259,7 @@ jobs: - name: Run Doc Tests if: ${{ matrix.default-features == 'all' }} run: cargo test --doc --workspace + continue-on-error: ${{ matrix.rust-toolchain == 'nightly' }} run-cargo-tests: needs: changes diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3dc02788..650f0d50 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -85,13 +85,13 @@ with `experimental` and `buildkit` set to `true`, for example: - Build a multi-plaform Docker image via [buildx][buildx]: ```console - docker buildx build --file docker/Dockerfile --platform=linux/amd64,linux/arm64 -t homestar --progress=plain . + docker buildx build --build-arg git_sha=$(git rev-parse HEAD) --file docker/Dockerfile --platform=linux/amd64,linux/arm64 -t homestar --progress=plain . ``` - Run a Docker image (depending on your platform): ```console - docker run --platform=linux/arm64 -t homestar + docker run -it --rm --platform=linux/arm64 -p 3030:3030 -p 1337:1337 -t homestar ``` ## Nix diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c0bd64a..f7c307d8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,10 +10,17 @@ ARG TARGETARCH FROM builder-$TARGETARCH as builder ARG database_url="homestar.db" +ARG git_sha +ARG git_timestamp + ENV DATABASE_URL=${database_url} +ENV VERGEN_GIT_SHA=${git_sha} +ENV VERGEN_GIT_COMMIT_TIMESTAMP=${git_timestamp} RUN adduser --disabled-password --disabled-login --gecos "" --no-create-home homestar +RUN apt install -y git + RUN cargo init RUN rm -Rf src @@ -27,7 +34,7 @@ RUN echo "fn main() {}" > ./homestar-runtime/src/main.rs RUN mkdir -p ./homestar-runtime/src/test_utils/proc_macro ./homestar-runtime/migrations ./examples ./homestar-functions RUN bash -c 'pushd ./examples && cargo init dummy-app-examples && popd' RUN bash -c 'pushd ./homestar-functions && cargo init dummy-app-fns && popd' -RUN cargo init homestar-workspace-hack +RUN cargo init --lib homestar-workspace-hack # copy cargo.* COPY Cargo.lock Cargo.toml diesel.toml ./ @@ -90,7 +97,7 @@ COPY --chown=homestar:homestar --from=builder /etc/*.db ./ COPY --chown=homestar:homestar --from=builder /etc/config ./config ARG database_url="homestar.db" -ARG rpc_host="127.0.0.1" +ARG rpc_host="0.0.0.0" ARG rpc_port=3030 ARG ws_port=1337 @@ -101,4 +108,4 @@ ENV DATABASE_URL=${database_url} \ EXPOSE ${rpc_port} ${ws_port} -ENTRYPOINT ["./homestar", "start", "--db", "homestar.db"] +ENTRYPOINT ["./homestar", "start", "--db", "homestar.db", "-c", "./config/settings.toml"] diff --git a/flake.lock b/flake.lock index 3103b057..769f8d26 100644 --- a/flake.lock +++ b/flake.lock @@ -79,11 +79,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1706509323, - "narHash": "sha256-oGp1mhDfVtsacopNkv46lmMyI2a4klHBJbtDsj4FG3g=", + "lastModified": 1706941198, + "narHash": "sha256-t6/qloMYdknVJ9a3QzjylQIZnQfgefJ5kMim50B7dwA=", "owner": "nix-community", "repo": "fenix", - "rev": "2fccc3f4ab4f1853e56f858a969b85f48005a2a8", + "rev": "28dbd8b43ea328ee708f7da538c63e03d5ed93c8", "type": "github" }, "original": { @@ -226,11 +226,11 @@ "systems": "systems_3" }, "locked": { - "lastModified": 1681202837, - "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -445,11 +445,11 @@ }, "nixlib_3": { "locked": { - "lastModified": 1706402708, - "narHash": "sha256-v6z1V+BwolqR9w0sbRkZ9DnnviMcZdZzPJe+4K4h+d4=", + "lastModified": 1707007541, + "narHash": "sha256-fuFppCuZO4wJAfodUkiWhtSxTb+pkBW+lJP2S51jRNU=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "4833b4eb30dfe3abad5a21775bc5460322c8d337", + "rev": "948ff77600f9fff8c904d1e1ffb87a60773991af", "type": "github" }, "original": { @@ -460,11 +460,11 @@ }, "nixos-unstable": { "locked": { - "lastModified": 1706547092, - "narHash": "sha256-Gs3RWi83YRmQFnEKDI2vK5XWTOpUK21uDHH/GEMWIio=", + "lastModified": 1707211987, + "narHash": "sha256-pDXvs0ltoqfnVbNjreyD5qQd2PK7NCnkmucB2CN9vSw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8131c739ff0b40820a7d7888666dbf6e43809233", + "rev": "ff912d3d37789a57062b4e510e921076aeaecb7c", "type": "github" }, "original": { @@ -555,11 +555,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1706373441, - "narHash": "sha256-S1hbgNbVYhuY2L05OANWqmRzj4cElcbLuIkXTb69xkk=", + "lastModified": 1707091808, + "narHash": "sha256-LahKBAfGbY836gtpVNnWwBTIzN7yf/uYM/S0g393r0Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "56911ef3403a9318b7621ce745f5452fb9ef6867", + "rev": "9f2ee8c91ac42da3ae6c6a1d21555f283458247e", "type": "github" }, "original": { @@ -584,11 +584,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1706352756, - "narHash": "sha256-6K5rK1b2APQfXOrC+Hm+0QcyfPVt+TV81Q6Fd/QjMlQ=", + "lastModified": 1706875368, + "narHash": "sha256-KOBXxNurIU2lEmO6lR2A5El32X9x8ITt25McxKZ/Ew0=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "7219414e81810fd4d967136c4a0650523892c157", + "rev": "8f6a72871ec87ed53cfe43a09fb284168a284e7e", "type": "github" }, "original": { @@ -694,11 +694,11 @@ ] }, "locked": { - "lastModified": 1706580650, - "narHash": "sha256-e6q4Pn1dp3NoQJdMYdyNdDHU5IRBW9i3bHSJ3jThEL0=", + "lastModified": 1707185459, + "narHash": "sha256-+GdqwnFZQ9G6pOrTGA+XzD4hg+DZZwkx2/x46fuNkL8=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "39e20b3c02caa91c9970beef325a04975d83d77f", + "rev": "be473291b92e4e84e84a1e04e57481c848060c6c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 8220d4f1..ec429747 100644 --- a/flake.nix +++ b/flake.nix @@ -144,7 +144,9 @@ dockerBuild = arch: pkgs.writeScriptBin "docker-${arch}" '' #!${pkgs.stdenv.shell} - docker buildx build --file docker/Dockerfile --platform=linux/${arch} -t homestar-runtime --progress=plain . + docker buildx build --build-arg git_sha=$(git rev-parse HEAD) \ + --build-arg git_timestamp=$(git log -1 --pretty=format:'%cI') \ + --file docker/Dockerfile --platform=linux/${arch} -t homestar --progress=plain . ''; xFunc = cmd: @@ -289,6 +291,7 @@ direnv unstable.nodejs_20 unstable.nodePackages.pnpm + action-validator kubo self.packages.${system}.irust ] diff --git a/homestar-runtime/build.rs b/homestar-runtime/build.rs index 5b672ed4..2ed68771 100644 --- a/homestar-runtime/build.rs +++ b/homestar-runtime/build.rs @@ -3,8 +3,9 @@ fn main() -> Result<(), Box> { vergen::EmitBuilder::builder() .fail_on_error() - .use_local_build() .git_sha(true) + // TODO: Look into why this returns old date/times under nix. + //.git_commit_timestamp() .cargo_features() .emit()?; diff --git a/homestar-runtime/config/settings.toml b/homestar-runtime/config/settings.toml index 59a93a6a..50673b42 100644 --- a/homestar-runtime/config/settings.toml +++ b/homestar-runtime/config/settings.toml @@ -1,4 +1,16 @@ [node] +[node.network.libp2p] +listen_address = "/ip4/0.0.0.0/tcp/7001" + +[node.network.libp2p.rendezvous] +enable_server = true + [node.network.metrics] port = 4000 + +[node.network.rpc] +port = 3030 + +[node.network.webserver] +port = 1337 diff --git a/homestar-runtime/src/event_handler.rs b/homestar-runtime/src/event_handler.rs index 997538e6..0e18d786 100644 --- a/homestar-runtime/src/event_handler.rs +++ b/homestar-runtime/src/event_handler.rs @@ -34,10 +34,12 @@ pub(crate) use event::Event; type P2PSender = channel::AsyncChannelSender; +/// Quorum configuration specifies the minimum number of distinct nodes that +/// must be successfully contacted in order for a query to succeed. struct Quorum { - /// Minimum number of peers required to receive a receipt. + /// Minimum number of peers required to achieve quorum for a receipt. receipt: usize, - /// Minimum number of peers required to receive workflow information. + /// Minimum number of peers required to achieve quorum for workflow information. workflow: usize, } diff --git a/homestar-runtime/src/runner/response.rs b/homestar-runtime/src/runner/response.rs index d00acb85..d2aecc79 100644 --- a/homestar-runtime/src/runner/response.rs +++ b/homestar-runtime/src/runner/response.rs @@ -222,6 +222,8 @@ impl show::ConsoleTable for AckNodeInfo { pub struct Info { version: String, git_sha: String, + // TODO: Look into why this returns old date/times under nix. + //timestamp: String, features: String, } @@ -236,7 +238,12 @@ impl Info { pub(crate) fn new() -> Self { Self { version: env!("CARGO_PKG_VERSION").to_string(), - git_sha: env!("VERGEN_GIT_SHA").to_string(), + git_sha: option_env!("VERGEN_GIT_SHA") + .unwrap_or("unknown") + .to_string(), + // timestamp: option_env!("VERGEN_GIT_COMMIT_TIMESTAMP") + // .unwrap_or("unknown") + // .to_string(), features: env!("VERGEN_CARGO_FEATURES").to_string(), } } From 564ec8c8b0fcdf4ce9877db0ae05b7f15bebfd14 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 6 Feb 2024 15:02:37 -0500 Subject: [PATCH 2/4] chore: fix defaults --- flake.nix | 2 +- homestar-runtime/config/settings.toml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index ec429747..23999c34 100644 --- a/flake.nix +++ b/flake.nix @@ -242,7 +242,7 @@ testCleanup = pkgs.writeScriptBin "test-clean" '' #!${pkgs.stdenv.shell} - rm -rf homestar-runtime/tests/fixtures/*.db + rm -rf homestar-runtime/tests/fixtures/*.db* rm -rf homestar-runtime/tests/fixtures/*.toml ''; diff --git a/homestar-runtime/config/settings.toml b/homestar-runtime/config/settings.toml index 50673b42..4ac4d219 100644 --- a/homestar-runtime/config/settings.toml +++ b/homestar-runtime/config/settings.toml @@ -1,10 +1,7 @@ [node] [node.network.libp2p] -listen_address = "/ip4/0.0.0.0/tcp/7001" - -[node.network.libp2p.rendezvous] -enable_server = true +listen_address = "/ip4/0.0.0.0/tcp/0" [node.network.metrics] port = 4000 From d7ede6211a0679b5e3655f05bc0b008131c1dd6e Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 6 Feb 2024 15:32:34 -0500 Subject: [PATCH 3/4] chore: dev readme --- DEVELOPMENT.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 650f0d50..db66030a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -85,7 +85,9 @@ with `experimental` and `buildkit` set to `true`, for example: - Build a multi-plaform Docker image via [buildx][buildx]: ```console - docker buildx build --build-arg git_sha=$(git rev-parse HEAD) --file docker/Dockerfile --platform=linux/amd64,linux/arm64 -t homestar --progress=plain . + docker buildx build --build-arg git_sha=$(git rev-parse HEAD) \ + --build-arg git_timestamp=$(git log -1 --pretty=format:'%cI') \ + --file docker/Dockerfile --platform=linux/amd64,linux/arm64 -t homestar --progress=plain . ``` - Run a Docker image (depending on your platform): From 5b783a6ad9d233884571d0c1884a29aae2b91ba0 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 6 Feb 2024 19:37:48 -0500 Subject: [PATCH 4/4] chore: fix nix things --- flake.nix | 2 ++ homestar-runtime/build.rs | 4 +--- homestar-runtime/src/runner/response.rs | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 23999c34..b99ee5e9 100644 --- a/flake.nix +++ b/flake.nix @@ -315,6 +315,8 @@ ipfs --repo-dir ./.ipfs --offline init fi + unset SOURCE_DATE_EPOCH + # Run Kubo / IPFS echo -e "To run Kubo as a local IPFS node, use the following command:" echo -e "ipfs --repo-dir ./.ipfs --offline daemon" diff --git a/homestar-runtime/build.rs b/homestar-runtime/build.rs index 2ed68771..5d2e32be 100644 --- a/homestar-runtime/build.rs +++ b/homestar-runtime/build.rs @@ -2,10 +2,8 @@ fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=migrations"); vergen::EmitBuilder::builder() - .fail_on_error() .git_sha(true) - // TODO: Look into why this returns old date/times under nix. - //.git_commit_timestamp() + .git_commit_timestamp() .cargo_features() .emit()?; diff --git a/homestar-runtime/src/runner/response.rs b/homestar-runtime/src/runner/response.rs index d2aecc79..833867f5 100644 --- a/homestar-runtime/src/runner/response.rs +++ b/homestar-runtime/src/runner/response.rs @@ -222,8 +222,7 @@ impl show::ConsoleTable for AckNodeInfo { pub struct Info { version: String, git_sha: String, - // TODO: Look into why this returns old date/times under nix. - //timestamp: String, + timestamp: String, features: String, } @@ -241,9 +240,9 @@ impl Info { git_sha: option_env!("VERGEN_GIT_SHA") .unwrap_or("unknown") .to_string(), - // timestamp: option_env!("VERGEN_GIT_COMMIT_TIMESTAMP") - // .unwrap_or("unknown") - // .to_string(), + timestamp: option_env!("VERGEN_GIT_COMMIT_TIMESTAMP") + .unwrap_or("unknown") + .to_string(), features: env!("VERGEN_CARGO_FEATURES").to_string(), } }