Build tooling updates and fixes. (#974)

* Build tooling updates and fixes.

I stared with documenting how to regenerate protos, and found a few
things that were blocking release, and also was just broken.

So this PR includes:

* Documenting how to regenerate protos with `cargo` and `make`.
* `make` targets for generating protobuf
* Fixes and updates for build tooling for release.
* Updates to rust version - which aligns with the available darwin
  builder image tags.
* Replace `live-server` which broke as unmaintained with `browser-sync`
  which is also way better.

* * Fixes for lint issues.

* Rollback to rust 1.77.1

* There's a joseluisq/rust-linux-darwin-builder:1.77.1 image, but no
  1.77.0 image 🤷🏻‍♂️ - so need to update so we can release macos
  binaries.
* Can't upgrade to 1.78.0 because of
  tikv/pprof-rs#232

build/Makefile

@@ -131,14 +131,19 @@ build: binary-archive build-image
 # Build all binaries
 build-all-binaries: ensure-build-image build-linux-binary build-macos-binary build-windows-binary
 
+# Checks for changes to protobuf definitions and generates rust code if there is any.
+gen-protobuf: ensure-build-image
+ docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) \
+ --entrypoint=cargo $(BUILD_IMAGE_TAG) run -p proto-gen -- generate
+
 # Build an archive all binaries
 binary-archive: ensure-build-image build-licence-report build-all-binaries
  docker run --rm $(common_rust_args) -w $(CARGO_TARGET_DIR) \
  --entrypoint=bash $(BUILD_IMAGE_TAG) -c 'cp ../../license.html . && zip ../../quilkin-$(package_version).zip ./*/release/quilkin ./*/release/quilkin.exe ./license.html'
 
 # Build binary for x86_64-unknown-linux-gnu.
 # Use BUILD_LOCAL=1 to build through local cargo rather than through the build container.
-build-linux-binary: ensure-build-image
+build-linux-binary: ensure-build-image gen-protobuf
 ifdef BUILD_LOCAL
  cargo $(cargo_build_x86_64_linux)
 else
@@ -148,7 +153,7 @@ endif
 
 # Build binary for x86_64-pc-windows-gnu
 # Use BUILD_LOCAL=1 to build through local cargo rather than through the build container.
-build-windows-binary: ensure-build-image
+build-windows-binary: ensure-build-image gen-protobuf
 ifdef BUILD_LOCAL
  cargo $(cargo_build_x86_64_windows)
 else
@@ -158,7 +163,7 @@ endif
 
 # Build binary for x86_64-apple-darwin and aarch64-apple-darwin
 # Use BUILD_LOCAL=1 to build through local cargo rather than through the build container.
-build-macos-binary:
+build-macos-binary: gen-protobuf
 ifdef BUILD_LOCAL
  cargo $(cargo_build_x86_64_apple)
  cargo $(cargo_build_aarch64-apple)
@@ -169,7 +174,8 @@ else
  -e "CC=o64-clang" -e "CXX=o64-clang++" \
  -e "PROTOC=/opt/protoc/bin/protoc" \
  joseluisq/rust-linux-darwin-builder:$(rust_toolchain) \
- sh -c "/workspace/build/darwin-builder/setup.sh && cargo $(cargo_build_x86_64_apple) --no-default-features && \
+ sh -c "rustup target add x86_64-apple-darwin aarch64-apple-darwin && \
+ cargo $(cargo_build_x86_64_apple) --no-default-features && \
  CC=oa64-clang CXX=oa64-clang++ LIBZ_SYS_STATIC=1 cargo $(cargo_build_aarch64-apple) --no-default-features"
 endif
 
@@ -242,24 +248,23 @@ minikube-test-agones: minikube-push
  $(MAKE) DOCKER_RUN_ARGS="$(minikube_args)" run-test-agones
 
 # Runs mdbook and cargo doc in the same directory structure as what is hosted on Github pages.
-# Open http://localhost:3000/book/index.html or http://localhost:3000/api/quilkin/index.html after running. Pages will live reload on change.
-# (the .html extension is required for hot reload, but pages will display without it)
+# Open http://localhost:3000/ after running and browse to the docs you are editing. Pages will live reload on change.
 # Use `GITHUB_REF_NAME` (default: `main`) to specify the branch or tag to point Github links to and `QUILKIN_VERSION`
 # (default `$(make version)) to specify which Quilkin version the documentation uses for examples.
 docs: ensure-build-image
 docs: GITHUB_REF_NAME ?= main
 docs: QUILKIN_VERSION ?= $(shell $(MAKE) version)
 docs:
- @echo "📭 Open browser to http://localhost:3000/book/index.html or http://localhost:3000/api/quilkin/index.html (the .html extension is required for hot reload)"
+ @echo "📭 Open browser to http://localhost:3000/ and browse as needed"
  docker run -it --rm $(common_rust_args) -p 3000:3000 \
  -e GITHUB_REF_NAME=$(GITHUB_REF_NAME) -e QUILKIN_VERSION=$(QUILKIN_VERSION) \
  --entrypoint=bash $(BUILD_IMAGE_TAG) -c \
- 'mkdir /tmp/docs && (live-server -p 3000 /tmp/docs &) && \
+ 'mkdir /tmp/docs && (browser-sync start --server "/tmp/docs" --directory --index index.html --port 3000 --no-open --files "/tmp/docs" &) && \
  mkdir -p "$(CARGO_TARGET_DIR)/doc"; ln -s "$(CARGO_TARGET_DIR)/doc" /tmp/docs/api && \
  cargo watch -s "cargo doc --workspace --no-deps && cd docs && mdbook build --dest-dir /tmp/docs/book"'
 
 # Start an interactive shell inside the build image
-# Useful for testing, or adhoc cargo, gcloud, kubectl or terraform commands
+# Useful for testing, or adhoc cargo, gcloud, kubectl or other commands
 shell: ensure-gcloud-dirs ensure-kube-dirs ensure-build-image
  # we --network=host because docker containers are not great at ipv6.
  docker run --rm -it $(DOCKER_RUN_ARGS) $(common_rust_args) \

build/README.md

@@ -37,6 +37,17 @@ To build a production release, run:
 
 `cargo build --release`
 
+##### Generating Protobuf Files
+
+If you are adding or editing the `.proto` files in this repository, they will need to be regenerated. To do this,
+you will need [protoc](https://grpc.io/docs/protoc-installation/) installed locally.
+
+Then run:
+
+`cargo run -p proto-gen -- generate`
+
+Which will identify changes in protobuf definitions and regenerate code appropriately. 
+
 #### Testing
 
 To run the unit, integration and docs tests:
@@ -53,7 +64,7 @@ To test dependency licences and security advisories:
 
 `cargo deny check`
 
-See the [agones](../agones) folder for the [Agones](https://agones.dev) integration testing tooling.
+See the [agones](../crates/agones) folder for the [Agones](https://agones.dev) integration testing tooling.
 
 ### Developing with Make + Docker 
 
@@ -70,13 +81,6 @@ To use the tooling for Make + Docker testing and development, you will need:
 * Make installed
 * [Docker installed](https://docs.docker.com/get-docker/)
 
-#### Known issues
-
-* If you are running on an arm64 machine, such as an M1 Mac, `make build-macos-binary` to build an amd64 macOS 
- binary will fail. Depending on your setup, it may be possible to use `BUILD_LOCAL=1 make build-macos-binary` to 
- attempt to build the binary with local `cargo` tooling. This is generally only a release time task, so we expect 
- it to be of minimal impact. See [#608](https://github.com/googleforgames/quilkin/issues/608) for more details.
-
 #### Run tests
 
 `make test` will run all tests for this project, except the [Agones](https:/agones.dev) integration tests.

build/build-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:bullseye
+FROM debian:bookworm
 
 ARG RUST_TOOLCHAIN
 
@@ -25,9 +25,9 @@ ENV RUSTUP_HOME=/usr/local/rustup \
 # Install packages
 RUN set -eux && \
  apt-get update && \
- apt-get install -y lsb-release jq curl wget zip git build-essential software-properties-common \
- libssl-dev pkg-config python3-pip bash-completion g++-x86-64-linux-gnu g++-mingw-w64-x86-64 && \
- pip3 install live-server && \
+ apt-get install -y lsb-release jq curl wget zip git build-essential software-properties-common protobuf-compiler \
+ libssl-dev pkg-config nodejs npm bash-completion g++-x86-64-linux-gnu g++-mingw-w64-x86-64 && \
+ npm install -g browser-sync && \
  echo "source /etc/bash_completion" >> /root/.bashrc
 
 # install gcloud
@@ -38,7 +38,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.
  echo "source <(kubectl completion bash)" >> /root/.bashrc
 
 # install terraform. There is no apt support for arm64, so direct downloading instead.
-RUN wget --quiet -O terraform.zip "https://releases.hashicorp.com/terraform/1.3.1/terraform_1.3.1_linux_$(dpkg --print-architecture).zip" && \
+RUN wget --quiet -O terraform.zip "https://releases.hashicorp.com/terraform/1.5.4/terraform_1.5.4_linux_$(dpkg --print-architecture).zip" && \
  unzip terraform.zip && rm terraform.zip && mv terraform /usr/local/bin/
 
 # install helm

rust-toolchain.toml

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 [toolchain]
-channel = "1.77.0"
+channel = "1.77.1"
 components = ["rustfmt", "clippy"]

src/filters/registry.rs

@@ -70,6 +70,7 @@ mod tests {
  use crate::filters::{Filter, FilterError, FilterRegistry, ReadContext, WriteContext};
  use crate::net::endpoint::{Endpoint, EndpointAddress};
 
+ #[allow(dead_code)]
  struct TestFilter {}
 
  #[async_trait::async_trait]

src/net/phoenix.rs

@@ -570,6 +570,7 @@ mod tests {
  use tokio::sync::Mutex;
 
  #[derive(Clone)]
+ #[allow(dead_code)]
  struct LoggingMockMeasurement {
  latencies: HashMap<SocketAddr, DistanceMeasure>,
  probed_addresses: Arc<Mutex<HashSet<SocketAddr>>>,