classify tests and extend coverage

.github/workflows/ci.yml

@@ -81,8 +81,8 @@ jobs:
         command: check
         args: --all
 
-  integration:
-    # Integration tests are linux only
+  e2e:
+    # e2e tests are docker on linux
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -113,17 +113,17 @@ jobs:
               --mount type=bind,source=$HOME/.cargo/registry,target=/root/.cargo/registry \
               --mount type=bind,source=$HOME/.cargo/git,target=/root/.cargo/git \
               clux/muslrust:stable \
-              cargo build -p integration --release -v
-          cp target/x86_64-unknown-linux-musl/release/dapp integration/
+              cargo build -p e2e --release -v
+          cp target/x86_64-unknown-linux-musl/release/dapp e2e/
 
       - name: Build image
-        run: "docker build -t clux/kube-dapp:${{ github.sha }} integration/"
+        run: "docker build -t clux/kube-dapp:${{ github.sha }} e2e/"
       - name: Import image
         run: "k3d image import clux/kube-dapp:${{ github.sha }} --cluster kube"
-      - run: sed -i 's/latest/${{ github.sha }}/g' integration/deployment.yaml
+      - run: sed -i 's/latest/${{ github.sha }}/g' e2e/deployment.yaml
 
       - name: Create resource
-        run: kubectl apply -f integration/deployment.yaml -n apps
+        run: kubectl apply -f e2e/deployment.yaml -n apps
       - run: kubectl get all -n apps
       - run: kubectl describe jobs/dapp -n apps
       - name: Wait for job to complete

CONTRIBUTING.md

@@ -32,20 +32,67 @@ Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 - **Channel**: Code is built and tested using the **stable** channel of Rust, but documented and formatted with **nightly** <sup>[*](https://github.com/kube-rs/kube-rs/issues/707)</sup>
 - **Formatting**: To format the codebase, run `make fmt`
 - **Documentation** To check documentation, run `make doc`
-- **Testing**: To run tests, run `make test`.
+- **Testing**: To run tests, run `make test` and see below.
 
 ## Testing
 
-Most tests can be run with `cargo test --all`, but because of features, some tests must be run a little more precisely.
-For the complete variations see the `make test` target in the `Makefile`.
+We have 3 classes of tests.
 
-Some tests and examples require an accessible kubernetes cluster via a `KUBECONFIG` environment variable.
+- Unit tests
+- Integration tests (requires Kubernetes)
+- End to End tests (requires Kubernetes)
 
-- unit tests marked as `#[ignore]` run via `cargo test --all --lib -- --ignored`
-- examples run with `cargo run --example=...`
-- [integration tests](https://github.com/kube-rs/kube-rs/tree/master/integration)
+The last two will try to access the Kubernetes cluster that is your `current-context`; i.e. via your local `KUBECONFIG` evar or `~/.kube/config` file.
+
+The easiest way set up a minimal Kubernetes cluster for these is with [`k3d`](https://k3d.io/).
+
+### Unit Tests
+
+**Most** unit tests are run with `cargo test --lib --doc --all`
+
+but because of feature-sets, doc tests, and examples, you will need a couple of extra invocations to replicate our CI.
+
+For the complete variations, run the `make test` target in the `Makefile`.
+
+### Integration Tests
+
+These are tests within the crates marked with an *`#[ignore]`* decoration.
+
+These are slower as they will generally :warn: try to create/modify/delete resources on your `current-context` :warn:.
+
+Most integration tests are run with `cargo test --all --lib -- --ignored`, but because of feature-sets, you will need a few invocations of these to replicate our CI. See `make test-integration`
+
+#### End to End Tests
+
+We have a small set of [e2e tests](https://github.com/kube-rs/kube-rs/tree/master/integration) that tests difference between in-cluster and local configuration.
+
+These tests are the heaviest tests we have because they require a full `docker build`, image import (or push/pull flow), yaml construction, and `kubectl` usage to verify that the outcome was sufficient.
+
+To run E2E tests, use (or follow) `make e2e` as appropriate.
+
+### Test Guidelines
+
+#### When to add a test
+
+When adding new non-trivial pieces of logic that results in a drop in coverage you should add a test.
+
+Cross-reference with the coverage build [![coverage build](https://codecov.io/gh/kube-rs/kube-rs/branch/master/graph/badge.svg?token=9FCqEcyDTZ)](https://codecov.io/gh/kube-rs/kube-rs) and go to your branch. Coverage can also be run locally with [`cargo tarpaulin`](https://github.com/xd009642/tarpaulin) at project root. This will use our [tarpaulin.toml](./tarpaulin.toml) config, and **will run both unit and integration** tests.
+
+#### What type of test
+
+- Unit tests **MUST NOT** try to contact a Kubernetes cluster
+- Integration tests **MUST NOT** be used when a unit test is sufficient
+- Integration tests **MUST NOT** assume existence of non-standard objects in the cluster
+- Integration tests **MUST NOT** cross-depend on other unit tests completing (and installing what you need)
+- E2E tests **MUST NOT** be used where an integration test is sufficient
+
+In general: **use the least powerful method** of testing available to you:
+
+- use unit tests in `kube-core`
+- use unit tests in `kube-client` (and in rare cases integration tests)
+- use unit tests in `kube-runtime` (and occassionally integration tests)
+- use e2e tests when testing differences between in-cluster and local configuration
 
-The easiest way set up a minimal kubernetes cluster for these is with [`k3d`](https://k3d.io/).
 
 ## Support
 ### Documentation

Cargo.toml

@@ -8,6 +8,6 @@ members = [
   "kube-runtime",
 
   # internal
-  "integration",
+  "e2e",
   "examples",
 ]

Makefile

@@ -23,22 +23,23 @@ test:
 	cargo test -p kube --lib --no-default-features
 	cargo test -p kube-examples --example crd_api --no-default-features --features=deprecated,kubederive,native-tls
 
-test-kubernetes:
+test-integration:
 	cargo test --lib --all -- --ignored # also run tests that fail on github actions
 	cargo test -p kube --lib --features=derive,runtime -- --ignored
+	cargo test -p kube-client --lib --features=rustls-tls,ws -- --ignored
 	cargo run -p kube-examples --example crd_derive
 	cargo run -p kube-examples --example crd_api
 
 readme:
 	rustdoc README.md --test --edition=2021
 
-integration: dapp
-	ls -lah integration/
-	docker build -t clux/kube-dapp:$(VERSION) integration/
+e2e: dapp
+	ls -lah e2e/
+	docker build -t clux/kube-dapp:$(VERSION) e2e/
 	k3d image import clux/kube-dapp:$(VERSION) --cluster main
-	sed -i 's/latest/$(VERSION)/g' integration/deployment.yaml
-	kubectl apply -f integration/deployment.yaml
-	sed -i 's/$(VERSION)/latest/g' integration/deployment.yaml
+	sed -i 's/latest/$(VERSION)/g' e2e/deployment.yaml
+	kubectl apply -f e2e/deployment.yaml
+	sed -i 's/$(VERSION)/latest/g' e2e/deployment.yaml
 	kubectl get all -n apps
 	kubectl describe jobs/dapp -n apps
 	kubectl wait --for=condition=complete job/dapp -n apps --timeout=50s || kubectl logs -f job/dapp -n apps
@@ -49,13 +50,13 @@ dapp:
 	docker run \
 		-v cargo-cache:/root/.cargo/registry \
 		-v "$$PWD:/volume" -w /volume \
-		--rm -it clux/muslrust:stable cargo build --release -p integration
-	cp target/x86_64-unknown-linux-musl/release/dapp integration/dapp
-	chmod +x integration/dapp
+		--rm -it clux/muslrust:stable cargo build --release -p e2e
+	cp target/x86_64-unknown-linux-musl/release/dapp e2e/dapp
+	chmod +x e2e/dapp
 
 k3d:
 	k3d cluster create --servers 1 --agents 1 main \
 		--k3s-agent-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%' \
 		--k3s-agent-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%'
 
-.PHONY: doc build fmt clippy test readme k3d integration
+.PHONY: doc build fmt clippy test readme k3d e2e

deny.toml

@@ -29,6 +29,7 @@ copyleft = "deny"
 default = "deny"
 unlicensed = "deny"
 
+# We are beholden to https://github.com/cncf/foundation/blob/master/allowed-third-party-license-policy.md#approved-licenses-for-allowlist
 allow = [
     "MIT",
     "Apache-2.0",

integration/Cargo.toml → e2e/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "integration"
+name = "e2e"
 version = "0.1.0"
 authors = ["clux <sszynrae@gmail.com>"]
 publish = false

integration/README.md → e2e/README.md

@@ -1,11 +1,17 @@
-# integration tests
+# E2E tests
 
-This is a working example of a kubernetes application `dapp`, that is deployed on CI during the `integration` job via [our ci workflow](https://github.com/kube-rs/kube-rs/blob/2b5e4ad788366125448ad40eadaf68cf9ceeaf31/.github/workflows/ci.yml#L58-L107). It is here to ensure in-cluster configuration is working.
+Small set of tests to verify differences between local and in-cluster development.
 
-## Behavior
+In general, [you do not want to run or develop E2E tests](../CONTRIBUTING.md#test-guidelines).
+
+## dapp
+
+A working example of a kubernetes application `dapp` deployed on CI during the `e2e` job via [our ci workflow](https://github.com/kube-rs/kube-rs/blob/2b5e4ad788366125448ad40eadaf68cf9ceeaf31/.github/workflows/ci.yml#L58-L107). It is here to ensure in-cluster configuration is working.
+
+### Behavior
 The app currently only does what the `job_api` example does, but from within the cluster, so it needs the rbac permissions to `create` a `job` in `batch`.
 
-## Github Actions
+### Github Actions
 General process, optimized for time.
 
 - compile the image with [muslrust](https://github.com/clux/muslrust)

examples/pod_api.rs

@@ -1,9 +1,9 @@
-use futures::{StreamExt, TryStreamExt};
 use k8s_openapi::api::core::v1::Pod;
 use serde_json::json;
 
 use kube::{
-    api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams, ResourceExt, WatchEvent},
+    api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams, ResourceExt},
+    runtime::wait::{await_condition, conditions::is_pod_running},
     Client,
 };
 
@@ -12,10 +12,9 @@ async fn main() -> anyhow::Result<()> {
     std::env::set_var("RUST_LOG", "info,kube=debug");
     tracing_subscriber::fmt::init();
     let client = Client::try_default().await?;
-    let namespace = std::env::var("NAMESPACE").unwrap_or_else(|_| "default".into());
 
     // Manage pods
-    let pods: Api<Pod> = Api::namespaced(client, &namespace);
+    let pods: Api<Pod> = Api::default_namespaced(client);
 
     // Create Pod blog
     tracing::info!("Creating Pod instance blog");
@@ -37,31 +36,14 @@ async fn main() -> anyhow::Result<()> {
             let name = o.name();
             assert_eq!(p.name(), name);
             tracing::info!("Created {}", name);
-            // wait for it..
-            std::thread::sleep(std::time::Duration::from_millis(5_000));
         }
         Err(kube::Error::Api(ae)) => assert_eq!(ae.code, 409), // if you skipped delete, for instance
         Err(e) => return Err(e.into()),                        // any other case is probably bad
     }
 
     // Watch it phase for a few seconds
-    let lp = ListParams::default()
-        .fields(&format!("metadata.name={}", "blog"))
-        .timeout(10);
-    let mut stream = pods.watch(&lp, "0").await?.boxed();
-    while let Some(status) = stream.try_next().await? {
-        match status {
-            WatchEvent::Added(o) => tracing::info!("Added {}", o.name()),
-            WatchEvent::Modified(o) => {
-                let s = o.status.as_ref().expect("status exists on pod");
-                let phase = s.phase.clone().unwrap_or_default();
-                tracing::info!("Modified: {} with phase: {}", o.name(), phase);
-            }
-            WatchEvent::Deleted(o) => tracing::info!("Deleted {}", o.name()),
-            WatchEvent::Error(e) => tracing::error!("Error {}", e),
-            _ => {}
-        }
-    }
+    let establish = await_condition(pods.clone(), "blog", is_pod_running());
+    let _ = tokio::time::timeout(std::time::Duration::from_secs(15), establish).await?;
 
     // Verify we can get it
     tracing::info!("Get Pod blog");

examples/pod_evict.rs

@@ -57,7 +57,7 @@ async fn main() -> anyhow::Result<()> {
         }
     }
 
-    // Clean up the old job record..
+    // Evict the pod
     let ep = EvictParams::default();
     let eres = pods.evict(pod_name, &ep).await?;
     println!("{:?}", eres);