Port k8s tests to run real pods (#140)

* port k8s tests to run real pods
* check dups in events log (ensure there are none)
* fixup tests for newer kube-rs and add pod log printer
* propagate error rather than unwrapping

Dockerfile

@@ -35,6 +35,8 @@ COPY . .
 RUN env
 # Rebuild the agent
 RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
+    --mount=type=cache,target=/opt/rust/cargo/registry  \
+    --mount=type=cache,target=/opt/logdna-agent-v2/target \
     if [ -z "$SCCACHE_BUCKET" ]; then unset RUSTC_WRAPPER; fi; \
     if [ -n "${TARGET}" ]; then export TARGET_ARG="--target ${TARGET}"; fi; \
     export ${BUILD_ENVS?}; cargo build --manifest-path bin/Cargo.toml --no-default-features ${FEATURES} --release $TARGET_ARG && \

Dockerfile.debian

@@ -0,0 +1,73 @@
+# syntax = docker/dockerfile:1.0-experimental
+ARG BUILD_IMAGE
+
+FROM ${BUILD_IMAGE} as build
+
+ENV _RJEM_MALLOC_CONF="narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0"
+ENV JEMALLOC_SYS_WITH_MALLOC_CONF="narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0"
+
+ARG FEATURES
+
+ARG SCCACHE_BUCKET
+ENV SCCACHE_BUCKET=${SCCACHE_BUCKET}
+
+ARG SCCACHE_REGION
+ENV SCCACHE_REGION=${SCCACHE_REGION}
+
+# Create the directory for agent repo
+WORKDIR /opt/logdna-agent-v2
+
+# Add the actual agent source files
+COPY . .
+
+# Rebuild the agent
+RUN --mount=type=cache,target=/opt/rust/cargo/registry  \
+    --mount=type=cache,target=/opt/logdna-agent-v2/target \
+    if [ -z "$SCCACHE_BUCKET" ]; then unset RUSTC_WRAPPER; fi; \
+    cargo build --manifest-path bin/Cargo.toml ${FEATURES} --release && \
+    strip ./target/release/logdna-agent && \
+    cp ./target/release/logdna-agent logdna-agent && \
+    sccache --show-stats
+
+# Use Debian as agent base image
+FROM debian:buster
+
+ARG REPO
+ARG BUILD_TIMESTAMP
+ARG VCS_REF
+ARG VCS_URL
+ARG BUILD_VERSION
+
+LABEL org.opencontainers.image.created="${BUILD_TIMESTAMP}"
+LABEL org.opencontainers.image.authors="LogDNA <support@logdna.com>"
+LABEL org.opencontainers.image.url="https://logdna.com"
+LABEL org.opencontainers.image.documentation=""
+LABEL org.opencontainers.image.source="${VCS_URL}"
+LABEL org.opencontainers.image.version="${BUILD_VERSION}"
+LABEL org.opencontainers.image.revision="${VCS_REF}"
+LABEL org.opencontainers.image.vendor="LogDNA Inc."
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.ref.name=""
+LABEL org.opencontainers.image.title="LogDNA Agent"
+LABEL org.opencontainers.image.description="The blazingly fast, resource efficient log collection client"
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV _RJEM_MALLOC_CONF="narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0"
+ENV JEMALLOC_SYS_WITH_MALLOC_CONF="narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0"
+
+# Copy the agent binary from the build stage
+COPY --from=build /opt/logdna-agent-v2/logdna-agent /work/
+WORKDIR /work/
+
+RUN apt update -y \
+    && apt upgrade -y \
+    && apt auto-remove -y \
+    && apt install ca-certificates libcap2 netcat-openbsd nmap \
+                   dnsutils vim curl -y \
+    && rm -rf /var/cache/apt \
+    && chmod -R 777 . \
+    && setcap "cap_dac_read_search+eip" /work/logdna-agent \
+    && groupadd -g 5000 logdna \
+    && useradd -u 5000 -g logdna logdna
+
+CMD ["./logdna-agent"]

Makefile

@@ -21,7 +21,7 @@ WORKDIR :=/build
 DOCKER := DOCKER_BUILDKIT=1 docker
 DOCKER_DISPATCH := ./docker/dispatch.sh "$(WORKDIR)" "$(shell pwd):/build:Z"
 DOCKER_JOURNALD_DISPATCH := ./docker/journald_dispatch.sh "$(WORKDIR)" "$(shell pwd):/build:Z"
-DOCKER_KIND_DISPATCH := ./docker/kind_dispatch.sh "$(WORKDIR)"
+DOCKER_KIND_DISPATCH := ./docker/kind_dispatch.sh "$(WORKDIR)" "$(shell pwd):/build:Z"
 DOCKER_PRIVATE_IMAGE := us.gcr.io/logdna-k8s/logdna-agent-v2
 DOCKER_PUBLIC_IMAGE := docker.io/logdna/logdna-agent
 DOCKER_IBM_IMAGE := icr.io/ext/logdna-agent
@@ -146,7 +146,7 @@ integration-test: ## Run integration tests using image with additional tools
 
 .PHONY:k8s-test
 k8s-test: ## Run integration tests using k8s kind
-	$(DOCKER_KIND_DISPATCH) $(K8S_TEST_CREATE_CLUSTER)
+	$(DOCKER_KIND_DISPATCH) $(K8S_TEST_CREATE_CLUSTER) $(RUST_IMAGE) "--env RUST_LOG=$(RUST_LOG)" "cargo test --manifest-path bin/Cargo.toml --features k8s_tests -- --nocapture"
 
 .PHONY:test-journald
 test-journald: ## Run journald unit tests

bin/Cargo.toml

@@ -57,6 +57,11 @@ journald_tests = ["journald/journald_tests"]
 [dev-dependencies]
 lazy_static = "*"
 assert_cmd = "1"
+escargot = "0.5"
+kube = { version = "0.59", default-features = false, features = ["rustls-tls"] }
+kube-runtime = "0.59"
+k8s-openapi = { version = "0.13", default_features = false, features = ["v1_16"] }
+pnet = "0.28"
 itertools = "0.10"
 predicates = "2"
 tempfile = "3"
@@ -66,11 +71,18 @@ logdna_mock_ingester = { package = "logdna-mock-ingester", path = "../common/tes
 test_types = { package = "types", path = "../common/test/types" }
 proptest = "1"
 tokio-test = "0.4"
+futures = "0.3"
 rand = "0.8"
 systemd = "0.7"
 nix = "0.20"
 wait-timeout = "0.2"
 hyper = { version = "0.14", features = ["http1"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
 
 [build-dependencies]
 auditable-build = "0.1"
+
+[profile.release]
+opt-level = 2
+debug = true

bin/tests/common.rs

@@ -1,4 +1,3 @@
-use assert_cmd::cargo::CommandCargoExt;
 use core::time;
 
 use rand::seq::IteratorRandom;
@@ -153,7 +152,18 @@ impl<'a> AgentSettings<'a> {
 }
 
 pub fn spawn_agent(settings: AgentSettings) -> Child {
-    let mut cmd = Command::cargo_bin("logdna-agent").unwrap();
+    let mut manifest_path = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
+    manifest_path.pop();
+    manifest_path.push("bin/Cargo.toml");
+    let cargo_build = escargot::CargoBuild::new()
+        .manifest_path(manifest_path)
+        .bin("logdna-agent")
+        .release()
+        .current_target()
+        .run()
+        .unwrap();
+
+    let mut cmd = cargo_build.command();
 
     let ingestion_key = if let Some(key) = settings.ingester_key {
         key.to_string()

bin/tests/http.rs

@@ -28,7 +28,8 @@ async fn test_http_buffer_size() {
             {
                 let mut counter = counter.lock().unwrap();
                 *counter += 1;
-            }
+            };
+            None
         }));
 
     let mut settings = AgentSettings::with_mock_ingester(dir.to_str().unwrap(), &address);