From 4f5980f36e014afaaee514e641b9e4b57e621d46 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Sun, 9 Jun 2024 17:18:27 +0300 Subject: [PATCH] delete from master --- .../workflows/logzio-logs-collector-test.yaml | 74 -------- .github/workflows/logzio-telemetry-test.yaml | 86 --------- go.mod | 8 - tests/common.go | 51 ----- tests/logs_e2e_test.go | 94 ---------- tests/metrics_e2e_test.go | 177 ------------------ tests/resources/logsgen.yaml | 22 --- tests/resources/tracegen.yaml | 24 --- tests/traces_e2e_test.go | 124 ------------ 9 files changed, 660 deletions(-) delete mode 100644 .github/workflows/logzio-logs-collector-test.yaml delete mode 100644 .github/workflows/logzio-telemetry-test.yaml delete mode 100644 go.mod delete mode 100644 tests/common.go delete mode 100644 tests/logs_e2e_test.go delete mode 100644 tests/metrics_e2e_test.go delete mode 100644 tests/resources/logsgen.yaml delete mode 100644 tests/resources/tracegen.yaml delete mode 100644 tests/traces_e2e_test.go diff --git a/.github/workflows/logzio-logs-collector-test.yaml b/.github/workflows/logzio-logs-collector-test.yaml deleted file mode 100644 index bfab472e..00000000 --- a/.github/workflows/logzio-logs-collector-test.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: Test Logzio Logs Collector Helm Chart on Kind Kubernetes Environment - -on: - pull_request: - branches: - - main - paths: - - 'charts/logzio-logs-collector/**' -jobs: - test-helm-chart: - name: Test Helm Chart on Kind - runs-on: ubuntu-latest - steps: - - name: Generate random id - id: random_id - run: echo "::set-output name=rand::$(echo $RANDOM)" - - - name: Set ENV_ID - run: echo "ENV_ID=logs-test-run-${{ steps.random_id.outputs.rand }}" >> $GITHUB_ENV - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.20' - - - name: Set up Helm - uses: azure/setup-helm@v4.2.0 - - - name: Set up kubectl - uses: azure/setup-kubectl@v3 - - - name: Install Kind - run: | - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-Linux-amd64" - chmod +x ./kind - mv ./kind /usr/local/bin/kind - - - name: Create Kind cluster - run: | - kind create cluster --name kind-${{ github.run_id }} - kubectl cluster-info - - - name: Deploy Helm Chart - run: | - cd charts/logzio-logs-collector - helm upgrade --install \ - --set secrets.env_id=${{ env.ENV_ID }} \ - --set secrets.logzioLogsToken=${{ secrets.LOGZIO_LOGS_TOKEN }} \ - --set secrets.logzioRegion=us \ - logzio-logs-collector . - - - name: run log generator - run: | - kubectl apply -f tests/resources/logsgen.yaml - kubectl rollout status deployment/log-generator --timeout=300s - - - name: sleep - run: sleep 120 - - - name: Run Go Tests - run: | - go get go.uber.org/zap - go test -v ./tests/logs_e2e_test.go ./tests/common.go - - - name: Cleanup Environment - run: | - helm uninstall logzio-logs-collector - - - name: Delete Kind cluster - if: always() - run: kind delete cluster --name kind-${{ github.run_id }} \ No newline at end of file diff --git a/.github/workflows/logzio-telemetry-test.yaml b/.github/workflows/logzio-telemetry-test.yaml deleted file mode 100644 index 8a04ad5c..00000000 --- a/.github/workflows/logzio-telemetry-test.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: Test Logzio Telemetry Helm Chart on Kind Kubernetes Environment - -on: - pull_request: - branches: - - main - paths: - - 'charts/logzio-k8s-telemetry/**' -jobs: - test-helm-chart: - name: Test Helm Chart on Kind - runs-on: ubuntu-latest - strategy: - matrix: - mode: ['daemonset', 'standalone'] - steps: - - name: Generate random id - id: random_id - run: echo "::set-output name=rand::$(echo $RANDOM)" - - - name: Set ENV_ID - run: echo "ENV_ID=telemetry-test-run-${{ steps.random_id.outputs.rand }}-${{ matrix.mode }}" >> $GITHUB_ENV - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.20' - - - name: Set up Helm - uses: azure/setup-helm@v4.2.0 - - - name: Set up kubectl - uses: azure/setup-kubectl@v3 - - - name: Install Kind - run: | - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.11.1/kind-Linux-amd64" - chmod +x ./kind - mv ./kind /usr/local/bin/kind - - - name: Create Kind cluster - run: | - kind create cluster --name kind-${{ github.run_id }}-${{ matrix.mode }} - kubectl cluster-info - - - name: Deploy Helm Chart - run: | - cd charts/logzio-telemetry - helm dependency build - helm upgrade --install \ - --set traces.enabled=true \ - --set spm.enabled=true \ - --set metrics.enabled=true \ - --set secrets.TracesToken=${{ secrets.LOGZIO_TRACES_TOKEN }} \ - --set secrets.SpmToken=${{ secrets.LOGZIO_METRICS_TOKEN }} \ - --set secrets.MetricsToken=${{ secrets.LOGZIO_METRICS_TOKEN }} \ - --set secrets.ListenerHost=https://listener.logz.io:8053 \ - --set secrets.p8s_logzio_name=${{ env.ENV_ID }} \ - --set secrets.env_id=${{ env.ENV_ID }} \ - --set mode=${{ matrix.mode }} \ - logzio-k8s-telemetry . - kubectl rollout status deployment/logzio-k8s-telemetry-otel-collector-standalone --timeout=300s - kubectl rollout status deployment/logzio-k8s-telemetry-otel-collector-spm --timeout=300s - - - name: run trace generator - run: | - kubectl apply -f tests/resources/tracegen.yaml - kubectl rollout status deployment/trace-gen --timeout=300s - - - name: sleep for 2 minutes - run: sleep 120 - - - name: Run Go Tests - run: | - go get go.uber.org/zap - go test -v ./tests/traces_e2e_test.go ./tests/common.go - go test -v ./tests/metrics_e2e_test.go ./tests/common.go - - name: Cleanup Environment - run: | - helm uninstall logzio-k8s-telemetry - - - name: Delete Kind cluster - if: always() - run: kind delete cluster --name kind-${{ github.run_id }}-${{ matrix.mode }} \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 85975d64..00000000 --- a/go.mod +++ /dev/null @@ -1,8 +0,0 @@ -module github.com/logzio/logzio-helm - -go 1.19 - -require ( - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect -) diff --git a/tests/common.go b/tests/common.go deleted file mode 100644 index 5e3ad75e..00000000 --- a/tests/common.go +++ /dev/null @@ -1,51 +0,0 @@ -package tests - -import ( - "go.uber.org/zap" - "strings" -) - -const ( - BaseLogzioApiUrl = "https://api.logz.io/v1" - QueryTemplate = `{ - "query": { - "query_string": { - "query": "{{QUERY}}" - } - }, - "from": 0, - "size": 100, - "sort": [ - { - "@timestamp": { - "order": "desc" - } - } - ], - "_source": true, - "docvalue_fields": [ - "@timestamp" - ], - "version": true, - "stored_fields": [ - "*" - ], - "highlight": {}, - "aggregations": { - "byType": { - "terms": { - "field": "type", - "size": 5 - } - } - } -}` -) - -func formatQuery(query string) string { - return strings.Replace(QueryTemplate, "{{QUERY}}", query, 1) -} - -var ( - logger, _ = zap.NewProduction() -) diff --git a/tests/logs_e2e_test.go b/tests/logs_e2e_test.go deleted file mode 100644 index b8e0636f..00000000 --- a/tests/logs_e2e_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package tests - -import ( - "bytes" - "encoding/json" - "fmt" - "go.uber.org/zap" - "io" - "net/http" - "os" - "testing" -) - -type LogResponse struct { - Hits struct { - Total int `json:"total"` - Hits []struct { - Source struct { - ContainerImageTag string `json:"container_image_tag"` - ContainerImageName string `json:"container_image_name"` - ContainerName string `json:"k8s_container_name"` - NamespaceName string `json:"k8s_namespace_name"` - PodName string `json:"k8s_pod_name"` - PodUID string `json:"k8s_pod_uid"` - NodeName string `json:"k8s_node_name"` - LogLevel string `json:"log_level"` - } `json:"_source"` - } `json:"hits"` - } `json:"hits"` -} - -func TestLogzioMonitoringLogs(t *testing.T) { - logsApiKey := os.Getenv("LOGZIO_LOGS_API_KEY") - if logsApiKey == "" { - t.Fatalf("LOGZIO_LOGS_API_KEY environment variable not set") - } - - logResponse, err := fetchLogs(logsApiKey) - if err != nil { - t.Fatalf("Failed to fetch logs: %v", err) - } - - if logResponse.Hits.Total == 0 { - t.Errorf("No logs found") - } - - for _, hit := range logResponse.Hits.Hits { - kubernetes := hit.Source - if kubernetes.ContainerImageTag == "" || kubernetes.ContainerName == "" || kubernetes.NamespaceName == "" || kubernetes.PodName == "" || kubernetes.PodUID == "" || kubernetes.NodeName == "" || kubernetes.ContainerImageName == "" || kubernetes.LogLevel == "" { - logger.Error("Missing log fields", zap.Any("log", hit)) - t.Errorf("Missing log fields") - break - } - } -} - -func fetchLogs(logsApiKey string) (*LogResponse, error) { - url := fmt.Sprintf("%s/search", BaseLogzioApiUrl) - client := &http.Client{} - envID := os.Getenv("ENV_ID") - query := fmt.Sprintf("env_id:%s AND type:agent-k8s AND k8s_deployment_name:log-generator", envID) - formattedQuery := formatQuery(query) - logger.Info("sending api request", zap.String("url", url), zap.String("query", query)) - req, err := http.NewRequest("POST", url, bytes.NewBufferString(formattedQuery)) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("X-API-TOKEN", logsApiKey) - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var logResponse LogResponse - err = json.Unmarshal(body, &logResponse) - if err != nil { - return nil, err - } - - return &logResponse, nil -} diff --git a/tests/metrics_e2e_test.go b/tests/metrics_e2e_test.go deleted file mode 100644 index e036fd7f..00000000 --- a/tests/metrics_e2e_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package tests - -import ( - "encoding/json" - "fmt" - "go.uber.org/zap" - "io" - "net/http" - "os" - "strings" - "testing" -) - -// MetricResponse represents the structure of the API response -type MetricResponse struct { - Status string `json:"status"` - Data struct { - ResultType string `json:"resultType"` - Result []struct { - Metric map[string]string `json:"metric"` - Value []interface{} `json:"value"` - } `json:"result"` - } `json:"data"` -} - -func TestContainerMetrics(t *testing.T) { - requiredMetrics := map[string][]string{ - "container_cpu_usage_seconds_total": {"p8s_logzio_name", "namespace", "pod", "container"}, - "container_memory_working_set_bytes": {"p8s_logzio_name", "namespace", "pod", "container"}, - "container_network_transmit_bytes_total": {"p8s_logzio_name", "namespace", "pod"}, - "container_network_receive_bytes_total": {"p8s_logzio_name", "namespace", "pod"}, - } - envId := os.Getenv("ENV_ID") - query := fmt.Sprintf(`{env_id='%s',pod=~'.+'}`, envId) - testMetrics(t, requiredMetrics, query) -} - -func TestInfrastructureMetrics(t *testing.T) { - requiredMetrics := map[string][]string{ - "kube_pod_status_phase": {"p8s_logzio_name", "namespace", "pod", "phase", "uid"}, - "kube_pod_info": {"p8s_logzio_name", "namespace", "host_ip", "node", "pod"}, - "kube_pod_container_resource_limits": {"p8s_logzio_name", "namespace", "pod", "resource"}, - "kube_pod_container_info": {"p8s_logzio_name", "namespace", "pod"}, - "kube_pod_created": {"p8s_logzio_name", "namespace", "pod"}, - "kube_pod_owner": {"p8s_logzio_name", "namespace", "pod", "owner_kind", "owner_name"}, - "kube_pod_container_status_restarts_total": {"p8s_logzio_name", "namespace", "pod"}, - "kube_pod_status_reason": {"p8s_logzio_name", "namespace", "pod", "reason"}, - "node_cpu_seconds_total": {"p8s_logzio_name", "instance", "kubernetes_node"}, - "kube_node_status_allocatable": {"p8s_logzio_name", "node", "resource"}, - "node_memory_MemAvailable_bytes": {"p8s_logzio_name", "instance", "kubernetes_node"}, - "node_memory_MemTotal_bytes": {"p8s_logzio_name", "instance", "kubernetes_node"}, - "kube_node_role": {"p8s_logzio_name", "status", "role", "node"}, - "kube_node_status_condition": {"p8s_logzio_name", "status", "node"}, - "kube_node_created": {"p8s_logzio_name", "node"}, - "node_filesystem_avail_bytes": {"p8s_logzio_name", "instance", "kubernetes_node"}, - "node_filesystem_size_bytes": {"p8s_logzio_name", "instance", "kubernetes_node"}, - "kube_replicaset_owner": {"p8s_logzio_name", "namespace", "owner_kind", "owner_name", "replicaset"}, - "kube_deployment_created": {"p8s_logzio_name", "namespace", "deployment"}, - "kube_deployment_status_condition": {"p8s_logzio_name", "namespace", "deployment", "status"}, - } - envId := os.Getenv("ENV_ID") - query := fmt.Sprintf(`{env_id='%s'}`, envId) - testMetrics(t, requiredMetrics, query) -} - -func TestSpmMetrics(t *testing.T) { - requiredMetrics := map[string][]string{ - "calls_total": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, - "latency_sum": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, - "latency_count": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, - "latency_bucket": {"k8s_node_name", "k8s_namespace_name", "k8s_pod_name", "span_kind", "operation"}, - } - envId := os.Getenv("ENV_ID") - query := fmt.Sprintf(`{env_id='%s'}`, envId) - testMetrics(t, requiredMetrics, query) -} - -func testMetrics(t *testing.T, requiredMetrics map[string][]string, query string) { - metricsApiKey := os.Getenv("LOGZIO_METRICS_API_KEY") - if metricsApiKey == "" { - t.Fatalf("LOGZIO_METRICS_API_KEY environment variable not set") - } - - metricResponse, err := fetchMetrics(metricsApiKey, query) - if err != nil { - t.Fatalf("Failed to fetch metrics: %v", err) - } - - if metricResponse.Status != "success" { - t.Errorf("No metrics found") - } - logger.Info("Found metrics", zap.Int("metrics_count", len(metricResponse.Data.Result))) - // Verify required metrics - missingMetrics := verifyMetrics(metricResponse, requiredMetrics) - if len(missingMetrics) > 0 { - var sb strings.Builder - for _, metric := range missingMetrics { - sb.WriteString(metric + "\n") - } - t.Errorf("Missing metrics or labels:\n%s", sb.String()) - } -} - -// fetchMetrics fetches the metrics from the logz.io API -func fetchMetrics(metricsApiKey string, query string) (*MetricResponse, error) { - url := fmt.Sprintf("%s/metrics/prometheus/api/v1/query?query=%s", BaseLogzioApiUrl, query) - client := &http.Client{} - logger.Info("sending api request", zap.String("url", url)) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("X-API-TOKEN", metricsApiKey) - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var metricResponse MetricResponse - err = json.Unmarshal(body, &metricResponse) - if err != nil { - return nil, err - } - - return &metricResponse, nil -} - -// verifyMetrics checks if the required metrics and their labels are present in the response -func verifyMetrics(metricResponse *MetricResponse, requiredMetrics map[string][]string) []string { - missingMetrics := []string{} - - for metricName, requiredLabels := range requiredMetrics { - found := false - for _, result := range metricResponse.Data.Result { - if result.Metric["__name__"] == metricName { - found = true - for _, label := range requiredLabels { - if _, exists := result.Metric[label]; !exists { - missingMetrics = append(missingMetrics, fmt.Sprintf("%s (missing label: %s)", metricName, label)) - } - } - } - } - if !found { - missingMetrics = append(missingMetrics, metricName+" (not found)") - } - } - return deduplicate(missingMetrics) -} - -// deduplicate removes duplicate strings from the input array. -func deduplicate(data []string) []string { - uniqueMap := make(map[string]bool) - var uniqueList []string - - for _, item := range data { - trimmedItem := strings.TrimSpace(item) - if _, exists := uniqueMap[trimmedItem]; !exists { - uniqueMap[trimmedItem] = true - uniqueList = append(uniqueList, trimmedItem) - } - } - - return uniqueList -} diff --git a/tests/resources/logsgen.yaml b/tests/resources/logsgen.yaml deleted file mode 100644 index b81554f2..00000000 --- a/tests/resources/logsgen.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: log-generator - labels: - app: log-generator -spec: - replicas: 1 - selector: - matchLabels: - app: log-generator - template: - metadata: - labels: - app: log-generator - spec: - containers: - - name: random-logger - image: chentex/random-logger:latest - args: - - "100" - - "400" \ No newline at end of file diff --git a/tests/resources/tracegen.yaml b/tests/resources/tracegen.yaml deleted file mode 100644 index e9367b10..00000000 --- a/tests/resources/tracegen.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trace-gen -spec: - selector: - matchLabels: - app: trace-gen - template: - metadata: - labels: - app: trace-gen - spec: - containers: - - name: trace-gen - image: ghcr.io/frzifus/jaeger-otel-test:latest - args: - [ - '-otel.agent.host=logzio-k8s-telemetry-otel-collector', - '-otel.agent.port=4317', - ] - env: - - name: OTEL_SERVICE_NAME - value: 'local-test-service' \ No newline at end of file diff --git a/tests/traces_e2e_test.go b/tests/traces_e2e_test.go deleted file mode 100644 index 67dc0473..00000000 --- a/tests/traces_e2e_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package tests - -import ( - "bytes" - "encoding/json" - "fmt" - "go.uber.org/zap" - "io" - "net/http" - "os" - "testing" -) - -// TraceResponse represents the structure of the traces API response -type TraceResponse struct { - Hits struct { - Total int `json:"total"` - Hits []struct { - Source struct { - Process struct { - Tag struct { - KubernetesNamespace string `json:"kubernetes_namespace"` - KubernetesNode string `json:"kubernetes_node"` - Pod string `json:"pod"` - } `json:"tag"` - } `json:"process"` - } `json:"_source"` - } `json:"hits"` - } `json:"hits"` -} - -func TestLogzioMonitoringTraces(t *testing.T) { - tracesApiKey := os.Getenv("LOGZIO_TRACES_API_KEY") - if tracesApiKey == "" { - t.Fatalf("LOGZIO_TRACES_API_KEY environment variable not set") - } - - traceResponse, err := fetchTraces(tracesApiKey) - if err != nil { - t.Fatalf("Failed to fetch traces: %v", err) - } - if traceResponse.Hits.Total == 0 { - t.Errorf("No traces found") - } - // Verify required fields - requiredFields := []string{"kubernetes_namespace", "kubernetes_node", "pod"} - missingFields := verifyTraces(traceResponse, requiredFields) - if len(missingFields) > 0 { - t.Errorf("Missing trace fields: %v", missingFields) - } -} - -// verifyTraces checks if the traces contain the required Kubernetes fields -func verifyTraces(traceResponse *TraceResponse, requiredFields []string) []string { - missingFieldsMap := make(map[string]bool, len(requiredFields)) - for _, field := range requiredFields { - missingFieldsMap[field] = false - } - - for _, hit := range traceResponse.Hits.Hits { - tag := hit.Source.Process.Tag - if tag.KubernetesNamespace == "" { - missingFieldsMap["kubernetes_namespace"] = true - break - } - if tag.KubernetesNode == "" { - missingFieldsMap["kubernetes_node"] = true - break - } - if tag.Pod == "" { - missingFieldsMap["pod"] = true - break - } - } - - var missingFields []string - for field, value := range missingFieldsMap { - if value { - missingFields = append(missingFields, field) - } - } - - return missingFields -} - -// fetchTraces fetches the traces from the logz.io API -func fetchTraces(tracesApiKey string) (*TraceResponse, error) { - url := fmt.Sprintf("%s/search", BaseLogzioApiUrl) - client := &http.Client{} - envID := os.Getenv("ENV_ID") - query := fmt.Sprintf(`JaegerTag.env_id:%s AND type:jaegerSpan`, envID) - logger.Info("sending api request", zap.String("url", url), zap.String("query", query)) - formattedQuery := formatQuery(query) - req, err := http.NewRequest("POST", url, bytes.NewBufferString(formattedQuery)) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("X-API-TOKEN", tracesApiKey) - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var traceResponse TraceResponse - err = json.Unmarshal(body, &traceResponse) - if err != nil { - return nil, err - } - - return &traceResponse, nil -}