diff --git a/.chloggen/3149-add-must-gather.yaml b/.chloggen/3149-add-must-gather.yaml new file mode 100755 index 0000000000..d42c553265 --- /dev/null +++ b/.chloggen/3149-add-must-gather.yaml @@ -0,0 +1,25 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action) +component: auto-instrumentation, collector + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add a must gather utility to help troubleshoot" + +# One or more tracking issues related to the change +issues: [3149] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + The new utility is available as part of a new container image. + + To use the image in a running OpenShift cluster, you need to run the following command: + + ```sh + oc adm must-gather --image=ghcr.io/open-telemetry/opentelemetry-operator/must-gather -- /usr/bin/must-gather --operator-namespace opentelemetry-operator-system + ``` + + See the [README](https://github.com/open-telemetry/opentelemetry-operator/blob/main/cmd/gather/README.md) for more details. diff --git a/.github/workflows/publish-must-gather.yaml b/.github/workflows/publish-must-gather.yaml new file mode 100644 index 0000000000..e1d507d24c --- /dev/null +++ b/.github/workflows/publish-must-gather.yaml @@ -0,0 +1,86 @@ +name: "Publish must-gather image" + +on: + push: + branches: [ main ] + tags: [ 'v*' ] + + workflow_dispatch: + +env: + PLATFORMS: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le + +jobs: + publish: + name: Publish must-gather container image + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '~1.22.4' + + - name: Unshallow + run: git fetch --prune --unshallow + + - name: Build the binary for each supported architecture + run: | + for platform in $(echo $PLATFORMS | tr "," "\n"); do + arch=${platform#*/} + echo "Building must-gather for $arch" + make must-gather ARCH=$arch + done + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/open-telemetry/opentelemetry-operator/must-gather + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{raw}} + type=ref,event=branch + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Log into Docker.io + uses: docker/login-action@v3 + if: ${{ github.event_name == 'push' }} + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Package Registry + uses: docker/login-action@v3 + if: ${{ github.event_name == 'push' }} + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push must-gather image + uses: docker/build-push-action@v6 + with: + context: . + file: ./cmd/gather/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/Makefile b/Makefile index 3a8396009a..939af881d5 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,8 @@ OPERATOROPAMPBRIDGE_IMG ?= ${IMG_PREFIX}/${OPERATOROPAMPBRIDGE_IMG_REPO}:$(addpr BRIDGETESTSERVER_IMG_REPO ?= e2e-test-app-bridge-server BRIDGETESTSERVER_IMG ?= ${IMG_PREFIX}/${BRIDGETESTSERVER_IMG_REPO}:ve2e +MUSTGATHER_IMG ?= ${IMG_PREFIX}/must-gather + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -143,6 +145,10 @@ ci: generate fmt vet test ensure-generate-is-noop manager: generate CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(ARCH) go build -o bin/manager_${ARCH} -ldflags "${COMMON_LDFLAGS} ${OPERATOR_LDFLAGS}" main.go +.PHONY: must-gather +must-gather: + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(ARCH) go build -o bin/must-gather_${ARCH} -ldflags "${COMMON_LDFLAGS}" ./cmd/gather/main.go + # Build target allocator binary .PHONY: targetallocator targetallocator: @@ -362,6 +368,15 @@ container-bridge-test-server: GOOS = linux container-bridge-test-server: docker build --load -t ${BRIDGETESTSERVER_IMG} tests/test-e2e-apps/bridge-server +.PHONY: container-must-gather +container-must-gather: GOOS = linux +container-must-gather: must-gather + docker build -f cmd/gather/Dockerfile --load -t ${MUSTGATHER_IMG} . + +.PHONY: container-must-gather-push +container-must-gather-push: + docker push ${MUSTGATHER_IMG} + .PHONY: start-kind start-kind: kind ifeq (true,$(START_KIND_CLUSTER)) @@ -388,7 +403,6 @@ else $(MAKE) container-push endif - .PHONY: load-image-target-allocator load-image-target-allocator: container-target-allocator kind ifeq (true,$(START_KIND_CLUSTER)) diff --git a/cmd/gather/Dockerfile b/cmd/gather/Dockerfile new file mode 100644 index 0000000000..3eb3d5c54b --- /dev/null +++ b/cmd/gather/Dockerfile @@ -0,0 +1,16 @@ +FROM registry.access.redhat.com/ubi9-minimal:9.2 + +RUN INSTALL_PKGS=" \ + rsync \ + tar \ + " && \ + microdnf install -y $INSTALL_PKGS && \ + microdnf clean all +WORKDIR / + +ARG TARGETARCH +COPY bin/must-gather_${TARGETARCH} /usr/bin/must-gather + +USER 65532:65532 + +ENTRYPOINT ["/usr/bin/must-gather"] diff --git a/cmd/gather/README.md b/cmd/gather/README.md new file mode 100644 index 0000000000..0599ef4c12 --- /dev/null +++ b/cmd/gather/README.md @@ -0,0 +1,36 @@ +# OpenTelemetry Operator Must-Gather + +The OpenTelemetry Operator `must-gather` tool is designed to collect comprehensive information about OpenTelemetry components within an OpenShift cluster. This utility extends the functionality of [OpenShift must-gather](https://github.com/openshift/must-gather) by specifically targeting and retrieving data related to the OpenTelemetry Operator, helping in diagnostics and troubleshooting. + +Note that you can use this utility too to gather information about the objects deployed by the OpenTelemetry Operator if you don't use OpenShift. + +## What is a Must-Gather? + +The `must-gather` tool is a utility that collects logs, cluster information, and resource configurations related to a specific operator or application in an OpenShift cluster. It helps cluster administrators and developers diagnose issues by providing a snapshot of the cluster's state related to the targeted component. More information [in the official documentation](https://docs.openshift.com/container-platform/4.16/support/gathering-cluster-data.html). + +## Usage + +First, you will need to build and push the image: +```sh +make container-must-gather container-must-gather-push +``` + +To run the must-gather tool for the OpenTelemetry Operator, use one of the following commands, depending on how you want to source the image and the namespace where the operator is deployed. + +### Using the image from the Operator deployment + +This is the recommended way to do it if you are not using OpenShift. + +If you want to use the image in a running cluster, you need to run the following command: + +```sh +oc adm must-gather --image=ghcr.io/open-telemetry/opentelemetry-operator/must-gather -- /usr/bin/must-gather --operator-namespace opentelemetry-operator-system +``` + +### Using it as a CLI + +You only need to build and run: +```sh +make must-gather +./bin/must-gather_$(go env GOARCH) --help +``` diff --git a/cmd/gather/cluster/cluster.go b/cmd/gather/cluster/cluster.go new file mode 100644 index 0000000000..3d289fb6ae --- /dev/null +++ b/cmd/gather/cluster/cluster.go @@ -0,0 +1,419 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "reflect" + "strings" + + routev1 "github.com/openshift/api/route/v1" + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + policy1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + otelv1alpha1 "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" + "github.com/open-telemetry/opentelemetry-operator/cmd/gather/config" +) + +type Cluster struct { + config *config.Config + apiAvailabilityCache map[schema.GroupVersionResource]bool +} + +func NewCluster(cfg *config.Config) Cluster { + return Cluster{ + config: cfg, + apiAvailabilityCache: make(map[schema.GroupVersionResource]bool), + } +} +func (c *Cluster) getOperatorNamespace() (string, error) { + if c.config.OperatorNamespace != "" { + return c.config.OperatorNamespace, nil + } + + deployment, err := c.getOperatorDeployment() + if err != nil { + return "", err + } + + c.config.OperatorNamespace = deployment.Namespace + + return c.config.OperatorNamespace, nil +} + +func (c *Cluster) getOperatorDeployment() (appsv1.Deployment, error) { + operatorDeployments := appsv1.DeploymentList{} + err := c.config.KubernetesClient.List(context.TODO(), &operatorDeployments, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + "app.kubernetes.io/name": "opentelemetry-operator", + }), + }) + + if err != nil { + return appsv1.Deployment{}, err + } + + if len(operatorDeployments.Items) == 0 { + return appsv1.Deployment{}, fmt.Errorf("operator not found") + } + + return operatorDeployments.Items[0], nil + +} + +func (c *Cluster) GetOperatorLogs() error { + deployment, err := c.getOperatorDeployment() + if err != nil { + return err + } + + labelSelector := labels.Set(deployment.Spec.Selector.MatchLabels).AsSelectorPreValidated() + operatorPods := corev1.PodList{} + err = c.config.KubernetesClient.List(context.TODO(), &operatorPods, &client.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return err + } + + pod := operatorPods.Items[0] + c.getPodLogs(pod.Name, pod.Namespace, "manager") + return nil +} + +func (c *Cluster) getPodLogs(podName, namespace, container string) { + pods := c.config.KubernetesClientSet.CoreV1().Pods(namespace) + writeLogToFile(c.config.CollectionDir, podName, container, pods) +} + +func (c *Cluster) GetOperatorDeploymentInfo() error { + err := os.MkdirAll(c.config.CollectionDir, os.ModePerm) + if err != nil { + return err + } + + deployment, err := c.getOperatorDeployment() + if err != nil { + return err + } + + writeToFile(c.config.CollectionDir, &deployment) + + return nil +} + +func (c *Cluster) GetOLMInfo() error { + if !c.isAPIAvailable(schema.GroupVersionResource{ + Group: operatorsv1.SchemeGroupVersion.Group, + Version: operatorsv1.SchemeGroupVersion.Version, + Resource: "Operator", + }) { + log.Println("OLM info not available") + return nil + } + + outputDir := filepath.Join(c.config.CollectionDir, "olm") + err := os.MkdirAll(outputDir, os.ModePerm) + if err != nil { + return err + } + + operatorNamespace, err := c.getOperatorNamespace() + if err != nil { + return err + } + + // Operators + operators := operatorsv1.OperatorList{} + err = c.config.KubernetesClient.List(context.TODO(), &operators, &client.ListOptions{ + Namespace: operatorNamespace, + }) + if err != nil { + return err + } + for _, o := range operators.Items { + o := o + writeToFile(outputDir, &o) + + } + + // OperatorGroups + operatorGroups := operatorsv1.OperatorGroupList{} + err = c.config.KubernetesClient.List(context.TODO(), &operatorGroups, &client.ListOptions{ + Namespace: operatorNamespace, + }) + if err != nil { + return err + } + for _, o := range operatorGroups.Items { + o := o + if strings.Contains(o.Name, "opentelemetry") { + writeToFile(outputDir, &o) + } + } + + // Subscription + subscriptions := operatorsv1alpha1.SubscriptionList{} + err = c.config.KubernetesClient.List(context.TODO(), &subscriptions, &client.ListOptions{ + Namespace: operatorNamespace, + }) + if err != nil { + return err + } + for _, o := range subscriptions.Items { + o := o + writeToFile(outputDir, &o) + + } + + // InstallPlan + ips := operatorsv1alpha1.InstallPlanList{} + err = c.config.KubernetesClient.List(context.TODO(), &ips, &client.ListOptions{ + Namespace: operatorNamespace, + }) + if err != nil { + return err + } + for _, o := range ips.Items { + o := o + writeToFile(outputDir, &o) + } + + // ClusterServiceVersion + csvs := operatorsv1alpha1.ClusterServiceVersionList{} + err = c.config.KubernetesClient.List(context.TODO(), &csvs, &client.ListOptions{ + Namespace: operatorNamespace, + }) + if err != nil { + return err + } + for _, o := range csvs.Items { + o := o + if strings.Contains(o.Name, "opentelemetry") { + writeToFile(outputDir, &o) + } + } + + return nil +} + +func (c *Cluster) GetOpenTelemetryCollectors() error { + otelCols := otelv1beta1.OpenTelemetryCollectorList{} + + err := c.config.KubernetesClient.List(context.TODO(), &otelCols) + if err != nil { + return err + } + + log.Println("OpenTelemetryCollectors found:", len(otelCols.Items)) + + errorDetected := false + + for _, otelCol := range otelCols.Items { + otelCol := otelCol + err := c.processOTELCollector(&otelCol) + if err != nil { + log.Fatalln(err) + errorDetected = true + } + } + + if errorDetected { + return fmt.Errorf("something failed while getting the opentelemtrycollectors") + } + return nil +} + +func (c *Cluster) GetInstrumentations() error { + instrumentations := otelv1alpha1.InstrumentationList{} + + err := c.config.KubernetesClient.List(context.TODO(), &instrumentations) + if err != nil { + return err + } + + log.Println("Instrumentations found:", len(instrumentations.Items)) + + errorDetected := false + + for _, instr := range instrumentations.Items { + instr := instr + outputDir := filepath.Join(c.config.CollectionDir, instr.Namespace) + err := os.MkdirAll(outputDir, os.ModePerm) + if err != nil { + log.Fatalln(err) + errorDetected = true + continue + } + + writeToFile(outputDir, &instr) + } + + if errorDetected { + return fmt.Errorf("something failed while getting the opentelemtrycollectors") + } + return nil +} + +func (c *Cluster) processOTELCollector(otelCol *otelv1beta1.OpenTelemetryCollector) error { + log.Printf("Processing OpenTelemetryCollector %s/%s", otelCol.Namespace, otelCol.Name) + folder, err := createOTELFolder(c.config.CollectionDir, otelCol) + if err != nil { + return err + } + writeToFile(folder, otelCol) + + err = c.processOwnedResources(otelCol, folder) + if err != nil { + return err + } + + return nil +} + +func (c *Cluster) processOwnedResources(owner interface{}, folder string) error { + resourceTypes := []struct { + list client.ObjectList + apiCheck func() bool + }{ + {&appsv1.DaemonSetList{}, func() bool { return true }}, + {&appsv1.DeploymentList{}, func() bool { return true }}, + {&appsv1.StatefulSetList{}, func() bool { return true }}, + {&rbacv1.ClusterRoleList{}, func() bool { return true }}, + {&rbacv1.ClusterRoleBindingList{}, func() bool { return true }}, + {&corev1.ConfigMapList{}, func() bool { return true }}, + {&corev1.PersistentVolumeList{}, func() bool { return true }}, + {&corev1.PersistentVolumeClaimList{}, func() bool { return true }}, + {&corev1.PodList{}, func() bool { return true }}, + {&corev1.ServiceList{}, func() bool { return true }}, + {&corev1.ServiceAccountList{}, func() bool { return true }}, + {&autoscalingv2.HorizontalPodAutoscalerList{}, func() bool { return true }}, + {&networkingv1.IngressList{}, func() bool { return true }}, + {&policy1.PodDisruptionBudgetList{}, func() bool { return true }}, + {&monitoringv1.PodMonitorList{}, c.isMonitoringAPIAvailable}, + {&monitoringv1.ServiceMonitorList{}, c.isMonitoringAPIAvailable}, + {&routev1.RouteList{}, c.isRouteAPIAvailable}, + } + + for _, rt := range resourceTypes { + if rt.apiCheck() { + if err := c.processResourceType(rt.list, owner, folder); err != nil { + return err + } + } + } + + return nil +} + +func (c *Cluster) getOwnerResources(objList client.ObjectList, owner interface{}) ([]client.Object, error) { + err := c.config.KubernetesClient.List(context.TODO(), objList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + "app.kubernetes.io/managed-by": "opentelemetry-operator", + }), + }) + if err != nil { + return nil, err + } + + resources := []client.Object{} + + items := reflect.ValueOf(objList).Elem().FieldByName("Items") + for i := 0; i < items.Len(); i++ { + item := items.Index(i).Addr().Interface().(client.Object) + if hasOwnerReference(item, owner) { + resources = append(resources, item) + } + } + return resources, nil + +} + +func (c *Cluster) processResourceType(list client.ObjectList, owner interface{}, folder string) error { + resources, err := c.getOwnerResources(list, owner) + if err != nil { + return fmt.Errorf("failed to get resources: %w", err) + } + for _, resource := range resources { + writeToFile(folder, resource) + } + return nil +} + +func (c *Cluster) isMonitoringAPIAvailable() bool { + return c.isAPIAvailable(schema.GroupVersionResource{ + Group: monitoringv1.SchemeGroupVersion.Group, + Version: monitoringv1.SchemeGroupVersion.Version, + Resource: "ServiceMonitor", + }) +} + +func (c *Cluster) isRouteAPIAvailable() bool { + return c.isAPIAvailable(schema.GroupVersionResource{ + Group: routev1.GroupName, + Version: routev1.GroupVersion.Version, + Resource: "Route", + }) +} + +func (c *Cluster) isAPIAvailable(gvr schema.GroupVersionResource) bool { + if result, ok := c.apiAvailabilityCache[gvr]; ok { + return result + } + + rm := c.config.KubernetesClient.RESTMapper() + + gvk, err := rm.KindFor(gvr) + result := err == nil && !gvk.Empty() + c.apiAvailabilityCache[gvr] = result + + return result +} + +func hasOwnerReference(obj client.Object, owner interface{}) bool { + var ownerKind string + var ownerUID types.UID + + switch o := owner.(type) { + case *otelv1beta1.OpenTelemetryCollector: + ownerKind = o.Kind + ownerUID = o.UID + default: + return false + } + + for _, ownerRef := range obj.GetOwnerReferences() { + if ownerRef.Kind == ownerKind && ownerRef.UID == ownerUID { + return true + } + } + return false +} diff --git a/cmd/gather/cluster/cluster_test.go b/cmd/gather/cluster/cluster_test.go new file mode 100644 index 0000000000..8749d688e5 --- /dev/null +++ b/cmd/gather/cluster/cluster_test.go @@ -0,0 +1,176 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/open-telemetry/opentelemetry-operator/cmd/gather/config" +) + +// MockClient is a mock implementation of client.Client. +type MockClient struct { + mock.Mock +} + +func (m *MockClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, _ ...client.GetOption) error { + args := m.Called(ctx, key, obj) + return args.Error(0) +} + +func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + args := m.Called(ctx, list, opts) + return args.Error(0) +} + +func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + args := m.Called(ctx, obj, patch, opts) + return args.Error(0) +} + +func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + args := m.Called(ctx, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Status() client.StatusWriter { + args := m.Called() + return args.Get(0).(client.StatusWriter) +} + +func (m *MockClient) Scheme() *runtime.Scheme { + args := m.Called() + return args.Get(0).(*runtime.Scheme) +} + +func (m *MockClient) RESTMapper() meta.RESTMapper { + args := m.Called() + return args.Get(0).(meta.RESTMapper) +} + +func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{}, nil +} + +func (m *MockClient) IsObjectNamespaced(_ runtime.Object) (bool, error) { + return true, nil +} + +func (m *MockClient) SubResource(string) client.SubResourceClient { + return nil +} + +func TestGetOperatorNamespace(t *testing.T) { + mockClient := new(MockClient) + cfg := &config.Config{ + KubernetesClient: mockClient, + } + cluster := NewCluster(cfg) + + // Test when OperatorNamespace is already set + cfg.OperatorNamespace = "test-namespace" + ns, err := cluster.getOperatorNamespace() + assert.NoError(t, err) + assert.Equal(t, "test-namespace", ns) + + // Test when OperatorNamespace is not set + cfg.OperatorNamespace = "" + mockClient.On("List", mock.Anything, &appsv1.DeploymentList{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(1).(*appsv1.DeploymentList) + arg.Items = []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "operator-namespace", + }, + }, + } + }) + + ns, err = cluster.getOperatorNamespace() + assert.NoError(t, err) + assert.Equal(t, "operator-namespace", ns) + mockClient.AssertExpectations(t) +} + +func TestGetOperatorDeployment(t *testing.T) { + + mockClient := new(MockClient) + cfg := &config.Config{ + KubernetesClient: mockClient, + } + cluster := NewCluster(cfg) + // Test successful case + mockClient.On("List", mock.Anything, &appsv1.DeploymentList{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(1).(*appsv1.DeploymentList) + arg.Items = []appsv1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "opentelemetry-operator", + }, + }, + } + }) + + deployment, err := cluster.getOperatorDeployment() + assert.NoError(t, err) + assert.Equal(t, "opentelemetry-operator", deployment.Name) + + mockClient.AssertExpectations(t) +} + +func TestGetOperatorDeploymentNotFound(t *testing.T) { + + mockClient := new(MockClient) + cfg := &config.Config{ + KubernetesClient: mockClient, + } + cluster := NewCluster(cfg) + + // Test when no operator is found + mockClient.On("List", mock.Anything, &appsv1.DeploymentList{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + arg := args.Get(1).(*appsv1.DeploymentList) + arg.Items = []appsv1.Deployment{} + }) + + _, err := cluster.getOperatorDeployment() + assert.Error(t, err) + assert.Equal(t, "operator not found", err.Error()) +} diff --git a/cmd/gather/cluster/write.go b/cmd/gather/cluster/write.go new file mode 100644 index 0000000000..eafd05b795 --- /dev/null +++ b/cmd/gather/cluster/write.go @@ -0,0 +1,114 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + cgocorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" +) + +func createOTELFolder(collectionDir string, otelCol *v1beta1.OpenTelemetryCollector) (string, error) { + outputDir := filepath.Join(collectionDir, "namespaces", otelCol.Namespace, otelCol.Name) + err := os.MkdirAll(outputDir, os.ModePerm) + if err != nil { + return "", err + } + return outputDir, nil +} + +func createFile(outputDir string, obj client.Object) (*os.File, error) { + kind := obj.GetObjectKind().GroupVersionKind().Kind + + if kind == "" { + // reflect.TypeOf(obj) will return something like *v1.Deployment. We remove the first part + prefix, typeName, found := strings.Cut(reflect.TypeOf(obj).String(), ".") + if found { + kind = typeName + } else { + kind = prefix + } + } + + kind = strings.ToLower(kind) + name := strings.ReplaceAll(obj.GetName(), ".", "-") + + path := filepath.Join(outputDir, fmt.Sprintf("%s-%s.yaml", kind, name)) + return os.Create(path) +} + +func writeLogToFile(outputDir, podName, container string, p cgocorev1.PodInterface) { + req := p.GetLogs(podName, &corev1.PodLogOptions{Container: container}) + podLogs, err := req.Stream(context.Background()) + if err != nil { + log.Fatalf("Error getting pod logs: %v\n", err) + return + } + defer podLogs.Close() + + err = os.MkdirAll(outputDir, os.ModePerm) + if err != nil { + log.Fatalln(err) + return + } + + outputFile, err := os.Create(filepath.Join(outputDir, podName)) + if err != nil { + log.Fatalf("Error getting pod logs: %v\n", err) + return + } + + _, err = io.Copy(outputFile, podLogs) + if err != nil { + log.Fatalf("Error copying logs to file: %v\n", err) + } +} + +func writeToFile(outputDir string, o client.Object) { + // Open or create the file for writing + outputFile, err := createFile(outputDir, o) + if err != nil { + log.Fatalf("Failed to create file: %v", err) + } + defer outputFile.Close() + + unstructuredDeployment, err := runtime.DefaultUnstructuredConverter.ToUnstructured(o) + if err != nil { + log.Fatalf("Error converting deployment to unstructured: %v", err) + } + + unstructuredObj := &unstructured.Unstructured{Object: unstructuredDeployment} + + // Serialize the unstructured object to YAML + serializer := json.NewYAMLSerializer(json.DefaultMetaFactory, nil, nil) + err = serializer.Encode(unstructuredObj, outputFile) + if err != nil { + log.Fatalf("Error encoding to YAML: %v", err) + } +} diff --git a/cmd/gather/cluster/write_test.go b/cmd/gather/cluster/write_test.go new file mode 100644 index 0000000000..262931abb3 --- /dev/null +++ b/cmd/gather/cluster/write_test.go @@ -0,0 +1,261 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + + "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" +) + +type MockObject struct { + mock.Mock +} + +// Implement all methods required by client.Object and runtime.Object + +// GetObjectKind mocks the GetObjectKind method. +func (m *MockObject) GetObjectKind() schema.ObjectKind { + args := m.Called() + return args.Get(0).(schema.ObjectKind) +} + +// GetName mocks the GetName method. +func (m *MockObject) GetName() string { + args := m.Called() + return args.String(0) +} + +// SetName mocks the SetName method. +func (m *MockObject) SetName(name string) { + m.Called(name) +} + +// GetNamespace mocks the GetNamespace method. +func (m *MockObject) GetNamespace() string { + args := m.Called() + return args.String(0) +} + +// SetNamespace mocks the SetNamespace method. +func (m *MockObject) SetNamespace(namespace string) { + m.Called(namespace) +} + +// GetAnnotations mocks the GetAnnotations method. +func (m *MockObject) GetAnnotations() map[string]string { + args := m.Called() + return args.Get(0).(map[string]string) +} + +// SetAnnotations mocks the SetAnnotations method. +func (m *MockObject) SetAnnotations(annotations map[string]string) { + m.Called(annotations) +} + +// GetCreationTimestamp mocks the GetCreationTimestamp method. +func (m *MockObject) GetCreationTimestamp() v1.Time { + args := m.Called() + return args.Get(0).(v1.Time) +} + +// SetCreationTimestamp mocks the SetCreationTimestamp method. +func (m *MockObject) SetCreationTimestamp(timestamp v1.Time) { + m.Called(timestamp) +} + +// GetDeletionGracePeriodSeconds mocks the GetDeletionGracePeriodSeconds method. +func (m *MockObject) GetDeletionGracePeriodSeconds() *int64 { + args := m.Called() + return args.Get(0).(*int64) +} + +// GetDeletionTimestamp mocks the GetDeletionTimestamp method. +func (m *MockObject) GetDeletionTimestamp() *v1.Time { + args := m.Called() + return args.Get(0).(*v1.Time) +} + +// GetLabels mocks the GetLabels method. +func (m *MockObject) GetLabels() map[string]string { + args := m.Called() + return args.Get(0).(map[string]string) +} + +// SetLabels mocks the SetLabels method. +func (m *MockObject) SetLabels(labels map[string]string) { + m.Called(labels) +} + +// GetFinalizers mocks the GetFinalizers method. +func (m *MockObject) GetFinalizers() []string { + args := m.Called() + return args.Get(0).([]string) +} + +// SetFinalizers mocks the SetFinalizers method. +func (m *MockObject) SetFinalizers(finalizers []string) { + m.Called(finalizers) +} + +// GetGenerateName mocks the GetGenerateName method. +func (m *MockObject) GetGenerateName() string { + args := m.Called() + return args.String(0) +} + +// SetGenerateName mocks the SetGenerateName method. +func (m *MockObject) SetGenerateName(name string) { + m.Called(name) +} + +// DeepCopyObject mocks the DeepCopyObject method. +func (m *MockObject) DeepCopyObject() runtime.Object { + args := m.Called() + return args.Get(0).(runtime.Object) +} + +func (m *MockObject) GetManagedFields() []v1.ManagedFieldsEntry { + args := m.Called() + return args.Get(0).([]v1.ManagedFieldsEntry) +} + +func (m *MockObject) GetOwnerReferences() []v1.OwnerReference { + args := m.Called() + return args.Get(0).([]v1.OwnerReference) +} + +func (m *MockObject) GetGeneration() int64 { + args := m.Called() + return args.Get(0).(int64) +} + +func (m *MockObject) GetResourceVersion() string { + args := m.Called() + return args.String(0) +} + +func (m *MockObject) GetSelfLink() string { + args := m.Called() + return args.String(0) +} + +type MockPodInterface struct { + mock.Mock +} + +func (m *MockPodInterface) GetLogs(podName string, options *corev1.PodLogOptions) *rest.Request { + args := m.Called(podName, options) + return args.Get(0).(*rest.Request) +} + +type MockRequest struct { + mock.Mock +} + +func (m *MockRequest) Stream(ctx context.Context) (io.ReadCloser, error) { + args := m.Called(ctx) + return args.Get(0).(io.ReadCloser), args.Error(1) +} + +func TestCreateOTELFolder(t *testing.T) { + collectionDir := "/tmp/test-dir" + otelCol := &v1beta1.OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-otel", + }, + } + + outputDir, err := createOTELFolder(collectionDir, otelCol) + + expectedDir := filepath.Join(collectionDir, "namespaces", otelCol.Namespace, otelCol.Name) + assert.NoError(t, err) + assert.Equal(t, expectedDir, outputDir) + + // Clean up after the test + os.RemoveAll(collectionDir) +} + +func TestCreateFile(t *testing.T) { + outputDir := "/tmp/test-dir" + err := os.MkdirAll(outputDir, os.ModePerm) + assert.NoError(t, err) + defer os.RemoveAll(outputDir) + + mockObj := &MockObject{} + mockObj.On("GetObjectKind").Return(schema.EmptyObjectKind) + mockObj.On("GetName").Return("test-deployment") + mockObj.On("DeepCopyObject").Return(mockObj) + + file, err := createFile(outputDir, mockObj) + assert.NoError(t, err) + defer file.Close() + + expectedPath := filepath.Join(outputDir, "mockobject-test-deployment.yaml") + _, err = os.Stat(expectedPath) + assert.NoError(t, err) +} + +func (m *MockObject) SetUID(uid types.UID) { + m.Called(uid) +} + +func (m *MockObject) GetUID() types.UID { + args := m.Called() + return args.Get(0).(types.UID) +} + +func (m *MockObject) SetDeletionGracePeriodSeconds(seconds *int64) { + m.Called(seconds) +} + +func (m *MockObject) SetDeletionTimestamp(timestamp *v1.Time) { + m.Called(timestamp) +} + +func (m *MockObject) SetGeneration(generation int64) { + m.Called(generation) +} + +func (m *MockObject) SetManagedFields(fields []v1.ManagedFieldsEntry) { + m.Called(fields) +} + +func (m *MockObject) SetOwnerReferences(references []v1.OwnerReference) { + m.Called(references) +} + +func (m *MockObject) SetResourceVersion(version string) { + m.Called(version) +} + +func (m *MockObject) SetSelfLink(selfLink string) { + m.Called(selfLink) +} diff --git a/cmd/gather/config/config.go b/cmd/gather/config/config.go new file mode 100644 index 0000000000..4a1ba48ff0 --- /dev/null +++ b/cmd/gather/config/config.go @@ -0,0 +1,75 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "path/filepath" + + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Config struct { + CollectionDir string + OperatorName string + OperatorNamespace string + KubernetesClient client.Client + KubernetesClientSet *kubernetes.Clientset +} + +func NewConfig(scheme *runtime.Scheme) (Config, error) { + var operatorName, operatorNamespace, collectionDir, kubeconfigPath string + + pflag.StringVar(&operatorName, "operator-name", "opentelemetry-operator", "Operator name") + pflag.StringVar(&operatorNamespace, "operator-namespace", "", "Namespace where the operator was deployed") + pflag.StringVar(&collectionDir, "collection-dir", filepath.Join(homedir.HomeDir(), "/must-gather"), "Absolute path to the KubeconfigPath file") + pflag.StringVar(&kubeconfigPath, "kubeconfig", "", "Path to the kubeconfig file") + pflag.Parse() + + config, err := rest.InClusterConfig() + if err != nil { + if kubeconfigPath == "" { + kubeconfigPath = filepath.Join(homedir.HomeDir(), ".kube", "config") + } + config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return Config{}, fmt.Errorf("failed to create Kubernetes client config: %w", err) + } + } + + clusterClient, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + return Config{}, fmt.Errorf("creating the Kubernetes client: %w\n", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return Config{}, fmt.Errorf("creating the Kubernetes clienset: %w\n", err) + } + + return Config{ + CollectionDir: collectionDir, + KubernetesClient: clusterClient, + KubernetesClientSet: clientset, + OperatorName: operatorName, + OperatorNamespace: operatorNamespace, + }, nil +} diff --git a/cmd/gather/main.go b/cmd/gather/main.go new file mode 100644 index 0000000000..692bf14a38 --- /dev/null +++ b/cmd/gather/main.go @@ -0,0 +1,86 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + "os" + + routev1 "github.com/openshift/api/route/v1" + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + policyV1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + otelv1alpha1 "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" + otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1" + "github.com/open-telemetry/opentelemetry-operator/cmd/gather/cluster" + "github.com/open-telemetry/opentelemetry-operator/cmd/gather/config" +) + +var scheme *k8sruntime.Scheme + +func init() { + scheme = k8sruntime.NewScheme() + utilruntime.Must(otelv1alpha1.AddToScheme(scheme)) + utilruntime.Must(otelv1beta1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + utilruntime.Must(networkingv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) + utilruntime.Must(rbacv1.AddToScheme(scheme)) + utilruntime.Must(policyV1.AddToScheme(scheme)) + utilruntime.Must(monitoringv1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) + utilruntime.Must(operatorsv1.AddToScheme(scheme)) + utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) +} + +func main() { + config, err := config.NewConfig(scheme) + if err != nil { + log.Fatalln(err) + os.Exit(1) + } + + cluster := cluster.NewCluster(&config) + err = cluster.GetOperatorLogs() + if err != nil { + log.Fatalln(err) + } + err = cluster.GetOperatorDeploymentInfo() + if err != nil { + log.Fatalln(err) + } + err = cluster.GetOLMInfo() + if err != nil { + log.Fatalln(err) + } + err = cluster.GetOpenTelemetryCollectors() + if err != nil { + log.Fatalln(err) + } + err = cluster.GetInstrumentations() + if err != nil { + log.Fatalln(err) + } +} diff --git a/go.mod b/go.mod index 2e8d4b2cf6..3947fed58d 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/oklog/run v1.1.0 github.com/open-telemetry/opamp-go v0.15.0 github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e + github.com/operator-framework/api v0.27.0 github.com/operator-framework/operator-lib v0.15.0 github.com/prometheus-operator/prometheus-operator v0.76.0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.2 @@ -184,7 +185,9 @@ require ( github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 1583007822..f1069f8cf6 100644 --- a/go.sum +++ b/go.sum @@ -508,6 +508,8 @@ github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e h1:cxgCNo/R769CO23AK github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/operator-framework/api v0.27.0 h1:OrVaGKZJvbZo58HTv2guz7aURkhVKYhFqZ/6VpifiXI= +github.com/operator-framework/api v0.27.0/go.mod h1:lg2Xx+S8NQWGYlEOvFwQvH46E5EK5IrAIL7HWfAhciM= github.com/operator-framework/operator-lib v0.15.0 h1:0QeRM4PMtThqINpcFGCEBnIV3Z8u7/8fYLEx6mUtdcM= github.com/operator-framework/operator-lib v0.15.0/go.mod h1:ZxLvFuQ7bRWiTNBOqodbuNvcsy/Iq0kOygdxhlbNdI0= github.com/ovh/go-ovh v1.6.0 h1:ixLOwxQdzYDx296sXcgS35TOPEahJkpjMGtzPadCjQI= @@ -840,6 +842,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=